From 44e5eded77da32847b70f8ace8beb6df5426c61b Mon Sep 17 00:00:00 2001
From: Mark Lee <youtube-dl@lazymalevolence.com>
Date: Tue, 25 Mar 2014 19:49:17 -0700
Subject: [PATCH 0001/1105] Add video joiner class to postprocessor module

---
 youtube_dl/postprocessor/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py
index 7f19f717f..2ccbd5d48 100644
--- a/youtube_dl/postprocessor/__init__.py
+++ b/youtube_dl/postprocessor/__init__.py
@@ -5,6 +5,7 @@ from .ffmpeg import (
     FFmpegVideoConvertor,
     FFmpegExtractAudioPP,
     FFmpegEmbedSubtitlePP,
+    FFmpegJoinVideosPP,
 )
 from .xattrpp import XAttrMetadataPP
 
@@ -14,5 +15,6 @@ __all__ = [
     'FFmpegVideoConvertor',
     'FFmpegExtractAudioPP',
     'FFmpegEmbedSubtitlePP',
+    'FFmpegJoinVideosPP',
     'XAttrMetadataPP',
 ]

From 25ca480a513f53c9159ef277937e8b53e3834fa8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 26 Mar 2014 12:01:08 +0100
Subject: [PATCH 0002/1105] Makefile: include the docs in the tarball

---
 Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index c6d09932b..f7d917d09 100644
--- a/Makefile
+++ b/Makefile
@@ -72,8 +72,9 @@ youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-
 		--exclude '__pycache' \
 		--exclude '.git' \
 		--exclude 'testdata' \
+		--exclude 'docs/_build' \
 		-- \
-		bin devscripts test youtube_dl \
+		bin devscripts test youtube_dl docs \
 		CHANGELOG LICENSE README.md README.txt \
 		Makefile MANIFEST.in youtube-dl.1 youtube-dl.bash-completion setup.py \
 		youtube-dl

From f1eb03211a7b666b14043ff7fba46f0ee6c864dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Mar 2014 19:51:54 +0700
Subject: [PATCH 0003/1105] [cspan] Roll back unfinished rtmp support

---
 youtube_dl/extractor/cspan.py | 57 ++++-------------------------------
 1 file changed, 6 insertions(+), 51 deletions(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 795ccd926..2a8eda9ef 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -56,61 +56,16 @@ class CSpanIE(InfoExtractor):
 
         url = unescapeHTML(data['video']['files'][0]['path']['#text'])
 
-        doc = self._download_xml('http://www.c-span.org/common/services/flashXml.php?programid=' + video_id + '&version=2014-01-23',
+        doc = self._download_xml('http://www.c-span.org/common/services/flashXml.php?programid=' + video_id,
             video_id)
 
-        formats = [
-            {
-                'url': url,
-            }
-        ]
-
-        def find_string(node, s):
-            return find_xpath_attr(node, './/string', 'name', s).text
-
-        def find_number(node, s):
-            return int(find_xpath_attr(node, './/number', 'name', s).text)
-
-        def find_array(node, s):
-            return find_xpath_attr(node, './/array', 'name', s)
-
-        def process_files(files, url, formats):
-            for file in files:
-                path = find_string(file, 'path')
-                #duration = find_number(file, './number', 'name', 'length')
-                hd = find_number(file, 'hd')
-                formats.append({
-                    'url': url,
-                    'play_path': path,
-                    'ext': 'flv',
-                    'quality': hd,
-                })
-
-        def process_node(node, formats):
-            url = find_xpath_attr(node, './string', 'name', 'url')
-            if url is None:
-                url = find_xpath_attr(node, './string', 'name', 'URL')
-                if url is None:
-                    return
-            url = url.text.replace('$(protocol)', 'rtmp').replace('$(port)', '1935')
-            files = find_array(node, 'files')
-            if files is None:
-                return
-            process_files(files, url, formats)
-
-        process_node(doc.find('./media-link'), formats)
-
-        streams = find_array(doc, 'streams')
-        if streams is not None:
-            for stream in streams:
-                if find_string(stream, 'name') != 'vod':
-                    continue
-                process_node(stream, formats)
+        def find_string(s):
+            return find_xpath_attr(doc, './/string', 'name', s).text
 
         return {
             'id': video_id,
-            'title': find_string(doc, 'title'),
+            'title': find_string('title'),
+            'url': url,
             'description': description,
-            'thumbnail': find_string(doc, 'poster'),
-            'formats': formats,
+            'thumbnail': find_string('poster'),
         }

From 87a26277000b425972ec427f86b68fef2097c1ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 26 Mar 2014 15:03:34 +0100
Subject: [PATCH 0004/1105] [vice] Remove extractor

The generic ooyala detection works fine.
---
 youtube_dl/extractor/__init__.py |  1 -
 youtube_dl/extractor/vice.py     | 38 --------------------------------
 2 files changed, 39 deletions(-)
 delete mode 100644 youtube_dl/extractor/vice.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 56b382aed..d4b89fd5c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -263,7 +263,6 @@ from .veehd import VeeHDIE
 from .veoh import VeohIE
 from .vesti import VestiIE
 from .vevo import VevoIE
-from .vice import ViceIE
 from .viddler import ViddlerIE
 from .videobam import VideoBamIE
 from .videodetective import VideoDetectiveIE
diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
deleted file mode 100644
index 87812d6af..000000000
--- a/youtube_dl/extractor/vice.py
+++ /dev/null
@@ -1,38 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from .ooyala import OoyalaIE
-from ..utils import ExtractorError
-
-
-class ViceIE(InfoExtractor):
-    _VALID_URL = r'http://www\.vice\.com/.*?/(?P<name>.+)'
-
-    _TEST = {
-        u'url': u'http://www.vice.com/Fringes/cowboy-capitalists-part-1',
-        u'file': u'43cW1mYzpia9IlestBjVpd23Yu3afAfp.mp4',
-        u'info_dict': {
-            u'title': u'VICE_COWBOYCAPITALISTS_PART01_v1_VICE_WM_1080p.mov',
-        },
-        u'params': {
-            # Requires ffmpeg (m3u8 manifest)
-            u'skip_download': True,
-        },
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        name = mobj.group('name')
-        webpage = self._download_webpage(url, name)
-        try:
-            ooyala_url = self._og_search_video_url(webpage)
-        except ExtractorError:
-            try:
-                embed_code = self._search_regex(
-                    r'OO.Player.create\(\'ooyalaplayer\', \'(.+?)\'', webpage,
-                    u'ooyala embed code')
-                ooyala_url = OoyalaIE._url_for_embed_code(embed_code)
-            except ExtractorError:
-                raise ExtractorError(u'The page doesn\'t contain a video', expected=True)
-        return self.url_result(ooyala_url, ie='Ooyala')
-

From 44f373f3caf55c9ddc2e01bd605a7984fe9c4837 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 26 Mar 2014 15:09:14 +0100
Subject: [PATCH 0005/1105] [slashdot] Remove extractor

The generic ooyala detection works fine.
---
 youtube_dl/extractor/__init__.py |  1 -
 youtube_dl/extractor/slashdot.py | 24 ------------------------
 2 files changed, 25 deletions(-)
 delete mode 100644 youtube_dl/extractor/slashdot.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d4b89fd5c..685fc749d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -208,7 +208,6 @@ from .rutv import RUTVIE
 from .savefrom import SaveFromIE
 from .servingsys import ServingSysIE
 from .sina import SinaIE
-from .slashdot import SlashdotIE
 from .slideshare import SlideshareIE
 from .smotri import (
     SmotriIE,
diff --git a/youtube_dl/extractor/slashdot.py b/youtube_dl/extractor/slashdot.py
deleted file mode 100644
index d68646d24..000000000
--- a/youtube_dl/extractor/slashdot.py
+++ /dev/null
@@ -1,24 +0,0 @@
-import re
-
-from .common import InfoExtractor
-
-
-class SlashdotIE(InfoExtractor):
-    _VALID_URL = r'https?://tv\.slashdot\.org/video/\?embed=(?P<id>.*?)(&|$)'
-
-    _TEST = {
-        u'add_ie': ['Ooyala'],
-        u'url': u'http://tv.slashdot.org/video/?embed=JscHMzZDplD0p-yNLOzTfzC3Q3xzJaUz',
-        u'file': u'JscHMzZDplD0p-yNLOzTfzC3Q3xzJaUz.mp4',
-        u'md5': u'd2222e7a4a4c1541b3e0cf732fb26735',
-        u'info_dict': {
-            u'title': u' Meet the Stampede Supercomputing Cluster\'s Administrator',
-        },
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        webpage = self._download_webpage(url, video_id)
-        ooyala_url = self._search_regex(r'<script src="(.*?)"', webpage, 'ooyala url')
-        return self.url_result(ooyala_url, 'Ooyala')

From 216c85318a9e5e0a2e62544372d51061bcb2d6f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 25 Mar 2014 21:22:27 +0700
Subject: [PATCH 0006/1105] Use explicitly set Referer header for downloading

---
 youtube_dl/downloader/http.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 348097dab..16a2dce3f 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -23,6 +23,8 @@ class HttpFD(FileDownloader):
         headers = {'Youtubedl-no-compression': 'True'}
         if 'user_agent' in info_dict:
             headers['Youtubedl-user-agent'] = info_dict['user_agent']
+        if 'referer' in info_dict:
+            headers['Referer'] = info_dict['referer']
         basic_request = compat_urllib_request.Request(url, None, headers)
         request = compat_urllib_request.Request(url, None, headers)
 

From aff4f1b31fa7c091a445fbfe4deadd5b2b1150ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 25 Mar 2014 21:22:41 +0700
Subject: [PATCH 0007/1105] [auengine] Add referer

---
 youtube_dl/extractor/auengine.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/auengine.py b/youtube_dl/extractor/auengine.py
index 94caad0cf..926d733c5 100644
--- a/youtube_dl/extractor/auengine.py
+++ b/youtube_dl/extractor/auengine.py
@@ -51,4 +51,5 @@ class AUEngineIE(InfoExtractor):
             'url': video_url,
             'title': title,
             'thumbnail': thumbnail,
+            'referer': 'http://www.auengine.com/flowplayer/flowplayer.commercial-3.2.14.swf',
         }

From eb2396d87a7febd97be4433c9d9584ec1492395f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 25 Mar 2014 21:53:26 +0700
Subject: [PATCH 0008/1105] Replace 'referer' with 'http_referer'

---
 youtube_dl/downloader/http.py    | 4 ++--
 youtube_dl/extractor/auengine.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 16a2dce3f..cc8b9c9a7 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -23,8 +23,8 @@ class HttpFD(FileDownloader):
         headers = {'Youtubedl-no-compression': 'True'}
         if 'user_agent' in info_dict:
             headers['Youtubedl-user-agent'] = info_dict['user_agent']
-        if 'referer' in info_dict:
-            headers['Referer'] = info_dict['referer']
+        if 'http_referer' in info_dict:
+            headers['Referer'] = info_dict['http_referer']
         basic_request = compat_urllib_request.Request(url, None, headers)
         request = compat_urllib_request.Request(url, None, headers)
 
diff --git a/youtube_dl/extractor/auengine.py b/youtube_dl/extractor/auengine.py
index 926d733c5..20bf12550 100644
--- a/youtube_dl/extractor/auengine.py
+++ b/youtube_dl/extractor/auengine.py
@@ -51,5 +51,5 @@ class AUEngineIE(InfoExtractor):
             'url': video_url,
             'title': title,
             'thumbnail': thumbnail,
-            'referer': 'http://www.auengine.com/flowplayer/flowplayer.commercial-3.2.14.swf',
+            'http_referer': 'http://www.auengine.com/flowplayer/flowplayer.commercial-3.2.14.swf',
         }

From 7ee7436c6e9769ca9e40a93df1692d5f8da4045c Mon Sep 17 00:00:00 2001
From: Mark Lee <youtube-dl@lazymalevolence.com>
Date: Wed, 26 Mar 2014 10:47:50 -0700
Subject: [PATCH 0009/1105] [common] add method to convert entries with formats
 to formats with parts

---
 youtube_dl/__init__.py         |  1 +
 youtube_dl/extractor/common.py | 24 ++++++++++++++++++++++++
 2 files changed, 25 insertions(+)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 6bbe9d3df..ebd333f79 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -745,6 +745,7 @@ def _real_main(argv=None):
         'playliststart': opts.playliststart,
         'playlistend': opts.playlistend,
         'noplaylist': opts.noplaylist,
+        'joinparts': opts.joinparts,
         'logtostderr': opts.outtmpl == '-',
         'consoletitle': opts.consoletitle,
         'nopart': opts.nopart,
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index facfbf2f5..c47fc58b9 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1,4 +1,5 @@
 import base64
+from collections import defaultdict
 import hashlib
 import json
 import os
@@ -534,6 +535,29 @@ class InfoExtractor(object):
             )
         formats.sort(key=_formats_key)
 
+    def _entry_formats_to_parts(self, entries):
+        '''Transforms entries with formats to formats with parts. Used when joinparts is set.'''
+        ekeys = None
+        fmt_map = defaultdict(dict)
+        fkeys = None
+        for entry in entries:
+            if ekeys is None:
+                ekeys = set(entry.keys()) - set(['formats'])
+            for fmt in entry['formats']:
+                if fkeys is None:
+                    fkeys = set(fmt.keys()) - set(['url'])
+                fid = fmt['format_id']
+                if 'parts' not in fmt_map[fid]:
+                    for k in fkeys:
+                        fmt_map[fid][k] = fmt[k]
+                    fmt_map[fid]['parts'] = []
+                part = dict([(k, entry[k]) for k in ekeys])
+                part['url'] = fmt['url']
+                fmt_map[fid]['parts'].append(part)
+        formats = fmt_map.values()
+        self._sort_formats(formats)
+        return formats
+
 
 class SearchInfoExtractor(InfoExtractor):
     """

From 6edb1a73ee5fac3c3518e04791aca16eedb2dbd3 Mon Sep 17 00:00:00 2001
From: Mark Lee <youtube-dl@lazymalevolence.com>
Date: Wed, 26 Mar 2014 10:48:13 -0700
Subject: [PATCH 0010/1105] [comedycentral] join playlists into one video when
 requested

---
 youtube_dl/extractor/comedycentral.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 483ae5761..775eb8ebf 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -203,9 +203,18 @@ class ComedyCentralShowsIE(InfoExtractor):
                 'description': description,
             })
 
+        playlist_title = show_name + ' ' + title
+        if self._downloader.params.get('joinparts') and len(entries) > 1:
+            return {
+                'id': epTitle,
+                'title': playlist_title,
+                'description': description,
+                'formats': self._entry_formats_to_parts(entries)
+            }
+
         return {
             '_type': 'playlist',
             'entries': entries,
-            'title': show_name + ' ' + title,
+            'title': playlist_title,
             'description': description,
         }

From 839d05db5d08379f57b1b620849da9178734c56c Mon Sep 17 00:00:00 2001
From: Mark Lee <youtube-dl@lazymalevolence.com>
Date: Wed, 26 Mar 2014 10:48:49 -0700
Subject: [PATCH 0011/1105] [mtv] join playlists into one video when requested

---
 youtube_dl/extractor/mtv.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index d75241d3f..ca42491c6 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -156,7 +156,22 @@ class MTVServicesInfoExtractor(InfoExtractor):
             mgid = self._search_regex(
                 [r'data-mgid="(.*?)"', r'swfobject.embedSWF\(".*?(mgid:.*?)"'],
                 webpage, u'mgid')
-        return self._get_videos_info(mgid)
+        videos_info = self._get_videos_info(mgid)
+        if self._downloader.params.get('joinparts'):
+            show_name = self._html_search_regex(
+                r'<h2.*?class="[^"]*show_name[^"]*".*?>(.*?)</h2>',
+                webpage, 'show name')
+            video_name = self._og_search_title(webpage)
+            description = self._og_search_description(webpage)
+            thumbnail = self._og_search_thumbnail(webpage)
+            return {
+                'id': title,
+                'title': '{0} - {1}'.format(show_name, video_name),
+                'description': description,
+                'thumbnail': thumbnail,
+                'formats': self._entry_formats_to_parts(videos_info),
+            }
+        return videos_info
 
 
 class MTVIE(MTVServicesInfoExtractor):

From d7b31b5b35d04dc33e6ff00ccbf083724f6eefc6 Mon Sep 17 00:00:00 2001
From: Mark Lee <youtube-dl@lazymalevolence.com>
Date: Tue, 15 Apr 2014 12:39:00 -0700
Subject: [PATCH 0012/1105] Check parts properly inside a format key

---
 youtube_dl/YoutubeDL.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 95b7a9f31..b939314ea 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -739,8 +739,13 @@ class YoutubeDL(object):
 
         # We check that all the formats have the format and format_id fields
         for i, format in enumerate(formats):
-            if 'url' not in format:
-                raise ExtractorError('Missing "url" key in result (index %d)' % i)
+            if 'parts' in format:
+                for j, part in enumerate(format['parts']):
+                    if 'url' not in part:
+                        raise ExtractorError('Missing "url" key in result (index %d, part %d)' % (i, j))
+            else:
+                if 'url' not in format:
+                    raise ExtractorError('Missing "url" key in result (index %d)' % i)
 
             if format.get('format_id') is None:
                 format['format_id'] = compat_str(i)

From 3f6ea3d07baf995e33bac9f5e6d933b751cbe043 Mon Sep 17 00:00:00 2001
From: Mark Lee <youtube-dl@lazymalevolence.com>
Date: Tue, 22 Jul 2014 20:50:49 -0700
Subject: [PATCH 0013/1105] [mtv] Update show name regex

---
 youtube_dl/extractor/mtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 0f53cd79e..df9b259f7 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -167,7 +167,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         videos_info = self._get_videos_info(mgid)
         if self._downloader.params.get('joinparts'):
             show_name = self._html_search_regex(
-                r'<h2.*?class="[^"]*show_name[^"]*".*?>(.*?)</h2>',
+                r'<span.*?class="[^"]*title[^"]*".*?>(.*?)</span>',
                 webpage, 'show name')
             video_name = self._og_search_title(webpage)
             description = self._og_search_description(webpage)

From 16d46b732f49a9183ce6659e4a7ba2183c1fdab8 Mon Sep 17 00:00:00 2001
From: Mark Lee <youtube-dl@lazymalevolence.com>
Date: Mon, 1 Dec 2014 19:10:45 -0800
Subject: [PATCH 0014/1105] Fix merge error

---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 2f3246d89..45bc2121c 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -86,7 +86,7 @@ class FFmpegPostProcessor(PostProcessor):
         files_cmd = []
         for path in input_paths:
             files_cmd.extend(['-i', encodeFilename(path, True)])
-        cmd = ([self._get_executable(), '-y'] +
+        cmd = ([self._executable, '-y'] +
                self._encode_opts(input_opts) + files_cmd + self._encode_opts(opts) +
                [encodeFilename(self._ffmpeg_filename_argument(out_path), True)])
 

From 3a7fd2035e4caa03b4e910fde65e4c10994cf578 Mon Sep 17 00:00:00 2001
From: Mark Lee <youtube-dl@lazymalevolence.com>
Date: Thu, 11 Dec 2014 22:38:46 -0800
Subject: [PATCH 0015/1105] Fix MTV + join-parts

---
 youtube_dl/extractor/mtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 302391b6b..226f1d6f7 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -178,7 +178,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
                 'title': '{0} - {1}'.format(show_name, video_name),
                 'description': description,
                 'thumbnail': thumbnail,
-                'formats': self._entry_formats_to_parts(videos_info),
+                'formats': self._entry_formats_to_parts(videos_info['entries']),
             }
         return videos_info
 

From 42a2155c1d7ad33d69c6481825d82828d970a413 Mon Sep 17 00:00:00 2001
From: Mark Lee <youtube-dl@lazymalevolence.com>
Date: Fri, 20 Feb 2015 21:34:53 -0800
Subject: [PATCH 0016/1105] [mtv] fix show name regex

---
 youtube_dl/extractor/mtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 226f1d6f7..f953a622e 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -168,7 +168,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         videos_info = self._get_videos_info(mgid)
         if self._downloader.params.get('joinparts'):
             show_name = self._html_search_regex(
-                r'<span.*?class="[^"]*title[^"]*".*?>(.*?)</span>',
+                r'<h1.*?class="[^"]*title[^"]*".*?>(.*?)</h1>',
                 webpage, 'show name')
             video_name = self._og_search_title(webpage)
             description = self._og_search_description(webpage)

From e664c6a8e6143e5c14d09a342a5800d32b3170b9 Mon Sep 17 00:00:00 2001
From: Mark Lee <youtube-dl@lazymalevolence.com>
Date: Fri, 20 Feb 2015 22:17:18 -0800
Subject: [PATCH 0017/1105] Reorder for better merging

---
 youtube_dl/postprocessor/ffmpeg.py | 48 +++++++++++++++---------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 3a062fb00..e07f7cc97 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -583,6 +583,30 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
         return True, info
 
 
+class FFmpegJoinVideosPP(FFmpegPostProcessor):
+    def run(self, information):
+        filename = information['filepath']
+        parts = information.get('parts')
+        if parts is None or len(parts) == 1:
+            return (True, information)
+        parts_files = [build_part_filename(filename, i) for (i, _) in enumerate(parts)]
+        files_file = u'%s.videos' % filename
+        with io.open(encodeFilename(files_file), 'w', encoding='utf-8') as f:
+            for video in parts_files:
+                f.write(u'file \'%s\'\n' % video)
+        self._downloader.to_screen(u'[ffmpeg] Joining video parts, destination: %s' % filename)
+        try:
+            self.run_ffmpeg(files_file, filename, ['-c', 'copy'], ['-f', 'concat'])
+        except FFmpegPostProcessorError:
+            return False, information
+        os.remove(encodeFilename(files_file))
+        # We have to manually remove the parts if requested
+        if not self._downloader.params.get('keepvideo', False):
+            for part_file in parts_files:
+                os.remove(encodeFilename(part_file))
+        return (True, information)
+
+
 class FFmpegMergerPP(FFmpegPostProcessor):
     def run(self, info):
         filename = info['filepath']
@@ -642,27 +666,3 @@ class FFmpegFixupM4aPP(FFmpegPostProcessor):
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
         return True, info
-
-
-class FFmpegJoinVideosPP(FFmpegPostProcessor):
-    def run(self, information):
-        filename = information['filepath']
-        parts = information.get('parts')
-        if parts is None or len(parts) == 1:
-            return (True, information)
-        parts_files = [build_part_filename(filename, i) for (i, _) in enumerate(parts)]
-        files_file = u'%s.videos' % filename
-        with io.open(encodeFilename(files_file), 'w', encoding='utf-8') as f:
-                    for video in parts_files:
-                        f.write(u'file \'%s\'\n' % video)
-        self._downloader.to_screen(u'[ffmpeg] Joining video parts, destination: %s' % filename)
-        try:
-            self.run_ffmpeg(files_file, filename, ['-c', 'copy'], ['-f', 'concat'])
-        except FFmpegPostProcessorError:
-            return False, information
-        os.remove(encodeFilename(files_file))
-        # We have to manually remove the parts if requested
-        if not self._downloader.params.get('keepvideo', False):
-            for part_file in parts_files:
-                os.remove(encodeFilename(part_file))
-        return (True, information)

From fe07a74fcbf7ce58fe48c5d648ce9bfe59956a80 Mon Sep 17 00:00:00 2001
From: Mark Lee <youtube-dl@lazymalevolence.com>
Date: Fri, 20 Feb 2015 22:17:40 -0800
Subject: [PATCH 0018/1105] Fix merge

---
 youtube_dl/YoutubeDL.py | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5f855c6f9..15eca7e56 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1300,10 +1300,22 @@ class YoutubeDL(object):
                                             ' The formats won\'t be merged')
                     else:
                         postprocessors = [merger]
+                    for f in info_dict['requested_formats']:
+                        new_info = dict(info_dict)
+                        new_info.update(f)
+                        fname = self.prepare_filename(new_info)
+                        fname = prepend_extension(fname, 'f%s' % f['format_id'])
+                        downloaded.append(fname)
+                        partial_success = dl(fname, new_info)
+                        success = success and partial_success
+                    info_dict['__postprocessors'] = postprocessors
+                    info_dict['__files_to_merge'] = downloaded
+                else:
                     parts = info_dict.get('parts', [])
                     if not parts:
                         success = dl(filename, info_dict)
                     elif len(parts) == 1:
+                        # Just a single file
                         info_dict.update(parts[0])
                         success = dl(filename, info_dict)
                     else:
@@ -1320,19 +1332,6 @@ class YoutubeDL(object):
                                 part_filename = build_part_filename(filename, i)
                                 parts_success.append(dl(part_filename, part_info))
                             success = all(parts_success)
-                    for f in info_dict['requested_formats']:
-                        new_info = dict(info_dict)
-                        new_info.update(f)
-                        fname = self.prepare_filename(new_info)
-                        fname = prepend_extension(fname, 'f%s' % f['format_id'])
-                        downloaded.append(fname)
-                        partial_success = dl(fname, new_info)
-                        success = success and partial_success
-                    info_dict['__postprocessors'] = postprocessors
-                    info_dict['__files_to_merge'] = downloaded
-                else:
-                    # Just a single file
-                    success = dl(filename, info_dict)
             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
                 self.report_error('unable to download video data: %s' % str(err))
                 return

From 15e4f0ff84108707d90b37666ed325b5c059cfc1 Mon Sep 17 00:00:00 2001
From: Mark Lee <youtube-dl@lazymalevolence.com>
Date: Tue, 24 Feb 2015 18:44:04 -0800
Subject: [PATCH 0019/1105] [adultswim] add joinparts support

---
 youtube_dl/extractor/adultswim.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 34b8b0115..ce1a67f3e 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -160,8 +160,7 @@ class AdultSwimIE(InfoExtractor):
                 'description': episode_description
             })
 
-        return {
-            '_type': 'playlist',
+        data = {
             'id': episode_id,
             'display_id': episode_path,
             'entries': entries,
@@ -169,3 +168,13 @@ class AdultSwimIE(InfoExtractor):
             'description': episode_description,
             'duration': episode_duration
         }
+
+        if self._downloader.params.get('joinparts') and len(entries) > 1:
+            # convert entries => formats into formats => parts
+            self.to_screen('Found {} segments to join'.format(len(entries)))
+            data['formats'] = self._entry_formats_to_parts(entries)
+        else:
+            data['_type'] = 'playlist'
+            data['entries'] = entries
+
+        return data

From be3544d25155b4825b4913e708803b20b7fe6d88 Mon Sep 17 00:00:00 2001
From: Mark Lee <youtube-dl@lazymalevolence.com>
Date: Tue, 24 Feb 2015 18:44:25 -0800
Subject: [PATCH 0020/1105] Continue to fix merge

---
 youtube_dl/YoutubeDL.py            | 14 +++++++++-----
 youtube_dl/__init__.py             |  2 ++
 youtube_dl/postprocessor/ffmpeg.py |  2 +-
 3 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 15eca7e56..8b4a2ff5f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1031,6 +1031,9 @@ class YoutubeDL(object):
                 for j, part in enumerate(format['parts']):
                     if 'url' not in part:
                         raise ExtractorError('Missing "url" key in result (index %d, part %d)' % (i, j))
+                    full_format_info = info_dict.copy()
+                    full_format_info.update(part)
+                    part['http_headers'] = self._calc_headers(full_format_info)
             else:
                 if 'url' not in format:
                     raise ExtractorError('Missing "url" key in result (index %d)' % i)
@@ -1046,11 +1049,12 @@ class YoutubeDL(object):
             # Automatically determine file extension if missing
             if 'ext' not in format:
                 format['ext'] = determine_ext(format['url']).lower()
-            # Add HTTP headers, so that external programs can use them from the
-            # json output
-            full_format_info = info_dict.copy()
-            full_format_info.update(format)
-            format['http_headers'] = self._calc_headers(full_format_info)
+            if 'parts' not in format:
+                # Add HTTP headers, so that external programs can use them from the
+                # json output
+                full_format_info = info_dict.copy()
+                full_format_info.update(format)
+                format['http_headers'] = self._calc_headers(full_format_info)
 
         format_limit = self.params.get('format_limit', None)
         if format_limit:
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 09de067e9..5182dac15 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -234,6 +234,8 @@ def _real_main(argv=None):
         if not opts.addmetadata:
             postprocessors.append({'key': 'FFmpegAudioFix'})
         postprocessors.append({'key': 'AtomicParsley'})
+    if opts.joinparts:
+        postprocessors.append({'key': 'FFmpegJoinVideos'})
     # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
     # So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
     if opts.exec_cmd:
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index e07f7cc97..0087786ff 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -134,7 +134,7 @@ class FFmpegPostProcessor(PostProcessor):
         files_cmd = []
         for path in input_paths:
             files_cmd.extend([encodeArgument('-i'), encodeFilename(path, True)])
-        cmd = ([encodeFilename(self._executable, True), encodeArgument('-y')] +
+        cmd = ([encodeFilename(self.executable, True), encodeArgument('-y')] +
                [encodeArgument(o) for o in input_opts] + files_cmd + [encodeArgument(o) for o in opts] +
                [encodeFilename(self._ffmpeg_filename_argument(out_path), True)])
 

From 9bcd7ddecf84074d6fb64f29a96d09ae9562c41a Mon Sep 17 00:00:00 2001
From: Mark Lee <youtube-dl@lazymalevolence.com>
Date: Sun, 1 Mar 2015 10:49:03 -0800
Subject: [PATCH 0021/1105] Remove unicode markers (inconsistent with rest of
 file)

---
 youtube_dl/extractor/tudou.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index e85bb4259..a8fc77901 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -11,14 +11,14 @@ from .common import InfoExtractor
 class TudouIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:listplay|programs(?:/view)?|albumplay)/.*?/(?P<id>[^/?#]+?)(?:\.html)?/?(?:$|[?#])'
     _TESTS = [{
-        u'url': u'http://www.tudou.com/listplay/zzdE77v6Mmo/2xN2duXMxmw.html',
-        u'file': u'2xN2duXMxmw.f4v',
-        u'md5': u'140a49ed444bd22f93330985d8475fcb',
-        u'info_dict': {
-            u'id': u'159448201',
-            u'ext': u'f4v',
-            u'title': u'卡马乔国足开大脚长传冲吊集锦',
-            u'thumbnail': u're:^https?://.*\.jpg$',
+        'url': 'http://www.tudou.com/listplay/zzdE77v6Mmo/2xN2duXMxmw.html',
+        'file': '2xN2duXMxmw.f4v',
+        'md5': '140a49ed444bd22f93330985d8475fcb',
+        'info_dict': {
+            'id': '159448201',
+            'ext': 'f4v',
+            'title': '卡马乔国足开大脚长传冲吊集锦',
+            'thumbnail': 're:^https?://.*\.jpg$',
         }
     }, {
         'url': 'http://www.tudou.com/programs/view/ajX3gyhL0pc/',

From 7818cb23c9131fe647db31eab56fb214f811f013 Mon Sep 17 00:00:00 2001
From: Mark Lee <youtube-dl@lazymalevolence.com>
Date: Sun, 15 Mar 2015 21:20:44 -0700
Subject: [PATCH 0022/1105] [multipart_video] fix handling video parts with
 single quotes in the filename

---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index aeb2594dc..47146c71a 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -595,7 +595,7 @@ class FFmpegJoinVideosPP(FFmpegPostProcessor):
         files_file = u'%s.videos' % filename
         with io.open(encodeFilename(files_file), 'w', encoding='utf-8') as f:
             for video in parts_files:
-                f.write(u'file \'%s\'\n' % video)
+                f.write(u'file \'%s\'\n' % video.replace("'", r"'\''"))
         self._downloader.to_screen(u'[ffmpeg] Joining video parts, destination: %s' % filename)
         try:
             self.run_ffmpeg(files_file, filename, ['-c', 'copy'], ['-f', 'concat'])

From 984e4d487520bd2a860b31b3165416c879b28096 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 24 Jun 2015 01:13:23 +0100
Subject: [PATCH 0023/1105] [googledrive] Add new extractor

---
 youtube_dl/extractor/__init__.py    |   1 +
 youtube_dl/extractor/googledrive.py | 106 ++++++++++++++++++++++++++++
 2 files changed, 107 insertions(+)
 create mode 100644 youtube_dl/extractor/googledrive.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3cfa804ec..6655d7eb5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -209,6 +209,7 @@ from .globo import GloboIE
 from .godtube import GodTubeIE
 from .goldenmoustache import GoldenMoustacheIE
 from .golem import GolemIE
+from .googledrive import GoogleDriveIE
 from .googleplus import GooglePlusIE
 from .googlesearch import GoogleSearchIE
 from .gorillavid import GorillaVidIE
diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
new file mode 100644
index 000000000..8c611fa47
--- /dev/null
+++ b/youtube_dl/extractor/googledrive.py
@@ -0,0 +1,106 @@
+from .common import InfoExtractor
+from ..utils import RegexNotFoundError
+
+class GoogleDriveIE(InfoExtractor):
+    _VALID_URL = r'(?:https?://)?(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/))(?P<id>.+?)(?:&|/|$)'
+    _TEST = {
+        'url': 'https://drive.google.com/file/d/0BzpExh0WzJF0NlR5WUlxdEVsY0U/edit?pli=1',
+        'info_dict': {
+            'id': '0BzpExh0WzJF0NlR5WUlxdEVsY0U',
+            'ext': 'mp4',
+            'title': '[AHSH] Fairy Tail S2 - 01 [720p].mp4',
+        }
+    }
+    _formats = {
+        '5': {'ext': 'flv'},
+        '6': {'ext': 'flv'},
+        '13': {'ext': '3gp'},
+        '17': {'ext': '3gp'},
+        '18': {'ext': 'mp4'},
+        '22': {'ext': 'mp4'},
+        '34': {'ext': 'flv'},
+        '35': {'ext': 'flv'},
+        '36': {'ext': '3gp'},
+        '37': {'ext': 'mp4'},
+        '38': {'ext': 'mp4'},
+        '43': {'ext': 'webm'},
+        '44': {'ext': 'webm'},
+        '45': {'ext': 'webm'},
+        '46': {'ext': 'webm'},
+        '59': {'ext': 'mp4'}
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(
+            'http://docs.google.com/file/d/'+video_id, video_id, encoding='unicode_escape'
+        )
+        try:
+            title = self._html_search_regex(
+                r'"title","(?P<title>.*?)"',
+                webpage,
+                'title',
+                group='title'
+            )
+            fmt_stream_map = self._html_search_regex(
+                r'"fmt_stream_map","(?P<fmt_stream_map>.*?)"',
+                webpage,
+                'fmt_stream_map',
+                group='fmt_stream_map'
+            )
+            fmt_list = self._html_search_regex(
+                r'"fmt_list","(?P<fmt_list>.*?)"',
+                webpage,
+                'fmt_list',
+                group='fmt_list'
+            )
+#			timestamp = self._html_search_regex(
+#				r'"timestamp","(?P<timestamp>.*?)"',
+#				webpage,
+#				'timestamp',
+#				group='timestamp'
+#			)
+            length_seconds = self._html_search_regex(
+                r'"length_seconds","(?P<length_seconds>.*?)"',
+                webpage,
+                'length_seconds',
+                group='length_seconds'
+            )
+        except RegexNotFoundError:
+            try:
+                reason = self._html_search_regex(
+                    r'"reason","(?P<reason>.*?)"',
+                    webpage,
+                    'reason',
+                    group='reason'
+                )
+                self.report_warning(reason)
+                return
+            except RegexNotFoundError:
+                self.report_warning('not a video')
+                return
+
+        fmt_stream_map = fmt_stream_map.split(',')
+        fmt_list = fmt_list.split(',')
+        formats = []
+        for i in range(len(fmt_stream_map)):
+            fmt_id, fmt_url = fmt_stream_map[i].split('|')
+            resolution = fmt_list[i].split('/')[1]
+            width, height = resolution.split('x')
+            formats.append({
+                'url': fmt_url,
+                'format_id': fmt_id,
+                'resolution': resolution,
+                'width': int(width),
+                'height': int(height),
+                'ext': self._formats[fmt_id]['ext']
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+#           'timestamp': int(timestamp),
+            'duration': int(length_seconds),
+            'formats': formats
+        }

From f120a7ab5e9c560a8114f9662e2f213243a945b0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 24 Jun 2015 14:56:19 +0100
Subject: [PATCH 0024/1105] change the _TEST info

---
 youtube_dl/extractor/googledrive.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index 8c611fa47..e3d5c3418 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -4,11 +4,11 @@ from ..utils import RegexNotFoundError
 class GoogleDriveIE(InfoExtractor):
     _VALID_URL = r'(?:https?://)?(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/))(?P<id>.+?)(?:&|/|$)'
     _TEST = {
-        'url': 'https://drive.google.com/file/d/0BzpExh0WzJF0NlR5WUlxdEVsY0U/edit?pli=1',
+        'url': 'https://drive.google.com/file/d/0ByeS4oOUV-49Zzh4R1J6R09zazQ/edit?pli=1',
         'info_dict': {
-            'id': '0BzpExh0WzJF0NlR5WUlxdEVsY0U',
+            'id': '0ByeS4oOUV-49Zzh4R1J6R09zazQ',
             'ext': 'mp4',
-            'title': '[AHSH] Fairy Tail S2 - 01 [720p].mp4',
+            'title': 'Big Buck Bunny.mp4',
         }
     }
     _formats = {

From 3e5f3df1729846a33631dd38a887cd1d81a727c1 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 29 Jun 2015 07:53:21 +0100
Subject: [PATCH 0025/1105] move the embed to a separate class

---
 youtube_dl/extractor/googledrive.py | 31 ++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index e3d5c3418..ac891b275 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -1,8 +1,37 @@
+import re
+
 from .common import InfoExtractor
 from ..utils import RegexNotFoundError
 
+class GoogleDriveEmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9-]{28})(?:/preview)'
+    _TEST = {
+        'url': 'https://docs.google.com/file/d/0B8KB9DRosYGKMXNoeWxqa3JYclE/preview',
+        'info_dict': {
+            'id': '0B8KB9DRosYGKMXNoeWxqa3JYclE',
+            'ext': 'mp4',
+            'title': 'Jimmy Fallon Sings Since You\'ve Been Gone.wmv',
+        }
+    }
+
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<iframe src="https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9-]{28})(?:/preview)',
+            webpage)
+        if mobj:
+            return 'https://drive.google.com/file/d/%s' % mobj.group('id')
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        return {
+            '_type': 'url',
+            'ie-key': 'GoogleDrive',
+            'url': 'https://drive.google.com/file/d/%s' % video_id
+        }
+
 class GoogleDriveIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://)?(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/))(?P<id>.+?)(?:&|/|$)'
+    _VALID_URL = r'https?://(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/)(?P<id>[a-zA-Z0-9-]{28})'
     _TEST = {
         'url': 'https://drive.google.com/file/d/0ByeS4oOUV-49Zzh4R1J6R09zazQ/edit?pli=1',
         'info_dict': {

From 2d651a2d02885cddf1752b45497e9113d3a3d403 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 29 Jun 2015 07:55:44 +0100
Subject: [PATCH 0026/1105] import google drive embed class

---
 youtube_dl/extractor/__init__.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6655d7eb5..02e18a0da 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -209,7 +209,10 @@ from .globo import GloboIE
 from .godtube import GodTubeIE
 from .goldenmoustache import GoldenMoustacheIE
 from .golem import GolemIE
-from .googledrive import GoogleDriveIE
+from .googledrive import (
+    GoogleDriveEmbedIE,
+    GoogleDriveIE,
+)
 from .googleplus import GooglePlusIE
 from .googlesearch import GoogleSearchIE
 from .gorillavid import GorillaVidIE

From 653789afc72d1a225b971541fb633dd768d58942 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 29 Jun 2015 08:01:30 +0100
Subject: [PATCH 0027/1105] add google drive embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6d2efb22e..3f7b094db 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -48,6 +48,7 @@ from .vimeo import VimeoIE
 from .dailymotion import DailymotionCloudIE
 from .onionstudios import OnionStudiosIE
 from .snagfilms import SnagFilmsEmbedIE
+from .googledrive import GoogleDriveEmbedIE
 
 
 class GenericIE(InfoExtractor):
@@ -1599,6 +1600,11 @@ class GenericIE(InfoExtractor):
         if nbc_sports_url:
             return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
 
+        # Look for Google Drive embeds
+        google_drive_url = GoogleDriveEmbedIE._extract_url(webpage)
+        if google_drive_url:
+            return self.url_result(google_drive_url, 'GoogleDrive')
+
         # Look for UDN embeds
         mobj = re.search(
             r'<iframe[^>]+src="(?P<url>%s)"' % UDNEmbedIE._VALID_URL, webpage)

From 3b3d531965f0f36c20f5fa8557481c144170653f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 17 Jul 2015 14:17:19 +0100
Subject: [PATCH 0028/1105] fix embed regex

---
 youtube_dl/extractor/googledrive.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index ac891b275..c82c9037f 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -4,7 +4,7 @@ from .common import InfoExtractor
 from ..utils import RegexNotFoundError
 
 class GoogleDriveEmbedIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9-]{28})(?:/preview)'
+    _VALID_URL = r'https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9_-]{28})'
     _TEST = {
         'url': 'https://docs.google.com/file/d/0B8KB9DRosYGKMXNoeWxqa3JYclE/preview',
         'info_dict': {
@@ -17,7 +17,7 @@ class GoogleDriveEmbedIE(InfoExtractor):
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<iframe src="https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9-]{28})(?:/preview)',
+            r'<iframe src="https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9_-]{28})',
             webpage)
         if mobj:
             return 'https://drive.google.com/file/d/%s' % mobj.group('id')
@@ -31,7 +31,7 @@ class GoogleDriveEmbedIE(InfoExtractor):
         }
 
 class GoogleDriveIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/)(?P<id>[a-zA-Z0-9-]{28})'
+    _VALID_URL = r'https?://(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/)(?P<id>[a-zA-Z0-9_-]{28})'
     _TEST = {
         'url': 'https://drive.google.com/file/d/0ByeS4oOUV-49Zzh4R1J6R09zazQ/edit?pli=1',
         'info_dict': {

From d1cc05e17eccccb7ee6473574c6a4f887104baeb Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 17 Jul 2015 14:37:21 +0100
Subject: [PATCH 0029/1105] remove unnecessary regex group names

---
 youtube_dl/extractor/googledrive.py | 32 ++++++++++++-----------------
 1 file changed, 13 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index c82c9037f..6d9bcfefd 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -62,46 +62,40 @@ class GoogleDriveIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(
-            'http://docs.google.com/file/d/'+video_id, video_id, encoding='unicode_escape'
+            'http://docs.google.com/file/d/' + video_id, video_id, encoding='unicode_escape'
         )
         try:
             title = self._html_search_regex(
-                r'"title","(?P<title>.*?)"',
+                r'"title"\s+,\s+"[^"]+',
                 webpage,
-                'title',
-                group='title'
+                'title'
             )
             fmt_stream_map = self._html_search_regex(
-                r'"fmt_stream_map","(?P<fmt_stream_map>.*?)"',
+                r'"fmt_stream_map"\s+,\s+"[^"]+',
                 webpage,
-                'fmt_stream_map',
-                group='fmt_stream_map'
+                'fmt_stream_map'
             )
             fmt_list = self._html_search_regex(
-                r'"fmt_list","(?P<fmt_list>.*?)"',
+                r'"fmt_list"\s+,\s+"[^"]+',
                 webpage,
-                'fmt_list',
-                group='fmt_list'
+                'fmt_list'
             )
 #			timestamp = self._html_search_regex(
-#				r'"timestamp","(?P<timestamp>.*?)"',
+#				r'"timestamp"\s+,\s+"[^"]+',
 #				webpage,
-#				'timestamp',
-#				group='timestamp'
+#				'timestamp'
 #			)
             length_seconds = self._html_search_regex(
-                r'"length_seconds","(?P<length_seconds>.*?)"',
+                r'"length_seconds"\s+,\s+"[^"]+',
                 webpage,
-                'length_seconds',
-                group='length_seconds'
+                'length_seconds'
             )
         except RegexNotFoundError:
             try:
                 reason = self._html_search_regex(
-                    r'"reason","(?P<reason>.*?)"',
+                    r'"reason","[^"]+',
                     webpage,
-                    'reason',
-                    group='reason'
+                    'reason'
                 )
                 self.report_warning(reason)
                 return

From 36dbca87848fc5698d3e0b89380c7bcec741ceaf Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 17 Jul 2015 14:52:01 +0100
Subject: [PATCH 0030/1105] fix recursive error

---
 youtube_dl/extractor/googledrive.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index 6d9bcfefd..a3d9b4450 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -26,7 +26,7 @@ class GoogleDriveEmbedIE(InfoExtractor):
         video_id = self._match_id(url)
         return {
             '_type': 'url',
-            'ie-key': 'GoogleDrive',
+            'ie_key': 'GoogleDrive',
             'url': 'https://drive.google.com/file/d/%s' % video_id
         }
 
@@ -66,34 +66,34 @@ class GoogleDriveIE(InfoExtractor):
         )
         try:
             title = self._html_search_regex(
-                r'"title"\s+,\s+"[^"]+',
+                r'"title"\s*,\s*"([^"]+)',
                 webpage,
                 'title'
             )
             fmt_stream_map = self._html_search_regex(
-                r'"fmt_stream_map"\s+,\s+"[^"]+',
+                r'"fmt_stream_map"\s*,\s*"([^"]+)',
                 webpage,
                 'fmt_stream_map'
             )
             fmt_list = self._html_search_regex(
-                r'"fmt_list"\s+,\s+"[^"]+',
+                r'"fmt_list"\s*,\s*"([^"]+)',
                 webpage,
                 'fmt_list'
             )
 #			timestamp = self._html_search_regex(
-#				r'"timestamp"\s+,\s+"[^"]+',
+#				r'"timestamp"\s*,\s*"([^"]+)',
 #				webpage,
 #				'timestamp'
 #			)
             length_seconds = self._html_search_regex(
-                r'"length_seconds"\s+,\s+"[^"]+',
+                r'"length_seconds"\s*,\s*"([^"]+)',
                 webpage,
                 'length_seconds'
             )
         except RegexNotFoundError:
             try:
                 reason = self._html_search_regex(
-                    r'"reason","[^"]+',
+                    r'"reason","([^"]+)',
                     webpage,
                     'reason'
                 )

From 8e92d21ebf6f17e14c9e916f22e49f27529556af Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 18 Jul 2015 23:31:14 +0100
Subject: [PATCH 0031/1105] [googledrive] raise ExtractorError instead of
 warning

---
 youtube_dl/extractor/googledrive.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index a3d9b4450..7bc7b7a0d 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -1,7 +1,10 @@
 import re
 
 from .common import InfoExtractor
-from ..utils import RegexNotFoundError
+from ..utils import (
+    RegexNotFoundError,
+    ExtractorError,
+)
 
 class GoogleDriveEmbedIE(InfoExtractor):
     _VALID_URL = r'https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9_-]{28})'
@@ -97,10 +100,10 @@ class GoogleDriveIE(InfoExtractor):
                     webpage,
                     'reason'
                 )
-                self.report_warning(reason)
+                raise ExtractorError(reason)
                 return
             except RegexNotFoundError:
-                self.report_warning('not a video')
+                raise ExtractorError('not a video')
                 return
 
         fmt_stream_map = fmt_stream_map.split(',')

From f18e093d94d40ccda8be8e3c3a96a7f6d977f564 Mon Sep 17 00:00:00 2001
From: Mark Lee <youtube-dl@lazymalevolence.com>
Date: Mon, 10 Aug 2015 21:15:51 -0700
Subject: [PATCH 0032/1105] Update join video postprocessor due to API change

---
 youtube_dl/postprocessor/ffmpeg.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 7a38c4477..52aa2204b 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -403,7 +403,7 @@ class FFmpegJoinVideosPP(FFmpegPostProcessor):
         filename = information['filepath']
         parts = information.get('parts')
         if parts is None or len(parts) == 1:
-            return (True, information)
+            return [], information
         parts_files = [build_part_filename(filename, i) for (i, _) in enumerate(parts)]
         files_file = u'%s.videos' % filename
         with io.open(encodeFilename(files_file), 'w', encoding='utf-8') as f:
@@ -413,13 +413,11 @@ class FFmpegJoinVideosPP(FFmpegPostProcessor):
         try:
             self.run_ffmpeg(files_file, filename, ['-c', 'copy'], ['-f', 'concat'])
         except FFmpegPostProcessorError:
-            return False, information
+            return [filename], information
         os.remove(encodeFilename(files_file))
-        # We have to manually remove the parts if requested
         if not self._downloader.params.get('keepvideo', False):
-            for part_file in parts_files:
-                os.remove(encodeFilename(part_file))
-        return (True, information)
+            return parts_files, information
+        return [], information
 
 
 class FFmpegMergerPP(FFmpegPostProcessor):

From 9f4921bfa0ce3a48d2f93b4946f361116cfde5e9 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 3 Sep 2015 00:29:53 +0100
Subject: [PATCH 0033/1105] [dcn] add show extraction and support for other
 types of urls

---
 youtube_dl/extractor/__init__.py |  6 ++-
 youtube_dl/extractor/dcn.py      | 81 ++++++++++++++++++++++++++++++--
 2 files changed, 83 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 39b05ce8f..d4a3e8ab0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -118,7 +118,11 @@ from .dailymotion import (
 )
 from .daum import DaumIE
 from .dbtv import DBTVIE
-from .dcn import DCNIE
+from .dcn import (
+    DCNGeneralIE,
+    DCNVideoIE,
+    DCNShowIE,
+)
 from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
 from .dfb import DFBIE
diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 82261e25c..352d35c7a 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
@@ -12,10 +14,33 @@ from ..utils import (
 )
 
 
-class DCNIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/.+|show/\d+/.+?)/(?P<id>\d+)'
+class DCNGeneralIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?show/(?P<show_id>\d+)/[^/]+(?:/(?P<video_id>\d+)/(?P<season_id>\d+))?'
+
+    def _real_extract(self, url):
+        show_id, video_id, season_id = re.match(self._VALID_URL, url).groups()
+        url = ''
+        ie_key = ''
+        if video_id and int(video_id) > 0:
+            url = 'http://www.dcndigital.ae/#/media/%s' % video_id
+            ie_key = 'DCNVideo'
+        else:
+            ie_key = 'DCNShow'
+            if season_id and int(season_id) > 0:
+                url = 'http://www.dcndigital.ae/#/program/season/%s' % season_id
+            else:
+                url = 'http://www.dcndigital.ae/#/program/%s' % show_id
+        return {
+            'url': url,
+            '_type': 'url',
+            'ie_key': ie_key
+        }
+
+
+class DCNVideoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/[^/]+|media)/(?P<id>\d+)'
     _TEST = {
-        'url': 'http://www.dcndigital.ae/#/show/199074/%D8%B1%D8%AD%D9%84%D8%A9-%D8%A7%D9%84%D8%B9%D9%85%D8%B1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/17375/6887',
+        'url': 'http://www.dcndigital.ae/#/video/%D8%B1%D8%AD%D9%84%D8%A9-%D8%A7%D9%84%D8%B9%D9%85%D8%B1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/17375',
         'info_dict':
         {
             'id': '17375',
@@ -82,3 +107,53 @@ class DCNIE(InfoExtractor):
             'timestamp': timestamp,
             'formats': formats,
         }
+
+
+class DCNShowIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?program/(?:(?P<show_id>\d+)|season/(?P<season_id>\d+))'
+    _TEST = {
+        'url': 'http://dcndigital.ae/#/program/205024/%D9%85%D8%AD%D8%A7%D8%B6%D8%B1%D8%A7%D8%AA-%D8%A7%D9%84%D8%B4%D9%8A%D8%AE-%D8%A7%D9%84%D8%B4%D8%B9%D8%B1%D8%A7%D9%88%D9%8A',
+        'info_dict':
+        {
+            'id': '205024',
+            'title': 'محاضرات الشيخ الشعراوي',
+            'description': '',
+        },
+        'playlist_mincount': 27,
+    }
+
+    def _real_extract(self, url):
+        show_id, season_id = re.match(self._VALID_URL, url).groups()
+        data = {}
+        if season_id:
+            request = compat_urllib_request.Request(
+                'http://admin.mangomolo.com/analytics/index.php/plus/season_info?id=%s' % season_id,
+                headers={'Origin': 'http://www.dcndigital.ae'})
+            season = self._download_json(request, season_id)
+            show_id = season['id']
+            data['season'] = season_id
+        data['show_id'] = show_id
+        request = compat_urllib_request.Request(
+            'http://admin.mangomolo.com/analytics/index.php/plus/show',
+            compat_urllib_parse.urlencode(data),
+            {
+                'Origin': 'http://www.dcndigital.ae',
+                'Content-Type': 'application/x-www-form-urlencoded'
+            })
+        show = self._download_json(request, show_id)
+        title = show['cat'].get('title_en') or show['cat']['title_ar']
+        description = show['cat'].get('description_en') or show['cat'].get('description_ar')
+        entries = []
+        for video in show['videos']:
+            entries.append({
+                'url': 'http://www.dcndigital.ae/#/media/%s' % video['id'],
+                '_type': 'url',
+                'ie_key': 'DCNVideo',
+            })
+        return {
+            'id': show_id,
+            'title': title,
+            'description': description,
+            'entries': entries,
+            '_type': 'playlist',
+        }

From b477da2094db30a232f67edf3b342dc460aa14d4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 3 Sep 2015 16:59:10 +0100
Subject: [PATCH 0034/1105] correct the extractor name and id and remove
 unnecessary request

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/dcn.py      | 28 ++++++++++++++++++----------
 2 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d4a3e8ab0..4e41d9bf9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -121,7 +121,7 @@ from .dbtv import DBTVIE
 from .dcn import (
     DCNGeneralIE,
     DCNVideoIE,
-    DCNShowIE,
+    DCNSeasonIE,
 )
 from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 352d35c7a..8a36c10f6 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -11,6 +11,8 @@ from ..compat import (
 from ..utils import (
     int_or_none,
     parse_iso8601,
+    smuggle_url,
+    unsmuggle_url,
 )
 
 
@@ -25,9 +27,9 @@ class DCNGeneralIE(InfoExtractor):
             url = 'http://www.dcndigital.ae/#/media/%s' % video_id
             ie_key = 'DCNVideo'
         else:
-            ie_key = 'DCNShow'
+            ie_key = 'DCNSeason'
             if season_id and int(season_id) > 0:
-                url = 'http://www.dcndigital.ae/#/program/season/%s' % season_id
+                url = smuggle_url('http://www.dcndigital.ae/#/program/season/%s' % season_id, {'show_id': show_id})
             else:
                 url = 'http://www.dcndigital.ae/#/program/%s' % show_id
         return {
@@ -38,6 +40,7 @@ class DCNGeneralIE(InfoExtractor):
 
 
 class DCNVideoIE(InfoExtractor):
+    IE_NAME = 'dcn:video'
     _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/[^/]+|media)/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.dcndigital.ae/#/video/%D8%B1%D8%AD%D9%84%D8%A9-%D8%A7%D9%84%D8%B9%D9%85%D8%B1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/17375',
@@ -109,13 +112,14 @@ class DCNVideoIE(InfoExtractor):
         }
 
 
-class DCNShowIE(InfoExtractor):
+class DCNSeasonIE(InfoExtractor):
+    IE_NAME = 'dcn:season'
     _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?program/(?:(?P<show_id>\d+)|season/(?P<season_id>\d+))'
     _TEST = {
         'url': 'http://dcndigital.ae/#/program/205024/%D9%85%D8%AD%D8%A7%D8%B6%D8%B1%D8%A7%D8%AA-%D8%A7%D9%84%D8%B4%D9%8A%D8%AE-%D8%A7%D9%84%D8%B4%D8%B9%D8%B1%D8%A7%D9%88%D9%8A',
         'info_dict':
         {
-            'id': '205024',
+            'id': '7910',
             'title': 'محاضرات الشيخ الشعراوي',
             'description': '',
         },
@@ -123,15 +127,18 @@ class DCNShowIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
         show_id, season_id = re.match(self._VALID_URL, url).groups()
         data = {}
         if season_id:
-            request = compat_urllib_request.Request(
-                'http://admin.mangomolo.com/analytics/index.php/plus/season_info?id=%s' % season_id,
-                headers={'Origin': 'http://www.dcndigital.ae'})
-            season = self._download_json(request, season_id)
-            show_id = season['id']
             data['season'] = season_id
+            show_id = smuggled_data.get('show_id')
+            if show_id is None:
+                request = compat_urllib_request.Request(
+                    'http://admin.mangomolo.com/analytics/index.php/plus/season_info?id=%s' % season_id,
+                    headers={'Origin': 'http://www.dcndigital.ae'})
+                season = self._download_json(request, season_id)
+                show_id = season['id']
         data['show_id'] = show_id
         request = compat_urllib_request.Request(
             'http://admin.mangomolo.com/analytics/index.php/plus/show',
@@ -141,6 +148,7 @@ class DCNShowIE(InfoExtractor):
                 'Content-Type': 'application/x-www-form-urlencoded'
             })
         show = self._download_json(request, show_id)
+        season_id = season_id or show['default_season']
         title = show['cat'].get('title_en') or show['cat']['title_ar']
         description = show['cat'].get('description_en') or show['cat'].get('description_ar')
         entries = []
@@ -151,7 +159,7 @@ class DCNShowIE(InfoExtractor):
                 'ie_key': 'DCNVideo',
             })
         return {
-            'id': show_id,
+            'id': season_id,
             'title': title,
             'description': description,
             'entries': entries,

From 8e2898edf930830260ab6b294c8866e7651a01a6 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Sep 2015 15:42:09 +0100
Subject: [PATCH 0035/1105] [dcn] add support for live streams and catchup
 videos

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/dcn.py      | 62 +++++++++++++++++++++++++++++++-
 2 files changed, 62 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 4e41d9bf9..677c75564 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -121,6 +121,7 @@ from .dbtv import DBTVIE
 from .dcn import (
     DCNGeneralIE,
     DCNVideoIE,
+    DCNLiveIE,
     DCNSeasonIE,
 )
 from .dctp import DctpTvIE
diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 8a36c10f6..2e8fff660 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import re
+import base64
 
 from .common import InfoExtractor
 from ..compat import (
@@ -41,7 +42,7 @@ class DCNGeneralIE(InfoExtractor):
 
 class DCNVideoIE(InfoExtractor):
     IE_NAME = 'dcn:video'
-    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/[^/]+|media)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/[^/]+|media|catchup/[^/]+/[^/]+)/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.dcndigital.ae/#/video/%D8%B1%D8%AD%D9%84%D8%A9-%D8%A7%D9%84%D8%B9%D9%85%D8%B1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/17375',
         'info_dict':
@@ -112,6 +113,65 @@ class DCNVideoIE(InfoExtractor):
         }
 
 
+class DCNLiveIE(InfoExtractor):
+    IE_NAME = 'dcn:live'
+    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?live/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.dcndigital.ae/#/live/6/dubai-tv',
+        'info_dict':
+        {
+            'id': '6',
+            'ext': 'mp4',
+            'title': 'Dubai Al Oula',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        channel_id = self._match_id(url)
+
+        request = compat_urllib_request.Request(
+            'http://admin.mangomolo.com/analytics/index.php/plus/getchanneldetails?channel_id=%s' % channel_id,
+            headers={'Origin': 'http://www.dcndigital.ae'})
+
+        channel = self._download_json(request, channel_id)
+        title = channel.get('title_en') or channel['title_ar']
+
+        webpage = self._download_webpage(
+            'http://admin.mangomolo.com/analytics/index.php/customers/embed/index?'
+            + compat_urllib_parse.urlencode({
+                'id': base64.b64encode(channel['user_id'].encode()).decode(),
+                'channelid': base64.b64encode(channel['id'].encode()).decode(),
+                'signature': channel['signature'],
+                'countries': 'Q0M=',
+                'filter': 'DENY',
+            }), channel_id)
+
+        m3u8_url = self._html_search_regex(r'file:\s*"([^"]+)', webpage, 'm3u8 url')
+        formats = self._extract_m3u8_formats(
+            m3u8_url, channel_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
+
+        rtsp_url = self._search_regex(
+            r'<a[^>]+href="(rtsp://[^"]+)"', webpage, 'rtsp url', fatal=False)
+        if rtsp_url:
+            formats.append({
+                'url': rtsp_url,
+                'format_id': 'rtsp',
+            })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': channel_id,
+            'title': title,
+            'formats': formats,
+            'is_live': True,
+        }
+
+
 class DCNSeasonIE(InfoExtractor):
     IE_NAME = 'dcn:season'
     _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?program/(?:(?P<show_id>\d+)|season/(?P<season_id>\d+))'

From 486375154cb7d79bd084879467bc70550104b555 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Sep 2015 11:30:42 +0100
Subject: [PATCH 0036/1105] correct season info extraction and simplify

---
 youtube_dl/extractor/dcn.py | 64 ++++++++++++++++++-------------------
 1 file changed, 31 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 2e8fff660..8b360a9d7 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -25,19 +25,13 @@ class DCNGeneralIE(InfoExtractor):
         url = ''
         ie_key = ''
         if video_id and int(video_id) > 0:
-            url = 'http://www.dcndigital.ae/#/media/%s' % video_id
-            ie_key = 'DCNVideo'
+            return self.url_result('http://www.dcndigital.ae/#/media/%s' % video_id, 'DCNVideo')
         else:
-            ie_key = 'DCNSeason'
             if season_id and int(season_id) > 0:
                 url = smuggle_url('http://www.dcndigital.ae/#/program/season/%s' % season_id, {'show_id': show_id})
             else:
                 url = 'http://www.dcndigital.ae/#/program/%s' % show_id
-        return {
-            'url': url,
-            '_type': 'url',
-            'ie_key': ie_key
-        }
+            return self.url_result(url, 'DCNSeason')
 
 
 class DCNVideoIE(InfoExtractor):
@@ -71,6 +65,11 @@ class DCNVideoIE(InfoExtractor):
 
         video = self._download_json(request, video_id)
         title = video.get('title_en') or video['title_ar']
+        img = video.get('img')
+        thumbnail = 'http://admin.mangomolo.com/analytics/%s' % img if img else None
+        duration = int_or_none(video.get('duration'))
+        description = video.get('description_en') or video.get('description_ar')
+        timestamp = parse_iso8601(video.get('create_time') or video.get('update_time'), ' ')
 
         webpage = self._download_webpage(
             'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?'
@@ -96,12 +95,6 @@ class DCNVideoIE(InfoExtractor):
 
         self._sort_formats(formats)
 
-        img = video.get('img')
-        thumbnail = 'http://admin.mangomolo.com/analytics/%s' % img if img else None
-        duration = int_or_none(video.get('duration'))
-        description = video.get('description_en') or video.get('description_ar')
-        timestamp = parse_iso8601(video.get('create_time') or video.get('update_time'), ' ')
-
         return {
             'id': video_id,
             'title': title,
@@ -122,7 +115,9 @@ class DCNLiveIE(InfoExtractor):
         {
             'id': '6',
             'ext': 'mp4',
-            'title': 'Dubai Al Oula',
+            'title': 're:^Dubai Al Oula [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'thumbnail': 're:^https?://.*\.png$',
+            'is_live': True,
         },
         'params': {
             # m3u8 download
@@ -139,10 +134,14 @@ class DCNLiveIE(InfoExtractor):
 
         channel = self._download_json(request, channel_id)
         title = channel.get('title_en') or channel['title_ar']
+        img = channel.get('thumbnail')
+        thumbnail = 'http://admin.mangomolo.com/analytics/%s' % img if img else None
+        description = channel.get('description_en') or channel.get('description_ar')
+        timestamp = parse_iso8601(channel.get('create_time') or channel.get('update_time'), ' ')
 
         webpage = self._download_webpage(
-            'http://admin.mangomolo.com/analytics/index.php/customers/embed/index?'
-            + compat_urllib_parse.urlencode({
+            'http://admin.mangomolo.com/analytics/index.php/customers/embed/index?' +
+            compat_urllib_parse.urlencode({
                 'id': base64.b64encode(channel['user_id'].encode()).decode(),
                 'channelid': base64.b64encode(channel['id'].encode()).decode(),
                 'signature': channel['signature'],
@@ -166,7 +165,9 @@ class DCNLiveIE(InfoExtractor):
 
         return {
             'id': channel_id,
-            'title': title,
+            'title': self._live_title(title),
+            'description': description,
+            'thumbnail': thumbnail,
             'formats': formats,
             'is_live': True,
         }
@@ -181,7 +182,6 @@ class DCNSeasonIE(InfoExtractor):
         {
             'id': '7910',
             'title': 'محاضرات الشيخ الشعراوي',
-            'description': '',
         },
         'playlist_mincount': 27,
     }
@@ -189,6 +189,7 @@ class DCNSeasonIE(InfoExtractor):
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})
         show_id, season_id = re.match(self._VALID_URL, url).groups()
+
         data = {}
         if season_id:
             data['season'] = season_id
@@ -207,21 +208,18 @@ class DCNSeasonIE(InfoExtractor):
                 'Origin': 'http://www.dcndigital.ae',
                 'Content-Type': 'application/x-www-form-urlencoded'
             })
+
         show = self._download_json(request, show_id)
         season_id = season_id or show['default_season']
-        title = show['cat'].get('title_en') or show['cat']['title_ar']
-        description = show['cat'].get('description_en') or show['cat'].get('description_ar')
+        season = {}
+        for _ in show['seasons']:
+            if _['id'] == season_id:
+                season = _
+                break
+        title = season.get('title_en') or season['title_ar']
+
         entries = []
         for video in show['videos']:
-            entries.append({
-                'url': 'http://www.dcndigital.ae/#/media/%s' % video['id'],
-                '_type': 'url',
-                'ie_key': 'DCNVideo',
-            })
-        return {
-            'id': season_id,
-            'title': title,
-            'description': description,
-            'entries': entries,
-            '_type': 'playlist',
-        }
+            entries.append(self.url_result('http://www.dcndigital.ae/#/media/%s' % video['id'], 'DCNVideo'))
+
+        return self.playlist_result(entries, season_id, title)

From 8b55cadc83f198e0fa6bac7158f9b05826f39257 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 7 Sep 2015 16:39:01 +0100
Subject: [PATCH 0037/1105] [canal13cl] fix info extraction

---
 youtube_dl/extractor/__init__.py  |  2 +-
 youtube_dl/extractor/canal13cl.py | 48 -------------------
 youtube_dl/extractor/tele13.py    | 77 +++++++++++++++++++++++++++++++
 3 files changed, 78 insertions(+), 49 deletions(-)
 delete mode 100644 youtube_dl/extractor/canal13cl.py
 create mode 100644 youtube_dl/extractor/tele13.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5d2ea39d0..661b53e63 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -67,7 +67,6 @@ from .camdemy import (
     CamdemyIE,
     CamdemyFolderIE
 )
-from .canal13cl import Canal13clIE
 from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
 from .cbs import CBSIE
@@ -612,6 +611,7 @@ from .teachingchannel import TeachingChannelIE
 from .teamcoco import TeamcocoIE
 from .techtalks import TechTalksIE
 from .ted import TEDIE
+from .tele13 import Tele13IE
 from .telebruxelles import TeleBruxellesIE
 from .telecinco import TelecincoIE
 from .telegraaf import TelegraafIE
diff --git a/youtube_dl/extractor/canal13cl.py b/youtube_dl/extractor/canal13cl.py
deleted file mode 100644
index 93241fefe..000000000
--- a/youtube_dl/extractor/canal13cl.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-
-
-class Canal13clIE(InfoExtractor):
-    _VALID_URL = r'^http://(?:www\.)?13\.cl/(?:[^/?#]+/)*(?P<id>[^/?#]+)'
-    _TEST = {
-        'url': 'http://www.13.cl/t13/nacional/el-circulo-de-hierro-de-michelle-bachelet-en-su-regreso-a-la-moneda',
-        'md5': '4cb1fa38adcad8fea88487a078831755',
-        'info_dict': {
-            'id': '1403022125',
-            'display_id': 'el-circulo-de-hierro-de-michelle-bachelet-en-su-regreso-a-la-moneda',
-            'ext': 'mp4',
-            'title': 'El "círculo de hierro" de Michelle Bachelet en su regreso a La Moneda',
-            'description': '(Foto: Agencia Uno) En nueve días más, Michelle Bachelet va a asumir por segunda vez como presidenta de la República. Entre aquellos que la acompañarán hay caras que se repiten y otras que se consolidan en su entorno de colaboradores más cercanos.',
-        }
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('id')
-
-        webpage = self._download_webpage(url, display_id)
-
-        title = self._html_search_meta(
-            'twitter:title', webpage, 'title', fatal=True)
-        description = self._html_search_meta(
-            'twitter:description', webpage, 'description')
-        url = self._html_search_regex(
-            r'articuloVideo = \"(.*?)\"', webpage, 'url')
-        real_id = self._search_regex(
-            r'[^0-9]([0-9]{7,})[^0-9]', url, 'id', default=display_id)
-        thumbnail = self._html_search_regex(
-            r'articuloImagen = \"(.*?)\"', webpage, 'thumbnail')
-
-        return {
-            'id': real_id,
-            'display_id': display_id,
-            'url': url,
-            'title': title,
-            'description': description,
-            'ext': 'mp4',
-            'thumbnail': thumbnail,
-        }
diff --git a/youtube_dl/extractor/tele13.py b/youtube_dl/extractor/tele13.py
new file mode 100644
index 000000000..5d89e757f
--- /dev/null
+++ b/youtube_dl/extractor/tele13.py
@@ -0,0 +1,77 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import js_to_json
+
+
+class Tele13IE(InfoExtractor):
+    _VALID_URL = r'^http://(?:www\.)?t13\.cl/videos(?:/[^/]+)+/(?P<id>[\w-]+)'
+    _TESTS = [
+        {
+            'url': 'http://www.t13.cl/videos/actualidad/el-circulo-de-hierro-de-michelle-bachelet-en-su-regreso-a-la-moneda',
+            'md5': '4cb1fa38adcad8fea88487a078831755',
+            'info_dict': {
+                'id': 'el-circulo-de-hierro-de-michelle-bachelet-en-su-regreso-a-la-moneda',
+                'ext': 'mp4',
+                'title': 'El c\u00edrculo de hierro de Michelle Bachelet en su regreso a La Moneda',
+            }
+        },
+        {
+            'url': 'http://www.t13.cl/videos/mundo/tendencias/video-captan-misteriosa-bola-fuego-cielos-bangkok',
+            'md5': '65d1ae54812c96f4b345dd21d3bb1adc',
+            'info_dict': {
+                'id': 'rOoKv2OMpOw',
+                'ext': 'mp4',
+                'title': 'Shooting star seen on 7-Sep-2015',
+                'description': 'md5:a1cd2e74f6ee6851552c9cf5851d6b06',
+                'uploader': 'Porjai Jaturongkhakun',
+                'upload_date': '20150906',
+                'uploader_id': 'UCnLY_3ezwNcDSC_Wc6suZxw',
+            },
+            'add_ie': ['Youtube'],
+        }
+    ]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        setup_js = self._parse_json(
+            js_to_json(
+                self._search_regex(
+                    r"jwplayer\('player-vivo'\).setup\((\{.*?\})\)",
+                    webpage,
+                    'setup code',
+                    flags=re.DOTALL
+                ).replace('\n//', '')
+            ),
+            display_id
+        )
+        title = setup_js['title']
+        thumbnail = setup_js.get('image') or setup_js['playlist'][0].get('image')
+        description = self._html_search_meta(
+            'description', webpage, 'description')
+
+        formats = []
+        for f in setup_js['playlist'][0]['sources']:
+            format_url = f['file']
+            if format_url != '':
+                if '.m3u8' in format_url:
+                    formats.extend(self._extract_m3u8_formats(format_url, display_id))
+                else:
+                    if 'youtube.com' in format_url:
+                        return self.url_result(format_url, 'Youtube')
+                    else:
+                        formats.append({'url': format_url, 'format_id': f.get('label')})
+
+        return {
+            'id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'formats': formats,
+        }

From 436416afe2ea70dd6b55f8c9d699ddb0bdc1ec5f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 7 Sep 2015 21:13:49 +0100
Subject: [PATCH 0038/1105] [tele13] skip test

---
 youtube_dl/extractor/tele13.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tele13.py b/youtube_dl/extractor/tele13.py
index 5d89e757f..f1764eb2f 100644
--- a/youtube_dl/extractor/tele13.py
+++ b/youtube_dl/extractor/tele13.py
@@ -16,8 +16,12 @@ class Tele13IE(InfoExtractor):
             'info_dict': {
                 'id': 'el-circulo-de-hierro-de-michelle-bachelet-en-su-regreso-a-la-moneda',
                 'ext': 'mp4',
-                'title': 'El c\u00edrculo de hierro de Michelle Bachelet en su regreso a La Moneda',
-            }
+                'title': 'El círculo de hierro de Michelle Bachelet en su regreso a La Moneda',
+            },
+            'params': {
+                # HTTP Error 404: Not Found
+                'skip_download': True,
+            },
         },
         {
             'url': 'http://www.t13.cl/videos/mundo/tendencias/video-captan-misteriosa-bola-fuego-cielos-bangkok',

From b306c439d7f2997ebf2a88385c73fe2d92227b76 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 23 Sep 2015 13:28:05 +0100
Subject: [PATCH 0039/1105] [cnet] fix extraction and extract more formats

---
 youtube_dl/extractor/cnet.py | 54 +++++++++++++++---------------------
 1 file changed, 23 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/cnet.py b/youtube_dl/extractor/cnet.py
index 5dd69bff7..2fac0d79d 100644
--- a/youtube_dl/extractor/cnet.py
+++ b/youtube_dl/extractor/cnet.py
@@ -4,9 +4,7 @@ from __future__ import unicode_literals
 import json
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-)
+from .theplatform import ThePlatformIE
 
 
 class CNETIE(InfoExtractor):
@@ -15,29 +13,22 @@ class CNETIE(InfoExtractor):
         'url': 'http://www.cnet.com/videos/hands-on-with-microsofts-windows-8-1-update/',
         'info_dict': {
             'id': '56f4ea68-bd21-4852-b08c-4de5b8354c60',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Hands-on with Microsoft Windows 8.1 Update',
             'description': 'The new update to the Windows 8 OS brings improved performance for mouse and keyboard users.',
-            'thumbnail': 're:^http://.*/flmswindows8.jpg$',
             'uploader_id': '6085384d-619e-11e3-b231-14feb5ca9861',
             'uploader': 'Sarah Mitroff',
         },
-        'params': {
-            'skip_download': 'requires rtmpdump',
-        }
     }, {
         'url': 'http://www.cnet.com/videos/whiny-pothole-tweets-at-local-government-when-hit-by-cars-tomorrow-daily-187/',
         'info_dict': {
             'id': '56527b93-d25d-44e3-b738-f989ce2e49ba',
-            'ext': 'flv',
+            'ext': 'mp4',
             'description': 'Khail and Ashley wonder what other civic woes can be solved by self-tweeting objects, investigate a new kind of VR camera and watch an origami robot self-assemble, walk, climb, dig and dissolve. #TDPothole',
             'uploader_id': 'b163284d-6b73-44fc-b3e6-3da66c392d40',
             'uploader': 'Ashley Esqueda',
             'title': 'Whiny potholes tweet at local government when hit by cars (Tomorrow Daily 187)',
         },
-        'params': {
-            'skip_download': True,  # requires rtmpdump
-        },
     }]
 
     def _real_extract(self, url):
@@ -45,26 +36,13 @@ class CNETIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         data_json = self._html_search_regex(
-            r"<div class=\"cnetVideoPlayer\"\s+.*?data-cnet-video-options='([^']+)'",
+            r"<div class=\"videoPlayer\"\s+.*?data-cnet-video-uvp-options='([^']+)'",
             webpage, 'data json')
         data = json.loads(data_json)
-        vdata = data['video']
-        if not vdata:
-            vdata = data['videos'][0]
-        if not vdata:
-            raise ExtractorError('Cannot find video data')
-
-        mpx_account = data['config']['players']['default']['mpx_account']
-        vid = vdata['files'].get('rtmp', vdata['files']['hds'])
-        tp_link = 'http://link.theplatform.com/s/%s/%s' % (mpx_account, vid)
+        vdata = data['videos'][0]
 
         video_id = vdata['id']
-        title = vdata.get('headline')
-        if title is None:
-            title = vdata.get('title')
-        if title is None:
-            raise ExtractorError('Cannot find title!')
-        thumbnail = vdata.get('image', {}).get('path')
+        title = vdata['title']
         author = vdata.get('author')
         if author:
             uploader = '%s %s' % (author['firstName'], author['lastName'])
@@ -73,13 +51,27 @@ class CNETIE(InfoExtractor):
             uploader = None
             uploader_id = None
 
+        mpx_account = data['config']['uvpConfig']['default']['mpx_account']
+        tp = ThePlatformIE(self._downloader)
+        formats = []
+        subtitles = {}
+        description = vdata.get('description')
+
+        for vid in vdata['files'].values():
+            result = tp.extract(('http://link.theplatform.com/s/%s/%s' % (mpx_account, vid)))
+            formats.extend(result['formats'])
+            subtitles = self._merge_subtitles(subtitles, result['subtitles'])
+            description = description or result.get('description')
+
+        self._sort_formats(formats)
+
         return {
-            '_type': 'url_transparent',
-            'url': tp_link,
             'id': video_id,
             'display_id': display_id,
             'title': title,
+            'description': description,
             'uploader': uploader,
             'uploader_id': uploader_id,
-            'thumbnail': thumbnail,
+            'subtitles': subtitles,
+            'formats': formats,
         }

From 6aeba407db84a636fc2522b4f2344eac9e0c1fdb Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 25 Sep 2015 10:52:48 +0100
Subject: [PATCH 0040/1105] [jukebox] remove extractor and handle it using
 generic extractor

---
 youtube_dl/extractor/__init__.py  |  1 -
 youtube_dl/extractor/generic.py   | 21 +++++++
 youtube_dl/extractor/jukebox.py   | 59 ------------------
 youtube_dl/extractor/ultimedia.py | 99 +++++++++++++------------------
 4 files changed, 61 insertions(+), 119 deletions(-)
 delete mode 100644 youtube_dl/extractor/jukebox.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7272859db..1813c7e1b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -262,7 +262,6 @@ from .izlesene import IzleseneIE
 from .jadorecettepub import JadoreCettePubIE
 from .jeuxvideo import JeuxVideoIE
 from .jove import JoveIE
-from .jukebox import JukeboxIE
 from .jpopsukitv import JpopsukiIE
 from .kaltura import KalturaIE
 from .kanalplay import KanalPlayIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8881a8a23..4d1f75e63 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -50,6 +50,7 @@ from .dailymotion import DailymotionCloudIE
 from .onionstudios import OnionStudiosIE
 from .snagfilms import SnagFilmsEmbedIE
 from .screenwavemedia import ScreenwaveMediaIE
+from .ultimedia import UltimediaIE
 
 
 class GenericIE(InfoExtractor):
@@ -1029,6 +1030,21 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'cinemasnob',
             },
+        },
+        # Ultimedia embed
+        {
+            'url': 'http://www.jukebox.es/kosheen/videoclip,pride,r303r.html',
+            'md5': '25551df6e7c7ab8096ceeeae048c5f64',
+            'info_dict': {
+                'id': 'r303r',
+                'ext': 'mp4',
+                'title': 'Kosheen - Pride (live)',
+                'thumbnail': 're:^https?://.*\.jpg',
+                'duration': 293,
+                'upload_date': '20081103',
+                'timestamp': 1225733392,
+                'uploader_id': '33m03',
+            },
         }
     ]
 
@@ -1751,6 +1767,11 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(unescapeHTML(mobj.group('url')), 'ScreenwaveMedia')
 
+        # Look for Ulltimedia embeds
+        ultimedia_url = UltimediaIE._extract_url(webpage)
+        if ultimedia_url:
+            return self.url_result(self._proto_relative_url(ultimedia_url), 'Ultimedia')
+
         # Look for AdobeTVVideo embeds
         mobj = re.search(
             r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',
diff --git a/youtube_dl/extractor/jukebox.py b/youtube_dl/extractor/jukebox.py
deleted file mode 100644
index da8068efc..000000000
--- a/youtube_dl/extractor/jukebox.py
+++ /dev/null
@@ -1,59 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    RegexNotFoundError,
-    unescapeHTML,
-)
-
-
-class JukeboxIE(InfoExtractor):
-    _VALID_URL = r'^http://www\.jukebox?\..+?\/.+[,](?P<id>[a-z0-9\-]+)\.html'
-    _TEST = {
-        'url': 'http://www.jukebox.es/kosheen/videoclip,pride,r303r.html',
-        'info_dict': {
-            'id': 'r303r',
-            'ext': 'flv',
-            'title': 'Kosheen-En Vivo Pride',
-            'uploader': 'Kosheen',
-        },
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        html = self._download_webpage(url, video_id)
-        iframe_url = unescapeHTML(self._search_regex(r'<iframe .*src="([^"]*)"', html, 'iframe url'))
-
-        iframe_html = self._download_webpage(iframe_url, video_id, 'Downloading iframe')
-        if re.search(r'class="jkb_waiting"', iframe_html) is not None:
-            raise ExtractorError('Video is not available(in your country?)!')
-
-        self.report_extraction(video_id)
-
-        try:
-            video_url = self._search_regex(r'"config":{"file":"(?P<video_url>http:[^"]+\?mdtk=[0-9]+)"',
-                                           iframe_html, 'video url')
-            video_url = unescapeHTML(video_url).replace('\/', '/')
-        except RegexNotFoundError:
-            youtube_url = self._search_regex(
-                r'config":{"file":"(http:\\/\\/www\.youtube\.com\\/watch\?v=[^"]+)"',
-                iframe_html, 'youtube url')
-            youtube_url = unescapeHTML(youtube_url).replace('\/', '/')
-            self.to_screen('Youtube video detected')
-            return self.url_result(youtube_url, ie='Youtube')
-
-        title = self._html_search_regex(r'<h1 class="inline">([^<]+)</h1>',
-                                        html, 'title')
-        artist = self._html_search_regex(r'<span id="infos_article_artist">([^<]+)</span>',
-                                         html, 'artist')
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'title': artist + '-' + title,
-            'uploader': artist,
-        }
diff --git a/youtube_dl/extractor/ultimedia.py b/youtube_dl/extractor/ultimedia.py
index c4751050e..45201332d 100644
--- a/youtube_dl/extractor/ultimedia.py
+++ b/youtube_dl/extractor/ultimedia.py
@@ -4,102 +4,83 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlparse
-from ..utils import (
-    ExtractorError,
-    qualities,
-    unified_strdate,
-    clean_html,
-)
+from ..utils import int_or_none
 
 
 class UltimediaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ultimedia\.com/default/index/video[^/]+/id/(?P<id>[\d+a-z]+)'
+    _VALID_URL = r'https?://(?:www\.)?ultimedia\.com/deliver/(?P<type>generic|musique)(?:/[^/]+)*/(?:src|article)/(?P<id>[\d+a-z]+)'
     _TESTS = [{
         # news
-        'url': 'https://www.ultimedia.com/default/index/videogeneric/id/s8uk0r',
+        'url': 'https://www.ultimedia.com/deliver/generic/iframe/mdtk/01601930/zone/1/src/s8uk0r/autoplay/yes/ad/no/width/714/height/435',
         'md5': '276a0e49de58c7e85d32b057837952a2',
         'info_dict': {
             'id': 's8uk0r',
             'ext': 'mp4',
             'title': 'Loi sur la fin de vie: le texte prévoit un renforcement des directives anticipées',
-            'description': 'md5:3e5c8fd65791487333dda5db8aed32af',
             'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 74,
             'upload_date': '20150317',
+            'timestamp': 1426604939,
+            'uploader_id': '3fszv',
         },
     }, {
         # music
-        'url': 'https://www.ultimedia.com/default/index/videomusic/id/xvpfp8',
+        'url': 'https://www.ultimedia.com/deliver/musique/iframe/mdtk/01601930/zone/1/article/xvpfp8/autoplay/yes/ad/no/width/714/height/435',
         'md5': '2ea3513813cf230605c7e2ffe7eca61c',
         'info_dict': {
             'id': 'xvpfp8',
             'ext': 'mp4',
-            'title': "Two - C'est la vie (Clip)",
-            'description': 'Two',
+            'title': 'Two - C\'est La Vie (clip)',
             'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 233,
             'upload_date': '20150224',
+            'timestamp': 1424760500,
+            'uploader_id': '3rfzk',
         },
     }]
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<(?:iframe|script)[^>]+src=["\'](?P<url>(?:https?:)?//(?:www\.)?ultimedia\.com/deliver/(?:generic|musique)(?:/[^/]+)*/(?:src|article)/[\d+a-z]+)',
+            webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        video_type, video_id = re.match(self._VALID_URL, url).groups()
 
-        deliver_url = self._proto_relative_url(self._search_regex(
-            r'<iframe[^>]+src="((?:https?:)?//(?:www\.)?ultimedia\.com/deliver/[^"]+)"',
-            webpage, 'deliver URL'), compat_urllib_parse_urlparse(url).scheme + ':')
-
-        deliver_page = self._download_webpage(
-            deliver_url, video_id, 'Downloading iframe page')
-
-        if '>This video is currently not available' in deliver_page:
-            raise ExtractorError(
-                'Video %s is currently not available' % video_id, expected=True)
-
-        player = self._parse_json(
-            self._search_regex(
-                r"jwplayer\('player(?:_temp)?'\)\.setup\(({.+?})\)\.on",
-                deliver_page, 'player'),
+        deliver_info = self._download_json(
+            'http://www.ultimedia.com/deliver/video?video=%s&topic=%s' % (video_id, video_type),
             video_id)
 
-        quality = qualities(['flash', 'html5'])
+        yt_id = deliver_info.get('yt_id')
+        if yt_id:
+            return self.url_result(yt_id, 'Youtube')
+
+        jwconf = deliver_info['jwconf']
+
         formats = []
-        for mode in player['modes']:
-            video_url = mode.get('config', {}).get('file')
-            if not video_url:
-                continue
-            if re.match(r'https?://www\.youtube\.com/.+?', video_url):
-                return self.url_result(video_url, 'Youtube')
+        for source in jwconf['playlist'][0]['sources']:
             formats.append({
-                'url': video_url,
-                'format_id': mode.get('type'),
-                'quality': quality(mode.get('type')),
+                'url': source['file'],
+                'format_id': source.get('label'),
             })
+
         self._sort_formats(formats)
 
-        thumbnail = player.get('image')
-
-        title = clean_html((
-            self._html_search_regex(
-                r'(?s)<div\s+id="catArticle">.+?</div>(.+?)</h1>',
-                webpage, 'title', default=None) or
-            self._search_regex(
-                r"var\s+nameVideo\s*=\s*'([^']+)'",
-                deliver_page, 'title')))
-
-        description = clean_html(self._html_search_regex(
-            r'(?s)<span>Description</span>(.+?)</p>', webpage,
-            'description', fatal=False))
-
-        upload_date = unified_strdate(self._search_regex(
-            r'Ajouté le\s*<span>([^<]+)', webpage,
-            'upload date', fatal=False))
+        title = deliver_info['title']
+        thumbnail = jwconf.get('image')
+        duration = int_or_none(deliver_info.get('duration'))
+        timestamp = int_or_none(deliver_info.get('release_time'))
+        uploader_id = deliver_info.get('owner_id')
 
         return {
             'id': video_id,
             'title': title,
-            'description': description,
             'thumbnail': thumbnail,
-            'upload_date': upload_date,
+            'duration': duration,
+            'timestamp': timestamp,
+            'uploader_id': uploader_id,
             'formats': formats,
         }

From adccf33632c51def397cdfb08c1271de6d6ec95e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 2 Oct 2015 21:58:20 +0100
Subject: [PATCH 0041/1105] [ign] add support for pcmag and extract all formats
 and more metadata

---
 youtube_dl/extractor/__init__.py |   6 +-
 youtube_dl/extractor/ign.py      | 117 +++++++++++++++++++++++--------
 2 files changed, 93 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a73a1317e..191661390 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -231,7 +231,11 @@ from .howstuffworks import HowStuffWorksIE
 from .huffpost import HuffPostIE
 from .hypem import HypemIE
 from .iconosquare import IconosquareIE
-from .ign import IGNIE, OneUPIE
+from .ign import (
+    IGNIE,
+    OneUPIE,
+    PCMagIE,
+)
 from .imdb import (
     ImdbIE,
     ImdbListIE
diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index bf2d2041b..fa4e67394 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -3,6 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+)
 
 
 class IGNIE(InfoExtractor):
@@ -11,25 +15,23 @@ class IGNIE(InfoExtractor):
     Some videos of it.ign.com are also supported
     """
 
-    _VALID_URL = r'https?://.+?\.ign\.com/(?P<type>videos|show_videos|articles|(?:[^/]*/feature))(/.+)?/(?P<name_or_id>.+)'
+    _VALID_URL = r'https?://.+?\.ign\.com/(?:[^/]+/)?(?P<type>videos|show_videos|articles|feature|(?:[^/]+/\d+/video))(/.+)?/(?P<name_or_id>.+)'
     IE_NAME = 'ign.com'
 
-    _CONFIG_URL_TEMPLATE = 'http://www.ign.com/videos/configs/id/%s.config'
-    _DESCRIPTION_RE = [
-        r'<span class="page-object-description">(.+?)</span>',
-        r'id="my_show_video">.*?<p>(.*?)</p>',
-        r'<meta name="description" content="(.*?)"',
-    ]
+    _API_URL_TEMPLATE = 'http://apis.ign.com/video/v3/videos/%s'
+    _EMBED_RE = r'<iframe[^>]+?["\']((?:https?:)?//.+?\.ign\.com.+?/embed.+?)["\']'
 
     _TESTS = [
         {
             'url': 'http://www.ign.com/videos/2013/06/05/the-last-of-us-review',
-            'md5': 'eac8bdc1890980122c3b66f14bdd02e9',
+            'md5': 'febda82c4bafecd2d44b6e1a18a595f8',
             'info_dict': {
                 'id': '8f862beef863986b2785559b9e1aa599',
                 'ext': 'mp4',
                 'title': 'The Last of Us Review',
                 'description': 'md5:c8946d4260a4d43a00d5ae8ed998870c',
+                'timestamp': 1370440800,
+                'upload_date': '20130605',
             }
         },
         {
@@ -44,6 +46,8 @@ class IGNIE(InfoExtractor):
                         'ext': 'mp4',
                         'title': 'GTA 5 Video Review',
                         'description': 'Rockstar drops the mic on this generation of games. Watch our review of the masterly Grand Theft Auto V.',
+                        'timestamp': 1379339880,
+                        'upload_date': '20130916',
                     },
                 },
                 {
@@ -52,6 +56,8 @@ class IGNIE(InfoExtractor):
                         'ext': 'mp4',
                         'title': '26 Twisted Moments from GTA 5 in Slow Motion',
                         'description': 'The twisted beauty of GTA 5 in stunning slow motion.',
+                        'timestamp': 1386878820,
+                        'upload_date': '20131212',
                     },
                 },
             ],
@@ -66,10 +72,9 @@ class IGNIE(InfoExtractor):
                 'id': '078fdd005f6d3c02f63d795faa1b984f',
                 'ext': 'mp4',
                 'title': 'Rewind Theater - Wild Trailer Gamescom 2014',
-                'description': (
-                    'Giant skeletons, bloody hunts, and captivating'
-                    ' natural beauty take our breath away.'
-                ),
+                'description': 'Brian and Jared explore Michel Ancel\'s captivating new preview.',
+                'timestamp': 1408047180,
+                'upload_date': '20140814',
             },
         },
     ]
@@ -82,7 +87,7 @@ class IGNIE(InfoExtractor):
             r'<object id="vid_(.+?)"',
             r'<meta name="og:image" content=".*/(.+?)-(.+?)/.+.jpg"',
         ]
-        return self._search_regex(res_id, webpage, 'video id')
+        return self._search_regex(res_id, webpage, 'video id', default=None)
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -102,22 +107,45 @@ class IGNIE(InfoExtractor):
                 }
 
         video_id = self._find_video_id(webpage)
-        result = self._get_video_info(video_id)
-        description = self._html_search_regex(self._DESCRIPTION_RE,
-                                              webpage, 'video description', flags=re.DOTALL)
-        result['description'] = description
-        return result
+        if not video_id:
+            return self.url_result(self._search_regex(self._EMBED_RE, webpage, 'embed url'))
+        return self._get_video_info(video_id)
 
     def _get_video_info(self, video_id):
-        config_url = self._CONFIG_URL_TEMPLATE % video_id
-        config = self._download_json(config_url, video_id)
-        media = config['playlist']['media']
+        api_data = self._download_json(self._API_URL_TEMPLATE % video_id, video_id)
+
+        formats = []
+        m3u8_url = api_data['refs'].get('m3uUrl')
+        if m3u8_url:
+            formats.extend(self._extract_m3u8_formats(m3u8_url, video_id))
+        f4m_url = api_data['refs'].get('f4mUrl')
+        if f4m_url:
+            formats.extend(self._extract_f4m_formats(f4m_url, video_id))
+        for asset in api_data['assets']:
+            formats.append({
+                'url': asset['url'],
+                'tbr': asset.get('actual_bitrate_kbps'),
+                'fps': asset.get('frame_rate'),
+                'height': int_or_none(asset.get('height')),
+                'width': int_or_none(asset.get('width')),
+            })
+        self._sort_formats(formats)
+
+        thumbnails = []
+        for thumbnail in api_data['thumbnails']:
+            thumbnails.append({'url': thumbnail['url']})
+
+        metadata = api_data['metadata']
 
         return {
-            'id': media['metadata']['videoId'],
-            'url': media['url'],
-            'title': media['metadata']['title'],
-            'thumbnail': media['poster'][0]['url'].replace('{size}', 'grande'),
+            'id': api_data.get('videoId') or video_id,
+            'title': metadata.get('longTitle') or metadata.get('name') or metadata.get['title'],
+            'description': metadata.get('description'),
+            'timestamp': parse_iso8601(metadata.get('publishDate')),
+            'duration': int_or_none(metadata.get('duration')),
+            'display_id': metadata.get('slug') or video_id,
+            'thumbnails': thumbnails,
+            'formats': formats,
         }
 
 
@@ -125,16 +153,16 @@ class OneUPIE(IGNIE):
     _VALID_URL = r'https?://gamevideos\.1up\.com/(?P<type>video)/id/(?P<name_or_id>.+)\.html'
     IE_NAME = '1up.com'
 
-    _DESCRIPTION_RE = r'<div id="vid_summary">(.+?)</div>'
-
     _TESTS = [{
         'url': 'http://gamevideos.1up.com/video/id/34976.html',
-        'md5': '68a54ce4ebc772e4b71e3123d413163d',
+        'md5': 'c9cc69e07acb675c31a16719f909e347',
         'info_dict': {
             'id': '34976',
             'ext': 'mp4',
             'title': 'Sniper Elite V2 - Trailer',
-            'description': 'md5:5d289b722f5a6d940ca3136e9dae89cf',
+            'description': 'md5:bf0516c5ee32a3217aa703e9b1bc7826',
+            'timestamp': 1313099220,
+            'upload_date': '20110811',
         }
     }]
 
@@ -143,3 +171,34 @@ class OneUPIE(IGNIE):
         result = super(OneUPIE, self)._real_extract(url)
         result['id'] = mobj.group('name_or_id')
         return result
+
+
+class PCMagIE(IGNIE):
+    _VALID_URL = r'https?://(?:www\.)?pcmag\.com/(?P<type>videos|article2)(/.+)?/(?P<name_or_id>.+)'
+    IE_NAME = 'pcmag'
+
+    _EMBED_RE = r'iframe.setAttribute\("src",\s*__util.objToUrlString\("http://widgets\.ign\.com/video/embed/content.html?[^"]*url=([^"]+)["&]'
+
+    _TESTS = [{
+        'url': 'http://www.pcmag.com/videos/2015/01/06/010615-whats-new-now-is-gogo-snooping-on-your-data',
+        'md5': '212d6154fd0361a2781075f1febbe9ad',
+        'info_dict': {
+            'id': 'ee10d774b508c9b8ec07e763b9125b91',
+            'ext': 'mp4',
+            'title': '010615_What\'s New Now: Is GoGo Snooping on Your Data?',
+            'description': 'md5:a7071ae64d2f68cc821c729d4ded6bb3',
+            'timestamp': 1420571160,
+            'upload_date': '20150106',
+        }
+    },{
+        'url': 'http://www.pcmag.com/article2/0,2817,2470156,00.asp',
+        'md5': '94130c1ca07ba0adb6088350681f16c1',
+        'info_dict': {
+            'id': '042e560ba94823d43afcb12ddf7142ca',
+            'ext': 'mp4',
+            'title': 'HTC\'s Weird New Re Camera - What\'s New Now',
+            'description': 'md5:53433c45df96d2ea5d0fda18be2ca908',
+            'timestamp': 1412953920,
+            'upload_date': '20141010',
+        }
+    }]

From 1ef1563649374568870e9334cce7055f7c83a817 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 9 Oct 2015 20:08:37 +0100
Subject: [PATCH 0042/1105] [srgssr] Add generic extractor for SRGSSR Group
 sites

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/srf.py      | 104 -------------------------
 youtube_dl/extractor/srgssr.py   | 130 +++++++++++++++++++++++++++++++
 3 files changed, 134 insertions(+), 105 deletions(-)
 delete mode 100644 youtube_dl/extractor/srf.py
 create mode 100644 youtube_dl/extractor/srgssr.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3ace1cc2c..042ad3678 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -591,7 +591,10 @@ from .sportbox import (
     SportBoxEmbedIE,
 )
 from .sportdeutschland import SportDeutschlandIE
-from .srf import SrfIE
+from .srgssr import (
+    SRGSSRIE,
+    SRGSSRPlayIE,
+)
 from .srmediathek import SRMediathekIE
 from .ssa import SSAIE
 from .stanfordoc import StanfordOpenClassroomIE
diff --git a/youtube_dl/extractor/srf.py b/youtube_dl/extractor/srf.py
deleted file mode 100644
index 77eec0bc7..000000000
--- a/youtube_dl/extractor/srf.py
+++ /dev/null
@@ -1,104 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-from .common import InfoExtractor
-from ..utils import (
-    determine_ext,
-    parse_iso8601,
-    xpath_text,
-)
-
-
-class SrfIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.srf\.ch/play(?:er)?/tv/[^/]+/video/(?P<display_id>[^?]+)\?id=|tp\.srgssr\.ch/p/flash\?urn=urn:srf:ais:video:)(?P<id>[0-9a-f\-]{36})'
-    _TESTS = [{
-        'url': 'http://www.srf.ch/play/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
-        'md5': '4cd93523723beff51bb4bee974ee238d',
-        'info_dict': {
-            'id': '28e1a57d-5b76-4399-8ab3-9097f071e6c5',
-            'display_id': 'snowden-beantragt-asyl-in-russland',
-            'ext': 'm4v',
-            'upload_date': '20130701',
-            'title': 'Snowden beantragt Asyl in Russland',
-            'timestamp': 1372713995,
-        }
-    }, {
-        # No Speichern (Save) button
-        'url': 'http://www.srf.ch/play/tv/top-gear/video/jaguar-xk120-shadow-und-tornado-dampflokomotive?id=677f5829-e473-4823-ac83-a1087fe97faa',
-        'md5': 'd97e236e80d1d24729e5d0953d276a4f',
-        'info_dict': {
-            'id': '677f5829-e473-4823-ac83-a1087fe97faa',
-            'display_id': 'jaguar-xk120-shadow-und-tornado-dampflokomotive',
-            'ext': 'flv',
-            'upload_date': '20130710',
-            'title': 'Jaguar XK120, Shadow und Tornado-Dampflokomotive',
-            'timestamp': 1373493600,
-        },
-    }, {
-        'url': 'http://www.srf.ch/player/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
-        'only_matching': True,
-    }, {
-        'url': 'https://tp.srgssr.ch/p/flash?urn=urn:srf:ais:video:28e1a57d-5b76-4399-8ab3-9097f071e6c5',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        display_id = re.match(self._VALID_URL, url).group('display_id') or video_id
-
-        video_data = self._download_xml(
-            'http://il.srgssr.ch/integrationlayer/1.0/ue/srf/video/play/%s.xml' % video_id,
-            display_id)
-
-        title = xpath_text(
-            video_data, './AssetMetadatas/AssetMetadata/title', fatal=True)
-        thumbnails = [{
-            'url': s.text
-        } for s in video_data.findall('.//ImageRepresentation/url')]
-        timestamp = parse_iso8601(xpath_text(video_data, './createdDate'))
-        # The <duration> field in XML is different from the exact duration, skipping
-
-        formats = []
-        for item in video_data.findall('./Playlists/Playlist') + video_data.findall('./Downloads/Download'):
-            for url_node in item.findall('url'):
-                quality = url_node.attrib['quality']
-                full_url = url_node.text
-                original_ext = determine_ext(full_url)
-                format_id = '%s-%s' % (quality, item.attrib['protocol'])
-                if original_ext == 'f4m':
-                    formats.extend(self._extract_f4m_formats(
-                        full_url + '?hdcore=3.4.0', display_id, f4m_id=format_id))
-                elif original_ext == 'm3u8':
-                    formats.extend(self._extract_m3u8_formats(
-                        full_url, display_id, 'mp4', m3u8_id=format_id))
-                else:
-                    formats.append({
-                        'url': full_url,
-                        'ext': original_ext,
-                        'format_id': format_id,
-                        'quality': 0 if 'HD' in quality else -1,
-                        'preference': 1,
-                    })
-
-        self._sort_formats(formats)
-
-        subtitles = {}
-        subtitles_data = video_data.find('Subtitles')
-        if subtitles_data is not None:
-            subtitles_list = [{
-                'url': sub.text,
-                'ext': determine_ext(sub.text),
-            } for sub in subtitles_data]
-            if subtitles_list:
-                subtitles['de'] = subtitles_list
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'formats': formats,
-            'title': title,
-            'thumbnails': thumbnails,
-            'timestamp': timestamp,
-            'subtitles': subtitles,
-        }
diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py
new file mode 100644
index 000000000..addf4d26e
--- /dev/null
+++ b/youtube_dl/extractor/srgssr.py
@@ -0,0 +1,130 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    parse_iso8601,
+    qualities,
+)
+
+
+class SRGSSRIE(InfoExtractor):
+    _VALID_URL = r'(?:https?://tp\.srgssr\.ch/p(?:/[^/]+)+\?urn=)?urn:(?P<bu>srf|rts|rsi|rtr|swi):(?:[^:]+:)?(?P<type>video|audio):(?P<id>[0-9a-f\-]{36}|\d+)'
+
+    _ERRORS = {
+        'AGERATING12': 'To protect children under the age of 12, this video is only available between 8 p.m. and 6 a.m.',
+        'AGERATING18': 'To protect children under the age of 18, this video is only available between 11 p.m. and 5 a.m.',
+#        'ENDDATE': 'For legal reasons, this video was only available for a specified period of time.',
+        'GEOBLOCK': 'For legal reasons, this video is only available in Switzerland.',
+        'LEGAL': 'The video cannot be transmitted for legal reasons.',
+        'STARTDATE': 'This video is not yet available. Please try again later.',
+    }
+
+    def _real_extract(self, url):
+        bu, media_type, media_id = re.match(self._VALID_URL, url).groups()
+
+        media_data = self._download_json(
+            'http://il.srgssr.ch/integrationlayer/1.0/ue/%s/%s/play/%s.json' % (bu, media_type, media_id),
+            media_id)[media_type.capitalize()]
+
+        if media_data.get('block') and media_data['block'] in self._ERRORS:
+            raise ExtractorError(
+                    '%s said: %s' % (
+                        self.IE_NAME,
+                        self._ERRORS[media_data['block']]),
+                    expected=True)
+
+        metadata = media_data['AssetMetadatas']['AssetMetadata'][0]
+        title = metadata['title']
+        description = metadata.get('description')
+        created_date = media_data.get('createdDate') or metadata.get('createdDate')
+        timestamp = parse_iso8601(created_date)
+
+        thumbnails = []
+        for image in media_data['Image']['ImageRepresentations']['ImageRepresentation']:
+            thumbnails.append({
+                'id': image.get('id'),
+                'url': image['url'],
+            })
+
+        preference = qualities(['LQ', 'MQ', 'SD', 'HQ', 'HD'])
+        formats = []
+        for source in media_data['Playlists']['Playlist']:
+            assets = {}
+            protocol = source.get('@protocol')
+            if protocol in ('HTTP-HDS', 'HTTP-HLS'):
+                for quality in source['url']:
+                    assets[quality['@quality']] = quality['text']
+                asset_url = assets.get('HD') or assets.get('HQ') or assets.get('SD') or assets.get('MQ') or assets.get('LQ')
+                if '.f4m' in asset_url:
+                    formats.extend(self._extract_f4m_formats(asset_url + '?hdcore=3.4.0', media_id, f4m_id='hds'))
+                elif '.m3u8' in asset_url:
+                    formats.extend(self._extract_m3u8_formats(asset_url, media_id, m3u8_id='hls'))
+            else:
+                for asset in source['url']:
+                    asset_url = asset['text']
+                    ext = None
+                    if asset_url.startswith('rtmp'):
+                        ext = self._search_regex(r'([a-z0-9]+):[^/]+', asset_url, 'ext')
+                    formats.append({
+                        'url': asset_url,
+                        'preference': preference(asset['@quality']),
+                        'ext': ext,
+                    })
+
+        downloads = media_data.get('Downloads')
+        if downloads:
+            for source in downloads['Download']:
+                for asset in source['url']:
+                    formats.append({
+                        'url': asset['text'],
+                        'preference': preference(asset['@quality'])
+                    })
+        self._sort_formats(formats)
+
+        return {
+            'id': media_id,
+            'title': title,
+            'description': description,
+            'timestamp': timestamp,
+            'thumbnails': thumbnails,
+            'formats': formats,
+        }
+
+
+class SRGSSRPlayIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:(?:www|play)\.)?(?P<bu>srf|rts|rsi|rtr|swi)\.ch/play/(?:tv|radio)/[^/]+/(?P<type>video|audio)/[^?]+\?id=(?P<id>[0-9a-f\-]{36}|\d+)'
+
+    _TESTS = [{
+        'url': 'http://www.srf.ch/play/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
+        'md5': '4cd93523723beff51bb4bee974ee238d',
+        'info_dict': {
+            'id': '28e1a57d-5b76-4399-8ab3-9097f071e6c5',
+            'ext': 'm4v',
+            'upload_date': '20130701',
+            'title': 'Snowden beantragt Asyl in Russland',
+            'timestamp': 1372713995,
+        }
+    }, {
+        # No Speichern (Save) button
+        'url': 'http://www.srf.ch/play/tv/top-gear/video/jaguar-xk120-shadow-und-tornado-dampflokomotive?id=677f5829-e473-4823-ac83-a1087fe97faa',
+        'md5': '0a274ce38fda48c53c01890651985bc6',
+        'info_dict': {
+            'id': '677f5829-e473-4823-ac83-a1087fe97faa',
+            'ext': 'flv',
+            'upload_date': '20130710',
+            'title': 'Jaguar XK120, Shadow und Tornado-Dampflokomotive',
+            'description': 'md5:88604432b60d5a38787f152dec89cd56',
+            'timestamp': 1373493600,
+        },
+    }, {
+        'url': 'http://www.srf.ch/player/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        bu, media_type, media_id = re.match(self._VALID_URL, url).groups()
+        return self.url_result('urn:%s:%s:%s' % (bu, media_type, media_id), 'SRGSSR')

From 05ad5409b4fd044169ea0f67b9ae92d555564c4e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 9 Oct 2015 20:34:03 +0100
Subject: [PATCH 0043/1105] [srgssr] fix regex for swissinfo.ch

---
 youtube_dl/extractor/srgssr.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py
index addf4d26e..3b5dcc503 100644
--- a/youtube_dl/extractor/srgssr.py
+++ b/youtube_dl/extractor/srgssr.py
@@ -96,7 +96,7 @@ class SRGSSRIE(InfoExtractor):
 
 
 class SRGSSRPlayIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www|play)\.)?(?P<bu>srf|rts|rsi|rtr|swi)\.ch/play/(?:tv|radio)/[^/]+/(?P<type>video|audio)/[^?]+\?id=(?P<id>[0-9a-f\-]{36}|\d+)'
+    _VALID_URL = r'https?://(?:(?:www|play)\.)?(?P<bu>srf|rts|rsi|rtr|swissinfo)\.ch/play/(?:tv|radio)/[^/]+/(?P<type>video|audio)/[^?]+\?id=(?P<id>[0-9a-f\-]{36}|\d+)'
 
     _TESTS = [{
         'url': 'http://www.srf.ch/play/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
@@ -120,11 +120,8 @@ class SRGSSRPlayIE(InfoExtractor):
             'description': 'md5:88604432b60d5a38787f152dec89cd56',
             'timestamp': 1373493600,
         },
-    }, {
-        'url': 'http://www.srf.ch/player/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
-        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         bu, media_type, media_id = re.match(self._VALID_URL, url).groups()
-        return self.url_result('urn:%s:%s:%s' % (bu, media_type, media_id), 'SRGSSR')
+        return self.url_result('urn:%s:%s:%s' % (bu[:3], media_type, media_id), 'SRGSSR')

From e09f58b3bc3af6ce1e541fb7d034fe869fba6e82 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 14 Oct 2015 10:40:54 +0100
Subject: [PATCH 0044/1105] [srgssr] change the url chortcut, fix image
 extraction ,add a test and extract format id

---
 youtube_dl/extractor/srgssr.py | 35 ++++++++++++++++++++++++----------
 1 file changed, 25 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py
index 3b5dcc503..f759e5600 100644
--- a/youtube_dl/extractor/srgssr.py
+++ b/youtube_dl/extractor/srgssr.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class SRGSSRIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://tp\.srgssr\.ch/p(?:/[^/]+)+\?urn=)?urn:(?P<bu>srf|rts|rsi|rtr|swi):(?:[^:]+:)?(?P<type>video|audio):(?P<id>[0-9a-f\-]{36}|\d+)'
+    _VALID_URL = r'(?:https?://tp\.srgssr\.ch/p(?:/[^/]+)+\?urn=urn|srgssr):(?P<bu>srf|rts|rsi|rtr|swi):(?:[^:]+:)?(?P<type>video|audio):(?P<id>[0-9a-f\-]{36}|\d+)'
 
     _ERRORS = {
         'AGERATING12': 'To protect children under the age of 12, this video is only available between 8 p.m. and 6 a.m.',
@@ -44,11 +44,12 @@ class SRGSSRIE(InfoExtractor):
         timestamp = parse_iso8601(created_date)
 
         thumbnails = []
-        for image in media_data['Image']['ImageRepresentations']['ImageRepresentation']:
-            thumbnails.append({
-                'id': image.get('id'),
-                'url': image['url'],
-            })
+        if 'Image' in media_data:
+            for image in media_data['Image']['ImageRepresentations']['ImageRepresentation']:
+                thumbnails.append({
+                    'id': image.get('id'),
+                    'url': image['url'],
+                })
 
         preference = qualities(['LQ', 'MQ', 'SD', 'HQ', 'HD'])
         formats = []
@@ -70,16 +71,17 @@ class SRGSSRIE(InfoExtractor):
                     if asset_url.startswith('rtmp'):
                         ext = self._search_regex(r'([a-z0-9]+):[^/]+', asset_url, 'ext')
                     formats.append({
+                        'format_id': asset['@quality'],
                         'url': asset_url,
                         'preference': preference(asset['@quality']),
                         'ext': ext,
                     })
 
-        downloads = media_data.get('Downloads')
-        if downloads:
-            for source in downloads['Download']:
+        if 'Downloads' in media_data:
+            for source in media_data['Downloads']['Download']:
                 for asset in source['url']:
                     formats.append({
+                        'format_id': asset['@quality'],
                         'url': asset['text'],
                         'preference': preference(asset['@quality'])
                     })
@@ -120,8 +122,21 @@ class SRGSSRPlayIE(InfoExtractor):
             'description': 'md5:88604432b60d5a38787f152dec89cd56',
             'timestamp': 1373493600,
         },
+    },{
+        'url': 'http://www.rtr.ch/play/radio/actualitad/audio/saira-tujetsch-tuttina-cuntinuar-cun-sedrun-muster-turissem?id=63cb0778-27f8-49af-9284-8c7a8c6d15fc',
+        'info_dict': {
+            'id': '63cb0778-27f8-49af-9284-8c7a8c6d15fc',
+            'ext': 'mp3',
+            'upload_date': '20151013',
+            'title': 'Saira: Tujetsch - tuttina cuntinuar cun Sedrun Mustér Turissem',
+            'timestamp': 1444750398,
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
         bu, media_type, media_id = re.match(self._VALID_URL, url).groups()
-        return self.url_result('urn:%s:%s:%s' % (bu[:3], media_type, media_id), 'SRGSSR')
+        return self.url_result('srgssr:%s:%s:%s' % (bu[:3], media_type, media_id), 'SRGSSR')

From db8e38b8cff2e67a9ff51104c4a7b33c20650204 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 14 Oct 2015 11:55:03 +0100
Subject: [PATCH 0045/1105] [ign] add tests for me.ign specific language urls

---
 youtube_dl/extractor/ign.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index fa4e67394..fb2753738 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -77,6 +77,14 @@ class IGNIE(InfoExtractor):
                 'upload_date': '20140814',
             },
         },
+        {
+            'url': 'http://me.ign.com/en/videos/112203/video/how-hitman-aims-to-be-different-than-every-other-s',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://me.ign.com/ar/angry-birds-2/106533/video/lrd-ldyy-lwl-lfylm-angry-birds',
+            'only_matching': True,
+        },
     ]
 
     def _find_video_id(self, webpage):

From 77302fe5c989b9cafcb675c0a03642b80fa557ff Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 15 Oct 2015 23:27:46 +0100
Subject: [PATCH 0046/1105] [bliptv] remove extractor and add support for site
 replacement(makertv)

---
 youtube_dl/extractor/__init__.py     |   3 +-
 youtube_dl/extractor/bliptv.py       | 292 ---------------------------
 youtube_dl/extractor/cinemassacre.py |  18 +-
 youtube_dl/extractor/generic.py      |   6 -
 youtube_dl/extractor/jwplatform.py   |  67 ++++++
 youtube_dl/extractor/makertv.py      |  27 +++
 6 files changed, 103 insertions(+), 310 deletions(-)
 delete mode 100644 youtube_dl/extractor/bliptv.py
 create mode 100644 youtube_dl/extractor/jwplatform.py
 create mode 100644 youtube_dl/extractor/makertv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 462717b1e..f9c40e6cd 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -54,7 +54,6 @@ from .bet import BetIE
 from .bild import BildIE
 from .bilibili import BiliBiliIE
 from .blinkx import BlinkxIE
-from .bliptv import BlipTVIE, BlipTVUserIE
 from .bloomberg import BloombergIE
 from .bpb import BpbIE
 from .br import BRIE
@@ -263,6 +262,7 @@ from .jadorecettepub import JadoreCettePubIE
 from .jeuxvideo import JeuxVideoIE
 from .jove import JoveIE
 from .jukebox import JukeboxIE
+from .jwplatform import JWPlatformIE
 from .jpopsukitv import JpopsukiIE
 from .kaltura import KalturaIE
 from .kanalplay import KanalPlayIE
@@ -317,6 +317,7 @@ from .lynda import (
 from .m6 import M6IE
 from .macgamestore import MacGameStoreIE
 from .mailru import MailRuIE
+from .makertv import MakerTVIE
 from .malemotion import MalemotionIE
 from .mdr import MDRIE
 from .metacafe import MetacafeIE
diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
deleted file mode 100644
index c3296283d..000000000
--- a/youtube_dl/extractor/bliptv.py
+++ /dev/null
@@ -1,292 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-
-from ..compat import (
-    compat_urllib_request,
-    compat_urlparse,
-)
-from ..utils import (
-    clean_html,
-    int_or_none,
-    parse_iso8601,
-    unescapeHTML,
-    xpath_text,
-    xpath_with_ns,
-)
-
-
-class BlipTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:\w+\.)?blip\.tv/(?:(?:.+-|rss/flash/)(?P<id>\d+)|((?:play/|api\.swf#)(?P<lookup_id>[\da-zA-Z+_]+)))'
-
-    _TESTS = [
-        {
-            'url': 'http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352',
-            'md5': '80baf1ec5c3d2019037c1c707d676b9f',
-            'info_dict': {
-                'id': '5779306',
-                'ext': 'm4v',
-                'title': 'CBR EXCLUSIVE: "Gotham City Imposters" Bats VS Jokerz Short 3',
-                'description': 'md5:9bc31f227219cde65e47eeec8d2dc596',
-                'timestamp': 1323138843,
-                'upload_date': '20111206',
-                'uploader': 'cbr',
-                'uploader_id': '679425',
-                'duration': 81,
-            }
-        },
-        {
-            # https://github.com/rg3/youtube-dl/pull/2274
-            'note': 'Video with subtitles',
-            'url': 'http://blip.tv/play/h6Uag5OEVgI.html',
-            'md5': '309f9d25b820b086ca163ffac8031806',
-            'info_dict': {
-                'id': '6586561',
-                'ext': 'mp4',
-                'title': 'Red vs. Blue Season 11 Episode 1',
-                'description': 'One-Zero-One',
-                'timestamp': 1371261608,
-                'upload_date': '20130615',
-                'uploader': 'redvsblue',
-                'uploader_id': '792887',
-                'duration': 279,
-            }
-        },
-        {
-            # https://bugzilla.redhat.com/show_bug.cgi?id=967465
-            'url': 'http://a.blip.tv/api.swf#h6Uag5KbVwI',
-            'md5': '314e87b1ebe7a48fcbfdd51b791ce5a6',
-            'info_dict': {
-                'id': '6573122',
-                'ext': 'mov',
-                'upload_date': '20130520',
-                'description': 'Two hapless space marines argue over what to do when they realize they have an astronomically huge problem on their hands.',
-                'title': 'Red vs. Blue Season 11 Trailer',
-                'timestamp': 1369029609,
-                'uploader': 'redvsblue',
-                'uploader_id': '792887',
-            }
-        },
-        {
-            'url': 'http://blip.tv/play/gbk766dkj4Yn',
-            'md5': 'fe0a33f022d49399a241e84a8ea8b8e3',
-            'info_dict': {
-                'id': '1749452',
-                'ext': 'mp4',
-                'upload_date': '20090208',
-                'description': 'Witness the first appearance of the Nostalgia Critic character, as Doug reviews the movie Transformers.',
-                'title': 'Nostalgia Critic: Transformers',
-                'timestamp': 1234068723,
-                'uploader': 'NostalgiaCritic',
-                'uploader_id': '246467',
-            }
-        },
-        {
-            # https://github.com/rg3/youtube-dl/pull/4404
-            'note': 'Audio only',
-            'url': 'http://blip.tv/hilarios-productions/weekly-manga-recap-kingdom-7119982',
-            'md5': '76c0a56f24e769ceaab21fbb6416a351',
-            'info_dict': {
-                'id': '7103299',
-                'ext': 'flv',
-                'title': 'Weekly Manga Recap: Kingdom',
-                'description': 'And then Shin breaks the enemy line, and he&apos;s all like HWAH! And then he slices a guy and it&apos;s all like FWASHING! And... it&apos;s really hard to describe the best parts of this series without breaking down into sound effects, okay?',
-                'timestamp': 1417660321,
-                'upload_date': '20141204',
-                'uploader': 'The Rollo T',
-                'uploader_id': '407429',
-                'duration': 7251,
-                'vcodec': 'none',
-            }
-        },
-        {
-            # missing duration
-            'url': 'http://blip.tv/rss/flash/6700880',
-            'info_dict': {
-                'id': '6684191',
-                'ext': 'm4v',
-                'title': 'Cowboy Bebop: Gateway Shuffle Review',
-                'description': 'md5:3acc480c0f9ae157f5fe88547ecaf3f8',
-                'timestamp': 1386639757,
-                'upload_date': '20131210',
-                'uploader': 'sfdebris',
-                'uploader_id': '706520',
-            }
-        }
-    ]
-
-    @staticmethod
-    def _extract_url(webpage):
-        mobj = re.search(r'<meta\s[^>]*https?://api\.blip\.tv/\w+/redirect/\w+/(\d+)', webpage)
-        if mobj:
-            return 'http://blip.tv/a/a-' + mobj.group(1)
-        mobj = re.search(r'<(?:iframe|embed|object)\s[^>]*(https?://(?:\w+\.)?blip\.tv/(?:play/|api\.swf#)[a-zA-Z0-9_]+)', webpage)
-        if mobj:
-            return mobj.group(1)
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        lookup_id = mobj.group('lookup_id')
-
-        # See https://github.com/rg3/youtube-dl/issues/857 and
-        # https://github.com/rg3/youtube-dl/issues/4197
-        if lookup_id:
-            urlh = self._request_webpage(
-                'http://blip.tv/play/%s' % lookup_id, lookup_id, 'Resolving lookup id')
-            url = compat_urlparse.urlparse(urlh.geturl())
-            qs = compat_urlparse.parse_qs(url.query)
-            mobj = re.match(self._VALID_URL, qs['file'][0])
-
-        video_id = mobj.group('id')
-
-        rss = self._download_xml('http://blip.tv/rss/flash/%s' % video_id, video_id, 'Downloading video RSS')
-
-        def _x(p):
-            return xpath_with_ns(p, {
-                'blip': 'http://blip.tv/dtd/blip/1.0',
-                'media': 'http://search.yahoo.com/mrss/',
-                'itunes': 'http://www.itunes.com/dtds/podcast-1.0.dtd',
-            })
-
-        item = rss.find('channel/item')
-
-        video_id = xpath_text(item, _x('blip:item_id'), 'video id') or lookup_id
-        title = xpath_text(item, 'title', 'title', fatal=True)
-        description = clean_html(xpath_text(item, _x('blip:puredescription'), 'description'))
-        timestamp = parse_iso8601(xpath_text(item, _x('blip:datestamp'), 'timestamp'))
-        uploader = xpath_text(item, _x('blip:user'), 'uploader')
-        uploader_id = xpath_text(item, _x('blip:userid'), 'uploader id')
-        duration = int_or_none(xpath_text(item, _x('blip:runtime'), 'duration'))
-        media_thumbnail = item.find(_x('media:thumbnail'))
-        thumbnail = (media_thumbnail.get('url') if media_thumbnail is not None
-                     else xpath_text(item, 'image', 'thumbnail'))
-        categories = [category.text for category in item.findall('category') if category is not None]
-
-        formats = []
-        subtitles_urls = {}
-
-        media_group = item.find(_x('media:group'))
-        for media_content in media_group.findall(_x('media:content')):
-            url = media_content.get('url')
-            role = media_content.get(_x('blip:role'))
-            msg = self._download_webpage(
-                url + '?showplayer=20140425131715&referrer=http://blip.tv&mask=7&skin=flashvars&view=url',
-                video_id, 'Resolving URL for %s' % role)
-            real_url = compat_urlparse.parse_qs(msg.strip())['message'][0]
-
-            media_type = media_content.get('type')
-            if media_type == 'text/srt' or url.endswith('.srt'):
-                LANGS = {
-                    'english': 'en',
-                }
-                lang = role.rpartition('-')[-1].strip().lower()
-                langcode = LANGS.get(lang, lang)
-                subtitles_urls[langcode] = url
-            elif media_type.startswith('video/'):
-                formats.append({
-                    'url': real_url,
-                    'format_id': role,
-                    'format_note': media_type,
-                    'vcodec': media_content.get(_x('blip:vcodec')) or 'none',
-                    'acodec': media_content.get(_x('blip:acodec')),
-                    'filesize': media_content.get('filesize'),
-                    'width': int_or_none(media_content.get('width')),
-                    'height': int_or_none(media_content.get('height')),
-                })
-        self._check_formats(formats, video_id)
-        self._sort_formats(formats)
-
-        subtitles = self.extract_subtitles(video_id, subtitles_urls)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'timestamp': timestamp,
-            'uploader': uploader,
-            'uploader_id': uploader_id,
-            'duration': duration,
-            'thumbnail': thumbnail,
-            'categories': categories,
-            'formats': formats,
-            'subtitles': subtitles,
-        }
-
-    def _get_subtitles(self, video_id, subtitles_urls):
-        subtitles = {}
-        for lang, url in subtitles_urls.items():
-            # For some weird reason, blip.tv serves a video instead of subtitles
-            # when we request with a common UA
-            req = compat_urllib_request.Request(url)
-            req.add_header('User-Agent', 'youtube-dl')
-            subtitles[lang] = [{
-                # The extension is 'srt' but it's actually an 'ass' file
-                'ext': 'ass',
-                'data': self._download_webpage(req, None, note=False),
-            }]
-        return subtitles
-
-
-class BlipTVUserIE(InfoExtractor):
-    _VALID_URL = r'(?:(?:https?://(?:\w+\.)?blip\.tv/)|bliptvuser:)(?!api\.swf)([^/]+)/*$'
-    _PAGE_SIZE = 12
-    IE_NAME = 'blip.tv:user'
-    _TEST = {
-        'url': 'http://blip.tv/actone',
-        'info_dict': {
-            'id': 'actone',
-            'title': 'Act One: The Series',
-        },
-        'playlist_count': 5,
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        username = mobj.group(1)
-
-        page_base = 'http://m.blip.tv/pr/show_get_full_episode_list?users_id=%s&lite=0&esi=1'
-
-        page = self._download_webpage(url, username, 'Downloading user page')
-        mobj = re.search(r'data-users-id="([^"]+)"', page)
-        page_base = page_base % mobj.group(1)
-        title = self._og_search_title(page)
-
-        # Download video ids using BlipTV Ajax calls. Result size per
-        # query is limited (currently to 12 videos) so we need to query
-        # page by page until there are no video ids - it means we got
-        # all of them.
-
-        video_ids = []
-        pagenum = 1
-
-        while True:
-            url = page_base + "&page=" + str(pagenum)
-            page = self._download_webpage(
-                url, username, 'Downloading video ids from page %d' % pagenum)
-
-            # Extract video identifiers
-            ids_in_page = []
-
-            for mobj in re.finditer(r'href="/([^"]+)"', page):
-                if mobj.group(1) not in ids_in_page:
-                    ids_in_page.append(unescapeHTML(mobj.group(1)))
-
-            video_ids.extend(ids_in_page)
-
-            # A little optimization - if current page is not
-            # "full", ie. does not contain PAGE_SIZE video ids then
-            # we can assume that this page is the last one - there
-            # are no more ids on further pages - no need to query
-            # again.
-
-            if len(ids_in_page) < self._PAGE_SIZE:
-                break
-
-            pagenum += 1
-
-        urls = ['http://blip.tv/%s' % video_id for video_id in video_ids]
-        url_entries = [self.url_result(vurl, 'BlipTV') for vurl in urls]
-        return self.playlist_result(
-            url_entries, playlist_title=title, playlist_id=username)
diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
index fd1770dac..6d9cd8abd 100644
--- a/youtube_dl/extractor/cinemassacre.py
+++ b/youtube_dl/extractor/cinemassacre.py
@@ -5,7 +5,6 @@ import re
 
 from .common import InfoExtractor
 from ..utils import ExtractorError
-from .bliptv import BlipTVIE
 from .screenwavemedia import ScreenwaveMediaIE
 
 
@@ -34,18 +33,17 @@ class CinemassacreIE(InfoExtractor):
             },
         },
         {
-            # blip.tv embedded video
+            # Youtube embedded video
             'url': 'http://cinemassacre.com/2006/12/07/chronologically-confused-about-bad-movie-and-video-game-sequel-titles/',
-            'md5': 'ca9b3c8dd5a66f9375daeb5135f5a3de',
+            'md5': 'df4cf8a1dcedaec79a73d96d83b99023',
             'info_dict': {
-                'id': '4065369',
-                'ext': 'flv',
+                'id': 'OEVzPCY2T-g',
+                'ext': 'mp4',
                 'title': 'AVGN: Chronologically Confused about Bad Movie and Video Game Sequel Titles',
                 'upload_date': '20061207',
-                'uploader': 'cinemassacre',
-                'uploader_id': '250778',
-                'timestamp': 1283233867,
-                'description': 'md5:0a108c78d130676b207d0f6d029ecffd',
+                'uploader': 'Cinemassacre',
+                'uploader_id': 'JamesNintendoNerd',
+                'description': 'md5:784734696c2b8b7f4b8625cc799e07f6',
             }
         },
         {
@@ -88,8 +86,6 @@ class CinemassacreIE(InfoExtractor):
                 r'<iframe[^>]+src="(?P<url>(?:https?:)?//(?:[^.]+\.)?youtube\.com/.+?)"',
             ],
             webpage, 'player data URL', default=None, group='url')
-        if not playerdata_url:
-            playerdata_url = BlipTVIE._extract_url(webpage)
         if not playerdata_url:
             raise ExtractorError('Unable to find player data')
 
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ca5fbafb2..285c0ff66 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -41,7 +41,6 @@ from .myvi import MyviIE
 from .condenast import CondeNastIE
 from .udn import UDNEmbedIE
 from .senateisvp import SenateISVPIE
-from .bliptv import BlipTVIE
 from .svt import SVTIE
 from .pornhub import PornHubIE
 from .xhamster import XHamsterEmbedIE
@@ -1389,11 +1388,6 @@ class GenericIE(InfoExtractor):
                 'id': match.group('id')
             }
 
-        # Look for embedded blip.tv player
-        bliptv_url = BlipTVIE._extract_url(webpage)
-        if bliptv_url:
-            return self.url_result(bliptv_url, 'BlipTV')
-
         # Look for SVT player
         svt_url = SVTIE._extract_url(webpage)
         if svt_url:
diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
new file mode 100644
index 000000000..3a3dc439a
--- /dev/null
+++ b/youtube_dl/extractor/jwplatform.py
@@ -0,0 +1,67 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class JWPlatformIE(InfoExtractor):
+    _VALID_URL = r'(?:https?://content\.jwplatform\.com/(?:feeds|players|jw6)/|jwplatform:)(?P<id>[a-zA-Z0-9]{8})'
+    _TEST = {
+        'url': 'http://content.jwplatform.com/players/nPripu9l-ALJ3XQCI.js',
+        'md5': 'fa8899fa601eb7c83a64e9d568bdf325',
+        'info_dict': {
+            'id': 'nPripu9l',
+            'ext': 'mov',
+            'title': 'Big Buck Bunny Trailer',
+            'description': 'Big Buck Bunny is a short animated film by the Blender Institute. It is made using free and open source software.',
+            'upload_date': '20081127',
+            'timestamp': 1227796140,
+        }
+    }
+
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<script[^>]+?src=["\'](?P<url>(?:https?:)?//content.jwplatform.com/players/[a-zA-Z0-9]{8}',
+            webpage)
+        if mobj:
+            return mobj.group('url')
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        json_data = self._download_json('http://content.jwplatform.com/feeds/%s.json' % video_id, video_id)
+        video_data = json_data['playlist'][0]
+        subtitles = {}
+        for track in video_data['tracks']:
+            if track['kind'] == 'captions':
+                subtitles[track['label']] = [{'url': self._proto_relative_url(track['file'])}]
+
+        formats = []
+        for source in video_data['sources']:
+            source_url = self._proto_relative_url(source['file'])
+            source_type = source.get('type') or ''
+            if source_type == 'application/vnd.apple.mpegurl':
+                formats.extend(self._extract_m3u8_formats(source_url, video_id, 'mp4', 'm3u8_native', fatal=None))
+            elif source_type.startswith('audio'):
+                formats.append({
+                    'url': source_url,
+                    'vcodec': 'none',
+                })
+            else:
+                formats.append({
+                    'url': source_url,
+                    'width': int_or_none(source.get('width')),
+                    'height': int_or_none(source.get('height')),
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_data['mediaid'],
+            'title': video_data['title'],
+            'description': video_data.get('description'),
+            'thumbnail': self._proto_relative_url(video_data.get('image')),
+            'timestamp': int_or_none(video_data.get('pubdate')),
+            'subtitles': subtitles,
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/makertv.py b/youtube_dl/extractor/makertv.py
new file mode 100644
index 000000000..0256e4e24
--- /dev/null
+++ b/youtube_dl/extractor/makertv.py
@@ -0,0 +1,27 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class MakerTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:(?:www\.)?maker\.tv/(?:[^/]+/)?video|http://makerplayer.com/embed/maker)/(?P<id>[a-zA-Z0-9]{12})'
+    _TEST = {
+        'url': 'http://www.maker.tv/video/Fh3QgymL9gsc',
+        'md5': 'ca237a53a8eb20b6dc5bd60564d4ab3e',
+        'info_dict': {
+            'id': 'brOEcGut',
+            'ext': 'mp4',
+            'title': 'Maze Runner: The Scorch Trials Official Movie Review',
+            'description': 'md5:11ff3362d7ef1d679fdb649f6413975a',
+            'upload_date': '20150918',
+            'timestamp': 1442549540,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        jwplatform_id = self._search_regex([r'jwid="([^"]+)"', r'Maker.jw_id\s*=\s*"([^"]+)";'], webpage, 'jwplatform id')
+
+        return self.url_result('jwplatform:%s' % jwplatform_id, 'JWPlatform')

From 00d24327efcac74b11dbc4d813aed74da9a501e0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 30 Oct 2015 09:48:56 +0100
Subject: [PATCH 0047/1105] [vgtv] extract videos from FTV, Aftenposten,
 Aftonbladet using VGTVIE

---
 youtube_dl/extractor/__init__.py    |  1 -
 youtube_dl/extractor/aftenposten.py | 23 -----------
 youtube_dl/extractor/vgtv.py        | 60 ++++++++++++++++++++++-------
 3 files changed, 46 insertions(+), 38 deletions(-)
 delete mode 100644 youtube_dl/extractor/aftenposten.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f98e6487e..f7dcabcf7 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -9,7 +9,6 @@ from .adobetv import (
     AdobeTVVideoIE,
 )
 from .adultswim import AdultSwimIE
-from .aftenposten import AftenpostenIE
 from .aftonbladet import AftonbladetIE
 from .airmozilla import AirMozillaIE
 from .aljazeera import AlJazeeraIE
diff --git a/youtube_dl/extractor/aftenposten.py b/youtube_dl/extractor/aftenposten.py
deleted file mode 100644
index 0c00acfb5..000000000
--- a/youtube_dl/extractor/aftenposten.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-
-
-class AftenpostenIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?aftenposten\.no/webtv/(?:#!/)?video/(?P<id>\d+)'
-    _TEST = {
-        'url': 'http://www.aftenposten.no/webtv/#!/video/21039/trailer-sweatshop-i-can-t-take-any-more',
-        'md5': 'fd828cd29774a729bf4d4425fe192972',
-        'info_dict': {
-            'id': '21039',
-            'ext': 'mov',
-            'title': 'TRAILER: "Sweatshop" - I can´t take any more',
-            'description': 'md5:21891f2b0dd7ec2f78d84a50e54f8238',
-            'timestamp': 1416927969,
-            'upload_date': '20141125',
-        }
-    }
-
-    def _real_extract(self, url):
-        return self.url_result('xstream:ap:%s' % self._match_id(url), 'Xstream')
diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index f38a72fde..17213d9b6 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -11,16 +11,17 @@ from ..utils import (
 
 
 class VGTVIE(InfoExtractor):
-    IE_DESC = 'VGTV and BTTV'
+    IE_DESC = 'VGTV, BTTV, FTV, Aftenposten, Aftonbladet'
     _VALID_URL = r'''(?x)
                     (?:
                         vgtv:|
                         http://(?:www\.)?
                     )
-                    (?P<host>vgtv|bt)
+                    (?P<host>vgtv.no|(?:bt.no|aftenbladet.no)/tv|fvn.no/fvntv|aftenposten.no/webtv)
                     (?:
                         :|
-                        \.no/(?:tv/)?\#!/(?:video|live)/
+                        /\#!/(?:video|live)/|
+                        /embed?id=
                     )
                     (?P<id>[0-9]+)
                     '''
@@ -59,17 +60,18 @@ class VGTVIE(InfoExtractor):
                 # m3u8 download
                 'skip_download': True,
             },
+            'skip': 'Video is no longer available',
         },
         {
-            # streamType: live
+            # streamType: wasLive
             'url': 'http://www.vgtv.no/#!/live/113063/direkte-v75-fra-solvalla',
             'info_dict': {
                 'id': '113063',
-                'ext': 'flv',
-                'title': 're:^DIREKTE: V75 fra Solvalla [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+                'ext': 'mp4',
+                'title': 'V75 fra Solvalla 30.05.15',
                 'description': 'md5:b3743425765355855f88e096acc93231',
                 'thumbnail': 're:^https?://.*\.jpg',
-                'duration': 0,
+                'duration': 25966,
                 'timestamp': 1432975582,
                 'upload_date': '20150530',
                 'view_count': int,
@@ -78,26 +80,56 @@ class VGTVIE(InfoExtractor):
                 # m3u8 download
                 'skip_download': True,
             },
+        },{
+            'url': 'http://www.aftenposten.no/webtv/#!/video/21039/trailer-sweatshop-i-can-t-take-any-more',
+            'md5': '7fbc265a3ca4933a423c7a66aa879a67',
+            'info_dict': {
+                'id': '21039',
+                'ext': 'mp4',
+                'title': 'TRAILER: «SWEATSHOP» - I can´t take any more',
+                'description': 'md5:21891f2b0dd7ec2f78d84a50e54f8238',
+                'duration': 66,
+                'timestamp': 1417002452,
+                'upload_date': '20141126',
+                'view_count': int,
+            }
         },
         {
             'url': 'http://www.bt.no/tv/#!/video/100250/norling-dette-er-forskjellen-paa-1-divisjon-og-eliteserien',
             'only_matching': True,
         },
     ]
+    _HOST_WEBSITES = {
+        'vgtv.no': {
+            'vendor': 'vgtv',
+            'appname': 'vgtv',
+        },
+        'bt.no/tv': {
+            'vendor': 'bt',
+            'appname': 'bttv',
+        },
+        'aftenbladet.no/tv': {
+            'vendor': 'sa',
+            'appname': 'satv',
+        },
+        'fvn.no/fvntv': {
+            'vendor': 'fvn',
+            'appname': 'fvntv',
+        },
+        'aftenposten.no/webtv': {
+            'vendor': 'ap',
+            'appname': 'aptv',
+        },
+    }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         host = mobj.group('host')
 
-        HOST_WEBSITES = {
-            'vgtv': 'vgtv',
-            'bt': 'bttv',
-        }
-
         data = self._download_json(
             'http://svp.vg.no/svp/api/v1/%s/assets/%s?appName=%s-website'
-            % (host, video_id, HOST_WEBSITES[host]),
+            % (self._HOST_WEBSITES[host]['vendor'], video_id, self._HOST_WEBSITES[host]['appname']),
             video_id, 'Downloading media JSON')
 
         if data.get('status') == 'inactive':
@@ -144,7 +176,7 @@ class VGTVIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': self._live_title(data['title']),
+            'title': self._live_title(data['title']) if stream_type == 'live' else data['title'],
             'description': data['description'],
             'thumbnail': data['images']['main'] + '?t[]=900x506q80',
             'timestamp': data['published'],

From 804afc5871a88eaa32a6c161df67e6b37383d7d1 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 30 Oct 2015 10:20:38 +0100
Subject: [PATCH 0048/1105] [vgtv] improve _VALID_URL regex

---
 youtube_dl/extractor/vgtv.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 17213d9b6..e8039ec7f 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -11,17 +11,19 @@ from ..utils import (
 
 
 class VGTVIE(InfoExtractor):
-    IE_DESC = 'VGTV, BTTV, FTV, Aftenposten, Aftonbladet'
+    IE_DESC = 'VGTV, BTTV, FTV, Aftenposten and Aftonbladet'
     _VALID_URL = r'''(?x)
-                    (?:
-                        vgtv:|
-                        http://(?:www\.)?
+                    http://(?:www\.)?
+                    (?P<host>
+                        vgtv.no|
+                        (?:bt|aftenbladet).no/tv|
+                        fvn.no/fvntv|
+                        aftenposten.no/webtv
                     )
-                    (?P<host>vgtv.no|(?:bt.no|aftenbladet.no)/tv|fvn.no/fvntv|aftenposten.no/webtv)
+                    /
                     (?:
-                        :|
-                        /\#!/(?:video|live)/|
-                        /embed?id=
+                        \#!/(?:video|live)/|
+                        embed?.*id=
                     )
                     (?P<id>[0-9]+)
                     '''
@@ -211,7 +213,7 @@ class BTArticleIE(InfoExtractor):
         webpage = self._download_webpage(url, self._match_id(url))
         video_id = self._search_regex(
             r'SVP\.Player\.load\(\s*(\d+)', webpage, 'video id')
-        return self.url_result('vgtv:bt:%s' % video_id, 'VGTV')
+        return self.url_result('http://bt.no/tv/embed?id=%s' % video_id, 'VGTV')
 
 
 class BTVestlendingenIE(InfoExtractor):

From 957e0db1d2d1d4b2e9fdd1c314b0c46e68ca0cb3 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 30 Oct 2015 13:56:21 +0100
Subject: [PATCH 0049/1105] [baidu] improve info extraction

---
 youtube_dl/extractor/baidu.py | 45 +++++++++++++----------------------
 1 file changed, 16 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py
index e37ee4440..84fab551b 100644
--- a/youtube_dl/extractor/baidu.py
+++ b/youtube_dl/extractor/baidu.py
@@ -14,8 +14,8 @@ class BaiduVideoIE(InfoExtractor):
         'url': 'http://v.baidu.com/comic/1069.htm?frp=bdbrand&q=%E4%B8%AD%E5%8D%8E%E5%B0%8F%E5%BD%93%E5%AE%B6',
         'info_dict': {
             'id': '1069',
-            'title': '中华小当家 TV版 (全52集)',
-            'description': 'md5:395a419e41215e531c857bb037bbaf80',
+            'title': '中华小当家 TV版国语',
+            'description': 'md5:40a9c1b1c7f4e05d642e7bb1c84eeda0',
         },
         'playlist_count': 52,
     }, {
@@ -25,45 +25,32 @@ class BaiduVideoIE(InfoExtractor):
             'title': 're:^奔跑吧兄弟',
             'description': 'md5:1bf88bad6d850930f542d51547c089b8',
         },
-        'playlist_mincount': 3,
+        'playlist_mincount': 12,
     }]
 
+    def _call_api(self, path, category, playlist_id):
+        return self._download_json('http://app.video.baidu.com/%s/?worktype=adnative%s&id=%s' % (path, category, playlist_id), playlist_id)
+
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        playlist_id = mobj.group('id')
-        category = category2 = mobj.group('type')
+        category, playlist_id = re.match(self._VALID_URL, url).groups()
         if category == 'show':
-            category2 = 'tvshow'
+            category = 'tvshow'
+        if category == 'tv':
+            category = 'tvplay'
 
-        webpage = self._download_webpage(url, playlist_id)
+        playlist_detail = self._call_api('xqinfo', category, playlist_id)
 
-        playlist_title = self._html_search_regex(
-            r'title\s*:\s*(["\'])(?P<title>[^\']+)\1', webpage,
-            'playlist title', group='title')
-        playlist_description = self._html_search_regex(
-            r'<input[^>]+class="j-data-intro"[^>]+value="([^"]+)"/>', webpage,
-            playlist_id, 'playlist description')
+        playlist_title = playlist_detail['title']
+        playlist_description = playlist_detail.get('intro')
 
-        site = self._html_search_regex(
-            r'filterSite\s*:\s*["\']([^"]*)["\']', webpage,
-            'primary provider site')
-        api_result = self._download_json(
-            'http://v.baidu.com/%s_intro/?dtype=%sPlayUrl&id=%s&site=%s' % (
-                category, category2, playlist_id, site),
-            playlist_id, 'Get playlist links')
+        episodes_detail = self._call_api('xqsingle', category, playlist_id)
 
         entries = []
-        for episode in api_result[0]['episodes']:
+        for episode in episodes_detail['videos']:
             episode_id = '%s_%s' % (playlist_id, episode['episode'])
 
-            redirect_page = self._download_webpage(
-                compat_urlparse.urljoin(url, episode['url']), episode_id,
-                note='Download Baidu redirect page')
-            real_url = self._html_search_regex(
-                r'location\.replace\("([^"]+)"\)', redirect_page, 'real URL')
-
             entries.append(self.url_result(
-                real_url, video_title=episode['single_title']))
+                episode['url'], video_title=episode['title']))
 
         return self.playlist_result(
             entries, playlist_id, playlist_title, playlist_description)

From 720334659ac13d7f5bc0759662ba0534f48a1ce4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 31 Oct 2015 01:08:37 +0100
Subject: [PATCH 0050/1105] [daum] improve info extraction

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/daum.py     | 123 +++++++++++++++++++------------
 2 files changed, 79 insertions(+), 49 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f98e6487e..99b4be002 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -119,7 +119,10 @@ from .dailymotion import (
     DailymotionUserIE,
     DailymotionCloudIE,
 )
-from .daum import DaumIE
+from .daum import (
+    DaumIE,
+    DaumClipIE,
+)
 from .dbtv import DBTVIE
 from .dcn import DCNIE
 from .dctp import DctpTvIE
diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index 934da765e..0f5686e07 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -2,16 +2,73 @@
 
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse
+from ..utils import int_or_none
 
 
 class DaumIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/(?:v/|.*?clipid=)(?P<id>[^?#&]+)'
+    _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/v/(?P<id>[^?#&]+)'
+    IE_NAME = 'daum.net'
+
+    _TESTS = [{
+        'url': 'http://tvpot.daum.net/v/vab4dyeDBysyBssyukBUjBz',
+        'info_dict': {
+            'id': 'vab4dyeDBysyBssyukBUjBz',
+            'ext': 'mp4',
+            'title': '마크 헌트 vs 안토니오 실바',
+            'description': 'Mark Hunt vs Antonio Silva',
+            'upload_date': '20131217',
+            'duration': 2117,
+        },
+    }, {
+        'url': 'http://tvpot.daum.net/v/07dXWRka62Y%24',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        query = compat_urllib_parse.urlencode({'vid': video_id})
+        info = self._download_xml(
+            'http://tvpot.daum.net/clip/ClipInfoXml.do?' + query, video_id,
+            'Downloading video info')
+        movie_data = self._download_json(
+            'http://videofarm.daum.net/controller/api/closed/v1_2/IntegratedMovieData.json?' + query,
+            video_id, 'Downloading video formats info')
+
+        formats = []
+        for format_el in movie_data['output_list']['output_list']:
+            profile = format_el['profile']
+            format_query = compat_urllib_parse.urlencode({
+                'vid': video_id,
+                'profile': profile,
+            })
+            url_doc = self._download_xml(
+                'http://videofarm.daum.net/controller/api/open/v1_2/MovieLocation.apixml?' + format_query,
+                video_id, note='Downloading video data for %s format' % profile)
+            format_url = url_doc.find('result/url').text
+            formats.append({
+                'url': format_url,
+                'format_id': profile,
+                'width': int_or_none(format_el.get('width')),
+                'height': int_or_none(format_el.get('height')),
+                'filesize': int_or_none(format_el.get('filesize')),
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': info.find('TITLE').text,
+            'formats': formats,
+            'thumbnail': info.find('THUMB_URL').text,
+            'description': info.find('CONTENTS').text,
+            'duration': int_or_none(info.find('DURATION').text),
+            'upload_date': info.find('REGDTTM').text[:8],
+        }
+
+
+class DaumClipIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/.*?clipid=(?P<id>\d+)'
     IE_NAME = 'daum.net'
 
     _TESTS = [{
@@ -23,53 +80,23 @@ class DaumIE(InfoExtractor):
             'description': 'DOTA 2GETHER 시즌2 6회 - 2부',
             'upload_date': '20130831',
             'duration': 3868,
+            'view_count': int,
         },
-    }, {
-        'url': 'http://tvpot.daum.net/v/vab4dyeDBysyBssyukBUjBz',
-        'only_matching': True,
-    }, {
-        'url': 'http://tvpot.daum.net/v/07dXWRka62Y%24',
-        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        canonical_url = 'http://tvpot.daum.net/v/%s' % video_id
-        webpage = self._download_webpage(canonical_url, video_id)
-        full_id = self._search_regex(
-            r'src=["\']http://videofarm\.daum\.net/controller/video/viewer/Video\.html\?.*?vid=(.+?)[&"\']',
-            webpage, 'full id')
-        query = compat_urllib_parse.urlencode({'vid': full_id})
-        info = self._download_xml(
-            'http://tvpot.daum.net/clip/ClipInfoXml.do?' + query, video_id,
-            'Downloading video info')
-        urls = self._download_xml(
-            'http://videofarm.daum.net/controller/api/open/v1_2/MovieData.apixml?' + query,
-            video_id, 'Downloading video formats info')
-
-        formats = []
-        for format_el in urls.findall('result/output_list/output_list'):
-            profile = format_el.attrib['profile']
-            format_query = compat_urllib_parse.urlencode({
-                'vid': full_id,
-                'profile': profile,
-            })
-            url_doc = self._download_xml(
-                'http://videofarm.daum.net/controller/api/open/v1_2/MovieLocation.apixml?' + format_query,
-                video_id, note='Downloading video data for %s format' % profile)
-            format_url = url_doc.find('result/url').text
-            formats.append({
-                'url': format_url,
-                'format_id': profile,
-            })
+        video_id = self._match_id(url)
+        clip_info = self._download_json('http://tvpot.daum.net/mypot/json/GetClipInfo.do?clipid=%s' % video_id, video_id)['clip_bean']
 
         return {
+            '_type': 'url_transparent',
             'id': video_id,
-            'title': info.find('TITLE').text,
-            'formats': formats,
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'description': info.find('CONTENTS').text,
-            'duration': int(info.find('DURATION').text),
-            'upload_date': info.find('REGDTTM').text[:8],
+            'url': 'http://tvpot.daum.net/v/%s' % clip_info['vid'],
+            'title': clip_info['title'],
+            'thumbnail': clip_info.get('thumb_url'),
+            'description': clip_info.get('contents'),
+            'duration': int_or_none(clip_info.get('duration')),
+            'upload_date': clip_info.get('up_date')[:8],
+            'view_count': int_or_none(clip_info.get('play_count')),
+            'ie_key': 'Daum',
         }

From 50b9dd734423231f7a01ed8a156d09ca04a23a31 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 31 Oct 2015 15:40:11 +0100
Subject: [PATCH 0051/1105] [dcn] improve season info extraction

---
 youtube_dl/extractor/dcn.py | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 8b360a9d7..a9a5e94f5 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -210,16 +210,14 @@ class DCNSeasonIE(InfoExtractor):
             })
 
         show = self._download_json(request, show_id)
-        season_id = season_id or show['default_season']
-        season = {}
-        for _ in show['seasons']:
-            if _['id'] == season_id:
-                season = _
-                break
-        title = season.get('title_en') or season['title_ar']
+        if not season_id:
+            season_id = show['default_season']
+        for season in show['seasons']:
+            if season['id'] == season_id:
+                title = season.get('title_en') or season['title_ar']
 
-        entries = []
-        for video in show['videos']:
-            entries.append(self.url_result('http://www.dcndigital.ae/#/media/%s' % video['id'], 'DCNVideo'))
+                entries = []
+                for video in show['videos']:
+                    entries.append(self.url_result('http://www.dcndigital.ae/#/media/%s' % video['id'], 'DCNVideo'))
 
-        return self.playlist_result(entries, season_id, title)
+                return self.playlist_result(entries, season_id, title)

From 02fb9804513ce1bfe28ec7c285526db7989e5844 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 1 Nov 2015 02:08:19 +0100
Subject: [PATCH 0052/1105] [flickr] extract more info and formats

---
 youtube_dl/extractor/flickr.py | 110 ++++++++++++++++++---------------
 1 file changed, 60 insertions(+), 50 deletions(-)

diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index 2fe76d661..5ca754105 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -1,67 +1,77 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
+from ..compat import compat_urllib_parse
 from ..utils import (
-    ExtractorError,
-    find_xpath_attr,
+    int_or_none,
+    qualities,
 )
 
 
 class FlickrIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.|secure\.)?flickr\.com/photos/(?P<uploader_id>[\w\-_@]+)/(?P<id>\d+).*'
+    _VALID_URL = r'https?://(?:www\.|secure\.)?flickr\.com/photos/[\w\-_@]+/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.flickr.com/photos/forestwander-nature-pictures/5645318632/in/photostream/',
-        'md5': '6fdc01adbc89d72fc9c4f15b4a4ba87b',
+        'md5': '164fe3fa6c22e18d448d4d5af2330f31',
         'info_dict': {
             'id': '5645318632',
-            'ext': 'mp4',
-            "description": "Waterfalls in the Springtime at Dark Hollow Waterfalls. These are located just off of Skyline Drive in Virginia. They are only about 6/10 of a mile hike but it is a pretty steep hill and a good climb back up.",
-            "uploader_id": "forestwander-nature-pictures",
-            "title": "Dark Hollow Waterfalls"
+            'ext': 'mpg',
+            'description': 'Waterfalls in the Springtime at Dark Hollow Waterfalls. These are located just off of Skyline Drive in Virginia. They are only about 6/10 of a mile hike but it is a pretty steep hill and a good climb back up.',
+            'uploader_id': 'forestwander-nature-pictures',
+            'title': 'Dark Hollow Waterfalls',
+            'duration': 19,
+            'timestamp': 1303528740,
+            'upload_date': '20110423',
+            'uploader_id': '10922353@N03',
+            'uploader': 'Forest Wander',
+            'comment_count': int,
         }
     }
 
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
+    _API_BASE_URL = 'https://api.flickr.com/services/rest?'
+    _API_KEY = '61b16865f916058e63580a912d9143be'
 
-        video_id = mobj.group('id')
-        video_uploader_id = mobj.group('uploader_id')
-        webpage_url = 'http://www.flickr.com/photos/' + video_uploader_id + '/' + video_id
-        req = compat_urllib_request.Request(webpage_url)
-        req.add_header(
-            'User-Agent',
-            # it needs a more recent version
-            'Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20150101 Firefox/38.0 (Chrome)')
-        webpage = self._download_webpage(req, video_id)
-
-        secret = self._search_regex(r'secret"\s*:\s*"(\w+)"', webpage, 'secret')
-
-        first_url = 'https://secure.flickr.com/apps/video/video_mtl_xml.gne?v=x&photo_id=' + video_id + '&secret=' + secret + '&bitrate=700&target=_self'
-        first_xml = self._download_xml(first_url, video_id, 'Downloading first data webpage')
-
-        node_id = find_xpath_attr(
-            first_xml, './/{http://video.yahoo.com/YEP/1.0/}Item', 'id',
-            'id').text
-
-        second_url = 'https://secure.flickr.com/video_playlist.gne?node_id=' + node_id + '&tech=flash&mode=playlist&bitrate=700&secret=' + secret + '&rd=video.yahoo.com&noad=1'
-        second_xml = self._download_xml(second_url, video_id, 'Downloading second data webpage')
-
-        self.report_extraction(video_id)
-
-        stream = second_xml.find('.//STREAM')
-        if stream is None:
-            raise ExtractorError('Unable to extract video url')
-        video_url = stream.attrib['APP'] + stream.attrib['FULLPATH']
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'ext': 'mp4',
-            'title': self._og_search_title(webpage),
-            'description': self._og_search_description(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'uploader_id': video_uploader_id,
+    def _call_api(self, method, video_id, secret=None):
+        query = {
+            'photo_id': video_id,
+            'method': 'flickr.%s' % method,
+            'api_key': self._API_KEY,
+            'format': 'json',
+            'nojsoncallback': 1,
         }
+        if secret:
+            query['secret'] = secret
+        return self._download_json(self._API_BASE_URL + compat_urllib_parse.urlencode(query), video_id)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        video_info = self._call_api('photos.getInfo', video_id)['photo']
+        if video_info['media'] == 'video':
+            streams = self._call_api('video.getStreamInfo', video_id, video_info['secret'])['streams']
+
+            preference = qualities(['iphone_wifi', '700', 'appletv', 'orig'])
+
+            formats = []
+            for stream in streams['stream']:
+                stream_type = str(stream.get('type'))
+                formats.append({
+                    'format_id': stream_type,
+                    'url': stream['_content'],
+                    'preference': preference(stream_type),
+                })
+            self._sort_formats(formats)
+
+            owner = video_info.get('owner', {})
+
+            return {
+                'id': video_id,
+                'title': video_info['title']['_content'],
+                'description': video_info.get('description', {}).get('_content'),
+                'formats': formats,
+                'timestamp': int_or_none(video_info.get('dateuploaded')),
+                'duration': int_or_none(video_info.get('video', {}).get('duration')),
+                'uploader_id': owner.get('nsid'),
+                'uploader': owner.get('realname'),
+                'comment_count': int_or_none(video_info.get('comments', {}).get('_content')),
+            }

From 146672254e409bf97c82a302095fbfabf2c48928 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 1 Nov 2015 13:23:23 +0100
Subject: [PATCH 0053/1105] [flickr] extract fresh api key and remove
 duplication in test

---
 youtube_dl/extractor/flickr.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index 5ca754105..0d5d6b0b9 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -17,7 +17,6 @@ class FlickrIE(InfoExtractor):
             'id': '5645318632',
             'ext': 'mpg',
             'description': 'Waterfalls in the Springtime at Dark Hollow Waterfalls. These are located just off of Skyline Drive in Virginia. They are only about 6/10 of a mile hike but it is a pretty steep hill and a good climb back up.',
-            'uploader_id': 'forestwander-nature-pictures',
             'title': 'Dark Hollow Waterfalls',
             'duration': 19,
             'timestamp': 1303528740,
@@ -29,26 +28,27 @@ class FlickrIE(InfoExtractor):
     }
 
     _API_BASE_URL = 'https://api.flickr.com/services/rest?'
-    _API_KEY = '61b16865f916058e63580a912d9143be'
 
-    def _call_api(self, method, video_id, secret=None):
+    def _call_api(self, method, video_id, api_key, note, secret=None):
         query = {
             'photo_id': video_id,
             'method': 'flickr.%s' % method,
-            'api_key': self._API_KEY,
+            'api_key': api_key,
             'format': 'json',
             'nojsoncallback': 1,
         }
         if secret:
             query['secret'] = secret
-        return self._download_json(self._API_BASE_URL + compat_urllib_parse.urlencode(query), video_id)
+        return self._download_json(self._API_BASE_URL + compat_urllib_parse.urlencode(query), video_id, note)
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        video_info = self._call_api('photos.getInfo', video_id)['photo']
+        api_key = self._download_json('https://www.flickr.com/hermes_error_beacon.gne', video_id, 'Downloading api key',)['site_key']
+
+        video_info = self._call_api('photos.getInfo', video_id, api_key, 'Downloading video info')['photo']
         if video_info['media'] == 'video':
-            streams = self._call_api('video.getStreamInfo', video_id, video_info['secret'])['streams']
+            streams = self._call_api('video.getStreamInfo', video_id, api_key, 'Downloading streams info', video_info['secret'])['streams']
 
             preference = qualities(['iphone_wifi', '700', 'appletv', 'orig'])
 

From f3003531a5622cc01501325b9f35dcb2424cfb70 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 1 Nov 2015 13:38:11 +0100
Subject: [PATCH 0054/1105] [flickr] handle error message

---
 youtube_dl/extractor/flickr.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index 0d5d6b0b9..e97754d36 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse
 from ..utils import (
+    ExtractorError,
     int_or_none,
     qualities,
 )
@@ -39,7 +40,10 @@ class FlickrIE(InfoExtractor):
         }
         if secret:
             query['secret'] = secret
-        return self._download_json(self._API_BASE_URL + compat_urllib_parse.urlencode(query), video_id, note)
+        data = self._download_json(self._API_BASE_URL + compat_urllib_parse.urlencode(query), video_id, note)
+        if data['stat'] != 'ok':
+            raise ExtractorError(data['message'])
+        return data
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 967c9076a31ca2a2b43fb71082ad1a8db88116bd Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 5 Nov 2015 18:01:13 +0100
Subject: [PATCH 0055/1105] raise ExtractorError if the page doesn't contain a
 video

---
 youtube_dl/extractor/flickr.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index e97754d36..92d2ac553 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -79,3 +79,5 @@ class FlickrIE(InfoExtractor):
                 'uploader': owner.get('realname'),
                 'comment_count': int_or_none(video_info.get('comments', {}).get('_content')),
             }
+        else:
+            raise ExtractorError('not a video', expected=True)

From a641b2459263228fb1dd86dfe05d6047cedbf345 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 6 Nov 2015 07:23:03 +0100
Subject: [PATCH 0056/1105] [cnet] skip hls_phone if hls_tablet is present

---
 youtube_dl/extractor/cnet.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cnet.py b/youtube_dl/extractor/cnet.py
index 2fac0d79d..3ecf0efd4 100644
--- a/youtube_dl/extractor/cnet.py
+++ b/youtube_dl/extractor/cnet.py
@@ -57,7 +57,9 @@ class CNETIE(InfoExtractor):
         subtitles = {}
         description = vdata.get('description')
 
-        for vid in vdata['files'].values():
+        for (fkey, vid) in vdata['files'].items():
+            if fkey == 'hls_phone' and 'hls_tablet' in vdata['files']:
+                continue
             result = tp.extract(('http://link.theplatform.com/s/%s/%s' % (mpx_account, vid)))
             formats.extend(result['formats'])
             subtitles = self._merge_subtitles(subtitles, result['subtitles'])

From 3793090b1b1c1e3462b80dd3045a3573545cfb29 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 7 Nov 2015 16:54:35 +0100
Subject: [PATCH 0057/1105] [amp] Add generic extractor for Akamai AMP feeds
 and use it in dramafever and foxnews extractors

---
 youtube_dl/extractor/amp.py        | 84 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/dramafever.py | 65 ++++-------------------
 youtube_dl/extractor/foxnews.py    | 64 ++++-------------------
 3 files changed, 105 insertions(+), 108 deletions(-)
 create mode 100644 youtube_dl/extractor/amp.py

diff --git a/youtube_dl/extractor/amp.py b/youtube_dl/extractor/amp.py
new file mode 100644
index 000000000..b573b9280
--- /dev/null
+++ b/youtube_dl/extractor/amp.py
@@ -0,0 +1,84 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+)
+
+
+class AMPIE(InfoExtractor):
+    def _get_media_node(self, item, name, default=None):
+        media_name = 'media-%s' % name
+        media_group = item.get('media-group') or item
+        return media_group.get(media_name) or item.get(media_name) or item.get(name, default)
+
+    # parse Akamai Adaptive Media Player feed
+    def _extract_feed_info(self, url):
+        item = self._download_json(
+            url, None,
+            'Downloading Akamai AMP feed',
+            'Unable to download Akamai AMP feed'
+            )['channel']['item']
+
+        video_id = item['guid']
+        
+        thumbnails = []
+        media_thumbnail = self._get_media_node(item, 'thumbnail')
+        if media_thumbnail:
+            if isinstance(media_thumbnail, dict):
+                media_thumbnail = [media_thumbnail]
+            for thumbnail_data in media_thumbnail:
+                thumbnail = thumbnail_data['@attributes']
+                thumbnails.append({
+                    'url': self._proto_relative_url(thumbnail['url'], 'http:'),
+                    'width': int_or_none(thumbnail.get('width')),
+                    'height': int_or_none(thumbnail.get('height')),
+                })
+
+        subtitles = {}
+        media_subtitle = self._get_media_node(item, 'subTitle')
+        if media_subtitle:
+            if isinstance(media_subtitle, dict):
+                media_subtitle = [media_subtitle]
+            for subtitle_data in media_subtitle:
+                subtitle = subtitle_data['@attributes']
+                lang = subtitle.get('lang') or 'en'
+                subtitles[lang] = [{'url': subtitle['href']}]
+
+        formats = []
+        media_content = self._get_media_node(item, 'content')
+        if isinstance(media_content, dict):
+            media_content = [media_content]
+        for media_data in media_content:
+            media = media_data['@attributes']
+            media_type = media['type']
+            if media_type == 'video/f4m':
+                f4m_formats = self._extract_f4m_formats(media['url'] + '?hdcore=3.4.0&plugin=aasp-3.4.0.132.124', video_id, f4m_id='hds', fatal=False)
+                if f4m_formats:
+                    formats.extend(f4m_formats)
+            elif media_type == 'application/x-mpegURL':
+                m3u8_formats = self._extract_m3u8_formats(media['url'], video_id, m3u8_id='hls', fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
+            else:
+                formats.append({
+                    'format_id': media_data['media-category']['@attributes']['label'],
+                    'url': media['url'],
+                    'preference': 1,
+                    'vbr': int_or_none(media.get('bitrate')),
+                    'filesize': int_or_none(media.get('fileSize')),
+                })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': self._get_media_node(item, 'title'),
+            'description': self._get_media_node(item, 'description'),
+            'thumbnails': thumbnails,
+            'timestamp': parse_iso8601(item.get('pubDate'), ' '),
+            'duration': int_or_none(media_content[0].get('@attributes', {}).get('duration')),
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 38e6597c8..80a928827 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 
 import itertools
 
-from .common import InfoExtractor
+from .amp import AMPIE
 from ..compat import (
     compat_HTTPError,
     compat_urllib_parse,
@@ -19,7 +19,7 @@ from ..utils import (
 )
 
 
-class DramaFeverBaseIE(InfoExtractor):
+class DramaFeverBaseIE(AMPIE):
     _LOGIN_URL = 'https://www.dramafever.com/accounts/login/'
     _NETRC_MACHINE = 'dramafever'
 
@@ -80,60 +80,24 @@ class DramaFeverIE(DramaFeverBaseIE):
             'timestamp': 1404336058,
             'upload_date': '20140702',
             'duration': 343,
-        }
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url).replace('/', '.')
 
         try:
-            feed = self._download_json(
-                'http://www.dramafever.com/amp/episode/feed.json?guid=%s' % video_id,
-                video_id, 'Downloading episode JSON')['channel']['item']
+            info = self._extract_feed_info('http://www.dramafever.com/amp/episode/feed.json?guid=%s' % video_id)
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError):
                 raise ExtractorError(
                     'Currently unavailable in your country.', expected=True)
             raise
 
-        media_group = feed.get('media-group', {})
-
-        formats = []
-        for media_content in media_group['media-content']:
-            src = media_content.get('@attributes', {}).get('url')
-            if not src:
-                continue
-            ext = determine_ext(src)
-            if ext == 'f4m':
-                formats.extend(self._extract_f4m_formats(
-                    src, video_id, f4m_id='hds'))
-            elif ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    src, video_id, 'mp4', m3u8_id='hls'))
-            else:
-                formats.append({
-                    'url': src,
-                })
-        self._sort_formats(formats)
-
-        title = media_group.get('media-title')
-        description = media_group.get('media-description')
-        duration = int_or_none(media_group['media-content'][0].get('@attributes', {}).get('duration'))
-        thumbnail = self._proto_relative_url(
-            media_group.get('media-thumbnail', {}).get('@attributes', {}).get('url'))
-        timestamp = parse_iso8601(feed.get('pubDate'), ' ')
-
-        subtitles = {}
-        for media_subtitle in media_group.get('media-subTitle', []):
-            lang = media_subtitle.get('@attributes', {}).get('lang')
-            href = media_subtitle.get('@attributes', {}).get('href')
-            if not lang or not href:
-                continue
-            subtitles[lang] = [{
-                'ext': 'ttml',
-                'url': href,
-            }]
-
         series_id, episode_number = video_id.split('.')
         episode_info = self._download_json(
             # We only need a single episode info, so restricting page size to one episode
@@ -146,21 +110,12 @@ class DramaFeverIE(DramaFeverBaseIE):
             if value:
                 subfile = value[0].get('subfile') or value[0].get('new_subfile')
                 if subfile and subfile != 'http://www.dramafever.com/st/':
-                    subtitles.setdefault('English', []).append({
+                    info['subtitiles'].setdefault('English', []).append({
                         'ext': 'srt',
                         'url': subfile,
                     })
 
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'timestamp': timestamp,
-            'duration': duration,
-            'formats': formats,
-            'subtitles': subtitles,
-        }
+        return info
 
 
 class DramaFeverSeriesIE(DramaFeverBaseIE):
diff --git a/youtube_dl/extractor/foxnews.py b/youtube_dl/extractor/foxnews.py
index 3a4a59135..0cd0f9fa8 100644
--- a/youtube_dl/extractor/foxnews.py
+++ b/youtube_dl/extractor/foxnews.py
@@ -2,14 +2,14 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
+from .amp import AMPIE
 from ..utils import (
     parse_iso8601,
     int_or_none,
 )
 
 
-class FoxNewsIE(InfoExtractor):
+class FoxNewsIE(AMPIE):
     IE_DESC = 'Fox News and Fox Business Video'
     _VALID_URL = r'https?://(?P<host>video\.fox(?:news|business)\.com)/v/(?:video-embed\.html\?video_id=)?(?P<id>\d+)'
     _TESTS = [
@@ -20,10 +20,10 @@ class FoxNewsIE(InfoExtractor):
                 'id': '3937480',
                 'ext': 'flv',
                 'title': 'Frozen in Time',
-                'description': 'Doctors baffled by 16-year-old girl that is the size of a toddler',
+                'description': '16-year-old girl is size of toddler',
                 'duration': 265,
-                'timestamp': 1304411491,
-                'upload_date': '20110503',
+                #'timestamp': 1304411491,
+                #'upload_date': '20110503',
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
@@ -34,10 +34,10 @@ class FoxNewsIE(InfoExtractor):
                 'id': '3922535568001',
                 'ext': 'mp4',
                 'title': "Rep. Luis Gutierrez on if Obama's immigration plan is legal",
-                'description': "Congressman discusses the president's executive action",
+                'description': "Congressman discusses president's plan",
                 'duration': 292,
-                'timestamp': 1417662047,
-                'upload_date': '20141204',
+                #'timestamp': 1417662047,
+                #'upload_date': '20141204',
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
@@ -56,48 +56,6 @@ class FoxNewsIE(InfoExtractor):
         video_id = mobj.group('id')
         host = mobj.group('host')
 
-        video = self._download_json(
-            'http://%s/v/feed/video/%s.js?template=fox' % (host, video_id), video_id)
-
-        item = video['channel']['item']
-        title = item['title']
-        description = item['description']
-        timestamp = parse_iso8601(item['dc-date'])
-
-        media_group = item['media-group']
-        duration = None
-        formats = []
-        for media in media_group['media-content']:
-            attributes = media['@attributes']
-            video_url = attributes['url']
-            if video_url.endswith('.f4m'):
-                formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.0&plugin=aasp-3.4.0.132.124', video_id))
-            elif video_url.endswith('.m3u8'):
-                formats.extend(self._extract_m3u8_formats(video_url, video_id, 'flv'))
-            elif not video_url.endswith('.smil'):
-                duration = int_or_none(attributes.get('duration'))
-                formats.append({
-                    'url': video_url,
-                    'format_id': media['media-category']['@attributes']['label'],
-                    'preference': 1,
-                    'vbr': int_or_none(attributes.get('bitrate')),
-                    'filesize': int_or_none(attributes.get('fileSize'))
-                })
-        self._sort_formats(formats)
-
-        media_thumbnail = media_group['media-thumbnail']['@attributes']
-        thumbnails = [{
-            'url': media_thumbnail['url'],
-            'width': int_or_none(media_thumbnail.get('width')),
-            'height': int_or_none(media_thumbnail.get('height')),
-        }] if media_thumbnail else []
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'duration': duration,
-            'timestamp': timestamp,
-            'formats': formats,
-            'thumbnails': thumbnails,
-        }
+        info = self._extract_feed_info('http://%s/v/feed/video/%s.js?template=fox' % (host, video_id))
+        info['id'] = video_id
+        return info

From 63b728f06f00c2f1a45a67eddebd18bcdc36a753 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 7 Nov 2015 16:56:21 +0100
Subject: [PATCH 0058/1105] [bleacherreport] Add new Extractor

---
 youtube_dl/extractor/__init__.py       |   4 +
 youtube_dl/extractor/bleacherreport.py | 121 +++++++++++++++++++++++++
 2 files changed, 125 insertions(+)
 create mode 100644 youtube_dl/extractor/bleacherreport.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 94150a28f..4d65ece94 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -54,6 +54,10 @@ from .beatportpro import BeatportProIE
 from .bet import BetIE
 from .bild import BildIE
 from .bilibili import BiliBiliIE
+from .bleacherreport import (
+    BleacherReportIE,
+    BleacherReportCMSIE,
+)
 from .blinkx import BlinkxIE
 from .bliptv import BlipTVIE, BlipTVUserIE
 from .bloomberg import BloombergIE
diff --git a/youtube_dl/extractor/bleacherreport.py b/youtube_dl/extractor/bleacherreport.py
new file mode 100644
index 000000000..a55e696d2
--- /dev/null
+++ b/youtube_dl/extractor/bleacherreport.py
@@ -0,0 +1,121 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from .amp import AMPIE
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    parse_iso8601,
+)
+
+
+class BleacherReportIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?bleacherreport\.com/articles/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://bleacherreport.com/articles/2496438-fsu-stat-projections-is-jalen-ramsey-best-defensive-player-in-college-football',
+        'md5': 'a3ffc3dc73afdbc2010f02d98f990f20',
+        'info_dict': {
+            'id': '2496438',
+            'ext': 'mp4',
+            'title': 'FSU Stat Projections: Is Jalen Ramsey Best Defensive Player in College Football?',
+            'uploader_id': 3992341,
+            'description': 'CFB, ACC, Florida State',
+            'timestamp': 1434380212,
+            'upload_date': '20150615',
+            'uploader': 'Team Stream Now ',
+        },
+        'add_ie': ['Ooyala'],
+    },{
+        'url': 'http://bleacherreport.com/articles/2586817-aussie-golfers-get-fright-of-their-lives-after-being-chased-by-angry-kangaroo',
+        'md5': 'af5f90dc9c7ba1c19d0a3eac806bbf50',
+        'info_dict': {
+            'id': '2586817',
+            'ext': 'mp4',
+            'title': 'Aussie Golfers Get Fright of Their Lives After Being Chased by Angry Kangaroo',
+            'timestamp': 1446839961,
+            'uploader': 'Sean Fay',
+            'description': 'md5:e95afafa43619816552723878b3b0a84',
+            'uploader_id': 6466954,
+            'upload_date': '20151011',
+        },
+        'add_ie': ['Youtube'],
+    },{
+        'url': 'http://bleacherreport.com/articles/2496438-fsu-stat-projections-is-jalen-ramsey-best-defensive-player-in-college-football',
+        'md5': 'a3ffc3dc73afdbc2010f02d98f990f20',
+        'info_dict': {
+            'id': '2496438',
+            'ext': 'mp4',
+            'title': 'FSU Stat Projections: Is Jalen Ramsey Best Defensive Player in College Football?',
+            'upload_date': '20150615',
+            'uploader': 'Team Stream Now ',
+            'timestamp': 1434380212,
+            'description': 'CFB, ACC, Florida State',
+            'uploader_id': 3992341,
+        },
+        'add_ie': ['Vine'],
+    }]
+
+    def _real_extract(self, url):
+        article_id = self._match_id(url)
+
+        article_data = self._download_json('http://api.bleacherreport.com/api/v1/articles/%s' % article_id, article_id)['article']
+
+        thumbnails = []
+        primary_photo = article_data.get('primaryPhoto')
+        if primary_photo:
+            thumbnails = [{
+                'url': primary_photo['url'],
+                'width': primary_photo.get('width'),
+                'height': primary_photo.get('height'),
+            }]
+
+        info = {
+            '_type': 'url_transparent',
+            'id': article_id,
+            'title': article_data['title'],
+            'uploader': article_data.get('author', {}).get('name'),
+            'uploader_id': article_data.get('authorId'),
+            'timestamp': parse_iso8601(article_data.get('createdAt')),
+            'thumbnails': thumbnails,
+            'comment_count': int_or_none(article_data.get('commentsCount')),
+            'view_count': int_or_none(article_data.get('hitCount')),
+        }
+
+        video = article_data.get('video')
+        if video:
+            video_type = video['type']
+            if video_type == 'cms.bleacherreport.com':
+                info['url'] = 'http://bleacherreport.com/video_embed?id=%s' % video['id']
+            elif video_type == 'ooyala.com':
+                info['url'] = 'ooyala:%s' % video['id']
+            elif video_type == 'youtube.com':
+                info['url'] = video['id']
+            elif video_type == 'vine.co':
+                info['url'] = 'https://vine.co/v/%s' % video['id']
+            else:
+                info['url'] = video_type + video['id']
+            return info
+        else:
+            raise ExtractorError('no video in the article', expected=True)
+
+
+class BleacherReportCMSIE(AMPIE):
+    _VALID_URL = r'https?://(?:www\.)?bleacherreport\.com/video_embed\?id=(?P<id>[0-9a-f-]{36})'
+    _TESTS = [{
+        'url': 'http://bleacherreport.com/video_embed?id=8fd44c2f-3dc5-4821-9118-2c825a98c0e1',
+        'md5': 'f0ca220af012d4df857b54f792c586bb',
+        'info_dict': {
+            'id': '8fd44c2f-3dc5-4821-9118-2c825a98c0e1',
+            'ext': 'flv',
+            'title': 'Cena vs. Rollins Would Expose the Heavyweight Division',
+            'description': 'md5:984afb4ade2f9c0db35f3267ed88b36e',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        info = self._extract_feed_info('http://cms.bleacherreport.com/media/items/%s/akamai.json' % video_id)
+        info['id'] = video_id
+        return info

From 7db2897ded3d5970131fb34210f8510a02c672b8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Dec 2015 15:57:10 +0100
Subject: [PATCH 0059/1105] [srgssr] handle all play urls only in SRGSSRIE and
 keep RTSIE for articles

---
 youtube_dl/extractor/rts.py    | 97 +++++++++++++++-------------------
 youtube_dl/extractor/srgssr.py | 49 ++++++++++++-----
 2 files changed, 79 insertions(+), 67 deletions(-)

diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index 12639f08b..dc10fd88b 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
+from .srgssr import SRGSSRIE
 from ..compat import (
     compat_str,
     compat_urllib_parse_urlparse,
@@ -17,27 +17,18 @@ from ..utils import (
 )
 
 
-class RTSIE(InfoExtractor):
+class RTSIE(SRGSSRIE):
     IE_DESC = 'RTS.ch'
-    _VALID_URL = r'''(?x)
-                    (?:
-                        rts:(?P<rts_id>\d+)|
-                        https?://
-                            (?:www\.)?rts\.ch/
-                            (?:
-                                (?:[^/]+/){2,}(?P<id>[0-9]+)-(?P<display_id>.+?)\.html|
-                                play/tv/[^/]+/video/(?P<display_id_new>.+?)\?id=(?P<id_new>[0-9]+)
-                            )
-                    )'''
+    _VALID_URL = r'rts:(?P<rts_id>\d+)|https?://(?:www\.)?rts\.ch/(?:[^/]+/){2,}(?P<id>[0-9]+)-(?P<display_id>.+?)\.html'
 
     _TESTS = [
         {
             'url': 'http://www.rts.ch/archives/tv/divers/3449373-les-enfants-terribles.html',
-            'md5': '753b877968ad8afaeddccc374d4256a5',
+            'md5': 'f254c4b26fb1d3c183793d52bc40d3e7',
             'info_dict': {
                 'id': '3449373',
                 'display_id': 'les-enfants-terribles',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'duration': 1488,
                 'title': 'Les Enfants Terribles',
                 'description': 'France Pommier et sa soeur Luce Feral, les deux filles de ce groupe de 5.',
@@ -50,11 +41,11 @@ class RTSIE(InfoExtractor):
         },
         {
             'url': 'http://www.rts.ch/emissions/passe-moi-les-jumelles/5624067-entre-ciel-et-mer.html',
-            'md5': 'c148457a27bdc9e5b1ffe081a7a8337b',
+            'md5': 'f1077ac5af686c76528dc8d7c5df29ba',
             'info_dict': {
-                'id': '5624067',
-                'display_id': 'entre-ciel-et-mer',
-                'ext': 'mp4',
+                'id': '5742494',
+                'display_id': '5742494',
+                'ext': 'flv',
                 'duration': 3720,
                 'title': 'Les yeux dans les cieux - Mon homard au Canada',
                 'description': 'md5:d22ee46f5cc5bac0912e5a0c6d44a9f7',
@@ -85,11 +76,11 @@ class RTSIE(InfoExtractor):
         },
         {
             'url': 'http://www.rts.ch/video/info/journal-continu/5745356-londres-cachee-par-un-epais-smog.html',
-            'md5': '9bb06503773c07ce83d3cbd793cebb91',
+            'md5': '9f713382f15322181bb366cc8c3a4ff0',
             'info_dict': {
                 'id': '5745356',
                 'display_id': 'londres-cachee-par-un-epais-smog',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'duration': 33,
                 'title': 'Londres cachée par un épais smog',
                 'description': 'Un important voile de smog recouvre Londres depuis mercredi, provoqué par la pollution et du sable du Sahara.',
@@ -114,23 +105,6 @@ class RTSIE(InfoExtractor):
                 'timestamp': 1396551600,
             },
         },
-        {
-            'url': 'http://www.rts.ch/play/tv/-/video/le-19h30?id=6348260',
-            'md5': '968777c8779e5aa2434be96c54e19743',
-            'info_dict': {
-                'id': '6348260',
-                'display_id': 'le-19h30',
-                'ext': 'mp4',
-                'duration': 1796,
-                'title': 'Le 19h30',
-                'description': '',
-                'uploader': 'Le 19h30',
-                'upload_date': '20141201',
-                'timestamp': 1417458600,
-                'thumbnail': 're:^https?://.*\.image',
-                'view_count': int,
-            },
-        },
         {
             # article with videos on rhs
             'url': 'http://www.rts.ch/sport/hockey/6693917-hockey-davos-decroche-son-31e-titre-de-champion-de-suisse.html',
@@ -139,42 +113,47 @@ class RTSIE(InfoExtractor):
                 'title': 'Hockey: Davos décroche son 31e titre de champion de Suisse',
             },
             'playlist_mincount': 5,
-        },
-        {
-            'url': 'http://www.rts.ch/play/tv/le-19h30/video/le-chantier-du-nouveau-parlement-vaudois-a-permis-une-trouvaille-historique?id=6348280',
-            'only_matching': True,
         }
     ]
 
     def _real_extract(self, url):
         m = re.match(self._VALID_URL, url)
-        video_id = m.group('rts_id') or m.group('id') or m.group('id_new')
-        display_id = m.group('display_id') or m.group('display_id_new')
+        media_id = m.group('rts_id') or m.group('id')
+        display_id = m.group('display_id') or media_id
 
         def download_json(internal_id):
             return self._download_json(
                 'http://www.rts.ch/a/%s.html?f=json/article' % internal_id,
                 display_id)
 
-        all_info = download_json(video_id)
+        all_info = download_json(media_id)
 
-        # video_id extracted out of URL is not always a real id
+        # media_id extracted out of URL is not always a real id
         if 'video' not in all_info and 'audio' not in all_info:
             page = self._download_webpage(url, display_id)
 
             # article with videos on rhs
             videos = re.findall(
-                r'<article[^>]+class="content-item"[^>]*>\s*<a[^>]+data-video-urn="urn:rts:video:(\d+)"',
+                r'<article[^>]+class="content-item"[^>]*>\s*<a[^>]+data-video-urn="urn:([^"]+)"',
                 page)
+            if not videos:
+                videos = re.findall(
+                    r'(?s)<iframe[^>]+class="srg-player"[^>]+src="[^"]+urn:([^"]+)"',
+                    page)
             if videos:
-                entries = [self.url_result('rts:%s' % video_urn, 'RTS') for video_urn in videos]
-                return self.playlist_result(entries, video_id, self._og_search_title(page))
+                entries = [self.url_result('srgssr:%s' % video_urn, 'SRGSSR') for video_urn in videos]
+                return self.playlist_result(entries, media_id, self._og_search_title(page))
 
             internal_id = self._html_search_regex(
                 r'<(?:video|audio) data-id="([0-9]+)"', page,
                 'internal video id')
             all_info = download_json(internal_id)
 
+        media_type = 'video' if 'video' in all_info else 'audio'
+
+        # check for errors
+        self.get_media_data('rts', media_type, media_id)
+
         info = all_info['video']['JSONinfo'] if 'video' in all_info else all_info['audio']
 
         upload_timestamp = parse_iso8601(info.get('broadcast_date'))
@@ -190,19 +169,27 @@ class RTSIE(InfoExtractor):
 
         formats = []
         for format_id, format_url in info['streams'].items():
+            if format_id == 'hds_sd' and 'hds' in info['streams']:
+                continue
+            if format_id == 'hls_sd' and 'hls' in info['streams']:
+                continue
             if format_url.endswith('.f4m'):
                 token = self._download_xml(
                     'http://tp.srgssr.ch/token/akahd.xml?stream=%s/*' % compat_urllib_parse_urlparse(format_url).path,
-                    video_id, 'Downloading %s token' % format_id)
+                    media_id, 'Downloading %s token' % format_id)
                 auth_params = xpath_text(token, './/authparams', 'auth params')
                 if not auth_params:
                     continue
-                formats.extend(self._extract_f4m_formats(
+                f4m_formats = self._extract_f4m_formats(
                     '%s?%s&hdcore=3.4.0&plugin=aasp-3.4.0.132.66' % (format_url, auth_params),
-                    video_id, f4m_id=format_id))
+                    media_id, f4m_id=format_id, fatal=False)
+                if f4m_formats:
+                    formats.extend(f4m_formats)
             elif format_url.endswith('.m3u8'):
-                formats.extend(self._extract_m3u8_formats(
-                    format_url, video_id, 'mp4', m3u8_id=format_id))
+                m3u8_formats = self._extract_m3u8_formats(
+                    format_url, media_id, 'mp4', m3u8_id=format_id, fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
             else:
                 formats.append({
                     'format_id': format_id,
@@ -217,11 +204,11 @@ class RTSIE(InfoExtractor):
                 'tbr': media['rate'] or extract_bitrate(media['url']),
             } for media in info['media'] if media.get('rate')])
 
-        self._check_formats(formats, video_id)
+        self._check_formats(formats, media_id)
         self._sort_formats(formats)
 
         return {
-            'id': video_id,
+            'id': media_id,
             'display_id': display_id,
             'formats': formats,
             'title': info['title'],
diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py
index f759e5600..563851b3f 100644
--- a/youtube_dl/extractor/srgssr.py
+++ b/youtube_dl/extractor/srgssr.py
@@ -17,25 +17,29 @@ class SRGSSRIE(InfoExtractor):
     _ERRORS = {
         'AGERATING12': 'To protect children under the age of 12, this video is only available between 8 p.m. and 6 a.m.',
         'AGERATING18': 'To protect children under the age of 18, this video is only available between 11 p.m. and 5 a.m.',
-#        'ENDDATE': 'For legal reasons, this video was only available for a specified period of time.',
+        # 'ENDDATE': 'For legal reasons, this video was only available for a specified period of time.',
         'GEOBLOCK': 'For legal reasons, this video is only available in Switzerland.',
         'LEGAL': 'The video cannot be transmitted for legal reasons.',
         'STARTDATE': 'This video is not yet available. Please try again later.',
     }
 
-    def _real_extract(self, url):
-        bu, media_type, media_id = re.match(self._VALID_URL, url).groups()
-
+    def get_media_data(self, bu, media_type, media_id):
         media_data = self._download_json(
             'http://il.srgssr.ch/integrationlayer/1.0/ue/%s/%s/play/%s.json' % (bu, media_type, media_id),
             media_id)[media_type.capitalize()]
 
         if media_data.get('block') and media_data['block'] in self._ERRORS:
-            raise ExtractorError(
-                    '%s said: %s' % (
-                        self.IE_NAME,
-                        self._ERRORS[media_data['block']]),
-                    expected=True)
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, self._ERRORS[media_data['block']]), expected=True)
+
+        return media_data
+
+    def _real_extract(self, url):
+        bu, media_type, media_id = re.match(self._VALID_URL, url).groups()
+
+        if bu == 'rts':
+            return self.url_result('rts:%s' % media_id, 'RTS')
+
+        media_data = self.get_media_data(bu, media_type, media_id)
 
         metadata = media_data['AssetMetadatas']['AssetMetadata'][0]
         title = metadata['title']
@@ -61,9 +65,13 @@ class SRGSSRIE(InfoExtractor):
                     assets[quality['@quality']] = quality['text']
                 asset_url = assets.get('HD') or assets.get('HQ') or assets.get('SD') or assets.get('MQ') or assets.get('LQ')
                 if '.f4m' in asset_url:
-                    formats.extend(self._extract_f4m_formats(asset_url + '?hdcore=3.4.0', media_id, f4m_id='hds'))
+                    f4m_formats = formats.extend(self._extract_f4m_formats(asset_url + '?hdcore=3.4.0', media_id, f4m_id='hds', fatal=False))
+                    if f4m_formats:
+                        formats.extend(f4m_formats)
                 elif '.m3u8' in asset_url:
-                    formats.extend(self._extract_m3u8_formats(asset_url, media_id, m3u8_id='hls'))
+                    m3u8_formats = formats.extend(self._extract_m3u8_formats(asset_url, media_id, m3u8_id='hls', fatal=False))
+                    if m3u8_formats:
+                        formats.extend(m3u8_formats)
             else:
                 for asset in source['url']:
                     asset_url = asset['text']
@@ -122,7 +130,7 @@ class SRGSSRPlayIE(InfoExtractor):
             'description': 'md5:88604432b60d5a38787f152dec89cd56',
             'timestamp': 1373493600,
         },
-    },{
+    }, {
         'url': 'http://www.rtr.ch/play/radio/actualitad/audio/saira-tujetsch-tuttina-cuntinuar-cun-sedrun-muster-turissem?id=63cb0778-27f8-49af-9284-8c7a8c6d15fc',
         'info_dict': {
             'id': '63cb0778-27f8-49af-9284-8c7a8c6d15fc',
@@ -135,8 +143,25 @@ class SRGSSRPlayIE(InfoExtractor):
             # rtmp download
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.rts.ch/play/tv/-/video/le-19h30?id=6348260',
+        'md5': '67a2a9ae4e8e62a68d0e9820cc9782df',
+        'info_dict': {
+            'id': '6348260',
+            'display_id': '6348260',
+            'ext': 'flv',
+            'duration': 1796,
+            'title': 'Le 19h30',
+            'description': '',
+            'uploader': '19h30',
+            'upload_date': '20141201',
+            'timestamp': 1417458600,
+            'thumbnail': 're:^https?://.*\.image',
+            'view_count': int,
+        },
     }]
 
     def _real_extract(self, url):
         bu, media_type, media_id = re.match(self._VALID_URL, url).groups()
+        # other info can be extracted from url + '&layout=json'
         return self.url_result('srgssr:%s:%s:%s' % (bu[:3], media_type, media_id), 'SRGSSR')

From 9f6b517671a7a9b16ccf2bc3f5153673a73895ff Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 6 Dec 2015 07:43:46 +0100
Subject: [PATCH 0060/1105] [vgtv] extract all formats and improve extraction

---
 youtube_dl/extractor/vgtv.py | 51 +++++++++++++++++++-----------------
 1 file changed, 27 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index e8039ec7f..8ce3d8b8f 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -13,7 +13,7 @@ from ..utils import (
 class VGTVIE(InfoExtractor):
     IE_DESC = 'VGTV, BTTV, FTV, Aftenposten and Aftonbladet'
     _VALID_URL = r'''(?x)
-                    http://(?:www\.)?
+                    https?://(?:www\.)?
                     (?P<host>
                         vgtv.no|
                         (?:bt|aftenbladet).no/tv|
@@ -25,7 +25,7 @@ class VGTVIE(InfoExtractor):
                         \#!/(?:video|live)/|
                         embed?.*id=
                     )
-                    (?P<id>[0-9]+)
+                    (?P<id>\d+)
                     '''
     _TESTS = [
         {
@@ -82,7 +82,8 @@ class VGTVIE(InfoExtractor):
                 # m3u8 download
                 'skip_download': True,
             },
-        },{
+        },
+        {
             'url': 'http://www.aftenposten.no/webtv/#!/video/21039/trailer-sweatshop-i-can-t-take-any-more',
             'md5': '7fbc265a3ca4933a423c7a66aa879a67',
             'info_dict': {
@@ -145,35 +146,37 @@ class VGTVIE(InfoExtractor):
 
         hls_url = streams.get('hls')
         if hls_url:
-            formats.extend(self._extract_m3u8_formats(
-                hls_url, video_id, 'mp4', m3u8_id='hls'))
+            m3u8_formats = self._extract_m3u8_formats(hls_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+            if m3u8_formats:
+                formats.extend(m3u8_formats)
 
         hds_url = streams.get('hds')
         # wasLive hds are always 404
         if hds_url and stream_type != 'wasLive':
-            formats.extend(self._extract_f4m_formats(
-                hds_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18',
-                video_id, f4m_id='hds'))
+            f4m_formats = self._extract_f4m_formats(hds_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18', video_id, f4m_id='hds', fatal=False)
+            if f4m_formats:
+                formats.extend(f4m_formats)
 
+        mp4_urls = streams.get('pseudostreaming') or []
         mp4_url = streams.get('mp4')
         if mp4_url:
-            _url = hls_url or hds_url
-            MP4_URL_TEMPLATE = '%s/%%s.%s' % (mp4_url.rpartition('/')[0], mp4_url.rpartition('.')[-1])
-            for mp4_format in _url.split(','):
-                m = re.search('(?P<width>\d+)_(?P<height>\d+)_(?P<vbr>\d+)', mp4_format)
-                if not m:
-                    continue
-                width = int(m.group('width'))
-                height = int(m.group('height'))
-                vbr = int(m.group('vbr'))
-                formats.append({
-                    'url': MP4_URL_TEMPLATE % mp4_format,
-                    'format_id': 'mp4-%s' % vbr,
-                    'width': width,
-                    'height': height,
+            mp4_urls.append(mp4_url)
+        for mp4_url in mp4_urls:
+            format_info = {
+                'url': mp4_url,
+                'preference': 1,
+            }
+            mobj = re.search('(\d+)_(\d+)_(\d+)', mp4_url)
+            if mobj:
+                vbr = int(mobj.group(3))
+                format_info.update({
+                    'width': int(mobj.group(1)),
+                    'height': int(mobj.group(2)),
                     'vbr': vbr,
-                    'preference': 1,
+                    'format_id': 'mp4-%s' % vbr,
                 })
+            formats.append(format_info)
+
         self._sort_formats(formats)
 
         return {
@@ -234,4 +237,4 @@ class BTVestlendingenIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        return self.url_result('xstream:btno:%s' % self._match_id(url), 'Xstream')
+        return self.url_result('http://bt.no/tv/embed?id=%s' % self._match_id(url), 'VGTV')

From 41c3b34b1f1f94c723f3af760ccddc4d7119464f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 9 Dec 2015 10:50:11 +0100
Subject: [PATCH 0061/1105] [vgtv] add sortcut expressions to use the extractor

---
 youtube_dl/extractor/vgtv.py | 68 ++++++++++++++++++------------------
 1 file changed, 34 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 8ce3d8b8f..1fba37578 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -12,21 +12,39 @@ from ..utils import (
 
 class VGTVIE(InfoExtractor):
     IE_DESC = 'VGTV, BTTV, FTV, Aftenposten and Aftonbladet'
+
+    _HOST_TO_APPNAME = {
+        'vgtv.no': 'vgtv',
+        'bt.no/tv': 'bttv',
+        'aftenbladet.no/tv': 'satv',
+        'fvn.no/fvntv': 'fvntv',
+        'aftenposten.no/webtv': 'aptv',
+    }
+
+    _APP_NAME_TO_VENDOR = {
+        'vgtv': 'vgtv',
+        'bttv': 'bt',
+        'satv': 'sa',
+        'fvntv': 'fvn',
+        'aptv': 'ap',
+    }
+
     _VALID_URL = r'''(?x)
-                    https?://(?:www\.)?
+                    (?:https?://(?:www\.)?
                     (?P<host>
-                        vgtv.no|
-                        (?:bt|aftenbladet).no/tv|
-                        fvn.no/fvntv|
-                        aftenposten.no/webtv
+                        %s
                     )
                     /
                     (?:
                         \#!/(?:video|live)/|
                         embed?.*id=
-                    )
+                    )|
+                    (?P<appname>
+                        %s
+                    ):)
                     (?P<id>\d+)
-                    '''
+                    ''' % ('|'.join(_HOST_TO_APPNAME.keys()), '|'.join(_APP_NAME_TO_VENDOR.keys()))
+
     _TESTS = [
         {
             # streamType: vod
@@ -102,37 +120,17 @@ class VGTVIE(InfoExtractor):
             'only_matching': True,
         },
     ]
-    _HOST_WEBSITES = {
-        'vgtv.no': {
-            'vendor': 'vgtv',
-            'appname': 'vgtv',
-        },
-        'bt.no/tv': {
-            'vendor': 'bt',
-            'appname': 'bttv',
-        },
-        'aftenbladet.no/tv': {
-            'vendor': 'sa',
-            'appname': 'satv',
-        },
-        'fvn.no/fvntv': {
-            'vendor': 'fvn',
-            'appname': 'fvntv',
-        },
-        'aftenposten.no/webtv': {
-            'vendor': 'ap',
-            'appname': 'aptv',
-        },
-    }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         host = mobj.group('host')
+        appname = self._HOST_TO_APPNAME[host] if host else mobj.group('appname')
+        vendor = self._APP_NAME_TO_VENDOR[appname]
 
         data = self._download_json(
             'http://svp.vg.no/svp/api/v1/%s/assets/%s?appName=%s-website'
-            % (self._HOST_WEBSITES[host]['vendor'], video_id, self._HOST_WEBSITES[host]['appname']),
+            % (vendor, video_id, appname),
             video_id, 'Downloading media JSON')
 
         if data.get('status') == 'inactive':
@@ -146,14 +144,16 @@ class VGTVIE(InfoExtractor):
 
         hls_url = streams.get('hls')
         if hls_url:
-            m3u8_formats = self._extract_m3u8_formats(hls_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+            m3u8_formats = self._extract_m3u8_formats(
+                hls_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
             if m3u8_formats:
                 formats.extend(m3u8_formats)
 
         hds_url = streams.get('hds')
         # wasLive hds are always 404
         if hds_url and stream_type != 'wasLive':
-            f4m_formats = self._extract_f4m_formats(hds_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18', video_id, f4m_id='hds', fatal=False)
+            f4m_formats = self._extract_f4m_formats(
+                hds_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18', video_id, f4m_id='hds', fatal=False)
             if f4m_formats:
                 formats.extend(f4m_formats)
 
@@ -216,7 +216,7 @@ class BTArticleIE(InfoExtractor):
         webpage = self._download_webpage(url, self._match_id(url))
         video_id = self._search_regex(
             r'SVP\.Player\.load\(\s*(\d+)', webpage, 'video id')
-        return self.url_result('http://bt.no/tv/embed?id=%s' % video_id, 'VGTV')
+        return self.url_result('bttv:%s' % video_id, 'VGTV')
 
 
 class BTVestlendingenIE(InfoExtractor):
@@ -237,4 +237,4 @@ class BTVestlendingenIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        return self.url_result('http://bt.no/tv/embed?id=%s' % self._match_id(url), 'VGTV')
+        return self.url_result('bttv:%s' % self._match_id(url), 'VGTV')

From d50116b8ac2484ce9e8bc6a3d885c5a4a09b4e47 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 10 Dec 2015 22:18:42 +0100
Subject: [PATCH 0062/1105] [vgtv] extract 5 digit length video ids using both
 xstream and vgtv

---
 youtube_dl/extractor/vgtv.py    | 22 +++++++++++++++++-----
 youtube_dl/extractor/xstream.py | 14 +++++++++-----
 2 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 1fba37578..347410a78 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -4,13 +4,14 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from .xstream import XstreamIE
 from ..utils import (
     ExtractorError,
     float_or_none,
 )
 
 
-class VGTVIE(InfoExtractor):
+class VGTVIE(XstreamIE):
     IE_DESC = 'VGTV, BTTV, FTV, Aftenposten and Aftonbladet'
 
     _HOST_TO_APPNAME = {
@@ -137,6 +138,15 @@ class VGTVIE(InfoExtractor):
             raise ExtractorError(
                 'Video %s is no longer available' % video_id, expected=True)
 
+        info = {
+            'formats': [],
+        }
+        if len(video_id) == 5:
+            if appname == 'bttv':
+                info = self._extract_video_info('btno', video_id)
+            elif appname == 'aptv':
+                info = self._extract_video_info('ap', video_id)
+
         streams = data['streamUrls']
         stream_type = data.get('streamType')
 
@@ -177,9 +187,11 @@ class VGTVIE(InfoExtractor):
                 })
             formats.append(format_info)
 
-        self._sort_formats(formats)
+        info['formats'].extend(formats)
 
-        return {
+        self._sort_formats(info['formats'])
+
+        info.update({
             'id': video_id,
             'title': self._live_title(data['title']) if stream_type == 'live' else data['title'],
             'description': data['description'],
@@ -187,9 +199,9 @@ class VGTVIE(InfoExtractor):
             'timestamp': data['published'],
             'duration': float_or_none(data['duration'], 1000),
             'view_count': data['displays'],
-            'formats': formats,
             'is_live': True if stream_type == 'live' else False,
-        }
+        })
+        return info
 
 
 class BTArticleIE(InfoExtractor):
diff --git a/youtube_dl/extractor/xstream.py b/youtube_dl/extractor/xstream.py
index 71584c291..436f8978b 100644
--- a/youtube_dl/extractor/xstream.py
+++ b/youtube_dl/extractor/xstream.py
@@ -42,11 +42,7 @@ class XstreamIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        partner_id = mobj.group('partner_id')
-        video_id = mobj.group('id')
-
+    def _extract_video_info(self, partner_id, video_id):
         data = self._download_xml(
             'http://frontend.xstream.dk/%s/feed/video/?platform=web&id=%s'
             % (partner_id, video_id),
@@ -97,6 +93,7 @@ class XstreamIE(InfoExtractor):
             formats.append({
                 'url': link.get('href'),
                 'format_id': link.get('rel'),
+                'preference': 2,
             })
 
         thumbnails = [{
@@ -113,3 +110,10 @@ class XstreamIE(InfoExtractor):
             'formats': formats,
             'thumbnails': thumbnails,
         }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        partner_id = mobj.group('partner_id')
+        video_id = mobj.group('id')
+
+        return self._extract_video_info(partner_id, video_id)

From 3da3999612311ff193890072727208892b9e93f0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 10 Dec 2015 23:04:28 +0100
Subject: [PATCH 0063/1105] [ultimedia] keep direct support for ultimedia
 videos

---
 youtube_dl/extractor/ultimedia.py | 31 +++++++++++++++++++++++++++----
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/ultimedia.py b/youtube_dl/extractor/ultimedia.py
index 45201332d..60328123c 100644
--- a/youtube_dl/extractor/ultimedia.py
+++ b/youtube_dl/extractor/ultimedia.py
@@ -8,10 +8,29 @@ from ..utils import int_or_none
 
 
 class UltimediaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ultimedia\.com/deliver/(?P<type>generic|musique)(?:/[^/]+)*/(?:src|article)/(?P<id>[\d+a-z]+)'
+    _VALID_URL = r'''(?x)
+        https?://(?:www\.)?ultimedia\.com/
+        (?:
+            deliver/
+            (?P<embed_type>
+                generic|
+                musique
+            )
+            (?:/[^/]+)*/
+            (?:
+                src|
+                article
+            )|
+            default/index/video
+            (?P<site_type>
+                generic|
+                music
+            )
+            /id
+        )/(?P<id>[\d+a-z]+)'''
     _TESTS = [{
         # news
-        'url': 'https://www.ultimedia.com/deliver/generic/iframe/mdtk/01601930/zone/1/src/s8uk0r/autoplay/yes/ad/no/width/714/height/435',
+        'url': 'https://www.ultimedia.com/default/index/videogeneric/id/s8uk0r',
         'md5': '276a0e49de58c7e85d32b057837952a2',
         'info_dict': {
             'id': 's8uk0r',
@@ -25,7 +44,7 @@ class UltimediaIE(InfoExtractor):
         },
     }, {
         # music
-        'url': 'https://www.ultimedia.com/deliver/musique/iframe/mdtk/01601930/zone/1/article/xvpfp8/autoplay/yes/ad/no/width/714/height/435',
+        'url': 'https://www.ultimedia.com/default/index/videomusic/id/xvpfp8',
         'md5': '2ea3513813cf230605c7e2ffe7eca61c',
         'info_dict': {
             'id': 'xvpfp8',
@@ -48,7 +67,11 @@ class UltimediaIE(InfoExtractor):
             return mobj.group('url')
 
     def _real_extract(self, url):
-        video_type, video_id = re.match(self._VALID_URL, url).groups()
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        video_type = mobj.group('embed_type') or mobj.group('site_type')
+        if video_type == 'music':
+            video_type = 'musique'
 
         deliver_info = self._download_json(
             'http://www.ultimedia.com/deliver/video?video=%s&topic=%s' % (video_id, video_type),

From e3e166d8cfa2be039b320d0c733b8233e95d3dcf Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 11 Dec 2015 00:08:54 +0100
Subject: [PATCH 0064/1105] [ign] improve extraction and extract uploader_id

---
 youtube_dl/extractor/ign.py | 26 +++++++++++++++++++-------
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index fb2753738..6160f03d1 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -32,6 +32,7 @@ class IGNIE(InfoExtractor):
                 'description': 'md5:c8946d4260a4d43a00d5ae8ed998870c',
                 'timestamp': 1370440800,
                 'upload_date': '20130605',
+                'uploader_id': 'cberidon@ign.com',
             }
         },
         {
@@ -48,6 +49,7 @@ class IGNIE(InfoExtractor):
                         'description': 'Rockstar drops the mic on this generation of games. Watch our review of the masterly Grand Theft Auto V.',
                         'timestamp': 1379339880,
                         'upload_date': '20130916',
+                        'uploader_id': 'danieljkrupa@gmail.com',
                     },
                 },
                 {
@@ -58,6 +60,7 @@ class IGNIE(InfoExtractor):
                         'description': 'The twisted beauty of GTA 5 in stunning slow motion.',
                         'timestamp': 1386878820,
                         'upload_date': '20131212',
+                        'uploader_id': 'togilvie@ign.com',
                     },
                 },
             ],
@@ -75,6 +78,7 @@ class IGNIE(InfoExtractor):
                 'description': 'Brian and Jared explore Michel Ancel\'s captivating new preview.',
                 'timestamp': 1408047180,
                 'upload_date': '20140814',
+                'uploader_id': 'jamesduggan1990@gmail.com',
             },
         },
         {
@@ -104,8 +108,8 @@ class IGNIE(InfoExtractor):
         webpage = self._download_webpage(url, name_or_id)
         if page_type != 'video':
             multiple_urls = re.findall(
-                '<param name="flashvars"[^>]*value="[^"]*?url=(https?://www\.ign\.com/videos/.*?)["&]',
-                webpage)
+                r'<param name="flashvars"[^>]*value="[^"]*?url=(https?://www\.ign\.com/videos/.*?)["&]',
+                 webpage)
             if multiple_urls:
                 entries = [self.url_result(u, ie='IGN') for u in multiple_urls]
                 return {
@@ -125,10 +129,14 @@ class IGNIE(InfoExtractor):
         formats = []
         m3u8_url = api_data['refs'].get('m3uUrl')
         if m3u8_url:
-            formats.extend(self._extract_m3u8_formats(m3u8_url, video_id))
+            m3u8_formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+            if m3u8_formats:
+                formats.extend(m3u8_formats)
         f4m_url = api_data['refs'].get('f4mUrl')
         if f4m_url:
-            formats.extend(self._extract_f4m_formats(f4m_url, video_id))
+            f4m_formats = self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False)
+            if f4m_formats:
+                formats.extend(f4m_formats)
         for asset in api_data['assets']:
             formats.append({
                 'url': asset['url'],
@@ -139,9 +147,9 @@ class IGNIE(InfoExtractor):
             })
         self._sort_formats(formats)
 
-        thumbnails = []
-        for thumbnail in api_data['thumbnails']:
-            thumbnails.append({'url': thumbnail['url']})
+        thumbnails = [{
+            'url': thumbnail['url']
+        } for thumbnail in api_data.get('thumbnails', [])]
 
         metadata = api_data['metadata']
 
@@ -152,6 +160,7 @@ class IGNIE(InfoExtractor):
             'timestamp': parse_iso8601(metadata.get('publishDate')),
             'duration': int_or_none(metadata.get('duration')),
             'display_id': metadata.get('slug') or video_id,
+            'uploader_id': metadata.get('creator'),
             'thumbnails': thumbnails,
             'formats': formats,
         }
@@ -171,6 +180,7 @@ class OneUPIE(IGNIE):
             'description': 'md5:bf0516c5ee32a3217aa703e9b1bc7826',
             'timestamp': 1313099220,
             'upload_date': '20110811',
+            'uploader_id': 'IGN',
         }
     }]
 
@@ -197,6 +207,7 @@ class PCMagIE(IGNIE):
             'description': 'md5:a7071ae64d2f68cc821c729d4ded6bb3',
             'timestamp': 1420571160,
             'upload_date': '20150106',
+            'uploader_id': 'cozzipix@gmail.com',
         }
     },{
         'url': 'http://www.pcmag.com/article2/0,2817,2470156,00.asp',
@@ -208,5 +219,6 @@ class PCMagIE(IGNIE):
             'description': 'md5:53433c45df96d2ea5d0fda18be2ca908',
             'timestamp': 1412953920,
             'upload_date': '20141010',
+            'uploader_id': 'chris_snyder@pcmag.com',
         }
     }]

From bb4b8c57b986ce9aa1f21f437c8e1de98f8cb7bf Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 11 Dec 2015 23:40:12 +0100
Subject: [PATCH 0065/1105] [kaltura] extract more formats

---
 youtube_dl/extractor/ellentv.py |  6 +++---
 youtube_dl/extractor/kaltura.py | 23 +++++++++++++++--------
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 02c6a4615..476cce2d0 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -13,12 +13,12 @@ class EllenTVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?(?:ellentv|ellentube)\.com/videos/(?P<id>[a-z0-9_-]+)'
     _TEST = {
         'url': 'http://www.ellentv.com/videos/0-ipq1gsai/',
-        'md5': '8e3c576bf2e9bfff4d76565f56f94c9c',
+        'md5': '4294cf98bc165f218aaa0b89e0fd8042',
         'info_dict': {
             'id': '0_ipq1gsai',
-            'ext': 'mp4',
+            'ext': 'mov',
             'title': 'Fast Fingers of Fate',
-            'description': 'md5:587e79fbbd0d73b148bc596d99ce48e6',
+            'description': 'md5:3539013ddcbfa64b2a6d1b38d910868a',
             'timestamp': 1428035648,
             'upload_date': '20150403',
             'uploader_id': 'batchUser',
diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 583b1a5ad..353e960b0 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -45,7 +45,7 @@ class KalturaIE(InfoExtractor):
             'info_dict': {
                 'id': '1_1jc2y3e4',
                 'ext': 'mp4',
-                'title': 'Track 4',
+                'title': 'Straight from the Heart',
                 'upload_date': '20131219',
                 'uploader_id': 'mlundberg@wolfgangsvault.com',
                 'description': 'The Allman Brothers Band, 12/16/1981',
@@ -115,12 +115,9 @@ class KalturaIE(InfoExtractor):
                 'version': '-1',
             },
             {
-                'action': 'getContextData',
-                'contextDataParams:objectType': 'KalturaEntryContextDataParams',
-                'contextDataParams:referrer': 'http://www.kaltura.com/',
-                'contextDataParams:streamerType': 'http',
+                'action': 'getbyentryid',
                 'entryId': video_id,
-                'service': 'baseentry',
+                'service': 'flavorAsset',
             },
         ]
         return self._kaltura_api_call(
@@ -133,7 +130,7 @@ class KalturaIE(InfoExtractor):
         partner_id = mobj.group('partner_id_s') or mobj.group('partner_id') or mobj.group('partner_id_html5')
         entry_id = mobj.group('id_s') or mobj.group('id') or mobj.group('id_html5')
 
-        info, source_data = self._get_video_info(entry_id, partner_id)
+        info, flavor_assets = self._get_video_info(entry_id, partner_id)
 
         source_url = smuggled_data.get('source_url')
         if source_url:
@@ -144,7 +141,10 @@ class KalturaIE(InfoExtractor):
             referrer = None
 
         formats = []
-        for f in source_data['flavorAssets']:
+        for f in flavor_assets:
+            # Continue if asset is not ready
+            if f['status'] != 2:
+                continue
             video_url = '%s/flavorId/%s' % (info['dataUrl'], f['id'])
             if referrer:
                 video_url += '?referrer=%s' % referrer
@@ -160,6 +160,13 @@ class KalturaIE(InfoExtractor):
                 'width': int_or_none(f.get('width')),
                 'url': video_url,
             })
+
+        m3u8_formats = self._extract_m3u8_formats(
+            info['dataUrl'].replace('format/url', 'format/applehttp'),
+            entry_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+        if m3u8_formats:
+            formats.extend(m3u8_formats)
+
         self._check_formats(formats, entry_id)
         self._sort_formats(formats)
 

From 2d3b70271ca427cb4bbe3cf7bb8b6dc0b471cae5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Dec 2015 04:44:17 +0600
Subject: [PATCH 0066/1105] [rutube] Extend _VALID_URL

---
 youtube_dl/extractor/rutube.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index 6b09550b0..9db62adb1 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -17,9 +17,9 @@ from ..utils import (
 class RutubeIE(InfoExtractor):
     IE_NAME = 'rutube'
     IE_DESC = 'Rutube videos'
-    _VALID_URL = r'https?://rutube\.ru/video/(?P<id>[\da-z]{32})'
+    _VALID_URL = r'https?://rutube\.ru/(?:video|play/embed)/(?P<id>[\da-z]{32})'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://rutube.ru/video/3eac3b4561676c17df9132a9a1e62e3e/',
         'info_dict': {
             'id': '3eac3b4561676c17df9132a9a1e62e3e',
@@ -36,7 +36,10 @@ class RutubeIE(InfoExtractor):
             # It requires ffmpeg (m3u8 download)
             'skip_download': True,
         },
-    }
+    }, {
+        'url': 'http://rutube.ru/play/embed/a10e53b86e8f349080f718582ce4c661',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From eb4f27405b172e6ab19ed622a0296ec8d65abe0f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 16 Dec 2015 09:43:53 +0100
Subject: [PATCH 0067/1105] [vimeo] extract source file(closes #1072)

---
 youtube_dl/extractor/vimeo.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index f392ccf1c..cf854a0f6 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -23,6 +23,7 @@ from ..utils import (
     unsmuggle_url,
     urlencode_postdata,
     unescapeHTML,
+    parse_filesize,
 )
 
 
@@ -392,6 +393,20 @@ class VimeoIE(VimeoBaseInfoExtractor):
             comment_count = None
 
         formats = []
+        download_request = sanitized_Request('https://vimeo.com/%s?action=load_download_config' % video_id, headers={
+            'X-Requested-With': 'XMLHttpRequest'})
+        download_data = self._download_json(download_request, video_id, fatal=False)
+        if download_data:
+            source_file = download_data.get('source_file')
+            if source_file and not source_file.get('is_cold') and not source_file.get('is_defrosting'):
+                formats.append({
+                    'url': source_file['download_url'],
+                    'ext': source_file['extension'].lower(),
+                    'width': int_or_none(source_file.get('width')),
+                    'height': int_or_none(source_file.get('height')),
+                    'filesize': parse_filesize(source_file.get('size')),
+                    'format_id': source_file.get('public_name', 'Original'),
+                })
         config_files = config['video'].get('files') or config['request'].get('files', {})
         for f in config_files.get('progressive', []):
             video_url = f.get('url')

From 8534bf1f0051640399b7c65c7d33e38b02f598fd Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 16 Dec 2015 16:36:25 +0100
Subject: [PATCH 0068/1105] [vimeo] prefer original format

---
 youtube_dl/extractor/vimeo.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index cf854a0f6..715ede34e 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -406,6 +406,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                     'height': int_or_none(source_file.get('height')),
                     'filesize': parse_filesize(source_file.get('size')),
                     'format_id': source_file.get('public_name', 'Original'),
+                    'preference': 1,
                 })
         config_files = config['video'].get('files') or config['request'].get('files', {})
         for f in config_files.get('progressive', []):
@@ -423,12 +424,12 @@ class VimeoIE(VimeoBaseInfoExtractor):
         m3u8_url = config_files.get('hls', {}).get('url')
         if m3u8_url:
             m3u8_formats = self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
+                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
             if m3u8_formats:
                 formats.extend(m3u8_formats)
         # Bitrates are completely broken. Single m3u8 may contain entries in kbps and bps
         # at the same time without actual units specified. This lead to wrong sorting.
-        self._sort_formats(formats, field_preference=('height', 'width', 'fps', 'format_id'))
+        self._sort_formats(formats, field_preference=('preference', 'height', 'width', 'fps', 'format_id'))
 
         subtitles = {}
         text_tracks = config['request'].get('text_tracks')

From 323f82a7e05afcc3518706dc18016ebd97fbc052 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 16 Dec 2015 17:00:17 +0100
Subject: [PATCH 0069/1105] [vimeo] add test for original format

---
 youtube_dl/extractor/vimeo.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 715ede34e..ce08e6955 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -185,6 +185,20 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'uploader_id': 'user28849593',
             },
         },
+        {
+            # contains original format
+            'url': 'https://vimeo.com/33951933',
+            'md5': '53c688fa95a55bf4b7293d37a89c5c53',
+            'info_dict': {
+                'id': '33951933',
+                'ext': 'mp4',
+                'title': 'FOX CLASSICS - Forever Classic ID - A Full Minute',
+                'uploader': 'The DMCI',
+                'uploader_id': 'dmci',
+                'upload_date': '20111220',
+                'description': 'md5:ae23671e82d05415868f7ad1aec21147',
+            },
+        },
         {
             'url': 'https://vimeo.com/109815029',
             'note': 'Video not completely processed, "failed" seed status',

From 35e22b6b32d0e662d486c9be8c76f6ea86f7cdd4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 17 Dec 2015 12:51:50 +0100
Subject: [PATCH 0070/1105] [youku] check for the correct variable

---
 youtube_dl/extractor/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index d33caa79e..3a3432be8 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -221,7 +221,7 @@ class YoukuIE(InfoExtractor):
                     'Youku said: Sorry, this video is available in China only', expected=True)
             else:
                 msg = 'Youku server reported error %i' % error.get('code')
-                if error is not None:
+                if error_note is not None:
                     msg += ': ' + error_note
                 raise ExtractorError(msg)
 

From 8f0afda028e64f7620d8ce21e9cfa4d686e3e6c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Dec 2015 20:24:33 +0600
Subject: [PATCH 0071/1105] [pbs] Extend _VALID_URL (Closes #7889)

---
 youtube_dl/extractor/pbs.py | 320 ++++++++++++++++++------------------
 1 file changed, 160 insertions(+), 160 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 604a0dd22..744e4a09a 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -16,165 +16,165 @@ from ..utils import (
 
 class PBSIE(InfoExtractor):
     _STATIONS = (
-        ('video.pbs.org', 'PBS: Public Broadcasting Service'),  # http://www.pbs.org/
-        ('video.aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # http://aptv.org/
-        ('video.gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # http://www.gpb.org/
-        ('video.mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # http://www.mpbonline.org
-        ('video.wnpt.org', 'Nashville Public Television (WNPT)'),  # http://www.wnpt.org
-        ('video.wfsu.org', 'WFSU-TV (WFSU)'),  # http://wfsu.org/
-        ('video.wsre.org', 'WSRE (WSRE)'),  # http://www.wsre.org
-        ('video.wtcitv.org', 'WTCI (WTCI)'),  # http://www.wtcitv.org
-        ('video.pba.org', 'WPBA/Channel 30 (WPBA)'),  # http://pba.org/
-        ('video.alaskapublic.org', 'Alaska Public Media (KAKM)'),  # http://alaskapublic.org/kakm
-        # ('kuac.org', 'KUAC (KUAC)'),  # http://kuac.org/kuac-tv/
-        # ('ktoo.org', '360 North (KTOO)'),  # http://www.ktoo.org/
-        # ('azpm.org', 'KUAT 6 (KUAT)'),  # http://www.azpm.org/
-        ('video.azpbs.org', 'Arizona PBS (KAET)'),  # http://www.azpbs.org
-        ('portal.knme.org', 'KNME-TV/Channel 5 (KNME)'),  # http://www.newmexicopbs.org/
-        ('video.vegaspbs.org', 'Vegas PBS (KLVX)'),  # http://vegaspbs.org/
-        ('watch.aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # http://www.aetn.org/
-        ('video.ket.org', 'KET (WKLE)'),  # http://www.ket.org/
-        ('video.wkno.org', 'WKNO/Channel 10 (WKNO)'),  # http://www.wkno.org/
-        ('video.lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # http://www.lpb.org/
-        ('videos.oeta.tv', 'OETA (KETA)'),  # http://www.oeta.tv
-        ('video.optv.org', 'Ozarks Public Television (KOZK)'),  # http://www.optv.org/
-        ('watch.wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # http://www.wsiu.org/
-        ('video.keet.org', 'KEET TV (KEET)'),  # http://www.keet.org
-        ('pbs.kixe.org', 'KIXE/Channel 9 (KIXE)'),  # http://kixe.org/
-        ('video.kpbs.org', 'KPBS San Diego (KPBS)'),  # http://www.kpbs.org/
-        ('video.kqed.org', 'KQED (KQED)'),  # http://www.kqed.org
-        ('vids.kvie.org', 'KVIE Public Television (KVIE)'),  # http://www.kvie.org
-        ('video.pbssocal.org', 'PBS SoCal/KOCE (KOCE)'),  # http://www.pbssocal.org/
-        ('video.valleypbs.org', 'ValleyPBS (KVPT)'),  # http://www.valleypbs.org/
-        ('video.cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # http://cptv.org
-        ('watch.knpb.org', 'KNPB Channel 5 (KNPB)'),  # http://www.knpb.org/
-        ('video.soptv.org', 'SOPTV (KSYS)'),  # http://www.soptv.org
-        # ('klcs.org', 'KLCS/Channel 58 (KLCS)'),  # http://www.klcs.org
-        # ('krcb.org', 'KRCB Television & Radio (KRCB)'),  # http://www.krcb.org
-        # ('kvcr.org', 'KVCR TV/DT/FM :: Vision for the Future (KVCR)'),  # http://kvcr.org
-        ('video.rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # http://www.rmpbs.org
-        ('video.kenw.org', 'KENW-TV3 (KENW)'),  # http://www.kenw.org
-        ('video.kued.org', 'KUED Channel 7 (KUED)'),  # http://www.kued.org
-        ('video.wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # http://www.wyomingpbs.org
-        ('video.cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # http://www.cpt12.org/
-        ('video.kbyueleven.org', 'KBYU-TV (KBYU)'),  # http://www.kbyutv.org/
-        ('video.thirteen.org', 'Thirteen/WNET New York (WNET)'),  # http://www.thirteen.org
-        ('video.wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # http://wgbh.org
-        ('video.wgby.org', 'WGBY (WGBY)'),  # http://www.wgby.org
-        ('watch.njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # http://www.njtvonline.org/
-        # ('ripbs.org', 'Rhode Island PBS (WSBE)'),  # http://www.ripbs.org/home/
-        ('watch.wliw.org', 'WLIW21 (WLIW)'),  # http://www.wliw.org/
-        ('video.mpt.tv', 'mpt/Maryland Public Television (WMPB)'),  # http://www.mpt.org
-        ('watch.weta.org', 'WETA Television and Radio (WETA)'),  # http://www.weta.org
-        ('video.whyy.org', 'WHYY (WHYY)'),  # http://www.whyy.org
-        ('video.wlvt.org', 'PBS 39 (WLVT)'),  # http://www.wlvt.org/
-        ('video.wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # http://www.wvpt.net
-        ('video.whut.org', 'Howard University Television (WHUT)'),  # http://www.whut.org
-        ('video.wedu.org', 'WEDU PBS (WEDU)'),  # http://www.wedu.org
-        ('video.wgcu.org', 'WGCU Public Media (WGCU)'),  # http://www.wgcu.org/
-        # ('wjct.org', 'WJCT Public Broadcasting (WJCT)'),  # http://www.wjct.org
-        ('video.wpbt2.org', 'WPBT2 (WPBT)'),  # http://www.wpbt2.org
-        ('video.wucftv.org', 'WUCF TV (WUCF)'),  # http://wucftv.org
-        ('video.wuft.org', 'WUFT/Channel 5 (WUFT)'),  # http://www.wuft.org
-        ('watch.wxel.org', 'WXEL/Channel 42 (WXEL)'),  # http://www.wxel.org/home/
-        ('video.wlrn.org', 'WLRN/Channel 17 (WLRN)'),  # http://www.wlrn.org/
-        ('video.wusf.usf.edu', 'WUSF Public Broadcasting (WUSF)'),  # http://wusf.org/
-        ('video.scetv.org', 'ETV (WRLK)'),  # http://www.scetv.org
-        ('video.unctv.org', 'UNC-TV (WUNC)'),  # http://www.unctv.org/
-        # ('pbsguam.org', 'PBS Guam (KGTF)'),  # http://www.pbsguam.org/
-        ('video.pbshawaii.org', 'PBS Hawaii - Oceanic Cable Channel 10 (KHET)'),  # http://www.pbshawaii.org/
-        ('video.idahoptv.org', 'Idaho Public Television (KAID)'),  # http://idahoptv.org
-        ('video.ksps.org', 'KSPS (KSPS)'),  # http://www.ksps.org/home/
-        ('watch.opb.org', 'OPB (KOPB)'),  # http://www.opb.org
-        ('watch.nwptv.org', 'KWSU/Channel 10 & KTNW/Channel 31 (KWSU)'),  # http://www.kwsu.org
-        ('video.will.illinois.edu', 'WILL-TV (WILL)'),  # http://will.illinois.edu/
-        ('video.networkknowledge.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # http://www.wsec.tv
-        ('video.wttw.com', 'WTTW11 (WTTW)'),  # http://www.wttw.com/
-        # ('wtvp.org', 'WTVP & WTVP.org, Public Media for Central Illinois (WTVP)'),  # http://www.wtvp.org/
-        ('video.iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # http://www.iptv.org/
-        ('video.ninenet.org', 'Nine Network (KETC)'),  # http://www.ninenet.org
-        ('video.wfwa.org', 'PBS39 Fort Wayne (WFWA)'),  # http://wfwa.org/
-        ('video.wfyi.org', 'WFYI Indianapolis (WFYI)'),  # http://www.wfyi.org
-        ('video.mptv.org', 'Milwaukee Public Television (WMVS)'),  # http://www.mptv.org
-        ('video.wnin.org', 'WNIN (WNIN)'),  # http://www.wnin.org/
-        ('video.wnit.org', 'WNIT Public Television (WNIT)'),  # http://www.wnit.org/
-        ('video.wpt.org', 'WPT (WPNE)'),  # http://www.wpt.org/
-        ('video.wvut.org', 'WVUT/Channel 22 (WVUT)'),  # http://wvut.org/
-        ('video.weiu.net', 'WEIU/Channel 51 (WEIU)'),  # http://www.weiu.net
-        ('video.wqpt.org', 'WQPT-TV (WQPT)'),  # http://www.wqpt.org
-        ('video.wycc.org', 'WYCC PBS Chicago (WYCC)'),  # http://www.wycc.org
-        # ('lakeshorepublicmedia.org', 'Lakeshore Public Television (WYIN)'),  # http://lakeshorepublicmedia.org/
-        ('video.wipb.org', 'WIPB-TV (WIPB)'),  # http://wipb.org
-        ('video.indianapublicmedia.org', 'WTIU (WTIU)'),  # http://indianapublicmedia.org/tv/
-        ('watch.cetconnect.org', 'CET  (WCET)'),  # http://www.cetconnect.org
-        ('video.thinktv.org', 'ThinkTVNetwork (WPTD)'),  # http://www.thinktv.org
-        ('video.wbgu.org', 'WBGU-TV (WBGU)'),  # http://wbgu.org
-        ('video.wgvu.org', 'WGVU TV (WGVU)'),  # http://www.wgvu.org/
-        ('video.netnebraska.org', 'NET1 (KUON)'),  # http://netnebraska.org
-        ('video.pioneer.org', 'Pioneer Public Television (KWCM)'),  # http://www.pioneer.org
-        ('watch.sdpb.org', 'SDPB Television (KUSD)'),  # http://www.sdpb.org
-        ('video.tpt.org', 'TPT (KTCA)'),  # http://www.tpt.org
-        ('watch.ksmq.org', 'KSMQ (KSMQ)'),  # http://www.ksmq.org/
-        ('watch.kpts.org', 'KPTS/Channel 8 (KPTS)'),  # http://www.kpts.org/
-        ('watch.ktwu.org', 'KTWU/Channel 11 (KTWU)'),  # http://ktwu.org
-        # ('shptv.org', 'Smoky Hills Public Television (KOOD)'),  # http://www.shptv.org
-        # ('kcpt.org', 'KCPT Kansas City Public Television (KCPT)'),  # http://kcpt.org/
-        # ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # http://www.blueridgepbs.org/
-        ('watch.easttennesseepbs.org', 'East Tennessee PBS (WSJK)'),  # http://easttennesseepbs.org
-        ('video.wcte.tv', 'WCTE-TV (WCTE)'),  # http://www.wcte.org
-        ('video.wljt.org', 'WLJT, Channel 11 (WLJT)'),  # http://wljt.org/
-        ('video.wosu.org', 'WOSU TV (WOSU)'),  # http://wosu.org/
-        ('video.woub.org', 'WOUB/WOUC (WOUB)'),  # http://woub.org/tv/index.php?section=5
-        ('video.wvpublic.org', 'WVPB (WVPB)'),  # http://wvpublic.org/
-        ('video.wkyupbs.org', 'WKYU-PBS (WKYU)'),  # http://www.wkyupbs.org
-        # ('wyes.org', 'WYES-TV/New Orleans (WYES)'),  # http://www.wyes.org
-        ('video.kera.org', 'KERA 13 (KERA)'),  # http://www.kera.org/
-        ('video.mpbn.net', 'MPBN (WCBB)'),  # http://www.mpbn.net/
-        ('video.mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # http://www.mountainlake.org/
-        ('video.nhptv.org', 'NHPTV (WENH)'),  # http://nhptv.org/
-        ('video.vpt.org', 'Vermont PBS (WETK)'),  # http://www.vpt.org
-        ('video.witf.org', 'witf (WITF)'),  # http://www.witf.org
-        ('watch.wqed.org', 'WQED Multimedia (WQED)'),  # http://www.wqed.org/
-        ('video.wmht.org', 'WMHT Educational Telecommunications (WMHT)'),  # http://www.wmht.org/home/
-        ('video.deltabroadcasting.org', 'Q-TV (WDCQ)'),  # http://www.deltabroadcasting.org
-        ('video.dptv.org', 'WTVS Detroit Public TV (WTVS)'),  # http://www.dptv.org/
-        ('video.wcmu.org', 'CMU Public Television (WCMU)'),  # http://www.wcmu.org
-        ('video.wkar.org', 'WKAR-TV (WKAR)'),  # http://wkar.org/
-        ('wnmuvideo.nmu.edu', 'WNMU-TV Public TV 13 (WNMU)'),  # http://wnmutv.nmu.edu
-        ('video.wdse.org', 'WDSE - WRPT (WDSE)'),  # http://www.wdse.org/
-        ('video.wgte.org', 'WGTE TV (WGTE)'),  # http://www.wgte.org
-        ('video.lptv.org', 'Lakeland Public Television (KAWE)'),  # http://www.lakelandptv.org
-        # ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # http://www.prairiepublic.org/
-        ('video.kmos.org', 'KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS)'),  # http://www.kmos.org/
-        ('watch.montanapbs.org', 'MontanaPBS (KUSM)'),  # http://montanapbs.org
-        ('video.krwg.org', 'KRWG/Channel 22 (KRWG)'),  # http://www.krwg.org
-        ('video.kacvtv.org', 'KACV (KACV)'),  # http://www.panhandlepbs.org/home/
-        ('video.kcostv.org', 'KCOS/Channel 13 (KCOS)'),  # www.kcostv.org
-        ('video.wcny.org', 'WCNY/Channel 24 (WCNY)'),  # http://www.wcny.org
-        ('video.wned.org', 'WNED (WNED)'),  # http://www.wned.org/
-        ('watch.wpbstv.org', 'WPBS (WPBS)'),  # http://www.wpbstv.org
-        ('video.wskg.org', 'WSKG Public TV (WSKG)'),  # http://wskg.org
-        ('video.wxxi.org', 'WXXI (WXXI)'),  # http://wxxi.org
-        ('video.wpsu.org', 'WPSU (WPSU)'),  # http://www.wpsu.org
-        # ('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # http://www.wqln.org
-        ('on-demand.wvia.org', 'WVIA Public Media Studios (WVIA)'),  # http://www.wvia.org/
-        ('video.wtvi.org', 'WTVI (WTVI)'),  # http://www.wtvi.org/
-        # ('whro.org', 'WHRO (WHRO)'),  # http://whro.org
-        ('video.westernreservepublicmedia.org', 'Western Reserve PBS (WNEO)'),  # http://www.WesternReservePublicMedia.org/
-        ('video.ideastream.org', 'WVIZ/PBS ideastream (WVIZ)'),  # http://www.wviz.org/
-        ('video.kcts9.org', 'KCTS 9 (KCTS)'),  # http://kcts9.org/
-        ('video.basinpbs.org', 'Basin PBS (KPBT)'),  # http://www.basinpbs.org
-        ('video.houstonpbs.org', 'KUHT / Channel 8 (KUHT)'),  # http://www.houstonpublicmedia.org/
-        # ('tamu.edu', 'KAMU - TV (KAMU)'),  # http://KAMU.tamu.edu
-        # ('kedt.org', 'KEDT/Channel 16 (KEDT)'),  # http://www.kedt.org
-        ('video.klrn.org', 'KLRN (KLRN)'),  # http://www.klrn.org
-        ('video.klru.tv', 'KLRU (KLRU)'),  # http://www.klru.org
-        # ('kmbh.org', 'KMBH-TV (KMBH)'),  # http://www.kmbh.org
-        # ('knct.org', 'KNCT (KNCT)'),  # http://www.knct.org
-        # ('ktxt.org', 'KTTZ-TV (KTXT)'),  # http://www.ktxt.org
-        ('video.wtjx.org', 'WTJX Channel 12 (WTJX)'),  # http://www.wtjx.org/
-        ('video.ideastations.org', 'WCVE PBS (WCVE)'),  # http://ideastations.org/
-        ('video.kbtc.org', 'KBTC Public Television (KBTC)'),  # http://kbtc.org
+        (r'(?:video|www)\.pbs\.org', 'PBS: Public Broadcasting Service'),  # http://www.pbs.org/
+        (r'video\.aptv\.org', 'APT - Alabama Public Television (WBIQ)'),  # http://aptv.org/
+        (r'video\.gpb\.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # http://www.gpb.org/
+        (r'video\.mpbonline\.org', 'Mississippi Public Broadcasting (WMPN)'),  # http://www.mpbonline.org
+        (r'video\.wnpt\.org', 'Nashville Public Television (WNPT)'),  # http://www.wnpt.org
+        (r'video\.wfsu\.org', 'WFSU-TV (WFSU)'),  # http://wfsu.org/
+        (r'video\.wsre\.org', 'WSRE (WSRE)'),  # http://www.wsre.org
+        (r'video\.wtcitv\.org', 'WTCI (WTCI)'),  # http://www.wtcitv.org
+        (r'video\.pba\.org', 'WPBA/Channel 30 (WPBA)'),  # http://pba.org/
+        (r'video\.alaskapublic\.org', 'Alaska Public Media (KAKM)'),  # http://alaskapublic.org/kakm
+        # (r'kuac\.org', 'KUAC (KUAC)'),  # http://kuac.org/kuac-tv/
+        # (r'ktoo\.org', '360 North (KTOO)'),  # http://www.ktoo.org/
+        # (r'azpm\.org', 'KUAT 6 (KUAT)'),  # http://www.azpm.org/
+        (r'video\.azpbs\.org', 'Arizona PBS (KAET)'),  # http://www.azpbs.org
+        (r'portal\.knme\.org', 'KNME-TV/Channel 5 (KNME)'),  # http://www.newmexicopbs.org/
+        (r'video\.vegaspbs\.org', 'Vegas PBS (KLVX)'),  # http://vegaspbs.org/
+        (r'watch\.aetn\.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # http://www.aetn.org/
+        (r'video\.ket\.org', 'KET (WKLE)'),  # http://www.ket.org/
+        (r'video\.wkno\.org', 'WKNO/Channel 10 (WKNO)'),  # http://www.wkno.org/
+        (r'video\.lpb\.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # http://www.lpb.org/
+        (r'videos\.oeta\.tv', 'OETA (KETA)'),  # http://www.oeta.tv
+        (r'video\.optv\.org', 'Ozarks Public Television (KOZK)'),  # http://www.optv.org/
+        (r'watch\.wsiu\.org', 'WSIU Public Broadcasting (WSIU)'),  # http://www.wsiu.org/
+        (r'video\.keet\.org', 'KEET TV (KEET)'),  # http://www.keet.org
+        (r'pbs\.kixe\.org', 'KIXE/Channel 9 (KIXE)'),  # http://kixe.org/
+        (r'video\.kpbs\.org', 'KPBS San Diego (KPBS)'),  # http://www.kpbs.org/
+        (r'video\.kqed\.org', 'KQED (KQED)'),  # http://www.kqed.org
+        (r'vids\.kvie\.org', 'KVIE Public Television (KVIE)'),  # http://www.kvie.org
+        (r'video\.pbssocal\.org', 'PBS SoCal/KOCE (KOCE)'),  # http://www.pbssocal.org/
+        (r'video\.valleypbs\.org', 'ValleyPBS (KVPT)'),  # http://www.valleypbs.org/
+        (r'video\.cptv\.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # http://cptv.org
+        (r'watch\.knpb\.org', 'KNPB Channel 5 (KNPB)'),  # http://www.knpb.org/
+        (r'video\.soptv\.org', 'SOPTV (KSYS)'),  # http://www.soptv.org
+        # (r'klcs\.org', 'KLCS/Channel 58 (KLCS)'),  # http://www.klcs.org
+        # (r'krcb\.org', 'KRCB Television & Radio (KRCB)'),  # http://www.krcb.org
+        # (r'kvcr\.org', 'KVCR TV/DT/FM :: Vision for the Future (KVCR)'),  # http://kvcr.org
+        (r'video\.rmpbs\.org', 'Rocky Mountain PBS (KRMA)'),  # http://www.rmpbs.org
+        (r'video\.kenw\.org', 'KENW-TV3 (KENW)'),  # http://www.kenw.org
+        (r'video\.kued\.org', 'KUED Channel 7 (KUED)'),  # http://www.kued.org
+        (r'video\.wyomingpbs\.org', 'Wyoming PBS (KCWC)'),  # http://www.wyomingpbs.org
+        (r'video\.cpt12\.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # http://www.cpt12.org/
+        (r'video\.kbyueleven\.org', 'KBYU-TV (KBYU)'),  # http://www.kbyutv.org/
+        (r'video\.thirteen\.org', 'Thirteen/WNET New York (WNET)'),  # http://www.thirteen.org
+        (r'video\.wgbh\.org', 'WGBH/Channel 2 (WGBH)'),  # http://wgbh.org
+        (r'video\.wgby\.org', 'WGBY (WGBY)'),  # http://www.wgby.org
+        (r'watch\.njtvonline\.org', 'NJTV Public Media NJ (WNJT)'),  # http://www.njtvonline.org/
+        # (r'ripbs\.org', 'Rhode Island PBS (WSBE)'),  # http://www.ripbs.org/home/
+        (r'watch\.wliw\.org', 'WLIW21 (WLIW)'),  # http://www.wliw.org/
+        (r'video\.mpt\.tv', 'mpt/Maryland Public Television (WMPB)'),  # http://www.mpt.org
+        (r'watch\.weta\.org', 'WETA Television and Radio (WETA)'),  # http://www.weta.org
+        (r'video\.whyy\.org', 'WHYY (WHYY)'),  # http://www.whyy.org
+        (r'video\.wlvt\.org', 'PBS 39 (WLVT)'),  # http://www.wlvt.org/
+        (r'video\.wvpt\.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # http://www.wvpt.net
+        (r'video\.whut\.org', 'Howard University Television (WHUT)'),  # http://www.whut.org
+        (r'video\.wedu\.org', 'WEDU PBS (WEDU)'),  # http://www.wedu.org
+        (r'video\.wgcu\.org', 'WGCU Public Media (WGCU)'),  # http://www.wgcu.org/
+        # (r'wjct\.org', 'WJCT Public Broadcasting (WJCT)'),  # http://www.wjct.org
+        (r'video\.wpbt2\.org', 'WPBT2 (WPBT)'),  # http://www.wpbt2.org
+        (r'video\.wucftv\.org', 'WUCF TV (WUCF)'),  # http://wucftv.org
+        (r'video\.wuft\.org', 'WUFT/Channel 5 (WUFT)'),  # http://www.wuft.org
+        (r'watch\.wxel\.org', 'WXEL/Channel 42 (WXEL)'),  # http://www.wxel.org/home/
+        (r'video\.wlrn\.org', 'WLRN/Channel 17 (WLRN)'),  # http://www.wlrn.org/
+        (r'video\.wusf\.usf\.edu', 'WUSF Public Broadcasting (WUSF)'),  # http://wusf.org/
+        (r'video\.scetv\.org', 'ETV (WRLK)'),  # http://www.scetv.org
+        (r'video\.unctv\.org', 'UNC-TV (WUNC)'),  # http://www.unctv.org/
+        # (r'pbsguam\.org', 'PBS Guam (KGTF)'),  # http://www.pbsguam.org/
+        (r'video\.pbshawaii\.org', 'PBS Hawaii - Oceanic Cable Channel 10 (KHET)'),  # http://www.pbshawaii.org/
+        (r'video\.idahoptv\.org', 'Idaho Public Television (KAID)'),  # http://idahoptv.org
+        (r'video\.ksps\.org', 'KSPS (KSPS)'),  # http://www.ksps.org/home/
+        (r'watch\.opb\.org', 'OPB (KOPB)'),  # http://www.opb.org
+        (r'watch\.nwptv\.org', 'KWSU/Channel 10 & KTNW/Channel 31 (KWSU)'),  # http://www.kwsu.org
+        (r'video\.will\.illinois\.edu', 'WILL-TV (WILL)'),  # http://will.illinois.edu/
+        (r'video\.networkknowledge\.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # http://www.wsec.tv
+        (r'video\.wttw\.com', 'WTTW11 (WTTW)'),  # http://www.wttw.com/
+        # (r'wtvp\.org', 'WTVP & WTVP.org, Public Media for Central Illinois (WTVP)'),  # http://www.wtvp.org/
+        (r'video\.iptv\.org', 'Iowa Public Television/IPTV (KDIN)'),  # http://www.iptv.org/
+        (r'video\.ninenet\.org', 'Nine Network (KETC)'),  # http://www.ninenet.org
+        (r'video\.wfwa\.org', 'PBS39 Fort Wayne (WFWA)'),  # http://wfwa.org/
+        (r'video\.wfyi\.org', 'WFYI Indianapolis (WFYI)'),  # http://www.wfyi.org
+        (r'video\.mptv\.org', 'Milwaukee Public Television (WMVS)'),  # http://www.mptv.org
+        (r'video\.wnin\.org', 'WNIN (WNIN)'),  # http://www.wnin.org/
+        (r'video\.wnit\.org', 'WNIT Public Television (WNIT)'),  # http://www.wnit.org/
+        (r'video\.wpt\.org', 'WPT (WPNE)'),  # http://www.wpt.org/
+        (r'video\.wvut\.org', 'WVUT/Channel 22 (WVUT)'),  # http://wvut.org/
+        (r'video\.weiu\.net', 'WEIU/Channel 51 (WEIU)'),  # http://www.weiu.net
+        (r'video\.wqpt\.org', 'WQPT-TV (WQPT)'),  # http://www.wqpt.org
+        (r'video\.wycc\.org', 'WYCC PBS Chicago (WYCC)'),  # http://www.wycc.org
+        # (r'lakeshorepublicmedia\.org', 'Lakeshore Public Television (WYIN)'),  # http://lakeshorepublicmedia.org/
+        (r'video\.wipb\.org', 'WIPB-TV (WIPB)'),  # http://wipb.org
+        (r'video\.indianapublicmedia\.org', 'WTIU (WTIU)'),  # http://indianapublicmedia.org/tv/
+        (r'watch\.cetconnect\.org', 'CET  (WCET)'),  # http://www.cetconnect.org
+        (r'video\.thinktv\.org', 'ThinkTVNetwork (WPTD)'),  # http://www.thinktv.org
+        (r'video\.wbgu\.org', 'WBGU-TV (WBGU)'),  # http://wbgu.org
+        (r'video\.wgvu\.org', 'WGVU TV (WGVU)'),  # http://www.wgvu.org/
+        (r'video\.netnebraska\.org', 'NET1 (KUON)'),  # http://netnebraska.org
+        (r'video\.pioneer\.org', 'Pioneer Public Television (KWCM)'),  # http://www.pioneer.org
+        (r'watch\.sdpb\.org', 'SDPB Television (KUSD)'),  # http://www.sdpb.org
+        (r'video\.tpt\.org', 'TPT (KTCA)'),  # http://www.tpt.org
+        (r'watch\.ksmq\.org', 'KSMQ (KSMQ)'),  # http://www.ksmq.org/
+        (r'watch\.kpts\.org', 'KPTS/Channel 8 (KPTS)'),  # http://www.kpts.org/
+        (r'watch\.ktwu\.org', 'KTWU/Channel 11 (KTWU)'),  # http://ktwu.org
+        # (r'shptv\.org', 'Smoky Hills Public Television (KOOD)'),  # http://www.shptv.org
+        # (r'kcpt\.org', 'KCPT Kansas City Public Television (KCPT)'),  # http://kcpt.org/
+        # (r'blueridgepbs\.org', 'Blue Ridge PBS (WBRA)'),  # http://www.blueridgepbs.org/
+        (r'watch\.easttennesseepbs\.org', 'East Tennessee PBS (WSJK)'),  # http://easttennesseepbs.org
+        (r'video\.wcte\.tv', 'WCTE-TV (WCTE)'),  # http://www.wcte.org
+        (r'video\.wljt\.org', 'WLJT, Channel 11 (WLJT)'),  # http://wljt.org/
+        (r'video\.wosu\.org', 'WOSU TV (WOSU)'),  # http://wosu.org/
+        (r'video\.woub\.org', 'WOUB/WOUC (WOUB)'),  # http://woub.org/tv/index.php?section=5
+        (r'video\.wvpublic\.org', 'WVPB (WVPB)'),  # http://wvpublic.org/
+        (r'video\.wkyupbs\.org', 'WKYU-PBS (WKYU)'),  # http://www.wkyupbs.org
+        # (r'wyes\.org', 'WYES-TV/New Orleans (WYES)'),  # http://www.wyes.org
+        (r'video\.kera\.org', 'KERA 13 (KERA)'),  # http://www.kera.org/
+        (r'video\.mpbn\.net', 'MPBN (WCBB)'),  # http://www.mpbn.net/
+        (r'video\.mountainlake\.org', 'Mountain Lake PBS (WCFE)'),  # http://www.mountainlake.org/
+        (r'video\.nhptv\.org', 'NHPTV (WENH)'),  # http://nhptv.org/
+        (r'video\.vpt\.org', 'Vermont PBS (WETK)'),  # http://www.vpt.org
+        (r'video\.witf\.org', 'witf (WITF)'),  # http://www.witf.org
+        (r'watch\.wqed\.org', 'WQED Multimedia (WQED)'),  # http://www.wqed.org/
+        (r'video\.wmht\.org', 'WMHT Educational Telecommunications (WMHT)'),  # http://www.wmht.org/home/
+        (r'video\.deltabroadcasting\.org', 'Q-TV (WDCQ)'),  # http://www.deltabroadcasting.org
+        (r'video\.dptv\.org', 'WTVS Detroit Public TV (WTVS)'),  # http://www.dptv.org/
+        (r'video\.wcmu\.org', 'CMU Public Television (WCMU)'),  # http://www.wcmu.org
+        (r'video\.wkar\.org', 'WKAR-TV (WKAR)'),  # http://wkar.org/
+        (r'wnmuvideo\.nmu\.edu', 'WNMU-TV Public TV 13 (WNMU)'),  # http://wnmutv.nmu.edu
+        (r'video\.wdse\.org', 'WDSE - WRPT (WDSE)'),  # http://www.wdse.org/
+        (r'video\.wgte\.org', 'WGTE TV (WGTE)'),  # http://www.wgte.org
+        (r'video\.lptv\.org', 'Lakeland Public Television (KAWE)'),  # http://www.lakelandptv.org
+        # (r'prairiepublic\.org', 'PRAIRIE PUBLIC (KFME)'),  # http://www.prairiepublic.org/
+        (r'video\.kmos\.org', 'KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS)'),  # http://www.kmos.org/
+        (r'watch\.montanapbs\.org', 'MontanaPBS (KUSM)'),  # http://montanapbs.org
+        (r'video\.krwg\.org', 'KRWG/Channel 22 (KRWG)'),  # http://www.krwg.org
+        (r'video\.kacvtv\.org', 'KACV (KACV)'),  # http://www.panhandlepbs.org/home/
+        (r'video\.kcostv\.org', 'KCOS/Channel 13 (KCOS)'),  # www.kcostv.org
+        (r'video\.wcny\.org', 'WCNY/Channel 24 (WCNY)'),  # http://www.wcny.org
+        (r'video\.wned\.org', 'WNED (WNED)'),  # http://www.wned.org/
+        (r'watch\.wpbstv\.org', 'WPBS (WPBS)'),  # http://www.wpbstv.org
+        (r'video\.wskg\.org', 'WSKG Public TV (WSKG)'),  # http://wskg.org
+        (r'video\.wxxi\.org', 'WXXI (WXXI)'),  # http://wxxi.org
+        (r'video\.wpsu\.org', 'WPSU (WPSU)'),  # http://www.wpsu.org
+        # (r'wqln\.org', 'WQLN/Channel 54 (WQLN)'),  # http://www.wqln.org
+        (r'on-demand\.wvia\.org', 'WVIA Public Media Studios (WVIA)'),  # http://www.wvia.org/
+        (r'video\.wtvi\.org', 'WTVI (WTVI)'),  # http://www.wtvi.org/
+        # (r'whro\.org', 'WHRO (WHRO)'),  # http://whro.org
+        (r'video\.westernreservepublicmedia\.org', 'Western Reserve PBS (WNEO)'),  # http://www.WesternReservePublicMedia.org/
+        (r'video\.ideastream\.org', 'WVIZ/PBS ideastream (WVIZ)'),  # http://www.wviz.org/
+        (r'video\.kcts9\.org', 'KCTS 9 (KCTS)'),  # http://kcts9.org/
+        (r'video\.basinpbs\.org', 'Basin PBS (KPBT)'),  # http://www.basinpbs.org
+        (r'video\.houstonpbs\.org', 'KUHT / Channel 8 (KUHT)'),  # http://www.houstonpublicmedia.org/
+        # (r'tamu\.edu', 'KAMU - TV (KAMU)'),  # http://KAMU.tamu.edu
+        # (r'kedt\.org', 'KEDT/Channel 16 (KEDT)'),  # http://www.kedt.org
+        (r'video\.klrn\.org', 'KLRN (KLRN)'),  # http://www.klrn.org
+        (r'video\.klru\.tv', 'KLRU (KLRU)'),  # http://www.klru.org
+        # (r'kmbh\.org', 'KMBH-TV (KMBH)'),  # http://www.kmbh.org
+        # (r'knct\.org', 'KNCT (KNCT)'),  # http://www.knct.org
+        # (r'ktxt\.org', 'KTTZ-TV (KTXT)'),  # http://www.ktxt.org
+        (r'video\.wtjx\.org', 'WTJX Channel 12 (WTJX)'),  # http://www.wtjx.org/
+        (r'video\.ideastations\.org', 'WCVE PBS (WCVE)'),  # http://ideastations.org/
+        (r'video\.kbtc\.org', 'KBTC Public Television (KBTC)'),  # http://kbtc.org
     )
 
     IE_NAME = 'pbs'
@@ -189,7 +189,7 @@ class PBSIE(InfoExtractor):
            # Player
            (?:video|player)\.pbs\.org/(?:widget/)?partnerplayer/(?P<player_id>[^/]+)/
         )
-    ''' % '|'.join(re.escape(p) for p in list(zip(*_STATIONS))[0])
+    ''' % '|'.join(list(zip(*_STATIONS))[0])
 
     _TESTS = [
         {

From 2469a6aecbc819b87915d31dd2fb5940d84d11d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Dec 2015 22:16:22 +0600
Subject: [PATCH 0072/1105] [noco] Adjust timestamp according to server time
 (Closes #7864)

---
 youtube_dl/extractor/noco.py | 27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 76bd21e6d..63b97f170 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -9,6 +9,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_str,
     compat_urllib_parse,
+    compat_urlparse,
 )
 from ..utils import (
     clean_html,
@@ -82,14 +83,21 @@ class NocoIE(InfoExtractor):
         if 'erreur' in login:
             raise ExtractorError('Unable to login: %s' % clean_html(login['erreur']), expected=True)
 
+    @staticmethod
+    def _ts():
+        return int(time.time() * 1000)
+
     def _call_api(self, path, video_id, note, sub_lang=None):
-        ts = compat_str(int(time.time() * 1000))
+        ts = compat_str(self._ts() + self._ts_offset)
         tk = hashlib.md5((hashlib.md5(ts.encode('ascii')).hexdigest() + '#8S?uCraTedap6a').encode('ascii')).hexdigest()
         url = self._API_URL_TEMPLATE % (path, ts, tk)
         if sub_lang:
             url += self._SUB_LANG_TEMPLATE % sub_lang
 
-        resp = self._download_json(url, video_id, note)
+        request = sanitized_Request(url)
+        request.add_header('Referer', self._referer)
+
+        resp = self._download_json(request, video_id, note)
 
         if isinstance(resp, dict) and resp.get('error'):
             self._raise_error(resp['error'], resp['description'])
@@ -105,6 +113,21 @@ class NocoIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
+        # Timestamp adjustment offset between server time and local time
+        # must be calculated in order to use timestamps closest to server's
+        # in all API requests (see https://github.com/rg3/youtube-dl/issues/7864)
+        webpage = self._download_webpage(url, video_id)
+
+        player_url = self._search_regex(
+            r'(["\'])(?P<player>https?://noco\.tv/(?:[^/]+/)+NocoPlayer.+?\.swf.*?)\1',
+            webpage, 'noco player', group='player',
+            default='http://noco.tv/cdata/js/player/NocoPlayer-v1.2.40.swf')
+
+        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(player_url).query)
+        ts = int_or_none(qs.get('ts', [None])[0])
+        self._ts_offset = ts - self._ts() if ts else 0
+        self._referer = player_url
+
         medias = self._call_api(
             'shows/%s/medias' % video_id,
             video_id, 'Downloading video JSON')

From 7824e1f6a61e97c3c79c21db65de33a6380e29c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Dec 2015 22:16:58 +0600
Subject: [PATCH 0073/1105] [noco] Modernize

---
 youtube_dl/extractor/noco.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 63b97f170..daba2e12f 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -110,8 +110,7 @@ class NocoIE(InfoExtractor):
             expected=True)
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         # Timestamp adjustment offset between server time and local time
         # must be calculated in order to use timestamps closest to server's

From 9dc1d94a0ccf39c4dab686cbd5941d75208248fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Dec 2015 22:18:28 +0600
Subject: [PATCH 0074/1105] [noco] Fix bitrates

---
 youtube_dl/extractor/noco.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index daba2e12f..d440313d5 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -177,8 +177,8 @@ class NocoIE(InfoExtractor):
                         'format_id': format_id_extended,
                         'width': int_or_none(fmt.get('res_width')),
                         'height': int_or_none(fmt.get('res_lines')),
-                        'abr': int_or_none(fmt.get('audiobitrate')),
-                        'vbr': int_or_none(fmt.get('videobitrate')),
+                        'abr': int_or_none(fmt.get('audiobitrate'), 1000),
+                        'vbr': int_or_none(fmt.get('videobitrate'), 1000),
                         'filesize': int_or_none(fmt.get('filesize')),
                         'format_note': qualities[format_id].get('quality_name'),
                         'quality': qualities[format_id].get('priority'),

From 64ccbf18c0e4849618d82add2ff13879f9db67ab Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 17 Dec 2015 20:56:54 +0100
Subject: [PATCH 0075/1105] [livestream] improve extraction, add support for
 live streams and extract more info and formats

---
 youtube_dl/extractor/livestream.py | 276 +++++++++++++++++------------
 1 file changed, 167 insertions(+), 109 deletions(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 6d7733e41..b95c23c8c 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -1,27 +1,29 @@
 from __future__ import unicode_literals
 
 import re
-import json
 import itertools
 
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
-    compat_urllib_parse_urlparse,
     compat_urlparse,
 )
 from ..utils import (
-    ExtractorError,
     find_xpath_attr,
-    int_or_none,
-    orderedSet,
+    xpath_attr,
     xpath_with_ns,
+    xpath_text,
+    orderedSet,
+    int_or_none,
+    float_or_none,
+    parse_iso8601,
+    determine_ext,
 )
 
 
 class LivestreamIE(InfoExtractor):
     IE_NAME = 'livestream'
-    _VALID_URL = r'https?://(?:new\.)?livestream\.com/.*?/(?P<event_name>.*?)(/videos/(?P<id>[0-9]+)(?:/player)?)?/?(?:$|[?#])'
+    _VALID_URL = r'https?://(?:new\.)?livestream\.com/(?:accounts/(?P<account_id>\d+)|(?P<account_name>[^/]+))/(?:events/(?P<event_id>\d+)|(?P<event_name>[^/]+))(?:/videos/(?P<id>\d+))?'
     _TESTS = [{
         'url': 'http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370',
         'md5': '53274c76ba7754fb0e8d072716f2292b',
@@ -29,7 +31,9 @@ class LivestreamIE(InfoExtractor):
             'id': '4719370',
             'ext': 'mp4',
             'title': 'Live from Webster Hall NYC',
+            'timestamp': 1350008072,
             'upload_date': '20121012',
+            'duration': 5968.0,
             'like_count': int,
             'view_count': int,
             'thumbnail': 're:^http://.*\.jpg$'
@@ -55,39 +59,19 @@ class LivestreamIE(InfoExtractor):
         'url': 'http://livestream.com/bsww/concacafbeachsoccercampeonato2015',
         'only_matching': True,
     }]
+    _API_URL_TEMPLATE = 'http://livestream.com/api/accounts/%s/events/%s'
+
+    def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
+        base_ele = find_xpath_attr(smil, self._xpath_ns('.//meta', namespace), 'name', 'httpBase')
+        base = base_ele.get('content') if base_ele else 'http://livestreamvod-f.akamaihd.net/'
 
-    def _parse_smil(self, video_id, smil_url):
         formats = []
-        _SWITCH_XPATH = (
-            './/{http://www.w3.org/2001/SMIL20/Language}body/'
-            '{http://www.w3.org/2001/SMIL20/Language}switch')
-        smil_doc = self._download_xml(
-            smil_url, video_id,
-            note='Downloading SMIL information',
-            errnote='Unable to download SMIL information',
-            fatal=False)
-        if smil_doc is False:  # Download failed
-            return formats
-        title_node = find_xpath_attr(
-            smil_doc, './/{http://www.w3.org/2001/SMIL20/Language}meta',
-            'name', 'title')
-        if title_node is None:
-            self.report_warning('Cannot find SMIL id')
-            switch_node = smil_doc.find(_SWITCH_XPATH)
-        else:
-            title_id = title_node.attrib['content']
-            switch_node = find_xpath_attr(
-                smil_doc, _SWITCH_XPATH, 'id', title_id)
-        if switch_node is None:
-            raise ExtractorError('Cannot find switch node')
-        video_nodes = switch_node.findall(
-            '{http://www.w3.org/2001/SMIL20/Language}video')
+        video_nodes = smil.findall(self._xpath_ns('.//video', namespace))
 
         for vn in video_nodes:
-            tbr = int_or_none(vn.attrib.get('system-bitrate'))
+            tbr = int_or_none(vn.attrib.get('system-bitrate'), 1000)
             furl = (
-                'http://livestream-f.akamaihd.net/%s?v=3.0.3&fp=WIN%%2014,0,0,145' %
-                (vn.attrib['src']))
+                '%s%s?v=3.0.3&fp=WIN%%2014,0,0,145' % (base, vn.attrib['src']))
             if 'clipBegin' in vn.attrib:
                 furl += '&ssek=' + vn.attrib['clipBegin']
             formats.append({
@@ -106,89 +90,136 @@ class LivestreamIE(InfoExtractor):
             ('sd', 'progressive_url'),
             ('hd', 'progressive_url_hd'),
         )
-        formats = [{
-            'format_id': format_id,
-            'url': video_data[key],
-            'quality': i + 1,
-        } for i, (format_id, key) in enumerate(FORMAT_KEYS)
-            if video_data.get(key)]
+
+        formats = []
+        for format_id, key in FORMAT_KEYS:
+            video_url = video_data.get(key)
+            if video_url:
+                ext = determine_ext(video_url)
+                bitrate = int_or_none(self._search_regex(r'(\d+)\.%s' % ext, video_url, 'bitrate', default=None))
+                formats.append({
+                    'url': video_url,
+                    'format_id': format_id,
+                    'tbr': bitrate,
+                    'ext': ext,
+                })
 
         smil_url = video_data.get('smil_url')
         if smil_url:
-            formats.extend(self._parse_smil(video_id, smil_url))
+            smil_formats = self._extract_smil_formats(smil_url, video_id)
+            if smil_formats:
+                formats.extend(smil_formats)
+
+        m3u8_url = video_data.get('m3u8_url')
+        if m3u8_url:
+            m3u8_formats = self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+            if m3u8_formats:
+                formats.extend(m3u8_formats)
+
+        f4m_url = video_data.get('f4m_url')
+        if f4m_url:
+            f4m_formats = self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False)
+            if f4m_formats:
+                formats.extend(f4m_formats)
         self._sort_formats(formats)
 
+        comments = [{
+            'author_id': comment.get('author_id'),
+            'author': comment.get('author', {}).get('full_name'),
+            'id': comment.get('id'),
+            'text': comment['text'],
+            'timestamp': parse_iso8601(comment.get('created_at')),
+        } for comment in video_data.get('comments', {}).get('data', [])]
+
         return {
             'id': video_id,
             'formats': formats,
             'title': video_data['caption'],
+            'description': video_data.get('description'),
             'thumbnail': video_data.get('thumbnail_url'),
-            'upload_date': video_data['updated_at'].replace('-', '')[:8],
+            'duration': float_or_none(video_data.get('duration'), 1000),
+            'timestamp': parse_iso8601(video_data.get('publish_at')),
             'like_count': video_data.get('likes', {}).get('total'),
+            'comment_count': video_data.get('comments', {}).get('total'),
             'view_count': video_data.get('views'),
+            'comments': comments,
         }
 
-    def _extract_event(self, info):
-        event_id = compat_str(info['id'])
-        account = compat_str(info['owner_account_id'])
-        root_url = (
-            'https://new.livestream.com/api/accounts/{account}/events/{event}/'
-            'feed.json'.format(account=account, event=event_id))
+    def _extract_stream_info(self, stream_info):
+        broadcast_id = stream_info['broadcast_id']
+        is_live = stream_info.get('is_live')
 
-        def _extract_videos():
-            last_video = None
-            for i in itertools.count(1):
-                if last_video is None:
-                    info_url = root_url
-                else:
-                    info_url = '{root}?&id={id}&newer=-1&type=video'.format(
-                        root=root_url, id=last_video)
-                videos_info = self._download_json(info_url, event_id, 'Downloading page {0}'.format(i))['data']
-                videos_info = [v['data'] for v in videos_info if v['type'] == 'video']
-                if not videos_info:
-                    break
-                for v in videos_info:
-                    yield self._extract_video_info(v)
-                last_video = videos_info[-1]['id']
-        return self.playlist_result(_extract_videos(), event_id, info['full_name'])
+        formats = []
+        smil_url = stream_info.get('play_url')
+        if smil_url:
+            smil_formats = self._extract_smil_formats(smil_url, broadcast_id)
+            if smil_formats:
+                formats.extend(smil_formats)
+
+        m3u8_url = stream_info.get('m3u8_url')
+        if m3u8_url:
+            m3u8_formats = self._extract_m3u8_formats(
+                m3u8_url, broadcast_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+            if m3u8_formats:
+                formats.extend(m3u8_formats)
+
+        rtsp_url = stream_info.get('rtsp_url')
+        if rtsp_url:
+            formats.append({
+                'url': rtsp_url,
+                'format_id': 'rtsp',
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': broadcast_id,
+            'formats': formats,
+            'title': self._live_title(stream_info['stream_title']) if is_live else stream_info['stream_title'],
+            'thumbnail': stream_info.get('thumbnail_url'),
+            'is_live': is_live,
+        }
+
+    def _extract_event(self, event_data):
+        event_id = compat_str(event_data['id'])
+        account_id = compat_str(event_data['owner_account_id'])
+        feed_root_url = self._API_URL_TEMPLATE % (account_id, event_id) + '/feed.json'
+
+        stream_info = event_data.get('stream_info')
+        if stream_info:
+            return self._extract_stream_info(stream_info)
+
+        last_video = None
+        entries = []
+        for i in itertools.count(1):
+            if last_video is None:
+                info_url = feed_root_url
+            else:
+                info_url = '{root}?&id={id}&newer=-1&type=video'.format(
+                    root=feed_root_url, id=last_video)
+            videos_info = self._download_json(info_url, event_id, 'Downloading page {0}'.format(i))['data']
+            videos_info = [v['data'] for v in videos_info if v['type'] == 'video']
+            if not videos_info:
+                break
+            for v in videos_info:
+                entries.append(self.url_result(
+                    'http://livestream.com/accounts/%s/events/%s/videos/%s' % (account_id, event_id, v['id']),
+                    'Livestream', v['id'], v['caption']))
+            last_video = videos_info[-1]['id']
+        return self.playlist_result(entries, event_id, event_data['full_name'])
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-        event_name = mobj.group('event_name')
-        webpage = self._download_webpage(url, video_id or event_name)
-
-        og_video = self._og_search_video_url(
-            webpage, 'player url', fatal=False, default=None)
-        if og_video is not None:
-            query_str = compat_urllib_parse_urlparse(og_video).query
-            query = compat_urlparse.parse_qs(query_str)
-            if 'play_url' in query:
-                api_url = query['play_url'][0].replace('.smil', '')
-                info = json.loads(self._download_webpage(
-                    api_url, video_id, 'Downloading video info'))
-                return self._extract_video_info(info)
-
-        config_json = self._search_regex(
-            r'window.config = ({.*?});', webpage, 'window config')
-        info = json.loads(config_json)['event']
-
-        def is_relevant(vdata, vid):
-            result = vdata['type'] == 'video'
-            if video_id is not None:
-                result = result and compat_str(vdata['data']['id']) == vid
-            return result
-
-        if video_id is None:
-            # This is an event page:
-            return self._extract_event(info)
+        event = mobj.group('event_id') or mobj.group('event_name')
+        account = mobj.group('account_id') or mobj.group('account_name')
+        api_url = self._API_URL_TEMPLATE % (account, event)
+        if video_id:
+            video_data = self._download_json(api_url + '/videos/%s' % video_id, video_id)
+            return self._extract_video_info(video_data)
         else:
-            videos = [self._extract_video_info(video_data['data'])
-                      for video_data in info['feed']['data']
-                      if is_relevant(video_data, video_id)]
-            if not videos:
-                raise ExtractorError('Cannot find video %s' % video_id)
-            return videos[0]
+            event_data = self._download_json(api_url, video_id)
+            return self._extract_event(event_data)
 
 
 # The original version of Livestream uses a different system
@@ -204,6 +235,8 @@ class LivestreamOriginalIE(InfoExtractor):
             'id': 'pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
             'ext': 'mp4',
             'title': 'Spark 1 (BitCoin) with Cameron Winklevoss & Tyler Winklevoss of Winklevoss Capital',
+            'duration': 771.301,
+            'view_count': int,
         },
     }, {
         'url': 'https://original.livestream.com/newplay/folder?dirId=a07bf706-d0e4-4e75-a747-b021d84f2fd3',
@@ -220,15 +253,42 @@ class LivestreamOriginalIE(InfoExtractor):
         # this url is used on mobile devices
         stream_url = 'http://x{0}x.api.channel.livestream.com/3.0/getstream.json?id={1}'.format(user, video_id)
         stream_info = self._download_json(stream_url, video_id)
+        is_live = stream_info.get('isLive')
         item = info.find('channel').find('item')
-        ns = {'media': 'http://search.yahoo.com/mrss'}
-        thumbnail_url = item.find(xpath_with_ns('media:thumbnail', ns)).attrib['url']
+        media_ns = {'media': 'http://search.yahoo.com/mrss'}
+        thumbnail_url = xpath_attr(item, xpath_with_ns('media:thumbnail', media_ns), 'url')
+        duration = float_or_none(xpath_attr(item, xpath_with_ns('media:content', media_ns), 'duration'))
+        ls_ns = {'ls': 'http://api.channel.livestream.com/2.0'}
+        view_count = int_or_none(xpath_text(item, xpath_with_ns('ls:viewsCount', ls_ns)))
+
+        formats = [{
+            'url': stream_info['progressiveUrl'],
+            'format_id': 'http',
+        }]
+
+        m3u8_url = stream_info.get('httpUrl')
+        if m3u8_url:
+            m3u8_formats = self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+            if m3u8_formats:
+                formats.extend(m3u8_formats)
+
+        rtsp_url = stream_info.get('rtspUrl')
+        if rtsp_url:
+            formats.append({
+                'url': rtsp_url,
+                'format_id': 'rtsp',
+            })
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': item.find('title').text,
-            'url': stream_info['progressiveUrl'],
+            'title': self._live_title(xpath_text(item, 'title')) if is_live else xpath_text(item, 'title'),
+            'formats': formats,
             'thumbnail': thumbnail_url,
+            'duration': duration,
+            'view_count': view_count,
+            'is_live': is_live,
         }
 
     def _extract_folder(self, url, folder_id):
@@ -239,14 +299,12 @@ class LivestreamOriginalIE(InfoExtractor):
                 <a\s+href="(?=https?://livestre\.am/)
             )([^"]+)"''', webpage))
 
-        return {
-            '_type': 'playlist',
-            'id': folder_id,
-            'entries': [{
-                '_type': 'url',
-                'url': compat_urlparse.urljoin(url, p),
-            } for p in paths],
-        }
+        entries = [{
+            '_type': 'url',
+            'url': compat_urlparse.urljoin(url, p),
+        } for p in paths]
+
+        return self.playlist_result(entries, folder_id)
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 45dad8bab98e30eb3fb1cf0f20770c1dce74610b Mon Sep 17 00:00:00 2001
From: Gautam M <Blue9@users.noreply.github.com>
Date: Fri, 18 Dec 2015 03:16:36 -0500
Subject: [PATCH 0076/1105] Fix hyperlink to youtube-dl options

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 5c3915d64..7002f45e0 100644
--- a/README.md
+++ b/README.md
@@ -757,7 +757,7 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
     ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
 ```
 
-Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L117-L265). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
+Most likely, you'll want to use various options. For a list of what can be done, have a look at [`youtube_dl/YoutubeDL.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L121-L269). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
 
 Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
 

From 10171468d94e68f92505a77938be11d266ccabd8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 18 Dec 2015 18:20:41 +0800
Subject: [PATCH 0077/1105] [iqiyi] Update key (closes #7896)

---
 youtube_dl/extractor/iqiyi.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index f96e12e69..c3731a110 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -205,8 +205,8 @@ class IqiyiIE(InfoExtractor):
 
     def get_enc_key(self, swf_url, video_id):
         # TODO: automatic key extraction
-        # last update at 2015-12-06 for Zombie::bite
-        enc_key = '3719f6a1da83ee0aee3488d8802d7696'[::-1]
+        # last update at 2015-12-18 for Zombie::bite
+        enc_key = '8b6b683780897eb8d9a48a02ccc4817d'[::-1]
         return enc_key
 
     def _real_extract(self, url):

From b95779be21929c1aed1c0edbc3f9a5d11826973f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Dec 2015 18:57:49 +0600
Subject: [PATCH 0078/1105] [jsinterp] Extend function regex (Closes #7900,
 closes #7901)

---
 youtube_dl/jsinterp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 2191e8b89..a7440c582 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -232,7 +232,7 @@ class JSInterpreter(object):
     def extract_function(self, funcname):
         func_m = re.search(
             r'''(?x)
-                (?:function\s+%s|[{;]%s\s*=\s*function|var\s+%s\s*=\s*function)\s*
+                (?:function\s+%s|[{;,]%s\s*=\s*function|var\s+%s\s*=\s*function)\s*
                 \((?P<args>[^)]*)\)\s*
                 \{(?P<code>[^}]+)\}''' % (
                 re.escape(funcname), re.escape(funcname), re.escape(funcname)),

From 016dd820505e635ed1d4e5890f370cf800b25c7f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 18 Dec 2015 14:21:30 +0100
Subject: [PATCH 0079/1105] release 2015.12.18

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 20b44b94d..01607693e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.12.13'
+__version__ = '2015.12.18'

From 9796a9b20ccc46217c27523774f5088d74e672bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Dec 2015 21:34:17 +0600
Subject: [PATCH 0080/1105] [ndr] Fix description and upload date extraction
 (Closes #7893)

---
 youtube_dl/extractor/ndr.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 16213eed9..894c51399 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -88,10 +88,10 @@ class NDRIE(NDRBaseIE):
             'embedURL', webpage, 'embed URL', fatal=True)
         description = self._search_regex(
             r'<p[^>]+itemprop="description">([^<]+)</p>',
-            webpage, 'description', fatal=False)
+            webpage, 'description', default=None) or self._og_search_description(webpage)
         timestamp = parse_iso8601(
             self._search_regex(
-                r'<span itemprop="datePublished" content="([^"]+)">',
+                r'<span[^>]+itemprop="(?:datePublished|uploadDate)"[^>]+content="([^"]+)"',
                 webpage, 'upload date', fatal=False))
         return {
             '_type': 'url_transparent',

From 7234d1d9c795bd43799de47952f65f72952e0564 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Dec 2015 22:05:32 +0600
Subject: [PATCH 0081/1105] [brightcove:new] Add _extract_url

---
 youtube_dl/extractor/brightcove.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index f5ebae1e6..66b8d2dff 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -389,6 +389,11 @@ class BrightcoveNewIE(InfoExtractor):
         }
     }]
 
+    @staticmethod
+    def _extract_url(webpage):
+        urls = BrightcoveNewIE._extract_urls(webpage)
+        return urls[0] if urls else None
+
     @staticmethod
     def _extract_urls(webpage):
         # Reference:

From 15d50aca64421c97d251bcd86e9bb308265dfdaa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Dec 2015 22:05:56 +0600
Subject: [PATCH 0082/1105] [nowness] Add support for brightcove:new videos
 (Closes #7884)

---
 youtube_dl/extractor/nowness.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index d480fb58c..446f5901c 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -1,7 +1,10 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-from .brightcove import BrightcoveLegacyIE
+from .brightcove import (
+    BrightcoveLegacyIE,
+    BrightcoveNewIE,
+)
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
@@ -23,9 +26,12 @@ class NownessBaseIE(InfoExtractor):
                             note='Downloading player JavaScript',
                             errnote='Unable to download player JavaScript')
                         bc_url = BrightcoveLegacyIE._extract_brightcove_url(player_code)
-                        if bc_url is None:
-                            raise ExtractorError('Could not find player definition')
-                        return self.url_result(bc_url, 'BrightcoveLegacy')
+                        if bc_url:
+                            return self.url_result(bc_url, BrightcoveLegacyIE.ie_key())
+                        bc_url = BrightcoveNewIE._extract_url(player_code)
+                        if bc_url:
+                            return self.url_result(bc_url, BrightcoveNewIE.ie_key())
+                        raise ExtractorError('Could not find player definition')
                     elif source == 'vimeo':
                         return self.url_result('http://vimeo.com/%s' % video_id, 'Vimeo')
                     elif source == 'youtube':

From 9fd0f676788f94768b80fe5ff87895b607c84345 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Dec 2015 22:18:55 +0600
Subject: [PATCH 0083/1105] [brightcove:new] Add support for ref: preffixed
 video ids (Closes #7794)

---
 youtube_dl/extractor/brightcove.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 66b8d2dff..d77590195 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -355,7 +355,7 @@ class BrightcoveLegacyIE(InfoExtractor):
 
 class BrightcoveNewIE(InfoExtractor):
     IE_NAME = 'brightcove:new'
-    _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>\d+)'
+    _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>(?:ref:)?\d+)'
     _TESTS = [{
         'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
         'md5': 'c8100925723840d4b0d243f7025703be',
@@ -414,7 +414,7 @@ class BrightcoveNewIE(InfoExtractor):
                 # may be optional and what to do when it is
                 r'''(?sx)
                     <video[^>]+
-                        data-video-id=["\'](\d+)["\'][^>]*>.*?
+                        data-video-id=["\']((?:ref:)?\d+)["\'][^>]*>.*?
                     </video>.*?
                     <script[^>]+
                         src=["\'](?:https?:)?//players\.brightcove\.net/

From f81ccbb3df205c612275f8881ddf21115eb590b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Dec 2015 22:20:44 +0600
Subject: [PATCH 0084/1105] [brightcove:new] Fix typo

---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index d77590195..b0536ddbd 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -398,7 +398,7 @@ class BrightcoveNewIE(InfoExtractor):
     def _extract_urls(webpage):
         # Reference:
         # 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe
-        # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript)
+        # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript
         # 3. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/embed-in-page.html
 
         entries = []

From 5b72fda140488685be55215d584f764c39577b02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Dec 2015 22:22:41 +0600
Subject: [PATCH 0085/1105] [brightcove:new] Clarify ref: prefix

---
 youtube_dl/extractor/brightcove.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index b0536ddbd..4e983fa10 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -400,6 +400,7 @@ class BrightcoveNewIE(InfoExtractor):
         # 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe
         # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript
         # 3. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/embed-in-page.html
+        # 4. https://support.brightcove.com/en/video-cloud/docs/dynamically-assigning-videos-player
 
         entries = []
 
@@ -412,6 +413,7 @@ class BrightcoveNewIE(InfoExtractor):
         for video_id, account_id, player_id, embed in re.findall(
                 # According to examples from [3] it's unclear whether video id
                 # may be optional and what to do when it is
+                # According to [4] data-video-id may be prefixed with ref:
                 r'''(?sx)
                     <video[^>]+
                         data-video-id=["\']((?:ref:)?\d+)["\'][^>]*>.*?

From 4f29fa99069760dc47ef9ca5dbf607a567d2982f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Dec 2015 22:31:48 +0600
Subject: [PATCH 0086/1105] [brightcove:new] Add test for ref: prefixed video
 id

---
 youtube_dl/extractor/brightcove.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 4e983fa10..03a4f446e 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -387,6 +387,10 @@ class BrightcoveNewIE(InfoExtractor):
         'params': {
             'skip_download': True,
         }
+    }, {
+        # ref: prefixed video id
+        'url': 'http://players.brightcove.net/3910869709001/21519b5c-4b3b-4363-accb-bdc8f358f823_default/index.html?videoId=ref:7069442',
+        'only_matching': True,
     }]
 
     @staticmethod

From d631d5f9f27f93767226192e4288990413fa9dbd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 19 Dec 2015 18:21:42 +0800
Subject: [PATCH 0087/1105] [utils] Fix TTML conversion

Tolerate invalid timestamps (closes #7909)
---
 test/test_utils.py  |  7 +++++--
 youtube_dl/utils.py | 11 ++++++++---
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 8fc74e591..86045e680 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -661,8 +661,8 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
             {'like_count': 190, 'dislike_count': 10}))
 
     def test_parse_dfxp_time_expr(self):
-        self.assertEqual(parse_dfxp_time_expr(None), 0.0)
-        self.assertEqual(parse_dfxp_time_expr(''), 0.0)
+        self.assertEqual(parse_dfxp_time_expr(None), None)
+        self.assertEqual(parse_dfxp_time_expr(''), None)
         self.assertEqual(parse_dfxp_time_expr('0.1'), 0.1)
         self.assertEqual(parse_dfxp_time_expr('0.1s'), 0.1)
         self.assertEqual(parse_dfxp_time_expr('00:00:01'), 1.0)
@@ -676,6 +676,9 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
                     <p begin="0" end="1">The following line contains Chinese characters and special symbols</p>
                     <p begin="1" end="2">第二行<br/>♪♪</p>
                     <p begin="2" dur="1"><span>Third<br/>Line</span></p>
+                    <p begin="3" end="-1">Lines with invalid timestamps are ignored</p>
+                    <p begin="-1" end="-1">Ignore, two</p>
+                    <p begin="3" dur="-1">Ignored, three</p>
                 </div>
             </body>
             </tt>'''
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 91917fc96..ee20c3d9b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1976,7 +1976,7 @@ def match_filter_func(filter_str):
 
 def parse_dfxp_time_expr(time_expr):
     if not time_expr:
-        return 0.0
+        return
 
     mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
     if mobj:
@@ -2020,10 +2020,15 @@ def dfxp2srt(dfxp_data):
         raise ValueError('Invalid dfxp/TTML subtitle')
 
     for para, index in zip(paras, itertools.count(1)):
-        begin_time = parse_dfxp_time_expr(para.attrib['begin'])
+        begin_time = parse_dfxp_time_expr(para.attrib.get('begin'))
         end_time = parse_dfxp_time_expr(para.attrib.get('end'))
+        dur = parse_dfxp_time_expr(para.attrib.get('dur'))
+        if begin_time is None:
+            continue
         if not end_time:
-            end_time = begin_time + parse_dfxp_time_expr(para.attrib['dur'])
+            if not dur:
+                continue
+            end_time = begin_time + dur
         out.append('%d\n%s --> %s\n%s\n\n' % (
             index,
             srt_subtitles_timecode(begin_time),

From db2fe38b5508cbd28b89893219d9cccd41406851 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 19 Dec 2015 19:29:51 +0800
Subject: [PATCH 0088/1105] [utils] Support alternative timestamp format in
 TTML

Fixes #7608
---
 test/test_utils.py  | 1 +
 youtube_dl/utils.py | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 86045e680..e4e8d3825 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -667,6 +667,7 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
         self.assertEqual(parse_dfxp_time_expr('0.1s'), 0.1)
         self.assertEqual(parse_dfxp_time_expr('00:00:01'), 1.0)
         self.assertEqual(parse_dfxp_time_expr('00:00:01.100'), 1.1)
+        self.assertEqual(parse_dfxp_time_expr('00:00:01:100'), 1.1)
 
     def test_dfxp2srt(self):
         dfxp_data = '''<?xml version="1.0" encoding="UTF-8"?>
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index ee20c3d9b..5b396ede8 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1982,9 +1982,9 @@ def parse_dfxp_time_expr(time_expr):
     if mobj:
         return float(mobj.group('time_offset'))
 
-    mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:\.\d+)?)$', time_expr)
+    mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:(?:\.|:)\d+)?)$', time_expr)
     if mobj:
-        return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3))
+        return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3).replace(':', '.'))
 
 
 def srt_subtitles_timecode(seconds):

From ee0f0393cfd3daeff729f5f5594e4c5b1bec5436 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 17 Sep 2015 13:51:50 +0800
Subject: [PATCH 0089/1105] [togglesg] New extractor for toggle.sg

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/togglesg.py | 159 +++++++++++++++++++++++++++++++
 2 files changed, 160 insertions(+)
 create mode 100644 youtube_dl/extractor/togglesg.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3cd95ba01..b88fbcc4d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -675,6 +675,7 @@ from .tnaflix import (
     EMPFlixIE,
     MovieFapIE,
 )
+from .togglesg import ToggleSgIE
 from .thvideo import (
     THVideoIE,
     THVideoPlaylistIE
diff --git a/youtube_dl/extractor/togglesg.py b/youtube_dl/extractor/togglesg.py
new file mode 100644
index 000000000..56ef4b464
--- /dev/null
+++ b/youtube_dl/extractor/togglesg.py
@@ -0,0 +1,159 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+import re
+import itertools
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    determine_ext,
+    parse_iso8601,
+    remove_end
+)
+from ..compat import compat_urllib_request
+
+
+class ToggleSgIE(InfoExtractor):
+    IE_NAME = 'togglesg'
+    _VALID_URL = r'https?://video\.toggle\.sg/(?:(en|zh))/(?:(series|clips|movies))/.+?/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://video.toggle.sg/en/series/lion-moms-tif/trailers/lion-moms-premier/343115',
+        'info_dict': {
+            'id': '343115',
+            'ext': 'mp4',
+            'title': 'Lion Moms Premiere',
+            'description': 'md5:aea1149404bff4d7f7b6da11fafd8e6b',
+            'upload_date': '20150910',
+            'timestamp': 1441858274,
+        },
+        'params': {
+            'skip_download': 'm3u8 download',
+        }
+    }, {
+        'note': 'DRM-protected video',
+        'url': 'http://video.toggle.sg/en/movies/dug-s-special-mission/341413',
+        'info_dict': {
+            'id': '341413',
+            'ext': 'wvm',
+            'title': 'Dug\'s Special Mission',
+            'description': 'md5:e86c6f4458214905c1772398fabc93e0',
+            'upload_date': '20150827',
+            'timestamp': 1440644006,
+        },
+        'params': {
+            'skip_download': 'DRM-protected wvm download',
+        }
+    }, {
+        'note': 'm3u8 links are geo-restricted, but Android/mp4 is okay',
+        'url': 'http://video.toggle.sg/en/series/28th-sea-games-5-show/ep11/332861',
+        'info_dict': {
+            'id': '332861',
+            'ext': 'mp4',
+            'title': '28th SEA Games (5 Show) -  Episode  11',
+            'description': 'md5:3cd4f5f56c7c3b1340c50a863f896faa',
+            'upload_date': '20150605',
+            'timestamp': 1433480166,
+        },
+        'params': {
+            'skip_download': 'DRM-protected wvm download',
+        },
+        'skip': 'm3u8 links are geo-restricted'
+    }, {
+        'url': 'http://video.toggle.sg/en/clips/seraph-sun-aloysius-will-suddenly-sing-some-old-songs-in-high-pitch-on-set/343331',
+        'only_matching': True,
+    }, {
+        'url': 'http://video.toggle.sg/zh/series/zero-calling-s2-hd/ep13/336367',
+        'only_matching': True,
+    }, {
+        'url': 'http://video.toggle.sg/en/series/vetri-s2/webisodes/jeeva-is-an-orphan-vetri-s2-webisode-7/342302',
+        'only_matching': True,
+    }, {
+        'url': 'http://video.toggle.sg/en/movies/seven-days/321936',
+        'only_matching': True,
+    }]
+
+    _FORMAT_PREFERENCES = {
+        'wvm-STBMain': -10,
+        'wvm-iPadMain': -20,
+        'wvm-iPhoneMain': -30,
+        'wvm-Android': -40,
+    }
+    _API_USER = 'tvpapi_147'
+    _API_PASS = '11111'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id, note='Downloading video page')
+
+        api_user = self._search_regex(
+            r'apiUser:\s*"([^"]+)"', webpage, 'apiUser', default=self._API_USER, fatal=False)
+        api_pass = self._search_regex(
+            r'apiPass:\s*"([^"]+)"', webpage, 'apiPass', default=self._API_PASS, fatal=False)
+
+        params = {
+            'initObj': {
+                'Locale': {
+                    'LocaleLanguage': '', 'LocaleCountry': '',
+                    'LocaleDevice': '', 'LocaleUserState': 0
+                },
+                'Platform': 0, 'SiteGuid': 0, 'DomainID': '0', 'UDID': '',
+                'ApiUser': api_user, 'ApiPass': api_pass
+            },
+            'MediaID': video_id,
+            'mediaType': 0,
+        }
+
+        req = compat_urllib_request.Request(
+            'http://tvpapi.as.tvinci.com/v2_9/gateways/jsonpostgw.aspx?m=GetMediaInfo',
+            json.dumps(params).encode('utf-8'))
+        info = self._download_json(req, video_id, 'Downloading video info json')
+
+        title = info['MediaName']
+        duration = int_or_none(info.get('Duration'))
+        thumbnail = info.get('PicURL')
+        description = info.get('Description')
+        created_at = parse_iso8601(info.get('CreationDate') or None)
+        formats = []
+
+        for video_file in info.get('Files', []):
+            ext = determine_ext(video_file['URL'])
+            vid_format = video_file['Format'].replace(' ', '')
+            # if geo-restricted, m3u8 is inaccessible, but mp4 is okay
+            if ext == 'm3u8':
+                m3u8_formats = self._extract_m3u8_formats(
+                    video_file['URL'], video_id, ext='mp4', m3u8_id=vid_format,
+                    note='Downloading %s m3u8 information' % vid_format,
+                    errnote='Failed to download %s m3u8 information' % vid_format,
+                    fatal=False
+                )
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
+            if ext in ['mp4', 'wvm']:
+                # wvm are drm-protected files
+                formats.append({
+                    'ext': ext,
+                    'url': video_file['URL'],
+                    'format_id': vid_format,
+                    'preference': self._FORMAT_PREFERENCES.get(ext + '-' + vid_format) or -1,
+                    'format_note': 'DRM-protected video' if ext == 'wvm' else None
+                })
+
+        if not formats:
+            # Most likely because geo-blocked
+            raise ExtractorError('No downloadable videos found', expected=True)
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'timestamp': created_at,
+            'thumbnail': thumbnail,
+            'formats': formats,
+        }

From ed370ff0e6f52aaf3647b32cdbba9d25ce8533e5 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Fri, 18 Sep 2015 00:51:41 +0800
Subject: [PATCH 0090/1105] [togglesg] Fixes

---
 youtube_dl/extractor/togglesg.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/togglesg.py b/youtube_dl/extractor/togglesg.py
index 56ef4b464..9f958d453 100644
--- a/youtube_dl/extractor/togglesg.py
+++ b/youtube_dl/extractor/togglesg.py
@@ -18,7 +18,7 @@ from ..compat import compat_urllib_request
 
 class ToggleSgIE(InfoExtractor):
     IE_NAME = 'togglesg'
-    _VALID_URL = r'https?://video\.toggle\.sg/(?:(en|zh))/(?:(series|clips|movies))/.+?/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://video\.toggle\.sg/(?:en|zh)/(?:series|clips|movies)/.+?/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://video.toggle.sg/en/series/lion-moms-tif/trailers/lion-moms-premier/343115',
         'info_dict': {
@@ -90,9 +90,9 @@ class ToggleSgIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id, note='Downloading video page')
 
         api_user = self._search_regex(
-            r'apiUser:\s*"([^"]+)"', webpage, 'apiUser', default=self._API_USER, fatal=False)
+            r'apiUser:\s*"([^"]+)"', webpage, 'apiUser', default=self._API_USER)
         api_pass = self._search_regex(
-            r'apiPass:\s*"([^"]+)"', webpage, 'apiPass', default=self._API_PASS, fatal=False)
+            r'apiPass:\s*"([^"]+)"', webpage, 'apiPass', default=self._API_PASS)
 
         params = {
             'initObj': {

From f8253af561655c7c470323e631e8b66283bcf623 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Dec 2015 19:03:55 +0600
Subject: [PATCH 0091/1105] [toggle] Use sanitized_Request

---
 youtube_dl/extractor/togglesg.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/togglesg.py b/youtube_dl/extractor/togglesg.py
index 9f958d453..05a9e5439 100644
--- a/youtube_dl/extractor/togglesg.py
+++ b/youtube_dl/extractor/togglesg.py
@@ -11,7 +11,8 @@ from ..utils import (
     int_or_none,
     determine_ext,
     parse_iso8601,
-    remove_end
+    remove_end,
+    sanitized_Request,
 )
 from ..compat import compat_urllib_request
 
@@ -107,7 +108,7 @@ class ToggleSgIE(InfoExtractor):
             'mediaType': 0,
         }
 
-        req = compat_urllib_request.Request(
+        req = sanitized_Request(
             'http://tvpapi.as.tvinci.com/v2_9/gateways/jsonpostgw.aspx?m=GetMediaInfo',
             json.dumps(params).encode('utf-8'))
         info = self._download_json(req, video_id, 'Downloading video info json')

From c82a8dd14c813e1b022188925fc33fb4d831bc3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Dec 2015 19:04:38 +0600
Subject: [PATCH 0092/1105] [toggle] Remove unused imports

---
 youtube_dl/extractor/togglesg.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/togglesg.py b/youtube_dl/extractor/togglesg.py
index 05a9e5439..ea660960e 100644
--- a/youtube_dl/extractor/togglesg.py
+++ b/youtube_dl/extractor/togglesg.py
@@ -2,19 +2,15 @@
 from __future__ import unicode_literals
 
 import json
-import re
-import itertools
 
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
     ExtractorError,
     int_or_none,
-    determine_ext,
     parse_iso8601,
-    remove_end,
     sanitized_Request,
 )
-from ..compat import compat_urllib_request
 
 
 class ToggleSgIE(InfoExtractor):

From 74c730174fa872ae37b4709a3a452391ee6273fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Dec 2015 19:06:05 +0600
Subject: [PATCH 0093/1105] [toggle] Style

---
 youtube_dl/extractor/togglesg.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/togglesg.py b/youtube_dl/extractor/togglesg.py
index ea660960e..d3903b58d 100644
--- a/youtube_dl/extractor/togglesg.py
+++ b/youtube_dl/extractor/togglesg.py
@@ -94,11 +94,17 @@ class ToggleSgIE(InfoExtractor):
         params = {
             'initObj': {
                 'Locale': {
-                    'LocaleLanguage': '', 'LocaleCountry': '',
-                    'LocaleDevice': '', 'LocaleUserState': 0
+                    'LocaleLanguage': '',
+                    'LocaleCountry': '',
+                    'LocaleDevice': '',
+                    'LocaleUserState': 0
                 },
-                'Platform': 0, 'SiteGuid': 0, 'DomainID': '0', 'UDID': '',
-                'ApiUser': api_user, 'ApiPass': api_pass
+                'Platform': 0,
+                'SiteGuid': 0,
+                'DomainID': '0',
+                'UDID': '',
+                'ApiUser': api_user,
+                'ApiPass': api_pass
             },
             'MediaID': video_id,
             'mediaType': 0,

From ffaf6e66e3830d0e4750aec2cfdaff1a6bd9c2ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Dec 2015 19:08:47 +0600
Subject: [PATCH 0094/1105] [toggle] Improve

---
 youtube_dl/extractor/togglesg.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/togglesg.py b/youtube_dl/extractor/togglesg.py
index d3903b58d..244c79e8d 100644
--- a/youtube_dl/extractor/togglesg.py
+++ b/youtube_dl/extractor/togglesg.py
@@ -84,12 +84,15 @@ class ToggleSgIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id, note='Downloading video page')
+        webpage = self._download_webpage(
+            url, video_id, note='Downloading video page')
 
         api_user = self._search_regex(
-            r'apiUser:\s*"([^"]+)"', webpage, 'apiUser', default=self._API_USER)
+            r'apiUser\s*:\s*(["\'])(?P<user>.+?)\1', webpage, 'apiUser',
+            default=self._API_USER, group='user')
         api_pass = self._search_regex(
-            r'apiPass:\s*"([^"]+)"', webpage, 'apiPass', default=self._API_PASS)
+            r'apiPass\s*:\s*(["\'])(?P<pass>.+?)\1', webpage, 'apiPass',
+            default=self._API_PASS, group='pass')
 
         params = {
             'initObj': {
@@ -131,11 +134,10 @@ class ToggleSgIE(InfoExtractor):
                     video_file['URL'], video_id, ext='mp4', m3u8_id=vid_format,
                     note='Downloading %s m3u8 information' % vid_format,
                     errnote='Failed to download %s m3u8 information' % vid_format,
-                    fatal=False
-                )
+                    fatal=False)
                 if m3u8_formats:
                     formats.extend(m3u8_formats)
-            if ext in ['mp4', 'wvm']:
+            elif ext in ('mp4', 'wvm'):
                 # wvm are drm-protected files
                 formats.append({
                     'ext': ext,

From c40dbb19ab475e8a0e1b29548130adf9ce13ea43 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Dec 2015 19:19:26 +0600
Subject: [PATCH 0095/1105] [toggle] Extract thumbnails

---
 youtube_dl/extractor/togglesg.py | 34 ++++++++++++++++++++++++--------
 1 file changed, 26 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/togglesg.py b/youtube_dl/extractor/togglesg.py
index 244c79e8d..a2b89d6bb 100644
--- a/youtube_dl/extractor/togglesg.py
+++ b/youtube_dl/extractor/togglesg.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import json
+import re
 
 from .common import InfoExtractor
 from ..utils import (
@@ -119,12 +120,8 @@ class ToggleSgIE(InfoExtractor):
         info = self._download_json(req, video_id, 'Downloading video info json')
 
         title = info['MediaName']
-        duration = int_or_none(info.get('Duration'))
-        thumbnail = info.get('PicURL')
-        description = info.get('Description')
-        created_at = parse_iso8601(info.get('CreationDate') or None)
-        formats = []
 
+        formats = []
         for video_file in info.get('Files', []):
             ext = determine_ext(video_file['URL'])
             vid_format = video_file['Format'].replace(' ', '')
@@ -146,19 +143,40 @@ class ToggleSgIE(InfoExtractor):
                     'preference': self._FORMAT_PREFERENCES.get(ext + '-' + vid_format) or -1,
                     'format_note': 'DRM-protected video' if ext == 'wvm' else None
                 })
-
         if not formats:
             # Most likely because geo-blocked
             raise ExtractorError('No downloadable videos found', expected=True)
-
         self._sort_formats(formats)
 
+        duration = int_or_none(info.get('Duration'))
+        description = info.get('Description')
+        created_at = parse_iso8601(info.get('CreationDate') or None)
+
+        thumbnails = []
+        for picture in info.get('Pictures', []):
+            if not isinstance(picture, dict):
+                continue
+            pic_url = picture.get('URL')
+            if not pic_url:
+                continue
+            thumbnail = {
+                'url': pic_url,
+            }
+            pic_size = picture.get('PicSize', '')
+            m = re.search(r'(?P<width>\d+)[xX](?P<height>\d+)', pic_size)
+            if m:
+                thumbnail.update({
+                    'width': int(m.group('width')),
+                    'height': int(m.group('height')),
+                })
+            thumbnails.append(thumbnail)
+
         return {
             'id': video_id,
             'title': title,
             'description': description,
             'duration': duration,
             'timestamp': created_at,
-            'thumbnail': thumbnail,
+            'thumbnails': thumbnails,
             'formats': formats,
         }

From 8f097af4ec5d6ff72a80c744d67f947336fecb28 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Dec 2015 19:23:28 +0600
Subject: [PATCH 0096/1105] [toggle] Extract counters

---
 youtube_dl/extractor/togglesg.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/togglesg.py b/youtube_dl/extractor/togglesg.py
index a2b89d6bb..36367971f 100644
--- a/youtube_dl/extractor/togglesg.py
+++ b/youtube_dl/extractor/togglesg.py
@@ -8,6 +8,7 @@ from .common import InfoExtractor
 from ..utils import (
     determine_ext,
     ExtractorError,
+    float_or_none,
     int_or_none,
     parse_iso8601,
     sanitized_Request,
@@ -152,6 +153,10 @@ class ToggleSgIE(InfoExtractor):
         description = info.get('Description')
         created_at = parse_iso8601(info.get('CreationDate') or None)
 
+        average_rating = float_or_none(info.get('Rating'))
+        view_count = int_or_none(info.get('ViewCounter') or info.get('view_counter'))
+        like_count = int_or_none(info.get('LikeCounter') or info.get('like_counter'))
+
         thumbnails = []
         for picture in info.get('Pictures', []):
             if not isinstance(picture, dict):
@@ -177,6 +182,9 @@ class ToggleSgIE(InfoExtractor):
             'description': description,
             'duration': duration,
             'timestamp': created_at,
+            'average_rating': average_rating,
+            'view_count': view_count,
+            'like_count': like_count,
             'thumbnails': thumbnails,
             'formats': formats,
         }

From 989e9f8eadc936ab47fcd0b7ba63aac13020a526 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Dec 2015 19:52:37 +0600
Subject: [PATCH 0097/1105] [toggle] Improve formats extraction robustness

---
 youtube_dl/extractor/togglesg.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/togglesg.py b/youtube_dl/extractor/togglesg.py
index 36367971f..47b2bfcb8 100644
--- a/youtube_dl/extractor/togglesg.py
+++ b/youtube_dl/extractor/togglesg.py
@@ -124,12 +124,15 @@ class ToggleSgIE(InfoExtractor):
 
         formats = []
         for video_file in info.get('Files', []):
-            ext = determine_ext(video_file['URL'])
-            vid_format = video_file['Format'].replace(' ', '')
+            video_url, vid_format = video_file.get('URL'), video_file.get('Format')
+            if not video_url or not vid_format:
+                continue
+            ext = determine_ext(video_url)
+            vid_format = vid_format.replace(' ', '')
             # if geo-restricted, m3u8 is inaccessible, but mp4 is okay
             if ext == 'm3u8':
                 m3u8_formats = self._extract_m3u8_formats(
-                    video_file['URL'], video_id, ext='mp4', m3u8_id=vid_format,
+                    video_url, video_id, ext='mp4', m3u8_id=vid_format,
                     note='Downloading %s m3u8 information' % vid_format,
                     errnote='Failed to download %s m3u8 information' % vid_format,
                     fatal=False)
@@ -139,7 +142,7 @@ class ToggleSgIE(InfoExtractor):
                 # wvm are drm-protected files
                 formats.append({
                     'ext': ext,
-                    'url': video_file['URL'],
+                    'url': video_url,
                     'format_id': vid_format,
                     'preference': self._FORMAT_PREFERENCES.get(ext + '-' + vid_format) or -1,
                     'format_note': 'DRM-protected video' if ext == 'wvm' else None

From e33c9cba7c6c9d4a04d749c88ab447a57bcfe423 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Dec 2015 19:58:18 +0600
Subject: [PATCH 0098/1105] [toggle] Improve _VALID_URL

---
 youtube_dl/extractor/togglesg.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/togglesg.py b/youtube_dl/extractor/togglesg.py
index 47b2bfcb8..22c6c91b9 100644
--- a/youtube_dl/extractor/togglesg.py
+++ b/youtube_dl/extractor/togglesg.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 class ToggleSgIE(InfoExtractor):
     IE_NAME = 'togglesg'
-    _VALID_URL = r'https?://video\.toggle\.sg/(?:en|zh)/(?:series|clips|movies)/.+?/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://video\.toggle\.sg/(?:en|zh)/(?:series|clips|movies)/(?:[^/]+/)+(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://video.toggle.sg/en/series/lion-moms-tif/trailers/lion-moms-premier/343115',
         'info_dict': {
@@ -46,8 +46,9 @@ class ToggleSgIE(InfoExtractor):
             'skip_download': 'DRM-protected wvm download',
         }
     }, {
+        # this also tests correct video id extraction
         'note': 'm3u8 links are geo-restricted, but Android/mp4 is okay',
-        'url': 'http://video.toggle.sg/en/series/28th-sea-games-5-show/ep11/332861',
+        'url': 'http://video.toggle.sg/en/series/28th-sea-games-5-show/28th-sea-games-5-show-ep11/332861',
         'info_dict': {
             'id': '332861',
             'ext': 'mp4',

From cc0f378d5429225caf0477f5d10b342b1ef136d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Dec 2015 19:59:00 +0600
Subject: [PATCH 0099/1105] [toggle] Rename to toggle

---
 youtube_dl/extractor/__init__.py                | 2 +-
 youtube_dl/extractor/{togglesg.py => toggle.py} | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename youtube_dl/extractor/{togglesg.py => toggle.py} (99%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b88fbcc4d..fef2cff62 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -675,7 +675,7 @@ from .tnaflix import (
     EMPFlixIE,
     MovieFapIE,
 )
-from .togglesg import ToggleSgIE
+from .toggle import ToggleIE
 from .thvideo import (
     THVideoIE,
     THVideoPlaylistIE
diff --git a/youtube_dl/extractor/togglesg.py b/youtube_dl/extractor/toggle.py
similarity index 99%
rename from youtube_dl/extractor/togglesg.py
rename to youtube_dl/extractor/toggle.py
index 22c6c91b9..c9d6e52e1 100644
--- a/youtube_dl/extractor/togglesg.py
+++ b/youtube_dl/extractor/toggle.py
@@ -15,7 +15,7 @@ from ..utils import (
 )
 
 
-class ToggleSgIE(InfoExtractor):
+class ToggleIE(InfoExtractor):
     IE_NAME = 'togglesg'
     _VALID_URL = r'https?://video\.toggle\.sg/(?:en|zh)/(?:series|clips|movies)/(?:[^/]+/)+(?P<id>[0-9]+)'
     _TESTS = [{

From 0f206ee81447349f9ec8c430de4868a86020564c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Dec 2015 23:11:23 +0600
Subject: [PATCH 0100/1105] [toggle] Change IE_NAME

---
 youtube_dl/extractor/toggle.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/toggle.py b/youtube_dl/extractor/toggle.py
index c9d6e52e1..a47239952 100644
--- a/youtube_dl/extractor/toggle.py
+++ b/youtube_dl/extractor/toggle.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class ToggleIE(InfoExtractor):
-    IE_NAME = 'togglesg'
+    IE_NAME = 'toggle'
     _VALID_URL = r'https?://video\.toggle\.sg/(?:en|zh)/(?:series|clips|movies)/(?:[^/]+/)+(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://video.toggle.sg/en/series/lion-moms-tif/trailers/lion-moms-premier/343115',

From e0f06eae432e592b159237f2ce9813449431cc7b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 19 Dec 2015 18:26:28 +0100
Subject: [PATCH 0101/1105] [fktv] fix info extraction

---
 youtube_dl/extractor/fktv.py | 30 +++++++++++++-----------------
 1 file changed, 13 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index 40ea27895..5f6e65dae 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -1,12 +1,10 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
     determine_ext,
-    ExtractorError,
+    js_to_json,
 )
 
 
@@ -32,24 +30,22 @@ class FKTVIE(InfoExtractor):
             'http://fernsehkritik.tv/folge-%s/play' % episode, episode)
         title = clean_html(self._html_search_regex(
             '<h3>([^<]+)</h3>', webpage, 'title'))
-        matches = re.search(
-            r'(?s)<video(?:(?!poster)[^>])+(?:poster="([^"]+)")?[^>]*>(.*)</video>',
-            webpage)
-        if matches is None:
-            raise ExtractorError('Unable to extract the video')
+        thumbnail = self._search_regex(r'POSTER\s*=\s*"([^"]+)', webpage, 'thumbnail', fatal=False)
+        sources = self._parse_json(self._search_regex(r'(?s)MEDIA\s*=\s*(\[.+?\]);', webpage, 'media'), episode, js_to_json)
 
-        poster, sources = matches.groups()
-        if poster is None:
-            self.report_warning('unable to extract thumbnail')
+        formats = []
+        for source in sources:
+            furl = source.get('src')
+            if furl:
+                formats.append({
+                    'url': furl,
+                    'format_id': determine_ext(furl),
+                })
+        self._sort_formats(formats)
 
-        urls = re.findall(r'<source[^>]+src="([^"]+)"', sources)
-        formats = [{
-            'url': furl,
-            'format_id': determine_ext(furl),
-        } for furl in urls]
         return {
             'id': episode,
             'title': title,
             'formats': formats,
-            'thumbnail': poster,
+            'thumbnail': thumbnail,
         }

From fa64a8431166ad7ca7bd2881306ee379b6e2abfd Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 19 Dec 2015 19:02:04 +0100
Subject: [PATCH 0102/1105] [faz] fix info extraction

---
 youtube_dl/extractor/faz.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/faz.py b/youtube_dl/extractor/faz.py
index cebdd0193..d9a868119 100644
--- a/youtube_dl/extractor/faz.py
+++ b/youtube_dl/extractor/faz.py
@@ -38,7 +38,7 @@ class FazIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
         config_xml_url = self._search_regex(
-            r'writeFLV\(\'(.+?)\',', webpage, 'config xml url')
+            r'(?:var\s+)?videoXMLURL\s*=\s*"([^"]+)', webpage, 'config xml url')
         config = self._download_xml(
             config_xml_url, video_id, 'Downloading config xml')
 

From dd85e4d70787dd0c106fb05bdb0a38f51037fd39 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 02:43:50 +0600
Subject: [PATCH 0103/1105] [extractor/common] Properly decode error string on
 python 2 (Closes #1354, closes #3957, closes #4037, closes #6449)

---
 youtube_dl/extractor/common.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6ab2d68d6..3ab72ff76 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -34,6 +34,7 @@ from ..utils import (
     fix_xml_ampersands,
     float_or_none,
     int_or_none,
+    preferredencoding,
     RegexNotFoundError,
     sanitize_filename,
     sanitized_Request,
@@ -332,7 +333,12 @@ class InfoExtractor(object):
                 return False
             if errnote is None:
                 errnote = 'Unable to download webpage'
-            errmsg = '%s: %s' % (errnote, compat_str(err))
+            err_str = str(err)
+            # On python 2 error byte string must be decoded with proper
+            # encoding rather than ascii
+            if sys.version_info[0] < 3:
+                err_str = err_str.decode(preferredencoding())
+            errmsg = '%s: %s' % (errnote, err_str)
             if fatal:
                 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
             else:

From ec6504b39c9b3d9766c551691a96e481278fa659 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 19 Dec 2015 23:28:54 +0100
Subject: [PATCH 0104/1105] [gputechconf] Add new extractor(closes #5775)

---
 youtube_dl/extractor/__init__.py    |  1 +
 youtube_dl/extractor/gputechconf.py | 57 +++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+)
 create mode 100644 youtube_dl/extractor/gputechconf.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fef2cff62..65c5e1c92 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -235,6 +235,7 @@ from .golem import GolemIE
 from .googleplus import GooglePlusIE
 from .googlesearch import GoogleSearchIE
 from .goshgay import GoshgayIE
+from .gputechconf import GPUTechConfIE
 from .groupon import GrouponIE
 from .hark import HarkIE
 from .hearthisat import HearThisAtIE
diff --git a/youtube_dl/extractor/gputechconf.py b/youtube_dl/extractor/gputechconf.py
new file mode 100644
index 000000000..becc61f1e
--- /dev/null
+++ b/youtube_dl/extractor/gputechconf.py
@@ -0,0 +1,57 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    xpath_element,
+    xpath_text,
+    int_or_none,
+    parse_duration,
+)
+
+
+class GPUTechConfIE(InfoExtractor):
+    _VALID_URL = r'https?://on-demand\.gputechconf\.com/gtc/2015/video/S(?P<id>\d+)\.html'
+    _TEST = {
+        'url': 'http://on-demand.gputechconf.com/gtc/2015/video/S5156.html',
+        'md5': 'a8862a00a0fd65b8b43acc5b8e33f798',
+        'info_dict': {
+            'id': '5156',
+            'ext': 'mp4',
+            'title': 'Coordinating More Than 3 Million CUDA Threads for Social Network Analysis',
+            'duration': 1219,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        root_path = self._search_regex(r'var\s+rootPath\s*=\s*"([^"]+)', webpage, 'root path', 'http://evt.dispeak.com/nvidia/events/gtc15/')
+        xml_file_id = self._search_regex(r'var\s+xmlFileId\s*=\s*"([^"]+)', webpage, 'xml file id')
+
+        doc = self._download_xml('%sxml/%s.xml' % (root_path, xml_file_id), video_id)
+
+        metadata = xpath_element(doc, 'metadata')
+        http_host = xpath_text(metadata, 'httpHost')
+        mbr_videos = xpath_element(metadata, 'MBRVideos')
+
+        formats = []
+        for mbr_video in mbr_videos.findall('MBRVideo'):
+            stream_name = xpath_text(mbr_video, 'streamName')
+            if stream_name:
+                bitrate = int_or_none(xpath_text(mbr_video, 'bitrate'))
+                formats.append({
+                    'url': 'http://%s/%s' % (http_host, stream_name.replace('mp4:', '')),
+                    'tbr': bitrate,
+                    'format_id': 'http-%d' % bitrate,
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': xpath_text(metadata, 'title'),
+            'duration': parse_duration(xpath_text(metadata, 'endTime')),
+            'creator': xpath_text(metadata, 'speaker'),
+            'formats': formats,
+        }

From 1deb710f26b0ccd14db13103cb400b663e243c57 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 19 Dec 2015 23:59:00 +0100
Subject: [PATCH 0105/1105] [gputechconf] improve extraction

---
 youtube_dl/extractor/gputechconf.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/gputechconf.py b/youtube_dl/extractor/gputechconf.py
index becc61f1e..145b55bf3 100644
--- a/youtube_dl/extractor/gputechconf.py
+++ b/youtube_dl/extractor/gputechconf.py
@@ -33,18 +33,16 @@ class GPUTechConfIE(InfoExtractor):
         doc = self._download_xml('%sxml/%s.xml' % (root_path, xml_file_id), video_id)
 
         metadata = xpath_element(doc, 'metadata')
-        http_host = xpath_text(metadata, 'httpHost')
+        http_host = xpath_text(metadata, 'httpHost', 'http host', True)
         mbr_videos = xpath_element(metadata, 'MBRVideos')
 
         formats = []
         for mbr_video in mbr_videos.findall('MBRVideo'):
             stream_name = xpath_text(mbr_video, 'streamName')
             if stream_name:
-                bitrate = int_or_none(xpath_text(mbr_video, 'bitrate'))
                 formats.append({
                     'url': 'http://%s/%s' % (http_host, stream_name.replace('mp4:', '')),
-                    'tbr': bitrate,
-                    'format_id': 'http-%d' % bitrate,
+                    'tbr': int_or_none(xpath_text(mbr_video, 'bitrate')),
                 })
         self._sort_formats(formats)
 

From fdae2358581be6f80ac5fd20b1062e87a8797f46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 05:26:47 +0600
Subject: [PATCH 0106/1105] [utils] Add error_to_str

---
 youtube_dl/utils.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 5b396ede8..6d3119760 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1806,6 +1806,15 @@ def args_to_str(args):
     return ' '.join(shlex_quote(a) for a in args)
 
 
+def error_to_str(err):
+    err_str = str(err)
+    # On python 2 error byte string must be decoded with proper
+    # encoding rather than ascii
+    if sys.version_info[0] < 3:
+        err_str = err_str.decode(preferredencoding())
+    return err_str
+
+
 def mimetype2ext(mt):
     _, _, res = mt.rpartition('/')
 

From 7f8b271465df75bb1f83cb181dc45fee5fe02cf9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 05:27:38 +0600
Subject: [PATCH 0107/1105] Properly convert errors to strings

---
 youtube_dl/YoutubeDL.py             |  7 ++++---
 youtube_dl/downloader/common.py     |  4 ++--
 youtube_dl/extractor/common.py      | 12 ++++--------
 youtube_dl/extractor/dailymotion.py |  6 +++---
 youtube_dl/extractor/facebook.py    |  4 ++--
 youtube_dl/extractor/youtube.py     |  3 ++-
 6 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index c642a1fbf..26b3adb02 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -48,6 +48,7 @@ from .utils import (
     determine_ext,
     DownloadError,
     encodeFilename,
+    error_to_str,
     ExtractorError,
     format_bytes,
     formatSeconds,
@@ -681,7 +682,7 @@ class YoutubeDL(object):
                 raise
             except Exception as e:
                 if self.params.get('ignoreerrors', False):
-                    self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
+                    self.report_error(error_to_str(e), tb=compat_str(traceback.format_exc()))
                     break
                 else:
                     raise
@@ -1459,7 +1460,7 @@ class YoutubeDL(object):
             if dn and not os.path.exists(dn):
                 os.makedirs(dn)
         except (OSError, IOError) as err:
-            self.report_error('unable to create directory ' + compat_str(err))
+            self.report_error('unable to create directory ' + error_to_str(err))
             return
 
         if self.params.get('writedescription', False):
@@ -2039,4 +2040,4 @@ class YoutubeDL(object):
                                    (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
                     self.report_warning('Unable to download thumbnail "%s": %s' %
-                                        (t['url'], compat_str(err)))
+                                        (t['url'], error_to_str(err)))
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index b8bf8daf8..eb63ccffd 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -5,9 +5,9 @@ import re
 import sys
 import time
 
-from ..compat import compat_str
 from ..utils import (
     encodeFilename,
+    error_to_str,
     decodeArgument,
     format_bytes,
     timeconvert,
@@ -186,7 +186,7 @@ class FileDownloader(object):
                 return
             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
         except (IOError, OSError) as err:
-            self.report_error('unable to rename file: %s' % compat_str(err))
+            self.report_error('unable to rename file: %s' % error_to_str(err))
 
     def try_utime(self, filename, last_modified_hdr):
         """Try to set the last-modified time of the given file."""
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 3ab72ff76..5659e40c8 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -30,11 +30,11 @@ from ..utils import (
     clean_html,
     compiled_regex_type,
     determine_ext,
+    error_to_str,
     ExtractorError,
     fix_xml_ampersands,
     float_or_none,
     int_or_none,
-    preferredencoding,
     RegexNotFoundError,
     sanitize_filename,
     sanitized_Request,
@@ -333,12 +333,8 @@ class InfoExtractor(object):
                 return False
             if errnote is None:
                 errnote = 'Unable to download webpage'
-            err_str = str(err)
-            # On python 2 error byte string must be decoded with proper
-            # encoding rather than ascii
-            if sys.version_info[0] < 3:
-                err_str = err_str.decode(preferredencoding())
-            errmsg = '%s: %s' % (errnote, err_str)
+
+            errmsg = '%s: %s' % (errnote, error_to_str(err))
             if fatal:
                 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
             else:
@@ -628,7 +624,7 @@ class InfoExtractor(object):
                 else:
                     raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
             except (IOError, netrc.NetrcParseError) as err:
-                self._downloader.report_warning('parsing .netrc: %s' % compat_str(err))
+                self._downloader.report_warning('parsing .netrc: %s' % error_to_str(err))
 
         return (username, password)
 
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 428556213..7d66baf96 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -7,10 +7,10 @@ import itertools
 
 from .common import InfoExtractor
 
-from ..compat import compat_str
 from ..utils import (
-    ExtractorError,
     determine_ext,
+    error_to_str,
+    ExtractorError,
     int_or_none,
     parse_iso8601,
     sanitized_Request,
@@ -278,7 +278,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                 'https://api.dailymotion.com/video/%s/subtitles?fields=id,language,url' % video_id,
                 video_id, note=False)
         except ExtractorError as err:
-            self._downloader.report_warning('unable to download video subtitles: %s' % compat_str(err))
+            self._downloader.report_warning('unable to download video subtitles: %s' % error_to_str(err))
             return {}
         info = json.loads(sub_list)
         if (info['total'] > 0):
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 321eec59e..296d3dd77 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -7,11 +7,11 @@ import socket
 from .common import InfoExtractor
 from ..compat import (
     compat_http_client,
-    compat_str,
     compat_urllib_error,
     compat_urllib_parse_unquote,
 )
 from ..utils import (
+    error_to_str,
     ExtractorError,
     limit_length,
     sanitized_Request,
@@ -116,7 +116,7 @@ class FacebookIE(InfoExtractor):
             if re.search(r'id="checkpointSubmitButton"', check_response) is not None:
                 self._downloader.report_warning('Unable to confirm login, you have to login in your brower and authorize the login.')
         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-            self._downloader.report_warning('unable to log in: %s' % compat_str(err))
+            self._downloader.report_warning('unable to log in: %s' % error_to_str(err))
             return
 
     def _real_initialize(self):
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4556a16fb..d63aa5f4e 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -26,6 +26,7 @@ from ..compat import (
 from ..utils import (
     clean_html,
     encode_dict,
+    error_to_str,
     ExtractorError,
     float_or_none,
     get_element_by_attribute,
@@ -903,7 +904,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
                 video_id, note=False)
         except ExtractorError as err:
-            self._downloader.report_warning('unable to download video subtitles: %s' % compat_str(err))
+            self._downloader.report_warning('unable to download video subtitles: %s' % error_to_str(err))
             return {}
 
         sub_lang_list = {}

From fb043a6e4ee98a2ea9fbb0975e701341bbd34bc3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 06:16:19 +0600
Subject: [PATCH 0108/1105] [YoutubeDL] Use error_to_str

---
 youtube_dl/YoutubeDL.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 26b3adb02..216b5a5db 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -675,8 +675,8 @@ class YoutubeDL(object):
                     return self.process_ie_result(ie_result, download, extra_info)
                 else:
                     return ie_result
-            except ExtractorError as de:  # An error we somewhat expected
-                self.report_error(compat_str(de), de.format_traceback())
+            except ExtractorError as e:  # An error we somewhat expected
+                self.report_error(error_to_str(e), e.format_traceback())
                 break
             except MaxDownloadsReached:
                 raise

From 8900ab4d9bc6bf8a05a1a2608efaff08561945e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 06:22:01 +0600
Subject: [PATCH 0109/1105] [YoutubeDL] More error_to_str

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 216b5a5db..57df6a279 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1511,7 +1511,7 @@ class YoutubeDL(object):
                             sub_info['url'], info_dict['id'], note=False)
                     except ExtractorError as err:
                         self.report_warning('Unable to download subtitle for "%s": %s' %
-                                            (sub_lang, compat_str(err.cause)))
+                                            (sub_lang, error_to_str(err.cause)))
                         continue
                 try:
                     sub_filename = subtitles_filename(filename, sub_lang, sub_format)

From 8e60dc7526596f456c0b5d7dc48daa6cae08ebb6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 06:26:26 +0600
Subject: [PATCH 0110/1105] [utils] Add encode_compat_str

---
 youtube_dl/utils.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 6d3119760..9a2dd1439 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1712,6 +1712,10 @@ def encode_dict(d, encoding='utf-8'):
     return dict((encode(k), encode(v)) for k, v in d.items())
 
 
+def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):
+    return string if isinstance(string, compat_str) else compat_str(string, encoding, errors)
+
+
 US_RATINGS = {
     'G': 0,
     'PG': 10,

From c0384f221e5a8383c377a3c43f634cf9807eb634 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 06:29:36 +0600
Subject: [PATCH 0111/1105] Use proper encoding on compat_str construction when
 necessary

---
 youtube_dl/YoutubeDL.py |  5 +++--
 youtube_dl/update.py    | 16 ++++++++--------
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 57df6a279..1795097ae 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -47,6 +47,7 @@ from .utils import (
     DEFAULT_OUTTMPL,
     determine_ext,
     DownloadError,
+    encode_compat_str,
     encodeFilename,
     error_to_str,
     ExtractorError,
@@ -496,7 +497,7 @@ class YoutubeDL(object):
                     tb = ''
                     if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
                         tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
-                    tb += compat_str(traceback.format_exc())
+                    tb += encode_compat_str(traceback.format_exc())
                 else:
                     tb_data = traceback.format_list(traceback.extract_stack())
                     tb = ''.join(tb_data)
@@ -682,7 +683,7 @@ class YoutubeDL(object):
                 raise
             except Exception as e:
                 if self.params.get('ignoreerrors', False):
-                    self.report_error(error_to_str(e), tb=compat_str(traceback.format_exc()))
+                    self.report_error(error_to_str(e), tb=encode_compat_str(traceback.format_exc()))
                     break
                 else:
                     raise
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index 074eb64a7..995b8ed96 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -9,7 +9,7 @@ import subprocess
 import sys
 from zipimport import zipimporter
 
-from .compat import compat_str
+from .utils import encode_compat_str
 
 from .version import __version__
 
@@ -61,7 +61,7 @@ def update_self(to_screen, verbose, opener):
         newversion = opener.open(VERSION_URL).read().decode('utf-8').strip()
     except Exception:
         if verbose:
-            to_screen(compat_str(traceback.format_exc()))
+            to_screen(encode_compat_str(traceback.format_exc()))
         to_screen('ERROR: can\'t find the current version. Please try again later.')
         return
     if newversion == __version__:
@@ -74,7 +74,7 @@ def update_self(to_screen, verbose, opener):
         versions_info = json.loads(versions_info)
     except Exception:
         if verbose:
-            to_screen(compat_str(traceback.format_exc()))
+            to_screen(encode_compat_str(traceback.format_exc()))
         to_screen('ERROR: can\'t obtain versions info. Please try again later.')
         return
     if 'signature' not in versions_info:
@@ -123,7 +123,7 @@ def update_self(to_screen, verbose, opener):
             urlh.close()
         except (IOError, OSError):
             if verbose:
-                to_screen(compat_str(traceback.format_exc()))
+                to_screen(encode_compat_str(traceback.format_exc()))
             to_screen('ERROR: unable to download latest version')
             return
 
@@ -137,7 +137,7 @@ def update_self(to_screen, verbose, opener):
                 outf.write(newcontent)
         except (IOError, OSError):
             if verbose:
-                to_screen(compat_str(traceback.format_exc()))
+                to_screen(encode_compat_str(traceback.format_exc()))
             to_screen('ERROR: unable to write the new version')
             return
 
@@ -157,7 +157,7 @@ start /b "" cmd /c del "%%~f0"&exit /b"
             return  # Do not show premature success messages
         except (IOError, OSError):
             if verbose:
-                to_screen(compat_str(traceback.format_exc()))
+                to_screen(encode_compat_str(traceback.format_exc()))
             to_screen('ERROR: unable to overwrite current version')
             return
 
@@ -169,7 +169,7 @@ start /b "" cmd /c del "%%~f0"&exit /b"
             urlh.close()
         except (IOError, OSError):
             if verbose:
-                to_screen(compat_str(traceback.format_exc()))
+                to_screen(encode_compat_str(traceback.format_exc()))
             to_screen('ERROR: unable to download latest version')
             return
 
@@ -183,7 +183,7 @@ start /b "" cmd /c del "%%~f0"&exit /b"
                 outf.write(newcontent)
         except (IOError, OSError):
             if verbose:
-                to_screen(compat_str(traceback.format_exc()))
+                to_screen(encode_compat_str(traceback.format_exc()))
             to_screen('ERROR: unable to overwrite current version')
             return
 

From 2c74e6fa77935804b8e63807e23762e97a25f4d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 06:35:58 +0600
Subject: [PATCH 0112/1105] [YoutubeDL] Revert error_to_str for ExtractorError

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 1795097ae..bf05e9340 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -677,7 +677,7 @@ class YoutubeDL(object):
                 else:
                     return ie_result
             except ExtractorError as e:  # An error we somewhat expected
-                self.report_error(error_to_str(e), e.format_traceback())
+                self.report_error(compat_str(e), e.format_traceback())
                 break
             except MaxDownloadsReached:
                 raise

From d890b4cc0ac6b88daa917a1a791afe7541cd0411 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 06:43:42 +0600
Subject: [PATCH 0113/1105] [nbc:news] Remove unnecessary compat_str

---
 youtube_dl/extractor/nbc.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 4c1eca96f..340c922bd 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -3,10 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-    compat_HTTPError,
-)
+from ..compat import compat_HTTPError
 from ..utils import (
     ExtractorError,
     find_xpath_attr,
@@ -189,7 +186,7 @@ class NBCNewsIE(InfoExtractor):
                 'title': info.find('headline').text,
                 'ext': 'flv',
                 'url': find_xpath_attr(info, 'media', 'type', 'flashVideo').text,
-                'description': compat_str(info.find('caption').text),
+                'description': info.find('caption').text,
                 'thumbnail': find_xpath_attr(info, 'media', 'type', 'thumbnail').text,
             }
         else:

From 9b9c5355e4c0fe6f9028aaa3b1b8beb28a085433 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 07:00:39 +0600
Subject: [PATCH 0114/1105] Rename error_to_str to error_to_compat_str

---
 youtube_dl/YoutubeDL.py             | 10 +++++-----
 youtube_dl/downloader/common.py     |  4 ++--
 youtube_dl/extractor/common.py      |  6 +++---
 youtube_dl/extractor/dailymotion.py |  4 ++--
 youtube_dl/extractor/facebook.py    |  4 ++--
 youtube_dl/extractor/youtube.py     |  4 ++--
 youtube_dl/utils.py                 |  2 +-
 7 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index bf05e9340..50425b8d7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -49,7 +49,7 @@ from .utils import (
     DownloadError,
     encode_compat_str,
     encodeFilename,
-    error_to_str,
+    error_to_compat_str,
     ExtractorError,
     format_bytes,
     formatSeconds,
@@ -683,7 +683,7 @@ class YoutubeDL(object):
                 raise
             except Exception as e:
                 if self.params.get('ignoreerrors', False):
-                    self.report_error(error_to_str(e), tb=encode_compat_str(traceback.format_exc()))
+                    self.report_error(error_to_compat_str(e), tb=encode_compat_str(traceback.format_exc()))
                     break
                 else:
                     raise
@@ -1461,7 +1461,7 @@ class YoutubeDL(object):
             if dn and not os.path.exists(dn):
                 os.makedirs(dn)
         except (OSError, IOError) as err:
-            self.report_error('unable to create directory ' + error_to_str(err))
+            self.report_error('unable to create directory ' + error_to_compat_str(err))
             return
 
         if self.params.get('writedescription', False):
@@ -1512,7 +1512,7 @@ class YoutubeDL(object):
                             sub_info['url'], info_dict['id'], note=False)
                     except ExtractorError as err:
                         self.report_warning('Unable to download subtitle for "%s": %s' %
-                                            (sub_lang, error_to_str(err.cause)))
+                                            (sub_lang, error_to_compat_str(err.cause)))
                         continue
                 try:
                     sub_filename = subtitles_filename(filename, sub_lang, sub_format)
@@ -2041,4 +2041,4 @@ class YoutubeDL(object):
                                    (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
                     self.report_warning('Unable to download thumbnail "%s": %s' %
-                                        (t['url'], error_to_str(err)))
+                                        (t['url'], error_to_compat_str(err)))
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index eb63ccffd..beae8c4d0 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -7,7 +7,7 @@ import time
 
 from ..utils import (
     encodeFilename,
-    error_to_str,
+    error_to_compat_str,
     decodeArgument,
     format_bytes,
     timeconvert,
@@ -186,7 +186,7 @@ class FileDownloader(object):
                 return
             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
         except (IOError, OSError) as err:
-            self.report_error('unable to rename file: %s' % error_to_str(err))
+            self.report_error('unable to rename file: %s' % error_to_compat_str(err))
 
     def try_utime(self, filename, last_modified_hdr):
         """Try to set the last-modified time of the given file."""
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5659e40c8..828f58f12 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -30,7 +30,7 @@ from ..utils import (
     clean_html,
     compiled_regex_type,
     determine_ext,
-    error_to_str,
+    error_to_compat_str,
     ExtractorError,
     fix_xml_ampersands,
     float_or_none,
@@ -334,7 +334,7 @@ class InfoExtractor(object):
             if errnote is None:
                 errnote = 'Unable to download webpage'
 
-            errmsg = '%s: %s' % (errnote, error_to_str(err))
+            errmsg = '%s: %s' % (errnote, error_to_compat_str(err))
             if fatal:
                 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
             else:
@@ -624,7 +624,7 @@ class InfoExtractor(object):
                 else:
                     raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
             except (IOError, netrc.NetrcParseError) as err:
-                self._downloader.report_warning('parsing .netrc: %s' % error_to_str(err))
+                self._downloader.report_warning('parsing .netrc: %s' % error_to_compat_str(err))
 
         return (username, password)
 
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 7d66baf96..0c5b6617f 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -9,7 +9,7 @@ from .common import InfoExtractor
 
 from ..utils import (
     determine_ext,
-    error_to_str,
+    error_to_compat_str,
     ExtractorError,
     int_or_none,
     parse_iso8601,
@@ -278,7 +278,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                 'https://api.dailymotion.com/video/%s/subtitles?fields=id,language,url' % video_id,
                 video_id, note=False)
         except ExtractorError as err:
-            self._downloader.report_warning('unable to download video subtitles: %s' % error_to_str(err))
+            self._downloader.report_warning('unable to download video subtitles: %s' % error_to_compat_str(err))
             return {}
         info = json.loads(sub_list)
         if (info['total'] > 0):
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 296d3dd77..39c481068 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -11,7 +11,7 @@ from ..compat import (
     compat_urllib_parse_unquote,
 )
 from ..utils import (
-    error_to_str,
+    error_to_compat_str,
     ExtractorError,
     limit_length,
     sanitized_Request,
@@ -116,7 +116,7 @@ class FacebookIE(InfoExtractor):
             if re.search(r'id="checkpointSubmitButton"', check_response) is not None:
                 self._downloader.report_warning('Unable to confirm login, you have to login in your brower and authorize the login.')
         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-            self._downloader.report_warning('unable to log in: %s' % error_to_str(err))
+            self._downloader.report_warning('unable to log in: %s' % error_to_compat_str(err))
             return
 
     def _real_initialize(self):
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index d63aa5f4e..89759a1cb 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -26,7 +26,7 @@ from ..compat import (
 from ..utils import (
     clean_html,
     encode_dict,
-    error_to_str,
+    error_to_compat_str,
     ExtractorError,
     float_or_none,
     get_element_by_attribute,
@@ -904,7 +904,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
                 video_id, note=False)
         except ExtractorError as err:
-            self._downloader.report_warning('unable to download video subtitles: %s' % error_to_str(err))
+            self._downloader.report_warning('unable to download video subtitles: %s' % error_to_compat_str(err))
             return {}
 
         sub_lang_list = {}
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 9a2dd1439..1737ac5f6 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1810,7 +1810,7 @@ def args_to_str(args):
     return ' '.join(shlex_quote(a) for a in args)
 
 
-def error_to_str(err):
+def error_to_compat_str(err):
     err_str = str(err)
     # On python 2 error byte string must be decoded with proper
     # encoding rather than ascii

From 6b77d52b1f8ca3777284e4164e5363c110aeba66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 07:07:14 +0600
Subject: [PATCH 0115/1105] [test_utils] Add tests for encode_compat_str

---
 test/test_utils.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index e4e8d3825..1c3290d9b 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -22,6 +22,7 @@ from youtube_dl.utils import (
     DateRange,
     detect_exe_version,
     determine_ext,
+    encode_compat_str,
     encodeFilename,
     escape_rfc3986,
     escape_url,
@@ -449,6 +450,10 @@ class TestUtil(unittest.TestCase):
         data = urlencode_postdata({'username': 'foo@bar.com', 'password': '1234'})
         self.assertTrue(isinstance(data, bytes))
 
+    def test_encode_compat_str(self):
+        self.assertEqual(encode_compat_str(b'\xd1\x82\xd0\xb5\xd1\x81\xd1\x82', 'utf-8'), 'тест')
+        self.assertEqual(encode_compat_str('тест', 'utf-8'), 'тест')
+
     def test_parse_iso8601(self):
         self.assertEqual(parse_iso8601('2014-03-23T23:04:26+0100'), 1395612266)
         self.assertEqual(parse_iso8601('2014-03-23T22:04:26+0000'), 1395612266)

From e462474e1d944e8b1547edc619a13a7ee2abbc2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 07:48:16 +0600
Subject: [PATCH 0116/1105] [youtube] Generalize playlists extractor

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/youtube.py  | 19 +++++++++++++++----
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 65c5e1c92..eac50eda5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -852,7 +852,7 @@ from .youtube import (
     YoutubeTruncatedIDIE,
     YoutubeTruncatedURLIE,
     YoutubeUserIE,
-    YoutubeUserPlaylistsIE,
+    YoutubePlaylistsIE,
     YoutubeWatchLaterIE,
 )
 from .zapiks import ZapiksIE
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 89759a1cb..4aac2cc03 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1775,6 +1775,10 @@ class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
         },
     }]
 
+    @classmethod
+    def suitable(cls, url):
+        return False if YoutubePlaylistsIE.suitable(url) else super(YoutubeChannelIE, cls).suitable(url)
+
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
@@ -1848,10 +1852,10 @@ class YoutubeUserIE(YoutubeChannelIE):
             return super(YoutubeUserIE, cls).suitable(url)
 
 
-class YoutubeUserPlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
-    IE_DESC = 'YouTube.com user playlists'
-    _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/user/(?P<id>[^/]+)/playlists'
-    IE_NAME = 'youtube:user:playlists'
+class YoutubePlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
+    IE_DESC = 'YouTube.com user/channel playlists'
+    _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/(?:user|channel)/(?P<id>[^/]+)/playlists'
+    IE_NAME = 'youtube:playlists'
 
     _TESTS = [{
         'url': 'http://www.youtube.com/user/ThirstForScience/playlists',
@@ -1868,6 +1872,13 @@ class YoutubeUserPlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
             'id': 'igorkle1',
             'title': 'Игорь Клейнер',
         },
+    }, {
+        'url': 'https://www.youtube.com/channel/UCiU1dHvZObB2iP6xkJ__Icw/playlists',
+        'playlist_mincount': 17,
+        'info_dict': {
+            'id': 'UCiU1dHvZObB2iP6xkJ__Icw',
+            'title': 'Chem Player',
+        },
     }]
 
 

From c6ed6fadc248bd4957fd7cccc1eeb0f109ac3ffa Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 20 Dec 2015 12:43:00 +0100
Subject: [PATCH 0117/1105] [cnet] improve extraction

- relex data json regex
- extract the platform metadata once
- extract hds formats
- extract duration
- extract thumbnail
---
 youtube_dl/extractor/cnet.py | 41 ++++++++++++++++++++----------------
 1 file changed, 23 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/cnet.py b/youtube_dl/extractor/cnet.py
index 3ecf0efd4..5c3908f72 100644
--- a/youtube_dl/extractor/cnet.py
+++ b/youtube_dl/extractor/cnet.py
@@ -1,33 +1,33 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
-
-from .common import InfoExtractor
 from .theplatform import ThePlatformIE
+from ..utils import int_or_none
 
 
-class CNETIE(InfoExtractor):
+class CNETIE(ThePlatformIE):
     _VALID_URL = r'https?://(?:www\.)?cnet\.com/videos/(?P<id>[^/]+)/'
     _TESTS = [{
         'url': 'http://www.cnet.com/videos/hands-on-with-microsofts-windows-8-1-update/',
         'info_dict': {
             'id': '56f4ea68-bd21-4852-b08c-4de5b8354c60',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Hands-on with Microsoft Windows 8.1 Update',
             'description': 'The new update to the Windows 8 OS brings improved performance for mouse and keyboard users.',
             'uploader_id': '6085384d-619e-11e3-b231-14feb5ca9861',
             'uploader': 'Sarah Mitroff',
+            'duration': 70,
         },
     }, {
         'url': 'http://www.cnet.com/videos/whiny-pothole-tweets-at-local-government-when-hit-by-cars-tomorrow-daily-187/',
         'info_dict': {
             'id': '56527b93-d25d-44e3-b738-f989ce2e49ba',
-            'ext': 'mp4',
+            'ext': 'flv',
+            'title': 'Whiny potholes tweet at local government when hit by cars (Tomorrow Daily 187)',
             'description': 'Khail and Ashley wonder what other civic woes can be solved by self-tweeting objects, investigate a new kind of VR camera and watch an origami robot self-assemble, walk, climb, dig and dissolve. #TDPothole',
             'uploader_id': 'b163284d-6b73-44fc-b3e6-3da66c392d40',
             'uploader': 'Ashley Esqueda',
-            'title': 'Whiny potholes tweet at local government when hit by cars (Tomorrow Daily 187)',
+            'duration': 1482,
         },
     }]
 
@@ -36,10 +36,10 @@ class CNETIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         data_json = self._html_search_regex(
-            r"<div class=\"videoPlayer\"\s+.*?data-cnet-video-uvp-options='([^']+)'",
+            r"data-cnet-video(?:-uvp)?-options='([^']+)'",
             webpage, 'data json')
-        data = json.loads(data_json)
-        vdata = data['videos'][0]
+        data = self._parse_json(data_json, display_id)
+        vdata = data.get('video') or data['videos'][0]
 
         video_id = vdata['id']
         title = vdata['title']
@@ -52,19 +52,22 @@ class CNETIE(InfoExtractor):
             uploader_id = None
 
         mpx_account = data['config']['uvpConfig']['default']['mpx_account']
-        tp = ThePlatformIE(self._downloader)
+
+        metadata = self.get_metadata('%s/%s' % (mpx_account, list(vdata['files'].values())[0]), video_id)
+        description = vdata.get('description') or metadata.get('description')
+        duration = int_or_none(vdata.get('duration')) or metadata.get('duration')
+
         formats = []
         subtitles = {}
-        description = vdata.get('description')
-
         for (fkey, vid) in vdata['files'].items():
             if fkey == 'hls_phone' and 'hls_tablet' in vdata['files']:
                 continue
-            result = tp.extract(('http://link.theplatform.com/s/%s/%s' % (mpx_account, vid)))
-            formats.extend(result['formats'])
-            subtitles = self._merge_subtitles(subtitles, result['subtitles'])
-            description = description or result.get('description')
-
+            release_url = 'http://link.theplatform.com/s/%s/%s?format=SMIL&mbr=true' % (mpx_account, vid)
+            if fkey == 'hds':
+                release_url += '&manifest=f4m'
+            tp_formats, tp_subtitles = self._extract_theplatform_smil(release_url, video_id, 'Downloading %s SMIL data' % fkey)
+            formats.extend(tp_formats)
+            subtitles = self._merge_subtitles(subtitles, tp_subtitles)
         self._sort_formats(formats)
 
         return {
@@ -72,6 +75,8 @@ class CNETIE(InfoExtractor):
             'display_id': display_id,
             'title': title,
             'description': description,
+            'thumbnail': metadata.get('thumbnail'),
+            'duration': duration,
             'uploader': uploader,
             'uploader_id': uploader_id,
             'subtitles': subtitles,

From 6882c0870eb89a17eb81d3d273a9f9e42a8bea5e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 20 Dec 2015 15:48:19 +0100
Subject: [PATCH 0118/1105] [tele13] improve extraction

- improve jwplayer setup regex
- sort formats
- remove duplicate formats
- update youtube test
---
 youtube_dl/extractor/tele13.py | 64 +++++++++++++++++-----------------
 1 file changed, 32 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/tele13.py b/youtube_dl/extractor/tele13.py
index f1764eb2f..a363b4d40 100644
--- a/youtube_dl/extractor/tele13.py
+++ b/youtube_dl/extractor/tele13.py
@@ -1,10 +1,13 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import js_to_json
+from .youtube import YoutubeIE
+from ..utils import (
+    js_to_json,
+    qualities,
+    determine_ext,
+)
 
 
 class Tele13IE(InfoExtractor):
@@ -25,12 +28,12 @@ class Tele13IE(InfoExtractor):
         },
         {
             'url': 'http://www.t13.cl/videos/mundo/tendencias/video-captan-misteriosa-bola-fuego-cielos-bangkok',
-            'md5': '65d1ae54812c96f4b345dd21d3bb1adc',
+            'md5': '867adf6a3b3fef932c68a71d70b70946',
             'info_dict': {
                 'id': 'rOoKv2OMpOw',
                 'ext': 'mp4',
                 'title': 'Shooting star seen on 7-Sep-2015',
-                'description': 'md5:a1cd2e74f6ee6851552c9cf5851d6b06',
+                'description': 'md5:7292ff2a34b2f673da77da222ae77e1e',
                 'uploader': 'Porjai Jaturongkhakun',
                 'upload_date': '20150906',
                 'uploader_id': 'UCnLY_3ezwNcDSC_Wc6suZxw',
@@ -41,41 +44,38 @@ class Tele13IE(InfoExtractor):
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-
         webpage = self._download_webpage(url, display_id)
 
-        setup_js = self._parse_json(
-            js_to_json(
-                self._search_regex(
-                    r"jwplayer\('player-vivo'\).setup\((\{.*?\})\)",
-                    webpage,
-                    'setup code',
-                    flags=re.DOTALL
-                ).replace('\n//', '')
-            ),
-            display_id
-        )
-        title = setup_js['title']
-        thumbnail = setup_js.get('image') or setup_js['playlist'][0].get('image')
-        description = self._html_search_meta(
-            'description', webpage, 'description')
+        setup_js = self._search_regex(r"(?s)jwplayer\('player-vivo'\).setup\((\{.*?\})\)", webpage, 'setup code')
+        sources = self._parse_json(self._search_regex(r'sources\s*:\s*(\[[^\]]+\])', setup_js, 'sources'), display_id, js_to_json)
 
+        preference = qualities(['Móvil', 'SD', 'HD'])
         formats = []
-        for f in setup_js['playlist'][0]['sources']:
+        urls = []
+        for f in sources:
             format_url = f['file']
-            if format_url != '':
-                if '.m3u8' in format_url:
-                    formats.extend(self._extract_m3u8_formats(format_url, display_id))
+            if format_url and format_url not in urls:
+                ext = determine_ext(format_url)
+                if ext == 'm3u8':
+                    m3u8_formats = self._extract_m3u8_formats(format_url, display_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+                    if m3u8_formats:
+                        formats.extend(m3u8_formats)
+                elif YoutubeIE.suitable(format_url):
+                    return self.url_result(format_url, 'Youtube')
                 else:
-                    if 'youtube.com' in format_url:
-                        return self.url_result(format_url, 'Youtube')
-                    else:
-                        formats.append({'url': format_url, 'format_id': f.get('label')})
+                    formats.append({
+                        'url': format_url,
+                        'format_id': f.get('label'),
+                        'preference': preference(f.get('label')),
+                        'ext': ext,
+                    })
+                urls.append(format_url)
+        self._sort_formats(formats)
 
         return {
             'id': display_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
+            'title': self._search_regex(r'title\s*:\s*"([^"]+)"', setup_js, 'title'),
+            'description': self._html_search_meta('description', webpage, 'description'),
+            'thumbnail': self._search_regex(r'image\s*:\s*"([^"]+)"', setup_js, 'thumbnail', default=None),
             'formats': formats,
         }

From ecbccea703cc7812b66c6dd3a543d60b5be8aa48 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 20 Dec 2015 21:38:30 +0100
Subject: [PATCH 0119/1105] [faz] extract duration and bitrate and use
 xpath_element and xpath_text for extraction

---
 youtube_dl/extractor/faz.py | 36 +++++++++++++++++++++---------------
 1 file changed, 21 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/faz.py b/youtube_dl/extractor/faz.py
index d9a868119..6f9b003c2 100644
--- a/youtube_dl/extractor/faz.py
+++ b/youtube_dl/extractor/faz.py
@@ -2,6 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import (
+    xpath_element,
+    xpath_text,
+    int_or_none,
+)
 
 
 class FazIE(InfoExtractor):
@@ -37,31 +42,32 @@ class FazIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
+        description = self._og_search_description(webpage)
         config_xml_url = self._search_regex(
-            r'(?:var\s+)?videoXMLURL\s*=\s*"([^"]+)', webpage, 'config xml url')
+            r'videoXMLURL\s*=\s*"([^"]+)', webpage, 'config xml url')
         config = self._download_xml(
             config_xml_url, video_id, 'Downloading config xml')
 
-        encodings = config.find('ENCODINGS')
+        encodings = xpath_element(config, 'ENCODINGS', 'encodings', True)
         formats = []
         for pref, code in enumerate(['LOW', 'HIGH', 'HQ']):
-            encoding = encodings.find(code)
-            if encoding is None:
-                continue
-            encoding_url = encoding.find('FILENAME').text
-            formats.append({
-                'url': encoding_url,
-                'format_id': code.lower(),
-                'quality': pref,
-            })
+            encoding = xpath_element(encodings, code)
+            if encoding:
+                encoding_url = xpath_text(encoding, 'FILENAME')
+                if encoding_url:
+                    formats.append({
+                        'url': encoding_url,
+                        'format_id': code.lower(),
+                        'quality': pref,
+                        'tbr': int_or_none(xpath_text(encoding, 'AVERAGEBITRATE')),
+                    })
         self._sort_formats(formats)
 
-        descr = self._html_search_regex(
-            r'<p class="Content Copy">(.*?)</p>', webpage, 'description', fatal=False)
         return {
             'id': video_id,
             'title': self._og_search_title(webpage),
             'formats': formats,
-            'description': descr,
-            'thumbnail': config.find('STILL/STILL_BIG').text,
+            'description': description.strip() if description else None,
+            'thumbnail': xpath_text(config, 'STILL/STILL_BIG'),
+            'duration': int_or_none(xpath_text(config, 'DURATION')),
         }

From 5b95419ca59e6238ab0041d48a53a9e51485e1cc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Dec 2015 00:20:22 +0100
Subject: [PATCH 0120/1105] [flickr] extract views_count and tags

---
 youtube_dl/extractor/flickr.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index 92d2ac553..452b27b26 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -25,6 +25,8 @@ class FlickrIE(InfoExtractor):
             'uploader_id': '10922353@N03',
             'uploader': 'Forest Wander',
             'comment_count': int,
+            'view_count': int,
+            'tags': list,
         }
     }
 
@@ -78,6 +80,8 @@ class FlickrIE(InfoExtractor):
                 'uploader_id': owner.get('nsid'),
                 'uploader': owner.get('realname'),
                 'comment_count': int_or_none(video_info.get('comments', {}).get('_content')),
+                'view_count': int_or_none(video_info.get('views')),
+                'tags': [tag.get('_content') for tag in video_info.get('tags', {}).get('tag', [])]
             }
         else:
             raise ExtractorError('not a video', expected=True)

From 5b251628e9f45c89c1becb3f62c4212874eb74ea Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Dec 2015 03:05:34 +0100
Subject: [PATCH 0121/1105] [googledrive] Modernize

---
 youtube_dl/extractor/__init__.py    |   5 +-
 youtube_dl/extractor/generic.py     |   4 +-
 youtube_dl/extractor/googledrive.py | 146 ++++++++++------------------
 3 files changed, 54 insertions(+), 101 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 02e18a0da..6655d7eb5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -209,10 +209,7 @@ from .globo import GloboIE
 from .godtube import GodTubeIE
 from .goldenmoustache import GoldenMoustacheIE
 from .golem import GolemIE
-from .googledrive import (
-    GoogleDriveEmbedIE,
-    GoogleDriveIE,
-)
+from .googledrive import GoogleDriveIE
 from .googleplus import GooglePlusIE
 from .googlesearch import GoogleSearchIE
 from .gorillavid import GorillaVidIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 3f7b094db..abd98e500 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -48,7 +48,7 @@ from .vimeo import VimeoIE
 from .dailymotion import DailymotionCloudIE
 from .onionstudios import OnionStudiosIE
 from .snagfilms import SnagFilmsEmbedIE
-from .googledrive import GoogleDriveEmbedIE
+from .googledrive import GoogleDriveIE
 
 
 class GenericIE(InfoExtractor):
@@ -1601,7 +1601,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
 
         # Look for Google Drive embeds
-        google_drive_url = GoogleDriveEmbedIE._extract_url(webpage)
+        google_drive_url = GoogleDriveIE._extract_url(webpage)
         if google_drive_url:
             return self.url_result(google_drive_url, 'GoogleDrive')
 
diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index 7bc7b7a0d..f354c9c7a 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -1,132 +1,88 @@
+from __future__ import unicode_literals
+
 import re
 
 from .common import InfoExtractor
 from ..utils import (
-    RegexNotFoundError,
     ExtractorError,
+    int_or_none,
 )
 
-class GoogleDriveEmbedIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9_-]{28})'
+
+class GoogleDriveIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/)|video\.google\.com/get_player\?.*?docid=)(?P<id>[a-zA-Z0-9_-]{28})'
     _TEST = {
-        'url': 'https://docs.google.com/file/d/0B8KB9DRosYGKMXNoeWxqa3JYclE/preview',
+        'url': 'https://drive.google.com/file/d/0ByeS4oOUV-49Zzh4R1J6R09zazQ/edit?pli=1',
+        'md5': '881f7700aec4f538571fa1e0eed4a7b6',
         'info_dict': {
-            'id': '0B8KB9DRosYGKMXNoeWxqa3JYclE',
+            'id': '0ByeS4oOUV-49Zzh4R1J6R09zazQ',
             'ext': 'mp4',
-            'title': 'Jimmy Fallon Sings Since You\'ve Been Gone.wmv',
+            'title': 'Big Buck Bunny.mp4',
+            'duration': 46,
         }
     }
+    _FORMATS_EXT = {
+        '5': 'flv',
+        '6': 'flv',
+        '13': '3gp',
+        '17': '3gp',
+        '18': 'mp4',
+        '22': 'mp4',
+        '34': 'flv',
+        '35': 'flv',
+        '36': '3gp',
+        '37': 'mp4',
+        '38': 'mp4',
+        '43': 'webm',
+        '44': 'webm',
+        '45': 'webm',
+        '46': 'webm',
+        '59': 'mp4',
+    }
 
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<iframe src="https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9_-]{28})',
+            r'<iframe[^>]+src="https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9_-]{28})',
             webpage)
         if mobj:
             return 'https://drive.google.com/file/d/%s' % mobj.group('id')
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        return {
-            '_type': 'url',
-            'ie_key': 'GoogleDrive',
-            'url': 'https://drive.google.com/file/d/%s' % video_id
-        }
-
-class GoogleDriveIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/)(?P<id>[a-zA-Z0-9_-]{28})'
-    _TEST = {
-        'url': 'https://drive.google.com/file/d/0ByeS4oOUV-49Zzh4R1J6R09zazQ/edit?pli=1',
-        'info_dict': {
-            'id': '0ByeS4oOUV-49Zzh4R1J6R09zazQ',
-            'ext': 'mp4',
-            'title': 'Big Buck Bunny.mp4',
-        }
-    }
-    _formats = {
-        '5': {'ext': 'flv'},
-        '6': {'ext': 'flv'},
-        '13': {'ext': '3gp'},
-        '17': {'ext': '3gp'},
-        '18': {'ext': 'mp4'},
-        '22': {'ext': 'mp4'},
-        '34': {'ext': 'flv'},
-        '35': {'ext': 'flv'},
-        '36': {'ext': '3gp'},
-        '37': {'ext': 'mp4'},
-        '38': {'ext': 'mp4'},
-        '43': {'ext': 'webm'},
-        '44': {'ext': 'webm'},
-        '45': {'ext': 'webm'},
-        '46': {'ext': 'webm'},
-        '59': {'ext': 'mp4'}
-    }
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(
-            'http://docs.google.com/file/d/' + video_id, video_id, encoding='unicode_escape'
-        )
-        try:
-            title = self._html_search_regex(
-                r'"title"\s*,\s*"([^"]+)',
-                webpage,
-                'title'
-            )
-            fmt_stream_map = self._html_search_regex(
-                r'"fmt_stream_map"\s*,\s*"([^"]+)',
-                webpage,
-                'fmt_stream_map'
-            )
-            fmt_list = self._html_search_regex(
-                r'"fmt_list"\s*,\s*"([^"]+)',
-                webpage,
-                'fmt_list'
-            )
-#			timestamp = self._html_search_regex(
-#				r'"timestamp"\s*,\s*"([^"]+)',
-#				webpage,
-#				'timestamp'
-#			)
-            length_seconds = self._html_search_regex(
-                r'"length_seconds"\s*,\s*"([^"]+)',
-                webpage,
-                'length_seconds'
-            )
-        except RegexNotFoundError:
-            try:
-                reason = self._html_search_regex(
-                    r'"reason","([^"]+)',
-                    webpage,
-                    'reason'
-                )
-                raise ExtractorError(reason)
-                return
-            except RegexNotFoundError:
-                raise ExtractorError('not a video')
-                return
+            'http://docs.google.com/file/d/%s' % video_id, video_id, encoding='unicode_escape')
+
+        reason = self._search_regex(r'"reason"\s*,\s*"([^"]+)', webpage, 'reason', default=None)
+        if reason:
+            raise ExtractorError(reason)
+
+        title = self._search_regex(r'"title"\s*,\s*"([^"]+)', webpage, 'title')
+        duration = int_or_none(self._search_regex(
+            r'"length_seconds"\s*,\s*"([^"]+)', webpage, 'length seconds', default=None))
+        fmt_stream_map = self._search_regex(
+            r'"fmt_stream_map"\s*,\s*"([^"]+)', webpage, 'fmt stream map').split(',')
+        fmt_list = self._search_regex(r'"fmt_list"\s*,\s*"([^"]+)', webpage, 'fmt_list').split(',')
 
-        fmt_stream_map = fmt_stream_map.split(',')
-        fmt_list = fmt_list.split(',')
         formats = []
-        for i in range(len(fmt_stream_map)):
-            fmt_id, fmt_url = fmt_stream_map[i].split('|')
-            resolution = fmt_list[i].split('/')[1]
+        for fmt, fmt_stream in zip(fmt_list, fmt_stream_map):
+            fmt_id, fmt_url = fmt_stream.split('|')
+            resolution = fmt.split('/')[1]
             width, height = resolution.split('x')
             formats.append({
                 'url': fmt_url,
                 'format_id': fmt_id,
                 'resolution': resolution,
-                'width': int(width),
-                'height': int(height),
-                'ext': self._formats[fmt_id]['ext']
+                'width': int_or_none(width),
+                'height': int_or_none(height),
+                'ext': self._FORMATS_EXT[fmt_id],
             })
         self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': title,
-#           'timestamp': int(timestamp),
-            'duration': int(length_seconds),
-            'formats': formats
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'duration': duration,
+            'formats': formats,
         }

From 7cb09524749d4a061acc801d0a2d6ad08463e549 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Dec 2015 04:24:58 +0100
Subject: [PATCH 0122/1105] [makertv] improve extraction

---
 youtube_dl/extractor/generic.py    |  6 ++++++
 youtube_dl/extractor/jwplatform.py | 10 +++++++---
 youtube_dl/extractor/makertv.py    | 13 +++++++++----
 3 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 4d38b0c9d..f5dd88f54 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -53,6 +53,7 @@ from .onionstudios import OnionStudiosIE
 from .snagfilms import SnagFilmsEmbedIE
 from .screenwavemedia import ScreenwaveMediaIE
 from .mtv import MTVServicesEmbeddedIE
+from .jwplatform import JWPlatformIE
 
 
 class GenericIE(InfoExtractor):
@@ -1787,6 +1788,11 @@ class GenericIE(InfoExtractor):
         if snagfilms_url:
             return self.url_result(snagfilms_url)
 
+        # Look for JWPlatform embeds
+        jwplatform_url = JWPlatformIE._extract_url(webpage)
+        if jwplatform_url:
+            return self.url_result(jwplatform_url, 'JWPlatform')
+
         # Look for ScreenwaveMedia embeds
         mobj = re.search(ScreenwaveMediaIE.EMBED_PATTERN, webpage)
         if mobj is not None:
diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index 3a3dc439a..cdc095a79 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import int_or_none
 
@@ -23,7 +25,7 @@ class JWPlatformIE(InfoExtractor):
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<script[^>]+?src=["\'](?P<url>(?:https?:)?//content.jwplatform.com/players/[a-zA-Z0-9]{8}',
+            r'<script[^>]+?src=["\'](?P<url>(?:https?:)?//content.jwplatform.com/players/[a-zA-Z0-9]{8})',
             webpage)
         if mobj:
             return mobj.group('url')
@@ -42,7 +44,9 @@ class JWPlatformIE(InfoExtractor):
             source_url = self._proto_relative_url(source['file'])
             source_type = source.get('type') or ''
             if source_type == 'application/vnd.apple.mpegurl':
-                formats.extend(self._extract_m3u8_formats(source_url, video_id, 'mp4', 'm3u8_native', fatal=None))
+                m3u8_formats = self._extract_m3u8_formats(source_url, video_id, 'mp4', 'm3u8_native', fatal=None)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
             elif source_type.startswith('audio'):
                 formats.append({
                     'url': source_url,
@@ -57,7 +61,7 @@ class JWPlatformIE(InfoExtractor):
         self._sort_formats(formats)
 
         return {
-            'id': video_data['mediaid'],
+            'id': video_id,
             'title': video_data['title'],
             'description': video_data.get('description'),
             'thumbnail': self._proto_relative_url(video_data.get('image')),
diff --git a/youtube_dl/extractor/makertv.py b/youtube_dl/extractor/makertv.py
index 0256e4e24..3c34d4604 100644
--- a/youtube_dl/extractor/makertv.py
+++ b/youtube_dl/extractor/makertv.py
@@ -5,12 +5,12 @@ from .common import InfoExtractor
 
 
 class MakerTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www\.)?maker\.tv/(?:[^/]+/)?video|http://makerplayer.com/embed/maker)/(?P<id>[a-zA-Z0-9]{12})'
+    _VALID_URL = r'https?://(?:(?:www\.)?maker\.tv/(?:[^/]+/)*video|makerplayer.com/embed/maker)/(?P<id>[a-zA-Z0-9]{12})'
     _TEST = {
         'url': 'http://www.maker.tv/video/Fh3QgymL9gsc',
         'md5': 'ca237a53a8eb20b6dc5bd60564d4ab3e',
         'info_dict': {
-            'id': 'brOEcGut',
+            'id': 'Fh3QgymL9gsc',
             'ext': 'mp4',
             'title': 'Maze Runner: The Scorch Trials Official Movie Review',
             'description': 'md5:11ff3362d7ef1d679fdb649f6413975a',
@@ -22,6 +22,11 @@ class MakerTVIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        jwplatform_id = self._search_regex([r'jwid="([^"]+)"', r'Maker.jw_id\s*=\s*"([^"]+)";'], webpage, 'jwplatform id')
+        jwplatform_id = self._search_regex(r'jw_?id="([^"]+)"', webpage, 'jwplatform id')
 
-        return self.url_result('jwplatform:%s' % jwplatform_id, 'JWPlatform')
+        return {
+            '_type': 'url_transparent',
+            'id': video_id,
+            'url': 'jwplatform:%s' % jwplatform_id,
+            'ie_key': 'JWPlatform',
+        }

From c7fa5fa42cb91aa32c5b21c6821f6d190ec047a2 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Dec 2015 11:12:58 +0100
Subject: [PATCH 0123/1105] [bleacherreport] fix style issues and simplify

---
 youtube_dl/extractor/amp.py            | 38 +++++++++++++-------------
 youtube_dl/extractor/bleacherreport.py | 19 ++-----------
 youtube_dl/extractor/dramafever.py     |  6 ++--
 youtube_dl/extractor/foxnews.py        | 19 +++++--------
 4 files changed, 30 insertions(+), 52 deletions(-)

diff --git a/youtube_dl/extractor/amp.py b/youtube_dl/extractor/amp.py
index b573b9280..dcc3c97f1 100644
--- a/youtube_dl/extractor/amp.py
+++ b/youtube_dl/extractor/amp.py
@@ -9,23 +9,21 @@ from ..utils import (
 
 
 class AMPIE(InfoExtractor):
-    def _get_media_node(self, item, name, default=None):
-        media_name = 'media-%s' % name
-        media_group = item.get('media-group') or item
-        return media_group.get(media_name) or item.get(media_name) or item.get(name, default)
-
     # parse Akamai Adaptive Media Player feed
     def _extract_feed_info(self, url):
         item = self._download_json(
-            url, None,
-            'Downloading Akamai AMP feed',
-            'Unable to download Akamai AMP feed'
-            )['channel']['item']
+            url, None, 'Downloading Akamai AMP feed',
+            'Unable to download Akamai AMP feed')['channel']['item']
 
         video_id = item['guid']
-        
+
+        def get_media_node(name, default=None):
+            media_name = 'media-%s' % name
+            media_group = item.get('media-group') or item
+            return media_group.get(media_name) or item.get(media_name) or item.get(name, default)
+
         thumbnails = []
-        media_thumbnail = self._get_media_node(item, 'thumbnail')
+        media_thumbnail = get_media_node('thumbnail')
         if media_thumbnail:
             if isinstance(media_thumbnail, dict):
                 media_thumbnail = [media_thumbnail]
@@ -38,7 +36,7 @@ class AMPIE(InfoExtractor):
                 })
 
         subtitles = {}
-        media_subtitle = self._get_media_node(item, 'subTitle')
+        media_subtitle = get_media_node('subTitle')
         if media_subtitle:
             if isinstance(media_subtitle, dict):
                 media_subtitle = [media_subtitle]
@@ -48,26 +46,28 @@ class AMPIE(InfoExtractor):
                 subtitles[lang] = [{'url': subtitle['href']}]
 
         formats = []
-        media_content = self._get_media_node(item, 'content')
+        media_content = get_media_node('content')
         if isinstance(media_content, dict):
             media_content = [media_content]
         for media_data in media_content:
             media = media_data['@attributes']
             media_type = media['type']
             if media_type == 'video/f4m':
-                f4m_formats = self._extract_f4m_formats(media['url'] + '?hdcore=3.4.0&plugin=aasp-3.4.0.132.124', video_id, f4m_id='hds', fatal=False)
+                f4m_formats = self._extract_f4m_formats(
+                    media['url'] + '?hdcore=3.4.0&plugin=aasp-3.4.0.132.124',
+                    video_id, f4m_id='hds', fatal=False)
                 if f4m_formats:
                     formats.extend(f4m_formats)
             elif media_type == 'application/x-mpegURL':
-                m3u8_formats = self._extract_m3u8_formats(media['url'], video_id, m3u8_id='hls', fatal=False)
+                m3u8_formats = self._extract_m3u8_formats(
+                    media['url'], video_id, 'mp4', m3u8_id='hls', fatal=False)
                 if m3u8_formats:
                     formats.extend(m3u8_formats)
             else:
                 formats.append({
                     'format_id': media_data['media-category']['@attributes']['label'],
                     'url': media['url'],
-                    'preference': 1,
-                    'vbr': int_or_none(media.get('bitrate')),
+                    'tbr': int_or_none(media.get('bitrate')),
                     'filesize': int_or_none(media.get('fileSize')),
                 })
 
@@ -75,8 +75,8 @@ class AMPIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': self._get_media_node(item, 'title'),
-            'description': self._get_media_node(item, 'description'),
+            'title': get_media_node('title'),
+            'description': get_media_node('description'),
             'thumbnails': thumbnails,
             'timestamp': parse_iso8601(item.get('pubDate'), ' '),
             'duration': int_or_none(media_content[0].get('@attributes', {}).get('duration')),
diff --git a/youtube_dl/extractor/bleacherreport.py b/youtube_dl/extractor/bleacherreport.py
index a55e696d2..bd2a6340b 100644
--- a/youtube_dl/extractor/bleacherreport.py
+++ b/youtube_dl/extractor/bleacherreport.py
@@ -26,7 +26,7 @@ class BleacherReportIE(InfoExtractor):
             'uploader': 'Team Stream Now ',
         },
         'add_ie': ['Ooyala'],
-    },{
+    }, {
         'url': 'http://bleacherreport.com/articles/2586817-aussie-golfers-get-fright-of-their-lives-after-being-chased-by-angry-kangaroo',
         'md5': 'af5f90dc9c7ba1c19d0a3eac806bbf50',
         'info_dict': {
@@ -35,25 +35,11 @@ class BleacherReportIE(InfoExtractor):
             'title': 'Aussie Golfers Get Fright of Their Lives After Being Chased by Angry Kangaroo',
             'timestamp': 1446839961,
             'uploader': 'Sean Fay',
-            'description': 'md5:e95afafa43619816552723878b3b0a84',
+            'description': 'md5:825e94e0f3521df52fa83b2ed198fa20',
             'uploader_id': 6466954,
             'upload_date': '20151011',
         },
         'add_ie': ['Youtube'],
-    },{
-        'url': 'http://bleacherreport.com/articles/2496438-fsu-stat-projections-is-jalen-ramsey-best-defensive-player-in-college-football',
-        'md5': 'a3ffc3dc73afdbc2010f02d98f990f20',
-        'info_dict': {
-            'id': '2496438',
-            'ext': 'mp4',
-            'title': 'FSU Stat Projections: Is Jalen Ramsey Best Defensive Player in College Football?',
-            'upload_date': '20150615',
-            'uploader': 'Team Stream Now ',
-            'timestamp': 1434380212,
-            'description': 'CFB, ACC, Florida State',
-            'uploader_id': 3992341,
-        },
-        'add_ie': ['Vine'],
     }]
 
     def _real_extract(self, url):
@@ -115,7 +101,6 @@ class BleacherReportCMSIE(AMPIE):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
         info = self._extract_feed_info('http://cms.bleacherreport.com/media/items/%s/akamai.json' % video_id)
         info['id'] = video_id
         return info
diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 80a928827..2a4310754 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -13,9 +13,6 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     clean_html,
-    determine_ext,
-    int_or_none,
-    parse_iso8601,
 )
 
 
@@ -91,7 +88,8 @@ class DramaFeverIE(DramaFeverBaseIE):
         video_id = self._match_id(url).replace('/', '.')
 
         try:
-            info = self._extract_feed_info('http://www.dramafever.com/amp/episode/feed.json?guid=%s' % video_id)
+            info = self._extract_feed_info(
+                'http://www.dramafever.com/amp/episode/feed.json?guid=%s' % video_id)
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError):
                 raise ExtractorError(
diff --git a/youtube_dl/extractor/foxnews.py b/youtube_dl/extractor/foxnews.py
index 0cd0f9fa8..318ac013d 100644
--- a/youtube_dl/extractor/foxnews.py
+++ b/youtube_dl/extractor/foxnews.py
@@ -3,10 +3,6 @@ from __future__ import unicode_literals
 import re
 
 from .amp import AMPIE
-from ..utils import (
-    parse_iso8601,
-    int_or_none,
-)
 
 
 class FoxNewsIE(AMPIE):
@@ -22,8 +18,8 @@ class FoxNewsIE(AMPIE):
                 'title': 'Frozen in Time',
                 'description': '16-year-old girl is size of toddler',
                 'duration': 265,
-                #'timestamp': 1304411491,
-                #'upload_date': '20110503',
+                # 'timestamp': 1304411491,
+                # 'upload_date': '20110503',
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
@@ -36,8 +32,8 @@ class FoxNewsIE(AMPIE):
                 'title': "Rep. Luis Gutierrez on if Obama's immigration plan is legal",
                 'description': "Congressman discusses president's plan",
                 'duration': 292,
-                #'timestamp': 1417662047,
-                #'upload_date': '20141204',
+                # 'timestamp': 1417662047,
+                # 'upload_date': '20141204',
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
@@ -52,10 +48,9 @@ class FoxNewsIE(AMPIE):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        host = mobj.group('host')
+        host, video_id = re.match(self._VALID_URL, url).groups()
 
-        info = self._extract_feed_info('http://%s/v/feed/video/%s.js?template=fox' % (host, video_id))
+        info = self._extract_feed_info(
+            'http://%s/v/feed/video/%s.js?template=fox' % (host, video_id))
         info['id'] = video_id
         return info

From 0197004f786eba88238c5dd638dc1923b85b2b4c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 21 Dec 2015 11:42:25 +0100
Subject: [PATCH 0124/1105] release 2015.12.21

---
 docs/supportedsites.md | 13 +++++++++----
 youtube_dl/version.py  |  2 +-
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 8253335e3..299bc5e72 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -65,9 +65,9 @@
  - **Bet**
  - **Bild**: Bild.de
  - **BiliBili**
+ - **BleacherReport**
+ - **BleacherReportCMS**
  - **blinkx**
- - **blip.tv:user**
- - **BlipTV**
  - **Bloomberg**
  - **Bpb**: Bundeszentrale für politische Bildung
  - **BR**: Bayerischer Rundfunk Mediathek
@@ -80,7 +80,6 @@
  - **BYUtv**
  - **Camdemy**
  - **CamdemyFolder**
- - **Canal13cl**
  - **canalc2.tv**
  - **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv
  - **CBS**
@@ -210,7 +209,9 @@
  - **GodTube**
  - **GoldenMoustache**
  - **Golem**
+ - **GoogleDrive**
  - **Goshgay**
+ - **GPUTechConf**
  - **Groupon**
  - **Hark**
  - **HearThisAt**
@@ -252,6 +253,7 @@
  - **Jove**
  - **jpopsuki.tv**
  - **Jukebox**
+ - **JWPlatform**
  - **Kaltura**
  - **KanalPlay**: Kanal 5/9/11 Play
  - **Kankan**
@@ -292,6 +294,7 @@
  - **m6**
  - **macgamestore**: MacGameStore trailers
  - **mailru**: Видео@Mail.Ru
+ - **MakerTV**
  - **Malemotion**
  - **MDR**: MDR.DE and KiKA
  - **media.ccc.de**
@@ -551,6 +554,7 @@
  - **TechTalks**
  - **techtv.mit.edu**
  - **ted**
+ - **Tele13**
  - **TeleBruxelles**
  - **Telecinco**: telecinco.es, cuatro.com and mediaset.es
  - **Telegraaf**
@@ -573,6 +577,7 @@
  - **TMZ**
  - **TMZArticle**
  - **TNAFlix**
+ - **toggle**
  - **tou.tv**
  - **Toypics**: Toypics user profile
  - **ToypicsUser**: Toypics user profile
@@ -711,6 +716,7 @@
  - **youtube:favorites**: YouTube.com favourite videos, ":ytfav" for short (requires authentication)
  - **youtube:history**: Youtube watch history, ":ythistory" for short (requires authentication)
  - **youtube:playlist**: YouTube.com playlists
+ - **youtube:playlists**: YouTube.com user/channel playlists
  - **youtube:recommended**: YouTube.com recommended videos, ":ytrec" for short (requires authentication)
  - **youtube:search**: YouTube.com searches
  - **youtube:search:date**: YouTube.com searches, newest videos first
@@ -718,7 +724,6 @@
  - **youtube:show**: YouTube.com (multi-season) shows
  - **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)
  - **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword)
- - **youtube:user:playlists**: YouTube.com user playlists
  - **youtube:watchlater**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)
  - **Zapiks**
  - **ZDF**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 01607693e..7095033c5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.12.18'
+__version__ = '2015.12.21'

From 5c5a3ecf1b5337510378b782a14525584769e6d6 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Dec 2015 13:07:52 +0100
Subject: [PATCH 0125/1105] [abc] detect expired state and update tests

---
 youtube_dl/extractor/abc.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index c0e5d1abf..6a29e587f 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -23,6 +23,7 @@ class ABCIE(InfoExtractor):
             'title': 'Australia to help staff Ebola treatment centre in Sierra Leone',
             'description': 'md5:809ad29c67a05f54eb41f2a105693a67',
         },
+        'skip': 'this video has expired',
     }, {
         'url': 'http://www.abc.net.au/news/2015-08-17/warren-entsch-introduces-same-sex-marriage-bill/6702326',
         'md5': 'db2a5369238b51f9811ad815b69dc086',
@@ -36,6 +37,7 @@ class ABCIE(InfoExtractor):
             'title': 'Marriage Equality: Warren Entsch introduces same sex marriage bill',
         },
         'add_ie': ['Youtube'],
+        'skip': 'Not accessible from Travis CI server',
     }, {
         'url': 'http://www.abc.net.au/news/2015-10-23/nab-lifts-interest-rates-following-westpac-and-cba/6880080',
         'md5': 'b96eee7c9edf4fc5a358a0252881cc1f',
@@ -58,6 +60,9 @@ class ABCIE(InfoExtractor):
             r'inline(?P<type>Video|Audio|YouTube)Data\.push\((?P<json_data>[^)]+)\);',
             webpage)
         if mobj is None:
+            expired = self._html_search_regex(r'(?s)class="expired-(?:video|audio)".+?<span>(.+?)</span>', webpage, 'expired', None)
+            if expired:
+                raise ExtractorError('%s said: %s' % (self.IE_NAME, expired), expected=True)
             raise ExtractorError('Unable to extract video urls')
 
         urls_info = self._parse_json(

From 61ebb401b7ff8f15850bfc5778cdc08ab72145e1 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Dec 2015 16:26:40 +0100
Subject: [PATCH 0126/1105] [atresplayer] improve extraction

- select hashlib.md5 constructor as digestmod(in python 3.4+ MD5 as
implicit default digest for digestmod is deprecated.)
- extract hls formats
- update tests
- extract errors
---
 youtube_dl/extractor/atresplayer.py | 117 +++++++++++++++++++---------
 1 file changed, 81 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 50e47ba0a..7ac3044c7 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -2,6 +2,8 @@ from __future__ import unicode_literals
 
 import time
 import hmac
+import hashlib
+import re
 
 from .common import InfoExtractor
 from ..compat import (
@@ -32,6 +34,19 @@ class AtresPlayerIE(InfoExtractor):
                 'duration': 5527.6,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
+            'skip': 'This video is only available for registered users'
+        },
+        {
+            'url': 'http://www.atresplayer.com/television/especial/videoencuentros/temporada-1/capitulo-112-david-bustamante_2014121600375.html',
+            'md5': '0d0e918533bbd4b263f2de4d197d4aac',
+            'info_dict': {
+                'id': 'capitulo-112-david-bustamante',
+                'ext': 'flv',
+                'title': 'David Bustamante',
+                'description': 'md5:f33f1c0a05be57f6708d4dd83a3b81c6',
+                'duration': 1439.0,
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
         },
         {
             'url': 'http://www.atresplayer.com/television/series/el-secreto-de-puente-viejo/el-chico-de-los-tres-lunares/capitulo-977-29-12-14_2014122400174.html',
@@ -50,6 +65,13 @@ class AtresPlayerIE(InfoExtractor):
 
     _LOGIN_URL = 'https://servicios.atresplayer.com/j_spring_security_check'
 
+    _ERRORS = {
+        'UNPUBLISHED': 'We\'re sorry, but this video is not yet available.',
+        'DELETED': 'This video has expired and is no longer available for online streaming.',
+        'GEOUNPUBLISHED': 'We\'re sorry, but this video is not available in your region due to right restrictions.',
+        # 'PREMIUM': 'PREMIUM',
+    }
+
     def _real_initialize(self):
         self._login()
 
@@ -83,58 +105,81 @@ class AtresPlayerIE(InfoExtractor):
         episode_id = self._search_regex(
             r'episode="([^"]+)"', webpage, 'episode id')
 
+        request = sanitized_Request(
+            self._PLAYER_URL_TEMPLATE % episode_id,
+            headers={'User-Agent': self._USER_AGENT})
+        player = self._download_json(request, episode_id, 'Downloading player JSON')
+
+        episode_type = player.get('typeOfEpisode')
+        error_message = self._ERRORS.get(episode_type)
+        if error_message:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error_message), expected=True)
+
+        formats = []
+        video_url = player.get('urlVideo')
+        if video_url:
+            format_info = {
+                'url': video_url,
+                'format_id': 'http',
+            }
+            mobj = re.search(r'(?P<bitrate>\d+)K_(?P<width>\d+)x(?P<height>\d+)', video_url)
+            if mobj:
+                format_info.update({
+                    'width': int_or_none(mobj.group('width')),
+                    'height': int_or_none(mobj.group('height')),
+                    'tbr': int_or_none(mobj.group('bitrate')),
+                })
+            formats.append(format_info)
+
+        m3u8_url = player.get('urlVideoHls')
+        if m3u8_url:
+            m3u8_formats = self._extract_m3u8_formats(
+                m3u8_url, episode_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+            if m3u8_formats:
+                formats.extend(m3u8_formats)
+
         timestamp = int_or_none(self._download_webpage(
             self._TIME_API_URL,
             video_id, 'Downloading timestamp', fatal=False), 1000, time.time())
         timestamp_shifted = compat_str(timestamp + self._TIMESTAMP_SHIFT)
         token = hmac.new(
             self._MAGIC.encode('ascii'),
-            (episode_id + timestamp_shifted).encode('utf-8')
+            (episode_id + timestamp_shifted).encode('utf-8'), hashlib.md5
         ).hexdigest()
 
-        formats = []
-        for fmt in ['windows', 'android_tablet']:
-            request = sanitized_Request(
-                self._URL_VIDEO_TEMPLATE.format(fmt, episode_id, timestamp_shifted, token))
-            request.add_header('User-Agent', self._USER_AGENT)
+        request = sanitized_Request(
+            self._URL_VIDEO_TEMPLATE.format('windows', episode_id, timestamp_shifted, token),
+            headers={'User-Agent': self._USER_AGENT})
 
-            fmt_json = self._download_json(
-                request, video_id, 'Downloading %s video JSON' % fmt)
+        fmt_json = self._download_json(
+            request, video_id, 'Downloading windows video JSON')
 
-            result = fmt_json.get('resultDes')
-            if result.lower() != 'ok':
-                raise ExtractorError(
-                    '%s returned error: %s' % (self.IE_NAME, result), expected=True)
+        result = fmt_json.get('resultDes')
+        if result.lower() != 'ok':
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, result), expected=True)
 
-            for format_id, video_url in fmt_json['resultObject'].items():
-                if format_id == 'token' or not video_url.startswith('http'):
-                    continue
-                if video_url.endswith('/Manifest'):
-                    if 'geodeswowsmpra3player' in video_url:
-                        f4m_path = video_url.split('smil:', 1)[-1].split('free_', 1)[0]
-                        f4m_url = 'http://drg.antena3.com/{0}hds/es/sd.f4m'.format(f4m_path)
-                        # this videos are protected by DRM, the f4m downloader doesn't support them
-                        continue
-                    else:
-                        f4m_url = video_url[:-9] + '/manifest.f4m'
-                    formats.extend(self._extract_f4m_formats(f4m_url, video_id))
-                else:
-                    formats.append({
-                        'url': video_url,
-                        'format_id': 'android-%s' % format_id,
-                        'preference': 1,
-                    })
+        for format_id, video_url in fmt_json['resultObject'].items():
+            if format_id == 'token' or not video_url.startswith('http'):
+                continue
+            if 'geodeswowsmpra3player' in video_url:
+                f4m_path = video_url.split('smil:', 1)[-1].split('free_', 1)[0]
+                f4m_url = 'http://drg.antena3.com/{0}hds/es/sd.f4m'.format(f4m_path)
+                # this videos are protected by DRM, the f4m downloader doesn't support them
+                continue
+            else:
+                f4m_url = video_url[:-9] + '/manifest.f4m'
+            f4m_formats = self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False)
+            if f4m_formats:
+                formats.extend(f4m_formats)
         self._sort_formats(formats)
 
-        player = self._download_json(
-            self._PLAYER_URL_TEMPLATE % episode_id,
-            episode_id)
-
         path_data = player.get('pathData')
 
         episode = self._download_xml(
-            self._EPISODE_URL_TEMPLATE % path_data,
-            video_id, 'Downloading episode XML')
+            self._EPISODE_URL_TEMPLATE % path_data, video_id,
+            'Downloading episode XML')
 
         duration = float_or_none(xpath_text(
             episode, './media/asset/info/technical/contentDuration', 'duration'))

From f11d00fa4109ce754c58c4a7b8bef04fc14a257a Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Dec 2015 16:52:47 +0100
Subject: [PATCH 0127/1105] [test_subtitles] remove BlipTV test

---
 test/test_subtitles.py | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 75f0ea75f..9ed9fe622 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -11,7 +11,6 @@ from test.helper import FakeYDL, md5
 
 
 from youtube_dl.extractor import (
-    BlipTVIE,
     YoutubeIE,
     DailymotionIE,
     TEDIE,
@@ -145,18 +144,6 @@ class TestTedSubtitles(BaseTestSubtitles):
             self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
 
 
-class TestBlipTVSubtitles(BaseTestSubtitles):
-    url = 'http://blip.tv/a/a-6603250'
-    IE = BlipTVIE
-
-    def test_allsubtitles(self):
-        self.DL.params['writesubtitles'] = True
-        self.DL.params['allsubtitles'] = True
-        subtitles = self.getSubtitles()
-        self.assertEqual(set(subtitles.keys()), set(['en']))
-        self.assertEqual(md5(subtitles['en']), '5b75c300af65fe4476dff79478bb93e4')
-
-
 class TestVimeoSubtitles(BaseTestSubtitles):
     url = 'http://vimeo.com/76979871'
     IE = VimeoIE

From 0f15ad7b9b6cae4e8719d2b960d8e882963f7a5b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Dec 2015 17:07:19 +0100
Subject: [PATCH 0128/1105] [adultswim] update test

---
 youtube_dl/extractor/adultswim.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 3ae618e71..bf21a6887 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -68,7 +68,7 @@ class AdultSwimIE(InfoExtractor):
                 'md5': '3e346a2ab0087d687a05e1e7f3b3e529',
                 'info_dict': {
                     'id': 'sY3cMUR_TbuE4YmdjzbIcQ-0',
-                    'ext': 'flv',
+                    'ext': 'mp4',
                     'title': 'Tim and Eric Awesome Show Great Job! - Dr. Steve Brule, For Your Wine',
                     'description': 'Dr. Brule reports live from Wine Country with a special report on wines.  \r\nWatch Tim and Eric Awesome Show Great Job! episode #20, "Embarrassed" on Adult Swim.\r\n\r\n',
                 },
@@ -79,6 +79,10 @@ class AdultSwimIE(InfoExtractor):
             'title': 'Tim and Eric Awesome Show Great Job! - Dr. Steve Brule, For Your Wine',
             'description': 'Dr. Brule reports live from Wine Country with a special report on wines.  \r\nWatch Tim and Eric Awesome Show Great Job! episode #20, "Embarrassed" on Adult Swim.\r\n\r\n',
         },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
     }]
 
     @staticmethod

From 5ef5d25b150a383362aab046394849ed523b3730 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Dec 2015 22:51:58 +0600
Subject: [PATCH 0129/1105] [audiomack] Fix typo (Closes #7936)

---
 youtube_dl/extractor/audiomack.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index 693ba22c6..3eed91279 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -56,7 +56,7 @@ class AudiomackIE(InfoExtractor):
 
         # API is inconsistent with errors
         if 'url' not in api_response or not api_response['url'] or 'error' in api_response:
-            raise ExtractorError('Invalid url %s', url)
+            raise ExtractorError('Invalid url %s' % url)
 
         # Audiomack wraps a lot of soundcloud tracks in their branded wrapper
         # if so, pass the work off to the soundcloud extractor

From 5625bd0617d6c661c66049b420556ae442102ffc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Dec 2015 21:06:10 +0100
Subject: [PATCH 0130/1105] [br] add support for br-klassik.de and improve
 extraction

- extend _VALID_URL to match both br.de and br-klassik.de
- extract all formats(hls,hds and rtmp)
- use xpath_element and xpath_text for xml info extraction
---
 youtube_dl/extractor/br.py | 127 +++++++++++++++++++++++--------------
 1 file changed, 78 insertions(+), 49 deletions(-)

diff --git a/youtube_dl/extractor/br.py b/youtube_dl/extractor/br.py
index 66e394e10..e66854538 100644
--- a/youtube_dl/extractor/br.py
+++ b/youtube_dl/extractor/br.py
@@ -1,18 +1,21 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     int_or_none,
     parse_duration,
+    xpath_element,
+    xpath_text,
 )
 
 
 class BRIE(InfoExtractor):
     IE_DESC = 'Bayerischer Rundfunk Mediathek'
-    _VALID_URL = r'https?://(?:www\.)?br\.de/(?:[a-z0-9\-_]+/)+(?P<id>[a-z0-9\-_]+)\.html'
-    _BASE_URL = 'http://www.br.de'
+    _VALID_URL = r'(?P<base_url>https?://(?:www\.)?br(?:-klassik)?\.de)/(?:[a-z0-9\-_]+/)+(?P<id>[a-z0-9\-_]+)\.html'
 
     _TESTS = [
         {
@@ -22,7 +25,7 @@ class BRIE(InfoExtractor):
                 'id': '48f656ef-287e-486f-be86-459122db22cc',
                 'ext': 'mp4',
                 'title': 'Die böse Überraschung',
-                'description': 'Betriebliche Altersvorsorge: Die böse Überraschung',
+                'description': 'md5:ce9ac81b466ce775b8018f6801b48ac9',
                 'duration': 180,
                 'uploader': 'Reinhard Weber',
                 'upload_date': '20150422',
@@ -30,23 +33,23 @@ class BRIE(InfoExtractor):
         },
         {
             'url': 'http://www.br.de/nachrichten/oberbayern/inhalt/muenchner-polizeipraesident-schreiber-gestorben-100.html',
-            'md5': 'a44396d73ab6a68a69a568fae10705bb',
+            'md5': 'af3a3a4aa43ff0ce6a89504c67f427ef',
             'info_dict': {
                 'id': 'a4b83e34-123d-4b81-9f4e-c0d3121a4e05',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'Manfred Schreiber ist tot',
-                'description': 'Abendschau kompakt: Manfred Schreiber ist tot',
+                'description': 'md5:b454d867f2a9fc524ebe88c3f5092d97',
                 'duration': 26,
             }
         },
         {
-            'url': 'http://www.br.de/radio/br-klassik/sendungen/allegro/premiere-urauffuehrung-the-land-2015-dance-festival-muenchen-100.html',
+            'url': 'https://www.br-klassik.de/audio/peeping-tom-premierenkritik-dance-festival-muenchen-100.html',
             'md5': '8b5b27c0b090f3b35eac4ab3f7a73d3d',
             'info_dict': {
                 'id': '74c603c9-26d3-48bb-b85b-079aeed66e0b',
                 'ext': 'aac',
                 'title': 'Kurzweilig und sehr bewegend',
-                'description': '"The Land" von Peeping Tom: Kurzweilig und sehr bewegend',
+                'description': 'md5:0351996e3283d64adeb38ede91fac54e',
                 'duration': 296,
             }
         },
@@ -57,7 +60,7 @@ class BRIE(InfoExtractor):
                 'id': '6ba73750-d405-45d3-861d-1ce8c524e059',
                 'ext': 'mp4',
                 'title': 'Umweltbewusster Häuslebauer',
-                'description': 'Uwe Erdelt: Umweltbewusster Häuslebauer',
+                'description': 'md5:d52dae9792d00226348c1dbb13c9bae2',
                 'duration': 116,
             }
         },
@@ -68,7 +71,7 @@ class BRIE(InfoExtractor):
                 'id': 'd982c9ce-8648-4753-b358-98abb8aec43d',
                 'ext': 'mp4',
                 'title': 'Folge 1 - Metaphysik',
-                'description': 'Kant für Anfänger: Folge 1 - Metaphysik',
+                'description': 'md5:bb659990e9e59905c3d41e369db1fbe3',
                 'duration': 893,
                 'uploader': 'Eva Maria Steimle',
                 'upload_date': '20140117',
@@ -77,28 +80,31 @@ class BRIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        base_url, display_id = re.search(self._VALID_URL, url).groups()
         page = self._download_webpage(url, display_id)
         xml_url = self._search_regex(
             r"return BRavFramework\.register\(BRavFramework\('avPlayer_(?:[a-f0-9-]{36})'\)\.setup\({dataURL:'(/(?:[a-z0-9\-]+/)+[a-z0-9/~_.-]+)'}\)\);", page, 'XMLURL')
-        xml = self._download_xml(self._BASE_URL + xml_url, None)
+        xml = self._download_xml(base_url + xml_url, display_id)
 
         medias = []
 
         for xml_media in xml.findall('video') + xml.findall('audio'):
+            media_id = xml_media.get('externalId')
             media = {
-                'id': xml_media.get('externalId'),
-                'title': xml_media.find('title').text,
-                'duration': parse_duration(xml_media.find('duration').text),
-                'formats': self._extract_formats(xml_media.find('assets')),
-                'thumbnails': self._extract_thumbnails(xml_media.find('teaserImage/variants')),
-                'description': ' '.join(xml_media.find('shareTitle').text.splitlines()),
-                'webpage_url': xml_media.find('permalink').text
+                'id': media_id,
+                'title': xpath_text(xml_media, 'title', 'title', True),
+                'duration': parse_duration(xpath_text(xml_media, 'duration')),
+                'formats': self._extract_formats(xpath_element(
+                    xml_media, 'assets'), media_id),
+                'thumbnails': self._extract_thumbnails(xpath_element(
+                    xml_media, 'teaserImage/variants'), base_url),
+                'description': xpath_text(xml_media, 'desc'),
+                'webpage_url': xpath_text(xml_media, 'permalink'),
+                'uploader': xpath_text(xml_media, 'author'),
             }
-            if xml_media.find('author').text:
-                media['uploader'] = xml_media.find('author').text
-            if xml_media.find('broadcastDate').text:
-                media['upload_date'] = ''.join(reversed(xml_media.find('broadcastDate').text.split('.')))
+            broadcast_date = xpath_text(xml_media, 'broadcastDate')
+            if broadcast_date:
+                media['upload_date'] = ''.join(reversed(broadcast_date.split('.')))
             medias.append(media)
 
         if len(medias) > 1:
@@ -109,35 +115,58 @@ class BRIE(InfoExtractor):
             raise ExtractorError('No media entries found')
         return medias[0]
 
-    def _extract_formats(self, assets):
-
-        def text_or_none(asset, tag):
-            elem = asset.find(tag)
-            return None if elem is None else elem.text
-
-        formats = [{
-            'url': text_or_none(asset, 'downloadUrl'),
-            'ext': text_or_none(asset, 'mediaType'),
-            'format_id': asset.get('type'),
-            'width': int_or_none(text_or_none(asset, 'frameWidth')),
-            'height': int_or_none(text_or_none(asset, 'frameHeight')),
-            'tbr': int_or_none(text_or_none(asset, 'bitrateVideo')),
-            'abr': int_or_none(text_or_none(asset, 'bitrateAudio')),
-            'vcodec': text_or_none(asset, 'codecVideo'),
-            'acodec': text_or_none(asset, 'codecAudio'),
-            'container': text_or_none(asset, 'mediaType'),
-            'filesize': int_or_none(text_or_none(asset, 'size')),
-        } for asset in assets.findall('asset')
-            if asset.find('downloadUrl') is not None]
-
+    def _extract_formats(self, assets, media_id):
+        formats = []
+        for asset in assets.findall('asset'):
+            format_url = xpath_text(asset, ['downloadUrl', 'url'])
+            asset_type = asset.get('type')
+            if asset_type == 'HDS':
+                f4m_formats = self._extract_f4m_formats(
+                    format_url + '?hdcore=3.2.0', media_id, f4m_id='hds', fatal=False)
+                if f4m_formats:
+                    formats.extend(f4m_formats)
+            elif asset_type == 'HLS':
+                m3u8_formats = self._extract_m3u8_formats(
+                    format_url, media_id, 'mp4', 'm3u8_native', m3u8_id='hds', fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
+            else:
+                format_info = {
+                    'ext': xpath_text(asset, 'mediaType'),
+                    'width': int_or_none(xpath_text(asset, 'frameWidth')),
+                    'height': int_or_none(xpath_text(asset, 'frameHeight')),
+                    'tbr': int_or_none(xpath_text(asset, 'bitrateVideo')),
+                    'abr': int_or_none(xpath_text(asset, 'bitrateAudio')),
+                    'vcodec': xpath_text(asset, 'codecVideo'),
+                    'acodec': xpath_text(asset, 'codecAudio'),
+                    'container': xpath_text(asset, 'mediaType'),
+                    'filesize': int_or_none(xpath_text(asset, 'size')),
+                }
+                format_url = self._proto_relative_url(format_url)
+                if format_url:
+                    http_format_info = format_info.copy()
+                    http_format_info.update({
+                        'url': format_url,
+                        'format_id': 'http-%s' % asset_type,
+                    })
+                    formats.append(http_format_info)
+                server_prefix = xpath_text(asset, 'serverPrefix')
+                if server_prefix:
+                    rtmp_format_info = format_info.copy()
+                    rtmp_format_info.update({
+                        'url': server_prefix,
+                        'play_path': xpath_text(asset, 'fileName'),
+                        'format_id': 'rtmp-%s' % asset_type,
+                    })
+                    formats.append(rtmp_format_info)
         self._sort_formats(formats)
         return formats
 
-    def _extract_thumbnails(self, variants):
+    def _extract_thumbnails(self, variants, base_url):
         thumbnails = [{
-            'url': self._BASE_URL + variant.find('url').text,
-            'width': int_or_none(variant.find('width').text),
-            'height': int_or_none(variant.find('height').text),
-        } for variant in variants.findall('variant')]
+            'url': base_url + xpath_text(variant, 'url'),
+            'width': int_or_none(xpath_text(variant, 'width')),
+            'height': int_or_none(xpath_text(variant, 'height')),
+        } for variant in variants.findall('variant') if xpath_text(variant, 'url')]
         thumbnails.sort(key=lambda x: x['width'] * x['height'], reverse=True)
         return thumbnails

From eed30fea755f454582eb5ff555517cce3870dc87 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Dec 2015 22:10:16 +0100
Subject: [PATCH 0131/1105] [flickr] fix format sorting

---
 youtube_dl/extractor/flickr.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index 452b27b26..18f439df9 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -50,13 +50,19 @@ class FlickrIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        api_key = self._download_json('https://www.flickr.com/hermes_error_beacon.gne', video_id, 'Downloading api key',)['site_key']
+        api_key = self._download_json(
+            'https://www.flickr.com/hermes_error_beacon.gne', video_id,
+            'Downloading api key')['site_key']
 
-        video_info = self._call_api('photos.getInfo', video_id, api_key, 'Downloading video info')['photo']
+        video_info = self._call_api(
+            'photos.getInfo', video_id, api_key, 'Downloading video info')['photo']
         if video_info['media'] == 'video':
-            streams = self._call_api('video.getStreamInfo', video_id, api_key, 'Downloading streams info', video_info['secret'])['streams']
+            streams = self._call_api(
+                'video.getStreamInfo', video_id, api_key,
+                'Downloading streams info', video_info['secret'])['streams']
 
-            preference = qualities(['iphone_wifi', '700', 'appletv', 'orig'])
+            preference = qualities(
+                ['288p', 'iphone_wifi', '100', '300', '700', '360p', 'appletv', '720p', '1080p', 'orig'])
 
             formats = []
             for stream in streams['stream']:

From c7224074d61adf0dda362332a27f8ae20c4502cd Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Dec 2015 23:02:55 +0100
Subject: [PATCH 0132/1105] [audimedia] correct test case id

---
 youtube_dl/extractor/audimedia.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/audimedia.py b/youtube_dl/extractor/audimedia.py
index b0b089dee..4382a302b 100644
--- a/youtube_dl/extractor/audimedia.py
+++ b/youtube_dl/extractor/audimedia.py
@@ -15,7 +15,7 @@ class AudiMediaIE(InfoExtractor):
         'url': 'https://audimedia.tv/en/vid/60-seconds-of-audi-sport-104-2015-wec-bahrain-rookie-test',
         'md5': '79a8b71c46d49042609795ab59779b66',
         'info_dict': {
-            'id': '1564',
+            'id': '1565',
             'ext': 'mp4',
             'title': '60 Seconds of Audi Sport 104/2015 - WEC Bahrain, Rookie Test',
             'description': 'md5:60e5d30a78ced725f7b8d34370762941',

From 5f432ac8f552b06e715c5e17165328dc76d9c1b5 Mon Sep 17 00:00:00 2001
From: Founder Fang <founder.fang@gmail.com>
Date: Sun, 20 Dec 2015 19:09:45 +0800
Subject: [PATCH 0133/1105] [Weiqitv] Add new extractor

---
 youtube_dl/extractor/__init__.py |  4 ++-
 youtube_dl/extractor/letv.py     | 57 ++++++++++++++++++++++++++++++++
 youtube_dl/extractor/weiqitv.py  | 54 ++++++++++++++++++++++++++++++
 3 files changed, 114 insertions(+), 1 deletion(-)
 create mode 100644 youtube_dl/extractor/weiqitv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e46d73ed7..9dcd252f8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -331,7 +331,8 @@ from .lecture2go import Lecture2GoIE
 from .letv import (
     LetvIE,
     LetvTvIE,
-    LetvPlaylistIE
+    LetvPlaylistIE,
+    LetvCloudIE,
 )
 from .libsyn import LibsynIE
 from .lifenews import (
@@ -834,6 +835,7 @@ from .webofstories import (
     WebOfStoriesPlaylistIE,
 )
 from .weibo import WeiboIE
+from .weiqitv import WeiqitvIE
 from .wimp import WimpIE
 from .wistia import WistiaIE
 from .worldstarhiphop import WorldStarHipHopIE
diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index be648000e..c096cb1ab 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import datetime
 import re
 import time
+import base64
 
 from .common import InfoExtractor
 from ..compat import (
@@ -16,6 +17,7 @@ from ..utils import (
     parse_iso8601,
     sanitized_Request,
     int_or_none,
+    str_or_none,
     encode_data_uri,
 )
 
@@ -239,3 +241,58 @@ class LetvPlaylistIE(LetvTvIE):
         },
         'playlist_mincount': 7
     }]
+
+
+class LetvCloudIE(InfoExtractor):
+    IE_DESC = '乐视云'
+    _VALID_URL = r'http://yuntv\.letv\.com/bcloud.html\?.*$'
+
+    _TESTS = [{
+        'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=467623dedf',
+        'md5': '26450599afd64c513bc77030ad15db44',
+        'info_dict': {
+            'id': 'p7jnfw5hw9_467623dedf',
+            'ext': 'mp4',
+            'title': 'p7jnfw5hw9_467623dedf',
+        },
+    }, {
+        'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=ec93197892&pu=2c7cd40209&auto_play=1&gpcflag=1&width=640&height=360',
+        'info_dict': {
+            'id': 'p7jnfw5hw9_ec93197892',
+            'ext': 'mp4',
+            'title': 'p7jnfw5hw9_ec93197892',
+        },
+    }, {
+        'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=187060b6fd',
+        'info_dict': {
+            'id': 'p7jnfw5hw9_187060b6fd',
+            'ext': 'mp4',
+            'title': 'p7jnfw5hw9_187060b6fd',
+        },
+    }]
+
+    def _real_extract(self, url):
+        uu = re.search('uu=([\w]+)', url).group(1)
+        vu = re.search('vu=([\w]+)', url).group(1)
+        media_id = uu + '_' + vu
+
+        play_json_req = sanitized_Request(
+            'http://api.letvcloud.com/gpc.php?cf=html5&sign=signxxxxx&ver=2.2&format=json&' +
+            "uu=" + uu + "&vu=" + vu)
+        play_json = self._download_json(play_json_req, media_id, 'Downloading playJson data')
+
+        formats = [{
+            'url': base64.b64decode(media['play_url']['main_url'].encode('utf-8')).decode("utf-8"),
+            'ext': 'mp4',
+            'format_id': int_or_none(media.get('play_url', {}).get('vtype')),
+            'format_note': str_or_none(media.get('play_url', {}).get('definition')),
+            'width': int_or_none(media.get('play_url', {}).get('vwidth')),
+            'height': int_or_none(media.get('play_url', {}).get('vheight')),
+        } for media in play_json['data']['video_info']['media'].values()]
+        self._sort_formats(formats)
+
+        return {
+            'id': media_id,
+            'title': media_id,
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/weiqitv.py b/youtube_dl/extractor/weiqitv.py
new file mode 100644
index 000000000..da3b3d145
--- /dev/null
+++ b/youtube_dl/extractor/weiqitv.py
@@ -0,0 +1,54 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class WeiqitvIE(InfoExtractor):
+    IE_DESC = 'WQTV'
+    _VALID_URL = r'http://www\.weiqitv\.com/index/video_play\?videoId=(?P<id>[A-Za-z0-9]+)'
+
+    _TESTS = [{
+        'url': 'http://www.weiqitv.com/index/video_play?videoId=53c744f09874f0e76a8b46f3',
+        'md5': '26450599afd64c513bc77030ad15db44',
+        'info_dict': {
+            'id': '53c744f09874f0e76a8b46f3',
+            'ext': 'mp4',
+            'title': '2013年度盘点',
+        },
+    }, {
+        'url': 'http://www.weiqitv.com/index/video_play?videoId=567379a2d4c36cca518b4569',
+        'info_dict': {
+            'id': '567379a2d4c36cca518b4569',
+            'ext': 'mp4',
+            'title': '民国围棋史',
+        },
+    }, {
+        'url': 'http://www.weiqitv.com/index/video_play?videoId=5430220a9874f088658b4567',
+        'info_dict': {
+            'id': '5430220a9874f088658b4567',
+            'ext': 'mp4',
+            'title': '二路托过的手段和运用',
+        },
+    }]
+
+    def _real_extract(self, url):
+        media_id = self._match_id(url)
+        page = self._download_webpage(url, media_id)
+
+        info_json_str = self._search_regex(
+            'var\s+video\s*=\s*(.+});',
+            page, 'info_json_str')
+        info_json = self._parse_json(info_json_str, media_id)
+
+        letvcloud_url = self._search_regex(
+            'var\s+letvurl\s*=\s*"([^"]+)',
+            page, 'letvcloud_url')
+
+        return {
+            '_type': 'url_transparent',
+            "ie_key": 'LetvCloud',
+            'url': letvcloud_url,
+            'title': info_json['name'],
+            'id': media_id,
+        }

From ed63cbd6ee392a01d10f0ca3f36ac9442622f0a9 Mon Sep 17 00:00:00 2001
From: Austin Adams <git@austinjadams.com>
Date: Mon, 21 Dec 2015 20:26:15 -0500
Subject: [PATCH 0134/1105] [comcarcoff] adjust for json updates

---
 youtube_dl/extractor/comcarcoff.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/comcarcoff.py b/youtube_dl/extractor/comcarcoff.py
index 81f3d7697..4391b7ce4 100644
--- a/youtube_dl/extractor/comcarcoff.py
+++ b/youtube_dl/extractor/comcarcoff.py
@@ -32,8 +32,8 @@ class ComCarCoffIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         full_data = json.loads(self._search_regex(
-            r'<script type="application/json" id="videoData">(?P<json>.+?)</script>',
-            webpage, 'full data json'))
+            r'\nwindow.app = (?P<json>.+?);\n',
+            webpage, 'full data json'))['videoData']
 
         video_id = full_data['activeVideo']['video']
         video_data = full_data.get('videos', {}).get(video_id) or full_data['singleshots'][video_id]

From ff43d2365ff5569c98df8e01250d34706e266c44 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 22 Dec 2015 07:58:33 +0100
Subject: [PATCH 0135/1105] [soompi] remove extractor

http://tv.soompi.com now redirect to viki.com because Viki has acquired
Soompi
http://www.soompi.com/2015/08/19/we-got-married-soompi-joins-viki/
---
 youtube_dl/extractor/__init__.py |   4 -
 youtube_dl/extractor/soompi.py   | 146 -------------------------------
 2 files changed, 150 deletions(-)
 delete mode 100644 youtube_dl/extractor/soompi.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 760b65441..702cbc6e2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -591,10 +591,6 @@ from .snagfilms import (
 )
 from .snotr import SnotrIE
 from .sohu import SohuIE
-from .soompi import (
-    SoompiIE,
-    SoompiShowIE,
-)
 from .soundcloud import (
     SoundcloudIE,
     SoundcloudSetIE,
diff --git a/youtube_dl/extractor/soompi.py b/youtube_dl/extractor/soompi.py
deleted file mode 100644
index 5da66ca9e..000000000
--- a/youtube_dl/extractor/soompi.py
+++ /dev/null
@@ -1,146 +0,0 @@
-# encoding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .crunchyroll import CrunchyrollIE
-
-from .common import InfoExtractor
-from ..compat import compat_HTTPError
-from ..utils import (
-    ExtractorError,
-    int_or_none,
-    remove_start,
-    xpath_text,
-)
-
-
-class SoompiBaseIE(InfoExtractor):
-    def _get_episodes(self, webpage, episode_filter=None):
-        episodes = self._parse_json(
-            self._search_regex(
-                r'VIDEOS\s*=\s*(\[.+?\]);', webpage, 'episodes JSON'),
-            None)
-        return list(filter(episode_filter, episodes))
-
-
-class SoompiIE(SoompiBaseIE, CrunchyrollIE):
-    IE_NAME = 'soompi'
-    _VALID_URL = r'https?://tv\.soompi\.com/(?:en/)?watch/(?P<id>[0-9]+)'
-    _TESTS = [{
-        'url': 'http://tv.soompi.com/en/watch/29235',
-        'info_dict': {
-            'id': '29235',
-            'ext': 'mp4',
-            'title': 'Episode 1096',
-            'description': '2015-05-20'
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }]
-
-    def _get_episode(self, webpage, video_id):
-        return self._get_episodes(webpage, lambda x: x['id'] == video_id)[0]
-
-    def _get_subtitles(self, config, video_id):
-        sub_langs = {}
-        for subtitle in config.findall('./{default}preload/subtitles/subtitle'):
-            sub_langs[subtitle.attrib['id']] = subtitle.attrib['title']
-
-        subtitles = {}
-        for s in config.findall('./{default}preload/subtitle'):
-            lang_code = sub_langs.get(s.attrib['id'])
-            if not lang_code:
-                continue
-            sub_id = s.get('id')
-            data = xpath_text(s, './data', 'data')
-            iv = xpath_text(s, './iv', 'iv')
-            if not id or not iv or not data:
-                continue
-            subtitle = self._decrypt_subtitles(data, iv, sub_id).decode('utf-8')
-            subtitles[lang_code] = self._extract_subtitles(subtitle)
-        return subtitles
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        try:
-            webpage = self._download_webpage(
-                url, video_id, 'Downloading episode page')
-        except ExtractorError as ee:
-            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
-                webpage = ee.cause.read()
-                block_message = self._html_search_regex(
-                    r'(?s)<div class="block-message">(.+?)</div>', webpage,
-                    'block message', default=None)
-                if block_message:
-                    raise ExtractorError(block_message, expected=True)
-            raise
-
-        formats = []
-        config = None
-        for format_id in re.findall(r'\?quality=([0-9a-zA-Z]+)', webpage):
-            config = self._download_xml(
-                'http://tv.soompi.com/en/show/_/%s-config.xml?mode=hls&quality=%s' % (video_id, format_id),
-                video_id, 'Downloading %s XML' % format_id)
-            m3u8_url = xpath_text(
-                config, './{default}preload/stream_info/file',
-                '%s m3u8 URL' % format_id)
-            if not m3u8_url:
-                continue
-            formats.extend(self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', m3u8_id=format_id))
-        self._sort_formats(formats)
-
-        episode = self._get_episode(webpage, video_id)
-
-        title = episode['name']
-        description = episode.get('description')
-        duration = int_or_none(episode.get('duration'))
-
-        thumbnails = [{
-            'id': thumbnail_id,
-            'url': thumbnail_url,
-        } for thumbnail_id, thumbnail_url in episode.get('img_url', {}).items()]
-
-        subtitles = self.extract_subtitles(config, video_id)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnails': thumbnails,
-            'duration': duration,
-            'formats': formats,
-            'subtitles': subtitles
-        }
-
-
-class SoompiShowIE(SoompiBaseIE):
-    IE_NAME = 'soompi:show'
-    _VALID_URL = r'https?://tv\.soompi\.com/en/shows/(?P<id>[0-9a-zA-Z\-_]+)'
-    _TESTS = [{
-        'url': 'http://tv.soompi.com/en/shows/liar-game',
-        'info_dict': {
-            'id': 'liar-game',
-            'title': 'Liar Game',
-            'description': 'md5:52c02bce0c1a622a95823591d0589b66',
-        },
-        'playlist_count': 14,
-    }]
-
-    def _real_extract(self, url):
-        show_id = self._match_id(url)
-
-        webpage = self._download_webpage(
-            url, show_id, 'Downloading show page')
-
-        title = remove_start(self._og_search_title(webpage), 'SoompiTV | ')
-        description = self._og_search_description(webpage)
-
-        entries = [
-            self.url_result('http://tv.soompi.com/en/watch/%s' % episode['id'], 'Soompi')
-            for episode in self._get_episodes(webpage)]
-
-        return self.playlist_result(entries, show_id, title, description)

From dc016bf5216d4c0d5b5fb2cd707e1d08fa4b0517 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 22 Dec 2015 09:55:25 +0100
Subject: [PATCH 0136/1105] [viki] detect errors and fix formats extraction

---
 youtube_dl/extractor/viki.py | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index a63c23617..ca3f20a3d 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -30,6 +30,12 @@ class VikiBaseIE(InfoExtractor):
 
     _token = None
 
+    _ERRORS = {
+        'geo': 'Sorry, this content is not available in your region.',
+        'upcoming': 'Sorry, this content is not yet available.',
+        # 'paywall': 'paywall',
+    }
+
     def _prepare_call(self, path, timestamp=None, post_data=None):
         path += '?' if '?' not in path else '&'
         if not timestamp:
@@ -67,6 +73,12 @@ class VikiBaseIE(InfoExtractor):
             '%s returned error: %s' % (self.IE_NAME, error),
             expected=True)
 
+    def _check_errors(self, data):
+        for reason, status in data.get('blocking', {}).items():
+            if status and reason in self._ERRORS:
+                raise ExtractorError('%s said: %s' % (
+                    self.IE_NAME, self._ERRORS[reason]), expected=True)
+
     def _real_initialize(self):
         self._login()
 
@@ -193,6 +205,7 @@ class VikiIE(VikiBaseIE):
             'timestamp': 1321985454,
             'description': 'md5:44b1e46619df3a072294645c770cef36',
             'title': 'Love In Magic',
+            'age_limit': 13,
         },
     }]
 
@@ -202,6 +215,8 @@ class VikiIE(VikiBaseIE):
         video = self._call_api(
             'videos/%s.json' % video_id, video_id, 'Downloading video JSON')
 
+        self._check_errors(video)
+
         title = self.dict_selection(video.get('titles', {}), 'en')
         if not title:
             title = 'Episode %d' % video.get('number') if video.get('type') == 'episode' else video.get('id') or video_id
@@ -262,8 +277,11 @@ class VikiIE(VikiBaseIE):
                 r'^(\d+)[pP]$', format_id, 'height', default=None))
             for protocol, format_dict in stream_dict.items():
                 if format_id == 'm3u8':
-                    formats = self._extract_m3u8_formats(
-                        format_dict['url'], video_id, 'mp4', m3u8_id='m3u8-%s' % protocol)
+                    m3u8_formats = self._extract_m3u8_formats(
+                        format_dict['url'], video_id, 'mp4', 'm3u8_native',
+                        m3u8_id='m3u8-%s' % protocol, fatal=None)
+                    if m3u8_formats:
+                        formats.extend(m3u8_formats)
                 else:
                     formats.append({
                         'url': format_dict['url'],
@@ -315,6 +333,8 @@ class VikiChannelIE(VikiBaseIE):
             'containers/%s.json' % channel_id, channel_id,
             'Downloading channel JSON')
 
+        self._check_errors(channel)
+
         title = self.dict_selection(channel['titles'], 'en')
 
         description = self.dict_selection(channel['descriptions'], 'en')

From 48a6c984b806141dc6d3da0a96df2e553bb815e0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 22 Dec 2015 10:14:57 +0100
Subject: [PATCH 0137/1105] [bleacherreport] update test

---
 youtube_dl/extractor/bleacherreport.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bleacherreport.py b/youtube_dl/extractor/bleacherreport.py
index bd2a6340b..38bda3af5 100644
--- a/youtube_dl/extractor/bleacherreport.py
+++ b/youtube_dl/extractor/bleacherreport.py
@@ -90,7 +90,7 @@ class BleacherReportCMSIE(AMPIE):
     _VALID_URL = r'https?://(?:www\.)?bleacherreport\.com/video_embed\?id=(?P<id>[0-9a-f-]{36})'
     _TESTS = [{
         'url': 'http://bleacherreport.com/video_embed?id=8fd44c2f-3dc5-4821-9118-2c825a98c0e1',
-        'md5': 'f0ca220af012d4df857b54f792c586bb',
+        'md5': '8c2c12e3af7805152675446c905d159b',
         'info_dict': {
             'id': '8fd44c2f-3dc5-4821-9118-2c825a98c0e1',
             'ext': 'flv',

From 220bc3f0e3777b89de335cdbc58a7d105584f06b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 22 Dec 2015 11:27:18 +0100
Subject: [PATCH 0138/1105] [franceinter] fix title extraction

---
 youtube_dl/extractor/franceinter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/franceinter.py b/youtube_dl/extractor/franceinter.py
index 6613ee17a..90a17815d 100644
--- a/youtube_dl/extractor/franceinter.py
+++ b/youtube_dl/extractor/franceinter.py
@@ -33,7 +33,7 @@ class FranceInterIE(InfoExtractor):
         video_url = 'http://www.franceinter.fr/' + path
 
         title = self._html_search_regex(
-            r'<span class="title">(.+?)</span>', webpage, 'title')
+            r'<span class="title-diffusion">(.+?)</span>', webpage, 'title')
         description = self._html_search_regex(
             r'<span class="description">(.*?)</span>',
             webpage, 'description', fatal=False)

From 2db5806991145ee293c964ecd85623c093d2e429 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 22 Dec 2015 11:30:35 +0100
Subject: [PATCH 0139/1105] [franceinter] use _match_id

---
 youtube_dl/extractor/franceinter.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/franceinter.py b/youtube_dl/extractor/franceinter.py
index 90a17815d..fdc51f44f 100644
--- a/youtube_dl/extractor/franceinter.py
+++ b/youtube_dl/extractor/franceinter.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import int_or_none
 
@@ -23,8 +21,7 @@ class FranceInterIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 

From 2be689b7e27df89648d1d98fa74c297f0e06cbc1 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Mon, 21 Dec 2015 02:26:37 +0100
Subject: [PATCH 0140/1105] [theintercept] Add new extractor

---
 youtube_dl/extractor/__init__.py     |  1 +
 youtube_dl/extractor/theintercept.py | 68 ++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+)
 create mode 100644 youtube_dl/extractor/theintercept.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index eac50eda5..042b1e921 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -657,6 +657,7 @@ from .tenplay import TenPlayIE
 from .testurl import TestURLIE
 from .testtube import TestTubeIE
 from .tf1 import TF1IE
+from .theintercept import TheInterceptIE
 from .theonion import TheOnionIE
 from .theplatform import (
     ThePlatformIE,
diff --git a/youtube_dl/extractor/theintercept.py b/youtube_dl/extractor/theintercept.py
new file mode 100644
index 000000000..b096a28de
--- /dev/null
+++ b/youtube_dl/extractor/theintercept.py
@@ -0,0 +1,68 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+from ..utils import (
+    ExtractorError,
+)
+
+class TheInterceptIE(InfoExtractor):
+    _VALID_URL = r'https://theintercept.com/fieldofvision/(?P<id>.+?)/'
+    _TESTS = [{
+        'url': 'https://theintercept.com/fieldofvision/thisisacoup-episode-four-surrender-or-die/',
+        'info_dict': {
+            'id': 'thisisacoup-episode-four-surrender-or-die',
+            'ext': 'mp4',
+            'title': '#ThisIsACoup – Episode Four: Surrender or Die',
+            'upload_date': '20151218',
+            'description': 'md5:74dd27f0e2fbd50817829f97eaa33140',
+        }
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        mobj = re.search(r'initialStoreTree =(?P<json_data>.+})', webpage)
+        if mobj is None:
+            raise ExtractorError('Unable to extract initialStoreTree')
+        json_data = self._parse_json(mobj.group('json_data'), display_id)
+
+        info = None
+        for post in json_data['resources']['posts'].values():
+            if post['slug'] == display_id:
+                info = post
+                break
+        if info is None:
+            raise ExtractorError('Unable to find info for %s'%display_id)
+
+        title = info['title']
+        description = info['excerpt']
+        upload_date = info['date'][:10].replace('-', '')
+        video_id = info['fov_videoid']
+        creator = ','.join([a['display_name'] for a in info['authors']])
+        thumbnail = self._og_search_property('image', webpage)
+        content_id = thumbnail.split('/')[-1].split('.')[0]
+        content_url = 'https://content.jwplatform.com/jw6/{content_id}.xml'.format(content_id=content_id)
+        content = self._download_xml(content_url, video_id)
+
+        formats = []
+        for source in content.findall('.//{http://rss.jwpcdn.com/}source'):
+            if source.attrib['file'].endswith('.m3u8'):
+                formats.extend(self._extract_m3u8_formats(
+                    source.attrib['file'], video_id, 'mp4', preference=1, m3u8_id='hls'))
+
+        return {
+            'creator': creator,
+            'description': description,
+            'display_id': display_id,
+            'formats': formats,
+            'id': video_id,
+            'id': video_id,
+            'thumbnail': thumbnail,
+            'title': title,
+            'upload_date': upload_date,
+        }

From 3b68efdc6ae109a840ff5f15f0e28910c2463b3f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 22 Dec 2015 15:54:51 +0100
Subject: [PATCH 0141/1105] [vgtv] update tests and correct format sorting

---
 youtube_dl/extractor/vgtv.py    | 31 +++++++++++++++++++++----------
 youtube_dl/extractor/xstream.py |  2 +-
 2 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 347410a78..811ee197d 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -104,10 +104,10 @@ class VGTVIE(XstreamIE):
         },
         {
             'url': 'http://www.aftenposten.no/webtv/#!/video/21039/trailer-sweatshop-i-can-t-take-any-more',
-            'md5': '7fbc265a3ca4933a423c7a66aa879a67',
+            'md5': 'fd828cd29774a729bf4d4425fe192972',
             'info_dict': {
                 'id': '21039',
-                'ext': 'mp4',
+                'ext': 'mov',
                 'title': 'TRAILER: «SWEATSHOP» - I can´t take any more',
                 'description': 'md5:21891f2b0dd7ec2f78d84a50e54f8238',
                 'duration': 66,
@@ -174,16 +174,15 @@ class VGTVIE(XstreamIE):
         for mp4_url in mp4_urls:
             format_info = {
                 'url': mp4_url,
-                'preference': 1,
             }
             mobj = re.search('(\d+)_(\d+)_(\d+)', mp4_url)
             if mobj:
-                vbr = int(mobj.group(3))
+                tbr = int(mobj.group(3))
                 format_info.update({
                     'width': int(mobj.group(1)),
                     'height': int(mobj.group(2)),
-                    'vbr': vbr,
-                    'format_id': 'mp4-%s' % vbr,
+                    'tbr': tbr,
+                    'format_id': 'mp4-%s' % tbr,
                 })
             formats.append(format_info)
 
@@ -210,7 +209,7 @@ class BTArticleIE(InfoExtractor):
     _VALID_URL = 'http://(?:www\.)?bt\.no/(?:[^/]+/)+(?P<id>[^/]+)-\d+\.html'
     _TEST = {
         'url': 'http://www.bt.no/nyheter/lokalt/Kjemper-for-internatet-1788214.html',
-        'md5': 'd055e8ee918ef2844745fcfd1a4175fb',
+        'md5': '2acbe8ad129b3469d5ae51b1158878df',
         'info_dict': {
             'id': '23199',
             'ext': 'mp4',
@@ -227,7 +226,7 @@ class BTArticleIE(InfoExtractor):
     def _real_extract(self, url):
         webpage = self._download_webpage(url, self._match_id(url))
         video_id = self._search_regex(
-            r'SVP\.Player\.load\(\s*(\d+)', webpage, 'video id')
+            r'<video[^>]+data-id="(\d+)"', webpage, 'video id')
         return self.url_result('bttv:%s' % video_id, 'VGTV')
 
 
@@ -235,7 +234,7 @@ class BTVestlendingenIE(InfoExtractor):
     IE_NAME = 'bt:vestlendingen'
     IE_DESC = 'Bergens Tidende - Vestlendingen'
     _VALID_URL = 'http://(?:www\.)?bt\.no/spesial/vestlendingen/#!/(?P<id>\d+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.bt.no/spesial/vestlendingen/#!/86588',
         'md5': 'd7d17e3337dc80de6d3a540aefbe441b',
         'info_dict': {
@@ -246,7 +245,19 @@ class BTVestlendingenIE(InfoExtractor):
             'timestamp': 1430473209,
             'upload_date': '20150501',
         },
-    }
+        'skip': '404 Error',
+    }, {
+        'url': 'http://www.bt.no/spesial/vestlendingen/#!/86255',
+        'md5': 'a2893f8632e96389f4bdf36aa9463ceb',
+        'info_dict': {
+            'id': '86255',
+            'ext': 'mov',
+            'title': 'Du må tåle å fryse og være sulten',
+            'description': 'md5:b8046f4d022d5830ddab04865791d063',
+            'upload_date': '20150321',
+            'timestamp': 1426942023,
+        },
+    }]
 
     def _real_extract(self, url):
         return self.url_result('bttv:%s' % self._match_id(url), 'VGTV')
diff --git a/youtube_dl/extractor/xstream.py b/youtube_dl/extractor/xstream.py
index 436f8978b..76c91bd92 100644
--- a/youtube_dl/extractor/xstream.py
+++ b/youtube_dl/extractor/xstream.py
@@ -93,7 +93,7 @@ class XstreamIE(InfoExtractor):
             formats.append({
                 'url': link.get('href'),
                 'format_id': link.get('rel'),
-                'preference': 2,
+                'preference': 1,
             })
 
         thumbnails = [{

From dbee18b5521edbfa1642c683ad2d317ba06e9d5b Mon Sep 17 00:00:00 2001
From: Abhishek Kedia <kedia.abhishek10@gmail.com>
Date: Mon, 21 Dec 2015 01:50:07 +0100
Subject: [PATCH 0142/1105] Improve extraction (Closes #7918)

remove outer parentheses in if

Conflicts:
	youtube_dl/extractor/imgur.py

checked code with flake8

not returning list in case of single images.

using the fact that id with length 5 are albums and more are single videos.
Also for single videos ie ImgurIE both urls - http://imgur.com/gallery/oWeAMW2 and http://imgur.com/oWeAMW2 are equally fine. Change regex to allow thuis.
For albums urls - http://imgur.com/gallery/Q95ko and http://imgur.com/Q95ko are ok. Change regex to allow this also.

update description in ImgurIE Tests.
Also move single video test 'https://imgur.com/gallery/YcAQlkx' from ImgurAlbumIE to ImgurIE.
---
 youtube_dl/extractor/imgur.py | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index 70c8ca64e..88423f179 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class ImgurIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?!gallery)(?P<id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(gallery/)?(?P<id>[a-zA-Z0-9]{6,})'
 
     _TESTS = [{
         'url': 'https://i.imgur.com/A61SaA1.gifv',
@@ -21,7 +21,7 @@ class ImgurIE(InfoExtractor):
             'id': 'A61SaA1',
             'ext': 'mp4',
             'title': 're:Imgur GIF$|MRW gifv is up and running without any bugs$',
-            'description': 're:The origin of the Internet\'s most viral images$|The Internet\'s visual storytelling community\. Explore, share, and discuss the best visual stories the Internet has to offer\.$',
+            'description': 'Imgur: The most awesome images on the Internet.',
         },
     }, {
         'url': 'https://imgur.com/A61SaA1',
@@ -29,8 +29,17 @@ class ImgurIE(InfoExtractor):
             'id': 'A61SaA1',
             'ext': 'mp4',
             'title': 're:Imgur GIF$|MRW gifv is up and running without any bugs$',
-            'description': 're:The origin of the Internet\'s most viral images$|The Internet\'s visual storytelling community\. Explore, share, and discuss the best visual stories the Internet has to offer\.$',
+            'description': 'Imgur: The most awesome images on the Internet.',
         },
+    }, {
+        'url': 'https://imgur.com/gallery/YcAQlkx',
+        'info_dict': {
+            'id': 'YcAQlkx',
+            'ext': 'mp4',
+            'title': 'Classic Steve Carell gif...cracks me up everytime....damn the repost downvotes....',
+            'description': 'Imgur: The most awesome images on the Internet.'
+
+        }
     }]
 
     def _real_extract(self, url):
@@ -100,7 +109,7 @@ class ImgurIE(InfoExtractor):
 
 
 class ImgurAlbumIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:i\.)?imgur\.com/gallery/(?P<id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(gallery/)?(?P<id>[a-zA-Z0-9]{5})(?![a-zA-Z0-9])'
 
     _TEST = {
         'url': 'http://imgur.com/gallery/Q95ko',
@@ -113,12 +122,15 @@ class ImgurAlbumIE(InfoExtractor):
     def _real_extract(self, url):
         album_id = self._match_id(url)
 
-        album_images = self._download_json(
-            'http://imgur.com/gallery/%s/album_images/hit.json?all=true' % album_id,
-            album_id)['data']['images']
+        album_img_data = self._download_json(
+            'http://imgur.com/gallery/%s/album_images/hit.json?all=true' % album_id, album_id)['data']
 
-        entries = [
-            self.url_result('http://imgur.com/%s' % image['hash'])
-            for image in album_images if image.get('hash')]
+        if len(album_img_data) == 0:
+            return self.url_result('http://imgur.com/%s' % album_id)
+        else:
+            album_images = album_img_data['images']
+            entries = [
+                self.url_result('http://imgur.com/%s' % image['hash'])
+                for image in album_images if image.get('hash')]
 
         return self.playlist_result(entries, album_id)

From 774ce35571c08a1532fe4079224239adfdb80e43 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 22 Dec 2015 21:48:48 +0600
Subject: [PATCH 0143/1105] [imgur] Improve (Closes #7928)

---
 youtube_dl/extractor/imgur.py | 41 +++++++++++++++++++++++------------
 1 file changed, 27 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index 88423f179..85e9344aa 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class ImgurIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(gallery/)?(?P<id>[a-zA-Z0-9]{6,})'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?:(?:gallery|topic/[^/]+)/)?(?P<id>[a-zA-Z0-9]{6,})(?:[/?#&]+|\.[a-z]+)?$'
 
     _TESTS = [{
         'url': 'https://i.imgur.com/A61SaA1.gifv',
@@ -40,6 +40,9 @@ class ImgurIE(InfoExtractor):
             'description': 'Imgur: The most awesome images on the Internet.'
 
         }
+    }, {
+        'url': 'http://imgur.com/topic/Funny/N8rOudd',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -109,28 +112,38 @@ class ImgurIE(InfoExtractor):
 
 
 class ImgurAlbumIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(gallery/)?(?P<id>[a-zA-Z0-9]{5})(?![a-zA-Z0-9])'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?:(?:a|gallery|topic/[^/]+)/)?(?P<id>[a-zA-Z0-9]{5})(?:[/?#&]+)?$'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://imgur.com/gallery/Q95ko',
         'info_dict': {
             'id': 'Q95ko',
         },
         'playlist_count': 25,
-    }
+    }, {
+        'url': 'http://imgur.com/a/j6Orj',
+        'only_matching': True,
+    }, {
+        'url': 'http://imgur.com/topic/Aww/ll5Vk',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         album_id = self._match_id(url)
 
-        album_img_data = self._download_json(
-            'http://imgur.com/gallery/%s/album_images/hit.json?all=true' % album_id, album_id)['data']
+        album_images = self._download_json(
+            'http://imgur.com/gallery/%s/album_images/hit.json?all=true' % album_id,
+            album_id, fatal=False)
 
-        if len(album_img_data) == 0:
-            return self.url_result('http://imgur.com/%s' % album_id)
-        else:
-            album_images = album_img_data['images']
-            entries = [
-                self.url_result('http://imgur.com/%s' % image['hash'])
-                for image in album_images if image.get('hash')]
+        if album_images:
+            data = album_images.get('data')
+            if data and isinstance(data, dict):
+                images = data.get('images')
+                if images and isinstance(images, list):
+                    entries = [
+                        self.url_result('http://imgur.com/%s' % image['hash'])
+                        for image in images if image.get('hash')]
+                    return self.playlist_result(entries, album_id)
 
-        return self.playlist_result(entries, album_id)
+        # Fallback to single video
+        return self.url_result('http://imgur.com/%s' % album_id, ImgurIE.ie_key())

From 4c24ed94640b148882f1ceb400127b3b3afcafd4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Dec 2015 01:10:31 +0600
Subject: [PATCH 0144/1105] [comcarcoff] Improve json data regex and modernize

---
 youtube_dl/extractor/comcarcoff.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/comcarcoff.py b/youtube_dl/extractor/comcarcoff.py
index 4391b7ce4..edf5b29a0 100644
--- a/youtube_dl/extractor/comcarcoff.py
+++ b/youtube_dl/extractor/comcarcoff.py
@@ -31,9 +31,10 @@ class ComCarCoffIE(InfoExtractor):
             display_id = 'comediansincarsgettingcoffee.com'
         webpage = self._download_webpage(url, display_id)
 
-        full_data = json.loads(self._search_regex(
-            r'\nwindow.app = (?P<json>.+?);\n',
-            webpage, 'full data json'))['videoData']
+        full_data = self._parse_json(
+            self._search_regex(
+                r'window\.app\s*=\s*({.+?});\n', webpage, 'full data json'),
+            display_id)['videoData']
 
         video_id = full_data['activeVideo']['video']
         video_data = full_data.get('videos', {}).get(video_id) or full_data['singleshots'][video_id]

From cfe9e5aa6c5b14016ae454649b1a9df9c7c18b3c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Dec 2015 01:18:14 +0600
Subject: [PATCH 0145/1105] [comcarcoff] Extract duration

---
 youtube_dl/extractor/comcarcoff.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/comcarcoff.py b/youtube_dl/extractor/comcarcoff.py
index edf5b29a0..2efa200b5 100644
--- a/youtube_dl/extractor/comcarcoff.py
+++ b/youtube_dl/extractor/comcarcoff.py
@@ -1,10 +1,12 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import json
-
 from .common import InfoExtractor
-from ..utils import parse_iso8601
+from ..utils import (
+    int_or_none,
+    parse_duration,
+    parse_iso8601,
+)
 
 
 class ComCarCoffIE(InfoExtractor):
@@ -16,6 +18,7 @@ class ComCarCoffIE(InfoExtractor):
             'ext': 'mp4',
             'upload_date': '20141127',
             'timestamp': 1417107600,
+            'duration': 1232,
             'title': 'Happy Thanksgiving Miranda',
             'description': 'Jerry Seinfeld and his special guest Miranda Sings cruise around town in search of coffee, complaining and apologizing along the way.',
             'thumbnail': 'http://ccc.crackle.com/images/s5e4_thumb.jpg',
@@ -46,12 +49,18 @@ class ComCarCoffIE(InfoExtractor):
         formats = self._extract_m3u8_formats(
             video_data['mediaUrl'], video_id, ext='mp4')
 
+        timestamp = int_or_none(video_data.get('pubDateTime')) or parse_iso8601(
+            video_data.get('pubDate'))
+        duration = int_or_none(video_data.get('durationSeconds')) or parse_duration(
+            video_data.get('duration'))
+
         return {
             'id': video_id,
             'display_id': display_id,
             'title': video_data['title'],
             'description': video_data.get('description'),
-            'timestamp': parse_iso8601(video_data.get('pubDate')),
+            'timestamp': timestamp,
+            'duration': duration,
             'thumbnails': thumbnails,
             'formats': formats,
             'webpage_url': 'http://comediansincarsgettingcoffee.com/%s' % (video_data.get('urlSlug', video_data.get('slug'))),

From 89abf7bf4d5dfc8c161924067f4430b7d81a8b32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Dec 2015 02:09:50 +0600
Subject: [PATCH 0146/1105] [periscope] Fix token based extraction (Closes
 #7943)

---
 youtube_dl/extractor/periscope.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 63cc764bb..514e9b433 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -31,9 +31,8 @@ class PeriscopeIE(InfoExtractor):
     }]
 
     def _call_api(self, method, value):
-        attribute = 'token' if len(value) > 13 else 'broadcast_id'
         return self._download_json(
-            'https://api.periscope.tv/api/v2/%s?%s=%s' % (method, attribute, value), value)
+            'https://api.periscope.tv/api/v2/%s?broadcast_id=%s' % (method, value), value)
 
     def _real_extract(self, url):
         token = self._match_id(url)

From 3a70ed9ebeac782b922cc3cb3b74cd999e60845a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Dec 2015 02:54:32 +0600
Subject: [PATCH 0147/1105] [daum] Fix extraction (Closes #7949)

---
 youtube_dl/extractor/daum.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index 934da765e..e3fc639b0 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -37,9 +37,11 @@ class DaumIE(InfoExtractor):
         video_id = mobj.group('id')
         canonical_url = 'http://tvpot.daum.net/v/%s' % video_id
         webpage = self._download_webpage(canonical_url, video_id)
+        og_url = self._og_search_url(webpage, default=None) or self._search_regex(
+            r'<link[^>]+rel=(["\'])canonical\1[^>]+href=(["\'])(?P<url>.+?)\2',
+            webpage, 'canonical url', group='url')
         full_id = self._search_regex(
-            r'src=["\']http://videofarm\.daum\.net/controller/video/viewer/Video\.html\?.*?vid=(.+?)[&"\']',
-            webpage, 'full id')
+            r'tvpot\.daum\.net/v/([^/]+)', og_url, 'full id')
         query = compat_urllib_parse.urlencode({'vid': full_id})
         info = self._download_xml(
             'http://tvpot.daum.net/clip/ClipInfoXml.do?' + query, video_id,

From 178b47e6af73521da50e9eec04af7fdceb236e39 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Dec 2015 02:59:49 +0600
Subject: [PATCH 0148/1105] [daum] Add test for #7949

---
 youtube_dl/extractor/daum.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index e3fc639b0..9a94cf361 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -24,6 +24,18 @@ class DaumIE(InfoExtractor):
             'upload_date': '20130831',
             'duration': 3868,
         },
+    }, {
+        # Test for https://github.com/rg3/youtube-dl/issues/7949
+        'url': 'http://tvpot.daum.net/mypot/View.do?ownerid=M1O35s8HPOo0&clipid=73147290',
+        'md5': 'c92d78bcee4424451f1667f275c1dc97',
+        'info_dict': {
+            'id': '73147290',
+            'ext': 'mp4',
+            'title': '싸이 - 나팔바지 [유희열의 스케치북] 299회 20151218',
+            'description': '싸이 - 나팔바지',
+            'upload_date': '20151219',
+            'duration': 232,
+        },
     }, {
         'url': 'http://tvpot.daum.net/v/vab4dyeDBysyBssyukBUjBz',
         'only_matching': True,

From 60427f63d129919d5c98e1176ea8136d0eedd0f4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 23 Dec 2015 10:40:45 +0100
Subject: [PATCH 0149/1105] [appletrailers] Add support for AppleTrailers
 Section

---
 youtube_dl/extractor/__init__.py      |  5 +-
 youtube_dl/extractor/appletrailers.py | 74 +++++++++++++++++++++++++++
 2 files changed, 78 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fede5ff0d..3b541a538 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -25,7 +25,10 @@ from .aol import AolIE
 from .allocine import AllocineIE
 from .aparat import AparatIE
 from .appleconnect import AppleConnectIE
-from .appletrailers import AppleTrailersIE
+from .appletrailers import (
+    AppleTrailersIE,
+    AppleTrailersSectionIE,
+)
 from .archiveorg import ArchiveOrgIE
 from .ard import (
     ARDIE,
diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index f68dc3236..ca9a70924 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -11,6 +11,7 @@ from ..utils import (
 
 
 class AppleTrailersIE(InfoExtractor):
+    IE_NAME = 'appletrailers'
     _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/(?:trailers|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'
     _TESTS = [{
         'url': 'http://trailers.apple.com/trailers/wb/manofsteel/',
@@ -144,3 +145,76 @@ class AppleTrailersIE(InfoExtractor):
             'id': movie,
             'entries': playlist,
         }
+
+
+class AppleTrailersSectionIE(InfoExtractor):
+    IE_NAME = 'appletrailers:section'
+    _SECTIONS = {
+        'justadded': {
+            'feed_path': 'just_added',
+            'title': 'Just Added',
+        },
+        'exclusive': {
+            'feed_path': 'exclusive',
+            'title': 'Exclusive',
+        },
+        'justhd': {
+            'feed_path': 'just_hd',
+            'title': 'Just HD',
+        },
+        'mostpopular': {
+            'feed_path': 'most_pop',
+            'title': 'Most Popular',
+        },
+        'moviestudios': {
+            'feed_path': 'studios',
+            'title': 'Movie Studios',
+        },
+    }
+    _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/#section=(?P<id>%s)' % '|'.join(_SECTIONS)
+    _TESTS = [{
+        'url': 'http://trailers.apple.com/#section=justadded',
+        'info_dict': {
+            'title': 'Just Added',
+            'id': 'justadded',
+        },
+        'playlist_mincount': 80,
+    }, {
+        'url': 'http://trailers.apple.com/#section=exclusive',
+        'info_dict': {
+            'title': 'Exclusive',
+            'id': 'exclusive',
+        },
+        'playlist_mincount': 80,
+    }, {
+        'url': 'http://trailers.apple.com/#section=justhd',
+        'info_dict': {
+            'title': 'Just HD',
+            'id': 'justhd',
+        },
+        'playlist_mincount': 80,
+    }, {
+        'url': 'http://trailers.apple.com/#section=mostpopular',
+        'info_dict': {
+            'title': 'Most Popular',
+            'id': 'mostpopular',
+        },
+        'playlist_mincount': 80,
+    }, {
+        'url': 'http://trailers.apple.com/#section=moviestudios',
+        'info_dict': {
+            'title': 'Movie Studios',
+            'id': 'moviestudios',
+        },
+        'playlist_mincount': 80,
+    }]
+
+    def _real_extract(self, url):
+        section = self._match_id(url)
+        section_data = self._download_json(
+            'http://trailers.apple.com/trailers/home/feeds/%s.json' % self._SECTIONS[section]['feed_path'],
+            section)
+        entries = [
+            self.url_result('http://trailers.apple.com' + e['location'])
+            for e in section_data]
+        return self.playlist_result(entries, section, self._SECTIONS[section]['title'])

From f10c27b8cb35c72b5f9633956f71f96da72ada31 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 23 Dec 2015 14:05:06 +0100
Subject: [PATCH 0150/1105] release 2015.12.23

---
 docs/supportedsites.md | 8 +++-----
 youtube_dl/version.py  | 2 +-
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 299bc5e72..1a5c7cde9 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -23,7 +23,6 @@
  - **AdobeTVShow**
  - **AdobeTVVideo**
  - **AdultSwim**
- - **Aftenposten**
  - **Aftonbladet**
  - **AirMozilla**
  - **AlJazeera**
@@ -34,7 +33,8 @@
  - **Aparat**
  - **AppleConnect**
  - **AppleDaily**: 臺灣蘋果日報
- - **AppleTrailers**
+ - **appletrailers**
+ - **appletrailers:section**
  - **archive.org**: archive.org videos
  - **ARD**
  - **ARD:mediathek**
@@ -502,8 +502,6 @@
  - **SnagFilmsEmbed**
  - **Snotr**
  - **Sohu**
- - **soompi**
- - **soompi:show**
  - **soundcloud**
  - **soundcloud:playlist**
  - **soundcloud:search**: Soundcloud search
@@ -627,7 +625,7 @@
  - **Vessel**
  - **Vesti**: Вести.Ru
  - **Vevo**
- - **VGTV**: VGTV and BTTV
+ - **VGTV**: VGTV, BTTV, FTV, Aftenposten and Aftonbladet
  - **vh1.com**
  - **Vice**
  - **Viddler**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7095033c5..255d64269 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.12.21'
+__version__ = '2015.12.23'

From 7fe37d8a05609229332d5a156cb9b7cf4bba2790 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 23 Dec 2015 14:48:40 +0100
Subject: [PATCH 0151/1105] [appletrailers] Improve regex for fixing '<img>'
 tags (#7953)

---
 youtube_dl/extractor/appletrailers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index ca9a70924..82beed2ce 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -80,7 +80,7 @@ class AppleTrailersIE(InfoExtractor):
 
         def fix_html(s):
             s = re.sub(r'(?s)<script[^<]*?>.*?</script>', '', s)
-            s = re.sub(r'<img ([^<]*?)>', r'<img \1/>', s)
+            s = re.sub(r'<img ([^<]*?)/?>', r'<img \1/>', s)
             # The ' in the onClick attributes are not escaped, it couldn't be parsed
             # like: http://trailers.apple.com/trailers/wb/gravity/
 

From 747b028412828c66080c7f165b461a7ea490fead Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Dec 2015 20:42:36 +0600
Subject: [PATCH 0152/1105] [24video] Fix extraction (Closes #7956)

---
 youtube_dl/extractor/twentyfourvideo.py | 35 +++++++++----------------
 1 file changed, 12 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py
index c1ee1decc..cb9e5f1b5 100644
--- a/youtube_dl/extractor/twentyfourvideo.py
+++ b/youtube_dl/extractor/twentyfourvideo.py
@@ -64,33 +64,22 @@ class TwentyFourVideoIE(InfoExtractor):
             r'<div class="comments-title" id="comments-count">(\d+) комментари',
             webpage, 'comment count', fatal=False))
 
-        formats = []
+        # Sets some cookies
+        self._download_xml(
+            r'http://www.24video.net/video/xml/%s?mode=init' % video_id,
+            video_id, 'Downloading init XML')
 
-        pc_video = self._download_xml(
+        video = self._download_xml(
             'http://www.24video.net/video/xml/%s?mode=play' % video_id,
-            video_id, 'Downloading PC video URL').find('.//video')
+            video_id, 'Downloading video XML').find('.//video')
 
-        formats.append({
-            'url': pc_video.attrib['url'],
-            'format_id': 'pc',
-            'quality': 1,
-        })
+        formats = [{
+            'url': video.attrib['url'],
+        }]
 
-        like_count = int_or_none(pc_video.get('ratingPlus'))
-        dislike_count = int_or_none(pc_video.get('ratingMinus'))
-        age_limit = 18 if pc_video.get('adult') == 'true' else 0
-
-        mobile_video = self._download_xml(
-            'http://www.24video.net/video/xml/%s' % video_id,
-            video_id, 'Downloading mobile video URL').find('.//video')
-
-        formats.append({
-            'url': mobile_video.attrib['url'],
-            'format_id': 'mobile',
-            'quality': 0,
-        })
-
-        self._sort_formats(formats)
+        like_count = int_or_none(video.get('ratingPlus'))
+        dislike_count = int_or_none(video.get('ratingMinus'))
+        age_limit = 18 if video.get('adult') == 'true' else 0
 
         return {
             'id': video_id,

From 128eb31d90583113083ba1fe329eb4cf42c2989f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Dec 2015 20:49:41 +0600
Subject: [PATCH 0153/1105] [24video] Fix extraction on python 2.6

---
 youtube_dl/extractor/twentyfourvideo.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py
index cb9e5f1b5..68e2277a4 100644
--- a/youtube_dl/extractor/twentyfourvideo.py
+++ b/youtube_dl/extractor/twentyfourvideo.py
@@ -5,6 +5,8 @@ from .common import InfoExtractor
 from ..utils import (
     parse_iso8601,
     int_or_none,
+    xpath_attr,
+    xpath_element,
 )
 
 
@@ -69,12 +71,14 @@ class TwentyFourVideoIE(InfoExtractor):
             r'http://www.24video.net/video/xml/%s?mode=init' % video_id,
             video_id, 'Downloading init XML')
 
-        video = self._download_xml(
+        video_xml = self._download_xml(
             'http://www.24video.net/video/xml/%s?mode=play' % video_id,
-            video_id, 'Downloading video XML').find('.//video')
+            video_id, 'Downloading video XML')
+
+        video = xpath_element(video_xml, './/video', 'video', fatal=True)
 
         formats = [{
-            'url': video.attrib['url'],
+            'url': xpath_attr(video, '', 'url', 'video URL', fatal=True),
         }]
 
         like_count = int_or_none(video.get('ratingPlus'))

From be514c856cf9f95fe3e0d45b1df0319a0872b911 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Dec 2015 20:49:52 +0600
Subject: [PATCH 0154/1105] [24video] Fix test

---
 youtube_dl/extractor/twentyfourvideo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py
index 68e2277a4..e03e2dbaa 100644
--- a/youtube_dl/extractor/twentyfourvideo.py
+++ b/youtube_dl/extractor/twentyfourvideo.py
@@ -17,7 +17,7 @@ class TwentyFourVideoIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://www.24video.net/video/view/1044982',
-            'md5': 'd041af8b5b4246ea466226a0d6693345',
+            'md5': 'e09fc0901d9eaeedac872f154931deeb',
             'info_dict': {
                 'id': '1044982',
                 'ext': 'mp4',

From dcdc352371115007028632da6ae377d0ad39b62a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Dec 2015 21:13:31 +0600
Subject: [PATCH 0155/1105] [instagram:user] Improve _VALID_URL (Closes #7955)

---
 youtube_dl/extractor/instagram.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index c158f2064..e5e16ca3b 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -47,7 +47,7 @@ class InstagramIE(InfoExtractor):
 
 
 class InstagramUserIE(InfoExtractor):
-    _VALID_URL = r'https://instagram\.com/(?P<username>[^/]{2,})/?(?:$|[?#])'
+    _VALID_URL = r'https?://(?:www\.)?instagram\.com/(?P<username>[^/]{2,})/?(?:$|[?#])'
     IE_DESC = 'Instagram user profile'
     IE_NAME = 'instagram:user'
     _TEST = {

From 261b4c23c70c7c5dc4fe9fd22cf9e867b7456c40 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 23 Dec 2015 17:48:37 +0100
Subject: [PATCH 0156/1105] [appletrailers] skip clips with empty url

---
 youtube_dl/extractor/appletrailers.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 82beed2ce..62ed0c918 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -64,6 +64,12 @@ class AppleTrailersIE(InfoExtractor):
                 },
             },
         ]
+    }, {
+        'url': 'http://trailers.apple.com/trailers/magnolia/blackthorn/',
+        'info_dict': {
+            'id': 'blackthorn',
+        },
+        'playlist_mincount': 2,
     }, {
         'url': 'http://trailers.apple.com/ca/metropole/autrui/',
         'only_matching': True,
@@ -97,6 +103,9 @@ class AppleTrailersIE(InfoExtractor):
             trailer_info_json = self._search_regex(self._JSON_RE,
                                                    on_click, 'trailer info')
             trailer_info = json.loads(trailer_info_json)
+            first_url = trailer_info.get('url')
+            if not first_url:
+                continue
             title = trailer_info['title']
             video_id = movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', title).lower()
             thumbnail = li.find('.//img').attrib['src']
@@ -108,7 +117,6 @@ class AppleTrailersIE(InfoExtractor):
             if m:
                 duration = 60 * int(m.group('minutes')) + int(m.group('seconds'))
 
-            first_url = trailer_info['url']
             trailer_id = first_url.split('/')[-1].rpartition('_')[0].lower()
             settings_json_url = compat_urlparse.urljoin(url, 'includes/settings/%s.json' % trailer_id)
             settings = self._download_json(settings_json_url, trailer_id, 'Downloading settings json')

From a8f1d167f6741485b7cbec2cb355315c3774d5bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 23 Dec 2015 17:55:58 +0100
Subject: [PATCH 0157/1105] [arte] Prefer json URLs that contain the video id
 from the 'vid' parameter in the URL (fixes #7920)

---
 youtube_dl/extractor/arte.py | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 2a00da3ee..10301a8ea 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -68,9 +68,13 @@ class ArteTVPlus7IE(InfoExtractor):
     def _extract_url_info(cls, url):
         mobj = re.match(cls._VALID_URL, url)
         lang = mobj.group('lang')
-        # This is not a real id, it can be for example AJT for the news
-        # http://www.arte.tv/guide/fr/emissions/AJT/arte-journal
-        video_id = mobj.group('id')
+        query = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
+        if 'vid' in query:
+            video_id = query['vid'][0]
+        else:
+            # This is not a real id, it can be for example AJT for the news
+            # http://www.arte.tv/guide/fr/emissions/AJT/arte-journal
+            video_id = mobj.group('id')
         return video_id, lang
 
     def _real_extract(self, url):
@@ -79,9 +83,15 @@ class ArteTVPlus7IE(InfoExtractor):
         return self._extract_from_webpage(webpage, video_id, lang)
 
     def _extract_from_webpage(self, webpage, video_id, lang):
+        patterns_templates = (r'arte_vp_url=["\'](.*?%s.*?)["\']', r'data-url=["\']([^"]+%s[^"]+)["\']')
+        ids = (video_id, '')
+        # some pages contain multiple videos (like
+        # http://www.arte.tv/guide/de/sendungen/XEN/xenius/?vid=055918-015_PLUS7-D),
+        # so we first try to look for json URLs that contain the video id from
+        # the 'vid' parameter.
+        patterns = [t % re.escape(_id) for _id in ids for t in patterns_templates]
         json_url = self._html_search_regex(
-            [r'arte_vp_url=["\'](.*?)["\']', r'data-url=["\']([^"]+)["\']'],
-            webpage, 'json vp url', default=None)
+            patterns, webpage, 'json vp url', default=None)
         if not json_url:
             iframe_url = self._html_search_regex(
                 r'<iframe[^>]+src=(["\'])(?P<url>.+\bjson_url=.+?)\1',

From 2c566d02febb0cf137a8dce8646957beb1415770 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Dec 2015 23:22:47 +0600
Subject: [PATCH 0158/1105] [pbs] Extend PBS station regex (Closes #7964)

---
 youtube_dl/extractor/pbs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 744e4a09a..97e8ffc97 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 class PBSIE(InfoExtractor):
     _STATIONS = (
-        (r'(?:video|www)\.pbs\.org', 'PBS: Public Broadcasting Service'),  # http://www.pbs.org/
+        (r'(?:video|www|player)\.pbs\.org', 'PBS: Public Broadcasting Service'),  # http://www.pbs.org/
         (r'video\.aptv\.org', 'APT - Alabama Public Television (WBIQ)'),  # http://aptv.org/
         (r'video\.gpb\.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # http://www.gpb.org/
         (r'video\.mpbonline\.org', 'Mississippi Public Broadcasting (WMPN)'),  # http://www.mpbonline.org

From fc383f199e73358f88ecf24b7e804dda7400afae Mon Sep 17 00:00:00 2001
From: Jakub Wilk <jwilk@jwilk.net>
Date: Wed, 23 Dec 2015 17:35:10 +0100
Subject: [PATCH 0159/1105] Fix typos

---
 CONTRIBUTING.md | 2 +-
 README.md       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index f3fe0d432..d15267d7e 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -28,7 +28,7 @@ So please elaborate on what feature you are requesting, or what bug you want to
 - How it could be fixed
 - How your proposed solution would look like
 
-If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a commiter myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over.
+If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a committer myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over.
 
 For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the `-v` flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
 
diff --git a/README.md b/README.md
index 7002f45e0..3a4707227 100644
--- a/README.md
+++ b/README.md
@@ -830,7 +830,7 @@ So please elaborate on what feature you are requesting, or what bug you want to
 - How it could be fixed
 - How your proposed solution would look like
 
-If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a commiter myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over.
+If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a committer myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over.
 
 For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the `-v` flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
 

From 96db61ffb83de9d912003a4778e9ce7c4d46e848 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 23 Dec 2015 22:36:53 +0100
Subject: [PATCH 0160/1105] [theintercept] improve extraction

---
 youtube_dl/extractor/theintercept.py | 67 ++++++++++------------------
 1 file changed, 24 insertions(+), 43 deletions(-)

diff --git a/youtube_dl/extractor/theintercept.py b/youtube_dl/extractor/theintercept.py
index b096a28de..8cb3c3669 100644
--- a/youtube_dl/extractor/theintercept.py
+++ b/youtube_dl/extractor/theintercept.py
@@ -1,24 +1,28 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-
+from ..compat import compat_str
 from ..utils import (
+    parse_iso8601,
+    int_or_none,
     ExtractorError,
 )
 
+
 class TheInterceptIE(InfoExtractor):
-    _VALID_URL = r'https://theintercept.com/fieldofvision/(?P<id>.+?)/'
+    _VALID_URL = r'https://theintercept.com/fieldofvision/(?P<id>[^/?#]+)'
     _TESTS = [{
         'url': 'https://theintercept.com/fieldofvision/thisisacoup-episode-four-surrender-or-die/',
+        'md5': '145f28b41d44aab2f87c0a4ac8ec95bd',
         'info_dict': {
-            'id': 'thisisacoup-episode-four-surrender-or-die',
+            'id': '46214',
             'ext': 'mp4',
             'title': '#ThisIsACoup – Episode Four: Surrender or Die',
-            'upload_date': '20151218',
             'description': 'md5:74dd27f0e2fbd50817829f97eaa33140',
+            'timestamp': 1450429239,
+            'upload_date': '20151218',
+            'comment_count': int,
         }
     }]
 
@@ -26,43 +30,20 @@ class TheInterceptIE(InfoExtractor):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
 
-        mobj = re.search(r'initialStoreTree =(?P<json_data>.+})', webpage)
-        if mobj is None:
-            raise ExtractorError('Unable to extract initialStoreTree')
-        json_data = self._parse_json(mobj.group('json_data'), display_id)
+        json_data = self._parse_json(self._search_regex(
+            r'initialStoreTree\s*=\s*(?P<json_data>{.+})', webpage,
+            'initialStoreTree'), display_id)
 
-        info = None
         for post in json_data['resources']['posts'].values():
             if post['slug'] == display_id:
-                info = post
-                break
-        if info is None:
-            raise ExtractorError('Unable to find info for %s'%display_id)
-
-        title = info['title']
-        description = info['excerpt']
-        upload_date = info['date'][:10].replace('-', '')
-        video_id = info['fov_videoid']
-        creator = ','.join([a['display_name'] for a in info['authors']])
-        thumbnail = self._og_search_property('image', webpage)
-        content_id = thumbnail.split('/')[-1].split('.')[0]
-        content_url = 'https://content.jwplatform.com/jw6/{content_id}.xml'.format(content_id=content_id)
-        content = self._download_xml(content_url, video_id)
-
-        formats = []
-        for source in content.findall('.//{http://rss.jwpcdn.com/}source'):
-            if source.attrib['file'].endswith('.m3u8'):
-                formats.extend(self._extract_m3u8_formats(
-                    source.attrib['file'], video_id, 'mp4', preference=1, m3u8_id='hls'))
-
-        return {
-            'creator': creator,
-            'description': description,
-            'display_id': display_id,
-            'formats': formats,
-            'id': video_id,
-            'id': video_id,
-            'thumbnail': thumbnail,
-            'title': title,
-            'upload_date': upload_date,
-        }
+                return {
+                    '_type': 'url_transparent',
+                    'url': 'jwplatform:%s' % post['fov_videoid'],
+                    'id': compat_str(post['ID']),
+                    'display_id': display_id,
+                    'title': post['title'],
+                    'description': post.get('excerpt'),
+                    'timestamp': parse_iso8601(post.get('date')),
+                    'comment_count': int_or_none(post.get('comments_number')),
+                }
+        raise ExtractorError('Unable to find the current post')

From 8a609c32fdf8d99f7c868c74d64a05d9a936044d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Dec 2015 20:09:48 +0600
Subject: [PATCH 0161/1105] [chaturbate] Improve error extraction (Closes
 #7989)

---
 youtube_dl/extractor/chaturbate.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/chaturbate.py b/youtube_dl/extractor/chaturbate.py
index 0b67ba67d..242fba311 100644
--- a/youtube_dl/extractor/chaturbate.py
+++ b/youtube_dl/extractor/chaturbate.py
@@ -23,6 +23,8 @@ class ChaturbateIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    _ROOM_OFFLINE = 'Room is currently offline'
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -34,9 +36,16 @@ class ChaturbateIE(InfoExtractor):
 
         if not m3u8_url:
             error = self._search_regex(
-                r'<span[^>]+class=(["\'])desc_span\1[^>]*>(?P<error>[^<]+)</span>',
-                webpage, 'error', group='error')
-            raise ExtractorError(error, expected=True)
+                [r'<span[^>]+class=(["\'])desc_span\1[^>]*>(?P<error>[^<]+)</span>',
+                 r'<div[^>]+id=(["\'])defchat\1[^>]*>\s*<p><strong>(?P<error>[^<]+)<'],
+                webpage, 'error', group='error', default=None)
+            if not error:
+                if any(p not in webpage for p in (
+                        self._ROOM_OFFLINE, 'offline_tipping', 'tip_offline')):
+                    error = self._ROOM_OFFLINE
+            if error:
+                raise ExtractorError(error, expected=True)
+            raise ExtractorError('Unable to find stream URL')
 
         formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
 

From e41604227ce14ded0b87385a17490bfa65d35fde Mon Sep 17 00:00:00 2001
From: Boris Wachtmeister <boris-code@gmx.com>
Date: Thu, 24 Dec 2015 08:27:25 +0100
Subject: [PATCH 0162/1105] [zdf] expand valid-url pattern for channels

The webpage also creates URLs which include additional text that defines
the sorting order on the page like "aktuellste" (most current) and
"meist-gesehen" (most seen), e.g.:

http://www.zdf.de/ZDFmediathek/kanaluebersicht/aktuellste/332
http://www.zdf.de/ZDFmediathek/kanaluebersicht/meist-gesehen/332
---
 youtube_dl/extractor/zdf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index a795f56b3..b581813d6 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -131,7 +131,7 @@ class ZDFIE(InfoExtractor):
 
 
 class ZDFChannelIE(InfoExtractor):
-    _VALID_URL = r'(?:zdf:topic:|https?://www\.zdf\.de/ZDFmediathek(?:#)?/.*kanaluebersicht/)(?P<id>[0-9]+)'
+    _VALID_URL = r'(?:zdf:topic:|https?://www\.zdf\.de/ZDFmediathek(?:#)?/.*kanaluebersicht/(?:(?:aktuellste|meist-gesehen)/)?)(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.zdf.de/ZDFmediathek#/kanaluebersicht/1586442/sendung/Titanic',
         'info_dict': {

From 67ba388efb5a6fa925ba3472697484553f6ddcc9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Dec 2015 20:42:29 +0600
Subject: [PATCH 0163/1105] [zdf:channel] Relax _VALID_URL

---
 youtube_dl/extractor/zdf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index b581813d6..d852ffd07 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -131,7 +131,7 @@ class ZDFIE(InfoExtractor):
 
 
 class ZDFChannelIE(InfoExtractor):
-    _VALID_URL = r'(?:zdf:topic:|https?://www\.zdf\.de/ZDFmediathek(?:#)?/.*kanaluebersicht/(?:(?:aktuellste|meist-gesehen)/)?)(?P<id>[0-9]+)'
+    _VALID_URL = r'(?:zdf:topic:|https?://www\.zdf\.de/ZDFmediathek(?:#)?/.*kanaluebersicht/(?:[^/]+/)?)(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.zdf.de/ZDFmediathek#/kanaluebersicht/1586442/sendung/Titanic',
         'info_dict': {

From c24044635b6e7f67274899e7c5e7a5efcce947af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Dec 2015 20:44:49 +0600
Subject: [PATCH 0164/1105] [zdf:channel] Add more tests

---
 youtube_dl/extractor/zdf.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index d852ffd07..9a3331a69 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -132,13 +132,22 @@ class ZDFIE(InfoExtractor):
 
 class ZDFChannelIE(InfoExtractor):
     _VALID_URL = r'(?:zdf:topic:|https?://www\.zdf\.de/ZDFmediathek(?:#)?/.*kanaluebersicht/(?:[^/]+/)?)(?P<id>[0-9]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.zdf.de/ZDFmediathek#/kanaluebersicht/1586442/sendung/Titanic',
         'info_dict': {
             'id': '1586442',
         },
         'playlist_count': 3,
-    }
+    }, {
+        'url': 'http://www.zdf.de/ZDFmediathek/kanaluebersicht/aktuellste/332',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.zdf.de/ZDFmediathek/kanaluebersicht/meist-gesehen/332',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.zdf.de/ZDFmediathek/kanaluebersicht/_/1798716?bc=nrt;nrm?flash=off',
+        'only_matching': True,
+    }]
     _PAGE_SIZE = 50
 
     def _fetch_page(self, channel_id, page):

From fb8e402ad252bebc54c36c5acca1b7ef05416c9b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 25 Dec 2015 01:59:56 +0100
Subject: [PATCH 0165/1105] [hotstar] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/hotstar.py  | 79 ++++++++++++++++++++++++++++++++
 2 files changed, 80 insertions(+)
 create mode 100644 youtube_dl/extractor/hotstar.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 365c0b86f..dd7e23d80 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -253,6 +253,7 @@ from .history import HistoryIE
 from .hitbox import HitboxIE, HitboxLiveIE
 from .hornbunny import HornBunnyIE
 from .hotnewhiphop import HotNewHipHopIE
+from .hotstar import HotStarIE
 from .howcast import HowcastIE
 from .howstuffworks import HowStuffWorksIE
 from .huffpost import HuffPostIE
diff --git a/youtube_dl/extractor/hotstar.py b/youtube_dl/extractor/hotstar.py
new file mode 100644
index 000000000..05d27e75d
--- /dev/null
+++ b/youtube_dl/extractor/hotstar.py
@@ -0,0 +1,79 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    determine_ext,
+    int_or_none,
+)
+
+
+class HotStarIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?hotstar\.com/.*?[/-](?P<id>\d{10})'
+    _TEST = {
+        'url': 'http://www.hotstar.com/on-air-with-aib--english-1000076273',
+        'info_dict': {
+            'id': '1000076273',
+            'ext': 'mp4',
+            'title': 'On Air With AIB - English',
+            'description': 'md5:c957d8868e9bc793ccb813691cc4c434',
+            'timestamp': 1447227000,
+            'upload_date': '20151111',
+            'duration': 381,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }
+
+    _GET_CONTENT_TEMPLATE = 'http://account.hotstar.com/AVS/besc?action=GetAggregatedContentDetails&channel=PCTV&contentId=%s'
+    _GET_CDN_TEMPLATE = 'http://getcdn.hotstar.com/AVS/besc?action=GetCDN&asJson=Y&channel=%s&id=%s&type=%s'
+
+    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', fatal=True):
+        json_data = super(HotStarIE, self)._download_json(url_or_request, video_id, note, fatal=fatal)
+        if json_data['resultCode'] != 'OK':
+            if fatal:
+                raise ExtractorError(json_data['errorDescription'])
+            return None
+        return json_data['resultObj']
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_data = self._download_json(
+            self._GET_CONTENT_TEMPLATE % video_id,
+            video_id)['contentInfo'][0]
+
+        formats = []
+        # PCTV for extracting f4m manifest
+        for f in ('TABLET',):
+            format_data = self._download_json(
+                self._GET_CDN_TEMPLATE % (f, video_id, 'VOD'),
+                video_id, 'Downloading %s JSON metadata' % f, fatal=False)
+            if format_data:
+                format_url = format_data['src']
+                ext = determine_ext(format_url)
+                if ext == 'm3u8':
+                    m3u8_formats = self._extract_m3u8_formats(format_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+                    if m3u8_formats:
+                        formats.extend(m3u8_formats)
+                elif ext == 'f4m':
+                    # produce broken files
+                    continue
+                else:
+                    formats.append({
+                        'url': format_url,
+                        'width': int_or_none(format_data.get('width')),
+                        'height': int_or_none(format_data.get('height')),
+                    })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': video_data['episodeTitle'],
+            'description': video_data.get('description'),
+            'duration': int_or_none(video_data.get('duration')),
+            'timestamp': int_or_none(video_data.get('broadcastDate')),
+            'formats': formats,
+        }

From 06d5556dface3901a86419b6b125ef377116448f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 25 Dec 2015 15:38:12 +0100
Subject: [PATCH 0166/1105] [rai] improve extraction

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/rai.py      | 203 ++++++++++++++++++-------------
 2 files changed, 121 insertions(+), 87 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index dd7e23d80..4784db70b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -528,7 +528,10 @@ from .radiode import RadioDeIE
 from .radiojavan import RadioJavanIE
 from .radiobremen import RadioBremenIE
 from .radiofrance import RadioFranceIE
-from .rai import RaiIE
+from .rai import (
+    RaiTVIE,
+    RaiIE,
+)
 from .rbmaradio import RBMARadioIE
 from .rds import RDSIE
 from .redtube import RedTubeIE
diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 7ff1d06c4..14f1ccbb4 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -8,20 +8,24 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
+    ExtractorError,
+    determine_ext,
     parse_duration,
     unified_strdate,
+    int_or_none,
+    xpath_text,
 )
 
 
-class RaiIE(InfoExtractor):
-    _VALID_URL = r'(?P<url>(?P<host>http://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it))/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html)'
+class RaiTVIE(InfoExtractor):
+    _VALID_URL = r'http://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/(?:[^/]+/)+media/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
     _TESTS = [
         {
             'url': 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
-            'md5': 'c064c0b2d09c278fb293116ef5d0a32d',
+            'md5': '96382709b61dd64a6b88e0f791e6df4c',
             'info_dict': {
                 'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'Report del 07/04/2014',
                 'description': 'md5:f27c544694cacb46a078db84ec35d2d9',
                 'upload_date': '20140407',
@@ -30,16 +34,14 @@ class RaiIE(InfoExtractor):
         },
         {
             'url': 'http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html',
-            'md5': '8bb9c151924ce241b74dd52ef29ceafa',
+            'md5': 'd9751b78eac9710d62c2447b224dea39',
             'info_dict': {
                 'id': '04a9f4bd-b563-40cf-82a6-aad3529cb4a9',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'TG PRIMO TEMPO',
-                'description': '',
                 'upload_date': '20140612',
                 'duration': 1758,
             },
-            'skip': 'Error 404',
         },
         {
             'url': 'http://www.rainews.it/dl/rainews/media/state-of-the-net-Antonella-La-Carpia-regole-virali-7aafdea9-0e5d-49d5-88a6-7e65da67ae13.html',
@@ -55,110 +57,106 @@ class RaiIE(InfoExtractor):
         },
         {
             'url': 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-b4a49761-e0cc-4b14-8736-2729f6f73132-tg2.html',
-            'md5': '35694f062977fe6619943f08ed935730',
             'info_dict': {
                 'id': 'b4a49761-e0cc-4b14-8736-2729f6f73132',
                 'ext': 'mp4',
                 'title': 'Alluvione in Sardegna e dissesto idrogeologico',
                 'description': 'Edizione delle ore 20:30 ',
-            }
+            },
+            'skip': 'invalid urls',
         },
         {
             'url': 'http://www.ilcandidato.rai.it/dl/ray/media/Il-Candidato---Primo-episodio-Le-Primarie-28e5525a-b495-45e8-a7c3-bc48ba45d2b6.html',
-            'md5': '02b64456f7cc09f96ff14e7dd489017e',
+            'md5': '496ab63e420574447f70d02578333437',
             'info_dict': {
                 'id': '28e5525a-b495-45e8-a7c3-bc48ba45d2b6',
                 'ext': 'flv',
                 'title': 'Il Candidato - Primo episodio: "Le Primarie"',
-                'description': 'Primo appuntamento con "Il candidato" con Filippo Timi, alias Piero Zucca presidente!',
-                'uploader': 'RaiTre',
+                'description': 'md5:364b604f7db50594678f483353164fb8',
+                'upload_date': '20140923',
+                'duration': 386,
             }
         },
-        {
-            'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html',
-            'md5': '037104d2c14132887e5e4cf114569214',
-            'info_dict': {
-                'id': '0c7a664b-d0f4-4b2c-8835-3f82e46f433e',
-                'ext': 'flv',
-                'title': 'Il pacco',
-                'description': 'md5:4b1afae1364115ce5d78ed83cd2e5b3a',
-                'uploader': 'RaiTre',
-                'upload_date': '20141221',
-            },
-        }
     ]
 
-    def _extract_relinker_url(self, webpage):
-        return self._proto_relative_url(self._search_regex(
-            [r'name="videourl" content="([^"]+)"', r'var\s+videoURL(?:_MP4)?\s*=\s*"([^"]+)"'],
-            webpage, 'relinker url', default=None))
-
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        host = mobj.group('host')
+        video_id = self._match_id(url)
+        media = self._download_json(
+            'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-%s.html?json' % video_id,
+            video_id, 'Downloading video JSON')
 
-        webpage = self._download_webpage(url, video_id)
+        thumbnails = []
+        for image_type in ('image', 'image_medium', 'image_300'):
+            thumbnail_url = media.get(image_type)
+            if thumbnail_url:
+                thumbnails.append({
+                    'url': thumbnail_url,
+                })
 
-        relinker_url = self._extract_relinker_url(webpage)
+        subtitles = []
+        formats = []
+        media_type = media['type']
+        if 'Audio' in media_type:
+            formats.append({
+                'format_id': media.get('formatoAudio'),
+                'url': media['audioUrl'],
+                'ext': media.get('formatoAudio'),
+            })
+        elif 'Video' in media_type:
+            def fix_xml(xml):
+                return xml.replace(' tag elementi', '').replace('>/', '</')
 
-        if not relinker_url:
-            iframe_url = self._search_regex(
-                [r'<iframe[^>]+src="([^"]*/dl/[^"]+\?iframe\b[^"]*)"',
-                 r'drawMediaRaiTV\(["\'](.+?)["\']'],
-                webpage, 'iframe')
-            if not iframe_url.startswith('http'):
-                iframe_url = compat_urlparse.urljoin(url, iframe_url)
-            webpage = self._download_webpage(
-                iframe_url, video_id)
-            relinker_url = self._extract_relinker_url(webpage)
+            relinker = self._download_xml(
+                media['mediaUri'] + '&output=43', video_id, transform_source=fix_xml)
 
-        relinker = self._download_json(
-            '%s&output=47' % relinker_url, video_id)
+            has_subtitle = False
 
-        media_url = relinker['video'][0]
-        ct = relinker.get('ct')
-        if ct == 'f4m':
-            formats = self._extract_f4m_formats(
-                media_url + '&hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id)
+            for element in relinker.findall('element'):
+                media_url = xpath_text(element, 'url')
+                ext = determine_ext(media_url)
+                content_type = xpath_text(element, 'content-type')
+                if ext == 'm3u8':
+                    m3u8_formats = self._extract_m3u8_formats(
+                        media_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls',
+                        fatal=None)
+                    if m3u8_formats:
+                        formats.extend(m3u8_formats)
+                elif ext == 'f4m':
+                    f4m_formats = self._extract_f4m_formats(
+                        media_url + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44',
+                        video_id, f4m_id='hds', fatal=None)
+                    if f4m_formats:
+                        formats.extend(f4m_formats)
+                elif ext == 'stl':
+                    has_subtitle = True
+                elif content_type.startswith('video/'):
+                    bitrate = int_or_none(xpath_text(element, 'bitrate'))
+                    formats.append({
+                        'url': media_url,
+                        'tbr': bitrate if bitrate > 0 else None,
+                        'format_id': 'http-%d' % bitrate if bitrate > 0 else 'http',
+                    })
+                elif content_type.startswith('image/'):
+                    thumbnails.append({
+                        'url': media_url,
+                    })
+
+            self._sort_formats(formats)
+
+            if has_subtitle:
+                webpage = self._download_webpage(url, video_id)
+                subtitles = self._get_subtitles(video_id, webpage)
         else:
-            formats = [{
-                'url': media_url,
-                'format_id': ct,
-            }]
-
-        json_link = self._html_search_meta(
-            'jsonlink', webpage, 'JSON link', default=None)
-        if json_link:
-            media = self._download_json(
-                host + json_link, video_id, 'Downloading video JSON')
-            title = media.get('name')
-            description = media.get('desc')
-            thumbnail = media.get('image_300') or media.get('image_medium') or media.get('image')
-            duration = parse_duration(media.get('length'))
-            uploader = media.get('author')
-            upload_date = unified_strdate(media.get('date'))
-        else:
-            title = (self._search_regex(
-                r'var\s+videoTitolo\s*=\s*"(.+?)";',
-                webpage, 'title', default=None) or self._og_search_title(webpage)).replace('\\"', '"')
-            description = self._og_search_description(webpage)
-            thumbnail = self._og_search_thumbnail(webpage)
-            duration = None
-            uploader = self._html_search_meta('Editore', webpage, 'uploader')
-            upload_date = unified_strdate(self._html_search_meta(
-                'item-date', webpage, 'upload date', default=None))
-
-        subtitles = self.extract_subtitles(video_id, webpage)
+            raise ExtractorError('not a media file')
 
         return {
             'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'uploader': uploader,
-            'upload_date': upload_date,
-            'duration': duration,
+            'title': media['name'],
+            'description': media.get('desc'),
+            'thumbnails': thumbnails,
+            'uploader': media.get('author'),
+            'upload_date': unified_strdate(media.get('date')),
+            'duration': parse_duration(media.get('length')),
             'formats': formats,
             'subtitles': subtitles,
         }
@@ -177,3 +175,36 @@ class RaiIE(InfoExtractor):
                 'url': 'http://www.rai.tv%s' % compat_urllib_parse.quote(captions),
             }]
         return subtitles
+
+
+class RaiIE(InfoExtractor):
+    _VALID_URL = r'http://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
+    _TESTS = [
+        {
+            'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html',
+            'md5': 'e0e7a8a131e249d1aa0ebf270d1d8db7',
+            'info_dict': {
+                'id': '59d69d28-6bb6-409d-a4b5-ed44096560af',
+                'ext': 'flv',
+                'title': 'Il pacco',
+                'description': 'md5:4b1afae1364115ce5d78ed83cd2e5b3a',
+                'upload_date': '20141221',
+            },
+        }
+    ]
+
+    @classmethod
+    def suitable(cls, url):
+        return False if RaiTVIE.suitable(url) else super(RaiIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        iframe_url = self._search_regex(
+            [r'<iframe[^>]+src="([^"]*/dl/[^"]+\?iframe\b[^"]*)"',
+             r'drawMediaRaiTV\(["\'](.+?)["\']'],
+            webpage, 'iframe')
+        if not iframe_url.startswith('http'):
+            iframe_url = compat_urlparse.urljoin(url, iframe_url)
+        return self.url_result(iframe_url)

From 6418b2439b81ceec77b50879b4d9d395893d8eba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Dec 2015 21:14:00 +0600
Subject: [PATCH 0167/1105] [rutv] Fix extraction (Closes #8004)

---
 youtube_dl/extractor/rutv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index d9df06861..f7fe1fece 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -131,7 +131,7 @@ class RUTVIE(InfoExtractor):
         is_live = video_type == 'live'
 
         json_data = self._download_json(
-            'http://player.rutv.ru/iframe/%splay/id/%s' % ('live-' if is_live else '', video_id),
+            'http://player.rutv.ru/iframe/data%s/id/%s' % ('live' if is_live else 'video', video_id),
             video_id, 'Downloading JSON')
 
         if json_data['errors']:

From 1fc0b47fdf9367aa71e6b81076666f137e68f637 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 25 Dec 2015 17:37:50 +0100
Subject: [PATCH 0168/1105] [srmediathek] improve extraction

---
 youtube_dl/extractor/ard.py         | 14 ++++++---
 youtube_dl/extractor/srmediathek.py | 49 ++++++++++++++++++-----------
 2 files changed, 41 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 73be6d204..687eb9f82 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -110,13 +110,19 @@ class ARDMediathekIE(InfoExtractor):
                 server = stream.get('_server')
                 for stream_url in stream_urls:
                     ext = determine_ext(stream_url)
+                    if quality != 'auto' and ext in ('f4m', 'm3u8'):
+                        continue
                     if ext == 'f4m':
-                        formats.extend(self._extract_f4m_formats(
+                        f4m_formats = self._extract_f4m_formats(
                             stream_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124',
-                            video_id, preference=-1, f4m_id='hds'))
+                            video_id, preference=-1, f4m_id='hds', fatal=False)
+                        if f4m_formats:
+                            formats.extend(f4m_formats)
                     elif ext == 'm3u8':
-                        formats.extend(self._extract_m3u8_formats(
-                            stream_url, video_id, 'mp4', preference=1, m3u8_id='hls'))
+                        m3u8_formats = self._extract_m3u8_formats(
+                            stream_url, video_id, 'mp4', preference=1, m3u8_id='hls', fatal=False)
+                        if m3u8_formats:
+                            formats.extend(m3u8_formats)
                     else:
                         if server and server.startswith('rtmp'):
                             f = {
diff --git a/youtube_dl/extractor/srmediathek.py b/youtube_dl/extractor/srmediathek.py
index 5d583c720..74d01183f 100644
--- a/youtube_dl/extractor/srmediathek.py
+++ b/youtube_dl/extractor/srmediathek.py
@@ -1,17 +1,18 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import json
-
-from .common import InfoExtractor
-from ..utils import js_to_json
+from .ard import ARDMediathekIE
+from ..utils import (
+    ExtractorError,
+    get_element_by_attribute,
+)
 
 
-class SRMediathekIE(InfoExtractor):
+class SRMediathekIE(ARDMediathekIE):
     IE_DESC = 'Saarländischer Rundfunk'
     _VALID_URL = r'https?://sr-mediathek\.sr-online\.de/index\.php\?.*?&id=(?P<id>[0-9]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://sr-mediathek.sr-online.de/index.php?seite=7&id=28455',
         'info_dict': {
             'id': '28455',
@@ -20,24 +21,36 @@ class SRMediathekIE(InfoExtractor):
             'description': 'Ringen: KSV Köllerbach gegen Aachen-Walheim; Frauen-Fußball: 1. FC Saarbrücken gegen Sindelfingen; Motorsport: Rallye in Losheim; dazu: Interview mit Timo Bernhard; Turnen: TG Saar; Reitsport: Deutscher Voltigier-Pokal; Badminton: Interview mit Michael Fuchs ',
             'thumbnail': 're:^https?://.*\.jpg$',
         },
-    }
+        'skip': 'no longer available',
+    }, {
+        'url': 'http://sr-mediathek.sr-online.de/index.php?seite=7&id=37682',
+        'info_dict': {
+            'id': '37682',
+            'ext': 'mp4',
+            'title': 'Love, Cakes and Rock\'n\'Roll',
+            'description': 'md5:18bf9763631c7d326c22603681e1123d',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+        'expected_warnings': ['Unable to download f4m manifest']
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        murls = json.loads(js_to_json(self._search_regex(
-            r'var mediaURLs\s*=\s*(.*?);\n', webpage, 'video URLs')))
-        formats = [{'url': murl} for murl in murls]
-        self._sort_formats(formats)
+        if '>Der gew&uuml;nschte Beitrag ist leider nicht mehr verf&uuml;gbar.<' in webpage:
+            raise ExtractorError('Video %s is no longer available' % video_id, expected=True)
 
-        title = json.loads(js_to_json(self._search_regex(
-            r'var mediaTitles\s*=\s*(.*?);\n', webpage, 'title')))[0]
-
-        return {
+        media_collection_url = self._search_regex(
+            r'data-mediacollection-ardplayer="([^"]+)"', webpage, 'media collection url')
+        info = self._extract_media_info(media_collection_url, webpage, video_id)
+        info.update({
             'id': video_id,
-            'title': title,
-            'formats': formats,
+            'title': get_element_by_attribute('class', 'ardplayer-title', webpage),
             'description': self._og_search_description(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
-        }
+        })
+        return info

From 607d65fbceebe7df6d45bb02d6eab378896f099b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Dec 2015 03:17:56 +0600
Subject: [PATCH 0169/1105] [ign] flake8

---
 youtube_dl/extractor/ign.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index 6160f03d1..a2e18c8a7 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -109,7 +109,7 @@ class IGNIE(InfoExtractor):
         if page_type != 'video':
             multiple_urls = re.findall(
                 r'<param name="flashvars"[^>]*value="[^"]*?url=(https?://www\.ign\.com/videos/.*?)["&]',
-                 webpage)
+                webpage)
             if multiple_urls:
                 entries = [self.url_result(u, ie='IGN') for u in multiple_urls]
                 return {
@@ -209,7 +209,7 @@ class PCMagIE(IGNIE):
             'upload_date': '20150106',
             'uploader_id': 'cozzipix@gmail.com',
         }
-    },{
+    }, {
         'url': 'http://www.pcmag.com/article2/0,2817,2470156,00.asp',
         'md5': '94130c1ca07ba0adb6088350681f16c1',
         'info_dict': {

From 85367c3a478241a9d47be76ff5418b512cd40eed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?=
 <giedrius.statkevicius@gmail.com>
Date: Sat, 28 Nov 2015 23:14:38 +0200
Subject: [PATCH 0170/1105] [lrt] fix duration parsing

---
 youtube_dl/extractor/lrt.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lrt.py b/youtube_dl/extractor/lrt.py
index e3236f7b5..24a9a4c91 100644
--- a/youtube_dl/extractor/lrt.py
+++ b/youtube_dl/extractor/lrt.py
@@ -37,8 +37,7 @@ class LRTIE(InfoExtractor):
         thumbnail = self._og_search_thumbnail(webpage)
         description = self._og_search_description(webpage)
         duration = parse_duration(self._search_regex(
-            r"'duration':\s*'([^']+)',", webpage,
-            'duration', fatal=False, default=None))
+            r"var record_len = '([0-9]+:[0-9]+:[0-9]+)';", webpage, 'record_len', fatal=False, default=None))
 
         formats = []
         for js in re.findall(r'(?s)config:\s*(\{.*?\})', webpage):

From 339b1944e7c8c28d1fc7cc8721af73fd23937a1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?=
 <giedrius.statkevicius@gmail.com>
Date: Sun, 29 Nov 2015 02:58:52 +0200
Subject: [PATCH 0171/1105] [lrt] fix the rest of extractor

Closes #7690.
---
 youtube_dl/extractor/lrt.py | 25 +++----------------------
 1 file changed, 3 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/lrt.py b/youtube_dl/extractor/lrt.py
index 24a9a4c91..42e768ce7 100644
--- a/youtube_dl/extractor/lrt.py
+++ b/youtube_dl/extractor/lrt.py
@@ -1,12 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
-    determine_ext,
-    js_to_json,
     parse_duration,
     remove_end,
 )
@@ -25,7 +21,7 @@ class LRTIE(InfoExtractor):
             'duration': 1783,
         },
         'params': {
-            'skip_download': True,  # HLS download
+            'skip_download': True,  # m3u8 download
         },
     }
 
@@ -39,23 +35,8 @@ class LRTIE(InfoExtractor):
         duration = parse_duration(self._search_regex(
             r"var record_len = '([0-9]+:[0-9]+:[0-9]+)';", webpage, 'record_len', fatal=False, default=None))
 
-        formats = []
-        for js in re.findall(r'(?s)config:\s*(\{.*?\})', webpage):
-            data = self._parse_json(js, video_id, transform_source=js_to_json)
-            if 'provider' not in data:
-                continue
-            if data['provider'] == 'rtmp':
-                formats.append({
-                    'format_id': 'rtmp',
-                    'ext': determine_ext(data['file']),
-                    'url': data['streamer'],
-                    'play_path': 'mp4:%s' % data['file'],
-                    'preference': -1,
-                    'rtmp_real_time': True,
-                })
-            else:
-                formats.extend(
-                    self._extract_m3u8_formats(data['file'], video_id, 'mp4'))
+        link = self._search_regex(r'file: "(.*)" \+ location\.hash\.substring\(1\)', webpage, 'link to m3u8')
+        formats = self._extract_m3u8_formats(link, video_id, "mp4")
 
         return {
             'id': video_id,

From f7e1d82d407ccb0be23999ba8af27ae2549d39aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Dec 2015 12:16:55 +0600
Subject: [PATCH 0172/1105] [lrt] Improve

---
 youtube_dl/extractor/lrt.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/lrt.py b/youtube_dl/extractor/lrt.py
index 42e768ce7..f8d27598a 100644
--- a/youtube_dl/extractor/lrt.py
+++ b/youtube_dl/extractor/lrt.py
@@ -30,13 +30,16 @@ class LRTIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         title = remove_end(self._og_search_title(webpage), ' - LRT')
+        m3u8_url = self._search_regex(
+            r'file\s*:\s*(["\'])(?P<url>.+?)\1\s*\+\s*location\.hash\.substring\(1\)',
+            webpage, 'm3u8 url', group='url')
+        formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4')
+
         thumbnail = self._og_search_thumbnail(webpage)
         description = self._og_search_description(webpage)
         duration = parse_duration(self._search_regex(
-            r"var record_len = '([0-9]+:[0-9]+:[0-9]+)';", webpage, 'record_len', fatal=False, default=None))
-
-        link = self._search_regex(r'file: "(.*)" \+ location\.hash\.substring\(1\)', webpage, 'link to m3u8')
-        formats = self._extract_m3u8_formats(link, video_id, "mp4")
+            r'var\s+record_len\s*=\s*(["\'])(?P<duration>[0-9]+:[0-9]+:[0-9]+)\1',
+            webpage, 'duration', default=None, group='duration'))
 
         return {
             'id': video_id,

From 15aad84dc5ea9d60ffd4a1fd5d67f90afd11b4c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Dec 2015 12:26:48 +0600
Subject: [PATCH 0173/1105] [lrt] Extract counters

---
 youtube_dl/extractor/lrt.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/lrt.py b/youtube_dl/extractor/lrt.py
index f8d27598a..863efd896 100644
--- a/youtube_dl/extractor/lrt.py
+++ b/youtube_dl/extractor/lrt.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    int_or_none,
     parse_duration,
     remove_end,
 )
@@ -19,6 +20,8 @@ class LRTIE(InfoExtractor):
             'title': 'Septynios Kauno dienos',
             'description': 'md5:24d84534c7dc76581e59f5689462411a',
             'duration': 1783,
+            'view_count': int,
+            'like_count': int,
         },
         'params': {
             'skip_download': True,  # m3u8 download
@@ -41,6 +44,13 @@ class LRTIE(InfoExtractor):
             r'var\s+record_len\s*=\s*(["\'])(?P<duration>[0-9]+:[0-9]+:[0-9]+)\1',
             webpage, 'duration', default=None, group='duration'))
 
+        view_count = int_or_none(self._html_search_regex(
+            r'<div[^>]+class=(["\']).*?record-desc-seen.*?\1[^>]*>(?P<count>.+?)</div>',
+            webpage, 'view count', fatal=False, group='count'))
+        like_count = int_or_none(self._search_regex(
+            r'<span[^>]+id=(["\'])flikesCount.*?\1>(?P<count>\d+)<',
+            webpage, 'like count', fatal=False, group='count'))
+
         return {
             'id': video_id,
             'title': title,
@@ -48,4 +58,6 @@ class LRTIE(InfoExtractor):
             'thumbnail': thumbnail,
             'description': description,
             'duration': duration,
+            'view_count': view_count,
+            'like_count': like_count,
         }

From 6afe044b51e99a3a0e638492e118634a2ca3cdad Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 27 Dec 2015 09:56:15 +0100
Subject: [PATCH 0174/1105] [dcn] improve extraction

---
 youtube_dl/extractor/__init__.py |   2 +-
 youtube_dl/extractor/dcn.py      | 148 ++++++++++++++-----------------
 2 files changed, 67 insertions(+), 83 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 677c75564..abf36caf3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -119,7 +119,7 @@ from .dailymotion import (
 from .daum import DaumIE
 from .dbtv import DBTVIE
 from .dcn import (
-    DCNGeneralIE,
+    DCNIE,
     DCNVideoIE,
     DCNLiveIE,
     DCNSeasonIE,
diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index a9a5e94f5..3857ba334 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -17,24 +17,61 @@ from ..utils import (
 )
 
 
-class DCNGeneralIE(InfoExtractor):
+class DCNIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?show/(?P<show_id>\d+)/[^/]+(?:/(?P<video_id>\d+)/(?P<season_id>\d+))?'
 
     def _real_extract(self, url):
         show_id, video_id, season_id = re.match(self._VALID_URL, url).groups()
-        url = ''
-        ie_key = ''
         if video_id and int(video_id) > 0:
-            return self.url_result('http://www.dcndigital.ae/#/media/%s' % video_id, 'DCNVideo')
+            return self.url_result(
+                'http://www.dcndigital.ae/media/%s' % video_id, 'DCNVideo')
+        elif season_id and int(season_id) > 0:
+            return self.url_result(smuggle_url(
+                'http://www.dcndigital.ae/program/season/%s' % season_id,
+                {'show_id': show_id}), 'DCNSeason')
         else:
-            if season_id and int(season_id) > 0:
-                url = smuggle_url('http://www.dcndigital.ae/#/program/season/%s' % season_id, {'show_id': show_id})
-            else:
-                url = 'http://www.dcndigital.ae/#/program/%s' % show_id
-            return self.url_result(url, 'DCNSeason')
+            return self.url_result(
+                'http://www.dcndigital.ae/program/%s' % show_id, 'DCNSeason')
 
 
-class DCNVideoIE(InfoExtractor):
+class DCNBaseIE(InfoExtractor):
+    def _extract_video_info(self, video_data, video_id, is_live):
+        title = video_data.get('title_en') or video_data['title_ar']
+        img = video_data.get('img')
+        thumbnail = 'http://admin.mangomolo.com/analytics/%s' % img if img else None
+        duration = int_or_none(video_data.get('duration'))
+        description = video_data.get('description_en') or video_data.get('description_ar')
+        timestamp = parse_iso8601(video_data.get('create_time'), ' ')
+
+        return {
+            'id': video_id,
+            'title': self._live_title(title) if is_live else title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'timestamp': timestamp,
+            'is_live': is_live,
+        }
+
+    def _extract_video_formats(self, webpage, video_id, entry_protocol):
+        m3u8_url = self._html_search_regex(
+            r'file\s*:\s*"([^"]+)', webpage, 'm3u8 url')
+        formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, 'mp4', entry_protocol, m3u8_id='hls')
+
+        rtsp_url = self._search_regex(
+            r'<a[^>]+href="(rtsp://[^"]+)"', webpage, 'rtsp url', fatal=False)
+        if rtsp_url:
+            formats.append({
+                'url': rtsp_url,
+                'format_id': 'rtsp',
+            })
+
+        self._sort_formats(formats)
+        return formats
+
+
+class DCNVideoIE(DCNBaseIE):
     IE_NAME = 'dcn:video'
     _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/[^/]+|media|catchup/[^/]+/[^/]+)/(?P<id>\d+)'
     _TEST = {
@@ -45,7 +82,6 @@ class DCNVideoIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'رحلة العمر : الحلقة 1',
             'description': 'md5:0156e935d870acb8ef0a66d24070c6d6',
-            'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 2041,
             'timestamp': 1227504126,
             'upload_date': '20081124',
@@ -62,51 +98,23 @@ class DCNVideoIE(InfoExtractor):
         request = compat_urllib_request.Request(
             'http://admin.mangomolo.com/analytics/index.php/plus/video?id=%s' % video_id,
             headers={'Origin': 'http://www.dcndigital.ae'})
-
-        video = self._download_json(request, video_id)
-        title = video.get('title_en') or video['title_ar']
-        img = video.get('img')
-        thumbnail = 'http://admin.mangomolo.com/analytics/%s' % img if img else None
-        duration = int_or_none(video.get('duration'))
-        description = video.get('description_en') or video.get('description_ar')
-        timestamp = parse_iso8601(video.get('create_time') or video.get('update_time'), ' ')
+        video_data = self._download_json(request, video_id)
+        info = self._extract_video_info(video_data, video_id, False)
 
         webpage = self._download_webpage(
-            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?'
-            + compat_urllib_parse.urlencode({
-                'id': video['id'],
-                'user_id': video['user_id'],
-                'signature': video['signature'],
+            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?' +
+            compat_urllib_parse.urlencode({
+                'id': video_data['id'],
+                'user_id': video_data['user_id'],
+                'signature': video_data['signature'],
                 'countries': 'Q0M=',
                 'filter': 'DENY',
             }), video_id)
-
-        m3u8_url = self._html_search_regex(r'file:\s*"([^"]+)', webpage, 'm3u8 url')
-        formats = self._extract_m3u8_formats(
-            m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
-
-        rtsp_url = self._search_regex(
-            r'<a[^>]+href="(rtsp://[^"]+)"', webpage, 'rtsp url', fatal=False)
-        if rtsp_url:
-            formats.append({
-                'url': rtsp_url,
-                'format_id': 'rtsp',
-            })
-
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'timestamp': timestamp,
-            'formats': formats,
-        }
+        info['formats'] = self._extract_video_formats(webpage, video_id, 'm3u8_native')
+        return info
 
 
-class DCNLiveIE(InfoExtractor):
+class DCNLiveIE(DCNBaseIE):
     IE_NAME = 'dcn:live'
     _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?live/(?P<id>\d+)'
     _TEST = {
@@ -132,45 +140,20 @@ class DCNLiveIE(InfoExtractor):
             'http://admin.mangomolo.com/analytics/index.php/plus/getchanneldetails?channel_id=%s' % channel_id,
             headers={'Origin': 'http://www.dcndigital.ae'})
 
-        channel = self._download_json(request, channel_id)
-        title = channel.get('title_en') or channel['title_ar']
-        img = channel.get('thumbnail')
-        thumbnail = 'http://admin.mangomolo.com/analytics/%s' % img if img else None
-        description = channel.get('description_en') or channel.get('description_ar')
-        timestamp = parse_iso8601(channel.get('create_time') or channel.get('update_time'), ' ')
+        channel_data = self._download_json(request, channel_id)
+        info = self._extract_video_info(channel_data, channel_id, True)
 
         webpage = self._download_webpage(
             'http://admin.mangomolo.com/analytics/index.php/customers/embed/index?' +
             compat_urllib_parse.urlencode({
-                'id': base64.b64encode(channel['user_id'].encode()).decode(),
-                'channelid': base64.b64encode(channel['id'].encode()).decode(),
-                'signature': channel['signature'],
+                'id': base64.b64encode(channel_data['user_id'].encode()).decode(),
+                'channelid': base64.b64encode(channel_data['id'].encode()).decode(),
+                'signature': channel_data['signature'],
                 'countries': 'Q0M=',
                 'filter': 'DENY',
             }), channel_id)
-
-        m3u8_url = self._html_search_regex(r'file:\s*"([^"]+)', webpage, 'm3u8 url')
-        formats = self._extract_m3u8_formats(
-            m3u8_url, channel_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
-
-        rtsp_url = self._search_regex(
-            r'<a[^>]+href="(rtsp://[^"]+)"', webpage, 'rtsp url', fatal=False)
-        if rtsp_url:
-            formats.append({
-                'url': rtsp_url,
-                'format_id': 'rtsp',
-            })
-
-        self._sort_formats(formats)
-
-        return {
-            'id': channel_id,
-            'title': self._live_title(title),
-            'description': description,
-            'thumbnail': thumbnail,
-            'formats': formats,
-            'is_live': True,
-        }
+        info['formats'] = self._extract_video_formats(webpage, channel_id, 'm3u8')
+        return info
 
 
 class DCNSeasonIE(InfoExtractor):
@@ -218,6 +201,7 @@ class DCNSeasonIE(InfoExtractor):
 
                 entries = []
                 for video in show['videos']:
-                    entries.append(self.url_result('http://www.dcndigital.ae/#/media/%s' % video['id'], 'DCNVideo'))
+                    entries.append(self.url_result(
+                        'http://www.dcndigital.ae/media/%s' % video['id'], 'DCNVideo'))
 
                 return self.playlist_result(entries, season_id, title)

From 608cc3b85cbe5acc045274c22e3fdaca08b90d45 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 27 Dec 2015 10:19:44 +0100
Subject: [PATCH 0175/1105] [kaltura] add referrer to m3u8 url

---
 youtube_dl/extractor/kaltura.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 353e960b0..4807c8110 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -160,10 +160,11 @@ class KalturaIE(InfoExtractor):
                 'width': int_or_none(f.get('width')),
                 'url': video_url,
             })
-
+        m3u8_url = info['dataUrl'].replace('format/url', 'format/applehttp')
+        if referrer:
+            m3u8_url += '?referrer=%s' % referrer
         m3u8_formats = self._extract_m3u8_formats(
-            info['dataUrl'].replace('format/url', 'format/applehttp'),
-            entry_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+            m3u8_url, entry_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
         if m3u8_formats:
             formats.extend(m3u8_formats)
 

From 5b025168c73e00a45f1da857694835c52b73f0fa Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 27 Dec 2015 12:57:39 +0100
Subject: [PATCH 0176/1105] [livestream] improve extraction

- split long lines
- use m3u8 entry protocol for live streams
- extend _VALID_URL regex for livestream original
- extract livestream original live streams
---
 youtube_dl/extractor/livestream.py | 114 +++++++++++++++++++----------
 1 file changed, 76 insertions(+), 38 deletions(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index b95c23c8c..11168fc4d 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -62,7 +62,8 @@ class LivestreamIE(InfoExtractor):
     _API_URL_TEMPLATE = 'http://livestream.com/api/accounts/%s/events/%s'
 
     def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
-        base_ele = find_xpath_attr(smil, self._xpath_ns('.//meta', namespace), 'name', 'httpBase')
+        base_ele = find_xpath_attr(
+            smil, self._xpath_ns('.//meta', namespace), 'name', 'httpBase')
         base = base_ele.get('content') if base_ele else 'http://livestreamvod-f.akamaihd.net/'
 
         formats = []
@@ -96,7 +97,8 @@ class LivestreamIE(InfoExtractor):
             video_url = video_data.get(key)
             if video_url:
                 ext = determine_ext(video_url)
-                bitrate = int_or_none(self._search_regex(r'(\d+)\.%s' % ext, video_url, 'bitrate', default=None))
+                bitrate = int_or_none(self._search_regex(
+                    r'(\d+)\.%s' % ext, video_url, 'bitrate', default=None))
                 formats.append({
                     'url': video_url,
                     'format_id': format_id,
@@ -119,7 +121,8 @@ class LivestreamIE(InfoExtractor):
 
         f4m_url = video_data.get('f4m_url')
         if f4m_url:
-            f4m_formats = self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False)
+            f4m_formats = self._extract_f4m_formats(
+                f4m_url, video_id, f4m_id='hds', fatal=False)
             if f4m_formats:
                 formats.extend(f4m_formats)
         self._sort_formats(formats)
@@ -157,10 +160,11 @@ class LivestreamIE(InfoExtractor):
             if smil_formats:
                 formats.extend(smil_formats)
 
+        entry_protocol = 'm3u8' if is_live else 'm3u8_native'
         m3u8_url = stream_info.get('m3u8_url')
         if m3u8_url:
             m3u8_formats = self._extract_m3u8_formats(
-                m3u8_url, broadcast_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+                m3u8_url, broadcast_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=False)
             if m3u8_formats:
                 formats.extend(m3u8_formats)
 
@@ -197,7 +201,8 @@ class LivestreamIE(InfoExtractor):
             else:
                 info_url = '{root}?&id={id}&newer=-1&type=video'.format(
                     root=feed_root_url, id=last_video)
-            videos_info = self._download_json(info_url, event_id, 'Downloading page {0}'.format(i))['data']
+            videos_info = self._download_json(
+                info_url, event_id, 'Downloading page {0}'.format(i))['data']
             videos_info = [v['data'] for v in videos_info if v['type'] == 'video']
             if not videos_info:
                 break
@@ -215,7 +220,8 @@ class LivestreamIE(InfoExtractor):
         account = mobj.group('account_id') or mobj.group('account_name')
         api_url = self._API_URL_TEMPLATE % (account, event)
         if video_id:
-            video_data = self._download_json(api_url + '/videos/%s' % video_id, video_id)
+            video_data = self._download_json(
+                api_url + '/videos/%s' % video_id, video_id)
             return self._extract_video_info(video_data)
         else:
             event_data = self._download_json(api_url, video_id)
@@ -226,8 +232,8 @@ class LivestreamIE(InfoExtractor):
 class LivestreamOriginalIE(InfoExtractor):
     IE_NAME = 'livestream:original'
     _VALID_URL = r'''(?x)https?://original\.livestream\.com/
-        (?P<user>[^/]+)/(?P<type>video|folder)
-        (?:\?.*?Id=|/)(?P<id>.*?)(&|$)
+        (?P<user>[^/\?#]+)(?:/(?P<type>video|folder)
+        (?:(?:\?.*?Id=|/)(?P<id>.*?)(&|$))?)?
         '''
     _TESTS = [{
         'url': 'http://original.livestream.com/dealbook/video?clipId=pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
@@ -244,52 +250,61 @@ class LivestreamOriginalIE(InfoExtractor):
             'id': 'a07bf706-d0e4-4e75-a747-b021d84f2fd3',
         },
         'playlist_mincount': 4,
+    }, {
+        # live stream
+        'url': 'http://www.livestream.com/znsbahamas',
+        'only_matching': True,
     }]
 
-    def _extract_video(self, user, video_id):
-        api_url = 'http://x{0}x.api.channel.livestream.com/2.0/clipdetails?extendedInfo=true&id={1}'.format(user, video_id)
-
+    def _extract_video_info(self, user, video_id):
+        api_url = 'http://x%sx.api.channel.livestream.com/2.0/clipdetails?extendedInfo=true&id=%s' % (user, video_id)
         info = self._download_xml(api_url, video_id)
-        # this url is used on mobile devices
-        stream_url = 'http://x{0}x.api.channel.livestream.com/3.0/getstream.json?id={1}'.format(user, video_id)
-        stream_info = self._download_json(stream_url, video_id)
-        is_live = stream_info.get('isLive')
+
         item = info.find('channel').find('item')
+        title = xpath_text(item, 'title')
         media_ns = {'media': 'http://search.yahoo.com/mrss'}
-        thumbnail_url = xpath_attr(item, xpath_with_ns('media:thumbnail', media_ns), 'url')
-        duration = float_or_none(xpath_attr(item, xpath_with_ns('media:content', media_ns), 'duration'))
+        thumbnail_url = xpath_attr(
+            item, xpath_with_ns('media:thumbnail', media_ns), 'url')
+        duration = float_or_none(xpath_attr(
+            item, xpath_with_ns('media:content', media_ns), 'duration'))
         ls_ns = {'ls': 'http://api.channel.livestream.com/2.0'}
-        view_count = int_or_none(xpath_text(item, xpath_with_ns('ls:viewsCount', ls_ns)))
+        view_count = int_or_none(xpath_text(
+            item, xpath_with_ns('ls:viewsCount', ls_ns)))
 
-        formats = [{
-            'url': stream_info['progressiveUrl'],
-            'format_id': 'http',
-        }]
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail_url,
+            'duration': duration,
+            'view_count': view_count,
+        }
 
-        m3u8_url = stream_info.get('httpUrl')
+    def _extract_video_formats(self, video_data, video_id, entry_protocol):
+        formats = []
+
+        progressive_url = video_data.get('progressiveUrl')
+        if progressive_url:
+            formats.append({
+                'url': progressive_url,
+                'format_id': 'http',
+            })
+
+        m3u8_url = video_data.get('httpUrl')
         if m3u8_url:
             m3u8_formats = self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+                m3u8_url, video_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=False)
             if m3u8_formats:
                 formats.extend(m3u8_formats)
 
-        rtsp_url = stream_info.get('rtspUrl')
+        rtsp_url = video_data.get('rtspUrl')
         if rtsp_url:
             formats.append({
                 'url': rtsp_url,
                 'format_id': 'rtsp',
             })
-        self._sort_formats(formats)
 
-        return {
-            'id': video_id,
-            'title': self._live_title(xpath_text(item, 'title')) if is_live else xpath_text(item, 'title'),
-            'formats': formats,
-            'thumbnail': thumbnail_url,
-            'duration': duration,
-            'view_count': view_count,
-            'is_live': is_live,
-        }
+        self._sort_formats(formats)
+        return formats
 
     def _extract_folder(self, url, folder_id):
         webpage = self._download_webpage(url, folder_id)
@@ -308,13 +323,36 @@ class LivestreamOriginalIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        id = mobj.group('id')
         user = mobj.group('user')
         url_type = mobj.group('type')
+        content_id = mobj.group('id')
         if url_type == 'folder':
-            return self._extract_folder(url, id)
+            return self._extract_folder(url, content_id)
         else:
-            return self._extract_video(user, id)
+            # this url is used on mobile devices
+            stream_url = 'http://x%sx.api.channel.livestream.com/3.0/getstream.json' % user
+            info = {}
+            if content_id:
+                stream_url += '?id=%s' % content_id
+                info = self._extract_video_info(user, content_id)
+            else:
+                content_id = user
+                webpage = self._download_webpage(url, content_id)
+                info = {
+                    'title': self._og_search_title(webpage),
+                    'description': self._og_search_description(webpage),
+                    'thumbnail': self._search_regex(r'channelLogo.src\s*=\s*"([^"]+)"', webpage, 'thumbnail', None),
+                }
+            video_data = self._download_json(stream_url, content_id)
+            is_live = video_data.get('isLive')
+            entry_protocol = 'm3u8' if is_live else 'm3u8_native'
+            info.update({
+                'id': content_id,
+                'title': self._live_title(info['title']) if is_live else info['title'],
+                'formats': self._extract_video_formats(video_data, content_id, entry_protocol),
+                'is_live': is_live,
+            })
+            return info
 
 
 # The server doesn't support HEAD request, the generic extractor can't detect

From 9f610f3a9e0e7536fe9c0ca167ea084aa2cd54df Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 27 Dec 2015 13:11:53 +0100
Subject: [PATCH 0177/1105] [sportdeutschland] Do not abort if meta info is
 missing

This fixes http://sportdeutschland.tv/badminton/yonex-copenhagen-masters-2015 . No testcase though since the event will be over by 2016.
---
 youtube_dl/extractor/sportdeutschland.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sportdeutschland.py b/youtube_dl/extractor/sportdeutschland.py
index ebb75f059..a9927f6e2 100644
--- a/youtube_dl/extractor/sportdeutschland.py
+++ b/youtube_dl/extractor/sportdeutschland.py
@@ -70,10 +70,12 @@ class SportDeutschlandIE(InfoExtractor):
 
             smil_doc = self._download_xml(
                 smil_url, video_id, note='Downloading SMIL metadata')
-            base_url = smil_doc.find('./head/meta').attrib['base']
+            base_url_el = smil_doc.find('./head/meta')
+            if base_url_el:
+                base_url = base_url_el.attrib['base']
             formats.extend([{
                 'format_id': 'rmtp',
-                'url': base_url,
+                'url': base_url if base_url_el else n.attrib['src'],
                 'play_path': n.attrib['src'],
                 'ext': 'flv',
                 'preference': -100,

From 7b81316508d676393d89a99b42fc15e3dcfeab73 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 27 Dec 2015 14:58:57 +0100
Subject: [PATCH 0178/1105] [livestream] skip m3u8 manifest in progressive_urls

---
 youtube_dl/extractor/livestream.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 11168fc4d..9c8d826c4 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -97,6 +97,8 @@ class LivestreamIE(InfoExtractor):
             video_url = video_data.get(key)
             if video_url:
                 ext = determine_ext(video_url)
+                if ext == 'm3u8':
+                    continue
                 bitrate = int_or_none(self._search_regex(
                     r'(\d+)\.%s' % ext, video_url, 'bitrate', default=None))
                 formats.append({

From 8d29e47f543152bf91db0167a313e56ea2f132e3 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 27 Dec 2015 15:33:39 +0100
Subject: [PATCH 0179/1105] [common] simplify the use of _extract_m3u8_formats
 and _extract_f4m_formats

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 828f58f12..3d39781a4 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -885,7 +885,7 @@ class InfoExtractor(object):
             fatal=fatal)
 
         if manifest is False:
-            return manifest
+            return []
 
         formats = []
         manifest_version = '1.0'
@@ -956,7 +956,7 @@ class InfoExtractor(object):
             errnote=errnote or 'Failed to download m3u8 information',
             fatal=fatal)
         if res is False:
-            return res
+            return []
         m3u8_doc, urlh = res
         m3u8_url = urlh.geturl()
         last_info = None

From e73277c7e811a93a318a7707f82636c8093942b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Dec 2015 21:41:21 +0600
Subject: [PATCH 0180/1105] [abc7news] Remove redundant formats sorting

---
 youtube_dl/extractor/abc7news.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/abc7news.py b/youtube_dl/extractor/abc7news.py
index c04949c21..122dc9099 100644
--- a/youtube_dl/extractor/abc7news.py
+++ b/youtube_dl/extractor/abc7news.py
@@ -44,7 +44,6 @@ class Abc7NewsIE(InfoExtractor):
             'contentURL', webpage, 'm3u8 url', fatal=True)
 
         formats = self._extract_m3u8_formats(m3u8, display_id, 'mp4')
-        self._sort_formats(formats)
 
         title = self._og_search_title(webpage).strip()
         description = self._og_search_description(webpage).strip()

From 62bdc9feccbab0265243f8554bdbc7a84b5cca65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Dec 2015 21:41:36 +0600
Subject: [PATCH 0181/1105] [esri] Fix typo

---
 youtube_dl/extractor/esri.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/esri.py b/youtube_dl/extractor/esri.py
index bf5d2019f..d4205d7fb 100644
--- a/youtube_dl/extractor/esri.py
+++ b/youtube_dl/extractor/esri.py
@@ -61,7 +61,7 @@ class EsriVideoIE(InfoExtractor):
             webpage, 'duration', fatal=False))
 
         upload_date = unified_strdate(self._html_search_meta(
-            'last-modified', webpage, 'upload date', fatal=None))
+            'last-modified', webpage, 'upload date', fatal=False))
 
         return {
             'id': video_id,

From e0898585a188eb0538624b227eb00c90c2da26d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Dec 2015 21:41:55 +0600
Subject: [PATCH 0182/1105] [jwplatform] Fix typo

---
 youtube_dl/extractor/jwplatform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index cdc095a79..29eba77fc 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -44,7 +44,7 @@ class JWPlatformIE(InfoExtractor):
             source_url = self._proto_relative_url(source['file'])
             source_type = source.get('type') or ''
             if source_type == 'application/vnd.apple.mpegurl':
-                m3u8_formats = self._extract_m3u8_formats(source_url, video_id, 'mp4', 'm3u8_native', fatal=None)
+                m3u8_formats = self._extract_m3u8_formats(source_url, video_id, 'mp4', 'm3u8_native', fatal=False)
                 if m3u8_formats:
                     formats.extend(m3u8_formats)
             elif source_type.startswith('audio'):

From 30f51acbc8b0f4f3161102501527ed81d162d9e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Dec 2015 21:42:12 +0600
Subject: [PATCH 0183/1105] [rai] Fix typos

---
 youtube_dl/extractor/rai.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 14f1ccbb4..278b1d2bf 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -118,13 +118,13 @@ class RaiTVIE(InfoExtractor):
                 if ext == 'm3u8':
                     m3u8_formats = self._extract_m3u8_formats(
                         media_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls',
-                        fatal=None)
+                        fatal=False)
                     if m3u8_formats:
                         formats.extend(m3u8_formats)
                 elif ext == 'f4m':
                     f4m_formats = self._extract_f4m_formats(
                         media_url + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44',
-                        video_id, f4m_id='hds', fatal=None)
+                        video_id, f4m_id='hds', fatal=False)
                     if f4m_formats:
                         formats.extend(f4m_formats)
                 elif ext == 'stl':

From 0b0a17ae9d8c53b75f7f3cfa7e6fd236ba83e280 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Dec 2015 21:42:26 +0600
Subject: [PATCH 0184/1105] [viki] Fix typo

---
 youtube_dl/extractor/viki.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index ca3f20a3d..9a1c377a4 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -279,7 +279,7 @@ class VikiIE(VikiBaseIE):
                 if format_id == 'm3u8':
                     m3u8_formats = self._extract_m3u8_formats(
                         format_dict['url'], video_id, 'mp4', 'm3u8_native',
-                        m3u8_id='m3u8-%s' % protocol, fatal=None)
+                        m3u8_id='m3u8-%s' % protocol, fatal=False)
                     if m3u8_formats:
                         formats.extend(m3u8_formats)
                 else:

From a95c26a06a88ab464638d16b42d1659a15e71c3c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Dec 2015 21:43:14 +0600
Subject: [PATCH 0185/1105] [jwplatform] Carry long line

---
 youtube_dl/extractor/jwplatform.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index 29eba77fc..a92adf2b3 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -44,7 +44,8 @@ class JWPlatformIE(InfoExtractor):
             source_url = self._proto_relative_url(source['file'])
             source_type = source.get('type') or ''
             if source_type == 'application/vnd.apple.mpegurl':
-                m3u8_formats = self._extract_m3u8_formats(source_url, video_id, 'mp4', 'm3u8_native', fatal=False)
+                m3u8_formats = self._extract_m3u8_formats(
+                    source_url, video_id, 'mp4', 'm3u8_native', fatal=False)
                 if m3u8_formats:
                     formats.extend(m3u8_formats)
             elif source_type.startswith('audio'):

From 82e02ea5fce6bbc595dbfefc8eafce21e2bb53cc Mon Sep 17 00:00:00 2001
From: gam2046 <gam2046@qq.com>
Date: Fri, 18 Dec 2015 14:18:56 +0800
Subject: [PATCH 0186/1105] Update iqiyi.py

Fix part of the address can not be resolved.
eg:http://www.iqiyi.com/w_19rt6o8t9p.html
---
 youtube_dl/extractor/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index c3731a110..f0e2e5104 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -16,7 +16,7 @@ class IqiyiIE(InfoExtractor):
     IE_NAME = 'iqiyi'
     IE_DESC = '爱奇艺'
 
-    _VALID_URL = r'http://(?:www\.)iqiyi.com/v_.+?\.html'
+    _VALID_URL = r'http://(?:www\.)iqiyi.com/._.+?\.html'
 
     _TESTS = [{
         'url': 'http://www.iqiyi.com/v_19rrojlavg.html',

From bee83e84f64248a47f2f0b79e8c2698bc89abf55 Mon Sep 17 00:00:00 2001
From: forDream <gam2046@users.noreply.github.com>
Date: Tue, 22 Dec 2015 09:28:12 +0800
Subject: [PATCH 0187/1105] [iqiyi]fix valid url

eg:
http://yule.iqiyi.com/zbj.html
---
 youtube_dl/extractor/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index f0e2e5104..38348e0e7 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -16,7 +16,7 @@ class IqiyiIE(InfoExtractor):
     IE_NAME = 'iqiyi'
     IE_DESC = '爱奇艺'
 
-    _VALID_URL = r'http://(?:www\.)iqiyi.com/._.+?\.html'
+    _VALID_URL = r'http://(?:[^.]+\.)?iqiyi\.com/.+\.html'
 
     _TESTS = [{
         'url': 'http://www.iqiyi.com/v_19rrojlavg.html',

From 59185202c6854b39ceb5fe6ea960ad1b308ceca8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 28 Dec 2015 00:19:36 +0800
Subject: [PATCH 0188/1105] [iqiyi] Add tests for #7894

---
 youtube_dl/extractor/iqiyi.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 38348e0e7..66a70a181 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -84,6 +84,15 @@ class IqiyiIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.iqiyi.com/w_19rt6o8t9p.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.iqiyi.com/a_19rrhbc6kt.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://yule.iqiyi.com/pcb.html',
+        'only_matching': True,
     }]
 
     _FORMATS_MAP = [

From c71d2e2087e6daf2196bb26ba7fa7e00c10f2d53 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 27 Dec 2015 17:27:20 +0100
Subject: [PATCH 0189/1105] [livestream] change test url

---
 youtube_dl/extractor/livestream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 9c8d826c4..688eb2308 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -254,7 +254,7 @@ class LivestreamOriginalIE(InfoExtractor):
         'playlist_mincount': 4,
     }, {
         # live stream
-        'url': 'http://www.livestream.com/znsbahamas',
+        'url': 'http://original.livestream.com/znsbahamas',
         'only_matching': True,
     }]
 

From 97f18fac3a90da163bb6692038be560a5b93b525 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 27 Dec 2015 17:44:55 +0100
Subject: [PATCH 0190/1105] [vgtv] fix f4m downloading(fixes #7843)

---
 youtube_dl/extractor/vgtv.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 811ee197d..129668a99 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -86,9 +86,10 @@ class VGTVIE(XstreamIE):
         {
             # streamType: wasLive
             'url': 'http://www.vgtv.no/#!/live/113063/direkte-v75-fra-solvalla',
+            'md5': '458f4841239dab414343b50e5af8869c',
             'info_dict': {
                 'id': '113063',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'V75 fra Solvalla 30.05.15',
                 'description': 'md5:b3743425765355855f88e096acc93231',
                 'thumbnail': 're:^https?://.*\.jpg',
@@ -97,10 +98,6 @@ class VGTVIE(XstreamIE):
                 'upload_date': '20150530',
                 'view_count': int,
             },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            },
         },
         {
             'url': 'http://www.aftenposten.no/webtv/#!/video/21039/trailer-sweatshop-i-can-t-take-any-more',
@@ -160,12 +157,15 @@ class VGTVIE(XstreamIE):
                 formats.extend(m3u8_formats)
 
         hds_url = streams.get('hds')
-        # wasLive hds are always 404
-        if hds_url and stream_type != 'wasLive':
+        if hds_url:
+            hdcore_sign = 'hdcore=3.7.0'
             f4m_formats = self._extract_f4m_formats(
-                hds_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18', video_id, f4m_id='hds', fatal=False)
+                hds_url + '?%s' % hdcore_sign, video_id, f4m_id='hds', fatal=False)
             if f4m_formats:
-                formats.extend(f4m_formats)
+                for entry in f4m_formats:
+                    # URLs without the extra param induce an 404 error
+                    entry.update({'extra_param_to_segment_url': hdcore_sign})
+                    formats.append(entry)
 
         mp4_urls = streams.get('pseudostreaming') or []
         mp4_url = streams.get('mp4')

From c047270c02bef6c8303c985d1e1b8da5be394235 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 28 Dec 2015 01:09:18 +0800
Subject: [PATCH 0191/1105] [utils] Remove Content-encoding from headers after
 decompression

With cn_verification_proxy, our http_response() is called twice, one from
PerRequestProxyHandler.proxy_open() and another from normal
YoutubeDL.urlopen(). As a result, for proxies honoring Accept-Encoding, the
following bug occurs:

$ youtube-dl -vs --cn-verification-proxy https://secure.uku.im:993 "test:letv"
[debug] System config: []
[debug] User config: []
[debug] Command-line args: ['-vs', '--cn-verification-proxy', 'https://secure.uku.im:993', 'test:letv']
[debug] Encodings: locale UTF-8, fs utf-8, out UTF-8, pref UTF-8
[debug] youtube-dl version 2015.12.23
[debug] Git HEAD: 97f18fa
[debug] Python version 3.5.1 - Linux-4.3.3-1-ARCH-x86_64-with-arch-Arch-Linux
[debug] exe versions: ffmpeg 2.8.4, ffprobe 2.8.4, rtmpdump 2.4
[debug] Proxy map: {}
[TestURL] Test URL: http://www.letv.com/ptv/vplay/22005890.html
[Letv] 22005890: Downloading webpage
[Letv] 22005890: Downloading playJson data
ERROR: Unable to download JSON metadata: Not a gzipped file (b'{"') (caused by OSError('Not a gzipped file (b\'{"\')',)); please report this issue on https://yt-dl.org/bug . Make sure you are using the latest version; see  https://yt-dl.org/update  on how to update. Be sure to call youtube-dl with the --verbose flag and include its complete output.
  File "/home/yen/Executables/Multimedia/youtube-dl/youtube_dl/extractor/common.py", line 330, in _request_webpage
    return self._downloader.urlopen(url_or_request)
  File "/home/yen/Executables/Multimedia/youtube-dl/youtube_dl/YoutubeDL.py", line 1886, in urlopen
    return self._opener.open(req, timeout=self._socket_timeout)
  File "/usr/lib/python3.5/urllib/request.py", line 471, in open
    response = meth(req, response)
  File "/home/yen/Executables/Multimedia/youtube-dl/youtube_dl/utils.py", line 773, in http_response
    raise original_ioerror
  File "/home/yen/Executables/Multimedia/youtube-dl/youtube_dl/utils.py", line 761, in http_response
    uncompressed = io.BytesIO(gz.read())
  File "/usr/lib/python3.5/gzip.py", line 274, in read
    return self._buffer.read(size)
  File "/usr/lib/python3.5/gzip.py", line 461, in read
    if not self._read_gzip_header():
  File "/usr/lib/python3.5/gzip.py", line 409, in _read_gzip_header
    raise OSError('Not a gzipped file (%r)' % magic)
---
 youtube_dl/utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 1737ac5f6..0ed6c45c8 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -773,11 +773,13 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
                     raise original_ioerror
             resp = self.addinfourl_wrapper(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
             resp.msg = old_resp.msg
+            del resp.headers['Content-encoding']
         # deflate
         if resp.headers.get('Content-encoding', '') == 'deflate':
             gz = io.BytesIO(self.deflate(resp.read()))
             resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
             resp.msg = old_resp.msg
+            del resp.headers['Content-encoding']
         # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
         # https://github.com/rg3/youtube-dl/issues/6457).
         if 300 <= resp.code < 400:

From bd3f9ecabeb5e189f16580c61c56d93148079363 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 28 Dec 2015 00:36:57 +0100
Subject: [PATCH 0192/1105] [tunein] add support for tunein topic,clip and
 program(fixes #7348)

---
 youtube_dl/extractor/__init__.py |   8 +-
 youtube_dl/extractor/tunein.py   | 187 +++++++++++++++++++++----------
 2 files changed, 136 insertions(+), 59 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 165835f63..c1dbf9f21 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -703,7 +703,13 @@ from .tube8 import Tube8IE
 from .tubitv import TubiTvIE
 from .tudou import TudouIE
 from .tumblr import TumblrIE
-from .tunein import TuneInIE
+from .tunein import (
+    TuneInClipIE,
+    TuneInStationIE,
+    TuneInProgramIE,
+    TuneInTopicIE,
+    TuneInShortenerIE,
+)
 from .turbo import TurboIE
 from .tutv import TutvIE
 from .tv2 import (
diff --git a/youtube_dl/extractor/tunein.py b/youtube_dl/extractor/tunein.py
index b6b1f2568..8322cc14d 100644
--- a/youtube_dl/extractor/tunein.py
+++ b/youtube_dl/extractor/tunein.py
@@ -2,74 +2,33 @@
 from __future__ import unicode_literals
 
 import json
-import re
 
 from .common import InfoExtractor
 from ..utils import ExtractorError
+from ..compat import compat_urlparse
 
 
-class TuneInIE(InfoExtractor):
-    _VALID_URL = r'''(?x)https?://(?:www\.)?
-    (?:
-        tunein\.com/
-        (?:
-            radio/.*?-s|
-            station/.*?StationId\=
-        )(?P<id>[0-9]+)
-        |tun\.in/(?P<redirect_id>[A-Za-z0-9]+)
-    )
-    '''
-    _API_URL_TEMPLATE = 'http://tunein.com/tuner/tune/?stationId={0:}&tuneType=Station'
-
-    _INFO_DICT = {
-        'id': '34682',
-        'title': 'Jazz 24 on 88.5 Jazz24 - KPLU-HD2',
-        'ext': 'aac',
-        'thumbnail': 're:^https?://.*\.png$',
-        'location': 'Tacoma, WA',
-    }
-    _TESTS = [
-        {
-            'url': 'http://tunein.com/radio/Jazz24-885-s34682/',
-            'info_dict': _INFO_DICT,
-            'params': {
-                'skip_download': True,  # live stream
-            },
-        },
-        {  # test redirection
-            'url': 'http://tun.in/ser7s',
-            'info_dict': _INFO_DICT,
-            'params': {
-                'skip_download': True,  # live stream
-            },
-        },
-    ]
+class TuneInBaseIE(InfoExtractor):
+    _API_BASE_URL = 'http://tunein.com/tuner/tune/'
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        redirect_id = mobj.group('redirect_id')
-        if redirect_id:
-            # The server doesn't support HEAD requests
-            urlh = self._request_webpage(
-                url, redirect_id, note='Downloading redirect page')
-            url = urlh.geturl()
-            self.to_screen('Following redirect: %s' % url)
-            mobj = re.match(self._VALID_URL, url)
-        station_id = mobj.group('id')
+        content_id = self._match_id(url)
 
-        station_info = self._download_json(
-            self._API_URL_TEMPLATE.format(station_id),
-            station_id, note='Downloading station JSON')
+        content_info = self._download_json(
+            self._API_BASE_URL + self._API_URL_QUERY % content_id,
+            content_id, note='Downloading JSON metadata')
 
-        title = station_info['Title']
-        thumbnail = station_info.get('Logo')
-        location = station_info.get('Location')
-        streams_url = station_info.get('StreamUrl')
+        title = content_info['Title']
+        thumbnail = content_info.get('Logo')
+        location = content_info.get('Location')
+        streams_url = content_info.get('StreamUrl')
         if not streams_url:
-            raise ExtractorError('No downloadable streams found',
-                                 expected=True)
+            raise ExtractorError('No downloadable streams found', expected=True)
+        if not streams_url.startswith('http://'):
+            streams_url = compat_urlparse.urljoin(url, streams_url)
+
         stream_data = self._download_webpage(
-            streams_url, station_id, note='Downloading stream data')
+            streams_url, content_id, note='Downloading stream data')
         streams = json.loads(self._search_regex(
             r'\((.*)\);', stream_data, 'stream info'))['Streams']
 
@@ -97,10 +56,122 @@ class TuneInIE(InfoExtractor):
         self._sort_formats(formats)
 
         return {
-            'id': station_id,
+            'id': content_id,
             'title': title,
             'formats': formats,
             'thumbnail': thumbnail,
             'location': location,
             'is_live': is_live,
         }
+
+
+class TuneInClipIE(TuneInBaseIE):
+    IE_NAME = 'tunein:clip'
+    _VALID_URL = r'https?://(?:www\.)?tunein\.com/station/.*?audioClipId\=(?P<id>\d+)'
+    _API_URL_QUERY = '?tuneType=AudioClip&audioclipId=%s'
+
+    _TESTS = [
+        {
+            'url': 'http://tunein.com/station/?stationId=246119&audioClipId=816',
+            'md5': '99f00d772db70efc804385c6b47f4e77',
+            'info_dict': {
+                'id': '816',
+                'title': '32m',
+                'ext': 'mp3',
+            },
+        },
+    ]
+
+
+class TuneInStationIE(TuneInBaseIE):
+    IE_NAME = 'tunein:station'
+    _VALID_URL = r'https?://(?:www\.)?tunein\.com/(?:radio/.*?-s|station/.*?StationId\=)(?P<id>\d+)'
+    _API_URL_QUERY = '?tuneType=Station&stationId=%s'
+
+    @classmethod
+    def suitable(cls, url):
+        return False if TuneInClipIE.suitable(url) else super(TuneInStationIE, cls).suitable(url)
+
+    _TESTS = [
+        {
+            'url': 'http://tunein.com/radio/Jazz24-885-s34682/',
+            'info_dict': {
+                'id': '34682',
+                'title': 'Jazz 24 on 88.5 Jazz24 - KPLU-HD2',
+                'ext': 'mp3',
+                'location': 'Tacoma, WA',
+            },
+            'params': {
+                'skip_download': True,  # live stream
+            },
+        },
+    ]
+
+
+class TuneInProgramIE(TuneInBaseIE):
+    IE_NAME = 'tunein:program'
+    _VALID_URL = r'https?://(?:www\.)?tunein\.com/(?:radio/.*?-p|program/.*?ProgramId\=)(?P<id>\d+)'
+    _API_URL_QUERY = '?tuneType=Program&programId=%s'
+
+    _TESTS = [
+        {
+            'url': 'http://tunein.com/radio/Jazz-24-p2506/',
+            'info_dict': {
+                'id': '2506',
+                'title': 'Jazz 24 on 91.3 WUKY-HD3',
+                'ext': 'mp3',
+                'location': 'Lexington, KY',
+            },
+            'params': {
+                'skip_download': True,  # live stream
+            },
+        },
+    ]
+
+
+class TuneInTopicIE(TuneInBaseIE):
+    IE_NAME = 'tunein:topic'
+    _VALID_URL = r'https?://(?:www\.)?tunein\.com/topic/.*?TopicId\=(?P<id>\d+)'
+    _API_URL_QUERY = '?tuneType=Topic&topicId=%s'
+
+    _TESTS = [
+        {
+            'url': 'http://tunein.com/topic/?TopicId=101830576',
+            'md5': 'c31a39e6f988d188252eae7af0ef09c9',
+            'info_dict': {
+                'id': '101830576',
+                'title': 'Votez pour moi du 29 octobre 2015 (29/10/15)',
+                'ext': 'mp3',
+                'location': 'Belgium',
+            },
+        },
+    ]
+
+
+class TuneInShortenerIE(InfoExtractor):
+    IE_NAME = 'tunein:shortener'
+    IE_DESC = False  # Do not list
+    _VALID_URL = r'https?://tun\.in/(?P<id>[A-Za-z0-9]+)'
+
+    _TEST = {
+        # test redirection
+        'url': 'http://tun.in/ser7s',
+        'info_dict': {
+            'id': '34682',
+            'title': 'Jazz 24 on 88.5 Jazz24 - KPLU-HD2',
+            'ext': 'mp3',
+            'location': 'Tacoma, WA',
+        },
+        'params': {
+            'skip_download': True,  # live stream
+        },
+    }
+
+    def _real_extract(self, url):
+        redirect_id = self._match_id(url)
+        # The server doesn't support HEAD requests
+        urlh = self._request_webpage(
+            url, redirect_id, note='Downloading redirect page')
+        url = urlh.geturl()
+        self.to_screen('Following redirect: %s' % url)
+        return self.url_result(url)

From bca9bea1c1a13d2f9ad3244a2b11979220c30484 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 28 Dec 2015 10:27:17 +0100
Subject: [PATCH 0193/1105] [dcn] make m3u8 formats extraction non fatal

---
 youtube_dl/extractor/dcn.py | 25 +++++++------------------
 1 file changed, 7 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 3857ba334..d9485cd86 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -54,10 +54,14 @@ class DCNBaseIE(InfoExtractor):
         }
 
     def _extract_video_formats(self, webpage, video_id, entry_protocol):
+        formats = []
         m3u8_url = self._html_search_regex(
-            r'file\s*:\s*"([^"]+)', webpage, 'm3u8 url')
-        formats = self._extract_m3u8_formats(
-            m3u8_url, video_id, 'mp4', entry_protocol, m3u8_id='hls')
+            r'file\s*:\s*"([^"]+)', webpage, 'm3u8 url', fatal=False)
+        if m3u8_url:
+            m3u8_formats = self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=None)
+            if m3u8_formats:
+                formats.extend(m3u8_formats)
 
         rtsp_url = self._search_regex(
             r'<a[^>]+href="(rtsp://[^"]+)"', webpage, 'rtsp url', fatal=False)
@@ -117,21 +121,6 @@ class DCNVideoIE(DCNBaseIE):
 class DCNLiveIE(DCNBaseIE):
     IE_NAME = 'dcn:live'
     _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?live/(?P<id>\d+)'
-    _TEST = {
-        'url': 'http://www.dcndigital.ae/#/live/6/dubai-tv',
-        'info_dict':
-        {
-            'id': '6',
-            'ext': 'mp4',
-            'title': 're:^Dubai Al Oula [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
-            'thumbnail': 're:^https?://.*\.png$',
-            'is_live': True,
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)

From 04e24906be1dd0b6626b50de7e65609219a88116 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 28 Dec 2015 13:06:30 +0100
Subject: [PATCH 0194/1105] [cspan] Initialize 'video_type' to avoid
 'UnboundLocalError' exceptions (#8032)

---
 youtube_dl/extractor/cspan.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 7b685d157..154bddb43 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -58,6 +58,7 @@ class CSpanIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        video_type = None
         webpage = self._download_webpage(url, video_id)
         matches = re.search(r'data-(prog|clip)id=\'([0-9]+)\'', webpage)
         if matches:
@@ -70,6 +71,8 @@ class CSpanIE(InfoExtractor):
                 title = self._og_search_title(webpage)
                 surl = smuggle_url(senate_isvp_url, {'force_title': title})
                 return self.url_result(surl, 'SenateISVP', video_id, title)
+        if video_type is None or video_id is None:
+            raise ExtractorError('unable to find video id and type')
 
         def get_text_attr(d, attr):
             return d.get(attr, {}).get('#text')

From 6c6b8bd5ccef10aad73ee5a556830f9b00b93661 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 28 Dec 2015 13:48:10 +0100
Subject: [PATCH 0195/1105] [cspan] Fix extraction (fixes #8032)

---
 youtube_dl/extractor/cspan.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 154bddb43..b3ee67018 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -60,11 +60,13 @@ class CSpanIE(InfoExtractor):
         video_id = self._match_id(url)
         video_type = None
         webpage = self._download_webpage(url, video_id)
-        matches = re.search(r'data-(prog|clip)id=\'([0-9]+)\'', webpage)
-        if matches:
+        # We first look for clipid, because clipprog always appears before
+        patterns = [r'id=\'clip(%s)\'\s*value=\'([0-9]+)\'' % t for t in ('id', 'prog')]
+        results = list(filter(None, (re.search(p, webpage) for p in patterns)))
+        if results:
+            matches = results[0]
             video_type, video_id = matches.groups()
-            if video_type == 'prog':
-                video_type = 'program'
+            video_type = 'clip' if video_type == 'id' else 'program'
         else:
             senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
             if senate_isvp_url:

From 9f66931e160ec7c4cc3be8bf731da32b0bfa9043 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 28 Dec 2015 21:20:09 +0600
Subject: [PATCH 0196/1105] [facebook] Extract login error

---
 youtube_dl/extractor/facebook.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 39c481068..1692ec1cb 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -100,6 +100,11 @@ class FacebookIE(InfoExtractor):
             login_results = self._download_webpage(request, None,
                                                    note='Logging in', errnote='unable to fetch login page')
             if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
+                error = self._html_search_regex(
+                    r'(?s)<div[^>]+class=(["\']).*?login_error_box.*?\1[^>]*><div[^>]*>.*?</div><div[^>]*>(?P<error>.+?)</div>',
+                    login_results, 'login error', default=None, group='error')
+                if error:
+                    raise ExtractorError('Unable to login: %s' % error, expected=True)
                 self._downloader.report_warning('unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
                 return
 

From c1f49e16843bb9bdfdf132a472eb0fc2c6b920e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 28 Dec 2015 21:37:02 +0600
Subject: [PATCH 0197/1105] [facebook] Fix authentication

---
 youtube_dl/extractor/facebook.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 1692ec1cb..5e43f2359 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -74,7 +74,7 @@ class FacebookIE(InfoExtractor):
             return
 
         login_page_req = sanitized_Request(self._LOGIN_URL)
-        login_page_req.add_header('Cookie', 'locale=en_US')
+        self._set_cookie('facebook.com', 'locale', 'en_US')
         login_page = self._download_webpage(login_page_req, None,
                                             note='Downloading login page',
                                             errnote='Unable to download login page')
@@ -108,10 +108,17 @@ class FacebookIE(InfoExtractor):
                 self._downloader.report_warning('unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
                 return
 
+            fb_dtsg = self._search_regex(
+                r'name="fb_dtsg" value="(.+?)"', login_results, 'fb_dtsg', default=None)
+            h = self._search_regex(
+                r'name="h"\s+(?:\w+="[^"]+"\s+)*?value="([^"]+)"', login_results, 'h', default=None)
+
+            if not fb_dtsg or not h:
+                return
+
             check_form = {
-                'fb_dtsg': self._search_regex(r'name="fb_dtsg" value="(.+?)"', login_results, 'fb_dtsg'),
-                'h': self._search_regex(
-                    r'name="h"\s+(?:\w+="[^"]+"\s+)*?value="([^"]+)"', login_results, 'h'),
+                'fb_dtsg': fb_dtsg,
+                'h': h,
                 'name_action_selected': 'dont_save',
             }
             check_req = sanitized_Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))

From a5c1d95500898541cd6b19b32963883db421c4b4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 28 Dec 2015 17:49:59 +0100
Subject: [PATCH 0198/1105] [zdf] fix formats extraction

---
 youtube_dl/extractor/phoenix.py |   9 +-
 youtube_dl/extractor/zdf.py     | 253 ++++++++++++++++++++------------
 2 files changed, 161 insertions(+), 101 deletions(-)

diff --git a/youtube_dl/extractor/phoenix.py b/youtube_dl/extractor/phoenix.py
index 46cebc0d7..6ce2ec19d 100644
--- a/youtube_dl/extractor/phoenix.py
+++ b/youtube_dl/extractor/phoenix.py
@@ -1,10 +1,9 @@
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
-from .zdf import extract_from_xml_url
+from .zdf import ZDFIE
 
 
-class PhoenixIE(InfoExtractor):
+class PhoenixIE(ZDFIE):
     _VALID_URL = r'''(?x)https?://(?:www\.)?phoenix\.de/content/
         (?:
             phoenix/die_sendungen/(?:[^/]+/)?
@@ -41,5 +40,5 @@ class PhoenixIE(InfoExtractor):
             r'<div class="phx_vod" id="phx_vod_([0-9]+)"',
             webpage, 'internal video ID')
 
-        api_url = 'http://www.phoenix.de/php/zdfplayer-v1.3/data/beitragsDetails.php?ak=web&id=%s' % internal_id
-        return extract_from_xml_url(self, video_id, api_url)
+        api_url = 'http://www.phoenix.de/php/mediaplayer/data/beitrags_details.php?ak=web&id=%s' % internal_id
+        return self.extract_from_xml_url(video_id, api_url)
diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index 9a3331a69..92c12bac6 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -10,106 +10,16 @@ from ..utils import (
     unified_strdate,
     OnDemandPagedList,
     xpath_text,
+    determine_ext,
+    qualities,
+    float_or_none,
 )
 
 
-def extract_from_xml_url(ie, video_id, xml_url):
-    doc = ie._download_xml(
-        xml_url, video_id,
-        note='Downloading video info',
-        errnote='Failed to download video info')
-
-    title = doc.find('.//information/title').text
-    description = xpath_text(doc, './/information/detail', 'description')
-    duration = int_or_none(xpath_text(doc, './/details/lengthSec', 'duration'))
-    uploader = xpath_text(doc, './/details/originChannelTitle', 'uploader')
-    uploader_id = xpath_text(doc, './/details/originChannelId', 'uploader id')
-    upload_date = unified_strdate(xpath_text(doc, './/details/airtime', 'upload date'))
-
-    def xml_to_format(fnode):
-        video_url = fnode.find('url').text
-        is_available = 'http://www.metafilegenerator' not in video_url
-
-        format_id = fnode.attrib['basetype']
-        format_m = re.match(r'''(?x)
-            (?P<vcodec>[^_]+)_(?P<acodec>[^_]+)_(?P<container>[^_]+)_
-            (?P<proto>[^_]+)_(?P<index>[^_]+)_(?P<indexproto>[^_]+)
-        ''', format_id)
-
-        ext = format_m.group('container')
-        proto = format_m.group('proto').lower()
-
-        quality = xpath_text(fnode, './quality', 'quality')
-        abr = int_or_none(xpath_text(fnode, './audioBitrate', 'abr'), 1000)
-        vbr = int_or_none(xpath_text(fnode, './videoBitrate', 'vbr'), 1000)
-
-        width = int_or_none(xpath_text(fnode, './width', 'width'))
-        height = int_or_none(xpath_text(fnode, './height', 'height'))
-
-        filesize = int_or_none(xpath_text(fnode, './filesize', 'filesize'))
-
-        format_note = ''
-        if not format_note:
-            format_note = None
-
-        return {
-            'format_id': format_id + '-' + quality,
-            'url': video_url,
-            'ext': ext,
-            'acodec': format_m.group('acodec'),
-            'vcodec': format_m.group('vcodec'),
-            'abr': abr,
-            'vbr': vbr,
-            'width': width,
-            'height': height,
-            'filesize': filesize,
-            'format_note': format_note,
-            'protocol': proto,
-            '_available': is_available,
-        }
-
-    def xml_to_thumbnails(fnode):
-        thumbnails = []
-        for node in fnode:
-            thumbnail_url = node.text
-            if not thumbnail_url:
-                continue
-            thumbnail = {
-                'url': thumbnail_url,
-            }
-            if 'key' in node.attrib:
-                m = re.match('^([0-9]+)x([0-9]+)$', node.attrib['key'])
-                if m:
-                    thumbnail['width'] = int(m.group(1))
-                    thumbnail['height'] = int(m.group(2))
-            thumbnails.append(thumbnail)
-        return thumbnails
-
-    thumbnails = xml_to_thumbnails(doc.findall('.//teaserimages/teaserimage'))
-
-    format_nodes = doc.findall('.//formitaeten/formitaet')
-    formats = list(filter(
-        lambda f: f['_available'],
-        map(xml_to_format, format_nodes)))
-    ie._sort_formats(formats)
-
-    return {
-        'id': video_id,
-        'title': title,
-        'description': description,
-        'duration': duration,
-        'thumbnails': thumbnails,
-        'uploader': uploader,
-        'uploader_id': uploader_id,
-        'upload_date': upload_date,
-        'formats': formats,
-    }
-
-
 class ZDFIE(InfoExtractor):
     _VALID_URL = r'(?:zdf:|zdf:video:|https?://www\.zdf\.de/ZDFmediathek(?:#)?/(.*beitrag/(?:video/)?))(?P<id>[0-9]+)(?:/[^/?]+)?(?:\?.*)?'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.zdf.de/ZDFmediathek/beitrag/video/2037704/ZDFspezial---Ende-des-Machtpokers--?bc=sts;stt',
         'info_dict': {
             'id': '2037704',
@@ -122,12 +32,163 @@ class ZDFIE(InfoExtractor):
             'upload_date': '20131127',
         },
         'skip': 'Videos on ZDF.de are depublicised in short order',
-    }
+    }]
+
+    def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
+        param_groups = {}
+        for param_group in smil.findall(self._xpath_ns('./head/paramGroup', namespace)):
+            group_id = param_group.attrib.get(self._xpath_ns('id', 'http://www.w3.org/XML/1998/namespace'))
+            params = {}
+            for param in param_group:
+                params[param.get('name')] = param.get('value')
+            param_groups[group_id] = params
+
+        formats = []
+        for video in smil.findall(self._xpath_ns('.//video', namespace)):
+            src = video.get('src')
+            if not src:
+                continue
+            bitrate = float_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
+            group_id = video.get('paramGroup')
+            param_group = param_groups[group_id]
+            for proto in param_group['protocols'].split(','):
+                formats.append({
+                    'url': '%s://%s' % (proto, param_group['host']),
+                    'app': param_group['app'],
+                    'play_path': src,
+                    'ext': 'flv',
+                    'format_id': '%s-%d' % (proto, bitrate),
+                    'tbr': bitrate,
+                    'protocol': proto,
+                })
+        self._sort_formats(formats)
+        return formats
+
+    def extract_from_xml_url(self, video_id, xml_url):
+        doc = self._download_xml(
+            xml_url, video_id,
+            note='Downloading video info',
+            errnote='Failed to download video info')
+
+        title = doc.find('.//information/title').text
+        description = xpath_text(doc, './/information/detail', 'description')
+        duration = int_or_none(xpath_text(doc, './/details/lengthSec', 'duration'))
+        uploader = xpath_text(doc, './/details/originChannelTitle', 'uploader')
+        uploader_id = xpath_text(doc, './/details/originChannelId', 'uploader id')
+        upload_date = unified_strdate(xpath_text(doc, './/details/airtime', 'upload date'))
+
+        def xml_to_thumbnails(fnode):
+            thumbnails = []
+            for node in fnode:
+                thumbnail_url = node.text
+                if not thumbnail_url:
+                    continue
+                thumbnail = {
+                    'url': thumbnail_url,
+                }
+                if 'key' in node.attrib:
+                    m = re.match('^([0-9]+)x([0-9]+)$', node.attrib['key'])
+                    if m:
+                        thumbnail['width'] = int(m.group(1))
+                        thumbnail['height'] = int(m.group(2))
+                thumbnails.append(thumbnail)
+            return thumbnails
+
+        thumbnails = xml_to_thumbnails(doc.findall('.//teaserimages/teaserimage'))
+
+        format_nodes = doc.findall('.//formitaeten/formitaet')
+        quality = qualities(['veryhigh', 'high', 'med', 'low'])
+
+        def get_quality(elem):
+            return quality(xpath_text(elem, 'quality'))
+        format_nodes.sort(key=get_quality)
+        format_ids = []
+        formats = []
+        for fnode in format_nodes:
+            video_url = fnode.find('url').text
+            is_available = 'http://www.metafilegenerator' not in video_url
+            if not is_available:
+                continue
+            format_id = fnode.attrib['basetype']
+            quality = xpath_text(fnode, './quality', 'quality')
+            format_m = re.match(r'''(?x)
+                (?P<vcodec>[^_]+)_(?P<acodec>[^_]+)_(?P<container>[^_]+)_
+                (?P<proto>[^_]+)_(?P<index>[^_]+)_(?P<indexproto>[^_]+)
+            ''', format_id)
+
+            ext = determine_ext(video_url, None) or format_m.group('container')
+            if ext not in ('smil', 'f4m', 'm3u8'):
+                format_id = format_id + '-' + quality
+            if format_id in format_ids:
+                continue
+
+            if ext == 'meta':
+                continue
+            elif ext == 'smil':
+                smil_formats = self._extract_smil_formats(
+                    video_url, video_id, fatal=False)
+                if smil_formats:
+                    formats.extend(smil_formats)
+            elif ext == 'm3u8':
+                m3u8_formats = self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
+            elif ext == 'f4m':
+                f4m_formats = self._extract_f4m_formats(
+                    video_url, video_id, f4m_id='hds', fatal=False)
+                if f4m_formats:
+                    formats.extend(f4m_formats)
+            else:
+                proto = format_m.group('proto').lower()
+
+                abr = int_or_none(xpath_text(fnode, './audioBitrate', 'abr'), 1000)
+                vbr = int_or_none(xpath_text(fnode, './videoBitrate', 'vbr'), 1000)
+
+                width = int_or_none(xpath_text(fnode, './width', 'width'))
+                height = int_or_none(xpath_text(fnode, './height', 'height'))
+
+                filesize = int_or_none(xpath_text(fnode, './filesize', 'filesize'))
+
+                format_note = ''
+                if not format_note:
+                    format_note = None
+
+                formats.append({
+                    'format_id': format_id,
+                    'url': video_url,
+                    'ext': ext,
+                    'acodec': format_m.group('acodec'),
+                    'vcodec': format_m.group('vcodec'),
+                    'abr': abr,
+                    'vbr': vbr,
+                    'width': width,
+                    'height': height,
+                    'filesize': filesize,
+                    'format_note': format_note,
+                    'protocol': proto,
+                    '_available': is_available,
+                })
+            format_ids.append(format_id)
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'thumbnails': thumbnails,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'upload_date': upload_date,
+            'formats': formats,
+        }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         xml_url = 'http://www.zdf.de/ZDFmediathek/xmlservice/web/beitragsDetails?ak=web&id=%s' % video_id
-        return extract_from_xml_url(self, video_id, xml_url)
+        return self.extract_from_xml_url(video_id, xml_url)
 
 
 class ZDFChannelIE(InfoExtractor):

From d497a201cab0eb588e9b7f75494f5494c9663e67 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Dec 2015 21:14:43 +0100
Subject: [PATCH 0199/1105] [common] use specific variable for protocol
 preference in _sort_formats

---
 youtube_dl/extractor/common.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6ab2d68d6..76f5b8b05 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -18,7 +18,6 @@ from ..compat import (
     compat_http_client,
     compat_urllib_error,
     compat_urllib_parse,
-    compat_urllib_parse_urlparse,
     compat_urlparse,
     compat_str,
     compat_etree_fromstring,
@@ -42,6 +41,7 @@ from ..utils import (
     url_basename,
     xpath_text,
     xpath_with_ns,
+    determine_protocol,
 )
 
 
@@ -776,14 +776,12 @@ class InfoExtractor(object):
 
             preference = f.get('preference')
             if preference is None:
-                proto = f.get('protocol')
-                if proto is None:
-                    proto = compat_urllib_parse_urlparse(f.get('url', '')).scheme
-
-                preference = 0 if proto in ['http', 'https'] else -0.1
+                preference = 0
                 if f.get('ext') in ['f4f', 'f4m']:  # Not yet supported
                     preference -= 0.5
 
+            proto_preference = 0 if determine_protocol(f) in ['http', 'https'] else -0.1
+
             if f.get('vcodec') == 'none':  # audio only
                 if self._downloader.params.get('prefer_free_formats'):
                     ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus']
@@ -814,6 +812,7 @@ class InfoExtractor(object):
                 f.get('vbr') if f.get('vbr') is not None else -1,
                 f.get('height') if f.get('height') is not None else -1,
                 f.get('width') if f.get('width') is not None else -1,
+                proto_preference,
                 ext_preference,
                 f.get('abr') if f.get('abr') is not None else -1,
                 audio_ext_preference,

From 7e5edcfd333f05c610bd70c33f3d3fd0ff6a7cec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Dec 2015 00:58:24 +0600
Subject: [PATCH 0200/1105] Simplify formats accumulation for f4m/m3u8/smil
 formats

Now all _extract_*_formats routines return a list
---
 youtube_dl/extractor/amp.py         | 12 ++++-------
 youtube_dl/extractor/ard.py         | 12 ++++-------
 youtube_dl/extractor/atresplayer.py | 10 +++------
 youtube_dl/extractor/audimedia.py   |  8 ++------
 youtube_dl/extractor/bbc.py         |  6 ++----
 youtube_dl/extractor/bloomberg.py   | 12 ++++-------
 youtube_dl/extractor/br.py          | 12 ++++-------
 youtube_dl/extractor/brightcove.py  |  6 ++----
 youtube_dl/extractor/common.py      | 16 +++++----------
 youtube_dl/extractor/dailymotion.py | 12 ++++-------
 youtube_dl/extractor/dcn.py         |  6 ++----
 youtube_dl/extractor/eitb.py        | 12 ++++-------
 youtube_dl/extractor/funimation.py  |  6 ++----
 youtube_dl/extractor/funnyordie.py  |  6 ++----
 youtube_dl/extractor/globo.py       |  6 ++----
 youtube_dl/extractor/hotstar.py     |  4 +---
 youtube_dl/extractor/ign.py         |  8 ++------
 youtube_dl/extractor/jwplatform.py  |  6 ++----
 youtube_dl/extractor/kaltura.py     |  6 ++----
 youtube_dl/extractor/livestream.py  | 32 +++++++++--------------------
 youtube_dl/extractor/nba.py         |  8 ++------
 youtube_dl/extractor/ooyala.py      | 12 +++--------
 youtube_dl/extractor/rai.py         | 12 ++++-------
 youtube_dl/extractor/rutube.py      | 12 ++++-------
 youtube_dl/extractor/ruutu.py       | 12 ++++-------
 youtube_dl/extractor/tele13.py      |  4 +---
 youtube_dl/extractor/toggle.py      |  6 ++----
 youtube_dl/extractor/vgtv.py        |  6 ++----
 youtube_dl/extractor/viki.py        |  6 ++----
 youtube_dl/extractor/vimeo.py       |  6 ++----
 youtube_dl/extractor/wdr.py         | 14 ++++---------
 youtube_dl/extractor/zdf.py         | 18 ++++++----------
 32 files changed, 99 insertions(+), 215 deletions(-)

diff --git a/youtube_dl/extractor/amp.py b/youtube_dl/extractor/amp.py
index dcc3c97f1..1035d1c48 100644
--- a/youtube_dl/extractor/amp.py
+++ b/youtube_dl/extractor/amp.py
@@ -53,16 +53,12 @@ class AMPIE(InfoExtractor):
             media = media_data['@attributes']
             media_type = media['type']
             if media_type == 'video/f4m':
-                f4m_formats = self._extract_f4m_formats(
+                formats.extend(self._extract_f4m_formats(
                     media['url'] + '?hdcore=3.4.0&plugin=aasp-3.4.0.132.124',
-                    video_id, f4m_id='hds', fatal=False)
-                if f4m_formats:
-                    formats.extend(f4m_formats)
+                    video_id, f4m_id='hds', fatal=False))
             elif media_type == 'application/x-mpegURL':
-                m3u8_formats = self._extract_m3u8_formats(
-                    media['url'], video_id, 'mp4', m3u8_id='hls', fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                formats.extend(self._extract_m3u8_formats(
+                    media['url'], video_id, 'mp4', m3u8_id='hls', fatal=False))
             else:
                 formats.append({
                     'format_id': media_data['media-category']['@attributes']['label'],
diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 687eb9f82..9fb84911a 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -113,16 +113,12 @@ class ARDMediathekIE(InfoExtractor):
                     if quality != 'auto' and ext in ('f4m', 'm3u8'):
                         continue
                     if ext == 'f4m':
-                        f4m_formats = self._extract_f4m_formats(
+                        formats.extend(self._extract_f4m_formats(
                             stream_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124',
-                            video_id, preference=-1, f4m_id='hds', fatal=False)
-                        if f4m_formats:
-                            formats.extend(f4m_formats)
+                            video_id, preference=-1, f4m_id='hds', fatal=False))
                     elif ext == 'm3u8':
-                        m3u8_formats = self._extract_m3u8_formats(
-                            stream_url, video_id, 'mp4', preference=1, m3u8_id='hls', fatal=False)
-                        if m3u8_formats:
-                            formats.extend(m3u8_formats)
+                        formats.extend(self._extract_m3u8_formats(
+                            stream_url, video_id, 'mp4', preference=1, m3u8_id='hls', fatal=False))
                     else:
                         if server and server.startswith('rtmp'):
                             f = {
diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 7ac3044c7..3fb042cea 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -134,10 +134,8 @@ class AtresPlayerIE(InfoExtractor):
 
         m3u8_url = player.get('urlVideoHls')
         if m3u8_url:
-            m3u8_formats = self._extract_m3u8_formats(
-                m3u8_url, episode_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
-            if m3u8_formats:
-                formats.extend(m3u8_formats)
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, episode_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
 
         timestamp = int_or_none(self._download_webpage(
             self._TIME_API_URL,
@@ -170,9 +168,7 @@ class AtresPlayerIE(InfoExtractor):
                 continue
             else:
                 f4m_url = video_url[:-9] + '/manifest.f4m'
-            f4m_formats = self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False)
-            if f4m_formats:
-                formats.extend(f4m_formats)
+            formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False))
         self._sort_formats(formats)
 
         path_data = player.get('pathData')
diff --git a/youtube_dl/extractor/audimedia.py b/youtube_dl/extractor/audimedia.py
index 4382a302b..9b037bb0c 100644
--- a/youtube_dl/extractor/audimedia.py
+++ b/youtube_dl/extractor/audimedia.py
@@ -45,15 +45,11 @@ class AudiMediaIE(InfoExtractor):
 
             stream_url_hls = json_data.get('stream_url_hls')
             if stream_url_hls:
-                m3u8_formats = self._extract_m3u8_formats(stream_url_hls, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                formats.extend(self._extract_m3u8_formats(stream_url_hls, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
 
             stream_url_hds = json_data.get('stream_url_hds')
             if stream_url_hds:
-                f4m_formats = self._extract_f4m_formats(json_data.get('stream_url_hds') + '?hdcore=3.4.0', video_id, -1, f4m_id='hds', fatal=False)
-                if f4m_formats:
-                    formats.extend(f4m_formats)
+                formats.extend(self._extract_f4m_formats(json_data.get('stream_url_hds') + '?hdcore=3.4.0', video_id, -1, f4m_id='hds', fatal=False))
 
             for video_version in json_data.get('video_versions'):
                 video_version_url = video_version.get('download_url') or video_version.get('stream_url')
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 691aecc0d..923273fb2 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -223,11 +223,9 @@ class BBCCoUkIE(InfoExtractor):
             elif transfer_format == 'dash':
                 pass
             elif transfer_format == 'hls':
-                m3u8_formats = self._extract_m3u8_formats(
+                formats.extend(self._extract_m3u8_formats(
                     href, programme_id, ext='mp4', entry_protocol='m3u8_native',
-                    m3u8_id=supplier, fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                    m3u8_id=supplier, fatal=False))
             # Direct link
             else:
                 formats.append({
diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index ebeef8f2a..13343bc25 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -41,15 +41,11 @@ class BloombergIE(InfoExtractor):
             if not stream_url:
                 continue
             if stream['muxing_format'] == 'TS':
-                m3u8_formats = self._extract_m3u8_formats(
-                    stream_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                formats.extend(self._extract_m3u8_formats(
+                    stream_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
             else:
-                f4m_formats = self._extract_f4m_formats(
-                    stream_url, video_id, f4m_id='hds', fatal=False)
-                if f4m_formats:
-                    formats.extend(f4m_formats)
+                formats.extend(self._extract_f4m_formats(
+                    stream_url, video_id, f4m_id='hds', fatal=False))
         self._sort_formats(formats)
 
         return {
diff --git a/youtube_dl/extractor/br.py b/youtube_dl/extractor/br.py
index e66854538..11cf49851 100644
--- a/youtube_dl/extractor/br.py
+++ b/youtube_dl/extractor/br.py
@@ -121,15 +121,11 @@ class BRIE(InfoExtractor):
             format_url = xpath_text(asset, ['downloadUrl', 'url'])
             asset_type = asset.get('type')
             if asset_type == 'HDS':
-                f4m_formats = self._extract_f4m_formats(
-                    format_url + '?hdcore=3.2.0', media_id, f4m_id='hds', fatal=False)
-                if f4m_formats:
-                    formats.extend(f4m_formats)
+                formats.extend(self._extract_f4m_formats(
+                    format_url + '?hdcore=3.2.0', media_id, f4m_id='hds', fatal=False))
             elif asset_type == 'HLS':
-                m3u8_formats = self._extract_m3u8_formats(
-                    format_url, media_id, 'mp4', 'm3u8_native', m3u8_id='hds', fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, media_id, 'mp4', 'm3u8_native', m3u8_id='hds', fatal=False))
             else:
                 format_info = {
                     'ext': xpath_text(asset, 'mediaType'),
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 03a4f446e..c947337f9 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -469,11 +469,9 @@ class BrightcoveNewIE(InfoExtractor):
             if source_type == 'application/x-mpegURL':
                 if not src:
                     continue
-                m3u8_formats = self._extract_m3u8_formats(
+                formats.extend(self._extract_m3u8_formats(
                     src, video_id, 'mp4', entry_protocol='m3u8_native',
-                    m3u8_id='hls', fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                    m3u8_id='hls', fatal=False))
             else:
                 streaming_src = source.get('streaming_src')
                 stream_name, app_name = source.get('stream_name'), source.get('app_name')
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 34a28c126..655207447 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -910,10 +910,8 @@ class InfoExtractor(object):
                 # may differ leading to inability to resolve the format by requested
                 # bitrate in f4m downloader
                 if determine_ext(manifest_url) == 'f4m':
-                    f4m_formats = self._extract_f4m_formats(
-                        manifest_url, video_id, preference, f4m_id, fatal=fatal)
-                    if f4m_formats:
-                        formats.extend(f4m_formats)
+                    formats.extend(self._extract_f4m_formats(
+                        manifest_url, video_id, preference, f4m_id, fatal=fatal))
                     continue
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({
@@ -1147,10 +1145,8 @@ class InfoExtractor(object):
             src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
 
             if proto == 'm3u8' or src_ext == 'm3u8':
-                m3u8_formats = self._extract_m3u8_formats(
-                    src_url, video_id, ext or 'mp4', m3u8_id='hls', fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                formats.extend(self._extract_m3u8_formats(
+                    src_url, video_id, ext or 'mp4', m3u8_id='hls', fatal=False))
                 continue
 
             if src_ext == 'f4m':
@@ -1162,9 +1158,7 @@ class InfoExtractor(object):
                     }
                 f4m_url += '&' if '?' in f4m_url else '?'
                 f4m_url += compat_urllib_parse.urlencode(f4m_params)
-                f4m_formats = self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False)
-                if f4m_formats:
-                    formats.extend(f4m_formats)
+                formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False))
                 continue
 
             if src_url.startswith('http') and self._is_valid_url(src, video_id):
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 0c5b6617f..439fd42e8 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -148,15 +148,11 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                         continue
                     ext = determine_ext(media_url)
                     if type_ == 'application/x-mpegURL' or ext == 'm3u8':
-                        m3u8_formats = self._extract_m3u8_formats(
-                            media_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
-                        if m3u8_formats:
-                            formats.extend(m3u8_formats)
+                        formats.extend(self._extract_m3u8_formats(
+                            media_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
                     elif type_ == 'application/f4m' or ext == 'f4m':
-                        f4m_formats = self._extract_f4m_formats(
-                            media_url, video_id, preference=-1, f4m_id='hds', fatal=False)
-                        if f4m_formats:
-                            formats.extend(f4m_formats)
+                        formats.extend(self._extract_f4m_formats(
+                            media_url, video_id, preference=-1, f4m_id='hds', fatal=False))
                     else:
                         f = {
                             'url': media_url,
diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 0d140f12f..8f48571de 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -56,10 +56,8 @@ class DCNBaseIE(InfoExtractor):
         m3u8_url = self._html_search_regex(
             r'file\s*:\s*"([^"]+)', webpage, 'm3u8 url', fatal=False)
         if m3u8_url:
-            m3u8_formats = self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=None)
-            if m3u8_formats:
-                formats.extend(m3u8_formats)
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=None))
 
         rtsp_url = self._search_regex(
             r'<a[^>]+href="(rtsp://[^"]+)"', webpage, 'rtsp url', fatal=False)
diff --git a/youtube_dl/extractor/eitb.py b/youtube_dl/extractor/eitb.py
index c83845fc2..713cb7b32 100644
--- a/youtube_dl/extractor/eitb.py
+++ b/youtube_dl/extractor/eitb.py
@@ -65,18 +65,14 @@ class EitbIE(InfoExtractor):
             if token_data:
                 token = token_data.get('token')
                 if token:
-                    m3u8_formats = self._extract_m3u8_formats(
-                        '%s?hdnts=%s' % (hls_url, token), video_id, m3u8_id='hls', fatal=False)
-                    if m3u8_formats:
-                        formats.extend(m3u8_formats)
+                    formats.extend(self._extract_m3u8_formats(
+                        '%s?hdnts=%s' % (hls_url, token), video_id, m3u8_id='hls', fatal=False))
 
         hds_url = media.get('HDS_SURL')
         if hds_url:
-            f4m_formats = self._extract_f4m_formats(
+            formats.extend(self._extract_f4m_formats(
                 '%s?hdcore=3.7.0' % hds_url.replace('euskalsvod', 'euskalvod'),
-                video_id, f4m_id='hds', fatal=False)
-            if f4m_formats:
-                formats.extend(f4m_formats)
+                video_id, f4m_id='hds', fatal=False))
 
         self._sort_formats(formats)
 
diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index d1a95d87f..0f37ed786 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -151,11 +151,9 @@ class FunimationIE(InfoExtractor):
                         errors.append(format_url)
                         continue
                     if determine_ext(format_url) == 'm3u8':
-                        m3u8_formats = self._extract_m3u8_formats(
+                        formats.extend(self._extract_m3u8_formats(
                             format_url + auth_token, display_id, 'mp4', entry_protocol='m3u8_native',
-                            preference=preference, m3u8_id='%s-hls' % funimation_id, fatal=False)
-                        if m3u8_formats:
-                            formats.extend(m3u8_formats)
+                            preference=preference, m3u8_id='%s-hls' % funimation_id, fatal=False))
                     else:
                         tbr = int_or_none(self._search_regex(
                             r'-(\d+)[Kk]', format_url, 'tbr', default=None))
diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index 7f21d7410..4c4a87e2a 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -51,10 +51,8 @@ class FunnyOrDieIE(InfoExtractor):
 
         formats = []
 
-        m3u8_formats = self._extract_m3u8_formats(
-            m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
-        if m3u8_formats:
-            formats.extend(m3u8_formats)
+        formats.extend(self._extract_m3u8_formats(
+            m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
 
         bitrates = [int(bitrate) for bitrate in re.findall(r'[,/]v(\d+)[,/]', m3u8_url)]
         bitrates.sort()
diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index c65ef6bcf..b241c4868 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -369,11 +369,9 @@ class GloboIE(InfoExtractor):
             resource_url = resource['url']
             signed_url = '%s?h=%s&k=%s' % (resource_url, signed_hash, 'flash')
             if resource_id.endswith('m3u8') or resource_url.endswith('.m3u8'):
-                m3u8_formats = self._extract_m3u8_formats(
+                formats.extend(self._extract_m3u8_formats(
                     signed_url, resource_id, 'mp4', entry_protocol='m3u8_native',
-                    m3u8_id='hls', fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                    m3u8_id='hls', fatal=False))
             else:
                 formats.append({
                     'url': signed_url,
diff --git a/youtube_dl/extractor/hotstar.py b/youtube_dl/extractor/hotstar.py
index 05d27e75d..a7c3ce4ab 100644
--- a/youtube_dl/extractor/hotstar.py
+++ b/youtube_dl/extractor/hotstar.py
@@ -55,9 +55,7 @@ class HotStarIE(InfoExtractor):
                 format_url = format_data['src']
                 ext = determine_ext(format_url)
                 if ext == 'm3u8':
-                    m3u8_formats = self._extract_m3u8_formats(format_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
-                    if m3u8_formats:
-                        formats.extend(m3u8_formats)
+                    formats.extend(self._extract_m3u8_formats(format_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
                 elif ext == 'f4m':
                     # produce broken files
                     continue
diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index a2e18c8a7..d1c1c210c 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -129,14 +129,10 @@ class IGNIE(InfoExtractor):
         formats = []
         m3u8_url = api_data['refs'].get('m3uUrl')
         if m3u8_url:
-            m3u8_formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
-            if m3u8_formats:
-                formats.extend(m3u8_formats)
+            formats.extend(self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
         f4m_url = api_data['refs'].get('f4mUrl')
         if f4m_url:
-            f4m_formats = self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False)
-            if f4m_formats:
-                formats.extend(f4m_formats)
+            formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False))
         for asset in api_data['assets']:
             formats.append({
                 'url': asset['url'],
diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index a92adf2b3..8e90d5986 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -44,10 +44,8 @@ class JWPlatformIE(InfoExtractor):
             source_url = self._proto_relative_url(source['file'])
             source_type = source.get('type') or ''
             if source_type == 'application/vnd.apple.mpegurl':
-                m3u8_formats = self._extract_m3u8_formats(
-                    source_url, video_id, 'mp4', 'm3u8_native', fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                formats.extend(self._extract_m3u8_formats(
+                    source_url, video_id, 'mp4', 'm3u8_native', fatal=False))
             elif source_type.startswith('audio'):
                 formats.append({
                     'url': source_url,
diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 4807c8110..ccbc39c66 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -163,10 +163,8 @@ class KalturaIE(InfoExtractor):
         m3u8_url = info['dataUrl'].replace('format/url', 'format/applehttp')
         if referrer:
             m3u8_url += '?referrer=%s' % referrer
-        m3u8_formats = self._extract_m3u8_formats(
-            m3u8_url, entry_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
-        if m3u8_formats:
-            formats.extend(m3u8_formats)
+        formats.extend(self._extract_m3u8_formats(
+            m3u8_url, entry_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
 
         self._check_formats(formats, entry_id)
         self._sort_formats(formats)
diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 688eb2308..38fb3d9e4 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -110,23 +110,17 @@ class LivestreamIE(InfoExtractor):
 
         smil_url = video_data.get('smil_url')
         if smil_url:
-            smil_formats = self._extract_smil_formats(smil_url, video_id)
-            if smil_formats:
-                formats.extend(smil_formats)
+            formats.extend(self._extract_smil_formats(smil_url, video_id))
 
         m3u8_url = video_data.get('m3u8_url')
         if m3u8_url:
-            m3u8_formats = self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
-            if m3u8_formats:
-                formats.extend(m3u8_formats)
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
 
         f4m_url = video_data.get('f4m_url')
         if f4m_url:
-            f4m_formats = self._extract_f4m_formats(
-                f4m_url, video_id, f4m_id='hds', fatal=False)
-            if f4m_formats:
-                formats.extend(f4m_formats)
+            formats.extend(self._extract_f4m_formats(
+                f4m_url, video_id, f4m_id='hds', fatal=False))
         self._sort_formats(formats)
 
         comments = [{
@@ -158,17 +152,13 @@ class LivestreamIE(InfoExtractor):
         formats = []
         smil_url = stream_info.get('play_url')
         if smil_url:
-            smil_formats = self._extract_smil_formats(smil_url, broadcast_id)
-            if smil_formats:
-                formats.extend(smil_formats)
+            formats.extend(self._extract_smil_formats(smil_url, broadcast_id))
 
         entry_protocol = 'm3u8' if is_live else 'm3u8_native'
         m3u8_url = stream_info.get('m3u8_url')
         if m3u8_url:
-            m3u8_formats = self._extract_m3u8_formats(
-                m3u8_url, broadcast_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=False)
-            if m3u8_formats:
-                formats.extend(m3u8_formats)
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, broadcast_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=False))
 
         rtsp_url = stream_info.get('rtsp_url')
         if rtsp_url:
@@ -293,10 +283,8 @@ class LivestreamOriginalIE(InfoExtractor):
 
         m3u8_url = video_data.get('httpUrl')
         if m3u8_url:
-            m3u8_formats = self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=False)
-            if m3u8_formats:
-                formats.extend(m3u8_formats)
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=False))
 
         rtsp_url = video_data.get('rtspUrl')
         if rtsp_url:
diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 7c6b7841d..9d26030d3 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -68,13 +68,9 @@ class NBAIE(InfoExtractor):
             if video_url.startswith('/'):
                 continue
             if video_url.endswith('.m3u8'):
-                m3u8_formats = self._extract_m3u8_formats(video_url, video_id, m3u8_id='hls', fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                formats.extend(self._extract_m3u8_formats(video_url, video_id, m3u8_id='hls', fatal=False))
             elif video_url.endswith('.f4m'):
-                f4m_formats = self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id, f4m_id='hds', fatal=False)
-                if f4m_formats:
-                    formats.extend(f4m_formats)
+                formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id, f4m_id='hds', fatal=False))
             else:
                 key = video_file.attrib.get('bitrate')
                 format_info = {
diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 8603fd692..3960d522e 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -44,17 +44,11 @@ class OoyalaBaseIE(InfoExtractor):
                     urls.append(url)
                     delivery_type = stream['delivery_type']
                     if delivery_type == 'hls' or '.m3u8' in url:
-                        m3u8_formats = self._extract_m3u8_formats(url, embed_code, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
-                        if m3u8_formats:
-                            formats.extend(m3u8_formats)
+                        formats.extend(self._extract_m3u8_formats(url, embed_code, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
                     elif delivery_type == 'hds' or '.f4m' in url:
-                        f4m_formats = self._extract_f4m_formats(url, embed_code, f4m_id='hds', fatal=False)
-                        if f4m_formats:
-                            formats.extend(f4m_formats)
+                        formats.extend(self._extract_f4m_formats(url, embed_code, f4m_id='hds', fatal=False))
                     elif '.smil' in url:
-                        smil_formats = self._extract_smil_formats(url, embed_code, fatal=False)
-                        if smil_formats:
-                            formats.extend(smil_formats)
+                        formats.extend(self._extract_smil_formats(url, embed_code, fatal=False))
                     else:
                         formats.append({
                             'url': url,
diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 278b1d2bf..f2679591b 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -116,17 +116,13 @@ class RaiTVIE(InfoExtractor):
                 ext = determine_ext(media_url)
                 content_type = xpath_text(element, 'content-type')
                 if ext == 'm3u8':
-                    m3u8_formats = self._extract_m3u8_formats(
+                    formats.extend(self._extract_m3u8_formats(
                         media_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls',
-                        fatal=False)
-                    if m3u8_formats:
-                        formats.extend(m3u8_formats)
+                        fatal=False))
                 elif ext == 'f4m':
-                    f4m_formats = self._extract_f4m_formats(
+                    formats.extend(self._extract_f4m_formats(
                         media_url + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44',
-                        video_id, f4m_id='hds', fatal=False)
-                    if f4m_formats:
-                        formats.extend(f4m_formats)
+                        video_id, f4m_id='hds', fatal=False))
                 elif ext == 'stl':
                     has_subtitle = True
                 elif content_type.startswith('video/'):
diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index 9db62adb1..c5c47d01e 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -58,15 +58,11 @@ class RutubeIE(InfoExtractor):
         for format_id, format_url in options['video_balancer'].items():
             ext = determine_ext(format_url)
             if ext == 'm3u8':
-                m3u8_formats = self._extract_m3u8_formats(
-                    format_url, video_id, 'mp4', m3u8_id=format_id, fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', m3u8_id=format_id, fatal=False))
             elif ext == 'f4m':
-                f4m_formats = self._extract_f4m_formats(
-                    format_url, video_id, f4m_id=format_id, fatal=False)
-                if f4m_formats:
-                    formats.extend(f4m_formats)
+                formats.extend(self._extract_f4m_formats(
+                    format_url, video_id, f4m_id=format_id, fatal=False))
             else:
                 formats.append({
                     'url': format_url,
diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index e417bf661..41fddc375 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -63,15 +63,11 @@ class RuutuIE(InfoExtractor):
                     processed_urls.append(video_url)
                     ext = determine_ext(video_url)
                     if ext == 'm3u8':
-                        m3u8_formats = self._extract_m3u8_formats(
-                            video_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
-                        if m3u8_formats:
-                            formats.extend(m3u8_formats)
+                        formats.extend(self._extract_m3u8_formats(
+                            video_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
                     elif ext == 'f4m':
-                        f4m_formats = self._extract_f4m_formats(
-                            video_url, video_id, f4m_id='hds', fatal=False)
-                        if f4m_formats:
-                            formats.extend(f4m_formats)
+                        formats.extend(self._extract_f4m_formats(
+                            video_url, video_id, f4m_id='hds', fatal=False))
                     else:
                         proto = compat_urllib_parse_urlparse(video_url).scheme
                         if not child.tag.startswith('HTTP') and proto != 'rtmp':
diff --git a/youtube_dl/extractor/tele13.py b/youtube_dl/extractor/tele13.py
index a363b4d40..fe11c20df 100644
--- a/youtube_dl/extractor/tele13.py
+++ b/youtube_dl/extractor/tele13.py
@@ -57,9 +57,7 @@ class Tele13IE(InfoExtractor):
             if format_url and format_url not in urls:
                 ext = determine_ext(format_url)
                 if ext == 'm3u8':
-                    m3u8_formats = self._extract_m3u8_formats(format_url, display_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
-                    if m3u8_formats:
-                        formats.extend(m3u8_formats)
+                    formats.extend(self._extract_m3u8_formats(format_url, display_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
                 elif YoutubeIE.suitable(format_url):
                     return self.url_result(format_url, 'Youtube')
                 else:
diff --git a/youtube_dl/extractor/toggle.py b/youtube_dl/extractor/toggle.py
index a47239952..c54b876d3 100644
--- a/youtube_dl/extractor/toggle.py
+++ b/youtube_dl/extractor/toggle.py
@@ -132,13 +132,11 @@ class ToggleIE(InfoExtractor):
             vid_format = vid_format.replace(' ', '')
             # if geo-restricted, m3u8 is inaccessible, but mp4 is okay
             if ext == 'm3u8':
-                m3u8_formats = self._extract_m3u8_formats(
+                formats.extend(self._extract_m3u8_formats(
                     video_url, video_id, ext='mp4', m3u8_id=vid_format,
                     note='Downloading %s m3u8 information' % vid_format,
                     errnote='Failed to download %s m3u8 information' % vid_format,
-                    fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                    fatal=False))
             elif ext in ('mp4', 'wvm'):
                 # wvm are drm-protected files
                 formats.append({
diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 129668a99..86ba70ed9 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -151,10 +151,8 @@ class VGTVIE(XstreamIE):
 
         hls_url = streams.get('hls')
         if hls_url:
-            m3u8_formats = self._extract_m3u8_formats(
-                hls_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
-            if m3u8_formats:
-                formats.extend(m3u8_formats)
+            formats.extend(self._extract_m3u8_formats(
+                hls_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
 
         hds_url = streams.get('hds')
         if hds_url:
diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 9a1c377a4..433fc9914 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -277,11 +277,9 @@ class VikiIE(VikiBaseIE):
                 r'^(\d+)[pP]$', format_id, 'height', default=None))
             for protocol, format_dict in stream_dict.items():
                 if format_id == 'm3u8':
-                    m3u8_formats = self._extract_m3u8_formats(
+                    formats.extend(self._extract_m3u8_formats(
                         format_dict['url'], video_id, 'mp4', 'm3u8_native',
-                        m3u8_id='m3u8-%s' % protocol, fatal=False)
-                    if m3u8_formats:
-                        formats.extend(m3u8_formats)
+                        m3u8_id='m3u8-%s' % protocol, fatal=False))
                 else:
                     formats.append({
                         'url': format_dict['url'],
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index ce08e6955..7af699982 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -437,10 +437,8 @@ class VimeoIE(VimeoBaseInfoExtractor):
             })
         m3u8_url = config_files.get('hls', {}).get('url')
         if m3u8_url:
-            m3u8_formats = self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
-            if m3u8_formats:
-                formats.extend(m3u8_formats)
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
         # Bitrates are completely broken. Single m3u8 may contain entries in kbps and bps
         # at the same time without actual units specified. This lead to wrong sorting.
         self._sort_formats(formats, field_preference=('preference', 'height', 'width', 'fps', 'format_id'))
diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index ef096cbd2..e8511398c 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -157,16 +157,12 @@ class WDRIE(InfoExtractor):
         preference = qualities(['S', 'M', 'L', 'XL'])
 
         if video_url.endswith('.f4m'):
-            f4m_formats = self._extract_f4m_formats(video_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18', page_id, f4m_id='hds', fatal=False)
-            if f4m_formats:
-                formats.extend(f4m_formats)
+            formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18', page_id, f4m_id='hds', fatal=False))
         elif video_url.endswith('.smil'):
-            smil_formats = self._extract_smil_formats(video_url, page_id, False, {
+            formats.extend(self._extract_smil_formats(video_url, page_id, False, {
                 'hdcore': '3.3.0',
                 'plugin': 'aasp-3.3.0.99.43',
-            })
-            if smil_formats:
-                formats.extend(smil_formats)
+            }))
         else:
             formats.append({
                 'url': video_url,
@@ -177,9 +173,7 @@ class WDRIE(InfoExtractor):
 
         m3u8_url = self._search_regex(r'rel="adaptiv"[^>]+href="([^"]+)"', webpage, 'm3u8 url', default=None)
         if m3u8_url:
-            m3u8_formats = self._extract_m3u8_formats(m3u8_url, page_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
-            if m3u8_formats:
-                formats.extend(m3u8_formats)
+            formats.extend(self._extract_m3u8_formats(m3u8_url, page_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
 
         direct_urls = re.findall(r'rel="web(S|M|L|XL)"[^>]+href="([^"]+)"', webpage)
         if direct_urls:
diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index 92c12bac6..2a1f2f6d1 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -125,20 +125,14 @@ class ZDFIE(InfoExtractor):
             if ext == 'meta':
                 continue
             elif ext == 'smil':
-                smil_formats = self._extract_smil_formats(
-                    video_url, video_id, fatal=False)
-                if smil_formats:
-                    formats.extend(smil_formats)
+                formats.extend(self._extract_smil_formats(
+                    video_url, video_id, fatal=False))
             elif ext == 'm3u8':
-                m3u8_formats = self._extract_m3u8_formats(
-                    video_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
             elif ext == 'f4m':
-                f4m_formats = self._extract_f4m_formats(
-                    video_url, video_id, f4m_id='hds', fatal=False)
-                if f4m_formats:
-                    formats.extend(f4m_formats)
+                formats.extend(self._extract_f4m_formats(
+                    video_url, video_id, f4m_id='hds', fatal=False))
             else:
                 proto = format_m.group('proto').lower()
 

From 7447661e9b433d8871d0a898c9af271d5a49ec82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 28 Dec 2015 19:23:54 +0100
Subject: [PATCH 0201/1105] [franceculture] Fix test

---
 youtube_dl/extractor/franceculture.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/franceculture.py b/youtube_dl/extractor/franceculture.py
index 1e83a4e7e..2ac6157cd 100644
--- a/youtube_dl/extractor/franceculture.py
+++ b/youtube_dl/extractor/franceculture.py
@@ -22,8 +22,8 @@ class FranceCultureIE(InfoExtractor):
             'alt_title': 'Carnet nomade | 13-14',
             'vcodec': 'none',
             'upload_date': '20140301',
-            'thumbnail': r're:^http://www\.franceculture\.fr/.*/images/player/Carnet-nomade\.jpg$',
-            'description': 'startswith:Avec :Jean-Baptiste Péretié pour son documentaire sur Arte "La revanche des « geeks », une enquête menée aux Etats',
+            'thumbnail': r're:^http://static\.franceculture\.fr/.*/images/player/Carnet-nomade\.jpg$',
+            'description': 'startswith:Avec :Jean-Baptiste Péretié pour son documentaire sur Arte "La revanche',
             'timestamp': 1393700400,
         }
     }

From ecf17d165317078d0e31191e40903a74fc3da3e8 Mon Sep 17 00:00:00 2001
From: flatgreen <flatgreen@gmail.com>
Date: Sun, 27 Dec 2015 15:30:45 +0100
Subject: [PATCH 0202/1105] [franceculture] Add extractor for '/emission-*'
 urls (closes #3777, closes #8022)

---
 youtube_dl/extractor/__init__.py      |  5 +++-
 youtube_dl/extractor/franceculture.py | 40 +++++++++++++++++++++++++--
 2 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 971047ad4..68c9c1288 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -203,7 +203,10 @@ from .fourtube import FourTubeIE
 from .foxgay import FoxgayIE
 from .foxnews import FoxNewsIE
 from .foxsports import FoxSportsIE
-from .franceculture import FranceCultureIE
+from .franceculture import (
+    FranceCultureIE,
+    FranceCultureEmissionIE,
+)
 from .franceinter import FranceInterIE
 from .francetv import (
     PluzzIE,
diff --git a/youtube_dl/extractor/franceculture.py b/youtube_dl/extractor/franceculture.py
index 2ac6157cd..e2ca96283 100644
--- a/youtube_dl/extractor/franceculture.py
+++ b/youtube_dl/extractor/franceculture.py
@@ -8,6 +8,7 @@ from ..compat import (
 from ..utils import (
     determine_ext,
     int_or_none,
+    ExtractorError,
 )
 
 
@@ -28,8 +29,7 @@ class FranceCultureIE(InfoExtractor):
         }
     }
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
+    def _extract_from_player(self, url, video_id):
         webpage = self._download_webpage(url, video_id)
 
         video_path = self._search_regex(
@@ -42,6 +42,9 @@ class FranceCultureIE(InfoExtractor):
             r'<a id="player".*?>\s+<img src="([^"]+)"',
             webpage, 'thumbnail', fatal=False)
 
+        display_id = self._search_regex(
+            r'<span class="path-diffusion">emission-(.*?)</span>', webpage, 'display_id')
+
         title = self._html_search_regex(
             r'<span class="title-diffusion">(.*?)</span>', webpage, 'title')
         alt_title = self._html_search_regex(
@@ -66,4 +69,37 @@ class FranceCultureIE(InfoExtractor):
             'alt_title': alt_title,
             'thumbnail': thumbnail,
             'description': description,
+            'display_id': display_id,
         }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        return self._extract_from_player(url, video_id)
+
+
+class FranceCultureEmissionIE(FranceCultureIE):
+    _VALID_URL = r'https?://(?:www\.)?franceculture\.fr/emission-(?P<id>[^?#]+)'
+    _TEST = {
+        'url': 'http://www.franceculture.fr/emission-les-carnets-de-la-creation-jean-gabriel-periot-cineaste-2015-10-13',
+        'info_dict': {
+            'title': 'Jean-Gabriel Périot, cinéaste',
+            'alt_title': 'Les Carnets de la création',
+            'id': '5093239',
+            'display_id': 'les-carnets-de-la-creation-jean-gabriel-periot-cineaste-2015-10-13',
+            'ext': 'mp3',
+            'timestamp': 1444762500,
+            'upload_date': '20151013',
+            'description': 'startswith:Aujourd\'hui dans "Les carnets de la création", le cinéaste',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        video_path = self._html_search_regex(
+            r'<a class="rf-player-open".*?href="([^"]+)"', webpage, 'video path', 'no_path_player')
+        if video_path == 'no_path_player':
+            raise ExtractorError('no player : no sound in this page.', expected=True)
+        new_id = self._search_regex('play=(?P<id>[0-9]+)', video_path, 'new_id', group='id')
+        video_url = compat_urlparse.urljoin(url, video_path)
+        return self._extract_from_player(video_url, new_id)

From e974356f32aa31ac8d9e5f2a664c6c521fb45892 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 29 Dec 2015 11:02:58 +0100
Subject: [PATCH 0203/1105] release 2015.12.29

---
 docs/supportedsites.md | 16 ++++++++++++++--
 youtube_dl/version.py  |  2 +-
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 1a5c7cde9..1646277ec 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -37,6 +37,7 @@
  - **appletrailers:section**
  - **archive.org**: archive.org videos
  - **ARD**
+ - **ARD:mediathek**: Saarländischer Rundfunk
  - **ARD:mediathek**
  - **arte.tv**
  - **arte.tv:+7**
@@ -124,8 +125,12 @@
  - **dailymotion:user**
  - **DailymotionCloud**
  - **daum.net**
+ - **daum.net**
  - **DBTV**
  - **DCN**
+ - **dcn:live**
+ - **dcn:season**
+ - **dcn:video**
  - **DctpTv**
  - **DeezerPlaylist**
  - **defense.gouv.fr**
@@ -181,6 +186,7 @@
  - **FoxSports**
  - **france2.fr:generation-quoi**
  - **FranceCulture**
+ - **FranceCultureEmission**
  - **FranceInter**
  - **francetv**: France 2, 3, 4, 5 and Ô
  - **francetvinfo.fr**
@@ -225,6 +231,7 @@
  - **hitbox:live**
  - **HornBunny**
  - **HotNewHipHop**
+ - **HotStar**
  - **Howcast**
  - **HowStuffWorks**
  - **HuffPost**: Huffington Post
@@ -405,6 +412,7 @@
  - **parliamentlive.tv**: UK parliament videos
  - **Patreon**
  - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), CONNECTICUT PUBLIC TELEVISION (WEDH), KNPB Channel 5 (KNPB), SOPTV (KSYS), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), PBS 39 (WLVT), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WGCU Public Media (WGCU), WPBT2 (WPBT), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), ETV (WRLK), UNC-TV (WUNC), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), OPB (KOPB), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WTTW11 (WTTW), Iowa Public Television/IPTV (KDIN), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), WIPB-TV (WIPB), WTIU (WTIU), CET  (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), KSMQ (KSMQ), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), East Tennessee PBS (WSJK), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVPB (WVPB), WKYU-PBS (WKYU), KERA 13 (KERA), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), witf (WITF), WQED Multimedia (WQED), WMHT Educational Telecommunications (WMHT), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WGTE TV (WGTE), Lakeland Public Television (KAWE), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), MontanaPBS (KUSM), KRWG/Channel 22 (KRWG), KACV (KACV), KCOS/Channel 13 (KCOS), WCNY/Channel 24 (WCNY), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), WPSU (WPSU), WVIA Public Media Studios (WVIA), WTVI (WTVI), Western Reserve PBS (WNEO), WVIZ/PBS ideastream (WVIZ), KCTS 9 (KCTS), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KLRN (KLRN), KLRU (KLRU), WTJX Channel 12 (WTJX), WCVE PBS (WCVE), KBTC Public Television (KBTC)
+ - **pcmag**
  - **Periscope**: Periscope
  - **PhilharmonieDeParis**: Philharmonie de Paris
  - **Phoenix**
@@ -445,6 +453,7 @@
  - **radiofrance**
  - **RadioJavan**
  - **Rai**
+ - **RaiTV**
  - **RBMARadio**
  - **RDS**: RDS.ca
  - **RedTube**
@@ -527,7 +536,6 @@
  - **SportDeutschland**
  - **Sportschau**
  - **Srf**
- - **SRMediathek**: Saarländischer Rundfunk
  - **SSA**
  - **stanfordoc**: Stanford Open ClassRoom
  - **Steam**
@@ -561,6 +569,7 @@
  - **TenPlay**
  - **TestTube**
  - **TF1**
+ - **TheIntercept**
  - **TheOnion**
  - **ThePlatform**
  - **ThePlatformFeed**
@@ -586,7 +595,10 @@
  - **TubiTv**
  - **Tudou**
  - **Tumblr**
- - **TuneIn**
+ - **tunein:clip**
+ - **tunein:program**
+ - **tunein:station**
+ - **tunein:topic**
  - **Turbo**
  - **Tutv**
  - **tv.dfb.de**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 255d64269..a62baa305 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.12.23'
+__version__ = '2015.12.29'

From 422f7c112c12d92bf06c95ff33782c8e9a0df43e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 11:36:04 +0100
Subject: [PATCH 0204/1105] [srgssr] update tests

---
 youtube_dl/extractor/rts.py    | 30 +++++++++++++++++----------
 youtube_dl/extractor/srgssr.py | 38 ++++++++++++----------------------
 2 files changed, 32 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index dc10fd88b..f063ab5dd 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -28,7 +28,7 @@ class RTSIE(SRGSSRIE):
             'info_dict': {
                 'id': '3449373',
                 'display_id': 'les-enfants-terribles',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'duration': 1488,
                 'title': 'Les Enfants Terribles',
                 'description': 'France Pommier et sa soeur Luce Feral, les deux filles de ce groupe de 5.',
@@ -38,6 +38,10 @@ class RTSIE(SRGSSRIE):
                 'thumbnail': 're:^https?://.*\.image',
                 'view_count': int,
             },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            }
         },
         {
             'url': 'http://www.rts.ch/emissions/passe-moi-les-jumelles/5624067-entre-ciel-et-mer.html',
@@ -45,7 +49,7 @@ class RTSIE(SRGSSRIE):
             'info_dict': {
                 'id': '5742494',
                 'display_id': '5742494',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'duration': 3720,
                 'title': 'Les yeux dans les cieux - Mon homard au Canada',
                 'description': 'md5:d22ee46f5cc5bac0912e5a0c6d44a9f7',
@@ -55,6 +59,10 @@ class RTSIE(SRGSSRIE):
                 'thumbnail': 're:^https?://.*\.image',
                 'view_count': int,
             },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            }
         },
         {
             'url': 'http://www.rts.ch/video/sport/hockey/5745975-1-2-kloten-fribourg-5-2-second-but-pour-gotteron-par-kwiatowski.html',
@@ -80,7 +88,7 @@ class RTSIE(SRGSSRIE):
             'info_dict': {
                 'id': '5745356',
                 'display_id': 'londres-cachee-par-un-epais-smog',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'duration': 33,
                 'title': 'Londres cachée par un épais smog',
                 'description': 'Un important voile de smog recouvre Londres depuis mercredi, provoqué par la pollution et du sable du Sahara.',
@@ -90,6 +98,10 @@ class RTSIE(SRGSSRIE):
                 'thumbnail': 're:^https?://.*\.image',
                 'view_count': int,
             },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            }
         },
         {
             'url': 'http://www.rts.ch/audio/couleur3/programmes/la-belle-video-de-stephane-laurenceau/5706148-urban-hippie-de-damien-krisl-03-04-2014.html',
@@ -180,16 +192,12 @@ class RTSIE(SRGSSRIE):
                 auth_params = xpath_text(token, './/authparams', 'auth params')
                 if not auth_params:
                     continue
-                f4m_formats = self._extract_f4m_formats(
+                formats.extend(self._extract_f4m_formats(
                     '%s?%s&hdcore=3.4.0&plugin=aasp-3.4.0.132.66' % (format_url, auth_params),
-                    media_id, f4m_id=format_id, fatal=False)
-                if f4m_formats:
-                    formats.extend(f4m_formats)
+                    media_id, f4m_id=format_id, fatal=False))
             elif format_url.endswith('.m3u8'):
-                m3u8_formats = self._extract_m3u8_formats(
-                    format_url, media_id, 'mp4', m3u8_id=format_id, fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, media_id, 'mp4', m3u8_id=format_id, fatal=False))
             else:
                 formats.append({
                     'format_id': format_id,
diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py
index 563851b3f..4962fe885 100644
--- a/youtube_dl/extractor/srgssr.py
+++ b/youtube_dl/extractor/srgssr.py
@@ -47,31 +47,24 @@ class SRGSSRIE(InfoExtractor):
         created_date = media_data.get('createdDate') or metadata.get('createdDate')
         timestamp = parse_iso8601(created_date)
 
-        thumbnails = []
-        if 'Image' in media_data:
-            for image in media_data['Image']['ImageRepresentations']['ImageRepresentation']:
-                thumbnails.append({
-                    'id': image.get('id'),
-                    'url': image['url'],
-                })
+        thumbnails = [{
+            'id': image.get('id'),
+            'url': image['url'],
+        } for image in media_data.get('Image', {}).get('ImageRepresentations', {}).get('ImageRepresentation', [])]
 
         preference = qualities(['LQ', 'MQ', 'SD', 'HQ', 'HD'])
         formats = []
-        for source in media_data['Playlists']['Playlist']:
-            assets = {}
+        for source in media_data.get('Playlists', {}).get('Playlist', []) + media_data.get('Downloads', {}).get('Download', []):
             protocol = source.get('@protocol')
             if protocol in ('HTTP-HDS', 'HTTP-HLS'):
+                assets = {}
                 for quality in source['url']:
                     assets[quality['@quality']] = quality['text']
                 asset_url = assets.get('HD') or assets.get('HQ') or assets.get('SD') or assets.get('MQ') or assets.get('LQ')
                 if '.f4m' in asset_url:
-                    f4m_formats = formats.extend(self._extract_f4m_formats(asset_url + '?hdcore=3.4.0', media_id, f4m_id='hds', fatal=False))
-                    if f4m_formats:
-                        formats.extend(f4m_formats)
+                    formats.extend(self._extract_f4m_formats(asset_url + '?hdcore=3.4.0', media_id, f4m_id='hds', fatal=False))
                 elif '.m3u8' in asset_url:
-                    m3u8_formats = formats.extend(self._extract_m3u8_formats(asset_url, media_id, m3u8_id='hls', fatal=False))
-                    if m3u8_formats:
-                        formats.extend(m3u8_formats)
+                    formats.extend(self._extract_m3u8_formats(asset_url, media_id, m3u8_id='hls', fatal=False))
             else:
                 for asset in source['url']:
                     asset_url = asset['text']
@@ -84,15 +77,6 @@ class SRGSSRIE(InfoExtractor):
                         'preference': preference(asset['@quality']),
                         'ext': ext,
                     })
-
-        if 'Downloads' in media_data:
-            for source in media_data['Downloads']['Download']:
-                for asset in source['url']:
-                    formats.append({
-                        'format_id': asset['@quality'],
-                        'url': asset['text'],
-                        'preference': preference(asset['@quality'])
-                    })
         self._sort_formats(formats)
 
         return {
@@ -149,7 +133,7 @@ class SRGSSRPlayIE(InfoExtractor):
         'info_dict': {
             'id': '6348260',
             'display_id': '6348260',
-            'ext': 'flv',
+            'ext': 'mp4',
             'duration': 1796,
             'title': 'Le 19h30',
             'description': '',
@@ -159,6 +143,10 @@ class SRGSSRPlayIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.image',
             'view_count': int,
         },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
     }]
 
     def _real_extract(self, url):

From db710571fd7bf6b697a7999fa1ea76dd2b8b1382 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 11:59:27 +0100
Subject: [PATCH 0205/1105] [daum] fix IE_NAME

---
 youtube_dl/extractor/daum.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index a083cc0dc..f08f57157 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -77,7 +77,7 @@ class DaumIE(InfoExtractor):
 
 class DaumClipIE(InfoExtractor):
     _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/(?:clip/ClipView.do|mypot/View.do)\?.*?clipid=(?P<id>\d+)'
-    IE_NAME = 'daum.net'
+    IE_NAME = 'daum.net:clip'
 
     _TESTS = [{
         'url': 'http://tvpot.daum.net/clip/ClipView.do?clipid=52554690',

From 6c4d6609ad14d33ef1a9b13a938a9413ac438e2c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 12:00:52 +0100
Subject: [PATCH 0206/1105] [phoenix] fix IE_NAME

---
 youtube_dl/extractor/phoenix.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/phoenix.py b/youtube_dl/extractor/phoenix.py
index 6ce2ec19d..ac009f60f 100644
--- a/youtube_dl/extractor/phoenix.py
+++ b/youtube_dl/extractor/phoenix.py
@@ -4,6 +4,7 @@ from .zdf import ZDFIE
 
 
 class PhoenixIE(ZDFIE):
+    IE_NAME = 'phoenix.de'
     _VALID_URL = r'''(?x)https?://(?:www\.)?phoenix\.de/content/
         (?:
             phoenix/die_sendungen/(?:[^/]+/)?

From ce7d243c7e6b6f5eb5129a3e85aa75a3773b3a73 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 12:01:22 +0100
Subject: [PATCH 0207/1105] [srgssr] fix IE_DESC

---
 youtube_dl/extractor/srgssr.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py
index 4962fe885..2c8b4276b 100644
--- a/youtube_dl/extractor/srgssr.py
+++ b/youtube_dl/extractor/srgssr.py
@@ -90,6 +90,7 @@ class SRGSSRIE(InfoExtractor):
 
 
 class SRGSSRPlayIE(InfoExtractor):
+    IE_DESC = 'srf.ch, rts.ch, rsi.ch, rtr.ch and swissinfo.ch play sites'
     _VALID_URL = r'https?://(?:(?:www|play)\.)?(?P<bu>srf|rts|rsi|rtr|swissinfo)\.ch/play/(?:tv|radio)/[^/]+/(?P<type>video|audio)/[^?]+\?id=(?P<id>[0-9a-f\-]{36}|\d+)'
 
     _TESTS = [{

From eccde5e9de26dbe0c289b5bd1bf40b0482da59b1 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 14:53:06 +0100
Subject: [PATCH 0208/1105] [audimedia] split long lines

---
 youtube_dl/extractor/audimedia.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/audimedia.py b/youtube_dl/extractor/audimedia.py
index 9b037bb0c..3b2effa15 100644
--- a/youtube_dl/extractor/audimedia.py
+++ b/youtube_dl/extractor/audimedia.py
@@ -45,11 +45,15 @@ class AudiMediaIE(InfoExtractor):
 
             stream_url_hls = json_data.get('stream_url_hls')
             if stream_url_hls:
-                formats.extend(self._extract_m3u8_formats(stream_url_hls, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
+                formats.extend(self._extract_m3u8_formats(
+                    stream_url_hls, video_id, 'mp4',
+                    entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
 
             stream_url_hds = json_data.get('stream_url_hds')
             if stream_url_hds:
-                formats.extend(self._extract_f4m_formats(json_data.get('stream_url_hds') + '?hdcore=3.4.0', video_id, -1, f4m_id='hds', fatal=False))
+                formats.extend(self._extract_f4m_formats(
+                    stream_url_hds + '?hdcore=3.4.0',
+                    video_id, f4m_id='hds', fatal=False))
 
             for video_version in json_data.get('video_versions'):
                 video_version_url = video_version.get('download_url') or video_version.get('stream_url')

From f889ac45b83cd771919fba29c14cee9d24e637b7 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 14:55:36 +0100
Subject: [PATCH 0209/1105] [ign] split long lines

---
 youtube_dl/extractor/ign.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index d1c1c210c..c45c68c1d 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -120,19 +120,24 @@ class IGNIE(InfoExtractor):
 
         video_id = self._find_video_id(webpage)
         if not video_id:
-            return self.url_result(self._search_regex(self._EMBED_RE, webpage, 'embed url'))
+            return self.url_result(self._search_regex(
+                self._EMBED_RE, webpage, 'embed url'))
         return self._get_video_info(video_id)
 
     def _get_video_info(self, video_id):
-        api_data = self._download_json(self._API_URL_TEMPLATE % video_id, video_id)
+        api_data = self._download_json(
+            self._API_URL_TEMPLATE % video_id, video_id)
 
         formats = []
         m3u8_url = api_data['refs'].get('m3uUrl')
         if m3u8_url:
-            formats.extend(self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', 'm3u8_native',
+                m3u8_id='hls', fatal=False))
         f4m_url = api_data['refs'].get('f4mUrl')
         if f4m_url:
-            formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False))
+            formats.extend(self._extract_f4m_formats(
+                f4m_url, video_id, f4m_id='hds', fatal=False))
         for asset in api_data['assets']:
             formats.append({
                 'url': asset['url'],

From f2017cb020efe0956049bda11df4eec7c5985fda Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 14:58:22 +0100
Subject: [PATCH 0210/1105] [srgssr] split long lines and use m3u8_native

---
 youtube_dl/extractor/rts.py    |  2 +-
 youtube_dl/extractor/srgssr.py | 11 ++++++++---
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index f063ab5dd..3cc32847b 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -197,7 +197,7 @@ class RTSIE(SRGSSRIE):
                     media_id, f4m_id=format_id, fatal=False))
             elif format_url.endswith('.m3u8'):
                 formats.extend(self._extract_m3u8_formats(
-                    format_url, media_id, 'mp4', m3u8_id=format_id, fatal=False))
+                    format_url, media_id, 'mp4', 'm3u8_native', m3u8_id=format_id, fatal=False))
             else:
                 formats.append({
                     'format_id': format_id,
diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py
index 2c8b4276b..d06d34316 100644
--- a/youtube_dl/extractor/srgssr.py
+++ b/youtube_dl/extractor/srgssr.py
@@ -29,7 +29,8 @@ class SRGSSRIE(InfoExtractor):
             media_id)[media_type.capitalize()]
 
         if media_data.get('block') and media_data['block'] in self._ERRORS:
-            raise ExtractorError('%s said: %s' % (self.IE_NAME, self._ERRORS[media_data['block']]), expected=True)
+            raise ExtractorError('%s said: %s' % (
+                self.IE_NAME, self._ERRORS[media_data['block']]), expected=True)
 
         return media_data
 
@@ -62,9 +63,13 @@ class SRGSSRIE(InfoExtractor):
                     assets[quality['@quality']] = quality['text']
                 asset_url = assets.get('HD') or assets.get('HQ') or assets.get('SD') or assets.get('MQ') or assets.get('LQ')
                 if '.f4m' in asset_url:
-                    formats.extend(self._extract_f4m_formats(asset_url + '?hdcore=3.4.0', media_id, f4m_id='hds', fatal=False))
+                    formats.extend(self._extract_f4m_formats(
+                        asset_url + '?hdcore=3.4.0', media_id,
+                        f4m_id='hds', fatal=False))
                 elif '.m3u8' in asset_url:
-                    formats.extend(self._extract_m3u8_formats(asset_url, media_id, m3u8_id='hls', fatal=False))
+                    formats.extend(self._extract_m3u8_formats(
+                        asset_url, media_id, 'mp4', 'm3u8_native',
+                        m3u8_id='hls', fatal=False))
             else:
                 for asset in source['url']:
                     asset_url = asset['text']

From 98d7c0f4f7d88cdfed8dfd3ad81efec36090c7fb Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 15:02:18 +0100
Subject: [PATCH 0211/1105] [tele13] split long lines

---
 youtube_dl/extractor/tele13.py | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/tele13.py b/youtube_dl/extractor/tele13.py
index fe11c20df..4e860db0a 100644
--- a/youtube_dl/extractor/tele13.py
+++ b/youtube_dl/extractor/tele13.py
@@ -46,8 +46,12 @@ class Tele13IE(InfoExtractor):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
 
-        setup_js = self._search_regex(r"(?s)jwplayer\('player-vivo'\).setup\((\{.*?\})\)", webpage, 'setup code')
-        sources = self._parse_json(self._search_regex(r'sources\s*:\s*(\[[^\]]+\])', setup_js, 'sources'), display_id, js_to_json)
+        setup_js = self._search_regex(
+            r"(?s)jwplayer\('player-vivo'\).setup\((\{.*?\})\)",
+            webpage, 'setup code')
+        sources = self._parse_json(self._search_regex(
+            r'sources\s*:\s*(\[[^\]]+\])', setup_js, 'sources'),
+            display_id, js_to_json)
 
         preference = qualities(['Móvil', 'SD', 'HD'])
         formats = []
@@ -57,7 +61,9 @@ class Tele13IE(InfoExtractor):
             if format_url and format_url not in urls:
                 ext = determine_ext(format_url)
                 if ext == 'm3u8':
-                    formats.extend(self._extract_m3u8_formats(format_url, display_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+                    formats.extend(self._extract_m3u8_formats(
+                        format_url, display_id, 'mp4', 'm3u8_native',
+                        m3u8_id='hls', fatal=False))
                 elif YoutubeIE.suitable(format_url):
                     return self.url_result(format_url, 'Youtube')
                 else:
@@ -72,8 +78,11 @@ class Tele13IE(InfoExtractor):
 
         return {
             'id': display_id,
-            'title': self._search_regex(r'title\s*:\s*"([^"]+)"', setup_js, 'title'),
-            'description': self._html_search_meta('description', webpage, 'description'),
-            'thumbnail': self._search_regex(r'image\s*:\s*"([^"]+)"', setup_js, 'thumbnail', default=None),
+            'title': self._search_regex(
+                r'title\s*:\s*"([^"]+)"', setup_js, 'title'),
+            'description': self._html_search_meta(
+                'description', webpage, 'description'),
+            'thumbnail': self._search_regex(
+                r'image\s*:\s*"([^"]+)"', setup_js, 'thumbnail', default=None),
             'formats': formats,
         }

From bf96b45238b221a01ea9e07d7747313272cde142 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 15:03:14 +0100
Subject: [PATCH 0212/1105] [rai] split long lines

---
 youtube_dl/extractor/rai.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index f2679591b..a4dc5c335 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -107,7 +107,8 @@ class RaiTVIE(InfoExtractor):
                 return xml.replace(' tag elementi', '').replace('>/', '</')
 
             relinker = self._download_xml(
-                media['mediaUri'] + '&output=43', video_id, transform_source=fix_xml)
+                media['mediaUri'] + '&output=43',
+                video_id, transform_source=fix_xml)
 
             has_subtitle = False
 
@@ -117,8 +118,8 @@ class RaiTVIE(InfoExtractor):
                 content_type = xpath_text(element, 'content-type')
                 if ext == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
-                        media_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls',
-                        fatal=False))
+                        media_url, video_id, 'mp4', 'm3u8_native',
+                        m3u8_id='hls', fatal=False))
                 elif ext == 'f4m':
                     formats.extend(self._extract_f4m_formats(
                         media_url + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44',

From 6101f45ef90e5b893477c684baa0d797d15bff29 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 15:05:21 +0100
Subject: [PATCH 0213/1105] [ooyala] split long lines, fix test duration and
 add hdcode param to hds url

---
 youtube_dl/extractor/ooyala.py | 31 ++++++++++++++++++++++---------
 1 file changed, 22 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 3960d522e..20b984288 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -13,6 +13,9 @@ from ..compat import compat_urllib_parse
 
 
 class OoyalaBaseIE(InfoExtractor):
+    _PLAYER_BASE = 'http://player.ooyala.com/'
+    _CONTENT_TREE_BASE = _PLAYER_BASE + 'player_api/v1/content_tree/'
+    _AUTHORIZATION_URL_TEMPLATE = _PLAYER_BASE + 'sas/player_api/v1/authorization/embed_code/%s/%s?'
 
     def _extract(self, content_tree_url, video_id, domain='example.org'):
         content_tree = self._download_json(content_tree_url, video_id)['content_tree']
@@ -31,24 +34,33 @@ class OoyalaBaseIE(InfoExtractor):
         formats = []
         for supported_format in ('mp4', 'm3u8', 'hds', 'rtmp'):
             auth_data = self._download_json(
-                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?' % (pcode, embed_code) + compat_urllib_parse.urlencode({'domain': domain, 'supportedFormats': supported_format}),
+                self._AUTHORIZATION_URL_TEMPLATE % (pcode, embed_code) +
+                compat_urllib_parse.urlencode({
+                    'domain': domain,
+                    'supportedFormats': supported_format
+                }),
                 video_id, 'Downloading %s JSON' % supported_format)
 
             cur_auth_data = auth_data['authorization_data'][embed_code]
 
             if cur_auth_data['authorized']:
                 for stream in cur_auth_data['streams']:
-                    url = base64.b64decode(stream['url']['data'].encode('ascii')).decode('utf-8')
+                    url = base64.b64decode(
+                        stream['url']['data'].encode('ascii')).decode('utf-8')
                     if url in urls:
                         continue
                     urls.append(url)
                     delivery_type = stream['delivery_type']
                     if delivery_type == 'hls' or '.m3u8' in url:
-                        formats.extend(self._extract_m3u8_formats(url, embed_code, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+                        formats.extend(self._extract_m3u8_formats(
+                            url, embed_code, 'mp4', 'm3u8_native',
+                            m3u8_id='hls', fatal=False))
                     elif delivery_type == 'hds' or '.f4m' in url:
-                        formats.extend(self._extract_f4m_formats(url, embed_code, f4m_id='hds', fatal=False))
+                        formats.extend(self._extract_f4m_formats(
+                            url + '?hdcore=3.7.0', embed_code, f4m_id='hds', fatal=False))
                     elif '.smil' in url:
-                        formats.extend(self._extract_smil_formats(url, embed_code, fatal=False))
+                        formats.extend(self._extract_smil_formats(
+                            url, embed_code, fatal=False))
                     else:
                         formats.append({
                             'url': url,
@@ -62,7 +74,8 @@ class OoyalaBaseIE(InfoExtractor):
                             'fps': float_or_none(stream.get('framerate')),
                         })
             else:
-                raise ExtractorError('%s said: %s' % (self.IE_NAME, cur_auth_data['message']), expected=True)
+                raise ExtractorError('%s said: %s' % (
+                    self.IE_NAME, cur_auth_data['message']), expected=True)
         self._sort_formats(formats)
 
         video_info['formats'] = formats
@@ -120,7 +133,7 @@ class OoyalaIE(OoyalaBaseIE):
         url, smuggled_data = unsmuggle_url(url, {})
         embed_code = self._match_id(url)
         domain = smuggled_data.get('domain')
-        content_tree_url = 'http://player.ooyala.com/player_api/v1/content_tree/embed_code/%s/%s' % (embed_code, embed_code)
+        content_tree_url = self._CONTENT_TREE_BASE + 'embed_code/%s/%s' % (embed_code, embed_code)
         return self._extract(content_tree_url, embed_code, domain)
 
 
@@ -147,7 +160,7 @@ class OoyalaExternalIE(OoyalaBaseIE):
             'id': 'FkYWtmazr6Ed8xmvILvKLWjd4QvYZpzG',
             'ext': 'mp4',
             'title': 'dm_140128_30for30Shorts___JudgingJewellv2',
-            'duration': 1302000,
+            'duration': 1302.0,
         },
         'params': {
             # m3u8 download
@@ -157,5 +170,5 @@ class OoyalaExternalIE(OoyalaBaseIE):
 
     def _real_extract(self, url):
         partner_id, video_id, pcode = re.match(self._VALID_URL, url).groups()
-        content_tree_url = 'http://player.ooyala.com/player_api/v1/content_tree/external_id/%s/%s:%s' % (pcode, partner_id, video_id)
+        content_tree_url = self._CONTENT_TREE_BASE + 'external_id/%s/%s:%s' % (pcode, partner_id, video_id)
         return self._extract(content_tree_url, video_id)

From 0368181998cd5ec365bb8ed10c5da0188fbed798 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 15:07:21 +0100
Subject: [PATCH 0214/1105] [wdr] split long lines

---
 youtube_dl/extractor/wdr.py | 35 +++++++++++++++++++++++------------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index e8511398c..a851578e0 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -108,7 +108,9 @@ class WDRIE(InfoExtractor):
         if mobj.group('player') is None:
             entries = [
                 self.url_result(page_url + href, 'WDR')
-                for href in re.findall(r'<a href="/?(.+?%s\.html)" rel="nofollow"' % self._PLAYER_REGEX, webpage)
+                for href in re.findall(
+                    r'<a href="/?(.+?%s\.html)" rel="nofollow"' % self._PLAYER_REGEX,
+                    webpage)
             ]
 
             if entries:  # Playlist page
@@ -133,8 +135,8 @@ class WDRIE(InfoExtractor):
                     note='Downloading playlist page %d' % page_num)
             return self.playlist_result(entries, page_id)
 
-        flashvars = compat_parse_qs(
-            self._html_search_regex(r'<param name="flashvars" value="([^"]+)"', webpage, 'flashvars'))
+        flashvars = compat_parse_qs(self._html_search_regex(
+            r'<param name="flashvars" value="([^"]+)"', webpage, 'flashvars'))
 
         page_id = flashvars['trackerClipId'][0]
         video_url = flashvars['dslSrc'][0]
@@ -148,7 +150,8 @@ class WDRIE(InfoExtractor):
         if 'trackerClipAirTime' in flashvars:
             upload_date = flashvars['trackerClipAirTime'][0]
         else:
-            upload_date = self._html_search_meta('DC.Date', webpage, 'upload date')
+            upload_date = self._html_search_meta(
+                'DC.Date', webpage, 'upload date')
 
         if upload_date:
             upload_date = unified_strdate(upload_date)
@@ -157,12 +160,15 @@ class WDRIE(InfoExtractor):
         preference = qualities(['S', 'M', 'L', 'XL'])
 
         if video_url.endswith('.f4m'):
-            formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18', page_id, f4m_id='hds', fatal=False))
+            formats.extend(self._extract_f4m_formats(
+                video_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18', page_id,
+                f4m_id='hds', fatal=False))
         elif video_url.endswith('.smil'):
-            formats.extend(self._extract_smil_formats(video_url, page_id, False, {
-                'hdcore': '3.3.0',
-                'plugin': 'aasp-3.3.0.99.43',
-            }))
+            formats.extend(self._extract_smil_formats(
+                video_url, page_id, False, {
+                    'hdcore': '3.3.0',
+                    'plugin': 'aasp-3.3.0.99.43',
+                }))
         else:
             formats.append({
                 'url': video_url,
@@ -171,11 +177,16 @@ class WDRIE(InfoExtractor):
                 },
             })
 
-        m3u8_url = self._search_regex(r'rel="adaptiv"[^>]+href="([^"]+)"', webpage, 'm3u8 url', default=None)
+        m3u8_url = self._search_regex(
+            r'rel="adaptiv"[^>]+href="([^"]+)"',
+            webpage, 'm3u8 url', default=None)
         if m3u8_url:
-            formats.extend(self._extract_m3u8_formats(m3u8_url, page_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, page_id, 'mp4', 'm3u8_native',
+                m3u8_id='hls', fatal=False))
 
-        direct_urls = re.findall(r'rel="web(S|M|L|XL)"[^>]+href="([^"]+)"', webpage)
+        direct_urls = re.findall(
+            r'rel="web(S|M|L|XL)"[^>]+href="([^"]+)"', webpage)
         if direct_urls:
             for quality, video_url in direct_urls:
                 formats.append({

From 53365f74a7142c0e16eeb8db2c08f1f200de2406 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 29 Dec 2015 16:14:05 +0100
Subject: [PATCH 0215/1105] Credit @flatgreen for FranceCultureEmission (#8022)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index ce350e96c..20f51009e 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -149,3 +149,4 @@ Marco Ferragina
 reiv
 Muratcan Simsek
 Evan Lu
+flatgreen

From 0d8a0fdc30845aecfb9c98ccd4884dd1456dda86 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 16:38:06 +0100
Subject: [PATCH 0216/1105] [srgssr] use SRFIE format ids

---
 youtube_dl/extractor/srgssr.py | 27 ++++++++++++---------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py
index d06d34316..4707029ca 100644
--- a/youtube_dl/extractor/srgssr.py
+++ b/youtube_dl/extractor/srgssr.py
@@ -57,29 +57,26 @@ class SRGSSRIE(InfoExtractor):
         formats = []
         for source in media_data.get('Playlists', {}).get('Playlist', []) + media_data.get('Downloads', {}).get('Download', []):
             protocol = source.get('@protocol')
-            if protocol in ('HTTP-HDS', 'HTTP-HLS'):
-                assets = {}
-                for quality in source['url']:
-                    assets[quality['@quality']] = quality['text']
-                asset_url = assets.get('HD') or assets.get('HQ') or assets.get('SD') or assets.get('MQ') or assets.get('LQ')
-                if '.f4m' in asset_url:
+            for asset in source['url']:
+                asset_url = asset['text']
+                quality = asset['@quality']
+                format_id = '%s-%s' % (protocol, quality)
+                if protocol == 'HTTP-HDS':
                     formats.extend(self._extract_f4m_formats(
                         asset_url + '?hdcore=3.4.0', media_id,
-                        f4m_id='hds', fatal=False))
-                elif '.m3u8' in asset_url:
+                        f4m_id=format_id, fatal=False))
+                elif protocol == 'HTTP-HLS':
                     formats.extend(self._extract_m3u8_formats(
                         asset_url, media_id, 'mp4', 'm3u8_native',
-                        m3u8_id='hls', fatal=False))
-            else:
-                for asset in source['url']:
-                    asset_url = asset['text']
+                        m3u8_id=format_id, fatal=False))
+                else:
                     ext = None
-                    if asset_url.startswith('rtmp'):
+                    if protocol == 'RTMP':
                         ext = self._search_regex(r'([a-z0-9]+):[^/]+', asset_url, 'ext')
                     formats.append({
-                        'format_id': asset['@quality'],
+                        'format_id': format_id,
                         'url': asset_url,
-                        'preference': preference(asset['@quality']),
+                        'preference': preference(quality),
                         'ext': ext,
                     })
         self._sort_formats(formats)

From 030dfb04e0deff7e824194a7ddea2b55cf050e34 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Dec 2015 23:57:46 +0600
Subject: [PATCH 0217/1105] [videomore] Add extractor (Closes #8040)

---
 youtube_dl/extractor/__init__.py  |   5 +
 youtube_dl/extractor/videomore.py | 186 ++++++++++++++++++++++++++++++
 2 files changed, 191 insertions(+)
 create mode 100644 youtube_dl/extractor/videomore.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 75d191d5d..fb7151443 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -781,6 +781,11 @@ from .viddler import ViddlerIE
 from .videodetective import VideoDetectiveIE
 from .videofyme import VideofyMeIE
 from .videomega import VideoMegaIE
+from .videomore import (
+    VideomoreIE,
+    VideomoreVideoIE,
+    VideomoreSeasonIE,
+)
 from .videopremium import VideoPremiumIE
 from .videott import VideoTtIE
 from .vidme import VidmeIE
diff --git a/youtube_dl/extractor/videomore.py b/youtube_dl/extractor/videomore.py
new file mode 100644
index 000000000..dbc9dbdb7
--- /dev/null
+++ b/youtube_dl/extractor/videomore.py
@@ -0,0 +1,186 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_age_limit,
+    parse_iso8601,
+    xpath_text,
+)
+
+
+class VideomoreIE(InfoExtractor):
+    _VALID_URL = r'videomore:(?P<sid>\d+)$|https?://videomore\.ru/(?:(?:embed|[^/]+/[^/]+)/|[^/]+\?.*\btrack_id=)(?P<id>\d+)(?:[/?#&]|\.(?:xml|json)|$)'
+    _TESTS = [{
+        'url': 'http://videomore.ru/kino_v_detalayah/5_sezon/367617',
+        'md5': '70875fbf57a1cd004709920381587185',
+        'info_dict': {
+            'id': '367617',
+            'ext': 'flv',
+            'title': 'В гостях Алексей Чумаков и Юлия Ковальчук',
+            'description': 'В гостях – лучшие романтические комедии года, «Выживший» Иньярриту и «Стив Джобс» Дэнни Бойла.',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 2910,
+            'age_limit': 16,
+            'view_count': int,
+        },
+    }, {
+        'url': 'http://videomore.ru/elki_3?track_id=364623',
+        'only_matching': True,
+    }, {
+        'url': 'http://videomore.ru/embed/364623',
+        'only_matching': True,
+    }, {
+        'url': 'http://videomore.ru/video/tracks/364623.xml',
+        'only_matching': True,
+    }, {
+        'url': 'http://videomore.ru/video/tracks/364623.json',
+        'only_matching': True,
+    }, {
+        'url': 'http://videomore.ru/video/tracks/158031/quotes/33248',
+        'only_matching': True,
+    }, {
+        'url': 'videomore:367617',
+        'only_matching': True,
+    }]
+
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<object[^>]+data=(["\'])https?://videomore.ru/player\.swf\?.*config=(?P<url>https?://videomore\.ru/(?:[^/]+/)+\d+\.xml).*\1',
+            webpage)
+        if mobj:
+            return mobj.group('url')
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('sid') or mobj.group('id')
+
+        video = self._download_xml(
+            'http://videomore.ru/video/tracks/%s.xml' % video_id,
+            video_id, 'Downloading video XML')
+
+        video_url = xpath_text(video, './/video_url', 'video url', fatal=True)
+        formats = self._extract_f4m_formats(video_url, video_id, f4m_id='hds')
+
+        data = self._download_json(
+            'http://videomore.ru/video/tracks/%s.json' % video_id,
+            video_id, 'Downloadinng video JSON')
+
+        title = data.get('title') or data['project_title']
+        description = data.get('description') or data.get('description_raw')
+        timestamp = parse_iso8601(data.get('published_at'))
+        duration = int_or_none(data.get('duration'))
+        view_count = int_or_none(data.get('views'))
+        age_limit = parse_age_limit(data.get('min_age'))
+        thumbnails = [{
+            'url': thumbnail,
+        } for thumbnail in data.get('big_thumbnail_urls', [])]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnails': thumbnails,
+            'timestamp': timestamp,
+            'duration': duration,
+            'view_count': view_count,
+            'age_limit': age_limit,
+            'formats': formats,
+        }
+
+
+class VideomoreVideoIE(InfoExtractor):
+    IE_NAME = 'videomore:video'
+    _VALID_URL = r'https?://videomore\.ru/(?:(?:[^/]+/){2})?(?P<id>[^/?#&]+)[/?#&]*$'
+    _TESTS = [{
+        # single video with og:video:iframe
+        'url': 'http://videomore.ru/elki_3',
+        'info_dict': {
+            'id': '364623',
+            'ext': 'flv',
+            'title': 'Ёлки 3',
+            'description': '',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 5579,
+            'age_limit': 6,
+            'view_count': int,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        # season single serie with og:video:iframe
+        'url': 'http://videomore.ru/poslednii_ment/1_sezon/14_seriya',
+        'only_matching': True,
+    }, {
+        'url': 'http://videomore.ru/sejchas_v_seti/serii_221-240/226_vypusk',
+        'only_matching': True,
+    }, {
+        # single video without og:video:iframe
+        'url': 'http://videomore.ru/marin_i_ego_druzya',
+        'info_dict': {
+            'id': '359073',
+            'ext': 'flv',
+            'title': '1 серия. Здравствуй, Аквавилль!',
+            'description': 'md5:c6003179538b5d353e7bcd5b1372b2d7',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 754,
+            'age_limit': 6,
+            'view_count': int,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return False if VideomoreIE.suitable(url) else super(VideomoreVideoIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_url = self._og_search_property(
+            'video:iframe', webpage, 'video url', default=None)
+
+        if not video_url:
+            video_id = self._search_regex(
+                (r'config\s*:\s*["\']https?://videomore\.ru/video/tracks/(\d+)\.xml',
+                 r'track-id=["\'](\d+)',
+                 r'xcnt_product_id\s*=\s*(\d+)'), webpage, 'video id')
+            video_url = 'videomore:%s' % video_id
+
+        return self.url_result(video_url, VideomoreIE.ie_key())
+
+
+class VideomoreSeasonIE(InfoExtractor):
+    IE_NAME = 'videomore:season'
+    _VALID_URL = r'https?://videomore\.ru/(?!embed)(?P<id>[^/]+/[^/?#&]+)[/?#&]*$'
+    _TESTS = [{
+        'url': 'http://videomore.ru/molodezhka/sezon_promo',
+        'info_dict': {
+            'id': 'molodezhka/sezon_promo',
+            'title': 'Молодежка Промо',
+        },
+        'playlist_mincount': 12,
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        title = self._og_search_title(webpage)
+
+        entries = [
+            self.url_result(item) for item in re.findall(
+                r'<a[^>]+href="((?:https?:)?//videomore\.ru/%s/[^/]+)"[^>]+class="widget-item-desc"'
+                % display_id, webpage)]
+
+        return self.playlist_result(entries, display_id, title)

From ff18735cb2440dd2e628f0bf497325b50848353b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Dec 2015 23:58:23 +0600
Subject: [PATCH 0218/1105] [extractor/generic] Add support for videomore
 embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 70a8d8eb9..d79e1adc9 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -54,6 +54,7 @@ from .snagfilms import SnagFilmsEmbedIE
 from .screenwavemedia import ScreenwaveMediaIE
 from .mtv import MTVServicesEmbeddedIE
 from .pladform import PladformIE
+from .videomore import VideomoreIE
 from .googledrive import GoogleDriveIE
 from .jwplatform import JWPlatformIE
 from .ultimedia import UltimediaIE
@@ -1743,6 +1744,11 @@ class GenericIE(InfoExtractor):
         if pladform_url:
             return self.url_result(pladform_url)
 
+        # Look for Videomore embeds
+        videomore_url = VideomoreIE._extract_url(webpage)
+        if videomore_url:
+            return self.url_result(videomore_url)
+
         # Look for Playwire embeds
         mobj = re.search(
             r'<script[^>]+data-config=(["\'])(?P<url>(?:https?:)?//config\.playwire\.com/.+?)\1', webpage)

From 453fe2a345358064609df9adeee302e19aa636bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Dec 2015 00:13:00 +0600
Subject: [PATCH 0219/1105] [dramafever] Fix subtitles extraction (Closes
 #8049)

---
 youtube_dl/extractor/dramafever.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 60ed438f8..b3b21d65f 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -108,7 +108,7 @@ class DramaFeverIE(DramaFeverBaseIE):
             if value:
                 subfile = value[0].get('subfile') or value[0].get('new_subfile')
                 if subfile and subfile != 'http://www.dramafever.com/st/':
-                    info['subtitiles'].setdefault('English', []).append({
+                    info.setdefault('subtitles', {}).setdefault('English', []).append({
                         'ext': 'srt',
                         'url': subfile,
                     })

From 69759a5990ad897bdb1af9c44d5c73a2be1e92c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Dec 2015 00:44:07 +0600
Subject: [PATCH 0220/1105] [videomore] Set IE_NAME

---
 youtube_dl/extractor/videomore.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/videomore.py b/youtube_dl/extractor/videomore.py
index dbc9dbdb7..16078ac19 100644
--- a/youtube_dl/extractor/videomore.py
+++ b/youtube_dl/extractor/videomore.py
@@ -13,6 +13,7 @@ from ..utils import (
 
 
 class VideomoreIE(InfoExtractor):
+    IE_NAME = 'videomore'
     _VALID_URL = r'videomore:(?P<sid>\d+)$|https?://videomore\.ru/(?:(?:embed|[^/]+/[^/]+)/|[^/]+\?.*\btrack_id=)(?P<id>\d+)(?:[/?#&]|\.(?:xml|json)|$)'
     _TESTS = [{
         'url': 'http://videomore.ru/kino_v_detalayah/5_sezon/367617',

From f5bc4b5f95393b5fc83f5c64d44ee78093f9d798 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Dec 2015 23:12:35 +0600
Subject: [PATCH 0221/1105] [options] Prefer --convert-subs spelling

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index c46e136bf..ade58c375 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -752,7 +752,7 @@ def parseOpts(overrideArguments=None):
         metavar='CMD', dest='exec_cmd',
         help='Execute a command on the file after downloading, similar to find\'s -exec syntax. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\'')
     postproc.add_option(
-        '--convert-subtitles', '--convert-subs',
+        '--convert-subs', '--convert-subtitles',
         metavar='FORMAT', dest='convertsubtitles', default=None,
         help='Convert the subtitles to other format (currently supported: srt|ass|vtt)')
 

From 3092fc4035dd3a0841a92227e9bd80b7b9e4d871 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Dec 2015 01:09:21 +0600
Subject: [PATCH 0222/1105] [udemy] Fix typo

---
 youtube_dl/extractor/udemy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 59832b1ec..ce7e211fc 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -147,7 +147,7 @@ class UdemyIE(InfoExtractor):
             # Error could possibly mean we are not enrolled in the course
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
                 self._enroll_course(webpage, course_id)
-                lecture_id = self._download_lecture(course_id, lecture_id)
+                lecture = self._download_lecture(course_id, lecture_id)
             else:
                 raise
 

From 7109903e6140e01b5d5980fc80c6ef1b6c86797a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Dec 2015 03:10:44 +0600
Subject: [PATCH 0223/1105] [extractor/common] Document chapter and series
 fields

---
 youtube_dl/extractor/common.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 655207447..c443daf20 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -200,6 +200,26 @@ class InfoExtractor(object):
     end_time:       Time in seconds where the reproduction should end, as
                     specified in the URL.
 
+    The following fields should only be used when the video belongs to some logical
+    chapter or section:
+
+    chapter:        Name or title of the chapter the video belongs to.
+    chapter_id:     Number or id of the chapter the video belongs to, as an integer
+                    or unicode string.
+
+    The following fields should only be used when the video is an episode of some
+    series or programme:
+
+    series:         Title of the series or programme the video episode belongs to.
+    season:         Title of the season the video episode belongs to.
+    season_id:      Number or id of the season the video episode belongs to, as an
+                    integer or unicode string.
+    episode:        Title of the video episode. Unlike mandatory video title field,
+                    this field should denote the exact title of the video episode
+                    without any kind of decoration.
+    episode_id:     Number or id of the video episode within a season, as an integer
+                    or unicode string.
+
     Unless mentioned otherwise, the fields should be Unicode strings.
 
     Unless mentioned otherwise, None is equivalent to absence of information.

From 4d402db52140458b1b4707c30fb01bb92861a8cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Dec 2015 03:11:21 +0600
Subject: [PATCH 0224/1105] [udemy] Extract chapter info

---
 youtube_dl/extractor/udemy.py | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index ce7e211fc..41097829d 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -244,10 +244,25 @@ class UdemyCourseIE(UdemyIE):
             'https://www.udemy.com/api-1.1/courses/%s/curriculum' % course_id,
             course_id, 'Downloading course curriculum')
 
-        entries = [
-            self.url_result(
-                'https://www.udemy.com/%s/#/lecture/%s' % (course_path, asset['id']), 'Udemy')
-            for asset in response if asset.get('assetType') or asset.get('asset_type') == 'Video'
-        ]
+        entries = []
+        chapter, chapter_id = None, None
+        for asset in response:
+            asset_type = asset.get('assetType') or asset.get('asset_type')
+            if asset_type == 'Video':
+                asset_id = asset.get('id')
+                if asset_id:
+                    entry = {
+                        '_type': 'url_transparent',
+                        'url': 'https://www.udemy.com/%s/#/lecture/%s' % (course_path, asset['id']),
+                        'ie_key': UdemyIE.ie_key(),
+                    }
+                    if chapter_id:
+                        entry['chapter_id'] = chapter_id
+                    if chapter:
+                        entry['chapter'] = chapter
+                    entries.append(entry)
+            elif asset.get('type') == 'chapter':
+                chapter_id = asset.get('index') or asset.get('object_index')
+                chapter = asset.get('title')
 
         return self.playlist_result(entries, course_id, course_title)

From 7a0b07c719b621b1572120ed3f04870fa5737bfb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Dec 2015 03:13:02 +0600
Subject: [PATCH 0225/1105] [videomore] Extract series info

---
 youtube_dl/extractor/videomore.py | 56 +++++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/youtube_dl/extractor/videomore.py b/youtube_dl/extractor/videomore.py
index 16078ac19..3bd96e445 100644
--- a/youtube_dl/extractor/videomore.py
+++ b/youtube_dl/extractor/videomore.py
@@ -23,11 +23,56 @@ class VideomoreIE(InfoExtractor):
             'ext': 'flv',
             'title': 'В гостях Алексей Чумаков и Юлия Ковальчук',
             'description': 'В гостях – лучшие романтические комедии года, «Выживший» Иньярриту и «Стив Джобс» Дэнни Бойла.',
+            'series': 'Кино в деталях',
+            'episode': 'В гостях Алексей Чумаков и Юлия Ковальчук',
+            'episode_id': None,
+            'season': 'Сезон 2015',
+            'season_id': 5,
             'thumbnail': 're:^https?://.*\.jpg',
             'duration': 2910,
             'age_limit': 16,
             'view_count': int,
         },
+    }, {
+        'url': 'http://videomore.ru/embed/259974',
+        'info_dict': {
+            'id': '259974',
+            'ext': 'flv',
+            'title': '80 серия',
+            'description': '«Медведей» ждет решающий матч. Макеев выясняет отношения со Стрельцовым. Парни узнают подробности прошлого Макеева.',
+            'series': 'Молодежка',
+            'episode': '80 серия',
+            'episode_id': 40,
+            'season': '2 сезон',
+            'season_id': 2,
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 2809,
+            'age_limit': 16,
+            'view_count': int,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://videomore.ru/molodezhka/sezon_promo/341073',
+        'info_dict': {
+            'id': '341073',
+            'ext': 'flv',
+            'title': 'Команда проиграла из-за Бакина?',
+            'description': 'Молодежка 3 сезон скоро',
+            'series': 'Молодежка',
+            'episode': 'Команда проиграла из-за Бакина?',
+            'episode_id': None,
+            'season': 'Промо',
+            'season_id': 99,
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 29,
+            'age_limit': 16,
+            'view_count': int,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'http://videomore.ru/elki_3?track_id=364623',
         'only_matching': True,
@@ -81,10 +126,21 @@ class VideomoreIE(InfoExtractor):
             'url': thumbnail,
         } for thumbnail in data.get('big_thumbnail_urls', [])]
 
+        series = data.get('project_title')
+        episode = data.get('title')
+        episode_id = data.get('episode_of_season') or None
+        season = data.get('season_title')
+        season_id = data.get('season_pos') or None
+
         return {
             'id': video_id,
             'title': title,
             'description': description,
+            'series': series,
+            'episode': episode,
+            'episode_id': episode_id,
+            'season': season,
+            'season_id': season_id,
             'thumbnails': thumbnails,
             'timestamp': timestamp,
             'duration': duration,

From c7932289e73f62d5d73b942175c66510acc2be7e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 30 Dec 2015 23:57:19 +0100
Subject: [PATCH 0226/1105] [cbsnews] Fix extraction of the URL for the
 'RtmpDesktop' format (fixes #8048)

---
 youtube_dl/extractor/cbsnews.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index f9a64a0a2..d211ec23b 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -5,6 +5,7 @@ import re
 import json
 
 from .common import InfoExtractor
+from ..utils import remove_start
 
 
 class CBSNewsIE(InfoExtractor):
@@ -62,6 +63,7 @@ class CBSNewsIE(InfoExtractor):
             uri = item.get('media' + format_id + 'URI')
             if not uri:
                 continue
+            uri = remove_start(uri, '{manifest:none}')
             fmt = {
                 'url': uri,
                 'format_id': format_id,
@@ -70,6 +72,8 @@ class CBSNewsIE(InfoExtractor):
                 play_path = re.sub(
                     r'{slistFilePath}', '',
                     uri.split('<break>')[-1].split('{break}')[-1])
+                play_path = re.sub(
+                    r'{manifest:.+}.*$', '', play_path)
                 fmt.update({
                     'app': 'ondemand?auth=cbs',
                     'play_path': 'mp4:' + play_path,

From 29f36839016e989f4a335fdd51d63f083e26d555 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Dec 2015 05:31:01 +0600
Subject: [PATCH 0227/1105] [espn] Remove broken flag

---
 youtube_dl/extractor/espn.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/espn.py b/youtube_dl/extractor/espn.py
index e6f8f0337..7d758102c 100644
--- a/youtube_dl/extractor/espn.py
+++ b/youtube_dl/extractor/espn.py
@@ -5,7 +5,6 @@ from .common import InfoExtractor
 
 class ESPNIE(InfoExtractor):
     _VALID_URL = r'https?://espn\.go\.com/(?:[^/]+/)*(?P<id>[^/]+)'
-    _WORKING = False
     _TESTS = [{
         'url': 'http://espn.go.com/video/clip?id=10365079',
         'info_dict': {

From df827a983aa94ba99542230c6f7585b36ab49713 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Dec 2015 06:06:36 +0600
Subject: [PATCH 0228/1105] [discovery] Allow https (Closes #8065)

---
 youtube_dl/extractor/discovery.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index d6723ecf2..507f4bdad 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -9,7 +9,7 @@ from ..compat import compat_str
 
 
 class DiscoveryIE(InfoExtractor):
-    _VALID_URL = r'http://www\.discovery\.com\/[a-zA-Z0-9\-]*/[a-zA-Z0-9\-]*/videos/(?P<id>[a-zA-Z0-9_\-]*)(?:\.htm)?'
+    _VALID_URL = r'https?://www\.discovery\.com\/[a-zA-Z0-9\-]*/[a-zA-Z0-9\-]*/videos/(?P<id>[a-zA-Z0-9_\-]*)(?:\.htm)?'
     _TESTS = [{
         'url': 'http://www.discovery.com/tv-shows/mythbusters/videos/mission-impossible-outtakes.htm',
         'info_dict': {

From d5f6429de87da4bffa0be7703d774681393f1ffb Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 04:02:08 +0100
Subject: [PATCH 0229/1105] [canalplus] improve extraction(fixes #6301)

- extract data from json instead of xml
- fix http format urls
- extract more metadata
- update tests
- make m3u8 and f4m format extraction non fatal
- use m3u8_native implementation
---
 youtube_dl/extractor/canalplus.py | 59 ++++++++++++++++++-------------
 1 file changed, 34 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 004372f8d..25b2d4efe 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -10,13 +10,14 @@ from ..utils import (
     unified_strdate,
     url_basename,
     qualities,
+    int_or_none,
 )
 
 
 class CanalplusIE(InfoExtractor):
     IE_DESC = 'canalplus.fr, piwiplus.fr and d8.tv'
     _VALID_URL = r'https?://(?:www\.(?P<site>canalplus\.fr|piwiplus\.fr|d8\.tv|itele\.fr)/.*?/(?P<path>.*)|player\.canalplus\.fr/#/(?P<id>[0-9]+))'
-    _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/%s/%s'
+    _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/%s/%s?format=json'
     _SITE_ID_MAP = {
         'canalplus.fr': 'cplus',
         'piwiplus.fr': 'teletoon',
@@ -26,10 +27,10 @@ class CanalplusIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.canalplus.fr/c-emissions/pid1830-c-zapping.html?vid=1263092',
-        'md5': 'b3481d7ca972f61e37420798d0a9d934',
+        'md5': '12164a6f14ff6df8bd628e8ba9b10b78',
         'info_dict': {
             'id': '1263092',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Le Zapping - 13/05/15',
             'description': 'md5:09738c0d06be4b5d06a0940edb0da73f',
             'upload_date': '20150513',
@@ -56,10 +57,10 @@ class CanalplusIE(InfoExtractor):
         'skip': 'videos get deleted after a while',
     }, {
         'url': 'http://www.itele.fr/france/video/aubervilliers-un-lycee-en-colere-111559',
-        'md5': 'f3a46edcdf28006598ffaf5b30e6a2d4',
+        'md5': '38b8f7934def74f0d6f3ba6c036a5f82',
         'info_dict': {
             'id': '1213714',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Aubervilliers : un lycée en colère - Le 11/02/2015 à 06h45',
             'description': 'md5:8216206ec53426ea6321321f3b3c16db',
             'upload_date': '20150211',
@@ -82,15 +83,16 @@ class CanalplusIE(InfoExtractor):
                 webpage, 'video id', group='id')
 
         info_url = self._VIDEO_INFO_TEMPLATE % (site_id, video_id)
-        doc = self._download_xml(info_url, video_id, 'Downloading video XML')
+        video_data = self._download_json(info_url, video_id, 'Downloading video JSON')
 
-        video_info = [video for video in doc if video.find('ID').text == video_id][0]
-        media = video_info.find('MEDIA')
-        infos = video_info.find('INFOS')
+        if isinstance(video_data, list):
+            video_data = [video for video in video_data if video.get('ID') == video_id][0]
+        media = video_data['MEDIA']
+        infos = video_data['INFOS']
 
-        preference = qualities(['MOBILE', 'BAS_DEBIT', 'HAUT_DEBIT', 'HD', 'HLS', 'HDS'])
+        preference = qualities(['MOBILE', 'BAS_DEBIT', 'HAUT_DEBIT', 'HD'])
 
-        fmt_url = next(iter(media.find('VIDEOS'))).text
+        fmt_url = next(iter(media.get('VIDEOS')))
         if '/geo' in fmt_url.lower():
             response = self._request_webpage(
                 HEADRequest(fmt_url), video_id,
@@ -101,35 +103,42 @@ class CanalplusIE(InfoExtractor):
                     expected=True)
 
         formats = []
-        for fmt in media.find('VIDEOS'):
-            format_url = fmt.text
+        for format_id, format_url in media['VIDEOS'].items():
             if not format_url:
                 continue
-            format_id = fmt.tag
             if format_id == 'HLS':
                 formats.extend(self._extract_m3u8_formats(
-                    format_url, video_id, 'mp4', preference=preference(format_id)))
+                    format_url, video_id, 'mp4', 'm3u8_native', m3u8_id=format_id, fatal=False))
             elif format_id == 'HDS':
                 formats.extend(self._extract_f4m_formats(
-                    format_url + '?hdcore=2.11.3', video_id, preference=preference(format_id)))
+                    format_url + '?hdcore=2.11.3', video_id, f4m_id=format_id, fatal=False))
             else:
                 formats.append({
-                    'url': format_url,
+                    # the secret extracted ya function in http://player.canalplus.fr/common/js/canalPlayer.js
+                    'url': format_url + '?secret=pqzerjlsmdkjfoiuerhsdlfknaes',
                     'format_id': format_id,
                     'preference': preference(format_id),
                 })
         self._sort_formats(formats)
 
+        thumbnails = [{
+            'id': image_id,
+            'url': image_url,
+        } for image_id, image_url in media.get('images', {}).items()]
+
+        titrage = infos['TITRAGE']
+
         return {
             'id': video_id,
             'display_id': display_id,
-            'title': '%s - %s' % (infos.find('TITRAGE/TITRE').text,
-                                  infos.find('TITRAGE/SOUS_TITRE').text),
-            'upload_date': unified_strdate(infos.find('PUBLICATION/DATE').text),
-            'thumbnail': media.find('IMAGES/GRAND').text,
-            'description': infos.find('DESCRIPTION').text,
-            'view_count': int(infos.find('NB_VUES').text),
-            'like_count': int(infos.find('NB_LIKES').text),
-            'comment_count': int(infos.find('NB_COMMENTS').text),
+            'title': '%s - %s' % (titrage['TITRE'],
+                                  titrage['SOUS_TITRE']),
+            'upload_date': unified_strdate(infos.get('PUBLICATION', {}).get('DATE')),
+            'thumbnails': thumbnails,
+            'description': infos.get('DESCRIPTION'),
+            'duration': int_or_none(infos.get('DURATION')),
+            'view_count': int_or_none(infos.get('NB_VUES')),
+            'like_count': int_or_none(infos.get('NB_LIKES')),
+            'comment_count': int_or_none(infos.get('NB_COMMENTS')),
             'formats': formats,
         }

From 9787c5f4c89714acd36685b84671e6ea69307a6f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 12:02:33 +0100
Subject: [PATCH 0230/1105] [fox] Add new extractor(closes #3063)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/fox.py      | 39 ++++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100644 youtube_dl/extractor/fox.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fb7151443..2063ef633 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -203,6 +203,7 @@ from .flickr import FlickrIE
 from .folketinget import FolketingetIE
 from .footyroom import FootyRoomIE
 from .fourtube import FourTubeIE
+from .fox import FOXIE
 from .foxgay import FoxgayIE
 from .foxnews import FoxNewsIE
 from .foxsports import FoxSportsIE
diff --git a/youtube_dl/extractor/fox.py b/youtube_dl/extractor/fox.py
new file mode 100644
index 000000000..ab97b3196
--- /dev/null
+++ b/youtube_dl/extractor/fox.py
@@ -0,0 +1,39 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import smuggle_url
+
+
+class FOXIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?fox\.com/watch/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.fox.com/watch/255180355939/7684182528',
+        'info_dict': {
+            'id': '255180355939',
+            'ext': 'mp4',
+            'title': 'Official Trailer: Gotham',
+            'description': 'Tracing the rise of the great DC Comics Super-Villains and vigilantes, Gotham reveals an entirely new chapter that has never been told.',
+            'duration': 129,
+        },
+        'add_ie': ['ThePlatform'],
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        release_url = self._parse_json(self._search_regex(
+            r'"fox_pdk_player"\s*:\s*({[^}]+?})', webpage, 'fox_pdk_player'),
+            video_id)['release_url'] + '&manifest=m3u'
+
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'ThePlatform',
+            'url': smuggle_url(release_url, {'force_smil_url': True}),
+            'id': video_id,
+        }

From 80b8b72cb847bd286eea01819bf43fd8bbe6aa92 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 13:36:07 +0100
Subject: [PATCH 0231/1105] [animalplanet] Add new extractor(closes #5303)

---
 youtube_dl/extractor/__init__.py     |  1 +
 youtube_dl/extractor/animalplanet.py | 53 ++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+)
 create mode 100644 youtube_dl/extractor/animalplanet.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 2063ef633..abfabc7da 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -19,6 +19,7 @@ from .aftonbladet import AftonbladetIE
 from .airmozilla import AirMozillaIE
 from .aljazeera import AlJazeeraIE
 from .alphaporno import AlphaPornoIE
+from .animalplanet import AnimalPlanetIE
 from .anitube import AnitubeIE
 from .anysex import AnySexIE
 from .aol import AolIE
diff --git a/youtube_dl/extractor/animalplanet.py b/youtube_dl/extractor/animalplanet.py
new file mode 100644
index 000000000..0cebc8185
--- /dev/null
+++ b/youtube_dl/extractor/animalplanet.py
@@ -0,0 +1,53 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    parse_duration,
+    parse_iso8601,
+)
+
+
+class AnimalPlanetIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?animalplanet\.com/([^/]+/)*(?P<id>[^/\?#]+)'
+    _TESTS = [{
+        'url': 'http://www.animalplanet.com/tv-shows/i-shouldnt-be-alive/videos/dog-saves-injured-owner/',
+        'info_dict': {
+            'id': '10608',
+            'ext': 'mp4',
+            'title': 'Dog Saves Injured Owner',
+            'description': 'A world class athlete is put to the test when she falls into a canyon and breaks her hip. Her only companion is her dog, Taz, who is on a mission to save her!',
+            'upload_date': '20100410',
+            'timestamp': 1270857727,
+            'duration': 220,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://www.animalplanet.com/longfin-eels-maneaters/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        video_data = self._parse_json(self._search_regex(
+            r'initialVideoData\s*=\s*({.+?});',
+            webpage, 'initialVideoData'), display_id)['playlist'][0]
+
+        return {
+            'id': compat_str(video_data['id']),
+            'display_id': display_id,
+            'title': video_data['title'],
+            'description': video_data.get('description'),
+            'thumbnail': video_data.get('thumbnailURL'),
+            'duration': parse_duration(video_data.get('video_length')),
+            'timestamp': parse_iso8601(video_data.get('publishedDate')),
+            'formats': self._extract_m3u8_formats(
+                video_data['src'], display_id, 'mp4',
+                'm3u8_native', m3u8_id='hls')
+        }

From 9d46608efabe25454288b5decaa96f9216506770 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 16:35:51 +0100
Subject: [PATCH 0232/1105] [ora] Add new extractor(closes #7732)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ora.py      | 75 ++++++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+)
 create mode 100644 youtube_dl/extractor/ora.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index abfabc7da..3df80bf00 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -491,6 +491,7 @@ from .ooyala import (
     OoyalaIE,
     OoyalaExternalIE,
 )
+from .ora import OraTVIE
 from .orf import (
     ORFTVthekIE,
     ORFOE1IE,
diff --git a/youtube_dl/extractor/ora.py b/youtube_dl/extractor/ora.py
new file mode 100644
index 000000000..9c4255a2d
--- /dev/null
+++ b/youtube_dl/extractor/ora.py
@@ -0,0 +1,75 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+    get_element_by_attribute,
+    qualities,
+    unescapeHTML,
+)
+
+
+class OraTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ora\.tv/([^/]+/)*(?P<id>[^/\?#]+)'
+    _TEST = {
+        'url': 'https://www.ora.tv/larrykingnow/2015/12/16/vine-youtube-stars-zach-king-king-bach-on-their-viral-videos-0_36jupg6090pq',
+        'md5': 'fa33717591c631ec93b04b0e330df786',
+        'info_dict': {
+            'id': '50178',
+            'ext': 'mp4',
+            'title': 'Vine & YouTube Stars Zach King & King Bach On Their Viral Videos!',
+            'description': 'md5:ebbc5b1424dd5dba7be7538148287ac1',
+            'duration': 1477,
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        video_data = self._search_regex(
+            r'"current"\s*:\s*({[^}]+?})', webpage, 'current video')
+        m3u8_url = self._search_regex(
+            r'"hls_stream"\s*:\s*"([^"]+)', video_data, 'm3u8 url', None)
+        if m3u8_url:
+            formats = self._extract_m3u8_formats(
+                m3u8_url, display_id, 'mp4', 'm3u8_native',
+                m3u8_id='hls', fatal=False)
+            # simular to GameSpotIE
+            m3u8_path = compat_urlparse.urlparse(m3u8_url).path
+            QUALITIES_RE = r'((,[a-z]+\d+)+,?)'
+            available_qualities = self._search_regex(
+                QUALITIES_RE, m3u8_path, 'qualities').strip(',').split(',')
+            http_path = m3u8_path[1:].split('/', 1)[1]
+            http_template = re.sub(QUALITIES_RE, r'%s', http_path)
+            http_template = http_template.replace('.csmil/master.m3u8', '')
+            http_template = compat_urlparse.urljoin(
+                'http://videocdn-pmd.ora.tv/', http_template)
+            preference = qualities(
+                ['mobile400', 'basic400', 'basic600', 'sd900', 'sd1200', 'sd1500', 'hd720', 'hd1080'])
+            for q in available_qualities:
+                formats.append({
+                    'url': http_template % q,
+                    'format_id': q,
+                    'preference': preference(q),
+                })
+            self._sort_formats(formats)
+        else:
+            return self.url_result(self._search_regex(
+                r'"youtube_id"\s*:\s*"([^"]+)', webpage, 'youtube id'), 'Youtube')
+
+        return {
+            'id': self._search_regex(
+                r'"video_id"\s*:\s*(\d+)', video_data, 'video id'),
+            'display_id': display_id,
+            'title': unescapeHTML(self._og_search_title(webpage)),
+            'description': get_element_by_attribute(
+                'class', 'video_txt_decription', webpage),
+            'thumbnail': self._proto_relative_url(self._search_regex(
+                r'"thumb"\s*:\s*"([^"]+)', video_data, 'thumbnail', None)),
+            'duration': int(self._search_regex(
+                r'"duration"\s*:\s*(\d+)', video_data, 'duration')),
+            'formats': formats,
+        }

From 7de81fcc530bfad9023a1a89622c15e4443de5eb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 31 Dec 2015 16:50:53 +0100
Subject: [PATCH 0233/1105] release 2015.12.31

---
 README.md              |  2 +-
 docs/supportedsites.md | 16 +++++++++++-----
 youtube_dl/version.py  |  2 +-
 3 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 3a4707227..4fc83b8e3 100644
--- a/README.md
+++ b/README.md
@@ -400,7 +400,7 @@ which means you can modify it, redistribute it or use it however you like.
                                      downloading, similar to find's -exec
                                      syntax. Example: --exec 'adb push {}
                                      /sdcard/Music/ && rm {}'
-    --convert-subtitles FORMAT       Convert the subtitles to other format
+    --convert-subs FORMAT            Convert the subtitles to other format
                                      (currently supported: srt|ass|vtt)
 
 # CONFIGURATION
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 1646277ec..2297edc6b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -28,6 +28,7 @@
  - **AlJazeera**
  - **Allocine**
  - **AlphaPorno**
+ - **AnimalPlanet**
  - **anitube.se**
  - **AnySex**
  - **Aparat**
@@ -125,7 +126,7 @@
  - **dailymotion:user**
  - **DailymotionCloud**
  - **daum.net**
- - **daum.net**
+ - **daum.net:clip**
  - **DBTV**
  - **DCN**
  - **dcn:live**
@@ -164,7 +165,7 @@
  - **Eporner**
  - **EroProfile**
  - **Escapist**
- - **ESPN** (Currently broken)
+ - **ESPN**
  - **EsriVideo**
  - **Europa**
  - **EveryonesMixtape**
@@ -181,6 +182,7 @@
  - **Flickr**
  - **Folketinget**: Folketinget (ft.dk; Danish parliament)
  - **FootyRoom**
+ - **FOX**
  - **Foxgay**
  - **FoxNews**: Fox News and Fox Business Video
  - **FoxSports**
@@ -259,7 +261,6 @@
  - **JeuxVideo**
  - **Jove**
  - **jpopsuki.tv**
- - **Jukebox**
  - **JWPlatform**
  - **Kaltura**
  - **KanalPlay**: Kanal 5/9/11 Play
@@ -405,6 +406,7 @@
  - **OnionStudios**
  - **Ooyala**
  - **OoyalaExternal**
+ - **OraTV**
  - **orf:fm4**: radio FM4
  - **orf:iptv**: iptv.ORF.at
  - **orf:oe1**: Radio Österreich 1
@@ -415,7 +417,7 @@
  - **pcmag**
  - **Periscope**: Periscope
  - **PhilharmonieDeParis**: Philharmonie de Paris
- - **Phoenix**
+ - **phoenix.de**
  - **Photobucket**
  - **Pinkbike**
  - **Pladform**
@@ -535,7 +537,8 @@
  - **SportBoxEmbed**
  - **SportDeutschland**
  - **Sportschau**
- - **Srf**
+ - **SRGSSR**
+ - **SRGSSRPlay**: srf.ch, rts.ch, rsi.ch, rtr.ch and swissinfo.ch play sites
  - **SSA**
  - **stanfordoc**: Stanford Open ClassRoom
  - **Steam**
@@ -646,6 +649,9 @@
  - **VideoDetective**
  - **videofy.me**
  - **VideoMega**
+ - **videomore**
+ - **videomore:season**
+ - **videomore:video**
  - **VideoPremium**
  - **VideoTt**: video.tt - Your True Tube
  - **videoweed**: VideoWeed
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a62baa305..cf6c4d74d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.12.29'
+__version__ = '2015.12.31'

From c43fda4c1a6cc6d1a1726c3f37c82a0b256625e8 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 20 Dec 2015 03:17:23 +0100
Subject: [PATCH 0234/1105] [regiotv] Add new extractor (closes #7797)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/regiotv.py  | 61 ++++++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+)
 create mode 100644 youtube_dl/extractor/regiotv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3df80bf00..732229c14 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -552,6 +552,7 @@ from .rai import (
 from .rbmaradio import RBMARadioIE
 from .rds import RDSIE
 from .redtube import RedTubeIE
+from .regiotv import RegioTVIE
 from .restudy import RestudyIE
 from .reverbnation import ReverbNationIE
 from .ringtv import RingTVIE
diff --git a/youtube_dl/extractor/regiotv.py b/youtube_dl/extractor/regiotv.py
new file mode 100644
index 000000000..a5c21c085
--- /dev/null
+++ b/youtube_dl/extractor/regiotv.py
@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+from ..utils import (
+    sanitized_Request,
+    xpath_with_ns,
+)
+
+
+class RegioTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?regio-tv\.de/video/(?P<id>[0-9]+).html'
+    _TESTS = [
+        {
+            'url': 'http://www.regio-tv.de/video/395808.html',
+            'info_dict': {
+                'id': '395808',
+                'ext': 'mp4',
+                'title': u'Wir in Ludwigsburg',
+                'description': u'Mit unseren zuckers\xfc\xdfen Adventskindern, au\xdferdem besuchen wir die Abendsterne!',
+            }
+        },
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        webpage = self._download_webpage(url, video_id)
+        key = self._html_search_regex(r''',key: "(.*?)"''', webpage, 'key')
+
+        title = self._html_search_regex(
+            r'<meta property="og:title" content="\s*(.*?)\s*"\s*/?\s*>',
+            webpage, 'title')
+
+        soapxml = '<?xml version="1.0" encoding="utf-8"?><soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/"><soap:Body><getHTML5VideoData xmlns="http://v.telvi.de/"><key xsi:type="xsd:string">%s</key></getHTML5VideoData></soap:Body></soap:Envelope>' % key
+        request = sanitized_Request('http://v.telvi.de/?wsdl', soapxml)
+        request.add_header('Origin', 'http://www.regio-tv.de')
+        request.add_header('Referer', url)
+        video_data = self._download_xml(request, video_id, 'video data')
+
+        NS_MAP = {
+            'xsi': 'http://www.w3.org/2001/XMLSchema-instance',
+            'soap': 'http://schemas.xmlsoap.org/soap/envelope/',
+        }
+
+        url = video_data.find(xpath_with_ns('.//video', NS_MAP)).text
+        thumbnail = video_data.find(xpath_with_ns('.//image', NS_MAP)).text
+
+        description = self._html_search_meta('description', webpage)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': url,
+            'thumbnail': thumbnail,
+            'description': description,
+        }

From 34a9da136f1a29889af01815b696672b9dd5b57d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Dec 2015 22:12:47 +0600
Subject: [PATCH 0235/1105] [regiotv] Improve extraction (Closes #7915)

---
 youtube_dl/extractor/regiotv.py | 65 +++++++++++++++++----------------
 1 file changed, 33 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/regiotv.py b/youtube_dl/extractor/regiotv.py
index a5c21c085..e250a52f0 100644
--- a/youtube_dl/extractor/regiotv.py
+++ b/youtube_dl/extractor/regiotv.py
@@ -1,61 +1,62 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 from ..utils import (
     sanitized_Request,
+    xpath_text,
     xpath_with_ns,
 )
 
 
 class RegioTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?regio-tv\.de/video/(?P<id>[0-9]+).html'
-    _TESTS = [
-        {
-            'url': 'http://www.regio-tv.de/video/395808.html',
-            'info_dict': {
-                'id': '395808',
-                'ext': 'mp4',
-                'title': u'Wir in Ludwigsburg',
-                'description': u'Mit unseren zuckers\xfc\xdfen Adventskindern, au\xdferdem besuchen wir die Abendsterne!',
-            }
-        },
-    ]
+    _VALID_URL = r'https?://(?:www\.)?regio-tv\.de/video/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://www.regio-tv.de/video/395808.html',
+        'info_dict': {
+            'id': '395808',
+            'ext': 'mp4',
+            'title': 'Wir in Ludwigsburg',
+            'description': 'Mit unseren zuckersüßen Adventskindern, außerdem besuchen wir die Abendsterne!',
+        }
+    }, {
+        'url': 'http://www.regio-tv.de/video/395808',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
-        key = self._html_search_regex(r''',key: "(.*?)"''', webpage, 'key')
 
-        title = self._html_search_regex(
-            r'<meta property="og:title" content="\s*(.*?)\s*"\s*/?\s*>',
-            webpage, 'title')
+        key = self._search_regex(
+            r'key\s*:\s*(["\'])(?P<key>.+?)\1', webpage, 'key', group='key')
+        title = self._og_search_title(webpage)
 
-        soapxml = '<?xml version="1.0" encoding="utf-8"?><soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/"><soap:Body><getHTML5VideoData xmlns="http://v.telvi.de/"><key xsi:type="xsd:string">%s</key></getHTML5VideoData></soap:Body></soap:Envelope>' % key
-        request = sanitized_Request('http://v.telvi.de/?wsdl', soapxml)
-        request.add_header('Origin', 'http://www.regio-tv.de')
-        request.add_header('Referer', url)
-        video_data = self._download_xml(request, video_id, 'video data')
+        SOAP_TEMPLATE = '<?xml version="1.0" encoding="utf-8"?><soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/"><soap:Body><{0} xmlns="http://v.telvi.de/"><key xsi:type="xsd:string">{1}</key></{0}></soap:Body></soap:Envelope>'
+
+        request = sanitized_Request(
+            'http://v.telvi.de/',
+            SOAP_TEMPLATE.format('GetHTML5VideoData', key).encode('utf-8'))
+        video_data = self._download_xml(request, video_id, 'Downloading video XML')
 
         NS_MAP = {
             'xsi': 'http://www.w3.org/2001/XMLSchema-instance',
             'soap': 'http://schemas.xmlsoap.org/soap/envelope/',
         }
 
-        url = video_data.find(xpath_with_ns('.//video', NS_MAP)).text
-        thumbnail = video_data.find(xpath_with_ns('.//image', NS_MAP)).text
-
-        description = self._html_search_meta('description', webpage)
+        video_url = xpath_text(
+            video_data, xpath_with_ns('.//video', NS_MAP), 'video url', fatal=True)
+        thumbnail = xpath_text(
+            video_data, xpath_with_ns('.//image', NS_MAP), 'thumbnail')
+        description = self._og_search_description(
+            webpage) or self._html_search_meta('description', webpage)
 
         return {
             'id': video_id,
+            'url': video_url,
             'title': title,
-            'url': url,
-            'thumbnail': thumbnail,
             'description': description,
+            'thumbnail': thumbnail,
         }

From fec040e7549d06e91dae31da53bee855630d0adf Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 17:29:37 +0100
Subject: [PATCH 0236/1105] [discovery] add support for discovery related sites

- investigationdiscovery.com
- discoverylife.com
- animalplanet.com
- ahctv.com
- destinationamerica.com
- sciencechannel.com
- tlc.com
- velocity.com
---
 youtube_dl/extractor/__init__.py     |  1 -
 youtube_dl/extractor/animalplanet.py | 53 ----------------------------
 youtube_dl/extractor/discovery.py    | 39 +++++++++++++++-----
 3 files changed, 30 insertions(+), 63 deletions(-)
 delete mode 100644 youtube_dl/extractor/animalplanet.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 732229c14..bae151e58 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -19,7 +19,6 @@ from .aftonbladet import AftonbladetIE
 from .airmozilla import AirMozillaIE
 from .aljazeera import AlJazeeraIE
 from .alphaporno import AlphaPornoIE
-from .animalplanet import AnimalPlanetIE
 from .anitube import AnitubeIE
 from .anysex import AnySexIE
 from .aol import AolIE
diff --git a/youtube_dl/extractor/animalplanet.py b/youtube_dl/extractor/animalplanet.py
deleted file mode 100644
index 0cebc8185..000000000
--- a/youtube_dl/extractor/animalplanet.py
+++ /dev/null
@@ -1,53 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..compat import compat_str
-from ..utils import (
-    parse_duration,
-    parse_iso8601,
-)
-
-
-class AnimalPlanetIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?animalplanet\.com/([^/]+/)*(?P<id>[^/\?#]+)'
-    _TESTS = [{
-        'url': 'http://www.animalplanet.com/tv-shows/i-shouldnt-be-alive/videos/dog-saves-injured-owner/',
-        'info_dict': {
-            'id': '10608',
-            'ext': 'mp4',
-            'title': 'Dog Saves Injured Owner',
-            'description': 'A world class athlete is put to the test when she falls into a canyon and breaks her hip. Her only companion is her dog, Taz, who is on a mission to save her!',
-            'upload_date': '20100410',
-            'timestamp': 1270857727,
-            'duration': 220,
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        }
-    }, {
-        'url': 'http://www.animalplanet.com/longfin-eels-maneaters/',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-
-        video_data = self._parse_json(self._search_regex(
-            r'initialVideoData\s*=\s*({.+?});',
-            webpage, 'initialVideoData'), display_id)['playlist'][0]
-
-        return {
-            'id': compat_str(video_data['id']),
-            'display_id': display_id,
-            'title': video_data['title'],
-            'description': video_data.get('description'),
-            'thumbnail': video_data.get('thumbnailURL'),
-            'duration': parse_duration(video_data.get('video_length')),
-            'timestamp': parse_iso8601(video_data.get('publishedDate')),
-            'formats': self._extract_m3u8_formats(
-                video_data['src'], display_id, 'mp4',
-                'm3u8_native', m3u8_id='hls')
-        }
diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index 507f4bdad..2626d85e3 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -9,7 +9,17 @@ from ..compat import compat_str
 
 
 class DiscoveryIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.discovery\.com\/[a-zA-Z0-9\-]*/[a-zA-Z0-9\-]*/videos/(?P<id>[a-zA-Z0-9_\-]*)(?:\.htm)?'
+    _VALID_URL = r'''(?x)http://(?:www\.)?(?:
+            discovery|
+            investigationdiscovery|
+            discoverylife|
+            animalplanet|
+            ahctv|
+            destinationamerica|
+            sciencechannel|
+            tlc|
+            velocity
+        )\.com/([^/]+/)*(?P<id>[^\./\?#]+)'''
     _TESTS = [{
         'url': 'http://www.discovery.com/tv-shows/mythbusters/videos/mission-impossible-outtakes.htm',
         'info_dict': {
@@ -21,8 +31,8 @@ class DiscoveryIE(InfoExtractor):
                             'don\'t miss Adam moon-walking as Jamie ... behind Jamie\'s'
                             ' back.'),
             'duration': 156,
-            'timestamp': 1303099200,
-            'upload_date': '20110418',
+            'timestamp': 1302032462,
+            'upload_date': '20110405',
         },
         'params': {
             'skip_download': True,  # requires ffmpeg
@@ -33,27 +43,38 @@ class DiscoveryIE(InfoExtractor):
             'id': 'mythbusters-the-simpsons',
             'title': 'MythBusters: The Simpsons',
         },
-        'playlist_count': 9,
+        'playlist_mincount': 10,
+    }, {
+        'url': 'http://www.animalplanet.com/longfin-eels-maneaters/',
+        'info_dict': {
+            'id': '78326',
+            'ext': 'mp4',
+            'title': 'Longfin Eels: Maneaters?',
+            'description': 'Jeremy Wade tests whether or not New Zealand\'s longfin eels are man-eaters by covering himself in fish guts and getting in the water with them.',
+            'upload_date': '20140725',
+            'timestamp': 1406246400,
+            'duration': 116,
+        },
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        info = self._download_json(url + '?flat=1', video_id)
+        display_id = self._match_id(url)
+        info = self._download_json(url + '?flat=1', display_id)
 
         video_title = info.get('playlist_title') or info.get('video_title')
 
         entries = [{
             'id': compat_str(video_info['id']),
             'formats': self._extract_m3u8_formats(
-                video_info['src'], video_id, ext='mp4',
+                video_info['src'], display_id, 'mp4', 'm3u8_native', m3u8_id='hls',
                 note='Download m3u8 information for video %d' % (idx + 1)),
             'title': video_info['title'],
             'description': video_info.get('description'),
             'duration': parse_duration(video_info.get('video_length')),
-            'webpage_url': video_info.get('href'),
+            'webpage_url': video_info.get('href') or video_info.get('url'),
             'thumbnail': video_info.get('thumbnailURL'),
             'alt_title': video_info.get('secondary_title'),
             'timestamp': parse_iso8601(video_info.get('publishedDate')),
         } for idx, video_info in enumerate(info['playlist'])]
 
-        return self.playlist_result(entries, video_id, video_title)
+        return self.playlist_result(entries, display_id, video_title)

From b05641ce405ef281b395c26b094208c17141d027 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 18:24:49 +0100
Subject: [PATCH 0237/1105] [discovery] improve _VALID_URL regex

---
 youtube_dl/extractor/discovery.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index 2626d85e3..ce680a9f3 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -19,7 +19,7 @@ class DiscoveryIE(InfoExtractor):
             sciencechannel|
             tlc|
             velocity
-        )\.com/([^/]+/)*(?P<id>[^\./\?#]+)'''
+        )\.com/(?:[^/]+/)*(?P<id>[^./?#]+)'''
     _TESTS = [{
         'url': 'http://www.discovery.com/tv-shows/mythbusters/videos/mission-impossible-outtakes.htm',
         'info_dict': {

From f1e21efe63bc08abbc5cec0f8d9c3112c580bd48 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 18:33:40 +0100
Subject: [PATCH 0238/1105] [tlc] remove TlcIE

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/tlc.py      | 23 -----------------------
 2 files changed, 1 insertion(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bae151e58..f3f9cd978 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -695,7 +695,7 @@ from .thesixtyone import TheSixtyOneIE
 from .thisamericanlife import ThisAmericanLifeIE
 from .thisav import ThisAVIE
 from .tinypic import TinyPicIE
-from .tlc import TlcIE, TlcDeIE
+from .tlc import TlcDeIE
 from .tmz import (
     TMZIE,
     TMZArticleIE,
diff --git a/youtube_dl/extractor/tlc.py b/youtube_dl/extractor/tlc.py
index d6d038a8d..adc05ed5f 100644
--- a/youtube_dl/extractor/tlc.py
+++ b/youtube_dl/extractor/tlc.py
@@ -4,32 +4,9 @@ import re
 
 from .common import InfoExtractor
 from .brightcove import BrightcoveLegacyIE
-from .discovery import DiscoveryIE
 from ..compat import compat_urlparse
 
 
-class TlcIE(DiscoveryIE):
-    IE_NAME = 'tlc.com'
-    _VALID_URL = r'http://www\.tlc\.com\/[a-zA-Z0-9\-]*/[a-zA-Z0-9\-]*/videos/(?P<id>[a-zA-Z0-9\-]*)(.htm)?'
-
-    # DiscoveryIE has _TESTS
-    _TESTS = [{
-        'url': 'http://www.tlc.com/tv-shows/cake-boss/videos/too-big-to-fly.htm',
-        'info_dict': {
-            'id': '104493',
-            'ext': 'mp4',
-            'title': 'Too Big to Fly',
-            'description': 'Buddy has taken on a high flying task.',
-            'duration': 119,
-            'timestamp': 1393365060,
-            'upload_date': '20140225',
-        },
-        'params': {
-            'skip_download': True,  # requires ffmpef
-        },
-    }]
-
-
 class TlcDeIE(InfoExtractor):
     IE_NAME = 'tlc.de'
     _VALID_URL = r'http://www\.tlc\.de/sendungen/[^/]+/videos/(?P<title>[^/?]+)'

From 9accfed4e724d3048e894c2323b3a478951d670d Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 20 Dec 2015 15:37:57 +0100
Subject: [PATCH 0239/1105] [pandoratv] Add new extractor (closes #6884)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/pandoratv.py | 56 +++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100644 youtube_dl/extractor/pandoratv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f3f9cd978..6aed59d16 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -497,6 +497,7 @@ from .orf import (
     ORFFM4IE,
     ORFIPTVIE,
 )
+from .pandoratv import PandoraTVIE
 from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
diff --git a/youtube_dl/extractor/pandoratv.py b/youtube_dl/extractor/pandoratv.py
new file mode 100644
index 000000000..a0a0c114a
--- /dev/null
+++ b/youtube_dl/extractor/pandoratv.py
@@ -0,0 +1,56 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..compat import (
+    compat_urlparse,
+)
+from ..utils import (
+    ExtractorError,
+)
+
+
+class PandoraTVIE(InfoExtractor):
+    _VALID_URL = r'http://(?:.+?\.)?channel.pandora.tv/channel/video.ptv\?'
+    _TESTS = [{
+        'url': 'http://jp.channel.pandora.tv/channel/video.ptv?c1=&prgid=53294230&ch_userid=mikakim&ref=main&lot=cate_01_2',
+        'info_dict': {
+            'description': '\u982d\u3092\u64ab\u3067\u3066\u304f\u308c\u308b\uff1f',
+            'ext': 'mp4',
+            'id': '53294230',
+            'title': '\u982d\u3092\u64ab\u3067\u3066\u304f\u308c\u308b\uff1f',
+            'upload_date': '20151218',
+        }
+    }]
+
+
+    def _real_extract(self, url):
+        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+        video_id = qs.get('prgid', [None])[0]
+        user_id = qs.get('ch_userid', [None])[0]
+        if any(not f for f in (video_id, user_id,)):
+            raise ExtractorError('Invalid URL', expected=True)
+
+        data_url ='http://m.pandora.tv/?c=view&m=viewJsonApi&ch_userid={userid}&prgid={prgid}'.format(userid=user_id,prgid=video_id)
+        data = self._download_json(data_url, video_id)
+        info = data['data']['rows']['vod_play_info']['result']
+
+        formats = []
+        for format_id in sorted([k for k in info if k.startswith('v') and k.endswith('Url') and info[k]]):
+            formats.append({
+                'format_id': format_id,
+                'url': info[format_id],
+                'ext': 'mp4',
+                'height': int(format_id[1:-3]),
+            })
+
+        return {
+            'description': info['body'],
+            'thumbnail': info['thumbnail'],
+            'formats': formats,
+            'id': video_id,
+            'title': info['subject'],
+            'upload_date': info['fid'][:8],
+            'view_count': info['hit'],
+        }

From e4bd63f9c061cbf5c57e7aba29d0f46bffbf05d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jan 2016 00:40:27 +0600
Subject: [PATCH 0240/1105] [pandoratv] Improve extraction (Closes #7921)

---
 youtube_dl/extractor/pandoratv.py | 60 ++++++++++++++++++++-----------
 1 file changed, 40 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/pandoratv.py b/youtube_dl/extractor/pandoratv.py
index a0a0c114a..34cede4c1 100644
--- a/youtube_dl/extractor/pandoratv.py
+++ b/youtube_dl/extractor/pandoratv.py
@@ -2,28 +2,36 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-
 from ..compat import (
+    compat_str,
     compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
+    float_or_none,
+    parse_duration,
+    str_to_int,
 )
 
 
 class PandoraTVIE(InfoExtractor):
-    _VALID_URL = r'http://(?:.+?\.)?channel.pandora.tv/channel/video.ptv\?'
-    _TESTS = [{
+    _VALID_URL = r'https?://(?:.+?\.)?channel\.pandora\.tv/channel/video\.ptv\?'
+    _TEST = {
         'url': 'http://jp.channel.pandora.tv/channel/video.ptv?c1=&prgid=53294230&ch_userid=mikakim&ref=main&lot=cate_01_2',
         'info_dict': {
-            'description': '\u982d\u3092\u64ab\u3067\u3066\u304f\u308c\u308b\uff1f',
-            'ext': 'mp4',
             'id': '53294230',
-            'title': '\u982d\u3092\u64ab\u3067\u3066\u304f\u308c\u308b\uff1f',
+            'ext': 'flv',
+            'title': '頭を撫でてくれる？',
+            'description': '頭を撫でてくれる？',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 39,
             'upload_date': '20151218',
+            'uploader': 'カワイイ動物まとめ',
+            'uploader_id': 'mikakim',
+            'view_count': int,
+            'like_count': int,
         }
-    }]
-
+    }
 
     def _real_extract(self, url):
         qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
@@ -32,25 +40,37 @@ class PandoraTVIE(InfoExtractor):
         if any(not f for f in (video_id, user_id,)):
             raise ExtractorError('Invalid URL', expected=True)
 
-        data_url ='http://m.pandora.tv/?c=view&m=viewJsonApi&ch_userid={userid}&prgid={prgid}'.format(userid=user_id,prgid=video_id)
-        data = self._download_json(data_url, video_id)
+        data = self._download_json(
+            'http://m.pandora.tv/?c=view&m=viewJsonApi&ch_userid=%s&prgid=%s'
+            % (user_id, video_id), video_id)
+
         info = data['data']['rows']['vod_play_info']['result']
 
         formats = []
-        for format_id in sorted([k for k in info if k.startswith('v') and k.endswith('Url') and info[k]]):
+        for format_id, format_url in info.items():
+            if not format_url:
+                continue
+            height = self._search_regex(
+                r'^v(\d+)[Uu]rl$', format_id, 'height', default=None)
+            if not height:
+                continue
             formats.append({
-                'format_id': format_id,
-                'url': info[format_id],
-                'ext': 'mp4',
-                'height': int(format_id[1:-3]),
+                'format_id': '%sp' % height,
+                'url': format_url,
+                'height': int(height),
             })
+        self._sort_formats(formats)
 
         return {
-            'description': info['body'],
-            'thumbnail': info['thumbnail'],
-            'formats': formats,
             'id': video_id,
             'title': info['subject'],
-            'upload_date': info['fid'][:8],
-            'view_count': info['hit'],
+            'description': info.get('body'),
+            'thumbnail': info.get('thumbnail') or info.get('poster'),
+            'duration': float_or_none(info.get('runtime'), 1000) or parse_duration(info.get('time')),
+            'upload_date': info['fid'][:8] if isinstance(info.get('fid'), compat_str) else None,
+            'uploader': info.get('nickname'),
+            'uploader_id': info.get('upload_userid'),
+            'view_count': str_to_int(info.get('hit')),
+            'like_count': str_to_int(info.get('likecnt')),
+            'formats': formats,
         }

From 72528252e303a084c4b95ae07c7a7213e53cad8a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jan 2016 00:42:42 +0600
Subject: [PATCH 0241/1105] [pandoratv] Add IE names

---
 youtube_dl/extractor/pandoratv.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/pandoratv.py b/youtube_dl/extractor/pandoratv.py
index 34cede4c1..8d49f5c4a 100644
--- a/youtube_dl/extractor/pandoratv.py
+++ b/youtube_dl/extractor/pandoratv.py
@@ -15,6 +15,8 @@ from ..utils import (
 
 
 class PandoraTVIE(InfoExtractor):
+    IE_NAME = 'pandora.tv'
+    IE_DESC = '판도라TV'
     _VALID_URL = r'https?://(?:.+?\.)?channel\.pandora\.tv/channel/video\.ptv\?'
     _TEST = {
         'url': 'http://jp.channel.pandora.tv/channel/video.ptv?c1=&prgid=53294230&ch_userid=mikakim&ref=main&lot=cate_01_2',

From 9f9f7664b543f2691a9100920a8b5962d4608bf9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 31 Dec 2015 19:52:48 +0100
Subject: [PATCH 0242/1105] [espn] Update test

---
 youtube_dl/extractor/espn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/espn.py b/youtube_dl/extractor/espn.py
index 7d758102c..24780b82d 100644
--- a/youtube_dl/extractor/espn.py
+++ b/youtube_dl/extractor/espn.py
@@ -11,7 +11,7 @@ class ESPNIE(InfoExtractor):
             'id': 'FkYWtmazr6Ed8xmvILvKLWjd4QvYZpzG',
             'ext': 'mp4',
             'title': 'dm_140128_30for30Shorts___JudgingJewellv2',
-            'description': '',
+            'description': None,
         },
         'params': {
             # m3u8 download

From 930087f2f69d887d768f234c5dc4563c5ad86ad6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 31 Dec 2015 19:56:54 +0100
Subject: [PATCH 0243/1105] [espn] Support 'intl' videos (#7858)

---
 youtube_dl/extractor/espn.py | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/espn.py b/youtube_dl/extractor/espn.py
index 24780b82d..0a917f9f6 100644
--- a/youtube_dl/extractor/espn.py
+++ b/youtube_dl/extractor/espn.py
@@ -17,6 +17,18 @@ class ESPNIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        # intl video, from http://www.espnfc.us/video/mls-highlights/150/video/2743663/must-see-moments-best-of-the-mls-season
+        'url': 'http://espn.go.com/video/clip?id=2743663',
+        'info_dict': {
+            'id': '50NDFkeTqRHB0nXBOK-RGdSG5YQPuxHg',
+            'ext': 'mp4',
+            'title': 'int_151206_Must_See_Moments_Best_of_MLS_2015_season',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         'url': 'https://espn.go.com/video/iframe/twitter/?cms=espn&id=10365079',
         'only_matching': True,
@@ -43,12 +55,16 @@ class ESPNIE(InfoExtractor):
             r'class="video-play-button"[^>]+data-id="(\d+)',
             webpage, 'video id')
 
+        cms = 'espn'
+        if 'data-source="intl"' in webpage:
+            cms = 'intl'
+        player_url = 'https://espn.go.com/video/iframe/twitter/?id=%s&cms=%s' % (video_id, cms)
         player = self._download_webpage(
-            'https://espn.go.com/video/iframe/twitter/?id=%s' % video_id, video_id)
+            player_url, video_id)
 
         pcode = self._search_regex(
             r'["\']pcode=([^"\']+)["\']', player, 'pcode')
 
         return self.url_result(
-            'ooyalaexternal:espn:%s:%s' % (video_id, pcode),
+            'ooyalaexternal:%s:%s:%s' % (cms, video_id, pcode),
             'OoyalaExternal')

From 69f85952568def9ce13f71f3e8bac65f0c5f5f36 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 31 Dec 2015 20:06:21 +0100
Subject: [PATCH 0244/1105] [espn] Extract better titles

---
 youtube_dl/extractor/espn.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/espn.py b/youtube_dl/extractor/espn.py
index 0a917f9f6..3762d8748 100644
--- a/youtube_dl/extractor/espn.py
+++ b/youtube_dl/extractor/espn.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import remove_end
 
 
 class ESPNIE(InfoExtractor):
@@ -10,7 +11,7 @@ class ESPNIE(InfoExtractor):
         'info_dict': {
             'id': 'FkYWtmazr6Ed8xmvILvKLWjd4QvYZpzG',
             'ext': 'mp4',
-            'title': 'dm_140128_30for30Shorts___JudgingJewellv2',
+            'title': '30 for 30 Shorts: Judging Jewell',
             'description': None,
         },
         'params': {
@@ -23,7 +24,7 @@ class ESPNIE(InfoExtractor):
         'info_dict': {
             'id': '50NDFkeTqRHB0nXBOK-RGdSG5YQPuxHg',
             'ext': 'mp4',
-            'title': 'int_151206_Must_See_Moments_Best_of_MLS_2015_season',
+            'title': 'Must-See Moments: Best of the MLS season',
         },
         'params': {
             # m3u8 download
@@ -65,6 +66,13 @@ class ESPNIE(InfoExtractor):
         pcode = self._search_regex(
             r'["\']pcode=([^"\']+)["\']', player, 'pcode')
 
-        return self.url_result(
-            'ooyalaexternal:%s:%s:%s' % (cms, video_id, pcode),
-            'OoyalaExternal')
+        title = remove_end(
+            self._og_search_title(webpage),
+            '- ESPN Video').strip()
+
+        return {
+            '_type': 'url_transparent',
+            'url': 'ooyalaexternal:%s:%s:%s' % (cms, video_id, pcode),
+            'ie_key': 'OoyalaExternal',
+            'title': title,
+        }

From 91e274546c4492dfa8f216f311356f641859a7cc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 20:23:48 +0100
Subject: [PATCH 0245/1105] [tvland] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/tvland.py   | 70 ++++++++++++++++++++++++++++++++
 2 files changed, 71 insertions(+)
 create mode 100644 youtube_dl/extractor/tvland.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6aed59d16..e20d90bac 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -739,6 +739,7 @@ from .tvc import (
     TVCArticleIE,
 )
 from .tvigle import TvigleIE
+from .tvland import TVLandIE
 from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
 from .tweakers import TweakersIE
diff --git a/youtube_dl/extractor/tvland.py b/youtube_dl/extractor/tvland.py
new file mode 100644
index 000000000..7f049e076
--- /dev/null
+++ b/youtube_dl/extractor/tvland.py
@@ -0,0 +1,70 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .mtv import MTVIE
+
+
+class TVLandIE(MTVIE):
+    IE_NAME = 'tvland.com'
+    _VALID_URL = r'https?://(?:www\.)?tvland\.com/(?:video-clips|episodes)/(?P<id>[^/?#.]+)'
+    _FEED_URL = 'http://www.tvland.com/feeds/mrss/?uri='
+    _TESTS = [{
+        'url': 'http://www.tvland.com/episodes/hqhps2/everybody-loves-raymond-the-invasion-ep-048',
+        'playlist': [
+            {
+                'md5': '227e9723b9669c05bf51098b10287aa7',
+                'info_dict': {
+                    'id': 'bcbd3a83-3aca-4dca-809b-f78a87dcccdd',
+                    'ext': 'mp4',
+                    'title': 'Everybody Loves Raymond|Everybody Loves Raymond 048 HD, Part 1 of 5',
+                }
+            },
+            {
+                'md5': '9fa2b764ec0e8194fb3ebb01a83df88b',
+                'info_dict': {
+                    'id': 'f4279548-6e13-40dd-92e8-860d27289197',
+                    'ext': 'mp4',
+                    'title': 'Everybody Loves Raymond|Everybody Loves Raymond 048 HD, Part 2 of 5',
+                }
+            },
+            {
+                'md5': 'fde4c3bccd7cc7e3576b338734153cec',
+                'info_dict': {
+                    'id': '664e4a38-53ef-4115-9bc9-d0f789ec6334',
+                    'ext': 'mp4',
+                    'title': 'Everybody Loves Raymond|Everybody Loves Raymond 048 HD, Part 3 of 5',
+                }
+            },
+            {
+                'md5': '247f6780cda6891f2e49b8ae2b10e017',
+                'info_dict': {
+                    'id': '9146ecf5-b15a-4d78-879c-6679b77f4960',
+                    'ext': 'mp4',
+                    'title': 'Everybody Loves Raymond|Everybody Loves Raymond 048 HD, Part 4 of 5',
+                }
+            },
+            {
+                'md5': 'fd269f33256e47bad5eb6c40de089ff6',
+                'info_dict': {
+                    'id': '04334a2e-9a47-4214-a8c2-ae5792e2fab7',
+                    'ext': 'mp4',
+                    'title': 'Everybody Loves Raymond|Everybody Loves Raymond 048 HD, Part 5 of 5',
+                }
+            }
+        ],
+    }, {
+        'url': 'http://www.tvland.com/video-clips/zea2ev/younger-younger--hilary-duff---little-lies',
+        'md5': 'e2c6389401cf485df26c79c247b08713',
+        'info_dict': {
+            'id': 'b8697515-4bbe-4e01-83d5-fa705ce5fa88',
+            'ext': 'mp4',
+            'title': 'Younger|Younger: Hilary Duff - Little Lies',
+            'description': 'md5:7d192f56ca8d958645c83f0de8ef0269'
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        mgid = self._search_regex(r'data-mgid="([^"]+)"', webpage, 'mgid')
+        return self._get_videos_info_from_url(self._FEED_URL + mgid, video_id)

From 7f9134fb2db5559124a0f658bacb6bad54261043 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 20:52:47 +0100
Subject: [PATCH 0246/1105] [tvland] inherit from MTVServicesInfoExtractor

---
 youtube_dl/extractor/tvland.py | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/tvland.py b/youtube_dl/extractor/tvland.py
index 7f049e076..b73279dec 100644
--- a/youtube_dl/extractor/tvland.py
+++ b/youtube_dl/extractor/tvland.py
@@ -1,13 +1,13 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .mtv import MTVIE
+from .mtv import MTVServicesInfoExtractor
 
 
-class TVLandIE(MTVIE):
+class TVLandIE(MTVServicesInfoExtractor):
     IE_NAME = 'tvland.com'
     _VALID_URL = r'https?://(?:www\.)?tvland\.com/(?:video-clips|episodes)/(?P<id>[^/?#.]+)'
-    _FEED_URL = 'http://www.tvland.com/feeds/mrss/?uri='
+    _FEED_URL = 'http://www.tvland.com/feeds/mrss/'
     _TESTS = [{
         'url': 'http://www.tvland.com/episodes/hqhps2/everybody-loves-raymond-the-invasion-ep-048',
         'playlist': [
@@ -62,9 +62,3 @@ class TVLandIE(MTVIE):
             'description': 'md5:7d192f56ca8d958645c83f0de8ef0269'
         },
     }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        mgid = self._search_regex(r'data-mgid="([^"]+)"', webpage, 'mgid')
-        return self._get_videos_info_from_url(self._FEED_URL + mgid, video_id)

From 0416006a3051b15e4bebbb096960ca4fb8ffd0a9 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 15 Dec 2015 21:37:47 +0100
Subject: [PATCH 0247/1105] Fix einthusan parser

---
 youtube_dl/extractor/einthusan.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/einthusan.py b/youtube_dl/extractor/einthusan.py
index 5dfea0d39..bc6def65e 100644
--- a/youtube_dl/extractor/einthusan.py
+++ b/youtube_dl/extractor/einthusan.py
@@ -41,9 +41,12 @@ class EinthusanIE(InfoExtractor):
         video_title = self._html_search_regex(
             r'<h1><a class="movie-title".*?>(.*?)</a></h1>', webpage, 'title')
 
-        video_url = self._html_search_regex(
-            r'''(?s)jwplayer\("mediaplayer"\)\.setup\({.*?'file': '([^']+)'.*?}\);''',
-            webpage, 'video url')
+        movieid = self._html_search_regex(
+            r'data-movieid="(.*?)"', webpage, 'movieid')
+
+        location = 'Washington'
+        geturl = 'http://cdn.einthusan.com/geturl/%s/hd/%s' % (movieid, location)
+        video_url = self._download_webpage(geturl, video_id)
 
         description = self._html_search_meta('description', webpage)
         thumbnail = self._html_search_regex(

From b26afec81f408af37cca6298109ca2a59688bf79 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jan 2016 02:23:03 +0600
Subject: [PATCH 0248/1105] [einthusan] Improve extraction (Closes #7877)

---
 youtube_dl/extractor/einthusan.py | 34 ++++++++++++++++++-------------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/einthusan.py b/youtube_dl/extractor/einthusan.py
index bc6def65e..f7339702c 100644
--- a/youtube_dl/extractor/einthusan.py
+++ b/youtube_dl/extractor/einthusan.py
@@ -1,9 +1,12 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+    remove_start,
+    sanitized_Request,
+)
 
 
 class EinthusanIE(InfoExtractor):
@@ -34,30 +37,33 @@ class EinthusanIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        webpage = self._download_webpage(url, video_id)
+        video_id = self._match_id(url)
 
-        video_title = self._html_search_regex(
-            r'<h1><a class="movie-title".*?>(.*?)</a></h1>', webpage, 'title')
+        request = sanitized_Request(url)
+        request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0')
+        webpage = self._download_webpage(request, video_id)
 
-        movieid = self._html_search_regex(
-            r'data-movieid="(.*?)"', webpage, 'movieid')
+        title = self._html_search_regex(
+            r'<h1><a[^>]+class=["\']movie-title["\'][^>]*>(.+?)</a></h1>',
+            webpage, 'title')
 
-        location = 'Washington'
-        geturl = 'http://cdn.einthusan.com/geturl/%s/hd/%s' % (movieid, location)
-        video_url = self._download_webpage(geturl, video_id)
+        video_id = self._search_regex(
+            r'data-movieid=["\'](\d+)', webpage, 'video id', default=video_id)
+
+        video_url = self._download_webpage(
+            'http://cdn.einthusan.com/geturl/%s/hd/London,Washington,Toronto,Dallas,San,Sydney/'
+            % video_id, video_id)
 
         description = self._html_search_meta('description', webpage)
         thumbnail = self._html_search_regex(
             r'''<a class="movie-cover-wrapper".*?><img src=["'](.*?)["'].*?/></a>''',
             webpage, "thumbnail url", fatal=False)
         if thumbnail is not None:
-            thumbnail = thumbnail.replace('..', 'http://www.einthusan.com')
+            thumbnail = compat_urlparse.urljoin(url, remove_start(thumbnail, '..'))
 
         return {
             'id': video_id,
-            'title': video_title,
+            'title': title,
             'url': video_url,
             'thumbnail': thumbnail,
             'description': description,

From a0d7ede350f13096f94a94f982925ba4ffac91e3 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 15 Dec 2015 21:37:47 +0100
Subject: [PATCH 0249/1105] Fix einthusan parser

---
 youtube_dl/extractor/einthusan.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/einthusan.py b/youtube_dl/extractor/einthusan.py
index 5dfea0d39..bc6def65e 100644
--- a/youtube_dl/extractor/einthusan.py
+++ b/youtube_dl/extractor/einthusan.py
@@ -41,9 +41,12 @@ class EinthusanIE(InfoExtractor):
         video_title = self._html_search_regex(
             r'<h1><a class="movie-title".*?>(.*?)</a></h1>', webpage, 'title')
 
-        video_url = self._html_search_regex(
-            r'''(?s)jwplayer\("mediaplayer"\)\.setup\({.*?'file': '([^']+)'.*?}\);''',
-            webpage, 'video url')
+        movieid = self._html_search_regex(
+            r'data-movieid="(.*?)"', webpage, 'movieid')
+
+        location = 'Washington'
+        geturl = 'http://cdn.einthusan.com/geturl/%s/hd/%s' % (movieid, location)
+        video_url = self._download_webpage(geturl, video_id)
 
         description = self._html_search_meta('description', webpage)
         thumbnail = self._html_search_regex(

From fec09bf15d632bc1694b32981d1468011e9d7da6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jan 2016 02:39:00 +0600
Subject: [PATCH 0250/1105] [einthusan] Improve extraction (Closes #7877)

---
 youtube_dl/extractor/einthusan.py | 34 ++++++++++++++++++-------------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/einthusan.py b/youtube_dl/extractor/einthusan.py
index bc6def65e..f7339702c 100644
--- a/youtube_dl/extractor/einthusan.py
+++ b/youtube_dl/extractor/einthusan.py
@@ -1,9 +1,12 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+    remove_start,
+    sanitized_Request,
+)
 
 
 class EinthusanIE(InfoExtractor):
@@ -34,30 +37,33 @@ class EinthusanIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        webpage = self._download_webpage(url, video_id)
+        video_id = self._match_id(url)
 
-        video_title = self._html_search_regex(
-            r'<h1><a class="movie-title".*?>(.*?)</a></h1>', webpage, 'title')
+        request = sanitized_Request(url)
+        request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0')
+        webpage = self._download_webpage(request, video_id)
 
-        movieid = self._html_search_regex(
-            r'data-movieid="(.*?)"', webpage, 'movieid')
+        title = self._html_search_regex(
+            r'<h1><a[^>]+class=["\']movie-title["\'][^>]*>(.+?)</a></h1>',
+            webpage, 'title')
 
-        location = 'Washington'
-        geturl = 'http://cdn.einthusan.com/geturl/%s/hd/%s' % (movieid, location)
-        video_url = self._download_webpage(geturl, video_id)
+        video_id = self._search_regex(
+            r'data-movieid=["\'](\d+)', webpage, 'video id', default=video_id)
+
+        video_url = self._download_webpage(
+            'http://cdn.einthusan.com/geturl/%s/hd/London,Washington,Toronto,Dallas,San,Sydney/'
+            % video_id, video_id)
 
         description = self._html_search_meta('description', webpage)
         thumbnail = self._html_search_regex(
             r'''<a class="movie-cover-wrapper".*?><img src=["'](.*?)["'].*?/></a>''',
             webpage, "thumbnail url", fatal=False)
         if thumbnail is not None:
-            thumbnail = thumbnail.replace('..', 'http://www.einthusan.com')
+            thumbnail = compat_urlparse.urljoin(url, remove_start(thumbnail, '..'))
 
         return {
             'id': video_id,
-            'title': video_title,
+            'title': title,
             'url': video_url,
             'thumbnail': thumbnail,
             'description': description,

From c1e90619bde0c8b3b17d938195cfeb5ed7803125 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 22:10:00 +0100
Subject: [PATCH 0251/1105] [mtv] extract mgid extraction and query building
 into separate methods

---
 youtube_dl/extractor/mtv.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index d887583e6..e8bb527b8 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -167,14 +167,16 @@ class MTVServicesInfoExtractor(InfoExtractor):
             'description': description,
         }
 
+    def _get_feed_query(self, uri):
+        data = {'uri': uri}
+        if self._LANG:
+            data['lang'] = self._LANG
+        return compat_urllib_parse.urlencode(data)
+
     def _get_videos_info(self, uri):
         video_id = self._id_from_uri(uri)
         feed_url = self._get_feed_url(uri)
-        data = compat_urllib_parse.urlencode({'uri': uri})
-        info_url = feed_url + '?'
-        if self._LANG:
-            info_url += 'lang=%s&' % self._LANG
-        info_url += data
+        info_url = feed_url + '?' + self._get_feed_query(uri)
         return self._get_videos_info_from_url(info_url, video_id)
 
     def _get_videos_info_from_url(self, url, video_id):
@@ -184,9 +186,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         return self.playlist_result(
             [self._get_video_info(item) for item in idoc.findall('.//item')])
 
-    def _real_extract(self, url):
-        title = url_basename(url)
-        webpage = self._download_webpage(url, title)
+    def _extract_mgid(self, webpage):
         try:
             # the url can be http://media.mtvnservices.com/fb/{mgid}.swf
             # or http://media.mtvnservices.com/{mgid}
@@ -207,7 +207,12 @@ class MTVServicesInfoExtractor(InfoExtractor):
                 'sm4:video:embed', webpage, 'sm4 embed', default='')
             mgid = self._search_regex(
                 r'embed/(mgid:.+?)["\'&?/]', sm4_embed, 'mgid')
+        return mgid
 
+    def _real_extract(self, url):
+        title = url_basename(url)
+        webpage = self._download_webpage(url, title)
+        mgid = self._extract_mgid(webpage)
         videos_info = self._get_videos_info(mgid)
         return videos_info
 

From a0e5beb0fbfa5f300087c18b12ff67fade51d578 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 22:12:05 +0100
Subject: [PATCH 0252/1105] [nick] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/nick.py     | 63 ++++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 youtube_dl/extractor/nick.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e20d90bac..120ed9340 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -441,6 +441,7 @@ from .nhl import (
     NHLNewsIE,
     NHLVideocenterIE,
 )
+from .nick import NickIE
 from .niconico import NiconicoIE, NiconicoPlaylistIE
 from .ninegag import NineGagIE
 from .noco import NocoIE
diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
new file mode 100644
index 000000000..4840368ac
--- /dev/null
+++ b/youtube_dl/extractor/nick.py
@@ -0,0 +1,63 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .mtv import MTVServicesInfoExtractor
+from ..compat import compat_urllib_parse
+
+
+class NickIE(MTVServicesInfoExtractor):
+    IE_NAME = 'nick.com'
+    _VALID_URL = r'https?://(?:www\.)?nick\.com/videos/clip/(?P<id>[^/?#.]+)'
+    _FEED_URL = 'http://udat.mtvnservices.com/service1/dispatch.htm'
+    _TESTS = [{
+        'url': 'http://www.nick.com/videos/clip/alvinnn-and-the-chipmunks-112-full-episode.html',
+        'playlist': [
+            {
+                'md5': '6e5adc1e28253bbb1b28ab05403dd4d4',
+                'info_dict': {
+                    'id': 'be6a17b0-412d-11e5-8ff7-0026b9414f30',
+                    'ext': 'mp4',
+                    'title': 'ALVINNN!!! and The Chipmunks: "Mojo Missing/Who\'s The Animal" S1',
+                    'description': 'Alvin is convinced his mojo was in a cap he gave to a fan, and must find a way to get his hat back before the Chipmunks’ big concert.\nDuring a costume visit to the zoo, Alvin finds himself mistaken for the real Tasmanian devil.',
+
+                }
+            },
+            {
+                'md5': 'd7be441fc53a1d4882fa9508a1e5b3ce',
+                'info_dict': {
+                    'id': 'be6b8f96-412d-11e5-8ff7-0026b9414f30',
+                    'ext': 'mp4',
+                    'title': 'ALVINNN!!! and The Chipmunks: "Mojo Missing/Who\'s The Animal" S2',
+                    'description': 'Alvin is convinced his mojo was in a cap he gave to a fan, and must find a way to get his hat back before the Chipmunks’ big concert.\nDuring a costume visit to the zoo, Alvin finds himself mistaken for the real Tasmanian devil.',
+
+                }
+            },
+            {
+                'md5': 'efffe1728a234b2b0d2f2b343dd1946f',
+                'info_dict': {
+                    'id': 'be6cf7e6-412d-11e5-8ff7-0026b9414f30',
+                    'ext': 'mp4',
+                    'title': 'ALVINNN!!! and The Chipmunks: "Mojo Missing/Who\'s The Animal" S3',
+                    'description': 'Alvin is convinced his mojo was in a cap he gave to a fan, and must find a way to get his hat back before the Chipmunks’ big concert.\nDuring a costume visit to the zoo, Alvin finds himself mistaken for the real Tasmanian devil.',
+                }
+            },
+            {
+                'md5': '1ec6690733ab9f41709e274a1d5c7556',
+                'info_dict': {
+                    'id': 'be6e3354-412d-11e5-8ff7-0026b9414f30',
+                    'ext': 'mp4',
+                    'title': 'ALVINNN!!! and The Chipmunks: "Mojo Missing/Who\'s The Animal" S4',
+                    'description': 'Alvin is convinced his mojo was in a cap he gave to a fan, and must find a way to get his hat back before the Chipmunks’ big concert.\nDuring a costume visit to the zoo, Alvin finds himself mistaken for the real Tasmanian devil.',
+                }
+            },
+        ],
+    }]
+
+    def _get_feed_query(self, uri):
+        return compat_urllib_parse.urlencode({
+            'feed': 'nick_arc_player_prime',
+            'mgid': uri
+        })
+
+    def _extract_mgid(self, webpage):
+        return self._search_regex(r'data-contenturi="([^"]+)', webpage, 'mgid')

From e565cf6048739fdfdfbab19e4b35c37a53865807 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 22:47:18 +0100
Subject: [PATCH 0253/1105] [nextmovie] Add new extractor

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/nextmovie.py | 30 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/nick.py      |  2 +-
 3 files changed, 32 insertions(+), 1 deletion(-)
 create mode 100644 youtube_dl/extractor/nextmovie.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 120ed9340..b3f7059e4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -434,6 +434,7 @@ from .nextmedia import (
     NextMediaActionNewsIE,
     AppleDailyIE,
 )
+from .nextmovie import NextMovieIE
 from .nfb import NFBIE
 from .nfl import NFLIE
 from .nhl import (
diff --git a/youtube_dl/extractor/nextmovie.py b/youtube_dl/extractor/nextmovie.py
new file mode 100644
index 000000000..657ae77a0
--- /dev/null
+++ b/youtube_dl/extractor/nextmovie.py
@@ -0,0 +1,30 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .mtv import MTVServicesInfoExtractor
+from ..compat import compat_urllib_parse
+
+
+class NextMovieIE(MTVServicesInfoExtractor):
+    IE_NAME = 'nextmovie.com'
+    _VALID_URL = r'https?://(?:www\.)?nextmovie\.com/shows/[^/]+/\d{4}-\d{2}-\d{2}/(?P<id>[^/?#]+)'
+    _FEED_URL = 'http://lite.dextr.mtvi.com/service1/dispatch.htm'
+    _TESTS = [{
+        'url': 'http://www.nextmovie.com/shows/exclusives/2013-03-10/mgid:uma:videolist:nextmovie.com:1715019/',
+        'md5': '09a9199f2f11f10107d04fcb153218aa',
+        'info_dict': {
+            'id': '961726',
+            'ext': 'mp4',
+            'title': 'The Muppets\' Gravity',
+        },
+    }]
+
+    def _get_feed_query(self, uri):
+        return compat_urllib_parse.urlencode({
+            'feed': '1505',
+            'mgid': uri,
+        })
+
+    def _real_extract(self, url):
+        mgid = self._match_id(url)
+        return self._get_videos_info(mgid)
diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index 4840368ac..b62819ae5 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -56,7 +56,7 @@ class NickIE(MTVServicesInfoExtractor):
     def _get_feed_query(self, uri):
         return compat_urllib_parse.urlencode({
             'feed': 'nick_arc_player_prime',
-            'mgid': uri
+            'mgid': uri,
         })
 
     def _extract_mgid(self, webpage):

From 034caf70b24da1d1de12b00e9ac5620fb7664220 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 31 Dec 2015 13:05:46 +0800
Subject: [PATCH 0254/1105] [youku] Fix extraction (#8068)

---
 youtube_dl/extractor/youku.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 3a3432be8..f767fa15f 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -2,6 +2,9 @@
 from __future__ import unicode_literals
 
 import base64
+import random
+import string
+import time
 
 from .common import InfoExtractor
 from ..compat import (
@@ -141,6 +144,11 @@ class YoukuIE(InfoExtractor):
 
         return video_urls_dict
 
+    @staticmethod
+    def get_ysuid():
+        return '%d%s' % (int(time.time()), ''.join([
+            random.choice(string.ascii_letters) for i in range(3)]))
+
     def get_hd(self, fm):
         hd_id_dict = {
             '3gp': '0',
@@ -189,6 +197,8 @@ class YoukuIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
+        self._set_cookie('youku.com', '__ysuid', self.get_ysuid())
+
         def retrieve_data(req_url, note):
             headers = {
                 'Referer': req_url,

From 0d5095fc65d92251f7aeefd9cb504434590cee8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jan 2016 15:14:41 +0600
Subject: [PATCH 0255/1105] [ccc] Update _VALID_URL (Closes #8097)

---
 youtube_dl/extractor/ccc.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
index 6924eac70..2e0a0b29f 100644
--- a/youtube_dl/extractor/ccc.py
+++ b/youtube_dl/extractor/ccc.py
@@ -12,13 +12,13 @@ from ..utils import (
 
 class CCCIE(InfoExtractor):
     IE_NAME = 'media.ccc.de'
-    _VALID_URL = r'https?://(?:www\.)?media\.ccc\.de/[^?#]+/[^?#/]*?_(?P<id>[0-9]{8,})._[^?#/]*\.html'
+    _VALID_URL = r'https?://(?:www\.)?media\.ccc\.de/v/(?P<id>[^/?#&]+)'
 
-    _TEST = {
-        'url': 'http://media.ccc.de/browse/congress/2013/30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor.html#video',
+    _TESTS = [{
+        'url': 'https://media.ccc.de/v/30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor#video',
         'md5': '3a1eda8f3a29515d27f5adb967d7e740',
         'info_dict': {
-            'id': '20131228183',
+            'id': '30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor',
             'ext': 'mp4',
             'title': 'Introduction to Processor Design',
             'description': 'md5:5ddbf8c734800267f2cee4eab187bc1b',
@@ -26,7 +26,10 @@ class CCCIE(InfoExtractor):
             'view_count': int,
             'upload_date': '20131229',
         }
-    }
+    }, {
+        'url': 'https://media.ccc.de/v/32c3-7368-shopshifting#download',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From c9154514c495bd2259df2250d4cf72a2564a9136 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jan 2016 15:22:22 +0600
Subject: [PATCH 0256/1105] [ccc] Fix upload date extraction

---
 youtube_dl/extractor/ccc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
index 2e0a0b29f..5b549b343 100644
--- a/youtube_dl/extractor/ccc.py
+++ b/youtube_dl/extractor/ccc.py
@@ -46,7 +46,7 @@ class CCCIE(InfoExtractor):
             r"(?s)<p class='description'>(.*?)</p>",
             webpage, 'description', fatal=False)
         upload_date = unified_strdate(self._html_search_regex(
-            r"(?s)<span class='[^']*fa-calendar-o'></span>(.*?)</li>",
+            r"(?s)<span[^>]+class='[^']*fa-calendar-o'[^>]*>(.+?)</span>",
             webpage, 'upload date', fatal=False))
         view_count = int_or_none(self._html_search_regex(
             r"(?s)<span class='[^']*fa-eye'></span>(.*?)</li>",

From 8499d211583f3534c0d0e0f086d471c52ac6c803 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jan 2016 15:29:42 +0600
Subject: [PATCH 0257/1105] [ccc] Fix description extraction and update test

---
 youtube_dl/extractor/ccc.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
index 5b549b343..45115c838 100644
--- a/youtube_dl/extractor/ccc.py
+++ b/youtube_dl/extractor/ccc.py
@@ -21,10 +21,10 @@ class CCCIE(InfoExtractor):
             'id': '30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor',
             'ext': 'mp4',
             'title': 'Introduction to Processor Design',
-            'description': 'md5:5ddbf8c734800267f2cee4eab187bc1b',
+            'description': 'md5:80be298773966f66d56cb11260b879af',
             'thumbnail': 're:^https?://.*\.jpg$',
             'view_count': int,
-            'upload_date': '20131229',
+            'upload_date': '20131228',
         }
     }, {
         'url': 'https://media.ccc.de/v/32c3-7368-shopshifting#download',
@@ -43,7 +43,7 @@ class CCCIE(InfoExtractor):
         title = self._html_search_regex(
             r'(?s)<h1>(.*?)</h1>', webpage, 'title')
         description = self._html_search_regex(
-            r"(?s)<p class='description'>(.*?)</p>",
+            r"(?s)<h3>About</h3>(.+?)<h3>",
             webpage, 'description', fatal=False)
         upload_date = unified_strdate(self._html_search_regex(
             r"(?s)<span[^>]+class='[^']*fa-calendar-o'[^>]*>(.+?)</span>",

From 82597f0ec0b8ec208e77d27f62904de7804b914d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jan 2016 15:41:52 +0600
Subject: [PATCH 0258/1105] [ccc] Extract duration

---
 youtube_dl/extractor/ccc.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
index 45115c838..710e5919c 100644
--- a/youtube_dl/extractor/ccc.py
+++ b/youtube_dl/extractor/ccc.py
@@ -5,6 +5,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
+    parse_duration,
     qualities,
     unified_strdate,
 )
@@ -25,6 +26,7 @@ class CCCIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.jpg$',
             'view_count': int,
             'upload_date': '20131228',
+            'duration': 3660,
         }
     }, {
         'url': 'https://media.ccc.de/v/32c3-7368-shopshifting#download',
@@ -51,6 +53,9 @@ class CCCIE(InfoExtractor):
         view_count = int_or_none(self._html_search_regex(
             r"(?s)<span class='[^']*fa-eye'></span>(.*?)</li>",
             webpage, 'view count', fatal=False))
+        duration = parse_duration(self._html_search_regex(
+            r'(?s)<span[^>]+class=(["\']).*?fa-clock-o.*?\1[^>]*></span>(?P<duration>.+?)</li',
+            webpage, 'duration', fatal=False, group='duration'))
 
         matches = re.finditer(r'''(?xs)
             <(?:span|div)\s+class='label\s+filetype'>(?P<format>.*?)</(?:span|div)>\s*
@@ -98,5 +103,6 @@ class CCCIE(InfoExtractor):
             'thumbnail': thumbnail,
             'view_count': view_count,
             'upload_date': upload_date,
+            'duration': duration,
             'formats': formats,
         }

From 190ef0798103908d986e07f54323299084726d42 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 1 Jan 2016 12:17:10 +0100
Subject: [PATCH 0259/1105] release 2016.01.01

---
 docs/supportedsites.md | 7 +++++--
 youtube_dl/version.py  | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 2297edc6b..84c166805 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -28,7 +28,6 @@
  - **AlJazeera**
  - **Allocine**
  - **AlphaPorno**
- - **AnimalPlanet**
  - **anitube.se**
  - **AnySex**
  - **Aparat**
@@ -368,11 +367,13 @@
  - **Newstube**
  - **NextMedia**: 蘋果日報
  - **NextMediaActionNews**: 蘋果日報 - 動新聞
+ - **nextmovie.com**
  - **nfb**: National Film Board of Canada
  - **nfl.com**
  - **nhl.com**
  - **nhl.com:news**: NHL news
  - **nhl.com:videocenter**: NHL videocenter category
+ - **nick.com**
  - **niconico**: ニコニコ動画
  - **NiconicoPlaylist**
  - **njoy**: N-JOY
@@ -411,6 +412,7 @@
  - **orf:iptv**: iptv.ORF.at
  - **orf:oe1**: Radio Österreich 1
  - **orf:tvthek**: ORF TVthek
+ - **pandora.tv**: 판도라TV
  - **parliamentlive.tv**: UK parliament videos
  - **Patreon**
  - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), CONNECTICUT PUBLIC TELEVISION (WEDH), KNPB Channel 5 (KNPB), SOPTV (KSYS), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), PBS 39 (WLVT), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WGCU Public Media (WGCU), WPBT2 (WPBT), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), ETV (WRLK), UNC-TV (WUNC), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), OPB (KOPB), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WTTW11 (WTTW), Iowa Public Television/IPTV (KDIN), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), WIPB-TV (WIPB), WTIU (WTIU), CET  (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), KSMQ (KSMQ), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), East Tennessee PBS (WSJK), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVPB (WVPB), WKYU-PBS (WKYU), KERA 13 (KERA), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), witf (WITF), WQED Multimedia (WQED), WMHT Educational Telecommunications (WMHT), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WGTE TV (WGTE), Lakeland Public Television (KAWE), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), MontanaPBS (KUSM), KRWG/Channel 22 (KRWG), KACV (KACV), KCOS/Channel 13 (KCOS), WCNY/Channel 24 (WCNY), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), WPSU (WPSU), WVIA Public Media Studios (WVIA), WTVI (WTVI), Western Reserve PBS (WNEO), WVIZ/PBS ideastream (WVIZ), KCTS 9 (KCTS), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KLRN (KLRN), KLRU (KLRU), WTJX Channel 12 (WTJX), WCVE PBS (WCVE), KBTC Public Television (KBTC)
@@ -459,6 +461,7 @@
  - **RBMARadio**
  - **RDS**: RDS.ca
  - **RedTube**
+ - **RegioTV**
  - **Restudy**
  - **ReverbNation**
  - **RingTV**
@@ -582,7 +585,6 @@
  - **THVideo**
  - **THVideoPlaylist**
  - **tinypic**: tinypic.com videos
- - **tlc.com**
  - **tlc.de**
  - **TMZ**
  - **TMZArticle**
@@ -611,6 +613,7 @@
  - **TVC**
  - **TVCArticle**
  - **tvigle**: Интернет-телевидение Tvigle.ru
+ - **tvland.com**
  - **tvp.pl**
  - **tvp.pl:Series**
  - **TVPlay**: TV3Play and related services
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index cf6c4d74d..790bd5b3b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.12.31'
+__version__ = '2016.01.01'

From 32f9036447d1211f9ce0750203d71671f0ee99dc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 1 Jan 2016 13:28:45 +0100
Subject: [PATCH 0260/1105] [ccc] Add language information to formats

---
 youtube_dl/YoutubeDL.py        | 4 ++++
 youtube_dl/extractor/ccc.py    | 8 ++++++--
 youtube_dl/extractor/common.py | 5 +++--
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 50425b8d7..3b2be3159 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1791,6 +1791,10 @@ class YoutubeDL(object):
         res = ''
         if fdict.get('ext') in ['f4f', 'f4m']:
             res += '(unsupported) '
+        if fdict.get('language'):
+            if res:
+                res += ' '
+            res += '[%s]' % fdict['language']
         if fdict.get('format_note') is not None:
             res += fdict['format_note'] + ' '
         if fdict.get('tbr') is not None:
diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
index 710e5919c..e94b1e35b 100644
--- a/youtube_dl/extractor/ccc.py
+++ b/youtube_dl/extractor/ccc.py
@@ -58,11 +58,12 @@ class CCCIE(InfoExtractor):
             webpage, 'duration', fatal=False, group='duration'))
 
         matches = re.finditer(r'''(?xs)
-            <(?:span|div)\s+class='label\s+filetype'>(?P<format>.*?)</(?:span|div)>\s*
+            <(?:span|div)\s+class='label\s+filetype'>(?P<format>[^<]*)</(?:span|div)>\s*
+            <(?:span|div)\s+class='label\s+filetype'>(?P<lang>[^<]*)</(?:span|div)>\s*
             <a\s+download\s+href='(?P<http_url>[^']+)'>\s*
             (?:
                 .*?
-                <a\s+href='(?P<torrent_url>[^']+\.torrent)'
+                <a\s+(?:download\s+)?href='(?P<torrent_url>[^']+\.torrent)'
             )?''', webpage)
         formats = []
         for m in matches:
@@ -70,12 +71,15 @@ class CCCIE(InfoExtractor):
             format_id = self._search_regex(
                 r'.*/([a-z0-9_-]+)/[^/]*$',
                 m.group('http_url'), 'format id', default=None)
+            if format_id:
+                format_id = m.group('lang') + '-' + format_id
             vcodec = 'h264' if 'h264' in format_id else (
                 'none' if format_id in ('mp3', 'opus') else None
             )
             formats.append({
                 'format_id': format_id,
                 'format': format,
+                'language': m.group('lang'),
                 'url': m.group('http_url'),
                 'vcodec': vcodec,
                 'preference': preference(format_id),
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 655207447..2823b1d18 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -108,8 +108,9 @@ class InfoExtractor(object):
                                  -2 or smaller for less than default.
                                  < -1000 to hide the format (if there is
                                     another one which is strictly better)
-                    * language_preference  Is this in the correct requested
-                                 language?
+                    * language   Language code, e.g. "de" or "en-US".
+                    * language_preference  Is this in the language mentioned in
+                                 the URL?
                                  10 if it's what the URL is about,
                                  -1 for default (don't know),
                                  -10 otherwise, other values reserved for now.

From ca227c8698340ad9170698cef81ab4bf4d832a55 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 1 Jan 2016 14:32:00 +0100
Subject: [PATCH 0261/1105] [yahoo] Support pages that use an alias (fixes
 #8084)

---
 youtube_dl/extractor/yahoo.py | 34 ++++++++++++++++++++++++++--------
 1 file changed, 26 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index fca5ddc69..4a492f784 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -155,7 +155,16 @@ class YahooIE(InfoExtractor):
                 'description': 'md5:8fc39608213295748e1e289807838c97',
                 'duration': 1646,
             },
-        }
+        }, {
+            # it uses an alias to get the video_id
+            'url': 'https://www.yahoo.com/movies/the-stars-of-daddys-home-have-very-different-212843197.html',
+            'info_dict': {
+                'id': '40eda9c8-8e5f-3552-8745-830f67d0c737',
+                'ext': 'mp4',
+                'title': 'Will Ferrell & Mark Wahlberg Are Pro-Spanking',
+                'description': 'While they play feuding fathers in \'Daddy\'s Home,\' star Will Ferrell & Mark Wahlberg share their true feelings on parenthood.',
+            },
+        },
     ]
 
     def _real_extract(self, url):
@@ -199,13 +208,22 @@ class YahooIE(InfoExtractor):
             r'mediaItems: ({.*?})$', webpage, 'items', flags=re.MULTILINE,
             default=None)
         if items_json is None:
-            CONTENT_ID_REGEXES = [
-                r'YUI\.namespace\("Media"\)\.CONTENT_ID\s*=\s*"([^"]+)"',
-                r'root\.App\.Cache\.context\.videoCache\.curVideo = \{"([^"]+)"',
-                r'"first_videoid"\s*:\s*"([^"]+)"',
-                r'%s[^}]*"ccm_id"\s*:\s*"([^"]+)"' % re.escape(page_id),
-            ]
-            video_id = self._search_regex(CONTENT_ID_REGEXES, webpage, 'content ID')
+            alias = self._search_regex(
+                r'"aliases":{"video":"(.*?)"', webpage, 'alias', default=None)
+            if alias is not None:
+                alias_info = self._download_json(
+                    'https://www.yahoo.com/_td/api/resource/VideoService.videos;video_aliases=["%s"]' % alias,
+                    display_id, 'Downloading alias info')
+                video_id = alias_info[0]['id']
+            else:
+                CONTENT_ID_REGEXES = [
+                    r'YUI\.namespace\("Media"\)\.CONTENT_ID\s*=\s*"([^"]+)"',
+                    r'root\.App\.Cache\.context\.videoCache\.curVideo = \{"([^"]+)"',
+                    r'"first_videoid"\s*:\s*"([^"]+)"',
+                    r'%s[^}]*"ccm_id"\s*:\s*"([^"]+)"' % re.escape(page_id),
+                ]
+                video_id = self._search_regex(
+                    CONTENT_ID_REGEXES, webpage, 'content ID')
         else:
             items = json.loads(items_json)
             info = items['mediaItems']['query']['results']['mediaObj'][0]

From 27bfd4e5266623d9000ba17e6bb2a72eefb0e22d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jan 2016 20:26:56 +0600
Subject: [PATCH 0262/1105] [extractor/common] Introduce number fields for
 chapters and series

---
 youtube_dl/extractor/common.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index c443daf20..c63157619 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -204,21 +204,21 @@ class InfoExtractor(object):
     chapter or section:
 
     chapter:        Name or title of the chapter the video belongs to.
-    chapter_id:     Number or id of the chapter the video belongs to, as an integer
-                    or unicode string.
+    chapter_number: Number of the chapter the video belongs to, as an integer.
+    chapter_id:     Id of the chapter the video belongs to, as a unicode string.
 
     The following fields should only be used when the video is an episode of some
     series or programme:
 
     series:         Title of the series or programme the video episode belongs to.
     season:         Title of the season the video episode belongs to.
-    season_id:      Number or id of the season the video episode belongs to, as an
-                    integer or unicode string.
+    season_number:  Number of the season the video episode belongs to, as an integer.
+    season_id:      Id of the season the video episode belongs to, as a unicode string.
     episode:        Title of the video episode. Unlike mandatory video title field,
                     this field should denote the exact title of the video episode
                     without any kind of decoration.
-    episode_id:     Number or id of the video episode within a season, as an integer
-                    or unicode string.
+    episode_number: Number of the video episode within a season, as an integer.
+    episode_id:     Id of the video episode, as a unicode string.
 
     Unless mentioned otherwise, the fields should be Unicode strings.
 

From 306c51c66922abb1846640b81c9b7c4d6a570d37 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jan 2016 20:30:08 +0600
Subject: [PATCH 0263/1105] [videomore] Use number fields for series

---
 youtube_dl/extractor/videomore.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/videomore.py b/youtube_dl/extractor/videomore.py
index 3bd96e445..a66d6de23 100644
--- a/youtube_dl/extractor/videomore.py
+++ b/youtube_dl/extractor/videomore.py
@@ -25,9 +25,9 @@ class VideomoreIE(InfoExtractor):
             'description': 'В гостях – лучшие романтические комедии года, «Выживший» Иньярриту и «Стив Джобс» Дэнни Бойла.',
             'series': 'Кино в деталях',
             'episode': 'В гостях Алексей Чумаков и Юлия Ковальчук',
-            'episode_id': None,
+            'episode_number': None,
             'season': 'Сезон 2015',
-            'season_id': 5,
+            'season_number': 5,
             'thumbnail': 're:^https?://.*\.jpg',
             'duration': 2910,
             'age_limit': 16,
@@ -42,9 +42,9 @@ class VideomoreIE(InfoExtractor):
             'description': '«Медведей» ждет решающий матч. Макеев выясняет отношения со Стрельцовым. Парни узнают подробности прошлого Макеева.',
             'series': 'Молодежка',
             'episode': '80 серия',
-            'episode_id': 40,
+            'episode_number': 40,
             'season': '2 сезон',
-            'season_id': 2,
+            'season_number': 2,
             'thumbnail': 're:^https?://.*\.jpg',
             'duration': 2809,
             'age_limit': 16,
@@ -62,9 +62,9 @@ class VideomoreIE(InfoExtractor):
             'description': 'Молодежка 3 сезон скоро',
             'series': 'Молодежка',
             'episode': 'Команда проиграла из-за Бакина?',
-            'episode_id': None,
+            'episode_number': None,
             'season': 'Промо',
-            'season_id': 99,
+            'season_number': 99,
             'thumbnail': 're:^https?://.*\.jpg',
             'duration': 29,
             'age_limit': 16,
@@ -128,9 +128,9 @@ class VideomoreIE(InfoExtractor):
 
         series = data.get('project_title')
         episode = data.get('title')
-        episode_id = data.get('episode_of_season') or None
+        episode_number = int_or_none(data.get('episode_of_season') or None)
         season = data.get('season_title')
-        season_id = data.get('season_pos') or None
+        season_number = int_or_none(data.get('season_pos') or None)
 
         return {
             'id': video_id,
@@ -138,9 +138,9 @@ class VideomoreIE(InfoExtractor):
             'description': description,
             'series': series,
             'episode': episode,
-            'episode_id': episode_id,
+            'episode_number': episode_number,
             'season': season,
-            'season_id': season_id,
+            'season_number': season_number,
             'thumbnails': thumbnails,
             'timestamp': timestamp,
             'duration': duration,

From 5bafcf65255261c2dc6482de6b1e545e5de1cbee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jan 2016 20:34:29 +0600
Subject: [PATCH 0264/1105] [udemy] Use chapter_number

---
 youtube_dl/extractor/udemy.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 41097829d..1df636779 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -245,7 +245,7 @@ class UdemyCourseIE(UdemyIE):
             course_id, 'Downloading course curriculum')
 
         entries = []
-        chapter, chapter_id = None, None
+        chapter, chapter_number = None, None
         for asset in response:
             asset_type = asset.get('assetType') or asset.get('asset_type')
             if asset_type == 'Video':
@@ -256,13 +256,13 @@ class UdemyCourseIE(UdemyIE):
                         'url': 'https://www.udemy.com/%s/#/lecture/%s' % (course_path, asset['id']),
                         'ie_key': UdemyIE.ie_key(),
                     }
-                    if chapter_id:
-                        entry['chapter_id'] = chapter_id
+                    if chapter_number:
+                        entry['chapter_number'] = chapter_number
                     if chapter:
                         entry['chapter'] = chapter
                     entries.append(entry)
             elif asset.get('type') == 'chapter':
-                chapter_id = asset.get('index') or asset.get('object_index')
+                chapter_number = asset.get('index') or asset.get('object_index')
                 chapter = asset.get('title')
 
         return self.playlist_result(entries, course_id, course_title)

From 054479754c661f40140f5db7f58916a0006c6b10 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 1 Jan 2016 21:03:16 +0100
Subject: [PATCH 0265/1105] [revision3] Add new extractor(closes #6388)

- revision3.com
- testtube.com
- animalist.com
---
 youtube_dl/extractor/__init__.py  |   1 +
 youtube_dl/extractor/revision3.py | 103 ++++++++++++++++++++++++++++++
 2 files changed, 104 insertions(+)
 create mode 100644 youtube_dl/extractor/revision3.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b3f7059e4..7adce5499 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -557,6 +557,7 @@ from .redtube import RedTubeIE
 from .regiotv import RegioTVIE
 from .restudy import RestudyIE
 from .reverbnation import ReverbNationIE
+from .revision3 import Revision3IE
 from .ringtv import RingTVIE
 from .ro220 import Ro220IE
 from .rottentomatoes import RottenTomatoesIE
diff --git a/youtube_dl/extractor/revision3.py b/youtube_dl/extractor/revision3.py
new file mode 100644
index 000000000..25fe4ef16
--- /dev/null
+++ b/youtube_dl/extractor/revision3.py
@@ -0,0 +1,103 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+    unescapeHTML,
+)
+
+
+class Revision3IE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:revision3|testtube|animalist)\.com)/(?P<id>[^/]+(?:/[^/?#]+)?)'
+    _TESTS = [{
+        'url': 'http://www.revision3.com/technobuffalo/5-google-predictions-for-2016',
+        'md5': 'd94a72d85d0a829766de4deb8daaf7df',
+        'info_dict': {
+            'id': '73034',
+            'ext': 'webm',
+            'title': '5 Google Predictions for 2016',
+            'description': 'Google had a great 2015, but it\'s already time to look ahead. Here are our five predictions for 2016.',
+            'upload_date': '20151228',
+            'timestamp': 1451325600,
+            'duration': 187,
+        }
+    }, {
+        'url': 'http://testtube.com/brainstuff',
+        'info_dict': {
+            'id': '251',
+            'title': 'BrainStuff',
+            'description': 'Whether the topic is popcorn or particle physics, you can count on the HowStuffWorks team to explore-and explain-the everyday science in the world around us on BrainStuff.',
+        },
+        'playlist_mincount': 93,
+    }]
+    _PAGE_DATA_TEMPLATE = 'http://www.%s/apiProxy/ddn/%s?domain=%s'
+    _API_KEY = 'ba9c741bce1b9d8e3defcc22193f3651b8867e62'
+
+    def _real_extract(self, url):
+        domain, display_id = re.match(self._VALID_URL, url).groups()
+        page_info = self._download_json(
+            self._PAGE_DATA_TEMPLATE % (domain, display_id, domain), display_id)
+
+        if page_info['data']['type'] == 'episode':
+            episode_data = page_info['data']
+            video_id = compat_str(episode_data['video']['data']['id'])
+            video_data = self._download_json(
+                'http://revision3.com/api/getPlaylist.json?api_key=%s&codecs=h264,vp8,theora&video_id=%s' % (self._API_KEY, video_id),
+                video_id)['items'][0]
+
+            formats = []
+            for media_type, media in video_data['media'].items():
+                for quality_id, quality in media.items():
+                    if quality_id == 'hls':
+                        formats.extend(self._extract_m3u8_formats(
+                            quality['url'], video_id, 'mp4',
+                            'm3u8_native', m3u8_id='hls', fatal=False))
+                    else:
+                        formats.append({
+                            'url': quality['url'],
+                            'format_id': '%s-%s' % (media_type, quality_id),
+                            'tbr': int_or_none(quality.get('bitrate')),
+                        })
+            self._sort_formats(formats)
+
+            thumbnails = [{
+                'url': image_url,
+                'id': image_id,
+            } for image_id, image_url in video_data.get('images', {}).items()]
+
+            return {
+                'id': video_id,
+                'title': unescapeHTML(video_data['title']),
+                'description': unescapeHTML(video_data.get('summary')),
+                'timestamp': parse_iso8601(episode_data.get('publishTime'), ' '),
+                'author': episode_data.get('author'),
+                'duration': int_or_none(video_data.get('duration')),
+                'thumbnails': thumbnails,
+                'formats': formats,
+            }
+        else:
+            show_data = page_info['show']['data']
+            episodes_data = page_info['episodes']['data']
+            num_episodes = page_info['meta']['totalEpisodes']
+            processed_episodes = 0
+            entries = []
+            page_num = 1
+            while True:
+                entries.extend([self.url_result(
+                    url + '/%s' % episode['slug']) for episode in episodes_data])
+                processed_episodes += len(episodes_data)
+                if processed_episodes == num_episodes:
+                    break
+                page_num += 1
+                episodes_data = self._download_json(self._PAGE_DATA_TEMPLATE % (
+                    domain, display_id + '/' + compat_str(page_num), domain),
+                    display_id)['episodes']['data']
+
+            return self.playlist_result(
+                entries, compat_str(show_data['id']),
+                show_data.get('name'), show_data.get('summary'))

From 8af2804a5d09e08d8916e7a7d2eab40864e9ecd8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 1 Jan 2016 21:53:19 +0100
Subject: [PATCH 0266/1105] [testtube] Remove Extractor

---
 youtube_dl/extractor/revision3.py | 30 +++++++++--
 youtube_dl/extractor/testtube.py  | 90 -------------------------------
 2 files changed, 27 insertions(+), 93 deletions(-)
 delete mode 100644 youtube_dl/extractor/testtube.py

diff --git a/youtube_dl/extractor/revision3.py b/youtube_dl/extractor/revision3.py
index 25fe4ef16..b1b8800b9 100644
--- a/youtube_dl/extractor/revision3.py
+++ b/youtube_dl/extractor/revision3.py
@@ -9,6 +9,7 @@ from ..utils import (
     int_or_none,
     parse_iso8601,
     unescapeHTML,
+    qualities,
 )
 
 
@@ -19,12 +20,15 @@ class Revision3IE(InfoExtractor):
         'md5': 'd94a72d85d0a829766de4deb8daaf7df',
         'info_dict': {
             'id': '73034',
+            'display_id': 'technobuffalo/5-google-predictions-for-2016',
             'ext': 'webm',
             'title': '5 Google Predictions for 2016',
             'description': 'Google had a great 2015, but it\'s already time to look ahead. Here are our five predictions for 2016.',
             'upload_date': '20151228',
             'timestamp': 1451325600,
             'duration': 187,
+            'uploader': 'TechnoBuffalo',
+            'uploader_id': 'technobuffalo',
         }
     }, {
         'url': 'http://testtube.com/brainstuff',
@@ -34,6 +38,20 @@ class Revision3IE(InfoExtractor):
             'description': 'Whether the topic is popcorn or particle physics, you can count on the HowStuffWorks team to explore-and explain-the everyday science in the world around us on BrainStuff.',
         },
         'playlist_mincount': 93,
+    }, {
+        'url': 'https://testtube.com/dnews/5-weird-ways-plants-can-eat-animals?utm_source=FB&utm_medium=DNews&utm_campaign=DNewsSocial',
+        'info_dict': {
+            'id': '60163',
+            'display_id': 'dnews/5-weird-ways-plants-can-eat-animals',
+            'duration': 275,
+            'ext': 'webm',
+            'title': '5 Weird Ways Plants Can Eat Animals',
+            'description': 'Why have some plants evolved to eat meat?',
+            'upload_date': '20150120',
+            'timestamp': 1421763300,
+            'uploader': 'DNews',
+            'uploader_id': 'dnews',
+        },
     }]
     _PAGE_DATA_TEMPLATE = 'http://www.%s/apiProxy/ddn/%s?domain=%s'
     _API_KEY = 'ba9c741bce1b9d8e3defcc22193f3651b8867e62'
@@ -51,7 +69,7 @@ class Revision3IE(InfoExtractor):
                 video_id)['items'][0]
 
             formats = []
-            for media_type, media in video_data['media'].items():
+            for vcodec, media in video_data['media'].items():
                 for quality_id, quality in media.items():
                     if quality_id == 'hls':
                         formats.extend(self._extract_m3u8_formats(
@@ -60,22 +78,28 @@ class Revision3IE(InfoExtractor):
                     else:
                         formats.append({
                             'url': quality['url'],
-                            'format_id': '%s-%s' % (media_type, quality_id),
+                            'format_id': '%s-%s' % (vcodec, quality_id),
                             'tbr': int_or_none(quality.get('bitrate')),
+                            'vcodec': vcodec,
                         })
             self._sort_formats(formats)
 
+            preference = qualities(['mini', 'small', 'medium', 'large'])
             thumbnails = [{
                 'url': image_url,
                 'id': image_id,
+                'preference': preference(image_id)
             } for image_id, image_url in video_data.get('images', {}).items()]
 
             return {
                 'id': video_id,
+                'display_id': display_id,
                 'title': unescapeHTML(video_data['title']),
                 'description': unescapeHTML(video_data.get('summary')),
                 'timestamp': parse_iso8601(episode_data.get('publishTime'), ' '),
                 'author': episode_data.get('author'),
+                'uploader': video_data.get('show', {}).get('name'),
+                'uploader_id': video_data.get('show', {}).get('slug'),
                 'duration': int_or_none(video_data.get('duration')),
                 'thumbnails': thumbnails,
                 'formats': formats,
@@ -89,7 +113,7 @@ class Revision3IE(InfoExtractor):
             page_num = 1
             while True:
                 entries.extend([self.url_result(
-                    url + '/%s' % episode['slug']) for episode in episodes_data])
+                    'http://%s/%s/%s' % (domain, display_id, episode['slug'])) for episode in episodes_data])
                 processed_episodes += len(episodes_data)
                 if processed_episodes == num_episodes:
                     break
diff --git a/youtube_dl/extractor/testtube.py b/youtube_dl/extractor/testtube.py
deleted file mode 100644
index 26655d690..000000000
--- a/youtube_dl/extractor/testtube.py
+++ /dev/null
@@ -1,90 +0,0 @@
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    qualities,
-)
-
-
-class TestTubeIE(InfoExtractor):
-    _VALID_URL = r'https?://testtube\.com/[^/?#]+/(?P<id>[^/?#]+)'
-    _TESTS = [{
-        'url': 'https://testtube.com/dnews/5-weird-ways-plants-can-eat-animals?utm_source=FB&utm_medium=DNews&utm_campaign=DNewsSocial',
-        'info_dict': {
-            'id': '60163',
-            'display_id': '5-weird-ways-plants-can-eat-animals',
-            'duration': 275,
-            'ext': 'webm',
-            'title': '5 Weird Ways Plants Can Eat Animals',
-            'description': 'Why have some plants evolved to eat meat?',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'uploader': 'DNews',
-            'uploader_id': 'dnews',
-        },
-    }, {
-        'url': 'https://testtube.com/iflscience/insane-jet-ski-flipping',
-        'info_dict': {
-            'id': 'fAGfJ4YjVus',
-            'ext': 'mp4',
-            'title': 'Flipping Jet-Ski Skills | Outrageous Acts of Science',
-            'uploader': 'Science Channel',
-            'uploader_id': 'ScienceChannel',
-            'upload_date': '20150203',
-            'description': 'md5:e61374030015bae1d2e22f096d4769d6',
-        }
-    }]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, display_id)
-
-        youtube_url = self._html_search_regex(
-            r'<iframe[^>]+src="((?:https?:)?//www.youtube.com/embed/[^"]+)"',
-            webpage, 'youtube iframe', default=None)
-        if youtube_url:
-            return self.url_result(youtube_url, 'Youtube', video_id=display_id)
-
-        video_id = self._search_regex(
-            r"player\.loadRevision3Item\('video_id',\s*([0-9]+)\);",
-            webpage, 'video ID')
-
-        all_info = self._download_json(
-            'https://testtube.com/api/getPlaylist.json?api_key=ba9c741bce1b9d8e3defcc22193f3651b8867e62&codecs=h264,vp8,theora&video_id=%s' % video_id,
-            video_id)
-        info = all_info['items'][0]
-
-        formats = []
-        for vcodec, fdatas in info['media'].items():
-            for name, fdata in fdatas.items():
-                formats.append({
-                    'format_id': '%s-%s' % (vcodec, name),
-                    'url': fdata['url'],
-                    'vcodec': vcodec,
-                    'tbr': fdata.get('bitrate'),
-                })
-        self._sort_formats(formats)
-
-        duration = int_or_none(info.get('duration'))
-        images = info.get('images')
-        thumbnails = None
-        preference = qualities(['mini', 'small', 'medium', 'large'])
-        if images:
-            thumbnails = [{
-                'id': thumbnail_id,
-                'url': img_url,
-                'preference': preference(thumbnail_id)
-            } for thumbnail_id, img_url in images.items()]
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'title': info['title'],
-            'description': info.get('summary'),
-            'thumbnails': thumbnails,
-            'uploader': info.get('show', {}).get('name'),
-            'uploader_id': info.get('show', {}).get('slug'),
-            'duration': duration,
-            'formats': formats,
-        }

From 94de6cf59cb1933e395775f9ffca4fa311adb0dc Mon Sep 17 00:00:00 2001
From: pingtux <pingtux@users.noreply.github.com>
Date: Sat, 2 Jan 2016 01:35:09 +0100
Subject: [PATCH 0267/1105] Remove testtube import

Extractor got deleted in remitamine/youtube-dl@8af2804
---
 youtube_dl/extractor/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7adce5499..4c7e5223d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -687,7 +687,6 @@ from .telemb import TeleMBIE
 from .teletask import TeleTaskIE
 from .tenplay import TenPlayIE
 from .testurl import TestURLIE
-from .testtube import TestTubeIE
 from .tf1 import TF1IE
 from .theintercept import TheInterceptIE
 from .theonion import TheOnionIE

From 76a353c9e5511a522b5331cb3f74d4215341791c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Jan 2016 07:44:30 +0600
Subject: [PATCH 0268/1105] [ruutu] Fix extraction (Closes #8107)

---
 youtube_dl/extractor/ruutu.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index 41fddc375..ffea438cc 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -75,9 +75,12 @@ class RuutuIE(InfoExtractor):
                         preference = -1 if proto == 'rtmp' else 1
                         label = child.get('label')
                         tbr = int_or_none(child.get('bitrate'))
+                        format_id = '%s-%s' % (proto, label if label else tbr) if label or tbr else proto
+                        if not self._is_valid_url(video_url, video_id, format_id):
+                            continue
                         width, height = [int_or_none(x) for x in child.get('resolution', 'x').split('x')[:2]]
                         formats.append({
-                            'format_id': '%s-%s' % (proto, label if label else tbr),
+                            'format_id': format_id,
                             'url': video_url,
                             'width': width,
                             'height': height,

From f20a11ed257f21a1c658f827e0c2129a72582adf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Jan 2016 19:22:39 +0600
Subject: [PATCH 0269/1105] [bbccouk] Extend _VALID_URL (Closes #8116)

---
 youtube_dl/extractor/bbc.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 923273fb2..5c621a32b 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -23,7 +23,17 @@ class BBCCoUkIE(InfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
     _ID_REGEX = r'[pb][\da-z]{7}'
-    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:programmes/(?!articles/)|iplayer(?:/[^/]+)?/(?:episode/|playlist/))|music/clips[/#])(?P<id>%s)' % _ID_REGEX
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?bbc\.co\.uk/
+                        (?:
+                            programmes/(?!articles/)|
+                            iplayer(?:/[^/]+)?/(?:episode/|playlist/)|
+                            music/clips[/#]|
+                            radio/player/
+                        )
+                        (?P<id>%s)
+                    ''' % _ID_REGEX
 
     _MEDIASELECTOR_URLS = [
         # Provides HQ HLS streams with even better quality that pc mediaset but fails
@@ -193,6 +203,9 @@ class BBCCoUkIE(InfoExtractor):
         }, {
             'url': 'http://www.bbc.co.uk/iplayer/cbeebies/episode/b0480276/bing-14-atchoo',
             'only_matching': True,
+        }, {
+            'url': 'http://www.bbc.co.uk/radio/player/p03cchwf',
+            'only_matching': True,
         }
     ]
 

From c579c5e967cd223baa2c1f593b6ed65ac8643e42 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 Jan 2016 21:31:02 +0800
Subject: [PATCH 0270/1105] [baidu] Cleanups

---
 youtube_dl/extractor/baidu.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py
index 84fab551b..734b50d1d 100644
--- a/youtube_dl/extractor/baidu.py
+++ b/youtube_dl/extractor/baidu.py
@@ -4,7 +4,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
 
 
 class BaiduVideoIE(InfoExtractor):
@@ -47,8 +46,6 @@ class BaiduVideoIE(InfoExtractor):
 
         entries = []
         for episode in episodes_detail['videos']:
-            episode_id = '%s_%s' % (playlist_id, episode['episode'])
-
             entries.append(self.url_result(
                 episode['url'], video_title=episode['title']))
 

From a1d9f6c5dc15f1d62e0ab8c9dd6feaed4ff83a34 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 Jan 2016 21:36:35 +0800
Subject: [PATCH 0271/1105] [baidu] Improve playlist description

---
 youtube_dl/extractor/baidu.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py
index 734b50d1d..d60cb0fdd 100644
--- a/youtube_dl/extractor/baidu.py
+++ b/youtube_dl/extractor/baidu.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import unescapeHTML
 
 
 class BaiduVideoIE(InfoExtractor):
@@ -14,7 +15,7 @@ class BaiduVideoIE(InfoExtractor):
         'info_dict': {
             'id': '1069',
             'title': '中华小当家 TV版国语',
-            'description': 'md5:40a9c1b1c7f4e05d642e7bb1c84eeda0',
+            'description': 'md5:51be07afe461cf99fa61231421b5397c',
         },
         'playlist_count': 52,
     }, {
@@ -40,7 +41,7 @@ class BaiduVideoIE(InfoExtractor):
         playlist_detail = self._call_api('xqinfo', category, playlist_id)
 
         playlist_title = playlist_detail['title']
-        playlist_description = playlist_detail.get('intro')
+        playlist_description = unescapeHTML(playlist_detail.get('intro'))
 
         episodes_detail = self._call_api('xqsingle', category, playlist_id)
 

From 88fb59d91bf5fd33dbe7863fbf1e4a9a641fa483 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Jan 2016 19:42:11 +0600
Subject: [PATCH 0272/1105] [bbccouk] Extend title extraction

---
 youtube_dl/extractor/bbc.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 5c621a32b..7b169881a 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -482,7 +482,8 @@ class BBCCoUkIE(InfoExtractor):
 
         if programme_id:
             formats, subtitles = self._download_media_selector(programme_id)
-            title = self._og_search_title(webpage)
+            title = self._og_search_title(webpage, default=None) or self._html_search_regex(
+                r'<h2[^>]+id="parent-title"[^>]*>(.+?)</h2>', webpage, 'title')
             description = self._search_regex(
                 r'<p class="[^"]*medium-description[^"]*">([^<]+)</p>',
                 webpage, 'description', default=None)

From 03116772583103ba97eaf8ce2cbabba9742e1929 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 Jan 2016 21:44:49 +0800
Subject: [PATCH 0273/1105] [baidu] Add notes for API calls

---
 youtube_dl/extractor/baidu.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py
index d60cb0fdd..d35116a7c 100644
--- a/youtube_dl/extractor/baidu.py
+++ b/youtube_dl/extractor/baidu.py
@@ -28,8 +28,9 @@ class BaiduVideoIE(InfoExtractor):
         'playlist_mincount': 12,
     }]
 
-    def _call_api(self, path, category, playlist_id):
-        return self._download_json('http://app.video.baidu.com/%s/?worktype=adnative%s&id=%s' % (path, category, playlist_id), playlist_id)
+    def _call_api(self, path, category, playlist_id, note):
+        return self._download_json('http://app.video.baidu.com/%s/?worktype=adnative%s&id=%s' % (
+            path, category, playlist_id), playlist_id, note)
 
     def _real_extract(self, url):
         category, playlist_id = re.match(self._VALID_URL, url).groups()
@@ -38,12 +39,14 @@ class BaiduVideoIE(InfoExtractor):
         if category == 'tv':
             category = 'tvplay'
 
-        playlist_detail = self._call_api('xqinfo', category, playlist_id)
+        playlist_detail = self._call_api(
+            'xqinfo', category, playlist_id, 'Download playlist JSON metadata')
 
         playlist_title = playlist_detail['title']
         playlist_description = unescapeHTML(playlist_detail.get('intro'))
 
-        episodes_detail = self._call_api('xqsingle', category, playlist_id)
+        episodes_detail = self._call_api(
+            'xqsingle', category, playlist_id, 'Download episodes JSON metadata')
 
         entries = []
         for episode in episodes_detail['videos']:

From b7546397f080e65fed42c69701f423e4f57f43f5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 Jan 2016 21:46:40 +0800
Subject: [PATCH 0274/1105] [baidu] Use list comprehension

---
 youtube_dl/extractor/baidu.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py
index d35116a7c..76b21e596 100644
--- a/youtube_dl/extractor/baidu.py
+++ b/youtube_dl/extractor/baidu.py
@@ -48,10 +48,9 @@ class BaiduVideoIE(InfoExtractor):
         episodes_detail = self._call_api(
             'xqsingle', category, playlist_id, 'Download episodes JSON metadata')
 
-        entries = []
-        for episode in episodes_detail['videos']:
-            entries.append(self.url_result(
-                episode['url'], video_title=episode['title']))
+        entries = [self.url_result(
+            episode['url'], video_title=episode['title']
+        ) for episode in episodes_detail['videos']]
 
         return self.playlist_result(
             entries, playlist_id, playlist_title, playlist_description)

From 2fffb1dcd0790da264f86f4a8d9c54a127269aca Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 Jan 2016 22:33:33 +0800
Subject: [PATCH 0275/1105] [qqmusic:playlist] Capture errors and update tests

---
 youtube_dl/extractor/qqmusic.py | 29 +++++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 1ba3bbddf..03f04b724 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -11,6 +11,7 @@ from ..utils import (
     strip_jsonp,
     unescapeHTML,
     clean_html,
+    ExtractorError,
 )
 
 
@@ -315,7 +316,7 @@ class QQMusicPlaylistIE(QQPlaylistBaseIE):
     IE_DESC = 'QQ音乐 - 歌单'
     _VALID_URL = r'http://y\.qq\.com/#type=taoge&id=(?P<id>[0-9]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://y.qq.com/#type=taoge&id=3462654915',
         'info_dict': {
             'id': '3462654915',
@@ -323,7 +324,16 @@ class QQMusicPlaylistIE(QQPlaylistBaseIE):
             'description': 'md5:d2c9d758a96b9888cf4fe82f603121d4',
         },
         'playlist_count': 40,
-    }
+        'skip': 'playlist gone',
+    }, {
+        'url': 'http://y.qq.com/#type=taoge&id=1374105607',
+        'info_dict': {
+            'id': '1374105607',
+            'title': '易入人心的华语民谣',
+            'description': '民谣的歌曲易于传唱、、歌词朗朗伤口、旋律简单温馨。属于那种才入耳孔。却上心头的感觉。没有太多的复杂情绪。简单而直接地表达乐者的情绪，就是这样的简单才易入人心。',
+        },
+        'playlist_count': 20,
+    }]
 
     def _real_extract(self, url):
         list_id = self._match_id(url)
@@ -331,14 +341,21 @@ class QQMusicPlaylistIE(QQPlaylistBaseIE):
         list_json = self._download_json(
             'http://i.y.qq.com/qzone-music/fcg-bin/fcg_ucc_getcdinfo_byids_cp.fcg?type=1&json=1&utf8=1&onlysong=0&disstid=%s'
             % list_id, list_id, 'Download list page',
-            transform_source=strip_jsonp)['cdlist'][0]
+            transform_source=strip_jsonp)
+        if not len(list_json.get('cdlist', [])):
+            if list_json.get('code'):
+                raise ExtractorError(
+                    'QQ Music said: error %d in fetching playlist info' % list_json['code'],
+                    expected=True)
+            raise ExtractorError('Unable to get playlist info')
 
+        cdlist = list_json['cdlist'][0]
         entries = [
             self.url_result(
                 'http://y.qq.com/#type=song&mid=' + song['songmid'], 'QQMusic', song['songmid']
-            ) for song in list_json['songlist']
+            ) for song in cdlist['songlist']
         ]
 
-        list_name = list_json.get('dissname')
-        list_description = clean_html(unescapeHTML(list_json.get('desc')))
+        list_name = cdlist.get('dissname')
+        list_description = clean_html(unescapeHTML(cdlist.get('desc')))
         return self.playlist_result(entries, list_id, list_name, list_description)

From 141a273a8b7a95fac5ccc0b5141c8d2eac48e3f9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 Jan 2016 22:39:09 +0800
Subject: [PATCH 0276/1105] [qqmusic] Update tests

---
 youtube_dl/extractor/qqmusic.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 03f04b724..45a3c41c5 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -178,7 +178,7 @@ class QQMusicSingerIE(QQPlaylistBaseIE):
         'info_dict': {
             'id': '001BLpXF2DyJe2',
             'title': '林俊杰',
-            'description': 'md5:2a222d89ba4455a3af19940c0481bb78',
+            'description': 'md5:870ec08f7d8547c29c93010899103751',
         },
         'playlist_count': 12,
     }
@@ -273,7 +273,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         'url': 'http://y.qq.com/#type=toplist&p=top_3',
         'info_dict': {
             'id': 'top_3',
-            'title': 'QQ音乐巅峰榜·欧美',
+            'title': '巅峰榜·欧美',
             'description': 'QQ音乐巅峰榜·欧美根据用户收听行为自动生成，集结当下最流行的欧美新歌！:更新时间：每周四22点|统'
                            '计周期：一周（上周四至本周三）|统计对象：三个月内发行的欧美歌曲|统计数量：100首|统计算法：根据'
                            '歌曲在一周内的有效播放次数，由高到低取前100名（同一歌手最多允许5首歌曲同时上榜）|有效播放次数：'

From 6b461026616b0e3ede9a302c74fc437541e19343 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 2 Jan 2016 21:24:57 +0100
Subject: [PATCH 0277/1105] [zdf] fix rtmpt format downloading handle errors

---
 youtube_dl/extractor/zdf.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index 2a1f2f6d1..c619a75e2 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -13,6 +13,7 @@ from ..utils import (
     determine_ext,
     qualities,
     float_or_none,
+    ExtractorError,
 )
 
 
@@ -59,7 +60,6 @@ class ZDFIE(InfoExtractor):
                     'ext': 'flv',
                     'format_id': '%s-%d' % (proto, bitrate),
                     'tbr': bitrate,
-                    'protocol': proto,
                 })
         self._sort_formats(formats)
         return formats
@@ -70,6 +70,15 @@ class ZDFIE(InfoExtractor):
             note='Downloading video info',
             errnote='Failed to download video info')
 
+        status_code = doc.find('./status/statuscode')
+        if status_code is not None and status_code.text != 'ok':
+            code = status_code.text
+            if code == 'notVisibleAnymore':
+                message = 'Video %s is not available' % video_id
+            else:
+                message = '%s returned error: %s' % (self.IE_NAME, code)
+            raise ExtractorError(message, expected=True)
+
         title = doc.find('.//information/title').text
         description = xpath_text(doc, './/information/detail', 'description')
         duration = int_or_none(xpath_text(doc, './/details/lengthSec', 'duration'))
@@ -129,10 +138,10 @@ class ZDFIE(InfoExtractor):
                     video_url, video_id, fatal=False))
             elif ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
-                    video_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
+                    video_url, video_id, 'mp4', m3u8_id=format_id, fatal=False))
             elif ext == 'f4m':
                 formats.extend(self._extract_f4m_formats(
-                    video_url, video_id, f4m_id='hds', fatal=False))
+                    video_url, video_id, f4m_id=format_id, fatal=False))
             else:
                 proto = format_m.group('proto').lower()
 

From 4059eabd58dd5e68aa1229c0ba0ffce0262ef7d8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 2 Jan 2016 21:29:10 +0100
Subject: [PATCH 0278/1105] [dreisat] use extract_from_xml_url from ZDFIE for
 info extraction(fixes #7680)(fixes #8104)(closes #8121)

---
 youtube_dl/extractor/dreisat.py | 59 ++-------------------------------
 1 file changed, 3 insertions(+), 56 deletions(-)

diff --git a/youtube_dl/extractor/dreisat.py b/youtube_dl/extractor/dreisat.py
index 8ac8587be..028144f20 100644
--- a/youtube_dl/extractor/dreisat.py
+++ b/youtube_dl/extractor/dreisat.py
@@ -2,14 +2,10 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    unified_strdate,
-)
+from .zdf import ZDFIE
 
 
-class DreiSatIE(InfoExtractor):
+class DreiSatIE(ZDFIE):
     IE_NAME = '3sat'
     _VALID_URL = r'(?:http://)?(?:www\.)?3sat\.de/mediathek/(?:index\.php|mediathek\.php)?\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)$'
     _TESTS = [
@@ -35,53 +31,4 @@ class DreiSatIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         details_url = 'http://www.3sat.de/mediathek/xmlservice/web/beitragsDetails?ak=web&id=%s' % video_id
-        details_doc = self._download_xml(details_url, video_id, 'Downloading video details')
-
-        status_code = details_doc.find('./status/statuscode')
-        if status_code is not None and status_code.text != 'ok':
-            code = status_code.text
-            if code == 'notVisibleAnymore':
-                message = 'Video %s is not available' % video_id
-            else:
-                message = '%s returned error: %s' % (self.IE_NAME, code)
-            raise ExtractorError(message, expected=True)
-
-        thumbnail_els = details_doc.findall('.//teaserimage')
-        thumbnails = [{
-            'width': int(te.attrib['key'].partition('x')[0]),
-            'height': int(te.attrib['key'].partition('x')[2]),
-            'url': te.text,
-        } for te in thumbnail_els]
-
-        information_el = details_doc.find('.//information')
-        video_title = information_el.find('./title').text
-        video_description = information_el.find('./detail').text
-
-        details_el = details_doc.find('.//details')
-        video_uploader = details_el.find('./channel').text
-        upload_date = unified_strdate(details_el.find('./airtime').text)
-
-        format_els = details_doc.findall('.//formitaet')
-        formats = [{
-            'format_id': fe.attrib['basetype'],
-            'width': int(fe.find('./width').text),
-            'height': int(fe.find('./height').text),
-            'url': fe.find('./url').text,
-            'filesize': int(fe.find('./filesize').text),
-            'video_bitrate': int(fe.find('./videoBitrate').text),
-        } for fe in format_els
-            if not fe.find('./url').text.startswith('http://www.metafilegenerator.de/')]
-
-        self._sort_formats(formats)
-
-        return {
-            '_type': 'video',
-            'id': video_id,
-            'title': video_title,
-            'formats': formats,
-            'description': video_description,
-            'thumbnails': thumbnails,
-            'thumbnail': thumbnails[-1]['url'],
-            'uploader': video_uploader,
-            'upload_date': upload_date,
-        }
+        return self.extract_from_xml_url(video_id, details_url)

From e1a0bfdffe25dda494a9da8b02fba0c9ad39f4fe Mon Sep 17 00:00:00 2001
From: dyn888 <dyn.8.8.8+github@gmail.com>
Date: Sun, 3 Jan 2016 04:11:19 +0100
Subject: [PATCH 0279/1105] [youtube] added vcodec/acodec/abr for multiple
 itags

Should make downloading with filters more precise and easier, ie. bestvideo[vcodec=h264]. By default a lot of codecs are specified as avc1.xxxxxx and unique for each format, which makes them unusable for bestvideo selection.
---
 youtube_dl/extractor/youtube.py | 120 ++++++++++++++++----------------
 1 file changed, 60 insertions(+), 60 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4aac2cc03..64386f34a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -277,55 +277,55 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                      $"""
     _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
     _formats = {
-        '5': {'ext': 'flv', 'width': 400, 'height': 240},
-        '6': {'ext': 'flv', 'width': 450, 'height': 270},
-        '13': {'ext': '3gp'},
-        '17': {'ext': '3gp', 'width': 176, 'height': 144},
-        '18': {'ext': 'mp4', 'width': 640, 'height': 360},
-        '22': {'ext': 'mp4', 'width': 1280, 'height': 720},
-        '34': {'ext': 'flv', 'width': 640, 'height': 360},
-        '35': {'ext': 'flv', 'width': 854, 'height': 480},
-        '36': {'ext': '3gp', 'width': 320, 'height': 240},
-        '37': {'ext': 'mp4', 'width': 1920, 'height': 1080},
-        '38': {'ext': 'mp4', 'width': 4096, 'height': 3072},
-        '43': {'ext': 'webm', 'width': 640, 'height': 360},
-        '44': {'ext': 'webm', 'width': 854, 'height': 480},
-        '45': {'ext': 'webm', 'width': 1280, 'height': 720},
-        '46': {'ext': 'webm', 'width': 1920, 'height': 1080},
-        '59': {'ext': 'mp4', 'width': 854, 'height': 480},
-        '78': {'ext': 'mp4', 'width': 854, 'height': 480},
+        '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
+        '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
+        '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
+        '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
+        '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
+        '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
+        '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
+        '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
+        '36': {'ext': '3gp', 'width': 320, 'height': 240, 'acodec': 'aac', 'abr': 32, 'vcodec': 'mp4v'},
+        '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
+        '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
+        '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
+        '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
+        '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
+        '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
+        '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
+        '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 
 
-        # 3d videos
-        '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'preference': -20},
-        '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'preference': -20},
-        '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'preference': -20},
-        '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'preference': -20},
-        '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'preference': -20},
-        '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'preference': -20},
-        '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'preference': -20},
+        # 3D videos
+        '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
+        '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
+        '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
+        '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
+        '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
+        '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
+        '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 
         # Apple HTTP Live Streaming
-        '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'preference': -10},
-        '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'preference': -10},
-        '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'preference': -10},
-        '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'preference': -10},
-        '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'preference': -10},
-        '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'preference': -10},
-        '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'preference': -10},
+        '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
+        '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
+        '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
+        '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
+        '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
+        '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
+        '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 
         # DASH mp4 video
-        '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
-        '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
-        '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
-        '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'h264'},
+        '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
+        '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
+        '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
+        '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
+        '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
+        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
+        '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
+        '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
+        '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
+        '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
+        '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
 
         # Dash mp4 audio
         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 48, 'preference': -50, 'container': 'm4a_dash'},
@@ -339,26 +339,26 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'container': 'webm', 'vcodec': 'vp9'},
-        '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
+        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp9', 'preference': -40},
+        '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
+        '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
+        '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
+        '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
+        '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
+        '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
+        '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
+        '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
-        '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
-        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
-        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
-        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'vp9'},
-        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
+        '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
+        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
+        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
+        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
+        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
+        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
 
         # Dash webm audio
-        '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},
-        '172': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50},
+        '171': {'ext': 'webm', 'acodec': 'vorbis', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},
+        '172': {'ext': 'webm', 'acodec': 'vorbis', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50},
 
         # Dash webm audio with opus inside
         '249': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50, 'preference': -50},

From ea6abd740f8dd1c559e186bc1cbcfa73854a90d1 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 3 Jan 2016 10:12:13 +0500
Subject: [PATCH 0280/1105] [nowtv] Mark broken

---
 youtube_dl/extractor/nowtv.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index fd107aca2..916a102bf 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -71,6 +71,7 @@ class NowTVBaseIE(InfoExtractor):
 
 
 class NowTVIE(NowTVBaseIE):
+    _WORKING = False
     _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<show_id>[^/]+)/(?:(?:list/[^/]+|jahr/\d{4}/\d{1,2})/)?(?P<id>[^/]+)/(?:player|preview)'
 
     _TESTS = [{

From a7aaa39863a6b1ab89a56a55521fd4779c6ac900 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Jan 2016 01:08:34 +0600
Subject: [PATCH 0281/1105] [utils] Extract known extensions for reuse

---
 youtube_dl/utils.py | 31 +++++++++++++++++--------------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 0ed6c45c8..da4ec7f20 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -70,6 +70,21 @@ ENGLISH_MONTH_NAMES = [
     'January', 'February', 'March', 'April', 'May', 'June',
     'July', 'August', 'September', 'October', 'November', 'December']
 
+KNOWN_EXTENSIONS = (
+    'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
+    'flv', 'f4v', 'f4a', 'f4b',
+    'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
+    'mkv', 'mka', 'mk3d',
+    'avi', 'divx',
+    'mov',
+    'asf', 'wmv', 'wma',
+    '3gp', '3g2',
+    'mp3',
+    'flac',
+    'ape',
+    'wav',
+    'f4f', 'f4m', 'm3u8', 'smil')
+
 
 def preferredencoding():
     """Get preferred encoding.
@@ -942,20 +957,8 @@ def determine_ext(url, default_ext='unknown_video'):
     guess = url.partition('?')[0].rpartition('.')[2]
     if re.match(r'^[A-Za-z0-9]+$', guess):
         return guess
-    elif guess.rstrip('/') in (
-            'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
-            'flv', 'f4v', 'f4a', 'f4b',
-            'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
-            'mkv', 'mka', 'mk3d',
-            'avi', 'divx',
-            'mov',
-            'asf', 'wmv', 'wma',
-            '3gp', '3g2',
-            'mp3',
-            'flac',
-            'ape',
-            'wav',
-            'f4f', 'f4m', 'm3u8', 'smil'):
+    # Try extract ext from URLs like http://example.com/foo/bar.mp4/?download
+    elif guess.rstrip('/') in KNOWN_EXTENSIONS:
         return guess.rstrip('/')
     else:
         return default_ext

From e54c44eeab9e088c344ec379cdc4b1fbc63ff324 Mon Sep 17 00:00:00 2001
From: pingtux <pingtux@users.noreply.github.com>
Date: Sat, 2 Jan 2016 01:13:49 +0100
Subject: [PATCH 0282/1105] [20min.ch] Add new extractor (closes #5977)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/min20.py    | 40 ++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)
 create mode 100644 youtube_dl/extractor/min20.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 4c7e5223d..64e1fd334 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -364,6 +364,7 @@ from .mdr import MDRIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
+from .min20 import Min20IE
 from .minhateca import MinhatecaIE
 from .ministrygrid import MinistryGridIE
 from .miomio import MioMioIE
diff --git a/youtube_dl/extractor/min20.py b/youtube_dl/extractor/min20.py
new file mode 100644
index 000000000..a3f91671e
--- /dev/null
+++ b/youtube_dl/extractor/min20.py
@@ -0,0 +1,40 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class Min20IE(InfoExtractor):
+    _VALID_URL = r'http://www\.20min\.ch/.+?-(?P<id>[0-9]+)$'
+    _TEST = {
+        'url': 'http://www.20min.ch/schweiz/news/story/-Wir-muessen-mutig-nach-vorne-schauen--22050469',
+        'md5': 'cd4cbb99b94130cff423e967cd275e5e',
+        'info_dict': {
+            'id': '22050469',
+            'ext': 'flv',
+            'title': '«Wir müssen mutig nach vorne schauen»',
+            'description': 'Kein Land sei innovativer als die Schweiz, sagte Johann Schneider-Ammann in seiner Neujahrsansprache. Das Land müsse aber seine Hausaufgaben machen.',
+            'thumbnail': 'http://www.20min.ch/images/content/2/2/0/22050469/10/teaserbreit.jpg'
+        }
+    }
+
+    # location of the flv videos, can't be extracted from the web page
+    _BASE_URL = "http://flv-rr.20min-tv.ch/videos/"
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        title = self._html_search_regex(r'<h1><span>(.+?)</span></h1>', webpage, 'title')
+        flash_id = self._search_regex(r"so\.addVariable\(\"file1\",\"([0-9]+)\"\)", webpage, 'flash_id')
+
+        description = self._html_search_regex(r'<meta name="description" content="(.+?)" />', webpage, 'description')
+        thumbnail = self._html_search_regex(r'<meta property="og:image" content="(.+?)" />', webpage, 'thumbnail')
+        url = self._BASE_URL + flash_id + "m.flv"
+
+        return {
+            'id': video_id,
+            'url': url,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail
+        }

From 66295fa4a6a712d4d41ff54ee9c361c7c83b44bc Mon Sep 17 00:00:00 2001
From: pingtux <pingtux@users.noreply.github.com>
Date: Sat, 2 Jan 2016 16:19:05 +0100
Subject: [PATCH 0283/1105] [20min.ch] Added support for videoportal

---
 youtube_dl/extractor/min20.py | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/min20.py b/youtube_dl/extractor/min20.py
index a3f91671e..23aead19d 100644
--- a/youtube_dl/extractor/min20.py
+++ b/youtube_dl/extractor/min20.py
@@ -1,12 +1,14 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 
 
 class Min20IE(InfoExtractor):
-    _VALID_URL = r'http://www\.20min\.ch/.+?-(?P<id>[0-9]+)$'
-    _TEST = {
+    _VALID_URL = r'http://www\.20min\.ch/(videotv/\?vid=(?P<video_id>[0-9]+)|.+?-(?P<page_id>[0-9]+)$)'
+    _TESTS = [{
         'url': 'http://www.20min.ch/schweiz/news/story/-Wir-muessen-mutig-nach-vorne-schauen--22050469',
         'md5': 'cd4cbb99b94130cff423e967cd275e5e',
         'info_dict': {
@@ -16,15 +18,29 @@ class Min20IE(InfoExtractor):
             'description': 'Kein Land sei innovativer als die Schweiz, sagte Johann Schneider-Ammann in seiner Neujahrsansprache. Das Land müsse aber seine Hausaufgaben machen.',
             'thumbnail': 'http://www.20min.ch/images/content/2/2/0/22050469/10/teaserbreit.jpg'
         }
-    }
+    }, {
+        'url': 'http://www.20min.ch/videotv/?vid=469148&cid=2',
+        'md5': 'b52d6bc6ea6398e6a38f12cfd418149c',
+        'info_dict': {
+            'id': '469148',
+            'ext': 'flv',
+            'title': '85 000 Franken für 15 perfekte Minuten',
+            'description': 'Was die Besucher vom Silvesterzauber erwarten können. (Video: Alice Grosjean/Murat Temel)',
+            'thumbnail': 'http://thumbnails.20min-tv.ch/server063/469148/frame-72-469148.jpg'
+        }
+    }]
 
     # location of the flv videos, can't be extracted from the web page
     _BASE_URL = "http://flv-rr.20min-tv.ch/videos/"
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('page_id')
+        if video_id is None:
+            # URL from the videoportal
+            video_id = mobj.group('video_id')
         webpage = self._download_webpage(url, video_id)
-        title = self._html_search_regex(r'<h1><span>(.+?)</span></h1>', webpage, 'title')
+        title = self._html_search_regex(r'<h1>.*<span>(.+?)</span></h1>', webpage, 'title')
         flash_id = self._search_regex(r"so\.addVariable\(\"file1\",\"([0-9]+)\"\)", webpage, 'flash_id')
 
         description = self._html_search_regex(r'<meta name="description" content="(.+?)" />', webpage, 'description')

From 133b1886fc6721090ea5d3be3d382626e2602b48 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Jan 2016 02:33:08 +0600
Subject: [PATCH 0284/1105] [20min] Improve (Closes #8110)

---
 youtube_dl/extractor/__init__.py  |  2 +-
 youtube_dl/extractor/min20.py     | 56 ------------------------
 youtube_dl/extractor/twentymin.py | 73 +++++++++++++++++++++++++++++++
 3 files changed, 74 insertions(+), 57 deletions(-)
 delete mode 100644 youtube_dl/extractor/min20.py
 create mode 100644 youtube_dl/extractor/twentymin.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 64e1fd334..625b0bf16 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -364,7 +364,6 @@ from .mdr import MDRIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
-from .min20 import Min20IE
 from .minhateca import MinhatecaIE
 from .ministrygrid import MinistryGridIE
 from .miomio import MioMioIE
@@ -747,6 +746,7 @@ from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
 from .tweakers import TweakersIE
 from .twentyfourvideo import TwentyFourVideoIE
+from .twentymin import TwentyMinutenIE
 from .twentytwotracks import (
     TwentyTwoTracksIE,
     TwentyTwoTracksGenreIE
diff --git a/youtube_dl/extractor/min20.py b/youtube_dl/extractor/min20.py
deleted file mode 100644
index 23aead19d..000000000
--- a/youtube_dl/extractor/min20.py
+++ /dev/null
@@ -1,56 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-
-
-class Min20IE(InfoExtractor):
-    _VALID_URL = r'http://www\.20min\.ch/(videotv/\?vid=(?P<video_id>[0-9]+)|.+?-(?P<page_id>[0-9]+)$)'
-    _TESTS = [{
-        'url': 'http://www.20min.ch/schweiz/news/story/-Wir-muessen-mutig-nach-vorne-schauen--22050469',
-        'md5': 'cd4cbb99b94130cff423e967cd275e5e',
-        'info_dict': {
-            'id': '22050469',
-            'ext': 'flv',
-            'title': '«Wir müssen mutig nach vorne schauen»',
-            'description': 'Kein Land sei innovativer als die Schweiz, sagte Johann Schneider-Ammann in seiner Neujahrsansprache. Das Land müsse aber seine Hausaufgaben machen.',
-            'thumbnail': 'http://www.20min.ch/images/content/2/2/0/22050469/10/teaserbreit.jpg'
-        }
-    }, {
-        'url': 'http://www.20min.ch/videotv/?vid=469148&cid=2',
-        'md5': 'b52d6bc6ea6398e6a38f12cfd418149c',
-        'info_dict': {
-            'id': '469148',
-            'ext': 'flv',
-            'title': '85 000 Franken für 15 perfekte Minuten',
-            'description': 'Was die Besucher vom Silvesterzauber erwarten können. (Video: Alice Grosjean/Murat Temel)',
-            'thumbnail': 'http://thumbnails.20min-tv.ch/server063/469148/frame-72-469148.jpg'
-        }
-    }]
-
-    # location of the flv videos, can't be extracted from the web page
-    _BASE_URL = "http://flv-rr.20min-tv.ch/videos/"
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('page_id')
-        if video_id is None:
-            # URL from the videoportal
-            video_id = mobj.group('video_id')
-        webpage = self._download_webpage(url, video_id)
-        title = self._html_search_regex(r'<h1>.*<span>(.+?)</span></h1>', webpage, 'title')
-        flash_id = self._search_regex(r"so\.addVariable\(\"file1\",\"([0-9]+)\"\)", webpage, 'flash_id')
-
-        description = self._html_search_regex(r'<meta name="description" content="(.+?)" />', webpage, 'description')
-        thumbnail = self._html_search_regex(r'<meta property="og:image" content="(.+?)" />', webpage, 'thumbnail')
-        url = self._BASE_URL + flash_id + "m.flv"
-
-        return {
-            'id': video_id,
-            'url': url,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail
-        }
diff --git a/youtube_dl/extractor/twentymin.py b/youtube_dl/extractor/twentymin.py
new file mode 100644
index 000000000..ca7d953b8
--- /dev/null
+++ b/youtube_dl/extractor/twentymin.py
@@ -0,0 +1,73 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import remove_end
+
+
+class TwentyMinutenIE(InfoExtractor):
+    IE_NAME = '20min'
+    _VALID_URL = r'https?://(?:www\.)?20min\.ch/(?:videotv/*\?.*\bvid=(?P<id>\d+)|(?:[^/]+/)*(?P<display_id>[^/#?]+))'
+    _TESTS = [{
+        # regular video
+        'url': 'http://www.20min.ch/videotv/?vid=469148&cid=2',
+        'md5': 'b52d6bc6ea6398e6a38f12cfd418149c',
+        'info_dict': {
+            'id': '469148',
+            'ext': 'flv',
+            'title': '85 000 Franken für 15 perfekte Minuten',
+            'description': 'Was die Besucher vom Silvesterzauber erwarten können. (Video: Alice Grosjean/Murat Temel)',
+            'thumbnail': 'http://thumbnails.20min-tv.ch/server063/469148/frame-72-469148.jpg'
+        }
+    }, {
+        # news article with video
+        'url': 'http://www.20min.ch/schweiz/news/story/-Wir-muessen-mutig-nach-vorne-schauen--22050469',
+        'md5': 'cd4cbb99b94130cff423e967cd275e5e',
+        'info_dict': {
+            'id': '469408',
+            'display_id': '-Wir-muessen-mutig-nach-vorne-schauen--22050469',
+            'ext': 'flv',
+            'title': '«Wir müssen mutig nach vorne schauen»',
+            'description': 'Kein Land sei innovativer als die Schweiz, sagte Johann Schneider-Ammann in seiner Neujahrsansprache. Das Land müsse aber seine Hausaufgaben machen.',
+            'thumbnail': 'http://www.20min.ch/images/content/2/2/0/22050469/10/teaserbreit.jpg'
+        }
+    }, {
+        'url': 'http://www.20min.ch/videotv/?cid=44&vid=468738',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.20min.ch/ro/sortir/cinema/story/Grandir-au-bahut--c-est-dur-18927411',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id') or video_id
+
+        webpage = self._download_webpage(url, display_id)
+
+        title = self._html_search_regex(
+            r'<h1>.*?<span>(.+?)</span></h1>',
+            webpage, 'title', default=None)
+        if not title:
+            title = remove_end(re.sub(
+                r'^20 [Mm]inuten.*? -', '', self._og_search_title(webpage)), ' - News')
+
+        if not video_id:
+            video_id = self._search_regex(
+                r'"file\d?"\s*,\s*\"(\d+)', webpage, 'video id')
+
+        description = self._html_search_meta(
+            'description', webpage, 'description')
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'url': 'http://speed.20min-tv.ch/%sm.flv' % video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+        }

From 5aa535c32982f1f9b2f689097a64be78d25e7d90 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Jan 2016 02:55:25 +0600
Subject: [PATCH 0285/1105] [bbccouk] Update tests (Closes #8090)

---
 youtube_dl/extractor/bbc.py | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 7b169881a..ce99a34ab 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -124,14 +124,14 @@ class BBCCoUkIE(InfoExtractor):
             },
             'skip': 'Episode is no longer available on BBC iPlayer Radio',
         }, {
-            'url': 'http://www.bbc.co.uk/music/clips/p02frcc3',
+            'url': 'http://www.bbc.co.uk/music/clips/p022h44b',
             'note': 'Audio',
             'info_dict': {
-                'id': 'p02frcch',
+                'id': 'p022h44j',
                 'ext': 'flv',
-                'title': 'Pete Tong, Past, Present and Future Special, Madeon - After Hours mix',
-                'description': 'French house superstar Madeon takes us out of the club and onto the after party.',
-                'duration': 3507,
+                'title': 'BBC Proms Music Guides, Rachmaninov: Symphonic Dances',
+                'description': "In this Proms Music Guide, Andrew McGregor looks at Rachmaninov's Symphonic Dances.",
+                'duration': 227,
             },
             'params': {
                 # rtmp download
@@ -182,13 +182,12 @@ class BBCCoUkIE(InfoExtractor):
         }, {
             # iptv-all mediaset fails with geolocation however there is no geo restriction
             # for this programme at all
-            'url': 'http://www.bbc.co.uk/programmes/b06bp7lf',
+            'url': 'http://www.bbc.co.uk/programmes/b06rkn85',
             'info_dict': {
-                'id': 'b06bp7kf',
+                'id': 'b06rkms3',
                 'ext': 'flv',
-                'title': "Annie Mac's Friday Night, B.Traits sits in for Annie",
-                'description': 'B.Traits sits in for Annie Mac with a Mini-Mix from Disclosure.',
-                'duration': 10800,
+                'title': "Best of the Mini-Mixes 2015: Part 3, Annie Mac's Friday Night - BBC Radio 1",
+                'description': "Annie has part three in the Best of the Mini-Mixes 2015, plus the year's Most Played!",
             },
             'params': {
                 # rtmp download

From ab3176af3472917537635bd96b860a974094b0af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Jan 2016 03:34:15 +0600
Subject: [PATCH 0286/1105] [ivi] Fix extraction and modernize

---
 youtube_dl/extractor/ivi.py | 69 ++++++++++++++++++-------------------
 1 file changed, 33 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py
index 029878d24..216c534b5 100644
--- a/youtube_dl/extractor/ivi.py
+++ b/youtube_dl/extractor/ivi.py
@@ -7,6 +7,7 @@ import json
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    int_or_none,
     sanitized_Request,
 )
 
@@ -27,7 +28,7 @@ class IviIE(InfoExtractor):
                 'title': 'Иван Васильевич меняет профессию',
                 'description': 'md5:b924063ea1677c8fe343d8a72ac2195f',
                 'duration': 5498,
-                'thumbnail': 'http://thumbs.ivi.ru/f20.vcp.digitalaccess.ru/contents/d/1/c3c885163a082c29bceeb7b5a267a6.jpg',
+                'thumbnail': 're:^https?://.*\.jpg$',
             },
             'skip': 'Only works from Russia',
         },
@@ -38,33 +39,23 @@ class IviIE(InfoExtractor):
             'info_dict': {
                 'id': '9549',
                 'ext': 'mp4',
-                'title': 'Двое из ларца - Серия 1',
+                'title': 'Двое из ларца - Дело Гольдберга (1 часть)',
+                'series': 'Двое из ларца',
+                'episode': 'Дело Гольдберга (1 часть)',
+                'episode_number': 1,
                 'duration': 2655,
-                'thumbnail': 'http://thumbs.ivi.ru/f15.vcp.digitalaccess.ru/contents/8/4/0068dc0677041f3336b7c2baad8fc0.jpg',
+                'thumbnail': 're:^https?://.*\.jpg$',
             },
             'skip': 'Only works from Russia',
         }
     ]
 
     # Sorted by quality
-    _known_formats = ['MP4-low-mobile', 'MP4-mobile', 'FLV-lo', 'MP4-lo', 'FLV-hi', 'MP4-hi', 'MP4-SHQ']
-
-    # Sorted by size
-    _known_thumbnails = ['Thumb-120x90', 'Thumb-160', 'Thumb-640x480']
-
-    def _extract_description(self, html):
-        m = re.search(r'<meta name="description" content="(?P<description>[^"]+)"/>', html)
-        return m.group('description') if m is not None else None
-
-    def _extract_comment_count(self, html):
-        m = re.search('(?s)<a href="#" id="view-comments" class="action-button dim gradient">\s*Комментарии:\s*(?P<commentcount>\d+)\s*</a>', html)
-        return int(m.group('commentcount')) if m is not None else 0
+    _KNOWN_FORMATS = ['MP4-low-mobile', 'MP4-mobile', 'FLV-lo', 'MP4-lo', 'FLV-hi', 'MP4-hi', 'MP4-SHQ']
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        api_url = 'http://api.digitalaccess.ru/api/json/'
-
         data = {
             'method': 'da.content.get',
             'params': [
@@ -76,11 +67,10 @@ class IviIE(InfoExtractor):
             ]
         }
 
-        request = sanitized_Request(api_url, json.dumps(data))
-
-        video_json_page = self._download_webpage(
+        request = sanitized_Request(
+            'http://api.digitalaccess.ru/api/json/', json.dumps(data))
+        video_json = self._download_json(
             request, video_id, 'Downloading video JSON')
-        video_json = json.loads(video_json_page)
 
         if 'error' in video_json:
             error = video_json['error']
@@ -95,35 +85,42 @@ class IviIE(InfoExtractor):
         formats = [{
             'url': x['url'],
             'format_id': x['content_format'],
-            'preference': self._known_formats.index(x['content_format']),
-        } for x in result['files'] if x['content_format'] in self._known_formats]
+            'preference': self._KNOWN_FORMATS.index(x['content_format']),
+        } for x in result['files'] if x['content_format'] in self._KNOWN_FORMATS]
 
         self._sort_formats(formats)
 
-        if not formats:
-            raise ExtractorError('No media links available for %s' % video_id)
-
-        duration = result['duration']
-        compilation = result['compilation']
         title = result['title']
 
+        duration = int_or_none(result.get('duration'))
+        compilation = result.get('compilation')
+        episode = title if compilation else None
+
         title = '%s - %s' % (compilation, title) if compilation is not None else title
 
-        previews = result['preview']
-        previews.sort(key=lambda fmt: self._known_thumbnails.index(fmt['content_format']))
-        thumbnail = previews[-1]['url'] if len(previews) > 0 else None
+        thumbnails = [{
+            'url': preview['url'],
+            'id': preview.get('content_format'),
+        } for preview in result.get('preview', []) if preview.get('url')]
 
-        video_page = self._download_webpage(url, video_id, 'Downloading video page')
-        description = self._extract_description(video_page)
-        comment_count = self._extract_comment_count(video_page)
+        webpage = self._download_webpage(url, video_id)
+
+        episode_number = int_or_none(self._search_regex(
+            r'<meta[^>]+itemprop="episode"[^>]*>\s*<meta[^>]+itemprop="episodeNumber"[^>]+content="(\d+)',
+            webpage, 'episode number', default=None))
+
+        description = self._og_search_description(webpage, default=None) or self._html_search_meta(
+            'description', webpage, 'description', default=None)
 
         return {
             'id': video_id,
             'title': title,
-            'thumbnail': thumbnail,
+            'series': compilation,
+            'episode': episode,
+            'episode_number': episode_number,
+            'thumbnails': thumbnails,
             'description': description,
             'duration': duration,
-            'comment_count': comment_count,
             'formats': formats,
         }
 

From c6270b2ed5be9e83bcb8114511641f9c5ad9008d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Jan 2016 03:49:18 +0600
Subject: [PATCH 0287/1105] [ivi:compilation] Fix extraction

---
 youtube_dl/extractor/ivi.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py
index 216c534b5..9ccfee173 100644
--- a/youtube_dl/extractor/ivi.py
+++ b/youtube_dl/extractor/ivi.py
@@ -146,8 +146,11 @@ class IviCompilationIE(InfoExtractor):
     }]
 
     def _extract_entries(self, html, compilation_id):
-        return [self.url_result('http://www.ivi.ru/watch/%s/%s' % (compilation_id, serie), 'Ivi')
-                for serie in re.findall(r'<strong><a href="/watch/%s/(\d+)">(?:[^<]+)</a></strong>' % compilation_id, html)]
+        return [
+            self.url_result(
+                'http://www.ivi.ru/watch/%s/%s' % (compilation_id, serie), IviIE.ie_key())
+            for serie in re.findall(
+                r'<a href="/watch/%s/(\d+)"[^>]+data-id="\1"' % compilation_id, html)]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -155,7 +158,8 @@ class IviCompilationIE(InfoExtractor):
         season_id = mobj.group('seasonid')
 
         if season_id is not None:  # Season link
-            season_page = self._download_webpage(url, compilation_id, 'Downloading season %s web page' % season_id)
+            season_page = self._download_webpage(
+                url, compilation_id, 'Downloading season %s web page' % season_id)
             playlist_id = '%s/season%s' % (compilation_id, season_id)
             playlist_title = self._html_search_meta('title', season_page, 'title')
             entries = self._extract_entries(season_page, compilation_id)
@@ -163,8 +167,9 @@ class IviCompilationIE(InfoExtractor):
             compilation_page = self._download_webpage(url, compilation_id, 'Downloading compilation web page')
             playlist_id = compilation_id
             playlist_title = self._html_search_meta('title', compilation_page, 'title')
-            seasons = re.findall(r'<a href="/watch/%s/season(\d+)">[^<]+</a>' % compilation_id, compilation_page)
-            if len(seasons) == 0:  # No seasons in this compilation
+            seasons = re.findall(
+                r'<a href="/watch/%s/season(\d+)' % compilation_id, compilation_page)
+            if not seasons:  # No seasons in this compilation
                 entries = self._extract_entries(compilation_page, compilation_id)
             else:
                 entries = []

From 1463c5b9ac16bd50d5e30d2145ec584d1e6e74ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Jan 2016 03:54:52 +0600
Subject: [PATCH 0288/1105] [ivi] Extract season info

---
 youtube_dl/extractor/ivi.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py
index 9ccfee173..d0f00cdea 100644
--- a/youtube_dl/extractor/ivi.py
+++ b/youtube_dl/extractor/ivi.py
@@ -41,6 +41,8 @@ class IviIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Двое из ларца - Дело Гольдберга (1 часть)',
                 'series': 'Двое из ларца',
+                'season': 'Сезон 1',
+                'season_number': 1,
                 'episode': 'Дело Гольдберга (1 часть)',
                 'episode_number': 1,
                 'duration': 2655,
@@ -105,6 +107,13 @@ class IviIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
+        season = self._search_regex(
+            r'<li[^>]+class="season active"[^>]*><a[^>]+>([^<]+)',
+            webpage, 'season', default=None)
+        season_number = int_or_none(self._search_regex(
+            r'<li[^>]+class="season active"[^>]*><a[^>]+data-season(?:-index)?="(\d+)"',
+            webpage, 'season number', default=None))
+
         episode_number = int_or_none(self._search_regex(
             r'<meta[^>]+itemprop="episode"[^>]*>\s*<meta[^>]+itemprop="episodeNumber"[^>]+content="(\d+)',
             webpage, 'episode number', default=None))
@@ -116,6 +125,8 @@ class IviIE(InfoExtractor):
             'id': video_id,
             'title': title,
             'series': compilation,
+            'season': season,
+            'season_number': season_number,
             'episode': episode,
             'episode_number': episode_number,
             'thumbnails': thumbnails,

From 896c7a23cd9fbfc589f26e9a7bbada38a325e2ad Mon Sep 17 00:00:00 2001
From: bpfoley <bpfoley>
Date: Wed, 30 Dec 2015 21:50:26 +0000
Subject: [PATCH 0289/1105] [extractor/rte.py] Add support for RTE radio player

While here, stop RteIE changing filename extensions to .mp4. The files
saved are .flv containers with h264 video.
---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/rte.py      | 77 ++++++++++++++++++++++++++++----
 2 files changed, 70 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fb7151443..fe3c32874 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -556,7 +556,7 @@ from .ro220 import Ro220IE
 from .rottentomatoes import RottenTomatoesIE
 from .roxwel import RoxwelIE
 from .rtbf import RTBFIE
-from .rte import RteIE
+from .rte import RteIE, RteRadioIE
 from .rtlnl import RtlNlIE
 from .rtl2 import RTL2IE
 from .rtp import RTPIE
diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index d9cfbf180..25abcee92 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -5,16 +5,19 @@ from .common import InfoExtractor
 
 from ..utils import (
     float_or_none,
+    unescapeHTML,
 )
 
 
 class RteIE(InfoExtractor):
+    IE_NAME = 'rte'
+    IE_DESC = 'Raidió Teilifís Éireann TV'
     _VALID_URL = r'https?://(?:www\.)?rte\.ie/player/[^/]{2,3}/show/[^/]+/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.rte.ie/player/ie/show/iwitness-862/10478715/',
         'info_dict': {
             'id': '10478715',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Watch iWitness  online',
             'thumbnail': 're:^https?://.*\.jpg$',
             'description': 'iWitness : The spirit of Ireland, one voice and one minute at a time.',
@@ -44,13 +47,6 @@ class RteIE(InfoExtractor):
         # f4m_url = server + relative_url
         f4m_url = json_string['shows'][0]['media:group'][0]['rte:server'] + json_string['shows'][0]['media:group'][0]['url']
         f4m_formats = self._extract_f4m_formats(f4m_url, video_id)
-        f4m_formats = [{
-            'format_id': f['format_id'],
-            'url': f['url'],
-            'ext': 'mp4',
-            'width': f['width'],
-            'height': f['height'],
-        } for f in f4m_formats]
 
         return {
             'id': video_id,
@@ -60,3 +56,68 @@ class RteIE(InfoExtractor):
             'thumbnail': thumbnail,
             'duration': duration,
         }
+
+
+
+class RteRadioIE(InfoExtractor):
+    IE_NAME = 'rte:radio'
+    IE_DESC = 'Raidió Teilifís Éireann radio'
+    # Radioplayer URLs have the specifier #!rii=<channel_id>:<id>:<playable_item_id>:<date>:
+    # where the IDs are int/empty, the date is DD-MM-YYYY, and the specifier may be truncated.
+    # An <id> uniquely defines an individual recording, and is the only part we require.
+    _VALID_URL = r'https?://(?:www\.)?rte\.ie/radio/utils/radioplayer/rteradioweb\.html#!rii=(?:[0-9]*)(?:%3A|:)(?P<id>[0-9]+)'
+
+    _TEST = {
+        'url': 'http://www.rte.ie/radio/utils/radioplayer/rteradioweb.html#!rii=16:10507902:2414:27-12-2015:',
+        'info_dict': {
+            'id': '10507902',
+            'ext': 'flv',
+            'title': 'Gloria',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'Tim Thurston guides you through a millennium of sacred music featuring Gregorian chant, pure solo voices and choral masterpieces, framed around the glorious music of J.S. Bach.',
+            'duration': 7230.0,
+        },
+        'params': {
+            'skip_download': 'f4m fails with --test atm'
+        }
+    }
+
+    def _real_extract(self, url):
+        item_id = self._match_id(url)
+        feeds_url = 'http://www.rte.ie/rteavgen/getplaylist/?type=web&format=json&id=' + item_id
+        json_string = self._download_json(feeds_url, item_id)
+
+        # NB the string values in the JSON are stored using XML escaping(!)
+        show = json_string['shows'][0]
+        title = unescapeHTML(show['title'])
+        description = unescapeHTML(show.get('description'))
+        thumbnail = show.get('thumbnail')
+        duration = float_or_none(show.get('duration'), 1000)
+
+        mg = show['media:group'][0]
+
+        formats = []
+
+        if mg.get('url') and not mg['url'].startswith('rtmpe:'):
+            formats.append({'url': mg.get('url')})
+
+        if mg.get('hls_server') and mg.get('hls_url'):
+            hls_url = mg['hls_server'] +  mg['hls_url']
+            hls_formats = self._extract_m3u8_formats(
+                    hls_url, item_id, 'mp4', m3u8_id='hls', fatal=False)
+            formats.extend(hls_formats)
+
+        if mg.get('hds_server') and mg.get('hds_url'):
+            f4m_url = mg['hds_server'] + mg['hds_url']
+            f4m_formats = self._extract_f4m_formats(
+                    f4m_url, item_id, f4m_id='hds', fatal=False)
+            formats.extend(f4m_formats)
+
+        return {
+            'id': item_id,
+            'title': title,
+            'formats': formats,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+        }

From fb588f6a5663811da83cde68d17ff4eb041022ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Jan 2016 04:32:47 +0600
Subject: [PATCH 0290/1105] Credit @bpfoley for rte:radio (#8063)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 20f51009e..e39d07efe 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -150,3 +150,4 @@ reiv
 Muratcan Simsek
 Evan Lu
 flatgreen
+Brian Foley

From 0238451fc0f3c5f06827d5197a904981890a3d3b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Jan 2016 04:49:13 +0600
Subject: [PATCH 0291/1105] [rte] PEP 8

---
 youtube_dl/extractor/rte.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index 25abcee92..673399f04 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -58,7 +58,6 @@ class RteIE(InfoExtractor):
         }
 
 
-
 class RteRadioIE(InfoExtractor):
     IE_NAME = 'rte:radio'
     IE_DESC = 'Raidió Teilifís Éireann radio'
@@ -102,15 +101,15 @@ class RteRadioIE(InfoExtractor):
             formats.append({'url': mg.get('url')})
 
         if mg.get('hls_server') and mg.get('hls_url'):
-            hls_url = mg['hls_server'] +  mg['hls_url']
+            hls_url = mg['hls_server'] + mg['hls_url']
             hls_formats = self._extract_m3u8_formats(
-                    hls_url, item_id, 'mp4', m3u8_id='hls', fatal=False)
+                hls_url, item_id, 'mp4', m3u8_id='hls', fatal=False)
             formats.extend(hls_formats)
 
         if mg.get('hds_server') and mg.get('hds_url'):
             f4m_url = mg['hds_server'] + mg['hds_url']
             f4m_formats = self._extract_f4m_formats(
-                    f4m_url, item_id, f4m_id='hds', fatal=False)
+                f4m_url, item_id, f4m_id='hds', fatal=False)
             formats.extend(f4m_formats)
 
         return {

From 9746f4314ad0fe55076eb35fd70413623b0c8ab0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Jan 2016 05:01:32 +0600
Subject: [PATCH 0292/1105] [rte:radio] Simplify

---
 youtube_dl/extractor/rte.py | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index 673399f04..c17ef2cc2 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-
 from ..utils import (
     float_or_none,
     unescapeHTML,
@@ -98,19 +97,19 @@ class RteRadioIE(InfoExtractor):
         formats = []
 
         if mg.get('url') and not mg['url'].startswith('rtmpe:'):
-            formats.append({'url': mg.get('url')})
+            formats.append({'url': mg['url']})
 
         if mg.get('hls_server') and mg.get('hls_url'):
-            hls_url = mg['hls_server'] + mg['hls_url']
-            hls_formats = self._extract_m3u8_formats(
-                hls_url, item_id, 'mp4', m3u8_id='hls', fatal=False)
-            formats.extend(hls_formats)
+            formats.extend(self._extract_m3u8_formats(
+                mg['hls_server'] + mg['hls_url'], item_id, 'mp4',
+                entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
 
         if mg.get('hds_server') and mg.get('hds_url'):
-            f4m_url = mg['hds_server'] + mg['hds_url']
-            f4m_formats = self._extract_f4m_formats(
-                f4m_url, item_id, f4m_id='hds', fatal=False)
-            formats.extend(f4m_formats)
+            formats.extend(self._extract_f4m_formats(
+                mg['hds_server'] + mg['hds_url'], item_id,
+                f4m_id='hds', fatal=False))
+
+        self._sort_formats(formats)
 
         return {
             'id': item_id,

From 9938a17f92ad8242d8841528a89df647ff759183 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Jan 2016 05:04:48 +0600
Subject: [PATCH 0293/1105] [rte:radio] Extract timestamp

---
 youtube_dl/extractor/rte.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index c17ef2cc2..47c8331fe 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     float_or_none,
+    parse_iso8601,
     unescapeHTML,
 )
 
@@ -69,10 +70,12 @@ class RteRadioIE(InfoExtractor):
         'url': 'http://www.rte.ie/radio/utils/radioplayer/rteradioweb.html#!rii=16:10507902:2414:27-12-2015:',
         'info_dict': {
             'id': '10507902',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Gloria',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'Tim Thurston guides you through a millennium of sacred music featuring Gregorian chant, pure solo voices and choral masterpieces, framed around the glorious music of J.S. Bach.',
+            'description': 'md5:9ce124a7fb41559ec68f06387cabddf0',
+            'timestamp': 1451203200,
+            'upload_date': '20151227',
             'duration': 7230.0,
         },
         'params': {
@@ -82,8 +85,10 @@ class RteRadioIE(InfoExtractor):
 
     def _real_extract(self, url):
         item_id = self._match_id(url)
-        feeds_url = 'http://www.rte.ie/rteavgen/getplaylist/?type=web&format=json&id=' + item_id
-        json_string = self._download_json(feeds_url, item_id)
+
+        json_string = self._download_json(
+            'http://www.rte.ie/rteavgen/getplaylist/?type=web&format=json&id=' + item_id,
+            item_id)
 
         # NB the string values in the JSON are stored using XML escaping(!)
         show = json_string['shows'][0]
@@ -91,6 +96,7 @@ class RteRadioIE(InfoExtractor):
         description = unescapeHTML(show.get('description'))
         thumbnail = show.get('thumbnail')
         duration = float_or_none(show.get('duration'), 1000)
+        timestamp = parse_iso8601(show.get('published'))
 
         mg = show['media:group'][0]
 
@@ -114,8 +120,9 @@ class RteRadioIE(InfoExtractor):
         return {
             'id': item_id,
             'title': title,
-            'formats': formats,
             'description': description,
             'thumbnail': thumbnail,
+            'timestamp': timestamp,
             'duration': duration,
+            'formats': formats,
         }

From 3f17c357d9958bfae18f06676c89ffdb63553509 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 4 Jan 2016 18:35:31 +0100
Subject: [PATCH 0294/1105] [downloader/hls] Don't let ffmpeg read from stdin
 (#8139)

If you run 'while read aurl ; do youtube-dl "${aurl}"; done < path_to_batch_file'  (batch_file contains one url per line that uses the hls downloader) each call to youtube-dl consumed some characters and 'read' would assing to 'aurl' a non valid url

(This is the same problem that was fixed for the ffmpeg postprocessors in cffcbc02de504d84e1c2677bb525c00b03e04f53)
---
 youtube_dl/downloader/hls.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index b5a3e1167..d186d402d 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -46,7 +46,7 @@ class HlsFD(FileDownloader):
 
         self._debug_cmd(args)
 
-        retval = subprocess.call(args)
+        retval = subprocess.call(args, stdin=subprocess.PIPE)
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen('\r[%s] %s bytes' % (args[0], fsize))

From fff79f1867c5ac50eb7f88201dd3f728996d52a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Jan 2016 01:05:37 +0600
Subject: [PATCH 0295/1105] [amp] Add missing subtitles to info dict

---
 youtube_dl/extractor/amp.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/amp.py b/youtube_dl/extractor/amp.py
index 1035d1c48..69e6baff7 100644
--- a/youtube_dl/extractor/amp.py
+++ b/youtube_dl/extractor/amp.py
@@ -76,5 +76,6 @@ class AMPIE(InfoExtractor):
             'thumbnails': thumbnails,
             'timestamp': parse_iso8601(item.get('pubDate'), ' '),
             'duration': int_or_none(media_content[0].get('@attributes', {}).get('duration')),
+            'subtitles': subtitles,
             'formats': formats,
         }

From 1dcc38b233f33112e00e1ddf0de6eb8ba1ef028e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Jan 2016 01:11:07 +0600
Subject: [PATCH 0296/1105] [dramafever] Improve subtitles extraction (Closes
 #8136)

---
 youtube_dl/extractor/dramafever.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index b3b21d65f..0e9e6f7ba 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -105,13 +105,16 @@ class DramaFeverIE(DramaFeverBaseIE):
             video_id, 'Downloading episode info JSON', fatal=False)
         if episode_info:
             value = episode_info.get('value')
-            if value:
-                subfile = value[0].get('subfile') or value[0].get('new_subfile')
-                if subfile and subfile != 'http://www.dramafever.com/st/':
-                    info.setdefault('subtitles', {}).setdefault('English', []).append({
-                        'ext': 'srt',
-                        'url': subfile,
-                    })
+            if isinstance(value, list):
+                for v in value:
+                    if v.get('type') == 'Episode':
+                        subfile = v.get('subfile') or v.get('new_subfile')
+                        if subfile and subfile != 'http://www.dramafever.com/st/':
+                            info.setdefault('subtitles', {}).setdefault('English', []).append({
+                                'ext': 'srt',
+                                'url': subfile,
+                            })
+                        break
 
         return info
 

From 8f4c56f33412db909e92218f4933d5b08beb9f21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Jan 2016 01:17:33 +0600
Subject: [PATCH 0297/1105] [dramafever] Extract episode number

---
 youtube_dl/extractor/dramafever.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 0e9e6f7ba..8d71d6769 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -12,6 +12,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     clean_html,
+    int_or_none,
     sanitized_Request,
 )
 
@@ -114,6 +115,7 @@ class DramaFeverIE(DramaFeverBaseIE):
                                 'ext': 'srt',
                                 'url': subfile,
                             })
+                        info['episode_number'] = int_or_none(v.get('number'))
                         break
 
         return info

From bd19aa0ed32ca4d041e691d47697ad7b82454b69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Jan 2016 01:28:48 +0600
Subject: [PATCH 0298/1105] [dramafever] Extract episode

---
 youtube_dl/extractor/dramafever.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 8d71d6769..a14021a9e 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -115,7 +115,12 @@ class DramaFeverIE(DramaFeverBaseIE):
                                 'ext': 'srt',
                                 'url': subfile,
                             })
-                        info['episode_number'] = int_or_none(v.get('number'))
+                        episode_number = int_or_none(v.get('number'))
+                        episode_fallback = 'Episode'
+                        if episode_number:
+                            episode_fallback += ' %d' % episode_number
+                        info['episode'] = v.get('title', episode_fallback)
+                        info['episode_number'] = episode_number
                         break
 
         return info

From a2e51e7b49a193b326291842d03aa49dd08326f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Jan 2016 01:36:38 +0600
Subject: [PATCH 0299/1105] [dramafever] Fix episode fallback

---
 youtube_dl/extractor/dramafever.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index a14021a9e..9f5e82f56 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -119,7 +119,7 @@ class DramaFeverIE(DramaFeverBaseIE):
                         episode_fallback = 'Episode'
                         if episode_number:
                             episode_fallback += ' %d' % episode_number
-                        info['episode'] = v.get('title', episode_fallback)
+                        info['episode'] = v.get('title') or episode_fallback
                         info['episode_number'] = episode_number
                         break
 

From 33cee6c7f6260dd33a9fbafd9f374c70454ff0f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Jan 2016 01:41:18 +0600
Subject: [PATCH 0300/1105] [dramafever] Add test for custom episode title

---
 youtube_dl/extractor/dramafever.py | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 9f5e82f56..d35e88881 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -67,13 +67,15 @@ class DramaFeverBaseIE(AMPIE):
 class DramaFeverIE(DramaFeverBaseIE):
     IE_NAME = 'dramafever'
     _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)(?:/|$)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.dramafever.com/drama/4512/1/Cooking_with_Shin/',
         'info_dict': {
             'id': '4512.1',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Cooking with Shin 4512.1',
             'description': 'md5:a8eec7942e1664a6896fcd5e1287bfd0',
+            'episode': 'Episode 1',
+            'episode_number': 1,
             'thumbnail': 're:^https?://.*\.jpg',
             'timestamp': 1404336058,
             'upload_date': '20140702',
@@ -83,7 +85,25 @@ class DramaFeverIE(DramaFeverBaseIE):
             # m3u8 download
             'skip_download': True,
         },
-    }
+    }, {
+        'url': 'http://www.dramafever.com/drama/4826/4/Mnet_Asian_Music_Awards_2015/?ap=1',
+        'info_dict': {
+            'id': '4826.4',
+            'ext': 'mp4',
+            'title': 'Mnet Asian Music Awards 2015 4826.4',
+            'description': 'md5:3ff2ee8fedaef86e076791c909cf2e91',
+            'episode': 'Mnet Asian Music Awards 2015 - Part 3',
+            'episode_number': 4,
+            'thumbnail': 're:^https?://.*\.jpg',
+            'timestamp': 1450213200,
+            'upload_date': '20151215',
+            'duration': 5602,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url).replace('/', '.')

From 18c782ab26a53958a7d54d39e06eba0cd920178b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Jan 2016 01:58:25 +0600
Subject: [PATCH 0301/1105] [vrt] Extend _VALUD_URL

---
 youtube_dl/extractor/vrt.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vrt.py b/youtube_dl/extractor/vrt.py
index bbd3bbf7b..ee158b7b3 100644
--- a/youtube_dl/extractor/vrt.py
+++ b/youtube_dl/extractor/vrt.py
@@ -8,7 +8,7 @@ from ..utils import float_or_none
 
 
 class VRTIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:deredactie|sporza|cobra)\.be/cm/(?:[^/]+/)+(?P<id>[^/]+)/*'
+    _VALID_URL = r'https?://(?:deredactie|sporza|cobra(?:\.canvas)?)\.be/cm/(?:[^/]+/)+(?P<id>[^/]+)/*'
     _TESTS = [
         # deredactie.be
         {
@@ -52,6 +52,10 @@ class VRTIE(InfoExtractor):
                 'duration': 661,
             }
         },
+        {
+            'url': 'http://cobra.canvas.be/cm/cobra/videozone/rubriek/film-videozone/1.2377055',
+            'only_matching': True,
+        }
     ]
 
     def _real_extract(self, url):

From 2f546d0a3cc9b9fa7f022df68574f7009d7d1ffb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Jan 2016 01:59:45 +0600
Subject: [PATCH 0302/1105] [vrt] Prefix format ids

---
 youtube_dl/extractor/vrt.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vrt.py b/youtube_dl/extractor/vrt.py
index ee158b7b3..01891ac4c 100644
--- a/youtube_dl/extractor/vrt.py
+++ b/youtube_dl/extractor/vrt.py
@@ -73,11 +73,11 @@ class VRTIE(InfoExtractor):
         if mobj:
             formats.extend(self._extract_m3u8_formats(
                 '%s/%s' % (mobj.group('server'), mobj.group('path')),
-                video_id, 'mp4'))
+                video_id, 'mp4', m3u8_id='hls'))
         mobj = re.search(r'data-video-src="(?P<src>[^"]+)"', webpage)
         if mobj:
             formats.extend(self._extract_f4m_formats(
-                '%s/manifest.f4m' % mobj.group('src'), video_id))
+                '%s/manifest.f4m' % mobj.group('src'), video_id, f4m_id='hds'))
         self._sort_formats(formats)
 
         title = self._og_search_title(webpage)

From 40f796288afe634055c4600993b353da58e29a45 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 5 Jan 2016 02:17:12 +0600
Subject: [PATCH 0303/1105] [README.md] Clarify cookies usage

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 4fc83b8e3..75253199c 100644
--- a/README.md
+++ b/README.md
@@ -627,7 +627,7 @@ Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the opt
 
 Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`. Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows, `LF` (`\n`) for Linux and `CR` (`\r`) for Mac OS. `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
 
-Passing cookies to youtube-dl is a good way to workaround login when a particular extractor does not implement it explicitly.
+Passing cookies to youtube-dl is a good way to workaround login when a particular extractor does not implement it explicitly. Another use case is working around [CAPTCHA](https://en.wikipedia.org/wiki/CAPTCHA) some websites require you to solve in particular cases in order to get access (e.g. YouTube, CloudFlare).
 
 ### Can you add support for this anime video site, or site which shows current movies for free?
 

From 17b2d7ca772da5b709e00ca01e96e893807f7b66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 Jan 2016 00:02:21 +0600
Subject: [PATCH 0304/1105] [udemy] Detect non free courses (Closes #8138)

---
 youtube_dl/extractor/udemy.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 1df636779..e19c1f762 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -11,6 +11,7 @@ from ..utils import (
     float_or_none,
     int_or_none,
     sanitized_Request,
+    unescapeHTML,
 )
 
 
@@ -19,8 +20,6 @@ class UdemyIE(InfoExtractor):
     _VALID_URL = r'https?://www\.udemy\.com/(?:[^#]+#/lecture/|lecture/view/?\?lectureId=)(?P<id>\d+)'
     _LOGIN_URL = 'https://www.udemy.com/join/login-popup/?displayType=ajax&showSkipButton=1'
     _ORIGIN_URL = 'https://www.udemy.com'
-    _SUCCESSFULLY_ENROLLED = '>You have enrolled in this course!<'
-    _ALREADY_ENROLLED = '>You are already taking this course.<'
     _NETRC_MACHINE = 'udemy'
 
     _TESTS = [{
@@ -37,15 +36,21 @@ class UdemyIE(InfoExtractor):
     }]
 
     def _enroll_course(self, webpage, course_id):
-        enroll_url = self._search_regex(
+        checkout_url = unescapeHTML(self._search_regex(
+            r'href=(["\'])(?P<url>https?://(?:www\.)?udemy\.com/payment/checkout/.+?)\1',
+            webpage, 'checkout url', group='url', default=None))
+        if checkout_url:
+            raise ExtractorError(
+                'Course %s is not free. You have to pay for it before you can download.'
+                'Use this URL to confirm purchase: %s' % (course_id, checkout_url), expected=True)
+
+        enroll_url = unescapeHTML(self._search_regex(
             r'href=(["\'])(?P<url>https?://(?:www\.)?udemy\.com/course/subscribe/.+?)\1',
-            webpage, 'enroll url', group='url',
-            default='https://www.udemy.com/course/subscribe/?courseId=%s' % course_id)
-        webpage = self._download_webpage(enroll_url, course_id, 'Enrolling in the course')
-        if self._SUCCESSFULLY_ENROLLED in webpage:
-            self.to_screen('%s: Successfully enrolled in' % course_id)
-        elif self._ALREADY_ENROLLED in webpage:
-            self.to_screen('%s: Already enrolled in' % course_id)
+            webpage, 'enroll url', group='url', default=None))
+        if enroll_url:
+            webpage = self._download_webpage(enroll_url, course_id, 'Enrolling in the course')
+            if '>You have enrolled in' in webpage:
+                self.to_screen('%s: Successfully enrolled in the course' % course_id)
 
     def _download_lecture(self, course_id, lecture_id):
         return self._download_json(

From f20756fb10ec560177282f032684327f600acc34 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 Jan 2016 00:03:39 +0600
Subject: [PATCH 0305/1105] [udemy] Fix non free course message

---
 youtube_dl/extractor/udemy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index e19c1f762..f5b5e7fd6 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -41,7 +41,7 @@ class UdemyIE(InfoExtractor):
             webpage, 'checkout url', group='url', default=None))
         if checkout_url:
             raise ExtractorError(
-                'Course %s is not free. You have to pay for it before you can download.'
+                'Course %s is not free. You have to pay for it before you can download. '
                 'Use this URL to confirm purchase: %s' % (course_id, checkout_url), expected=True)
 
         enroll_url = unescapeHTML(self._search_regex(

From 76048b23e8a4aac93a33a96356fe64a9bcf78421 Mon Sep 17 00:00:00 2001
From: kaspi <je326@hotmail.com>
Date: Sat, 17 Oct 2015 23:27:03 -0400
Subject: [PATCH 0306/1105] [npr] Add extractor

removed md5 from _TEST

moved from xml data to json

test

changed _TEST url to one that will not expire, so tests would not be failing
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/npr.py      | 71 ++++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+)
 create mode 100644 youtube_dl/extractor/npr.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 19cea5e99..c46e19eae 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -473,6 +473,7 @@ from .npo import (
     VPROIE,
     WNLIE
 )
+from .npr import NprIE
 from .nrk import (
     NRKIE,
     NRKPlaylistIE,
diff --git a/youtube_dl/extractor/npr.py b/youtube_dl/extractor/npr.py
new file mode 100644
index 000000000..a823bc096
--- /dev/null
+++ b/youtube_dl/extractor/npr.py
@@ -0,0 +1,71 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import os.path
+import re
+
+from ..compat import compat_urllib_parse_unquote
+from ..utils import url_basename
+from .common import InfoExtractor
+
+class NprIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?npr\.org/player/v2/mediaPlayer.html?.*id=(?P<id>[0-9]+)'
+    _TEST = {
+    'url': 'http://www.npr.org/player/v2/mediaPlayer.html?id=449974205',
+    'info_dict': {
+        'id': '449974205',
+        'ext': 'mp4',
+        'title': 'New Music From Beach House, Chairlift, CMJ Discoveries And More'
+    }
+}
+
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        webpage_url = 'http://www.npr.org/player/v2/mediaPlayer.html?id=' + video_id
+        webpage = self._download_webpage(webpage_url, video_id)
+        key = 'MDAzMzQ2MjAyMDEyMzk4MTU1MDg3ZmM3MQ010'
+        xml_url = 'http://api.npr.org/query?id=%s&apiKey=%s' % (video_id, key)
+        json_url = 'http://api.npr.org/query?id=%s&apiKey=%s&format=json' % (video_id, key)
+
+        formats = []
+        entries = []
+
+        config = self._download_json(json_url, video_id)
+
+        content = config["list"]["story"]
+
+        album_title = config["list"]["story"][0]['song'][0]['album']['albumTitle']
+        print album_title['$text']
+
+        for key in content:
+            if "audio" in key:
+                for x in key['audio']:
+                    if x['type'] == 'standard':
+                        playlist = True
+                        song_duration = x["duration"]['$text']
+                        song_title = x["title"]["$text"]
+                        song_id = x["id"]
+
+                        for k in x["format"]:
+                            if type(x["format"][k]) is list:
+                                for z in x["format"][k]:
+                                    formats.append({ 'format': z['type'], 
+                                                     'url'   : z['$text']
+                                              })
+                            else:
+                                formats.append({ 'format': k, 
+                                                 'url'   : x["format"][k]['$text']
+                                      })
+
+                        entries.append({ "title":song_title,
+                                         "id":song_id,
+                                         "duration": song_duration ,
+                                         "formats":formats})
+                        formats = []
+
+        return {    '_type': 'playlist',
+                    'id' : video_id,
+                    'title' : album_title,
+                    'entries': entries  }

From 51d3045de26fccbdcf0076c5cab8ab8152548bba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 7 Jan 2016 01:57:36 +0600
Subject: [PATCH 0307/1105] [npr] Fix extractor (Closes #7218)

---
 youtube_dl/extractor/npr.py | 133 +++++++++++++++++++-----------------
 1 file changed, 72 insertions(+), 61 deletions(-)

diff --git a/youtube_dl/extractor/npr.py b/youtube_dl/extractor/npr.py
index a823bc096..125c7010b 100644
--- a/youtube_dl/extractor/npr.py
+++ b/youtube_dl/extractor/npr.py
@@ -1,71 +1,82 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
-import os.path
-import re
-
-from ..compat import compat_urllib_parse_unquote
-from ..utils import url_basename
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse
+from ..utils import (
+    int_or_none,
+    qualities,
+)
+
 
 class NprIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?npr\.org/player/v2/mediaPlayer.html?.*id=(?P<id>[0-9]+)'
-    _TEST = {
-    'url': 'http://www.npr.org/player/v2/mediaPlayer.html?id=449974205',
-    'info_dict': {
-        'id': '449974205',
-        'ext': 'mp4',
-        'title': 'New Music From Beach House, Chairlift, CMJ Discoveries And More'
-    }
-}
-
+    _VALID_URL = r'http://(?:www\.)?npr\.org/player/v2/mediaPlayer\.html\?.*\bid=(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://www.npr.org/player/v2/mediaPlayer.html?id=449974205',
+        'info_dict': {
+            'id': '449974205',
+            'title': 'New Music From Beach House, Chairlift, CMJ Discoveries And More'
+        },
+        'playlist_count': 7,
+    }, {
+        'url': 'http://www.npr.org/player/v2/mediaPlayer.html?action=1&t=1&islist=false&id=446928052&m=446929930&live=1',
+        'info_dict': {
+            'id': '446928052',
+            'title': "Songs We Love: Tigran Hamasyan, 'Your Mercy is Boundless'"
+        },
+        'playlist': [{
+            'md5': '12fa60cb2d3ed932f53609d4aeceabf1',
+            'info_dict': {
+                'id': '446929930',
+                'ext': 'mp3',
+                'title': 'Your Mercy is Boundless (Bazum en Qo gtutyunqd)',
+                'duration': 402,
+            },
+        }],
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        webpage_url = 'http://www.npr.org/player/v2/mediaPlayer.html?id=' + video_id
-        webpage = self._download_webpage(webpage_url, video_id)
-        key = 'MDAzMzQ2MjAyMDEyMzk4MTU1MDg3ZmM3MQ010'
-        xml_url = 'http://api.npr.org/query?id=%s&apiKey=%s' % (video_id, key)
-        json_url = 'http://api.npr.org/query?id=%s&apiKey=%s&format=json' % (video_id, key)
+        playlist_id = self._match_id(url)
+
+        config = self._download_json(
+            'http://api.npr.org/query?%s' % compat_urllib_parse.urlencode({
+                'id': playlist_id,
+                'fields': 'titles,audio,show',
+                'format': 'json',
+                'apiKey': 'MDAzMzQ2MjAyMDEyMzk4MTU1MDg3ZmM3MQ010',
+            }), playlist_id)
+
+        story = config['list']['story'][0]
+
+        KNOWN_FORMATS = ('threegp', 'mp4', 'mp3')
+        quality = qualities(KNOWN_FORMATS)
 
-        formats = []
         entries = []
+        for audio in story.get('audio', []):
+            title = audio.get('title', {}).get('$text')
+            duration = int_or_none(audio.get('duration', {}).get('$text'))
+            formats = []
+            for format_id, formats_entry in audio.get('format', {}).items():
+                if not formats_entry:
+                    continue
+                if isinstance(formats_entry, list):
+                    formats_entry = formats_entry[0]
+                format_url = formats_entry.get('$text')
+                if not format_url:
+                    continue
+                if format_id in KNOWN_FORMATS:
+                    formats.append({
+                        'url': format_url,
+                        'format_id': format_id,
+                        'ext': formats_entry.get('type'),
+                        'quality': quality(format_id),
+                    })
+            self._sort_formats(formats)
+            entries.append({
+                'id': audio['id'],
+                'title': title,
+                'duration': duration,
+                'formats': formats,
+            })
 
-        config = self._download_json(json_url, video_id)
-
-        content = config["list"]["story"]
-
-        album_title = config["list"]["story"][0]['song'][0]['album']['albumTitle']
-        print album_title['$text']
-
-        for key in content:
-            if "audio" in key:
-                for x in key['audio']:
-                    if x['type'] == 'standard':
-                        playlist = True
-                        song_duration = x["duration"]['$text']
-                        song_title = x["title"]["$text"]
-                        song_id = x["id"]
-
-                        for k in x["format"]:
-                            if type(x["format"][k]) is list:
-                                for z in x["format"][k]:
-                                    formats.append({ 'format': z['type'], 
-                                                     'url'   : z['$text']
-                                              })
-                            else:
-                                formats.append({ 'format': k, 
-                                                 'url'   : x["format"][k]['$text']
-                                      })
-
-                        entries.append({ "title":song_title,
-                                         "id":song_id,
-                                         "duration": song_duration ,
-                                         "formats":formats})
-                        formats = []
-
-        return {    '_type': 'playlist',
-                    'id' : video_id,
-                    'title' : album_title,
-                    'entries': entries  }
+        playlist_title = story.get('title', {}).get('$text')
+        return self.playlist_result(entries, playlist_id, playlist_title)

From e4f49a87533bd6e3ebf11b43c3baf0e32db2f5ca Mon Sep 17 00:00:00 2001
From: oittaa <oittaa@users.noreply.github.com>
Date: Thu, 7 Jan 2016 07:26:14 +0200
Subject: [PATCH 0308/1105] check video_play_path and use xpath_text

"This check should take place earlier and should be more general if not video_url:. Same should be done for video_play_path. Also these fields better extracted with xpath_text."

Suggestions by @dstftw
---
 youtube_dl/extractor/crunchyroll.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 00d943f77..785594df8 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -329,8 +329,10 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                 streamdata_req, video_id,
                 note='Downloading media info for %s' % video_format)
             stream_info = streamdata.find('./{default}preload/stream_info')
-            video_url = stream_info.find('./host').text
-            video_play_path = stream_info.find('./file').text
+            video_url = xpath_text(stream_info, './host')
+            video_play_path = xpath_text(stream_info, './file')
+            if not video_url or not video_play_path:
+                continue
             metadata = stream_info.find('./metadata')
             format_info = {
                 'format': video_format,

From 2e02ecbccc9751af54ae2aab9ce973712c78af74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 7 Jan 2016 12:24:32 +0600
Subject: [PATCH 0309/1105] [ivideon] Add extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ivideon.py  | 75 ++++++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+)
 create mode 100644 youtube_dl/extractor/ivideon.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c46e19eae..a9d23b8f4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -299,6 +299,7 @@ from .ivi import (
     IviIE,
     IviCompilationIE
 )
+from .ivideon import IvideonIE
 from .izlesene import IzleseneIE
 from .jadorecettepub import JadoreCettePubIE
 from .jeuxvideo import JeuxVideoIE
diff --git a/youtube_dl/extractor/ivideon.py b/youtube_dl/extractor/ivideon.py
new file mode 100644
index 000000000..aa05f5413
--- /dev/null
+++ b/youtube_dl/extractor/ivideon.py
@@ -0,0 +1,75 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse
+from ..utils import qualities
+
+
+class IvideonIE(InfoExtractor):
+    IE_NAME = 'ivideon'
+    IE_DESC = 'Ivideon TV'
+    _VALID_URL = r'https?://(?:www\.)?ivideon\.com/tv/camera/(?P<id>\d+-[\da-f]+)/(?P<camera_id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.ivideon.com/tv/camera/100-916ca13b5c4ad9f564266424a026386d/0/',
+        'info_dict': {
+            'id': '100-916ca13b5c4ad9f564266424a026386d',
+            'ext': 'flv',
+            'title': 're:^Касса [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 'Основное предназначение - запись действий кассиров. Плюс общий вид.',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }, {
+        'url': 'https://www.ivideon.com/tv/camera/100-c4ee4cb9ede885cf62dfbe93d7b53783/589824/?lang=ru',
+        'only_matching': True,
+    }]
+
+    _QUALITIES = ('low', 'mid', 'hi')
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        server_id, camera_id = mobj.group('id'), mobj.group('camera_id')
+        camera_name, description = None, None
+
+        webpage = self._download_webpage(url, server_id, fatal=False)
+        if webpage:
+            config_string = self._search_regex(
+                r'var\s+config\s*=\s*({.+?});', webpage, 'config', default=None)
+            if config_string:
+                config = self._parse_json(config_string, server_id, fatal=False)
+                camera_info = config.get('ivTvAppOptions', {}).get('currentCameraInfo')
+                if camera_info:
+                    camera_name = camera_info.get('camera_name')
+                    description = camera_info.get('misc', {}).get('description')
+            if not camera_name:
+                camera_name = self._html_search_meta(
+                    'name', webpage, 'camera name', default=None) or self._search_regex(
+                    r'<h1[^>]+class="b-video-title"[^>]*>([^<]+)', webpage, 'camera name', default=None)
+
+        quality = qualities(self._QUALITIES)
+
+        formats = [{
+            'url': 'https://streaming.ivideon.com/flv/live?%s' % compat_urllib_parse.urlencode({
+                'server': server_id,
+                'camera': camera_id,
+                'sessionId': 'demo',
+                'q': quality(format_id),
+            }),
+            'format_id': format_id,
+            'ext': 'flv',
+            'quality': quality(format_id),
+        } for format_id in self._QUALITIES]
+        self._sort_formats(formats)
+
+        return {
+            'id': server_id,
+            'title': self._live_title(camera_name or server_id),
+            'description': description,
+            'is_live': True,
+            'formats': formats,
+        }

From 23f13e97548cc4ff3d11408ee5bc77f682e642dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Jan 2016 00:52:55 +0600
Subject: [PATCH 0310/1105] [youtube] Support expanding alternative format of
 links in description (Closes #8164)

---
 youtube_dl/extractor/youtube.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4aac2cc03..897641e42 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1235,10 +1235,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             video_description = re.sub(r'''(?x)
                 <a\s+
                     (?:[a-zA-Z-]+="[^"]+"\s+)*?
-                    title="([^"]+)"\s+
+                    (?:title|href)="([^"]+)"\s+
                     (?:[a-zA-Z-]+="[^"]+"\s+)*?
-                    class="yt-uix-redirect-link"\s*>
-                [^<]+
+                    class="(?:yt-uix-redirect-link|yt-uix-sessionlink[^"]*)".*?>
+                [^<]+\.{3}\s*
                 </a>
             ''', r'\1', video_description)
             video_description = clean_html(video_description)

From 97afd99a18e4723e4ff588df456c7aec62967b35 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Jan 2016 01:54:31 +0600
Subject: [PATCH 0311/1105] [soundcloud:likes] Adapt to API changes (Closes
 #8166)

---
 youtube_dl/extractor/soundcloud.py | 38 +++++++++++++++---------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 02e64e094..b2d5487ca 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -384,27 +384,24 @@ class SoundcloudUserIE(SoundcloudIE):
         resource = mobj.group('rsrc') or 'all'
         base_url = self._BASE_URL_MAP[resource] % user['id']
 
-        next_href = None
+        COMMON_QUERY = {
+            'limit': 50,
+            'client_id': self._CLIENT_ID,
+            'linked_partitioning': '1',
+        }
+
+        query = COMMON_QUERY.copy()
+        query['offset'] = 0
+
+        next_href = base_url + '?' + compat_urllib_parse.urlencode(query)
 
         entries = []
         for i in itertools.count():
-            if not next_href:
-                data = compat_urllib_parse.urlencode({
-                    'offset': i * 50,
-                    'limit': 50,
-                    'client_id': self._CLIENT_ID,
-                    'linked_partitioning': '1',
-                    'representation': 'speedy',
-                })
-                next_href = base_url + '?' + data
-
             response = self._download_json(
                 next_href, uploader, 'Downloading track page %s' % (i + 1))
 
             collection = response['collection']
-
             if not collection:
-                self.to_screen('%s: End page received' % uploader)
                 break
 
             def resolve_permalink_url(candidates):
@@ -419,12 +416,15 @@ class SoundcloudUserIE(SoundcloudIE):
                 if permalink_url:
                     entries.append(self.url_result(permalink_url))
 
-            if 'next_href' in response:
-                next_href = response['next_href']
-                if not next_href:
-                    break
-            else:
-                next_href = None
+            next_href = response.get('next_href')
+            if not next_href:
+                break
+
+            parsed_next_href = compat_urlparse.urlparse(response['next_href'])
+            qs = compat_urlparse.parse_qs(parsed_next_href.query)
+            qs.update(COMMON_QUERY)
+            next_href = compat_urlparse.urlunparse(
+                parsed_next_href._replace(query=compat_urllib_parse.urlencode(qs, True)))
 
         return {
             '_type': 'playlist',

From 18e6c97c48f883911649d9b3d64127379a8b1df4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Jan 2016 03:19:47 +0600
Subject: [PATCH 0312/1105] [adultswim] Skip georestricted hls (Closes #8168)

---
 youtube_dl/extractor/adultswim.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index bf21a6887..8157da2cb 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -187,7 +187,8 @@ class AdultSwimIE(InfoExtractor):
                 media_url = file_el.text
                 if determine_ext(media_url) == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
-                        media_url, segment_title, 'mp4', preference=0, m3u8_id='hls'))
+                        media_url, segment_title, 'mp4', preference=0,
+                        m3u8_id='hls', fatal=False))
                 else:
                     formats.append({
                         'format_id': '%s_%s' % (bitrate, ftype),

From 4cf096a4a99c2cc29708f23244e4433d91c83b65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Jan 2016 05:11:23 +0600
Subject: [PATCH 0313/1105] [ivideon] Add support for map bound URLs

---
 youtube_dl/extractor/ivideon.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ivideon.py b/youtube_dl/extractor/ivideon.py
index aa05f5413..617dc8c07 100644
--- a/youtube_dl/extractor/ivideon.py
+++ b/youtube_dl/extractor/ivideon.py
@@ -4,14 +4,17 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import (
+    compat_urllib_parse,
+    compat_urlparse,
+)
 from ..utils import qualities
 
 
 class IvideonIE(InfoExtractor):
     IE_NAME = 'ivideon'
     IE_DESC = 'Ivideon TV'
-    _VALID_URL = r'https?://(?:www\.)?ivideon\.com/tv/camera/(?P<id>\d+-[\da-f]+)/(?P<camera_id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?ivideon\.com/tv/(?:[^/]+/)*camera/(?P<id>\d+-[\da-f]+)/(?P<camera_id>\d+)'
     _TESTS = [{
         'url': 'https://www.ivideon.com/tv/camera/100-916ca13b5c4ad9f564266424a026386d/0/',
         'info_dict': {
@@ -27,6 +30,9 @@ class IvideonIE(InfoExtractor):
     }, {
         'url': 'https://www.ivideon.com/tv/camera/100-c4ee4cb9ede885cf62dfbe93d7b53783/589824/?lang=ru',
         'only_matching': True,
+    }, {
+        'url': 'https://www.ivideon.com/tv/map/22.917923/-31.816406/16/camera/100-e7bc16c7d4b5bbd633fd5350b66dfa9a/0',
+        'only_matching': True,
     }]
 
     _QUALITIES = ('low', 'mid', 'hi')
@@ -35,8 +41,10 @@ class IvideonIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         server_id, camera_id = mobj.group('id'), mobj.group('camera_id')
         camera_name, description = None, None
+        camera_url = compat_urlparse.urljoin(
+            url, '/tv/camera/%s/%s/' % (server_id, camera_id))
 
-        webpage = self._download_webpage(url, server_id, fatal=False)
+        webpage = self._download_webpage(camera_url, server_id, fatal=False)
         if webpage:
             config_string = self._search_regex(
                 r'var\s+config\s*=\s*({.+?});', webpage, 'config', default=None)

From 5dbe81a1d35ae704b5ea208698a6bb785923d71a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Jan 2016 10:41:24 +0600
Subject: [PATCH 0314/1105] [vimeo] Automatically pickup full movie when rented
 (Closes #8171)

---
 youtube_dl/extractor/vimeo.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 7af699982..776e8cce4 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -217,7 +217,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/.+?)\1', webpage)
         if mobj:
             player_url = unescapeHTML(mobj.group('url'))
-            surl = smuggle_url(player_url, {'Referer': url})
+            surl = smuggle_url(player_url, {'http_headers': {'Referer': url}})
             return surl
         # Look for embedded (swf embed) Vimeo player
         mobj = re.search(
@@ -262,11 +262,11 @@ class VimeoIE(VimeoBaseInfoExtractor):
         self._login()
 
     def _real_extract(self, url):
-        url, data = unsmuggle_url(url)
+        url, data = unsmuggle_url(url, {})
         headers = std_headers
-        if data is not None:
+        if 'http_headers' in data:
             headers = headers.copy()
-            headers.update(data)
+            headers.update(data['http_headers'])
         if 'Referer' not in headers:
             headers['Referer'] = url
 
@@ -342,7 +342,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 raise ExtractorError('The author has restricted the access to this video, try with the "--referer" option')
 
             if re.search(r'<form[^>]+?id="pw_form"', webpage) is not None:
-                if data and '_video_password_verified' in data:
+                if '_video_password_verified' in data:
                     raise ExtractorError('video password verification failed!')
                 self._verify_video_password(url, video_id, webpage)
                 return self._real_extract(
@@ -354,6 +354,13 @@ class VimeoIE(VimeoBaseInfoExtractor):
             if config.get('view') == 4:
                 config = self._verify_player_video_password(url, video_id)
 
+        if '>You rented this title.<' in webpage:
+            feature_id = config.get('video', {}).get('vod', {}).get('feature_id')
+            if feature_id and not data.get('force_feature_id', False):
+                return self.url_result(smuggle_url(
+                    'https://player.vimeo.com/player/%s' % feature_id,
+                    {'force_feature_id': True}), 'Vimeo')
+
         # Extract title
         video_title = config["video"]["title"]
 

From 5777f5d386d14407a19bc86c31e0bf2b5ae4a87f Mon Sep 17 00:00:00 2001
From: atomic83 <atomic83@riseup.net>
Date: Fri, 8 Jan 2016 12:58:05 +0100
Subject: [PATCH 0315/1105] Extract xHamster title fix

---
 youtube_dl/extractor/xhamster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 8938c0e45..261d323a7 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -65,7 +65,7 @@ class XHamsterIE(InfoExtractor):
 
         title = self._html_search_regex(
             [r'<title>(?P<title>.+?)(?:, (?:[^,]+? )?Porn: xHamster| - xHamster\.com)</title>',
-             r'<h1>([^<]+)</h1>'], webpage, 'title')
+             r'<h1(?: itemprop="name")?>([^<]+)</h1>'], webpage, 'title')
 
         # Only a few videos have an description
         mobj = re.search(r'<span>Description: </span>([^<]+)', webpage)

From 6609b3ce371d08c0464a3caa30c21b61de221508 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jan 2016 00:19:36 +0600
Subject: [PATCH 0316/1105] [xhamster] Improve title extraction

---
 youtube_dl/extractor/xhamster.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 261d323a7..97355d17f 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -64,8 +64,9 @@ class XHamsterIE(InfoExtractor):
         webpage = self._download_webpage(mrss_url, video_id)
 
         title = self._html_search_regex(
-            [r'<title>(?P<title>.+?)(?:, (?:[^,]+? )?Porn: xHamster| - xHamster\.com)</title>',
-             r'<h1(?: itemprop="name")?>([^<]+)</h1>'], webpage, 'title')
+            [r'<title[^>]*>(.+?)(?:,\s*[^,]*?\s*Porn\s*[^,]*?:\s*xHamster[^<]*| - xHamster\.com)</title>',
+             r'<h1[^>]*>([^<]+)</h1>',
+             r'<meta[^>]+itemprop=".*?caption.*?"[^>]+content="(.+?)"'], webpage, 'title')
 
         # Only a few videos have an description
         mobj = re.search(r'<span>Description: </span>([^<]+)', webpage)

From 4763b624a6655bc2333157031c73858f6b918f61 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jan 2016 00:21:57 +0600
Subject: [PATCH 0317/1105] [xhamster] Fix upload date extraction

---
 youtube_dl/extractor/xhamster.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 97355d17f..560c38e26 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -72,10 +72,9 @@ class XHamsterIE(InfoExtractor):
         mobj = re.search(r'<span>Description: </span>([^<]+)', webpage)
         description = mobj.group(1) if mobj else None
 
-        upload_date = self._html_search_regex(r'hint=\'(\d{4}-\d{2}-\d{2}) \d{2}:\d{2}:\d{2} [A-Z]{3,4}\'',
-                                              webpage, 'upload date', fatal=False)
-        if upload_date:
-            upload_date = unified_strdate(upload_date)
+        upload_date = unified_strdate(self._search_regex(
+            r'hint=["\'](\d{4}-\d{2}-\d{2}) \d{2}:\d{2}:\d{2} [A-Z]{3,4}',
+            webpage, 'upload date', fatal=False))
 
         uploader = self._html_search_regex(
             r"<a href='[^']+xhamster\.com/user/[^>]+>(?P<uploader>[^<]+)",

From 44731e308cdcc89fda1e613e094aa23de33f5c9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jan 2016 00:26:37 +0600
Subject: [PATCH 0318/1105] [xhamster] Fix duration extraction

---
 youtube_dl/extractor/xhamster.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 560c38e26..2b9ac2419 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -4,10 +4,10 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    unified_strdate,
-    str_to_int,
+    float_or_none,
     int_or_none,
-    parse_duration,
+    str_to_int,
+    unified_strdate,
 )
 
 
@@ -85,8 +85,9 @@ class XHamsterIE(InfoExtractor):
              r'''<video[^>]+poster=(?P<q>["'])(?P<thumbnail>.+?)(?P=q)[^>]*>'''],
             webpage, 'thumbnail', fatal=False, group='thumbnail')
 
-        duration = parse_duration(self._html_search_regex(r'<span>Runtime:</span> (\d+:\d+)</div>',
-                                                          webpage, 'duration', fatal=False))
+        duration = float_or_none(self._search_regex(
+            r'(["\'])duration\1\s*:\s*(["\'])(?P<duration>.+?)\2',
+            webpage, 'duration', fatal=False, group='duration'))
 
         view_count = self._html_search_regex(r'<span>Views:</span> ([^<]+)</div>', webpage, 'view count', fatal=False)
         if view_count:

From 6a16fd4a1ad1dbd9372f75ddffce1e9fe95b002c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jan 2016 00:29:10 +0600
Subject: [PATCH 0319/1105] [xhamster] Fix view count extraction

---
 youtube_dl/extractor/xhamster.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 2b9ac2419..ccee77359 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -89,9 +89,9 @@ class XHamsterIE(InfoExtractor):
             r'(["\'])duration\1\s*:\s*(["\'])(?P<duration>.+?)\2',
             webpage, 'duration', fatal=False, group='duration'))
 
-        view_count = self._html_search_regex(r'<span>Views:</span> ([^<]+)</div>', webpage, 'view count', fatal=False)
-        if view_count:
-            view_count = str_to_int(view_count)
+        view_count = int_or_none(self._search_regex(
+            r'content=["\']User(?:View|Play)s:(\d+)',
+            webpage, 'view count', fatal=False))
 
         mobj = re.search(r"hint='(?P<likecount>\d+) Likes / (?P<dislikecount>\d+) Dislikes'", webpage)
         (like_count, dislike_count) = (mobj.group('likecount'), mobj.group('dislikecount')) if mobj else (None, None)

From 1a6d92847f4e5fe1d94f2f159f903a414d5ff62c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jan 2016 00:31:24 +0600
Subject: [PATCH 0320/1105] [xhamster] Change title regex precedence

---
 youtube_dl/extractor/xhamster.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index ccee77359..c9108c345 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -64,9 +64,10 @@ class XHamsterIE(InfoExtractor):
         webpage = self._download_webpage(mrss_url, video_id)
 
         title = self._html_search_regex(
-            [r'<title[^>]*>(.+?)(?:,\s*[^,]*?\s*Porn\s*[^,]*?:\s*xHamster[^<]*| - xHamster\.com)</title>',
-             r'<h1[^>]*>([^<]+)</h1>',
-             r'<meta[^>]+itemprop=".*?caption.*?"[^>]+content="(.+?)"'], webpage, 'title')
+            [r'<h1[^>]*>([^<]+)</h1>',
+             r'<meta[^>]+itemprop=".*?caption.*?"[^>]+content="(.+?)"',
+             r'<title[^>]*>(.+?)(?:,\s*[^,]*?\s*Porn\s*[^,]*?:\s*xHamster[^<]*| - xHamster\.com)</title>'],
+            webpage, 'title')
 
         # Only a few videos have an description
         mobj = re.search(r'<span>Description: </span>([^<]+)', webpage)

From bcac2a071040fd89585a806ca66e086c109406c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jan 2016 00:36:19 +0600
Subject: [PATCH 0321/1105] [xhamster] Fix uploader extraction

---
 youtube_dl/extractor/xhamster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index c9108c345..981ee284f 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -78,7 +78,7 @@ class XHamsterIE(InfoExtractor):
             webpage, 'upload date', fatal=False))
 
         uploader = self._html_search_regex(
-            r"<a href='[^']+xhamster\.com/user/[^>]+>(?P<uploader>[^<]+)",
+            r'<span[^>]+itemprop=["\']author[^>]+><a[^>]+href=["\'].+?xhamster\.com/user/[^>]+>(?P<uploader>.+?)</a>',
             webpage, 'uploader', default='anonymous')
 
         thumbnail = self._search_regex(

From 14b4f038c0123138a642a25bf6a3851dd610d1d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jan 2016 00:36:43 +0600
Subject: [PATCH 0322/1105] [xhamster] Update tests

---
 youtube_dl/extractor/xhamster.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 981ee284f..066af184f 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -22,7 +22,7 @@ class XHamsterIE(InfoExtractor):
                 'title': 'FemaleAgent Shy beauty takes the bait',
                 'upload_date': '20121014',
                 'uploader': 'Ruseful2011',
-                'duration': 893,
+                'duration': 893.52,
                 'age_limit': 18,
             }
         },
@@ -34,7 +34,7 @@ class XHamsterIE(InfoExtractor):
                 'title': 'Britney Spears  Sexy Booty',
                 'upload_date': '20130914',
                 'uploader': 'jojo747400',
-                'duration': 200,
+                'duration': 200.48,
                 'age_limit': 18,
             }
         },

From d5f071afb5ce302058d99a14989f3f6afb88b1a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jan 2016 03:06:09 +0600
Subject: [PATCH 0323/1105] [vimeo] Check source file URL (Closes #8187)

---
 youtube_dl/extractor/vimeo.py | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 776e8cce4..5cb01907b 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -11,6 +11,7 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
+    determine_ext,
     encode_dict,
     ExtractorError,
     InAdvancePagedList,
@@ -419,16 +420,21 @@ class VimeoIE(VimeoBaseInfoExtractor):
         download_data = self._download_json(download_request, video_id, fatal=False)
         if download_data:
             source_file = download_data.get('source_file')
-            if source_file and not source_file.get('is_cold') and not source_file.get('is_defrosting'):
-                formats.append({
-                    'url': source_file['download_url'],
-                    'ext': source_file['extension'].lower(),
-                    'width': int_or_none(source_file.get('width')),
-                    'height': int_or_none(source_file.get('height')),
-                    'filesize': parse_filesize(source_file.get('size')),
-                    'format_id': source_file.get('public_name', 'Original'),
-                    'preference': 1,
-                })
+            if isinstance(source_file, dict):
+                download_url = source_file.get('download_url')
+                if download_url and not source_file.get('is_cold') and not source_file.get('is_defrosting'):
+                    source_name = source_file.get('public_name', 'Original')
+                    if self._is_valid_url(download_url, video_id, '%s video' % source_name):
+                        ext = source_file.get('extension', determine_ext(download_url)).lower(),
+                        formats.append({
+                            'url': download_url,
+                            'ext': ext,
+                            'width': int_or_none(source_file.get('width')),
+                            'height': int_or_none(source_file.get('height')),
+                            'filesize': parse_filesize(source_file.get('size')),
+                            'format_id': source_name,
+                            'preference': 1,
+                        })
         config_files = config['video'].get('files') or config['request'].get('files', {})
         for f in config_files.get('progressive', []):
             video_url = f.get('url')

From 16f1131a4d46c4437485d4b075e9d1243ca8d60c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jan 2016 03:07:29 +0600
Subject: [PATCH 0324/1105] [vimeo] Add test for #8187

---
 youtube_dl/extractor/vimeo.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 5cb01907b..76e681bc3 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -209,6 +209,11 @@ class VimeoIE(VimeoBaseInfoExtractor):
             'url': 'https://vimeo.com/groups/travelhd/videos/22439234',
             'only_matching': True,
         },
+        {
+            # source file returns 403: Forbidden
+            'url': 'https://vimeo.com/7809605',
+            'only_matching': True,
+        },
     ]
 
     @staticmethod

From b374af6ebdf6298ab1593bee56dba5b0a1daf9b7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 9 Jan 2016 01:16:08 +0100
Subject: [PATCH 0325/1105] release 2016.01.09

---
 docs/supportedsites.md | 10 +++++++---
 youtube_dl/version.py  |  2 +-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 84c166805..8d0c7b97a 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -1,6 +1,7 @@
 # Supported sites
  - **1tv**: Первый канал
  - **1up.com**
+ - **20min**
  - **220.ro**
  - **22tracks:genre**
  - **22tracks:track**
@@ -255,6 +256,7 @@
  - **Ir90Tv**
  - **ivi**: ivi.ru
  - **ivi:compilation**: ivi.ru compilations
+ - **ivideon**: Ivideon TV
  - **Izlesene**
  - **JadoreCettePub**
  - **JeuxVideo**
@@ -386,13 +388,14 @@
  - **nowness**
  - **nowness:playlist**
  - **nowness:series**
- - **NowTV**
+ - **NowTV** (Currently broken)
  - **NowTVList**
  - **nowvideo**: NowVideo
  - **npo**: npo.nl and ntr.nl
  - **npo.nl:live**
  - **npo.nl:radio**
  - **npo.nl:radio:fragment**
+ - **Npr**
  - **NRK**
  - **NRKPlaylist**
  - **NRKTV**: NRK TV and NRK Radio
@@ -464,11 +467,13 @@
  - **RegioTV**
  - **Restudy**
  - **ReverbNation**
+ - **Revision3**
  - **RingTV**
  - **RottenTomatoes**
  - **Roxwel**
  - **RTBF**
- - **Rte**
+ - **rte**: Raidió Teilifís Éireann TV
+ - **rte:radio**: Raidió Teilifís Éireann radio
  - **rtl.nl**: rtl.nl and rtlxl.nl
  - **RTL2**
  - **RTP**
@@ -573,7 +578,6 @@
  - **TeleMB**
  - **TeleTask**
  - **TenPlay**
- - **TestTube**
  - **TF1**
  - **TheIntercept**
  - **TheOnion**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 790bd5b3b..7030903c0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.01.01'
+__version__ = '2016.01.09'

From 27783821af5a85fb20266834cff4f07eb0695cb5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 Jan 2016 11:16:23 +0100
Subject: [PATCH 0326/1105] [xhamster] Remove unused import

---
 youtube_dl/extractor/xhamster.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 066af184f..fd43e8854 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -6,7 +6,6 @@ from .common import InfoExtractor
 from ..utils import (
     float_or_none,
     int_or_none,
-    str_to_int,
     unified_strdate,
 )
 

From 7a34302e95ee66f770b1eba720847831c40121a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Jan 2016 01:37:10 +0600
Subject: [PATCH 0327/1105] [canalc2] Fix extraction (Closes #8191)

---
 youtube_dl/extractor/canalc2.py | 46 +++++++++++++++++++++------------
 1 file changed, 29 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index f6a1ff381..f1f128c45 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -9,9 +9,9 @@ from ..utils import parse_duration
 
 class Canalc2IE(InfoExtractor):
     IE_NAME = 'canalc2.tv'
-    _VALID_URL = r'https?://(?:www\.)?canalc2\.tv/video/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:(?:www\.)?canalc2\.tv/video/|archives-canalc2\.u-strasbg\.fr/video\.asp\?.*\bidVideo=)(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.canalc2.tv/video/12163',
         'md5': '060158428b650f896c542dfbb3d6487f',
         'info_dict': {
@@ -23,24 +23,36 @@ class Canalc2IE(InfoExtractor):
         'params': {
             'skip_download': True,  # Requires rtmpdump
         }
-    }
+    }, {
+        'url': 'http://archives-canalc2.u-strasbg.fr/video.asp?idVideo=11427&voir=oui',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        video_url = self._search_regex(
-            r'jwplayer\((["\'])Player\1\)\.setup\({[^}]*file\s*:\s*(["\'])(?P<file>.+?)\2',
-            webpage, 'video_url', group='file')
-        formats = [{'url': video_url}]
-        if video_url.startswith('rtmp://'):
-            rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+/))(?P<play_path>mp4:.+)$', video_url)
-            formats[0].update({
-                'url': rtmp.group('url'),
-                'ext': 'flv',
-                'app': rtmp.group('app'),
-                'play_path': rtmp.group('play_path'),
-                'page_url': url,
-            })
+
+        webpage = self._download_webpage(
+            'http://www.canalc2.tv/video/%s' % video_id, video_id)
+
+        formats = []
+        for _, video_url in re.findall(r'file\s*=\s*(["\'])(.+?)\1', webpage):
+            if video_url.startswith('rtmp://'):
+                rtmp = re.search(
+                    r'^(?P<url>rtmp://[^/]+/(?P<app>.+/))(?P<play_path>mp4:.+)$', video_url)
+                formats.append({
+                    'url': rtmp.group('url'),
+                    'format_id': 'rtmp',
+                    'ext': 'flv',
+                    'app': rtmp.group('app'),
+                    'play_path': rtmp.group('play_path'),
+                    'page_url': url,
+                })
+            else:
+                formats.append({
+                    'url': video_url,
+                    'format_id': 'http',
+                })
+        self._sort_formats(formats)
 
         title = self._html_search_regex(
             r'(?s)class="[^"]*col_description[^"]*">.*?<h3>(.*?)</h3>', webpage, 'title')

From 0a899a1448328648927e18e43c6e2274d2706396 Mon Sep 17 00:00:00 2001
From: Vignesh Venkat <vigneshacitizenofindia@gmail.com>
Date: Sat, 9 Jan 2016 15:31:50 -0800
Subject: [PATCH 0328/1105] [Bigflix] Add new extractor for bigflix.com

Add an IE to support bigflix.com. It uses some sort of silverlight
plugin whose video url is being populated using base64 encoded
flashvars. So it is quite straightforward to extract.
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/bigflix.py  | 42 ++++++++++++++++++++++++++++++++
 2 files changed, 43 insertions(+)
 create mode 100644 youtube_dl/extractor/bigflix.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a9d23b8f4..40c42d4d2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -61,6 +61,7 @@ from .beeg import BeegIE
 from .behindkink import BehindKinkIE
 from .beatportpro import BeatportProIE
 from .bet import BetIE
+from .bigflix import BigflixIE
 from .bild import BildIE
 from .bilibili import BiliBiliIE
 from .bleacherreport import (
diff --git a/youtube_dl/extractor/bigflix.py b/youtube_dl/extractor/bigflix.py
new file mode 100644
index 000000000..aeea1a002
--- /dev/null
+++ b/youtube_dl/extractor/bigflix.py
@@ -0,0 +1,42 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from base64 import b64decode
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote
+
+
+class BigflixIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?bigflix\.com/.*/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.bigflix.com/Hindi-movies/Action-movies/Singham-Returns/16537',
+        'md5': 'ec76aa9b1129e2e5b301a474e54fab74',
+        'info_dict': {
+            'id': '16537',
+            'ext': 'mp4',
+            'title': 'Singham Returns',
+            'description': 'md5:3d2ba5815f14911d5cc6a501ae0cf65d',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(
+            r'<div[^>]+class=["\']pagetitle["\'][^>]*>(.+?)</div>',
+            webpage, 'title')
+
+        video_url = b64decode(compat_urllib_parse_unquote(self._search_regex(
+            r'file=([^&]+)', webpage, 'video url')).encode('ascii')).decode('utf-8')
+
+        description = self._html_search_meta('description', webpage)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': video_url,
+            'description': description,
+        }

From 15b1c6656fd8b384bd974f4d943a04b8bf8ca915 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Jan 2016 10:03:56 +0600
Subject: [PATCH 0329/1105] Credit @vickyg3 for bigflix (#8194)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index e39d07efe..3d8bebbb0 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -151,3 +151,4 @@ Muratcan Simsek
 Evan Lu
 flatgreen
 Brian Foley
+Vignesh Venkat

From 6e99d5762a844b44cad4cae144045ed0537c084b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Jan 2016 10:31:36 +0600
Subject: [PATCH 0330/1105] [bigflix] Extract all formats

---
 youtube_dl/extractor/bigflix.py | 42 +++++++++++++++++++++++++++------
 1 file changed, 35 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/bigflix.py b/youtube_dl/extractor/bigflix.py
index aeea1a002..b7e498436 100644
--- a/youtube_dl/extractor/bigflix.py
+++ b/youtube_dl/extractor/bigflix.py
@@ -1,15 +1,16 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from base64 import b64decode
+import base64
+import re
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse_unquote
 
 
 class BigflixIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?bigflix\.com/.*/(?P<id>[0-9]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?bigflix\.com/.+/(?P<id>[0-9]+)'
+    _TESTS = [{
         'url': 'http://www.bigflix.com/Hindi-movies/Action-movies/Singham-Returns/16537',
         'md5': 'ec76aa9b1129e2e5b301a474e54fab74',
         'info_dict': {
@@ -18,7 +19,20 @@ class BigflixIE(InfoExtractor):
             'title': 'Singham Returns',
             'description': 'md5:3d2ba5815f14911d5cc6a501ae0cf65d',
         }
-    }
+    }, {
+        # multiple formats
+        'url': 'http://www.bigflix.com/Tamil-movies/Drama-movies/Madarasapatinam/16070',
+        'info_dict': {
+            'id': '16070',
+            'ext': 'mp4',
+            'title': 'Madarasapatinam',
+            'description': 'md5:63b9b8ed79189c6f0418c26d9a3452ca',
+            'formats': 'mincount:2',
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -29,14 +43,28 @@ class BigflixIE(InfoExtractor):
             r'<div[^>]+class=["\']pagetitle["\'][^>]*>(.+?)</div>',
             webpage, 'title')
 
-        video_url = b64decode(compat_urllib_parse_unquote(self._search_regex(
-            r'file=([^&]+)', webpage, 'video url')).encode('ascii')).decode('utf-8')
+        def decode_url(quoted_b64_url):
+            return base64.b64decode(compat_urllib_parse_unquote(
+                quoted_b64_url)).encode('ascii').decode('utf-8')
+
+        formats = [{
+            'url': decode_url(encoded_url),
+            'format_id': '%sp' % height,
+            'height': int(height),
+        } for height, encoded_url in re.findall(
+            r'ContentURL_(\d{3,4})[pP][^=]+=([^&]+)', webpage)]
+
+        if not formats:
+            formats.append({
+                'url': decode_url(self._search_regex(
+                    r'file=([^&]+)', webpage, 'video url')),
+            })
 
         description = self._html_search_meta('description', webpage)
 
         return {
             'id': video_id,
             'title': title,
-            'url': video_url,
             'description': description,
+            'formats': formats
         }

From a9bbd26f1d2bb45205f9fbd2626569522049e40e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Jan 2016 10:49:27 +0600
Subject: [PATCH 0331/1105] [bigflix] Improve formats extraction

---
 youtube_dl/extractor/bigflix.py | 41 ++++++++++++++++++++++-----------
 1 file changed, 28 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/bigflix.py b/youtube_dl/extractor/bigflix.py
index b7e498436..24720de86 100644
--- a/youtube_dl/extractor/bigflix.py
+++ b/youtube_dl/extractor/bigflix.py
@@ -20,7 +20,7 @@ class BigflixIE(InfoExtractor):
             'description': 'md5:3d2ba5815f14911d5cc6a501ae0cf65d',
         }
     }, {
-        # multiple formats
+        # 2 formats
         'url': 'http://www.bigflix.com/Tamil-movies/Drama-movies/Madarasapatinam/16070',
         'info_dict': {
             'id': '16070',
@@ -32,6 +32,10 @@ class BigflixIE(InfoExtractor):
         'params': {
             'skip_download': True,
         }
+    }, {
+        # multiple formats
+        'url': 'http://www.bigflix.com/Malayalam-movies/Drama-movies/Indian-Rupee/15967',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -45,20 +49,31 @@ class BigflixIE(InfoExtractor):
 
         def decode_url(quoted_b64_url):
             return base64.b64decode(compat_urllib_parse_unquote(
-                quoted_b64_url)).encode('ascii').decode('utf-8')
+                quoted_b64_url).encode('ascii')).decode('utf-8')
 
-        formats = [{
-            'url': decode_url(encoded_url),
-            'format_id': '%sp' % height,
-            'height': int(height),
-        } for height, encoded_url in re.findall(
-            r'ContentURL_(\d{3,4})[pP][^=]+=([^&]+)', webpage)]
+        formats = []
+        for height, encoded_url in re.findall(
+            r'ContentURL_(\d{3,4})[pP][^=]+=([^&]+)', webpage):
+            video_url = decode_url(encoded_url)
+            f = {
+                'url': video_url,
+                'format_id': '%sp' % height,
+                'height': int(height),
+            }
+            if video_url.startswith('rtmp'):
+                f['ext'] = 'flv'
+            formats.append(f)
 
-        if not formats:
-            formats.append({
-                'url': decode_url(self._search_regex(
-                    r'file=([^&]+)', webpage, 'video url')),
-            })
+        file_url = self._search_regex(
+            r'file=([^&]+)', webpage, 'video url', default=None)
+        if file_url:
+            video_url = decode_url(file_url)
+            if all(f['url'] != video_url for f in formats):
+                formats.append({
+                    'url': decode_url(file_url),
+                })
+
+        self._sort_formats(formats)
 
         description = self._html_search_meta('description', webpage)
 

From 3fc088f8c7f61026943a62fc28e051a7d3a6bdd5 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 10 Jan 2016 07:45:41 +0100
Subject: [PATCH 0332/1105] [dcn] extract video ids in season entries

---
 youtube_dl/extractor/dcn.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 8f48571de..15a1c40f7 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -5,7 +5,10 @@ import re
 import base64
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import (
+    compat_urllib_parse,
+    compat_str,
+)
 from ..utils import (
     int_or_none,
     parse_iso8601,
@@ -186,7 +189,8 @@ class DCNSeasonIE(InfoExtractor):
 
                 entries = []
                 for video in show['videos']:
+                    video_id = compat_str(video['id'])
                     entries.append(self.url_result(
-                        'http://www.dcndigital.ae/media/%s' % video['id'], 'DCNVideo'))
+                        'http://www.dcndigital.ae/media/%s' % video_id, 'DCNVideo', video_id))
 
                 return self.playlist_result(entries, season_id, title)

From 2334762b03dec5da4d6788539e3e11192eb97010 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 10 Jan 2016 07:55:58 +0100
Subject: [PATCH 0333/1105] [shahid] raise ExtractorError if the video is DRM
 protected

---
 youtube_dl/extractor/shahid.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index f76fb12c0..1178b7a27 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -73,6 +73,9 @@ class ShahidIE(InfoExtractor):
             'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-%s.type-%s.html'
             % (video_id, api_vars['type']), video_id, 'Downloading player JSON')
 
+        if player.get('drm'):
+            raise ExtractorError('This video is DRM protected.', expected=True)
+
         formats = self._extract_m3u8_formats(player['url'], video_id, 'mp4')
 
         video = self._download_json(

From 7e8a800f29d52cbc6057638595df05ac69f622bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 10 Jan 2016 14:26:27 +0100
Subject: [PATCH 0334/1105] [bigflix] Use correct indentation to make flake8
 happy

---
 youtube_dl/extractor/bigflix.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bigflix.py b/youtube_dl/extractor/bigflix.py
index 24720de86..33762ad93 100644
--- a/youtube_dl/extractor/bigflix.py
+++ b/youtube_dl/extractor/bigflix.py
@@ -53,7 +53,7 @@ class BigflixIE(InfoExtractor):
 
         formats = []
         for height, encoded_url in re.findall(
-            r'ContentURL_(\d{3,4})[pP][^=]+=([^&]+)', webpage):
+                r'ContentURL_(\d{3,4})[pP][^=]+=([^&]+)', webpage):
             video_url = decode_url(encoded_url)
             f = {
                 'url': video_url,

From 3c91e41614a0f3bfab8710e861c895b78d93bce8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 10 Jan 2016 14:32:53 +0100
Subject: [PATCH 0335/1105] [downloader/fragment] Don't fail if the
 'Content-Length' header is missing

In some dailymotion videos (like http://www.dailymotion.com/video/x3k0dtv from #8156) the segments URLs don't have the 'Content-Length' header and HttpFD sets the 'totat_bytes' field to None, so we also use '0' in that case (since we do different math operations with it).
---
 youtube_dl/downloader/fragment.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 5a64b29ee..3da554622 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -66,7 +66,7 @@ class FragmentFD(FileDownloader):
             if s['status'] not in ('downloading', 'finished'):
                 return
 
-            frag_total_bytes = s.get('total_bytes', 0)
+            frag_total_bytes = s.get('total_bytes') or 0
             if s['status'] == 'finished':
                 state['downloaded_bytes'] += frag_total_bytes
                 state['frag_index'] += 1

From dfb1b1468cef4ddc7ecc43776abce03763f8e426 Mon Sep 17 00:00:00 2001
From: Jakub Wilk <jwilk@jwilk.net>
Date: Sun, 10 Jan 2016 16:17:47 +0100
Subject: [PATCH 0336/1105] Fix typos

Closes #8200.
---
 devscripts/gh-pages/update-copyright.py | 2 +-
 test/test_write_annotations.py          | 2 +-
 youtube_dl/YoutubeDL.py                 | 2 +-
 youtube_dl/extractor/common.py          | 4 ++--
 youtube_dl/extractor/facebook.py        | 4 ++--
 youtube_dl/extractor/generic.py         | 4 ++--
 youtube_dl/extractor/ivi.py             | 2 +-
 youtube_dl/extractor/mdr.py             | 2 +-
 youtube_dl/extractor/nbc.py             | 2 +-
 youtube_dl/extractor/nhl.py             | 2 +-
 youtube_dl/extractor/ora.py             | 2 +-
 youtube_dl/extractor/testurl.py         | 2 +-
 youtube_dl/extractor/tv4.py             | 2 +-
 youtube_dl/extractor/videomore.py       | 2 +-
 youtube_dl/swfinterp.py                 | 2 +-
 youtube_dl/utils.py                     | 4 ++--
 16 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/devscripts/gh-pages/update-copyright.py b/devscripts/gh-pages/update-copyright.py
index 3663c8afe..e6c3abc8d 100755
--- a/devscripts/gh-pages/update-copyright.py
+++ b/devscripts/gh-pages/update-copyright.py
@@ -5,7 +5,7 @@ from __future__ import with_statement, unicode_literals
 
 import datetime
 import glob
-import io  # For Python 2 compatibilty
+import io  # For Python 2 compatibility
 import os
 import re
 
diff --git a/test/test_write_annotations.py b/test/test_write_annotations.py
index 84b8f39e0..8de08f2d6 100644
--- a/test/test_write_annotations.py
+++ b/test/test_write_annotations.py
@@ -66,7 +66,7 @@ class TestAnnotations(unittest.TestCase):
                 textTag = a.find('TEXT')
                 text = textTag.text
                 self.assertTrue(text in expected)  # assertIn only added in python 2.7
-                # remove the first occurance, there could be more than one annotation with the same text
+                # remove the first occurrence, there could be more than one annotation with the same text
                 expected.remove(text)
         # We should have seen (and removed) all the expected annotation texts.
         self.assertEqual(len(expected), 0, 'Not all expected annotations were found.')
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 3b2be3159..d50b7cfed 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1312,7 +1312,7 @@ class YoutubeDL(object):
             # only set the 'formats' fields if the original info_dict list them
             # otherwise we end up with a circular reference, the first (and unique)
             # element in the 'formats' field in info_dict is info_dict itself,
-            # wich can't be exported to json
+            # which can't be exported to json
             info_dict['formats'] = formats
         if self.params.get('listformats'):
             self.list_formats(info_dict)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 0719c7bcd..b05b22a94 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -313,9 +313,9 @@ class InfoExtractor(object):
         except ExtractorError:
             raise
         except compat_http_client.IncompleteRead as e:
-            raise ExtractorError('A network error has occured.', cause=e, expected=True)
+            raise ExtractorError('A network error has occurred.', cause=e, expected=True)
         except (KeyError, StopIteration) as e:
-            raise ExtractorError('An extractor error has occured.', cause=e)
+            raise ExtractorError('An extractor error has occurred.', cause=e)
 
     def set_downloader(self, downloader):
         """Sets the downloader for this IE."""
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 5e43f2359..ec699ba54 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -105,7 +105,7 @@ class FacebookIE(InfoExtractor):
                     login_results, 'login error', default=None, group='error')
                 if error:
                     raise ExtractorError('Unable to login: %s' % error, expected=True)
-                self._downloader.report_warning('unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
+                self._downloader.report_warning('unable to log in: bad username/password, or exceeded login rate limit (~3/min). Check credentials or wait.')
                 return
 
             fb_dtsg = self._search_regex(
@@ -126,7 +126,7 @@ class FacebookIE(InfoExtractor):
             check_response = self._download_webpage(check_req, None,
                                                     note='Confirming login')
             if re.search(r'id="checkpointSubmitButton"', check_response) is not None:
-                self._downloader.report_warning('Unable to confirm login, you have to login in your brower and authorize the login.')
+                self._downloader.report_warning('Unable to confirm login, you have to login in your browser and authorize the login.')
         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
             self._downloader.report_warning('unable to log in: %s' % error_to_compat_str(err))
             return
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index d79e1adc9..b3f8efc80 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -487,7 +487,7 @@ class GenericIE(InfoExtractor):
                 'description': 'md5:8145d19d320ff3e52f28401f4c4283b9',
             }
         },
-        # Embeded Ustream video
+        # Embedded Ustream video
         {
             'url': 'http://www.american.edu/spa/pti/nsa-privacy-janus-2014.cfm',
             'md5': '27b99cdb639c9b12a79bca876a073417',
@@ -1644,7 +1644,7 @@ class GenericIE(InfoExtractor):
         if myvi_url:
             return self.url_result(myvi_url)
 
-        # Look for embeded soundcloud player
+        # Look for embedded soundcloud player
         mobj = re.search(
             r'<iframe\s+(?:[a-zA-Z0-9_-]+="[^"]+"\s+)*src="(?P<url>https?://(?:w\.)?soundcloud\.com/player[^"]+)"',
             webpage)
diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py
index d0f00cdea..472d72b4c 100644
--- a/youtube_dl/extractor/ivi.py
+++ b/youtube_dl/extractor/ivi.py
@@ -32,7 +32,7 @@ class IviIE(InfoExtractor):
             },
             'skip': 'Only works from Russia',
         },
-        # Serial's serie
+        # Serial's series
         {
             'url': 'http://www.ivi.ru/watch/dvoe_iz_lartsa/9549',
             'md5': '221f56b35e3ed815fde2df71032f4b3e',
diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index 88334889e..425fc9e2a 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -17,7 +17,7 @@ class MDRIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?(?:mdr|kika)\.de/(?:.*)/[a-z]+(?P<id>\d+)(?:_.+?)?\.html'
 
     _TESTS = [{
-        # MDR regularily deletes its videos
+        # MDR regularly deletes its videos
         'url': 'http://www.mdr.de/fakt/video189002.html',
         'only_matching': True,
     }, {
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 340c922bd..1dd54c2f1 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -100,7 +100,7 @@ class NBCSportsVPlayerIE(InfoExtractor):
 
 
 class NBCSportsIE(InfoExtractor):
-    # Does not include https becuase its certificate is invalid
+    # Does not include https because its certificate is invalid
     _VALID_URL = r'http://www\.nbcsports\.com//?(?:[^/]+/)+(?P<id>[0-9a-z-]+)'
 
     _TEST = {
diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index e98a5ef89..8d5ce46ad 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -223,7 +223,7 @@ class NHLVideocenterIE(NHLBaseInfoExtractor):
         response = self._download_webpage(request_url, playlist_title)
         response = self._fix_json(response)
         if not response.strip():
-            self._downloader.report_warning('Got an empty reponse, trying '
+            self._downloader.report_warning('Got an empty response, trying '
                                             'adding the "newvideos" parameter')
             response = self._download_webpage(request_url + '&newvideos=true',
                                               playlist_title)
diff --git a/youtube_dl/extractor/ora.py b/youtube_dl/extractor/ora.py
index 9c4255a2d..02de1502a 100644
--- a/youtube_dl/extractor/ora.py
+++ b/youtube_dl/extractor/ora.py
@@ -37,7 +37,7 @@ class OraTVIE(InfoExtractor):
             formats = self._extract_m3u8_formats(
                 m3u8_url, display_id, 'mp4', 'm3u8_native',
                 m3u8_id='hls', fatal=False)
-            # simular to GameSpotIE
+            # similar to GameSpotIE
             m3u8_path = compat_urlparse.urlparse(m3u8_url).path
             QUALITIES_RE = r'((,[a-z]+\d+)+,?)'
             available_qualities = self._search_regex(
diff --git a/youtube_dl/extractor/testurl.py b/youtube_dl/extractor/testurl.py
index c7d559315..46918adb0 100644
--- a/youtube_dl/extractor/testurl.py
+++ b/youtube_dl/extractor/testurl.py
@@ -7,7 +7,7 @@ from ..utils import ExtractorError
 
 
 class TestURLIE(InfoExtractor):
-    """ Allows adressing of the test cases as test:yout.*be_1 """
+    """ Allows addressing of the test cases as test:yout.*be_1 """
 
     IE_DESC = False  # Do not list
     _VALID_URL = r'test(?:url)?:(?P<id>(?P<extractor>.+?)(?:_(?P<num>[0-9]+))?)$'
diff --git a/youtube_dl/extractor/tv4.py b/youtube_dl/extractor/tv4.py
index 1c4b6d635..343edf206 100644
--- a/youtube_dl/extractor/tv4.py
+++ b/youtube_dl/extractor/tv4.py
@@ -67,7 +67,7 @@ class TV4IE(InfoExtractor):
         info = self._download_json(
             'http://www.tv4play.se/player/assets/%s.json' % video_id, video_id, 'Downloading video info JSON')
 
-        # If is_geo_restricted is true, it doesn't neceserally mean we can't download it
+        # If is_geo_restricted is true, it doesn't necessarily mean we can't download it
         if info['is_geo_restricted']:
             self.report_warning('This content might not be available in your country due to licensing restrictions.')
         if info['requires_subscription']:
diff --git a/youtube_dl/extractor/videomore.py b/youtube_dl/extractor/videomore.py
index a66d6de23..fcee940e6 100644
--- a/youtube_dl/extractor/videomore.py
+++ b/youtube_dl/extractor/videomore.py
@@ -170,7 +170,7 @@ class VideomoreVideoIE(InfoExtractor):
             'skip_download': True,
         },
     }, {
-        # season single serie with og:video:iframe
+        # season single series with og:video:iframe
         'url': 'http://videomore.ru/poslednii_ment/1_sezon/14_seriya',
         'only_matching': True,
     }, {
diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index e60505ace..06c1d6cc1 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -689,7 +689,7 @@ class SWFInterpreter(object):
                     elif mname in _builtin_classes:
                         res = _builtin_classes[mname]
                     else:
-                        # Assume unitialized
+                        # Assume uninitialized
                         # TODO warn here
                         res = undefined
                     stack.append(res)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index da4ec7f20..9c1c0e0bd 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -984,7 +984,7 @@ def date_from_str(date_str):
         if sign == '-':
             time = -time
         unit = match.group('unit')
-        # A bad aproximation?
+        # A bad approximation?
         if unit == 'month':
             unit = 'day'
             time *= 30
@@ -1307,7 +1307,7 @@ def parse_filesize(s):
     if s is None:
         return None
 
-    # The lower-case forms are of course incorrect and inofficial,
+    # The lower-case forms are of course incorrect and unofficial,
     # but we support those too
     _UNIT_TABLE = {
         'B': 1,

From 36a0e46c39ea4f211dea9944177976e8f8364736 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 10 Jan 2016 17:55:41 +0100
Subject: [PATCH 0337/1105] fix typos

---
 youtube_dl/YoutubeDL.py | 4 ++--
 youtube_dl/utils.py     | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d50b7cfed..6afc1b730 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -241,7 +241,7 @@ class YoutubeDL(object):
                        - "detect_or_warn": check whether we can do anything
                                            about it, warn otherwise (default)
     source_address:    (Experimental) Client-side IP address to bind to.
-    call_home:         Boolean, true iff we are allowed to contact the
+    call_home:         Boolean, true if we are allowed to contact the
                        youtube-dl servers for debugging.
     sleep_interval:    Number of seconds to sleep before each download.
     listformats:       Print an overview of available video formats and exit.
@@ -590,7 +590,7 @@ class YoutubeDL(object):
             return None
 
     def _match_entry(self, info_dict, incomplete):
-        """ Returns None iff the file should be downloaded """
+        """ Returns None if the file should be downloaded """
 
         video_title = info_dict.get('title', info_dict.get('id', 'video'))
         if 'title' in info_dict:
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 9c1c0e0bd..e583299c5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1857,7 +1857,7 @@ def encode_data_uri(data, mime_type):
 
 
 def age_restricted(content_limit, age_limit):
-    """ Returns True iff the content should be blocked """
+    """ Returns True if the content should be blocked """
 
     if age_limit is None:  # No limit set
         return False

From 6ec6cb4e956abbd76a6ff42336821770a5fbbcc7 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 10 Jan 2016 19:27:22 +0100
Subject: [PATCH 0338/1105] Revert "fix typos"

This reverts commit 36a0e46c39ea4f211dea9944177976e8f8364736.
---
 youtube_dl/YoutubeDL.py | 4 ++--
 youtube_dl/utils.py     | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 6afc1b730..d50b7cfed 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -241,7 +241,7 @@ class YoutubeDL(object):
                        - "detect_or_warn": check whether we can do anything
                                            about it, warn otherwise (default)
     source_address:    (Experimental) Client-side IP address to bind to.
-    call_home:         Boolean, true if we are allowed to contact the
+    call_home:         Boolean, true iff we are allowed to contact the
                        youtube-dl servers for debugging.
     sleep_interval:    Number of seconds to sleep before each download.
     listformats:       Print an overview of available video formats and exit.
@@ -590,7 +590,7 @@ class YoutubeDL(object):
             return None
 
     def _match_entry(self, info_dict, incomplete):
-        """ Returns None if the file should be downloaded """
+        """ Returns None iff the file should be downloaded """
 
         video_title = info_dict.get('title', info_dict.get('id', 'video'))
         if 'title' in info_dict:
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e583299c5..9c1c0e0bd 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1857,7 +1857,7 @@ def encode_data_uri(data, mime_type):
 
 
 def age_restricted(content_limit, age_limit):
-    """ Returns True if the content should be blocked """
+    """ Returns True iff the content should be blocked """
 
     if age_limit is None:  # No limit set
         return False

From 192b9a571cd50b0fd924e9ecb926b28633e8c19c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 Jan 2016 21:56:19 +0600
Subject: [PATCH 0339/1105] [videomega] Mark broken

---
 youtube_dl/extractor/videomega.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index 87aca327b..5e2e7cbac 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -8,6 +8,7 @@ from ..utils import sanitized_Request
 
 
 class VideoMegaIE(InfoExtractor):
+    _WORKING = False
     _VALID_URL = r'(?:videomega:|https?://(?:www\.)?videomega\.tv/(?:(?:view|iframe|cdn)\.php)?\?ref=)(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA',

From b924bfad684ac7e9209e7165df3e13af6545596d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 Jan 2016 21:58:32 +0600
Subject: [PATCH 0340/1105] [videott] Mark broken

---
 youtube_dl/extractor/videott.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/videott.py b/youtube_dl/extractor/videott.py
index 591024ead..2cd36508a 100644
--- a/youtube_dl/extractor/videott.py
+++ b/youtube_dl/extractor/videott.py
@@ -11,6 +11,7 @@ from ..utils import (
 
 
 class VideoTtIE(InfoExtractor):
+    _WORKING = False
     ID_NAME = 'video.tt'
     IE_DESC = 'video.tt - Your True Tube'
     _VALID_URL = r'http://(?:www\.)?video\.tt/(?:(?:video|embed)/|watch_video\.php\?v=)(?P<id>[\da-zA-Z]{9})'

From be27283ef68e96d2461875a2bfb8c9ce962cfe61 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 Jan 2016 22:00:17 +0600
Subject: [PATCH 0341/1105] [iprima] Mark broken

---
 youtube_dl/extractor/iprima.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 36baf3245..073777f34 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -14,6 +14,7 @@ from ..utils import (
 
 
 class IPrimaIE(InfoExtractor):
+    _WORKING = False
     _VALID_URL = r'https?://play\.iprima\.cz/(?:[^/]+/)*(?P<id>[^?#]+)'
 
     _TESTS = [{

From 9cb1a06b6c8fbeb6cfdbf0533ea60a6624fa1246 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 12 Jan 2016 22:09:38 +0600
Subject: [PATCH 0342/1105] [downloader/fragment] Remove unused code and fix
 zero division error

---
 youtube_dl/downloader/fragment.py | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 3da554622..d236ac737 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -78,17 +78,10 @@ class FragmentFD(FileDownloader):
             state['total_bytes_estimate'] = estimated_size
             state['elapsed'] = time_now - start
 
-            if s['status'] == 'finished':
-                progress = self.calc_percent(state['frag_index'], total_frags)
-            else:
-                frag_downloaded_bytes = s['downloaded_bytes']
-                frag_progress = self.calc_percent(frag_downloaded_bytes,
-                                                  frag_total_bytes)
-                progress = self.calc_percent(state['frag_index'], total_frags)
-                progress += frag_progress / float(total_frags)
-
+            if s['status'] != 'finished':
                 state['eta'] = self.calc_eta(
-                    start, time_now, estimated_size, state['downloaded_bytes'] + frag_downloaded_bytes)
+                    start, time_now, estimated_size,
+                    state['downloaded_bytes'] + s['downloaded_bytes'])
                 state['speed'] = s.get('speed')
             self._hook_progress(state)
 

From 709185a2648c991cc143272dda67e4b42e89c03b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 12 Jan 2016 23:18:38 +0600
Subject: [PATCH 0343/1105] [downloader/fragment] More smooth calculations

`downloaded_bytes` is now updated on each fragment progress hook invocation
---
 youtube_dl/downloader/fragment.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index d236ac737..b2597f1e5 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -58,6 +58,11 @@ class FragmentFD(FileDownloader):
             'frag_count': total_frags,
             'filename': ctx['filename'],
             'tmpfilename': ctx['tmpfilename'],
+            # Total complete fragments downloaded so far in bytes
+            '_complete_frags_downloaded_bytes': 0,
+            # Amount of fragment's bytes downloaded by the time of the previous
+            # frag progress hook invocation
+            '_prev_frag_downloaded_bytes': 0,
         }
         start = time.time()
         ctx['started'] = start
@@ -67,22 +72,27 @@ class FragmentFD(FileDownloader):
                 return
 
             frag_total_bytes = s.get('total_bytes') or 0
-            if s['status'] == 'finished':
-                state['downloaded_bytes'] += frag_total_bytes
-                state['frag_index'] += 1
 
             estimated_size = (
-                (state['downloaded_bytes'] + frag_total_bytes) /
+                (state['_complete_frags_downloaded_bytes'] + frag_total_bytes) /
                 (state['frag_index'] + 1) * total_frags)
             time_now = time.time()
             state['total_bytes_estimate'] = estimated_size
             state['elapsed'] = time_now - start
 
-            if s['status'] != 'finished':
+            if s['status'] == 'finished':
+                state['frag_index'] += 1
+                state['downloaded_bytes'] += frag_total_bytes - state['_prev_frag_downloaded_bytes']
+                state['_complete_frags_downloaded_bytes'] = state['downloaded_bytes']
+                state['_prev_frag_downloaded_bytes'] = 0
+            else:
+                frag_downloaded_bytes = s['downloaded_bytes']
+                state['downloaded_bytes'] += frag_downloaded_bytes - state['_prev_frag_downloaded_bytes']
                 state['eta'] = self.calc_eta(
                     start, time_now, estimated_size,
-                    state['downloaded_bytes'] + s['downloaded_bytes'])
+                    state['downloaded_bytes'])
                 state['speed'] = s.get('speed')
+                state['_prev_frag_downloaded_bytes'] = frag_downloaded_bytes
             self._hook_progress(state)
 
         ctx['dl'].add_progress_hook(frag_progress_hook)

From 16a348475cb7ab90e80c31a01eec1280f4fc4fcb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 12 Jan 2016 23:23:39 +0600
Subject: [PATCH 0344/1105] [dailymotion] Prefer direct links (Closes #8156)

---
 youtube_dl/extractor/dailymotion.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 439fd42e8..b687ec4d6 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -149,14 +149,15 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                     ext = determine_ext(media_url)
                     if type_ == 'application/x-mpegURL' or ext == 'm3u8':
                         formats.extend(self._extract_m3u8_formats(
-                            media_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
+                            media_url, video_id, 'mp4', preference=-1,
+                            m3u8_id='hls', fatal=False))
                     elif type_ == 'application/f4m' or ext == 'f4m':
                         formats.extend(self._extract_f4m_formats(
                             media_url, video_id, preference=-1, f4m_id='hds', fatal=False))
                     else:
                         f = {
                             'url': media_url,
-                            'format_id': quality,
+                            'format_id': 'http-%s' % quality,
                         }
                         m = re.search(r'H264-(?P<width>\d+)x(?P<height>\d+)', media_url)
                         if m:

From b83b782dc4afeabbd356a329fd302b4410afa626 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 13 Jan 2016 00:00:31 +0600
Subject: [PATCH 0345/1105] [downloader/fragment] Move helper data to context
 dict

---
 youtube_dl/downloader/fragment.py | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index b2597f1e5..0c9113d0f 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -58,14 +58,17 @@ class FragmentFD(FileDownloader):
             'frag_count': total_frags,
             'filename': ctx['filename'],
             'tmpfilename': ctx['tmpfilename'],
+        }
+
+        start = time.time()
+        ctx.update({
+            'started': start,
             # Total complete fragments downloaded so far in bytes
-            '_complete_frags_downloaded_bytes': 0,
+            'complete_frags_downloaded_bytes': 0,
             # Amount of fragment's bytes downloaded by the time of the previous
             # frag progress hook invocation
-            '_prev_frag_downloaded_bytes': 0,
-        }
-        start = time.time()
-        ctx['started'] = start
+            'prev_frag_downloaded_bytes': 0,
+        })
 
         def frag_progress_hook(s):
             if s['status'] not in ('downloading', 'finished'):
@@ -74,7 +77,7 @@ class FragmentFD(FileDownloader):
             frag_total_bytes = s.get('total_bytes') or 0
 
             estimated_size = (
-                (state['_complete_frags_downloaded_bytes'] + frag_total_bytes) /
+                (ctx['complete_frags_downloaded_bytes'] + frag_total_bytes) /
                 (state['frag_index'] + 1) * total_frags)
             time_now = time.time()
             state['total_bytes_estimate'] = estimated_size
@@ -82,17 +85,17 @@ class FragmentFD(FileDownloader):
 
             if s['status'] == 'finished':
                 state['frag_index'] += 1
-                state['downloaded_bytes'] += frag_total_bytes - state['_prev_frag_downloaded_bytes']
-                state['_complete_frags_downloaded_bytes'] = state['downloaded_bytes']
-                state['_prev_frag_downloaded_bytes'] = 0
+                state['downloaded_bytes'] += frag_total_bytes - ctx['prev_frag_downloaded_bytes']
+                ctx['complete_frags_downloaded_bytes'] = state['downloaded_bytes']
+                ctx['prev_frag_downloaded_bytes'] = 0
             else:
                 frag_downloaded_bytes = s['downloaded_bytes']
-                state['downloaded_bytes'] += frag_downloaded_bytes - state['_prev_frag_downloaded_bytes']
+                state['downloaded_bytes'] += frag_downloaded_bytes - ctx['prev_frag_downloaded_bytes']
                 state['eta'] = self.calc_eta(
                     start, time_now, estimated_size,
                     state['downloaded_bytes'])
                 state['speed'] = s.get('speed')
-                state['_prev_frag_downloaded_bytes'] = frag_downloaded_bytes
+                ctx['prev_frag_downloaded_bytes'] = frag_downloaded_bytes
             self._hook_progress(state)
 
         ctx['dl'].add_progress_hook(frag_progress_hook)

From bc0550c262db0bc756bb7e08fb945f3285fb196e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 13 Jan 2016 08:18:37 +0600
Subject: [PATCH 0346/1105] [pluralsight] Fix new player (Closes #8215)

---
 youtube_dl/extractor/pluralsight.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 55c11b3bf..12e1c2862 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -232,7 +232,7 @@ class PluralsightIE(PluralsightBaseIE):
         # { a = author, cn = clip_id, lc = end, m = name }
 
         return {
-            'id': clip['clipName'],
+            'id': clip.get('clipName') or clip['name'],
             'title': '%s - %s' % (module['title'], clip['title']),
             'duration': int_or_none(clip.get('duration')) or parse_duration(clip.get('formattedDuration')),
             'creator': author,

From cc28492d31556b55bbd8fc574bf5d890305b22d2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 13 Jan 2016 17:05:38 +0800
Subject: [PATCH 0347/1105] [youtube] Fix acodec and vcodec order

In RFC6381, there's no rule stating that the first part of codecs should
be video and the second part should be audio, while it seems the case
for data reported by YouTube.
---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 897641e42..e4f227f19 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1487,7 +1487,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                             if codecs:
                                 codecs = codecs.split(',')
                                 if len(codecs) == 2:
-                                    acodec, vcodec = codecs[0], codecs[1]
+                                    acodec, vcodec = codecs[1], codecs[0]
                                 else:
                                     acodec, vcodec = (codecs[0], 'none') if kind == 'audio' else ('none', codecs[0])
                                 dct.update({

From 40cf7fcbd2e30747065ca7b8bf4467a9582a4aa9 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 13 Jan 2016 13:29:00 +0100
Subject: [PATCH 0348/1105] [tudou] Add support for Albums and Playlists and
 extract more metadata

---
 youtube_dl/extractor/__init__.py |  6 +-
 youtube_dl/extractor/tudou.py    | 94 ++++++++++++++++++++++++++------
 2 files changed, 81 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 40c42d4d2..e4ae9332d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -723,7 +723,11 @@ from .trilulilu import TriluliluIE
 from .trutube import TruTubeIE
 from .tube8 import Tube8IE
 from .tubitv import TubiTvIE
-from .tudou import TudouIE
+from .tudou import (
+    TudouIE,
+    TudouPlaylistIE,
+    TudouAlbumIE,
+)
 from .tumblr import TumblrIE
 from .tunein import (
     TuneInClipIE,
diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index 5f7ac4b35..da3cd76f7 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -4,10 +4,16 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    float_or_none,
+    unescapeHTML,
+)
 
 
 class TudouIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:listplay|programs(?:/view)?|albumplay)/([^/]+/)*(?P<id>[^/?#]+?)(?:\.html)?/?(?:$|[?#])'
+    IE_NAME = 'tudou'
+    _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:(?:programs|wlplay)/view|(?:listplay|albumplay)/[\w-]{11})/(?P<id>[\w-]{11})'
     _TESTS = [{
         'url': 'http://www.tudou.com/listplay/zzdE77v6Mmo/2xN2duXMxmw.html',
         'md5': '140a49ed444bd22f93330985d8475fcb',
@@ -16,6 +22,11 @@ class TudouIE(InfoExtractor):
             'ext': 'f4v',
             'title': '卡马乔国足开大脚长传冲吊集锦',
             'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1372113489000,
+            'description': '卡马乔卡家军，开大脚先进战术不完全集锦！',
+            'duration': 289.04,
+            'view_count': int,
+            'filesize': int,
         }
     }, {
         'url': 'http://www.tudou.com/programs/view/ajX3gyhL0pc/',
@@ -24,10 +35,12 @@ class TudouIE(InfoExtractor):
             'ext': 'f4v',
             'title': 'La Sylphide-Bolshoi-Ekaterina Krysanova & Vyacheslav Lopatin 2012',
             'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1349207518000,
+            'description': 'md5:294612423894260f2dcd5c6c04fe248b',
+            'duration': 5478.33,
+            'view_count': int,
+            'filesize': int,
         }
-    }, {
-        'url': 'http://www.tudou.com/albumplay/cJAHGih4yYg.html',
-        'only_matching': True,
     }]
 
     _PLAYER_URL = 'http://js.tudouui.com/bin/lingtong/PortalPlayer_177.swf'
@@ -42,24 +55,20 @@ class TudouIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        item_data = self._download_json(
+            'http://www.tudou.com/tvp/getItemInfo.action?ic=%s' % video_id, video_id)
 
-        youku_vcode = self._search_regex(
-            r'vcode\s*:\s*[\'"]([^\'"]*)[\'"]', webpage, 'youku vcode', default=None)
+        youku_vcode = item_data.get('vcode')
         if youku_vcode:
             return self.url_result('youku:' + youku_vcode, ie='Youku')
 
-        title = self._search_regex(
-            r',kw\s*:\s*[\'"]([^\'"]+)[\'"]', webpage, 'title')
-        thumbnail_url = self._search_regex(
-            r',pic\s*:\s*[\'"]([^\'"]+)[\'"]', webpage, 'thumbnail URL', fatal=False)
+        title = unescapeHTML(item_data['kw'])
+        description = item_data.get('desc')
+        thumbnail_url = item_data.get('pic')
+        view_count = int_or_none(item_data.get('playTimes'))
+        timestamp = int_or_none(item_data.get('pt'))
 
-        player_url = self._search_regex(
-            r'playerUrl\s*:\s*[\'"]([^\'"]+\.swf)[\'"]',
-            webpage, 'player URL', default=self._PLAYER_URL)
-
-        segments = self._parse_json(self._search_regex(
-            r'segs: \'([^\']+)\'', webpage, 'segments'), video_id)
+        segments = self._parse_json(item_data['itemSegs'], video_id)
         # It looks like the keys are the arguments that have to be passed as
         # the hd field in the request url, we pick the higher
         # Also, filter non-number qualities (see issue #3643).
@@ -80,8 +89,13 @@ class TudouIE(InfoExtractor):
                 'ext': ext,
                 'title': title,
                 'thumbnail': thumbnail_url,
+                'description': description,
+                'view_count': view_count,
+                'timestamp': timestamp,
+                'duration': float_or_none(part.get('seconds'), 1000),
+                'filesize': int_or_none(part.get('size')),
                 'http_headers': {
-                    'Referer': player_url,
+                    'Referer': self._PLAYER_URL,
                 },
             }
             result.append(part_info)
@@ -92,3 +106,47 @@ class TudouIE(InfoExtractor):
             'id': video_id,
             'title': title,
         }
+
+
+class TudouPlaylistIE(InfoExtractor):
+    IE_NAME = 'tudou:playlist'
+    _VALID_URL = r'https?://(?:www\.)?tudou\.com/listplay/(?P<id>[\w-]{11})\.html'
+    _TESTS = [{
+        'url': 'http://www.tudou.com/listplay/zzdE77v6Mmo.html',
+        'info_dict': {
+            'id': 'zzdE77v6Mmo',
+        },
+        'playlist_mincount': 209,
+    }]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        playlist_data = self._download_json(
+            'http://www.tudou.com/tvp/plist.action?lcode=%s' % playlist_id, playlist_id)
+        entries = [self.url_result(
+            'http://www.tudou.com/programs/view/%s' % item['icode'],
+            'Tudou', item['icode'],
+            item['kw']) for item in playlist_data['items']]
+        return self.playlist_result(entries, playlist_id)
+
+
+class TudouAlbumIE(InfoExtractor):
+    IE_NAME = 'tudou:album'
+    _VALID_URL = r'https?://(?:www\.)?tudou\.com/album(?:cover|play)/(?P<id>[\w-]{11})'
+    _TESTS = [{
+        'url': 'http://www.tudou.com/albumplay/v5qckFJvNJg.html',
+        'info_dict': {
+            'id': 'v5qckFJvNJg',
+        },
+        'playlist_mincount': 45,
+    }]
+
+    def _real_extract(self, url):
+        album_id = self._match_id(url)
+        album_data = self._download_json(
+            'http://www.tudou.com/tvp/alist.action?acode=%s' % album_id, album_id)
+        entries = [self.url_result(
+            'http://www.tudou.com/programs/view/%s' % item['icode'],
+            'Tudou', item['icode'],
+            item['kw']) for item in album_data['items']]
+        return self.playlist_result(entries, album_id)

From e37afbe0b8a1222cb214ad0bec9a53bb7953531d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 14 Jan 2016 00:16:23 +0100
Subject: [PATCH 0349/1105] [YoutubeDL] urlopen: disable the 'file:' protocol
 (#8227)

If someone is running youtube-dl on a server to deliver files, the user could input 'file:///some/important/file' and youtube-dl would save that file as a video giving access to sensitive information to the user.
'file:' urls can be filtered, but the user can use an URL to a crafted m3u8 manifest like:

    #EXTM3U
    #EXT-X-MEDIA-SEQUENCE:0
    #EXTINF:10.0
    file:///etc/passwd
    #EXT-X-ENDLIST

With this patch 'file:' URLs raise URLError like for unknown protocols.
---
 test/test_YoutubeDL.py  |  7 ++++++-
 youtube_dl/YoutubeDL.py | 10 ++++++++--
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 0388c0bf3..0caa43843 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -12,7 +12,7 @@ import copy
 
 from test.helper import FakeYDL, assertRegexpMatches
 from youtube_dl import YoutubeDL
-from youtube_dl.compat import compat_str
+from youtube_dl.compat import compat_str, compat_urllib_error
 from youtube_dl.extractor import YoutubeIE
 from youtube_dl.postprocessor.common import PostProcessor
 from youtube_dl.utils import ExtractorError, match_filter_func
@@ -631,6 +631,11 @@ class TestYoutubeDL(unittest.TestCase):
         result = get_ids({'playlist_items': '10'})
         self.assertEqual(result, [])
 
+    def test_urlopen_no_file_protocol(self):
+        # see https://github.com/rg3/youtube-dl/issues/8227
+        ydl = YDL()
+        self.assertRaises(compat_urllib_error.URLError, ydl.urlopen, 'file:///etc/passwd')
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d50b7cfed..e8ce58604 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1986,8 +1986,14 @@ class YoutubeDL(object):
         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
         data_handler = compat_urllib_request_DataHandler()
-        opener = compat_urllib_request.build_opener(
-            proxy_handler, https_handler, cookie_processor, ydlh, data_handler)
+        unknown_handler = compat_urllib_request.UnknownHandler()
+        handlers = (proxy_handler, https_handler, cookie_processor, ydlh, data_handler, unknown_handler)
+        # we don't use build_opener because it automatically adds FileHandler,
+        # which can be used for malicious purposes (see
+        # https://github.com/rg3/youtube-dl/issues/8227)
+        opener = compat_urllib_request.OpenerDirector()
+        for handler in handlers:
+            opener.add_handler(handler)
 
         # Delete the default user-agent header, which would otherwise apply in
         # cases where our custom HTTP handler doesn't come into play

From 6240b0a278781a3b584a9dd6d57191b2472c0fd8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 14 Jan 2016 08:14:01 +0100
Subject: [PATCH 0350/1105] [YoutubeDL] urlopen: use build_opener again

Otherwise we would need to manually add handlers like HTTPRedirectHandler, instead we add a customized FileHandler instance that raises an error.
---
 youtube_dl/YoutubeDL.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e8ce58604..ccad5f2ea 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1986,14 +1986,19 @@ class YoutubeDL(object):
         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
         data_handler = compat_urllib_request_DataHandler()
-        unknown_handler = compat_urllib_request.UnknownHandler()
-        handlers = (proxy_handler, https_handler, cookie_processor, ydlh, data_handler, unknown_handler)
-        # we don't use build_opener because it automatically adds FileHandler,
-        # which can be used for malicious purposes (see
+
+        # When passing our own FileHandler instance, build_opener won't add the
+        # default FileHandler and allows us to disable the file protocol, which
+        # can be used for malicious purposes (see
         # https://github.com/rg3/youtube-dl/issues/8227)
-        opener = compat_urllib_request.OpenerDirector()
-        for handler in handlers:
-            opener.add_handler(handler)
+        file_handler = compat_urllib_request.FileHandler()
+
+        def file_open(*args, **kwargs):
+            raise compat_urllib_error.URLError('file protocol is disabled')
+        file_handler.file_open = file_open
+
+        opener = compat_urllib_request.build_opener(
+            proxy_handler, https_handler, cookie_processor, ydlh, data_handler, file_handler)
 
         # Delete the default user-agent header, which would otherwise apply in
         # cases where our custom HTTP handler doesn't come into play

From 4240d504963bb6d1c7bd7c288a7874f9d8dc042b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 14 Jan 2016 14:07:54 +0100
Subject: [PATCH 0351/1105] [YoutubeDL] improve error message for file:/// URLs

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ccad5f2ea..4915fbd45 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1994,7 +1994,7 @@ class YoutubeDL(object):
         file_handler = compat_urllib_request.FileHandler()
 
         def file_open(*args, **kwargs):
-            raise compat_urllib_error.URLError('file protocol is disabled')
+            raise compat_urllib_error.URLError('file:/// protocol is explicitly disabled in youtube-dl for security reasons')
         file_handler.file_open = file_open
 
         opener = compat_urllib_request.build_opener(

From 4511c1976d0a06394a000333a020a4d3668072fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2016 19:57:20 +0600
Subject: [PATCH 0352/1105] [beeg] Fix extraction (Closes #8225)

---
 youtube_dl/extractor/beeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index c8d921daf..d0174b818 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -60,7 +60,7 @@ class BeegIE(InfoExtractor):
 
         def decrypt_url(encrypted_url):
             encrypted_url = self._proto_relative_url(
-                encrypted_url.replace('{DATA_MARKERS}', ''), 'http:')
+                encrypted_url.replace('{DATA_MARKERS}', ''), 'https:')
             key = self._search_regex(
                 r'/key=(.*?)%2Cend=', encrypted_url, 'key', default=None)
             if not key:

From abb893e6e45b0b0c6ec0e3a1d29dbd1746cbee96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2016 19:57:56 +0600
Subject: [PATCH 0353/1105] [beeg] Update API URL

---
 youtube_dl/extractor/beeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index d0174b818..34c2a756f 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -34,7 +34,7 @@ class BeegIE(InfoExtractor):
         video_id = self._match_id(url)
 
         video = self._download_json(
-            'http://beeg.com/api/v5/video/%s' % video_id, video_id)
+            'https://api.beeg.com/api/v5/video/%s' % video_id, video_id)
 
         def split(o, e):
             def cut(s, x):

From 11c60089a8772a2d12288f0ff382866e516f9a4b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 14 Jan 2016 15:43:21 +0100
Subject: [PATCH 0354/1105] release 2016.01.14

---
 docs/supportedsites.md | 11 +++++++----
 youtube_dl/version.py  |  2 +-
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 8d0c7b97a..eb160bd2f 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -65,6 +65,7 @@
  - **Beeg**
  - **BehindKink**
  - **Bet**
+ - **Bigflix**
  - **Bild**: Bild.de
  - **BiliBili**
  - **BleacherReport**
@@ -251,7 +252,7 @@
  - **Instagram**
  - **instagram:user**: Instagram user profile
  - **InternetVideoArchive**
- - **IPrima**
+ - **IPrima** (Currently broken)
  - **iqiyi**: 爱奇艺
  - **Ir90Tv**
  - **ivi**: ivi.ru
@@ -602,7 +603,9 @@
  - **TruTube**
  - **Tube8**
  - **TubiTv**
- - **Tudou**
+ - **tudou**
+ - **tudou:album**
+ - **tudou:playlist**
  - **Tumblr**
  - **tunein:clip**
  - **tunein:program**
@@ -655,12 +658,12 @@
  - **video.mit.edu**
  - **VideoDetective**
  - **videofy.me**
- - **VideoMega**
+ - **VideoMega** (Currently broken)
  - **videomore**
  - **videomore:season**
  - **videomore:video**
  - **VideoPremium**
- - **VideoTt**: video.tt - Your True Tube
+ - **VideoTt**: video.tt - Your True Tube (Currently broken)
  - **videoweed**: VideoWeed
  - **Vidme**
  - **Vidzi**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7030903c0..4d433b667 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.01.09'
+__version__ = '2016.01.14'

From 30e2f2d76f6dd52803effce14fa14f3a8051c84a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 14 Jan 2016 16:28:46 +0100
Subject: [PATCH 0355/1105] [YoutubeDL] use a more correct terminology in the
 error message for file:// URLs

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4915fbd45..6b73b8e06 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1994,7 +1994,7 @@ class YoutubeDL(object):
         file_handler = compat_urllib_request.FileHandler()
 
         def file_open(*args, **kwargs):
-            raise compat_urllib_error.URLError('file:/// protocol is explicitly disabled in youtube-dl for security reasons')
+            raise compat_urllib_error.URLError('file:// scheme is explicitly disabled in youtube-dl for security reasons')
         file_handler.file_open = file_open
 
         opener = compat_urllib_request.build_opener(

From fbd90643cb123011a224da58b4ff1c4ba1c4f8f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2016 21:48:08 +0600
Subject: [PATCH 0356/1105] [vodlocker] Fix extraction (Closes #8231)

---
 youtube_dl/extractor/vodlocker.py | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dl/extractor/vodlocker.py
index 357594a11..a97995a6d 100644
--- a/youtube_dl/extractor/vodlocker.py
+++ b/youtube_dl/extractor/vodlocker.py
@@ -5,12 +5,13 @@ from .common import InfoExtractor
 from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
+    NO_DEFAULT,
     sanitized_Request,
 )
 
 
 class VodlockerIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vodlocker\.com/(?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:\..*?)?'
+    _VALID_URL = r'https?://(?:www\.)?vodlocker\.(?:com|city)/(?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:\..*?)?'
 
     _TESTS = [{
         'url': 'http://vodlocker.com/e8wvyzz4sl42',
@@ -43,16 +44,31 @@ class VodlockerIE(InfoExtractor):
             webpage = self._download_webpage(
                 req, video_id, 'Downloading video page')
 
+        def extract_file_url(html, default=NO_DEFAULT):
+            return self._search_regex(
+                r'file:\s*"(http[^\"]+)",', html, 'file url', default=default)
+
+        video_url = extract_file_url(webpage, default=None)
+
+        if not video_url:
+            embed_url = self._search_regex(
+                r'<iframe[^>]+src=(["\'])(?P<url>(?:https?://)?vodlocker\.(?:com|city)/embed-.+?)\1',
+                webpage, 'embed url', group='url')
+            embed_webpage = self._download_webpage(
+                embed_url, video_id, 'Downloading embed webpage')
+            video_url = extract_file_url(embed_webpage)
+            thumbnail_webpage = embed_webpage
+        else:
+            thumbnail_webpage = webpage
+
         title = self._search_regex(
             r'id="file_title".*?>\s*(.*?)\s*<(?:br|span)', webpage, 'title')
         thumbnail = self._search_regex(
-            r'image:\s*"(http[^\"]+)",', webpage, 'thumbnail')
-        url = self._search_regex(
-            r'file:\s*"(http[^\"]+)",', webpage, 'file url')
+            r'image:\s*"(http[^\"]+)",', thumbnail_webpage, 'thumbnail', fatal=False)
 
         formats = [{
             'format_id': 'sd',
-            'url': url,
+            'url': video_url,
         }]
 
         return {

From 5cc9c5dfa8f731b6582b092e06f78cccbaefc3c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2016 21:53:24 +0600
Subject: [PATCH 0357/1105] [unistra] Fix extraction

---
 youtube_dl/extractor/unistra.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/unistra.py b/youtube_dl/extractor/unistra.py
index f70978299..594bee4f9 100644
--- a/youtube_dl/extractor/unistra.py
+++ b/youtube_dl/extractor/unistra.py
@@ -38,7 +38,7 @@ class UnistraIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        files = set(re.findall(r'file\s*:\s*"([^"]+)"', webpage))
+        files = set(re.findall(r'file\s*:\s*"(/[^"]+)"', webpage))
 
         quality = qualities(['SD', 'HD'])
         formats = []

From 163e8369b0d2f6b8cc59dd1e93b20a980590648f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2016 22:05:04 +0600
Subject: [PATCH 0358/1105] [ntvde] Fix extraction

---
 youtube_dl/extractor/ntvde.py | 31 +++++++++++++++++++------------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/ntvde.py b/youtube_dl/extractor/ntvde.py
index d2cfe0961..8268eace7 100644
--- a/youtube_dl/extractor/ntvde.py
+++ b/youtube_dl/extractor/ntvde.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
     int_or_none,
     js_to_json,
@@ -42,18 +43,24 @@ class NTVDeIE(InfoExtractor):
             webpage, 'player data'),
             video_id, transform_source=js_to_json)
         duration = parse_duration(vdata.get('duration'))
-        formats = [{
-            'format_id': 'flash',
-            'url': 'rtmp://fms.n-tv.de/' + vdata['video'],
-        }, {
-            'format_id': 'mobile',
-            'url': 'http://video.n-tv.de' + vdata['videoMp4'],
-            'tbr': 400,  # estimation
-        }]
-        m3u8_url = 'http://video.n-tv.de' + vdata['videoM3u8']
-        formats.extend(self._extract_m3u8_formats(
-            m3u8_url, video_id, ext='mp4',
-            entry_protocol='m3u8_native', preference=0))
+
+        formats = []
+        if vdata.get('video'):
+            formats.append({
+                'format_id': 'flash',
+                'url': 'rtmp://fms.n-tv.de/%s' % vdata['video'],
+            })
+        if vdata.get('videoMp4'):
+            formats.append({
+                'format_id': 'mobile',
+                'url': compat_urlparse.urljoin('http://video.n-tv.de', vdata['videoMp4']),
+                'tbr': 400,  # estimation
+            })
+        if vdata.get('videoM3u8'):
+            m3u8_url = compat_urlparse.urljoin('http://video.n-tv.de', vdata['videoM3u8'])
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, ext='mp4', entry_protocol='m3u8_native',
+                preference=0, m3u8_id='hls', fatal=False))
         self._sort_formats(formats)
 
         return {

From 4654c1d01613e26d782c95b13ce60e5fdd84892a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2016 22:07:42 +0600
Subject: [PATCH 0359/1105] [orf:fm4] Extend _VALID_URL (Closes #8234)

---
 youtube_dl/extractor/orf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 2e6c9872b..da598e7f7 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -170,7 +170,7 @@ class ORFOE1IE(InfoExtractor):
 class ORFFM4IE(InfoExtractor):
     IE_NAME = 'orf:fm4'
     IE_DESC = 'radio FM4'
-    _VALID_URL = r'http://fm4\.orf\.at/7tage/?#(?P<date>[0-9]+)/(?P<show>\w+)'
+    _VALID_URL = r'http://fm4\.orf\.at/(?:7tage/?#|player/)(?P<date>[0-9]+)/(?P<show>\w+)'
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 986986064ec102b0d97b4ab008ae38ede6358796 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2016 22:11:33 +0600
Subject: [PATCH 0360/1105] [orf:fm4] Add test

---
 youtube_dl/extractor/orf.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index da598e7f7..c54775d54 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -172,6 +172,20 @@ class ORFFM4IE(InfoExtractor):
     IE_DESC = 'radio FM4'
     _VALID_URL = r'http://fm4\.orf\.at/(?:7tage/?#|player/)(?P<date>[0-9]+)/(?P<show>\w+)'
 
+    _TEST = {
+        'url': 'http://fm4.orf.at/player/20160110/IS/',
+        'md5': '01e736e8f1cef7e13246e880a59ad298',
+        'info_dict': {
+            'id': '2016-01-10_2100_tl_54_7DaysSun13_11244',
+            'ext': 'mp3',
+            'title': 'Im Sumpf',
+            'description': 'md5:384c543f866c4e422a55f66a62d669cd',
+            'duration': 7173,
+            'timestamp': 1452456073,
+            'upload_date': '20160110',
+        },
+    }
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         show_date = mobj.group('date')

From 6b559c2fbcf70158bd84b3b5892ecd5fc4b03e91 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2016 22:12:24 +0600
Subject: [PATCH 0361/1105] [ntvde] Improve regex

---
 youtube_dl/extractor/ntvde.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ntvde.py b/youtube_dl/extractor/ntvde.py
index 8268eace7..a83e85cb8 100644
--- a/youtube_dl/extractor/ntvde.py
+++ b/youtube_dl/extractor/ntvde.py
@@ -35,7 +35,7 @@ class NTVDeIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         info = self._parse_json(self._search_regex(
-            r'(?s)ntv.pageInfo.article =\s(\{.*?\});', webpage, 'info'),
+            r'(?s)ntv\.pageInfo\.article\s*=\s*(\{.*?\});', webpage, 'info'),
             video_id, transform_source=js_to_json)
         timestamp = int_or_none(info.get('publishedDateAsUnixTimeStamp'))
         vdata = self._parse_json(self._search_regex(

From 0baedd1851692a4b9f94c08b3eae5d57acf07f09 Mon Sep 17 00:00:00 2001
From: Lucas <mikotosc@gmail.com>
Date: Wed, 13 Jan 2016 16:11:49 +0100
Subject: [PATCH 0362/1105] [prosiebensat1] add support for 7tv.de

---
 youtube_dl/extractor/prosiebensat1.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index baa54a3af..953df3efc 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -20,7 +20,7 @@ from ..utils import (
 class ProSiebenSat1IE(InfoExtractor):
     IE_NAME = 'prosiebensat1'
     IE_DESC = 'ProSiebenSat.1 Digital'
-    _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|the-voice-of-germany)\.(?:de|at|ch)|ran\.de|fem\.com)/(?P<id>.+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|the-voice-of-germany|7tv)\.(?:de|at|ch)|ran\.de|fem\.com)/(?P<id>.+)'
 
     _TESTS = [
         {
@@ -172,6 +172,20 @@ class ProSiebenSat1IE(InfoExtractor):
             },
             'playlist_count': 2,
         },
+        {
+            'url': 'http://www.7tv.de/circus-halligalli/615-best-of-circus-halligalli-ganze-folge',
+            'info_dict': {
+                'id': '4187506',
+                'ext': 'flv',
+                'title': 'Best of Circus HalliGalli',
+                'description': 'md5:8849752efd90b9772c9db6fdf87fb9e9',
+                'upload_date': '20151229',
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+        },
     ]
 
     _CLIPID_REGEXES = [
@@ -186,12 +200,14 @@ class ProSiebenSat1IE(InfoExtractor):
         r'<!-- start video -->\s*<h1>(.+?)</h1>',
         r'<h1 class="att-name">\s*(.+?)</h1>',
         r'<header class="module_header">\s*<h2>([^<]+)</h2>\s*</header>',
+        r'<h2 class="video-title" itemprop="name">\s*(.+?)</h2>',
     ]
     _DESCRIPTION_REGEXES = [
         r'<p itemprop="description">\s*(.+?)</p>',
         r'<div class="videoDecription">\s*<p><strong>Beschreibung</strong>: (.+?)</p>',
         r'<div class="g-plusone" data-size="medium"></div>\s*</div>\s*</header>\s*(.+?)\s*<footer>',
         r'<p class="att-description">\s*(.+?)\s*</p>',
+        r'<p class="video-description" itemprop="description">\s*(.+?)</p>',
     ]
     _UPLOAD_DATE_REGEXES = [
         r'<meta property="og:published_time" content="(.+?)">',

From 81549898c00a6f12eff4786f3e42e1a53e70c97c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2016 22:45:09 +0600
Subject: [PATCH 0363/1105] [prosiebensat1] Fix some extraction and update
 tests

---
 youtube_dl/extractor/prosiebensat1.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 953df3efc..670e6950f 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -32,7 +32,7 @@ class ProSiebenSat1IE(InfoExtractor):
             'url': 'http://www.prosieben.de/tv/circus-halligalli/videos/218-staffel-2-episode-18-jahresrueckblick-ganze-folge',
             'info_dict': {
                 'id': '2104602',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'Episode 18 - Staffel 2',
                 'description': 'md5:8733c81b702ea472e069bc48bb658fc1',
                 'upload_date': '20131231',
@@ -138,14 +138,13 @@ class ProSiebenSat1IE(InfoExtractor):
             'url': 'http://www.the-voice-of-germany.de/video/31-andreas-kuemmert-rocket-man-clip',
             'info_dict': {
                 'id': '2572814',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'Andreas Kümmert: Rocket Man',
                 'description': 'md5:6ddb02b0781c6adf778afea606652e38',
                 'upload_date': '20131017',
                 'duration': 469.88,
             },
             'params': {
-                # rtmp download
                 'skip_download': True,
             },
         },
@@ -153,13 +152,12 @@ class ProSiebenSat1IE(InfoExtractor):
             'url': 'http://www.fem.com/wellness/videos/wellness-video-clip-kurztripps-zum-valentinstag.html',
             'info_dict': {
                 'id': '2156342',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'Kurztrips zum Valentinstag',
-                'description': 'Romantischer Kurztrip zum Valentinstag? Wir verraten, was sich hier wirklich lohnt.',
+                'description': 'Romantischer Kurztrip zum Valentinstag? Nina Heinemann verrät, was sich hier wirklich lohnt.',
                 'duration': 307.24,
             },
             'params': {
-                # rtmp download
                 'skip_download': True,
             },
         },
@@ -182,7 +180,6 @@ class ProSiebenSat1IE(InfoExtractor):
                 'upload_date': '20151229',
             },
             'params': {
-                # rtmp download
                 'skip_download': True,
             },
         },
@@ -192,6 +189,7 @@ class ProSiebenSat1IE(InfoExtractor):
         r'"clip_id"\s*:\s+"(\d+)"',
         r'clipid: "(\d+)"',
         r'clip[iI]d=(\d+)',
+        r'clip[iI]d\s*=\s*["\'](\d+)',
         r"'itemImageUrl'\s*:\s*'/dynamic/thumbnails/full/\d+/(\d+)",
     ]
     _TITLE_REGEXES = [
@@ -201,6 +199,7 @@ class ProSiebenSat1IE(InfoExtractor):
         r'<h1 class="att-name">\s*(.+?)</h1>',
         r'<header class="module_header">\s*<h2>([^<]+)</h2>\s*</header>',
         r'<h2 class="video-title" itemprop="name">\s*(.+?)</h2>',
+        r'<div[^>]+id="veeseoTitle"[^>]*>(.+?)</div>',
     ]
     _DESCRIPTION_REGEXES = [
         r'<p itemprop="description">\s*(.+?)</p>',
@@ -208,6 +207,7 @@ class ProSiebenSat1IE(InfoExtractor):
         r'<div class="g-plusone" data-size="medium"></div>\s*</div>\s*</header>\s*(.+?)\s*<footer>',
         r'<p class="att-description">\s*(.+?)\s*</p>',
         r'<p class="video-description" itemprop="description">\s*(.+?)</p>',
+        r'<div[^>]+id="veeseoDescription"[^>]*>(.+?)</div>',
     ]
     _UPLOAD_DATE_REGEXES = [
         r'<meta property="og:published_time" content="(.+?)">',

From be2d40a58a4f89682859bb23789b75330ee756a6 Mon Sep 17 00:00:00 2001
From: Tom Gijselinck <tomgijselinck@gmail.com>
Date: Mon, 12 Oct 2015 09:07:18 +0200
Subject: [PATCH 0364/1105] [Canvas] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/canvas.py   | 80 ++++++++++++++++++++++++++++++++
 2 files changed, 81 insertions(+)
 create mode 100644 youtube_dl/extractor/canvas.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e4ae9332d..5621c9eb0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -86,6 +86,7 @@ from .camdemy import (
 )
 from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
+from .canvas import CanvasIE
 from .cbs import CBSIE
 from .cbsnews import CBSNewsIE
 from .cbssports import CBSSportsIE
diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
new file mode 100644
index 000000000..a37720e98
--- /dev/null
+++ b/youtube_dl/extractor/canvas.py
@@ -0,0 +1,80 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import os
+import urlparse
+
+from youtube_dl import utils
+from .common import InfoExtractor
+
+
+class CanvasIE(InfoExtractor):
+    _VALID_URL = r'(?:https?://)?(?:www\.)?canvas\.be/video/(?P<id>.+)'
+    _TEST = {
+        'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
+        'md5': 'ea838375a547ac787d4064d8c7860a6c',
+        'info_dict': {
+            'id': 'de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
+            'title': 'De afspraak veilt voor de Warmste Week',
+            'ext': 'mp4',
+            'duration': 49,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._search_regex(
+            r'<h1 class="video__body__header__title">(.+?)</h1>', webpage,
+            'title')
+        data_video = self._html_search_regex(
+            r'data-video=(["\'])(?P<id>.+?)\1', webpage, 'data-video', group='id')
+        json_url = 'https://mediazone.vrt.be/api/v1/canvas/assets/' + data_video
+        data = self._download_json(json_url, video_id)
+
+        formats = []
+        for target in data['targetUrls']:
+            if 'type' and 'url' in target:
+                extension = utils.determine_ext(target['url'])
+                if target['type'] == 'PROGRESSIVE_DOWNLOAD':
+                    formats.append({
+                        'format_id': extension,
+                        'url': target['url'],
+                        'protocol': 'http',
+                    })
+                elif target['type'] == 'HLS':
+                    formats.extend(self._extract_m3u8_formats(
+                        target['url'], video_id, entry_protocol='m3u8_native',
+                        ext='mp4',
+                        preference=0,
+                        fatal=False,
+                        m3u8_id='hls'))
+                elif target['type'] == 'HDS':
+                    formats.append({
+                        'format_id': extension,
+                        'url': target['url'],
+                        'protocol': 'HDS',
+                    })
+                elif target['type'] == 'RTMP':
+                    formats.append({
+                        'format_id': extension,
+                        'url': target['url'],
+                        'protocol': 'rtmp',
+                    })
+                elif target['type'] == 'RTSP':
+                    formats.append({
+                        'format_id': extension,
+                        'url': target['url'],
+                        'protocol': 'rtsp',
+                    })
+
+        self._sort_formats(formats)
+        duration = utils.int_or_none(data.get('duration')) / 1000
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'duration': duration,
+        }

From 4e2743abd9a3f685374cfc1eb4031fb0bc398c58 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2016 23:15:12 +0600
Subject: [PATCH 0365/1105] [canvas] Improve (Closes #7145)

---
 youtube_dl/extractor/canvas.py | 91 ++++++++++++++--------------------
 1 file changed, 38 insertions(+), 53 deletions(-)

diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
index a37720e98..ee19ff836 100644
--- a/youtube_dl/extractor/canvas.py
+++ b/youtube_dl/extractor/canvas.py
@@ -1,80 +1,65 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
-import os
-import urlparse
-
-from youtube_dl import utils
 from .common import InfoExtractor
+from ..utils import float_or_none
 
 
 class CanvasIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://)?(?:www\.)?canvas\.be/video/(?P<id>.+)'
+    _VALID_URL = r'https?://(?:www\.)?canvas\.be/video/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _TEST = {
         'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
         'md5': 'ea838375a547ac787d4064d8c7860a6c',
         'info_dict': {
-            'id': 'de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
-            'title': 'De afspraak veilt voor de Warmste Week',
+            'id': 'mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
+            'display_id': 'de-afspraak-veilt-voor-de-warmste-week',
             'ext': 'mp4',
-            'duration': 49,
+            'title': 'De afspraak veilt voor de Warmste Week',
+            'description': 'md5:24cb860c320dc2be7358e0e5aa317ba6',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 49.02,
         }
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        display_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url, display_id)
 
         title = self._search_regex(
-            r'<h1 class="video__body__header__title">(.+?)</h1>', webpage,
-            'title')
-        data_video = self._html_search_regex(
-            r'data-video=(["\'])(?P<id>.+?)\1', webpage, 'data-video', group='id')
-        json_url = 'https://mediazone.vrt.be/api/v1/canvas/assets/' + data_video
-        data = self._download_json(json_url, video_id)
+            r'<h1[^>]+class="video__body__header__title"[^>]*>(.+?)</h1>',
+            webpage, 'title', default=None) or self._og_search_title(webpage)
+
+        video_id = self._html_search_regex(
+            r'data-video=(["\'])(?P<id>.+?)\1', webpage, 'video id', group='id')
+
+        data = self._download_json(
+            'https://mediazone.vrt.be/api/v1/canvas/assets/%s' % video_id, display_id)
 
         formats = []
         for target in data['targetUrls']:
-            if 'type' and 'url' in target:
-                extension = utils.determine_ext(target['url'])
-                if target['type'] == 'PROGRESSIVE_DOWNLOAD':
-                    formats.append({
-                        'format_id': extension,
-                        'url': target['url'],
-                        'protocol': 'http',
-                    })
-                elif target['type'] == 'HLS':
-                    formats.extend(self._extract_m3u8_formats(
-                        target['url'], video_id, entry_protocol='m3u8_native',
-                        ext='mp4',
-                        preference=0,
-                        fatal=False,
-                        m3u8_id='hls'))
-                elif target['type'] == 'HDS':
-                    formats.append({
-                        'format_id': extension,
-                        'url': target['url'],
-                        'protocol': 'HDS',
-                    })
-                elif target['type'] == 'RTMP':
-                    formats.append({
-                        'format_id': extension,
-                        'url': target['url'],
-                        'protocol': 'rtmp',
-                    })
-                elif target['type'] == 'RTSP':
-                    formats.append({
-                        'format_id': extension,
-                        'url': target['url'],
-                        'protocol': 'rtsp',
-                    })
-
+            format_url, format_type = target.get('url'), target.get('type')
+            if not format_url or not format_type:
+                continue
+            if format_type == 'HLS':
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, display_id, entry_protocol='m3u8_native',
+                    ext='mp4', preference=0, fatal=False, m3u8_id=format_type))
+            elif format_type == 'HDS':
+                formats.extend(self._extract_f4m_formats(
+                    format_url, display_id, f4m_id=format_type, fatal=False))
+            else:
+                formats.append({
+                    'format_id': format_type,
+                    'url': format_url,
+                })
         self._sort_formats(formats)
-        duration = utils.int_or_none(data.get('duration')) / 1000
+
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': title,
+            'description': self._og_search_description(webpage),
             'formats': formats,
-            'duration': duration,
+            'duration': float_or_none(data.get('duration'), 1000),
+            'thumbnail': data.get('posterImageUrl'),
         }

From 5769b68bc018654b959580624b545a9e7e857c5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2016 23:15:26 +0600
Subject: [PATCH 0366/1105] Credit @TomGijselinck for canvas (#7145)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 3d8bebbb0..f4238e1d3 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -152,3 +152,4 @@ Evan Lu
 flatgreen
 Brian Foley
 Vignesh Venkat
+Tom Gijselinck

From ffbc0baf72f456b1f4f4ac989e4a7b75b69c7a54 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 15 Jan 2016 12:34:28 +0100
Subject: [PATCH 0367/1105] [history] fix signature and media url
 extraction(fixes #8240)

---
 youtube_dl/extractor/history.py     | 17 +++++++++++++----
 youtube_dl/extractor/theplatform.py |  2 +-
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/history.py b/youtube_dl/extractor/history.py
index f86164afe..1866ca679 100644
--- a/youtube_dl/extractor/history.py
+++ b/youtube_dl/extractor/history.py
@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import smuggle_url
+from ..compat import compat_urllib_parse
 
 
 class HistoryIE(InfoExtractor):
@@ -19,13 +20,21 @@ class HistoryIE(InfoExtractor):
         'add_ie': ['ThePlatform'],
     }]
 
+    _VIDEO_URL_RE = [
+        r"media_url\s*=\s*'[^']'",
+        r'data-mediaurl="([^"]+)"'
+    ]
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
-        video_url = self._search_regex(
-            r'data-href="[^"]*/%s"[^>]+data-release-url="([^"]+)"' % video_id,
-            webpage, 'video url')
+        media_url = self._search_regex(self._VIDEO_URL_RE, webpage, 'video url')
 
-        return self.url_result(smuggle_url(video_url, {'sig': {'key': 'crazyjava', 'secret': 's3cr3t'}}))
+        pdk_signature = webpage = self._download_webpage(
+            'https://signature.video.aetndigital.com/?' +
+            compat_urllib_parse.urlencode({'url': media_url}), video_id)
+
+        return self.url_result(smuggle_url(
+            media_url + '?mbr=true&format=smil&sig=' + pdk_signature, {'force_smil_url': True}))
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 0bf6726b5..10f2cad55 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -85,7 +85,7 @@ class ThePlatformBaseIE(InfoExtractor):
 class ThePlatformIE(ThePlatformBaseIE):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
-           (?:(?P<media>(?:[^/]+/)+select/media/)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
+           (?:(?P<media>(?:(?:[^/]+/)+select/)?media/)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
          |theplatform:)(?P<id>[^/\?&]+)'''
 
     _TESTS = [{

From fad7a336a162cda2a53a77bf24fe9f666ec3ce07 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 15 Jan 2016 14:54:39 +0100
Subject: [PATCH 0368/1105] Revert "[history] fix signature and media url
 extraction(fixes #8240)"

This reverts commit ffbc0baf72f456b1f4f4ac989e4a7b75b69c7a54.
---
 youtube_dl/extractor/history.py     | 17 ++++-------------
 youtube_dl/extractor/theplatform.py |  2 +-
 2 files changed, 5 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/history.py b/youtube_dl/extractor/history.py
index 1866ca679..f86164afe 100644
--- a/youtube_dl/extractor/history.py
+++ b/youtube_dl/extractor/history.py
@@ -2,7 +2,6 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import smuggle_url
-from ..compat import compat_urllib_parse
 
 
 class HistoryIE(InfoExtractor):
@@ -20,21 +19,13 @@ class HistoryIE(InfoExtractor):
         'add_ie': ['ThePlatform'],
     }]
 
-    _VIDEO_URL_RE = [
-        r"media_url\s*=\s*'[^']'",
-        r'data-mediaurl="([^"]+)"'
-    ]
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
-        media_url = self._search_regex(self._VIDEO_URL_RE, webpage, 'video url')
+        video_url = self._search_regex(
+            r'data-href="[^"]*/%s"[^>]+data-release-url="([^"]+)"' % video_id,
+            webpage, 'video url')
 
-        pdk_signature = webpage = self._download_webpage(
-            'https://signature.video.aetndigital.com/?' +
-            compat_urllib_parse.urlencode({'url': media_url}), video_id)
-
-        return self.url_result(smuggle_url(
-            media_url + '?mbr=true&format=smil&sig=' + pdk_signature, {'force_smil_url': True}))
+        return self.url_result(smuggle_url(video_url, {'sig': {'key': 'crazyjava', 'secret': 's3cr3t'}}))
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 10f2cad55..0bf6726b5 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -85,7 +85,7 @@ class ThePlatformBaseIE(InfoExtractor):
 class ThePlatformIE(ThePlatformBaseIE):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
-           (?:(?P<media>(?:(?:[^/]+/)+select/)?media/)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
+           (?:(?P<media>(?:[^/]+/)+select/media/)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
          |theplatform:)(?P<id>[^/\?&]+)'''
 
     _TESTS = [{

From 014b5c59d827c413663394c23384524ef6998f0b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 15 Jan 2016 15:12:35 +0100
Subject: [PATCH 0369/1105] [theplatform] extend _VALID_URL regex

---
 youtube_dl/extractor/theplatform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 0bf6726b5..10f2cad55 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -85,7 +85,7 @@ class ThePlatformBaseIE(InfoExtractor):
 class ThePlatformIE(ThePlatformBaseIE):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
-           (?:(?P<media>(?:[^/]+/)+select/media/)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
+           (?:(?P<media>(?:(?:[^/]+/)+select/)?media/)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
          |theplatform:)(?P<id>[^/\?&]+)'''
 
     _TESTS = [{

From 52767c1ba0ddd515f52bfdc9169cd2037d3eb403 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 15 Jan 2016 15:16:57 +0100
Subject: [PATCH 0370/1105] [history] add support for episode pages(fixes
 #8240)

---
 youtube_dl/extractor/history.py | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/history.py b/youtube_dl/extractor/history.py
index f86164afe..9ec2e92cf 100644
--- a/youtube_dl/extractor/history.py
+++ b/youtube_dl/extractor/history.py
@@ -9,13 +9,29 @@ class HistoryIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.history.com/topics/valentines-day/history-of-valentines-day/videos/bet-you-didnt-know-valentines-day?m=528e394da93ae&s=undefined&f=1&free=false',
-        'md5': '6fe632d033c92aa10b8d4a9be047a7c5',
         'info_dict': {
-            'id': 'bLx5Dv5Aka1G',
+            'id': 'g12m5Gyt3fdR',
             'ext': 'mp4',
             'title': "Bet You Didn't Know: Valentine's Day",
             'description': 'md5:7b57ea4829b391995b405fa60bd7b5f7',
         },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+        'add_ie': ['ThePlatform'],
+    }, {
+        'url': 'http://www.history.com/shows/mountain-men/season-1/episode-1',
+        'info_dict': {
+            'id': 'eg47EERs_JsZ',
+            'ext': 'mp4',
+            'title': "Winter Is Coming",
+            'description': 'md5:a40e370925074260b1c8a633c632c63a',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
         'add_ie': ['ThePlatform'],
     }]
 
@@ -24,8 +40,10 @@ class HistoryIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        video_url = self._search_regex(
+        video_url_re = [
             r'data-href="[^"]*/%s"[^>]+data-release-url="([^"]+)"' % video_id,
-            webpage, 'video url')
+            r"media_url\s*=\s*'([^']+)'"
+        ]
+        video_url = self._search_regex(video_url_re, webpage, 'video url')
 
         return self.url_result(smuggle_url(video_url, {'sig': {'key': 'crazyjava', 'secret': 's3cr3t'}}))

From 587dfd44a4df3fac573e1152ca70f34eabe42611 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 15 Jan 2016 16:18:07 +0100
Subject: [PATCH 0371/1105] [ae] Add support for fyi.tv, aetv.com and
 mylifetime.com(closes #3599)

---
 youtube_dl/extractor/{history.py => ae.py} | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)
 rename youtube_dl/extractor/{history.py => ae.py} (73%)

diff --git a/youtube_dl/extractor/history.py b/youtube_dl/extractor/ae.py
similarity index 73%
rename from youtube_dl/extractor/history.py
rename to youtube_dl/extractor/ae.py
index 9ec2e92cf..880c65350 100644
--- a/youtube_dl/extractor/history.py
+++ b/youtube_dl/extractor/ae.py
@@ -4,8 +4,8 @@ from .common import InfoExtractor
 from ..utils import smuggle_url
 
 
-class HistoryIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?history\.com/(?:[^/]+/)+(?P<id>[^/]+?)(?:$|[?#])'
+class AEIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?(?:(?:history|aetv|mylifetime)\.com|fyi.tv)/(?:[^/]+/)+(?P<id>[^/]+?)(?:$|[?#])'
 
     _TESTS = [{
         'url': 'http://www.history.com/topics/valentines-day/history-of-valentines-day/videos/bet-you-didnt-know-valentines-day?m=528e394da93ae&s=undefined&f=1&free=false',
@@ -33,6 +33,15 @@ class HistoryIE(InfoExtractor):
             'skip_download': True,
         },
         'add_ie': ['ThePlatform'],
+    }, {
+        'url': 'http://www.aetv.com/shows/duck-dynasty/video/inlawful-entry',
+        'only_matching': True
+    }, {
+        'url': 'http://www.fyi.tv/shows/tiny-house-nation/videos/207-sq-ft-minnesota-prairie-cottage',
+        'only_matching': True
+    }, {
+        'url': 'http://www.mylifetime.com/shows/project-runway-junior/video/season-1/episode-6/superstar-clients',
+        'only_matching': True
     }]
 
     def _real_extract(self, url):

From b3f502cdb971c8cc827d8646d8dcb659caa77149 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Jan 2016 21:28:36 +0600
Subject: [PATCH 0372/1105] [xtube] Add shortcut

---
 youtube_dl/extractor/xtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index a1fe24050..2254cd238 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class XTubeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?P<url>xtube\.com/watch\.php\?v=(?P<id>[^/?&#]+))'
+    _VALID_URL = r'(?:xtube:|https?://(?:www\.)?xtube\.com/watch\.php\?.*\bv=)(?P<id>[^/?&#]+)'
     _TEST = {
         'url': 'http://www.xtube.com/watch.php?v=kVTUy_G222_',
         'md5': '092fbdd3cbe292c920ef6fc6a8a9cdab',
@@ -30,7 +30,7 @@ class XTubeIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        req = sanitized_Request(url)
+        req = sanitized_Request('http://www.xtube.com/watch.php?v=%s' % video_id)
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 

From 34dd81c03a23ef7e4db31f8aa9a61473f2d8f727 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Jan 2016 21:35:20 +0600
Subject: [PATCH 0373/1105] [xtube:user] Fix extraction (Closes #8224)

---
 youtube_dl/extractor/xtube.py | 60 +++++++++++++++++------------------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 2254cd238..8cd3a0687 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -1,10 +1,12 @@
 from __future__ import unicode_literals
 
+import itertools
 import re
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse_unquote
 from ..utils import (
+    int_or_none,
     parse_duration,
     sanitized_Request,
     str_to_int,
@@ -88,45 +90,43 @@ class XTubeIE(InfoExtractor):
 
 class XTubeUserIE(InfoExtractor):
     IE_DESC = 'XTube user profile'
-    _VALID_URL = r'https?://(?:www\.)?xtube\.com/community/profile\.php\?(.*?)user=(?P<username>[^&#]+)(?:$|[&#])'
+    _VALID_URL = r'https?://(?:www\.)?xtube\.com/profile/(?P<id>[^/]+-\d+)'
     _TEST = {
-        'url': 'http://www.xtube.com/community/profile.php?user=greenshowers',
+        'url': 'http://www.xtube.com/profile/greenshowers-4056496',
         'info_dict': {
-            'id': 'greenshowers',
+            'id': 'greenshowers-4056496',
             'age_limit': 18,
         },
         'playlist_mincount': 155,
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        username = mobj.group('username')
+        user_id = self._match_id(url)
 
-        profile_page = self._download_webpage(
-            url, username, note='Retrieving profile page')
+        entries = []
+        for pagenum in itertools.count(1):
+            request = sanitized_Request(
+                'http://www.xtube.com/profile/%s/videos/%d' % (user_id, pagenum),
+                headers={
+                    'Cookie': 'popunder=4',
+                    'X-Requested-With': 'XMLHttpRequest',
+                    'Referer': url,
+                })
 
-        video_count = int(self._search_regex(
-            r'<strong>%s\'s Videos \(([0-9]+)\)</strong>' % username, profile_page,
-            'video count'))
+            page = self._download_json(
+                request, user_id, 'Downloading videos JSON page %d' % pagenum)
 
-        PAGE_SIZE = 25
-        urls = []
-        page_count = (video_count + PAGE_SIZE + 1) // PAGE_SIZE
-        for n in range(1, page_count + 1):
-            lpage_url = 'http://www.xtube.com/user_videos.php?page=%d&u=%s' % (n, username)
-            lpage = self._download_webpage(
-                lpage_url, username,
-                note='Downloading page %d/%d' % (n, page_count))
-            urls.extend(
-                re.findall(r'addthis:url="([^"]+)"', lpage))
+            html = page.get('html')
+            if not html:
+                break
 
-        return {
-            '_type': 'playlist',
-            'id': username,
-            'age_limit': 18,
-            'entries': [{
-                '_type': 'url',
-                'url': eurl,
-                'ie_key': 'XTube',
-            } for eurl in urls]
-        }
+            for _, video_id in re.findall(r'data-plid=(["\'])(.+?)\1', html):
+                entries.append(self.url_result('xtube:%s' % video_id, XTubeIE.ie_key()))
+
+            page_count = int_or_none(page.get('pageCount'))
+            if not page_count or pagenum == page_count:
+                break
+
+        playlist = self.playlist_result(entries, user_id)
+        playlist['age_limit'] = 18
+        return playlist

From 3a421c724f60b7cc9fe870e343e8eb62d587a620 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Jan 2016 22:10:07 +0600
Subject: [PATCH 0374/1105] [history] Remove import (Closes #8243)

---
 youtube_dl/extractor/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5621c9eb0..f50a5d625 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -263,7 +263,6 @@ from .hellporno import HellPornoIE
 from .helsinki import HelsinkiIE
 from .hentaistigma import HentaiStigmaIE
 from .historicfilms import HistoricFilmsIE
-from .history import HistoryIE
 from .hitbox import HitboxIE, HitboxLiveIE
 from .hornbunny import HornBunnyIE
 from .hotnewhiphop import HotNewHipHopIE

From 567f9a5809daf51ed48cf8292ab741f765937d56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Jan 2016 22:12:51 +0600
Subject: [PATCH 0375/1105] [ae] Add extractor import

---
 youtube_dl/extractor/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f50a5d625..3975354da 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -15,6 +15,7 @@ from .adobetv import (
     AdobeTVVideoIE,
 )
 from .adultswim import AdultSwimIE
+from .ae import AEIE
 from .aftonbladet import AftonbladetIE
 from .airmozilla import AirMozillaIE
 from .aljazeera import AlJazeeraIE

From a78d6a9bb1387db4aa29f1ac1e4edad8d2061783 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Jan 2016 22:13:48 +0600
Subject: [PATCH 0376/1105] [ae] Improve _VALID_URL

---
 youtube_dl/extractor/ae.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ae.py b/youtube_dl/extractor/ae.py
index 880c65350..a3b9f1ca4 100644
--- a/youtube_dl/extractor/ae.py
+++ b/youtube_dl/extractor/ae.py
@@ -5,7 +5,7 @@ from ..utils import smuggle_url
 
 
 class AEIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?:(?:history|aetv|mylifetime)\.com|fyi.tv)/(?:[^/]+/)+(?P<id>[^/]+?)(?:$|[?#])'
+    _VALID_URL = r'https?://(?:www\.)?(?:(?:history|aetv|mylifetime)\.com|fyi\.tv)/(?:[^/]+/)+(?P<id>[^/]+?)(?:$|[?#])'
 
     _TESTS = [{
         'url': 'http://www.history.com/topics/valentines-day/history-of-valentines-day/videos/bet-you-didnt-know-valentines-day?m=528e394da93ae&s=undefined&f=1&free=false',

From 27a95f51aaf75275ed0f7a7d2e8763d874c649a4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 15 Jan 2016 17:45:51 +0100
Subject: [PATCH 0377/1105] [cwtv] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/cwtv.py     | 88 ++++++++++++++++++++++++++++++++
 2 files changed, 89 insertions(+)
 create mode 100644 youtube_dl/extractor/cwtv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3975354da..d7a9df202 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -130,6 +130,7 @@ from .crunchyroll import (
 )
 from .cspan import CSpanIE
 from .ctsnews import CtsNewsIE
+from .cwtv import CWTVIE
 from .dailymotion import (
     DailymotionIE,
     DailymotionPlaylistIE,
diff --git a/youtube_dl/extractor/cwtv.py b/youtube_dl/extractor/cwtv.py
new file mode 100644
index 000000000..36af67013
--- /dev/null
+++ b/youtube_dl/extractor/cwtv.py
@@ -0,0 +1,88 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+)
+
+
+class CWTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?cw(?:tv|seed)\.com/shows/(?:[^/]+/){2}\?play=(?P<id>[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})'
+    _TESTS = [{
+        'url': 'http://cwtv.com/shows/arrow/legends-of-yesterday/?play=6b15e985-9345-4f60-baf8-56e96be57c63',
+        'info_dict': {
+            'id': '6b15e985-9345-4f60-baf8-56e96be57c63',
+            'ext': 'mp4',
+            'title': 'Legends of Yesterday',
+            'description': 'Oliver and Barry Allen take Kendra Saunders and Carter Hall to a remote location to keep them hidden from Vandal Savage while they figure out how to defeat him.',
+            'duration': 2665,
+            'series': 'Arrow',
+            'season_number': 4,
+            'season': '4',
+            'episode_number': 8,
+            'upload_date': '20151203',
+            'timestamp': 1449122100,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://www.cwseed.com/shows/whose-line-is-it-anyway/jeff-davis-4/?play=24282b12-ead2-42f2-95ad-26770c2c6088',
+        'info_dict': {
+            'id': '24282b12-ead2-42f2-95ad-26770c2c6088',
+            'ext': 'mp4',
+            'title': 'Jeff Davis 4',
+            'description': 'Jeff Davis is back to make you laugh.',
+            'duration': 1263,
+            'series': 'Whose Line Is It Anyway?',
+            'season_number': 11,
+            'season': '11',
+            'episode_number': 20,
+            'upload_date': '20151006',
+            'timestamp': 1444107300,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_data = self._download_json(
+            'http://metaframe.digitalsmiths.tv/v2/CWtv/assets/%s/partner/132?format=json' % video_id, video_id)
+
+        formats = self._extract_m3u8_formats(
+            video_data['videos']['variantplaylist']['uri'], video_id, 'mp4')
+
+        thumbnails = [{
+            'url': image['uri'],
+            'width': image.get('width'),
+            'height': image.get('height'),
+        } for image_id, image in video_data['images'].items() if image.get('uri')] if video_data.get('images') else None
+
+        video_metadata = video_data['assetFields']
+
+        subtitles = {
+            'en': [{
+                'url': video_metadata['UnicornCcUrl'],
+            }],
+        } if video_metadata.get('UnicornCcUrl') else None
+
+        return {
+            'id': video_id,
+            'title': video_metadata['title'],
+            'description': video_metadata.get('description'),
+            'duration': int_or_none(video_metadata.get('duration')),
+            'series': video_metadata.get('seriesName'),
+            'season_number': int_or_none(video_metadata.get('seasonNumber')),
+            'season': video_metadata.get('seasonName'),
+            'episode_number': int_or_none(video_metadata.get('episodeNumber')),
+            'timestamp': parse_iso8601(video_data.get('startTime')),
+            'thumbnails': thumbnails,
+            'formats': formats,
+            'subtitles': subtitles,
+        }

From 33d2fc2f640c32b4b5469ecb3a48e7e209844407 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Jan 2016 00:09:54 +0600
Subject: [PATCH 0378/1105] [YoutubeDL] Auto generate title fields
 corresponding to the *_number fields

Auto generate title fields corresponding to the *_number fields when missing in order to always have clean titles. This is very common for TV series.
---
 youtube_dl/YoutubeDL.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 6b73b8e06..0748fbba0 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1244,6 +1244,12 @@ class YoutubeDL(object):
             except (ValueError, OverflowError, OSError):
                 pass
 
+        # Auto generate title fields corresponding to the *_number fields when missing
+        # in order to always have clean titles. This is very common for TV series.
+        for field in ('chapter', 'season', 'episode'):
+            if info_dict.get('%s_number' % field) is not None and not info_dict.get(field):
+                info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field])
+
         subtitles = info_dict.get('subtitles')
         if subtitles:
             for _, subtitle in subtitles.items():

From 4ca2a3cf3cf0227025f09cfe6e75a37f884442a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Jan 2016 00:36:02 +0600
Subject: [PATCH 0379/1105] [extractor/common] Add initial support for JSON-LD
 metadata extraction into info_dict

---
 youtube_dl/extractor/common.py | 37 ++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b05b22a94..92e2e4f43 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -34,6 +34,7 @@ from ..utils import (
     fix_xml_ampersands,
     float_or_none,
     int_or_none,
+    parse_iso8601,
     RegexNotFoundError,
     sanitize_filename,
     sanitized_Request,
@@ -762,6 +763,42 @@ class InfoExtractor(object):
         return self._html_search_meta('twitter:player', html,
                                       'twitter card player')
 
+    def _search_json_ld(self, html, video_id, fatal=True):
+        json_ld = self._search_regex(
+            r'(?s)<script[^>]+type=(["\'])application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>',
+            html, 'JSON-LD', fatal=fatal, group='json_ld')
+        if not json_ld:
+            return {}
+        return self._json_ld(json_ld, video_id, fatal=fatal)
+
+    def _json_ld(self, json_ld, video_id, fatal=True):
+        if isinstance(json_ld, compat_str):
+            json_ld = self._parse_json(json_ld, video_id, fatal=fatal)
+        if not json_ld:
+            return {}
+        info = {}
+        if json_ld.get('@context') == 'http://schema.org':
+            item_type = json_ld.get('@type')
+            if item_type == 'TVEpisode':
+                info.update({
+                    'episode': unescapeHTML(json_ld.get('name')),
+                    'episode_number': int_or_none(json_ld.get('episodeNumber')),
+                    'description': unescapeHTML(json_ld.get('description')),
+                })
+                part_of_season = json_ld.get('partOfSeason')
+                if isinstance(part_of_season, dict) and part_of_season.get('@type') == 'TVSeason':
+                    info['season_number'] = int_or_none(part_of_season.get('seasonNumber'))
+                part_of_series = json_ld.get('partOfSeries')
+                if isinstance(part_of_series, dict) and part_of_series.get('@type') == 'TVSeries':
+                    info['series'] = unescapeHTML(part_of_series.get('name'))
+            elif item_type == 'Article':
+                info.update({
+                    'timestamp': parse_iso8601(json_ld.get('datePublished')),
+                    'title': unescapeHTML(json_ld.get('headline')),
+                    'description': unescapeHTML(json_ld.get('articleBody')),
+                })
+        return dict((k, v) for k, v in info.items() if v is not None)
+
     @staticmethod
     def _hidden_inputs(html):
         html = re.sub(r'<!--(?:(?!<!--).)*-->', '', html)

From b4e44234bc481747b88c330b0e30a13dbc0fc99a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Jan 2016 00:36:49 +0600
Subject: [PATCH 0380/1105] [ae] Use JSON-LD for TV series metadata

---
 youtube_dl/extractor/ae.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ae.py b/youtube_dl/extractor/ae.py
index a3b9f1ca4..3bc7c12fc 100644
--- a/youtube_dl/extractor/ae.py
+++ b/youtube_dl/extractor/ae.py
@@ -55,4 +55,9 @@ class AEIE(InfoExtractor):
         ]
         video_url = self._search_regex(video_url_re, webpage, 'video url')
 
-        return self.url_result(smuggle_url(video_url, {'sig': {'key': 'crazyjava', 'secret': 's3cr3t'}}))
+        info = self._search_json_ld(webpage, video_id, fatal=False)
+        info.update({
+            '_type': 'url_transparent',
+            'url': smuggle_url(video_url, {'sig': {'key': 'crazyjava', 'secret': 's3cr3t'}}),
+        })
+        return info

From f3ed5df6118169894a27ee70a15c6202ba3823b4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 15 Jan 2016 19:43:04 +0100
Subject: [PATCH 0381/1105] release 2016.01.15

---
 docs/supportedsites.md | 4 +++-
 youtube_dl/version.py  | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index eb160bd2f..99b1e2731 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -24,6 +24,7 @@
  - **AdobeTVShow**
  - **AdobeTVVideo**
  - **AdultSwim**
+ - **AE**
  - **Aftonbladet**
  - **AirMozilla**
  - **AlJazeera**
@@ -85,6 +86,7 @@
  - **CamdemyFolder**
  - **canalc2.tv**
  - **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv
+ - **Canvas**
  - **CBS**
  - **CBSNews**: CBS News
  - **CBSSports**
@@ -122,6 +124,7 @@
  - **CSpan**: C-SPAN
  - **CtsNews**: 華視新聞
  - **culturebox.francetvinfo.fr**
+ - **CWTV**
  - **dailymotion**
  - **dailymotion:playlist**
  - **dailymotion:user**
@@ -229,7 +232,6 @@
  - **Helsinki**: helsinki.fi
  - **HentaiStigma**
  - **HistoricFilms**
- - **History**
  - **hitbox**
  - **hitbox:live**
  - **HornBunny**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4d433b667..02c438f3a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.01.14'
+__version__ = '2016.01.15'

From 2ddfd26f1be72bc8e92ec539cdd969440f301f67 Mon Sep 17 00:00:00 2001
From: ckuu <cku@gmx.com>
Date: Sun, 10 Jan 2016 15:20:59 +0400
Subject: [PATCH 0382/1105] '[ZippCast] Add new extractor' Closes
 rg3/youtube-dl#6591

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/zippcast.py | 58 ++++++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+)
 create mode 100644 youtube_dl/extractor/zippcast.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d7a9df202..5b8e16a29 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -915,6 +915,7 @@ from .zingmp3 import (
     ZingMp3SongIE,
     ZingMp3AlbumIE,
 )
+from .zippcast import ZippCastIE
 
 _ALL_CLASSES = [
     klass
diff --git a/youtube_dl/extractor/zippcast.py b/youtube_dl/extractor/zippcast.py
new file mode 100644
index 000000000..afeab4114
--- /dev/null
+++ b/youtube_dl/extractor/zippcast.py
@@ -0,0 +1,58 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class ZippCastIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?zippcast\.com/video/(?P<id>[0-9a-zA-Z]+)'
+    _TESTS = [{
+        'url': 'http://www.zippcast.com/video/c9cfd5c7e44dbc29c81',
+        'md5': 'f2aea8659962d9155031aaeac53f7c54',
+        'info_dict': {
+            'id': 'c9cfd5c7e44dbc29c81',
+            'ext': 'mp4',
+            'title': '[Vinesauce] Vinny - Digital Space Traveler',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'vinesauce',
+            'description': 'Muted on youtube, but now uploaded in it\'s original form.',
+            'categories': ['Entertainment'],
+            'view_count': int,
+        },
+    }, {
+        'url': 'http://www.zippcast.com/video/b79c0a233e9c6581775',
+        'md5': 'b8631f0cc48ed15387f9179988d0c97c',
+        'info_dict': {
+            'id': 'b79c0a233e9c6581775',
+            'ext': 'mp4',
+            'title': 'Battlefield Hardline Trailer',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'IGXGaming',
+            'description': 'Battlefield Hardline Trailer',
+            'categories': ['Gaming'],
+            'view_count': int,
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r'title="(.+?)"', webpage, 'title')
+        uploader = self._html_search_regex(r'http://www.zippcast.com/profile/(.+?)">', webpage, 'uploader')
+        url = self._html_search_regex(r'<source src="(.+?)" type="', webpage, 'url')
+        description = self._html_search_regex(r'<span class="vdescr".+>(.+?)</span>', webpage, 'description')
+        thumbnail = self._html_search_regex(r'poster="(.+?)" controls>', webpage, 'thumbnail')
+        categories = self._html_search_regex(r'<a href="http://www.zippcast.com/categories/(.+?)"', webpage, 'categories')
+        view_count = self._html_search_regex(r'<td align="right"><h3>(.+?) views!', webpage, 'view_count')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': url,
+            'description': description,
+            'uploader': uploader,
+            'thumbnail': thumbnail,
+            'categories': [categories],
+            'view_count': int(view_count.replace(',', '')),
+        }

From 315bdae00a906b0e07d400b63754abadec34dab5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Jan 2016 06:25:25 +0600
Subject: [PATCH 0383/1105] [zippcast] Improve (Closes #8198)

---
 youtube_dl/extractor/zippcast.py | 92 ++++++++++++++++++++++----------
 1 file changed, 64 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/zippcast.py b/youtube_dl/extractor/zippcast.py
index afeab4114..de819376d 100644
--- a/youtube_dl/extractor/zippcast.py
+++ b/youtube_dl/extractor/zippcast.py
@@ -1,58 +1,94 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    str_to_int,
+)
 
 
 class ZippCastIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?zippcast\.com/video/(?P<id>[0-9a-zA-Z]+)'
+    _VALID_URL = r'https?://(?:www\.)?zippcast\.com/(?:video/|videoview\.php\?.*\bvplay=)(?P<id>[0-9a-zA-Z]+)'
     _TESTS = [{
+        # m3u8, hq direct link
         'url': 'http://www.zippcast.com/video/c9cfd5c7e44dbc29c81',
-        'md5': 'f2aea8659962d9155031aaeac53f7c54',
+        'md5': '5ea0263b5606866c4d6cda0fc5e8c6b6',
         'info_dict': {
             'id': 'c9cfd5c7e44dbc29c81',
             'ext': 'mp4',
             'title': '[Vinesauce] Vinny - Digital Space Traveler',
+            'description': 'Muted on youtube, but now uploaded in it\'s original form.',
             'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': 'vinesauce',
-            'description': 'Muted on youtube, but now uploaded in it\'s original form.',
-            'categories': ['Entertainment'],
             'view_count': int,
+            'categories': ['Entertainment'],
+            'tags': list,
         },
     }, {
+        # f4m, lq ipod direct link
         'url': 'http://www.zippcast.com/video/b79c0a233e9c6581775',
-        'md5': 'b8631f0cc48ed15387f9179988d0c97c',
-        'info_dict': {
-            'id': 'b79c0a233e9c6581775',
-            'ext': 'mp4',
-            'title': 'Battlefield Hardline Trailer',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'uploader': 'IGXGaming',
-            'description': 'Battlefield Hardline Trailer',
-            'categories': ['Gaming'],
-            'view_count': int,
-        },
+        'only_matching': True,
+    }, {
+        'url': 'http://www.zippcast.com/videoview.php?vplay=c9cfd5c7e44dbc29c81&auto=no',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'title="(.+?)"', webpage, 'title')
-        uploader = self._html_search_regex(r'http://www.zippcast.com/profile/(.+?)">', webpage, 'uploader')
-        url = self._html_search_regex(r'<source src="(.+?)" type="', webpage, 'url')
-        description = self._html_search_regex(r'<span class="vdescr".+>(.+?)</span>', webpage, 'description')
-        thumbnail = self._html_search_regex(r'poster="(.+?)" controls>', webpage, 'thumbnail')
-        categories = self._html_search_regex(r'<a href="http://www.zippcast.com/categories/(.+?)"', webpage, 'categories')
-        view_count = self._html_search_regex(r'<td align="right"><h3>(.+?) views!', webpage, 'view_count')
+        webpage = self._download_webpage(
+            'http://www.zippcast.com/video/%s' % video_id, video_id)
+
+        formats = []
+        video_url = self._search_regex(
+            r'<source[^>]+src=(["\'])(?P<url>.+?)\1', webpage,
+            'video url', default=None, group='url')
+        if video_url:
+            formats.append({
+                'url': video_url,
+                'format_id': 'http',
+                'preference': 0,  # direct link is almost always of worse quality
+            })
+        src_url = self._search_regex(
+            r'src\s*:\s*(?:escape\()?(["\'])(?P<url>http://.+?)\1',
+            webpage, 'src', default=None, group='url')
+        ext = determine_ext(src_url)
+        if ext == 'm3u8':
+            formats.extend(self._extract_m3u8_formats(
+                src_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                m3u8_id='hls', fatal=False))
+        elif ext == 'f4m':
+            formats.extend(self._extract_f4m_formats(
+                src_url, video_id, f4m_id='hds', fatal=False))
+        self._sort_formats(formats)
+
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage) or self._html_search_meta(
+            'description', webpage)
+        uploader = self._search_regex(
+            r'<a[^>]+href="https?://[^/]+/profile/[^>]+>([^<]+)</a>',
+            webpage, 'uploader', fatal=False)
+        thumbnail = self._og_search_thumbnail(webpage)
+        view_count = str_to_int(self._search_regex(
+            r'>([\d,.]+) views!', webpage, 'view count', fatal=False))
+
+        categories = re.findall(
+            r'<a[^>]+href="https?://[^/]+/categories/[^"]+">([^<]+),?<',
+            webpage)
+        tags = re.findall(
+            r'<a[^>]+href="https?://[^/]+/search/tags/[^"]+">([^<]+),?<',
+            webpage)
 
         return {
             'id': video_id,
             'title': title,
-            'url': url,
             'description': description,
-            'uploader': uploader,
             'thumbnail': thumbnail,
-            'categories': [categories],
-            'view_count': int(view_count.replace(',', '')),
+            'uploader': uploader,
+            'view_count': view_count,
+            'categories': categories,
+            'tags': tags,
+            'formats': formats,
         }

From a50a8003a08bc67bc99363f6738147a394a43210 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Jan 2016 07:10:51 +0600
Subject: [PATCH 0384/1105] [cultureunplugged] Improve (Closes #8060)

---
 youtube_dl/extractor/__init__.py         |  1 +
 youtube_dl/extractor/cultureunplugged.py | 63 ++++++++++++++++++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 youtube_dl/extractor/cultureunplugged.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5b8e16a29..2c56797a5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -130,6 +130,7 @@ from .crunchyroll import (
 )
 from .cspan import CSpanIE
 from .ctsnews import CtsNewsIE
+from .cultureunplugged import CultureUnpluggedIE
 from .cwtv import CWTVIE
 from .dailymotion import (
     DailymotionIE,
diff --git a/youtube_dl/extractor/cultureunplugged.py b/youtube_dl/extractor/cultureunplugged.py
new file mode 100644
index 000000000..9c764fe68
--- /dev/null
+++ b/youtube_dl/extractor/cultureunplugged.py
@@ -0,0 +1,63 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class CultureUnpluggedIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?cultureunplugged\.com/documentary/watch-online/play/(?P<id>\d+)(?:/(?P<display_id>[^/]+))?'
+    _TESTS = [{
+        'url': 'http://www.cultureunplugged.com/documentary/watch-online/play/53662/The-Next--Best-West',
+        'md5': 'ac6c093b089f7d05e79934dcb3d228fc',
+        'info_dict': {
+            'id': '53662',
+            'display_id': 'The-Next--Best-West',
+            'ext': 'mp4',
+            'title': 'The Next, Best West',
+            'description': 'md5:0423cd00833dea1519cf014e9d0903b1',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'creator': 'Coldstream Creative',
+            'duration': 2203,
+            'view_count': int,
+        }
+    }, {
+        'url': 'http://www.cultureunplugged.com/documentary/watch-online/play/53662',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id') or video_id
+
+        movie_data = self._download_json(
+            'http://www.cultureunplugged.com/movie-data/cu-%s.json' % video_id, display_id)
+
+        video_url = movie_data['url']
+        title = movie_data['title']
+
+        description = movie_data.get('synopsis')
+        creator = movie_data.get('producer')
+        duration = int_or_none(movie_data.get('duration'))
+        view_count = int_or_none(movie_data.get('views'))
+
+        thumbnails = [{
+            'url': movie_data['%s_thumb' % size],
+            'id': size,
+            'preference': preference,
+        } for preference, size in enumerate((
+            'small', 'large')) if movie_data.get('%s_thumb' % size)]
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'url': video_url,
+            'title': title,
+            'description': description,
+            'creator': creator,
+            'duration': duration,
+            'view_count': view_count,
+            'thumbnails': thumbnails,
+        }

From 10d33b347391ed0af453a88d2860a8ad7d8d18c7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 13 Jan 2016 16:24:48 +0800
Subject: [PATCH 0385/1105] [YoutubeDL] Introduce CSS3 like string operators

---
 youtube_dl/YoutubeDL.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0748fbba0..18b1443b8 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -898,6 +898,9 @@ class YoutubeDL(object):
             STR_OPERATORS = {
                 '=': operator.eq,
                 '!=': operator.ne,
+                '^=': lambda attr, value: attr.startswith(value),
+                '$=': lambda attr, value: attr.endswith(value),
+                '*=': lambda attr, value: value in attr,
             }
             str_operator_rex = re.compile(r'''(?x)
                 \s*(?P<key>ext|acodec|vcodec|container|protocol)

From 15870bbd018415231c40d9e4319ef43e563ab7b8 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sat, 16 Jan 2016 07:31:14 +0600
Subject: [PATCH 0386/1105] [README.md] Mention new string operators for format
 selection

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 75253199c..af6907be3 100644
--- a/README.md
+++ b/README.md
@@ -467,7 +467,7 @@ youtube-dl_test_video_.mp4          # A simple file name
 By default youtube-dl tries to download the best quality, but sometimes you may want to download in a different format.
 The simplest case is requesting a specific format, for example `-f 22`. You can get the list of available formats using `--list-formats`, you can also use a file extension (currently it supports aac, m4a, mp3, mp4, ogg, wav, webm) or the special names `best`, `bestvideo`, `bestaudio` and `worst`.
 
-If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`. Format selectors can also be grouped using parentheses, for example if you want to download the best mp4 and webm formats with a height lower than 480 you can use `-f '(mp4,webm)[height<480]'`.
+If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, !=, ^= (begins with), $= (ends with), *= (contains) . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`. Format selectors can also be grouped using parentheses, for example if you want to download the best mp4 and webm formats with a height lower than 480 you can use `-f '(mp4,webm)[height<480]'`.
 
 Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
 

From b2dca40d8138cb044bd6c6f75a4e87ecb633fc00 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sat, 16 Jan 2016 09:49:54 +0600
Subject: [PATCH 0387/1105] [README.md] Improve format selection documentation

---
 README.md | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 63 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index af6907be3..efb51027b 100644
--- a/README.md
+++ b/README.md
@@ -464,15 +464,74 @@ youtube-dl_test_video_.mp4          # A simple file name
 
 # FORMAT SELECTION
 
-By default youtube-dl tries to download the best quality, but sometimes you may want to download in a different format.
-The simplest case is requesting a specific format, for example `-f 22`. You can get the list of available formats using `--list-formats`, you can also use a file extension (currently it supports aac, m4a, mp3, mp4, ogg, wav, webm) or the special names `best`, `bestvideo`, `bestaudio` and `worst`.
+By default youtube-dl tries to download the best available quality, i.e. if you want the best quality you **don't need** to pass any special options, youtube-dl will guess it for you by **default**.
 
-If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, !=, ^= (begins with), $= (ends with), *= (contains) . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`. Format selectors can also be grouped using parentheses, for example if you want to download the best mp4 and webm formats with a height lower than 480 you can use `-f '(mp4,webm)[height<480]'`.
+But sometimes you may want to download in a different format, for example when you are on a slow or intermittent connection. The key mechanism for achieving this is so called *format selection* based on which you can explicitly specify desired format, select formats based on some criterion or criteria, setup precedence and much more.
 
-Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
+The general syntax for format selection is `--format FORMAT` or shorter `-f FORMAT` where `FORMAT` is a *selector expression*, i.e. an expression that describes format or formats you would like to download.
+
+The simplest case is requesting a specific format, for example with `-f 22` you can download the format with format code equal to 22. You can get the list of available format codes for particular video using `--list-formats` or `-F`. Note that these format codes are extractor specific. 
+
+You can also use a file extension (currently `3gp`, `aac`, `flv`, `m4a`, `mp3`, `mp4`, `ogg`, `wav`, `webm` are supported) to download best quality format of particular file extension served as a single file, e.g. `-f webm` will download best quality format with `webm` extension served as a single file.
+
+You can also use special names to select particular edge case format:
+ - `best`: Select best quality format represented by single file with video and audio
+ - `worst`: Select worst quality format represented by single file with video and audio
+ - `bestvideo`: Select best quality video only format (e.g. DASH video), may not be available
+ - `worstvideo`: Select worst quality video only format, may not be available
+ - `bestaudio`: Select best quality audio only format, may not be available
+ - `worstaudio`: Select worst quality audio only format, may not be available
+
+For example, to download worst quality video only format you can use `-f worstvideo`.
+
+If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes. Note that slash is left-associative, i.e. formats on the left hand side are preferred, for example `-f 22/17/18` will download format 22 if it's available, otherwise it will download format 17 if it's available, otherwise it will download format 18 if it's available, otherwise it will complain that no suitable formats are available for download.
+
+If you want to download several formats of the same video use comma as a separator, e.g. `-f 22,17,18` will download all these three formats, of course if they are available. Or more sophisticated example combined with precedence feature `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`.
+
+You can also filter the video formats by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).
+
+The following numeric meta fields can be used with comparisons `<`, `<=`, `>`, `>=`, `=` (equals), `!=` (not equals):
+ - `filesize`: The number of bytes, if known in advance
+ - `width`: Width of the video, if known
+ - `height`: Height of the video, if known
+ - `tbr`: Average bitrate of audio and video in KBit/s
+ - `abr`: Average audio bitrate in KBit/s
+ - `vbr`: Average video bitrate in KBit/s
+ - `asr`: Audio sampling rate in Hertz
+ - `fps`: Frame rate
+
+Also filtering work for comparisons `=` (equals), `!=` (not equals), `^=` (begins with), `$=` (ends with), `*=` (contains) and following string meta fields:
+ - `ext`: File extension
+ - `acodec`: Name of the audio codec in use
+ - `vcodec`: Name of the video codec in use
+ - `container`: Name of the container format
+ - `protocol`: The protocol that will be used for the actual download, lower-case. `http`, `https`, `rtsp`, `rtmp`, `rtmpe`, `m3u8`, or `m3u8_native`
+
+Note that none of the aforementioned meta fields are guaranteed to be present since this solely depends on the metadata obtained by particular extractor, i.e. the metadata offered by video hoster.
+
+Formats for which the value is not known are excluded unless you put a question mark (`?`) after the operator. You can combine format filters, so `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s.
+
+You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv installed), for example `-f bestvideo+bestaudio` will download best video only format, best audio only format and mux them together with ffmpeg/avconv.
+
+Format selectors can also be grouped using parentheses, for example if you want to download the best mp4 and webm formats with a height lower than 480 you can use `-f '(mp4,webm)[height<480]'`.
+
+Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some DASH formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
 
 If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download the best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 
+Examples (note on Windows you may need to use double quotes instead of single):
+```bash
+# Download best mp4 format available or any other best if no mp4 available
+$ youtube-dl -f 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best'
+
+# Download best format available but not better that 480p
+$ youtube-dl -f 'bestvideo[height<=480]+bestaudio/best[height<=480]'
+
+# Download best video only format but no bigger that 50 MB
+$ youtube-dl -f 'best[filesize<50M]'
+```
+
+
 # VIDEO SELECTION
 
 Videos can be filtered by their upload date using the options `--date`, `--datebefore` or `--dateafter`. They accept dates in two formats:

From b555942428726bd619744a208b5e6cfceb99d8d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Jan 2016 10:10:28 +0600
Subject: [PATCH 0388/1105] [YoutubeDL] Ensure protocol is always present

---
 youtube_dl/YoutubeDL.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 18b1443b8..09d2b18f2 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -46,6 +46,7 @@ from .utils import (
     DateRange,
     DEFAULT_OUTTMPL,
     determine_ext,
+    determine_protocol,
     DownloadError,
     encode_compat_str,
     encodeFilename,
@@ -1309,6 +1310,10 @@ class YoutubeDL(object):
             # Automatically determine file extension if missing
             if 'ext' not in format:
                 format['ext'] = determine_ext(format['url']).lower()
+            # Automatically determine protocol if missing (useful for format
+            # selection purposes)
+            if 'protocol' not in format:
+                format['protocol'] = determine_protocol(format)
             # Add HTTP headers, so that external programs can use them from the
             # json output
             full_format_info = info_dict.copy()

From 6be16ed24bba86c0bb22a3eca6640e727153f1fd Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sat, 16 Jan 2016 10:15:24 +0600
Subject: [PATCH 0389/1105] [README.md] Add protocol usage example in format
 selection

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index efb51027b..9dbeae1bc 100644
--- a/README.md
+++ b/README.md
@@ -529,6 +529,9 @@ $ youtube-dl -f 'bestvideo[height<=480]+bestaudio/best[height<=480]'
 
 # Download best video only format but no bigger that 50 MB
 $ youtube-dl -f 'best[filesize<50M]'
+
+# Download best format available via direct link over HTTP/HTTPS protocol
+$ youtube-dl -f '(bestvideo+bestaudio/best)[protocol^=http]'
 ```
 
 

From 3a0a78731b5223b232beaabc3cdc771b9cebb078 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sat, 16 Jan 2016 12:17:07 +0800
Subject: [PATCH 0390/1105] Fixes #8239

---
 youtube_dl/extractor/neteasemusic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index 15eca825a..848ebdea0 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -40,7 +40,7 @@ class NetEaseMusicBaseIE(InfoExtractor):
             if not details:
                 continue
             formats.append({
-                'url': 'http://m5.music.126.net/%s/%s.%s' %
+                'url': 'http://203.130.59.9/m1.music.126.net/%s/%s.%s' %
                        (cls._encrypt(details['dfsId']), details['dfsId'],
                         details['extension']),
                 'ext': details.get('extension'),

From 0b26ba3fc8eee0bb047c5e78e6f1a8ba59fa9457 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Jan 2016 20:45:36 +0800
Subject: [PATCH 0391/1105] [extractor/common] Allow passing more parameters to
 _search_json_ld

---
 youtube_dl/extractor/common.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 92e2e4f43..8da70ae14 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -763,13 +763,13 @@ class InfoExtractor(object):
         return self._html_search_meta('twitter:player', html,
                                       'twitter card player')
 
-    def _search_json_ld(self, html, video_id, fatal=True):
+    def _search_json_ld(self, html, video_id, **kwargs):
         json_ld = self._search_regex(
             r'(?s)<script[^>]+type=(["\'])application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>',
-            html, 'JSON-LD', fatal=fatal, group='json_ld')
+            html, 'JSON-LD', group='json_ld', **kwargs)
         if not json_ld:
             return {}
-        return self._json_ld(json_ld, video_id, fatal=fatal)
+        return self._json_ld(json_ld, video_id, fatal=kwargs.get('fatal', True))
 
     def _json_ld(self, json_ld, video_id, fatal=True):
         if isinstance(json_ld, compat_str):

From 350e02d40d93c341d73b23d630f458b14638fcb6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Jan 2016 20:46:28 +0800
Subject: [PATCH 0392/1105] [bbc] Use _search_json_ld

---
 youtube_dl/extractor/bbc.py | 17 ++++-------------
 1 file changed, 4 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index ce99a34ab..1c493b72d 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -718,19 +718,10 @@ class BBCIE(BBCCoUkIE):
 
         webpage = self._download_webpage(url, playlist_id)
 
-        timestamp = None
-        playlist_title = None
-        playlist_description = None
-
-        ld = self._parse_json(
-            self._search_regex(
-                r'(?s)<script type="application/ld\+json">(.+?)</script>',
-                webpage, 'ld json', default='{}'),
-            playlist_id, fatal=False)
-        if ld:
-            timestamp = parse_iso8601(ld.get('datePublished'))
-            playlist_title = ld.get('headline')
-            playlist_description = ld.get('articleBody')
+        json_ld_info = self._search_json_ld(webpage, playlist_id, default=None)
+        timestamp = json_ld_info.get('timestamp')
+        playlist_title = json_ld_info.get('title')
+        playlist_description = json_ld_info.get('description')
 
         if not timestamp:
             timestamp = parse_iso8601(self._search_regex(

From 1358b94163899d38cfe1106eb4d734996f42328e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Jan 2016 20:56:53 +0800
Subject: [PATCH 0393/1105] [ae] Fix _TESTS

---
 youtube_dl/extractor/ae.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ae.py b/youtube_dl/extractor/ae.py
index 3bc7c12fc..01c189d78 100644
--- a/youtube_dl/extractor/ae.py
+++ b/youtube_dl/extractor/ae.py
@@ -20,13 +20,14 @@ class AEIE(InfoExtractor):
             'skip_download': True,
         },
         'add_ie': ['ThePlatform'],
+        'expected_warnings': ['JSON-LD'],
     }, {
         'url': 'http://www.history.com/shows/mountain-men/season-1/episode-1',
         'info_dict': {
             'id': 'eg47EERs_JsZ',
             'ext': 'mp4',
             'title': "Winter Is Coming",
-            'description': 'md5:a40e370925074260b1c8a633c632c63a',
+            'description': 'md5:641f424b7a19d8e24f26dea22cf59d74',
         },
         'params': {
             # m3u8 download

From 614db89ae39a922ddbbfbd34e667933bc05c6b04 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Jan 2016 22:17:31 +0800
Subject: [PATCH 0394/1105] [compat] Clarify the versions requiring
 compat_kwargs

It's supported since 2.7.0 alpha 1 and 2.6.5 rc 1. See
https://hg.python.org/cpython/file/v2.7a1/Misc/NEWS#l337
https://hg.python.org/cpython/file/v2.6.5rc1/Misc/NEWS#l28
---
 youtube_dl/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index a3e85264a..8ab688001 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -433,7 +433,7 @@ if sys.version_info < (3, 0) and sys.platform == 'win32':
 else:
     compat_getpass = getpass.getpass
 
-# Old 2.6 and 2.7 releases require kwargs to be bytes
+# Python < 2.6.5 require kwargs to be bytes
 try:
     def _testfunc(x):
         pass

From 855f90fa6f348d8128d9c581db4756e26db02944 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Jan 2016 03:02:45 +0600
Subject: [PATCH 0395/1105] [ae] Rename to aenetworks and clarify extractor
 name and description

---
 youtube_dl/extractor/__init__.py              | 2 +-
 youtube_dl/extractor/{ae.py => aenetworks.py} | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)
 rename youtube_dl/extractor/{ae.py => aenetworks.py} (94%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 2c56797a5..0ba0d226f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -15,7 +15,7 @@ from .adobetv import (
     AdobeTVVideoIE,
 )
 from .adultswim import AdultSwimIE
-from .ae import AEIE
+from .aenetworks import AENetworksIE
 from .aftonbladet import AftonbladetIE
 from .airmozilla import AirMozillaIE
 from .aljazeera import AlJazeeraIE
diff --git a/youtube_dl/extractor/ae.py b/youtube_dl/extractor/aenetworks.py
similarity index 94%
rename from youtube_dl/extractor/ae.py
rename to youtube_dl/extractor/aenetworks.py
index 01c189d78..43d7b0523 100644
--- a/youtube_dl/extractor/ae.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -4,7 +4,9 @@ from .common import InfoExtractor
 from ..utils import smuggle_url
 
 
-class AEIE(InfoExtractor):
+class AENetworksIE(InfoExtractor):
+    IE_NAME = 'aenetworks'
+    IE_DESC = 'A+E Networks: A&E, Lifetime, History.com, FYI Network'
     _VALID_URL = r'https?://(?:www\.)?(?:(?:history|aetv|mylifetime)\.com|fyi\.tv)/(?:[^/]+/)+(?P<id>[^/]+?)(?:$|[?#])'
 
     _TESTS = [{

From e0ef13ddebfab39d8c59c8cd9c11882cfc604070 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sun, 17 Jan 2016 07:48:46 +0800
Subject: [PATCH 0396/1105] [neteasemusic] Fallback to alt hosts if
 m5.music.126.net doesn't work

---
 youtube_dl/extractor/neteasemusic.py | 35 +++++++++++++++++-----------
 1 file changed, 22 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index 848ebdea0..435cd8e2e 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -32,23 +32,32 @@ class NetEaseMusicBaseIE(InfoExtractor):
         result = b64encode(m.digest()).decode('ascii')
         return result.replace('/', '_').replace('+', '-')
 
-    @classmethod
-    def extract_formats(cls, info):
+    def extract_formats(self, info):
         formats = []
-        for song_format in cls._FORMATS:
+        for song_format in self._FORMATS:
             details = info.get(song_format)
             if not details:
                 continue
-            formats.append({
-                'url': 'http://203.130.59.9/m1.music.126.net/%s/%s.%s' %
-                       (cls._encrypt(details['dfsId']), details['dfsId'],
-                        details['extension']),
-                'ext': details.get('extension'),
-                'abr': details.get('bitrate', 0) / 1000,
-                'format_id': song_format,
-                'filesize': details.get('size'),
-                'asr': details.get('sr')
-            })
+            song_file_path = '/%s/%s.%s' % (
+                self._encrypt(details['dfsId']), details['dfsId'], details['extension'])
+
+            # 203.130.59.9, 124.40.233.182, 115.231.74.139, etc is a reverse proxy-like feature
+            # from NetEase's CDN provider that can be used if m5.music.126.net does not
+            # work, especially for users outside of Mainland China
+            # via: https://github.com/JixunMoe/unblock-163/issues/3#issuecomment-163115880
+            for host in ('http://m5.music.126.net', 'http://115.231.74.139/m1.music.126.net',
+                         'http://124.40.233.182/m1.music.126.net', 'http://203.130.59.9/m1.music.126.net'):
+                song_url = host + song_file_path
+                if self._is_valid_url(song_url, info['id'], 'song'):
+                    formats.append({
+                        'url': song_url,
+                        'ext': details.get('extension'),
+                        'abr': details.get('bitrate', 0) / 1000,
+                        'format_id': song_format,
+                        'filesize': details.get('size'),
+                        'asr': details.get('sr')
+                    })
+                    break
         return formats
 
     @classmethod

From e5a66240c0ddb326b3f885cc29b6f00058481739 Mon Sep 17 00:00:00 2001
From: Jakub Wilk <jwilk@jwilk.net>
Date: Sun, 17 Jan 2016 15:06:00 +0100
Subject: [PATCH 0397/1105] [ustream] Use HTTPS for GitHub URL

---
 youtube_dl/extractor/ustream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 73b05ecab..b5fe753d7 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -47,7 +47,7 @@ class UstreamIE(InfoExtractor):
         m = re.match(self._VALID_URL, url)
         video_id = m.group('id')
 
-        # some sites use this embed format (see: http://github.com/rg3/youtube-dl/issues/2990)
+        # some sites use this embed format (see: https://github.com/rg3/youtube-dl/issues/2990)
         if m.group('type') == 'embed/recorded':
             video_id = m.group('id')
             desktop_url = 'http://www.ustream.tv/recorded/' + video_id

From dae503afaa8dbbf745a3c796dab1506cd736de83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Jan 2016 22:14:07 +0600
Subject: [PATCH 0398/1105] [atresplayer] Skip HLS completely (Closes #8261)

---
 youtube_dl/extractor/atresplayer.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 3fb042cea..b8f9ae005 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -132,11 +132,6 @@ class AtresPlayerIE(InfoExtractor):
                 })
             formats.append(format_info)
 
-        m3u8_url = player.get('urlVideoHls')
-        if m3u8_url:
-            formats.extend(self._extract_m3u8_formats(
-                m3u8_url, episode_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
-
         timestamp = int_or_none(self._download_webpage(
             self._TIME_API_URL,
             video_id, 'Downloading timestamp', fatal=False), 1000, time.time())

From 096b5339823fc65beeca6cff96867ed03e6234db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 Jan 2016 23:17:45 +0600
Subject: [PATCH 0399/1105] [youtube] Fix URL expansion in video description

Fixes test_Youtube_18
---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e4f227f19..50c1363cd 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1237,7 +1237,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     (?:[a-zA-Z-]+="[^"]+"\s+)*?
                     (?:title|href)="([^"]+)"\s+
                     (?:[a-zA-Z-]+="[^"]+"\s+)*?
-                    class="(?:yt-uix-redirect-link|yt-uix-sessionlink[^"]*)".*?>
+                    class="(?:yt-uix-redirect-link|yt-uix-sessionlink[^"]*)"[^>]*>
                 [^<]+\.{3}\s*
                 </a>
             ''', r'\1', video_description)

From a8776b107b0137fc13d8fd4350035d02e7bb7761 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 Jan 2016 23:19:38 +0600
Subject: [PATCH 0400/1105] [youtube] Clarify test_Youtube_18

---
 youtube_dl/extractor/youtube.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 50c1363cd..2045f5386 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -706,6 +706,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         },
         {
             # Title with JS-like syntax "};" (see https://github.com/rg3/youtube-dl/issues/7468)
+            # Also tests cut-off URL expansion in video description (see
+            # https://github.com/rg3/youtube-dl/issues/1892,
+            # https://github.com/rg3/youtube-dl/issues/8164)
             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
             'info_dict': {
                 'id': 'lsguqyKfVQg',

From 8ceabd4df332d9c39f71a62bbd60cbea9fcb6621 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Jan 2016 20:54:43 +0600
Subject: [PATCH 0401/1105] [youtube] Capture and output unavailable message

---
 youtube_dl/extractor/youtube.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2045f5386..2a0c2e5a6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1508,6 +1508,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             for a_format in formats:
                 a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = 'True'
         else:
+            unavailable_message = self._html_search_regex(
+                r'(?s)<h1[^>]+id="unavailable-message"[^>]*>(.+?)</h1>',
+                video_webpage, 'unavailable message', default=None)
+            if unavailable_message:
+                raise ExtractorError(unavailable_message, expected=True)
             raise ExtractorError('no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
 
         # Look for the DASH manifest

From e323cf3ff37b1268f585b66159de20bd8ffe8dc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Jan 2016 20:56:04 +0600
Subject: [PATCH 0402/1105] [youtube] Skip test

---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2a0c2e5a6..d31161d21 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -613,7 +613,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             },
             'params': {
                 'skip_download': 'requires avconv',
-            }
+            },
+            'skip': 'This live event has ended.',
         },
         # Extraction from multiple DASH manifests (https://github.com/rg3/youtube-dl/pull/6097)
         {

From 9fbd0822aa9bccb319446e6d75a76ab4f23ddf46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Jan 2016 21:20:14 +0600
Subject: [PATCH 0403/1105] [dailymotion] Extend _VALID_URL

---
 youtube_dl/extractor/dailymotion.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index b687ec4d6..6e462af69 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -37,7 +37,7 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
 
 
 class DailymotionIE(DailymotionBaseInfoExtractor):
-    _VALID_URL = r'(?i)(?:https?://)?(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(embed|#)/)?video/(?P<id>[^/?_]+)'
+    _VALID_URL = r'(?i)(?:https?://)?(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(?:embed|swf|#)/)?video/(?P<id>[^/?_]+)'
     IE_NAME = 'dailymotion'
 
     _FORMATS = [
@@ -104,6 +104,10 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
         {
             'url': 'http://www.dailymotion.com/video/x20su5f_the-power-of-nightmares-1-the-rise-of-the-politics-of-fear-bbc-2004_news',
             'only_matching': True,
+        },
+        {
+            'url': 'http://www.dailymotion.com/swf/video/x3n92nf',
+            'only_matching': True,
         }
     ]
 
@@ -336,7 +340,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
 
 class DailymotionUserIE(DailymotionPlaylistIE):
     IE_NAME = 'dailymotion:user'
-    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?!(?:embed|#|video|playlist)/)(?:(?:old/)?user/)?(?P<user>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?!(?:embed|swf|#|video|playlist)/)(?:(?:old/)?user/)?(?P<user>[^/]+)'
     _PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
     _TESTS = [{
         'url': 'https://www.dailymotion.com/user/nqtv',

From 1fa2b9841ddafd41c3aebe40976445b3b7087f9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Jan 2016 21:20:45 +0600
Subject: [PATCH 0404/1105] [extractor/generic] Extend dailymotion embed regex

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b3f8efc80..0baa17e8d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1402,7 +1402,7 @@ class GenericIE(InfoExtractor):
 
         # Look for embedded Dailymotion player
         matches = re.findall(
-            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/embed/video/.+?)\1', webpage)
+            r'<(?:embed|iframe)[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/(?:embed|swf)/video/.+?)\1', webpage)
         if matches:
             return _playlist_from_matches(
                 matches, lambda m: unescapeHTML(m[1]))

From 942d46196ffc18bc7444bbc2c822824efcd12bdc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Jan 2016 21:47:06 +0600
Subject: [PATCH 0405/1105] [ultimedia] Extend _VALID_URL to support digiteka

---
 youtube_dl/extractor/ultimedia.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ultimedia.py b/youtube_dl/extractor/ultimedia.py
index 60328123c..eef8705e0 100644
--- a/youtube_dl/extractor/ultimedia.py
+++ b/youtube_dl/extractor/ultimedia.py
@@ -9,7 +9,7 @@ from ..utils import int_or_none
 
 class UltimediaIE(InfoExtractor):
     _VALID_URL = r'''(?x)
-        https?://(?:www\.)?ultimedia\.com/
+        https?://(?:www\.)?(?:digiteka\.net|ultimedia\.com)/
         (?:
             deliver/
             (?P<embed_type>
@@ -56,6 +56,9 @@ class UltimediaIE(InfoExtractor):
             'timestamp': 1424760500,
             'uploader_id': '3rfzk',
         },
+    }, {
+        'url': 'https://www.digiteka.net/deliver/generic/iframe/mdtk/01637594/src/lqm3kl/zone/1/showtitle/1/autoplay/yes',
+        'only_matching': True,
     }]
 
     @staticmethod

From aecfcd4e595121d17b173f1ee9f4444ee9c0fda2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Jan 2016 21:51:46 +0600
Subject: [PATCH 0406/1105] [ultimedia] Rename to digiteka

---
 youtube_dl/extractor/__init__.py                   |  2 +-
 youtube_dl/extractor/{ultimedia.py => digiteka.py} |  2 +-
 youtube_dl/extractor/generic.py                    | 10 +++++-----
 3 files changed, 7 insertions(+), 7 deletions(-)
 rename youtube_dl/extractor/{ultimedia.py => digiteka.py} (99%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0ba0d226f..1d838cdd0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -777,7 +777,7 @@ from .udemy import (
     UdemyCourseIE
 )
 from .udn import UDNEmbedIE
-from .ultimedia import UltimediaIE
+from .digiteka import DigitekaIE
 from .unistra import UnistraIE
 from .urort import UrortIE
 from .ustream import UstreamIE, UstreamChannelIE
diff --git a/youtube_dl/extractor/ultimedia.py b/youtube_dl/extractor/digiteka.py
similarity index 99%
rename from youtube_dl/extractor/ultimedia.py
rename to youtube_dl/extractor/digiteka.py
index eef8705e0..7bb79ffda 100644
--- a/youtube_dl/extractor/ultimedia.py
+++ b/youtube_dl/extractor/digiteka.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 from ..utils import int_or_none
 
 
-class UltimediaIE(InfoExtractor):
+class DigitekaIE(InfoExtractor):
     _VALID_URL = r'''(?x)
         https?://(?:www\.)?(?:digiteka\.net|ultimedia\.com)/
         (?:
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 0baa17e8d..26d3698c8 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -57,7 +57,7 @@ from .pladform import PladformIE
 from .videomore import VideomoreIE
 from .googledrive import GoogleDriveIE
 from .jwplatform import JWPlatformIE
-from .ultimedia import UltimediaIE
+from .digiteka import DigitekaIE
 
 
 class GenericIE(InfoExtractor):
@@ -1814,10 +1814,10 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(unescapeHTML(mobj.group('url')), 'ScreenwaveMedia')
 
-        # Look for Ulltimedia embeds
-        ultimedia_url = UltimediaIE._extract_url(webpage)
-        if ultimedia_url:
-            return self.url_result(self._proto_relative_url(ultimedia_url), 'Ultimedia')
+        # Look for Digiteka embeds
+        digiteka_url = DigitekaIE._extract_url(webpage)
+        if digiteka_url:
+            return self.url_result(self._proto_relative_url(digiteka_url), DigitekaIE.ie_key())
 
         # Look for AdobeTVVideo embeds
         mobj = re.search(

From b605ebb609dc208a814ab44fff84463bf0c5369a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Jan 2016 22:09:55 +0600
Subject: [PATCH 0407/1105] [lemonde] Add extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/lemonde.py  | 34 ++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)
 create mode 100644 youtube_dl/extractor/lemonde.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1d838cdd0..0cc339a37 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -333,6 +333,7 @@ from .kuwo import (
 from .la7 import LA7IE
 from .laola1tv import Laola1TvIE
 from .lecture2go import Lecture2GoIE
+from .lemonde import LemondeIE
 from .letv import (
     LetvIE,
     LetvTvIE,
diff --git a/youtube_dl/extractor/lemonde.py b/youtube_dl/extractor/lemonde.py
new file mode 100644
index 000000000..be66fff03
--- /dev/null
+++ b/youtube_dl/extractor/lemonde.py
@@ -0,0 +1,34 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class LemondeIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:.+?\.)?lemonde\.fr/(?:[^/]+/)*(?P<id>[^/]+)\.html'
+    _TESTS = [{
+        'url': 'http://www.lemonde.fr/police-justice/video/2016/01/19/comprendre-l-affaire-bygmalion-en-cinq-minutes_4849702_1653578.html',
+        'md5': '01fb3c92de4c12c573343d63e163d302',
+        'info_dict': {
+            'id': 'lqm3kl',
+            'ext': 'mp4',
+            'title': "Comprendre l'affaire Bygmalion en 5 minutes",
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 320,
+            'upload_date': '20160119',
+            'timestamp': 1453194778,
+            'uploader_id': '3pmkp',
+        },
+    }, {
+        'url': 'http://redaction.actu.lemonde.fr/societe/video/2016/01/18/calais-debut-des-travaux-de-defrichement-dans-la-jungle_4849233_3224.html',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        digiteka_url = self._proto_relative_url(self._search_regex(
+            r'url\s*:\s*(["\'])(?P<url>(?:https?://)?//(?:www\.)?(?:digiteka\.net|ultimedia\.com)/deliver/.+?)\1',
+            webpage, 'digiteka url', group='url'))
+        return self.url_result(digiteka_url, 'Digiteka')

From 485139c15c91af03bdb89c7ac07563d9fb979363 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 20 Jan 2016 00:02:46 +0600
Subject: [PATCH 0408/1105] [viewster] Tolerate missing synopsis (Closes #8274)

---
 youtube_dl/extractor/viewster.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 185b1c119..fe94a4793 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -155,10 +155,10 @@ class ViewsterIE(InfoExtractor):
 
         self._sort_formats(formats)
 
-        synopsis = info.get('Synopsis', {})
+        synopsis = info.get('Synopsis') or {}
         # Prefer title outside synopsis since it's less messy
         title = (info.get('Title') or synopsis['Title']).strip()
-        description = synopsis.get('Detailed') or info.get('Synopsis', {}).get('Short')
+        description = synopsis.get('Detailed') or (info.get('Synopsis') or {}).get('Short')
         duration = int_or_none(info.get('Duration'))
         timestamp = parse_iso8601(info.get('ReleaseDate'))
 

From 10defdd06a1ea878087f5bad6ee99da3b2f9d3f6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 03:17:35 +0800
Subject: [PATCH 0409/1105] [letv] Reduce duplicated codes

---
 youtube_dl/extractor/letv.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index c096cb1ab..d43b4e808 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -281,14 +281,17 @@ class LetvCloudIE(InfoExtractor):
             "uu=" + uu + "&vu=" + vu)
         play_json = self._download_json(play_json_req, media_id, 'Downloading playJson data')
 
-        formats = [{
-            'url': base64.b64decode(media['play_url']['main_url'].encode('utf-8')).decode("utf-8"),
-            'ext': 'mp4',
-            'format_id': int_or_none(media.get('play_url', {}).get('vtype')),
-            'format_note': str_or_none(media.get('play_url', {}).get('definition')),
-            'width': int_or_none(media.get('play_url', {}).get('vwidth')),
-            'height': int_or_none(media.get('play_url', {}).get('vheight')),
-        } for media in play_json['data']['video_info']['media'].values()]
+        formats = []
+        for media in play_json['data']['video_info']['media'].values():
+            play_url = media['play_url']
+            formats.append({
+                'url': base64.b64decode(play_url['main_url'].encode('utf-8')).decode("utf-8"),
+                'ext': 'mp4',
+                'format_id': int_or_none(play_url.get('vtype')),
+                'format_note': str_or_none(play_url.get('definition')),
+                'width': int_or_none(play_url.get('vwidth')),
+                'height': int_or_none(play_url.get('vheight')),
+            })
         self._sort_formats(formats)
 
         return {

From 8fff4f61e5741e56890ccb108f42b0c7dc6607e8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 03:18:54 +0800
Subject: [PATCH 0410/1105] [letv] Use single quotes

---
 youtube_dl/extractor/letv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index d43b4e808..be6c75967 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -278,14 +278,14 @@ class LetvCloudIE(InfoExtractor):
 
         play_json_req = sanitized_Request(
             'http://api.letvcloud.com/gpc.php?cf=html5&sign=signxxxxx&ver=2.2&format=json&' +
-            "uu=" + uu + "&vu=" + vu)
+            'uu=' + uu + '&vu=' + vu)
         play_json = self._download_json(play_json_req, media_id, 'Downloading playJson data')
 
         formats = []
         for media in play_json['data']['video_info']['media'].values():
             play_url = media['play_url']
             formats.append({
-                'url': base64.b64decode(play_url['main_url'].encode('utf-8')).decode("utf-8"),
+                'url': base64.b64decode(play_url['main_url'].encode('utf-8')).decode('utf-8'),
                 'ext': 'mp4',
                 'format_id': int_or_none(play_url.get('vtype')),
                 'format_note': str_or_none(play_url.get('definition')),

From e0690782b8531f9962950693fe33a5d4a4f494f6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 03:25:12 +0800
Subject: [PATCH 0411/1105] [letv] LetvCloud: guard against invalid URLs

---
 youtube_dl/extractor/letv.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index be6c75967..b8d4f5bb8 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -272,8 +272,14 @@ class LetvCloudIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        uu = re.search('uu=([\w]+)', url).group(1)
-        vu = re.search('vu=([\w]+)', url).group(1)
+        uu_mobj = re.search('uu=([\w]+)', url)
+        vu_mobj = re.search('vu=([\w]+)', url)
+
+        if not uu_mobj or not vu_mobj:
+            raise ExtractorError('Invalid URL: %s' % url, expected=True)
+
+        uu = uu_mobj.group(1)
+        vu = vu_mobj.group(1)
         media_id = uu + '_' + vu
 
         play_json_req = sanitized_Request(

From 26de1bba83b3269653b571682cf962899b6231b7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 03:31:34 +0800
Subject: [PATCH 0412/1105] [letv] LetvCloud: check error messages from server

---
 youtube_dl/extractor/letv.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index b8d4f5bb8..177d53bbb 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -287,6 +287,14 @@ class LetvCloudIE(InfoExtractor):
             'uu=' + uu + '&vu=' + vu)
         play_json = self._download_json(play_json_req, media_id, 'Downloading playJson data')
 
+        if not play_json.get('data'):
+            if play_json.get('message'):
+                raise ExtractorError('Letv cloud said: %s' % play_json['message'], expected=True)
+            elif play_json.get('code'):
+                raise ExtractorError('Letv cloud returned error %d' % play_json['code'], expected=True)
+            else:
+                raise ExtractorError('Letv cloud returned an unknwon error')
+
         formats = []
         for media in play_json['data']['video_info']['media'].values():
             play_url = media['play_url']

From 73e74424561b3e277c2f071fd88b17c845a1162f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 03:39:45 +0800
Subject: [PATCH 0413/1105] [letv] LetvCloud: simplify and improve _VALID_URL

---
 youtube_dl/extractor/letv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 177d53bbb..ff74b5141 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -245,7 +245,7 @@ class LetvPlaylistIE(LetvTvIE):
 
 class LetvCloudIE(InfoExtractor):
     IE_DESC = '乐视云'
-    _VALID_URL = r'http://yuntv\.letv\.com/bcloud.html\?.*$'
+    _VALID_URL = r'https?://yuntv\.letv\.com/bcloud.html\?.+'
 
     _TESTS = [{
         'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=467623dedf',

From 0428106da33419302ac8ad31306f9357549f086e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 03:53:17 +0800
Subject: [PATCH 0414/1105] [letv] LetvCloud: make title looks like a title

---
 youtube_dl/extractor/letv.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index ff74b5141..b15ac163e 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -253,21 +253,21 @@ class LetvCloudIE(InfoExtractor):
         'info_dict': {
             'id': 'p7jnfw5hw9_467623dedf',
             'ext': 'mp4',
-            'title': 'p7jnfw5hw9_467623dedf',
+            'title': 'Video p7jnfw5hw9_467623dedf',
         },
     }, {
         'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=ec93197892&pu=2c7cd40209&auto_play=1&gpcflag=1&width=640&height=360',
         'info_dict': {
             'id': 'p7jnfw5hw9_ec93197892',
             'ext': 'mp4',
-            'title': 'p7jnfw5hw9_ec93197892',
+            'title': 'Video p7jnfw5hw9_ec93197892',
         },
     }, {
         'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=187060b6fd',
         'info_dict': {
             'id': 'p7jnfw5hw9_187060b6fd',
             'ext': 'mp4',
-            'title': 'p7jnfw5hw9_187060b6fd',
+            'title': 'Video p7jnfw5hw9_187060b6fd',
         },
     }]
 
@@ -310,6 +310,6 @@ class LetvCloudIE(InfoExtractor):
 
         return {
             'id': media_id,
-            'title': media_id,
+            'title': 'Video %s' % media_id,
             'formats': formats,
         }

From bec30224ff86d11c3f61808dbe683f97940ba23e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 04:00:37 +0800
Subject: [PATCH 0415/1105] [letv] LetvCloud: Detect ext instead of the
 hardcoded one

---
 youtube_dl/extractor/letv.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index b15ac163e..08bdae8a2 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -19,6 +19,7 @@ from ..utils import (
     int_or_none,
     str_or_none,
     encode_data_uri,
+    url_basename,
 )
 
 
@@ -295,12 +296,17 @@ class LetvCloudIE(InfoExtractor):
             else:
                 raise ExtractorError('Letv cloud returned an unknwon error')
 
+        def b64decode(s):
+            return base64.b64decode(s.encode('utf-8')).decode('utf-8')
+
         formats = []
         for media in play_json['data']['video_info']['media'].values():
             play_url = media['play_url']
+            url = b64decode(play_url['main_url'])
+            decoded_url = b64decode(url_basename(url))
             formats.append({
-                'url': base64.b64decode(play_url['main_url'].encode('utf-8')).decode('utf-8'),
-                'ext': 'mp4',
+                'url': url,
+                'ext': determine_ext(decoded_url),
                 'format_id': int_or_none(play_url.get('vtype')),
                 'format_note': str_or_none(play_url.get('definition')),
                 'width': int_or_none(play_url.get('vwidth')),

From 65ced034b851f890f894ce79d8069de852208dfc Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 04:02:30 +0800
Subject: [PATCH 0416/1105] [weiqitv] Make codes shorter

---
 youtube_dl/extractor/weiqitv.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/weiqitv.py b/youtube_dl/extractor/weiqitv.py
index da3b3d145..b72df1e8d 100644
--- a/youtube_dl/extractor/weiqitv.py
+++ b/youtube_dl/extractor/weiqitv.py
@@ -37,13 +37,11 @@ class WeiqitvIE(InfoExtractor):
         page = self._download_webpage(url, media_id)
 
         info_json_str = self._search_regex(
-            'var\s+video\s*=\s*(.+});',
-            page, 'info_json_str')
+            'var\s+video\s*=\s*(.+});', page, 'info json str')
         info_json = self._parse_json(info_json_str, media_id)
 
         letvcloud_url = self._search_regex(
-            'var\s+letvurl\s*=\s*"([^"]+)',
-            page, 'letvcloud_url')
+            'var\s+letvurl\s*=\s*"([^"]+)', page, 'letvcloud url')
 
         return {
             '_type': 'url_transparent',

From f15a9ca301799c44300d16ac7e61e5b36d67fd31 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 04:03:57 +0800
Subject: [PATCH 0417/1105] [weiqitv] Rename the extractor - capitilize 'TV'

---
 youtube_dl/extractor/__init__.py | 2 +-
 youtube_dl/extractor/weiqitv.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index aeb67354e..5ec017d5d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -857,7 +857,7 @@ from .webofstories import (
     WebOfStoriesPlaylistIE,
 )
 from .weibo import WeiboIE
-from .weiqitv import WeiqitvIE
+from .weiqitv import WeiqiTVIE
 from .wimp import WimpIE
 from .wistia import WistiaIE
 from .worldstarhiphop import WorldStarHipHopIE
diff --git a/youtube_dl/extractor/weiqitv.py b/youtube_dl/extractor/weiqitv.py
index b72df1e8d..24389d4bb 100644
--- a/youtube_dl/extractor/weiqitv.py
+++ b/youtube_dl/extractor/weiqitv.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 
 
-class WeiqitvIE(InfoExtractor):
+class WeiqiTVIE(InfoExtractor):
     IE_DESC = 'WQTV'
     _VALID_URL = r'http://www\.weiqitv\.com/index/video_play\?videoId=(?P<id>[A-Za-z0-9]+)'
 

From 8652bd22f1ee955940793874b5c4060ef8e273b3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 04:04:39 +0800
Subject: [PATCH 0418/1105] [weiqitv] Use single quotes

---
 youtube_dl/extractor/weiqitv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/weiqitv.py b/youtube_dl/extractor/weiqitv.py
index 24389d4bb..e333ae345 100644
--- a/youtube_dl/extractor/weiqitv.py
+++ b/youtube_dl/extractor/weiqitv.py
@@ -45,7 +45,7 @@ class WeiqiTVIE(InfoExtractor):
 
         return {
             '_type': 'url_transparent',
-            "ie_key": 'LetvCloud',
+            'ie_key': 'LetvCloud',
             'url': letvcloud_url,
             'title': info_json['name'],
             'id': media_id,

From fe6856b0595b85c8070d7ccc154cd8ac2ff85e2c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 04:21:51 +0800
Subject: [PATCH 0419/1105] [neteasemusic] Use float_or_none

---
 youtube_dl/extractor/neteasemusic.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index 435cd8e2e..7830616f8 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -12,7 +12,10 @@ from ..compat import (
     compat_str,
     compat_itertools_count,
 )
-from ..utils import sanitized_Request
+from ..utils import (
+    sanitized_Request,
+    float_or_none,
+)
 
 
 class NetEaseMusicBaseIE(InfoExtractor):
@@ -52,7 +55,7 @@ class NetEaseMusicBaseIE(InfoExtractor):
                     formats.append({
                         'url': song_url,
                         'ext': details.get('extension'),
-                        'abr': details.get('bitrate', 0) / 1000,
+                        'abr': float_or_none(details.get('bitrate'), scale=1000),
                         'format_id': song_format,
                         'filesize': details.get('size'),
                         'asr': details.get('sr')

From 9d90e7de03c20b8365ffbcbb17cad58cbfbe35a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 16 Jan 2016 21:51:25 +0100
Subject: [PATCH 0420/1105] [downloader/hls] Ask ffmpeg to quit when
 interrupting youtube-dl with 'Ctrl+C' (#8252)

Otherwise the mp4 file can't be played.
---
 youtube_dl/downloader/hls.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index d186d402d..10b83c6b2 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -46,7 +46,16 @@ class HlsFD(FileDownloader):
 
         self._debug_cmd(args)
 
-        retval = subprocess.call(args, stdin=subprocess.PIPE)
+        proc = subprocess.Popen(args, stdin=subprocess.PIPE)
+        try:
+            retval = proc.wait()
+        except KeyboardInterrupt:
+            # subprocces.run would send the SIGKILL signal to ffmpeg and the
+            # mp4 file couldn't be played, but if we ask ffmpeg to quit it
+            # produces a file that is playable (this is mostly useful for live
+            # streams)
+            proc.communicate(b'q')
+            raise
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen('\r[%s] %s bytes' % (args[0], fsize))

From 56aa074538afc4941b437e2d91a3e298d5f20eb9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 13:20:03 +0800
Subject: [PATCH 0421/1105] Credit @FounderSG for WeiqiTV and LetvCloud (#7994)

[ci skip]
---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index f4238e1d3..73fcafeb5 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -153,3 +153,4 @@ flatgreen
 Brian Foley
 Vignesh Venkat
 Tom Gijselinck
+Founder Fang

From 27b83249c90fa0500bff68b5aee7874100bba5e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 20 Jan 2016 20:00:51 +0600
Subject: [PATCH 0422/1105] [tube8] Fix extraction and extract all formats
 (Closes #8281)

---
 youtube_dl/extractor/tube8.py | 34 +++++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index 46ef61ff5..6d7ca4874 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -1,10 +1,9 @@
 from __future__ import unicode_literals
 
-import json
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlparse
+from ..compat import compat_str
 from ..utils import (
     int_or_none,
     sanitized_Request,
@@ -44,14 +43,28 @@ class Tube8IE(InfoExtractor):
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, display_id)
 
-        flashvars = json.loads(self._html_search_regex(
-            r'flashvars\s*=\s*({.+?});\r?\n', webpage, 'flashvars'))
+        flashvars = self._parse_json(
+            self._search_regex(
+                r'flashvars\s*=\s*({.+?});\r?\n', webpage, 'flashvars'),
+            video_id)
 
-        video_url = flashvars['video_url']
-        if flashvars.get('encrypted') is True:
-            video_url = aes_decrypt_text(video_url, flashvars['video_title'], 32).decode('utf-8')
-        path = compat_urllib_parse_urlparse(video_url).path
-        format_id = '-'.join(path.split('/')[4].split('_')[:2])
+        formats = []
+        for key, video_url in flashvars.items():
+            if not isinstance(video_url, compat_str) or not video_url.startswith('http'):
+                continue
+            height = self._search_regex(
+                r'quality_(\d+)[pP]', key, 'height', default=None)
+            if not height:
+                continue
+            if flashvars.get('encrypted') is True:
+                video_url = aes_decrypt_text(
+                    video_url, flashvars['video_title'], 32).decode('utf-8')
+            formats.append({
+                'url': video_url,
+                'format_id': '%sp' % height,
+                'height': int(height),
+            })
+        self._sort_formats(formats)
 
         thumbnail = flashvars.get('image_url')
 
@@ -79,15 +92,14 @@ class Tube8IE(InfoExtractor):
         return {
             'id': video_id,
             'display_id': display_id,
-            'url': video_url,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
             'uploader': uploader,
-            'format_id': format_id,
             'view_count': view_count,
             'like_count': like_count,
             'dislike_count': dislike_count,
             'comment_count': comment_count,
             'age_limit': 18,
+            'formats': formats,
         }

From 7dde358adcc3d6ef9e7a174cbeadde4d27029669 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 20 Jan 2016 20:07:32 +0600
Subject: [PATCH 0423/1105] [tube8] Extract duration and modernize

---
 youtube_dl/extractor/tube8.py | 56 +++++++++++++++++------------------
 1 file changed, 27 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index 6d7ca4874..378cb0570 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -14,25 +14,23 @@ from ..aes import aes_decrypt_text
 
 class Tube8IE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?tube8\.com/(?:[^/]+/)+(?P<display_id>[^/]+)/(?P<id>\d+)'
-    _TESTS = [
-        {
-            'url': 'http://www.tube8.com/teen/kasia-music-video/229795/',
-            'md5': '44bf12b98313827dd52d35b8706a4ea0',
-            'info_dict': {
-                'id': '229795',
-                'display_id': 'kasia-music-video',
-                'ext': 'mp4',
-                'description': 'hot teen Kasia grinding',
-                'uploader': 'unknown',
-                'title': 'Kasia music video',
-                'age_limit': 18,
-            }
-        },
-        {
-            'url': 'http://www.tube8.com/shemale/teen/blonde-cd-gets-kidnapped-by-two-blacks-and-punished-for-being-a-slutty-girl/19569151/',
-            'only_matching': True,
-        },
-    ]
+    _TESTS = [{
+        'url': 'http://www.tube8.com/teen/kasia-music-video/229795/',
+        'md5': '65e20c48e6abff62ed0c3965fff13a39',
+        'info_dict': {
+            'id': '229795',
+            'display_id': 'kasia-music-video',
+            'ext': 'mp4',
+            'description': 'hot teen Kasia grinding',
+            'uploader': 'unknown',
+            'title': 'Kasia music video',
+            'age_limit': 18,
+            'duration': 230,
+        }
+    },{
+        'url': 'http://www.tube8.com/shemale/teen/blonde-cd-gets-kidnapped-by-two-blacks-and-punished-for-being-a-slutty-girl/19569151/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -75,19 +73,18 @@ class Tube8IE(InfoExtractor):
         uploader = self._html_search_regex(
             r'<span class="username">\s*(.+?)\s*<',
             webpage, 'uploader', fatal=False)
+        duration = int_or_none(flashvars.get('video_duration'))
 
-        like_count = int_or_none(self._html_search_regex(
+        like_count = int_or_none(self._search_regex(
             r'rupVar\s*=\s*"(\d+)"', webpage, 'like count', fatal=False))
-        dislike_count = int_or_none(self._html_search_regex(
+        dislike_count = int_or_none(self._search_regex(
             r'rdownVar\s*=\s*"(\d+)"', webpage, 'dislike count', fatal=False))
-        view_count = self._html_search_regex(
-            r'<strong>Views: </strong>([\d,\.]+)\s*</li>', webpage, 'view count', fatal=False)
-        if view_count:
-            view_count = str_to_int(view_count)
-        comment_count = self._html_search_regex(
-            r'<span id="allCommentsCount">(\d+)</span>', webpage, 'comment count', fatal=False)
-        if comment_count:
-            comment_count = str_to_int(comment_count)
+        view_count = str_to_int(self._search_regex(
+            r'<strong>Views: </strong>([\d,\.]+)\s*</li>',
+            webpage, 'view count', fatal=False))
+        comment_count = str_to_int(self._search_regex(
+            r'<span id="allCommentsCount">(\d+)</span>',
+            webpage, 'comment count', fatal=False))
 
         return {
             'id': video_id,
@@ -96,6 +93,7 @@ class Tube8IE(InfoExtractor):
             'description': description,
             'thumbnail': thumbnail,
             'uploader': uploader,
+            'duration': duration,
             'view_count': view_count,
             'like_count': like_count,
             'dislike_count': dislike_count,

From 1f78ed189aab9b7ea17594ac1f33662d37433455 Mon Sep 17 00:00:00 2001
From: CeruleanSky <CeruleanSky@users.noreply.github.com>
Date: Mon, 18 Jan 2016 17:10:16 -0500
Subject: [PATCH 0424/1105] [OraTV] update extractor

"current" is now "video"
"hls_stream" is now hls_stream without quotes
video_id is now id
duration for current video is not present(for other videos it is)

modified regex to find hls_stream variable to work reguardless of whether it is quoted or not.

[ora] Improve (Closes #8273)
---
 youtube_dl/extractor/ora.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/ora.py b/youtube_dl/extractor/ora.py
index 02de1502a..8545fb1b8 100644
--- a/youtube_dl/extractor/ora.py
+++ b/youtube_dl/extractor/ora.py
@@ -21,7 +21,6 @@ class OraTVIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Vine & YouTube Stars Zach King & King Bach On Their Viral Videos!',
             'description': 'md5:ebbc5b1424dd5dba7be7538148287ac1',
-            'duration': 1477,
         }
     }
 
@@ -30,9 +29,9 @@ class OraTVIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         video_data = self._search_regex(
-            r'"current"\s*:\s*({[^}]+?})', webpage, 'current video')
+            r'"(?:video|current)"\s*:\s*({[^}]+?})', webpage, 'current video')
         m3u8_url = self._search_regex(
-            r'"hls_stream"\s*:\s*"([^"]+)', video_data, 'm3u8 url', None)
+            r'hls_stream"?\s*:\s*"([^"]+)', video_data, 'm3u8 url', None)
         if m3u8_url:
             formats = self._extract_m3u8_formats(
                 m3u8_url, display_id, 'mp4', 'm3u8_native',
@@ -62,14 +61,12 @@ class OraTVIE(InfoExtractor):
 
         return {
             'id': self._search_regex(
-                r'"video_id"\s*:\s*(\d+)', video_data, 'video id'),
+                r'"id"\s*:\s*(\d+)', video_data, 'video id', default=display_id),
             'display_id': display_id,
             'title': unescapeHTML(self._og_search_title(webpage)),
             'description': get_element_by_attribute(
                 'class', 'video_txt_decription', webpage),
             'thumbnail': self._proto_relative_url(self._search_regex(
                 r'"thumb"\s*:\s*"([^"]+)', video_data, 'thumbnail', None)),
-            'duration': int(self._search_regex(
-                r'"duration"\s*:\s*(\d+)', video_data, 'duration')),
             'formats': formats,
         }

From a616f65471743abfacccb2639a299ea356839909 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 20 Jan 2016 21:30:29 +0600
Subject: [PATCH 0425/1105] [tube8] PEP 8

---
 youtube_dl/extractor/tube8.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index 378cb0570..1d9271d1e 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -27,7 +27,7 @@ class Tube8IE(InfoExtractor):
             'age_limit': 18,
             'duration': 230,
         }
-    },{
+    }, {
         'url': 'http://www.tube8.com/shemale/teen/blonde-cd-gets-kidnapped-by-two-blacks-and-punished-for-being-a-slutty-girl/19569151/',
         'only_matching': True,
     }]

From 4519c1f43cd2d0634e85041aa8f950ddb745e285 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 21 Jan 2016 12:43:45 +0100
Subject: [PATCH 0426/1105] [vimeo] 'ext' must be a string, not a tuple (fixes
 #8288)

There was an ',' at the end of the line.
---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 76e681bc3..2389e7f0f 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -430,7 +430,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 if download_url and not source_file.get('is_cold') and not source_file.get('is_defrosting'):
                     source_name = source_file.get('public_name', 'Original')
                     if self._is_valid_url(download_url, video_id, '%s video' % source_name):
-                        ext = source_file.get('extension', determine_ext(download_url)).lower(),
+                        ext = source_file.get('extension', determine_ext(download_url)).lower()
                         formats.append({
                             'url': download_url,
                             'ext': ext,

From 383496e65ed2a965ab82e5cc3012fd3e0ca2afbf Mon Sep 17 00:00:00 2001
From: igv <igv.versn@gmail.com>
Date: Thu, 21 Jan 2016 12:36:09 +0300
Subject: [PATCH 0427/1105] Additional regex for yahoo extractor

---
 youtube_dl/extractor/yahoo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 4a492f784..92b50f6f7 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -221,6 +221,7 @@ class YahooIE(InfoExtractor):
                     r'root\.App\.Cache\.context\.videoCache\.curVideo = \{"([^"]+)"',
                     r'"first_videoid"\s*:\s*"([^"]+)"',
                     r'%s[^}]*"ccm_id"\s*:\s*"([^"]+)"' % re.escape(page_id),
+                    r'yahoo:\/\/article\/view\?uuid=([^&]+)&',
                 ]
                 video_id = self._search_regex(
                     CONTENT_ID_REGEXES, webpage, 'content ID')

From e87d98b0ddbb2cfa873b4d0c13c751bb12224290 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 Jan 2016 22:42:50 +0600
Subject: [PATCH 0428/1105] [yahoo] Add improve content id regexes (Closes
 #8290)

---
 youtube_dl/extractor/yahoo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 92b50f6f7..4c6142927 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -221,7 +221,8 @@ class YahooIE(InfoExtractor):
                     r'root\.App\.Cache\.context\.videoCache\.curVideo = \{"([^"]+)"',
                     r'"first_videoid"\s*:\s*"([^"]+)"',
                     r'%s[^}]*"ccm_id"\s*:\s*"([^"]+)"' % re.escape(page_id),
-                    r'yahoo:\/\/article\/view\?uuid=([^&]+)&',
+                    r'<article[^>]data-uuid=["\']([^"\']+)',
+                    r'yahoo://article/view\?.*\buuid=([^&"\']+)',
                 ]
                 video_id = self._search_regex(
                     CONTENT_ID_REGEXES, webpage, 'content ID')

From 582f4f834e5cb6fc783eeb51bd977aa237330e79 Mon Sep 17 00:00:00 2001
From: Dimitre Liotev <dl@liotev.com>
Date: Thu, 21 Jan 2016 08:10:41 +0200
Subject: [PATCH 0429/1105] Fix issue #8109 (error when downloading automatic
 captions)

---
 youtube_dl/extractor/youtube.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index d31161d21..567877920 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -964,6 +964,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         try:
             args = player_config['args']
             caption_url = args['ttsurl']
+            if caption_url is None or caption_url == "" or caption_url.isspace():
+                self._downloader.report_warning("No automatic captions")
+                return {}
             timestamp = args['timestamp']
             # We get the available subtitles
             list_params = compat_urllib_parse.urlencode({

From 51290d8457aa8460382407796740063ced464481 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 Jan 2016 22:58:03 +0600
Subject: [PATCH 0430/1105] [youtube] Simplify automatic captions URL check
 (Closes #8287)

---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 567877920..8e8fc14d2 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -964,8 +964,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         try:
             args = player_config['args']
             caption_url = args['ttsurl']
-            if caption_url is None or caption_url == "" or caption_url.isspace():
-                self._downloader.report_warning("No automatic captions")
+            if not caption_url:
+                self._downloader.report_warning(err_msg)
                 return {}
             timestamp = args['timestamp']
             # We get the available subtitles

From 1e10d02fec559618f877e6694b6eeec53d59a65a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 Jan 2016 23:28:22 +0600
Subject: [PATCH 0431/1105] [hitbox] Skip subscribe only formats (Closes #8217)

---
 youtube_dl/extractor/hitbox.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/hitbox.py b/youtube_dl/extractor/hitbox.py
index 421f55bbe..ff797438d 100644
--- a/youtube_dl/extractor/hitbox.py
+++ b/youtube_dl/extractor/hitbox.py
@@ -159,6 +159,9 @@ class HitboxLiveIE(HitboxIE):
         cdns = player_config.get('cdns')
         servers = []
         for cdn in cdns:
+            # Subscribe URLs are not playable
+            if cdn.get('rtmpSubscribe') is True:
+                continue
             base_url = cdn.get('netConnectionUrl')
             host = re.search('.+\.([^\.]+\.[^\./]+)/.+', base_url).group(1)
             if base_url not in servers:

From 6b45f9aba2dad6e965ab51b4d18f4bb05336eaf1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 22 Jan 2016 02:14:47 +0800
Subject: [PATCH 0432/1105] [iqiyi] Update key (closes #8292)

---
 youtube_dl/extractor/iqiyi.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 66a70a181..691cb66d6 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -214,8 +214,8 @@ class IqiyiIE(InfoExtractor):
 
     def get_enc_key(self, swf_url, video_id):
         # TODO: automatic key extraction
-        # last update at 2015-12-18 for Zombie::bite
-        enc_key = '8b6b683780897eb8d9a48a02ccc4817d'[::-1]
+        # last update at 2016-01-22 for Zombie::bite
+        enc_key = '6ab6d0280511493ba85594779759d4ed'
         return enc_key
 
     def _real_extract(self, url):

From 4d318be1951d6bbae0eae7aff69a58de353c8337 Mon Sep 17 00:00:00 2001
From: Filippo Valsorda <filippo@cloudflare.com>
Date: Mon, 14 Dec 2015 02:18:13 +0000
Subject: [PATCH 0433/1105] [update] fix (unexploitable) BB'06 vulnerability in
 rsa_verify

The rsa_verify code was vulnerable to a BB'06 attack, allowing to forge
signatures for arbitrary messages if and only if the public key exponent is
3.  Since the updates key is hardcoded to 65537, there is no risk for
youtube-dl, but I don't want vulnerable code in the wild.

The new function adopts a way safer approach of encoding-and-comparing to
replace the dangerous parsing code.
---
 test/test_update.py  | 30 ++++++++++++++++++++++++++++++
 test/versions.json   | 34 ++++++++++++++++++++++++++++++++++
 youtube_dl/update.py | 32 ++++++++------------------------
 3 files changed, 72 insertions(+), 24 deletions(-)
 create mode 100644 test/test_update.py
 create mode 100644 test/versions.json

diff --git a/test/test_update.py b/test/test_update.py
new file mode 100644
index 000000000..d9c71511d
--- /dev/null
+++ b/test/test_update.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+import json
+from youtube_dl.update import rsa_verify
+
+
+class TestUpdate(unittest.TestCase):
+    def test_rsa_verify(self):
+        UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
+        with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'versions.json'), 'rb') as f:
+            versions_info = f.read().decode()
+        versions_info = json.loads(versions_info)
+        signature = versions_info['signature']
+        del versions_info['signature']
+        self.assertTrue(rsa_verify(
+            json.dumps(versions_info, sort_keys=True).encode('utf-8'),
+            signature, UPDATES_RSA_KEY))
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/test/versions.json b/test/versions.json
new file mode 100644
index 000000000..6cccc2259
--- /dev/null
+++ b/test/versions.json
@@ -0,0 +1,34 @@
+{
+    "latest": "2013.01.06", 
+    "signature": "72158cdba391628569ffdbea259afbcf279bbe3d8aeb7492690735dc1cfa6afa754f55c61196f3871d429599ab22f2667f1fec98865527b32632e7f4b3675a7ef0f0fbe084d359256ae4bba68f0d33854e531a70754712f244be71d4b92e664302aa99653ee4df19800d955b6c4149cd2b3f24288d6e4b40b16126e01f4c8ce6", 
+    "versions": {
+        "2013.01.02": {
+            "bin": [
+                "http://youtube-dl.org/downloads/2013.01.02/youtube-dl", 
+                "f5b502f8aaa77675c4884938b1e4871ebca2611813a0c0e74f60c0fbd6dcca6b"
+            ], 
+            "exe": [
+                "http://youtube-dl.org/downloads/2013.01.02/youtube-dl.exe", 
+                "75fa89d2ce297d102ff27675aa9d92545bbc91013f52ec52868c069f4f9f0422"
+            ], 
+            "tar": [
+                "http://youtube-dl.org/downloads/2013.01.02/youtube-dl-2013.01.02.tar.gz", 
+                "6a66d022ac8e1c13da284036288a133ec8dba003b7bd3a5179d0c0daca8c8196"
+            ]
+        }, 
+        "2013.01.06": {
+            "bin": [
+                "http://youtube-dl.org/downloads/2013.01.06/youtube-dl", 
+                "64b6ed8865735c6302e836d4d832577321b4519aa02640dc508580c1ee824049"
+            ], 
+            "exe": [
+                "http://youtube-dl.org/downloads/2013.01.06/youtube-dl.exe", 
+                "58609baf91e4389d36e3ba586e21dab882daaaee537e4448b1265392ae86ff84"
+            ], 
+            "tar": [
+                "http://youtube-dl.org/downloads/2013.01.06/youtube-dl-2013.01.06.tar.gz", 
+                "fe77ab20a95d980ed17a659aa67e371fdd4d656d19c4c7950e7b720b0c2f1a86"
+            ]
+        }
+    }
+}
\ No newline at end of file
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index 995b8ed96..e4a1aaa64 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -15,33 +15,17 @@ from .version import __version__
 
 
 def rsa_verify(message, signature, key):
-    from struct import pack
     from hashlib import sha256
-
     assert isinstance(message, bytes)
-    block_size = 0
-    n = key[0]
-    while n:
-        block_size += 1
-        n >>= 8
-    signature = pow(int(signature, 16), key[1], key[0])
-    raw_bytes = []
-    while signature:
-        raw_bytes.insert(0, pack("B", signature & 0xFF))
-        signature >>= 8
-    signature = (block_size - len(raw_bytes)) * b'\x00' + b''.join(raw_bytes)
-    if signature[0:2] != b'\x00\x01':
+    byte_size = (len(bin(key[0])) - 2 + 8 - 1) // 8
+    signature = ('%x' % pow(int(signature, 16), key[1], key[0])).encode()
+    signature = (byte_size * 2 - len(signature)) * b'0' + signature
+    asn1 = b'3031300d060960864801650304020105000420'
+    asn1 += sha256(message).hexdigest().encode()
+    if byte_size < len(asn1) // 2 + 11:
         return False
-    signature = signature[2:]
-    if b'\x00' not in signature:
-        return False
-    signature = signature[signature.index(b'\x00') + 1:]
-    if not signature.startswith(b'\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20'):
-        return False
-    signature = signature[19:]
-    if signature != sha256(message).digest():
-        return False
-    return True
+    expected = b'0001' + (byte_size - len(asn1) // 2 - 3) * b'ff' + b'00' + asn1
+    return expected == signature
 
 
 def update_self(to_screen, verbose, opener):

From 32d77eeb04657529b118159db4d23fd671fe04c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 22 Jan 2016 14:49:17 +0100
Subject: [PATCH 0434/1105] [downloader/common] report_retry: Don't crash when
 retries is infinite (fixes #8299)

---
 youtube_dl/downloader/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index beae8c4d0..fc7521598 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -295,7 +295,7 @@ class FileDownloader(object):
 
     def report_retry(self, count, retries):
         """Report retry in case of HTTP error 5xx"""
-        self.to_screen('[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
+        self.to_screen('[download] Got server HTTP error. Retrying (attempt %d of %.0f)...' % (count, retries))
 
     def report_file_already_downloaded(self, file_name):
         """Report file has already been fully downloaded."""

From 4118cc02c1694a7fd355b75c806a69d75b3850bb Mon Sep 17 00:00:00 2001
From: John Assael <iassael@gmail.com>
Date: Thu, 21 Jan 2016 19:51:48 +0000
Subject: [PATCH 0435/1105] [cbsnews] Extract subtitles

added test function for CBS News subtitles
---
 youtube_dl/extractor/cbsnews.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index d211ec23b..7b9dd67fd 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -41,6 +41,21 @@ class CBSNewsIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            'url': 'http://www.cbsnews.com/videos/mountain-lions-of-l-a/',
+            'info_dict': {
+                'id': 'Mountain Lions of L.A.',
+                'ext': 'flv',
+                'title': 'Fort Hood shooting: Army downplays mental illness as cause of attack',
+                'thumbnail': 're:^http?://.*\.jpg$',
+                'subtitles': 're:^http?://.*\.xml$',
+                'duration': 787,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+        },
     ]
 
     def _real_extract(self, url):
@@ -85,10 +100,21 @@ class CBSNewsIE(InfoExtractor):
                 fmt['ext'] = 'mp4'
             formats.append(fmt)
 
+        if 'mpxRefId' in video_info:
+            cap_url = 'http://www.cbsnews.com/videos/captions/%s.adb_xml' % video_info['mpxRefId']
+            subtitles = {
+                'en': [{
+                    'url': cap_url,
+                    'ext': 'xml'
+                }], }
+        else:
+            subtitles = {}
+
         return {
             'id': video_id,
             'title': title,
             'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,
+            'subtitles': subtitles,
         }

From 220ee33f2b8a22d2fe992c26b32a046b77ce2a4a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 Jan 2016 22:23:21 +0600
Subject: [PATCH 0436/1105] [cbsnews] Simplify subtitles extraction and fix
 test (Closes #8295)

---
 youtube_dl/extractor/cbsnews.py | 33 ++++++++++-----------------------
 1 file changed, 10 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 7b9dd67fd..480435e26 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -35,21 +35,11 @@ class CBSNewsIE(InfoExtractor):
                 'title': 'Fort Hood shooting: Army downplays mental illness as cause of attack',
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'duration': 205,
-            },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'http://www.cbsnews.com/videos/mountain-lions-of-l-a/',
-            'info_dict': {
-                'id': 'Mountain Lions of L.A.',
-                'ext': 'flv',
-                'title': 'Fort Hood shooting: Army downplays mental illness as cause of attack',
-                'thumbnail': 're:^http?://.*\.jpg$',
-                'subtitles': 're:^http?://.*\.xml$',
-                'duration': 787,
+                'subtitles': {
+                    'en': [{
+                        'ext': 'ttml',
+                    }],
+                },
             },
             'params': {
                 # rtmp download
@@ -100,15 +90,12 @@ class CBSNewsIE(InfoExtractor):
                 fmt['ext'] = 'mp4'
             formats.append(fmt)
 
+        subtitles = {}
         if 'mpxRefId' in video_info:
-            cap_url = 'http://www.cbsnews.com/videos/captions/%s.adb_xml' % video_info['mpxRefId']
-            subtitles = {
-                'en': [{
-                    'url': cap_url,
-                    'ext': 'xml'
-                }], }
-        else:
-            subtitles = {}
+            subtitles['en'] = [{
+                'ext': 'ttml',
+                'url': 'http://www.cbsnews.com/videos/captions/%s.adb_xml' % video_info['mpxRefId'],
+            }]
 
         return {
             'id': video_id,

From 24114fee747c5cec4aae41f93581561fcb902e10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fran=C3=A7ois=20Charlier?= <francois.charlier@redhat.com>
Date: Thu, 21 Jan 2016 18:47:43 +0100
Subject: [PATCH 0437/1105] [arte:future] Fix extraction

[arte] Add support for more "Arte Future" uri
---
 youtube_dl/extractor/arte.py | 35 ++++++++++++++++++-----------------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 10301a8ea..964d38fdf 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -199,25 +199,26 @@ class ArteTVCreativeIE(ArteTVPlus7IE):
 
 class ArteTVFutureIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:future'
-    _VALID_URL = r'https?://future\.arte\.tv/(?P<lang>fr|de)/(thema|sujet)/.*?#article-anchor-(?P<id>\d+)'
+    _VALID_URL = r'https?://future\.arte\.tv/(?P<lang>fr|de)/(?P<id>.+)'
 
-    _TEST = {
-        'url': 'http://future.arte.tv/fr/sujet/info-sciences#article-anchor-7081',
-        'info_dict': {
-            'id': '5201',
-            'ext': 'mp4',
-            'title': 'Les champignons au secours de la planète',
-            'upload_date': '20131101',
+    _TESTS = [
+        {
+            'url': 'http://future.arte.tv/fr/info-sciences/les-ecrevisses-aussi-sont-anxieuses',
+            'info_dict': {
+                'id': '050940-028-A',
+                'ext': 'mp4',
+                'title': 'Les écrevisses aussi peuvent être anxieuses',
+            },
         },
-    }
-
-    def _real_extract(self, url):
-        anchor_id, lang = self._extract_url_info(url)
-        webpage = self._download_webpage(url, anchor_id)
-        row = self._search_regex(
-            r'(?s)id="%s"[^>]*>.+?(<div[^>]*arte_vp_url[^>]*>)' % anchor_id,
-            webpage, 'row')
-        return self._extract_from_webpage(row, anchor_id, lang)
+        {
+            'url': 'http://future.arte.tv/fr/la-science-est-elle-responsable',
+            'info_dict': {
+                'id': '061982-002-A',
+                'ext': 'mp4',
+                'title': 'Brian P. Schmidt - Prix Nobel de physique 2011',
+            },
+        }
+    ]
 
 
 class ArteTVDDCIE(ArteTVPlus7IE):

From 9c54ae3387b883bec5f014e2ac864a9a6c109163 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 Jan 2016 23:00:05 +0600
Subject: [PATCH 0438/1105] [arte:future] Make duplicated test matching only

---
 youtube_dl/extractor/arte.py | 27 ++++++++++-----------------
 1 file changed, 10 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 964d38fdf..7ef42e092 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -201,24 +201,17 @@ class ArteTVFutureIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:future'
     _VALID_URL = r'https?://future\.arte\.tv/(?P<lang>fr|de)/(?P<id>.+)'
 
-    _TESTS = [
-        {
-            'url': 'http://future.arte.tv/fr/info-sciences/les-ecrevisses-aussi-sont-anxieuses',
-            'info_dict': {
-                'id': '050940-028-A',
-                'ext': 'mp4',
-                'title': 'Les écrevisses aussi peuvent être anxieuses',
-            },
+    _TESTS = [{
+        'url': 'http://future.arte.tv/fr/info-sciences/les-ecrevisses-aussi-sont-anxieuses',
+        'info_dict': {
+            'id': '050940-028-A',
+            'ext': 'mp4',
+            'title': 'Les écrevisses aussi peuvent être anxieuses',
         },
-        {
-            'url': 'http://future.arte.tv/fr/la-science-est-elle-responsable',
-            'info_dict': {
-                'id': '061982-002-A',
-                'ext': 'mp4',
-                'title': 'Brian P. Schmidt - Prix Nobel de physique 2011',
-            },
-        }
-    ]
+    }, {
+        'url': 'http://future.arte.tv/fr/la-science-est-elle-responsable',
+        'only_matching': True,
+    }]
 
 
 class ArteTVDDCIE(ArteTVPlus7IE):

From 4fcd9d147df9b06d954b8f8a1749b50609529ed4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 Jan 2016 23:00:50 +0600
Subject: [PATCH 0439/1105] [arte:cinema] Add extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/arte.py     | 17 +++++++++++++++++
 2 files changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bab3d7b46..cee5cfe7c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -42,6 +42,7 @@ from .arte import (
     ArteTVCreativeIE,
     ArteTVConcertIE,
     ArteTVFutureIE,
+    ArteTVCinemaIE,
     ArteTVDDCIE,
     ArteTVEmbedIE,
 )
diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 7ef42e092..b9e07f0ef 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -249,6 +249,23 @@ class ArteTVConcertIE(ArteTVPlus7IE):
     }
 
 
+class ArteTVCinemaIE(ArteTVPlus7IE):
+    IE_NAME = 'arte.tv:cinema'
+    _VALID_URL = r'https?://cinema\.arte\.tv/(?P<lang>de|fr)/(?P<id>.+)'
+
+    _TEST = {
+        'url': 'http://cinema.arte.tv/de/node/38291',
+        'md5': '6b275511a5107c60bacbeeda368c3aa1',
+        'info_dict': {
+            'id': '055876-000_PWA12025-D',
+            'ext': 'mp4',
+            'title': 'Tod auf dem Nil',
+            'upload_date': '20160122',
+            'description': 'md5:7f749bbb77d800ef2be11d54529b96bc',
+        },
+    }
+
+
 class ArteTVEmbedIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:embed'
     _VALID_URL = r'''(?x)

From d570746e45cff3c0f89654bf748e44a5da75a924 Mon Sep 17 00:00:00 2001
From: "Andrew \"Akari\" Alexeyew" <akari@dbc.1gb.ua>
Date: Wed, 2 Dec 2015 06:00:47 +0200
Subject: [PATCH 0440/1105] [nuevo] Generalize nuevo extractor and add support
 for trollvids

Supports only the nuevo player for now (most common).

[trollvids] convert duration to an int

[trollvids] added a test

[trollvids] made flake8 shut up

Generalized the Nuevo extractor

Affects: anitube, trollvids, trutube

[nuevo] Complied with the code comments.
---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/anitube.py   | 34 +++------------------
 youtube_dl/extractor/nuevo.py     | 37 +++++++++++++++++++++++
 youtube_dl/extractor/trollvids.py | 49 +++++++++++++++++++++++++++++++
 youtube_dl/extractor/trutube.py   | 23 +++++----------
 5 files changed, 98 insertions(+), 46 deletions(-)
 create mode 100644 youtube_dl/extractor/nuevo.py
 create mode 100644 youtube_dl/extractor/trollvids.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index cee5cfe7c..6f2b35cf1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -726,6 +726,7 @@ from .toutv import TouTvIE
 from .toypics import ToypicsUserIE, ToypicsIE
 from .traileraddict import TrailerAddictIE
 from .trilulilu import TriluliluIE
+from .trollvids import TrollvidsIE
 from .trutube import TruTubeIE
 from .tube8 import Tube8IE
 from .tubitv import TubiTvIE
diff --git a/youtube_dl/extractor/anitube.py b/youtube_dl/extractor/anitube.py
index 23f942ae2..73690df82 100644
--- a/youtube_dl/extractor/anitube.py
+++ b/youtube_dl/extractor/anitube.py
@@ -2,10 +2,10 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
+from .nuevo import NuevoBaseIE
 
 
-class AnitubeIE(InfoExtractor):
+class AnitubeIE(NuevoBaseIE):
     IE_NAME = 'anitube.se'
     _VALID_URL = r'https?://(?:www\.)?anitube\.se/video/(?P<id>\d+)'
 
@@ -29,31 +29,5 @@ class AnitubeIE(InfoExtractor):
         key = self._search_regex(
             r'src=["\']https?://[^/]+/embed/([A-Za-z0-9_-]+)', webpage, 'key')
 
-        config_xml = self._download_xml(
-            'http://www.anitube.se/nuevo/econfig.php?key=%s' % key, key)
-
-        video_title = config_xml.find('title').text
-        thumbnail = config_xml.find('image').text
-        duration = float(config_xml.find('duration').text)
-
-        formats = []
-        video_url = config_xml.find('file')
-        if video_url is not None:
-            formats.append({
-                'format_id': 'sd',
-                'url': video_url.text,
-            })
-        video_url = config_xml.find('filehd')
-        if video_url is not None:
-            formats.append({
-                'format_id': 'hd',
-                'url': video_url.text,
-            })
-
-        return {
-            'id': video_id,
-            'title': video_title,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'formats': formats
-        }
+        config_url = 'http://www.anitube.se/nuevo/econfig.php?key=%s' % key
+        return self._extract_nuevo(config_url, video_id)
diff --git a/youtube_dl/extractor/nuevo.py b/youtube_dl/extractor/nuevo.py
new file mode 100644
index 000000000..ccc697e4f
--- /dev/null
+++ b/youtube_dl/extractor/nuevo.py
@@ -0,0 +1,37 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..utils import (
+    float_or_none,
+    xpath_text
+)
+
+
+class NuevoBaseIE(InfoExtractor):
+    def _extract_nuevo(self, config_url, video_id):
+        tree = self._download_xml(config_url, video_id, transform_source=lambda s: s.strip())
+
+        title = xpath_text(tree, './title')
+        if title:
+            title = title.strip()
+
+        thumbnail = xpath_text(tree, './image')
+        duration = float_or_none(xpath_text(tree, './duration'))
+
+        formats = []
+        for element_name, format_id in (('file', 'sd'), ('filehd', 'hd')):
+            video_url = tree.find(element_name)
+            video_url is None or formats.append({
+                'format_id': format_id,
+                'url': video_url.text
+            })
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats
+        }
diff --git a/youtube_dl/extractor/trollvids.py b/youtube_dl/extractor/trollvids.py
new file mode 100644
index 000000000..e4fe620f7
--- /dev/null
+++ b/youtube_dl/extractor/trollvids.py
@@ -0,0 +1,49 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .nuevo import NuevoBaseIE
+
+from ..compat import (
+    compat_urllib_parse_unquote
+)
+
+import re
+
+
+class TrollvidsIE(NuevoBaseIE):
+    _VALID_URL = r'http://(?:www\.)?trollvids\.com/+video/+(?P<id>[0-9]+)/+(?P<title>[^?&]+)'
+    IE_NAME = 'trollvids'
+
+    def _real_extract(self, url):
+        match = re.match(self._VALID_URL, url)
+
+        video_id = match.group('id')
+        raw_video_title = match.group('title')
+        url = 'http://trollvids.com/video/%s/%s' % (video_id, raw_video_title)
+        config_url = 'http://trollvids.com/nuevo/player/config.php?v=%s' % video_id
+
+        info = self._extract_nuevo(config_url, video_id)
+
+        info.update({
+            'webpage_url': url,
+            'age_limit': 18
+        })
+
+        if 'title' not in info:
+            info['title'] = compat_urllib_parse_unquote(raw_video_title)
+
+        return info
+
+    _TESTS = [
+        {
+            'url': 'http://trollvids.com/video/2349002/%E3%80%90MMD-R-18%E3%80%91%E3%82%AC%E3%83%BC%E3%83%AB%E3%83%95%E3%83%AC%E3%83%B3%E3%83%89-carrymeoff',
+            'md5': '1d53866b2c514b23ed69e4352fdc9839',
+            'info_dict': {
+                'id': '2349002',
+                'ext': 'mp4',
+                'title': "【MMD R-18】ガールフレンド carry_me_off",
+                'age_limit': 18,
+                'duration': 216.78,
+            },
+        },
+    ]
diff --git a/youtube_dl/extractor/trutube.py b/youtube_dl/extractor/trutube.py
index e7b79243a..d7ec2ec26 100644
--- a/youtube_dl/extractor/trutube.py
+++ b/youtube_dl/extractor/trutube.py
@@ -1,10 +1,9 @@
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
-from ..utils import xpath_text
+from .nuevo import NuevoBaseIE
 
 
-class TruTubeIE(InfoExtractor):
+class TruTubeIE(NuevoBaseIE):
     _VALID_URL = r'https?://(?:www\.)?trutube\.tv/(?:video/|nuevo/player/embed\.php\?v=)(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://trutube.tv/video/14880/Ramses-II-Proven-To-Be-A-Red-Headed-Caucasoid-',
@@ -22,19 +21,11 @@ class TruTubeIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        config_url = 'https://trutube.tv/nuevo/player/config.php?v=%s' % video_id
 
-        config = self._download_xml(
-            'https://trutube.tv/nuevo/player/config.php?v=%s' % video_id,
-            video_id, transform_source=lambda s: s.strip())
+        info = self._extract_nuevo(config_url, video_id)
 
-        # filehd is always 404
-        video_url = xpath_text(config, './file', 'video URL', fatal=True)
-        title = xpath_text(config, './title', 'title').strip()
-        thumbnail = xpath_text(config, './image', ' thumbnail')
+        # filehd always 404s
+        info['formats'] = info['formats'][:1]
 
-        return {
-            'id': video_id,
-            'url': video_url,
-            'title': title,
-            'thumbnail': thumbnail,
-        }
+        return info

From 10677ece81b7ed05bb84a0dbaf5bd237107eeb62 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jan 2016 00:04:33 +0600
Subject: [PATCH 0441/1105] [nuevo] Simplify nuevo extractors (Closes #7728)

---
 youtube_dl/extractor/anitube.py   |  9 ++---
 youtube_dl/extractor/nuevo.py     | 25 +++++++-------
 youtube_dl/extractor/trollvids.py | 55 ++++++++++++-------------------
 youtube_dl/extractor/trutube.py   | 13 +++-----
 4 files changed, 41 insertions(+), 61 deletions(-)

diff --git a/youtube_dl/extractor/anitube.py b/youtube_dl/extractor/anitube.py
index 73690df82..2fd912da4 100644
--- a/youtube_dl/extractor/anitube.py
+++ b/youtube_dl/extractor/anitube.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .nuevo import NuevoBaseIE
 
 
@@ -22,12 +20,11 @@ class AnitubeIE(NuevoBaseIE):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
         key = self._search_regex(
             r'src=["\']https?://[^/]+/embed/([A-Za-z0-9_-]+)', webpage, 'key')
 
-        config_url = 'http://www.anitube.se/nuevo/econfig.php?key=%s' % key
-        return self._extract_nuevo(config_url, video_id)
+        return self._extract_nuevo(
+            'http://www.anitube.se/nuevo/econfig.php?key=%s' % key, video_id)
diff --git a/youtube_dl/extractor/nuevo.py b/youtube_dl/extractor/nuevo.py
index ccc697e4f..225da033c 100644
--- a/youtube_dl/extractor/nuevo.py
+++ b/youtube_dl/extractor/nuevo.py
@@ -11,22 +11,23 @@ from ..utils import (
 
 class NuevoBaseIE(InfoExtractor):
     def _extract_nuevo(self, config_url, video_id):
-        tree = self._download_xml(config_url, video_id, transform_source=lambda s: s.strip())
+        config = self._download_xml(
+            config_url, video_id, transform_source=lambda s: s.strip())
 
-        title = xpath_text(tree, './title')
-        if title:
-            title = title.strip()
-
-        thumbnail = xpath_text(tree, './image')
-        duration = float_or_none(xpath_text(tree, './duration'))
+        title = xpath_text(config, './title', 'title', fatal=True).strip()
+        video_id = xpath_text(config, './mediaid', default=video_id)
+        thumbnail = xpath_text(config, './image')
+        duration = float_or_none(xpath_text(config, './duration'))
 
         formats = []
         for element_name, format_id in (('file', 'sd'), ('filehd', 'hd')):
-            video_url = tree.find(element_name)
-            video_url is None or formats.append({
-                'format_id': format_id,
-                'url': video_url.text
-            })
+            video_url = xpath_text(config, element_name)
+            if video_url:
+                formats.append({
+                    'url': video_url,
+                    'format_id': format_id,
+                })
+        self._check_formats(formats, video_id)
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/trollvids.py b/youtube_dl/extractor/trollvids.py
index e4fe620f7..d239949a6 100644
--- a/youtube_dl/extractor/trollvids.py
+++ b/youtube_dl/extractor/trollvids.py
@@ -1,49 +1,36 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-from .nuevo import NuevoBaseIE
-
-from ..compat import (
-    compat_urllib_parse_unquote
-)
-
 import re
 
+from .nuevo import NuevoBaseIE
+
 
 class TrollvidsIE(NuevoBaseIE):
-    _VALID_URL = r'http://(?:www\.)?trollvids\.com/+video/+(?P<id>[0-9]+)/+(?P<title>[^?&]+)'
+    _VALID_URL = r'http://(?:www\.)?trollvids\.com/video/(?P<id>\d+)/(?P<display_id>[^/?#&]+)'
     IE_NAME = 'trollvids'
+    _TEST = {
+        'url': 'http://trollvids.com/video/2349002/%E3%80%90MMD-R-18%E3%80%91%E3%82%AC%E3%83%BC%E3%83%AB%E3%83%95%E3%83%AC%E3%83%B3%E3%83%89-carrymeoff',
+        'md5': '1d53866b2c514b23ed69e4352fdc9839',
+        'info_dict': {
+            'id': '2349002',
+            'ext': 'mp4',
+            'title': '【MMD R-18】ガールフレンド carry_me_off',
+            'age_limit': 18,
+            'duration': 216.78,
+        },
+    }
 
     def _real_extract(self, url):
-        match = re.match(self._VALID_URL, url)
-
-        video_id = match.group('id')
-        raw_video_title = match.group('title')
-        url = 'http://trollvids.com/video/%s/%s' % (video_id, raw_video_title)
-        config_url = 'http://trollvids.com/nuevo/player/config.php?v=%s' % video_id
-
-        info = self._extract_nuevo(config_url, video_id)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
+        info = self._extract_nuevo(
+            'http://trollvids.com/nuevo/player/config.php?v=%s' % video_id,
+            video_id)
         info.update({
-            'webpage_url': url,
+            'display_id': display_id,
             'age_limit': 18
         })
-
-        if 'title' not in info:
-            info['title'] = compat_urllib_parse_unquote(raw_video_title)
-
         return info
-
-    _TESTS = [
-        {
-            'url': 'http://trollvids.com/video/2349002/%E3%80%90MMD-R-18%E3%80%91%E3%82%AC%E3%83%BC%E3%83%AB%E3%83%95%E3%83%AC%E3%83%B3%E3%83%89-carrymeoff',
-            'md5': '1d53866b2c514b23ed69e4352fdc9839',
-            'info_dict': {
-                'id': '2349002',
-                'ext': 'mp4',
-                'title': "【MMD R-18】ガールフレンド carry_me_off",
-                'age_limit': 18,
-                'duration': 216.78,
-            },
-        },
-    ]
diff --git a/youtube_dl/extractor/trutube.py b/youtube_dl/extractor/trutube.py
index d7ec2ec26..d55e0c563 100644
--- a/youtube_dl/extractor/trutube.py
+++ b/youtube_dl/extractor/trutube.py
@@ -4,7 +4,7 @@ from .nuevo import NuevoBaseIE
 
 
 class TruTubeIE(NuevoBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?trutube\.tv/(?:video/|nuevo/player/embed\.php\?v=)(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?trutube\.tv/(?:video/|nuevo/player/embed\.php\?v=)(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://trutube.tv/video/14880/Ramses-II-Proven-To-Be-A-Red-Headed-Caucasoid-',
         'md5': 'c5b6e301b0a2040b074746cbeaa26ca1',
@@ -21,11 +21,6 @@ class TruTubeIE(NuevoBaseIE):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        config_url = 'https://trutube.tv/nuevo/player/config.php?v=%s' % video_id
-
-        info = self._extract_nuevo(config_url, video_id)
-
-        # filehd always 404s
-        info['formats'] = info['formats'][:1]
-
-        return info
+        return self._extract_nuevo(
+            'https://trutube.tv/nuevo/player/config.php?v=%s' % video_id,
+            video_id)

From b9698135480e156c89b30b4da60df1aae1c2a660 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jan 2016 00:10:49 +0600
Subject: [PATCH 0442/1105] Credit @nexAkari for trollvids and nuevo (#7728)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 73fcafeb5..bb1f2d8d9 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -154,3 +154,4 @@ Brian Foley
 Vignesh Venkat
 Tom Gijselinck
 Founder Fang
+Andrew Alexeyew

From 1257b049bcd9857e2993a4366af41219c36baa3e Mon Sep 17 00:00:00 2001
From: Dankryn <dankryn@weg-werf-email.de>
Date: Mon, 11 Jan 2016 21:17:30 +0100
Subject: [PATCH 0443/1105] [ruleporn] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ruleporn.py | 31 +++++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+)
 create mode 100644 youtube_dl/extractor/ruleporn.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6f2b35cf1..dc8679d46 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -580,6 +580,7 @@ from .rts import RTSIE
 from .rtve import RTVEALaCartaIE, RTVELiveIE, RTVEInfantilIE
 from .rtvnh import RTVNHIE
 from .ruhd import RUHDIE
+from .ruleporn import RulepornIE
 from .rutube import (
     RutubeIE,
     RutubeChannelIE,
diff --git a/youtube_dl/extractor/ruleporn.py b/youtube_dl/extractor/ruleporn.py
new file mode 100644
index 000000000..9e6a9125a
--- /dev/null
+++ b/youtube_dl/extractor/ruleporn.py
@@ -0,0 +1,31 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class RulepornIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ruleporn\.com/(?:[a-z]+(?:-[a-z]+)+)'
+    _TEST = {
+        'url': 'http://ruleporn.com/brunette-nympho-chick-takes-her-boyfriend-in-every-angle/',
+        'md5': '86861ebc624a1097c7c10eaf06d7d505',
+        'info_dict': {
+            'id': '48212',
+            'ext': 'mp4',
+            'title': 'Brunette Nympho Chick Takes Her Boyfriend In Every Angle',
+        }
+    }
+
+    def _real_extract(self, url):
+        webpage = self._download_webpage(url, None)
+
+        video_id = self._search_regex(r'http://lovehomeporn.com/embed/([0-9]+)', webpage, 'video_id', fatal=True)
+        title = self._search_regex(r'<h2 title="((?:\w|\s|\d)+)">', webpage, 'title', fatal=True)
+        info_xml = self._download_xml('http://lovehomeporn.com/media/nuevo/econfig.php?key=%s&rp=true' % video_id, video_id)
+        url = info_xml.find('file').text
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': url,
+        }

From ea178204326c0d4203cc3ff88057d2ffd25a68a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jan 2016 00:38:58 +0600
Subject: [PATCH 0444/1105] [nuevo] Improve thumbnail extraction

---
 youtube_dl/extractor/nuevo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nuevo.py b/youtube_dl/extractor/nuevo.py
index 225da033c..ef093dec2 100644
--- a/youtube_dl/extractor/nuevo.py
+++ b/youtube_dl/extractor/nuevo.py
@@ -16,7 +16,7 @@ class NuevoBaseIE(InfoExtractor):
 
         title = xpath_text(config, './title', 'title', fatal=True).strip()
         video_id = xpath_text(config, './mediaid', default=video_id)
-        thumbnail = xpath_text(config, './image')
+        thumbnail = xpath_text(config, ['./image', './thumb'])
         duration = float_or_none(xpath_text(config, './duration'))
 
         formats = []

From b2c6528baf990d1fc0ad7b435595102ae2d8ba2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jan 2016 00:40:11 +0600
Subject: [PATCH 0445/1105] [ruleporn] Rework in terms of nuevo (Closes #8206)

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/ruleporn.py | 39 +++++++++++++++++++++-----------
 2 files changed, 27 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index dc8679d46..4ea6a3f71 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -580,7 +580,7 @@ from .rts import RTSIE
 from .rtve import RTVEALaCartaIE, RTVELiveIE, RTVEInfantilIE
 from .rtvnh import RTVNHIE
 from .ruhd import RUHDIE
-from .ruleporn import RulepornIE
+from .ruleporn import RulePornIE
 from .rutube import (
     RutubeIE,
     RutubeChannelIE,
diff --git a/youtube_dl/extractor/ruleporn.py b/youtube_dl/extractor/ruleporn.py
index 9e6a9125a..ebf9808d5 100644
--- a/youtube_dl/extractor/ruleporn.py
+++ b/youtube_dl/extractor/ruleporn.py
@@ -1,31 +1,44 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
+from .nuevo import NuevoBaseIE
 
 
-class RulepornIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ruleporn\.com/(?:[a-z]+(?:-[a-z]+)+)'
+class RulePornIE(NuevoBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?ruleporn\.com/(?:[^/?#&]+/)*(?P<id>[^/?#&]+)'
     _TEST = {
         'url': 'http://ruleporn.com/brunette-nympho-chick-takes-her-boyfriend-in-every-angle/',
         'md5': '86861ebc624a1097c7c10eaf06d7d505',
         'info_dict': {
             'id': '48212',
+            'display_id': 'brunette-nympho-chick-takes-her-boyfriend-in-every-angle',
             'ext': 'mp4',
             'title': 'Brunette Nympho Chick Takes Her Boyfriend In Every Angle',
+            'description': 'md5:6d28be231b981fff1981deaaa03a04d5',
+            'age_limit': 18,
+            'duration': 635.1,
         }
     }
 
     def _real_extract(self, url):
-        webpage = self._download_webpage(url, None)
+        display_id = self._match_id(url)
 
-        video_id = self._search_regex(r'http://lovehomeporn.com/embed/([0-9]+)', webpage, 'video_id', fatal=True)
-        title = self._search_regex(r'<h2 title="((?:\w|\s|\d)+)">', webpage, 'title', fatal=True)
-        info_xml = self._download_xml('http://lovehomeporn.com/media/nuevo/econfig.php?key=%s&rp=true' % video_id, video_id)
-        url = info_xml.find('file').text
+        webpage = self._download_webpage(url, display_id)
 
-        return {
-            'id': video_id,
+        video_id = self._search_regex(
+            r'lovehomeporn\.com/embed/(\d+)', webpage, 'video id')
+
+        title = self._search_regex(
+            r'<h2[^>]+title=(["\'])(?P<url>.+?)\1',
+            webpage, 'title', group='url')
+        description = self._html_search_meta('description', webpage)
+
+        info = self._extract_nuevo(
+            'http://lovehomeporn.com/media/nuevo/econfig.php?key=%s&rp=true' % video_id,
+            video_id)
+        info.update({
+            'display_id': display_id,
             'title': title,
-            'url': url,
-        }
+            'description': description,
+            'age_limit': 18
+        })
+        return info

From 4c0d13df9bdf7222fbfa6dde543ffcdb47696392 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jan 2016 00:52:23 +0600
Subject: [PATCH 0446/1105] [lovehomeporn] Add extractor

---
 youtube_dl/extractor/__init__.py     |  1 +
 youtube_dl/extractor/lovehomeporn.py | 37 ++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+)
 create mode 100644 youtube_dl/extractor/lovehomeporn.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 4ea6a3f71..245e4d044 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -358,6 +358,7 @@ from .livestream import (
     LivestreamShortenerIE,
 )
 from .lnkgo import LnkGoIE
+from .lovehomeporn import LoveHomePornIE
 from .lrt import LRTIE
 from .lynda import (
     LyndaIE,
diff --git a/youtube_dl/extractor/lovehomeporn.py b/youtube_dl/extractor/lovehomeporn.py
new file mode 100644
index 000000000..8f65a3c03
--- /dev/null
+++ b/youtube_dl/extractor/lovehomeporn.py
@@ -0,0 +1,37 @@
+from __future__ import unicode_literals
+
+import re
+
+from .nuevo import NuevoBaseIE
+
+
+class LoveHomePornIE(NuevoBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?lovehomeporn\.com/video/(?P<id>\d+)(?:/(?P<display_id>[^/?#&]+))?'
+    _TEST = {
+        'url': 'http://lovehomeporn.com/video/48483/stunning-busty-brunette-girlfriend-sucking-and-riding-a-big-dick#menu',
+        'info_dict': {
+            'id': '48483',
+            'display_id': 'stunning-busty-brunette-girlfriend-sucking-and-riding-a-big-dick',
+            'ext': 'mp4',
+            'title': 'Stunning busty brunette girlfriend sucking and riding a big dick',
+            'age_limit': 18,
+            'duration': 238.47,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
+
+        info = self._extract_nuevo(
+            'http://lovehomeporn.com/media/nuevo/config.php?key=%s' % video_id,
+            video_id)
+        info.update({
+            'display_id': display_id,
+            'age_limit': 18
+        })
+        return info

From 1f16b958b11c4c8503a3555d46bd205540d7aae8 Mon Sep 17 00:00:00 2001
From: Marian Sigler <m@qjym.de>
Date: Tue, 12 Jan 2016 17:08:55 +0100
Subject: [PATCH 0447/1105] [SVTPlay] Add subtitle support

---
 youtube_dl/extractor/svt.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index fc20f664b..7ff5ee11a 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -37,6 +37,12 @@ class SVTBaseIE(InfoExtractor):
                 })
         self._sort_formats(formats)
 
+        # SVT does not tell us the language, so we assume swedish.
+        subtitles = {}
+        for sr in video_info['subtitleReferences']:
+            if 'url' in sr:
+                subtitles.setdefault('sv', []).append({'url': sr['url']})
+
         duration = video_info.get('materialLength')
         age_limit = 18 if video_info.get('inappropriateForChildren') else 0
 
@@ -44,6 +50,7 @@ class SVTBaseIE(InfoExtractor):
             'id': video_id,
             'title': title,
             'formats': formats,
+            'subtitles': subtitles,
             'thumbnail': thumbnail,
             'duration': duration,
             'age_limit': age_limit,

From 594c4d79a5cc988af99cbd0e3c00d1752e277dd1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jan 2016 01:47:54 +0600
Subject: [PATCH 0448/1105] [svt] Improve subtitles extraction and add test
 (Closes #8265)

---
 youtube_dl/extractor/svt.py | 43 ++++++++++++++++---------------------
 1 file changed, 19 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index 7ff5ee11a..399c3b8ee 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -37,11 +37,13 @@ class SVTBaseIE(InfoExtractor):
                 })
         self._sort_formats(formats)
 
-        # SVT does not tell us the language, so we assume swedish.
         subtitles = {}
-        for sr in video_info['subtitleReferences']:
-            if 'url' in sr:
-                subtitles.setdefault('sv', []).append({'url': sr['url']})
+        subtitle_references = video_info.get('subtitleReferences')
+        if isinstance(subtitle_references, list):
+            for sr in subtitle_references:
+                subtitle_url = sr.get('url')
+                if subtitle_url:
+                    subtitles.setdefault('sv', []).append({'url': subtitle_url})
 
         duration = video_info.get('materialLength')
         age_limit = 18 if video_info.get('inappropriateForChildren') else 0
@@ -90,30 +92,23 @@ class SVTIE(SVTBaseIE):
 class SVTPlayIE(SVTBaseIE):
     IE_DESC = 'SVT Play and Öppet arkiv'
     _VALID_URL = r'https?://(?:www\.)?(?P<host>svtplay|oppetarkiv)\.se/video/(?P<id>[0-9]+)'
-    _TESTS = [{
-        'url': 'http://www.svtplay.se/video/2609989/sm-veckan/sm-veckan-rally-final-sasong-1-sm-veckan-rally-final',
-        'md5': 'ade3def0643fa1c40587a422f98edfd9',
+    _TEST = {
+        'url': 'http://www.svtplay.se/video/5996901/flygplan-till-haile-selassie/flygplan-till-haile-selassie-2',
+        'md5': '2b6704fe4a28801e1a098bbf3c5ac611',
         'info_dict': {
-            'id': '2609989',
-            'ext': 'flv',
-            'title': 'SM veckan vinter, Örebro - Rally, final',
-            'duration': 4500,
+            'id': '5996901',
+            'ext': 'mp4',
+            'title': 'Flygplan till Haile Selassie',
+            'duration': 3527,
             'thumbnail': 're:^https?://.*[\.-]jpg$',
             'age_limit': 0,
+            'subtitles': {
+                'sv': [{
+                    'ext': 'wsrt',
+                }]
+            },
         },
-    }, {
-        'url': 'http://www.oppetarkiv.se/video/1058509/rederiet-sasong-1-avsnitt-1-av-318',
-        'md5': 'c3101a17ce9634f4c1f9800f0746c187',
-        'info_dict': {
-            'id': '1058509',
-            'ext': 'flv',
-            'title': 'Farlig kryssning',
-            'duration': 2566,
-            'thumbnail': 're:^https?://.*[\.-]jpg$',
-            'age_limit': 0,
-        },
-        'skip': 'Only works from Sweden',
-    }]
+    }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 1ca59daca964b789105b50aa42835aa87c982342 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jan 2016 01:50:06 +0600
Subject: [PATCH 0449/1105] [options] Clarify language tags

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index ade58c375..433245f00 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -380,7 +380,7 @@ def parseOpts(overrideArguments=None):
         '--sub-lang', '--sub-langs', '--srt-lang',
         action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
         default=[], callback=_comma_separated_values_options_callback,
-        help='Languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
+        help='Languages of the subtitles to download (optional) separated by commas, use --list-subs for available language tags')
 
     downloader = optparse.OptionGroup(parser, 'Download Options')
     downloader.add_option(

From 5ca01bb9e4803d82734420445d49ea7f98579fe6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jan 2016 01:51:18 +0600
Subject: [PATCH 0450/1105] [kanalplay] Use IETF language tag

---
 youtube_dl/extractor/kanalplay.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kanalplay.py b/youtube_dl/extractor/kanalplay.py
index 4597d1b96..6c3498c67 100644
--- a/youtube_dl/extractor/kanalplay.py
+++ b/youtube_dl/extractor/kanalplay.py
@@ -49,7 +49,7 @@ class KanalPlayIE(InfoExtractor):
         subs = self._download_json(
             'http://www.kanal%splay.se/api/subtitles/%s' % (channel_id, video_id),
             video_id, 'Downloading subtitles JSON', fatal=False)
-        return {'se': [{'ext': 'srt', 'data': self._fix_subtitles(subs)}]} if subs else {}
+        return {'sv': [{'ext': 'srt', 'data': self._fix_subtitles(subs)}]} if subs else {}
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 6fa73386cbb9b3d39dafde9c61bd950bd99b7155 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jan 2016 01:54:00 +0600
Subject: [PATCH 0451/1105] [drtv] Use IETF language tag

---
 youtube_dl/extractor/drtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index baa24c6d1..2d74ff855 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -91,7 +91,7 @@ class DRTVIE(InfoExtractor):
                 subtitles_list = asset.get('SubtitlesList')
                 if isinstance(subtitles_list, list):
                     LANGS = {
-                        'Danish': 'dk',
+                        'Danish': 'da',
                     }
                     for subs in subtitles_list:
                         lang = subs['Language']

From f733b05302f44776d369f45c730da4a5b558e21b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 23 Jan 2016 12:03:12 +0100
Subject: [PATCH 0452/1105] release 2016.01.23

---
 README.md              |  4 ++--
 docs/supportedsites.md | 13 +++++++++++--
 youtube_dl/version.py  |  2 +-
 3 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 9dbeae1bc..724fb17d1 100644
--- a/README.md
+++ b/README.md
@@ -339,8 +339,8 @@ which means you can modify it, redistribute it or use it however you like.
                                      preference, for example: "srt" or
                                      "ass/srt/best"
     --sub-lang LANGS                 Languages of the subtitles to download
-                                     (optional) separated by commas, use IETF
-                                     language tags like 'en,pt'
+                                     (optional) separated by commas, use --list-
+                                     subs for available language tags
 
 ## Authentication Options:
     -u, --username USERNAME          Login with this account ID
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 99b1e2731..e86467cfa 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -24,7 +24,7 @@
  - **AdobeTVShow**
  - **AdobeTVVideo**
  - **AdultSwim**
- - **AE**
+ - **aenetworks**: A+E Networks: A&E, Lifetime, History.com, FYI Network
  - **Aftonbladet**
  - **AirMozilla**
  - **AlJazeera**
@@ -43,6 +43,7 @@
  - **ARD:mediathek**
  - **arte.tv**
  - **arte.tv:+7**
+ - **arte.tv:cinema**
  - **arte.tv:concert**
  - **arte.tv:creative**
  - **arte.tv:ddc**
@@ -124,6 +125,7 @@
  - **CSpan**: C-SPAN
  - **CtsNews**: 華視新聞
  - **culturebox.francetvinfo.fr**
+ - **CultureUnplugged**
  - **CWTV**
  - **dailymotion**
  - **dailymotion:playlist**
@@ -141,6 +143,7 @@
  - **defense.gouv.fr**
  - **democracynow**
  - **DHM**: Filmarchiv - Deutsches Historisches Museum
+ - **Digiteka**
  - **Discovery**
  - **Dotsub**
  - **DouyuTV**: 斗鱼
@@ -287,7 +290,9 @@
  - **la7.tv**
  - **Laola1Tv**
  - **Lecture2Go**
+ - **Lemonde**
  - **Letv**: 乐视网
+ - **LetvCloud**: 乐视云
  - **LetvPlaylist**
  - **LetvTv**
  - **Libsyn**
@@ -300,6 +305,7 @@
  - **livestream**
  - **livestream:original**
  - **LnkGo**
+ - **LoveHomePorn**
  - **lrt.lt**
  - **lynda**: lynda.com videos
  - **lynda:course**: lynda.com online courses
@@ -486,6 +492,7 @@
  - **rtve.es:live**: RTVE.es live streams
  - **RTVNH**
  - **RUHD**
+ - **RulePorn**
  - **rutube**: Rutube videos
  - **rutube:channel**: Rutube channels
  - **rutube:embed**: Rutube embedded videos
@@ -602,6 +609,7 @@
  - **ToypicsUser**: Toypics user profile
  - **TrailerAddict** (Currently broken)
  - **Trilulilu**
+ - **trollvids**
  - **TruTube**
  - **Tube8**
  - **TubiTv**
@@ -640,7 +648,6 @@
  - **udemy**
  - **udemy:course**
  - **UDNEmbed**: 聯合影音
- - **Ultimedia**
  - **Unistra**
  - **Urort**: NRK P3 Urørt
  - **ustream**
@@ -707,6 +714,7 @@
  - **WebOfStories**
  - **WebOfStoriesPlaylist**
  - **Weibo**
+ - **WeiqiTV**: WQTV
  - **wholecloud**: WholeCloud
  - **Wimp**
  - **Wistia**
@@ -758,3 +766,4 @@
  - **ZDFChannel**
  - **zingmp3:album**: mp3.zing.vn albums
  - **zingmp3:song**: mp3.zing.vn songs
+ - **ZippCast**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 02c438f3a..d5bf73815 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.01.15'
+__version__ = '2016.01.23'

From f6861ec96f33722c9b1ba4f2d8ca307dcbe64ac1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 25 Jan 2016 00:58:53 +0800
Subject: [PATCH 0453/1105] [utils] Add more items to mimetype2ext (#8293)

These are used in Youtube formats
---
 youtube_dl/utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 9c1c0e0bd..178d1dcb3 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1831,6 +1831,8 @@ def mimetype2ext(mt):
         'x-ms-wmv': 'wmv',
         'x-mp4-fragmented': 'mp4',
         'ttml+xml': 'ttml',
+        '3gpp': '3gp',
+        'x-flv': 'flv',
     }.get(res, res)
 
 

From a0d8d704df5ff3098b86e741f238d53de1711198 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 25 Jan 2016 01:01:15 +0800
Subject: [PATCH 0454/1105] [utils] Reorder items in mimetype2ext
 alphabetically

---
 youtube_dl/utils.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 178d1dcb3..c63b61598 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1828,11 +1828,11 @@ def mimetype2ext(mt):
     _, _, res = mt.rpartition('/')
 
     return {
-        'x-ms-wmv': 'wmv',
-        'x-mp4-fragmented': 'mp4',
-        'ttml+xml': 'ttml',
         '3gpp': '3gp',
+        'ttml+xml': 'ttml',
         'x-flv': 'flv',
+        'x-mp4-fragmented': 'mp4',
+        'x-ms-wmv': 'wmv',
     }.get(res, res)
 
 

From 94278f720272c5ad2cd5900f59f8e71f31d46633 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 25 Jan 2016 01:02:19 +0800
Subject: [PATCH 0455/1105] [youtube] Prefer info from YouTube than _formats
 (#8293)

---
 youtube_dl/extractor/youtube.py | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8e8fc14d2..6f0665775 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -32,6 +32,7 @@ from ..utils import (
     get_element_by_attribute,
     get_element_by_id,
     int_or_none,
+    mimetype2ext,
     orderedSet,
     parse_duration,
     remove_quotes,
@@ -1090,9 +1091,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         full_info.update(f)
                         codecs = r.attrib.get('codecs')
                         if codecs:
-                            if full_info.get('acodec') == 'none' and 'vcodec' not in full_info:
+                            if full_info.get('acodec') == 'none':
                                 full_info['vcodec'] = codecs
-                            elif full_info.get('vcodec') == 'none' and 'acodec' not in full_info:
+                            elif full_info.get('vcodec') == 'none':
                                 full_info['acodec'] = codecs
                         formats.append(full_info)
                     else:
@@ -1461,15 +1462,21 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 if 'ratebypass' not in url:
                     url += '&ratebypass=yes'
 
+                dct = {
+                    'format_id': format_id,
+                    'url': url,
+                    'player_url': player_url,
+                }
+                if format_id in self._formats:
+                    dct.update(self._formats[format_id])
+
                 # Some itags are not included in DASH manifest thus corresponding formats will
                 # lack metadata (see https://github.com/rg3/youtube-dl/pull/5993).
                 # Trying to extract metadata from url_encoded_fmt_stream_map entry.
                 mobj = re.search(r'^(?P<width>\d+)[xX](?P<height>\d+)$', url_data.get('size', [''])[0])
                 width, height = (int(mobj.group('width')), int(mobj.group('height'))) if mobj else (None, None)
-                dct = {
-                    'format_id': format_id,
-                    'url': url,
-                    'player_url': player_url,
+
+                more_fields = {
                     'filesize': int_or_none(url_data.get('clen', [None])[0]),
                     'tbr': float_or_none(url_data.get('bitrate', [None])[0], 1000),
                     'width': width,
@@ -1477,13 +1484,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'fps': int_or_none(url_data.get('fps', [None])[0]),
                     'format_note': url_data.get('quality_label', [None])[0] or url_data.get('quality', [None])[0],
                 }
+                for key, value in more_fields.items():
+                    if value:
+                        dct[key] = value
                 type_ = url_data.get('type', [None])[0]
                 if type_:
                     type_split = type_.split(';')
                     kind_ext = type_split[0].split('/')
                     if len(kind_ext) == 2:
-                        kind, ext = kind_ext
-                        dct['ext'] = ext
+                        kind, _ = kind_ext
+                        dct['ext'] = mimetype2ext(type_split[0])
                         if kind in ('audio', 'video'):
                             codecs = None
                             for mobj in re.finditer(
@@ -1501,8 +1511,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                                     'acodec': acodec,
                                     'vcodec': vcodec,
                                 })
-                if format_id in self._formats:
-                    dct.update(self._formats[format_id])
                 formats.append(dct)
         elif video_info.get('hlsvp'):
             manifest_url = video_info['hlsvp'][0]

From 77f785076f9a2fcfc35de49c8e612716bbb37f6c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 25 Jan 2016 01:03:46 +0800
Subject: [PATCH 0456/1105] [common] Keep full codec name from m3u8 manifests

See #8293. This is for consistency between YouTube and HLS formats.
---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8da70ae14..2f574054d 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1058,9 +1058,9 @@ class InfoExtractor(object):
                     # TODO: looks like video codec is not always necessarily goes first
                     va_codecs = codecs.split(',')
                     if va_codecs[0]:
-                        f['vcodec'] = va_codecs[0].partition('.')[0]
+                        f['vcodec'] = va_codecs[0]
                     if len(va_codecs) > 1 and va_codecs[1]:
-                        f['acodec'] = va_codecs[1].partition('.')[0]
+                        f['acodec'] = va_codecs[1]
                 resolution = last_info.get('RESOLUTION')
                 if resolution:
                     width_str, height_str = resolution.split('x')

From 2e6e742c3cda5ed4846bff4ef894aac21434e3d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 25 Jan 2016 22:15:21 +0600
Subject: [PATCH 0457/1105] [facebook] Add shortcut and reformat _VALID_URL

---
 youtube_dl/extractor/facebook.py | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index ec699ba54..f9fd8ed4e 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -23,15 +23,23 @@ from ..utils import (
 
 class FacebookIE(InfoExtractor):
     _VALID_URL = r'''(?x)
-        https?://(?:\w+\.)?facebook\.com/
-        (?:[^#]*?\#!/)?
-        (?:
-            (?:video/video\.php|photo\.php|video\.php|video/embed)\?(?:.*?)
-            (?:v|video_id)=|
-            [^/]+/videos/(?:[^/]+/)?
-        )
-        (?P<id>[0-9]+)
-        (?:.*)'''
+                (?:
+                    https?://
+                        (?:\w+\.)?facebook\.com/
+                        (?:[^#]*?\#!/)?
+                        (?:
+                            (?:
+                                video/video\.php|
+                                photo\.php|
+                                video\.php|
+                                video/embed
+                            )\?(?:.*?)(?:v|video_id)=|
+                            [^/]+/videos/(?:[^/]+/)?
+                        )|
+                    facebook:
+                )
+                (?P<id>[0-9]+)
+                '''
     _LOGIN_URL = 'https://www.facebook.com/login.php?next=http%3A%2F%2Ffacebook.com%2Fhome.php&login_attempt=1'
     _CHECKPOINT_URL = 'https://www.facebook.com/checkpoint/?next=http%3A%2F%2Ffacebook.com%2Fhome.php&_fb_noscript=1'
     _NETRC_MACHINE = 'facebook'
@@ -66,6 +74,9 @@ class FacebookIE(InfoExtractor):
     }, {
         'url': 'https://www.facebook.com/ChristyClarkForBC/videos/vb.22819070941/10153870694020942/?type=2&theater',
         'only_matching': True,
+    }, {
+        'url': 'facebook:544765982287235',
+        'only_matching': True,
     }]
 
     def _login(self):

From de691a498dd334ee5f8d237a4fc7ced314d86f44 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 25 Jan 2016 22:18:34 +0600
Subject: [PATCH 0458/1105] [facebook:post] Add extractor (Closes #8321)

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/facebook.py | 30 ++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 245e4d044..532be7e4c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -196,7 +196,10 @@ from .everyonesmixtape import EveryonesMixtapeIE
 from .exfm import ExfmIE
 from .expotv import ExpoTVIE
 from .extremetube import ExtremeTubeIE
-from .facebook import FacebookIE
+from .facebook import (
+    FacebookIE,
+    FacebookPostIE,
+)
 from .faz import FazIE
 from .fc2 import FC2IE
 from .fczenit import FczenitIE
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index f9fd8ed4e..cb5dd57fb 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -199,3 +199,33 @@ class FacebookIE(InfoExtractor):
             'formats': formats,
             'uploader': uploader,
         }
+
+
+class FacebookPostIE(InfoExtractor):
+    IE_NAME = 'facebook:post'
+    _VALID_URL = r'https?://(?:\w+\.)?facebook\.com/[^/]+/posts/(?P<id>\d+)'
+    _TEST = {
+        'url': 'https://www.facebook.com/maxlayn/posts/10153807558977570',
+        'md5': '037b1fa7f3c2d02b7a0d7bc16031ecc6',
+        'info_dict': {
+            'id': '544765982287235',
+            'ext': 'mp4',
+            'title': '"What are you doing running in the snow?"',
+            'uploader': 'FailArmy',
+        }
+    }
+
+    def _real_extract(self, url):
+        post_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, post_id)
+
+        entries = [
+            self.url_result('facebook:%s' % video_id, FacebookIE.ie_key())
+            for video_id in self._parse_json(
+                self._search_regex(
+                    r'(["\'])video_ids\1\s*:\s*(?P<ids>\[.+?\])',
+                    webpage, 'video ids', group='ids'),
+                post_id)]
+
+        return self.playlist_result(entries, post_id)

From 2975fe1a7be39b66d14fc7ef68ea7db948f0e40f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 25 Jan 2016 22:35:06 +0100
Subject: [PATCH 0459/1105] [vevo] extract all formats and bypass geo
 restriction

---
 youtube_dl/extractor/vevo.py | 234 +++++++++++++++--------------------
 1 file changed, 102 insertions(+), 132 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 02dfd36f4..ff854e74b 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -3,22 +3,18 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_etree_fromstring,
-    compat_urlparse,
-)
+from ..compat import compat_etree_fromstring
 from ..utils import (
     ExtractorError,
     int_or_none,
-    sanitized_Request,
 )
 
 
 class VevoIE(InfoExtractor):
-    """
+    '''
     Accepts urls from vevo.com or in the format 'vevo:{id}'
     (currently used by MTVIE and MySpaceIE)
-    """
+    '''
     _VALID_URL = r'''(?x)
         (?:https?://www\.vevo\.com/watch/(?:[^/]+/(?:[^/]+/)?)?|
            https?://cache\.vevo\.com/m/html/embed\.html\?video=|
@@ -28,48 +24,51 @@ class VevoIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.vevo.com/watch/hurts/somebody-to-die-for/GB1101300280',
-        "md5": "95ee28ee45e70130e3ab02b0f579ae23",
+        'md5': '2dbc7e9fd4f1c60436c9aa73a5406193',
         'info_dict': {
-            'id': 'GB1101300280',
+            'id': 'Pt1kc_FniKM',
             'ext': 'mp4',
-            "upload_date": "20130624",
-            "uploader": "Hurts",
-            "title": "Somebody to Die For",
-            "duration": 230.12,
-            "width": 1920,
-            "height": 1080,
-            # timestamp and upload_date are often incorrect; seem to change randomly
-            'timestamp': int,
-        }
+            'title': 'Hurts - Somebody to Die For',
+            'description': 'md5:13e925b89af6b01c7e417332bd23c4bf',
+            'uploader_id': 'HurtsVEVO',
+            'uploader': 'HurtsVEVO',
+            'upload_date': '20130624',
+            'duration': 230,
+        },
+        'add_ie': ['Youtube'],
     }, {
         'note': 'v3 SMIL format',
         'url': 'http://www.vevo.com/watch/cassadee-pope/i-wish-i-could-break-your-heart/USUV71302923',
-        'md5': 'f6ab09b034f8c22969020b042e5ac7fc',
+        'md5': '13d5204f520af905eeffa675040b8e76',
         'info_dict': {
-            'id': 'USUV71302923',
+            'id': 'ByGmQn1uxJw',
             'ext': 'mp4',
+            'title': 'Cassadee Pope - I Wish I Could Break Your Heart',
+            'description': 'md5:5e9721c92ef117a6f69d00e9b42ceba7',
+            'uploader_id': 'CassadeeVEVO',
+            'uploader': 'CassadeeVEVO',
             'upload_date': '20140219',
-            'uploader': 'Cassadee Pope',
-            'title': 'I Wish I Could Break Your Heart',
-            'duration': 226.101,
+            'duration': 226,
             'age_limit': 0,
-            'timestamp': int,
-        }
+        },
+        'add_ie': ['Youtube'],
     }, {
         'note': 'Age-limited video',
         'url': 'https://www.vevo.com/watch/justin-timberlake/tunnel-vision-explicit/USRV81300282',
         'info_dict': {
-            'id': 'USRV81300282',
+            'id': '07FYdnEawAQ',
             'ext': 'mp4',
             'age_limit': 18,
-            'title': 'Tunnel Vision (Explicit)',
-            'uploader': 'Justin Timberlake',
-            'upload_date': 're:2013070[34]',
-            'timestamp': int,
+            'title': 'Justin Timberlake - Tunnel Vision (Explicit)',
+            'description': 'md5:64249768eec3bc4276236606ea996373',
+            'uploader_id': 'justintimberlakeVEVO',
+            'uploader': 'justintimberlakeVEVO',
+            'upload_date': '20130703',
         },
         'params': {
             'skip_download': 'true',
-        }
+        },
+        'add_ie': ['Youtube'],
     }, {
         'note': 'No video_info',
         'url': 'http://www.vevo.com/watch/k-camp-1/Till-I-Die/USUV71503000',
@@ -80,66 +79,40 @@ class VevoIE(InfoExtractor):
             'title': 'Till I Die - K Camp ft. T.I.',
             'duration': 193,
         },
-        'expected_warnings': ['Unable to download SMIL file'],
     }]
-    _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com/'
+    _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com'
+    _SOURCE_TYPES = {
+        0: 'youtube',
+        1: 'brightcove',
+        2: 'http',
+        3: 'hls_ios',
+        4: 'hls',
+        5: 'smil',  # http
+        7: 'f4m_cc',
+        8: 'f4m_ak',
+        9: 'f4m_l3',
+        10: 'ism',
+        13: 'smil',  # rtmp
+        18: 'dash',
+    }
+    _VERSIONS = {
+        0: 'youtube',
+        1: 'level3',
+        2: 'akamai',
+        3: 'level3',
+        4: 'amazon',
+    }
 
-    def _real_initialize(self):
-        req = sanitized_Request(
-            'http://www.vevo.com/auth', data=b'')
-        webpage = self._download_webpage(
-            req, None,
-            note='Retrieving oauth token',
-            errnote='Unable to retrieve oauth token',
-            fatal=False)
-        if webpage is False:
-            self._oauth_token = None
-        else:
-            if 'THIS PAGE IS CURRENTLY UNAVAILABLE IN YOUR REGION' in webpage:
-                raise ExtractorError('%s said: This page is currently unavailable in your region.' % self.IE_NAME, expected=True)
-
-            self._oauth_token = self._search_regex(
-                r'access_token":\s*"([^"]+)"',
-                webpage, 'access token', fatal=False)
-
-    def _formats_from_json(self, video_info):
-        if not video_info:
-            return []
-
-        last_version = {'version': -1}
-        for version in video_info['videoVersions']:
-            # These are the HTTP downloads, other types are for different manifests
-            if version['sourceType'] == 2:
-                if version['version'] > last_version['version']:
-                    last_version = version
-        if last_version['version'] == -1:
-            raise ExtractorError('Unable to extract last version of the video')
-
-        renditions = compat_etree_fromstring(last_version['data'])
+    def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
         formats = []
-        # Already sorted from worst to best quality
-        for rend in renditions.findall('rendition'):
-            attr = rend.attrib
-            format_note = '%(videoCodec)s@%(videoBitrate)4sk, %(audioCodec)s@%(audioBitrate)3sk' % attr
-            formats.append({
-                'url': attr['url'],
-                'format_id': attr['name'],
-                'format_note': format_note,
-                'height': int(attr['frameheight']),
-                'width': int(attr['frameWidth']),
-            })
-        return formats
-
-    def _formats_from_smil(self, smil_doc):
-        formats = []
-        els = smil_doc.findall('.//{http://www.w3.org/2001/SMIL20/Language}video')
+        els = smil.findall('.//{http://www.w3.org/2001/SMIL20/Language}video')
         for el in els:
             src = el.attrib['src']
             m = re.match(r'''(?xi)
                 (?P<ext>[a-z0-9]+):
                 (?P<path>
                     [/a-z0-9]+     # The directory and main part of the URL
-                    _(?P<cbr>[0-9]+)k
+                    _(?P<tbr>[0-9]+)k
                     _(?P<width>[0-9]+)x(?P<height>[0-9]+)
                     _(?P<vcodec>[a-z0-9]+)
                     _(?P<vbr>[0-9]+)
@@ -153,9 +126,10 @@ class VevoIE(InfoExtractor):
             format_url = self._SMIL_BASE_URL + m.group('path')
             formats.append({
                 'url': format_url,
-                'format_id': 'SMIL_' + m.group('cbr'),
+                'format_id': 'smil_' + m.group('tbr'),
                 'vcodec': m.group('vcodec'),
                 'acodec': m.group('acodec'),
+                'tbr': int(m.group('tbr')),
                 'vbr': int(m.group('vbr')),
                 'abr': int(m.group('abr')),
                 'ext': m.group('ext'),
@@ -164,26 +138,6 @@ class VevoIE(InfoExtractor):
             })
         return formats
 
-    def _download_api_formats(self, video_id, video_url):
-        if not self._oauth_token:
-            self._downloader.report_warning(
-                'No oauth token available, skipping API HLS download')
-            return []
-
-        api_url = compat_urlparse.urljoin(video_url, '//apiv2.vevo.com/video/%s/streams/hls?token=%s' % (
-            video_id, self._oauth_token))
-        api_data = self._download_json(
-            api_url, video_id,
-            note='Downloading HLS formats',
-            errnote='Failed to download HLS format list', fatal=False)
-        if api_data is None:
-            return []
-
-        m3u8_url = api_data[0]['url']
-        return self._extract_m3u8_formats(
-            m3u8_url, video_id, entry_protocol='m3u8_native', ext='mp4',
-            preference=0)
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -193,19 +147,57 @@ class VevoIE(InfoExtractor):
         response = self._download_json(json_url, video_id)
         video_info = response['video'] or {}
 
-        if not video_info and response.get('statusCode') != 909:
-            if 'statusMessage' in response:
-                raise ExtractorError('%s said: %s' % (self.IE_NAME, response['statusMessage']), expected=True)
-            raise ExtractorError('Unable to extract videos')
-
         if not video_info:
+            ytid = response.get('errorInfo', {}).get('ytid')
+            if ytid:
+                return self.url_result(ytid, 'Youtube', ytid)
+
+            if response.get('statusCode') != 909:
+                if 'statusMessage' in response:
+                    raise ExtractorError('%s said: %s' % (
+                        self.IE_NAME, response['statusMessage']), expected=True)
+                raise ExtractorError('Unable to extract videos')
+
             if url.startswith('vevo:'):
-                raise ExtractorError('Please specify full Vevo URL for downloading', expected=True)
+                raise ExtractorError(
+                    'Please specify full Vevo URL for downloading', expected=True)
             webpage = self._download_webpage(url, video_id)
 
         title = video_info.get('title') or self._og_search_title(webpage)
 
-        formats = self._formats_from_json(video_info)
+        smil_parsed = False
+        formats = []
+        for video_version in video_info['videoVersions']:
+            version = self._VERSIONS.get(video_version['version'])
+            if version == 'youtube':
+                return self.url_result(
+                    video_version['id'], 'Youtube', video_version['id'])
+            else:
+                source_type = self._SOURCE_TYPES.get(video_version['sourceType'])
+                renditions = compat_etree_fromstring(video_version['data'])
+                if source_type == 'http':
+                    for rend in renditions.findall('rendition'):
+                        attr = rend.attrib
+                        formats.append({
+                            'url': attr['url'],
+                            'format_id': '%s-%s' % (version, attr['name']),
+                            'height': int_or_none(attr.get('frameheight')),
+                            'width': int_or_none(attr.get('frameWidth')),
+                            'tbr': int_or_none(attr.get('totalBitrate')),
+                            'vbr': int_or_none(attr.get('videoBitrate')),
+                            'abr': int_or_none(attr.get('audioBitrate')),
+                            'vcodec': attr.get('videoCodec'),
+                            'acodec': attr.get('audioCodec'),
+                        })
+                elif source_type == 'hls':
+                    formats.extend(self._extract_m3u8_formats(
+                        renditions.find('rendition').attrib['url'], video_id,
+                        'mp4', 'm3u8_native', m3u8_id='hls-%s' % version, fatal=False))
+                elif source_type == 'smil' and not smil_parsed:
+                    formats.extend(self._extract_smil_formats(
+                        renditions.find('rendition').attrib['url'], video_id, False))
+                    smil_parsed = True
+        self._sort_formats(formats)
 
         is_explicit = video_info.get('isExplicit')
         if is_explicit is True:
@@ -215,28 +207,6 @@ class VevoIE(InfoExtractor):
         else:
             age_limit = None
 
-        # Download via HLS API
-        formats.extend(self._download_api_formats(video_id, url))
-
-        # Download SMIL
-        smil_blocks = sorted((
-            f for f in video_info.get('videoVersions', [])
-            if f['sourceType'] == 13),
-            key=lambda f: f['version'])
-        smil_url = '%s/Video/V2/VFILE/%s/%sr.smil' % (
-            self._SMIL_BASE_URL, video_id, video_id.lower())
-        if smil_blocks:
-            smil_url_m = self._search_regex(
-                r'url="([^"]+)"', smil_blocks[-1]['data'], 'SMIL URL',
-                default=None)
-            if smil_url_m is not None:
-                smil_url = smil_url_m
-        if smil_url:
-            smil_doc = self._download_smil(smil_url, video_id, fatal=False)
-            if smil_doc:
-                formats.extend(self._formats_from_smil(smil_doc))
-
-        self._sort_formats(formats)
         timestamp = int_or_none(self._search_regex(
             r'/Date\((\d+)\)/',
             video_info['launchDate'], 'launch date', fatal=False),

From 9165d6bab9ae3cc98984ebeb249ef320b9f4db83 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 26 Jan 2016 13:46:58 +0100
Subject: [PATCH 0460/1105] [vevo] extract metadata and formats from api if
 videoinfo is empty

these was fixed by @yan12125 in ff51983e1563db08734e43a07d5671a517f99ef6
i only added some code to extract video metadata and more formats from
api
---
 youtube_dl/extractor/vevo.py | 189 ++++++++++++++++++++++++++---------
 1 file changed, 140 insertions(+), 49 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index ff854e74b..9c955c895 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -7,6 +7,8 @@ from ..compat import compat_etree_fromstring
 from ..utils import (
     ExtractorError,
     int_or_none,
+    sanitized_Request,
+    parse_iso8601,
 )
 
 
@@ -64,6 +66,7 @@ class VevoIE(InfoExtractor):
             'uploader_id': 'justintimberlakeVEVO',
             'uploader': 'justintimberlakeVEVO',
             'upload_date': '20130703',
+            'duration': 419,
         },
         'params': {
             'skip_download': 'true',
@@ -72,13 +75,18 @@ class VevoIE(InfoExtractor):
     }, {
         'note': 'No video_info',
         'url': 'http://www.vevo.com/watch/k-camp-1/Till-I-Die/USUV71503000',
-        'md5': '8b83cc492d72fc9cf74a02acee7dc1b0',
+        'md5': 'a8b84d1d1957cd01046441b701b270fb',
         'info_dict': {
-            'id': 'USUV71503000',
+            'id': 'Lad2jHtJCqY',
             'ext': 'mp4',
-            'title': 'Till I Die - K Camp ft. T.I.',
+            'title': 'K Camp - Till I Die ft. T.I.',
+            'description': 'md5:0694920ededdee4a14cfc39695cc8ec3',
+            'uploader_id': 'KCampVEVO',
+            'uploader': 'KCampVEVO',
+            'upload_date': '20151207',
             'duration': 193,
         },
+        'add_ie': ['Youtube'],
     }]
     _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com'
     _SOURCE_TYPES = {
@@ -96,7 +104,7 @@ class VevoIE(InfoExtractor):
         18: 'dash',
     }
     _VERSIONS = {
-        0: 'youtube',
+        0: 'youtube',  # only in AuthenticateVideo videoVersions
         1: 'level3',
         2: 'akamai',
         3: 'level3',
@@ -138,14 +146,34 @@ class VevoIE(InfoExtractor):
             })
         return formats
 
+    def _initialize_api(self, video_url, video_id):
+        req = sanitized_Request(
+            'http://www.vevo.com/auth', data=b'')
+        webpage = self._download_webpage(
+            req, None,
+            note='Retrieving oauth token',
+            errnote='Unable to retrieve oauth token')
+
+        if 'THIS PAGE IS CURRENTLY UNAVAILABLE IN YOUR REGION' in webpage:
+            raise ExtractorError('%s said: This page is currently unavailable in your region.' % self.IE_NAME, expected=True)
+
+        auth_info = self._parse_json(webpage, video_id)
+        self._api_url_template = self.http_scheme() + '//apiv2.vevo.com/%s?token=' + auth_info['access_token']
+
+    def _call_api(self, path, video_id, note, errnote, fatal=True):
+        return self._download_json(self._api_url_template % path, video_id, note, errnote)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = None
-
         json_url = 'http://videoplayer.vevo.com/VideoService/AuthenticateVideo?isrc=%s' % video_id
-        response = self._download_json(json_url, video_id)
-        video_info = response['video'] or {}
+        response = self._download_json(json_url, video_id, 'Downloading video info', 'Unable to download info')
+        video_info = response.get('video') or {}
+        video_versions = video_info.get('videoVersions')
+        uploader = None
+        timestamp = None
+        view_count = None
+        formats = []
 
         if not video_info:
             ytid = response.get('errorInfo', {}).get('ytid')
@@ -161,44 +189,112 @@ class VevoIE(InfoExtractor):
             if url.startswith('vevo:'):
                 raise ExtractorError(
                     'Please specify full Vevo URL for downloading', expected=True)
-            webpage = self._download_webpage(url, video_id)
 
-        title = video_info.get('title') or self._og_search_title(webpage)
+            self._initialize_api(url, video_id)
+            video_info = self._call_api(
+                'video/%s' % video_id, video_id, 'Downloading api video info',
+                'Failed to download video info')
 
-        smil_parsed = False
-        formats = []
-        for video_version in video_info['videoVersions']:
-            version = self._VERSIONS.get(video_version['version'])
-            if version == 'youtube':
+            ytid = video_info.get('youTubeId')
+            if ytid:
                 return self.url_result(
-                    video_version['id'], 'Youtube', video_version['id'])
-            else:
-                source_type = self._SOURCE_TYPES.get(video_version['sourceType'])
-                renditions = compat_etree_fromstring(video_version['data'])
-                if source_type == 'http':
-                    for rend in renditions.findall('rendition'):
-                        attr = rend.attrib
-                        formats.append({
-                            'url': attr['url'],
-                            'format_id': '%s-%s' % (version, attr['name']),
-                            'height': int_or_none(attr.get('frameheight')),
-                            'width': int_or_none(attr.get('frameWidth')),
-                            'tbr': int_or_none(attr.get('totalBitrate')),
-                            'vbr': int_or_none(attr.get('videoBitrate')),
-                            'abr': int_or_none(attr.get('audioBitrate')),
-                            'vcodec': attr.get('videoCodec'),
-                            'acodec': attr.get('audioCodec'),
-                        })
-                elif source_type == 'hls':
+                    ytid, 'Youtube', ytid)
+
+            video_versions = self._call_api(
+                'video/%s/streams' % video_id, video_id,
+                'Downloading video versions info',
+                'Failed to download video versions info')
+
+            timestamp = parse_iso8601(video_info.get('releaseDate'))
+            artists = video_info.get('artists')
+            if artists:
+                uploader = artists[0]['name']
+            view_count = int_or_none(video_info.get('views', {}).get('total'))
+
+            for video_version in video_versions:
+                version = self._VERSIONS.get(video_version['version'])
+                version_url = video_version.get('url')
+                if not version_url:
+                        continue
+
+                if '.mpd' in version_url or '.ism' in version_url:
+                    continue
+                elif '.m3u8' in version_url:
                     formats.extend(self._extract_m3u8_formats(
-                        renditions.find('rendition').attrib['url'], video_id,
-                        'mp4', 'm3u8_native', m3u8_id='hls-%s' % version, fatal=False))
-                elif source_type == 'smil' and not smil_parsed:
-                    formats.extend(self._extract_smil_formats(
-                        renditions.find('rendition').attrib['url'], video_id, False))
-                    smil_parsed = True
+                        version_url, video_id, 'mp4', 'm3u8_native',
+                        m3u8_id='hls-%s' % version,
+                        note='Downloading %s m3u8 information' % version,
+                        errnote='Failed to download %s m3u8 information' % version,
+                        fatal=False))
+                else:
+                    m = re.search(r'''(?xi)
+                        _(?P<width>[0-9]+)x(?P<height>[0-9]+)
+                        _(?P<vcodec>[a-z0-9]+)
+                        _(?P<vbr>[0-9]+)
+                        _(?P<acodec>[a-z0-9]+)
+                        _(?P<abr>[0-9]+)
+                        \.(?P<ext>[a-z0-9]+)''', version_url)
+                    if not m:
+                        continue
+
+                    formats.append({
+                        'url': version_url,
+                        'format_id': 'http-%s-%s' % (version, video_version['quality']),
+                        'vcodec': m.group('vcodec'),
+                        'acodec': m.group('acodec'),
+                        'vbr': int(m.group('vbr')),
+                        'abr': int(m.group('abr')),
+                        'ext': m.group('ext'),
+                        'width': int(m.group('width')),
+                        'height': int(m.group('height')),
+                    })
+        else:
+            timestamp = int_or_none(self._search_regex(
+                r'/Date\((\d+)\)/',
+                video_info['releaseDate'], 'release date', fatal=False),
+                scale=1000)
+            artists = video_info.get('mainArtists')
+            if artists:
+                uploader = artists[0]['artistName']
+
+            smil_parsed = False
+            for video_version in video_info['videoVersions']:
+                version = self._VERSIONS.get(video_version['version'])
+                if version == 'youtube':
+                    return self.url_result(
+                        video_version['id'], 'Youtube', video_version['id'])
+                else:
+                    source_type = self._SOURCE_TYPES.get(video_version['sourceType'])
+                    renditions = compat_etree_fromstring(video_version['data'])
+                    if source_type == 'http':
+                        for rend in renditions.findall('rendition'):
+                            attr = rend.attrib
+                            formats.append({
+                                'url': attr['url'],
+                                'format_id': 'http-%s-%s' % (version, attr['name']),
+                                'height': int_or_none(attr.get('frameheight')),
+                                'width': int_or_none(attr.get('frameWidth')),
+                                'tbr': int_or_none(attr.get('totalBitrate')),
+                                'vbr': int_or_none(attr.get('videoBitrate')),
+                                'abr': int_or_none(attr.get('audioBitrate')),
+                                'vcodec': attr.get('videoCodec'),
+                                'acodec': attr.get('audioCodec'),
+                            })
+                    elif source_type == 'hls':
+                        formats.extend(self._extract_m3u8_formats(
+                            renditions.find('rendition').attrib['url'], video_id,
+                            'mp4', 'm3u8_native', m3u8_id='hls-%s' % version,
+                            note='Downloading %s m3u8 information' % version,
+                            errnote='Failed to download %s m3u8 information' % version,
+                            fatal=False))
+                    elif source_type == 'smil' and not smil_parsed:
+                        formats.extend(self._extract_smil_formats(
+                            renditions.find('rendition').attrib['url'], video_id, False))
+                        smil_parsed = True
         self._sort_formats(formats)
 
+        title = video_info['title']
+
         is_explicit = video_info.get('isExplicit')
         if is_explicit is True:
             age_limit = 18
@@ -207,21 +303,16 @@ class VevoIE(InfoExtractor):
         else:
             age_limit = None
 
-        timestamp = int_or_none(self._search_regex(
-            r'/Date\((\d+)\)/',
-            video_info['launchDate'], 'launch date', fatal=False),
-            scale=1000) if video_info else None
-
-        duration = video_info.get('duration') or int_or_none(
-            self._html_search_meta('video:duration', webpage))
+        duration = video_info.get('duration')
 
         return {
             'id': video_id,
             'title': title,
             'formats': formats,
-            'thumbnail': video_info.get('imageUrl'),
+            'thumbnail': video_info.get('imageUrl') or video_info.get('thumbnailUrl'),
             'timestamp': timestamp,
-            'uploader': video_info['mainArtists'][0]['artistName'] if video_info else None,
+            'uploader': uploader,
             'duration': duration,
+            'view_count': view_count,
             'age_limit': age_limit,
         }

From f693213567010ecc108447cba4615ae2932d1c18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 26 Jan 2016 20:42:20 +0600
Subject: [PATCH 0461/1105] [cspan] Fix clip/prog id extraction (#8317)

---
 youtube_dl/extractor/cspan.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index b3ee67018..f78cbbd7e 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -68,11 +68,16 @@ class CSpanIE(InfoExtractor):
             video_type, video_id = matches.groups()
             video_type = 'clip' if video_type == 'id' else 'program'
         else:
-            senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
-            if senate_isvp_url:
-                title = self._og_search_title(webpage)
-                surl = smuggle_url(senate_isvp_url, {'force_title': title})
-                return self.url_result(surl, 'SenateISVP', video_id, title)
+            m = re.search(r'data-(?P<type>clip|prog)id=["\'](?P<id>\d+)', webpage)
+            if m:
+                video_id = m.group('id')
+                video_type = 'program' if m.group('type') == 'prog' else 'clip'
+            else:
+                senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
+                if senate_isvp_url:
+                    title = self._og_search_title(webpage)
+                    surl = smuggle_url(senate_isvp_url, {'force_title': title})
+                    return self.url_result(surl, 'SenateISVP', video_id, title)
         if video_type is None or video_id is None:
             raise ExtractorError('unable to find video id and type')
 

From 682f8c43b50d8b5e2c02e34187cbddb0d5f8d3ed Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 26 Jan 2016 15:54:32 +0100
Subject: [PATCH 0462/1105] [vevo] fallback to youtube video only if vevo video
 is geo restricted(fixes 8263)(fixes 2874)

---
 youtube_dl/extractor/vevo.py | 93 ++++++++++++++----------------------
 1 file changed, 36 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 9c955c895..35fcff1b2 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -26,67 +26,52 @@ class VevoIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.vevo.com/watch/hurts/somebody-to-die-for/GB1101300280',
-        'md5': '2dbc7e9fd4f1c60436c9aa73a5406193',
+        'md5': '95ee28ee45e70130e3ab02b0f579ae23',
         'info_dict': {
-            'id': 'Pt1kc_FniKM',
+            'id': 'GB1101300280',
             'ext': 'mp4',
-            'title': 'Hurts - Somebody to Die For',
-            'description': 'md5:13e925b89af6b01c7e417332bd23c4bf',
-            'uploader_id': 'HurtsVEVO',
-            'uploader': 'HurtsVEVO',
+            'title': 'Somebody to Die For',
             'upload_date': '20130624',
-            'duration': 230,
+            'uploader': 'Hurts',
+            'timestamp': 1372057200,
         },
-        'add_ie': ['Youtube'],
     }, {
         'note': 'v3 SMIL format',
         'url': 'http://www.vevo.com/watch/cassadee-pope/i-wish-i-could-break-your-heart/USUV71302923',
-        'md5': '13d5204f520af905eeffa675040b8e76',
+        'md5': 'f6ab09b034f8c22969020b042e5ac7fc',
         'info_dict': {
-            'id': 'ByGmQn1uxJw',
+            'id': 'USUV71302923',
             'ext': 'mp4',
-            'title': 'Cassadee Pope - I Wish I Could Break Your Heart',
-            'description': 'md5:5e9721c92ef117a6f69d00e9b42ceba7',
-            'uploader_id': 'CassadeeVEVO',
-            'uploader': 'CassadeeVEVO',
+            'title': 'I Wish I Could Break Your Heart',
             'upload_date': '20140219',
-            'duration': 226,
-            'age_limit': 0,
+            'uploader': 'Cassadee Pope',
+            'timestamp': 1392796919,
         },
-        'add_ie': ['Youtube'],
     }, {
         'note': 'Age-limited video',
         'url': 'https://www.vevo.com/watch/justin-timberlake/tunnel-vision-explicit/USRV81300282',
         'info_dict': {
-            'id': '07FYdnEawAQ',
+            'id': 'USRV81300282',
             'ext': 'mp4',
-            'age_limit': 18,
-            'title': 'Justin Timberlake - Tunnel Vision (Explicit)',
-            'description': 'md5:64249768eec3bc4276236606ea996373',
-            'uploader_id': 'justintimberlakeVEVO',
-            'uploader': 'justintimberlakeVEVO',
+            'title': 'Tunnel Vision (Explicit)',
             'upload_date': '20130703',
-            'duration': 419,
+            'age_limit': 18,
+            'uploader': 'Justin Timberlake',
+            'timestamp': 1372888800,
         },
-        'params': {
-            'skip_download': 'true',
-        },
-        'add_ie': ['Youtube'],
     }, {
         'note': 'No video_info',
         'url': 'http://www.vevo.com/watch/k-camp-1/Till-I-Die/USUV71503000',
-        'md5': 'a8b84d1d1957cd01046441b701b270fb',
+        'md5': '8b83cc492d72fc9cf74a02acee7dc1b0',
         'info_dict': {
-            'id': 'Lad2jHtJCqY',
+            'id': 'USUV71503000',
             'ext': 'mp4',
-            'title': 'K Camp - Till I Die ft. T.I.',
-            'description': 'md5:0694920ededdee4a14cfc39695cc8ec3',
-            'uploader_id': 'KCampVEVO',
-            'uploader': 'KCampVEVO',
+            'title': 'Till I Die',
             'upload_date': '20151207',
-            'duration': 193,
+            'age_limit': 18,
+            'uploader': 'K Camp',
+            'timestamp': 1449468000,
         },
-        'add_ie': ['Youtube'],
     }]
     _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com'
     _SOURCE_TYPES = {
@@ -146,7 +131,7 @@ class VevoIE(InfoExtractor):
             })
         return formats
 
-    def _initialize_api(self, video_url, video_id):
+    def _initialize_api(self, video_id):
         req = sanitized_Request(
             'http://www.vevo.com/auth', data=b'')
         webpage = self._download_webpage(
@@ -155,7 +140,8 @@ class VevoIE(InfoExtractor):
             errnote='Unable to retrieve oauth token')
 
         if 'THIS PAGE IS CURRENTLY UNAVAILABLE IN YOUR REGION' in webpage:
-            raise ExtractorError('%s said: This page is currently unavailable in your region.' % self.IE_NAME, expected=True)
+            raise ExtractorError(
+                '%s said: This page is currently unavailable in your region.' % self.IE_NAME, expected=True)
 
         auth_info = self._parse_json(webpage, video_id)
         self._api_url_template = self.http_scheme() + '//apiv2.vevo.com/%s?token=' + auth_info['access_token']
@@ -167,7 +153,8 @@ class VevoIE(InfoExtractor):
         video_id = self._match_id(url)
 
         json_url = 'http://videoplayer.vevo.com/VideoService/AuthenticateVideo?isrc=%s' % video_id
-        response = self._download_json(json_url, video_id, 'Downloading video info', 'Unable to download info')
+        response = self._download_json(
+            json_url, video_id, 'Downloading video info', 'Unable to download info')
         video_info = response.get('video') or {}
         video_versions = video_info.get('videoVersions')
         uploader = None
@@ -176,30 +163,23 @@ class VevoIE(InfoExtractor):
         formats = []
 
         if not video_info:
-            ytid = response.get('errorInfo', {}).get('ytid')
-            if ytid:
-                return self.url_result(ytid, 'Youtube', ytid)
-
             if response.get('statusCode') != 909:
+                ytid = response.get('errorInfo', {}).get('ytid')
+                if ytid:
+                    self.report_warning(
+                        'Video is geoblocked, trying with the YouTube video %s' % ytid)
+                    return self.url_result(ytid, 'Youtube', ytid)
+
                 if 'statusMessage' in response:
                     raise ExtractorError('%s said: %s' % (
                         self.IE_NAME, response['statusMessage']), expected=True)
                 raise ExtractorError('Unable to extract videos')
 
-            if url.startswith('vevo:'):
-                raise ExtractorError(
-                    'Please specify full Vevo URL for downloading', expected=True)
-
-            self._initialize_api(url, video_id)
+            self._initialize_api(video_id)
             video_info = self._call_api(
                 'video/%s' % video_id, video_id, 'Downloading api video info',
                 'Failed to download video info')
 
-            ytid = video_info.get('youTubeId')
-            if ytid:
-                return self.url_result(
-                    ytid, 'Youtube', ytid)
-
             video_versions = self._call_api(
                 'video/%s/streams' % video_id, video_id,
                 'Downloading video versions info',
@@ -215,7 +195,7 @@ class VevoIE(InfoExtractor):
                 version = self._VERSIONS.get(video_version['version'])
                 version_url = video_version.get('url')
                 if not version_url:
-                        continue
+                    continue
 
                 if '.mpd' in version_url or '.ism' in version_url:
                     continue
@@ -261,8 +241,7 @@ class VevoIE(InfoExtractor):
             for video_version in video_info['videoVersions']:
                 version = self._VERSIONS.get(video_version['version'])
                 if version == 'youtube':
-                    return self.url_result(
-                        video_version['id'], 'Youtube', video_version['id'])
+                    continue
                 else:
                     source_type = self._SOURCE_TYPES.get(video_version['sourceType'])
                     renditions = compat_etree_fromstring(video_version['data'])
@@ -287,7 +266,7 @@ class VevoIE(InfoExtractor):
                             note='Downloading %s m3u8 information' % version,
                             errnote='Failed to download %s m3u8 information' % version,
                             fatal=False))
-                    elif source_type == 'smil' and not smil_parsed:
+                    elif source_type == 'smil' and version == 'level3' and not smil_parsed:
                         formats.extend(self._extract_smil_formats(
                             renditions.find('rendition').attrib['url'], video_id, False))
                         smil_parsed = True

From af9c2a07aea530b3bee560a953e94ac92fcd49c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 26 Jan 2016 21:29:42 +0600
Subject: [PATCH 0463/1105] [cspan] Extract from path when no qualities (Closes
 #8317)

---
 youtube_dl/extractor/cspan.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index f78cbbd7e..b78edf729 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -112,6 +112,13 @@ class CSpanIE(InfoExtractor):
                     'height': int_or_none(get_text_attr(quality, 'height')),
                     'tbr': int_or_none(get_text_attr(quality, 'bitrate')),
                 })
+            if not formats:
+                path = get_text_attr(f, 'path')
+                if not path:
+                    continue
+                formats = self._extract_m3u8_formats(
+                    path, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls') if determine_ext(path) == 'm3u8' else [{'url': path, }]
             self._sort_formats(formats)
             entries.append({
                 'id': '%s_%d' % (video_id, partnum + 1),

From fb4b3458000bff87f2b083f9a5e6853c043d032d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 26 Jan 2016 21:46:51 +0600
Subject: [PATCH 0464/1105] [instagram] Make description optional (Closes
 #8326)

---
 youtube_dl/extractor/instagram.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index e5e16ca3b..ed3e07118 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -21,6 +21,18 @@ class InstagramIE(InfoExtractor):
             'title': 'Video by naomipq',
             'description': 'md5:1f17f0ab29bd6fe2bfad705f58de3cb8',
         }
+    }, {
+        # missing description
+        'url': 'https://www.instagram.com/p/BA-pQFBG8HZ/?taken-by=britneyspears',
+        'info_dict': {
+            'id': 'BA-pQFBG8HZ',
+            'ext': 'mp4',
+            'uploader_id': 'britneyspears',
+            'title': 'Video by britneyspears',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'https://instagram.com/p/-Cmh1cukG2/',
         'only_matching': True,
@@ -32,8 +44,8 @@ class InstagramIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
         uploader_id = self._search_regex(r'"owner":{"username":"(.+?)"',
                                          webpage, 'uploader id', fatal=False)
-        desc = self._search_regex(r'"caption":"(.*?)"', webpage, 'description',
-                                  fatal=False)
+        desc = self._search_regex(
+            r'"caption":"(.+?)"', webpage, 'description', default=None)
 
         return {
             'id': video_id,

From b6c33fd544adddf399a80842eb9a7d2250aa04b6 Mon Sep 17 00:00:00 2001
From: ping <liping.ong@gmail.com>
Date: Wed, 27 Jan 2016 12:48:00 +0800
Subject: [PATCH 0465/1105] [daum.net] Fixes #8331

---
 youtube_dl/extractor/daum.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index f08f57157..c84302c43 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -2,6 +2,8 @@
 
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse
 from ..utils import (
@@ -28,6 +30,15 @@ class DaumIE(InfoExtractor):
             'comment_count': int,
         },
     }, {
+        'url': 'http://m.tvpot.daum.net/v/65139429',
+        'info_dict': {
+            'id': '65139429',
+            'ext': 'mp4',
+            'title': 'md5:a100d65d09cec246d8aa9bde7de45aed',
+            'description': 'md5:79794514261164ff27e36a21ad229fc5',
+            'upload_date': '20150604',
+            'duration': 154
+        },    }, {
         'url': 'http://tvpot.daum.net/v/07dXWRka62Y%24',
         'only_matching': True,
     }]
@@ -42,6 +53,10 @@ class DaumIE(InfoExtractor):
             'http://videofarm.daum.net/controller/api/closed/v1_2/IntegratedMovieData.json?' + query,
             video_id, 'Downloading video formats info')
 
+        # For urls like http://m.tvpot.daum.net/v/65139429, where the video_id is really a clipid
+        if not movie_data.get('output_list', {}).get('output_list') and re.match(r'^\d+$', video_id):
+            return self.url_result('http://tvpot.daum.net/clip/ClipView.do?clipid=%s' % video_id)
+
         formats = []
         for format_el in movie_data['output_list']['output_list']:
             profile = format_el['profile']
@@ -76,7 +91,7 @@ class DaumIE(InfoExtractor):
 
 
 class DaumClipIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/(?:clip/ClipView.do|mypot/View.do)\?.*?clipid=(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/(?:clip/ClipView.(?:do|tv)|mypot/View.do)\?.*?clipid=(?P<id>\d+)'
     IE_NAME = 'daum.net:clip'
 
     _TESTS = [{
@@ -90,6 +105,9 @@ class DaumClipIE(InfoExtractor):
             'duration': 3868,
             'view_count': int,
         },
+    }, {
+        'url': 'http://m.tvpot.daum.net/clip/ClipView.tv?clipid=54999425',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From fab6f0e65b0084ce8b29c0810b874d132ffcd47c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 27 Jan 2016 08:32:03 +0100
Subject: [PATCH 0466/1105] release 2016.01.27

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index e86467cfa..eb68c23b5 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -180,6 +180,7 @@
  - **ExpoTV**
  - **ExtremeTube**
  - **facebook**
+ - **facebook:post**
  - **faz.net**
  - **fc2**
  - **Fczenit**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d5bf73815..4ac7f9e93 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.01.23'
+__version__ = '2016.01.27'

From b0d21deda95e7e49e162c12aa305f27e477e0ce9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 Jan 2016 21:11:17 +0600
Subject: [PATCH 0467/1105] [extractor/common] Auto calculate tbr when missing

---
 youtube_dl/extractor/common.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2f574054d..f5a358388 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -825,6 +825,12 @@ class InfoExtractor(object):
         if not formats:
             raise ExtractorError('No video formats found')
 
+        for f in formats:
+            # Automatically determine tbr when missing based on abr and vbr (improves
+            # formats sorting in some cases)
+            if 'tbr' not in f and 'abr' in f and 'vbr' in f:
+                f['tbr'] = f['abr'] + f['vbr']
+
         def _formats_key(f):
             # TODO remove the following workaround
             from ..utils import determine_ext

From 9339774af274b04a054b0fa151347619e40b5b21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 Jan 2016 21:49:39 +0600
Subject: [PATCH 0468/1105] [spankbang] Fix formats extraction

---
 youtube_dl/extractor/spankbang.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index 7f060b15b..a67db5905 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -34,7 +34,7 @@ class SpankBangIE(InfoExtractor):
             'ext': 'mp4',
             'format_id': '%sp' % height,
             'height': int(height),
-        } for height in re.findall(r'<span[^>]+q_(\d+)p', webpage)]
+        } for height in re.findall(r'<(?:span|li)[^>]+q_(\d+)p', webpage)]
         self._sort_formats(formats)
 
         title = self._html_search_regex(

From c3111ab34fed3af16f645c6ee4dcd6e5d35b69dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 Jan 2016 21:49:56 +0600
Subject: [PATCH 0469/1105] [spankbang] Fix title extraction (Closes #8329)

---
 youtube_dl/extractor/spankbang.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index a67db5905..3cfa671ed 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -38,7 +38,7 @@ class SpankBangIE(InfoExtractor):
         self._sort_formats(formats)
 
         title = self._html_search_regex(
-            r'(?s)<h1>(.+?)</h1>', webpage, 'title')
+            r'(?s)<h1[^>]*>(.+?)</h1>', webpage, 'title')
         description = self._search_regex(
             r'class="desc"[^>]*>([^<]+)',
             webpage, 'description', default=None)

From 7f32e5dc350e2a641e6304855727623f95bef31c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 26 Jan 2016 17:44:44 +0100
Subject: [PATCH 0470/1105] [extractor/common] detect media playlist in
 _extract_m3u8_formats

---
 youtube_dl/extractor/common.py | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2f574054d..11191c173 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1014,6 +1014,18 @@ class InfoExtractor(object):
             return []
         m3u8_doc, urlh = res
         m3u8_url = urlh.geturl()
+        # A Media Playlist Tag MUST NOT appear in a Master Playlist
+        # https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3
+        # The EXT-X-TARGETDURATION tag is REQUIRED for every M3U8 Media Playlists
+        # https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.1
+        if '#EXT-X-TARGETDURATION' in m3u8_doc:
+            return [{
+                'url': m3u8_url,
+                'format_id': m3u8_id,
+                'ext': ext,
+                'protocol': entry_protocol,
+                'preference': preference,
+            }]
         last_info = None
         last_media = None
         kv_rex = re.compile(
@@ -1164,6 +1176,7 @@ class InfoExtractor(object):
         formats = []
         rtmp_count = 0
         http_count = 0
+        m3u8_count = 0
 
         videos = smil.findall(self._xpath_ns('.//video', namespace))
         for video in videos:
@@ -1203,8 +1216,17 @@ class InfoExtractor(object):
             src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
 
             if proto == 'm3u8' or src_ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    src_url, video_id, ext or 'mp4', m3u8_id='hls', fatal=False))
+                m3u8_formats = self._extract_m3u8_formats(
+                    src_url, video_id, ext or 'mp4', m3u8_id='hls', fatal=False)
+                if len(m3u8_formats) == 1:
+                    m3u8_count += 1
+                    m3u8_formats[0].update({
+                        'format_id': 'hls-%d' % (m3u8_count if bitrate is None else bitrate),
+                        'tbr': bitrate,
+                        'width': width,
+                        'height': height,
+                    })
+                formats.extend(m3u8_formats)
                 continue
 
             if src_ext == 'f4m':

From f125d9115b262ccead54a31c4e64b0c71ad65721 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 27 Jan 2016 19:10:34 +0100
Subject: [PATCH 0471/1105] [cbsnews] extract all formats

---
 youtube_dl/extractor/cbsnews.py | 46 +++++++++++----------------------
 1 file changed, 15 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 480435e26..6db66e886 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -4,11 +4,11 @@ from __future__ import unicode_literals
 import re
 import json
 
-from .common import InfoExtractor
+from .theplatform import ThePlatformIE
 from ..utils import remove_start
 
 
-class CBSNewsIE(InfoExtractor):
+class CBSNewsIE(ThePlatformIE):
     IE_DESC = 'CBS News'
     _VALID_URL = r'http://(?:www\.)?cbsnews\.com/(?:[^/]+/)+(?P<id>[\da-z_-]+)'
 
@@ -31,7 +31,7 @@ class CBSNewsIE(InfoExtractor):
             'url': 'http://www.cbsnews.com/videos/fort-hood-shooting-army-downplays-mental-illness-as-cause-of-attack/',
             'info_dict': {
                 'id': 'fort-hood-shooting-army-downplays-mental-illness-as-cause-of-attack',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Fort Hood shooting: Army downplays mental illness as cause of attack',
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'duration': 205,
@@ -42,7 +42,7 @@ class CBSNewsIE(InfoExtractor):
                 },
             },
             'params': {
-                # rtmp download
+                # m3u8 download
                 'skip_download': True,
             },
         },
@@ -63,33 +63,6 @@ class CBSNewsIE(InfoExtractor):
         duration = item.get('duration')
         thumbnail = item.get('mediaImage') or item.get('thumbnail')
 
-        formats = []
-        for format_id in ['RtmpMobileLow', 'RtmpMobileHigh', 'Hls', 'RtmpDesktop']:
-            uri = item.get('media' + format_id + 'URI')
-            if not uri:
-                continue
-            uri = remove_start(uri, '{manifest:none}')
-            fmt = {
-                'url': uri,
-                'format_id': format_id,
-            }
-            if uri.startswith('rtmp'):
-                play_path = re.sub(
-                    r'{slistFilePath}', '',
-                    uri.split('<break>')[-1].split('{break}')[-1])
-                play_path = re.sub(
-                    r'{manifest:.+}.*$', '', play_path)
-                fmt.update({
-                    'app': 'ondemand?auth=cbs',
-                    'play_path': 'mp4:' + play_path,
-                    'player_url': 'http://www.cbsnews.com/[[IMPORT]]/vidtech.cbsinteractive.com/player/3_3_0/CBSI_PLAYER_HD.swf',
-                    'page_url': 'http://www.cbsnews.com',
-                    'ext': 'flv',
-                })
-            elif uri.endswith('.m3u8'):
-                fmt['ext'] = 'mp4'
-            formats.append(fmt)
-
         subtitles = {}
         if 'mpxRefId' in video_info:
             subtitles['en'] = [{
@@ -97,6 +70,17 @@ class CBSNewsIE(InfoExtractor):
                 'url': 'http://www.cbsnews.com/videos/captions/%s.adb_xml' % video_info['mpxRefId'],
             }]
 
+        formats = []
+        for format_id in ['RtmpMobileLow', 'RtmpMobileHigh', 'Hls', 'RtmpDesktop']:
+            pid = item.get('media' + format_id)
+            if not pid:
+                continue
+            release_url = 'http://link.theplatform.com/s/dJ5BDC/%s?format=SMIL&mbr=true' % pid
+            tp_formats, tp_subtitles = self._extract_theplatform_smil(release_url, video_id, 'Downloading %s SMIL data' % pid)
+            formats.extend(tp_formats)
+            subtitles = self._merge_subtitles(subtitles, tp_subtitles)
+        self._sort_formats(formats)
+
         return {
             'id': video_id,
             'title': title,

From ed7cd1e859cf97e975a28a5e8c58a1d1aca819fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Jan 2016 00:42:04 +0600
Subject: [PATCH 0472/1105] [cbsnews] Remove unused import

---
 youtube_dl/extractor/cbsnews.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 6db66e886..cabf7e73b 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -5,7 +5,6 @@ import re
 import json
 
 from .theplatform import ThePlatformIE
-from ..utils import remove_start
 
 
 class CBSNewsIE(ThePlatformIE):

From b0df5223be5429da0378841dc33eecb532f6898f Mon Sep 17 00:00:00 2001
From: dyn888 <dyn.8.8.8+github@gmail.com>
Date: Thu, 28 Jan 2016 12:07:15 +0100
Subject: [PATCH 0473/1105] Update YoutubeDL.py

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 09d2b18f2..e1bd40843 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -906,7 +906,7 @@ class YoutubeDL(object):
             str_operator_rex = re.compile(r'''(?x)
                 \s*(?P<key>ext|acodec|vcodec|container|protocol)
                 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
-                \s*(?P<value>[a-zA-Z0-9_-]+)
+                \s*(?P<value>[a-zA-Z0-9._-]+)
                 \s*$
                 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
             m = str_operator_rex.search(filter_spec)

From 2b14cb566fde3e5482ce9a63b2be7103cec939e0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 28 Jan 2016 12:38:34 +0100
Subject: [PATCH 0474/1105] [utils] fix dfxp2srt text extraction(fixes #8055)

---
 youtube_dl/utils.py | 33 ++++++++++++++++++++-------------
 1 file changed, 20 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index c63b61598..18dbe28bb 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2017,20 +2017,27 @@ def dfxp2srt(dfxp_data):
         'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
     })
 
+    class TTMLPElementParser:
+        out = ''
+
+        def start(self, tag, attrib):
+            if tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
+                self.out += '\n'
+
+        def end(self, tag):
+            pass
+
+        def data(self, data):
+            self.out += data
+
+        def close(self):
+            return self.out.strip()
+
     def parse_node(node):
-        str_or_empty = functools.partial(str_or_none, default='')
-
-        out = str_or_empty(node.text)
-
-        for child in node:
-            if child.tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
-                out += '\n' + str_or_empty(child.tail)
-            elif child.tag in (_x('ttml:span'), _x('ttaf1:span'), 'span'):
-                out += str_or_empty(parse_node(child))
-            else:
-                out += str_or_empty(xml.etree.ElementTree.tostring(child))
-
-        return out
+        target = TTMLPElementParser()
+        parser = xml.etree.ElementTree.XMLParser(target=target)
+        parser.feed(xml.etree.ElementTree.tostring(node))
+        return parser.close()
 
     dfxp = compat_etree_fromstring(dfxp_data.encode('utf-8'))
     out = []

From b913348d5fa1130ab1c65576e458407f262df6e1 Mon Sep 17 00:00:00 2001
From: dyn888 <dyn.8.8.8+github@gmail.com>
Date: Thu, 28 Jan 2016 15:07:33 +0100
Subject: [PATCH 0475/1105] Test codec with a dot '.' in name selection.

---
 test/test_YoutubeDL.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 0caa43843..b53cfbe78 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -221,6 +221,16 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['format_id'], 'dash-video-low')
 
+        formats = [
+            {'format_id': 'vid-vcodec-dot', 'ext': 'mp4', 'preference': 1, 'vcodec': 'avc1.123456', 'acodec': 'none', 'url': TEST_URL},
+        ]
+        info_dict = _make_result(formats)
+
+        ydl = YDL({'format': 'bestvideo[vcodec=avc1.123456]'})
+        ydl.process_ie_result(info_dict.copy())
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'vid-vcodec-dot')
+
     def test_youtube_format_selection(self):
         order = [
             '38', '37', '46', '22', '45', '35', '44', '18', '34', '43', '6', '5', '36', '17', '13',

From 29f46c2bee3641efcf2c06c3d173baf7a41010af Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 28 Jan 2016 22:56:00 +0800
Subject: [PATCH 0476/1105] Credit @dyn888 for improving format selection

[ci skip]
---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index bb1f2d8d9..a46799506 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -155,3 +155,4 @@ Vignesh Venkat
 Tom Gijselinck
 Founder Fang
 Andrew Alexeyew
+Saso Bezlaj

From 38c84acae512f1b218e5146f4b2ddcac6b1a5e13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Jan 2016 22:50:18 +0600
Subject: [PATCH 0477/1105] [ndr:embed:base] Add missing ext for m3u8

---
 youtube_dl/extractor/ndr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 894c51399..0cded6b5c 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -193,7 +193,7 @@ class NDREmbedBaseIE(InfoExtractor):
                     src + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id, f4m_id='hds'))
             elif ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
-                    src, video_id, m3u8_id='hls', entry_protocol='m3u8_native'))
+                    src, video_id, 'mp4', m3u8_id='hls', entry_protocol='m3u8_native'))
             else:
                 quality = f.get('quality')
                 ff = {

From 10e6ed93417fb51c606af2e3e47b9a5a094dd6ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Jan 2016 22:56:49 +0600
Subject: [PATCH 0478/1105] [ok] Add support for mobile URLs (Closes #8345)

---
 youtube_dl/extractor/odnoklassniki.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 184c7a323..f9e064a60 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class OdnoklassnikiIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?:odnoklassniki|ok)\.ru/(?:video(?:embed)?|web-api/video/moviePlayer)/(?P<id>[\d-]+)'
+    _VALID_URL = r'https?://(?:(?:www|m|mobile)\.)?(?:odnoklassniki|ok)\.ru/(?:video(?:embed)?|web-api/video/moviePlayer)/(?P<id>[\d-]+)'
     _TESTS = [{
         # metadata in JSON
         'url': 'http://ok.ru/video/20079905452',
@@ -69,6 +69,12 @@ class OdnoklassnikiIE(InfoExtractor):
     }, {
         'url': 'http://www.ok.ru/videoembed/20648036891',
         'only_matching': True,
+    }, {
+        'url': 'http://m.ok.ru/video/20079905452',
+        'only_matching': True,
+    }, {
+        'url': 'http://mobile.ok.ru/video/20079905452',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 50e989e2636fc59ed896cc021b1b594bd10e9e17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Jan 2016 23:19:53 +0600
Subject: [PATCH 0479/1105] [bbc] Add another title regex (Closes #8340)

---
 youtube_dl/extractor/bbc.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 1c493b72d..210ce568b 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -482,7 +482,8 @@ class BBCCoUkIE(InfoExtractor):
         if programme_id:
             formats, subtitles = self._download_media_selector(programme_id)
             title = self._og_search_title(webpage, default=None) or self._html_search_regex(
-                r'<h2[^>]+id="parent-title"[^>]*>(.+?)</h2>', webpage, 'title')
+                (r'<h2[^>]+id="parent-title"[^>]*>(.+?)</h2>',
+                 r'<div[^>]+class="info"[^>]*>\s*<h1>(.+?)</h1>'), webpage, 'title')
             description = self._search_regex(
                 r'<p class="[^"]*medium-description[^"]*">([^<]+)</p>',
                 webpage, 'description', default=None)

From a853427427269549593a0217db203305332ac983 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Jan 2016 23:23:13 +0600
Subject: [PATCH 0480/1105] [bbc] Add another description regex

---
 youtube_dl/extractor/bbc.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 210ce568b..5be0ff476 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -485,7 +485,8 @@ class BBCCoUkIE(InfoExtractor):
                 (r'<h2[^>]+id="parent-title"[^>]*>(.+?)</h2>',
                  r'<div[^>]+class="info"[^>]*>\s*<h1>(.+?)</h1>'), webpage, 'title')
             description = self._search_regex(
-                r'<p class="[^"]*medium-description[^"]*">([^<]+)</p>',
+                (r'<p class="[^"]*medium-description[^"]*">([^<]+)</p>',
+                 r'<div[^>]+class="info_+synopsis"[^>]*>([^<]+)</div>'),
                 webpage, 'description', default=None)
             if not description:
                 description = self._html_search_meta('description', webpage)

From 1ac6e794cb36af612db97007006fc7cf1468e049 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Jan 2016 23:27:48 +0600
Subject: [PATCH 0481/1105] [bbc] Add test for #8147

---
 youtube_dl/extractor/bbc.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 5be0ff476..6ddee686c 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -193,6 +193,19 @@ class BBCCoUkIE(InfoExtractor):
                 # rtmp download
                 'skip_download': True,
             },
+        }, {
+            # compact player (https://github.com/rg3/youtube-dl/issues/8147)
+            'url': 'http://www.bbc.co.uk/programmes/p028bfkf/player',
+            'info_dict': {
+                'id': 'p028bfkj',
+                'ext': 'flv',
+                'title': 'Extract from BBC documentary Look Stranger - Giant Leeks and Magic Brews',
+                'description': 'Extract from BBC documentary Look Stranger - Giant Leeks and Magic Brews',
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
         }, {
             'url': 'http://www.bbc.co.uk/iplayer/playlist/p01dvks4',
             'only_matching': True,

From cf57433bbd190143694b44ac4cd5f56e6eb625ee Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 28 Jan 2016 18:57:32 +0100
Subject: [PATCH 0482/1105] [ffmpeg] fix adding metadata when using
 m3u8_native(fixes #8350)

---
 youtube_dl/postprocessor/ffmpeg.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index daca5d814..48f86e21c 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -391,6 +391,10 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
         for (name, value) in metadata.items():
             options.extend(['-metadata', '%s=%s' % (name, value)])
 
+        # https://github.com/rg3/youtube-dl/issues/8350
+        if info['protocol'] == 'm3u8_native':
+            options.extend(['-bsf:a', 'aac_adtstoasc'])
+
         self._downloader.to_screen('[ffmpeg] Adding metadata to \'%s\'' % filename)
         self.run_ffmpeg(filename, temp_filename, options)
         os.remove(encodeFilename(filename))

From 70029bc348f27294b7f3e369f953167c1893c2bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 29 Jan 2016 11:27:11 +0100
Subject: [PATCH 0483/1105] [youtube:user] Require 'https?://' in the url
 (fixes #8356)

It was matching www.youtube.com/embed/WpfukLMe1TM.
The generic extractor automatically adds http:// if it's missing.
---
 test/test_all_urls.py           | 2 +-
 youtube_dl/extractor/youtube.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index a0c11e6c1..f5af184e6 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -56,7 +56,7 @@ class TestAllURLsMatching(unittest.TestCase):
         assertChannel('https://www.youtube.com/channel/HCtnHdj3df7iM/videos')
 
     def test_youtube_user_matching(self):
-        self.assertMatch('www.youtube.com/NASAgovVideo/videos', ['youtube:user'])
+        self.assertMatch('http://www.youtube.com/NASAgovVideo/videos', ['youtube:user'])
 
     def test_youtube_feeds(self):
         self.assertMatch('https://www.youtube.com/feed/watch_later', ['youtube:watchlater'])
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 92b9f3ae4..a24c73584 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1846,7 +1846,7 @@ class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
 
 class YoutubeUserIE(YoutubeChannelIE):
     IE_DESC = 'YouTube.com user videos (URL or "ytuser" keyword)'
-    _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch|results)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)(?P<id>[A-Za-z0-9_-]+)'
+    _VALID_URL = r'(?:(?:https?://(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch|results)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)(?P<id>[A-Za-z0-9_-]+)'
     _TEMPLATE_URL = 'https://www.youtube.com/user/%s/videos'
     IE_NAME = 'youtube:user'
 

From 055f4172781dd2a43d60f17a91a1d0c1a5f3e6b9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 29 Jan 2016 12:20:08 +0100
Subject: [PATCH 0484/1105] release 2016.01.29

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4ac7f9e93..d9f1e22b0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.01.27'
+__version__ = '2016.01.29'

From 2b4f5e68d1517bcadac4b25ecbac3b143104b1c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 29 Jan 2016 15:36:33 +0100
Subject: [PATCH 0485/1105] [azubu] Add extractor for live streams (closes
 #8343)

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/azubu.py    | 40 +++++++++++++++++++++++++++++++-
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 532be7e4c..5e0d7d3dc 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -50,7 +50,7 @@ from .atresplayer import AtresPlayerIE
 from .atttechchannel import ATTTechChannelIE
 from .audimedia import AudiMediaIE
 from .audiomack import AudiomackIE, AudiomackAlbumIE
-from .azubu import AzubuIE
+from .azubu import AzubuIE, AzubuLiveIE
 from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
diff --git a/youtube_dl/extractor/azubu.py b/youtube_dl/extractor/azubu.py
index 0961d339f..011edf128 100644
--- a/youtube_dl/extractor/azubu.py
+++ b/youtube_dl/extractor/azubu.py
@@ -3,7 +3,11 @@ from __future__ import unicode_literals
 import json
 
 from .common import InfoExtractor
-from ..utils import float_or_none
+from ..utils import (
+    ExtractorError,
+    float_or_none,
+    sanitized_Request,
+)
 
 
 class AzubuIE(InfoExtractor):
@@ -91,3 +95,37 @@ class AzubuIE(InfoExtractor):
             'view_count': view_count,
             'formats': formats,
         }
+
+
+class AzubuLiveIE(InfoExtractor):
+    _VALID_URL = r'http://www.azubu.tv/(?P<id>[^/]+)$'
+
+    _TEST = {
+        'url': 'http://www.azubu.tv/MarsTVMDLen',
+        'only_matching': True,
+    }
+
+    def _real_extract(self, url):
+        user = self._match_id(url)
+
+        info = self._download_json(
+            'http://api.azubu.tv/public/modules/last-video/{0}/info'.format(user),
+            user)['data']
+        if info['type'] != 'STREAM':
+            raise ExtractorError('{0} is not streaming live'.format(user), expected=True)
+
+        req = sanitized_Request(
+            'https://edge-elb.api.brightcove.com/playback/v1/accounts/3361910549001/videos/ref:' + info['reference_id'])
+        req.add_header('Accept', 'application/json;pk=BCpkADawqM1gvI0oGWg8dxQHlgT8HkdE2LnAlWAZkOlznO39bSZX726u4JqnDsK3MDXcO01JxXK2tZtJbgQChxgaFzEVdHRjaDoxaOu8hHOO8NYhwdxw9BzvgkvLUlpbDNUuDoc4E4wxDToV')
+        bc_info = self._download_json(req, user)
+        m3u8_url = next(source['src'] for source in bc_info['sources'] if source['container'] == 'M2TS')
+        formats = self._extract_m3u8_formats(m3u8_url, user, ext='mp4')
+
+        return {
+            'id': info['id'],
+            'title': self._live_title(info['title']),
+            'uploader_id': user,
+            'formats': formats,
+            'is_live': True,
+            'thumbnail': bc_info['poster'],
+        }

From 68a0ea15b4c20ed0174a82ee79a6d3c3474b0f69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Jan 2016 00:26:33 +0600
Subject: [PATCH 0486/1105] [cspan] Unescape path (Closes #8365)

---
 youtube_dl/extractor/cspan.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index b78edf729..b8b9d058d 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -113,7 +113,7 @@ class CSpanIE(InfoExtractor):
                     'tbr': int_or_none(get_text_attr(quality, 'bitrate')),
                 })
             if not formats:
-                path = get_text_attr(f, 'path')
+                path = unescapeHTML(get_text_attr(f, 'path'))
                 if not path:
                     continue
                 formats = self._extract_m3u8_formats(

From 350cf045d8f86497e8d79ae193b40cc44c8e670c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Jan 2016 01:47:46 +0600
Subject: [PATCH 0487/1105] [extractor/common] Restrict checks when auto
 calculating tbr

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 33290fd74..b3d57dfce 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -828,7 +828,7 @@ class InfoExtractor(object):
         for f in formats:
             # Automatically determine tbr when missing based on abr and vbr (improves
             # formats sorting in some cases)
-            if 'tbr' not in f and 'abr' in f and 'vbr' in f:
+            if 'tbr' not in f and f.get('abr') is not None and f.get('vbr') is not None:
                 f['tbr'] = f['abr'] + f['vbr']
 
         def _formats_key(f):

From 83ab8a79ccc3b6ef143f7d636c0118f7c3e5777b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Jan 2016 01:48:54 +0600
Subject: [PATCH 0488/1105] [espn] Improve video id extraction (Closes #8368)

---
 youtube_dl/extractor/espn.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/espn.py b/youtube_dl/extractor/espn.py
index 3762d8748..db4b263bc 100644
--- a/youtube_dl/extractor/espn.py
+++ b/youtube_dl/extractor/espn.py
@@ -53,8 +53,8 @@ class ESPNIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         video_id = self._search_regex(
-            r'class="video-play-button"[^>]+data-id="(\d+)',
-            webpage, 'video id')
+            r'class=(["\']).*?video-play-button.*?\1[^>]+data-id=["\'](?P<id>\d+)',
+            webpage, 'video id', group='id')
 
         cms = 'espn'
         if 'data-source="intl"' in webpage:

From e047922be01ca346cee8f53b7972773d64eed6f4 Mon Sep 17 00:00:00 2001
From: ping <liping.ong@gmail.com>
Date: Sat, 30 Jan 2016 11:04:11 +0800
Subject: [PATCH 0489/1105] [daum] Fix copy-paste mistake

---
 youtube_dl/extractor/daum.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index c84302c43..0d74e5207 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -38,7 +38,8 @@ class DaumIE(InfoExtractor):
             'description': 'md5:79794514261164ff27e36a21ad229fc5',
             'upload_date': '20150604',
             'duration': 154
-        },    }, {
+        },
+    }, {
         'url': 'http://tvpot.daum.net/v/07dXWRka62Y%24',
         'only_matching': True,
     }]

From f5d30d521c63b4eec4aa4386365054222e354231 Mon Sep 17 00:00:00 2001
From: ping <liping.ong@gmail.com>
Date: Sat, 30 Jan 2016 11:09:30 +0800
Subject: [PATCH 0490/1105] [daum] Fix add view_count, comment_count to test

---
 youtube_dl/extractor/daum.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index 0d74e5207..567abdc39 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -37,7 +37,9 @@ class DaumIE(InfoExtractor):
             'title': 'md5:a100d65d09cec246d8aa9bde7de45aed',
             'description': 'md5:79794514261164ff27e36a21ad229fc5',
             'upload_date': '20150604',
-            'duration': 154
+            'duration': 154,
+            'view_count': int,
+            'comment_count': int,
         },
     }, {
         'url': 'http://tvpot.daum.net/v/07dXWRka62Y%24',

From 06ffa3348590f9aa6ddca7ca8f78424c49314560 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 16:23:37 +0800
Subject: [PATCH 0491/1105] [daum.net] Move the request to ClipInfoXml.do

To reduce the number of wasted requests
---
 youtube_dl/extractor/daum.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index 567abdc39..543eb5c3a 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -49,9 +49,6 @@ class DaumIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         query = compat_urllib_parse.urlencode({'vid': video_id})
-        info = self._download_xml(
-            'http://tvpot.daum.net/clip/ClipInfoXml.do?' + query, video_id,
-            'Downloading video info')
         movie_data = self._download_json(
             'http://videofarm.daum.net/controller/api/closed/v1_2/IntegratedMovieData.json?' + query,
             video_id, 'Downloading video formats info')
@@ -60,6 +57,10 @@ class DaumIE(InfoExtractor):
         if not movie_data.get('output_list', {}).get('output_list') and re.match(r'^\d+$', video_id):
             return self.url_result('http://tvpot.daum.net/clip/ClipView.do?clipid=%s' % video_id)
 
+        info = self._download_xml(
+            'http://tvpot.daum.net/clip/ClipInfoXml.do?' + query, video_id,
+            'Downloading video info')
+
         formats = []
         for format_el in movie_data['output_list']['output_list']:
             profile = format_el['profile']

From 830afe85dc79e034ffbe7f3ec1d049482ed651de Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 16:50:13 +0800
Subject: [PATCH 0492/1105] [daum.net] Support VodPlayer.swf URLs (closes
 #8173)

---
 youtube_dl/extractor/daum.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index 543eb5c3a..ea6a9d848 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -5,7 +5,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import (
+    compat_urllib_parse,
+    compat_urllib_parse_unquote,
+)
 from ..utils import (
     int_or_none,
     str_to_int,
@@ -14,7 +17,7 @@ from ..utils import (
 
 
 class DaumIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/v/(?P<id>[^?#&]+)'
+    _VALID_URL = r'https?://(?:(?:m\.)?tvpot\.daum\.net/v/|videofarm\.daum\.net/controller/player/VodPlayer\.swf\?vid=)(?P<id>[^?#&]+)'
     IE_NAME = 'daum.net'
 
     _TESTS = [{
@@ -44,10 +47,23 @@ class DaumIE(InfoExtractor):
     }, {
         'url': 'http://tvpot.daum.net/v/07dXWRka62Y%24',
         'only_matching': True,
+    }, {
+        'url': 'http://videofarm.daum.net/controller/player/VodPlayer.swf?vid=vwIpVpCQsT8%24&ref=',
+        'info_dict': {
+            'id': 'vwIpVpCQsT8$',
+            'ext': 'flv',
+            'title': '01-Korean War ( Trouble on the horizon )',
+            'description': '\nKorean War 01\nTrouble on the horizon\n전쟁의 먹구름',
+            'upload_date': '20080223',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 249,
+            'view_count': int,
+            'comment_count': int,
+        },
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        video_id = compat_urllib_parse_unquote(self._match_id(url))
         query = compat_urllib_parse.urlencode({'vid': video_id})
         movie_data = self._download_json(
             'http://videofarm.daum.net/controller/api/closed/v1_2/IntegratedMovieData.json?' + query,

From 0179f6a8301e8dee1f435289f89cf1c748a13b16 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 16:54:14 +0800
Subject: [PATCH 0493/1105] [daum] Add 'thumbnail' to all _TESTS

---
 youtube_dl/extractor/daum.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index ea6a9d848..9bc345f60 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -28,6 +28,7 @@ class DaumIE(InfoExtractor):
             'title': '마크 헌트 vs 안토니오 실바',
             'description': 'Mark Hunt vs Antonio Silva',
             'upload_date': '20131217',
+            'thumbnail': 're:^https?://.*\.(?:jpg|png)',
             'duration': 2117,
             'view_count': int,
             'comment_count': int,
@@ -40,6 +41,7 @@ class DaumIE(InfoExtractor):
             'title': 'md5:a100d65d09cec246d8aa9bde7de45aed',
             'description': 'md5:79794514261164ff27e36a21ad229fc5',
             'upload_date': '20150604',
+            'thumbnail': 're:^https?://.*\.(?:jpg|png)',
             'duration': 154,
             'view_count': int,
             'comment_count': int,
@@ -55,7 +57,7 @@ class DaumIE(InfoExtractor):
             'title': '01-Korean War ( Trouble on the horizon )',
             'description': '\nKorean War 01\nTrouble on the horizon\n전쟁의 먹구름',
             'upload_date': '20080223',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': 're:^https?://.*\.(?:jpg|png)',
             'duration': 249,
             'view_count': int,
             'comment_count': int,
@@ -122,6 +124,7 @@ class DaumClipIE(InfoExtractor):
             'title': 'DOTA 2GETHER 시즌2 6회 - 2부',
             'description': 'DOTA 2GETHER 시즌2 6회 - 2부',
             'upload_date': '20130831',
+            'thumbnail': 're:^https?://.*\.(?:jpg|png)',
             'duration': 3868,
             'view_count': int,
         },

From 7d106a65ca30cc3201f147cb96aa98a15a48d6d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 30 Jan 2016 12:26:40 +0100
Subject: [PATCH 0494/1105] Add --hls-use-mpegts option

When using the mpegts container hls vidoes can be played while being downloaded (useful if you are recording a live stream).
VLC and mpv play them file, but QuickTime doesn't.
---
 youtube_dl/YoutubeDL.py         | 2 +-
 youtube_dl/__init__.py          | 1 +
 youtube_dl/downloader/common.py | 1 +
 youtube_dl/downloader/hls.py    | 6 +++++-
 youtube_dl/options.py           | 5 +++++
 5 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e1bd40843..2a3d6cd4a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -263,7 +263,7 @@ class YoutubeDL(object):
     the downloader (see youtube_dl/downloader/common.py):
     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
     noresizebuffer, retries, continuedl, noprogress, consoletitle,
-    xattr_set_filesize, external_downloader_args.
+    xattr_set_filesize, external_downloader_args, hls_use_mpegts.
 
     The following options are used by the post processors:
     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 9f131f5db..f5f064241 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -369,6 +369,7 @@ def _real_main(argv=None):
         'no_color': opts.no_color,
         'ffmpeg_location': opts.ffmpeg_location,
         'hls_prefer_native': opts.hls_prefer_native,
+        'hls_use_mpegts': opts.hls_use_mpegts,
         'external_downloader_args': external_downloader_args,
         'postprocessor_args': postprocessor_args,
         'cn_verification_proxy': opts.cn_verification_proxy,
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index fc7521598..de815612c 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -45,6 +45,7 @@ class FileDownloader(object):
                         (experimental)
     external_downloader_args:  A list of additional command-line arguments for the
                         external downloader.
+    hls_use_mpegts:     Use the mpegts container for HLS videos.
 
     Subclasses of this one must re-define the real_download method.
     """
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 10b83c6b2..cb34dc4ab 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -39,7 +39,11 @@ class HlsFD(FileDownloader):
                 '-headers',
                 ''.join('%s: %s\r\n' % (key, val) for key, val in headers.items())]
 
-        args += ['-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc']
+        args += ['-i', url, '-c', 'copy']
+        if self.params.get('hls_use_mpegts', False):
+            args += ['-f', 'mpegts']
+        else:
+            args += ['-f', 'mp4', '-bsf:a', 'aac_adtstoasc']
 
         args = [encodeArgument(opt) for opt in args]
         args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True))
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 433245f00..39fc4306a 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -415,6 +415,11 @@ def parseOpts(overrideArguments=None):
         '--hls-prefer-native',
         dest='hls_prefer_native', action='store_true',
         help='Use the native HLS downloader instead of ffmpeg (experimental)')
+    downloader.add_option(
+        '--hls-use-mpegts',
+        dest='hls_use_mpegts', action='store_true',
+        help='Use the mpegts container for HLS videos, allowing to play the '
+             'video while downloading (some players may not be able to play it')
     downloader.add_option(
         '--external-downloader',
         dest='external_downloader', metavar='COMMAND',

From c1406299959b780e7b86b567bd4eeecc8556dce0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 19:30:39 +0800
Subject: [PATCH 0495/1105] [facebook] Support alternative webpage form

Fixes #8371
---
 youtube_dl/extractor/facebook.py | 29 ++++++++++++++++++++++++-----
 1 file changed, 24 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index cb5dd57fb..899b0896b 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -150,10 +150,32 @@ class FacebookIE(InfoExtractor):
         url = 'https://www.facebook.com/video/video.php?v=%s' % video_id
         webpage = self._download_webpage(url, video_id)
 
+        video_data = None
+
         BEFORE = '{swf.addParam(param[0], param[1]);});\n'
         AFTER = '.forEach(function(variable) {swf.addVariable(variable[0], variable[1]);});'
         m = re.search(re.escape(BEFORE) + '(.*?)' + re.escape(AFTER), webpage)
-        if not m:
+        if m:
+            data = dict(json.loads(m.group(1)))
+            params_raw = compat_urllib_parse_unquote(data['params'])
+            video_data = json.loads(params_raw)['video_data']
+
+        def video_data_list2dict(video_data):
+            ret = {}
+            for item in video_data:
+                format_id = item['stream_type']
+                ret.setdefault(format_id, []).append(item)
+            return ret
+
+        if not video_data:
+            server_js_data = self._parse_json(self._search_regex(
+                r'handleServerJS\(({.+})\);', webpage, 'server js data'), video_id)
+            for item in server_js_data['instances']:
+                if item[1][0] == 'VideoConfig':
+                    video_data = video_data_list2dict(item[2][0]['videoData'])
+                    break
+
+        if not video_data:
             m_msg = re.search(r'class="[^"]*uiInterstitialContent[^"]*"><div>(.*?)</div>', webpage)
             if m_msg is not None:
                 raise ExtractorError(
@@ -161,12 +183,9 @@ class FacebookIE(InfoExtractor):
                     expected=True)
             else:
                 raise ExtractorError('Cannot parse data')
-        data = dict(json.loads(m.group(1)))
-        params_raw = compat_urllib_parse_unquote(data['params'])
-        params = json.loads(params_raw)
 
         formats = []
-        for format_id, f in params['video_data'].items():
+        for format_id, f in video_data.items():
             if not f or not isinstance(f, list):
                 continue
             for quality in ('sd', 'hd'):

From 05d0d131a76e34af843d982dd47b556ddc8b9e44 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 20:05:56 +0800
Subject: [PATCH 0496/1105] [youtube] Move decrypt_sig out of
 _parse_dash_manifest

---
 youtube_dl/extractor/youtube.py | 32 ++++++++++++++------------------
 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a24c73584..08b7e15c4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1035,22 +1035,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
         return self._download_webpage(url, video_id, note='Searching for annotations.', errnote='Unable to download video annotations.')
 
-    def _parse_dash_manifest(
-            self, video_id, dash_manifest_url, player_url, age_gate, fatal=True):
-        def decrypt_sig(mobj):
-            s = mobj.group(1)
-            dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
-            return '/signature/%s' % dec_s
-        dash_manifest_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, dash_manifest_url)
-        dash_doc = self._download_xml(
-            dash_manifest_url, video_id,
-            note='Downloading DASH manifest',
-            errnote='Could not download DASH manifest',
-            fatal=fatal)
-
-        if dash_doc is False:
-            return []
-
+    def _parse_dash_manifest(self, video_id, dash_doc, fatal=True):
         formats = []
         for a in dash_doc.findall('.//{urn:mpeg:DASH:schema:MPD:2011}AdaptationSet'):
             mime_type = a.attrib.get('mimeType')
@@ -1533,8 +1518,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             for dash_manifest_url in dash_mpds:
                 dash_formats = {}
                 try:
-                    for df in self._parse_dash_manifest(
-                            video_id, dash_manifest_url, player_url, age_gate, dash_mpd_fatal):
+                    def decrypt_sig(mobj):
+                        s = mobj.group(1)
+                        dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
+                        return '/signature/%s' % dec_s
+
+                    dash_manifest_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, dash_manifest_url)
+                    dash_doc = self._download_xml(
+                        dash_manifest_url, video_id,
+                        note='Downloading DASH manifest',
+                        errnote='Could not download DASH manifest',
+                        fatal=dash_mpd_fatal)
+
+                    for df in self._parse_dash_manifest(video_id, dash_doc, dash_mpd_fatal):
                         # Do not overwrite DASH format found in some previous DASH manifest
                         if df['format_id'] not in dash_formats:
                             dash_formats[df['format_id']] = df

From 2f0a33d8a3c926f3a81d0e76bd669a8bc57a2cf8 Mon Sep 17 00:00:00 2001
From: ping <liping.ong@gmail.com>
Date: Sat, 30 Jan 2016 20:10:36 +0800
Subject: [PATCH 0497/1105] [daum.net] Support for playlists, user channels

---
 youtube_dl/extractor/__init__.py |   2 +
 youtube_dl/extractor/daum.py     | 119 ++++++++++++++++++++++++++++++-
 2 files changed, 120 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5e0d7d3dc..e05d986ef 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -142,6 +142,8 @@ from .dailymotion import (
 from .daum import (
     DaumIE,
     DaumClipIE,
+    DaumPlaylistIE,
+    DaumUserIE,
 )
 from .dbtv import DBTVIE
 from .dcn import (
diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index 9bc345f60..cf158c42d 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -3,6 +3,7 @@
 from __future__ import unicode_literals
 
 import re
+import itertools
 
 from .common import InfoExtractor
 from ..compat import (
@@ -13,6 +14,7 @@ from ..utils import (
     int_or_none,
     str_to_int,
     xpath_text,
+    unescapeHTML,
 )
 
 
@@ -143,7 +145,7 @@ class DaumClipIE(InfoExtractor):
             '_type': 'url_transparent',
             'id': video_id,
             'url': 'http://tvpot.daum.net/v/%s' % clip_info['vid'],
-            'title': clip_info['title'],
+            'title': unescapeHTML(clip_info['title']),
             'thumbnail': clip_info.get('thumb_url'),
             'description': clip_info.get('contents'),
             'duration': int_or_none(clip_info.get('duration')),
@@ -151,3 +153,118 @@ class DaumClipIE(InfoExtractor):
             'view_count': int_or_none(clip_info.get('play_count')),
             'ie_key': 'Daum',
         }
+
+
+class DaumListIE(InfoExtractor):
+    def _get_entries(self, list_id, list_id_type):
+        name = None
+        entries = []
+        for pagenum in itertools.count(1):
+            list_info = self._download_json(
+                'http://tvpot.daum.net/mypot/json/GetClipInfo.do?size=48&init=true&order=date&page=%d&%s=%s' % (
+                    pagenum, list_id_type, list_id), list_id,'Downloading list info - %s' % pagenum)
+            
+            entries.extend([
+                self.url_result(
+                    'http://tvpot.daum.net/v/%s' % clip['vid'])
+                for clip in list_info['clip_list']
+            ])
+
+            if not name:
+                name = list_info.get('playlist_bean', {}).get('name') or \
+                    list_info.get('potInfo', {}).get('name')
+
+            if not list_info.get('has_more'):
+                break
+
+        return name, entries
+
+
+class DaumPlaylistIE(DaumListIE):
+    _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/mypot/(?:View\.do|Top\.tv)\?.*?playlistid=(?P<id>[0-9]+)'
+    IE_NAME = 'daum.net:playlist'
+
+    _TESTS = [{
+        'note': 'Playlist url with clipid',
+        'url': 'http://tvpot.daum.net/mypot/View.do?playlistid=6213966&clipid=73806844',
+        'info_dict': {
+            'id': '6213966',
+            'title': 'Woorissica Official',
+        },
+        'playlist_mincount': 181
+    }, {
+        'note': 'Playlist url with clipid - noplaylist',
+        'url': 'http://tvpot.daum.net/mypot/View.do?playlistid=6213966&clipid=73806844',
+        'info_dict': {
+            'id': '73806844',
+            'ext': 'mp4',
+            'title': '151017 Airport',
+            'upload_date': '20160117',
+        },
+        'params': {
+            'noplaylist': True,
+            'skip_download': True,
+        }
+    }]
+
+    def _real_extract(self, url):
+        if DaumClipIE.suitable(url) and self._downloader.params.get('noplaylist'):
+            return self.url_result(url, 'DaumClip')
+
+        list_id = self._match_id(url)
+        self.to_screen('Downloading playlist %s - add --no-playlist to just download video' % list_id)
+
+        name, entries = self._get_entries(list_id, 'playlistid')
+
+        return self.playlist_result(entries, list_id, name)
+
+
+class DaumUserIE(DaumListIE):
+    _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/mypot/(?:View|Top)\.do\?.*?ownerid=(?P<id>[0-9a-zA-Z]+)'
+    IE_NAME = 'daum.net:user'
+
+    _TESTS = [{
+        'url': 'http://tvpot.daum.net/mypot/View.do?ownerid=o2scDLIVbHc0',
+        'info_dict': {
+            'id': 'o2scDLIVbHc0',
+            'title': '마이 리틀 텔레비전',
+        },
+        'playlist_mincount': 213
+    }, {
+        'url': 'http://tvpot.daum.net/mypot/View.do?ownerid=o2scDLIVbHc0&clipid=73801156',
+        'info_dict': {
+            'id': '73801156',
+            'ext': 'mp4',
+            'title': '[미공개] 김구라, 오만석이 부릅니다 \'오케피\' - 마이 리틀 텔레비전 20160116',
+            'upload_date': '20160117',
+            'description': 'md5:5e91d2d6747f53575badd24bd62b9f36'
+        },
+        'params': {
+            'noplaylist': True,
+            'skip_download': True,
+        }
+    }, {
+        'note': 'Playlist url has ownerid and playlistid, playlistid takes precedence',
+        'url': 'http://tvpot.daum.net/mypot/View.do?ownerid=o2scDLIVbHc0&playlistid=6196631',
+        'info_dict': {
+            'id': '6196631',
+            'title': '마이 리틀 텔레비전 - 20160109',
+        },
+        'playlist_count': 11
+    }, {
+        'url': 'http://tvpot.daum.net/mypot/Top.do?ownerid=o2scDLIVbHc0',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        if DaumClipIE.suitable(url) and self._downloader.params.get('noplaylist'):
+            return self.url_result(url, 'DaumClip')
+
+        if DaumPlaylistIE.suitable(url):
+            return self.url_result(url, 'DaumPlaylist')
+
+        list_id = self._match_id(url)
+        self.to_screen('Downloading playlist %s - add --no-playlist to just download video' % list_id)
+        name, entries = self._get_entries(list_id, 'ownerid')
+
+        return self.playlist_result(entries, list_id, name)

From 8ab3fe81d84cb541eac3af73404e71d979967dcf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Jan 2016 18:28:38 +0600
Subject: [PATCH 0498/1105] [downloader/f4m] Prefer bootstrap url attribute
 over inline bootstrap info

---
 youtube_dl/downloader/f4m.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index aaf0c49c8..f8da04f69 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -273,15 +273,21 @@ class F4mFD(FragmentFD):
         return fragments_list
 
     def _parse_bootstrap_node(self, node, base_url):
-        if node.text is None:
+        # Sometimes non empty inline bootstrap info can be specified along
+        # with bootstrap url attribute (e.g. dummy inline bootstrap info
+        # contains whitespace characters in [1]). We will prefer bootstrap
+        # url over inline bootstrap info when present.
+        # 1. http://live-1-1.rutube.ru/stream/1024/HDS/SD/C2NKsS85HQNckgn5HdEmOQ/1454167650/S-s604419906/move/four/dirs/upper/1024-576p.f4m
+        bootstrap_url = node.get('url')
+        if bootstrap_url:
             bootstrap_url = compat_urlparse.urljoin(
-                base_url, node.attrib['url'])
+                base_url, bootstrap_url)
             boot_info = self._get_bootstrap_from_url(bootstrap_url)
         else:
             bootstrap_url = None
             bootstrap = base64.b64decode(node.text.encode('ascii'))
             boot_info = read_bootstrap_info(bootstrap)
-        return (boot_info, bootstrap_url)
+        return boot_info, bootstrap_url
 
     def real_download(self, filename, info_dict):
         man_url = info_dict['url']

From c3deacd5621d9686a32614e550eb4726e66c3899 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Jan 2016 18:30:27 +0600
Subject: [PATCH 0499/1105] [matchtv] Add extractor (Closes #8313)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/matchtv.py  | 55 ++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/matchtv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5e0d7d3dc..e15495ec8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -372,6 +372,7 @@ from .macgamestore import MacGameStoreIE
 from .mailru import MailRuIE
 from .makertv import MakerTVIE
 from .malemotion import MalemotionIE
+from .matchtv import MatchTVIE
 from .mdr import MDRIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
diff --git a/youtube_dl/extractor/matchtv.py b/youtube_dl/extractor/matchtv.py
new file mode 100644
index 000000000..28e0dfe63
--- /dev/null
+++ b/youtube_dl/extractor/matchtv.py
@@ -0,0 +1,55 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import random
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse
+from ..utils import (
+    sanitized_Request,
+    xpath_text,
+)
+
+
+class MatchTVIE(InfoExtractor):
+    _VALID_URL = r'https?://matchtv\.ru/?#live-player'
+    _TEST = {
+        'url': 'http://matchtv.ru/#live-player',
+        'info_dict': {
+            'id': 'matchtv-live',
+            'ext': 'flv',
+            'title': 're:^Матч ТВ - Прямой эфир \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = 'matchtv-live'
+        request = sanitized_Request(
+            'http://player.matchtv.ntvplus.tv/player/smil?%s' % compat_urllib_parse.urlencode({
+                'ts': '',
+                'quality': 'SD',
+                'contentId': '561d2c0df7159b37178b4567',
+                'sign': '',
+                'includeHighlights': '0',
+                'userId': '',
+                'sessionId': random.randint(1, 1000000000),
+                'contentType': 'channel',
+                'timeShift': '0',
+                'platform': 'portal',
+            }),
+            headers={
+                'Referer': 'http://player.matchtv.ntvplus.tv/embed-player/NTVEmbedPlayer.swf',
+            })
+        video_url = self._download_json(request, video_id)['data']['videoUrl']
+        f4m_url = xpath_text(self._download_xml(video_url, video_id), './to')
+        formats = self._extract_f4m_formats(f4m_url, video_id)
+        return {
+            'id': video_id,
+            'title': self._live_title('Матч ТВ - Прямой эфир'),
+            'is_live': True,
+            'formats': formats,
+        }

From 53be8894e476f6e3a49373fbd34ecff157440607 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Jan 2016 18:44:22 +0600
Subject: [PATCH 0500/1105] [options] Add missing closing parenthesis

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 39fc4306a..2137dfb3f 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -419,7 +419,7 @@ def parseOpts(overrideArguments=None):
         '--hls-use-mpegts',
         dest='hls_use_mpegts', action='store_true',
         help='Use the mpegts container for HLS videos, allowing to play the '
-             'video while downloading (some players may not be able to play it')
+             'video while downloading (some players may not be able to play it)')
     downloader.add_option(
         '--external-downloader',
         dest='external_downloader', metavar='COMMAND',

From 17b598d30cae2c287f3556f874ddf0fc5d028aec Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 21:05:55 +0800
Subject: [PATCH 0501/1105] [common] _parse_dash_manifest() from youtube.py

---
 youtube_dl/extractor/common.py  | 52 +++++++++++++++++++++++++++++++++
 youtube_dl/extractor/youtube.py | 52 ---------------------------------
 2 files changed, 52 insertions(+), 52 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b3d57dfce..7ad255672 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1330,6 +1330,58 @@ class InfoExtractor(object):
             })
         return entries
 
+    def _parse_dash_manifest(self, video_id, dash_doc, fatal=True):
+        formats = []
+        for a in dash_doc.findall('.//{urn:mpeg:DASH:schema:MPD:2011}AdaptationSet'):
+            mime_type = a.attrib.get('mimeType')
+            for r in a.findall('{urn:mpeg:DASH:schema:MPD:2011}Representation'):
+                url_el = r.find('{urn:mpeg:DASH:schema:MPD:2011}BaseURL')
+                if url_el is None:
+                    continue
+                if mime_type == 'text/vtt':
+                    # TODO implement WebVTT downloading
+                    pass
+                elif mime_type.startswith('audio/') or mime_type.startswith('video/'):
+                    segment_list = r.find('{urn:mpeg:DASH:schema:MPD:2011}SegmentList')
+                    format_id = r.attrib['id']
+                    video_url = url_el.text
+                    filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength'))
+                    f = {
+                        'format_id': format_id,
+                        'url': video_url,
+                        'width': int_or_none(r.attrib.get('width')),
+                        'height': int_or_none(r.attrib.get('height')),
+                        'tbr': int_or_none(r.attrib.get('bandwidth'), 1000),
+                        'asr': int_or_none(r.attrib.get('audioSamplingRate')),
+                        'filesize': filesize,
+                        'fps': int_or_none(r.attrib.get('frameRate')),
+                    }
+                    if segment_list is not None:
+                        f.update({
+                            'initialization_url': segment_list.find('{urn:mpeg:DASH:schema:MPD:2011}Initialization').attrib['sourceURL'],
+                            'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')],
+                            'protocol': 'http_dash_segments',
+                        })
+                    try:
+                        existing_format = next(
+                            fo for fo in formats
+                            if fo['format_id'] == format_id)
+                    except StopIteration:
+                        full_info = self._formats.get(format_id, {}).copy()
+                        full_info.update(f)
+                        codecs = r.attrib.get('codecs')
+                        if codecs:
+                            if full_info.get('acodec') == 'none':
+                                full_info['vcodec'] = codecs
+                            elif full_info.get('vcodec') == 'none':
+                                full_info['acodec'] = codecs
+                        formats.append(full_info)
+                    else:
+                        existing_format.update(f)
+                else:
+                    self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
+        return formats
+
     def _live_title(self, name):
         """ Generate the title for a live video """
         now = datetime.datetime.now()
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 08b7e15c4..acd5cf2c3 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1035,58 +1035,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
         return self._download_webpage(url, video_id, note='Searching for annotations.', errnote='Unable to download video annotations.')
 
-    def _parse_dash_manifest(self, video_id, dash_doc, fatal=True):
-        formats = []
-        for a in dash_doc.findall('.//{urn:mpeg:DASH:schema:MPD:2011}AdaptationSet'):
-            mime_type = a.attrib.get('mimeType')
-            for r in a.findall('{urn:mpeg:DASH:schema:MPD:2011}Representation'):
-                url_el = r.find('{urn:mpeg:DASH:schema:MPD:2011}BaseURL')
-                if url_el is None:
-                    continue
-                if mime_type == 'text/vtt':
-                    # TODO implement WebVTT downloading
-                    pass
-                elif mime_type.startswith('audio/') or mime_type.startswith('video/'):
-                    segment_list = r.find('{urn:mpeg:DASH:schema:MPD:2011}SegmentList')
-                    format_id = r.attrib['id']
-                    video_url = url_el.text
-                    filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength'))
-                    f = {
-                        'format_id': format_id,
-                        'url': video_url,
-                        'width': int_or_none(r.attrib.get('width')),
-                        'height': int_or_none(r.attrib.get('height')),
-                        'tbr': int_or_none(r.attrib.get('bandwidth'), 1000),
-                        'asr': int_or_none(r.attrib.get('audioSamplingRate')),
-                        'filesize': filesize,
-                        'fps': int_or_none(r.attrib.get('frameRate')),
-                    }
-                    if segment_list is not None:
-                        f.update({
-                            'initialization_url': segment_list.find('{urn:mpeg:DASH:schema:MPD:2011}Initialization').attrib['sourceURL'],
-                            'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')],
-                            'protocol': 'http_dash_segments',
-                        })
-                    try:
-                        existing_format = next(
-                            fo for fo in formats
-                            if fo['format_id'] == format_id)
-                    except StopIteration:
-                        full_info = self._formats.get(format_id, {}).copy()
-                        full_info.update(f)
-                        codecs = r.attrib.get('codecs')
-                        if codecs:
-                            if full_info.get('acodec') == 'none':
-                                full_info['vcodec'] = codecs
-                            elif full_info.get('vcodec') == 'none':
-                                full_info['acodec'] = codecs
-                        formats.append(full_info)
-                    else:
-                        existing_format.update(f)
-                else:
-                    self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
-        return formats
-
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})
 

From 5fa1702ca6d8a57b4f0ff108c6fe7488d09b448c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Jan 2016 19:20:52 +0600
Subject: [PATCH 0502/1105] [downloader/fragment] Do not report total bytes
 estimation and eta for live streams

---
 youtube_dl/downloader/fragment.py | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 0c9113d0f..e2f34eec6 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -26,7 +26,11 @@ class FragmentFD(FileDownloader):
         self._start_frag_download(ctx)
 
     def _prepare_frag_download(self, ctx):
-        self.to_screen('[%s] Total fragments: %d' % (self.FD_NAME, ctx['total_frags']))
+        if 'live' not in ctx:
+            ctx['live'] = False
+        self.to_screen(
+            '[%s] Total fragments: %s'
+            % (self.FD_NAME, ctx['total_frags'] if not ctx['live'] else 'unknown (live)'))
         self.report_destination(ctx['filename'])
         dl = HttpQuietDownloader(
             self.ydl,
@@ -74,13 +78,15 @@ class FragmentFD(FileDownloader):
             if s['status'] not in ('downloading', 'finished'):
                 return
 
+            time_now = time.time()
+
             frag_total_bytes = s.get('total_bytes') or 0
 
-            estimated_size = (
-                (ctx['complete_frags_downloaded_bytes'] + frag_total_bytes) /
-                (state['frag_index'] + 1) * total_frags)
-            time_now = time.time()
-            state['total_bytes_estimate'] = estimated_size
+            if not ctx['live']:
+                estimated_size = (
+                    (ctx['complete_frags_downloaded_bytes'] + frag_total_bytes) /
+                    (state['frag_index'] + 1) * total_frags)
+                state['total_bytes_estimate'] = estimated_size
             state['elapsed'] = time_now - start
 
             if s['status'] == 'finished':
@@ -91,9 +97,10 @@ class FragmentFD(FileDownloader):
             else:
                 frag_downloaded_bytes = s['downloaded_bytes']
                 state['downloaded_bytes'] += frag_downloaded_bytes - ctx['prev_frag_downloaded_bytes']
-                state['eta'] = self.calc_eta(
-                    start, time_now, estimated_size,
-                    state['downloaded_bytes'])
+                if not ctx['live']:
+                    state['eta'] = self.calc_eta(
+                        start, time_now, estimated_size,
+                        state['downloaded_bytes'])
                 state['speed'] = s.get('speed')
                 ctx['prev_frag_downloaded_bytes'] = frag_downloaded_bytes
             self._hook_progress(state)

From 09104e9930cff6cc5e9d3ab1951fdbcbb5e96840 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Jan 2016 19:22:15 +0600
Subject: [PATCH 0503/1105] [downloader/f4m] Add live stream flag to context

Now download progress for f4m livestreams is reported correctly
---
 youtube_dl/downloader/f4m.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index f8da04f69..581fa7b71 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -332,6 +332,7 @@ class F4mFD(FragmentFD):
         ctx = {
             'filename': filename,
             'total_frags': total_frags,
+            'live': live,
         }
 
         self._prepare_frag_download(ctx)

From b323e1707d8f058b88a5f15f3418b31cf969399d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 21:27:43 +0800
Subject: [PATCH 0504/1105] [common] Modify _parse_dash_manifest for use in
 Facebook

---
 youtube_dl/extractor/common.py | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 7ad255672..83628a68f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1330,22 +1330,24 @@ class InfoExtractor(object):
             })
         return entries
 
-    def _parse_dash_manifest(self, video_id, dash_doc, fatal=True):
+    def _parse_dash_manifest(self, video_id, dash_doc, default_ns='urn:mpeg:DASH:schema:MPD:2011', formats_dict={}, fatal=True):
+        def _add_ns(tag):
+            return '{%s}%s' % (default_ns, tag)
+
         formats = []
-        for a in dash_doc.findall('.//{urn:mpeg:DASH:schema:MPD:2011}AdaptationSet'):
+        for a in dash_doc.findall('.//' + _add_ns('AdaptationSet')):
             mime_type = a.attrib.get('mimeType')
-            for r in a.findall('{urn:mpeg:DASH:schema:MPD:2011}Representation'):
-                url_el = r.find('{urn:mpeg:DASH:schema:MPD:2011}BaseURL')
-                if url_el is None:
-                    continue
+            for r in a.findall(_add_ns('Representation')):
+                mime_type = r.attrib.get('mimeType') or mime_type
+                url_el = r.find(_add_ns('BaseURL'))
                 if mime_type == 'text/vtt':
                     # TODO implement WebVTT downloading
                     pass
                 elif mime_type.startswith('audio/') or mime_type.startswith('video/'):
-                    segment_list = r.find('{urn:mpeg:DASH:schema:MPD:2011}SegmentList')
+                    segment_list = r.find(_add_ns('SegmentList'))
                     format_id = r.attrib['id']
-                    video_url = url_el.text
-                    filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength'))
+                    video_url = url_el.text if url_el else None
+                    filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength') if url_el else None)
                     f = {
                         'format_id': format_id,
                         'url': video_url,
@@ -1357,17 +1359,20 @@ class InfoExtractor(object):
                         'fps': int_or_none(r.attrib.get('frameRate')),
                     }
                     if segment_list is not None:
+                        initialization_url = segment_list.find(_add_ns('Initialization')).attrib['sourceURL']
                         f.update({
-                            'initialization_url': segment_list.find('{urn:mpeg:DASH:schema:MPD:2011}Initialization').attrib['sourceURL'],
-                            'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')],
+                            'initialization_url': initialization_url,
+                            'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall(_add_ns('SegmentURL'))],
                             'protocol': 'http_dash_segments',
                         })
+                        if not f.get('url'):
+                            f['url'] = initialization_url
                     try:
                         existing_format = next(
                             fo for fo in formats
                             if fo['format_id'] == format_id)
                     except StopIteration:
-                        full_info = self._formats.get(format_id, {}).copy()
+                        full_info = formats_dict.get(format_id, {}).copy()
                         full_info.update(f)
                         codecs = r.attrib.get('codecs')
                         if codecs:

From 2c2f1efdcd37f88196bb2b546e6d8bc7c22f3b6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Jan 2016 19:30:31 +0600
Subject: [PATCH 0505/1105] [downloader/fragment] Remove superfluous whitespace

---
 youtube_dl/downloader/fragment.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index e2f34eec6..8b96eceb9 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -79,15 +79,13 @@ class FragmentFD(FileDownloader):
                 return
 
             time_now = time.time()
-
+            state['elapsed'] = time_now - start
             frag_total_bytes = s.get('total_bytes') or 0
-
             if not ctx['live']:
                 estimated_size = (
                     (ctx['complete_frags_downloaded_bytes'] + frag_total_bytes) /
                     (state['frag_index'] + 1) * total_frags)
                 state['total_bytes_estimate'] = estimated_size
-            state['elapsed'] = time_now - start
 
             if s['status'] == 'finished':
                 state['frag_index'] += 1

From 0803753fea578355db68ae9f2d915db04cde7557 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 21:31:53 +0800
Subject: [PATCH 0506/1105] [facebook] Add support for DASH manifests

---
 youtube_dl/extractor/facebook.py | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 899b0896b..1f3e270d9 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -6,9 +6,11 @@ import socket
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_etree_fromstring,
     compat_http_client,
     compat_urllib_error,
     compat_urllib_parse_unquote,
+    compat_urllib_parse_unquote_plus,
 )
 from ..utils import (
     error_to_compat_str,
@@ -44,6 +46,9 @@ class FacebookIE(InfoExtractor):
     _CHECKPOINT_URL = 'https://www.facebook.com/checkpoint/?next=http%3A%2F%2Ffacebook.com%2Fhome.php&_fb_noscript=1'
     _NETRC_MACHINE = 'facebook'
     IE_NAME = 'facebook'
+
+    _CHROME_USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.97 Safari/537.36'
+
     _TESTS = [{
         'url': 'https://www.facebook.com/video.php?v=637842556329505&fref=nf',
         'md5': '6a40d33c0eccbb1af76cf0485a052659',
@@ -65,6 +70,15 @@ class FacebookIE(InfoExtractor):
         'expected_warnings': [
             'title'
         ]
+    }, {
+        'note': 'Video with DASH manifest',
+        'url': 'https://www.facebook.com/video.php?v=957955867617029',
+        'info_dict': {
+            'id': '957955867617029',
+            'ext': 'mp4',
+            'title': 'When you post epic content on instagram.com/433 8 million followers, this is ...',
+            'uploader': 'Demy de Zeeuw',
+        },
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',
         'only_matching': True,
@@ -147,8 +161,9 @@ class FacebookIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        url = 'https://www.facebook.com/video/video.php?v=%s' % video_id
-        webpage = self._download_webpage(url, video_id)
+        req = sanitized_Request('https://www.facebook.com/video/video.php?v=%s' % video_id)
+        req.add_header('User-Agent', self._CHROME_USER_AGENT)
+        webpage = self._download_webpage(req, video_id)
 
         video_data = None
 
@@ -197,9 +212,16 @@ class FacebookIE(InfoExtractor):
                             'url': src,
                             'preference': -10 if format_id == 'progressive' else 0,
                         })
+            dash_manifest = f[0].get('dash_manifest')
+            if dash_manifest:
+                formats.extend(self._parse_dash_manifest(
+                    video_id, compat_etree_fromstring(compat_urllib_parse_unquote_plus(dash_manifest)),
+                    default_ns='urn:mpeg:dash:schema:mpd:2011'))
         if not formats:
             raise ExtractorError('Cannot find video formats')
 
+        self._sort_formats(formats)
+
         video_title = self._html_search_regex(
             r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>([^<]*)</h2>', webpage, 'title',
             default=None)

From 5d2c0fd9ba2251a5c91b7d8b1dd285b427a2251e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 21:32:15 +0800
Subject: [PATCH 0507/1105] [youtube] Pass self._formats to
 _parse_dash_manifest

---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index acd5cf2c3..c1d570cb3 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1478,7 +1478,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         errnote='Could not download DASH manifest',
                         fatal=dash_mpd_fatal)
 
-                    for df in self._parse_dash_manifest(video_id, dash_doc, dash_mpd_fatal):
+                    for df in self._parse_dash_manifest(
+                            video_id, dash_doc, formats_dict=self._formats, fatal=dash_mpd_fatal):
                         # Do not overwrite DASH format found in some previous DASH manifest
                         if df['format_id'] not in dash_formats:
                             dash_formats[df['format_id']] = df

From 5ea1eb78f5e8d9b2ede35504dc3b999f5f89bbaa Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 21:36:01 +0800
Subject: [PATCH 0508/1105] [common] Fix for youtube

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 83628a68f..243db71dc 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1346,8 +1346,8 @@ class InfoExtractor(object):
                 elif mime_type.startswith('audio/') or mime_type.startswith('video/'):
                     segment_list = r.find(_add_ns('SegmentList'))
                     format_id = r.attrib['id']
-                    video_url = url_el.text if url_el else None
-                    filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength') if url_el else None)
+                    video_url = url_el.text if url_el is not None else None
+                    filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength') if url_el is not None else None)
                     f = {
                         'format_id': format_id,
                         'url': video_url,

From df374b52228e8a083d045f9bc56847e418ef452e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 21:42:27 +0800
Subject: [PATCH 0509/1105] [common] Prefer the manifest than formats_dict in
 determining codecs

---
 youtube_dl/extractor/common.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 243db71dc..f1313ef04 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1376,10 +1376,15 @@ class InfoExtractor(object):
                         full_info.update(f)
                         codecs = r.attrib.get('codecs')
                         if codecs:
-                            if full_info.get('acodec') == 'none':
-                                full_info['vcodec'] = codecs
-                            elif full_info.get('vcodec') == 'none':
-                                full_info['acodec'] = codecs
+                            if mime_type.startswith('video/'):
+                                vcodec, acodec = codecs, 'none'
+                            else:  # mime_type.startswith('audio/')
+                                vcodec, acodec = 'none', codecs
+
+                            full_info.update({
+                                'vcodec': vcodec,
+                                'acodec': acodec,
+                            })
                         formats.append(full_info)
                     else:
                         existing_format.update(f)

From b8c9926c0a9d68fb636bf99e873d6a88de41f76b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Jan 2016 19:43:25 +0600
Subject: [PATCH 0510/1105] [downloader/f4m] Do not update fragment list while
 test

---
 youtube_dl/downloader/f4m.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 581fa7b71..fc9642905 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -322,7 +322,8 @@ class F4mFD(FragmentFD):
             metadata = None
 
         fragments_list = build_fragments_list(boot_info)
-        if self.params.get('test', False):
+        test = self.params.get('test', False)
+        if test:
             # We only download the first fragment
             fragments_list = fragments_list[:1]
         total_frags = len(fragments_list)
@@ -387,7 +388,7 @@ class F4mFD(FragmentFD):
                 else:
                     raise
 
-            if not fragments_list and live and bootstrap_url:
+            if not fragments_list and not test and live and bootstrap_url:
                 fragments_list = self._update_live_fragments(bootstrap_url, frag_i)
                 total_frags += len(fragments_list)
                 if fragments_list and (fragments_list[0][1] > frag_i + 1):

From a6c2c24479e5f4827ceb06f64d855329c0a6f593 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 22:28:53 +0800
Subject: [PATCH 0511/1105] [youtube] Remove '(v|a)codec': 'none' entries

Not used anymore
---
 youtube_dl/extractor/youtube.py | 80 ++++++++++++++++-----------------
 1 file changed, 40 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c1d570cb3..f992c3624 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -316,55 +316,55 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 
         # DASH mp4 video
-        '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
-        '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
-        '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
-        '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
-        '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
-        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
-        '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
-        '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
-        '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
-        '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
-        '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
+        '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
+        '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
+        '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
+        '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
 
         # Dash mp4 audio
-        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 48, 'preference': -50, 'container': 'm4a_dash'},
-        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 128, 'preference': -50, 'container': 'm4a_dash'},
-        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
+        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'preference': -50, 'container': 'm4a_dash'},
+        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'preference': -50, 'container': 'm4a_dash'},
+        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
 
         # Dash webm
-        '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp9', 'preference': -40},
-        '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
-        '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
-        '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
-        '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
-        '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
-        '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
-        '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
-        '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
+        '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9', 'preference': -40},
+        '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
+        '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
+        '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
+        '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
+        '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
+        '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
+        '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
+        '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
-        '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
-        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
-        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
-        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
-        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
-        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
+        '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
+        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
+        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
+        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
+        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
+        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
 
         # Dash webm audio
-        '171': {'ext': 'webm', 'acodec': 'vorbis', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},
-        '172': {'ext': 'webm', 'acodec': 'vorbis', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50},
+        '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},
+        '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50},
 
         # Dash webm audio with opus inside
-        '249': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50, 'preference': -50},
-        '250': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70, 'preference': -50},
-        '251': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160, 'preference': -50},
+        '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50, 'preference': -50},
+        '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70, 'preference': -50},
+        '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160, 'preference': -50},
 
         # RTMP (unnamed)
         '_rtmp': {'protocol': 'rtmp'},

From 16f38a699f7c6d2820720d704a2373ba28c568b1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 22:40:13 +0800
Subject: [PATCH 0512/1105] [common] Rename to namespace

For consistency with _parse_smil_*
---
 youtube_dl/extractor/common.py   | 6 +++---
 youtube_dl/extractor/facebook.py | 2 +-
 youtube_dl/extractor/youtube.py  | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f1313ef04..a05efec9e 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1330,9 +1330,9 @@ class InfoExtractor(object):
             })
         return entries
 
-    def _parse_dash_manifest(self, video_id, dash_doc, default_ns='urn:mpeg:DASH:schema:MPD:2011', formats_dict={}, fatal=True):
-        def _add_ns(tag):
-            return '{%s}%s' % (default_ns, tag)
+    def _parse_dash_manifest(self, video_id, dash_doc, namespace=None, formats_dict={}, fatal=True):
+        def _add_ns(path):
+            return self._xpath_ns(path, namespace)
 
         formats = []
         for a in dash_doc.findall('.//' + _add_ns('AdaptationSet')):
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 1f3e270d9..af38eaff1 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -216,7 +216,7 @@ class FacebookIE(InfoExtractor):
             if dash_manifest:
                 formats.extend(self._parse_dash_manifest(
                     video_id, compat_etree_fromstring(compat_urllib_parse_unquote_plus(dash_manifest)),
-                    default_ns='urn:mpeg:dash:schema:mpd:2011'))
+                    namespace='urn:mpeg:dash:schema:mpd:2011'))
         if not formats:
             raise ExtractorError('Cannot find video formats')
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f992c3624..8d3fcfa5f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1479,7 +1479,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         fatal=dash_mpd_fatal)
 
                     for df in self._parse_dash_manifest(
-                            video_id, dash_doc, formats_dict=self._formats, fatal=dash_mpd_fatal):
+                            video_id, dash_doc, namespace='urn:mpeg:DASH:schema:MPD:2011', formats_dict=self._formats, fatal=dash_mpd_fatal):
                         # Do not overwrite DASH format found in some previous DASH manifest
                         if df['format_id'] not in dash_formats:
                             dash_formats[df['format_id']] = df

From c94678957fbe4483b2b7c8b3e6824cb7a215d42d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 22:45:16 +0800
Subject: [PATCH 0513/1105] [common] Remove unused arguments

---
 youtube_dl/extractor/common.py   | 2 +-
 youtube_dl/extractor/facebook.py | 2 +-
 youtube_dl/extractor/youtube.py  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a05efec9e..5a2b7a721 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1330,7 +1330,7 @@ class InfoExtractor(object):
             })
         return entries
 
-    def _parse_dash_manifest(self, video_id, dash_doc, namespace=None, formats_dict={}, fatal=True):
+    def _parse_dash_manifest(self, dash_doc, namespace=None, formats_dict={}):
         def _add_ns(path):
             return self._xpath_ns(path, namespace)
 
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index af38eaff1..bd65e43f2 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -215,7 +215,7 @@ class FacebookIE(InfoExtractor):
             dash_manifest = f[0].get('dash_manifest')
             if dash_manifest:
                 formats.extend(self._parse_dash_manifest(
-                    video_id, compat_etree_fromstring(compat_urllib_parse_unquote_plus(dash_manifest)),
+                    compat_etree_fromstring(compat_urllib_parse_unquote_plus(dash_manifest)),
                     namespace='urn:mpeg:dash:schema:mpd:2011'))
         if not formats:
             raise ExtractorError('Cannot find video formats')
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8d3fcfa5f..a2f776050 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1479,7 +1479,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         fatal=dash_mpd_fatal)
 
                     for df in self._parse_dash_manifest(
-                            video_id, dash_doc, namespace='urn:mpeg:DASH:schema:MPD:2011', formats_dict=self._formats, fatal=dash_mpd_fatal):
+                            dash_doc, namespace='urn:mpeg:DASH:schema:MPD:2011', formats_dict=self._formats):
                         # Do not overwrite DASH format found in some previous DASH manifest
                         if df['format_id'] not in dash_formats:
                             dash_formats[df['format_id']] = df

From 2d2fa82d172a10a49fb5449fa35bc409de778f05 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 22:52:23 +0800
Subject: [PATCH 0514/1105] [common] Add _extract_dash_manifest_formats

---
 youtube_dl/extractor/common.py  | 15 +++++++++++++++
 youtube_dl/extractor/youtube.py | 10 +++-------
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5a2b7a721..199a04d1c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1330,6 +1330,21 @@ class InfoExtractor(object):
             })
         return entries
 
+    def _download_dash_manifest(self, dash_manifest_url, video_id, fatal=True):
+        return self._download_xml(
+            dash_manifest_url, video_id,
+            note='Downloading DASH manifest',
+            errnote='Could not download DASH manifest',
+            fatal=fatal)
+
+    def _extract_dash_manifest_formats(self, dash_manifest_url, video_id, fatal=True, namespace=None, formats_dict={}):
+        dash_doc = self._download_dash_manifest(dash_manifest_url, video_id, fatal)
+        if dash_doc is False:
+            return []
+
+        return self._parse_dash_manifest(
+            dash_doc, namespace=namespace, formats_dict=formats_dict)
+
     def _parse_dash_manifest(self, dash_doc, namespace=None, formats_dict={}):
         def _add_ns(path):
             return self._xpath_ns(path, namespace)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a2f776050..d6fef39e9 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1472,14 +1472,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         return '/signature/%s' % dec_s
 
                     dash_manifest_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, dash_manifest_url)
-                    dash_doc = self._download_xml(
-                        dash_manifest_url, video_id,
-                        note='Downloading DASH manifest',
-                        errnote='Could not download DASH manifest',
-                        fatal=dash_mpd_fatal)
 
-                    for df in self._parse_dash_manifest(
-                            dash_doc, namespace='urn:mpeg:DASH:schema:MPD:2011', formats_dict=self._formats):
+                    for df in self._extract_dash_manifest_formats(
+                            dash_manifest_url, video_id, fatal=dash_mpd_fatal,
+                            namespace='urn:mpeg:DASH:schema:MPD:2011', formats_dict=self._formats):
                         # Do not overwrite DASH format found in some previous DASH manifest
                         if df['format_id'] not in dash_formats:
                             dash_formats[df['format_id']] = df

From 248ae880b6900a753e38253f149c013c3f95c0c5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 23:01:19 +0800
Subject: [PATCH 0515/1105] [facebook] Add md5 for the test case with DASH

---
 youtube_dl/extractor/facebook.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index bd65e43f2..b6d1180f0 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -73,6 +73,7 @@ class FacebookIE(InfoExtractor):
     }, {
         'note': 'Video with DASH manifest',
         'url': 'https://www.facebook.com/video.php?v=957955867617029',
+        'md5': '54706e4db4f5ad58fbad82dde1f1213f',
         'info_dict': {
             'id': '957955867617029',
             'ext': 'mp4',

From 1bf996fa5cd996c5f586089388853e324dc3850d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 30 Jan 2016 20:45:56 +0100
Subject: [PATCH 0516/1105] [generic] Add support for Limelight API

---
 youtube_dl/extractor/generic.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 26d3698c8..b18e734c4 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1819,6 +1819,17 @@ class GenericIE(InfoExtractor):
         if digiteka_url:
             return self.url_result(self._proto_relative_url(digiteka_url), DigitekaIE.ie_key())
 
+        # Look for Limelight embeds
+        mobj = re.search(r'LimelightPlayer\.doLoad(Media|Channel|ChannelList)\(["\'](?P<id>[a-z0-9]{32})', webpage)
+        if mobj:
+            lm = {
+                'Media': 'media',
+                'Channel': 'channel',
+                'ChannelList': 'channel_list',
+            }
+            return self.url_result('limelight:%s:%s' % (
+                lm[mobj.group(1)], mobj.group(2)), 'Limelight%s' % mobj.group(1), mobj.group(2))
+
         # Look for AdobeTVVideo embeds
         mobj = re.search(
             r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',

From 8f1fddc816dac7c0e9b216e85474aa8eb0d847bf Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 30 Jan 2016 20:51:47 +0100
Subject: [PATCH 0517/1105] [limelight] fix format sorting and make m3u8 and
 f4m extraction non fatal

---
 youtube_dl/extractor/limelight.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index fb03dd527..1a0625ac3 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -40,7 +40,8 @@ class LimelightBaseIE(InfoExtractor):
             if not stream_url:
                 continue
             if '.f4m' in stream_url:
-                formats.extend(self._extract_f4m_formats(stream_url, video_id))
+                formats.extend(self._extract_f4m_formats(
+                    stream_url, video_id, fatal=False))
             else:
                 fmt = {
                     'url': stream_url,
@@ -72,8 +73,8 @@ class LimelightBaseIE(InfoExtractor):
             format_id = mobile_url.get('targetMediaPlatform')
             if determine_ext(media_url) == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
-                    media_url, video_id, 'mp4', entry_protocol='m3u8_native',
-                    preference=-1, m3u8_id=format_id))
+                    media_url, video_id, 'mp4', 'm3u8_native',
+                    m3u8_id=format_id, fatal=False))
             else:
                 formats.append({
                     'url': media_url,

From a7685b3a6b02be62ebd53e2978ab43b1cde3ec4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 Jan 2016 02:38:28 +0600
Subject: [PATCH 0518/1105] [npo] Add extension for m3u8

---
 youtube_dl/extractor/npo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index eb12fb810..7cd5c9303 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -189,7 +189,7 @@ class NPOIE(NPOBaseIE):
                 if not video_url:
                     continue
                 if format_id == 'adaptive':
-                    formats.extend(self._extract_m3u8_formats(video_url, video_id))
+                    formats.extend(self._extract_m3u8_formats(video_url, video_id, 'mp4'))
                 else:
                     formats.append({
                         'url': video_url,

From 60ad3eb9706861d4182ab44ee19d64350ca2e36e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 03:21:33 +0800
Subject: [PATCH 0519/1105] [viidea] Skip download for the test case requiring
 ffmpeg

---
 youtube_dl/extractor/viidea.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/viidea.py b/youtube_dl/extractor/viidea.py
index 525e303d4..315984bf9 100644
--- a/youtube_dl/extractor/viidea.py
+++ b/youtube_dl/extractor/viidea.py
@@ -45,6 +45,10 @@ class ViideaIE(InfoExtractor):
             'upload_date': '20130627',
             'duration': 565,
         },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         # video with invalid direct format links (HTTP 403)
         'url': 'http://videolectures.net/russir2010_filippova_nlp/',

From 7c38af48b91d050a5694e878a789a028fbcc0a3b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 03:31:03 +0800
Subject: [PATCH 0520/1105] [vgtv] Fix test_VGTV_2

---
 youtube_dl/extractor/vgtv.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 86ba70ed9..14e945d49 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -86,10 +86,9 @@ class VGTVIE(XstreamIE):
         {
             # streamType: wasLive
             'url': 'http://www.vgtv.no/#!/live/113063/direkte-v75-fra-solvalla',
-            'md5': '458f4841239dab414343b50e5af8869c',
             'info_dict': {
                 'id': '113063',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'V75 fra Solvalla 30.05.15',
                 'description': 'md5:b3743425765355855f88e096acc93231',
                 'thumbnail': 're:^https?://.*\.jpg',
@@ -98,6 +97,10 @@ class VGTVIE(XstreamIE):
                 'upload_date': '20150530',
                 'view_count': int,
             },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
         },
         {
             'url': 'http://www.aftenposten.no/webtv/#!/video/21039/trailer-sweatshop-i-can-t-take-any-more',

From 809e1857c5155fb41d033b3c0215708eb41ddb55 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 03:38:20 +0800
Subject: [PATCH 0521/1105] [screenwavemedia] Fix HLS extension and
 test_TeamFour

---
 youtube_dl/extractor/screenwavemedia.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 05f93904c..e5d62a139 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -71,7 +71,7 @@ class ScreenwaveMediaIE(InfoExtractor):
         formats = []
         for source in sources:
             if source['type'] == 'hls':
-                formats.extend(self._extract_m3u8_formats(source['file'], video_id))
+                formats.extend(self._extract_m3u8_formats(source['file'], video_id, ext='mp4'))
             else:
                 file_ = source.get('file')
                 if not file_:
@@ -107,7 +107,11 @@ class TeamFourIE(InfoExtractor):
             'upload_date': '20130401',
             'description': 'Check out this and more on our website: http://teamfourstar.com\nTFS Store: http://sharkrobot.com/team-four-star\nFollow on Twitter: http://twitter.com/teamfourstar\nLike on FB: http://facebook.com/teamfourstar',
             'title': 'A Moment With TFS Episode 4',
-        }
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):

From ed1a390583f7adfcf38c4af5432b4b2e611004ab Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 03:42:34 +0800
Subject: [PATCH 0522/1105] [tv2] Fix test_TV2

---
 youtube_dl/extractor/tv2.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/tv2.py b/youtube_dl/extractor/tv2.py
index fa338b936..1457e524e 100644
--- a/youtube_dl/extractor/tv2.py
+++ b/youtube_dl/extractor/tv2.py
@@ -17,18 +17,21 @@ class TV2IE(InfoExtractor):
     _VALID_URL = 'http://(?:www\.)?tv2\.no/v/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.tv2.no/v/916509/',
-        'md5': '9cb9e3410b18b515d71892f27856e9b1',
         'info_dict': {
             'id': '916509',
-            'ext': 'flv',
-            'title': 'Se Gryttens hyllest av Steven Gerrard',
+            'ext': 'mp4',
+            'title': 'Se Frode Gryttens hyllest av Steven Gerrard',
             'description': 'TV 2 Sportens huspoet tar avskjed med Liverpools kaptein Steven Gerrard.',
             'timestamp': 1431715610,
             'upload_date': '20150515',
             'duration': 156.967,
             'view_count': int,
             'categories': list,
-        }
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):

From eb6fc7d32a6f909496cc1a4b46fdcbf2cc6f344e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 03:45:09 +0800
Subject: [PATCH 0523/1105] [senateisvp] Fix test_SenateISVP and
 test_SenateISVP_1

---
 youtube_dl/extractor/senateisvp.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
index 474ebb49b..990ea0fa8 100644
--- a/youtube_dl/extractor/senateisvp.py
+++ b/youtube_dl/extractor/senateisvp.py
@@ -53,17 +53,25 @@ class SenateISVPIE(InfoExtractor):
         'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png',
         'info_dict': {
             'id': 'judiciary031715',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Integrated Senate Video Player',
             'thumbnail': 're:^https?://.*\.(?:jpg|png)$',
-        }
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.senate.gov/isvp/?type=live&comm=commerce&filename=commerce011514.mp4&auto_play=false',
         'info_dict': {
             'id': 'commerce011514',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Integrated Senate Video Player'
-        }
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.senate.gov/isvp/?type=arch&comm=intel&filename=intel090613&hc_location=ufi',
         # checksum differs each time

From 670ad51ade60141a7948aa68686db49dd2e30145 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 04:01:45 +0800
Subject: [PATCH 0524/1105] [nrktv] Fix _TESTS

---
 youtube_dl/extractor/nrk.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 6ff13050d..a126f5054 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -133,26 +133,32 @@ class NRKTVIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'https://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014',
-            'md5': 'adf2c5454fa2bf032f47a9f8fb351342',
             'info_dict': {
                 'id': 'MUHH48000314',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': '20 spørsmål',
                 'description': 'md5:bdea103bc35494c143c6a9acdd84887a',
                 'upload_date': '20140523',
                 'duration': 1741.52,
             },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
         },
         {
             'url': 'https://tv.nrk.no/program/mdfp15000514',
-            'md5': '383650ece2b25ecec996ad7b5bb2a384',
             'info_dict': {
                 'id': 'mdfp15000514',
-                'ext': 'flv',
-                'title': 'Kunnskapskanalen: Grunnlovsjubiléet - Stor ståhei for ingenting',
+                'ext': 'mp4',
+                'title': 'Grunnlovsjubiléet - Stor ståhei for ingenting',
                 'description': 'md5:654c12511f035aed1e42bdf5db3b206a',
                 'upload_date': '20140524',
-                'duration': 4605.0,
+                'duration': 4605.08,
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
             },
         },
         {

From e881c4bcabb64f84e382a75bd9c03379859105d0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 04:34:46 +0800
Subject: [PATCH 0525/1105] [nbc] Use NBC's id and fix _TESTS

ThePlatform URL gives the same ID for all _TESTS
---
 youtube_dl/extractor/nbc.py | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 1dd54c2f1..18d01f423 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -19,32 +19,39 @@ class NBCIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://www.nbc.com/the-tonight-show/segments/112966',
-            # md5 checksum is not stable
             'info_dict': {
-                'id': 'c9xnCo0YPOPH',
-                'ext': 'flv',
+                'id': '112966',
+                'ext': 'mp4',
                 'title': 'Jimmy Fallon Surprises Fans at Ben & Jerry\'s',
                 'description': 'Jimmy gives out free scoops of his new "Tonight Dough" ice cream flavor by surprising customers at the Ben & Jerry\'s scoop shop.',
             },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
         },
         {
             'url': 'http://www.nbc.com/the-tonight-show/episodes/176',
             'info_dict': {
-                'id': 'XwU9KZkp98TH',
+                'id': '176',
                 'ext': 'flv',
                 'title': 'Ricky Gervais, Steven Van Zandt, ILoveMakonnen',
                 'description': 'A brand new episode of The Tonight Show welcomes Ricky Gervais, Steven Van Zandt and ILoveMakonnen.',
             },
-            'skip': 'Only works from US',
+            'skip': '404 Not Found',
         },
         {
             'url': 'http://www.nbc.com/saturday-night-live/video/star-wars-teaser/2832821',
             'info_dict': {
-                'id': '8iUuyzWDdYUZ',
-                'ext': 'flv',
+                'id': '2832821',
+                'ext': 'mp4',
                 'title': 'Star Wars Teaser',
                 'description': 'md5:0b40f9cbde5b671a7ff62fceccc4f442',
             },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
             'skip': 'Only works from US',
         },
         {
@@ -66,7 +73,11 @@ class NBCIE(InfoExtractor):
             webpage, 'theplatform url').replace('_no_endcard', '').replace('\\/', '/')))
         if theplatform_url.startswith('//'):
             theplatform_url = 'http:' + theplatform_url
-        return self.url_result(smuggle_url(theplatform_url, {'source_url': url}))
+        return {
+            '_type': 'url_transparent',
+            'url': smuggle_url(theplatform_url, {'source_url': url}),
+            'id': video_id,
+        }
 
 
 class NBCSportsVPlayerIE(InfoExtractor):

From db9b1dbcd918bee41b3b38f1f1106ad5dec07bb7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 04:49:04 +0800
Subject: [PATCH 0526/1105] [nba] Add ext for hls formats and fix test_NBA

---
 youtube_dl/extractor/nba.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 9d26030d3..a071378b6 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -18,13 +18,17 @@ class NBAIE(InfoExtractor):
         'md5': '9e7729d3010a9c71506fd1248f74e4f4',
         'info_dict': {
             'id': '0021200253-okc-bkn-recap',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Thunder vs. Nets',
             'description': 'Kevin Durant scores 32 points and dishes out six assists as the Thunder beat the Nets in Brooklyn.',
             'duration': 181,
             'timestamp': 1354638466,
             'upload_date': '20121204',
         },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.nba.com/video/games/hornets/2014/12/05/0021400276-nyk-cha-play5.nba/',
         'only_matching': True,
@@ -68,7 +72,7 @@ class NBAIE(InfoExtractor):
             if video_url.startswith('/'):
                 continue
             if video_url.endswith('.m3u8'):
-                formats.extend(self._extract_m3u8_formats(video_url, video_id, m3u8_id='hls', fatal=False))
+                formats.extend(self._extract_m3u8_formats(video_url, video_id, ext='mp4', m3u8_id='hls', fatal=False))
             elif video_url.endswith('.f4m'):
                 formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id, f4m_id='hds', fatal=False))
             else:

From 89f2602880198b94f98bc41109771a52105e3cf8 Mon Sep 17 00:00:00 2001
From: rrooij <rderooij685@gmail.com>
Date: Sat, 30 Jan 2016 20:31:31 +0100
Subject: [PATCH 0527/1105] [schooltv] Add extractor for SchoolTV playlists

This closes #8163
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/npo.py      | 31 +++++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e15495ec8..dbdfb86c0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -483,6 +483,7 @@ from .npo import (
     NPOLiveIE,
     NPORadioIE,
     NPORadioFragmentIE,
+    SchoolTVIE,
     VPROIE,
     WNLIE
 )
diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index eb12fb810..ab418edf4 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -406,6 +406,37 @@ class NPORadioFragmentIE(InfoExtractor):
         }
 
 
+class SchoolTVIE(InfoExtractor):
+    IE_NAME = 'schooltv'
+    _VALID_URL = r'https?://(?:www\.)?schooltv\.nl/video/(?P<id>[^/?#&]+)'
+
+    _TEST = {
+        'url': 'http://www.schooltv.nl/video/ademhaling-de-hele-dag-haal-je-adem-maar-wat-gebeurt-er-dan-eigenlijk-in-je-lichaam/',
+        'info_dict': {
+            'id': 'WO_NTR_429477',
+            'display_id': 'ademhaling-de-hele-dag-haal-je-adem-maar-wat-gebeurt-er-dan-eigenlijk-in-je-lichaam',
+            'title': 'Ademhaling: De hele dag haal je adem. Maar wat gebeurt er dan eigenlijk in je lichaam?',
+            'ext': 'mp4',
+            'description': 'md5:abfa0ff690adb73fd0297fd033aaa631'
+        },
+        'params': {
+            # Skip because of m3u8 download
+            'skip_download': True
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        video_id = self._search_regex(r'data-mid="([^"]+)"', webpage, 'video_id')
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'NPO',
+            'url': 'npo:%s' % video_id,
+            'display_id': display_id
+        }
+
+
 class VPROIE(NPOIE):
     IE_NAME = 'vpro'
     _VALID_URL = r'https?://(?:www\.)?(?:tegenlicht\.)?vpro\.nl/(?:[^/]+/){2,}(?P<id>[^/]+)\.html'

From 673fb82e65e28935ad16d85771cefe15013a12bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 Jan 2016 04:41:18 +0600
Subject: [PATCH 0528/1105] [schooltv] Improve video id regex

---
 youtube_dl/extractor/npo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index a4363c16e..87f5675c7 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -428,7 +428,8 @@ class SchoolTVIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-        video_id = self._search_regex(r'data-mid="([^"]+)"', webpage, 'video_id')
+        video_id = self._search_regex(
+            r'data-mid=(["\'])(?P<id>.+?)\1', webpage, 'video_id', group='id')
         return {
             '_type': 'url_transparent',
             'ie_key': 'NPO',

From 14823decf3f27d5e7f17abf85444575f7600f4e4 Mon Sep 17 00:00:00 2001
From: rrooij <rderooij685@gmail.com>
Date: Sun, 31 Jan 2016 00:03:23 +0100
Subject: [PATCH 0529/1105] [Gamekings] Fix url from .tv to .nl

Gamekings doesn't use the .tv top level domain anymore, but the regular
domain for Dutch sites.
---
 youtube_dl/extractor/gamekings.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index 027f55eb2..e7747a3ae 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -9,9 +9,9 @@ from ..utils import (
 
 
 class GamekingsIE(InfoExtractor):
-    _VALID_URL = r'http://www\.gamekings\.tv/(?:videos|nieuws)/(?P<id>[^/]+)'
+    _VALID_URL = r'http://www\.gamekings\.nl/(?:videos|nieuws)/(?P<id>[^/]+)'
     _TESTS = [{
-        'url': 'http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/',
+        'url': 'http://www.gamekings.nl/videos/phoenix-wright-ace-attorney-dual-destinies-review/',
         # MD5 is flaky, seems to change regularly
         # 'md5': '2f32b1f7b80fdc5cb616efb4f387f8a3',
         'info_dict': {
@@ -23,7 +23,7 @@ class GamekingsIE(InfoExtractor):
         },
     }, {
         # vimeo video
-        'url': 'http://www.gamekings.tv/videos/the-legend-of-zelda-majoras-mask/',
+        'url': 'http://www.gamekings.nl/videos/the-legend-of-zelda-majoras-mask/',
         'md5': '12bf04dfd238e70058046937657ea68d',
         'info_dict': {
             'id': 'the-legend-of-zelda-majoras-mask',
@@ -33,7 +33,7 @@ class GamekingsIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.jpg$',
         },
     }, {
-        'url': 'http://www.gamekings.tv/nieuws/gamekings-extra-shelly-en-david-bereiden-zich-voor-op-de-livestream/',
+        'url': 'http://www.gamekings.nl/nieuws/gamekings-extra-shelly-en-david-bereiden-zich-voor-op-de-livestream/',
         'only_matching': True,
     }]
 

From 7b7507d6e166bcbad4d42b7fac7608b2db276e3f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 07:13:21 +0800
Subject: [PATCH 0530/1105] [letv] Fix LetvCloud extraction

---
 youtube_dl/extractor/letv.py | 61 ++++++++++++++++++++++++++++--------
 1 file changed, 48 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 08bdae8a2..9665ece89 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -5,11 +5,13 @@ import datetime
 import re
 import time
 import base64
+import hashlib
 
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
     compat_ord,
+    compat_str,
 )
 from ..utils import (
     determine_ext,
@@ -258,6 +260,7 @@ class LetvCloudIE(InfoExtractor):
         },
     }, {
         'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=ec93197892&pu=2c7cd40209&auto_play=1&gpcflag=1&width=640&height=360',
+        'md5': 'e03d9cc8d9c13191e1caf277e42dbd31',
         'info_dict': {
             'id': 'p7jnfw5hw9_ec93197892',
             'ext': 'mp4',
@@ -265,6 +268,7 @@ class LetvCloudIE(InfoExtractor):
         },
     }, {
         'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=187060b6fd',
+        'md5': 'cb988699a776b22d4a41b9d43acfb3ac',
         'info_dict': {
             'id': 'p7jnfw5hw9_187060b6fd',
             'ext': 'mp4',
@@ -272,21 +276,37 @@ class LetvCloudIE(InfoExtractor):
         },
     }]
 
-    def _real_extract(self, url):
-        uu_mobj = re.search('uu=([\w]+)', url)
-        vu_mobj = re.search('vu=([\w]+)', url)
+    @staticmethod
+    def sign_data(obj):
+        if obj['cf'] == 'flash':
+            salt = '2f9d6924b33a165a6d8b5d3d42f4f987'
+            items = ['cf', 'format', 'ran', 'uu', 'ver', 'vu']
+        elif obj['cf'] == 'html5':
+            salt = 'fbeh5player12c43eccf2bec3300344'
+            items = ['cf', 'ran', 'uu', 'bver', 'vu']
+        input_data = ''.join([item + obj[item] for item in items]) + salt
+        obj['sign'] = hashlib.md5(input_data.encode('utf-8')).hexdigest()
 
-        if not uu_mobj or not vu_mobj:
-            raise ExtractorError('Invalid URL: %s' % url, expected=True)
+    def _get_formats(self, cf, uu, vu, media_id):
+        def get_play_json(cf, timestamp):
+            data = {
+                'cf': cf,
+                'ver': '2.2',
+                'bver': 'firefox44.0',
+                'format': 'json',
+                'uu': uu,
+                'vu': vu,
+                'ran': compat_str(timestamp),
+            }
+            self.sign_data(data)
+            return self._download_json(
+                'http://api.letvcloud.com/gpc.php?' + compat_urllib_parse.urlencode(data),
+                media_id, 'Downloading playJson data for type %s' % cf)
 
-        uu = uu_mobj.group(1)
-        vu = vu_mobj.group(1)
-        media_id = uu + '_' + vu
-
-        play_json_req = sanitized_Request(
-            'http://api.letvcloud.com/gpc.php?cf=html5&sign=signxxxxx&ver=2.2&format=json&' +
-            'uu=' + uu + '&vu=' + vu)
-        play_json = self._download_json(play_json_req, media_id, 'Downloading playJson data')
+        play_json = get_play_json(cf, time.time())
+        # The server time may be different from local time
+        if play_json.get('code') == 10071:
+            play_json = get_play_json(cf, play_json['timestamp'])
 
         if not play_json.get('data'):
             if play_json.get('message'):
@@ -312,6 +332,21 @@ class LetvCloudIE(InfoExtractor):
                 'width': int_or_none(play_url.get('vwidth')),
                 'height': int_or_none(play_url.get('vheight')),
             })
+
+        return formats
+
+    def _real_extract(self, url):
+        uu_mobj = re.search('uu=([\w]+)', url)
+        vu_mobj = re.search('vu=([\w]+)', url)
+
+        if not uu_mobj or not vu_mobj:
+            raise ExtractorError('Invalid URL: %s' % url, expected=True)
+
+        uu = uu_mobj.group(1)
+        vu = vu_mobj.group(1)
+        media_id = uu + '_' + vu
+
+        formats = self._get_formats('flash', uu, vu, media_id) + self._get_formats('html5', uu, vu, media_id)
         self._sort_formats(formats)
 
         return {

From ce5879fa1475d9b0a74bd4e644ea944083177b2d Mon Sep 17 00:00:00 2001
From: rrooij <rderooij685@gmail.com>
Date: Sun, 31 Jan 2016 00:12:45 +0100
Subject: [PATCH 0531/1105] [Gamekings] Fix viewing of old videos

Some old videos that aren't on Vimeo are being uploaded to YouTube under the
'Gamekings Vault' channel. They use YouTube now for some videos as video
hosting instead of Vimeo or their own hosting. The first test failed to
succeed under the existing code, but works now by using the YouTube
extractor.

The Regex is changed to find the new gogoVideo JavaScript line with the
YouTube embed. Checking if there is a YouTube embed is done by a String
find, which is probably not the best method of checking this.
---
 youtube_dl/extractor/gamekings.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index e7747a3ae..efe31c10d 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -15,11 +15,14 @@ class GamekingsIE(InfoExtractor):
         # MD5 is flaky, seems to change regularly
         # 'md5': '2f32b1f7b80fdc5cb616efb4f387f8a3',
         'info_dict': {
-            'id': 'phoenix-wright-ace-attorney-dual-destinies-review',
+            'id': 'HkSQKetlGOU',
             'ext': 'mp4',
-            'title': 'Phoenix Wright: Ace Attorney \u2013 Dual Destinies Review',
-            'description': 'md5:36fd701e57e8c15ac8682a2374c99731',
+            'title': 'Phoenix Wright: Ace Attorney - Dual Destinies Review',
+            'description': 'md5:db88c0e7f47e9ea50df3271b9dc72e1d',
             'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader_id': 'UCJugRGo4STYMeFr5RoOShtQ',
+            'uploader': 'Gamekings Vault',
+            'upload_date': '20151123',
         },
     }, {
         # vimeo video
@@ -43,7 +46,11 @@ class GamekingsIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         playlist_id = self._search_regex(
-            r'gogoVideo\(\s*\d+\s*,\s*"([^"]+)', webpage, 'playlist id')
+            r'gogoVideo\(.*,\s*"([^"]+)', webpage, 'playlist id')
+
+        # Check if a YouTube embed is used
+        if playlist_id.find('youtube') != -1:
+            return self.url_result(playlist_id, ie='Youtube')
 
         playlist = self._download_xml(
             'http://www.gamekings.tv/wp-content/themes/gk2010/rss_playlist.php?id=%s' % playlist_id,

From 8e7aad20756efffa649cf073d89f22a57f93048c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 Jan 2016 17:49:59 +0600
Subject: [PATCH 0532/1105] [youtube] Use authentication for entry list base
 extractor (Closes #8380)

---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a24c73584..2941da467 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -181,7 +181,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             return
 
 
-class YoutubeEntryListBaseInfoExtractor(InfoExtractor):
+class YoutubeEntryListBaseInfoExtractor(YoutubeBaseInfoExtractor):
     # Extract entries from page with "Load more" button
     def _entries(self, page, playlist_id):
         more_widget_html = content_html = page
@@ -1602,7 +1602,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         }
 
 
-class YoutubePlaylistIE(YoutubeBaseInfoExtractor, YoutubePlaylistBaseInfoExtractor):
+class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
     IE_DESC = 'YouTube.com playlists'
     _VALID_URL = r"""(?x)(?:
                         (?:https?://)?

From 9acd33094d36acde60a42837ed775c68ad3a327d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 Jan 2016 17:52:02 +0600
Subject: [PATCH 0533/1105] [youtube] Filter duplicates in playlists base
 extractor

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2941da467..9a64c1d11 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -233,7 +233,7 @@ class YoutubePlaylistBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
 
 class YoutubePlaylistsBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
     def _process_page(self, content):
-        for playlist_id in re.findall(r'href="/?playlist\?list=(.+?)"', content):
+        for playlist_id in set(re.findall(r'href="/?playlist\?list=([0-9A-Za-z-_]{10,})"', content)):
             yield self.url_result(
                 'https://www.youtube.com/playlist?list=%s' % playlist_id, 'YoutubePlaylist')
 

From a69bee4762044b657e1fd8bd4cf0baa58d950bc3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 31 Jan 2016 12:57:18 +0100
Subject: [PATCH 0534/1105] release 2016.01.31

---
 README.md              | 4 ++++
 docs/supportedsites.md | 3 +++
 youtube_dl/version.py  | 2 +-
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 724fb17d1..7c582511f 100644
--- a/README.md
+++ b/README.md
@@ -173,6 +173,10 @@ which means you can modify it, redistribute it or use it however you like.
                                      expected filesize (experimental)
     --hls-prefer-native              Use the native HLS downloader instead of
                                      ffmpeg (experimental)
+    --hls-use-mpegts                 Use the mpegts container for HLS videos,
+                                     allowing to play the video while
+                                     downloading (some players may not be able
+                                     to play it)
     --external-downloader COMMAND    Use the specified external downloader.
                                      Currently supports
                                      aria2c,axel,curl,httpie,wget
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index eb68c23b5..0644436a8 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -55,6 +55,7 @@
  - **audiomack**
  - **audiomack:album**
  - **Azubu**
+ - **AzubuLive**
  - **BaiduVideo**: 百度视频
  - **bambuser**
  - **bambuser:channel**
@@ -315,6 +316,7 @@
  - **mailru**: Видео@Mail.Ru
  - **MakerTV**
  - **Malemotion**
+ - **MatchTV**
  - **MDR**: MDR.DE and KiKA
  - **media.ccc.de**
  - **metacafe**
@@ -507,6 +509,7 @@
  - **Sapo**: SAPO Vídeos
  - **savefrom.net**
  - **SBS**: sbs.com.au
+ - **schooltv**
  - **SciVee**
  - **screen.yahoo:search**: Yahoo screen search
  - **Screencast**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d9f1e22b0..006b960b3 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.01.29'
+__version__ = '2016.01.31'

From a7aab0c23e537d9df3d0946c0b960cc92e35d2ff Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 19:49:54 +0800
Subject: [PATCH 0535/1105] [test_youtube_lists] Fix
 TestYoutubeLists.test_youtube_course

Youtube entries are now generators
---
 test/test_youtube_lists.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index 26aadb34f..47df0f348 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -34,7 +34,7 @@ class TestYoutubeLists(unittest.TestCase):
         ie = YoutubePlaylistIE(dl)
         # TODO find a > 100 (paginating?) videos course
         result = ie.extract('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
-        entries = result['entries']
+        entries = list(result['entries'])
         self.assertEqual(YoutubeIE().extract_id(entries[0]['url']), 'j9WZyLZCBzs')
         self.assertEqual(len(entries), 25)
         self.assertEqual(YoutubeIE().extract_id(entries[-1]['url']), 'rYefUsYuEp0')

From 189d72d5fd16176a8990a9447c0165478a384676 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 19:53:22 +0800
Subject: [PATCH 0536/1105] [test_subtitles] Fix TestRaiSubtitles

RaiIE is renamed to RaiTVIE in 06d5556dface3901a86419b6b125ef377116448f
---
 test/test_subtitles.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 9ed9fe622..9a695c4e8 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -21,7 +21,7 @@ from youtube_dl.extractor import (
     NPOIE,
     ComedyCentralIE,
     NRKTVIE,
-    RaiIE,
+    RaiTVIE,
     VikiIE,
     ThePlatformIE,
     ThePlatformFeedIE,
@@ -260,7 +260,7 @@ class TestNRKSubtitles(BaseTestSubtitles):
 
 class TestRaiSubtitles(BaseTestSubtitles):
     url = 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-cb27157f-9dd0-4aee-b788-b1f67643a391.html'
-    IE = RaiIE
+    IE = RaiTVIE
 
     def test_allsubtitles(self):
         self.DL.params['writesubtitles'] = True

From b99d88c6a1d91f57e1804bd512fed1bdbf31a384 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 20:00:07 +0800
Subject: [PATCH 0537/1105] [youporn] Fix uploader and description

---
 youtube_dl/extractor/youporn.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index dd724085a..b29baafc4 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -114,15 +114,13 @@ class YouPornIE(InfoExtractor):
             formats.append(f)
         self._sort_formats(formats)
 
-        description = self._html_search_regex(
-            r'(?s)<div[^>]+class=["\']video-description["\'][^>]*>(.+?)</div>',
-            webpage, 'description', default=None)
+        description = self._og_search_description(webpage, default=None)
         thumbnail = self._search_regex(
             r'(?:imageurl\s*=|poster\s*:)\s*(["\'])(?P<thumbnail>.+?)\1',
             webpage, 'thumbnail', fatal=False, group='thumbnail')
 
         uploader = self._html_search_regex(
-            r'(?s)<div[^>]+class=["\']videoInfoBy["\'][^>]*>\s*By:\s*</div>(.+?)</(?:a|div)>',
+            r'(?s)<div[^>]+class=["\']videoInfoBy(?:\s+[^"\']+)?["\'][^>]*>\s*By:\s*</div>(.+?)</(?:a|div)>',
             webpage, 'uploader', fatal=False)
         upload_date = unified_strdate(self._html_search_regex(
             r'(?s)<div[^>]+class=["\']videoInfoTime["\'][^>]*>(.+?)</div>',

From 231ea2a3bbc6324fbd0e5dddd224646c91b0f035 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 20:21:57 +0800
Subject: [PATCH 0538/1105] [xuite] Replace the test case with my uploaded one

---
 youtube_dl/extractor/xuite.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index 8bbac54e2..2466410fa 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -34,19 +34,20 @@ class XuiteIE(InfoExtractor):
         },
     }, {
         # Video with only one format
-        'url': 'http://vlog.xuite.net/play/TkRZNjhULTM0NDE2MjkuZmx2',
-        'md5': 'c45737fc8ac5dc8ac2f92ecbcecf505e',
+        'url': 'http://vlog.xuite.net/play/WUxxR2xCLTI1OTI1MDk5LmZsdg==',
+        'md5': '21f7b39c009b5a4615b4463df6eb7a46',
         'info_dict': {
-            'id': '3441629',
+            'id': '25925099',
             'ext': 'mp4',
-            'title': '孫燕姿 - 眼淚成詩',
+            'title': 'BigBuckBunny_320x180',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'duration': 217.399,
-            'timestamp': 1299383640,
-            'upload_date': '20110306',
-            'uploader': 'Valen',
-            'uploader_id': '10400126',
-            'categories': ['影視娛樂'],
+            'duration': 596.458,
+            'timestamp': 1454242500,
+            'upload_date': '20160131',
+            'uploader': 'yan12125',
+            'uploader_id': '12158353',
+            'categories': ['個人短片'],
+            'description': 'http://download.blender.org/peach/bigbuckbunny_movies/BigBuckBunny_320x180.mp4',
         },
     }, {
         # Video with two formats

From e04398e397d4a4f7e0d0994355d5a94c22441e7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 31 Jan 2016 14:22:36 +0100
Subject: [PATCH 0539/1105] [FFmpegSubtitlesConvertorPP] delete old subtitle
 files (fixes #8382)

---
 youtube_dl/postprocessor/ffmpeg.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index daca5d814..16a64802a 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -479,6 +479,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
             self._downloader.to_screen('[ffmpeg] There aren\'t any subtitles to convert')
             return [], info
         self._downloader.to_screen('[ffmpeg] Converting subtitles')
+        sub_filenames = []
         for lang, sub in subs.items():
             ext = sub['ext']
             if ext == new_ext:
@@ -486,6 +487,8 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                     '[ffmpeg] Subtitle file for %s is already in the requested'
                     'format' % new_ext)
                 continue
+            old_file = subtitles_filename(filename, lang, ext)
+            sub_filenames.append(old_file)
             new_file = subtitles_filename(filename, lang, new_ext)
 
             if ext == 'dfxp' or ext == 'ttml':
@@ -493,7 +496,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                     'You have requested to convert dfxp (TTML) subtitles into another format, '
                     'which results in style information loss')
 
-                dfxp_file = subtitles_filename(filename, lang, ext)
+                dfxp_file = old_file
                 srt_file = subtitles_filename(filename, lang, 'srt')
 
                 with io.open(dfxp_file, 'rt', encoding='utf-8') as f:
@@ -511,9 +514,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                 if new_ext == 'srt':
                     continue
 
-            self.run_ffmpeg(
-                subtitles_filename(filename, lang, ext),
-                new_file, ['-f', new_format])
+            self.run_ffmpeg(old_file, new_file, ['-f', new_format])
 
             with io.open(new_file, 'rt', encoding='utf-8') as f:
                 subs[lang] = {
@@ -521,4 +522,4 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                     'data': f.read(),
                 }
 
-        return [], info
+        return sub_filenames, info

From 3ccb0655c1f27953ba00ee3f8bc290ca9bfda7ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 31 Jan 2016 15:11:00 +0100
Subject: [PATCH 0540/1105] [youtube] Use 'orderedSet' instead of 'set' to
 preserve the order

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9a64c1d11..bd87c75b6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -233,7 +233,7 @@ class YoutubePlaylistBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
 
 class YoutubePlaylistsBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
     def _process_page(self, content):
-        for playlist_id in set(re.findall(r'href="/?playlist\?list=([0-9A-Za-z-_]{10,})"', content)):
+        for playlist_id in orderedSet(re.findall(r'href="/?playlist\?list=([0-9A-Za-z-_]{10,})"', content)):
             yield self.url_result(
                 'https://www.youtube.com/playlist?list=%s' % playlist_id, 'YoutubePlaylist')
 

From 163da6a484f57c52afdf300d368136d164316803 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 02:13:11 +0800
Subject: [PATCH 0541/1105] [gamekings] Add MD5 back

The test is now a YouTube video, whose MD5 should be stable
---
 youtube_dl/extractor/gamekings.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index efe31c10d..c011e9d22 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -12,8 +12,7 @@ class GamekingsIE(InfoExtractor):
     _VALID_URL = r'http://www\.gamekings\.nl/(?:videos|nieuws)/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://www.gamekings.nl/videos/phoenix-wright-ace-attorney-dual-destinies-review/',
-        # MD5 is flaky, seems to change regularly
-        # 'md5': '2f32b1f7b80fdc5cb616efb4f387f8a3',
+        'md5': '5208d3a17adeaef829a7861887cb9029',
         'info_dict': {
             'id': 'HkSQKetlGOU',
             'ext': 'mp4',

From eab3c2895c66a8d2f5da181d3ccba35a901b813f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 02:15:25 +0800
Subject: [PATCH 0542/1105] [gamekings] add_ie

---
 youtube_dl/extractor/gamekings.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index c011e9d22..df385e338 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -11,6 +11,7 @@ from ..utils import (
 class GamekingsIE(InfoExtractor):
     _VALID_URL = r'http://www\.gamekings\.nl/(?:videos|nieuws)/(?P<id>[^/]+)'
     _TESTS = [{
+        # YouTube embed video
         'url': 'http://www.gamekings.nl/videos/phoenix-wright-ace-attorney-dual-destinies-review/',
         'md5': '5208d3a17adeaef829a7861887cb9029',
         'info_dict': {
@@ -23,6 +24,7 @@ class GamekingsIE(InfoExtractor):
             'uploader': 'Gamekings Vault',
             'upload_date': '20151123',
         },
+        'add_ie': ['Youtube'],
     }, {
         # vimeo video
         'url': 'http://www.gamekings.nl/videos/the-legend-of-zelda-majoras-mask/',

From 0e1b1a011d1772bc1a7069bad8ad71a53798a212 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 02:19:03 +0800
Subject: [PATCH 0543/1105] [gamekings] Stricter checks

---
 youtube_dl/extractor/gamekings.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index df385e338..f6b9046f9 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -6,6 +6,7 @@ from ..utils import (
     xpath_text,
     xpath_with_ns,
 )
+from .youtube import YoutubeIE
 
 
 class GamekingsIE(InfoExtractor):
@@ -47,10 +48,10 @@ class GamekingsIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         playlist_id = self._search_regex(
-            r'gogoVideo\(.*,\s*"([^"]+)', webpage, 'playlist id')
+            r'gogoVideo\([^,]+,\s*"([^"]+)', webpage, 'playlist id')
 
         # Check if a YouTube embed is used
-        if playlist_id.find('youtube') != -1:
+        if YoutubeIE.suitable(playlist_id):
             return self.url_result(playlist_id, ie='Youtube')
 
         playlist = self._download_xml(

From a8aad210019b50540cf1bfd28390b7e7f2573a31 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 03:07:04 +0800
Subject: [PATCH 0544/1105] [acast] Fix extraction

---
 youtube_dl/extractor/acast.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/acast.py b/youtube_dl/extractor/acast.py
index be7913bc7..f2465f8f8 100644
--- a/youtube_dl/extractor/acast.py
+++ b/youtube_dl/extractor/acast.py
@@ -23,14 +23,19 @@ class ACastIE(ACastBaseIE):
             'ext': 'mp3',
             'title': '"Where Are You?": Taipei 101, Taiwan',
             'timestamp': 1196172000000,
-            'description': 'md5:0c5d8201dfea2b93218ea986c91eee6e',
+            'description': 'md5:a0b4ef3634e63866b542e5b1199a1a0e',
             'duration': 211,
         }
     }
 
     def _real_extract(self, url):
         channel, display_id = re.match(self._VALID_URL, url).groups()
-        cast_data = self._download_json(self._API_BASE_URL + 'channels/%s/acasts/%s/playback' % (channel, display_id), display_id)
+
+        embed_page = self._download_webpage(
+            re.sub('(?:www\.)?acast\.com', 'embedcdn.acast.com', url), display_id)
+        cast_data = self._parse_json(self._search_regex(
+            r'window\[\'acast/queries\'\]\s*=\s*([^;]+);', embed_page, 'acast data'),
+            display_id)['GetAcast/%s/%s' % (channel, display_id)]
 
         return {
             'id': compat_str(cast_data['id']),

From 9934fe76be616cf468038a6b04771b3fcc514765 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 03:08:46 +0800
Subject: [PATCH 0545/1105] [acast] Remove ACastBaseIE

No longer necessary as _API_BASE_URL is used by ACastChannelIE only
---
 youtube_dl/extractor/acast.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/acast.py b/youtube_dl/extractor/acast.py
index f2465f8f8..92eee8119 100644
--- a/youtube_dl/extractor/acast.py
+++ b/youtube_dl/extractor/acast.py
@@ -8,11 +8,7 @@ from ..compat import compat_str
 from ..utils import int_or_none
 
 
-class ACastBaseIE(InfoExtractor):
-    _API_BASE_URL = 'https://www.acast.com/api/'
-
-
-class ACastIE(ACastBaseIE):
+class ACastIE(InfoExtractor):
     IE_NAME = 'acast'
     _VALID_URL = r'https?://(?:www\.)?acast\.com/(?P<channel>[^/]+)/(?P<id>[^/#?]+)'
     _TEST = {
@@ -49,7 +45,7 @@ class ACastIE(ACastBaseIE):
         }
 
 
-class ACastChannelIE(ACastBaseIE):
+class ACastChannelIE(InfoExtractor):
     IE_NAME = 'acast:channel'
     _VALID_URL = r'https?://(?:www\.)?acast\.com/(?P<id>[^/#?]+)'
     _TEST = {
@@ -61,6 +57,7 @@ class ACastChannelIE(ACastBaseIE):
         },
         'playlist_mincount': 20,
     }
+    _API_BASE_URL = 'https://www.acast.com/api/'
 
     @classmethod
     def suitable(cls, url):

From 7a0ed06909f8951f8be3049ab089d61ead626158 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 03:31:58 +0800
Subject: [PATCH 0546/1105] [allocine] Fix extraction of test_allocine_1 and
 update tests

---
 youtube_dl/extractor/allocine.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/allocine.py b/youtube_dl/extractor/allocine.py
index 7d65b8193..f94da1a05 100644
--- a/youtube_dl/extractor/allocine.py
+++ b/youtube_dl/extractor/allocine.py
@@ -8,6 +8,7 @@ from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
     qualities,
+    unescapeHTML,
 )
 
 
@@ -31,7 +32,7 @@ class AllocineIE(InfoExtractor):
             'id': '19540403',
             'ext': 'mp4',
             'title': 'Planes 2 Bande-annonce VF',
-            'description': 'md5:eeaffe7c2d634525e21159b93acf3b1e',
+            'description': 'Regardez la bande annonce du film Planes 2 (Planes 2 Bande-annonce VF). Planes 2, un film de Roberts Gannaway',
             'thumbnail': 're:http://.*\.jpg',
         },
     }, {
@@ -41,7 +42,7 @@ class AllocineIE(InfoExtractor):
             'id': '19544709',
             'ext': 'mp4',
             'title': 'Dragons 2 - Bande annonce finale VF',
-            'description': 'md5:71742e3a74b0d692c7fce0dd2017a4ac',
+            'description': 'md5:601d15393ac40f249648ef000720e7e3',
             'thumbnail': 're:http://.*\.jpg',
         },
     }, {
@@ -59,10 +60,14 @@ class AllocineIE(InfoExtractor):
         if typ == 'film':
             video_id = self._search_regex(r'href="/video/player_gen_cmedia=([0-9]+).+"', webpage, 'video id')
         else:
-            player = self._search_regex(r'data-player=\'([^\']+)\'>', webpage, 'data player')
-
-            player_data = json.loads(player)
-            video_id = compat_str(player_data['refMedia'])
+            player = self._search_regex(r'data-player=\'([^\']+)\'>', webpage, 'data player', default=None)
+            if player:
+                player_data = json.loads(player)
+                video_id = compat_str(player_data['refMedia'])
+            else:
+                model = self._search_regex(r'data-model="([^"]+)">', webpage, 'data model')
+                model_data = self._parse_json(unescapeHTML(model), display_id)
+                video_id = compat_str(model_data['id'])
 
         xml = self._download_xml('http://www.allocine.fr/ws/AcVisiondataV4.ashx?media=%s' % video_id, display_id)
 

From f63757ec35bb310d05966de03dcfae72b427eac3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 03:34:02 +0800
Subject: [PATCH 0547/1105] [allocine] Fix for Python 2.6

Python 2.6 does not support .// syntax in find(). Fortunately, the
interested node is at the top level
---
 youtube_dl/extractor/allocine.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/allocine.py b/youtube_dl/extractor/allocine.py
index f94da1a05..f6cd3a85f 100644
--- a/youtube_dl/extractor/allocine.py
+++ b/youtube_dl/extractor/allocine.py
@@ -71,7 +71,7 @@ class AllocineIE(InfoExtractor):
 
         xml = self._download_xml('http://www.allocine.fr/ws/AcVisiondataV4.ashx?media=%s' % video_id, display_id)
 
-        video = xml.find('.//AcVisionVideo').attrib
+        video = xml.find('./AcVisionVideo').attrib
         quality = qualities(['ld', 'md', 'hd'])
 
         formats = []

From 566bda51f2d943d58fbe37d88ea0fceb8ddefea8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 05:00:09 +0800
Subject: [PATCH 0548/1105] [bpb] Fix extraction and update tests

---
 youtube_dl/extractor/bpb.py | 30 +++++++++++++++++++++++++-----
 1 file changed, 25 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/bpb.py b/youtube_dl/extractor/bpb.py
index 510813f76..c28e72927 100644
--- a/youtube_dl/extractor/bpb.py
+++ b/youtube_dl/extractor/bpb.py
@@ -1,7 +1,13 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..utils import (
+    js_to_json,
+    determine_ext,
+)
 
 
 class BpbIE(InfoExtractor):
@@ -10,7 +16,8 @@ class BpbIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://www.bpb.de/mediathek/297/joachim-gauck-zu-1989-und-die-erinnerung-an-die-ddr',
-        'md5': '0792086e8e2bfbac9cdf27835d5f2093',
+        # md5 fails in Python 2.6 due to buggy server response and wrong handling of urllib2
+        'md5': 'c4f84c8a8044ca9ff68bb8441d300b3f',
         'info_dict': {
             'id': '297',
             'ext': 'mp4',
@@ -25,13 +32,26 @@ class BpbIE(InfoExtractor):
 
         title = self._html_search_regex(
             r'<h2 class="white">(.*?)</h2>', webpage, 'title')
-        video_url = self._html_search_regex(
-            r'(http://film\.bpb\.de/player/dokument_[0-9]+\.mp4)',
-            webpage, 'video URL')
+        video_info_dicts = re.findall(
+            r"({\s*src:\s*'http://film\.bpb\.de/[^}]+})", webpage)
+
+        formats = []
+        for video_info in video_info_dicts:
+            video_info = self._parse_json(video_info, video_id, transform_source=js_to_json)
+            quality = video_info['quality']
+            video_url = video_info['src']
+            formats.append({
+                'url': video_url,
+                'preference': 10 if quality == 'high' else 0,
+                'format_note': quality,
+                'format_id': '%s-%s' % (quality, determine_ext(video_url)),
+            })
+
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'url': video_url,
+            'formats': formats,
             'title': title,
             'description': self._og_search_description(webpage),
         }

From dc4fe5c6d713ac09cd35f8b8dbfbe46a67a67f10 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 05:32:28 +0800
Subject: [PATCH 0549/1105] [allocine] Use xpath_element

---
 youtube_dl/extractor/allocine.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/allocine.py b/youtube_dl/extractor/allocine.py
index f6cd3a85f..190bc2cc8 100644
--- a/youtube_dl/extractor/allocine.py
+++ b/youtube_dl/extractor/allocine.py
@@ -9,6 +9,7 @@ from ..compat import compat_str
 from ..utils import (
     qualities,
     unescapeHTML,
+    xpath_element,
 )
 
 
@@ -71,7 +72,7 @@ class AllocineIE(InfoExtractor):
 
         xml = self._download_xml('http://www.allocine.fr/ws/AcVisiondataV4.ashx?media=%s' % video_id, display_id)
 
-        video = xml.find('./AcVisionVideo').attrib
+        video = xpath_element(xml, './/AcVisionVideo').attrib
         quality = qualities(['ld', 'md', 'hd'])
 
         formats = []

From 92769650fae8827f242a1517565cceef5604eeb0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 15:40:42 +0800
Subject: [PATCH 0550/1105] [vidzi] Fix extraction

Closes #8386.

Vidzi.tv now uses jwplayer, which can be handled by GenericIE
---
 youtube_dl/extractor/vidzi.py | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index 2ba9f31df..210a738a6 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import smuggle_url
 
 
 class VidziIE(InfoExtractor):
@@ -20,19 +21,14 @@ class VidziIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
-        video_host = self._html_search_regex(
-            r'id=\'vplayer\'><img src="http://(.*?)/i', webpage,
-            'video host')
-        video_hash = self._html_search_regex(
-            r'\|([a-z0-9]+)\|hls\|type', webpage, 'video_hash')
-        ext = self._html_search_regex(
-            r'\|tracks\|([a-z0-9]+)\|', webpage, 'video ext')
-        video_url = 'http://' + video_host + '/' + video_hash + '/v.' + ext
         title = self._html_search_regex(
             r'(?s)<h2 class="video-title">(.*?)</h2>', webpage, 'title')
 
+        # Vidzi now uses jwplayer, which can be handled by GenericIE
         return {
+            '_type': 'url_transparent',
             'id': video_id,
             'title': title,
-            'url': video_url,
+            'url': smuggle_url(url, {'to_generic': True}),
+            'ie_key': 'Generic',
         }

From bd93a12e852f34a1c9d8c86023972c6812dc80d5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 17:03:31 +0800
Subject: [PATCH 0551/1105] [vidzi] Fix _TESTS

---
 youtube_dl/extractor/vidzi.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index 210a738a6..7c6e98026 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -14,6 +14,11 @@ class VidziIE(InfoExtractor):
             'id': 'cghql9yq6emu',
             'ext': 'mp4',
             'title': 'youtube-dl test video  1\\\\2\'3/4<5\\\\6ä7↭',
+            'uploader': 'vidzi.tv',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
         },
     }
 

From b286f201a8424e00cc91d27adcf53a5249b8a3b9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 17:05:48 +0800
Subject: [PATCH 0552/1105] [YoutubeDL] Do not override ie_key in
 url_transparent

---
 test/test_YoutubeDL.py  | 37 +++++++++++++++++++++++++++++++++++++
 youtube_dl/YoutubeDL.py |  3 +--
 2 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index b53cfbe78..73910eaec 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -14,6 +14,7 @@ from test.helper import FakeYDL, assertRegexpMatches
 from youtube_dl import YoutubeDL
 from youtube_dl.compat import compat_str, compat_urllib_error
 from youtube_dl.extractor import YoutubeIE
+from youtube_dl.extractor.common import InfoExtractor
 from youtube_dl.postprocessor.common import PostProcessor
 from youtube_dl.utils import ExtractorError, match_filter_func
 
@@ -646,6 +647,42 @@ class TestYoutubeDL(unittest.TestCase):
         ydl = YDL()
         self.assertRaises(compat_urllib_error.URLError, ydl.urlopen, 'file:///etc/passwd')
 
+    def test_do_not_override_ie_key_in_url_transparent(self):
+        ydl = YDL()
+
+        class Foo1IE(InfoExtractor):
+            _VALID_URL = r'foo1:'
+
+            def _real_extract(self, url):
+                return {
+                    '_type': 'url_transparent',
+                    'url': 'foo2:',
+                    'ie_key': 'Foo2',
+                }
+
+        class Foo2IE(InfoExtractor):
+            _VALID_URL = r'foo2:'
+
+            def _real_extract(self, url):
+                return {
+                    '_type': 'url',
+                    'url': 'foo3:',
+                    'ie_key': 'Foo3',
+                }
+
+        class Foo3IE(InfoExtractor):
+            _VALID_URL = r'foo3:'
+
+            def _real_extract(self, url):
+                return _make_result([{'url': TEST_URL}])
+
+        ydl.add_info_extractor(Foo1IE(ydl))
+        ydl.add_info_extractor(Foo2IE(ydl))
+        ydl.add_info_extractor(Foo3IE(ydl))
+        ydl.extract_info('foo1:')
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['url'], TEST_URL)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 2a3d6cd4a..50228bb32 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -707,7 +707,6 @@ class YoutubeDL(object):
         It will also download the videos if 'download'.
         Returns the resolved ie_result.
         """
-
         result_type = ie_result.get('_type', 'video')
 
         if result_type in ('url', 'url_transparent'):
@@ -736,7 +735,7 @@ class YoutubeDL(object):
 
             force_properties = dict(
                 (k, v) for k, v in ie_result.items() if v is not None)
-            for f in ('_type', 'url'):
+            for f in ('_type', 'url', 'ie_key'):
                 if f in force_properties:
                     del force_properties[f]
             new_result = info.copy()

From abe694ca95fe3344db6b3ebcf483911203fcdc3f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 17:10:11 +0800
Subject: [PATCH 0553/1105] [kickstarter] Eliminate the warning message and
 add_ie

---
 youtube_dl/extractor/kickstarter.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/kickstarter.py b/youtube_dl/extractor/kickstarter.py
index 1d391e69f..e703fc597 100644
--- a/youtube_dl/extractor/kickstarter.py
+++ b/youtube_dl/extractor/kickstarter.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import smuggle_url
 
 
 class KickStarterIE(InfoExtractor):
@@ -27,7 +28,8 @@ class KickStarterIE(InfoExtractor):
             'uploader_id': 'pebble',
             'uploader': 'Pebble Technology',
             'title': 'Pebble iOS Notifications',
-        }
+        },
+        'add_ie': ['Vimeo'],
     }, {
         'url': 'https://www.kickstarter.com/projects/1420158244/power-drive-2000/widget/video.html',
         'info_dict': {
@@ -52,7 +54,7 @@ class KickStarterIE(InfoExtractor):
             return {
                 '_type': 'url_transparent',
                 'ie_key': 'Generic',
-                'url': url,
+                'url': smuggle_url(url, {'to_generic': True}),
                 'title': title,
             }
 

From f021acee496b0bd91ceb5e7a7e115c54c2f99f5b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 17:21:40 +0800
Subject: [PATCH 0554/1105] [kickstarter] Fix title and test_kickstarter

It's the description page that contains a video. The original URL is now
the calendar.
---
 youtube_dl/extractor/kickstarter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/kickstarter.py b/youtube_dl/extractor/kickstarter.py
index e703fc597..9f1ade2e4 100644
--- a/youtube_dl/extractor/kickstarter.py
+++ b/youtube_dl/extractor/kickstarter.py
@@ -8,7 +8,7 @@ from ..utils import smuggle_url
 class KickStarterIE(InfoExtractor):
     _VALID_URL = r'https?://www\.kickstarter\.com/projects/(?P<id>[^/]*)/.*'
     _TESTS = [{
-        'url': 'https://www.kickstarter.com/projects/1404461844/intersection-the-story-of-josh-grant?ref=home_location',
+        'url': 'https://www.kickstarter.com/projects/1404461844/intersection-the-story-of-josh-grant/description',
         'md5': 'c81addca81327ffa66c642b5d8b08cab',
         'info_dict': {
             'id': '1404461844',
@@ -45,7 +45,7 @@ class KickStarterIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_regex(
-            r'<title>\s*(.*?)(?:\s*&mdash; Kickstarter)?\s*</title>',
+            r'<title>\s*(.*?)(?:\s*&mdash;\s*Kickstarter)?\s*</title>',
             webpage, 'title')
         video_url = self._search_regex(
             r'data-video-url="(.*?)"',

From 7537b35fb80061287f6901bcdb6b3f6aae558036 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 17:40:35 +0800
Subject: [PATCH 0555/1105] [daum] PEP8

---
 youtube_dl/extractor/daum.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index cf158c42d..67c4c7c6b 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -162,8 +162,8 @@ class DaumListIE(InfoExtractor):
         for pagenum in itertools.count(1):
             list_info = self._download_json(
                 'http://tvpot.daum.net/mypot/json/GetClipInfo.do?size=48&init=true&order=date&page=%d&%s=%s' % (
-                    pagenum, list_id_type, list_id), list_id,'Downloading list info - %s' % pagenum)
-            
+                    pagenum, list_id_type, list_id), list_id, 'Downloading list info - %s' % pagenum)
+
             entries.extend([
                 self.url_result(
                     'http://tvpot.daum.net/v/%s' % clip['vid'])

From 11a4efc505fdb6456b00e848609523bb7b8e344c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 18:15:53 +0800
Subject: [PATCH 0556/1105] [daum] Do not match a single URL with multiple info
 extractors

---
 youtube_dl/extractor/daum.py | 47 +++++++++++++++++++++++++++---------
 1 file changed, 36 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index 67c4c7c6b..5a7565e96 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -7,8 +7,10 @@ import itertools
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_parse_qs,
     compat_urllib_parse,
     compat_urllib_parse_unquote,
+    compat_urlparse,
 )
 from ..utils import (
     int_or_none,
@@ -117,6 +119,7 @@ class DaumIE(InfoExtractor):
 class DaumClipIE(InfoExtractor):
     _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/(?:clip/ClipView.(?:do|tv)|mypot/View.do)\?.*?clipid=(?P<id>\d+)'
     IE_NAME = 'daum.net:clip'
+    _URL_TEMPLATE = 'http://tvpot.daum.net/clip/ClipView.do?clipid=%s'
 
     _TESTS = [{
         'url': 'http://tvpot.daum.net/clip/ClipView.do?clipid=52554690',
@@ -135,6 +138,10 @@ class DaumClipIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @classmethod
+    def suitable(cls, url):
+        return False if DaumPlaylistIE.suitable(url) or DaumUserIE.suitable(url) else super(DaumClipIE, cls).suitable(url)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         clip_info = self._download_json(
@@ -179,10 +186,21 @@ class DaumListIE(InfoExtractor):
 
         return name, entries
 
+    def _check_clip(self, url, list_id):
+        query_dict = compat_parse_qs(compat_urlparse.urlparse(url).query)
+        if 'clipid' in query_dict:
+            clip_id = query_dict['clipid'][0]
+            if self._downloader.params.get('noplaylist'):
+                self.to_screen('Downloading just video %s because of --no-playlist' % clip_id)
+                return self.url_result(DaumClipIE._URL_TEMPLATE % clip_id, 'DaumClip')
+            else:
+                self.to_screen('Downloading playlist %s - add --no-playlist to just download video' % list_id)
+
 
 class DaumPlaylistIE(DaumListIE):
     _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/mypot/(?:View\.do|Top\.tv)\?.*?playlistid=(?P<id>[0-9]+)'
     IE_NAME = 'daum.net:playlist'
+    _URL_TEMPLATE = 'http://tvpot.daum.net/mypot/View.do?playlistid=%s'
 
     _TESTS = [{
         'note': 'Playlist url with clipid',
@@ -207,12 +225,16 @@ class DaumPlaylistIE(DaumListIE):
         }
     }]
 
-    def _real_extract(self, url):
-        if DaumClipIE.suitable(url) and self._downloader.params.get('noplaylist'):
-            return self.url_result(url, 'DaumClip')
+    @classmethod
+    def suitable(cls, url):
+        return False if DaumUserIE.suitable(url) else super(DaumPlaylistIE, cls).suitable(url)
 
+    def _real_extract(self, url):
         list_id = self._match_id(url)
-        self.to_screen('Downloading playlist %s - add --no-playlist to just download video' % list_id)
+
+        clip_result = self._check_clip(url, list_id)
+        if clip_result:
+            return clip_result
 
         name, entries = self._get_entries(list_id, 'playlistid')
 
@@ -257,14 +279,17 @@ class DaumUserIE(DaumListIE):
     }]
 
     def _real_extract(self, url):
-        if DaumClipIE.suitable(url) and self._downloader.params.get('noplaylist'):
-            return self.url_result(url, 'DaumClip')
-
-        if DaumPlaylistIE.suitable(url):
-            return self.url_result(url, 'DaumPlaylist')
-
         list_id = self._match_id(url)
-        self.to_screen('Downloading playlist %s - add --no-playlist to just download video' % list_id)
+
+        clip_result = self._check_clip(url, list_id)
+        if clip_result:
+            return clip_result
+
+        query_dict = compat_parse_qs(compat_urlparse.urlparse(url).query)
+        if 'playlistid' in query_dict:
+            playlist_id = query_dict['playlistid'][0]
+            return self.url_result(DaumPlaylistIE._URL_TEMPLATE % playlist_id, 'DaumPlaylist')
+
         name, entries = self._get_entries(list_id, 'ownerid')
 
         return self.playlist_result(entries, list_id, name)

From 58250eff2b1cd8968d17d88c8e8fbefdfd9b0a68 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 18:19:02 +0800
Subject: [PATCH 0557/1105] [daum] Update test_daum_1

---
 youtube_dl/extractor/daum.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index 5a7565e96..9fce845e1 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -42,7 +42,7 @@ class DaumIE(InfoExtractor):
         'info_dict': {
             'id': '65139429',
             'ext': 'mp4',
-            'title': 'md5:a100d65d09cec246d8aa9bde7de45aed',
+            'title': '1297회, \'아빠 아들로 태어나길 잘 했어\' 민수, 감동의 눈물[아빠 어디가] 20150118',
             'description': 'md5:79794514261164ff27e36a21ad229fc5',
             'upload_date': '20150604',
             'thumbnail': 're:^https?://.*\.(?:jpg|png)',

From dc724e0c8bceee1eda28513d5e9531bc926d9a3b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 18:26:23 +0800
Subject: [PATCH 0558/1105] [daum.net:user] Match more URLs (#1952)

---
 youtube_dl/extractor/daum.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index 9fce845e1..c84c51058 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -242,7 +242,7 @@ class DaumPlaylistIE(DaumListIE):
 
 
 class DaumUserIE(DaumListIE):
-    _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/mypot/(?:View|Top)\.do\?.*?ownerid=(?P<id>[0-9a-zA-Z]+)'
+    _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/mypot/(?:View|Top)\.(?:do|tv)\?.*?ownerid=(?P<id>[0-9a-zA-Z]+)'
     IE_NAME = 'daum.net:user'
 
     _TESTS = [{
@@ -276,6 +276,9 @@ class DaumUserIE(DaumListIE):
     }, {
         'url': 'http://tvpot.daum.net/mypot/Top.do?ownerid=o2scDLIVbHc0',
         'only_matching': True,
+    }, {
+        'url': 'http://m.tvpot.daum.net/mypot/Top.tv?ownerid=45x1okb1If50&playlistid=3569733',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From ae0db349c16a205ec571c388239999873be4d7ba Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 1 Feb 2016 12:00:32 +0100
Subject: [PATCH 0559/1105] release 2016.02.01

---
 docs/supportedsites.md | 2 ++
 youtube_dl/version.py  | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 0644436a8..61be9990d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -134,6 +134,8 @@
  - **DailymotionCloud**
  - **daum.net**
  - **daum.net:clip**
+ - **daum.net:playlist**
+ - **daum.net:user**
  - **DBTV**
  - **DCN**
  - **dcn:live**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 006b960b3..6da42c5a5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.01.31'
+__version__ = '2016.02.01'

From 0436157b95ddd6d7c415549ba48b7c0305b2c0df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Feb 2016 00:52:37 +0600
Subject: [PATCH 0560/1105] [vk:uservideos] Improve _VALID_URL (Closes #8389)

---
 youtube_dl/extractor/vk.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 90557fa61..11014865a 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -321,7 +321,7 @@ class VKIE(InfoExtractor):
 class VKUserVideosIE(InfoExtractor):
     IE_NAME = 'vk:uservideos'
     IE_DESC = "VK - User's Videos"
-    _VALID_URL = r'https?://vk\.com/videos(?P<id>-?[0-9]+)$'
+    _VALID_URL = r'https?://vk\.com/videos(?P<id>-?[0-9]+)(?!\?.*\bz=video)(?:[/?#&]|$)'
     _TEMPLATE_URL = 'https://vk.com/videos'
     _TESTS = [{
         'url': 'http://vk.com/videos205387401',
@@ -333,6 +333,9 @@ class VKUserVideosIE(InfoExtractor):
     }, {
         'url': 'http://vk.com/videos-77521',
         'only_matching': True,
+    }, {
+        'url': 'http://vk.com/videos-97664626?section=all',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 1bac34556fcff324fa17c679004284f03123d64d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 2 Feb 2016 18:07:07 +0100
Subject: [PATCH 0561/1105] [common] add a generic support for mpd manifests

---
 youtube_dl/extractor/common.py | 206 ++++++++++++++++++++++-----------
 1 file changed, 136 insertions(+), 70 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 199a04d1c..1758ab29b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -10,6 +10,7 @@ import re
 import socket
 import sys
 import time
+import math
 
 from ..compat import (
     compat_cookiejar,
@@ -44,6 +45,7 @@ from ..utils import (
     xpath_text,
     xpath_with_ns,
     determine_protocol,
+    parse_duration,
 )
 
 
@@ -1330,81 +1332,145 @@ class InfoExtractor(object):
             })
         return entries
 
-    def _download_dash_manifest(self, dash_manifest_url, video_id, fatal=True):
-        return self._download_xml(
-            dash_manifest_url, video_id,
-            note='Downloading DASH manifest',
-            errnote='Could not download DASH manifest',
+    def _extract_mpd_formats(self, mpd_url, video_id, mpd_id=None, note=None, errnote=None, fatal=True, formats_dict={}):
+        res = self._download_webpage_handle(
+            mpd_url, video_id,
+            note=note or 'Downloading MPD manifest',
+            errnote=errnote or 'Failed to download MPD manifest',
             fatal=fatal)
+        if res is False:
+            return []
+        mpd, urlh = res
+        mpd_base_url = re.match(r'https?://.+/', urlh.geturl()).group()
 
-    def _extract_dash_manifest_formats(self, dash_manifest_url, video_id, fatal=True, namespace=None, formats_dict={}):
-        dash_doc = self._download_dash_manifest(dash_manifest_url, video_id, fatal)
-        if dash_doc is False:
+        return self._parse_mpd(
+            compat_etree_fromstring(mpd.encode('utf-8')), mpd_id, mpd_base_url, formats_dict=formats_dict)
+
+    def _parse_mpd(self, mpd_doc, mpd_id=None, mpd_base_url='', formats_dict={}):
+        if mpd_doc.get('type') == 'dynamic':
             return []
 
-        return self._parse_dash_manifest(
-            dash_doc, namespace=namespace, formats_dict=formats_dict)
-
-    def _parse_dash_manifest(self, dash_doc, namespace=None, formats_dict={}):
-        def _add_ns(path):
-            return self._xpath_ns(path, namespace)
-
-        formats = []
-        for a in dash_doc.findall('.//' + _add_ns('AdaptationSet')):
-            mime_type = a.attrib.get('mimeType')
-            for r in a.findall(_add_ns('Representation')):
-                mime_type = r.attrib.get('mimeType') or mime_type
-                url_el = r.find(_add_ns('BaseURL'))
-                if mime_type == 'text/vtt':
-                    # TODO implement WebVTT downloading
-                    pass
-                elif mime_type.startswith('audio/') or mime_type.startswith('video/'):
-                    segment_list = r.find(_add_ns('SegmentList'))
-                    format_id = r.attrib['id']
-                    video_url = url_el.text if url_el is not None else None
-                    filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength') if url_el is not None else None)
-                    f = {
-                        'format_id': format_id,
-                        'url': video_url,
-                        'width': int_or_none(r.attrib.get('width')),
-                        'height': int_or_none(r.attrib.get('height')),
-                        'tbr': int_or_none(r.attrib.get('bandwidth'), 1000),
-                        'asr': int_or_none(r.attrib.get('audioSamplingRate')),
-                        'filesize': filesize,
-                        'fps': int_or_none(r.attrib.get('frameRate')),
-                    }
-                    if segment_list is not None:
-                        initialization_url = segment_list.find(_add_ns('Initialization')).attrib['sourceURL']
-                        f.update({
-                            'initialization_url': initialization_url,
-                            'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall(_add_ns('SegmentURL'))],
-                            'protocol': 'http_dash_segments',
-                        })
-                        if not f.get('url'):
-                            f['url'] = initialization_url
-                    try:
-                        existing_format = next(
-                            fo for fo in formats
-                            if fo['format_id'] == format_id)
-                    except StopIteration:
-                        full_info = formats_dict.get(format_id, {}).copy()
-                        full_info.update(f)
-                        codecs = r.attrib.get('codecs')
-                        if codecs:
-                            if mime_type.startswith('video/'):
-                                vcodec, acodec = codecs, 'none'
-                            else:  # mime_type.startswith('audio/')
-                                vcodec, acodec = 'none', codecs
-
-                            full_info.update({
-                                'vcodec': vcodec,
-                                'acodec': acodec,
-                            })
-                        formats.append(full_info)
+        def extract_multisegment_info(element, ms_parent_info):
+            ms_info = ms_parent_info.copy()
+            segment_list = element.find(self._xpath_ns('SegmentList', namespace))
+            if segment_list is not None:
+                segment_urls_e = segment_list.findall(self._xpath_ns('SegmentURL', namespace))
+                if segment_urls_e:
+                    ms_info['segment_urls'] = [segment.attrib['media'] for segment in segment_urls_e]
+                initialization = segment_list.find(self._xpath_ns('Initialization', namespace))
+                if initialization is not None:
+                    ms_info['initialization_url'] = initialization.attrib['sourceURL']
+            else:
+                segment_template = element.find(self._xpath_ns('SegmentTemplate', namespace))
+                if segment_template is not None:
+                    start_number = segment_template.get('startNumber')
+                    if start_number:
+                        ms_info['start_number'] = int(start_number)
+                    segment_timeline = segment_template.find(self._xpath_ns('SegmentTimeline', namespace))
+                    if segment_timeline is not None:
+                        s_e = segment_timeline.findall(self._xpath_ns('S', namespace))
+                        if s_e:
+                            ms_info['total_number'] = 0
+                            for s in s_e:
+                                ms_info['total_number'] += 1 + int(s.get('r', '0'))
                     else:
-                        existing_format.update(f)
-                else:
-                    self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
+                        timescale = segment_template.get('timescale')
+                        if timescale:
+                            ms_info['timescale'] = int(timescale)
+                        segment_duration = segment_template.get('duration')
+                        if segment_duration:
+                            ms_info['segment_duration'] = int(segment_duration)
+                    media_template = segment_template.get('media')
+                    if media_template:
+                        ms_info['media_template'] = media_template
+                    initialization = segment_template.get('initialization')
+                    if initialization:
+                        ms_info['initialization_url'] = initialization
+                    else:
+                        initialization = segment_template.find(self._xpath_ns('Initialization', namespace))
+                        if initialization is not None:
+                            ms_info['initialization_url'] = initialization.attrib['sourceURL']
+            return ms_info
+
+        namespace = self._search_regex(r'(?i)^{([^}]+)?}MPD$', mpd_doc.tag, 'namespace')
+        mpd_duration = parse_duration(mpd_doc.get('mediaPresentationDuration'))
+        formats = []
+        for period in mpd_doc.findall(self._xpath_ns('Period', namespace)):
+            period_duration = parse_duration(period.get('duration')) or mpd_duration
+            period_ms_info = extract_multisegment_info(period, {
+                'start_number': 1,
+                'timescale': 1,
+            })
+            for adaptation_set in period.findall(self._xpath_ns('AdaptationSet', namespace)):
+                adaption_set_ms_info = extract_multisegment_info(adaptation_set, period_ms_info)
+                for representation in adaptation_set.findall(self._xpath_ns('Representation', namespace)):
+                    representation_attrib = adaptation_set.attrib.copy()
+                    representation_attrib.update(representation.attrib)
+                    mime_type = representation_attrib.get('mimeType')
+                    content_type = mime_type.split('/')[0] if mime_type else representation_attrib.get('contentType')
+                    if content_type == 'text':
+                        # TODO implement WebVTT downloading
+                        pass
+                    elif content_type == 'video' or content_type == 'audio':
+                        base_url = ''
+                        for element in (representation, adaptation_set, period, mpd_doc):
+                            base_url_e = element.find(self._xpath_ns('BaseURL', namespace))
+                            if base_url_e is not None:
+                                base_url = base_url_e.text + base_url
+                                if re.match(r'^https?://', base_url):
+                                    break
+                        if not re.match(r'^https?://', base_url):
+                            base_url = mpd_base_url + base_url
+                        representation_id = representation_attrib.get('id')
+                        f = {
+                            'format_id': mpd_id or representation_id,
+                            'url': base_url,
+                            'width': int_or_none(representation_attrib.get('width')),
+                            'height': int_or_none(representation_attrib.get('height')),
+                            'tbr': int_or_none(representation_attrib.get('bandwidth'), 1000),
+                            'asr': int_or_none(representation_attrib.get('audioSamplingRate')),
+                            'fps': int_or_none(representation_attrib.get('frameRate')),
+                            'vcodec': 'none' if content_type == 'audio' else representation_attrib.get('codecs'),
+                            'acodec': 'none' if content_type == 'video' else representation_attrib.get('codecs'),
+                            'language': representation_attrib.get('lang'),
+                            'format_note': 'DASH %s' % content_type,
+                        }
+                        representation_ms_info = extract_multisegment_info(representation, adaption_set_ms_info)
+                        if 'segment_urls' not in representation_ms_info and 'media_template' in representation_ms_info:
+                            if 'total_number' not in representation_ms_info and 'segment_duration':
+                                segment_duration = representation_ms_info['segment_duration'] / representation_ms_info['timescale']
+                                representation_ms_info['total_number'] = int(math.ceil(period_duration / segment_duration))
+                            media_template = representation_ms_info['media_template']
+                            media_template = media_template.replace('$RepresentationID$', representation_id)
+                            media_template = re.sub(r'\$(Bandwidth)(?:%(0\d+d))?\$', r'%(\1)\2', media_template)
+                            media_template = media_template % {'Bandwidth': representation_attrib.get('bandwidth')}
+                            media_template = re.sub(r'\$(Number)(?:%(0\d+d))?\$', r'%(\1)\2', media_template)
+                            media_template.replace('$$', '$')
+                            representation_ms_info['segment_urls'] = [media_template % {'Number': segment_number} for segment_number in range(representation_ms_info['start_number'], representation_ms_info['total_number'] + representation_ms_info['start_number'])]
+                        if 'segment_urls' in representation_ms_info:
+                            f.update({
+                                'segment_urls': representation_ms_info['segment_urls'],
+                                'protocol': 'http_dash_segments',
+                            })
+                            if 'initialization_url' in representation_ms_info:
+                                initialization_url = representation_ms_info['initialization_url'].replace('$RepresentationID$', representation_id)
+                                f.update({
+                                    'initialization_url': initialization_url,
+                                })
+                                if not f.get('url'):
+                                    f['url'] = initialization_url
+                        try:
+                            existing_format = next(
+                                fo for fo in formats
+                                if fo['format_id'] == representation_id)
+                        except StopIteration:
+                            full_info = formats_dict.get(representation_id, {}).copy()
+                            full_info.update(f)
+                            formats.append(full_info)
+                        else:
+                            existing_format.update(f)
+                    else:
+                        self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
         return formats
 
     def _live_title(self, name):

From 8ff648e4f97c9d2692cd3f97db404911c23792f5 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 2 Feb 2016 18:10:23 +0100
Subject: [PATCH 0562/1105] [youtube] use _extract_mpd_formats

---
 youtube_dl/extractor/youtube.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a7f8c968e..828f5d1f4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1463,7 +1463,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         # Look for the DASH manifest
         if self._downloader.params.get('youtube_include_dash_manifest', True):
             dash_mpd_fatal = True
-            for dash_manifest_url in dash_mpds:
+            for mpd_url in dash_mpds:
                 dash_formats = {}
                 try:
                     def decrypt_sig(mobj):
@@ -1471,11 +1471,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
                         return '/signature/%s' % dec_s
 
-                    dash_manifest_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, dash_manifest_url)
+                    mpd_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, mpd_url)
 
-                    for df in self._extract_dash_manifest_formats(
-                            dash_manifest_url, video_id, fatal=dash_mpd_fatal,
-                            namespace='urn:mpeg:DASH:schema:MPD:2011', formats_dict=self._formats):
+                    for df in self._extract_mpd_formats(
+                            mpd_url, video_id, fatal=dash_mpd_fatal,
+                            formats_dict=self._formats):
                         # Do not overwrite DASH format found in some previous DASH manifest
                         if df['format_id'] not in dash_formats:
                             dash_formats[df['format_id']] = df

From c11875b3287fe7dbacdb296eac4a21d93bc1830e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 2 Feb 2016 18:11:16 +0100
Subject: [PATCH 0563/1105] [facebook] use _parse_mpd

---
 youtube_dl/extractor/facebook.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index b6d1180f0..9c751178f 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -215,9 +215,8 @@ class FacebookIE(InfoExtractor):
                         })
             dash_manifest = f[0].get('dash_manifest')
             if dash_manifest:
-                formats.extend(self._parse_dash_manifest(
-                    compat_etree_fromstring(compat_urllib_parse_unquote_plus(dash_manifest)),
-                    namespace='urn:mpeg:dash:schema:mpd:2011'))
+                formats.extend(self._parse_mpd(
+                    compat_etree_fromstring(compat_urllib_parse_unquote_plus(dash_manifest))))
         if not formats:
             raise ExtractorError('Cannot find video formats')
 

From a1a22572fb968aa90e110798b34c444d894ad24a Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 2 Feb 2016 18:12:32 +0100
Subject: [PATCH 0564/1105] [downloader/dash] make initialization_url optional

---
 youtube_dl/downloader/dash.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 535f2a7fc..b0070aead 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -40,9 +40,10 @@ class DashSegmentsFD(FileDownloader):
             return '%s%s%s' % (base_url, '' if base_url.endswith('/') else '/', target_url)
 
         with open(tmpfilename, 'wb') as outf:
-            append_url_to_file(
-                outf, combine_url(base_url, info_dict['initialization_url']),
-                'initialization segment')
+            if info_dict.get('initialization_url'):
+                append_url_to_file(
+                    outf, combine_url(base_url, info_dict['initialization_url']),
+                    'initialization segment')
             for i, segment_url in enumerate(segment_urls):
                 segment_len = append_url_to_file(
                     outf, combine_url(base_url, segment_url),

From 5976e7ab579c4d1e2fae26acbaa628009b2d83a8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 2 Feb 2016 18:13:01 +0100
Subject: [PATCH 0565/1105] [vevo] add support for dash formats

---
 youtube_dl/extractor/vevo.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 35fcff1b2..152fef42e 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -197,8 +197,14 @@ class VevoIE(InfoExtractor):
                 if not version_url:
                     continue
 
-                if '.mpd' in version_url or '.ism' in version_url:
+                if '.ism' in version_url:
                     continue
+                elif '.mpd' in version_url:
+                    formats.extend(self._extract_mpd_formats(
+                        version_url, video_id, mpd_id='dash-%s' % version,
+                        note='Downloading %s MPD information' % version,
+                        errnote='Failed to download %s MPD information' % version,
+                        fatal=False))
                 elif '.m3u8' in version_url:
                     formats.extend(self._extract_m3u8_formats(
                         version_url, video_id, 'mp4', 'm3u8_native',

From 9c74423510003a5d6d03e6da1f73f9230df9281f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 2 Feb 2016 18:30:31 +0100
Subject: [PATCH 0566/1105] [common] fix media template regex

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 1758ab29b..eda16a315 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1442,9 +1442,9 @@ class InfoExtractor(object):
                                 representation_ms_info['total_number'] = int(math.ceil(period_duration / segment_duration))
                             media_template = representation_ms_info['media_template']
                             media_template = media_template.replace('$RepresentationID$', representation_id)
-                            media_template = re.sub(r'\$(Bandwidth)(?:%(0\d+d))?\$', r'%(\1)\2', media_template)
+                            media_template = re.sub(r'\$(Bandwidth)(?:%(0\d+)d)?\$', r'%(\1)\2d', media_template)
                             media_template = media_template % {'Bandwidth': representation_attrib.get('bandwidth')}
-                            media_template = re.sub(r'\$(Number)(?:%(0\d+d))?\$', r'%(\1)\2', media_template)
+                            media_template = re.sub(r'\$(Number)(?:%(0\d+)d)?\$', r'%(\1)\2d', media_template)
                             media_template.replace('$$', '$')
                             representation_ms_info['segment_urls'] = [media_template % {'Number': segment_number} for segment_number in range(representation_ms_info['start_number'], representation_ms_info['total_number'] + representation_ms_info['start_number'])]
                         if 'segment_urls' in representation_ms_info:

From 1df41411968b52d7fd6d0f0d233ad760aa5e6368 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 3 Feb 2016 03:42:37 +0800
Subject: [PATCH 0567/1105] [test_YoutubeDL] Fix test_youtube_format_selection

Broken since a6c2c24479e5f4827ceb06f64d855329c0a6f593. Thanks to
@jaimeMF and @anisse for pointing that out
---
 test/test_YoutubeDL.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 73910eaec..88c63010e 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -248,6 +248,17 @@ class TestFormatSelection(unittest.TestCase):
 
         def format_info(f_id):
             info = YoutubeIE._formats[f_id].copy()
+
+            # XXX: In real cases InfoExtractor._parse_mpd() fills up 'acodec'
+            # and 'vcodec', while in tests such information is incomplete since
+            # commit a6c2c24479e5f4827ceb06f64d855329c0a6f593
+            # test_YoutubeDL.test_youtube_format_selection is broken without
+            # this fix
+            if 'acodec' in info and 'vcodec' not in info:
+                info['vcodec'] = 'none'
+            elif 'vcodec' in info and 'acodec' not in info:
+                info['acodec'] = 'none'
+
             info['format_id'] = f_id
             info['url'] = 'url:' + f_id
             return info

From f14be228164a2da299d64e18f9081330d1104106 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 2 Feb 2016 22:02:08 +0100
Subject: [PATCH 0568/1105] [common] remove duplicate reference to namespace

---
 youtube_dl/extractor/common.py | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index eda16a315..caca32d06 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1350,25 +1350,30 @@ class InfoExtractor(object):
         if mpd_doc.get('type') == 'dynamic':
             return []
 
+        namespace = self._search_regex(r'(?i)^{([^}]+)?}MPD$', mpd_doc.tag, 'namespace')
+
+        def _add_ns(path):
+            return self._xpath_ns(path, namespace)
+
         def extract_multisegment_info(element, ms_parent_info):
             ms_info = ms_parent_info.copy()
-            segment_list = element.find(self._xpath_ns('SegmentList', namespace))
+            segment_list = element.find(_add_ns('SegmentList'))
             if segment_list is not None:
-                segment_urls_e = segment_list.findall(self._xpath_ns('SegmentURL', namespace))
+                segment_urls_e = segment_list.findall(_add_ns('SegmentURL'))
                 if segment_urls_e:
                     ms_info['segment_urls'] = [segment.attrib['media'] for segment in segment_urls_e]
-                initialization = segment_list.find(self._xpath_ns('Initialization', namespace))
+                initialization = segment_list.find(_add_ns('Initialization'))
                 if initialization is not None:
                     ms_info['initialization_url'] = initialization.attrib['sourceURL']
             else:
-                segment_template = element.find(self._xpath_ns('SegmentTemplate', namespace))
+                segment_template = element.find(_add_ns('SegmentTemplate'))
                 if segment_template is not None:
                     start_number = segment_template.get('startNumber')
                     if start_number:
                         ms_info['start_number'] = int(start_number)
-                    segment_timeline = segment_template.find(self._xpath_ns('SegmentTimeline', namespace))
+                    segment_timeline = segment_template.find(_add_ns('SegmentTimeline'))
                     if segment_timeline is not None:
-                        s_e = segment_timeline.findall(self._xpath_ns('S', namespace))
+                        s_e = segment_timeline.findall(_add_ns('S'))
                         if s_e:
                             ms_info['total_number'] = 0
                             for s in s_e:
@@ -1387,23 +1392,22 @@ class InfoExtractor(object):
                     if initialization:
                         ms_info['initialization_url'] = initialization
                     else:
-                        initialization = segment_template.find(self._xpath_ns('Initialization', namespace))
+                        initialization = segment_template.find(_add_ns('Initialization'))
                         if initialization is not None:
                             ms_info['initialization_url'] = initialization.attrib['sourceURL']
             return ms_info
 
-        namespace = self._search_regex(r'(?i)^{([^}]+)?}MPD$', mpd_doc.tag, 'namespace')
         mpd_duration = parse_duration(mpd_doc.get('mediaPresentationDuration'))
         formats = []
-        for period in mpd_doc.findall(self._xpath_ns('Period', namespace)):
+        for period in mpd_doc.findall(_add_ns('Period')):
             period_duration = parse_duration(period.get('duration')) or mpd_duration
             period_ms_info = extract_multisegment_info(period, {
                 'start_number': 1,
                 'timescale': 1,
             })
-            for adaptation_set in period.findall(self._xpath_ns('AdaptationSet', namespace)):
+            for adaptation_set in period.findall(_add_ns('AdaptationSet')):
                 adaption_set_ms_info = extract_multisegment_info(adaptation_set, period_ms_info)
-                for representation in adaptation_set.findall(self._xpath_ns('Representation', namespace)):
+                for representation in adaptation_set.findall(_add_ns('Representation')):
                     representation_attrib = adaptation_set.attrib.copy()
                     representation_attrib.update(representation.attrib)
                     mime_type = representation_attrib.get('mimeType')
@@ -1414,7 +1418,7 @@ class InfoExtractor(object):
                     elif content_type == 'video' or content_type == 'audio':
                         base_url = ''
                         for element in (representation, adaptation_set, period, mpd_doc):
-                            base_url_e = element.find(self._xpath_ns('BaseURL', namespace))
+                            base_url_e = element.find(_add_ns('BaseURL'))
                             if base_url_e is not None:
                                 base_url = base_url_e.text + base_url
                                 if re.match(r'^https?://', base_url):

From 6fba62c87a39686c0424ce9acd0fd29700a21076 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 2 Feb 2016 22:14:23 +0100
Subject: [PATCH 0569/1105] [ffmpeg] fix adding metadata when using
 --hls-prefer-native(#8350)

---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index d3d9d4f1d..e293e03e6 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -392,7 +392,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
             options.extend(['-metadata', '%s=%s' % (name, value)])
 
         # https://github.com/rg3/youtube-dl/issues/8350
-        if info['protocol'] == 'm3u8_native':
+        if info['protocol'] == 'm3u8_native' or self._downloader.params.get('hls_prefer_native', False):
             options.extend(['-bsf:a', 'aac_adtstoasc'])
 
         self._downloader.to_screen('[ffmpeg] Adding metadata to \'%s\'' % filename)

From 87de7069b90a5154c8fb625510891cf3fff6f713 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 2 Feb 2016 22:30:13 +0100
Subject: [PATCH 0570/1105] [utils] dfxp2srt: make TTMLPElementParser inherit
 from object

For consistency between python 2 and 3.
---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 18dbe28bb..4262ad6ac 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2017,7 +2017,7 @@ def dfxp2srt(dfxp_data):
         'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
     })
 
-    class TTMLPElementParser:
+    class TTMLPElementParser(object):
         out = ''
 
         def start(self, tag, attrib):

From fd3a1f3d608857ee36b1c5b455bbdbd3dedc17b1 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 2 Feb 2016 23:02:18 +0100
Subject: [PATCH 0571/1105] [cbsnews] add support for live videos(fixes #7010)

---
 youtube_dl/extractor/__init__.py |  5 ++-
 youtube_dl/extractor/cbsnews.py  | 52 +++++++++++++++++++++++++++-----
 2 files changed, 48 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e61a88de7..8d2fa604b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -90,7 +90,10 @@ from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
 from .canvas import CanvasIE
 from .cbs import CBSIE
-from .cbsnews import CBSNewsIE
+from .cbsnews import (
+    CBSNewsIE,
+    CBSNewsLiveVideoIE,
+)
 from .cbssports import CBSSportsIE
 from .ccc import CCCIE
 from .ceskatelevize import CeskaTelevizeIE
diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index cabf7e73b..8f864699f 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -1,15 +1,14 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-import json
-
+from .common import InfoExtractor
 from .theplatform import ThePlatformIE
+from ..utils import parse_duration
 
 
 class CBSNewsIE(ThePlatformIE):
     IE_DESC = 'CBS News'
-    _VALID_URL = r'http://(?:www\.)?cbsnews\.com/(?:[^/]+/)+(?P<id>[\da-z_-]+)'
+    _VALID_URL = r'http://(?:www\.)?cbsnews\.com/(?:news|videos)/(?P<id>[\da-z_-]+)'
 
     _TESTS = [
         {
@@ -48,14 +47,13 @@ class CBSNewsIE(ThePlatformIE):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
-        video_info = json.loads(self._html_search_regex(
+        video_info = self._parse_json(self._html_search_regex(
             r'(?:<ul class="media-list items" id="media-related-items"><li data-video-info|<div id="cbsNewsVideoPlayer" data-video-player-options)=\'({.+?})\'',
-            webpage, 'video JSON info'))
+            webpage, 'video JSON info'), video_id)
 
         item = video_info['item'] if 'item' in video_info else video_info
         title = item.get('articleTitle') or item.get('hed')
@@ -88,3 +86,41 @@ class CBSNewsIE(ThePlatformIE):
             'formats': formats,
             'subtitles': subtitles,
         }
+
+
+class CBSNewsLiveVideoIE(InfoExtractor):
+    IE_DESC = 'CBS News Live Videos'
+    _VALID_URL = r'http://(?:www\.)?cbsnews\.com/live/video/(?P<id>[\da-z_-]+)'
+
+    _TEST = {
+        'url': 'http://www.cbsnews.com/live/video/clinton-sanders-prepare-to-face-off-in-nh/',
+        'info_dict': {
+            'id': 'clinton-sanders-prepare-to-face-off-in-nh',
+            'ext': 'flv',
+            'title': 'Clinton, Sanders Prepare To Face Off In NH',
+            'duration': 334,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_info = self._parse_json(self._html_search_regex(
+            r'data-story-obj=\'({.+?})\'', webpage, 'video JSON info'), video_id)['story']
+
+        hdcore_sign = 'hdcore=3.3.1'
+        f4m_formats = self._extract_f4m_formats(video_info['url'] + '&' + hdcore_sign, video_id)
+        if f4m_formats:
+            for entry in f4m_formats:
+                # URLs without the extra param induce an 404 error
+                entry.update({'extra_param_to_segment_url': hdcore_sign})
+
+        return {
+            'id': video_id,
+            'title': video_info['headline'],
+            'thumbnail': video_info.get('thumbnail_url_hd') or video_info.get('thumbnail_url_sd'),
+            'duration': parse_duration(video_info.get('segmentDur')),
+            'formats': f4m_formats,
+        }

From 6ad2b01e14fe22e4607ea16cebbd13b4270505ca Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 2 Feb 2016 23:09:50 +0100
Subject: [PATCH 0572/1105] [srgssr] use flv as ext for rtmp formats

---
 youtube_dl/extractor/srgssr.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py
index 4707029ca..246970c4d 100644
--- a/youtube_dl/extractor/srgssr.py
+++ b/youtube_dl/extractor/srgssr.py
@@ -70,14 +70,11 @@ class SRGSSRIE(InfoExtractor):
                         asset_url, media_id, 'mp4', 'm3u8_native',
                         m3u8_id=format_id, fatal=False))
                 else:
-                    ext = None
-                    if protocol == 'RTMP':
-                        ext = self._search_regex(r'([a-z0-9]+):[^/]+', asset_url, 'ext')
                     formats.append({
                         'format_id': format_id,
                         'url': asset_url,
                         'preference': preference(quality),
-                        'ext': ext,
+                        'ext': 'flv' if protocol == 'RTMP' else None,
                     })
         self._sort_formats(formats)
 

From d577c79632e96e900f8ff5b89c98c3e60475d4b0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 3 Feb 2016 13:24:07 +0100
Subject: [PATCH 0573/1105] [common] ignore ISO 639-2 generic codes

---
 youtube_dl/extractor/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index caca32d06..ee0e3d8d1 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1426,6 +1426,7 @@ class InfoExtractor(object):
                         if not re.match(r'^https?://', base_url):
                             base_url = mpd_base_url + base_url
                         representation_id = representation_attrib.get('id')
+                        lang = representation_attrib.get('lang')
                         f = {
                             'format_id': mpd_id or representation_id,
                             'url': base_url,
@@ -1436,7 +1437,7 @@ class InfoExtractor(object):
                             'fps': int_or_none(representation_attrib.get('frameRate')),
                             'vcodec': 'none' if content_type == 'audio' else representation_attrib.get('codecs'),
                             'acodec': 'none' if content_type == 'video' else representation_attrib.get('codecs'),
-                            'language': representation_attrib.get('lang'),
+                            'language': lang if lang not in ('mul', 'und', 'zxx', 'mis') else None,
                             'format_note': 'DASH %s' % content_type,
                         }
                         representation_ms_info = extract_multisegment_info(representation, adaption_set_ms_info)

From c84d3a557de8eddc8e63adbd11ab40877f52c502 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Wed, 3 Feb 2016 19:18:25 +0500
Subject: [PATCH 0574/1105] [README.md] Clarify unavailable sequences in output
 format

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 7c582511f..79cd08df4 100644
--- a/README.md
+++ b/README.md
@@ -455,6 +455,8 @@ The `-o` option allows users to indicate a template for the output file names. T
  - `format_id`: The sequence will be replaced by the format code specified by `--format`.
  - `duration`: The sequence will be replaced by the length of the video in seconds.
 
+Note that some of the aforementioned sequences are not guaranteed to be present since they depend on the metadata obtained by particular extractor, such sequences will be replaced with `NA`.
+
 The current default template is `%(title)s-%(id)s.%(ext)s`.
 
 In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title:

From 58be922079a09bc56ff331864b094ba38beaf26e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 4 Feb 2016 01:26:25 +0800
Subject: [PATCH 0575/1105] [kuwo] Check for georestriction

---
 youtube_dl/extractor/kuwo.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 0c8ed5d07..f641edef8 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -31,6 +31,10 @@ class KuwoBaseIE(InfoExtractor):
                 (file_format['ext'], file_format.get('br', ''), song_id),
                 song_id, note='Download %s url info' % file_format['format'],
             )
+
+            if song_url == 'IPDeny':
+                raise ExtractorError('This song is blocked in this region', expected=True)
+
             if song_url.startswith('http://') or song_url.startswith('https://'):
                 formats.append({
                     'url': song_url,

From 675d001633c9446e9d53db2794614862d1d82607 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 3 Feb 2016 18:44:43 +0100
Subject: [PATCH 0576/1105] [common] skip drm protected dash formats

---
 youtube_dl/extractor/common.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index ee0e3d8d1..4859b911a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1355,6 +1355,9 @@ class InfoExtractor(object):
         def _add_ns(path):
             return self._xpath_ns(path, namespace)
 
+        def is_drm_protected(element):
+            return element.find(_add_ns('ContentProtection')) is not None
+
         def extract_multisegment_info(element, ms_parent_info):
             ms_info = ms_parent_info.copy()
             segment_list = element.find(_add_ns('SegmentList'))
@@ -1406,8 +1409,12 @@ class InfoExtractor(object):
                 'timescale': 1,
             })
             for adaptation_set in period.findall(_add_ns('AdaptationSet')):
+                if is_drm_protected(adaptation_set):
+                    continue
                 adaption_set_ms_info = extract_multisegment_info(adaptation_set, period_ms_info)
                 for representation in adaptation_set.findall(_add_ns('Representation')):
+                    if is_drm_protected(representation):
+                        continue
                     representation_attrib = adaptation_set.attrib.copy()
                     representation_attrib.update(representation.attrib)
                     mime_type = representation_attrib.get('mimeType')

From 53c269c6fd1af690570d35acae1d170b125d9d3c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 3 Feb 2016 23:54:34 +0100
Subject: [PATCH 0577/1105] [common] fix media_template string formating

---
 youtube_dl/extractor/common.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 4859b911a..b384e412e 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1454,11 +1454,9 @@ class InfoExtractor(object):
                                 representation_ms_info['total_number'] = int(math.ceil(period_duration / segment_duration))
                             media_template = representation_ms_info['media_template']
                             media_template = media_template.replace('$RepresentationID$', representation_id)
-                            media_template = re.sub(r'\$(Bandwidth)(?:%(0\d+)d)?\$', r'%(\1)\2d', media_template)
-                            media_template = media_template % {'Bandwidth': representation_attrib.get('bandwidth')}
-                            media_template = re.sub(r'\$(Number)(?:%(0\d+)d)?\$', r'%(\1)\2d', media_template)
+                            media_template = re.sub(r'\$(Number|Bandwidth)(?:%(0\d+)d)?\$', r'%(\1)\2d', media_template)
                             media_template.replace('$$', '$')
-                            representation_ms_info['segment_urls'] = [media_template % {'Number': segment_number} for segment_number in range(representation_ms_info['start_number'], representation_ms_info['total_number'] + representation_ms_info['start_number'])]
+                            representation_ms_info['segment_urls'] = [media_template % {'Number': segment_number, 'Bandwidth': representation_attrib.get('bandwidth')} for segment_number in range(representation_ms_info['start_number'], representation_ms_info['total_number'] + representation_ms_info['start_number'])]
                         if 'segment_urls' in representation_ms_info:
                             f.update({
                                 'segment_urls': representation_ms_info['segment_urls'],

From 255732f0d33268aeababb1b3ce37a1defb5bc965 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 3 Feb 2016 23:57:08 +0100
Subject: [PATCH 0578/1105] [common] fix segment duration calculation

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b384e412e..d9f31daaa 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1,4 +1,4 @@
-from __future__ import unicode_literals
+from __future__ import unicode_literals, division
 
 import base64
 import datetime

From eadc3ccd504a13487a97172cb8468e12fd8bc068 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 4 Feb 2016 01:25:36 +0100
Subject: [PATCH 0579/1105] [generic] extract m3u8 formats when mpegurl content
 type detected

---
 youtube_dl/extractor/generic.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b18e734c4..c02fe201c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1229,19 +1229,24 @@ class GenericIE(InfoExtractor):
 
         # Check for direct link to a video
         content_type = head_response.headers.get('Content-Type', '')
-        m = re.match(r'^(?P<type>audio|video|application(?=/ogg$))/(?P<format_id>.+)$', content_type)
+        m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>.+)$', content_type)
         if m:
             upload_date = unified_strdate(
                 head_response.headers.get('Last-Modified'))
+            formats = []
+            if m.group('format_id').endswith('mpegurl'):
+                formats = self._extract_m3u8_formats(url, video_id, 'mp4')
+            else:
+                formats = [{
+                    'format_id': m.group('format_id'),
+                    'url': url,
+                    'vcodec': 'none' if m.group('type') == 'audio' else None
+                }]
             return {
                 'id': video_id,
                 'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
                 'direct': True,
-                'formats': [{
-                    'format_id': m.group('format_id'),
-                    'url': url,
-                    'vcodec': 'none' if m.group('type') == 'audio' else None
-                }],
+                'formats': formats,
                 'upload_date': upload_date,
             }
 

From 09be85b8dd987ff4c3b3161dc58c36959c33da5e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 4 Feb 2016 08:28:37 +0100
Subject: [PATCH 0580/1105] [youtube] fix subtitle extraction(fixes #8415)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a7f8c968e..1fef4cfd4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -918,7 +918,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if lang in sub_lang_list:
                 continue
             sub_formats = []
-            for ext in ['sbv', 'vtt', 'srt']:
+            for ext in ['vtt', 'ttml']:
                 params = compat_urllib_parse.urlencode({
                     'lang': lang,
                     'v': video_id,

From 2fb35d1c28f6dcac4bf35cc533fbc1699983b740 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 4 Feb 2016 08:39:01 +0100
Subject: [PATCH 0581/1105] [youtube] fix subtitle order

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 1fef4cfd4..e9ed77e42 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -918,7 +918,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if lang in sub_lang_list:
                 continue
             sub_formats = []
-            for ext in ['vtt', 'ttml']:
+            for ext in ['ttml', 'vtt']:
                 params = compat_urllib_parse.urlencode({
                     'lang': lang,
                     'v': video_id,

From 920d21b9d375327760afbfbf257c8719106d7325 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 4 Feb 2016 08:50:55 +0100
Subject: [PATCH 0582/1105] [test_subtitles] update youtube subtitles tests

---
 test/test_subtitles.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 9a695c4e8..27e763edd 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -65,16 +65,16 @@ class TestYoutubeSubtitles(BaseTestSubtitles):
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(len(subtitles.keys()), 13)
-        self.assertEqual(md5(subtitles['en']), '4cd9278a35ba2305f47354ee13472260')
-        self.assertEqual(md5(subtitles['it']), '164a51f16f260476a05b50fe4c2f161d')
-        for lang in ['it', 'fr', 'de']:
+        self.assertEqual(md5(subtitles['en']), '3cb210999d3e021bd6c7f0ea751eab06')
+        self.assertEqual(md5(subtitles['it']), '6d752b98c31f1cf8d597050c7a2cb4b5')
+        for lang in ['fr', 'de']:
             self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
 
-    def test_youtube_subtitles_sbv_format(self):
+    def test_youtube_subtitles_ttml_format(self):
         self.DL.params['writesubtitles'] = True
-        self.DL.params['subtitlesformat'] = 'sbv'
+        self.DL.params['subtitlesformat'] = 'ttml'
         subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['en']), '13aeaa0c245a8bed9a451cb643e3ad8b')
+        self.assertEqual(md5(subtitles['en']), 'e306f8c42842f723447d9f63ad65df54')
 
     def test_youtube_subtitles_vtt_format(self):
         self.DL.params['writesubtitles'] = True

From f1ed3acae56a5874b6f3965e0735104bbccf8ce2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Feb 2016 13:39:26 +0100
Subject: [PATCH 0583/1105] release 2016.02.04

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 61be9990d..ee34adf26 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -91,6 +91,7 @@
  - **Canvas**
  - **CBS**
  - **CBSNews**: CBS News
+ - **CBSNewsLiveVideo**: CBS News Live Videos
  - **CBSSports**
  - **CeskaTelevize**
  - **channel9**: Channel 9
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6da42c5a5..d4135e88e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.02.01'
+__version__ = '2016.02.04'

From 3f7e8750d4658702b00237370da1a1c680fdb548 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 4 Feb 2016 20:16:47 +0100
Subject: [PATCH 0584/1105] [arte.tv:+7] Fix extraction (fixes #8427)

---
 youtube_dl/extractor/arte.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index b9e07f0ef..6ed855a57 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -13,6 +13,7 @@ from ..utils import (
     unified_strdate,
     get_element_by_attribute,
     int_or_none,
+    NO_DEFAULT,
     qualities,
 )
 
@@ -93,9 +94,18 @@ class ArteTVPlus7IE(InfoExtractor):
         json_url = self._html_search_regex(
             patterns, webpage, 'json vp url', default=None)
         if not json_url:
-            iframe_url = self._html_search_regex(
-                r'<iframe[^>]+src=(["\'])(?P<url>.+\bjson_url=.+?)\1',
-                webpage, 'iframe url', group='url')
+            def find_iframe_url(webpage, default=NO_DEFAULT):
+                return self._html_search_regex(
+                    r'<iframe[^>]+src=(["\'])(?P<url>.+\bjson_url=.+?)\1',
+                    webpage, 'iframe url', group='url', default=default)
+
+            iframe_url = find_iframe_url(webpage, None)
+            if not iframe_url:
+                embed_url = self._html_search_regex(
+                    r'arte_vp_url_oembed=\'([^\']+?)\'', webpage, 'embed url')
+                player = self._download_json(
+                    embed_url, video_id, 'Downloading player page')
+                iframe_url = find_iframe_url(player['html'])
             json_url = compat_parse_qs(
                 compat_urllib_parse_urlparse(iframe_url).query)['json_url'][0]
         return self._extract_from_json_url(json_url, video_id, lang)

From 008ab0f814a84caebeb7551b86cc0e7422eda6b7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 5 Feb 2016 11:04:00 +0100
Subject: [PATCH 0585/1105] release 2016.02.05

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d4135e88e..3b3e7e029 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.02.04'
+__version__ = '2016.02.05'

From 593e0f43b43d5a90d6e3430aeb94357ef81fb1ab Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 5 Feb 2016 15:12:06 +0100
Subject: [PATCH 0586/1105] [ffmpeg] fix condition(fixes #8440)

---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index e293e03e6..7e85a702d 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -392,7 +392,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
             options.extend(['-metadata', '%s=%s' % (name, value)])
 
         # https://github.com/rg3/youtube-dl/issues/8350
-        if info['protocol'] == 'm3u8_native' or self._downloader.params.get('hls_prefer_native', False):
+        if info.get('protocol') == 'm3u8_native' or self._downloader.params.get('hls_prefer_native', False):
             options.extend(['-bsf:a', 'aac_adtstoasc'])
 
         self._downloader.to_screen('[ffmpeg] Adding metadata to \'%s\'' % filename)

From e06e75c7e7794c759077f2bb462dcf5d94a21760 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 5 Feb 2016 15:17:31 +0100
Subject: [PATCH 0587/1105] release 2016.02.05.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3b3e7e029..3fec14ab1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.02.05'
+__version__ = '2016.02.05.1'

From 7d3a035ee0a940021cb9c1a8cf0a8e5ab67f0c43 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 5 Feb 2016 17:12:49 +0100
Subject: [PATCH 0588/1105] [ffmpeg] check for m3u8 protocol in
 FFmpegMetadataPP

---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 7e85a702d..22d7ac65a 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -392,7 +392,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
             options.extend(['-metadata', '%s=%s' % (name, value)])
 
         # https://github.com/rg3/youtube-dl/issues/8350
-        if info.get('protocol') == 'm3u8_native' or self._downloader.params.get('hls_prefer_native', False):
+        if info.get('protocol') == 'm3u8_native' or info.get('protocol') == 'm3u8' and self._downloader.params.get('hls_prefer_native', False):
             options.extend(['-bsf:a', 'aac_adtstoasc'])
 
         self._downloader.to_screen('[ffmpeg] Adding metadata to \'%s\'' % filename)

From 58ae24336a0177567ac142734ca8f302f019107b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 Feb 2016 22:16:12 +0600
Subject: [PATCH 0589/1105] [spankbang] Extend format id regex (Closes #8398)

---
 youtube_dl/extractor/spankbang.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index 3cfa671ed..b14c01230 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -34,7 +34,7 @@ class SpankBangIE(InfoExtractor):
             'ext': 'mp4',
             'format_id': '%sp' % height,
             'height': int(height),
-        } for height in re.findall(r'<(?:span|li)[^>]+q_(\d+)p', webpage)]
+        } for height in re.findall(r'<(?:span|li|p)[^>]+[qb]_(\d+)p', webpage)]
         self._sort_formats(formats)
 
         title = self._html_search_regex(

From c773c232d80580c6662b5c957c830f75ccbb766b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 Feb 2016 22:16:40 +0600
Subject: [PATCH 0590/1105] [spankbang] Check formats (#8398)

---
 youtube_dl/extractor/spankbang.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index b14c01230..6299bfd91 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -35,6 +35,7 @@ class SpankBangIE(InfoExtractor):
             'format_id': '%sp' % height,
             'height': int(height),
         } for height in re.findall(r'<(?:span|li|p)[^>]+[qb]_(\d+)p', webpage)]
+        self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
         title = self._html_search_regex(

From d9e543b68030c8bd24d1b0352938d54312823345 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 Feb 2016 22:16:56 +0600
Subject: [PATCH 0591/1105] [spankbang] Add test with single format (#8398)

---
 youtube_dl/extractor/spankbang.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index 6299bfd91..50433d0f6 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 
 class SpankBangIE(InfoExtractor):
     _VALID_URL = r'https?://(?:(?:www|[a-z]{2})\.)?spankbang\.com/(?P<id>[\da-z]+)/video'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://spankbang.com/3vvn/video/fantasy+solo',
         'md5': '1cc433e1d6aa14bc376535b8679302f7',
         'info_dict': {
@@ -19,7 +19,11 @@ class SpankBangIE(InfoExtractor):
             'uploader': 'silly2587',
             'age_limit': 18,
         }
-    }
+    }, {
+        # 480p only
+        'url': 'http://spankbang.com/1vt0/video/solvane+gangbang',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From b7b365067f492e00166ef7ee0ebebcc53b4b6b9c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 Feb 2016 23:32:22 +0600
Subject: [PATCH 0592/1105] [vidme:user] Add extractor (Closes #8416)

---
 youtube_dl/extractor/__init__.py |  5 +++-
 youtube_dl/extractor/vidme.py    | 48 +++++++++++++++++++++++++++++++-
 2 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8d2fa604b..bedd2397c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -822,7 +822,10 @@ from .videomore import (
 )
 from .videopremium import VideoPremiumIE
 from .videott import VideoTtIE
-from .vidme import VidmeIE
+from .vidme import (
+    VidmeIE,
+    VidmeUserIE,
+)
 from .vidzi import VidziIE
 from .vier import VierIE, VierVideosIE
 from .viewster import ViewsterIE
diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 3d63ed4f0..91c8349c2 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 
+import itertools
+
 from .common import InfoExtractor
 from ..compat import compat_HTTPError
 from ..utils import (
@@ -11,7 +13,7 @@ from ..utils import (
 
 
 class VidmeIE(InfoExtractor):
-    _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]+)'
+    _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]{,5})(?:[^\da-zA-Z]|$)'
     _TESTS = [{
         'url': 'https://vid.me/QNB',
         'md5': 'f42d05e7149aeaec5c037b17e5d3dc82',
@@ -202,3 +204,47 @@ class VidmeIE(InfoExtractor):
             'comment_count': comment_count,
             'formats': formats,
         }
+
+
+class VidmeUserIE(InfoExtractor):
+    _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]{6,})'
+    _TEST = {
+        'url': 'https://vid.me/EFARCHIVE',
+        'info_dict': {
+            'id': '3834632',
+            'title': 'EFARCHIVE',
+        },
+        'playlist_mincount': 238,
+    }
+
+    # Max possible limit according to https://docs.vid.me/#api-Videos-List
+    _LIMIT = 100
+
+    def _entries(self, user_id, user_name):
+        for page_num in itertools.count(1):
+            page = self._download_json(
+                'https://api.vid.me/videos/list?user=%s&limit=%d&offset=%d'
+                % (user_id, self._LIMIT, (page_num - 1) * self._LIMIT), user_name,
+                'Downloading user page %d' % page_num)
+
+            videos = page.get('videos', [])
+            if not videos:
+                break
+
+            for video in videos:
+                video_url = video.get('full_url') or video.get('embed_url')
+                if video_url:
+                    yield self.url_result(video_url, VidmeIE.ie_key())
+
+            total = int_or_none(page.get('page', {}).get('total'))
+            if total and self._LIMIT * page_num >= total:
+                break
+
+    def _real_extract(self, url):
+        user_name = self._match_id(url)
+
+        user_id = self._download_json(
+            'https://api.vid.me/userByUsername?username=%s' % user_name,
+            user_name)['user']['user_id']
+
+        return self.playlist_result(self._entries(user_id, user_name), user_id, user_name)

From eb2533ec4cc55f1a100e508ea683f345cc84dac9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 Feb 2016 23:57:35 +0600
Subject: [PATCH 0593/1105] [vidme:user:likes] Add extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/vidme.py    | 50 ++++++++++++++++++++++----------
 2 files changed, 36 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bedd2397c..2fbc7f812 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -825,6 +825,7 @@ from .videott import VideoTtIE
 from .vidme import (
     VidmeIE,
     VidmeUserIE,
+    VidmeUserLikesIE,
 )
 from .vidzi import VidziIE
 from .vier import VierIE, VierVideosIE
diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 91c8349c2..e39f52204 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -206,26 +206,16 @@ class VidmeIE(InfoExtractor):
         }
 
 
-class VidmeUserIE(InfoExtractor):
-    _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]{6,})'
-    _TEST = {
-        'url': 'https://vid.me/EFARCHIVE',
-        'info_dict': {
-            'id': '3834632',
-            'title': 'EFARCHIVE',
-        },
-        'playlist_mincount': 238,
-    }
-
+class VidmeListBaseIE(InfoExtractor):
     # Max possible limit according to https://docs.vid.me/#api-Videos-List
     _LIMIT = 100
 
     def _entries(self, user_id, user_name):
         for page_num in itertools.count(1):
             page = self._download_json(
-                'https://api.vid.me/videos/list?user=%s&limit=%d&offset=%d'
-                % (user_id, self._LIMIT, (page_num - 1) * self._LIMIT), user_name,
-                'Downloading user page %d' % page_num)
+                'https://api.vid.me/videos/%s?user=%s&limit=%d&offset=%d'
+                % (self._API_ITEM, user_id, self._LIMIT, (page_num - 1) * self._LIMIT),
+                user_name, 'Downloading user %s page %d' % (self._API_ITEM, page_num))
 
             videos = page.get('videos', [])
             if not videos:
@@ -247,4 +237,34 @@ class VidmeUserIE(InfoExtractor):
             'https://api.vid.me/userByUsername?username=%s' % user_name,
             user_name)['user']['user_id']
 
-        return self.playlist_result(self._entries(user_id, user_name), user_id, user_name)
+        return self.playlist_result(
+            self._entries(user_id, user_name), user_id,
+            '%s - %s' % (user_name, self._TITLE))
+
+
+class VidmeUserIE(VidmeListBaseIE):
+    _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]{6,})(?!/likes)(?:[^\da-zA-Z]|$)'
+    _API_ITEM = 'list'
+    _TITLE = 'Videos'
+    _TEST = {
+        'url': 'https://vid.me/EFARCHIVE',
+        'info_dict': {
+            'id': '3834632',
+            'title': 'EFARCHIVE - %s' % _TITLE,
+        },
+        'playlist_mincount': 238,
+    }
+
+
+class VidmeUserLikesIE(VidmeListBaseIE):
+    _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]{6,})/likes'
+    _API_ITEM = 'likes'
+    _TITLE = 'Likes'
+    _TEST = {
+        'url': 'https://vid.me/ErinAlexis/likes',
+        'info_dict': {
+            'id': '6483530',
+            'title': 'ErinAlexis - %s' % _TITLE,
+        },
+        'playlist_mincount': 415,
+    }

From d97b0e324152a1313675fcaea22af72b6593e365 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 Feb 2016 23:58:26 +0600
Subject: [PATCH 0594/1105] [vidme] Clarify IE_NAMEs

---
 youtube_dl/extractor/vidme.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index e39f52204..b1156d531 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -13,6 +13,7 @@ from ..utils import (
 
 
 class VidmeIE(InfoExtractor):
+    IE_NAME = 'vidme'
     _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]{,5})(?:[^\da-zA-Z]|$)'
     _TESTS = [{
         'url': 'https://vid.me/QNB',
@@ -243,6 +244,7 @@ class VidmeListBaseIE(InfoExtractor):
 
 
 class VidmeUserIE(VidmeListBaseIE):
+    IE_NAME = 'vidme:user'
     _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]{6,})(?!/likes)(?:[^\da-zA-Z]|$)'
     _API_ITEM = 'list'
     _TITLE = 'Videos'
@@ -257,6 +259,7 @@ class VidmeUserIE(VidmeListBaseIE):
 
 
 class VidmeUserLikesIE(VidmeListBaseIE):
+    IE_NAME = 'vidme:user:likes'
     _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]{6,})/likes'
     _API_ITEM = 'likes'
     _TITLE = 'Likes'

From 23d17e4bebdff9a5defa012e4fa5cd99db605919 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Feb 2016 06:44:38 +0600
Subject: [PATCH 0595/1105] [youtube] Fix automatic captions

---
 youtube_dl/extractor/youtube.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e9ed77e42..9b346d27a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -369,6 +369,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         # RTMP (unnamed)
         '_rtmp': {'protocol': 'rtmp'},
     }
+    _SUBTITLE_FORMATS = ('ttml', 'vtt')
 
     IE_NAME = 'youtube'
     _TESTS = [
@@ -918,7 +919,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if lang in sub_lang_list:
                 continue
             sub_formats = []
-            for ext in ['ttml', 'vtt']:
+            for ext in self._SUBTITLE_FORMATS:
                 params = compat_urllib_parse.urlencode({
                     'lang': lang,
                     'v': video_id,
@@ -988,7 +989,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             for lang_node in caption_list.findall('target'):
                 sub_lang = lang_node.attrib['lang_code']
                 sub_formats = []
-                for ext in ['sbv', 'vtt', 'srt']:
+                for ext in self._SUBTITLE_FORMATS:
                     params = compat_urllib_parse.urlencode({
                         'lang': original_lang,
                         'tlang': sub_lang,

From bcbbb98bfe5c5309f447901c2de4f793647752fc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 6 Feb 2016 06:47:38 +0100
Subject: [PATCH 0596/1105] [generic] extract dash formats detected using
 content type

---
 youtube_dl/extractor/generic.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c02fe201c..4c2c76dc5 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1229,13 +1229,15 @@ class GenericIE(InfoExtractor):
 
         # Check for direct link to a video
         content_type = head_response.headers.get('Content-Type', '')
-        m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>.+)$', content_type)
+        m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|dash\+xml|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>.+)$', content_type)
         if m:
             upload_date = unified_strdate(
                 head_response.headers.get('Last-Modified'))
             formats = []
             if m.group('format_id').endswith('mpegurl'):
                 formats = self._extract_m3u8_formats(url, video_id, 'mp4')
+            elif m.group('format_id').startswith('dash+xml'):
+                formats = self._extract_mpd_formats(url, video_id)
             else:
                 formats = [{
                     'format_id': m.group('format_id'),

From 0826a0b555a06e0cc9047a7779c0f55612c8f3a3 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 6 Feb 2016 06:52:48 +0100
Subject: [PATCH 0597/1105] [common] sort dash formats

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index d9f31daaa..5287fa4f6 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1481,6 +1481,7 @@ class InfoExtractor(object):
                             existing_format.update(f)
                     else:
                         self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
+        self._sort_formats(formats)
         return formats
 
     def _live_title(self, name):

From 91cb6b50652f1ff395b1c38834ce33e99fa1975c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 6 Feb 2016 14:03:48 +0100
Subject: [PATCH 0598/1105] rename _parse_mpd to _parse_mpd_formats and add
 default value for mpd namespace

---
 test/test_YoutubeDL.py           | 2 +-
 youtube_dl/extractor/common.py   | 6 +++---
 youtube_dl/extractor/facebook.py | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 88c63010e..02caf5908 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -249,7 +249,7 @@ class TestFormatSelection(unittest.TestCase):
         def format_info(f_id):
             info = YoutubeIE._formats[f_id].copy()
 
-            # XXX: In real cases InfoExtractor._parse_mpd() fills up 'acodec'
+            # XXX: In real cases InfoExtractor._parse_mpd_formats() fills up 'acodec'
             # and 'vcodec', while in tests such information is incomplete since
             # commit a6c2c24479e5f4827ceb06f64d855329c0a6f593
             # test_YoutubeDL.test_youtube_format_selection is broken without
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5287fa4f6..271ba3739 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1343,14 +1343,14 @@ class InfoExtractor(object):
         mpd, urlh = res
         mpd_base_url = re.match(r'https?://.+/', urlh.geturl()).group()
 
-        return self._parse_mpd(
+        return self._parse_mpd_formats(
             compat_etree_fromstring(mpd.encode('utf-8')), mpd_id, mpd_base_url, formats_dict=formats_dict)
 
-    def _parse_mpd(self, mpd_doc, mpd_id=None, mpd_base_url='', formats_dict={}):
+    def _parse_mpd_formats(self, mpd_doc, mpd_id=None, mpd_base_url='', formats_dict={}):
         if mpd_doc.get('type') == 'dynamic':
             return []
 
-        namespace = self._search_regex(r'(?i)^{([^}]+)?}MPD$', mpd_doc.tag, 'namespace')
+        namespace = self._search_regex(r'(?i)^{([^}]+)?}MPD$', mpd_doc.tag, 'namespace', default=None)
 
         def _add_ns(path):
             return self._xpath_ns(path, namespace)
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 9c751178f..ed237f081 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -215,7 +215,7 @@ class FacebookIE(InfoExtractor):
                         })
             dash_manifest = f[0].get('dash_manifest')
             if dash_manifest:
-                formats.extend(self._parse_mpd(
+                formats.extend(self._parse_mpd_formats(
                     compat_etree_fromstring(compat_urllib_parse_unquote_plus(dash_manifest))))
         if not formats:
             raise ExtractorError('Cannot find video formats')

From 6a3828fddd6f6cec28ec8676bcc3918d583db77c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 6 Feb 2016 14:27:04 +0100
Subject: [PATCH 0599/1105] [common] use float conversion instead of using
 division from __future__

---
 youtube_dl/extractor/common.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 271ba3739..9c0421de7 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1,4 +1,4 @@
-from __future__ import unicode_literals, division
+from __future__ import unicode_literals
 
 import base64
 import datetime
@@ -1450,8 +1450,8 @@ class InfoExtractor(object):
                         representation_ms_info = extract_multisegment_info(representation, adaption_set_ms_info)
                         if 'segment_urls' not in representation_ms_info and 'media_template' in representation_ms_info:
                             if 'total_number' not in representation_ms_info and 'segment_duration':
-                                segment_duration = representation_ms_info['segment_duration'] / representation_ms_info['timescale']
-                                representation_ms_info['total_number'] = int(math.ceil(period_duration / segment_duration))
+                                segment_duration = float(representation_ms_info['segment_duration']) / float(representation_ms_info['timescale'])
+                                representation_ms_info['total_number'] = int(math.ceil(float(period_duration) / segment_duration))
                             media_template = representation_ms_info['media_template']
                             media_template = media_template.replace('$RepresentationID$', representation_id)
                             media_template = re.sub(r'\$(Number|Bandwidth)(?:%(0\d+)d)?\$', r'%(\1)\2d', media_template)

From 1b840245bdcc7789f996c9bb163af79efbebc813 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Feb 2016 19:35:32 +0600
Subject: [PATCH 0600/1105] [extractor/generic] Detect DASH manifests and
 extract mpd formats

---
 youtube_dl/extractor/generic.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 4c2c76dc5..6b52e75dc 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1291,7 +1291,7 @@ class GenericIE(InfoExtractor):
 
         self.report_extraction(video_id)
 
-        # Is it an RSS feed, a SMIL file or a XSPF playlist?
+        # Is it an RSS feed, a SMIL file, an XSPF playlist or a MPD manifest?
         try:
             doc = compat_etree_fromstring(webpage.encode('utf-8'))
             if doc.tag == 'rss':
@@ -1300,6 +1300,12 @@ class GenericIE(InfoExtractor):
                 return self._parse_smil(doc, url, video_id)
             elif doc.tag == '{http://xspf.org/ns/0/}playlist':
                 return self.playlist_result(self._parse_xspf(doc, video_id), video_id)
+            elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag):
+                return {
+                    'id': video_id,
+                    'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
+                    'formats': self._parse_mpd_formats(doc, video_id),
+                }
         except compat_xml_parse_error:
             pass
 

From 79a350857911d21bb04c90b2db04d7f6f0a3676b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Feb 2016 19:42:03 +0600
Subject: [PATCH 0601/1105] [extractor/generic] Detect DASH manifests in found
 URLs and extract mpd formats

---
 youtube_dl/extractor/generic.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6b52e75dc..6de777c41 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1959,6 +1959,8 @@ class GenericIE(InfoExtractor):
                 return self.playlist_result(self._extract_xspf_playlist(video_url, video_id), video_id)
             elif ext == 'm3u8':
                 entry_info_dict['formats'] = self._extract_m3u8_formats(video_url, video_id, ext='mp4')
+            elif ext == 'mpd':
+                entry_info_dict['formats'] = self._extract_mpd_formats(video_url, video_id)
             else:
                 entry_info_dict['url'] = video_url
 

From 28ad7df65ddb78c16ac008886d14ae2914aea6be Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 6 Feb 2016 14:51:45 +0100
Subject: [PATCH 0602/1105] [generic] detect MPD manfiest only from the content

---
 youtube_dl/extractor/generic.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6de777c41..bf61ab2e7 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1229,15 +1229,13 @@ class GenericIE(InfoExtractor):
 
         # Check for direct link to a video
         content_type = head_response.headers.get('Content-Type', '')
-        m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|dash\+xml|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>.+)$', content_type)
+        m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>.+)$', content_type)
         if m:
             upload_date = unified_strdate(
                 head_response.headers.get('Last-Modified'))
             formats = []
             if m.group('format_id').endswith('mpegurl'):
                 formats = self._extract_m3u8_formats(url, video_id, 'mp4')
-            elif m.group('format_id').startswith('dash+xml'):
-                formats = self._extract_mpd_formats(url, video_id)
             else:
                 formats = [{
                     'format_id': m.group('format_id'),

From 3ceccade87bf9ef8bccf92b353381e04cb32604d Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sat, 6 Feb 2016 22:33:49 +0600
Subject: [PATCH 0603/1105] [README.md] Improve output template documentation
 and add more examples

---
 README.md | 56 ++++++++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 43 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index 79cd08df4..c73b250fb 100644
--- a/README.md
+++ b/README.md
@@ -442,30 +442,60 @@ On Windows you may also need to setup the `%HOME%` environment variable manually
 
 The `-o` option allows users to indicate a template for the output file names. The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "http://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences have the format `%(NAME)s`. To clarify, that is a percent symbol followed by a name in parentheses, followed by a lowercase S. Allowed names are:
 
- - `id`: The sequence will be replaced by the video identifier.
- - `url`: The sequence will be replaced by the video URL.
- - `uploader`: The sequence will be replaced by the nickname of the person who uploaded the video.
- - `upload_date`: The sequence will be replaced by the upload date in YYYYMMDD format.
- - `title`: The sequence will be replaced by the video title.
- - `ext`: The sequence will be replaced by the appropriate extension (like flv or mp4).
- - `epoch`: The sequence will be replaced by the Unix epoch when creating the file.
- - `autonumber`: The sequence will be replaced by a five-digit number that will be increased with each download, starting at zero.
- - `playlist`: The sequence will be replaced by the name or the id of the playlist that contains the video.
- - `playlist_index`: The sequence will be replaced by the index of the video in the playlist padded with leading zeros according to the total length of the playlist.
- - `format_id`: The sequence will be replaced by the format code specified by `--format`.
- - `duration`: The sequence will be replaced by the length of the video in seconds.
+ - `id`: Video identifier.
+ - `url`: Video URL.
+ - `uploader`: Nickname of the person who uploaded the video.
+ - `upload_date`: Upload date in YYYYMMDD format.
+ - `title`: Video title.
+ - `ext`: Video filename extension.
+ - `epoch`: Unix epoch when creating the file.
+ - `autonumber`: Five-digit number that will be increased with each download, starting at zero.
+ - `playlist`: Name or id of the playlist that contains the video.
+ - `playlist_index`: Index of the video in the playlist padded with leading zeros according to the total length of the playlist.
+ - `format_id`: Format code specified by `--format`.
+ - `duration`: Length of the video in seconds.
 
-Note that some of the aforementioned sequences are not guaranteed to be present since they depend on the metadata obtained by particular extractor, such sequences will be replaced with `NA`.
+Available for the video that belong to some logical chapter or section:
+ - `chapter`: Name or title of the chapter the video belongs to.
+ - `chapter_number`: Number of the chapter the video belongs to, as an integer.
+ - `chapter_id`: Id of the chapter the video belongs to, as a unicode string.
+
+Available for the video that is an episode of some series or programme:
+ - `series`: Title of the series or programme the video episode belongs to.
+ - `season`: Title of the season the video episode belongs to.
+ - `season_number`: Number of the season the video episode belongs to, as an integer.
+ - `season_id`:Id of the season the video episode belongs to, as a unicode string.
+ - `episode`: Title of the video episode.
+ - `episode_number`: Number of the video episode within a season, as an integer.
+ - `episode_id`: Id of the video episode, as a unicode string.
+
+Each aforementioned sequence when referenced in output template will be replaced by the actual value corresponding to the sequence name. Note that some of the sequences are not guaranteed to be present since they depend on the metadata obtained by particular extractor, such sequences will be replaced with `NA`.
+
+For example for `-o %(title)s-%(id)s.%(ext)s` and mp4 video with title `youtube-dl test video` and id `BaW_jenozKcj` this will result in a `youtube-dl test video-BaW_jenozKcj.mp4` file created in the current directory.
+
+Output template can also contain arbitrary hierarchical path, e.g. `-o %(playlist)s/%(playlist_index)s - %(title)s.%(ext)s` that will result in downloading each video in a directory corresponding to this path template. Any missing directory will be automatically created for you.
 
 The current default template is `%(title)s-%(id)s.%(ext)s`.
 
 In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title:
 
+Examples:
+
 ```bash
 $ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc
 youtube-dl test video ''_ä↭𝕐.mp4    # All kinds of weird characters
+
 $ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc --restrict-filenames
 youtube-dl_test_video_.mp4          # A simple file name
+
+# Download YouTube playlist videos in separate directory indexed by video order in a playlist
+$ youtube-dl -o '%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s' https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re
+
+# Download Udemy course keeping each chapter in separate directory under MyVideos directory in your home
+$ youtube-dl -u user -p password -o '~/MyVideos/%(playlist)s/%(chapter_number)s - %(chapter)s/%(title)s.%(ext)s' https://www.udemy.com/java-tutorial/
+
+# Download entire series season keeping each series and each season in separate directory under C:/MyVideos
+$ youtube-dl -o 'C:/MyVideos/%(series)s/%(season_number)s - %(season)s/%(episode_number)s - %(episode)s.%(ext)s' http://videomore.ru/kino_v_detalayah/5_sezon/367617
 ```
 
 # FORMAT SELECTION

From 4340727e6cbdb55c2d190fe41f38f614a3e3780b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Feb 2016 22:36:30 +0600
Subject: [PATCH 0604/1105] [videomore] Fix typo

---
 youtube_dl/extractor/videomore.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/videomore.py b/youtube_dl/extractor/videomore.py
index fcee940e6..0bd1e1eec 100644
--- a/youtube_dl/extractor/videomore.py
+++ b/youtube_dl/extractor/videomore.py
@@ -114,7 +114,7 @@ class VideomoreIE(InfoExtractor):
 
         data = self._download_json(
             'http://videomore.ru/video/tracks/%s.json' % video_id,
-            video_id, 'Downloadinng video JSON')
+            video_id, 'Downloading video JSON')
 
         title = data.get('title') or data['project_title']
         description = data.get('description') or data.get('description_raw')

From bd6b6f662268a8a59a1cb9e72ac38a84dfa07822 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sat, 6 Feb 2016 22:36:30 +0600
Subject: [PATCH 0605/1105] [README.md] Fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index c73b250fb..85cd02a01 100644
--- a/README.md
+++ b/README.md
@@ -455,7 +455,7 @@ The `-o` option allows users to indicate a template for the output file names. T
  - `format_id`: Format code specified by `--format`.
  - `duration`: Length of the video in seconds.
 
-Available for the video that belong to some logical chapter or section:
+Available for the video that belongs to some logical chapter or section:
  - `chapter`: Name or title of the chapter the video belongs to.
  - `chapter_number`: Number of the chapter the video belongs to, as an integer.
  - `chapter_id`: Id of the chapter the video belongs to, as a unicode string.

From 4b8874975cdcd9ea656ce603f87d94b871d2849c Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sat, 6 Feb 2016 22:39:50 +0600
Subject: [PATCH 0606/1105] [README.md] Remove non-relevant info

---
 README.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 85cd02a01..7b8a1a22f 100644
--- a/README.md
+++ b/README.md
@@ -457,16 +457,16 @@ The `-o` option allows users to indicate a template for the output file names. T
 
 Available for the video that belongs to some logical chapter or section:
  - `chapter`: Name or title of the chapter the video belongs to.
- - `chapter_number`: Number of the chapter the video belongs to, as an integer.
- - `chapter_id`: Id of the chapter the video belongs to, as a unicode string.
+ - `chapter_number`: Number of the chapter the video belongs to.
+ - `chapter_id`: Id of the chapter the video belongs to.
 
 Available for the video that is an episode of some series or programme:
  - `series`: Title of the series or programme the video episode belongs to.
  - `season`: Title of the season the video episode belongs to.
- - `season_number`: Number of the season the video episode belongs to, as an integer.
- - `season_id`:Id of the season the video episode belongs to, as a unicode string.
+ - `season_number`: Number of the season the video episode belongs to.
+ - `season_id`:Id of the season the video episode belongs to.
  - `episode`: Title of the video episode.
- - `episode_number`: Number of the video episode within a season, as an integer.
+ - `episode_number`: Number of the video episode within a season.
  - `episode_id`: Id of the video episode, as a unicode string.
 
 Each aforementioned sequence when referenced in output template will be replaced by the actual value corresponding to the sequence name. Note that some of the sequences are not guaranteed to be present since they depend on the metadata obtained by particular extractor, such sequences will be replaced with `NA`.

From 860db2d508f2a18c1b48adb9b1eddc753721bd47 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sat, 6 Feb 2016 22:40:20 +0600
Subject: [PATCH 0607/1105] [README.md] Fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7b8a1a22f..7a180dccf 100644
--- a/README.md
+++ b/README.md
@@ -464,7 +464,7 @@ Available for the video that is an episode of some series or programme:
  - `series`: Title of the series or programme the video episode belongs to.
  - `season`: Title of the season the video episode belongs to.
  - `season_number`: Number of the season the video episode belongs to.
- - `season_id`:Id of the season the video episode belongs to.
+ - `season_id`: Id of the season the video episode belongs to.
  - `episode`: Title of the video episode.
  - `episode_number`: Number of the video episode within a season.
  - `episode_id`: Id of the video episode, as a unicode string.

From 56401e1e5f53e126bf124699b2f62bb670078fd3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Feb 2016 23:24:22 +0600
Subject: [PATCH 0608/1105] [downloader/hls] Do not send 'q' to ffmpeg on
 Windows (Closes #8300)

---
 youtube_dl/downloader/hls.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index cb34dc4ab..2a775bf00 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import os
 import re
 import subprocess
+import sys
 
 from .common import FileDownloader
 from .fragment import FragmentFD
@@ -57,8 +58,10 @@ class HlsFD(FileDownloader):
             # subprocces.run would send the SIGKILL signal to ffmpeg and the
             # mp4 file couldn't be played, but if we ask ffmpeg to quit it
             # produces a file that is playable (this is mostly useful for live
-            # streams)
-            proc.communicate(b'q')
+            # streams). Note that Windows is not affected and produces playable
+            # files (see https://github.com/rg3/youtube-dl/issues/8300).
+            if sys.platform != 'win32':
+                proc.communicate(b'q')
             raise
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))

From 7e62c2eb6d13ebf0fdd24c90c2f439177d485081 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 6 Feb 2016 18:51:05 +0100
Subject: [PATCH 0609/1105] [FFmpegSubtitlesConvertorPP] fix not working when
 srt is used as the intermediate format between ttml/dfxp and other format

It was trying to use the ttml/dfxp file with ffmpeg, which doesn't have support for them.
I broke it in e04398e397d4a4f7e0d0994355d5a94c22441e7c.
---
 youtube_dl/postprocessor/ffmpeg.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 22d7ac65a..6daa6a8b4 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -508,6 +508,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
 
                 with io.open(srt_file, 'wt', encoding='utf-8') as f:
                     f.write(srt_data)
+                old_file = srt_file
 
                 ext = 'srt'
                 subs[lang] = {

From 3547d26587214f0e8ade1295dab0438d88a567de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 6 Feb 2016 18:58:18 +0100
Subject: [PATCH 0610/1105] [FFmpegSubtitlesConvertorPP] correctly update the
 extension (fixes #8444)

---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 6daa6a8b4..05f2f0623 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -523,7 +523,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
 
             with io.open(new_file, 'rt', encoding='utf-8') as f:
                 subs[lang] = {
-                    'ext': ext,
+                    'ext': new_ext,
                     'data': f.read(),
                 }
 

From 7b8b007cd9b55326b45598130c9cdde8839d5340 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 6 Feb 2016 19:04:18 +0100
Subject: [PATCH 0611/1105] [FFmpegSubtitlesConvertorPP] remove intermediate
 srt files

---
 youtube_dl/postprocessor/ffmpeg.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 05f2f0623..6955b01f9 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -518,6 +518,8 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
 
                 if new_ext == 'srt':
                     continue
+                else:
+                    sub_filenames.append(srt_file)
 
             self.run_ffmpeg(old_file, new_file, ['-f', new_format])
 

From d4f8e834042ba120db5662b0150e1a01afe81ca8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 6 Feb 2016 19:04:53 +0100
Subject: [PATCH 0612/1105] [FFmpegSubtitlesConvertorPP] remove unused variable

---
 youtube_dl/postprocessor/ffmpeg.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 6955b01f9..380bc6f29 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -510,7 +510,6 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                     f.write(srt_data)
                 old_file = srt_file
 
-                ext = 'srt'
                 subs[lang] = {
                     'ext': 'srt',
                     'data': srt_data

From 5dd4b3468f3a400b865db2e7868bbcbd7c559626 Mon Sep 17 00:00:00 2001
From: Brian Foley <bpfoley@gmail.com>
Date: Sat, 6 Feb 2016 18:42:57 +0000
Subject: [PATCH 0613/1105] [rte:radio] Add support for RTMP downloads,
 alternate URL style

This is useful as
a) RTMP downloads are a good deal faster to download
b) Older items are available only as RTMP streams
---
 youtube_dl/extractor/rte.py | 43 +++++++++++++++++++++++++++++++------
 1 file changed, 37 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index 47c8331fe..73c9788be 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     float_or_none,
@@ -61,12 +63,15 @@ class RteIE(InfoExtractor):
 class RteRadioIE(InfoExtractor):
     IE_NAME = 'rte:radio'
     IE_DESC = 'Raidió Teilifís Éireann radio'
-    # Radioplayer URLs have the specifier #!rii=<channel_id>:<id>:<playable_item_id>:<date>:
+    # Radioplayer URLs have two distinct specifier formats,
+    # the old format #!rii=<channel_id>:<id>:<playable_item_id>:<date>:
+    # the new format #!rii=b<channel_id>_<id>_<playable_item_id>_<date>_
     # where the IDs are int/empty, the date is DD-MM-YYYY, and the specifier may be truncated.
     # An <id> uniquely defines an individual recording, and is the only part we require.
-    _VALID_URL = r'https?://(?:www\.)?rte\.ie/radio/utils/radioplayer/rteradioweb\.html#!rii=(?:[0-9]*)(?:%3A|:)(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?rte\.ie/radio/utils/radioplayer/rteradioweb\.html#!rii=(?:b?[0-9]*)(?:%3A|:|%5F|_)(?P<id>[0-9]+)'
 
-    _TEST = {
+    _TESTS = [{
+        # Old-style player URL; HLS and RTMPE formats
         'url': 'http://www.rte.ie/radio/utils/radioplayer/rteradioweb.html#!rii=16:10507902:2414:27-12-2015:',
         'info_dict': {
             'id': '10507902',
@@ -81,7 +86,23 @@ class RteRadioIE(InfoExtractor):
         'params': {
             'skip_download': 'f4m fails with --test atm'
         }
-    }
+    }, {
+        # New-style player URL; RTMPE formats only
+        'url': 'http://rte.ie/radio/utils/radioplayer/rteradioweb.html#!rii=b16_3250678_8861_06-04-2012_',
+        'info_dict': {
+            'id': '3250678',
+            'ext': 'flv',
+            'title': 'The Lyric Concert with Paul Herriott',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': '',
+            'timestamp': 1333742400,
+            'upload_date': '20120406',
+            'duration': 7199.016,
+        },
+        'params': {
+            'skip_download': 'f4m fails with --test atm'
+        }
+    }]
 
     def _real_extract(self, url):
         item_id = self._match_id(url)
@@ -102,8 +123,18 @@ class RteRadioIE(InfoExtractor):
 
         formats = []
 
-        if mg.get('url') and not mg['url'].startswith('rtmpe:'):
-            formats.append({'url': mg['url']})
+        if mg.get('url'):
+            m = re.match(r'(?P<url>rtmpe?://[^/]+)/(?P<app>.+)/(?P<playpath>mp4:.*)', mg['url'])
+            if m:
+                m = m.groupdict()
+                formats.append({
+                    'url': m['url'] + '/' + m['app'],
+                    'app': m['app'],
+                    'play_path': m['playpath'],
+                    'player_url': url,
+                    'ext': 'flv',
+                    'format_id': 'rtmp',
+                })
 
         if mg.get('hls_server') and mg.get('hls_url'):
             formats.extend(self._extract_m3u8_formats(

From f406c78785b57afb584cc0ee8ef4e84e568b0614 Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Sat, 6 Feb 2016 21:23:41 +0100
Subject: [PATCH 0614/1105] [IPrima] Fix extractor (fixes #7617)

---
 youtube_dl/extractor/iprima.py | 99 ++++++++--------------------------
 1 file changed, 22 insertions(+), 77 deletions(-)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 073777f34..093a7e551 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -2,46 +2,30 @@
 from __future__ import unicode_literals
 
 import re
-from random import random
-from math import floor
+import time
 
 from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
-    remove_end,
     sanitized_Request,
 )
 
 
 class IPrimaIE(InfoExtractor):
-    _WORKING = False
-    _VALID_URL = r'https?://play\.iprima\.cz/(?:[^/]+/)*(?P<id>[^?#]+)'
+    _VALID_URL = r'https?://play\.iprima\.cz/(?:.+/)?(?P<id>[^?#]+)'
 
     _TESTS = [{
+        'url': 'http://play.iprima.cz/gondici-s-r-o-33',
+        'info_dict': {
+            'id': 'p136534',
+            'ext': 'mp4',
+            'title': 'Gondíci s. r. o. (34)',
+            'description': 'md5:16577c629d006aa91f59ca8d8e7f99bd',
+        },
+        'params': {
+            'skip_download': True,  # m3u8 download
+        },
+    }, {
         'url': 'http://play.iprima.cz/particka/particka-92',
-        'info_dict': {
-            'id': '39152',
-            'ext': 'flv',
-            'title': 'Partička (92)',
-            'description': 'md5:74e9617e51bca67c3ecfb2c6f9766f45',
-            'thumbnail': 'http://play.iprima.cz/sites/default/files/image_crops/image_620x349/3/491483_particka-92_image_620x349.jpg',
-        },
-        'params': {
-            'skip_download': True,  # requires rtmpdump
-        },
-    }, {
-        'url': 'http://play.iprima.cz/particka/tchibo-particka-jarni-moda',
-        'info_dict': {
-            'id': '9718337',
-            'ext': 'flv',
-            'title': 'Tchibo Partička - Jarní móda',
-            'thumbnail': 're:^http:.*\.jpg$',
-        },
-        'params': {
-            'skip_download': True,  # requires rtmpdump
-        },
-    }, {
-        'url': 'http://play.iprima.cz/zpravy-ftv-prima-2752015',
         'only_matching': True,
     }]
 
@@ -51,62 +35,23 @@ class IPrimaIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        if re.search(r'Nemáte oprávnění přistupovat na tuto stránku\.\s*</div>', webpage):
-            raise ExtractorError(
-                '%s said: You do not have permission to access this page' % self.IE_NAME, expected=True)
+        video_id = self._search_regex(r'data-product="([^"]+)">', webpage, 'real id')
 
-        player_url = (
-            'http://embed.livebox.cz/iprimaplay/player-embed-v2.js?__tok%s__=%s' %
-            (floor(random() * 1073741824), floor(random() * 1073741824))
-        )
-
-        req = sanitized_Request(player_url)
+        req = sanitized_Request('http://play.iprima.cz/prehravac/init?_infuse=1'
+            '&_ts=%s&productId=%s' % (round(time.time()), 'p22201'))
         req.add_header('Referer', url)
-        playerpage = self._download_webpage(req, video_id)
+        playerpage = self._download_webpage(req, video_id, note='Downloading player')
 
-        base_url = ''.join(re.findall(r"embed\['stream'\] = '(.+?)'.+'(\?auth=)'.+'(.+?)';", playerpage)[1])
+        m3u8_url = self._search_regex(r"'src': '([^']+\.m3u8)'", playerpage, 'm3u8 url')
 
-        zoneGEO = self._html_search_regex(r'"zoneGEO":(.+?),', webpage, 'zoneGEO')
-        if zoneGEO != '0':
-            base_url = base_url.replace('token', 'token_' + zoneGEO)
-
-        formats = []
-        for format_id in ['lq', 'hq', 'hd']:
-            filename = self._html_search_regex(
-                r'"%s_id":(.+?),' % format_id, webpage, 'filename')
-
-            if filename == 'null':
-                continue
-
-            real_id = self._search_regex(
-                r'Prima-(?:[0-9]{10}|WEB)-([0-9]+)[-_]',
-                filename, 'real video id')
-
-            if format_id == 'lq':
-                quality = 0
-            elif format_id == 'hq':
-                quality = 1
-            elif format_id == 'hd':
-                quality = 2
-                filename = 'hq/' + filename
-
-            formats.append({
-                'format_id': format_id,
-                'url': base_url,
-                'quality': quality,
-                'play_path': 'mp4:' + filename.replace('"', '')[:-4],
-                'rtmp_live': True,
-                'ext': 'flv',
-            })
+        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
 
         self._sort_formats(formats)
 
         return {
-            'id': real_id,
-            'title': remove_end(self._og_search_title(webpage), ' | Prima PLAY'),
+            'id': video_id,
+            'title': self._og_search_title(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
             'formats': formats,
-            'description': self._search_regex(
-                r'<p[^>]+itemprop="description"[^>]*>([^<]+)',
-                webpage, 'description', default=None),
+            'description': self._og_search_description(webpage),
         }

From 70328330117e3c4c0e413b8c41bfd9e8e55fc62a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 6 Feb 2016 21:37:28 +0100
Subject: [PATCH 0615/1105] [iprima] Follow pep8

---
 youtube_dl/extractor/iprima.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 093a7e551..d922f905a 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -37,7 +37,8 @@ class IPrimaIE(InfoExtractor):
 
         video_id = self._search_regex(r'data-product="([^"]+)">', webpage, 'real id')
 
-        req = sanitized_Request('http://play.iprima.cz/prehravac/init?_infuse=1'
+        req = sanitized_Request(
+            'http://play.iprima.cz/prehravac/init?_infuse=1'
             '&_ts=%s&productId=%s' % (round(time.time()), 'p22201'))
         req.add_header('Referer', url)
         playerpage = self._download_webpage(req, video_id, note='Downloading player')

From 59b35c674597b913454b3efc82d60f903fb83fcb Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Sat, 6 Feb 2016 21:42:24 +0100
Subject: [PATCH 0616/1105] [IPrima] Remove test video_id

---
 youtube_dl/extractor/iprima.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index d922f905a..61a0de472 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -39,7 +39,7 @@ class IPrimaIE(InfoExtractor):
 
         req = sanitized_Request(
             'http://play.iprima.cz/prehravac/init?_infuse=1'
-            '&_ts=%s&productId=%s' % (round(time.time()), 'p22201'))
+            '&_ts=%s&productId=%s' % (round(time.time()), video_id))
         req.add_header('Referer', url)
         playerpage = self._download_webpage(req, video_id, note='Downloading player')
 

From b8b465af3e83fb19c1818c2fa83f0c5f753dd917 Mon Sep 17 00:00:00 2001
From: Erwin de Haan <git@erayan.eu>
Date: Sat, 6 Feb 2016 23:37:55 +0100
Subject: [PATCH 0617/1105] [vlive] Updated to new V App/VLive api.

More robust with getting keys and ids from website.
---
 youtube_dl/extractor/vlive.py | 44 ++++++++++++++++-------------------
 1 file changed, 20 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index 86c1cb5ef..3e1f8ef07 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -9,17 +9,18 @@ from time import time
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    determine_ext
+    determine_ext,
+    int_or_none
 )
 from ..compat import compat_urllib_parse
 
 
 class VLiveIE(InfoExtractor):
     IE_NAME = 'vlive'
-    # www.vlive.tv/video/ links redirect to m.vlive.tv/video/ for mobile devices
+    # vlive.tv/video/ links redirect to www.vlive.tv/video/ 
     _VALID_URL = r'https?://(?:(www|m)\.)?vlive\.tv/video/(?P<id>[0-9]+)'
     _TEST = {
-        'url': 'http://m.vlive.tv/video/1326',
+        'url': 'http://www.vlive.tv/video/1326',
         'md5': 'cc7314812855ce56de70a06a27314983',
         'info_dict': {
             'id': '1326',
@@ -28,50 +29,45 @@ class VLiveIE(InfoExtractor):
             'creator': 'Girl\'s Day',
         },
     }
-    _SECRET = 'rFkwZet6pqk1vQt6SxxUkAHX7YL3lmqzUMrU4IDusTo4jEBdtOhNfT4BYYAdArwH'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(
-            'http://m.vlive.tv/video/%s' % video_id,
+            'http://www.vlive.tv/video/%s' % video_id,
             video_id, note='Download video page')
 
+        long_video_id = self._search_regex(
+            r'vlive\.tv\.video\.ajax\.request\.handler\.init\("[0-9]+",\s?"[^"]*",\s?"([^"]+)",\s?"[^"]+",\s?"[^"]*",\s?"[^"]*"\)', webpage, 'long_video_id')
+
+        key = self._search_regex(
+            r'vlive\.tv\.video\.ajax\.request\.handler\.init\("[0-9]+",\s?"[^"]*",\s?"[^"]+",\s?"([^"]+)",\s?"[^"]*",\s?"[^"]*"\)', webpage, 'key')
+
         title = self._og_search_title(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
         creator = self._html_search_regex(
-            r'<span[^>]+class="name">([^<>]+)</span>', webpage, 'creator')
+            r'<div class="info_area">\s*<strong[^>]+class="name">([^<>]+)</strong>', webpage, 'creator',fatal=False)
 
-        url = 'http://global.apis.naver.com/globalV/globalV/vod/%s/playinfo?' % video_id
-        msgpad = '%.0f' % (time() * 1000)
-        md = b64encode(
-            hmac.new(self._SECRET.encode('ascii'),
-                     (url[:255] + msgpad).encode('ascii'), sha1).digest()
-        )
-        url += '&' + compat_urllib_parse.urlencode({'msgpad': msgpad, 'md': md})
+        # doct = document type (xml or json), cpt = caption type (vtt or ttml)
+        url = "http://global.apis.naver.com/rmcnmv/rmcnmv/vod_play_videoInfo.json?videoId=%s&key=%s&ptc=http&doct=json&cpt=vtt" % (long_video_id, key)
+        
         playinfo = self._download_json(url, video_id, 'Downloading video json')
 
-        if playinfo.get('message', '') != 'success':
-            raise ExtractorError(playinfo.get('message', 'JSON request unsuccessful'))
-
-        if not playinfo.get('result'):
-            raise ExtractorError('No videos found.')
-
         formats = []
-        for vid in playinfo['result'].get('videos', {}).get('list', []):
+        for vid in playinfo.get('videos', {}).get('list', []):
             formats.append({
                 'url': vid['source'],
                 'ext': 'mp4',
                 'abr': vid.get('bitrate', {}).get('audio'),
                 'vbr': vid.get('bitrate', {}).get('video'),
-                'format_id': vid['encodingOption']['name'],
-                'height': vid.get('height'),
-                'width': vid.get('width'),
+                'format_id': vid.get('encodingOption', {}).get('name'),
+                'height': int_or_none(vid.get('encodingOption', {}).get('height')),
+                'width': int_or_none(vid.get('encodingOption', {}).get('width')),
             })
         self._sort_formats(formats)
 
         subtitles = {}
-        for caption in playinfo['result'].get('captions', {}).get('list', []):
+        for caption in playinfo.get('captions', {}).get('list', []):
             subtitles[caption['language']] = [
                 {'ext': determine_ext(caption['source'], default_ext='vtt'),
                  'url': caption['source']}]

From cbecc9b9039d5166185a41ca4d9d6c4d11595c52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 06:12:53 +0600
Subject: [PATCH 0618/1105] [utils] Add dict_get convenience method

---
 test/test_utils.py  | 14 ++++++++++++++
 youtube_dl/utils.py |  9 +++++++++
 2 files changed, 23 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 1c3290d9b..e3dd019af 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -22,6 +22,7 @@ from youtube_dl.utils import (
     DateRange,
     detect_exe_version,
     determine_ext,
+    dict_get,
     encode_compat_str,
     encodeFilename,
     escape_rfc3986,
@@ -450,6 +451,19 @@ class TestUtil(unittest.TestCase):
         data = urlencode_postdata({'username': 'foo@bar.com', 'password': '1234'})
         self.assertTrue(isinstance(data, bytes))
 
+    def test_dict_get(self):
+        d = {
+            'a': 42,
+        }
+        self.assertEqual(dict_get(d, 'a'), 42)
+        self.assertEqual(dict_get(d, 'b'), None)
+        self.assertEqual(dict_get(d, 'b', 42), 42)
+        self.assertEqual(dict_get(d, ('a', )), 42)
+        self.assertEqual(dict_get(d, ('b', 'a', )), 42)
+        self.assertEqual(dict_get(d, ('b', 'c', 'a', 'd', )), 42)
+        self.assertEqual(dict_get(d, ('b', 'c', )), None)
+        self.assertEqual(dict_get(d, ('b', 'c', ), 42), 42)
+
     def test_encode_compat_str(self):
         self.assertEqual(encode_compat_str(b'\xd1\x82\xd0\xb5\xd1\x81\xd1\x82', 'utf-8'), 'тест')
         self.assertEqual(encode_compat_str('тест', 'utf-8'), 'тест')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4262ad6ac..652dba59d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1717,6 +1717,15 @@ def encode_dict(d, encoding='utf-8'):
     return dict((encode(k), encode(v)) for k, v in d.items())
 
 
+def dict_get(d, key_or_keys, default=None):
+    if isinstance(key_or_keys, (list, tuple)):
+        for key in key_or_keys:
+            if d.get(key):
+                return d[key]
+        return default
+    return d.get(key_or_keys, default)
+
+
 def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):
     return string if isinstance(string, compat_str) else compat_str(string, encoding, errors)
 

From 81e0b4f2d182d05ccadc2ef5140fabf2e8cbd60f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 06:14:26 +0600
Subject: [PATCH 0619/1105] Credit @EraYaN for vlive update (#8446)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index a46799506..7cac337a7 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -156,3 +156,4 @@ Tom Gijselinck
 Founder Fang
 Andrew Alexeyew
 Saso Bezlaj
+Erwin de Haan

From 52f5889f77e04420634d70145d689660ea6bfe24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 06:17:40 +0600
Subject: [PATCH 0620/1105] [vlive] Improve and extract more metadata (Closes
 #8446)

---
 youtube_dl/extractor/vlive.py | 80 +++++++++++++++++++----------------
 1 file changed, 43 insertions(+), 37 deletions(-)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index 3e1f8ef07..9e2aa58bd 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -1,32 +1,27 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import hmac
-from hashlib import sha1
-from base64 import b64encode
-from time import time
-
 from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
-    determine_ext,
-    int_or_none
+    dict_get,
+    float_or_none,
+    int_or_none,
 )
 from ..compat import compat_urllib_parse
 
 
 class VLiveIE(InfoExtractor):
     IE_NAME = 'vlive'
-    # vlive.tv/video/ links redirect to www.vlive.tv/video/ 
-    _VALID_URL = r'https?://(?:(www|m)\.)?vlive\.tv/video/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?vlive\.tv/video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.vlive.tv/video/1326',
         'md5': 'cc7314812855ce56de70a06a27314983',
         'info_dict': {
             'id': '1326',
             'ext': 'mp4',
-            'title': '[V] Girl\'s Day\'s Broadcast',
-            'creator': 'Girl\'s Day',
+            'title': "[V] Girl's Day's Broadcast",
+            'creator': "Girl's Day",
+            'view_count': int,
         },
     }
 
@@ -34,49 +29,60 @@ class VLiveIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(
-            'http://www.vlive.tv/video/%s' % video_id,
-            video_id, note='Download video page')
+            'http://www.vlive.tv/video/%s' % video_id, video_id)
 
         long_video_id = self._search_regex(
-            r'vlive\.tv\.video\.ajax\.request\.handler\.init\("[0-9]+",\s?"[^"]*",\s?"([^"]+)",\s?"[^"]+",\s?"[^"]*",\s?"[^"]*"\)', webpage, 'long_video_id')
+            r'vlive\.tv\.video\.ajax\.request\.handler\.init\(\s*"[0-9]+"\s*,\s*"[^"]*"\s*,\s*"([^"]+)"',
+            webpage, 'long video id')
 
         key = self._search_regex(
-            r'vlive\.tv\.video\.ajax\.request\.handler\.init\("[0-9]+",\s?"[^"]*",\s?"[^"]+",\s?"([^"]+)",\s?"[^"]*",\s?"[^"]*"\)', webpage, 'key')
+            r'vlive\.tv\.video\.ajax\.request\.handler\.init\(\s*"[0-9]+"\s*,\s*"[^"]*"\s*,\s*"[^"]+"\s*,\s*"([^"]+)"',
+            webpage, 'key')
 
         title = self._og_search_title(webpage)
+
+        playinfo = self._download_json(
+            'http://global.apis.naver.com/rmcnmv/rmcnmv/vod_play_videoInfo.json?%s'
+            % compat_urllib_parse.urlencode({
+                'videoId': long_video_id,
+                'key': key,
+                'ptc': 'http',
+                'doct': 'json',  # document type (xml or json)
+                'cpt': 'vtt',  # captions type (vtt or ttml)
+            }), video_id)
+
+        formats = [{
+            'url': vid['source'],
+            'format_id': vid.get('encodingOption', {}).get('name'),
+            'abr': float_or_none(vid.get('bitrate', {}).get('audio')),
+            'vbr': float_or_none(vid.get('bitrate', {}).get('video')),
+            'width': int_or_none(vid.get('encodingOption', {}).get('width')),
+            'height': int_or_none(vid.get('encodingOption', {}).get('height')),
+            'filesize': int_or_none(vid.get('size')),
+        } for vid in playinfo.get('videos', {}).get('list', []) if vid.get('source')]
+        self._sort_formats(formats)
+
         thumbnail = self._og_search_thumbnail(webpage)
         creator = self._html_search_regex(
-            r'<div class="info_area">\s*<strong[^>]+class="name">([^<>]+)</strong>', webpage, 'creator',fatal=False)
+            r'<div[^>]+class="info_area"[^>]*>\s*<strong[^>]+class="name"[^>]*>([^<]+)</strong>',
+            webpage, 'creator', fatal=False)
 
-        # doct = document type (xml or json), cpt = caption type (vtt or ttml)
-        url = "http://global.apis.naver.com/rmcnmv/rmcnmv/vod_play_videoInfo.json?videoId=%s&key=%s&ptc=http&doct=json&cpt=vtt" % (long_video_id, key)
-        
-        playinfo = self._download_json(url, video_id, 'Downloading video json')
-
-        formats = []
-        for vid in playinfo.get('videos', {}).get('list', []):
-            formats.append({
-                'url': vid['source'],
-                'ext': 'mp4',
-                'abr': vid.get('bitrate', {}).get('audio'),
-                'vbr': vid.get('bitrate', {}).get('video'),
-                'format_id': vid.get('encodingOption', {}).get('name'),
-                'height': int_or_none(vid.get('encodingOption', {}).get('height')),
-                'width': int_or_none(vid.get('encodingOption', {}).get('width')),
-            })
-        self._sort_formats(formats)
+        view_count = int_or_none(playinfo.get('meta', {}).get('count'))
 
         subtitles = {}
         for caption in playinfo.get('captions', {}).get('list', []):
-            subtitles[caption['language']] = [
-                {'ext': determine_ext(caption['source'], default_ext='vtt'),
-                 'url': caption['source']}]
+            lang = dict_get(caption, ('language', 'locale', 'country', 'label'))
+            if lang and caption.get('source'):
+                subtitles[lang] = [{
+                    'ext': 'vtt',
+                    'url': caption['source']}]
 
         return {
             'id': video_id,
             'title': title,
             'creator': creator,
             'thumbnail': thumbnail,
+            'view_count': view_count,
             'formats': formats,
             'subtitles': subtitles,
         }

From 86296ad2cd5702a66b05aae79ec9196b62e41e9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 08:13:04 +0600
Subject: [PATCH 0621/1105] [utils] Add ability to control skipping false
 values in dict_get

---
 test/test_utils.py  | 13 +++++++++++--
 youtube_dl/utils.py |  7 ++++---
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index e3dd019af..909d0e51d 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -452,9 +452,15 @@ class TestUtil(unittest.TestCase):
         self.assertTrue(isinstance(data, bytes))
 
     def test_dict_get(self):
-        d = {
-            'a': 42,
+        FALSE_VALUES = {
+            'none': None,
+            'false': False,
+            'zero': 0,
+            'empty_string': '',
+            'empty_list': [],
         }
+        d = FALSE_VALUES.copy()
+        d['a'] = 42
         self.assertEqual(dict_get(d, 'a'), 42)
         self.assertEqual(dict_get(d, 'b'), None)
         self.assertEqual(dict_get(d, 'b', 42), 42)
@@ -463,6 +469,9 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(dict_get(d, ('b', 'c', 'a', 'd', )), 42)
         self.assertEqual(dict_get(d, ('b', 'c', )), None)
         self.assertEqual(dict_get(d, ('b', 'c', ), 42), 42)
+        for key, false_value in FALSE_VALUES.items():
+            self.assertEqual(dict_get(d, ('b', 'c', key, )), None)
+            self.assertEqual(dict_get(d, ('b', 'c', key, ), skip_false_values=False), false_value)
 
     def test_encode_compat_str(self):
         self.assertEqual(encode_compat_str(b'\xd1\x82\xd0\xb5\xd1\x81\xd1\x82', 'utf-8'), 'тест')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 652dba59d..f3b0180ab 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1717,11 +1717,12 @@ def encode_dict(d, encoding='utf-8'):
     return dict((encode(k), encode(v)) for k, v in d.items())
 
 
-def dict_get(d, key_or_keys, default=None):
+def dict_get(d, key_or_keys, default=None, skip_false_values=True):
     if isinstance(key_or_keys, (list, tuple)):
         for key in key_or_keys:
-            if d.get(key):
-                return d[key]
+            if key not in d or d[key] is None or skip_false_values and not d[key]:
+                continue
+            return d[key]
         return default
     return d.get(key_or_keys, default)
 

From f0905c6ec3e1ad20f284edcd1d9c58a43b23bac9 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 18:45:44 +0600
Subject: [PATCH 0622/1105] [README.md] Document more sequences in output
 template

---
 README.md | 58 ++++++++++++++++++++++++++++++++++---------------------
 1 file changed, 36 insertions(+), 22 deletions(-)

diff --git a/README.md b/README.md
index 7a180dccf..78fc42c28 100644
--- a/README.md
+++ b/README.md
@@ -442,32 +442,46 @@ On Windows you may also need to setup the `%HOME%` environment variable manually
 
 The `-o` option allows users to indicate a template for the output file names. The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "http://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences have the format `%(NAME)s`. To clarify, that is a percent symbol followed by a name in parentheses, followed by a lowercase S. Allowed names are:
 
- - `id`: Video identifier.
- - `url`: Video URL.
- - `uploader`: Nickname of the person who uploaded the video.
- - `upload_date`: Upload date in YYYYMMDD format.
- - `title`: Video title.
- - `ext`: Video filename extension.
- - `epoch`: Unix epoch when creating the file.
- - `autonumber`: Five-digit number that will be increased with each download, starting at zero.
- - `playlist`: Name or id of the playlist that contains the video.
- - `playlist_index`: Index of the video in the playlist padded with leading zeros according to the total length of the playlist.
- - `format_id`: Format code specified by `--format`.
- - `duration`: Length of the video in seconds.
+ - `id`: Video identifier
+ - `title`: Video title
+ - `url`: Video URL
+ - `ext`: Video filename extension
+ - `alt_title`: A secondary title of the video
+ - `display_id`: An alternative identifier for the video
+ - `uploader`: Full name of the video uploader
+ - `creator`: The main artist who created the video
+ - `release_date`: The date (YYYYMMDD) when the video was released
+ - `timestamp`: UNIX timestamp of the moment the video became available
+ - `upload_date`: Video upload date (YYYYMMDD)
+ - `uploader_id`: Nickname or id of the video uploader
+ - `location`: Physical location where the video was filmed
+ - `duration`: Length of the video in seconds
+ - `view_count`: How many users have watched the video on the platform
+ - `like_count`: Number of positive ratings of the video
+ - `dislike_count`: Number of negative ratings of the video
+ - `repost_count`: Number of reposts of the video
+ - `average_rating`: Average rating give by users, the scale used depends on the webpage
+ - `comment_count`: Number of comments on the video
+ - `age_limit`: Age restriction for the video (years)
+ - `epoch`: Unix epoch when creating the file
+ - `autonumber`: Five-digit number that will be increased with each download, starting at zero
+ - `playlist`: Name or id of the playlist that contains the video
+ - `playlist_index`: Index of the video in the playlist padded with leading zeros according to the total length of the playlist
+ - `format_id`: Format code specified by `--format`
 
 Available for the video that belongs to some logical chapter or section:
- - `chapter`: Name or title of the chapter the video belongs to.
- - `chapter_number`: Number of the chapter the video belongs to.
- - `chapter_id`: Id of the chapter the video belongs to.
+ - `chapter`: Name or title of the chapter the video belongs to
+ - `chapter_number`: Number of the chapter the video belongs to
+ - `chapter_id`: Id of the chapter the video belongs to
 
 Available for the video that is an episode of some series or programme:
- - `series`: Title of the series or programme the video episode belongs to.
- - `season`: Title of the season the video episode belongs to.
- - `season_number`: Number of the season the video episode belongs to.
- - `season_id`: Id of the season the video episode belongs to.
- - `episode`: Title of the video episode.
- - `episode_number`: Number of the video episode within a season.
- - `episode_id`: Id of the video episode, as a unicode string.
+ - `series`: Title of the series or programme the video episode belongs to
+ - `season`: Title of the season the video episode belongs to
+ - `season_number`: Number of the season the video episode belongs to
+ - `season_id`: Id of the season the video episode belongs to
+ - `episode`: Title of the video episode
+ - `episode_number`: Number of the video episode within a season
+ - `episode_id`: Id of the video episode
 
 Each aforementioned sequence when referenced in output template will be replaced by the actual value corresponding to the sequence name. Note that some of the sequences are not guaranteed to be present since they depend on the metadata obtained by particular extractor, such sequences will be replaced with `NA`.
 

From 0474512e3029a7a3121fbe8f08c9ff226deecc7d Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 19:00:59 +0600
Subject: [PATCH 0623/1105] [README.md] Document even more sequences in output
 template

---
 README.md | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 78fc42c28..75f53e51f 100644
--- a/README.md
+++ b/README.md
@@ -463,11 +463,27 @@ The `-o` option allows users to indicate a template for the output file names. T
  - `average_rating`: Average rating give by users, the scale used depends on the webpage
  - `comment_count`: Number of comments on the video
  - `age_limit`: Age restriction for the video (years)
+ - `format`: A human-readable description of the format 
+ - `format_id`: Format code specified by `--format`
+ - `format_note`: Additional info about the format
+ - `width`: Width of the video
+ - `height`: Height of the video
+ - `resolution`: Textual description of width and height
+ - `tbr`: Average bitrate of audio and video in KBit/s
+ - `abr`: Average audio bitrate in KBit/s
+ - `acodec`: Name of the audio codec in use
+ - `asr`: Audio sampling rate in Hertz
+ - `vbr`: Average video bitrate in KBit/s
+ - `fps`: Frame rate
+ - `vcodec`: Name of the video codec in use
+ - `container`: Name of the container format
+ - `filesize`: The number of bytes, if known in advance
+ - `filesize_approx`: An estimate for the number of bytes
+ - `protocol`: The protocol that will be used for the actual download
  - `epoch`: Unix epoch when creating the file
  - `autonumber`: Five-digit number that will be increased with each download, starting at zero
  - `playlist`: Name or id of the playlist that contains the video
  - `playlist_index`: Index of the video in the playlist padded with leading zeros according to the total length of the playlist
- - `format_id`: Format code specified by `--format`
 
 Available for the video that belongs to some logical chapter or section:
  - `chapter`: Name or title of the chapter the video belongs to

From 3b799ca14cfeb1d17da637b2539044758d7355e6 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 19:06:42 +0600
Subject: [PATCH 0624/1105] [README.md] Clarify percent literal and output to
 stdout

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 75f53e51f..93a88e79b 100644
--- a/README.md
+++ b/README.md
@@ -505,6 +505,8 @@ For example for `-o %(title)s-%(id)s.%(ext)s` and mp4 video with title `youtube-
 
 Output template can also contain arbitrary hierarchical path, e.g. `-o %(playlist)s/%(playlist_index)s - %(title)s.%(ext)s` that will result in downloading each video in a directory corresponding to this path template. Any missing directory will be automatically created for you.
 
+To specify percent literal in output template use `%%`. To output to stdout use `-o -`.
+
 The current default template is `%(title)s-%(id)s.%(ext)s`.
 
 In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title:

From 72b9ebc65da3f1cb2f85d85b002674c8e5ae36ce Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 19:08:54 +0600
Subject: [PATCH 0625/1105] [README.md] Document extractor sequences in output
 template

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 93a88e79b..fe301ef53 100644
--- a/README.md
+++ b/README.md
@@ -480,6 +480,8 @@ The `-o` option allows users to indicate a template for the output file names. T
  - `filesize`: The number of bytes, if known in advance
  - `filesize_approx`: An estimate for the number of bytes
  - `protocol`: The protocol that will be used for the actual download
+ - `extractor`: Name of the extractor
+ - `extractor_key`: Key name of the extractor
  - `epoch`: Unix epoch when creating the file
  - `autonumber`: Five-digit number that will be increased with each download, starting at zero
  - `playlist`: Name or id of the playlist that contains the video

From 8411229bd5e147ef1ac3420000c7c28ba660df2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 19:47:09 +0600
Subject: [PATCH 0626/1105] [utils] Allow dot in strip_jsonp

---
 test/test_utils.py  | 4 ++++
 youtube_dl/utils.py | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 909d0e51d..a1e416dd5 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -494,6 +494,10 @@ class TestUtil(unittest.TestCase):
         d = json.loads(stripped)
         self.assertEqual(d, {'STATUS': 'OK'})
 
+        stripped = strip_jsonp('ps.embedHandler({"status": "success"});')
+        d = json.loads(stripped)
+        self.assertEqual(d, {'status': 'success'})
+
     def test_uppercase_escape(self):
         self.assertEqual(uppercase_escape('aä'), 'aä')
         self.assertEqual(uppercase_escape('\\U0001d550'), '𝕐')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index f3b0180ab..dce67de8a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1749,7 +1749,7 @@ def parse_age_limit(s):
 
 def strip_jsonp(code):
     return re.sub(
-        r'(?s)^[a-zA-Z0-9_]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
+        r'(?s)^[a-zA-Z0-9_.]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
 
 
 def js_to_json(code):

From 8628d26f3888d2f0541a5d0e5d7c35fa5cac3fc0 Mon Sep 17 00:00:00 2001
From: ovitei <sh33p0@gmail.com>
Date: Sun, 31 Jan 2016 03:08:34 +0100
Subject: [PATCH 0627/1105] [KonserthusetPlay] Add new extractor (partial
 support)

---
 youtube_dl/extractor/__init__.py         |  1 +
 youtube_dl/extractor/konserthusetplay.py | 45 ++++++++++++++++++++++++
 2 files changed, 46 insertions(+)
 create mode 100644 youtube_dl/extractor/konserthusetplay.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 2fbc7f812..18951c287 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -328,6 +328,7 @@ from .keezmovies import KeezMoviesIE
 from .khanacademy import KhanAcademyIE
 from .kickstarter import KickStarterIE
 from .keek import KeekIE
+from .konserthusetplay import KonserthusetPlayIE
 from .kontrtube import KontrTubeIE
 from .krasview import KrasViewIE
 from .ku6 import Ku6IE
diff --git a/youtube_dl/extractor/konserthusetplay.py b/youtube_dl/extractor/konserthusetplay.py
new file mode 100644
index 000000000..8200942d3
--- /dev/null
+++ b/youtube_dl/extractor/konserthusetplay.py
@@ -0,0 +1,45 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class KonserthusetPlayIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?konserthusetplay\.se/\?m=(?P<id>[0-9A-Za-z_-]+)'
+
+    _TESTS = [{
+        'url': 'http://www.konserthusetplay.se/?m=CKDDnlCY-dhWAAqiMERd-A',
+        'md5': 'e272a765e0d12a0226199e5f32d76116',
+        'info_dict': {
+            'id': 'CKDDnlCY-dhWAAqiMERd-A',
+            'ext': 'mp4',
+            'title': 'Orkesterns instrument: Valthornen',
+            'description': 'md5:f10e1f0030202020396a4d712d2fa827',
+            'thumbnail': 'http://csp.picsearch.com/img/C/K/D/D/title_CKDDnlCY-dhWAAqiMERd-A'
+        }
+    }, {
+        'url': 'http://www.konserthusetplay.se/?m=IyQcMOEpmKqT91SVT5OP8Q',
+        'md5': 'c4adb8ca76fdd33d4cbdcc7c3d181f22',
+        'info_dict': {
+            'id': 'IyQcMOEpmKqT91SVT5OP8Q',
+            'ext': 'mp4',
+            'title': 'Eliasson Einsame Fahrt, violinkonsert',
+            'description': 'md5:a8dcc8dfd9473d52433b2c5f588ba191',
+            'thumbnail': 'http://csp.picsearch.com/img/I/y/Q/c/title_IyQcMOEpmKqT91SVT5OP8Q'
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        description = self._og_search_description(webpage)
+        title = self._og_search_title(webpage)
+        main_video = self._html_search_regex(r'<link rel="video_src" href="(.+?)" />', webpage, 'url')
+        thumbnail = self._og_search_thumbnail(webpage)
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'url': main_video,
+            'thumbnail': thumbnail
+        }

From 5a7699bb2ef77c2548bcc2112e042481a809ba35 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 21:11:59 +0600
Subject: [PATCH 0628/1105] [konserthusetplay] Improve and extract all formats
 (Closes #8381)

---
 youtube_dl/extractor/konserthusetplay.py | 111 ++++++++++++++++++-----
 1 file changed, 87 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/konserthusetplay.py b/youtube_dl/extractor/konserthusetplay.py
index 8200942d3..f21d050d7 100644
--- a/youtube_dl/extractor/konserthusetplay.py
+++ b/youtube_dl/extractor/konserthusetplay.py
@@ -2,44 +2,107 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    int_or_none,
+)
 
 
 class KonserthusetPlayIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?konserthusetplay\.se/\?m=(?P<id>[0-9A-Za-z_-]+)'
-
-    _TESTS = [{
+    _VALID_URL = r'https?://(?:www\.)?konserthusetplay\.se/\?m=(?P<id>[^&]+)'
+    _TEST = {
         'url': 'http://www.konserthusetplay.se/?m=CKDDnlCY-dhWAAqiMERd-A',
-        'md5': 'e272a765e0d12a0226199e5f32d76116',
         'info_dict': {
             'id': 'CKDDnlCY-dhWAAqiMERd-A',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Orkesterns instrument: Valthornen',
             'description': 'md5:f10e1f0030202020396a4d712d2fa827',
-            'thumbnail': 'http://csp.picsearch.com/img/C/K/D/D/title_CKDDnlCY-dhWAAqiMERd-A'
-        }
-    }, {
-        'url': 'http://www.konserthusetplay.se/?m=IyQcMOEpmKqT91SVT5OP8Q',
-        'md5': 'c4adb8ca76fdd33d4cbdcc7c3d181f22',
-        'info_dict': {
-            'id': 'IyQcMOEpmKqT91SVT5OP8Q',
-            'ext': 'mp4',
-            'title': 'Eliasson Einsame Fahrt, violinkonsert',
-            'description': 'md5:a8dcc8dfd9473d52433b2c5f588ba191',
-            'thumbnail': 'http://csp.picsearch.com/img/I/y/Q/c/title_IyQcMOEpmKqT91SVT5OP8Q'
-        }
-    }]
+            'thumbnail': 're:^https?://.*$',
+            'duration': 398.8,
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
-        description = self._og_search_description(webpage)
-        title = self._og_search_title(webpage)
-        main_video = self._html_search_regex(r'<link rel="video_src" href="(.+?)" />', webpage, 'url')
-        thumbnail = self._og_search_thumbnail(webpage)
+
+        e = self._search_regex(
+            r'https?://csp\.picsearch\.com/rest\?.*\be=(.+?)[&"\']', webpage, 'e')
+
+        rest = self._download_json(
+            'http://csp.picsearch.com/rest?e=%s&containerId=mediaplayer&i=object' % e,
+            video_id, transform_source=lambda s: s[s.index('{'):s.rindex('}') + 1])
+
+        media = rest['media']
+        player_config = media['playerconfig']
+        playlist = player_config['playlist']
+
+        source = next(f for f in playlist if f.get('bitrates'))
+
+        connection_url = (player_config.get('rtmp', {}).get(
+            'netConnectionUrl') or player_config.get(
+            'plugins', {}).get('bwcheck', {}).get('netConnectionUrl'))
+
+        FORMAT_ID_REGEX = r'_([^_]+)_h264m\.mp4'
+
+        formats = []
+
+        fallback_url = source.get('fallbackUrl')
+        fallback_format_id = None
+        if fallback_url:
+            fallback_format_id = self._search_regex(
+                FORMAT_ID_REGEX, fallback_url, 'format id', default=None)
+
+        if connection_url:
+            for f in source['bitrates']:
+                video_url = f.get('url')
+                if not video_url:
+                    continue
+                format_id = self._search_regex(
+                    FORMAT_ID_REGEX, video_url, 'format id', default=None)
+                f_common = {
+                    'vbr': int_or_none(f.get('bitrate')),
+                    'width': int_or_none(f.get('width')),
+                    'height': int_or_none(f.get('height')),
+                }
+                f = f_common.copy()
+                f.update({
+                    'url': connection_url,
+                    'play_path': video_url,
+                    'format_id': 'rtmp-%s' % format_id if format_id else 'rtmp',
+                    'ext': 'flv',
+                })
+                formats.append(f)
+                if format_id and format_id == fallback_format_id:
+                    f = f_common.copy()
+                    f.update({
+                        'url': fallback_url,
+                        'format_id': 'http-%s' % format_id if format_id else 'http',
+                    })
+                    formats.append(f)
+
+        if not formats and fallback_url:
+            formats.append({
+                'url': fallback_url,
+            })
+
+        self._sort_formats(formats)
+
+        title = player_config.get('title') or media['title']
+        description = player_config.get('mediaInfo', {}).get('description')
+        thumbnail = media.get('image')
+        duration = float_or_none(media.get('duration'), 1000)
+
         return {
             'id': video_id,
             'title': title,
             'description': description,
-            'url': main_video,
-            'thumbnail': thumbnail
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
         }

From f070197bd7b0de49672c4ff151887355d5e6a2ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 21:16:31 +0600
Subject: [PATCH 0629/1105] [konserthusetplay] Improve _VALID_URL

---
 youtube_dl/extractor/konserthusetplay.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/konserthusetplay.py b/youtube_dl/extractor/konserthusetplay.py
index f21d050d7..e1395b24e 100644
--- a/youtube_dl/extractor/konserthusetplay.py
+++ b/youtube_dl/extractor/konserthusetplay.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class KonserthusetPlayIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?konserthusetplay\.se/\?m=(?P<id>[^&]+)'
+    _VALID_URL = r'https?://(?:www\.)?konserthusetplay\.se/\?.*\bm=(?P<id>[^&]+)'
     _TEST = {
         'url': 'http://www.konserthusetplay.se/?m=CKDDnlCY-dhWAAqiMERd-A',
         'info_dict': {

From b6bfa6fb7953b8dc477a932acea2df2d73175bc2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 21:18:32 +0600
Subject: [PATCH 0630/1105] [konserthusetplay] Reorder code pieces

---
 youtube_dl/extractor/konserthusetplay.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/konserthusetplay.py b/youtube_dl/extractor/konserthusetplay.py
index e1395b24e..55291c66f 100644
--- a/youtube_dl/extractor/konserthusetplay.py
+++ b/youtube_dl/extractor/konserthusetplay.py
@@ -44,10 +44,6 @@ class KonserthusetPlayIE(InfoExtractor):
 
         source = next(f for f in playlist if f.get('bitrates'))
 
-        connection_url = (player_config.get('rtmp', {}).get(
-            'netConnectionUrl') or player_config.get(
-            'plugins', {}).get('bwcheck', {}).get('netConnectionUrl'))
-
         FORMAT_ID_REGEX = r'_([^_]+)_h264m\.mp4'
 
         formats = []
@@ -58,6 +54,9 @@ class KonserthusetPlayIE(InfoExtractor):
             fallback_format_id = self._search_regex(
                 FORMAT_ID_REGEX, fallback_url, 'format id', default=None)
 
+        connection_url = (player_config.get('rtmp', {}).get(
+            'netConnectionUrl') or player_config.get(
+            'plugins', {}).get('bwcheck', {}).get('netConnectionUrl'))
         if connection_url:
             for f in source['bitrates']:
                 video_url = f.get('url')

From 915dd777830e7e309349c589f91f21e29224530c Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 21:21:14 +0600
Subject: [PATCH 0631/1105] [README.md] Add output template example for
 streaming to stdout

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index fe301ef53..4070aebf8 100644
--- a/README.md
+++ b/README.md
@@ -530,6 +530,9 @@ $ youtube-dl -u user -p password -o '~/MyVideos/%(playlist)s/%(chapter_number)s
 
 # Download entire series season keeping each series and each season in separate directory under C:/MyVideos
 $ youtube-dl -o 'C:/MyVideos/%(series)s/%(season_number)s - %(season)s/%(episode_number)s - %(episode)s.%(ext)s' http://videomore.ru/kino_v_detalayah/5_sezon/367617
+
+# Stream the video being downloaded to stdout
+$ youtube-dl -o - BaW_jenozKc
 ```
 
 # FORMAT SELECTION

From 11c0339bec676a56ad8c5582bb145a084d04b94d Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 21:22:33 +0600
Subject: [PATCH 0632/1105] [README.md] Clarify quotes in output template

---
 README.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 4070aebf8..a027bb7f9 100644
--- a/README.md
+++ b/README.md
@@ -505,7 +505,7 @@ Each aforementioned sequence when referenced in output template will be replaced
 
 For example for `-o %(title)s-%(id)s.%(ext)s` and mp4 video with title `youtube-dl test video` and id `BaW_jenozKcj` this will result in a `youtube-dl test video-BaW_jenozKcj.mp4` file created in the current directory.
 
-Output template can also contain arbitrary hierarchical path, e.g. `-o %(playlist)s/%(playlist_index)s - %(title)s.%(ext)s` that will result in downloading each video in a directory corresponding to this path template. Any missing directory will be automatically created for you.
+Output template can also contain arbitrary hierarchical path, e.g. `-o '%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s'` that will result in downloading each video in a directory corresponding to this path template. Any missing directory will be automatically created for you.
 
 To specify percent literal in output template use `%%`. To output to stdout use `-o -`.
 
@@ -513,13 +513,13 @@ The current default template is `%(title)s-%(id)s.%(ext)s`.
 
 In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title:
 
-Examples:
+Examples (note on Windows you may need to use double quotes instead of single)::
 
 ```bash
-$ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc
+$ youtube-dl --get-filename -o '%(title)s.%(ext)s' BaW_jenozKc
 youtube-dl test video ''_ä↭𝕐.mp4    # All kinds of weird characters
 
-$ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc --restrict-filenames
+$ youtube-dl --get-filename -o '%(title)s.%(ext)s' BaW_jenozKc --restrict-filenames
 youtube-dl_test_video_.mp4          # A simple file name
 
 # Download YouTube playlist videos in separate directory indexed by video order in a playlist
@@ -529,7 +529,7 @@ $ youtube-dl -o '%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s' https://ww
 $ youtube-dl -u user -p password -o '~/MyVideos/%(playlist)s/%(chapter_number)s - %(chapter)s/%(title)s.%(ext)s' https://www.udemy.com/java-tutorial/
 
 # Download entire series season keeping each series and each season in separate directory under C:/MyVideos
-$ youtube-dl -o 'C:/MyVideos/%(series)s/%(season_number)s - %(season)s/%(episode_number)s - %(episode)s.%(ext)s' http://videomore.ru/kino_v_detalayah/5_sezon/367617
+$ youtube-dl -o "C:/MyVideos/%(series)s/%(season_number)s - %(season)s/%(episode_number)s - %(episode)s.%(ext)s" http://videomore.ru/kino_v_detalayah/5_sezon/367617
 
 # Stream the video being downloaded to stdout
 $ youtube-dl -o - BaW_jenozKc

From 8d0a2a2a4e3e930f9916a96d8ef7bf9c970be088 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 21:23:29 +0600
Subject: [PATCH 0633/1105] [README.md] Fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a027bb7f9..65e0ee963 100644
--- a/README.md
+++ b/README.md
@@ -513,7 +513,7 @@ The current default template is `%(title)s-%(id)s.%(ext)s`.
 
 In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title:
 
-Examples (note on Windows you may need to use double quotes instead of single)::
+Examples (note on Windows you may need to use double quotes instead of single):
 
 ```bash
 $ youtube-dl --get-filename -o '%(title)s.%(ext)s' BaW_jenozKc

From 3834d3e35c5ccd957a3bc7bb1c12607f9d96ccbd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 Feb 2016 01:30:57 +0600
Subject: [PATCH 0634/1105] [youtube] Clarify itag 36 height and abr (Closes
 #8457)

---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 63abe5477..b9a91dea2 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -286,7 +286,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
-        '36': {'ext': '3gp', 'width': 320, 'height': 240, 'acodec': 'aac', 'abr': 32, 'vcodec': 'mp4v'},
+        # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
+        '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},

From 35972ba17214608d8929d6532831abf7402afcf8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 Feb 2016 21:30:23 +0600
Subject: [PATCH 0635/1105] [vk] Improve rutube embeds detection (Closes #8461)

---
 youtube_dl/extractor/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 11014865a..0805e3c08 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -265,7 +265,7 @@ class VKIE(InfoExtractor):
             return self.url_result(pladform_url)
 
         m_rutube = re.search(
-            r'\ssrc="((?:https?:)?//rutube\.ru\\?/video\\?/embed(?:.*?))\\?"', info_page)
+            r'\ssrc="((?:https?:)?//rutube\.ru\\?/(?:video|play)\\?/embed(?:.*?))\\?"', info_page)
         if m_rutube is not None:
             rutube_url = self._proto_relative_url(
                 m_rutube.group(1).replace('\\', ''))

From 085ad71157dd6588cece2df0bfba6815f234564f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 9 Feb 2016 12:57:51 +0100
Subject: [PATCH 0636/1105] release 2016.02.09

---
 docs/supportedsites.md | 7 +++++--
 youtube_dl/version.py  | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index ee34adf26..c48561eff 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -262,7 +262,7 @@
  - **Instagram**
  - **instagram:user**: Instagram user profile
  - **InternetVideoArchive**
- - **IPrima** (Currently broken)
+ - **IPrima**
  - **iqiyi**: 爱奇艺
  - **Ir90Tv**
  - **ivi**: ivi.ru
@@ -283,6 +283,7 @@
  - **KeezMovies**
  - **KhanAcademy**
  - **KickStarter**
+ - **KonserthusetPlay**
  - **kontrtube**: KontrTube.ru - Труба зовёт
  - **KrasView**: Красвью
  - **Ku6**
@@ -681,7 +682,9 @@
  - **VideoPremium**
  - **VideoTt**: video.tt - Your True Tube (Currently broken)
  - **videoweed**: VideoWeed
- - **Vidme**
+ - **vidme**
+ - **vidme:user**
+ - **vidme:user:likes**
  - **Vidzi**
  - **vier**
  - **vier:videos**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3fec14ab1..53b210290 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.02.05.1'
+__version__ = '2016.02.09'

From 16f1430ba6fc1dc3fe0baf587f4461f37b62466a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Feb 2016 21:14:02 +0600
Subject: [PATCH 0637/1105] [mailru] Prefer metaUrl API (Closes #8474)

---
 youtube_dl/extractor/mailru.py | 36 ++++++++++++++++++++++++++++++++--
 1 file changed, 34 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mailru.py b/youtube_dl/extractor/mailru.py
index ab1300185..09424620b 100644
--- a/youtube_dl/extractor/mailru.py
+++ b/youtube_dl/extractor/mailru.py
@@ -42,6 +42,22 @@ class MailRuIE(InfoExtractor):
             },
             'skip': 'Not accessible from Travis CI server',
         },
+        {
+            # only available via metaUrl API
+            'url': 'http://my.mail.ru/mail/720pizle/video/_myvideo/502.html',
+            'md5': '3b26d2491c6949d031a32b96bd97c096',
+            'info_dict': {
+                'id': '56664382_502',
+                'ext': 'mp4',
+                'title': ':8336',
+                'timestamp': 1449094163,
+                'upload_date': '20151202',
+                'uploader': '720pizle@mail.ru',
+                'uploader_id': '720pizle@mail.ru',
+                'duration': 6001,
+            },
+            'skip': 'Not accessible from Travis CI server',
+        }
     ]
 
     def _real_extract(self, url):
@@ -51,8 +67,24 @@ class MailRuIE(InfoExtractor):
         if not video_id:
             video_id = mobj.group('idv2prefix') + mobj.group('idv2suffix')
 
-        video_data = self._download_json(
-            'http://api.video.mail.ru/videos/%s.json?new=1' % video_id, video_id, 'Downloading video JSON')
+        webpage = self._download_webpage(url, video_id)
+
+        video_data = None
+
+        page_config = self._parse_json(self._search_regex(
+            r'(?s)<script[^>]+class="sp-video__page-config"[^>]*>(.+?)</script>',
+            webpage, 'page config', default='{}'), video_id, fatal=False)
+        if page_config:
+            meta_url = page_config.get('metaUrl') or page_config.get('video', {}).get('metaUrl')
+            if meta_url:
+                video_data = self._download_json(
+                    meta_url, video_id, 'Downloading video meta JSON', fatal=False)
+
+        # Fallback old approach
+        if not video_data:
+            video_data = self._download_json(
+                'http://api.video.mail.ru/videos/%s.json?new=1' % video_id,
+                video_id, 'Downloading video JSON')
 
         author = video_data['author']
         uploader = author['name']

From b081350bd943d7595acda70240d2aaa749fc34b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Feb 2016 21:30:48 +0600
Subject: [PATCH 0638/1105] [mailru] Improve and modernize

---
 youtube_dl/extractor/mailru.py | 51 +++++++++++++++++++++-------------
 1 file changed, 31 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/mailru.py b/youtube_dl/extractor/mailru.py
index 09424620b..b7671f16e 100644
--- a/youtube_dl/extractor/mailru.py
+++ b/youtube_dl/extractor/mailru.py
@@ -4,6 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    remove_end,
+)
 
 
 class MailRuIE(InfoExtractor):
@@ -86,29 +90,36 @@ class MailRuIE(InfoExtractor):
                 'http://api.video.mail.ru/videos/%s.json?new=1' % video_id,
                 video_id, 'Downloading video JSON')
 
-        author = video_data['author']
-        uploader = author['name']
-        uploader_id = author.get('id') or author.get('email')
-        view_count = video_data.get('views_count')
+        formats = []
+        for f in video_data['videos']:
+            video_url = f.get('url')
+            if not video_url:
+                continue
+            format_id = f.get('key')
+            height = int_or_none(self._search_regex(
+                r'^(\d+)[pP]$', format_id, 'height', default=None)) if format_id else None
+            formats.append({
+                'url': video_url,
+                'format_id': format_id,
+                'height': height,
+            })
+        self._sort_formats(formats)
 
         meta_data = video_data['meta']
-        content_id = '%s_%s' % (
-            meta_data.get('accId', ''), meta_data['itemId'])
-        title = meta_data['title']
-        if title.endswith('.mp4'):
-            title = title[:-4]
-        thumbnail = meta_data['poster']
-        duration = meta_data['duration']
-        timestamp = meta_data['timestamp']
+        title = remove_end(meta_data['title'], '.mp4')
 
-        formats = [
-            {
-                'url': video['url'],
-                'format_id': video['key'],
-                'height': int(video['key'].rstrip('p'))
-            } for video in video_data['videos']
-        ]
-        self._sort_formats(formats)
+        author = video_data.get('author')
+        uploader = author.get('name')
+        uploader_id = author.get('id') or author.get('email')
+        view_count = int_or_none(video_data.get('viewsCount') or video_data.get('views_count'))
+
+        acc_id = meta_data.get('accId')
+        item_id = meta_data.get('itemId')
+        content_id = '%s_%s' % (acc_id, item_id) if acc_id and item_id else video_id
+
+        thumbnail = meta_data.get('poster')
+        duration = int_or_none(meta_data.get('duration'))
+        timestamp = int_or_none(meta_data.get('timestamp'))
 
         return {
             'id': content_id,

From 3967a761f484b8092e83e8c255f75adeb9ce3d90 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Feb 2016 21:31:51 +0600
Subject: [PATCH 0639/1105] [mailru] Fix tests

---
 youtube_dl/extractor/mailru.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/mailru.py b/youtube_dl/extractor/mailru.py
index b7671f16e..71085f279 100644
--- a/youtube_dl/extractor/mailru.py
+++ b/youtube_dl/extractor/mailru.py
@@ -38,9 +38,9 @@ class MailRuIE(InfoExtractor):
                 'id': '46843144_1263',
                 'ext': 'mp4',
                 'title': 'Samsung Galaxy S5 Hammer Smash Fail Battery Explosion',
-                'timestamp': 1397217632,
-                'upload_date': '20140411',
-                'uploader': 'hitech',
+                'timestamp': 1397039888,
+                'upload_date': '20140409',
+                'uploader': 'hitech@corp.mail.ru',
                 'uploader_id': 'hitech@corp.mail.ru',
                 'duration': 245,
             },

From 1bedf4de06300ea67655aaec8d83cf4af36a156f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 9 Feb 2016 17:12:34 +0100
Subject: [PATCH 0640/1105] [fox] extract http formats

---
 youtube_dl/extractor/fox.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/fox.py b/youtube_dl/extractor/fox.py
index ab97b3196..4a8acd53d 100644
--- a/youtube_dl/extractor/fox.py
+++ b/youtube_dl/extractor/fox.py
@@ -29,7 +29,7 @@ class FOXIE(InfoExtractor):
 
         release_url = self._parse_json(self._search_regex(
             r'"fox_pdk_player"\s*:\s*({[^}]+?})', webpage, 'fox_pdk_player'),
-            video_id)['release_url'] + '&manifest=m3u'
+            video_id)['release_url'] + '&switch=http'
 
         return {
             '_type': 'url_transparent',

From d413095f7e14f146f66d5f0cb828fb55aa4cb323 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 9 Feb 2016 17:15:41 +0100
Subject: [PATCH 0641/1105] [extractor/common] remove duplicated formats and
 subtiles in smil manifests

---
 youtube_dl/extractor/common.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9c0421de7..1143f6dbb 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1186,6 +1186,7 @@ class InfoExtractor(object):
         http_count = 0
         m3u8_count = 0
 
+        src_urls = []
         videos = smil.findall(self._xpath_ns('.//video', namespace))
         for video in videos:
             src = video.get('src')
@@ -1222,6 +1223,9 @@ class InfoExtractor(object):
                 continue
 
             src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
+            if src_url in src_urls:
+                continue
+            src_urls.append(src_url)
 
             if proto == 'm3u8' or src_ext == 'm3u8':
                 m3u8_formats = self._extract_m3u8_formats(
@@ -1267,11 +1271,13 @@ class InfoExtractor(object):
         return formats
 
     def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
+        urls = []
         subtitles = {}
         for num, textstream in enumerate(smil.findall(self._xpath_ns('.//textstream', namespace))):
             src = textstream.get('src')
-            if not src:
+            if not src or src in urls:
                 continue
+            urls.append(src)
             ext = textstream.get('ext') or determine_ext(src)
             if not ext:
                 type_ = textstream.get('type')

From c43fe0268cc4f7eee0e189b1b3183950574e3e19 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 9 Feb 2016 17:25:02 +0100
Subject: [PATCH 0642/1105] [downloader/dash] Implement dashsegments fd in
 terms of fragment fd

---
 youtube_dl/downloader/dash.py | 79 ++++++++++++++++-------------------
 1 file changed, 35 insertions(+), 44 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index b0070aead..4f3eeb386 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -1,67 +1,58 @@
 from __future__ import unicode_literals
 
+import os
 import re
 
-from .common import FileDownloader
-from ..utils import sanitized_Request
+from .fragment import FragmentFD
+from ..utils import (
+    sanitize_open,
+    encodeFilename,
+)
 
 
-class DashSegmentsFD(FileDownloader):
+class DashSegmentsFD(FragmentFD):
     """
     Download segments in a DASH manifest
     """
+
+    FD_NAME = 'dashsegments'
+
     def real_download(self, filename, info_dict):
-        self.report_destination(filename)
-        tmpfilename = self.temp_name(filename)
         base_url = info_dict['url']
-        segment_urls = info_dict['segment_urls']
+        segment_urls = [info_dict['segment_urls'][0]] if self.params.get('test', False) else info_dict['segment_urls']
+        initialization_url = info_dict.get('initialization_url')
 
-        is_test = self.params.get('test', False)
-        remaining_bytes = self._TEST_FILE_SIZE if is_test else None
-        byte_counter = 0
+        ctx = {
+            'filename': filename,
+            'total_frags': len(segment_urls) + (1 if initialization_url else 0),
+        }
 
-        def append_url_to_file(outf, target_url, target_name, remaining_bytes=None):
-            self.to_screen('[DashSegments] %s: Downloading %s' % (info_dict['id'], target_name))
-            req = sanitized_Request(target_url)
-            if remaining_bytes is not None:
-                req.add_header('Range', 'bytes=0-%d' % (remaining_bytes - 1))
-
-            data = self.ydl.urlopen(req).read()
-
-            if remaining_bytes is not None:
-                data = data[:remaining_bytes]
-
-            outf.write(data)
-            return len(data)
+        self._prepare_and_start_frag_download(ctx)
 
         def combine_url(base_url, target_url):
             if re.match(r'^https?://', target_url):
                 return target_url
             return '%s%s%s' % (base_url, '' if base_url.endswith('/') else '/', target_url)
 
-        with open(tmpfilename, 'wb') as outf:
-            if info_dict.get('initialization_url'):
-                append_url_to_file(
-                    outf, combine_url(base_url, info_dict['initialization_url']),
-                    'initialization segment')
-            for i, segment_url in enumerate(segment_urls):
-                segment_len = append_url_to_file(
-                    outf, combine_url(base_url, segment_url),
-                    'segment %d / %d' % (i + 1, len(segment_urls)),
-                    remaining_bytes)
-                byte_counter += segment_len
-                if remaining_bytes is not None:
-                    remaining_bytes -= segment_len
-                    if remaining_bytes <= 0:
-                        break
+        segments_filenames = []
+        def append_url_to_file(target_url, target_filename):
+            success = ctx['dl'].download(target_filename, {'url': combine_url(base_url, target_url)})
+            if not success:
+                return False
+            down, target_sanitized = sanitize_open(target_filename, 'rb')
+            ctx['dest_stream'].write(down.read())
+            down.close()
+            segments_filenames.append(target_sanitized)
 
-        self.try_rename(tmpfilename, filename)
+        if initialization_url:
+            append_url_to_file(initialization_url, ctx['tmpfilename'] + '-Init')
+        for i, segment_url in enumerate(segment_urls):
+            segment_filename = '%s-Seg%d' % (ctx['tmpfilename'], i)
+            append_url_to_file(segment_url, segment_filename)
 
-        self._hook_progress({
-            'downloaded_bytes': byte_counter,
-            'total_bytes': byte_counter,
-            'filename': filename,
-            'status': 'finished',
-        })
+        self._finish_frag_download(ctx)
+
+        for segment_file in segments_filenames:
+            os.remove(encodeFilename(segment_file))
 
         return True

From 5e3a6fec3397e729991f360a4af9e2dd1e91d45c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 9 Feb 2016 17:30:42 +0100
Subject: [PATCH 0643/1105] [fox] update test

---
 youtube_dl/extractor/fox.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/fox.py b/youtube_dl/extractor/fox.py
index 4a8acd53d..fa05af50d 100644
--- a/youtube_dl/extractor/fox.py
+++ b/youtube_dl/extractor/fox.py
@@ -9,6 +9,7 @@ class FOXIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?fox\.com/watch/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.fox.com/watch/255180355939/7684182528',
+        'md5': 'ebd296fcc41dd4b19f8115d8461a3165',
         'info_dict': {
             'id': '255180355939',
             'ext': 'mp4',
@@ -17,10 +18,6 @@ class FOXIE(InfoExtractor):
             'duration': 129,
         },
         'add_ie': ['ThePlatform'],
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
     }
 
     def _real_extract(self, url):

From 51e9094f4a769187b11621c6477b19ad0e2418f9 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 9 Feb 2016 20:05:39 +0100
Subject: [PATCH 0644/1105] [extractor/common] extract youtube dash formats
 filesize(fixes #8480)

---
 youtube_dl/extractor/common.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 1143f6dbb..a9497851c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1440,6 +1440,8 @@ class InfoExtractor(object):
                             base_url = mpd_base_url + base_url
                         representation_id = representation_attrib.get('id')
                         lang = representation_attrib.get('lang')
+                        url_el = representation.find(_add_ns('BaseURL'))
+                        filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength') if url_el is not None else None)
                         f = {
                             'format_id': mpd_id or representation_id,
                             'url': base_url,
@@ -1452,6 +1454,7 @@ class InfoExtractor(object):
                             'acodec': 'none' if content_type == 'video' else representation_attrib.get('codecs'),
                             'language': lang if lang not in ('mul', 'und', 'zxx', 'mis') else None,
                             'format_note': 'DASH %s' % content_type,
+                            'filesize': filesize,
                         }
                         representation_ms_info = extract_multisegment_info(representation, adaption_set_ms_info)
                         if 'segment_urls' not in representation_ms_info and 'media_template' in representation_ms_info:

From 9c15869c2846d955f4a5968924feed81121cffbb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Feb 2016 01:09:07 +0600
Subject: [PATCH 0645/1105] [viddler] Add support for secret videos (Closes
 #8481)

---
 youtube_dl/extractor/viddler.py | 38 +++++++++++++++++++++++++++++----
 1 file changed, 34 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py
index 40ffbad2a..8ba54ad39 100644
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -1,6 +1,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse,
+    compat_urlparse,
+)
 from ..utils import (
     float_or_none,
     int_or_none,
@@ -54,16 +58,42 @@ class ViddlerIE(InfoExtractor):
             'view_count': int,
             'comment_count': int,
         }
+    }, {
+        # secret protected
+        'url': 'http://www.viddler.com/v/890c0985?secret=34051570',
+        'info_dict': {
+            'id': '890c0985',
+            'ext': 'mp4',
+            'title': 'Complete Property Training - Traineeships',
+            'description': ' ',
+            'upload_date': '20130606',
+            'uploader': 'TiffanyBowtell',
+            'timestamp': 1370496993,
+            'view_count': int,
+            'comment_count': int,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        json_url = (
-            'http://api.viddler.com/api/v2/viddler.videos.getPlaybackDetails.json?video_id=%s&key=v0vhrt7bg2xq1vyxhkct' %
-            video_id)
+        query = {
+            'video_id': video_id,
+            'key': 'v0vhrt7bg2xq1vyxhkct',
+        }
+
+        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+        secret = qs.get('secret', [None])[0]
+        if secret:
+            query['secret'] = secret
+
         headers = {'Referer': 'http://static.cdn-ec.viddler.com/js/arpeggio/v2/embed.html'}
-        request = sanitized_Request(json_url, None, headers)
+        request = sanitized_Request(
+            'http://api.viddler.com/api/v2/viddler.videos.getPlaybackDetails.json?%s'
+            % compat_urllib_parse.urlencode(query), None, headers)
         data = self._download_json(request, video_id)['video']
 
         formats = []

From 47246ae26c7e19764ffa5c5c922fd549f562d42c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Feb 2016 01:12:47 +0600
Subject: [PATCH 0646/1105] [viddler] Update tests

---
 youtube_dl/extractor/viddler.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py
index 8ba54ad39..6bfbd4d85 100644
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -16,10 +16,10 @@ class ViddlerIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?viddler\.com/(?:v|embed|player)/(?P<id>[a-z0-9]+)'
     _TESTS = [{
         'url': 'http://www.viddler.com/v/43903784',
-        'md5': 'ae43ad7cb59431ce043f0ff7fa13cbf4',
+        'md5': '9eee21161d2c7f5b39690c3e325fab2f',
         'info_dict': {
             'id': '43903784',
-            'ext': 'mp4',
+            'ext': 'mov',
             'title': 'Video Made Easy',
             'description': 'md5:6a697ebd844ff3093bd2e82c37b409cd',
             'uploader': 'viddler',
@@ -33,10 +33,10 @@ class ViddlerIE(InfoExtractor):
         }
     }, {
         'url': 'http://www.viddler.com/v/4d03aad9/',
-        'md5': 'faa71fbf70c0bee7ab93076fd007f4b0',
+        'md5': 'f12c5a7fa839c47a79363bfdf69404fb',
         'info_dict': {
             'id': '4d03aad9',
-            'ext': 'mp4',
+            'ext': 'ts',
             'title': 'WALL-TO-GORTAT',
             'upload_date': '20150126',
             'uploader': 'deadspin',
@@ -46,10 +46,10 @@ class ViddlerIE(InfoExtractor):
         }
     }, {
         'url': 'http://www.viddler.com/player/221ebbbd/0/',
-        'md5': '0defa2bd0ea613d14a6e9bd1db6be326',
+        'md5': '740511f61d3d1bb71dc14a0fe01a1c10',
         'info_dict': {
             'id': '221ebbbd',
-            'ext': 'mp4',
+            'ext': 'mov',
             'title': 'LETeens-Grammar-snack-third-conditional',
             'description': ' ',
             'upload_date': '20140929',

From b92071ef00d3a74b3c7145883587f5bf4d14498b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 9 Feb 2016 20:12:36 +0100
Subject: [PATCH 0647/1105] release 2016.02.09.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 53b210290..ce2c58e52 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.02.09'
+__version__ = '2016.02.09.1'

From 89d23f37f26c2a245c76fb6a0842cc945f68de19 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Feb 2016 04:43:00 +0600
Subject: [PATCH 0648/1105] [hotstar] Relax _VALID_URL (Closes #8487)

---
 youtube_dl/extractor/hotstar.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/hotstar.py b/youtube_dl/extractor/hotstar.py
index a7c3ce4ab..f05d765d6 100644
--- a/youtube_dl/extractor/hotstar.py
+++ b/youtube_dl/extractor/hotstar.py
@@ -10,8 +10,8 @@ from ..utils import (
 
 
 class HotStarIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?hotstar\.com/.*?[/-](?P<id>\d{10})'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?hotstar\.com/(?:.+?[/-])?(?P<id>\d{10})'
+    _TESTS = [{
         'url': 'http://www.hotstar.com/on-air-with-aib--english-1000076273',
         'info_dict': {
             'id': '1000076273',
@@ -26,7 +26,13 @@ class HotStarIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         }
-    }
+    }, {
+        'url': 'http://www.hotstar.com/sports/cricket/rajitha-sizzles-on-debut-with-329/2001477583',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.hotstar.com/1000000515',
+        'only_matching': True,
+    }]
 
     _GET_CONTENT_TEMPLATE = 'http://account.hotstar.com/AVS/besc?action=GetAggregatedContentDetails&channel=PCTV&contentId=%s'
     _GET_CDN_TEMPLATE = 'http://getcdn.hotstar.com/AVS/besc?action=GetCDN&asJson=Y&channel=%s&id=%s&type=%s'

From cc799437ea828af8ca74cd5ba7dcfaf763f6e960 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Feb 2016 20:05:17 +0600
Subject: [PATCH 0649/1105] [youku] Report private videos (Closes #8498)

---
 youtube_dl/extractor/youku.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index f767fa15f..49687371a 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -229,6 +229,9 @@ class YoukuIE(InfoExtractor):
             if error_note is not None and '因版权原因无法观看此视频' in error_note:
                 raise ExtractorError(
                     'Youku said: Sorry, this video is available in China only', expected=True)
+            elif error_note and '该视频被设为私密' in error_note:
+                raise ExtractorError(
+                    'Youku said: Sorry, this video is private', expected=True)
             else:
                 msg = 'Youku server reported error %i' % error.get('code')
                 if error_note is not None:

From 7fcea295c59f3321c5a70cb8171e6d5de1f378e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Feb 2016 20:46:38 +0600
Subject: [PATCH 0650/1105] [pbs] Switch to portal player by default (Closes
 #8491)

---
 youtube_dl/extractor/pbs.py | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 97e8ffc97..cbd1efea0 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -8,6 +8,7 @@ from ..utils import (
     ExtractorError,
     determine_ext,
     int_or_none,
+    js_to_json,
     strip_jsonp,
     unified_strdate,
     US_RATINGS,
@@ -432,9 +433,20 @@ class PBSIE(InfoExtractor):
                 for vid_id in video_id]
             return self.playlist_result(entries, display_id)
 
-        info = self._download_json(
-            'http://player.pbs.org/videoInfo/%s?format=json&type=partner' % video_id,
-            display_id)
+        player = self._download_webpage(
+            'http://player.pbs.org/portalplayer/%s' % video_id, display_id)
+
+        info = self._parse_json(
+            self._search_regex(
+                r'(?s)PBS\.videoData\s*=\s*({.+?});\n',
+                player, 'video data', default='{}'),
+            display_id, transform_source=js_to_json, fatal=False)
+
+        # Fallback to old videoInfo API
+        if not info:
+            info = self._download_json(
+                'http://player.pbs.org/videoInfo/%s?format=json&type=partner' % video_id,
+                display_id, 'Downloading video info JSON')
 
         formats = []
         for encoding_name in ('recommended_encoding', 'alternate_encoding'):

From e2effb08a46817104b56a6242616fd60848251d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Feb 2016 21:16:58 +0600
Subject: [PATCH 0651/1105] [YoutubeDL] Sanitize format_id (Closes #8494)

---
 youtube_dl/YoutubeDL.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 50228bb32..3cabfbc03 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1288,6 +1288,9 @@ class YoutubeDL(object):
 
             if format.get('format_id') is None:
                 format['format_id'] = compat_str(i)
+            else:
+                # Sanitize format_id from characters used in format selector expression
+                format['format_id'] = re.sub('[\s,/+\[\]()]', '_', format['format_id'])
             format_id = format['format_id']
             if format_id not in formats_dict:
                 formats_dict[format_id] = []

From f817d9bec180a932541564bdb9baf3db43e47faa Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 10 Feb 2016 16:17:38 +0100
Subject: [PATCH 0652/1105] release 2016.02.10

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index ce2c58e52..2d0d67b0f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.02.09.1'
+__version__ = '2016.02.10'

From 80f772c28a3277376620ed7f50308e12437e358d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 10 Feb 2016 22:16:21 +0100
Subject: [PATCH 0653/1105] [crackle] Add new extractor

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/comcarcoff.py | 16 +++---
 youtube_dl/extractor/crackle.py    | 92 ++++++++++++++++++++++++++++++
 3 files changed, 102 insertions(+), 7 deletions(-)
 create mode 100644 youtube_dl/extractor/crackle.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 18951c287..f08f27480 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -126,6 +126,7 @@ from .comcarcoff import ComCarCoffIE
 from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
 from .condenast import CondeNastIE
 from .cracked import CrackedIE
+from .crackle import CrackleIE
 from .criterion import CriterionIE
 from .crooksandliars import CrooksAndLiarsIE
 from .crunchyroll import (
diff --git a/youtube_dl/extractor/comcarcoff.py b/youtube_dl/extractor/comcarcoff.py
index 2efa200b5..7dff68492 100644
--- a/youtube_dl/extractor/comcarcoff.py
+++ b/youtube_dl/extractor/comcarcoff.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     int_or_none,
     parse_duration,
@@ -14,14 +15,13 @@ class ComCarCoffIE(InfoExtractor):
     _TESTS = [{
         'url': 'http://comediansincarsgettingcoffee.com/miranda-sings-happy-thanksgiving-miranda/',
         'info_dict': {
-            'id': 'miranda-sings-happy-thanksgiving-miranda',
+            'id': '2494164',
             'ext': 'mp4',
             'upload_date': '20141127',
             'timestamp': 1417107600,
             'duration': 1232,
             'title': 'Happy Thanksgiving Miranda',
             'description': 'Jerry Seinfeld and his special guest Miranda Sings cruise around town in search of coffee, complaining and apologizing along the way.',
-            'thumbnail': 'http://ccc.crackle.com/images/s5e4_thumb.jpg',
         },
         'params': {
             'skip_download': 'requires ffmpeg',
@@ -39,15 +39,14 @@ class ComCarCoffIE(InfoExtractor):
                 r'window\.app\s*=\s*({.+?});\n', webpage, 'full data json'),
             display_id)['videoData']
 
-        video_id = full_data['activeVideo']['video']
-        video_data = full_data.get('videos', {}).get(video_id) or full_data['singleshots'][video_id]
+        display_id = full_data['activeVideo']['video']
+        video_data = full_data.get('videos', {}).get(display_id) or full_data['singleshots'][display_id]
+        video_id = compat_str(video_data['mediaId'])
         thumbnails = [{
             'url': video_data['images']['thumb'],
         }, {
             'url': video_data['images']['poster'],
         }]
-        formats = self._extract_m3u8_formats(
-            video_data['mediaUrl'], video_id, ext='mp4')
 
         timestamp = int_or_none(video_data.get('pubDateTime')) or parse_iso8601(
             video_data.get('pubDate'))
@@ -55,6 +54,8 @@ class ComCarCoffIE(InfoExtractor):
             video_data.get('duration'))
 
         return {
+            '_type': 'url_transparent',
+            'url': 'crackle:%s' % video_id,
             'id': video_id,
             'display_id': display_id,
             'title': video_data['title'],
@@ -62,6 +63,7 @@ class ComCarCoffIE(InfoExtractor):
             'timestamp': timestamp,
             'duration': duration,
             'thumbnails': thumbnails,
-            'formats': formats,
+            'season_number': int_or_none(video_data.get('season')),
+            'episode_number': int_or_none(video_data.get('episode')),
             'webpage_url': 'http://comediansincarsgettingcoffee.com/%s' % (video_data.get('urlSlug', video_data.get('slug'))),
         }
diff --git a/youtube_dl/extractor/crackle.py b/youtube_dl/extractor/crackle.py
new file mode 100644
index 000000000..a478333a2
--- /dev/null
+++ b/youtube_dl/extractor/crackle.py
@@ -0,0 +1,92 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class CrackleIE(InfoExtractor):
+    _VALID_URL = r'(?:crackle:|https?://(?:www\.)?crackle\.com/(?:playlist/\d+/|(?:[^/]+/)+))(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.crackle.com/the-art-of-more/2496419',
+        'info_dict': {
+            'id': '2496419',
+            'ext': 'mp4',
+            'title': 'Heavy Lies the Head',
+            'description': 'md5:bb56aa0708fe7b9a4861535f15c3abca',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }
+
+    # extracted from http://legacyweb-us.crackle.com/flash/QueryReferrer.ashx
+    _SUBTITLE_SERVER = 'http://web-us-az.crackle.com'
+    _UPLYNK_OWNER_ID = 'e8773f7770a44dbd886eee4fca16a66b'
+    _THUMBNAIL_TEMPLATE = 'http://images-us-am.crackle.com/%stnl_1920x1080.jpg?ts=20140107233116?c=635333335057637614'
+
+    # extracted from http://legacyweb-us.crackle.com/flash/ReferrerRedirect.ashx
+    _MEDIA_FILE_SLOTS = {
+        'c544.flv': {
+            'width': 544,
+            'height': 306,
+        },
+        '360p.mp4': {
+            'width': 640,
+            'height': 360,
+        },
+        '480p.mp4': {
+            'width': 852,
+            'height': 478,
+        },
+        '480p_1mbps.mp4': {
+            'width': 852,
+            'height': 478,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        item = self._download_xml(
+            'http://legacyweb-us.crackle.com/app/revamp/vidwallcache.aspx?flags=-1&fm=%s' % video_id, video_id).find('i')
+        title = item.attrib['t']
+
+        thumbnail = None
+        subtitles = {}
+        formats = self._extract_m3u8_formats('http://content.uplynk.com/ext/%s/%s.m3u8' % (self._UPLYNK_OWNER_ID, video_id), video_id, 'mp4', fatal=None)
+        path = item.attrib.get('p')
+        if path:
+            thumbnail = self._THUMBNAIL_TEMPLATE % path
+            http_base_url = 'http://ahttp.crackle.com/' + path
+            for mfs_path, mfs_info in self._MEDIA_FILE_SLOTS.items():
+                formats.append({
+                    'url': http_base_url + mfs_path,
+                    'format_id': mfs_path.split('.')[0],
+                    'width': mfs_info['width'],
+                    'height': mfs_info['height'],
+                })
+            for cc in item.findall('cc'):
+                locale = cc.attrib.get('l')
+                v = cc.attrib.get('v')
+                if locale and v:
+                    if locale not in subtitles:
+                        subtitles[locale] = []
+                    subtitles[locale] = [{
+                        'url': '%s/%s%s_%s.xml' % (self._SUBTITLE_SERVER, path, locale, v),
+                        'ext': 'ttml',
+                    }]
+        self._sort_formats(formats, ('width', 'height', 'tbr'))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': item.attrib.get('d'),
+            'duration': int(item.attrib.get('r'), 16) if item.attrib.get('r') else None,
+            'series': item.attrib.get('sn'),
+            'season_number': int_or_none(item.attrib.get('se')),
+            'episode_number': int_or_none(item.attrib.get('ep')),
+            'thumbnail': thumbnail,
+            'subtitles': subtitles,
+            'formats': formats,
+        }

From 47d205a6460a696514f6485a516358513ab880b6 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 10 Feb 2016 22:23:56 +0100
Subject: [PATCH 0654/1105] [crackle] improve format sorting

---
 youtube_dl/extractor/crackle.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/crackle.py b/youtube_dl/extractor/crackle.py
index a478333a2..70f3f14c3 100644
--- a/youtube_dl/extractor/crackle.py
+++ b/youtube_dl/extractor/crackle.py
@@ -49,12 +49,15 @@ class CrackleIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         item = self._download_xml(
-            'http://legacyweb-us.crackle.com/app/revamp/vidwallcache.aspx?flags=-1&fm=%s' % video_id, video_id).find('i')
+            'http://legacyweb-us.crackle.com/app/revamp/vidwallcache.aspx?flags=-1&fm=%s' % video_id,
+            video_id).find('i')
         title = item.attrib['t']
 
         thumbnail = None
         subtitles = {}
-        formats = self._extract_m3u8_formats('http://content.uplynk.com/ext/%s/%s.m3u8' % (self._UPLYNK_OWNER_ID, video_id), video_id, 'mp4', fatal=None)
+        formats = self._extract_m3u8_formats(
+            'http://content.uplynk.com/ext/%s/%s.m3u8' % (self._UPLYNK_OWNER_ID, video_id),
+            video_id, 'mp4', fatal=None)
         path = item.attrib.get('p')
         if path:
             thumbnail = self._THUMBNAIL_TEMPLATE % path
@@ -76,7 +79,7 @@ class CrackleIE(InfoExtractor):
                         'url': '%s/%s%s_%s.xml' % (self._SUBTITLE_SERVER, path, locale, v),
                         'ext': 'ttml',
                     }]
-        self._sort_formats(formats, ('width', 'height', 'tbr'))
+        self._sort_formats(formats, ('width', 'height', 'tbr', 'format_id'))
 
         return {
             'id': video_id,

From a1188d0ed06a619f4d0274eca602114648d7f33b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 10 Feb 2016 22:39:33 +0100
Subject: [PATCH 0655/1105] [crackle] add prefix to format ids

---
 youtube_dl/extractor/crackle.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crackle.py b/youtube_dl/extractor/crackle.py
index 70f3f14c3..79238cce7 100644
--- a/youtube_dl/extractor/crackle.py
+++ b/youtube_dl/extractor/crackle.py
@@ -57,7 +57,7 @@ class CrackleIE(InfoExtractor):
         subtitles = {}
         formats = self._extract_m3u8_formats(
             'http://content.uplynk.com/ext/%s/%s.m3u8' % (self._UPLYNK_OWNER_ID, video_id),
-            video_id, 'mp4', fatal=None)
+            video_id, 'mp4', m3u8_id='hls', fatal=None)
         path = item.attrib.get('p')
         if path:
             thumbnail = self._THUMBNAIL_TEMPLATE % path
@@ -65,7 +65,7 @@ class CrackleIE(InfoExtractor):
             for mfs_path, mfs_info in self._MEDIA_FILE_SLOTS.items():
                 formats.append({
                     'url': http_base_url + mfs_path,
-                    'format_id': mfs_path.split('.')[0],
+                    'format_id': 'http-' + mfs_path.split('.')[0],
                     'width': mfs_info['width'],
                     'height': mfs_info['height'],
                 })

From d1ea5e171f817d94bc356a96e7eb6967919cc18d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 11 Feb 2016 10:30:31 +0100
Subject: [PATCH 0656/1105] [plays] Add new extractor(#8458)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/plays.py    | 50 ++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 youtube_dl/extractor/plays.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f08f27480..f1e5a5e86 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -534,6 +534,7 @@ from .planetaplay import PlanetaPlayIE
 from .pladform import PladformIE
 from .played import PlayedIE
 from .playfm import PlayFMIE
+from .plays import PlaysTVIE
 from .playtvak import PlaytvakIE
 from .playvid import PlayvidIE
 from .playwire import PlaywireIE
diff --git a/youtube_dl/extractor/plays.py b/youtube_dl/extractor/plays.py
new file mode 100644
index 000000000..2aba7cb9c
--- /dev/null
+++ b/youtube_dl/extractor/plays.py
@@ -0,0 +1,50 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class PlaysTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?plays\.tv/video/(?P<id>[0-9a-f]{18})'
+    _TEST = {
+        'url': 'http://plays.tv/video/56af17f56c95335490/when-you-outplay-the-azir-wall',
+        'md5': 'dfeac1198506652b5257a62762cec7bc',
+        'info_dict': {
+            'id': '56af17f56c95335490',
+            'ext': 'mp4',
+            'title': 'When you outplay the Azir wall',
+            'description': 'Posted by Bjergsen',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._og_search_title(webpage)
+        content = self._parse_json(
+            self._search_regex(r'R\.bindContent\(({.+?})\);', webpage,
+            'content'), video_id)['content']
+        mpd_url, sources = re.search(
+            r'(?s)<video[^>]+data-mpd="([^"]+)"[^>]*>(.+?)</video>',
+            content).groups()
+        formats = self._extract_mpd_formats(
+            self._proto_relative_url(mpd_url), video_id, mpd_id='DASH')
+        for format_id, height, format_url in re.findall(r'<source\s+res="((\d+)h?)"\s+src="([^"]+)"', sources):
+            formats.append({
+                'url': self._proto_relative_url(format_url),
+                'format_id': 'http-' + format_id,
+                'height': int_or_none(height),
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': self._og_search_description(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'formats': formats,
+        }

From 154c209e2db2bd6f51998503b15d9a050bc1e738 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 11 Feb 2016 10:33:26 +0100
Subject: [PATCH 0657/1105] [extractor/common] improve dash format ids

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a9497851c..00645feed 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1443,7 +1443,7 @@ class InfoExtractor(object):
                         url_el = representation.find(_add_ns('BaseURL'))
                         filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength') if url_el is not None else None)
                         f = {
-                            'format_id': mpd_id or representation_id,
+                            'format_id': '%s-%s' % (mpd_id, representation_id) if mpd_id else representation_id,
                             'url': base_url,
                             'width': int_or_none(representation_attrib.get('width')),
                             'height': int_or_none(representation_attrib.get('height')),

From dd867805969126ed0bd2ab8fe69eaf61fbf44ab7 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 11 Feb 2016 10:55:50 +0100
Subject: [PATCH 0658/1105] [extractor/common] fix dash formats sorting

---
 youtube_dl/extractor/common.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 00645feed..cd7087bec 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -851,6 +851,7 @@ class InfoExtractor(object):
             proto_preference = 0 if determine_protocol(f) in ['http', 'https'] else -0.1
 
             if f.get('vcodec') == 'none':  # audio only
+                preference -= 50
                 if self._downloader.params.get('prefer_free_formats'):
                     ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus']
                 else:
@@ -861,6 +862,8 @@ class InfoExtractor(object):
                 except ValueError:
                     audio_ext_preference = -1
             else:
+                if f.get('acodec') == 'none':  # video only
+                    preference -= 40
                 if self._downloader.params.get('prefer_free_formats'):
                     ORDER = ['flv', 'mp4', 'webm']
                 else:

From 9a61dfba0cfb9b978278fe2e1c823c205fcd72dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Feb 2016 21:22:57 +0600
Subject: [PATCH 0659/1105] [pbs] Revert prefer portalplayer

---
 youtube_dl/extractor/pbs.py | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index cbd1efea0..0d4d709de 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_HTTPError
 from ..utils import (
     ExtractorError,
     determine_ext,
@@ -433,20 +434,21 @@ class PBSIE(InfoExtractor):
                 for vid_id in video_id]
             return self.playlist_result(entries, display_id)
 
-        player = self._download_webpage(
-            'http://player.pbs.org/portalplayer/%s' % video_id, display_id)
-
-        info = self._parse_json(
-            self._search_regex(
-                r'(?s)PBS\.videoData\s*=\s*({.+?});\n',
-                player, 'video data', default='{}'),
-            display_id, transform_source=js_to_json, fatal=False)
-
-        # Fallback to old videoInfo API
-        if not info:
+        try:
             info = self._download_json(
                 'http://player.pbs.org/videoInfo/%s?format=json&type=partner' % video_id,
                 display_id, 'Downloading video info JSON')
+        except ExtractorError as e:
+            if not isinstance(e.cause, compat_HTTPError) or e.cause.code != 404:
+                raise
+            # videoInfo API may not work for some videos, fallback to portalplayer API
+            player = self._download_webpage(
+                'http://player.pbs.org/portalplayer/%s' % video_id, display_id)
+            info = self._parse_json(
+                self._search_regex(
+                    r'(?s)PBS\.videoData\s*=\s*({.+?});\n',
+                    player, 'video data', default='{}'),
+                display_id, transform_source=js_to_json, fatal=False)
 
         formats = []
         for encoding_name in ('recommended_encoding', 'alternate_encoding'):

From b14d5e26f6c5c63f8de600e5159bf9cf12813e8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Feb 2016 21:24:07 +0600
Subject: [PATCH 0660/1105] [pbs] Improve description extraction

---
 youtube_dl/extractor/pbs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 0d4d709de..0cdae6721 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -507,7 +507,7 @@ class PBSIE(InfoExtractor):
             'id': video_id,
             'display_id': display_id,
             'title': info['title'],
-            'description': info['program'].get('description'),
+            'description': info.get('description') or info.get('program', {}).get('description'),
             'thumbnail': info.get('image_url'),
             'duration': int_or_none(info.get('duration')),
             'age_limit': age_limit,

From a7ab46375b81dfab6b46535796f3f63296c7bb2c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Feb 2016 21:43:01 +0600
Subject: [PATCH 0661/1105] [pbs] Update some tests

---
 youtube_dl/extractor/pbs.py | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 0cdae6721..c159c0071 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -201,7 +201,7 @@ class PBSIE(InfoExtractor):
                 'id': '2365006249',
                 'ext': 'mp4',
                 'title': 'Constitution USA with Peter Sagal - A More Perfect Union',
-                'description': 'md5:ba0c207295339c8d6eced00b7c363c6a',
+                'description': 'md5:36f341ae62e251b8f5bd2b754b95a071',
                 'duration': 3190,
             },
             'params': {
@@ -215,7 +215,7 @@ class PBSIE(InfoExtractor):
                 'id': '2365297690',
                 'ext': 'mp4',
                 'title': 'FRONTLINE - Losing Iraq',
-                'description': 'md5:f5bfbefadf421e8bb8647602011caf8e',
+                'description': 'md5:4d3eaa01f94e61b3e73704735f1196d9',
                 'duration': 5050,
             },
             'params': {
@@ -229,7 +229,7 @@ class PBSIE(InfoExtractor):
                 'id': '2201174722',
                 'ext': 'mp4',
                 'title': 'PBS NewsHour - Cyber Schools Gain Popularity, but Quality Questions Persist',
-                'description': 'md5:5871c15cba347c1b3d28ac47a73c7c28',
+                'description': 'md5:95a19f568689d09a166dff9edada3301',
                 'duration': 801,
             },
         },
@@ -239,8 +239,8 @@ class PBSIE(InfoExtractor):
             'info_dict': {
                 'id': '2365297708',
                 'ext': 'mp4',
-                'description': 'md5:68d87ef760660eb564455eb30ca464fe',
                 'title': 'Great Performances - Dudamel Conducts Verdi Requiem at the Hollywood Bowl - Full',
+                'description': 'md5:657897370e09e2bc6bf0f8d2cd313c6b',
                 'duration': 6559,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
@@ -280,7 +280,7 @@ class PBSIE(InfoExtractor):
                 'display_id': 'player',
                 'ext': 'mp4',
                 'title': 'American Experience - Death and the Civil War, Chapter 1',
-                'description': 'American Experience, TV’s most-watched history series, brings to life the compelling stories from our past that inform our understanding of the world today.',
+                'description': 'md5:1b80a74e0380ed2a4fb335026de1600d',
                 'duration': 682,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
@@ -289,20 +289,19 @@ class PBSIE(InfoExtractor):
             },
         },
         {
-            'url': 'http://video.pbs.org/video/2365367186/',
+            'url': 'http://www.pbs.org/video/2365245528/',
             'info_dict': {
-                'id': '2365367186',
-                'display_id': '2365367186',
+                'id': '2365245528',
+                'display_id': '2365245528',
                 'ext': 'mp4',
-                'title': 'To Catch A Comet - Full Episode',
-                'description': 'On November 12, 2014, billions of kilometers from Earth, spacecraft orbiter Rosetta and lander Philae did what no other had dared to attempt \u2014 land on the volatile surface of a comet as it zooms around the sun at 67,000 km/hr. The European Space Agency hopes this mission can help peer into our past and unlock secrets of our origins.',
-                'duration': 3342,
+                'title': 'FRONTLINE - United States of Secrets (Part One)',
+                'description': 'md5:55756bd5c551519cc4b7703e373e217e',
+                'duration': 6851,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
             'params': {
                 'skip_download': True,  # requires ffmpeg
             },
-            'skip': 'Expired',
         },
         {
             # Video embedded in iframe containing angle brackets as attribute's value (e.g.
@@ -314,7 +313,7 @@ class PBSIE(InfoExtractor):
                 'display_id': 'a-chefs-life-season-3-episode-5-prickly-business',
                 'ext': 'mp4',
                 'title': "A Chef's Life - Season 3, Ep. 5: Prickly Business",
-                'description': 'md5:61db2ddf27c9912f09c241014b118ed1',
+                'description': 'md5:54033c6baa1f9623607c6e2ed245888b',
                 'duration': 1480,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
@@ -330,7 +329,7 @@ class PBSIE(InfoExtractor):
                 'display_id': 'the-atomic-artists',
                 'ext': 'mp4',
                 'title': 'FRONTLINE - The Atomic Artists',
-                'description': 'md5:f5bfbefadf421e8bb8647602011caf8e',
+                'description': 'md5:1a2481e86b32b2e12ec1905dd473e2c1',
                 'duration': 723,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },

From 6b3fbd3425180a00d99e91e6f2b9b5896c567ea4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Feb 2016 22:02:37 +0600
Subject: [PATCH 0662/1105] [pbs] Fix multi part videos extraction

---
 youtube_dl/extractor/pbs.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index c159c0071..cca012953 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -366,10 +366,14 @@ class PBSIE(InfoExtractor):
                 webpage, 'upload date', default=None))
 
             # tabbed frontline videos
-            tabbed_videos = re.findall(
-                r'<div[^>]+class="videotab[^"]*"[^>]+vid="(\d+)"', webpage)
-            if tabbed_videos:
-                return tabbed_videos, presumptive_id, upload_date
+            MULTI_PART_REGEXES = (
+                r'<div[^>]+class="videotab[^"]*"[^>]+vid="(\d+)"',
+                r'<a[^>]+href=["\']#video-\d+["\'][^>]+data-coveid=["\'](\d+)',
+            )
+            for p in MULTI_PART_REGEXES:
+                tabbed_videos = re.findall(p, webpage)
+                if tabbed_videos:
+                    return tabbed_videos, presumptive_id, upload_date
 
             MEDIA_ID_REGEXES = [
                 r"div\s*:\s*'videoembed'\s*,\s*mediaid\s*:\s*'(\d+)'",  # frontline video embed

From 28280e8ded11ca13d67b705a5a56fb930ba26205 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Feb 2016 22:02:57 +0600
Subject: [PATCH 0663/1105] [plays] PEP 8

---
 youtube_dl/extractor/plays.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/plays.py b/youtube_dl/extractor/plays.py
index 2aba7cb9c..c3c38cf4a 100644
--- a/youtube_dl/extractor/plays.py
+++ b/youtube_dl/extractor/plays.py
@@ -26,8 +26,9 @@ class PlaysTVIE(InfoExtractor):
 
         title = self._og_search_title(webpage)
         content = self._parse_json(
-            self._search_regex(r'R\.bindContent\(({.+?})\);', webpage,
-            'content'), video_id)['content']
+            self._search_regex(
+                r'R\.bindContent\(({.+?})\);', webpage,
+                'content'), video_id)['content']
         mpd_url, sources = re.search(
             r'(?s)<video[^>]+data-mpd="([^"]+)"[^>]*>(.+?)</video>',
             content).groups()

From b67d63149d79f98a84bec0ecf8028282d523d1f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Feb 2016 22:33:08 +0600
Subject: [PATCH 0664/1105] [youtube] Fix typos

---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b9a91dea2..382d52cb6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -375,7 +375,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
     IE_NAME = 'youtube'
     _TESTS = [
         {
-            'url': 'http://www.youtube.com/watch?v=BaW_jenozKcj&t=1s&end=9',
+            'url': 'http://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
             'info_dict': {
                 'id': 'BaW_jenozKc',
                 'ext': 'mp4',
@@ -441,7 +441,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             }
         },
         {
-            'url': 'http://www.youtube.com/watch?v=BaW_jenozKcj&v=UxxajLWwzqY',
+            'url': 'http://www.youtube.com/watch?v=BaW_jenozKc&v=UxxajLWwzqY',
             'note': 'Use the first video ID in the URL',
             'info_dict': {
                 'id': 'BaW_jenozKc',

From 388ae76b52ff85dba5c47f5958fbe3e662abd346 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Feb 2016 22:46:13 +0600
Subject: [PATCH 0665/1105] [YoutubeDL] Fix format resolution when height is
 missing

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 3cabfbc03..e5b75f0f5 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1798,7 +1798,7 @@ class YoutubeDL(object):
             else:
                 res = '%sp' % format['height']
         elif format.get('width') is not None:
-            res = '?x%d' % format['width']
+            res = '%dx?' % format['width']
         else:
             res = default
         return res

From 81e1c4e2fc260c77de8afabc73090b549d6ba259 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 11 Feb 2016 17:58:48 +0100
Subject: [PATCH 0666/1105] [extractor/common] remove duplicate rtmp formats in
 smil manifest

---
 youtube_dl/extractor/common.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 00645feed..6b40a8323 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1186,12 +1186,13 @@ class InfoExtractor(object):
         http_count = 0
         m3u8_count = 0
 
-        src_urls = []
+        srcs = []
         videos = smil.findall(self._xpath_ns('.//video', namespace))
         for video in videos:
             src = video.get('src')
-            if not src:
+            if not src or src in srcs:
                 continue
+            srcs.append(src)
 
             bitrate = float_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
             filesize = int_or_none(video.get('size') or video.get('fileSize'))
@@ -1223,9 +1224,6 @@ class InfoExtractor(object):
                 continue
 
             src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
-            if src_url in src_urls:
-                continue
-            src_urls.append(src_url)
 
             if proto == 'm3u8' or src_ext == 'm3u8':
                 m3u8_formats = self._extract_m3u8_formats(

From 8b85ac3fd95dbc3296801036f43c8d2ebfefcb6f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 11 Feb 2016 18:10:32 +0100
Subject: [PATCH 0667/1105] [cbc] Add new extractor(closes #3803)(closes
 #4731)(closes #5309)

---
 youtube_dl/extractor/__init__.py |   4 ++
 youtube_dl/extractor/cbc.py      | 113 +++++++++++++++++++++++++++++++
 2 files changed, 117 insertions(+)
 create mode 100644 youtube_dl/extractor/cbc.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f1e5a5e86..7056cf323 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -89,6 +89,10 @@ from .camdemy import (
 from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
 from .canvas import CanvasIE
+from .cbc import (
+    CBCIE,
+    CBCPlayerIE,
+)
 from .cbs import CBSIE
 from .cbsnews import (
     CBSNewsIE,
diff --git a/youtube_dl/extractor/cbc.py b/youtube_dl/extractor/cbc.py
new file mode 100644
index 000000000..d8aa31038
--- /dev/null
+++ b/youtube_dl/extractor/cbc.py
@@ -0,0 +1,113 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import js_to_json
+
+
+class CBCIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?cbc\.ca/(?:[^/]+/)+(?P<id>[^/?#]+)'
+    _TESTS = [{
+        # with mediaId
+        'url': 'http://www.cbc.ca/22minutes/videos/clips-season-23/don-cherry-play-offs',
+        'info_dict': {
+            'id': '2682904050',
+            'ext': 'flv',
+            'title': 'Don Cherry – All-Stars',
+            'description': 'Don Cherry has a bee in his bonnet about AHL player John Scott because that guy’s got heart.',
+            'timestamp': 1454475540,
+            'upload_date': '20160203',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }, {
+        # with clipId
+        'url': 'http://www.cbc.ca/archives/entry/1978-robin-williams-freestyles-on-90-minutes-live',
+        'info_dict': {
+            'id': '2487345465',
+            'ext': 'flv',
+            'title': 'Robin Williams freestyles on 90 Minutes Live',
+            'description': 'Wacky American comedian Robin Williams shows off his infamous "freestyle" comedic talents while being interviewed on CBC\'s 90 Minutes Live.',
+            'upload_date': '19700101',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }, {
+        # multiple iframes
+        'url': 'http://www.cbc.ca/natureofthings/blog/birds-eye-view-from-vancouvers-burrard-street-bridge-how-we-got-the-shot',
+        'playlist': [{
+            'info_dict': {
+                'id': '2680832926',
+                'ext': 'flv',
+                'title': 'An Eagle\'s-Eye View Off Burrard Bridge',
+                'description': 'Hercules the eagle flies from Vancouver\'s Burrard Bridge down to a nearby park with a mini-camera strapped to his back.',
+                'upload_date': '19700101',
+            },
+        }, {
+            'info_dict': {
+                'id': '2658915080',
+                'ext': 'flv',
+                'title': 'Fly like an eagle!',
+                'description': 'Eagle equipped with a mini camera flies from the world\'s tallest tower',
+                'upload_date': '19700101',
+            },
+        }],
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return False if CBCPlayerIE.suitable(url) else super(CBCIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        player_init = self._search_regex(
+            r'CBC\.APP\.Caffeine\.initInstance\(({.+?})\);', webpage, 'player init',
+            default=None)
+        if player_init:
+            player_info = self._parse_json(player_init, display_id, js_to_json)
+            media_id = player_info.get('mediaId')
+            if not media_id:
+                clip_id = player_info['clipId']
+                media_id = self._download_json(
+                    'http://feed.theplatform.com/f/h9dtGB/punlNGjMlc1F?fields=id&byContent=byReleases%3DbyId%253D' + clip_id,
+                    clip_id)['entries'][0]['id'].split('/')[-1]
+            return self.url_result('cbcplayer:%s' % media_id, 'CBCPlayer', media_id)
+        else:
+            entries = [self.url_result('cbcplayer:%s' % media_id, 'CBCPlayer', media_id) for media_id in re.findall(r'<iframe[^>]+src="[^"]+?mediaId=(\d+)"', webpage)]
+            return self.playlist_result(entries)
+
+
+class CBCPlayerIE(InfoExtractor):
+    _VALID_URL = r'(?:cbcplayer:|https?://(?:www\.)?cbc\.ca/(?:player/play/|i/caffeine/syndicate/\?mediaId=))(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.cbc.ca/player/play/2683190193',
+        'info_dict': {
+            'id': '2683190193',
+            'ext': 'flv',
+            'title': 'Gerry Runs a Sweat Shop',
+            'description': 'md5:b457e1c01e8ff408d9d801c1c2cd29b0',
+            'timestamp': 1455067800,
+            'upload_date': '20160210',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        return self.url_result(
+            'http://feed.theplatform.com/f/ExhSPC/vms_5akSXx4Ng_Zn?byGuid=%s' % video_id,
+            'ThePlatformFeed', video_id)

From 9c7b38981cb6429ba690aaef313e36fb694f9902 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Feb 2016 23:12:30 +0600
Subject: [PATCH 0668/1105] [utils] Bump Firefox version in User-Agent

Old version number causes Youtube not to serve some formats in ytplayer.config
---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index dce67de8a..a82a262a0 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -56,7 +56,7 @@ from .compat import (
 compiled_regex_type = type(re.compile(''))
 
 std_headers = {
-    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/20.0 (Chrome)',
+    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/44.0 (Chrome)',
     'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
     'Accept-Encoding': 'gzip, deflate',

From 3dc71d82ce628d08327a098d8e87af0905d5b59f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 11 Feb 2016 18:12:38 +0100
Subject: [PATCH 0669/1105] [theplatform] fix pid extraction in the platform
 feed

---
 youtube_dl/extractor/theplatform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 10f2cad55..f2e975ab0 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -284,7 +284,7 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
         duration = None
         for item in entry['media$content']:
             smil_url = item['plfile$url'] + '&format=SMIL&Tracking=true&Embedded=true&formats=MPEG4,F4M'
-            cur_video_id = url_basename(smil_url)
+            cur_video_id = ThePlatformIE._match_id(smil_url)
             if first_video_id is None:
                 first_video_id = cur_video_id
                 duration = float_or_none(item.get('plfile$duration'))

From a7cab4d03927c90b8db12fda627022996e560c78 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 11 Feb 2016 18:50:14 +0100
Subject: [PATCH 0670/1105] [theplatform] remove unused import and change smil
 url for ThePlatformFeedIE

---
 youtube_dl/extractor/theplatform.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index f2e975ab0..755f816ff 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -20,7 +20,6 @@ from ..utils import (
     int_or_none,
     sanitized_Request,
     unsmuggle_url,
-    url_basename,
     xpath_with_ns,
 )
 
@@ -283,7 +282,7 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
         first_video_id = None
         duration = None
         for item in entry['media$content']:
-            smil_url = item['plfile$url'] + '&format=SMIL&Tracking=true&Embedded=true&formats=MPEG4,F4M'
+            smil_url = item['plfile$url'] + '&format=SMIL&mbr=true'
             cur_video_id = ThePlatformIE._match_id(smil_url)
             if first_video_id is None:
                 first_video_id = cur_video_id

From b826035dd5948d598e29bc17c9d08e3c4d77abbc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Feb 2016 03:16:26 +0600
Subject: [PATCH 0671/1105] [vimeo] Fix authentication (Closes #8520)

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 2389e7f0f..6a8f9b49d 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -57,7 +57,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
 
     def _extract_xsrft_and_vuid(self, webpage):
         xsrft = self._search_regex(
-            r'xsrft\s*[=:]\s*(?P<q>["\'])(?P<xsrft>.+?)(?P=q)',
+            r'(?:(?P<q1>["\'])xsrft(?P=q1)\s*:|xsrft\s*[=:])\s*(?P<q>["\'])(?P<xsrft>.+?)(?P=q)',
             webpage, 'login token', group='xsrft')
         vuid = self._search_regex(
             r'["\']vuid["\']\s*:\s*(["\'])(?P<vuid>.+?)\1',

From 5a4905924d15bb2bdc0a80f6fd41a869a9787d38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Feb 2016 22:03:10 +0600
Subject: [PATCH 0672/1105] [extractor/generic] Improve dailymotion embed
 detection (Closes #8521, closes #8325)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index bf61ab2e7..ede9801bb 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1413,7 +1413,7 @@ class GenericIE(InfoExtractor):
 
         # Look for embedded Dailymotion player
         matches = re.findall(
-            r'<(?:embed|iframe)[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/(?:embed|swf)/video/.+?)\1', webpage)
+            r'<(?:(?:embed|iframe)[^>]+?src=|input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=)(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/(?:embed|swf)/video/.+?)\1', webpage)
         if matches:
             return _playlist_from_matches(
                 matches, lambda m: unescapeHTML(m[1]))

From c349456ef60d2f638be808bb023635c63df4107c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 12 Feb 2016 17:38:48 +0100
Subject: [PATCH 0673/1105] [extractor/common] strip http urls in smil manifest

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6b40a8323..359bdd299 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1224,6 +1224,7 @@ class InfoExtractor(object):
                 continue
 
             src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
+            src_url = src_url.strip()
 
             if proto == 'm3u8' or src_ext == 'm3u8':
                 m3u8_formats = self._extract_m3u8_formats(

From 3d98f97c6408326dd841fae767a721ed91a1cae8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 12 Feb 2016 17:39:32 +0100
Subject: [PATCH 0674/1105] [aenetworks] extract http formats

---
 youtube_dl/extractor/aenetworks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index 43d7b0523..e192b5ac5 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -61,6 +61,6 @@ class AENetworksIE(InfoExtractor):
         info = self._search_json_ld(webpage, video_id, fatal=False)
         info.update({
             '_type': 'url_transparent',
-            'url': smuggle_url(video_url, {'sig': {'key': 'crazyjava', 'secret': 's3cr3t'}}),
+            'url': smuggle_url(video_url + '?mbr=true&format=SMIL&assetTypes=medium_video_s3', {'sig': {'key': 'crazyjava', 'secret': 's3cr3t'}, 'force_smil_url': True}),
         })
         return info

From bcbb1b08b24215731d3444297508a1bd6b7011ff Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 12 Feb 2016 17:56:06 +0100
Subject: [PATCH 0675/1105] Revert "[aenetworks] extract http formats"

This reverts commit 3d98f97c6408326dd841fae767a721ed91a1cae8.
---
 youtube_dl/extractor/aenetworks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index e192b5ac5..43d7b0523 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -61,6 +61,6 @@ class AENetworksIE(InfoExtractor):
         info = self._search_json_ld(webpage, video_id, fatal=False)
         info.update({
             '_type': 'url_transparent',
-            'url': smuggle_url(video_url + '?mbr=true&format=SMIL&assetTypes=medium_video_s3', {'sig': {'key': 'crazyjava', 'secret': 's3cr3t'}, 'force_smil_url': True}),
+            'url': smuggle_url(video_url, {'sig': {'key': 'crazyjava', 'secret': 's3cr3t'}}),
         })
         return info

From bb20526b64c1624639b18e1b582fbdee1bd5da64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Feb 2016 00:13:56 +0600
Subject: [PATCH 0676/1105] [extractor/common] Improve base url construction

---
 youtube_dl/extractor/common.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 359bdd299..444d412d9 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1435,7 +1435,9 @@ class InfoExtractor(object):
                                 base_url = base_url_e.text + base_url
                                 if re.match(r'^https?://', base_url):
                                     break
-                        if not re.match(r'^https?://', base_url):
+                        if mpd_base_url and not re.match(r'^https?://', base_url):
+                            if not mpd_base_url.endswith('/') and not base_url.startswith('/'):
+                                mpd_base_url += '/'
                             base_url = mpd_base_url + base_url
                         representation_id = representation_attrib.get('id')
                         lang = representation_attrib.get('lang')

From 7be747b921f0e31f4db3db2fd7e1c90d265ed566 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Feb 2016 00:15:59 +0600
Subject: [PATCH 0677/1105] [extractor/generic] Pass mpd base url to
 _parse_mpd_formats

---
 youtube_dl/extractor/generic.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ede9801bb..d6d0a9ed0 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1302,7 +1302,8 @@ class GenericIE(InfoExtractor):
                 return {
                     'id': video_id,
                     'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
-                    'formats': self._parse_mpd_formats(doc, video_id),
+                    'formats': self._parse_mpd_formats(
+                        doc, video_id, mpd_base_url=url.rpartition('/')[0]),
                 }
         except compat_xml_parse_error:
             pass

From 4c77d3f52ad16d9d9ac0ff0f15a62beef3158ca4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Feb 2016 00:23:14 +0600
Subject: [PATCH 0678/1105] [YoutubeDL] Allow bestvideo+bestaudio for any
 extractor

---
 youtube_dl/YoutubeDL.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e5b75f0f5..0072c7d35 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1341,7 +1341,6 @@ class YoutubeDL(object):
         if req_format is None:
             req_format_list = []
             if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
-                    info_dict['extractor'] in ['youtube', 'ted'] and
                     not info_dict.get('is_live')):
                 merger = FFmpegMergerPP(self)
                 if merger.available and merger.can_merge():

From 9d939cec48f06a401fb79eb078c1fc50b2aefbe1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Feb 2016 00:36:47 +0600
Subject: [PATCH 0679/1105] [extractor/generic] Add direct mpd url test

---
 youtube_dl/extractor/generic.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index d6d0a9ed0..45adbb7a3 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -224,6 +224,20 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        # MPD from http://dash-mse-test.appspot.com/media.html
+        {
+            'url': 'http://yt-dash-mse-test.commondatastorage.googleapis.com/media/car-20120827-manifest.mpd',
+            'md5': '4b57baab2e30d6eb3a6a09f0ba57ef53',
+            'info_dict': {
+                'id': 'car-20120827-manifest',
+                'ext': 'mp4',
+                'title': 'car-20120827-manifest',
+                'formats': 'mincount:9',
+            },
+            'params': {
+                'format': 'bestvideo',
+            },
+        },
         # google redirect
         {
             'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',

From 6863631c2696f93123ae5ceca1e261311f0d8925 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Feb 2016 05:01:20 +0600
Subject: [PATCH 0680/1105] [youtube] Improve multifeed videos extraction
 (Closes #8536)

---
 youtube_dl/extractor/youtube.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 382d52cb6..97d621907 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1196,9 +1196,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if not self._downloader.params.get('noplaylist'):
                 entries = []
                 feed_ids = []
-                multifeed_metadata_list = compat_urllib_parse_unquote_plus(video_info['multifeed_metadata_list'][0])
+                multifeed_metadata_list = video_info['multifeed_metadata_list'][0]
                 for feed in multifeed_metadata_list.split(','):
-                    feed_data = compat_parse_qs(feed)
+                    # Unquote should take place before split on comma (,) since textual
+                    # fields may contain comma as well (see
+                    # https://github.com/rg3/youtube-dl/issues/8536)
+                    feed_data = compat_parse_qs(compat_urllib_parse_unquote_plus(feed))
                     entries.append({
                         '_type': 'url_transparent',
                         'ie_key': 'Youtube',

From f9f49d87c2d731e220ee4b5cf83fbbd17e3bfa5d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Feb 2016 05:18:58 +0600
Subject: [PATCH 0681/1105] [youtube] Add test for #8536

---
 youtube_dl/extractor/youtube.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 97d621907..18f7d37f4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -704,6 +704,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            # Multifeed video with comma in title (see https://github.com/rg3/youtube-dl/issues/8536)
+            'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
+            'info_dict': {
+                'id': 'gVfLd0zydlo',
+                'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
+            },
+            'playlist_count': 2,
+        },
         {
             'url': 'http://vid.plus/FlRa-iH7PGw',
             'only_matching': True,

From 3e18700d45fe4117bd2560a58129e763c408b2ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Feb 2016 07:45:32 +0600
Subject: [PATCH 0682/1105] [nbc] Correct test

---
 youtube_dl/extractor/nbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 18d01f423..2202cfa33 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -57,7 +57,7 @@ class NBCIE(InfoExtractor):
         {
             # This video has expired but with an escaped embedURL
             'url': 'http://www.nbc.com/parenthood/episode-guide/season-5/just-like-at-home/515',
-            'skip': 'Expired'
+            'only_matching': True,
         }
     ]
 

From 35ced3985acb23282b96771ac8b6570aab180768 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Feb 2016 08:25:05 +0100
Subject: [PATCH 0683/1105] release 2016.02.13

---
 docs/supportedsites.md | 4 ++++
 youtube_dl/version.py  | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index c48561eff..b384a3165 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -89,6 +89,8 @@
  - **canalc2.tv**
  - **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv
  - **Canvas**
+ - **CBC**
+ - **CBCPlayer**
  - **CBS**
  - **CBSNews**: CBS News
  - **CBSNewsLiveVideo**: CBS News Live Videos
@@ -120,6 +122,7 @@
  - **ComedyCentralShows**: The Daily Show / The Colbert Report
  - **CondeNast**: Condé Nast media group: Allure, Architectural Digest, Ars Technica, Bon Appétit, Brides, Condé Nast, Condé Nast Traveler, Details, Epicurious, GQ, Glamour, Golf Digest, SELF, Teen Vogue, The New Yorker, Vanity Fair, Vogue, W Magazine, WIRED
  - **Cracked**
+ - **Crackle**
  - **Criterion**
  - **CrooksAndLiars**
  - **Crunchyroll**
@@ -445,6 +448,7 @@
  - **PlanetaPlay**
  - **play.fm**
  - **played.to**
+ - **PlaysTV**
  - **Playtvak**: Playtvak.cz, iDNES.cz and Lidovky.cz
  - **Playvid**
  - **Playwire**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2d0d67b0f..9aca8001a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.02.10'
+__version__ = '2016.02.13'

From e2bd68c90197ae629c1bbea899b6fc6db7bfffdf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Feb 2016 13:30:31 +0600
Subject: [PATCH 0684/1105] [animeondemand][wip] Add extractor (#8518)

---
 youtube_dl/extractor/__init__.py      |   1 +
 youtube_dl/extractor/animeondemand.py | 159 ++++++++++++++++++++++++++
 2 files changed, 160 insertions(+)
 create mode 100644 youtube_dl/extractor/animeondemand.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7056cf323..6937f28d3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -20,6 +20,7 @@ from .aftonbladet import AftonbladetIE
 from .airmozilla import AirMozillaIE
 from .aljazeera import AlJazeeraIE
 from .alphaporno import AlphaPornoIE
+from .animeondemand import AnimeOnDemandIE
 from .anitube import AnitubeIE
 from .anysex import AnySexIE
 from .aol import AolIE
diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
new file mode 100644
index 000000000..0d0516ba6
--- /dev/null
+++ b/youtube_dl/extractor/animeondemand.py
@@ -0,0 +1,159 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+    determine_ext,
+    encode_dict,
+    ExtractorError,
+    sanitized_Request,
+    urlencode_postdata,
+)
+
+
+class AnimeOnDemandIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?anime-on-demand\.de/anime/(?P<id>\d+)'
+    _LOGIN_URL = 'https://www.anime-on-demand.de/users/sign_in'
+    _APPLY_HTML5_URL = 'https://www.anime-on-demand.de/html5apply'
+    _TEST = {
+        'url': 'https://www.anime-on-demand.de/anime/161',
+        'info_dict': {
+            'id': '161',
+            'title': 'Grimgar, Ashes and Illusions (OmU)',
+            'description': 'md5:6681ce3c07c7189d255ac6ab23812d31',
+        },
+        'playlist_mincount': 4,
+    }
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+
+        login_page = self._download_webpage(
+            self._LOGIN_URL, None, 'Downloading login page')
+
+        login_form = self._form_hidden_inputs('new_user', login_page)
+
+        login_form.update({
+            'user[login]': username,
+            'user[password]': password,
+        })
+
+        post_url = self._search_regex(
+            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
+            'post url', default=self._LOGIN_URL, group='url')
+
+        if not post_url.startswith('http'):
+            post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
+
+        request = sanitized_Request(
+            post_url, urlencode_postdata(encode_dict(login_form)))
+        request.add_header('Referer', self._LOGIN_URL)
+
+        response = self._download_webpage(
+            request, None, 'Logging in as %s' % username)
+
+        if all(p not in response for p in ('>Logout<', 'href="/users/sign_out"')):
+            error = self._search_regex(
+                r'<p class="alert alert-danger">(.+?)</p>',
+                response, 'error', default=None)
+            if error:
+                raise ExtractorError('Unable to login: %s' % error, expected=True)
+            raise ExtractorError('Unable to log in')
+
+    def _real_initialize(self):
+        self._login()
+
+    def _real_extract(self, url):
+        anime_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, anime_id)
+
+        if 'data-playlist=' not in webpage:
+            self._download_webpage(
+                self._APPLY_HTML5_URL, anime_id,
+                'Activating HTML5 beta', 'Unable to apply HTML5 beta')
+            webpage = self._download_webpage(url, anime_id)
+
+        csrf_token = self._html_search_meta(
+            'csrf-token', webpage, 'csrf token', fatal=True)
+
+        anime_title = self._html_search_regex(
+            r'(?s)<h1[^>]+itemprop="name"[^>]*>(.+?)</h1>',
+            webpage, 'anime name')
+        anime_description = self._html_search_regex(
+            r'(?s)<div[^>]+itemprop="description"[^>]*>(.+?)</div>',
+            webpage, 'anime description', default=None)
+
+        entries = []
+
+        for episode_html in re.findall(r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', webpage):
+            m = re.search(
+                r'class="episodebox-title"[^>]+title="Episode (?P<number>\d+) - (?P<title>.+?)"', episode_html)
+            if not m:
+                continue
+
+            episode_number = int(m.group('number'))
+            episode_title = m.group('title')
+            video_id = 'episode-%d' % episode_number
+
+            common_info = {
+                'id': video_id,
+                'series': anime_title,
+                'episode': episode_title,
+                'episode_number': episode_number,
+            }
+
+            formats = []
+
+            playlist_url = self._search_regex(
+                r'data-playlist=(["\'])(?P<url>.+?)\1',
+                episode_html, 'data playlist', default=None, group='url')
+            if playlist_url:
+                request = sanitized_Request(
+                    compat_urlparse.urljoin(url, playlist_url),
+                    headers={
+                        'X-Requested-With': 'XMLHttpRequest',
+                        'X-CSRF-Token': csrf_token,
+                        'Referer': url,
+                        'Accept': 'application/json, text/javascript, */*; q=0.01',
+                    })
+
+                playlist = self._download_json(
+                    request, video_id, 'Downloading playlist JSON', fatal=False)
+                if playlist:
+                    playlist = playlist['playlist'][0]
+                    title = playlist['title']
+                    description = playlist.get('description')
+                    for source in playlist.get('sources', []):
+                        file_ = source.get('file')
+                        if file_ and determine_ext(file_) == 'm3u8':
+                            formats = self._extract_m3u8_formats(
+                                file_, video_id, 'mp4',
+                                entry_protocol='m3u8_native', m3u8_id='hls')
+
+            if formats:
+                f = common_info.copy()
+                f.update({
+                    'title': title,
+                    'description': description,
+                    'formats': formats,
+                })
+                entries.append(f)
+
+            m = re.search(
+                r'data-dialog-header=(["\'])(?P<title>.+?)\1[^>]+href=(["\'])(?P<href>.+?)\3[^>]*>Teaser<',
+                episode_html)
+            if m:
+                f = common_info.copy()
+                f.update({
+                    'id': '%s-teaser' % f['id'],
+                    'title': m.group('title'),
+                    'url': compat_urlparse.urljoin(url, m.group('href')),
+                })
+                entries.append(f)
+
+        return self.playlist_result(entries, anime_id, anime_title, anime_description)

From 702325123964c767eafa5667aa3d3844f8133e8a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 13 Feb 2016 12:26:27 +0100
Subject: [PATCH 0685/1105] [comedycentral] Support /shows URLs (fixes #8405)

---
 youtube_dl/extractor/comedycentral.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 3e4bd10b6..055c9eec5 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -16,11 +16,11 @@ from ..utils import (
 
 class ComedyCentralIE(MTVServicesInfoExtractor):
     _VALID_URL = r'''(?x)https?://(?:www\.)?cc\.com/
-        (video-clips|episodes|cc-studios|video-collections|full-episodes)
+        (video-clips|episodes|cc-studios|video-collections|full-episodes|shows)
         /(?P<title>.*)'''
     _FEED_URL = 'http://comedycentral.com/feeds/mrss/'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.cc.com/video-clips/kllhuv/stand-up-greg-fitzsimmons--uncensored---too-good-of-a-mother',
         'md5': 'c4f48e9eda1b16dd10add0744344b6d8',
         'info_dict': {
@@ -29,7 +29,10 @@ class ComedyCentralIE(MTVServicesInfoExtractor):
             'title': 'CC:Stand-Up|Greg Fitzsimmons: Life on Stage|Uncensored - Too Good of a Mother',
             'description': 'After a certain point, breastfeeding becomes c**kblocking.',
         },
-    }
+    }, {
+        'url': 'http://www.cc.com/shows/the-daily-show-with-trevor-noah/interviews/6yx39d/exclusive-rand-paul-extended-interview',
+        'only_matching': True,
+    }]
 
 
 class ComedyCentralShowsIE(MTVServicesInfoExtractor):

From b4561e857f7bbeb6571d36c9991b3fc5c109352d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Feb 2016 22:41:58 +0600
Subject: [PATCH 0686/1105] [animeondemand] Add .netrc

---
 youtube_dl/extractor/animeondemand.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 0d0516ba6..a7d8daf7b 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -17,6 +17,7 @@ class AnimeOnDemandIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?anime-on-demand\.de/anime/(?P<id>\d+)'
     _LOGIN_URL = 'https://www.anime-on-demand.de/users/sign_in'
     _APPLY_HTML5_URL = 'https://www.anime-on-demand.de/html5apply'
+    _NETRC_MACHINE = 'animeondemand'
     _TEST = {
         'url': 'https://www.anime-on-demand.de/anime/161',
         'info_dict': {

From b45f2b1d6ec2fb4081d261f862e9413c5870bc16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Feb 2016 11:24:57 +0600
Subject: [PATCH 0687/1105] [myvideo] Mark broken

---
 youtube_dl/extractor/myvideo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/myvideo.py b/youtube_dl/extractor/myvideo.py
index 36ab388b2..1e21cf98a 100644
--- a/youtube_dl/extractor/myvideo.py
+++ b/youtube_dl/extractor/myvideo.py
@@ -19,6 +19,7 @@ from ..utils import (
 
 
 class MyVideoIE(InfoExtractor):
+    _WORKING = False
     _VALID_URL = r'http://(?:www\.)?myvideo\.de/(?:[^/]+/)?watch/(?P<id>[0-9]+)/[^?/]+.*'
     IE_NAME = 'myvideo'
     _TEST = {

From fca2e6d5a6fd402bc14dd381c1060f6a2cc38eb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Feb 2016 13:44:23 +0600
Subject: [PATCH 0688/1105] [dailymotion:cloud] Use idiomatic name for
 classmethod's first argument

---
 youtube_dl/extractor/dailymotion.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 6e462af69..7ae9f2359 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -396,13 +396,13 @@ class DailymotionCloudIE(DailymotionBaseInfoExtractor):
     }]
 
     @classmethod
-    def _extract_dmcloud_url(self, webpage):
-        mobj = re.search(r'<iframe[^>]+src=[\'"](%s)[\'"]' % self._VALID_EMBED_URL, webpage)
+    def _extract_dmcloud_url(cls, webpage):
+        mobj = re.search(r'<iframe[^>]+src=[\'"](%s)[\'"]' % cls._VALID_EMBED_URL, webpage)
         if mobj:
             return mobj.group(1)
 
         mobj = re.search(
-            r'<input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=[\'"](%s)[\'"]' % self._VALID_EMBED_URL,
+            r'<input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=[\'"](%s)[\'"]' % cls._VALID_EMBED_URL,
             webpage)
         if mobj:
             return mobj.group(1)

From 080e09557d0c323a9abf5f72e9d307f8bb545be6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Feb 2016 14:01:43 +0600
Subject: [PATCH 0689/1105] [aes] Switch to new-style classes

---
 youtube_dl/aes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/aes.py b/youtube_dl/aes.py
index 7817adcfd..a01c367de 100644
--- a/youtube_dl/aes.py
+++ b/youtube_dl/aes.py
@@ -161,7 +161,7 @@ def aes_decrypt_text(data, password, key_size_bytes):
     nonce = data[:NONCE_LENGTH_BYTES]
     cipher = data[NONCE_LENGTH_BYTES:]
 
-    class Counter:
+    class Counter(object):
         __value = nonce + [0] * (BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES)
 
         def next_value(self):

From e76394f36cdff2f701af7c2420b812e7ea045098 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Feb 2016 14:02:12 +0600
Subject: [PATCH 0690/1105] [globo] Switch to new-style classes

---
 youtube_dl/extractor/globo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index b241c4868..3de8356f6 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -65,7 +65,7 @@ class GloboIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    class MD5:
+    class MD5(object):
         HEX_FORMAT_LOWERCASE = 0
         HEX_FORMAT_UPPERCASE = 1
         BASE64_PAD_CHARACTER_DEFAULT_COMPLIANCE = ''

From c78c9cd10dfbb4fc3fd49df4f9c98e9c94c9aae9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Feb 2016 14:13:09 +0600
Subject: [PATCH 0691/1105] [downloader/dash] PEP 8

---
 youtube_dl/downloader/dash.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 4f3eeb386..8b1b17c6e 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -35,6 +35,7 @@ class DashSegmentsFD(FragmentFD):
             return '%s%s%s' % (base_url, '' if base_url.endswith('/') else '/', target_url)
 
         segments_filenames = []
+
         def append_url_to_file(target_url, target_filename):
             success = ctx['dl'].download(target_filename, {'url': combine_url(base_url, target_url)})
             if not success:

From d800609c62703e4e6edd2891a8432306462e4db3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Feb 2016 14:25:04 +0600
Subject: [PATCH 0692/1105] [refactor] Do not specify redundant None as second
 argument in dict.get()

---
 youtube_dl/YoutubeDL.py           | 10 +++++-----
 youtube_dl/downloader/common.py   |  2 +-
 youtube_dl/downloader/fragment.py |  2 +-
 youtube_dl/downloader/http.py     |  4 ++--
 youtube_dl/downloader/rtmp.py     | 16 ++++++++--------
 youtube_dl/extractor/common.py    |  8 ++++----
 youtube_dl/extractor/smotri.py    |  4 ++--
 youtube_dl/extractor/vimeo.py     |  6 +++---
 youtube_dl/extractor/youku.py     |  2 +-
 youtube_dl/utils.py               |  2 +-
 10 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0072c7d35..d34f77a6d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -605,12 +605,12 @@ class YoutubeDL(object):
             if rejecttitle:
                 if re.search(rejecttitle, title, re.IGNORECASE):
                     return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
-        date = info_dict.get('upload_date', None)
+        date = info_dict.get('upload_date')
         if date is not None:
             dateRange = self.params.get('daterange', DateRange())
             if date not in dateRange:
                 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
-        view_count = info_dict.get('view_count', None)
+        view_count = info_dict.get('view_count')
         if view_count is not None:
             min_views = self.params.get('min_views')
             if min_views is not None and view_count < min_views:
@@ -747,18 +747,18 @@ class YoutubeDL(object):
                 new_result, download=download, extra_info=extra_info)
         elif result_type == 'playlist' or result_type == 'multi_video':
             # We process each entry in the playlist
-            playlist = ie_result.get('title', None) or ie_result.get('id', None)
+            playlist = ie_result.get('title') or ie_result.get('id')
             self.to_screen('[download] Downloading playlist: %s' % playlist)
 
             playlist_results = []
 
             playliststart = self.params.get('playliststart', 1) - 1
-            playlistend = self.params.get('playlistend', None)
+            playlistend = self.params.get('playlistend')
             # For backwards compatibility, interpret -1 as whole list
             if playlistend == -1:
                 playlistend = None
 
-            playlistitems_str = self.params.get('playlist_items', None)
+            playlistitems_str = self.params.get('playlist_items')
             playlistitems = None
             if playlistitems_str is not None:
                 def iter_playlistitems(format):
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index de815612c..2d5154051 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -157,7 +157,7 @@ class FileDownloader(object):
 
     def slow_down(self, start_time, now, byte_counter):
         """Sleep if the download speed is over the rate limit."""
-        rate_limit = self.params.get('ratelimit', None)
+        rate_limit = self.params.get('ratelimit')
         if rate_limit is None or byte_counter == 0:
             return
         if now is None:
diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 8b96eceb9..5bc99492b 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -38,7 +38,7 @@ class FragmentFD(FileDownloader):
                 'continuedl': True,
                 'quiet': True,
                 'noprogress': True,
-                'ratelimit': self.params.get('ratelimit', None),
+                'ratelimit': self.params.get('ratelimit'),
                 'retries': self.params.get('retries', 0),
                 'test': self.params.get('test', False),
             }
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 56840e026..7089983ce 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -140,8 +140,8 @@ class HttpFD(FileDownloader):
 
         if data_len is not None:
             data_len = int(data_len) + resume_len
-            min_data_len = self.params.get("min_filesize", None)
-            max_data_len = self.params.get("max_filesize", None)
+            min_data_len = self.params.get("min_filesize")
+            max_data_len = self.params.get("max_filesize")
             if min_data_len is not None and data_len < min_data_len:
                 self.to_screen('\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
                 return False
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 14d56db47..9de6e70bb 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -94,15 +94,15 @@ class RtmpFD(FileDownloader):
             return proc.returncode
 
         url = info_dict['url']
-        player_url = info_dict.get('player_url', None)
-        page_url = info_dict.get('page_url', None)
-        app = info_dict.get('app', None)
-        play_path = info_dict.get('play_path', None)
-        tc_url = info_dict.get('tc_url', None)
-        flash_version = info_dict.get('flash_version', None)
+        player_url = info_dict.get('player_url')
+        page_url = info_dict.get('page_url')
+        app = info_dict.get('app')
+        play_path = info_dict.get('play_path')
+        tc_url = info_dict.get('tc_url')
+        flash_version = info_dict.get('flash_version')
         live = info_dict.get('rtmp_live', False)
-        conn = info_dict.get('rtmp_conn', None)
-        protocol = info_dict.get('rtmp_protocol', None)
+        conn = info_dict.get('rtmp_conn')
+        protocol = info_dict.get('rtmp_protocol')
         real_time = info_dict.get('rtmp_real_time', False)
         no_resume = info_dict.get('no_resume', False)
         continue_dl = self.params.get('continuedl', True)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 444d412d9..144d8c6b6 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -636,7 +636,7 @@ class InfoExtractor(object):
         downloader_params = self._downloader.params
 
         # Attempt to use provided username and password or .netrc data
-        if downloader_params.get('username', None) is not None:
+        if downloader_params.get('username') is not None:
             username = downloader_params['username']
             password = downloader_params['password']
         elif downloader_params.get('usenetrc', False):
@@ -663,7 +663,7 @@ class InfoExtractor(object):
             return None
         downloader_params = self._downloader.params
 
-        if downloader_params.get('twofactor', None) is not None:
+        if downloader_params.get('twofactor') is not None:
             return downloader_params['twofactor']
 
         return compat_getpass('Type %s and press [Return]: ' % note)
@@ -744,7 +744,7 @@ class InfoExtractor(object):
             'mature': 17,
             'restricted': 19,
         }
-        return RATING_TABLE.get(rating.lower(), None)
+        return RATING_TABLE.get(rating.lower())
 
     def _family_friendly_search(self, html):
         # See http://schema.org/VideoObject
@@ -759,7 +759,7 @@ class InfoExtractor(object):
             '0': 18,
             'false': 18,
         }
-        return RATING_TABLE.get(family_friendly.lower(), None)
+        return RATING_TABLE.get(family_friendly.lower())
 
     def _twitter_search_player(self, html):
         return self._html_search_meta('twitter:player', html,
diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index 30210c8a3..015ef75f3 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -170,7 +170,7 @@ class SmotriIE(InfoExtractor):
             'getvideoinfo': '1',
         }
 
-        video_password = self._downloader.params.get('videopassword', None)
+        video_password = self._downloader.params.get('videopassword')
         if video_password:
             video_form['pass'] = hashlib.md5(video_password.encode('utf-8')).hexdigest()
 
@@ -356,7 +356,7 @@ class SmotriBroadcastIE(InfoExtractor):
 
         url = 'http://smotri.com/broadcast/view/url/?ticket=%s' % ticket
 
-        broadcast_password = self._downloader.params.get('videopassword', None)
+        broadcast_password = self._downloader.params.get('videopassword')
         if broadcast_password:
             url += '&pass=%s' % hashlib.md5(broadcast_password.encode('utf-8')).hexdigest()
 
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 6a8f9b49d..c7df6b0c5 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -232,7 +232,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             return mobj.group(1)
 
     def _verify_video_password(self, url, video_id, webpage):
-        password = self._downloader.params.get('videopassword', None)
+        password = self._downloader.params.get('videopassword')
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
         token, vuid = self._extract_xsrft_and_vuid(webpage)
@@ -252,7 +252,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             'Verifying the password', 'Wrong password')
 
     def _verify_player_video_password(self, url, video_id):
-        password = self._downloader.params.get('videopassword', None)
+        password = self._downloader.params.get('videopassword')
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option')
         data = urlencode_postdata(encode_dict({'password': password}))
@@ -516,7 +516,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
         if not login_form:
             return webpage
 
-        password = self._downloader.params.get('videopassword', None)
+        password = self._downloader.params.get('videopassword')
         if password is None:
             raise ExtractorError('This album is protected by a password, use the --video-password option', expected=True)
         fields = self._hidden_inputs(login_form)
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 49687371a..5c1f84a09 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -214,7 +214,7 @@ class YoukuIE(InfoExtractor):
 
             return raw_data['data']
 
-        video_password = self._downloader.params.get('videopassword', None)
+        video_password = self._downloader.params.get('videopassword')
 
         # request basic data
         basic_data_url = "http://play.youku.com/play/get.json?vid=%s&ct=12" % video_id
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a82a262a0..3e4219b17 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1744,7 +1744,7 @@ def parse_age_limit(s):
     if s is None:
         return None
     m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
-    return int(m.group('age')) if m else US_RATINGS.get(s, None)
+    return int(m.group('age')) if m else US_RATINGS.get(s)
 
 
 def strip_jsonp(code):

From 611c1dd96efc36a788475e14cc4de64d554d28a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Feb 2016 15:37:17 +0600
Subject: [PATCH 0693/1105] [refactor] Single quotes consistency

---
 youtube_dl/YoutubeDL.py                       |   6 +-
 youtube_dl/__main__.py                        |   2 +-
 youtube_dl/compat.py                          |  14 +--
 youtube_dl/downloader/http.py                 |   4 +-
 youtube_dl/extractor/aenetworks.py            |   2 +-
 youtube_dl/extractor/bbc.py                   |   2 +-
 youtube_dl/extractor/ccc.py                   |   2 +-
 youtube_dl/extractor/ceskatelevize.py         |   8 +-
 youtube_dl/extractor/cnn.py                   |  12 +-
 youtube_dl/extractor/collegerama.py           |   6 +-
 youtube_dl/extractor/comedycentral.py         |   2 +-
 youtube_dl/extractor/common.py                |   8 +-
 youtube_dl/extractor/crunchyroll.py           |  74 ++++++------
 youtube_dl/extractor/drbonanza.py             |  10 +-
 youtube_dl/extractor/eighttracks.py           | 108 +++++++++---------
 youtube_dl/extractor/ellentv.py               |   2 +-
 youtube_dl/extractor/everyonesmixtape.py      |  12 +-
 youtube_dl/extractor/exfm.py                  |   2 +-
 youtube_dl/extractor/fc2.py                   |   2 +-
 youtube_dl/extractor/franceinter.py           |   2 +-
 youtube_dl/extractor/freevideo.py             |   4 +-
 youtube_dl/extractor/hentaistigma.py          |   4 +-
 youtube_dl/extractor/kankan.py                |   2 +-
 youtube_dl/extractor/liveleak.py              |   2 +-
 youtube_dl/extractor/mofosex.py               |   2 +-
 youtube_dl/extractor/myspass.py               |   4 +-
 youtube_dl/extractor/nerdcubed.py             |   4 +-
 youtube_dl/extractor/pornhub.py               |   8 +-
 youtube_dl/extractor/pornovoisines.py         |   2 +-
 youtube_dl/extractor/radiobremen.py           |  12 +-
 youtube_dl/extractor/radiofrance.py           |   6 +-
 youtube_dl/extractor/rbmaradio.py             |  10 +-
 youtube_dl/extractor/reverbnation.py          |  12 +-
 youtube_dl/extractor/ringtv.py                |  14 +--
 youtube_dl/extractor/rte.py                   |   2 +-
 youtube_dl/extractor/rtl2.py                  |   2 +-
 youtube_dl/extractor/screenwavemedia.py       |   2 +-
 youtube_dl/extractor/senateisvp.py            |  62 +++++-----
 youtube_dl/extractor/slutload.py              |   4 +-
 youtube_dl/extractor/snotr.py                 |   2 +-
 youtube_dl/extractor/soundcloud.py            |   2 +-
 youtube_dl/extractor/steam.py                 |  16 +--
 youtube_dl/extractor/tenplay.py               |   8 +-
 youtube_dl/extractor/thesixtyone.py           |  32 +++---
 youtube_dl/extractor/traileraddict.py         |   8 +-
 youtube_dl/extractor/tudou.py                 |   2 +-
 youtube_dl/extractor/vbox7.py                 |   2 +-
 youtube_dl/extractor/videopremium.py          |  10 +-
 youtube_dl/extractor/vimeo.py                 |  18 +--
 youtube_dl/extractor/vine.py                  |   6 +-
 youtube_dl/extractor/worldstarhiphop.py       |  14 +--
 youtube_dl/extractor/youjizz.py               |   4 +-
 youtube_dl/extractor/youku.py                 |   2 +-
 youtube_dl/options.py                         |   2 +-
 youtube_dl/postprocessor/execafterdownload.py |   2 +-
 youtube_dl/postprocessor/metadatafromtitle.py |   2 +-
 youtube_dl/postprocessor/xattrpp.py           |  10 +-
 youtube_dl/update.py                          |   8 +-
 youtube_dl/utils.py                           |  16 +--
 59 files changed, 302 insertions(+), 302 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d34f77a6d..f4324039c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -782,7 +782,7 @@ class YoutubeDL(object):
                     entries = ie_entries[playliststart:playlistend]
                 n_entries = len(entries)
                 self.to_screen(
-                    "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
+                    '[%s] playlist %s: Collected %d video ids (downloading %d of them)' %
                     (ie_result['extractor'], playlist, n_all_entries, n_entries))
             elif isinstance(ie_entries, PagedList):
                 if playlistitems:
@@ -796,7 +796,7 @@ class YoutubeDL(object):
                         playliststart, playlistend)
                 n_entries = len(entries)
                 self.to_screen(
-                    "[%s] playlist %s: Downloading %d videos" %
+                    '[%s] playlist %s: Downloading %d videos' %
                     (ie_result['extractor'], playlist, n_entries))
             else:  # iterable
                 if playlistitems:
@@ -807,7 +807,7 @@ class YoutubeDL(object):
                         ie_entries, playliststart, playlistend))
                 n_entries = len(entries)
                 self.to_screen(
-                    "[%s] playlist %s: Downloading %d videos" %
+                    '[%s] playlist %s: Downloading %d videos' %
                     (ie_result['extractor'], playlist, n_entries))
 
             if self.params.get('playlistreverse', False):
diff --git a/youtube_dl/__main__.py b/youtube_dl/__main__.py
index 42a0f8c6f..138f5fbec 100755
--- a/youtube_dl/__main__.py
+++ b/youtube_dl/__main__.py
@@ -7,7 +7,7 @@ from __future__ import unicode_literals
 
 import sys
 
-if __package__ is None and not hasattr(sys, "frozen"):
+if __package__ is None and not hasattr(sys, 'frozen'):
     # direct call of __main__.py
     import os.path
     path = os.path.realpath(os.path.abspath(__file__))
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 8ab688001..b497da696 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -181,20 +181,20 @@ except ImportError:  # Python < 3.4
             # parameter := attribute "=" value
             url = req.get_full_url()
 
-            scheme, data = url.split(":", 1)
-            mediatype, data = data.split(",", 1)
+            scheme, data = url.split(':', 1)
+            mediatype, data = data.split(',', 1)
 
             # even base64 encoded data URLs might be quoted so unquote in any case:
             data = compat_urllib_parse_unquote_to_bytes(data)
-            if mediatype.endswith(";base64"):
+            if mediatype.endswith(';base64'):
                 data = binascii.a2b_base64(data)
                 mediatype = mediatype[:-7]
 
             if not mediatype:
-                mediatype = "text/plain;charset=US-ASCII"
+                mediatype = 'text/plain;charset=US-ASCII'
 
             headers = email.message_from_string(
-                "Content-type: %s\nContent-length: %d\n" % (mediatype, len(data)))
+                'Content-type: %s\nContent-length: %d\n' % (mediatype, len(data)))
 
             return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url)
 
@@ -268,7 +268,7 @@ except ImportError:  # Python 2
             nv = name_value.split('=', 1)
             if len(nv) != 2:
                 if strict_parsing:
-                    raise ValueError("bad query field: %r" % (name_value,))
+                    raise ValueError('bad query field: %r' % (name_value,))
                 # Handle case of a control-name with no equal sign
                 if keep_blank_values:
                     nv.append('')
@@ -466,7 +466,7 @@ if sys.version_info < (2, 7):
         if err is not None:
             raise err
         else:
-            raise socket.error("getaddrinfo returns an empty list")
+            raise socket.error('getaddrinfo returns an empty list')
 else:
     compat_socket_create_connection = socket.create_connection
 
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 7089983ce..f8b69d186 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -140,8 +140,8 @@ class HttpFD(FileDownloader):
 
         if data_len is not None:
             data_len = int(data_len) + resume_len
-            min_data_len = self.params.get("min_filesize")
-            max_data_len = self.params.get("max_filesize")
+            min_data_len = self.params.get('min_filesize')
+            max_data_len = self.params.get('max_filesize')
             if min_data_len is not None and data_len < min_data_len:
                 self.to_screen('\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
                 return False
diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index 43d7b0523..6018ae79a 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -28,7 +28,7 @@ class AENetworksIE(InfoExtractor):
         'info_dict': {
             'id': 'eg47EERs_JsZ',
             'ext': 'mp4',
-            'title': "Winter Is Coming",
+            'title': 'Winter Is Coming',
             'description': 'md5:641f424b7a19d8e24f26dea22cf59d74',
         },
         'params': {
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 6ddee686c..9d0dfb961 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -86,7 +86,7 @@ class BBCCoUkIE(InfoExtractor):
                 'id': 'b00yng1d',
                 'ext': 'flv',
                 'title': 'The Voice UK: Series 3: Blind Auditions 5',
-                'description': "Emma Willis and Marvin Humes present the fifth set of blind auditions in the singing competition, as the coaches continue to build their teams based on voice alone.",
+                'description': 'Emma Willis and Marvin Humes present the fifth set of blind auditions in the singing competition, as the coaches continue to build their teams based on voice alone.',
                 'duration': 5100,
             },
             'params': {
diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
index e94b1e35b..dda2c0959 100644
--- a/youtube_dl/extractor/ccc.py
+++ b/youtube_dl/extractor/ccc.py
@@ -45,7 +45,7 @@ class CCCIE(InfoExtractor):
         title = self._html_search_regex(
             r'(?s)<h1>(.*?)</h1>', webpage, 'title')
         description = self._html_search_regex(
-            r"(?s)<h3>About</h3>(.+?)<h3>",
+            r'(?s)<h3>About</h3>(.+?)<h3>',
             webpage, 'description', fatal=False)
         upload_date = unified_strdate(self._html_search_regex(
             r"(?s)<span[^>]+class='[^']*fa-calendar-o'[^>]*>(.+?)</span>",
diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 6f7b2a70d..b27b4e670 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -177,16 +177,16 @@ class CeskaTelevizeIE(InfoExtractor):
             for divider in [1000, 60, 60, 100]:
                 components.append(msec % divider)
                 msec //= divider
-            return "{3:02}:{2:02}:{1:02},{0:03}".format(*components)
+            return '{3:02}:{2:02}:{1:02},{0:03}'.format(*components)
 
         def _fix_subtitle(subtitle):
             for line in subtitle.splitlines():
-                m = re.match(r"^\s*([0-9]+);\s*([0-9]+)\s+([0-9]+)\s*$", line)
+                m = re.match(r'^\s*([0-9]+);\s*([0-9]+)\s+([0-9]+)\s*$', line)
                 if m:
                     yield m.group(1)
                     start, stop = (_msectotimecode(int(t)) for t in m.groups()[1:])
-                    yield "{0} --> {1}".format(start, stop)
+                    yield '{0} --> {1}'.format(start, stop)
                 else:
                     yield line
 
-        return "\r\n".join(_fix_subtitle(subtitles))
+        return '\r\n'.join(_fix_subtitle(subtitles))
diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 3b1bd4033..53489a14e 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -26,14 +26,14 @@ class CNNIE(InfoExtractor):
             'upload_date': '20130609',
         },
     }, {
-        "url": "http://edition.cnn.com/video/?/video/us/2013/08/21/sot-student-gives-epic-speech.georgia-institute-of-technology&utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+rss%2Fcnn_topstories+%28RSS%3A+Top+Stories%29",
-        "md5": "b5cc60c60a3477d185af8f19a2a26f4e",
-        "info_dict": {
+        'url': 'http://edition.cnn.com/video/?/video/us/2013/08/21/sot-student-gives-epic-speech.georgia-institute-of-technology&utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+rss%2Fcnn_topstories+%28RSS%3A+Top+Stories%29',
+        'md5': 'b5cc60c60a3477d185af8f19a2a26f4e',
+        'info_dict': {
             'id': 'us/2013/08/21/sot-student-gives-epic-speech.georgia-institute-of-technology',
             'ext': 'mp4',
-            "title": "Student's epic speech stuns new freshmen",
-            "description": "A Georgia Tech student welcomes the incoming freshmen with an epic speech backed by music from \"2001: A Space Odyssey.\"",
-            "upload_date": "20130821",
+            'title': "Student's epic speech stuns new freshmen",
+            'description': "A Georgia Tech student welcomes the incoming freshmen with an epic speech backed by music from \"2001: A Space Odyssey.\"",
+            'upload_date': '20130821',
         }
     }, {
         'url': 'http://www.cnn.com/video/data/2.0/video/living/2014/12/22/growing-america-nashville-salemtown-board-episode-1.hln.html',
diff --git a/youtube_dl/extractor/collegerama.py b/youtube_dl/extractor/collegerama.py
index 40667a0f1..f9e84193d 100644
--- a/youtube_dl/extractor/collegerama.py
+++ b/youtube_dl/extractor/collegerama.py
@@ -46,9 +46,9 @@ class CollegeRamaIE(InfoExtractor):
         video_id = self._match_id(url)
 
         player_options_request = {
-            "getPlayerOptionsRequest": {
-                "ResourceId": video_id,
-                "QueryString": "",
+            'getPlayerOptionsRequest': {
+                'ResourceId': video_id,
+                'QueryString': '',
             }
         }
 
diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 055c9eec5..5b1b99675 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -195,7 +195,7 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
             if len(altMovieParams) == 0:
                 raise ExtractorError('unable to find Flash URL in webpage ' + url)
             else:
-                mMovieParams = [("http://media.mtvnservices.com/" + altMovieParams[0], altMovieParams[0])]
+                mMovieParams = [('http://media.mtvnservices.com/' + altMovieParams[0], altMovieParams[0])]
 
         uri = mMovieParams[0][1]
         # Correct cc.com in uri
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 144d8c6b6..f411ea763 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1497,7 +1497,7 @@ class InfoExtractor(object):
     def _live_title(self, name):
         """ Generate the title for a live video """
         now = datetime.datetime.now()
-        now_str = now.strftime("%Y-%m-%d %H:%M")
+        now_str = now.strftime('%Y-%m-%d %H:%M')
         return name + ' ' + now_str
 
     def _int(self, v, name, fatal=False, **kwargs):
@@ -1570,7 +1570,7 @@ class InfoExtractor(object):
         return {}
 
     def _get_subtitles(self, *args, **kwargs):
-        raise NotImplementedError("This method must be implemented by subclasses")
+        raise NotImplementedError('This method must be implemented by subclasses')
 
     @staticmethod
     def _merge_subtitle_items(subtitle_list1, subtitle_list2):
@@ -1596,7 +1596,7 @@ class InfoExtractor(object):
         return {}
 
     def _get_automatic_captions(self, *args, **kwargs):
-        raise NotImplementedError("This method must be implemented by subclasses")
+        raise NotImplementedError('This method must be implemented by subclasses')
 
 
 class SearchInfoExtractor(InfoExtractor):
@@ -1636,7 +1636,7 @@ class SearchInfoExtractor(InfoExtractor):
 
     def _get_n_results(self, query, n):
         """Get a specified number of results for a query"""
-        raise NotImplementedError("This method must be implemented by subclasses")
+        raise NotImplementedError('This method must be implemented by subclasses')
 
     @property
     def SEARCH_KEY(self):
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 785594df8..c7032ffa2 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -180,40 +180,40 @@ class CrunchyrollIE(CrunchyrollBaseIE):
             return assvalue
 
         output = '[Script Info]\n'
-        output += 'Title: %s\n' % sub_root.attrib["title"]
+        output += 'Title: %s\n' % sub_root.attrib['title']
         output += 'ScriptType: v4.00+\n'
-        output += 'WrapStyle: %s\n' % sub_root.attrib["wrap_style"]
-        output += 'PlayResX: %s\n' % sub_root.attrib["play_res_x"]
-        output += 'PlayResY: %s\n' % sub_root.attrib["play_res_y"]
+        output += 'WrapStyle: %s\n' % sub_root.attrib['wrap_style']
+        output += 'PlayResX: %s\n' % sub_root.attrib['play_res_x']
+        output += 'PlayResY: %s\n' % sub_root.attrib['play_res_y']
         output += """ScaledBorderAndShadow: yes
 
 [V4+ Styles]
 Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
 """
         for style in sub_root.findall('./styles/style'):
-            output += 'Style: ' + style.attrib["name"]
-            output += ',' + style.attrib["font_name"]
-            output += ',' + style.attrib["font_size"]
-            output += ',' + style.attrib["primary_colour"]
-            output += ',' + style.attrib["secondary_colour"]
-            output += ',' + style.attrib["outline_colour"]
-            output += ',' + style.attrib["back_colour"]
-            output += ',' + ass_bool(style.attrib["bold"])
-            output += ',' + ass_bool(style.attrib["italic"])
-            output += ',' + ass_bool(style.attrib["underline"])
-            output += ',' + ass_bool(style.attrib["strikeout"])
-            output += ',' + style.attrib["scale_x"]
-            output += ',' + style.attrib["scale_y"]
-            output += ',' + style.attrib["spacing"]
-            output += ',' + style.attrib["angle"]
-            output += ',' + style.attrib["border_style"]
-            output += ',' + style.attrib["outline"]
-            output += ',' + style.attrib["shadow"]
-            output += ',' + style.attrib["alignment"]
-            output += ',' + style.attrib["margin_l"]
-            output += ',' + style.attrib["margin_r"]
-            output += ',' + style.attrib["margin_v"]
-            output += ',' + style.attrib["encoding"]
+            output += 'Style: ' + style.attrib['name']
+            output += ',' + style.attrib['font_name']
+            output += ',' + style.attrib['font_size']
+            output += ',' + style.attrib['primary_colour']
+            output += ',' + style.attrib['secondary_colour']
+            output += ',' + style.attrib['outline_colour']
+            output += ',' + style.attrib['back_colour']
+            output += ',' + ass_bool(style.attrib['bold'])
+            output += ',' + ass_bool(style.attrib['italic'])
+            output += ',' + ass_bool(style.attrib['underline'])
+            output += ',' + ass_bool(style.attrib['strikeout'])
+            output += ',' + style.attrib['scale_x']
+            output += ',' + style.attrib['scale_y']
+            output += ',' + style.attrib['spacing']
+            output += ',' + style.attrib['angle']
+            output += ',' + style.attrib['border_style']
+            output += ',' + style.attrib['outline']
+            output += ',' + style.attrib['shadow']
+            output += ',' + style.attrib['alignment']
+            output += ',' + style.attrib['margin_l']
+            output += ',' + style.attrib['margin_r']
+            output += ',' + style.attrib['margin_v']
+            output += ',' + style.attrib['encoding']
             output += '\n'
 
         output += """
@@ -222,15 +222,15 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 """
         for event in sub_root.findall('./events/event'):
             output += 'Dialogue: 0'
-            output += ',' + event.attrib["start"]
-            output += ',' + event.attrib["end"]
-            output += ',' + event.attrib["style"]
-            output += ',' + event.attrib["name"]
-            output += ',' + event.attrib["margin_l"]
-            output += ',' + event.attrib["margin_r"]
-            output += ',' + event.attrib["margin_v"]
-            output += ',' + event.attrib["effect"]
-            output += ',' + event.attrib["text"]
+            output += ',' + event.attrib['start']
+            output += ',' + event.attrib['end']
+            output += ',' + event.attrib['style']
+            output += ',' + event.attrib['name']
+            output += ',' + event.attrib['margin_l']
+            output += ',' + event.attrib['margin_r']
+            output += ',' + event.attrib['margin_v']
+            output += ',' + event.attrib['effect']
+            output += ',' + event.attrib['text']
             output += '\n'
 
         return output
@@ -376,7 +376,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 
 
 class CrunchyrollShowPlaylistIE(CrunchyrollBaseIE):
-    IE_NAME = "crunchyroll:playlist"
+    IE_NAME = 'crunchyroll:playlist'
     _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login))(?P<id>[\w\-]+))/?(?:\?|$)'
 
     _TESTS = [{
diff --git a/youtube_dl/extractor/drbonanza.py b/youtube_dl/extractor/drbonanza.py
index 8b98b013a..01271f8f0 100644
--- a/youtube_dl/extractor/drbonanza.py
+++ b/youtube_dl/extractor/drbonanza.py
@@ -87,7 +87,7 @@ class DRBonanzaIE(InfoExtractor):
 
         formats = []
         for file in info['Files']:
-            if info['Type'] == "Video":
+            if info['Type'] == 'Video':
                 if file['Type'] in video_types:
                     format = parse_filename_info(file['Location'])
                     format.update({
@@ -101,10 +101,10 @@ class DRBonanzaIE(InfoExtractor):
                         if '/bonanza/' in rtmp_url:
                             format['play_path'] = rtmp_url.split('/bonanza/')[1]
                     formats.append(format)
-                elif file['Type'] == "Thumb":
+                elif file['Type'] == 'Thumb':
                     thumbnail = file['Location']
-            elif info['Type'] == "Audio":
-                if file['Type'] == "Audio":
+            elif info['Type'] == 'Audio':
+                if file['Type'] == 'Audio':
                     format = parse_filename_info(file['Location'])
                     format.update({
                         'url': file['Location'],
@@ -112,7 +112,7 @@ class DRBonanzaIE(InfoExtractor):
                         'vcodec': 'none',
                     })
                     formats.append(format)
-                elif file['Type'] == "Thumb":
+                elif file['Type'] == 'Thumb':
                     thumbnail = file['Location']
 
         description = '%s\n%s\n%s\n' % (
diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py
index 0b61ea0ba..9a44f89f3 100644
--- a/youtube_dl/extractor/eighttracks.py
+++ b/youtube_dl/extractor/eighttracks.py
@@ -17,85 +17,85 @@ class EightTracksIE(InfoExtractor):
     IE_NAME = '8tracks'
     _VALID_URL = r'https?://8tracks\.com/(?P<user>[^/]+)/(?P<id>[^/#]+)(?:#.*)?$'
     _TEST = {
-        "name": "EightTracks",
-        "url": "http://8tracks.com/ytdl/youtube-dl-test-tracks-a",
-        "info_dict": {
+        'name': 'EightTracks',
+        'url': 'http://8tracks.com/ytdl/youtube-dl-test-tracks-a',
+        'info_dict': {
             'id': '1336550',
             'display_id': 'youtube-dl-test-tracks-a',
-            "description": "test chars:  \"'/\\ä↭",
-            "title": "youtube-dl test tracks \"'/\\ä↭<>",
+            'description': "test chars:  \"'/\\ä↭",
+            'title': "youtube-dl test tracks \"'/\\ä↭<>",
         },
-        "playlist": [
+        'playlist': [
             {
-                "md5": "96ce57f24389fc8734ce47f4c1abcc55",
-                "info_dict": {
-                    "id": "11885610",
-                    "ext": "m4a",
-                    "title": "youtue-dl project<>\"' - youtube-dl test track 1 \"'/\\\u00e4\u21ad",
-                    "uploader_id": "ytdl"
+                'md5': '96ce57f24389fc8734ce47f4c1abcc55',
+                'info_dict': {
+                    'id': '11885610',
+                    'ext': 'm4a',
+                    'title': "youtue-dl project<>\"' - youtube-dl test track 1 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'
                 }
             },
             {
-                "md5": "4ab26f05c1f7291ea460a3920be8021f",
-                "info_dict": {
-                    "id": "11885608",
-                    "ext": "m4a",
-                    "title": "youtube-dl project - youtube-dl test track 2 \"'/\\\u00e4\u21ad",
-                    "uploader_id": "ytdl"
+                'md5': '4ab26f05c1f7291ea460a3920be8021f',
+                'info_dict': {
+                    'id': '11885608',
+                    'ext': 'm4a',
+                    'title': "youtube-dl project - youtube-dl test track 2 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'
                 }
             },
             {
-                "md5": "d30b5b5f74217410f4689605c35d1fd7",
-                "info_dict": {
-                    "id": "11885679",
-                    "ext": "m4a",
-                    "title": "youtube-dl project as well - youtube-dl test track 3 \"'/\\\u00e4\u21ad",
-                    "uploader_id": "ytdl"
+                'md5': 'd30b5b5f74217410f4689605c35d1fd7',
+                'info_dict': {
+                    'id': '11885679',
+                    'ext': 'm4a',
+                    'title': "youtube-dl project as well - youtube-dl test track 3 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'
                 }
             },
             {
-                "md5": "4eb0a669317cd725f6bbd336a29f923a",
-                "info_dict": {
-                    "id": "11885680",
-                    "ext": "m4a",
-                    "title": "youtube-dl project as well - youtube-dl test track 4 \"'/\\\u00e4\u21ad",
-                    "uploader_id": "ytdl"
+                'md5': '4eb0a669317cd725f6bbd336a29f923a',
+                'info_dict': {
+                    'id': '11885680',
+                    'ext': 'm4a',
+                    'title': "youtube-dl project as well - youtube-dl test track 4 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'
                 }
             },
             {
-                "md5": "1893e872e263a2705558d1d319ad19e8",
-                "info_dict": {
-                    "id": "11885682",
-                    "ext": "m4a",
-                    "title": "PH - youtube-dl test track 5 \"'/\\\u00e4\u21ad",
-                    "uploader_id": "ytdl"
+                'md5': '1893e872e263a2705558d1d319ad19e8',
+                'info_dict': {
+                    'id': '11885682',
+                    'ext': 'm4a',
+                    'title': "PH - youtube-dl test track 5 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'
                 }
             },
             {
-                "md5": "b673c46f47a216ab1741ae8836af5899",
-                "info_dict": {
-                    "id": "11885683",
-                    "ext": "m4a",
-                    "title": "PH - youtube-dl test track 6 \"'/\\\u00e4\u21ad",
-                    "uploader_id": "ytdl"
+                'md5': 'b673c46f47a216ab1741ae8836af5899',
+                'info_dict': {
+                    'id': '11885683',
+                    'ext': 'm4a',
+                    'title': "PH - youtube-dl test track 6 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'
                 }
             },
             {
-                "md5": "1d74534e95df54986da7f5abf7d842b7",
-                "info_dict": {
-                    "id": "11885684",
-                    "ext": "m4a",
-                    "title": "phihag - youtube-dl test track 7 \"'/\\\u00e4\u21ad",
-                    "uploader_id": "ytdl"
+                'md5': '1d74534e95df54986da7f5abf7d842b7',
+                'info_dict': {
+                    'id': '11885684',
+                    'ext': 'm4a',
+                    'title': "phihag - youtube-dl test track 7 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'
                 }
             },
             {
-                "md5": "f081f47af8f6ae782ed131d38b9cd1c0",
-                "info_dict": {
-                    "id": "11885685",
-                    "ext": "m4a",
-                    "title": "phihag - youtube-dl test track 8 \"'/\\\u00e4\u21ad",
-                    "uploader_id": "ytdl"
+                'md5': 'f081f47af8f6ae782ed131d38b9cd1c0',
+                'info_dict': {
+                    'id': '11885685',
+                    'ext': 'm4a',
+                    'title': "phihag - youtube-dl test track 8 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'
                 }
             }
         ]
diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 476cce2d0..4c8190d68 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -72,7 +72,7 @@ class EllenTVClipsIE(InfoExtractor):
     def _extract_playlist(self, webpage):
         json_string = self._search_regex(r'playerView.addClips\(\[\{(.*?)\}\]\);', webpage, 'json')
         try:
-            return json.loads("[{" + json_string + "}]")
+            return json.loads('[{' + json_string + '}]')
         except ValueError as ve:
             raise ExtractorError('Failed to download JSON', cause=ve)
 
diff --git a/youtube_dl/extractor/everyonesmixtape.py b/youtube_dl/extractor/everyonesmixtape.py
index 493d38af8..84a9b750e 100644
--- a/youtube_dl/extractor/everyonesmixtape.py
+++ b/youtube_dl/extractor/everyonesmixtape.py
@@ -14,14 +14,14 @@ class EveryonesMixtapeIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://everyonesmixtape.com/#/mix/m7m0jJAbMQi/5',
-        "info_dict": {
+        'info_dict': {
             'id': '5bfseWNmlds',
             'ext': 'mp4',
-            "title": "Passion Pit - \"Sleepyhead\" (Official Music Video)",
-            "uploader": "FKR.TV",
-            "uploader_id": "frenchkissrecords",
-            "description": "Music video for \"Sleepyhead\" from Passion Pit's debut EP Chunk Of Change.\nBuy on iTunes: https://itunes.apple.com/us/album/chunk-of-change-ep/id300087641\n\nDirected by The Wilderness.\n\nhttp://www.passionpitmusic.com\nhttp://www.frenchkissrecords.com",
-            "upload_date": "20081015"
+            'title': "Passion Pit - \"Sleepyhead\" (Official Music Video)",
+            'uploader': 'FKR.TV',
+            'uploader_id': 'frenchkissrecords',
+            'description': "Music video for \"Sleepyhead\" from Passion Pit's debut EP Chunk Of Change.\nBuy on iTunes: https://itunes.apple.com/us/album/chunk-of-change-ep/id300087641\n\nDirected by The Wilderness.\n\nhttp://www.passionpitmusic.com\nhttp://www.frenchkissrecords.com",
+            'upload_date': '20081015'
         },
         'params': {
             'skip_download': True,  # This is simply YouTube
diff --git a/youtube_dl/extractor/exfm.py b/youtube_dl/extractor/exfm.py
index 4de02aee9..0c0fe6d65 100644
--- a/youtube_dl/extractor/exfm.py
+++ b/youtube_dl/extractor/exfm.py
@@ -41,7 +41,7 @@ class ExfmIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         song_id = mobj.group('id')
-        info_url = "http://ex.fm/api/v3/song/%s" % song_id
+        info_url = 'http://ex.fm/api/v3/song/%s' % song_id
         info = self._download_json(info_url, song_id)['song']
         song_url = info['url']
         if re.match(self._SOUNDCLOUD_URL, song_url) is not None:
diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index 4c81271d3..9580f5c0c 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -87,7 +87,7 @@ class FC2IE(InfoExtractor):
         mimi = hashlib.md5((video_id + '_gGddgPfeaf_gzyr').encode('utf-8')).hexdigest()
 
         info_url = (
-            "http://video.fc2.com/ginfo.php?mimi={1:s}&href={2:s}&v={0:s}&fversion=WIN%2011%2C6%2C602%2C180&from=2&otag=0&upid={0:s}&tk=null&".
+            'http://video.fc2.com/ginfo.php?mimi={1:s}&href={2:s}&v={0:s}&fversion=WIN%2011%2C6%2C602%2C180&from=2&otag=0&upid={0:s}&tk=null&'.
             format(video_id, mimi, compat_urllib_request.quote(refer, safe=b'').replace('.', '%2E')))
 
         info_webpage = self._download_webpage(
diff --git a/youtube_dl/extractor/franceinter.py b/youtube_dl/extractor/franceinter.py
index fdc51f44f..0388ba00c 100644
--- a/youtube_dl/extractor/franceinter.py
+++ b/youtube_dl/extractor/franceinter.py
@@ -10,7 +10,7 @@ class FranceInterIE(InfoExtractor):
     _TEST = {
         'url': 'http://www.franceinter.fr/player/reecouter?play=793962',
         'md5': '4764932e466e6f6c79c317d2e74f6884',
-        "info_dict": {
+        'info_dict': {
             'id': '793962',
             'ext': 'mp3',
             'title': 'L’Histoire dans les jeux vidéo',
diff --git a/youtube_dl/extractor/freevideo.py b/youtube_dl/extractor/freevideo.py
index f755e3c4a..c7bec027b 100644
--- a/youtube_dl/extractor/freevideo.py
+++ b/youtube_dl/extractor/freevideo.py
@@ -12,8 +12,8 @@ class FreeVideoIE(InfoExtractor):
         'info_dict': {
             'id': 'vysukany-zadecek-22033',
             'ext': 'mp4',
-            "title": "vysukany-zadecek-22033",
-            "age_limit": 18,
+            'title': 'vysukany-zadecek-22033',
+            'age_limit': 18,
         },
         'skip': 'Blocked outside .cz',
     }
diff --git a/youtube_dl/extractor/hentaistigma.py b/youtube_dl/extractor/hentaistigma.py
index f5aa73d18..86a93de4d 100644
--- a/youtube_dl/extractor/hentaistigma.py
+++ b/youtube_dl/extractor/hentaistigma.py
@@ -11,8 +11,8 @@ class HentaiStigmaIE(InfoExtractor):
         'info_dict': {
             'id': 'inyouchuu-etsu-bonus',
             'ext': 'mp4',
-            "title": "Inyouchuu Etsu Bonus",
-            "age_limit": 18,
+            'title': 'Inyouchuu Etsu Bonus',
+            'age_limit': 18,
         }
     }
 
diff --git a/youtube_dl/extractor/kankan.py b/youtube_dl/extractor/kankan.py
index 364dc878e..a677ff447 100644
--- a/youtube_dl/extractor/kankan.py
+++ b/youtube_dl/extractor/kankan.py
@@ -28,7 +28,7 @@ class KankanIE(InfoExtractor):
 
         title = self._search_regex(r'(?:G_TITLE=|G_MOVIE_TITLE = )[\'"](.+?)[\'"]', webpage, 'video title')
         surls = re.search(r'surls:\[\'.+?\'\]|lurl:\'.+?\.flv\'', webpage).group(0)
-        gcids = re.findall(r"http://.+?/.+?/(.+?)/", surls)
+        gcids = re.findall(r'http://.+?/.+?/(.+?)/', surls)
         gcid = gcids[-1]
 
         info_url = 'http://p2s.cl.kankan.com/getCdnresource_flv?gcid=%s' % gcid
diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index 857edfde2..4684994e1 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -47,7 +47,7 @@ class LiveLeakIE(InfoExtractor):
         'info_dict': {
             'id': '801_1409392012',
             'ext': 'mp4',
-            'description': "Happened on 27.7.2014. \r\nAt 0:53 you can see people still swimming at near beach.",
+            'description': 'Happened on 27.7.2014. \r\nAt 0:53 you can see people still swimming at near beach.',
             'uploader': 'bony333',
             'title': 'Crazy Hungarian tourist films close call waterspout in Croatia'
         }
diff --git a/youtube_dl/extractor/mofosex.py b/youtube_dl/extractor/mofosex.py
index f8226cbb2..e47c80119 100644
--- a/youtube_dl/extractor/mofosex.py
+++ b/youtube_dl/extractor/mofosex.py
@@ -38,7 +38,7 @@ class MofosexIE(InfoExtractor):
         path = compat_urllib_parse_urlparse(video_url).path
         extension = os.path.splitext(path)[1][1:]
         format = path.split('/')[5].split('_')[:2]
-        format = "-".join(format)
+        format = '-'.join(format)
 
         age_limit = self._rta_search(webpage)
 
diff --git a/youtube_dl/extractor/myspass.py b/youtube_dl/extractor/myspass.py
index 4557a2b13..f936b92bb 100644
--- a/youtube_dl/extractor/myspass.py
+++ b/youtube_dl/extractor/myspass.py
@@ -18,8 +18,8 @@ class MySpassIE(InfoExtractor):
         'info_dict': {
             'id': '11741',
             'ext': 'mp4',
-            "description": "Wer kann in die Fu\u00dfstapfen von Wolfgang Kubicki treten und die Mehrheit der Zuschauer hinter sich versammeln? Wird vielleicht sogar die Absolute Mehrheit geknackt und der Jackpot von 200.000 Euro mit nach Hause genommen?",
-            "title": "Absolute Mehrheit vom 17.02.2013 - Die Highlights, Teil 2",
+            'description': 'Wer kann in die Fu\u00dfstapfen von Wolfgang Kubicki treten und die Mehrheit der Zuschauer hinter sich versammeln? Wird vielleicht sogar die Absolute Mehrheit geknackt und der Jackpot von 200.000 Euro mit nach Hause genommen?',
+            'title': 'Absolute Mehrheit vom 17.02.2013 - Die Highlights, Teil 2',
         },
     }
 
diff --git a/youtube_dl/extractor/nerdcubed.py b/youtube_dl/extractor/nerdcubed.py
index dff78e486..9feccc672 100644
--- a/youtube_dl/extractor/nerdcubed.py
+++ b/youtube_dl/extractor/nerdcubed.py
@@ -18,14 +18,14 @@ class NerdCubedFeedIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        feed = self._download_json(url, url, "Downloading NerdCubed JSON feed")
+        feed = self._download_json(url, url, 'Downloading NerdCubed JSON feed')
 
         entries = [{
             '_type': 'url',
             'title': feed_entry['title'],
             'uploader': feed_entry['source']['name'] if feed_entry['source'] else None,
             'upload_date': datetime.datetime.strptime(feed_entry['date'], '%Y-%m-%d').strftime('%Y%m%d'),
-            'url': "http://www.youtube.com/watch?v=" + feed_entry['youtube_id'],
+            'url': 'http://www.youtube.com/watch?v=' + feed_entry['youtube_id'],
         } for feed_entry in feed]
 
         return {
diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 08275687d..91e574dc2 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -27,9 +27,9 @@ class PornHubIE(InfoExtractor):
         'info_dict': {
             'id': '648719015',
             'ext': 'mp4',
-            "uploader": "Babes",
-            "title": "Seductive Indian beauty strips down and fingers her pink pussy",
-            "age_limit": 18
+            'uploader': 'Babes',
+            'title': 'Seductive Indian beauty strips down and fingers her pink pussy',
+            'age_limit': 18
         }
     }, {
         'url': 'http://www.pornhub.com/view_video.php?viewkey=ph557bbb6676d2d',
@@ -95,7 +95,7 @@ class PornHubIE(InfoExtractor):
             path = compat_urllib_parse_urlparse(video_url).path
             extension = os.path.splitext(path)[1][1:]
             format = path.split('/')[5].split('_')[:2]
-            format = "-".join(format)
+            format = '-'.join(format)
 
             m = re.match(r'^(?P<height>[0-9]+)[pP]-(?P<tbr>[0-9]+)[kK]$', format)
             if m is None:
diff --git a/youtube_dl/extractor/pornovoisines.py b/youtube_dl/extractor/pornovoisines.py
index eba4dfbb3..1a53fd71c 100644
--- a/youtube_dl/extractor/pornovoisines.py
+++ b/youtube_dl/extractor/pornovoisines.py
@@ -56,7 +56,7 @@ class PornoVoisinesIE(InfoExtractor):
             r'<h1>(.+?)</h1>', webpage, 'title', flags=re.DOTALL)
         description = self._html_search_regex(
             r'<article id="descriptif">(.+?)</article>',
-            webpage, "description", fatal=False, flags=re.DOTALL)
+            webpage, 'description', fatal=False, flags=re.DOTALL)
 
         thumbnail = self._search_regex(
             r'<div id="mediaspace%s">\s*<img src="/?([^"]+)"' % video_id,
diff --git a/youtube_dl/extractor/radiobremen.py b/youtube_dl/extractor/radiobremen.py
index 0d706312e..0cbb15f08 100644
--- a/youtube_dl/extractor/radiobremen.py
+++ b/youtube_dl/extractor/radiobremen.py
@@ -28,16 +28,16 @@ class RadioBremenIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        meta_url = "http://www.radiobremen.de/apps/php/mediathek/metadaten.php?id=%s" % video_id
+        meta_url = 'http://www.radiobremen.de/apps/php/mediathek/metadaten.php?id=%s' % video_id
         meta_doc = self._download_webpage(
             meta_url, video_id, 'Downloading metadata')
         title = self._html_search_regex(
-            r"<h1.*>(?P<title>.+)</h1>", meta_doc, "title")
+            r'<h1.*>(?P<title>.+)</h1>', meta_doc, 'title')
         description = self._html_search_regex(
-            r"<p>(?P<description>.*)</p>", meta_doc, "description", fatal=False)
+            r'<p>(?P<description>.*)</p>', meta_doc, 'description', fatal=False)
         duration = parse_duration(self._html_search_regex(
-            r"L&auml;nge:</td>\s+<td>(?P<duration>[0-9]+:[0-9]+)</td>",
-            meta_doc, "duration", fatal=False))
+            r'L&auml;nge:</td>\s+<td>(?P<duration>[0-9]+:[0-9]+)</td>',
+            meta_doc, 'duration', fatal=False))
 
         page_doc = self._download_webpage(
             url, video_id, 'Downloading video information')
@@ -51,7 +51,7 @@ class RadioBremenIE(InfoExtractor):
         formats = [{
             'url': video_url,
             'ext': 'mp4',
-            'width': int(mobj.group("width")),
+            'width': int(mobj.group('width')),
         }]
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/radiofrance.py b/youtube_dl/extractor/radiofrance.py
index 09352ed82..a8afc0014 100644
--- a/youtube_dl/extractor/radiofrance.py
+++ b/youtube_dl/extractor/radiofrance.py
@@ -16,9 +16,9 @@ class RadioFranceIE(InfoExtractor):
         'info_dict': {
             'id': 'one-one',
             'ext': 'ogg',
-            "title": "One to one",
-            "description": "Plutôt que d'imaginer la radio de demain comme technologie ou comme création de contenu, je veux montrer que quelles que soient ses évolutions, j'ai l'intime conviction que la radio continuera d'être un grand média de proximité pour les auditeurs.",
-            "uploader": "Thomas Hercouët",
+            'title': 'One to one',
+            'description': "Plutôt que d'imaginer la radio de demain comme technologie ou comme création de contenu, je veux montrer que quelles que soient ses évolutions, j'ai l'intime conviction que la radio continuera d'être un grand média de proximité pour les auditeurs.",
+            'uploader': 'Thomas Hercouët',
         },
     }
 
diff --git a/youtube_dl/extractor/rbmaradio.py b/youtube_dl/extractor/rbmaradio.py
index 0f8f3ebde..7932af6ef 100644
--- a/youtube_dl/extractor/rbmaradio.py
+++ b/youtube_dl/extractor/rbmaradio.py
@@ -18,11 +18,11 @@ class RBMARadioIE(InfoExtractor):
         'info_dict': {
             'id': 'ford-lopatin-live-at-primavera-sound-2011',
             'ext': 'mp3',
-            "uploader_id": "ford-lopatin",
-            "location": "Spain",
-            "description": "Joel Ford and Daniel ’Oneohtrix Point Never’ Lopatin fly their midified pop extravaganza to Spain. Live at Primavera Sound 2011.",
-            "uploader": "Ford & Lopatin",
-            "title": "Live at Primavera Sound 2011",
+            'uploader_id': 'ford-lopatin',
+            'location': 'Spain',
+            'description': 'Joel Ford and Daniel ’Oneohtrix Point Never’ Lopatin fly their midified pop extravaganza to Spain. Live at Primavera Sound 2011.',
+            'uploader': 'Ford & Lopatin',
+            'title': 'Live at Primavera Sound 2011',
         },
     }
 
diff --git a/youtube_dl/extractor/reverbnation.py b/youtube_dl/extractor/reverbnation.py
index ec7e7df7b..3c6725aeb 100644
--- a/youtube_dl/extractor/reverbnation.py
+++ b/youtube_dl/extractor/reverbnation.py
@@ -12,12 +12,12 @@ class ReverbNationIE(InfoExtractor):
         'url': 'http://www.reverbnation.com/alkilados/song/16965047-mona-lisa',
         'md5': '3da12ebca28c67c111a7f8b262d3f7a7',
         'info_dict': {
-            "id": "16965047",
-            "ext": "mp3",
-            "title": "MONA LISA",
-            "uploader": "ALKILADOS",
-            "uploader_id": "216429",
-            "thumbnail": "re:^https://gp1\.wac\.edgecastcdn\.net/.*?\.jpg$"
+            'id': '16965047',
+            'ext': 'mp3',
+            'title': 'MONA LISA',
+            'uploader': 'ALKILADOS',
+            'uploader_id': '216429',
+            'thumbnail': 're:^https://gp1\.wac\.edgecastcdn\.net/.*?\.jpg$'
         },
     }]
 
diff --git a/youtube_dl/extractor/ringtv.py b/youtube_dl/extractor/ringtv.py
index efa4afeb6..508758075 100644
--- a/youtube_dl/extractor/ringtv.py
+++ b/youtube_dl/extractor/ringtv.py
@@ -8,13 +8,13 @@ from .common import InfoExtractor
 class RingTVIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?ringtv\.craveonline\.com/(?P<type>news|videos/video)/(?P<id>[^/?#]+)'
     _TEST = {
-        "url": "http://ringtv.craveonline.com/news/310833-luis-collazo-says-victor-ortiz-better-not-quit-on-jan-30",
-        "md5": "d25945f5df41cdca2d2587165ac28720",
-        "info_dict": {
+        'url': 'http://ringtv.craveonline.com/news/310833-luis-collazo-says-victor-ortiz-better-not-quit-on-jan-30',
+        'md5': 'd25945f5df41cdca2d2587165ac28720',
+        'info_dict': {
             'id': '857645',
             'ext': 'mp4',
-            "title": 'Video: Luis Collazo says Victor Ortiz "better not quit on Jan. 30" - Ring TV',
-            "description": 'Luis Collazo is excited about his Jan. 30 showdown with fellow former welterweight titleholder Victor Ortiz at Barclays Center in his hometown of Brooklyn. The SuperBowl week fight headlines a Golden Boy Live! card on Fox Sports 1.',
+            'title': 'Video: Luis Collazo says Victor Ortiz "better not quit on Jan. 30" - Ring TV',
+            'description': 'Luis Collazo is excited about his Jan. 30 showdown with fellow former welterweight titleholder Victor Ortiz at Barclays Center in his hometown of Brooklyn. The SuperBowl week fight headlines a Golden Boy Live! card on Fox Sports 1.',
         }
     }
 
@@ -32,8 +32,8 @@ class RingTVIE(InfoExtractor):
         description = self._html_search_regex(
             r'addthis:description="([^"]+)"',
             webpage, 'description', fatal=False)
-        final_url = "http://ringtv.craveonline.springboardplatform.com/storage/ringtv.craveonline.com/conversion/%s.mp4" % video_id
-        thumbnail_url = "http://ringtv.craveonline.springboardplatform.com/storage/ringtv.craveonline.com/snapshots/%s.jpg" % video_id
+        final_url = 'http://ringtv.craveonline.springboardplatform.com/storage/ringtv.craveonline.com/conversion/%s.mp4' % video_id
+        thumbnail_url = 'http://ringtv.craveonline.springboardplatform.com/storage/ringtv.craveonline.com/snapshots/%s.jpg' % video_id
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index 73c9788be..042bc8dab 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -43,7 +43,7 @@ class RteIE(InfoExtractor):
             r'<meta name="thumbnail" content="uri:irus:(.*?)" />', webpage, 'thumbnail')
         thumbnail = 'http://img.rasset.ie/' + thumbnail_id + '.jpg'
 
-        feeds_url = self._html_search_meta("feeds-prefix", webpage, 'feeds url') + video_id
+        feeds_url = self._html_search_meta('feeds-prefix', webpage, 'feeds url') + video_id
         json_string = self._download_json(feeds_url, video_id)
 
         # f4m_url = server + relative_url
diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
index 25f7faf76..de004671d 100644
--- a/youtube_dl/extractor/rtl2.py
+++ b/youtube_dl/extractor/rtl2.py
@@ -63,7 +63,7 @@ class RTL2IE(InfoExtractor):
         download_url = video_info['streamurl']
         download_url = download_url.replace('\\', '')
         stream_url = 'mp4:' + self._html_search_regex(r'ondemand/(.*)', download_url, 'stream URL')
-        rtmp_conn = ["S:connect", "O:1", "NS:pageUrl:" + url, "NB:fpad:0", "NN:videoFunction:1", "O:0"]
+        rtmp_conn = ['S:connect', 'O:1', 'NS:pageUrl:' + url, 'NB:fpad:0', 'NN:videoFunction:1', 'O:0']
 
         formats = [{
             'url': download_url,
diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index e5d62a139..2cf210e0d 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -40,7 +40,7 @@ class ScreenwaveMediaIE(InfoExtractor):
                 re.sub(
                     r'(?s)/\*.*?\*/', '',
                     self._search_regex(
-                        r"sources\s*:\s*(\[[^\]]+?\])", playerconfig,
+                        r'sources\s*:\s*(\[[^\]]+?\])', playerconfig,
                         'sources',
                     ).replace(
                         "' + thisObj.options.videoserver + '",
diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
index 990ea0fa8..4d3b58522 100644
--- a/youtube_dl/extractor/senateisvp.py
+++ b/youtube_dl/extractor/senateisvp.py
@@ -15,37 +15,37 @@ from ..compat import (
 
 class SenateISVPIE(InfoExtractor):
     _COMM_MAP = [
-        ["ag", "76440", "http://ag-f.akamaihd.net"],
-        ["aging", "76442", "http://aging-f.akamaihd.net"],
-        ["approps", "76441", "http://approps-f.akamaihd.net"],
-        ["armed", "76445", "http://armed-f.akamaihd.net"],
-        ["banking", "76446", "http://banking-f.akamaihd.net"],
-        ["budget", "76447", "http://budget-f.akamaihd.net"],
-        ["cecc", "76486", "http://srs-f.akamaihd.net"],
-        ["commerce", "80177", "http://commerce1-f.akamaihd.net"],
-        ["csce", "75229", "http://srs-f.akamaihd.net"],
-        ["dpc", "76590", "http://dpc-f.akamaihd.net"],
-        ["energy", "76448", "http://energy-f.akamaihd.net"],
-        ["epw", "76478", "http://epw-f.akamaihd.net"],
-        ["ethics", "76449", "http://ethics-f.akamaihd.net"],
-        ["finance", "76450", "http://finance-f.akamaihd.net"],
-        ["foreign", "76451", "http://foreign-f.akamaihd.net"],
-        ["govtaff", "76453", "http://govtaff-f.akamaihd.net"],
-        ["help", "76452", "http://help-f.akamaihd.net"],
-        ["indian", "76455", "http://indian-f.akamaihd.net"],
-        ["intel", "76456", "http://intel-f.akamaihd.net"],
-        ["intlnarc", "76457", "http://intlnarc-f.akamaihd.net"],
-        ["jccic", "85180", "http://jccic-f.akamaihd.net"],
-        ["jec", "76458", "http://jec-f.akamaihd.net"],
-        ["judiciary", "76459", "http://judiciary-f.akamaihd.net"],
-        ["rpc", "76591", "http://rpc-f.akamaihd.net"],
-        ["rules", "76460", "http://rules-f.akamaihd.net"],
-        ["saa", "76489", "http://srs-f.akamaihd.net"],
-        ["smbiz", "76461", "http://smbiz-f.akamaihd.net"],
-        ["srs", "75229", "http://srs-f.akamaihd.net"],
-        ["uscc", "76487", "http://srs-f.akamaihd.net"],
-        ["vetaff", "76462", "http://vetaff-f.akamaihd.net"],
-        ["arch", "", "http://ussenate-f.akamaihd.net/"]
+        ['ag', '76440', 'http://ag-f.akamaihd.net'],
+        ['aging', '76442', 'http://aging-f.akamaihd.net'],
+        ['approps', '76441', 'http://approps-f.akamaihd.net'],
+        ['armed', '76445', 'http://armed-f.akamaihd.net'],
+        ['banking', '76446', 'http://banking-f.akamaihd.net'],
+        ['budget', '76447', 'http://budget-f.akamaihd.net'],
+        ['cecc', '76486', 'http://srs-f.akamaihd.net'],
+        ['commerce', '80177', 'http://commerce1-f.akamaihd.net'],
+        ['csce', '75229', 'http://srs-f.akamaihd.net'],
+        ['dpc', '76590', 'http://dpc-f.akamaihd.net'],
+        ['energy', '76448', 'http://energy-f.akamaihd.net'],
+        ['epw', '76478', 'http://epw-f.akamaihd.net'],
+        ['ethics', '76449', 'http://ethics-f.akamaihd.net'],
+        ['finance', '76450', 'http://finance-f.akamaihd.net'],
+        ['foreign', '76451', 'http://foreign-f.akamaihd.net'],
+        ['govtaff', '76453', 'http://govtaff-f.akamaihd.net'],
+        ['help', '76452', 'http://help-f.akamaihd.net'],
+        ['indian', '76455', 'http://indian-f.akamaihd.net'],
+        ['intel', '76456', 'http://intel-f.akamaihd.net'],
+        ['intlnarc', '76457', 'http://intlnarc-f.akamaihd.net'],
+        ['jccic', '85180', 'http://jccic-f.akamaihd.net'],
+        ['jec', '76458', 'http://jec-f.akamaihd.net'],
+        ['judiciary', '76459', 'http://judiciary-f.akamaihd.net'],
+        ['rpc', '76591', 'http://rpc-f.akamaihd.net'],
+        ['rules', '76460', 'http://rules-f.akamaihd.net'],
+        ['saa', '76489', 'http://srs-f.akamaihd.net'],
+        ['smbiz', '76461', 'http://smbiz-f.akamaihd.net'],
+        ['srs', '75229', 'http://srs-f.akamaihd.net'],
+        ['uscc', '76487', 'http://srs-f.akamaihd.net'],
+        ['vetaff', '76462', 'http://vetaff-f.akamaihd.net'],
+        ['arch', '', 'http://ussenate-f.akamaihd.net/']
     ]
     _IE_NAME = 'senate.gov'
     _VALID_URL = r'http://www\.senate\.gov/isvp/?\?(?P<qs>.+)'
diff --git a/youtube_dl/extractor/slutload.py b/youtube_dl/extractor/slutload.py
index 3df71304d..7efb29f65 100644
--- a/youtube_dl/extractor/slutload.py
+++ b/youtube_dl/extractor/slutload.py
@@ -13,8 +13,8 @@ class SlutloadIE(InfoExtractor):
         'info_dict': {
             'id': 'TD73btpBqSxc',
             'ext': 'mp4',
-            "title": "virginie baisee en cam",
-            "age_limit": 18,
+            'title': 'virginie baisee en cam',
+            'age_limit': 18,
             'thumbnail': 're:https?://.*?\.jpg'
         }
     }
diff --git a/youtube_dl/extractor/snotr.py b/youtube_dl/extractor/snotr.py
index da3b05a8d..0d1ab07f8 100644
--- a/youtube_dl/extractor/snotr.py
+++ b/youtube_dl/extractor/snotr.py
@@ -43,7 +43,7 @@ class SnotrIE(InfoExtractor):
         title = self._og_search_title(webpage)
 
         description = self._og_search_description(webpage)
-        video_url = "http://cdn.videos.snotr.com/%s.flv" % video_id
+        video_url = 'http://cdn.videos.snotr.com/%s.flv' % video_id
 
         view_count = str_to_int(self._html_search_regex(
             r'<p>\n<strong>Views:</strong>\n([\d,\.]+)</p>',
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index b2d5487ca..1efb2b980 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -222,7 +222,7 @@ class SoundcloudIE(InfoExtractor):
             full_title = track_id
             token = mobj.group('secret_token')
             if token:
-                info_json_url += "&secret_token=" + token
+                info_json_url += '&secret_token=' + token
         elif mobj.group('player'):
             query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
             real_url = query['url'][0]
diff --git a/youtube_dl/extractor/steam.py b/youtube_dl/extractor/steam.py
index 183dcb03c..1a831ef6d 100644
--- a/youtube_dl/extractor/steam.py
+++ b/youtube_dl/extractor/steam.py
@@ -22,23 +22,23 @@ class SteamIE(InfoExtractor):
     _VIDEO_PAGE_TEMPLATE = 'http://store.steampowered.com/video/%s/'
     _AGECHECK_TEMPLATE = 'http://store.steampowered.com/agecheck/video/%s/?snr=1_agecheck_agecheck__age-gate&ageDay=1&ageMonth=January&ageYear=1970'
     _TESTS = [{
-        "url": "http://store.steampowered.com/video/105600/",
-        "playlist": [
+        'url': 'http://store.steampowered.com/video/105600/',
+        'playlist': [
             {
-                "md5": "f870007cee7065d7c76b88f0a45ecc07",
-                "info_dict": {
+                'md5': 'f870007cee7065d7c76b88f0a45ecc07',
+                'info_dict': {
                     'id': '81300',
                     'ext': 'flv',
-                    "title": "Terraria 1.1 Trailer",
+                    'title': 'Terraria 1.1 Trailer',
                     'playlist_index': 1,
                 }
             },
             {
-                "md5": "61aaf31a5c5c3041afb58fb83cbb5751",
-                "info_dict": {
+                'md5': '61aaf31a5c5c3041afb58fb83cbb5751',
+                'info_dict': {
                     'id': '80859',
                     'ext': 'flv',
-                    "title": "Terraria Trailer",
+                    'title': 'Terraria Trailer',
                     'playlist_index': 2,
                 }
             }
diff --git a/youtube_dl/extractor/tenplay.py b/youtube_dl/extractor/tenplay.py
index f6694149b..02a31a609 100644
--- a/youtube_dl/extractor/tenplay.py
+++ b/youtube_dl/extractor/tenplay.py
@@ -27,10 +27,10 @@ class TenPlayIE(InfoExtractor):
     }
 
     _video_fields = [
-        "id", "name", "shortDescription", "longDescription", "creationDate",
-        "publishedDate", "lastModifiedDate", "customFields", "videoStillURL",
-        "thumbnailURL", "referenceId", "length", "playsTotal",
-        "playsTrailingWeek", "renditions", "captioning", "startDate", "endDate"]
+        'id', 'name', 'shortDescription', 'longDescription', 'creationDate',
+        'publishedDate', 'lastModifiedDate', 'customFields', 'videoStillURL',
+        'thumbnailURL', 'referenceId', 'length', 'playsTotal',
+        'playsTrailingWeek', 'renditions', 'captioning', 'startDate', 'endDate']
 
     def _real_extract(self, url):
         webpage = self._download_webpage(url, url)
diff --git a/youtube_dl/extractor/thesixtyone.py b/youtube_dl/extractor/thesixtyone.py
index 5d09eb9a8..d8b1fd281 100644
--- a/youtube_dl/extractor/thesixtyone.py
+++ b/youtube_dl/extractor/thesixtyone.py
@@ -48,22 +48,22 @@ class TheSixtyOneIE(InfoExtractor):
     ]
 
     _DECODE_MAP = {
-        "x": "a",
-        "m": "b",
-        "w": "c",
-        "q": "d",
-        "n": "e",
-        "p": "f",
-        "a": "0",
-        "h": "1",
-        "e": "2",
-        "u": "3",
-        "s": "4",
-        "i": "5",
-        "o": "6",
-        "y": "7",
-        "r": "8",
-        "c": "9"
+        'x': 'a',
+        'm': 'b',
+        'w': 'c',
+        'q': 'd',
+        'n': 'e',
+        'p': 'f',
+        'a': '0',
+        'h': '1',
+        'e': '2',
+        'u': '3',
+        's': '4',
+        'i': '5',
+        'o': '6',
+        'y': '7',
+        'r': '8',
+        'c': '9'
     }
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/traileraddict.py b/youtube_dl/extractor/traileraddict.py
index 1c53a3fd0..0e01b15fc 100644
--- a/youtube_dl/extractor/traileraddict.py
+++ b/youtube_dl/extractor/traileraddict.py
@@ -38,12 +38,12 @@ class TrailerAddictIE(InfoExtractor):
 
         # Presence of (no)watchplus function indicates HD quality is available
         if re.search(r'function (no)?watchplus()', webpage):
-            fvar = "fvarhd"
+            fvar = 'fvarhd'
         else:
-            fvar = "fvar"
+            fvar = 'fvar'
 
-        info_url = "http://www.traileraddict.com/%s.php?tid=%s" % (fvar, str(video_id))
-        info_webpage = self._download_webpage(info_url, video_id, "Downloading the info webpage")
+        info_url = 'http://www.traileraddict.com/%s.php?tid=%s' % (fvar, str(video_id))
+        info_webpage = self._download_webpage(info_url, video_id, 'Downloading the info webpage')
 
         final_url = self._search_regex(r'&fileurl=(.+)',
                                        info_webpage, 'Download url').replace('%3F', '?')
diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index da3cd76f7..f56b66d06 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -49,7 +49,7 @@ class TudouIE(InfoExtractor):
         info_url = 'http://v2.tudou.com/f?id=' + compat_str(video_id)
         if quality:
             info_url += '&hd' + quality
-        xml_data = self._download_xml(info_url, video_id, "Opening the info XML page")
+        xml_data = self._download_xml(info_url, video_id, 'Opening the info XML page')
         final_url = xml_data.text
         return final_url
 
diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index 1e740fbe6..3794bcded 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -47,7 +47,7 @@ class Vbox7IE(InfoExtractor):
         title = self._html_search_regex(r'<title>(.*)</title>',
                                         webpage, 'title').split('/')[0].strip()
 
-        info_url = "http://vbox7.com/play/magare.do"
+        info_url = 'http://vbox7.com/play/magare.do'
         data = compat_urllib_parse.urlencode({'as3': '1', 'vid': video_id})
         info_request = sanitized_Request(info_url, data)
         info_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
diff --git a/youtube_dl/extractor/videopremium.py b/youtube_dl/extractor/videopremium.py
index 3176e3b9d..5de8273c3 100644
--- a/youtube_dl/extractor/videopremium.py
+++ b/youtube_dl/extractor/videopremium.py
@@ -26,7 +26,7 @@ class VideoPremiumIE(InfoExtractor):
         webpage_url = 'http://videopremium.tv/' + video_id
         webpage = self._download_webpage(webpage_url, video_id)
 
-        if re.match(r"^<html><head><script[^>]*>window.location\s*=", webpage):
+        if re.match(r'^<html><head><script[^>]*>window.location\s*=', webpage):
             # Download again, we need a cookie
             webpage = self._download_webpage(
                 webpage_url, video_id,
@@ -37,10 +37,10 @@ class VideoPremiumIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'url': "rtmp://e%d.md.iplay.md/play" % random.randint(1, 16),
-            'play_path': "mp4:%s.f4v" % video_id,
-            'page_url': "http://videopremium.tv/" + video_id,
-            'player_url': "http://videopremium.tv/uplayer/uppod.swf",
+            'url': 'rtmp://e%d.md.iplay.md/play' % random.randint(1, 16),
+            'play_path': 'mp4:%s.f4v' % video_id,
+            'page_url': 'http://videopremium.tv/' + video_id,
+            'player_url': 'http://videopremium.tv/uplayer/uppod.swf',
             'ext': 'f4v',
             'title': video_title,
         }
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index c7df6b0c5..3049dffb6 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -368,16 +368,16 @@ class VimeoIE(VimeoBaseInfoExtractor):
                     {'force_feature_id': True}), 'Vimeo')
 
         # Extract title
-        video_title = config["video"]["title"]
+        video_title = config['video']['title']
 
         # Extract uploader and uploader_id
-        video_uploader = config["video"]["owner"]["name"]
-        video_uploader_id = config["video"]["owner"]["url"].split('/')[-1] if config["video"]["owner"]["url"] else None
+        video_uploader = config['video']['owner']['name']
+        video_uploader_id = config['video']['owner']['url'].split('/')[-1] if config['video']['owner']['url'] else None
 
         # Extract video thumbnail
-        video_thumbnail = config["video"].get("thumbnail")
+        video_thumbnail = config['video'].get('thumbnail')
         if video_thumbnail is None:
-            video_thumbs = config["video"].get("thumbs")
+            video_thumbs = config['video'].get('thumbs')
             if video_thumbs and isinstance(video_thumbs, dict):
                 _, video_thumbnail = sorted((int(width if width.isdigit() else 0), t_url) for (width, t_url) in video_thumbs.items())[-1]
 
@@ -401,7 +401,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             self._downloader.report_warning('Cannot find video description')
 
         # Extract video duration
-        video_duration = int_or_none(config["video"].get("duration"))
+        video_duration = int_or_none(config['video'].get('duration'))
 
         # Extract upload date
         video_upload_date = None
@@ -703,10 +703,10 @@ class VimeoLikesIE(InfoExtractor):
     _TEST = {
         'url': 'https://vimeo.com/user755559/likes/',
         'playlist_mincount': 293,
-        "info_dict": {
+        'info_dict': {
             'id': 'user755559_likes',
-            "description": "See all the videos urza likes",
-            "title": 'Videos urza likes',
+            'description': 'See all the videos urza likes',
+            'title': 'Videos urza likes',
         },
     }
 
diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index cb2a4b0b5..a6a6cc479 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -119,7 +119,7 @@ class VineIE(InfoExtractor):
 class VineUserIE(InfoExtractor):
     IE_NAME = 'vine:user'
     _VALID_URL = r'(?:https?://)?vine\.co/(?P<u>u/)?(?P<user>[^/]+)/?(\?.*)?$'
-    _VINE_BASE_URL = "https://vine.co/"
+    _VINE_BASE_URL = 'https://vine.co/'
     _TESTS = [
         {
             'url': 'https://vine.co/Visa',
@@ -139,7 +139,7 @@ class VineUserIE(InfoExtractor):
         user = mobj.group('user')
         u = mobj.group('u')
 
-        profile_url = "%sapi/users/profiles/%s%s" % (
+        profile_url = '%sapi/users/profiles/%s%s' % (
             self._VINE_BASE_URL, 'vanity/' if not u else '', user)
         profile_data = self._download_json(
             profile_url, user, note='Downloading user profile data')
@@ -147,7 +147,7 @@ class VineUserIE(InfoExtractor):
         user_id = profile_data['data']['userId']
         timeline_data = []
         for pagenum in itertools.count(1):
-            timeline_url = "%sapi/timelines/users/%s?page=%s&size=100" % (
+            timeline_url = '%sapi/timelines/users/%s?page=%s&size=100' % (
                 self._VINE_BASE_URL, user_id, pagenum)
             timeline_page = self._download_json(
                 timeline_url, user, note='Downloading page %d' % pagenum)
diff --git a/youtube_dl/extractor/worldstarhiphop.py b/youtube_dl/extractor/worldstarhiphop.py
index a3ea26feb..09415b589 100644
--- a/youtube_dl/extractor/worldstarhiphop.py
+++ b/youtube_dl/extractor/worldstarhiphop.py
@@ -8,12 +8,12 @@ from .common import InfoExtractor
 class WorldStarHipHopIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www|m)\.worldstar(?:candy|hiphop)\.com/(?:videos|android)/video\.php\?v=(?P<id>.*)'
     _TESTS = [{
-        "url": "http://www.worldstarhiphop.com/videos/video.php?v=wshh6a7q1ny0G34ZwuIO",
-        "md5": "9d04de741161603bf7071bbf4e883186",
-        "info_dict": {
-            "id": "wshh6a7q1ny0G34ZwuIO",
-            "ext": "mp4",
-            "title": "KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick!"
+        'url': 'http://www.worldstarhiphop.com/videos/video.php?v=wshh6a7q1ny0G34ZwuIO',
+        'md5': '9d04de741161603bf7071bbf4e883186',
+        'info_dict': {
+            'id': 'wshh6a7q1ny0G34ZwuIO',
+            'ext': 'mp4',
+            'title': 'KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick!'
         }
     }, {
         'url': 'http://m.worldstarhiphop.com/android/video.php?v=wshh6a7q1ny0G34ZwuIO',
@@ -21,7 +21,7 @@ class WorldStarHipHopIE(InfoExtractor):
         'info_dict': {
             'id': 'wshh6a7q1ny0G34ZwuIO',
             'ext': 'mp4',
-            "title": "KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick!"
+            'title': 'KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick!'
         }
     }]
 
diff --git a/youtube_dl/extractor/youjizz.py b/youtube_dl/extractor/youjizz.py
index c642075dc..4150b28da 100644
--- a/youtube_dl/extractor/youjizz.py
+++ b/youtube_dl/extractor/youjizz.py
@@ -16,8 +16,8 @@ class YouJizzIE(InfoExtractor):
         'info_dict': {
             'id': '2189178',
             'ext': 'flv',
-            "title": "Zeichentrick 1",
-            "age_limit": 18,
+            'title': 'Zeichentrick 1',
+            'age_limit': 18,
         }
     }
 
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 5c1f84a09..900eb2aba 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -217,7 +217,7 @@ class YoukuIE(InfoExtractor):
         video_password = self._downloader.params.get('videopassword')
 
         # request basic data
-        basic_data_url = "http://play.youku.com/play/get.json?vid=%s&ct=12" % video_id
+        basic_data_url = 'http://play.youku.com/play/get.json?vid=%s&ct=12' % video_id
         if video_password:
             basic_data_url += '&pwd=%s' % video_password
 
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 2137dfb3f..3afa8bb6f 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -85,7 +85,7 @@ def parseOpts(overrideArguments=None):
         if option.takes_value():
             opts.append(' %s' % option.metavar)
 
-        return "".join(opts)
+        return ''.join(opts)
 
     def _comma_separated_values_options_callback(option, opt_str, value, parser):
         setattr(parser.values, option.dest, value.split(','))
diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py
index 13794b7ba..74f66d669 100644
--- a/youtube_dl/postprocessor/execafterdownload.py
+++ b/youtube_dl/postprocessor/execafterdownload.py
@@ -19,7 +19,7 @@ class ExecAfterDownloadPP(PostProcessor):
 
         cmd = cmd.replace('{}', shlex_quote(information['filepath']))
 
-        self._downloader.to_screen("[exec] Executing command: %s" % cmd)
+        self._downloader.to_screen('[exec] Executing command: %s' % cmd)
         retCode = subprocess.call(cmd, shell=True)
         if retCode != 0:
             raise PostProcessingError(
diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dl/postprocessor/metadatafromtitle.py
index a56077f20..42377fa0f 100644
--- a/youtube_dl/postprocessor/metadatafromtitle.py
+++ b/youtube_dl/postprocessor/metadatafromtitle.py
@@ -24,7 +24,7 @@ class MetadataFromTitlePP(PostProcessor):
            '(?P<title>.+)\ \-\ (?P<artist>.+)'
         """
         lastpos = 0
-        regex = ""
+        regex = ''
         # replace %(..)s with regex group and escape other string parts
         for match in re.finditer(r'%\((\w+)\)s', fmt):
             regex += re.escape(fmt[lastpos:match.start()])
diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index 7d88e1308..480d48d05 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -80,15 +80,15 @@ class XAttrMetadataPP(PostProcessor):
                     assert ':' not in key
                     assert os.path.exists(path)
 
-                    ads_fn = path + ":" + key
+                    ads_fn = path + ':' + key
                     try:
-                        with open(ads_fn, "wb") as f:
+                        with open(ads_fn, 'wb') as f:
                             f.write(value)
                     except EnvironmentError as e:
                         raise XAttrMetadataError(e.errno, e.strerror)
             else:
-                user_has_setfattr = check_executable("setfattr", ['--version'])
-                user_has_xattr = check_executable("xattr", ['-h'])
+                user_has_setfattr = check_executable('setfattr', ['--version'])
+                user_has_xattr = check_executable('xattr', ['-h'])
 
                 if user_has_setfattr or user_has_xattr:
 
@@ -150,7 +150,7 @@ class XAttrMetadataPP(PostProcessor):
                 value = info.get(infoname)
 
                 if value:
-                    if infoname == "upload_date":
+                    if infoname == 'upload_date':
                         value = hyphenate_date(value)
 
                     byte_value = value.encode('utf-8')
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index e4a1aaa64..676ebe1c4 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -31,12 +31,12 @@ def rsa_verify(message, signature, key):
 def update_self(to_screen, verbose, opener):
     """Update the program file with the latest version from the repository"""
 
-    UPDATE_URL = "https://rg3.github.io/youtube-dl/update/"
+    UPDATE_URL = 'https://rg3.github.io/youtube-dl/update/'
     VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
     JSON_URL = UPDATE_URL + 'versions.json'
     UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
 
-    if not isinstance(globals().get('__loader__'), zipimporter) and not hasattr(sys, "frozen"):
+    if not isinstance(globals().get('__loader__'), zipimporter) and not hasattr(sys, 'frozen'):
         to_screen('It looks like you installed youtube-dl with a package manager, pip, setup.py or a tarball. Please use that to update.')
         return
 
@@ -85,7 +85,7 @@ def update_self(to_screen, verbose, opener):
 
     filename = sys.argv[0]
     # Py2EXE: Filename could be different
-    if hasattr(sys, "frozen") and not os.path.isfile(filename):
+    if hasattr(sys, 'frozen') and not os.path.isfile(filename):
         if os.path.isfile(filename + '.exe'):
             filename += '.exe'
 
@@ -94,7 +94,7 @@ def update_self(to_screen, verbose, opener):
         return
 
     # Py2EXE
-    if hasattr(sys, "frozen"):
+    if hasattr(sys, 'frozen'):
         exe = os.path.abspath(filename)
         directory = os.path.dirname(exe)
         if not os.access(directory, os.W_OK):
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 3e4219b17..672ce05ea 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -248,7 +248,7 @@ def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
 
 def get_element_by_id(id, html):
     """Return the content of the tag with the specified ID in the passed HTML document"""
-    return get_element_by_attribute("id", id, html)
+    return get_element_by_attribute('id', id, html)
 
 
 def get_element_by_attribute(attribute, value, html):
@@ -994,7 +994,7 @@ def date_from_str(date_str):
         unit += 's'
         delta = datetime.timedelta(**{unit: time})
         return today + delta
-    return datetime.datetime.strptime(date_str, "%Y%m%d").date()
+    return datetime.datetime.strptime(date_str, '%Y%m%d').date()
 
 
 def hyphenate_date(date_str):
@@ -1074,22 +1074,22 @@ def _windows_write_string(s, out):
 
     GetStdHandle = ctypes.WINFUNCTYPE(
         ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
-        (b"GetStdHandle", ctypes.windll.kernel32))
+        (b'GetStdHandle', ctypes.windll.kernel32))
     h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
 
     WriteConsoleW = ctypes.WINFUNCTYPE(
         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
         ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
-        ctypes.wintypes.LPVOID)((b"WriteConsoleW", ctypes.windll.kernel32))
+        ctypes.wintypes.LPVOID)((b'WriteConsoleW', ctypes.windll.kernel32))
     written = ctypes.wintypes.DWORD(0)
 
-    GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b"GetFileType", ctypes.windll.kernel32))
+    GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b'GetFileType', ctypes.windll.kernel32))
     FILE_TYPE_CHAR = 0x0002
     FILE_TYPE_REMOTE = 0x8000
     GetConsoleMode = ctypes.WINFUNCTYPE(
         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
         ctypes.POINTER(ctypes.wintypes.DWORD))(
-        (b"GetConsoleMode", ctypes.windll.kernel32))
+        (b'GetConsoleMode', ctypes.windll.kernel32))
     INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
 
     def not_a_console(handle):
@@ -1387,7 +1387,7 @@ def fix_xml_ampersands(xml_str):
 def setproctitle(title):
     assert isinstance(title, compat_str)
     try:
-        libc = ctypes.cdll.LoadLibrary("libc.so.6")
+        libc = ctypes.cdll.LoadLibrary('libc.so.6')
     except OSError:
         return
     title_bytes = title.encode('utf-8')
@@ -1427,7 +1427,7 @@ def url_basename(url):
 
 class HEADRequest(compat_urllib_request.Request):
     def get_method(self):
-        return "HEAD"
+        return 'HEAD'
 
 
 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):

From 5e19323ed91a2ac3014c8f803fa65d2774a6a4f4 Mon Sep 17 00:00:00 2001
From: Jens Wille <jens.wille@gmail.com>
Date: Tue, 9 Feb 2016 16:25:30 +0100
Subject: [PATCH 0694/1105] [laola1tv] Fixes for changed site layout.

* Fixed valid URLs (w/ tests).
* Fixed iframe URL extraction.
* Fixed token URL extraction.
* Fixed variable extraction.
* Fixed uploader spelling.
* Added upload_date to result dictionary.
---
 youtube_dl/extractor/laola1tv.py | 99 ++++++++++++++++++++------------
 1 file changed, 63 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index b459559b0..c1f248b6b 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -1,86 +1,113 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import random
 import re
 
 from .common import InfoExtractor
 from ..utils import (
+    compat_urlparse,
     ExtractorError,
+    sanitized_Request,
+    xpath_element,
     xpath_text,
+    unified_strdate,
+    urlencode_postdata,
 )
 
 
 class Laola1TvIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?laola1\.tv/(?P<lang>[a-z]+)-(?P<portal>[a-z]+)/.*?/(?P<id>[0-9]+)\.html'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?laola1\.tv/(?P<lang>[a-z]+)-(?P<portal>[a-z]+)/.*?/(?P<slug>[\w-]+)'
+    _TESTS = [{
         'url': 'http://www.laola1.tv/de-de/video/straubing-tigers-koelner-haie/227883.html',
         'info_dict': {
-            'id': '227883',
-            'ext': 'mp4',
-            'title': 'Straubing Tigers - Kölner Haie',
             'categories': ['Eishockey'],
+            'ext': 'flv',
+            'id': '227883',
             'is_live': False,
+            'title': 'Straubing Tigers - Kölner Haie',
+            'upload_date': '20140912',
         },
         'params': {
             'skip_download': True,
         }
-    }
+    }, {
+        'url': 'http://www.laola1.tv/de-de/video/straubing-tigers-koelner-haie',
+        'info_dict': {
+            'categories': ['Eishockey'],
+            'ext': 'flv',
+            'id': '464602',
+            'is_live': False,
+            'title': 'Straubing Tigers - Kölner Haie',
+            'upload_date': '20160129',
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
         lang = mobj.group('lang')
         portal = mobj.group('portal')
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url, mobj.group('slug'))
         iframe_url = self._search_regex(
-            r'<iframe[^>]*?class="main_tv_player"[^>]*?src="([^"]+)"',
+            r'<iframe[^>]*?id="videoplayer"[^>]*?src="([^"]+)"',
             webpage, 'iframe URL')
 
-        iframe = self._download_webpage(
-            iframe_url, video_id, note='Downloading iframe')
-        flashvars_m = re.findall(
-            r'flashvars\.([_a-zA-Z0-9]+)\s*=\s*"([^"]*)";', iframe)
-        flashvars = dict((m[0], m[1]) for m in flashvars_m)
+        video_id = self._search_regex(
+            r'videoid=(\d+)', iframe_url, 'video ID')
+
+        iframe = self._download_webpage(compat_urlparse.urljoin(
+            url, iframe_url), video_id, note='Downloading iframe')
 
         partner_id = self._search_regex(
-            r'partnerid\s*:\s*"([^"]+)"', iframe, 'partner id')
+            r'partnerid\s*:\s*"([^"]+)"', iframe, 'partner ID')
 
         xml_url = ('http://www.laola1.tv/server/hd_video.php?' +
                    'play=%s&partner=%s&portal=%s&v5ident=&lang=%s' % (
                        video_id, partner_id, portal, lang))
         hd_doc = self._download_xml(xml_url, video_id)
 
-        title = xpath_text(hd_doc, './/video/title', fatal=True)
-        flash_url = xpath_text(hd_doc, './/video/url', fatal=True)
-        uploader = xpath_text(hd_doc, './/video/meta_organistation')
-        is_live = xpath_text(hd_doc, './/video/islive') == 'true'
+        _v = lambda x, **k: xpath_text(hd_doc, './/video/' + x, **k)
+        title = _v('title', fatal=True)
 
-        categories = xpath_text(hd_doc, './/video/meta_sports')
+        categories = _v('meta_sports')
         if categories:
             categories = categories.split(',')
 
-        ident = random.randint(10000000, 99999999)
-        token_url = '%s&ident=%s&klub=0&unikey=0&timestamp=%s&auth=%s' % (
-            flash_url, ident, flashvars['timestamp'], flashvars['auth'])
+        time_date = _v('time_date')
+        time_start = _v('time_start')
+        upload_date = None
+        if time_date and time_start:
+            upload_date = unified_strdate(time_date + ' ' + time_start)
 
+        json_url = ('https://club.laola1.tv/sp/laola1/api/v3/user/session' +
+                    '/premium/player/stream-access?videoId=%s&target=2' +
+                    '&label=laola1tv&area=%s') % (video_id, _v('area'))
+        req = sanitized_Request(json_url, urlencode_postdata(
+            dict((i, v) for i, v in enumerate(_v('req_liga_abos').split(',')))))
+
+        token_url = self._download_json(req, video_id)['data']['stream-access'][0]
         token_doc = self._download_xml(
             token_url, video_id, note='Downloading token')
-        token_attrib = token_doc.find('.//token').attrib
-        if token_attrib.get('auth') in ('blocked', 'restricted'):
-            raise ExtractorError(
-                'Token error: %s' % token_attrib.get('comment'), expected=True)
 
-        video_url = '%s?hdnea=%s&hdcore=3.2.0' % (
-            token_attrib['url'], token_attrib['auth'])
+        token_attrib = xpath_element(token_doc, './/token').attrib
+        token_auth = token_attrib['auth']
+
+        if token_auth in ('blocked', 'restricted'):
+            raise ExtractorError(
+                'Token error: %s' % token_attrib['comment'], expected=True)
+
+        video_url = '%s?hdnea=%s&hdcore=3.2.0' % (token_attrib['url'], token_auth)
 
         return {
-            'id': video_id,
-            'is_live': is_live,
-            'title': title,
-            'url': video_url,
-            'uploader': uploader,
             'categories': categories,
-            'ext': 'mp4',
+            'formats': self._extract_f4m_formats(
+                video_url, video_id, f4m_id='hds'),
+            'id': video_id,
+            'is_live': _v('islive') == 'true',
+            'title': title,
+            'upload_date': upload_date,
+            'uploader': _v('meta_organisation'),
         }

From d9da74bc06d4b82d990bac501aedd52004d83b63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Feb 2016 23:39:49 +0600
Subject: [PATCH 0695/1105] Credit @blackwinter for laola1tv (#8478)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 7cac337a7..ad3fc021b 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -157,3 +157,4 @@ Founder Fang
 Andrew Alexeyew
 Saso Bezlaj
 Erwin de Haan
+Jens Wille

From 9dcefb23a12470390baa5e2ca5f3319d1479a571 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Feb 2016 23:40:26 +0600
Subject: [PATCH 0696/1105] [laola1tv] Improve (Closes #8478)

---
 youtube_dl/extractor/laola1tv.py | 102 +++++++++++++++++--------------
 1 file changed, 57 insertions(+), 45 deletions(-)

diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index c1f248b6b..5d8ebbeb3 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -1,31 +1,35 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
+    compat_urllib_parse,
     compat_urlparse,
+)
+from ..utils import (
     ExtractorError,
     sanitized_Request,
-    xpath_element,
-    xpath_text,
     unified_strdate,
     urlencode_postdata,
+    xpath_element,
+    xpath_text,
 )
 
 
 class Laola1TvIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?laola1\.tv/(?P<lang>[a-z]+)-(?P<portal>[a-z]+)/.*?/(?P<slug>[\w-]+)'
+    _VALID_URL = r'https?://(?:www\.)?laola1\.tv/(?P<lang>[a-z]+)-(?P<portal>[a-z]+)/[^/]+/(?P<slug>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://www.laola1.tv/de-de/video/straubing-tigers-koelner-haie/227883.html',
         'info_dict': {
-            'categories': ['Eishockey'],
-            'ext': 'flv',
             'id': '227883',
-            'is_live': False,
+            'display_id': 'straubing-tigers-koelner-haie',
+            'ext': 'flv',
             'title': 'Straubing Tigers - Kölner Haie',
             'upload_date': '20140912',
+            'is_live': False,
+            'categories': ['Eishockey'],
         },
         'params': {
             'skip_download': True,
@@ -33,12 +37,13 @@ class Laola1TvIE(InfoExtractor):
     }, {
         'url': 'http://www.laola1.tv/de-de/video/straubing-tigers-koelner-haie',
         'info_dict': {
-            'categories': ['Eishockey'],
-            'ext': 'flv',
             'id': '464602',
-            'is_live': False,
+            'display_id': 'straubing-tigers-koelner-haie',
+            'ext': 'flv',
             'title': 'Straubing Tigers - Kölner Haie',
             'upload_date': '20160129',
+            'is_live': False,
+            'categories': ['Eishockey'],
         },
         'params': {
             'skip_download': True,
@@ -47,67 +52,74 @@ class Laola1TvIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('slug')
         lang = mobj.group('lang')
         portal = mobj.group('portal')
 
-        webpage = self._download_webpage(url, mobj.group('slug'))
+        webpage = self._download_webpage(url, display_id)
+
         iframe_url = self._search_regex(
             r'<iframe[^>]*?id="videoplayer"[^>]*?src="([^"]+)"',
-            webpage, 'iframe URL')
+            webpage, 'iframe url')
 
         video_id = self._search_regex(
-            r'videoid=(\d+)', iframe_url, 'video ID')
+            r'videoid=(\d+)', iframe_url, 'video id')
 
         iframe = self._download_webpage(compat_urlparse.urljoin(
-            url, iframe_url), video_id, note='Downloading iframe')
+            url, iframe_url), display_id, 'Downloading iframe')
 
         partner_id = self._search_regex(
-            r'partnerid\s*:\s*"([^"]+)"', iframe, 'partner ID')
+            r'partnerid\s*:\s*(["\'])(?P<partner_id>.+?)\1',
+            iframe, 'partner id', group='partner_id')
 
-        xml_url = ('http://www.laola1.tv/server/hd_video.php?' +
-                   'play=%s&partner=%s&portal=%s&v5ident=&lang=%s' % (
-                       video_id, partner_id, portal, lang))
-        hd_doc = self._download_xml(xml_url, video_id)
+        hd_doc = self._download_xml(
+            'http://www.laola1.tv/server/hd_video.php?%s'
+            % compat_urllib_parse.urlencode({
+                'play': video_id,
+                'partner': partner_id,
+                'portal': portal,
+                'lang': lang,
+                'v5ident': '',
+            }), display_id)
 
         _v = lambda x, **k: xpath_text(hd_doc, './/video/' + x, **k)
         title = _v('title', fatal=True)
 
-        categories = _v('meta_sports')
-        if categories:
-            categories = categories.split(',')
+        req = sanitized_Request(
+            'https://club.laola1.tv/sp/laola1/api/v3/user/session/premium/player/stream-access?%s' %
+            compat_urllib_parse.urlencode({
+                'videoId': video_id,
+                'target': '2',
+                'label': 'laola1tv',
+                'area': _v('area'),
+            }),
+            urlencode_postdata(
+                dict((i, v) for i, v in enumerate(_v('req_liga_abos').split(',')))))
 
-        time_date = _v('time_date')
-        time_start = _v('time_start')
-        upload_date = None
-        if time_date and time_start:
-            upload_date = unified_strdate(time_date + ' ' + time_start)
-
-        json_url = ('https://club.laola1.tv/sp/laola1/api/v3/user/session' +
-                    '/premium/player/stream-access?videoId=%s&target=2' +
-                    '&label=laola1tv&area=%s') % (video_id, _v('area'))
-        req = sanitized_Request(json_url, urlencode_postdata(
-            dict((i, v) for i, v in enumerate(_v('req_liga_abos').split(',')))))
-
-        token_url = self._download_json(req, video_id)['data']['stream-access'][0]
-        token_doc = self._download_xml(
-            token_url, video_id, note='Downloading token')
+        token_url = self._download_json(req, display_id)['data']['stream-access'][0]
+        token_doc = self._download_xml(token_url, display_id, 'Downloading token')
 
         token_attrib = xpath_element(token_doc, './/token').attrib
         token_auth = token_attrib['auth']
 
-        if token_auth in ('blocked', 'restricted'):
+        if token_auth in ('blocked', 'restricted', 'error'):
             raise ExtractorError(
                 'Token error: %s' % token_attrib['comment'], expected=True)
 
-        video_url = '%s?hdnea=%s&hdcore=3.2.0' % (token_attrib['url'], token_auth)
+        formats = self._extract_f4m_formats(
+            '%s?hdnea=%s&hdcore=3.2.0' % (token_attrib['url'], token_auth),
+            video_id, f4m_id='hds')
+
+        categories_str = _v('meta_sports')
+        categories = categories_str.split(',') if categories_str else []
 
         return {
-            'categories': categories,
-            'formats': self._extract_f4m_formats(
-                video_url, video_id, f4m_id='hds'),
             'id': video_id,
-            'is_live': _v('islive') == 'true',
+            'display_id': display_id,
             'title': title,
-            'upload_date': upload_date,
+            'upload_date': unified_strdate(_v('time_date')),
             'uploader': _v('meta_organisation'),
+            'categories': categories,
+            'is_live': _v('islive') == 'true',
+            'formats': formats,
         }

From a517bb4b1e3eaea67973ee6a0718f2f4cf2b760e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 15 Feb 2016 00:07:16 +0100
Subject: [PATCH 0697/1105] [noz] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/noz.py      | 69 ++++++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+)
 create mode 100644 youtube_dl/extractor/noz.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6937f28d3..a02113c86 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -490,6 +490,7 @@ from .nowtv import (
     NowTVIE,
     NowTVListIE,
 )
+from .noz import NozIE
 from .npo import (
     NPOIE,
     NPOLiveIE,
diff --git a/youtube_dl/extractor/noz.py b/youtube_dl/extractor/noz.py
new file mode 100644
index 000000000..0ffb44b47
--- /dev/null
+++ b/youtube_dl/extractor/noz.py
@@ -0,0 +1,69 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote
+from ..utils import (
+    int_or_none,
+    xpath_text,
+)
+
+
+class NozIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?noz\.de/video/(?P<id>[0-9]+)/'
+    _TESTS = [{
+        'url': 'http://www.noz.de/video/25151/32-Deutschland-gewinnt-Badminton-Lnderspiel-in-Melle',
+        'info_dict': {
+            'id': '25151',
+            'ext': 'mp4',
+            'duration': 215,
+            'title': '3:2 - Deutschland gewinnt Badminton-Länderspiel in Melle',
+            'description': 'Vor rund 370 Zuschauern gewinnt die deutsche Badminton-Nationalmannschaft am Donnerstag ein EM-Vorbereitungsspiel gegen Frankreich in Melle. Video Moritz Frankenberg.',
+            'thumbnail': 're:^http://.*\.jpg',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        description = self._og_search_description(webpage)
+
+        edge_url = self._html_search_regex(
+            r'<script\s+(?:type="text/javascript"\s+)?src="(.*?/videojs_.*?)"',
+            webpage, 'edge URL')
+        edge_content = self._download_webpage(edge_url, 'meta configuration')
+
+        config_url_encoded = self._search_regex(
+            r'so\.addVariable\("config_url","[^,]*,(.*?)"',
+            edge_content, 'config URL'
+        )
+        config_url = compat_urllib_parse_unquote(config_url_encoded)
+
+        doc = self._download_xml(config_url, 'video configuration')
+        title = xpath_text(doc, './/title')
+        thumbnail = xpath_text(doc, './/article/thumbnail/url')
+        duration = int_or_none(xpath_text(
+            doc, './/article/movie/file/duration'))
+        formats = []
+        for qnode in doc.findall('.//article/movie/file/qualities/qual'):
+            video_node = qnode.find('./html_urls/video_url[@format="video/mp4"]')
+            if video_node is None:
+                continue  # auto
+            formats.append({
+                'url': video_node.text,
+                'format_name': xpath_text(qnode, './name'),
+                'format_id': xpath_text(qnode, './id'),
+                'height': int_or_none(xpath_text(qnode, './height')),
+                'width': int_or_none(xpath_text(qnode, './width')),
+                'tbr': int_or_none(xpath_text(qnode, './bitrate'), scale=1000),
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'title': title,
+            'duration': duration,
+            'description': description,
+            'thumbnail': thumbnail,
+        }

From 8e0548e180ee960eff6325b65e6cb2b91ee4d336 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Feb 2016 19:58:24 +0800
Subject: [PATCH 0698/1105] [iqiyi] Partial support for VIP-only videos

See #8569 and #8019. Currently only 6-min preview are supported
---
 youtube_dl/extractor/iqiyi.py | 106 ++++++++++++++++++++++++++++++----
 1 file changed, 94 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 691cb66d6..04db252df 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -3,13 +3,23 @@ from __future__ import unicode_literals
 
 import hashlib
 import math
+import os
 import random
 import time
 import uuid
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
-from ..utils import ExtractorError
+from ..compat import (
+    compat_parse_qs,
+    compat_urllib_parse,
+    compat_urllib_parse_urlparse,
+)
+from ..utils import (
+    ExtractorError,
+    sanitized_Request,
+    urlencode_postdata,
+    url_basename,
+)
 
 
 class IqiyiIE(InfoExtractor):
@@ -93,6 +103,29 @@ class IqiyiIE(InfoExtractor):
     }, {
         'url': 'http://yule.iqiyi.com/pcb.html',
         'only_matching': True,
+    }, {
+        # VIP-only video. The first 2 parts (6 minutes) are available without login
+        'url': 'http://www.iqiyi.com/v_19rrny4w8w.html',
+        'info_dict': {
+            'id': 'f3cf468b39dddb30d676f89a91200dc1',
+            'title': '泰坦尼克号',
+        },
+        'playlist': [{
+            'md5': '436bcde6e1307307d5ba1549715b0bbd',
+            'info_dict': {
+                'id': 'f3cf468b39dddb30d676f89a91200dc1_part1',
+                'ext': 'f4v',
+                'title': '泰坦尼克号',
+            },
+        }, {
+            'md5': 'bfc5e332f4900fde547c69372385649e',
+            'info_dict': {
+                'id': 'f3cf468b39dddb30d676f89a91200dc1_part2',
+                'ext': 'f4v',
+                'title': '泰坦尼克号',
+            },
+        }],
+        'expected_warnings': ['Needs a VIP account for full video'],
     }]
 
     _FORMATS_MAP = [
@@ -108,7 +141,40 @@ class IqiyiIE(InfoExtractor):
     def md5_text(text):
         return hashlib.md5(text.encode('utf-8')).hexdigest()
 
-    def construct_video_urls(self, data, video_id, _uuid):
+    def _authenticate_vip_video(self, api_video_url, video_id, tvid, _uuid, do_report_warning):
+        auth_params = {
+            # version and platform hard-coded in com/qiyi/player/core/model/remote/AuthenticationRemote.as
+            'version': '2.0',
+            'platform': 'b6c13e26323c537d',
+            'aid': tvid,
+            'tvid': tvid,
+            'uid': '',
+            'deviceId': _uuid,
+            'playType': 'main',  # XXX: always main?
+            'filename': os.path.splitext(url_basename(api_video_url))[0],
+        }
+
+        qd_items = compat_parse_qs(compat_urllib_parse_urlparse(api_video_url).query)
+        for key, val in qd_items.items():
+            auth_params[key] = val[0]
+
+        auth_req = sanitized_Request(
+            'http://api.vip.iqiyi.com/services/ckn.action',
+            urlencode_postdata(auth_params))
+        # iQiyi server throws HTTP 405 error without the following header
+        auth_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        auth_result = self._download_json(
+            auth_req, video_id,
+            note='Downloading video authentication JSON',
+            errnote='Unable to download video authentication JSON')
+        if auth_result['code'] == 'Q00506':  # requires a VIP account
+            if do_report_warning:
+                self.report_warning('Needs a VIP account for full video')
+            return False
+
+        return auth_result
+
+    def construct_video_urls(self, data, video_id, _uuid, tvid):
         def do_xor(x, y):
             a = y % 3
             if a == 1:
@@ -137,6 +203,7 @@ class IqiyiIE(InfoExtractor):
             return self.md5_text(t + mg + x)
 
         video_urls_dict = {}
+        need_vip_warning_report = True
         for format_item in data['vp']['tkl'][0]['vs']:
             if 0 < int(format_item['bid']) <= 10:
                 format_id = self.get_format(format_item['bid'])
@@ -155,11 +222,13 @@ class IqiyiIE(InfoExtractor):
                 vl = segment['l']
                 if not vl.startswith('/'):
                     vl = get_encode_code(vl)
-                key = get_path_key(
-                    vl.split('/')[-1].split('.')[0], format_id, segment_index)
+                is_vip_video = '/vip/' in vl
                 filesize = segment['b']
                 base_url = data['vp']['du'].split('/')
-                base_url.insert(-1, key)
+                if not is_vip_video:
+                    key = get_path_key(
+                        vl.split('/')[-1].split('.')[0], format_id, segment_index)
+                    base_url.insert(-1, key)
                 base_url = '/'.join(base_url)
                 param = {
                     'su': _uuid,
@@ -170,8 +239,23 @@ class IqiyiIE(InfoExtractor):
                     'ct': '',
                     'tn': str(int(time.time()))
                 }
-                api_video_url = base_url + vl + '?' + \
-                    compat_urllib_parse.urlencode(param)
+                api_video_url = base_url + vl
+                if is_vip_video:
+                    api_video_url = api_video_url.replace('.f4v', '.hml')
+                    auth_result = self._authenticate_vip_video(
+                        api_video_url, video_id, tvid, _uuid, need_vip_warning_report)
+                    if auth_result is False:
+                        need_vip_warning_report = False
+                        break
+                    param.update({
+                        't': auth_result['data']['t'],
+                        # cid is hard-coded in com/qiyi/player/core/player/RuntimeData.as
+                        'cid': 'afbe8fd3d73448c9',
+                        'vid': video_id,
+                        'QY00001': auth_result['data']['u'],
+                    })
+                api_video_url += '?' if '?' not in api_video_url else '&'
+                api_video_url += compat_urllib_parse.urlencode(param)
                 js = self._download_json(
                     api_video_url, video_id,
                     note='Download video info of segment %d for format %s' % (segment_index + 1, format_id))
@@ -205,6 +289,7 @@ class IqiyiIE(InfoExtractor):
             'tn': random.random(),
             'um': 0,
             'authkey': self.md5_text(self.md5_text('') + tail),
+            'k_tag': 1,
         }
 
         api_url = 'http://cache.video.qiyi.com/vms' + '?' + \
@@ -236,16 +321,13 @@ class IqiyiIE(InfoExtractor):
         if raw_data['code'] != 'A000000':
             raise ExtractorError('Unable to load data. Error code: ' + raw_data['code'])
 
-        if not raw_data['data']['vp']['tkl']:
-            raise ExtractorError('No support iQiqy VIP video')
-
         data = raw_data['data']
 
         title = data['vi']['vn']
 
         # generate video_urls_dict
         video_urls_dict = self.construct_video_urls(
-            data, video_id, _uuid)
+            data, video_id, _uuid, tvid)
 
         # construct info
         entries = []

From 8d3eeb36d781183d52c093153e7fab6c17427dc7 Mon Sep 17 00:00:00 2001
From: Robin Houtevelts <robin@houtevelts.com>
Date: Sat, 13 Feb 2016 01:12:44 +0100
Subject: [PATCH 0699/1105] [Canvas] Add subtitles

---
 youtube_dl/extractor/canvas.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
index ee19ff836..16f458d6c 100644
--- a/youtube_dl/extractor/canvas.py
+++ b/youtube_dl/extractor/canvas.py
@@ -53,6 +53,15 @@ class CanvasIE(InfoExtractor):
                     'url': format_url,
                 })
         self._sort_formats(formats)
+        
+        subtitles = {}
+        for target in data['subtitleUrls']:
+            format_url, format_type = target.get('url'), target.get('type')
+            if format_type == 'CLOSED':
+                subtitles['nl'] = [{
+                    'ext': 'vtt',
+                    'url': format_url,
+                }]
 
         return {
             'id': video_id,
@@ -62,4 +71,5 @@ class CanvasIE(InfoExtractor):
             'formats': formats,
             'duration': float_or_none(data.get('duration'), 1000),
             'thumbnail': data.get('posterImageUrl'),
+            'subtitles': subtitles,
         }

From fd7a3ea4a41ccad37c932662d39e159854311357 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Feb 2016 20:54:01 +0600
Subject: [PATCH 0700/1105] [canvas] Improve subtitles (Closes #8537)

---
 youtube_dl/extractor/canvas.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
index 16f458d6c..54b535ae9 100644
--- a/youtube_dl/extractor/canvas.py
+++ b/youtube_dl/extractor/canvas.py
@@ -53,15 +53,14 @@ class CanvasIE(InfoExtractor):
                     'url': format_url,
                 })
         self._sort_formats(formats)
-        
+
         subtitles = {}
-        for target in data['subtitleUrls']:
-            format_url, format_type = target.get('url'), target.get('type')
-            if format_type == 'CLOSED':
-                subtitles['nl'] = [{
-                    'ext': 'vtt',
-                    'url': format_url,
-                }]
+        subtitle_urls = data.get('subtitleUrls')
+        if isinstance(subtitle_urls, list):
+            for subtitle in subtitle_urls:
+                subtitle_url = subtitle.get('url')
+                if subtitle_url and subtitle.get('type') == 'CLOSED':
+                    subtitles.setdefault('nl', []).append({'url': subtitle_url})
 
         return {
             'id': video_id,

From 6eff2605d645664ba52fe873bb28cea77744e7f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Feb 2016 20:59:16 +0600
Subject: [PATCH 0701/1105] [canvas] Add subtitles test (#8537)

---
 youtube_dl/extractor/canvas.py | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
index 54b535ae9..ec6d24d96 100644
--- a/youtube_dl/extractor/canvas.py
+++ b/youtube_dl/extractor/canvas.py
@@ -6,7 +6,7 @@ from ..utils import float_or_none
 
 class CanvasIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?canvas\.be/video/(?:[^/]+/)*(?P<id>[^/?#&]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
         'md5': 'ea838375a547ac787d4064d8c7860a6c',
         'info_dict': {
@@ -18,7 +18,27 @@ class CanvasIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 49.02,
         }
-    }
+    }, {
+        # with subtitles
+        'url': 'http://www.canvas.be/video/panorama/2016/pieter-0167',
+        'info_dict': {
+            'id': 'mz-ast-5240ff21-2d30-4101-bba6-92b5ec67c625',
+            'display_id': 'pieter-0167',
+            'ext': 'mp4',
+            'title': 'Pieter 0167',
+            'description': 'md5:943cd30f48a5d29ba02c3a104dc4ec4e',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 2553.08,
+            'subtitles': {
+                'nl': [{
+                    'ext': 'vtt',
+                }],
+            },
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)

From b9e8d7140aa8584f8516c3575c20b21172278fef Mon Sep 17 00:00:00 2001
From: Patrick Griffis <tingping@tingping.se>
Date: Wed, 10 Feb 2016 15:33:19 -0500
Subject: [PATCH 0702/1105] [screenjunkies] Add new extractor

This doesn't handle the plus only videos yet

Closes #8492
---
 youtube_dl/extractor/__init__.py      |  1 +
 youtube_dl/extractor/screenjunkies.py | 74 +++++++++++++++++++++++++++
 2 files changed, 75 insertions(+)
 create mode 100644 youtube_dl/extractor/screenjunkies.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a02113c86..537d25777 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -621,6 +621,7 @@ from .sbs import SBSIE
 from .scivee import SciVeeIE
 from .screencast import ScreencastIE
 from .screencastomatic import ScreencastOMaticIE
+from .screenjunkies import ScreenJunkiesIE
 from .screenwavemedia import ScreenwaveMediaIE, TeamFourIE
 from .senateisvp import SenateISVPIE
 from .servingsys import ServingSysIE
diff --git a/youtube_dl/extractor/screenjunkies.py b/youtube_dl/extractor/screenjunkies.py
new file mode 100644
index 000000000..5da3f9bcd
--- /dev/null
+++ b/youtube_dl/extractor/screenjunkies.py
@@ -0,0 +1,74 @@
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import (
+    InfoExtractor,
+    ExtractorError,
+)
+
+from ..utils import (
+    int_or_none,
+)
+
+class ScreenJunkiesIE(InfoExtractor):
+    _VALID_URL = r'http://www.screenjunkies.com/video/(.+-(?P<id>\d+)|.+)'
+
+    _TESTS = [{
+        'url': 'http://www.screenjunkies.com/video/best-quentin-tarantino-movie-2841915',
+        'info_dict': {
+            'id': '2841915',
+            'ext': 'mp4',
+            'title': 'Best Quentin Tarantino Movie',
+        },
+    }, {
+        'url': 'http://www.screenjunkies.com/video/honest-trailers-the-dark-knight',
+        'info_dict': {
+            'id': '2348808',
+            'ext': 'mp4',
+            'title': "Honest Trailers - 'The Dark Knight'",
+        },
+    }, {
+        'url': 'http://www.screenjunkies.com/video/knocking-dead-ep-1-the-show-so-far-3003285',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        if not video_id: # Older urls didn't have the id in them, but we can grab it manually
+            webpage = self._download_webpage(url, url)
+            video_id = self._html_search_regex(r'src="/embed/(\d+)"', webpage, 'video id')
+
+        webpage = self._download_webpage('http://www.screenjunkies.com/embed/%s' %video_id, video_id)
+        video_vars_str = self._html_search_regex(r'embedVars = (\{.+\})\s*</script>', webpage, 'video variables', flags=re.S)
+        video_vars = self._parse_json(video_vars_str, video_id)
+
+        # TODO: Figure out required cookies
+        if video_vars['subscriptionLevel'] > 0:
+            raise ExtractorError('This video requires ScreenJunkiesPlus', expected=True)
+
+        formats = []
+        for f in video_vars['media']:
+            if not f['mediaPurpose'] == 'play':
+                continue
+
+            formats.append({
+                'url': f['uri'],
+                'width': int_or_none(f.get('width')),
+                'height': int_or_none(f.get('height')),
+                'tbr': int_or_none(f.get('bitRate')),
+                'format': 'mp4',
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': video_vars['contentName'],
+            'formats': formats,
+            'duration': int_or_none(video_vars.get('videoLengthInSeconds')),
+            'thumbnail': video_vars.get('thumbUri'),
+            'tags': video_vars['tags'].split(',') if 'tags' in video_vars else [],
+        }
+

From c8a80fd8184cddbd30ac8188ce04486886156365 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Feb 2016 22:29:28 +0600
Subject: [PATCH 0703/1105] [screenjunkies] Improve, extract more metadata and
 workaround subscription (Closes #8505)

---
 youtube_dl/extractor/screenjunkies.py | 122 ++++++++++++++++++++------
 1 file changed, 93 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/screenjunkies.py b/youtube_dl/extractor/screenjunkies.py
index 5da3f9bcd..e0db180b8 100644
--- a/youtube_dl/extractor/screenjunkies.py
+++ b/youtube_dl/extractor/screenjunkies.py
@@ -1,74 +1,138 @@
 from __future__ import unicode_literals
 
 import re
-import json
-
-from .common import (
-    InfoExtractor,
-    ExtractorError,
-)
 
+from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     int_or_none,
+    parse_age_limit,
 )
 
-class ScreenJunkiesIE(InfoExtractor):
-    _VALID_URL = r'http://www.screenjunkies.com/video/(.+-(?P<id>\d+)|.+)'
 
+class ScreenJunkiesIE(InfoExtractor):
+    _VALID_URL = r'http://www.screenjunkies.com/video/(?P<display_id>[^/]+?)(?:-(?P<id>\d+))?(?:[/?#&]|$)'
     _TESTS = [{
         'url': 'http://www.screenjunkies.com/video/best-quentin-tarantino-movie-2841915',
+        'md5': '5c2b686bec3d43de42bde9ec047536b0',
         'info_dict': {
             'id': '2841915',
+            'display_id': 'best-quentin-tarantino-movie',
             'ext': 'mp4',
             'title': 'Best Quentin Tarantino Movie',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 3671,
+            'age_limit': 13,
+            'tags': list,
         },
     }, {
         'url': 'http://www.screenjunkies.com/video/honest-trailers-the-dark-knight',
         'info_dict': {
             'id': '2348808',
+            'display_id': 'honest-trailers-the-dark-knight',
             'ext': 'mp4',
-            'title': "Honest Trailers - 'The Dark Knight'",
+            'title': "Honest Trailers: 'The Dark Knight'",
+            'thumbnail': 're:^https?://.*\.jpg',
+            'age_limit': 10,
+            'tags': list,
         },
     }, {
+        # requires subscription but worked around
         'url': 'http://www.screenjunkies.com/video/knocking-dead-ep-1-the-show-so-far-3003285',
-        'only_matching': True,
+        'info_dict': {
+            'id': '3003285',
+            'display_id': 'knocking-dead-ep-1-the-show-so-far',
+            'ext': 'mp4',
+            'title': 'Knocking Dead Ep 1: State of The Dead Recap',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 3307,
+            'age_limit': 13,
+            'tags': list,
+        },
     }]
 
+    _DEFAULT_BITRATES = (48, 150, 496, 864, 2240)
+
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        if not video_id: # Older urls didn't have the id in them, but we can grab it manually
-            webpage = self._download_webpage(url, url)
-            video_id = self._html_search_regex(r'src="/embed/(\d+)"', webpage, 'video id')
+        if not video_id:
+            webpage = self._download_webpage(url, display_id)
+            video_id = self._search_regex(
+                (r'src=["\']/embed/(\d+)', r'data-video-content-id=["\'](\d+)'),
+                webpage, 'video id')
 
-        webpage = self._download_webpage('http://www.screenjunkies.com/embed/%s' %video_id, video_id)
-        video_vars_str = self._html_search_regex(r'embedVars = (\{.+\})\s*</script>', webpage, 'video variables', flags=re.S)
-        video_vars = self._parse_json(video_vars_str, video_id)
+        webpage = self._download_webpage(
+            'http://www.screenjunkies.com/embed/%s' % video_id,
+            display_id, 'Downloading video embed page')
+        embed_vars = self._parse_json(
+            self._search_regex(
+                r'(?s)embedVars\s*=\s*({.+?})\s*</script>', webpage, 'embed vars'),
+            display_id)
 
-        # TODO: Figure out required cookies
-        if video_vars['subscriptionLevel'] > 0:
-            raise ExtractorError('This video requires ScreenJunkiesPlus', expected=True)
+        title = embed_vars['contentName']
 
         formats = []
-        for f in video_vars['media']:
-            if not f['mediaPurpose'] == 'play':
+        bitrates = []
+        for f in embed_vars.get('media', []):
+            if not f.get('uri') or f.get('mediaPurpose') != 'play':
                 continue
-
+            bitrate = int_or_none(f.get('bitRate'))
+            if bitrate:
+                bitrates.append(bitrate)
             formats.append({
                 'url': f['uri'],
+                'format_id': 'http-%d' % bitrate if bitrate else 'http',
                 'width': int_or_none(f.get('width')),
                 'height': int_or_none(f.get('height')),
-                'tbr': int_or_none(f.get('bitRate')),
+                'tbr': bitrate,
                 'format': 'mp4',
             })
+
+        if not bitrates:
+            # When subscriptionLevel > 0, i.e. plus subscription is required
+            # media list will be empty. However, hds and hls uri are still
+            # available. We can grabbing them assuming bitrates to be default.
+            bitrates = self._DEFAULT_BITRATES
+
+        auth_token = embed_vars.get('AuthToken')
+
+        def construct_manifest_url(base_url, ext):
+            pieces = [base_url]
+            pieces.extend([compat_str(b) for b in bitrates])
+            pieces.append('_kbps.mp4.%s?%s' % (ext, auth_token))
+            return ','.join(pieces)
+
+        if bitrates and auth_token:
+            hds_url = embed_vars.get('hdsUri')
+            if hds_url:
+                f4m_formats = self._extract_f4m_formats(
+                    construct_manifest_url(hds_url, 'f4m'),
+                    display_id, f4m_id='hds', fatal=False)
+                if len(f4m_formats) == len(bitrates):
+                    for f, bitrate in zip(f4m_formats, bitrates):
+                        if not f.get('tbr'):
+                            f['format_id'] = 'hds-%d' % bitrate
+                            f['tbr'] = bitrate
+                # TODO: fix f4m downloader to handle manifests without bitrates if possible
+                # formats.extend(f4m_formats)
+
+            hls_url = embed_vars.get('hlsUri')
+            if hls_url:
+                formats.extend(self._extract_m3u8_formats(
+                    construct_manifest_url(hls_url, 'm3u8'),
+                    display_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
         self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': video_vars['contentName'],
+            'display_id': display_id,
+            'title': title,
+            'thumbnail': embed_vars.get('thumbUri'),
+            'duration': int_or_none(embed_vars.get('videoLengthInSeconds')) or None,
+            'age_limit': parse_age_limit(embed_vars.get('audienceRating')),
+            'tags': embed_vars.get('tags', '').split(','),
             'formats': formats,
-            'duration': int_or_none(video_vars.get('videoLengthInSeconds')),
-            'thumbnail': video_vars.get('thumbUri'),
-            'tags': video_vars['tags'].split(',') if 'tags' in video_vars else [],
         }
-

From 8eacae8cf9a1226ccd9b186034b03756ca607921 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Feb 2016 22:33:32 +0600
Subject: [PATCH 0704/1105] Credit @RobinHoutevelts for canvas subtiltes
 (#8537)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index ad3fc021b..f8890eb27 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -158,3 +158,4 @@ Andrew Alexeyew
 Saso Bezlaj
 Erwin de Haan
 Jens Wille
+Robin Houtevelts

From d2c1f79f20007981432fbef6d8610094a45b0196 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Feb 2016 00:29:51 +0600
Subject: [PATCH 0705/1105] [youtube:searchurl] Extend _VALID_URL

---
 youtube_dl/extractor/youtube.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 18f7d37f4..45ad88152 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1912,13 +1912,16 @@ class YoutubeSearchDateIE(YoutubeSearchIE):
 class YoutubeSearchURLIE(InfoExtractor):
     IE_DESC = 'YouTube.com search URLs'
     IE_NAME = 'youtube:search_url'
-    _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?search_query=(?P<query>[^&]+)(?:[&]|$)'
+    _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?(?:search_query|q)=(?P<query>[^&]+)(?:[&]|$)'
     _TESTS = [{
         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
         'playlist_mincount': 5,
         'info_dict': {
             'title': 'youtube-dl test video',
         }
+    }, {
+        'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 1932476c137363d7674abb84286c9d1aca3a0e70 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 16 Feb 2016 01:47:07 +0800
Subject: [PATCH 0706/1105] [iqiyi] Omit MD5 sums for the VIP-only video

---
 youtube_dl/extractor/iqiyi.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 04db252df..9046705a5 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -105,20 +105,19 @@ class IqiyiIE(InfoExtractor):
         'only_matching': True,
     }, {
         # VIP-only video. The first 2 parts (6 minutes) are available without login
+        # MD5 sums omitted as values are different on Travis CI and my machine
         'url': 'http://www.iqiyi.com/v_19rrny4w8w.html',
         'info_dict': {
             'id': 'f3cf468b39dddb30d676f89a91200dc1',
             'title': '泰坦尼克号',
         },
         'playlist': [{
-            'md5': '436bcde6e1307307d5ba1549715b0bbd',
             'info_dict': {
                 'id': 'f3cf468b39dddb30d676f89a91200dc1_part1',
                 'ext': 'f4v',
                 'title': '泰坦尼克号',
             },
         }, {
-            'md5': 'bfc5e332f4900fde547c69372385649e',
             'info_dict': {
                 'id': 'f3cf468b39dddb30d676f89a91200dc1_part2',
                 'ext': 'f4v',

From fe80df30804736db5e644ad15ac09149c57ddf33 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Feb 2016 01:24:57 +0600
Subject: [PATCH 0707/1105] Credit @TingPing for screenjunkies (#8505)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index f8890eb27..0e90a3ecb 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -159,3 +159,4 @@ Saso Bezlaj
 Erwin de Haan
 Jens Wille
 Robin Houtevelts
+Patrick Griffis

From f66a3c7bc2c675b0a053c6cfab6f755cb9f5bbda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Feb 2016 01:30:00 +0600
Subject: [PATCH 0708/1105] [screenjunkies] Fix spelling

---
 youtube_dl/extractor/screenjunkies.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/screenjunkies.py b/youtube_dl/extractor/screenjunkies.py
index e0db180b8..f2af15f6b 100644
--- a/youtube_dl/extractor/screenjunkies.py
+++ b/youtube_dl/extractor/screenjunkies.py
@@ -93,8 +93,8 @@ class ScreenJunkiesIE(InfoExtractor):
 
         if not bitrates:
             # When subscriptionLevel > 0, i.e. plus subscription is required
-            # media list will be empty. However, hds and hls uri are still
-            # available. We can grabbing them assuming bitrates to be default.
+            # media list will be empty. However, hds and hls uris are still
+            # available. We can grab them assuming bitrates to be default.
             bitrates = self._DEFAULT_BITRATES
 
         auth_token = embed_vars.get('AuthToken')

From cb64ccc71531fd7491f65ad12fa1d68ac5de70e1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 16 Feb 2016 09:07:38 +0800
Subject: [PATCH 0709/1105] [facebook] Improve error handling (#8572)

---
 youtube_dl/extractor/facebook.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index ed237f081..0a9a5ca71 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -186,7 +186,7 @@ class FacebookIE(InfoExtractor):
         if not video_data:
             server_js_data = self._parse_json(self._search_regex(
                 r'handleServerJS\(({.+})\);', webpage, 'server js data'), video_id)
-            for item in server_js_data['instances']:
+            for item in server_js_data.get('instances', []):
                 if item[1][0] == 'VideoConfig':
                     video_data = video_data_list2dict(item[2][0]['videoData'])
                     break

From 86be3cdc2a623a522500ca073835579b853a1ae1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Feb 2016 22:05:23 +0600
Subject: [PATCH 0710/1105] [xtube] Fix extraction (Closes #8565)

---
 youtube_dl/extractor/xtube.py | 90 +++++++++++++++++------------------
 1 file changed, 44 insertions(+), 46 deletions(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 8cd3a0687..24df5c48e 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -7,15 +7,16 @@ from .common import InfoExtractor
 from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     int_or_none,
-    parse_duration,
     sanitized_Request,
     str_to_int,
 )
 
 
 class XTubeIE(InfoExtractor):
-    _VALID_URL = r'(?:xtube:|https?://(?:www\.)?xtube\.com/watch\.php\?.*\bv=)(?P<id>[^/?&#]+)'
-    _TEST = {
+    _VALID_URL = r'(?:xtube:|https?://(?:www\.)?xtube\.com/(?:watch\.php\?.*\bv=|video-watch/(?P<display_id>[^/]+)-))(?P<id>[^/?&#]+)'
+
+    _TESTS = [{
+        # old URL schema
         'url': 'http://www.xtube.com/watch.php?v=kVTUy_G222_',
         'md5': '092fbdd3cbe292c920ef6fc6a8a9cdab',
         'info_dict': {
@@ -27,63 +28,60 @@ class XTubeIE(InfoExtractor):
             'duration': 450,
             'age_limit': 18,
         }
-    }
+    }, {
+        # new URL schema
+        'url': 'http://www.xtube.com/video-watch/strange-erotica-625837',
+        'only_matching': True,
+    }, {
+        'url': 'xtube:625837',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        req = sanitized_Request('http://www.xtube.com/watch.php?v=%s' % video_id)
-        req.add_header('Cookie', 'age_verified=1')
-        webpage = self._download_webpage(req, video_id)
+        if not display_id:
+            display_id = video_id
+            url = 'http://www.xtube.com/watch.php?v=%s' % video_id
 
-        video_title = self._html_search_regex(
-            r'<p class="title">([^<]+)', webpage, 'title')
-        video_uploader = self._html_search_regex(
-            [r"var\s+contentOwnerId\s*=\s*'([^']+)",
-             r'By:\s*<a href="/community/profile\.php\?user=([^"]+)'],
+        req = sanitized_Request(url)
+        req.add_header('Cookie', 'age_verified=1; cookiesAccepted=1')
+        webpage = self._download_webpage(req, display_id)
+
+        flashvars = self._parse_json(
+            self._search_regex(
+                r'xt\.playerOps\s*=\s*({.+?});', webpage, 'player ops'),
+            video_id)['flashvars']
+
+        title = flashvars.get('title') or self._search_regex(
+            r'<h1>([^<]+)</h1>', webpage, 'title')
+        video_url = compat_urllib_parse_unquote(flashvars['video_url'])
+        duration = int_or_none(flashvars.get('video_duration'))
+
+        uploader = self._search_regex(
+            r'<input[^>]+name="contentOwnerId"[^>]+value="([^"]+)"',
             webpage, 'uploader', fatal=False)
-        video_description = self._html_search_regex(
-            r'<p class="fieldsDesc">([^<]+)',
-            webpage, 'description', fatal=False)
-        duration = parse_duration(self._html_search_regex(
-            r'<span class="bold">Runtime:</span> ([^<]+)</p>',
-            webpage, 'duration', fatal=False))
-        view_count = str_to_int(self._html_search_regex(
-            r'<span class="bold">Views:</span> ([\d,\.]+)</p>',
+        description = self._search_regex(
+            r'</h1>\s*<p>([^<]+)', webpage, 'description', fatal=False)
+        view_count = str_to_int(self._search_regex(
+            r'<dt>Views:</dt>\s*<dd>([\d,\.]+)</dd>',
             webpage, 'view count', fatal=False))
         comment_count = str_to_int(self._html_search_regex(
-            r'<div id="commentBar">([\d,\.]+) Comments</div>',
+            r'>Comments? \(([\d,\.]+)\)<',
             webpage, 'comment count', fatal=False))
 
-        formats = []
-        for format_id, video_url in re.findall(
-                r'flashvars\.quality_(.+?)\s*=\s*"([^"]+)"', webpage):
-            fmt = {
-                'url': compat_urllib_parse_unquote(video_url),
-                'format_id': format_id,
-            }
-            m = re.search(r'^(?P<height>\d+)[pP]', format_id)
-            if m:
-                fmt['height'] = int(m.group('height'))
-            formats.append(fmt)
-
-        if not formats:
-            video_url = compat_urllib_parse_unquote(self._search_regex(
-                r'flashvars\.video_url\s*=\s*"([^"]+)"',
-                webpage, 'video URL'))
-            formats.append({'url': video_url})
-
-        self._sort_formats(formats)
-
         return {
             'id': video_id,
-            'title': video_title,
-            'uploader': video_uploader,
-            'description': video_description,
+            'display_id': display_id,
+            'url': video_url,
+            'title': title,
+            'description': description,
+            'uploader': uploader,
             'duration': duration,
             'view_count': view_count,
             'comment_count': comment_count,
-            'formats': formats,
             'age_limit': 18,
         }
 

From f4db09178a2c531005a3502a497b8dbf0bfe20e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Feb 2016 22:06:26 +0600
Subject: [PATCH 0711/1105] [xtube:user] Remove duplicated video ids

---
 youtube_dl/extractor/xtube.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 24df5c48e..4075b8a4f 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     int_or_none,
+    orderedSet,
     sanitized_Request,
     str_to_int,
 )
@@ -118,7 +119,8 @@ class XTubeUserIE(InfoExtractor):
             if not html:
                 break
 
-            for _, video_id in re.findall(r'data-plid=(["\'])(.+?)\1', html):
+            for video_id in orderedSet([video_id for _, video_id in re.findall(
+                    r'data-plid=(["\'])(.+?)\1', html)]):
                 entries.append(self.url_result('xtube:%s' % video_id, XTubeIE.ie_key()))
 
             page_count = int_or_none(page.get('pageCount'))

From e277f2a63b836c774009c8f6c8e45a7867159b73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Feb 2016 22:23:38 +0600
Subject: [PATCH 0712/1105] [orf:tvthek] Check formats (Closes #8580)

---
 youtube_dl/extractor/orf.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index c54775d54..958eb398b 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -112,6 +112,7 @@ class ORFTVthekIE(InfoExtractor):
                             % geo_str),
                         fatal=False)
 
+            self._check_formats(formats, video_id)
             self._sort_formats(formats)
 
             upload_date = unified_strdate(sd['created_date'])

From 199e72429106375218902102812e26c2fc6624b5 Mon Sep 17 00:00:00 2001
From: mutantmonkey <mutantmonkey@mutantmonkey.in>
Date: Mon, 15 Feb 2016 17:30:53 -0800
Subject: [PATCH 0713/1105] [KUSI] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/kusi.py     | 61 ++++++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+)
 create mode 100644 youtube_dl/extractor/kusi.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 537d25777..bfc2008be 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -338,6 +338,7 @@ from .konserthusetplay import KonserthusetPlayIE
 from .kontrtube import KontrTubeIE
 from .krasview import KrasViewIE
 from .ku6 import Ku6IE
+from .kusi import KUSIIE
 from .kuwo import (
     KuwoIE,
     KuwoAlbumIE,
diff --git a/youtube_dl/extractor/kusi.py b/youtube_dl/extractor/kusi.py
new file mode 100644
index 000000000..20407411b
--- /dev/null
+++ b/youtube_dl/extractor/kusi.py
@@ -0,0 +1,61 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote_plus
+from ..utils import int_or_none
+
+
+class KUSIIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?kusi\.com/(?P<path>story/.+|video\?clipId=(?P<clipId>\d+))'
+    _TEST = {
+        'url': 'http://www.kusi.com/story/31183873/turko-files-case-closed-put-on-hold',
+        'md5': 'f926e7684294cf8cb7bdf8858e1b3988',
+        'info_dict': {
+            'id': '12203019',
+            'ext': 'mp4',
+            'title': 'Turko Files: Case Closed! & Put On Hold!',
+            'duration': 231000,
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+
+        if mobj.group('clipId') is not None:
+            video_id = mobj.group('clipId')
+        else:
+            webpage = self._download_webpage(url, mobj.group('path'))
+            video_id = self._html_search_regex(r'"clipId", "(\d+)"', webpage,
+                                               'clipId')
+
+        xml_url = 'http://www.kusi.com/build.asp?buildtype=buildfeaturexml'\
+                  'request&featureType=Clip&featureid={0}&affiliateno=956&'\
+                  'clientgroupid=1&rnd=562461'.format(video_id)
+        doc = self._download_xml(xml_url, video_id,
+                                 note='Downloading video info',
+                                 errnote='Failed to download video info')
+
+        video_title = doc.find('HEADLINE').text
+        duration = int_or_none(doc.find('DURATION'), get_attr='text')
+        description = doc.find('ABSTRACT')
+
+        quality_options = doc.find('{http://search.yahoo.com/mrss/}group').findall('{http://search.yahoo.com/mrss/}content')
+        formats = []
+        for quality in quality_options:
+            if 'height' in quality.attrib:
+                formats.append({
+                    'url': compat_urllib_parse_unquote_plus(quality.attrib['url']),
+                    'height': quality.attrib['height'],
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': video_title,
+            'description': description,
+            'duration': duration,
+            'formats': formats,
+        }

From 0f425e65ec6d9ac47fa7f348fae7c06be2557f8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Feb 2016 21:47:18 +0600
Subject: [PATCH 0714/1105] [arte:+7] Add support for en and es URLs

---
 youtube_dl/extractor/arte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 6ed855a57..67194bd31 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -63,7 +63,7 @@ class ArteTvIE(InfoExtractor):
 
 class ArteTVPlus7IE(InfoExtractor):
     IE_NAME = 'arte.tv:+7'
-    _VALID_URL = r'https?://(?:www\.)?arte\.tv/guide/(?P<lang>fr|de)/(?:(?:sendungen|emissions)/)?(?P<id>.*?)/(?P<name>.*?)(\?.*)?'
+    _VALID_URL = r'https?://(?:www\.)?arte\.tv/guide/(?P<lang>fr|de|en|es)/(?:(?:sendungen|emissions)/)?(?P<id>.*?)/(?P<name>.*?)(\?.*)?'
 
     @classmethod
     def _extract_url_info(cls, url):

From 9654fc875bdcd055031670b640531375bc513012 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Feb 2016 21:49:15 +0600
Subject: [PATCH 0715/1105] [arte:+7] Fix extraction for react-based layout

---
 youtube_dl/extractor/arte.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 67194bd31..497899f8d 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -102,10 +102,20 @@ class ArteTVPlus7IE(InfoExtractor):
             iframe_url = find_iframe_url(webpage, None)
             if not iframe_url:
                 embed_url = self._html_search_regex(
-                    r'arte_vp_url_oembed=\'([^\']+?)\'', webpage, 'embed url')
-                player = self._download_json(
-                    embed_url, video_id, 'Downloading player page')
-                iframe_url = find_iframe_url(player['html'])
+                    r'arte_vp_url_oembed=\'([^\']+?)\'', webpage, 'embed url', default=None)
+                if embed_url:
+                    player = self._download_json(
+                        embed_url, video_id, 'Downloading player page')
+                    iframe_url = find_iframe_url(player['html'])
+            # en and es URLs produce react-based pages with different layout (e.g.
+            # http://www.arte.tv/guide/en/053330-002-A/carnival-italy?zone=world)
+            if not iframe_url:
+                embed_html = self._parse_json(
+                    self._search_regex(
+                        r'program\s*:\s*({.+?["\']embed_html["\'].+?}),?\s*\n',
+                        webpage, 'program'),
+                    video_id)['embed_html']
+                iframe_url= find_iframe_url(embed_html)
             json_url = compat_parse_qs(
                 compat_urllib_parse_urlparse(iframe_url).query)['json_url'][0]
         return self._extract_from_json_url(json_url, video_id, lang)

From 44b97450005dc973175e6897e3422a2a63983b36 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Feb 2016 21:53:53 +0600
Subject: [PATCH 0716/1105] [arte] Extend more _VALID_URLs for en and es
 support

---
 youtube_dl/extractor/arte.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 497899f8d..1f4e0e6a2 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -23,7 +23,7 @@ from ..utils import (
 
 
 class ArteTvIE(InfoExtractor):
-    _VALID_URL = r'http://videos\.arte\.tv/(?P<lang>fr|de)/.*-(?P<id>.*?)\.html'
+    _VALID_URL = r'http://videos\.arte\.tv/(?P<lang>fr|de|en|es)/.*-(?P<id>.*?)\.html'
     IE_NAME = 'arte.tv'
 
     def _real_extract(self, url):
@@ -195,7 +195,7 @@ class ArteTVPlus7IE(InfoExtractor):
 # It also uses the arte_vp_url url from the webpage to extract the information
 class ArteTVCreativeIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:creative'
-    _VALID_URL = r'https?://creative\.arte\.tv/(?P<lang>fr|de)/(?:magazine?/)?(?P<id>[^?#]+)'
+    _VALID_URL = r'https?://creative\.arte\.tv/(?P<lang>fr|de|en|es)/(?:magazine?/)?(?P<id>[^?#]+)'
 
     _TESTS = [{
         'url': 'http://creative.arte.tv/de/magazin/agentur-amateur-corporate-design',
@@ -219,7 +219,7 @@ class ArteTVCreativeIE(ArteTVPlus7IE):
 
 class ArteTVFutureIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:future'
-    _VALID_URL = r'https?://future\.arte\.tv/(?P<lang>fr|de)/(?P<id>.+)'
+    _VALID_URL = r'https?://future\.arte\.tv/(?P<lang>fr|de|en|es)/(?P<id>.+)'
 
     _TESTS = [{
         'url': 'http://future.arte.tv/fr/info-sciences/les-ecrevisses-aussi-sont-anxieuses',
@@ -254,7 +254,7 @@ class ArteTVDDCIE(ArteTVPlus7IE):
 
 class ArteTVConcertIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:concert'
-    _VALID_URL = r'https?://concert\.arte\.tv/(?P<lang>de|fr)/(?P<id>.+)'
+    _VALID_URL = r'https?://concert\.arte\.tv/(?P<lang>fr|de|en|es)/(?P<id>.+)'
 
     _TEST = {
         'url': 'http://concert.arte.tv/de/notwist-im-pariser-konzertclub-divan-du-monde',
@@ -271,7 +271,7 @@ class ArteTVConcertIE(ArteTVPlus7IE):
 
 class ArteTVCinemaIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:cinema'
-    _VALID_URL = r'https?://cinema\.arte\.tv/(?P<lang>de|fr)/(?P<id>.+)'
+    _VALID_URL = r'https?://cinema\.arte\.tv/(?P<lang>fr|de|en|es)/(?P<id>.+)'
 
     _TEST = {
         'url': 'http://cinema.arte.tv/de/node/38291',

From 08d65046f057272155cf6be4a8329ecf3274596f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Feb 2016 22:37:05 +0600
Subject: [PATCH 0717/1105] [arte] Make sorting aware of en/es formats

---
 youtube_dl/extractor/arte.py | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 1f4e0e6a2..4a82a106f 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -142,27 +142,30 @@ class ArteTVPlus7IE(InfoExtractor):
         }
         qfunc = qualities(['HQ', 'MQ', 'EQ', 'SQ'])
 
+        LANGS = {
+            'fr': 'F',
+            'de': 'A',
+            'en': 'E[ANG]',
+            'es': 'E[ESP]',
+        }
+
         formats = []
         for format_id, format_dict in player_info['VSR'].items():
             f = dict(format_dict)
             versionCode = f.get('versionCode')
-
-            langcode = {
-                'fr': 'F',
-                'de': 'A',
-            }.get(lang, lang)
-            lang_rexs = [r'VO?%s' % langcode, r'VO?.-ST%s' % langcode]
-            lang_pref = (
-                None if versionCode is None else (
-                    10 if any(re.match(r, versionCode) for r in lang_rexs)
-                    else -10))
+            langcode = LANGS.get(lang, lang)
+            lang_rexs = [r'VO?%s-' % re.escape(langcode), r'VO?.-ST%s$' % re.escape(langcode)]
+            lang_pref = None
+            if versionCode:
+                matched_lang_rexs = [r for r in lang_rexs if re.match(r, versionCode)]
+                lang_pref = -10 if not matched_lang_rexs else 10 * len(matched_lang_rexs)
             source_pref = 0
             if versionCode is not None:
                 # The original version with subtitles has lower relevance
-                if re.match(r'VO-ST(F|A)', versionCode):
+                if re.match(r'VO-ST(F|A|E)', versionCode):
                     source_pref -= 10
                 # The version with sourds/mal subtitles has also lower relevance
-                elif re.match(r'VO?(F|A)-STM\1', versionCode):
+                elif re.match(r'VO?(F|A|E)-STM\1', versionCode):
                     source_pref -= 9
             format = {
                 'format_id': format_id,

From e725e4bced92f4eb6c406d6ce01fa361c972fe30 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Feb 2016 22:37:55 +0600
Subject: [PATCH 0718/1105] [arte] PEP 8

---
 youtube_dl/extractor/arte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 4a82a106f..5ec67bc9c 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -115,7 +115,7 @@ class ArteTVPlus7IE(InfoExtractor):
                         r'program\s*:\s*({.+?["\']embed_html["\'].+?}),?\s*\n',
                         webpage, 'program'),
                     video_id)['embed_html']
-                iframe_url= find_iframe_url(embed_html)
+                iframe_url = find_iframe_url(embed_html)
             json_url = compat_parse_qs(
                 compat_urllib_parse_urlparse(iframe_url).query)['json_url'][0]
         return self._extract_from_json_url(json_url, video_id, lang)

From 8bbd3d14765a836f72beb2d9dc53832872845669 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Feb 2016 22:51:08 +0600
Subject: [PATCH 0719/1105] [arte] Fix upload date extraction (Closes #8581)

---
 youtube_dl/extractor/arte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 5ec67bc9c..c416f82ee 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -126,7 +126,7 @@ class ArteTVPlus7IE(InfoExtractor):
 
         upload_date_str = player_info.get('shootingDate')
         if not upload_date_str:
-            upload_date_str = player_info.get('VDA', '').split(' ')[0]
+            upload_date_str = (player_info.get('VRA') or player_info.get('VDA') or '').split(' ')[0]
 
         title = player_info['VTI'].strip()
         subtitle = player_info.get('VSU', '').strip()

From 80e59a0d5d397c46c73d52a4e2c1ec6af3b9b1e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Feb 2016 23:18:23 +0600
Subject: [PATCH 0720/1105] [vrt] Make formats extraction non fatal (Closes
 #8587)

---
 youtube_dl/extractor/vrt.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vrt.py b/youtube_dl/extractor/vrt.py
index 01891ac4c..522e2bbc8 100644
--- a/youtube_dl/extractor/vrt.py
+++ b/youtube_dl/extractor/vrt.py
@@ -73,11 +73,12 @@ class VRTIE(InfoExtractor):
         if mobj:
             formats.extend(self._extract_m3u8_formats(
                 '%s/%s' % (mobj.group('server'), mobj.group('path')),
-                video_id, 'mp4', m3u8_id='hls'))
+                video_id, 'mp4', m3u8_id='hls', fatal=False))
         mobj = re.search(r'data-video-src="(?P<src>[^"]+)"', webpage)
         if mobj:
             formats.extend(self._extract_f4m_formats(
-                '%s/manifest.f4m' % mobj.group('src'), video_id, f4m_id='hds'))
+                '%s/manifest.f4m' % mobj.group('src'),
+                video_id, f4m_id='hds', fatal=False))
         self._sort_formats(formats)
 
         title = self._og_search_title(webpage)

From 63bf4f0dc0502a87c19863c2509a4bc28c7c1abb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Feb 2016 23:28:41 +0600
Subject: [PATCH 0721/1105] [vrt] Detect geo restriction

---
 youtube_dl/extractor/vrt.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/vrt.py b/youtube_dl/extractor/vrt.py
index 522e2bbc8..2b6bae89b 100644
--- a/youtube_dl/extractor/vrt.py
+++ b/youtube_dl/extractor/vrt.py
@@ -79,6 +79,10 @@ class VRTIE(InfoExtractor):
             formats.extend(self._extract_f4m_formats(
                 '%s/manifest.f4m' % mobj.group('src'),
                 video_id, f4m_id='hds', fatal=False))
+
+        if not formats and 'data-video-geoblocking="true"' in webpage:
+            self.raise_geo_restricted('This video is only available in Belgium')
+
         self._sort_formats(formats)
 
         title = self._og_search_title(webpage)

From 411cb8f4769215ea6835895206782d8ec00a5114 Mon Sep 17 00:00:00 2001
From: cazulu <jvlarapeinado@gmail.com>
Date: Tue, 16 Feb 2016 18:45:53 +0900
Subject: [PATCH 0722/1105] [dailymotion] Fix view count extraction

Fix view count parsing when the decimal marker is a whitespace, e.g. '101 101'
---
 youtube_dl/extractor/dailymotion.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 7ae9f2359..2e6226ea0 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -122,10 +122,13 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
         description = self._og_search_description(webpage) or self._html_search_meta(
             'description', webpage, 'description')
 
-        view_count = str_to_int(self._search_regex(
-            [r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserPlays:(\d+)"',
-             r'video_views_count[^>]+>\s+([\d\.,]+)'],
-            webpage, 'view count', fatal=False))
+        view_count_str = self._search_regex(
+            (r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserPlays:([\s\d,.]+)"',
+             r'video_views_count[^>]+>\s+([\s\d\,.]+)'),
+            webpage, 'view count', fatal=False)
+        if view_count_str:
+            view_count_str = re.sub(r'\s', '', view_count_str)
+        view_count = str_to_int(view_count_str)
         comment_count = int_or_none(self._search_regex(
             r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserComments:(\d+)"',
             webpage, 'comment count', fatal=False))

From 778433cb90064a3fbe21afb722c787841e85ddf4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 18 Feb 2016 15:43:28 +0100
Subject: [PATCH 0723/1105] [cbsnews] extract subtitle url from theplatform
 SMIL manifest(fixes #8568)

---
 youtube_dl/extractor/cbsnews.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 8f864699f..f0de73aeb 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -46,6 +46,15 @@ class CBSNewsIE(ThePlatformIE):
         },
     ]
 
+    def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
+        closed_caption_e = smil.find(self._xpath_ns('.//param[@name=\'ClosedCaptionURL\']', namespace))
+        return {
+            'en': [{
+                'ext': 'ttml',
+                'url': closed_caption_e.attrib['value'],
+            }]
+        } if closed_caption_e is not None and closed_caption_e.attrib.get('value') else []
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -61,12 +70,6 @@ class CBSNewsIE(ThePlatformIE):
         thumbnail = item.get('mediaImage') or item.get('thumbnail')
 
         subtitles = {}
-        if 'mpxRefId' in video_info:
-            subtitles['en'] = [{
-                'ext': 'ttml',
-                'url': 'http://www.cbsnews.com/videos/captions/%s.adb_xml' % video_info['mpxRefId'],
-            }]
-
         formats = []
         for format_id in ['RtmpMobileLow', 'RtmpMobileHigh', 'Hls', 'RtmpDesktop']:
             pid = item.get('media' + format_id)

From 8b809a079aa5b55d39029ee00c706e3c67ed311a Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 18 Feb 2016 16:10:09 +0100
Subject: [PATCH 0724/1105] [cbsnews] use find_xpath_attr

---
 youtube_dl/extractor/cbsnews.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index f0de73aeb..7319ee1b7 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from .theplatform import ThePlatformIE
-from ..utils import parse_duration
+from ..utils import (
+    parse_duration,
+    find_xpath_attr,
+)
 
 
 class CBSNewsIE(ThePlatformIE):
@@ -47,7 +50,7 @@ class CBSNewsIE(ThePlatformIE):
     ]
 
     def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
-        closed_caption_e = smil.find(self._xpath_ns('.//param[@name=\'ClosedCaptionURL\']', namespace))
+        closed_caption_e = find_xpath_attr(smil, self._xpath_ns('.//param', namespace), 'name', 'ClosedCaptionURL')
         return {
             'en': [{
                 'ext': 'ttml',

From 49625662a94bec2d289ed831f7e024c4c43098fd Mon Sep 17 00:00:00 2001
From: Thomas Jost <schnouki@schnouki.net>
Date: Mon, 8 Feb 2016 22:17:21 +0100
Subject: [PATCH 0725/1105] [arte:magazine] Add extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/arte.py     | 23 +++++++++++++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 537d25777..4049141d9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -45,6 +45,7 @@ from .arte import (
     ArteTVFutureIE,
     ArteTVCinemaIE,
     ArteTVDDCIE,
+    ArteTVMagazineIE,
     ArteTVEmbedIE,
 )
 from .atresplayer import AtresPlayerIE
diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index c416f82ee..b524866d9 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -289,6 +289,29 @@ class ArteTVCinemaIE(ArteTVPlus7IE):
     }
 
 
+class ArteTVMagazineIE(ArteTVPlus7IE):
+    IE_NAME = 'arte.tv:magazine'
+    _VALID_URL = r'https?://(?:www\.)?arte\.tv/magazine/(?P<name>.*?)/(?P<lang>fr|de)/(?P<id>.*?)'
+
+    _TESTS = [{
+        'url': 'http://www.arte.tv/magazine/trepalium/fr/entretien-avec-le-realisateur-vincent-lannoo-trepalium',
+        'md5': '66a093339c1278bb3719157ef07107b2',
+        'info_dict': {
+            'id': '065965-000-A',
+            'ext': 'mp4',
+            'title': 'Trepalium - Extrait Ep.01',
+        },
+    }, {
+        'url': 'http://www.arte.tv/magazine/metropolis/de/frank-woeste-german-paris-metropolis',
+        'md5': '53185a2b4876383552c037c21aeca8f7',
+        'info_dict': {
+            'id': '066149-000-A',
+            'ext': 'mp4',
+            'title': 'Bonus: Frank Woeste - ein deutscher Jazzmusiker in Paris',
+        },
+    }]
+
+
 class ArteTVEmbedIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:embed'
     _VALID_URL = r'''(?x)

From 479ec54a8d9631215483f144c3d1b7ea830178b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Feb 2016 21:29:07 +0600
Subject: [PATCH 0726/1105] [arte:magazine] Improve (Closes #8473)

---
 youtube_dl/extractor/arte.py | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index b524866d9..3337d0085 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -291,7 +291,7 @@ class ArteTVCinemaIE(ArteTVPlus7IE):
 
 class ArteTVMagazineIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:magazine'
-    _VALID_URL = r'https?://(?:www\.)?arte\.tv/magazine/(?P<name>.*?)/(?P<lang>fr|de)/(?P<id>.*?)'
+    _VALID_URL = r'https?://(?:www\.)?arte\.tv/magazine/[^/]+/(?P<lang>fr|de|en|es)/(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'http://www.arte.tv/magazine/trepalium/fr/entretien-avec-le-realisateur-vincent-lannoo-trepalium',
@@ -303,12 +303,7 @@ class ArteTVMagazineIE(ArteTVPlus7IE):
         },
     }, {
         'url': 'http://www.arte.tv/magazine/metropolis/de/frank-woeste-german-paris-metropolis',
-        'md5': '53185a2b4876383552c037c21aeca8f7',
-        'info_dict': {
-            'id': '066149-000-A',
-            'ext': 'mp4',
-            'title': 'Bonus: Frank Woeste - ein deutscher Jazzmusiker in Paris',
-        },
+        'only_matching': True,
     }]
 
 

From d48502b82a3466dc3f75dc35815bfe22a934aabf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Feb 2016 21:29:52 +0600
Subject: [PATCH 0727/1105] [arte] Improve _VALID_URLs

---
 youtube_dl/extractor/arte.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 3337d0085..793da2ee1 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -63,7 +63,7 @@ class ArteTvIE(InfoExtractor):
 
 class ArteTVPlus7IE(InfoExtractor):
     IE_NAME = 'arte.tv:+7'
-    _VALID_URL = r'https?://(?:www\.)?arte\.tv/guide/(?P<lang>fr|de|en|es)/(?:(?:sendungen|emissions)/)?(?P<id>.*?)/(?P<name>.*?)(\?.*)?'
+    _VALID_URL = r'https?://(?:www\.)?arte\.tv/guide/(?P<lang>fr|de|en|es)/(?:(?:sendungen|emissions)/)?(?P<id>[^/]+)/(?P<name>[^/?#&+])'
 
     @classmethod
     def _extract_url_info(cls, url):
@@ -198,7 +198,7 @@ class ArteTVPlus7IE(InfoExtractor):
 # It also uses the arte_vp_url url from the webpage to extract the information
 class ArteTVCreativeIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:creative'
-    _VALID_URL = r'https?://creative\.arte\.tv/(?P<lang>fr|de|en|es)/(?:magazine?/)?(?P<id>[^?#]+)'
+    _VALID_URL = r'https?://creative\.arte\.tv/(?P<lang>fr|de|en|es)/(?:magazine?/)?(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'http://creative.arte.tv/de/magazin/agentur-amateur-corporate-design',
@@ -222,7 +222,7 @@ class ArteTVCreativeIE(ArteTVPlus7IE):
 
 class ArteTVFutureIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:future'
-    _VALID_URL = r'https?://future\.arte\.tv/(?P<lang>fr|de|en|es)/(?P<id>.+)'
+    _VALID_URL = r'https?://future\.arte\.tv/(?P<lang>fr|de|en|es)/(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'http://future.arte.tv/fr/info-sciences/les-ecrevisses-aussi-sont-anxieuses',
@@ -239,7 +239,7 @@ class ArteTVFutureIE(ArteTVPlus7IE):
 
 class ArteTVDDCIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:ddc'
-    _VALID_URL = r'https?://ddc\.arte\.tv/(?P<lang>emission|folge)/(?P<id>.+)'
+    _VALID_URL = r'https?://ddc\.arte\.tv/(?P<lang>emission|folge)/(?P<id>[^/?#&]+)'
 
     def _real_extract(self, url):
         video_id, lang = self._extract_url_info(url)
@@ -257,7 +257,7 @@ class ArteTVDDCIE(ArteTVPlus7IE):
 
 class ArteTVConcertIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:concert'
-    _VALID_URL = r'https?://concert\.arte\.tv/(?P<lang>fr|de|en|es)/(?P<id>.+)'
+    _VALID_URL = r'https?://concert\.arte\.tv/(?P<lang>fr|de|en|es)/(?P<id>[^/?#&]+)'
 
     _TEST = {
         'url': 'http://concert.arte.tv/de/notwist-im-pariser-konzertclub-divan-du-monde',

From bc7a9cd8fb223481593840441d20c0f19f93c4c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Feb 2016 21:50:21 +0600
Subject: [PATCH 0728/1105] [youtube:watchlater] Improve _VALID_URL (Closes
 #8594)

---
 youtube_dl/extractor/youtube.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 45ad88152..408c8d52c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2026,9 +2026,15 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
 class YoutubeWatchLaterIE(YoutubePlaylistIE):
     IE_NAME = 'youtube:watchlater'
     IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
-    _VALID_URL = r'https?://www\.youtube\.com/(?:feed/watch_later|playlist\?list=WL)|:ytwatchlater'
+    _VALID_URL = r'https?://www\.youtube\.com/(?:feed/watch_later|(?:playlist|watch)\?(?:.+&)?list=WL)|:ytwatchlater'
 
-    _TESTS = []  # override PlaylistIE tests
+    _TESTS = [{
+        'url': 'https://www.youtube.com/playlist?list=WL',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.youtube.com/watch?v=bCNU9TrbiRk&index=1&list=WL',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         return self._extract_playlist('WL')

From ebf1b291d0fc16612fdac877c1242f1cb5761277 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Feb 2016 22:03:46 +0600
Subject: [PATCH 0729/1105] [youtube:watchlater] Respect --no-playlist

---
 youtube_dl/extractor/youtube.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 408c8d52c..e24dd3e5b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1701,13 +1701,7 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
 
         return self.playlist_result(self._entries(page, playlist_id), playlist_id, playlist_title)
 
-    def _real_extract(self, url):
-        # Extract playlist id
-        mobj = re.match(self._VALID_URL, url)
-        if mobj is None:
-            raise ExtractorError('Invalid URL: %s' % url)
-        playlist_id = mobj.group(1) or mobj.group(2)
-
+    def _check_download_just_video(self, url, playlist_id):
         # Check if it's a video-specific URL
         query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
         if 'v' in query_dict:
@@ -1718,6 +1712,17 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
             else:
                 self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
 
+    def _real_extract(self, url):
+        # Extract playlist id
+        mobj = re.match(self._VALID_URL, url)
+        if mobj is None:
+            raise ExtractorError('Invalid URL: %s' % url)
+        playlist_id = mobj.group(1) or mobj.group(2)
+
+        video = self._check_download_just_video(url, playlist_id)
+        if video:
+            return video
+
         if playlist_id.startswith('RD') or playlist_id.startswith('UL'):
             # Mixes require a custom extraction process
             return self._extract_mix(playlist_id)
@@ -2037,6 +2042,9 @@ class YoutubeWatchLaterIE(YoutubePlaylistIE):
     }]
 
     def _real_extract(self, url):
+        video = self._check_download_just_video(url, 'WL')
+        if video:
+            return video
         return self._extract_playlist('WL')
 
 

From f3f9cd9234aada1d677517e3042bd9312610081d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Feb 2016 22:09:21 +0600
Subject: [PATCH 0730/1105] [francetv] Improve video id regex (Closes #8563)

---
 youtube_dl/extractor/francetv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 8e60cf60f..3f4ac3093 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -289,7 +289,7 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         video_id, catalogue = self._html_search_regex(
-            r'href="http://videos?\.francetv\.fr/video/([^@]+@[^"]+)"',
+            r'(?:href=|player\.setVideo\(\s*)"http://videos?\.francetv\.fr/video/([^@]+@[^"]+)"',
             webpage, 'video ID').split('@')
         return self._extract_video(video_id, catalogue)
 

From 40e146aa1e1a8fd57d3f84b0a541174f56fa5dba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Feb 2016 22:29:17 +0600
Subject: [PATCH 0731/1105] [pornhub:user:videos] Add extractor (Closes #8548)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/pornhub.py  | 55 ++++++++++++++++++++++----------
 2 files changed, 40 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 4049141d9..1edbfbd28 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -555,6 +555,7 @@ from .pornhd import PornHdIE
 from .pornhub import (
     PornHubIE,
     PornHubPlaylistIE,
+    PornHubUserVideosIE,
 )
 from .pornotube import PornotubeIE
 from .pornovoisines import PornoVoisinesIE
diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 91e574dc2..405dbf006 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -129,7 +129,31 @@ class PornHubIE(InfoExtractor):
         }
 
 
-class PornHubPlaylistIE(InfoExtractor):
+class PornHubPlaylistBaseIE(InfoExtractor):
+    def _extract_entries(self, webpage):
+        return [
+            self.url_result('http://www.pornhub.com/%s' % video_url, 'PornHub')
+            for video_url in set(re.findall(
+                r'href="/?(view_video\.php\?.*\bviewkey=[\da-z]+[^"]*)"', webpage))
+        ]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = self._extract_entries(webpage)
+
+        playlist = self._parse_json(
+            self._search_regex(
+                r'playlistObject\s*=\s*({.+?});', webpage, 'playlist'),
+            playlist_id)
+
+        return self.playlist_result(
+            entries, playlist_id, playlist.get('title'), playlist.get('description'))
+
+
+class PornHubPlaylistIE(PornHubPlaylistBaseIE):
     _VALID_URL = r'https?://(?:www\.)?pornhub\.com/playlist/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.pornhub.com/playlist/6201671',
@@ -140,21 +164,20 @@ class PornHubPlaylistIE(InfoExtractor):
         'playlist_mincount': 35,
     }]
 
+
+class PornHubUserVideosIE(PornHubPlaylistBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?pornhub\.com/users/(?P<id>[^/]+)/videos'
+    _TESTS = [{
+        'url': 'http://www.pornhub.com/users/rushandlia/videos',
+        'info_dict': {
+            'id': 'rushandlia',
+        },
+        'playlist_mincount': 13,
+    }]
+
     def _real_extract(self, url):
-        playlist_id = self._match_id(url)
+        user_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, playlist_id)
+        webpage = self._download_webpage(url, user_id)
 
-        entries = [
-            self.url_result('http://www.pornhub.com/%s' % video_url, 'PornHub')
-            for video_url in set(re.findall(
-                r'href="/?(view_video\.php\?.*\bviewkey=[\da-z]+[^"]*)"', webpage))
-        ]
-
-        playlist = self._parse_json(
-            self._search_regex(
-                r'playlistObject\s*=\s*({.+?});', webpage, 'playlist'),
-            playlist_id)
-
-        return self.playlist_result(
-            entries, playlist_id, playlist.get('title'), playlist.get('description'))
+        return self.playlist_result(self._extract_entries(webpage), user_id)

From 88641243abfdc5673359269574bf7c76191966bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Feb 2016 22:30:19 +0600
Subject: [PATCH 0732/1105] [pornhub:playlistbase] Improve extract entries

---
 youtube_dl/extractor/pornhub.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 405dbf006..55af13324 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -132,7 +132,7 @@ class PornHubIE(InfoExtractor):
 class PornHubPlaylistBaseIE(InfoExtractor):
     def _extract_entries(self, webpage):
         return [
-            self.url_result('http://www.pornhub.com/%s' % video_url, 'PornHub')
+            self.url_result('http://www.pornhub.com/%s' % video_url, PornHubIE.ie_key())
             for video_url in set(re.findall(
                 r'href="/?(view_video\.php\?.*\bviewkey=[\da-z]+[^"]*)"', webpage))
         ]

From ed8648a322f8ec55957db0d4a959868807c38ebb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 19 Feb 2016 21:42:46 +0600
Subject: [PATCH 0733/1105] [pornhub] Fix thumbnail and duration extraction
 (Closes #8604)

---
 youtube_dl/extractor/pornhub.py | 31 ++++++++++++++++++++++++-------
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 55af13324..5a55c25e7 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -11,6 +11,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    int_or_none,
     sanitized_Request,
     str_to_int,
 )
@@ -23,13 +24,18 @@ class PornHubIE(InfoExtractor):
     _VALID_URL = r'https?://(?:[a-z]+\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)(?P<id>[0-9a-z]+)'
     _TESTS = [{
         'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
-        'md5': '882f488fa1f0026f023f33576004a2ed',
+        'md5': '1e19b41231a02eba417839222ac9d58e',
         'info_dict': {
             'id': '648719015',
             'ext': 'mp4',
-            'uploader': 'Babes',
             'title': 'Seductive Indian beauty strips down and fingers her pink pussy',
-            'age_limit': 18
+            'uploader': 'Babes',
+            'duration': 361,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'comment_count': int,
+            'age_limit': 18,
         }
     }, {
         'url': 'http://www.pornhub.com/view_video.php?viewkey=ph557bbb6676d2d',
@@ -67,13 +73,23 @@ class PornHubIE(InfoExtractor):
                 'PornHub said: %s' % error_msg,
                 expected=True, video_id=video_id)
 
-        video_title = self._html_search_regex(r'<h1 [^>]+>([^<]+)', webpage, 'title')
+        flashvars = self._parse_json(
+            self._search_regex(
+                r'var\s+flashv1ars_\d+\s*=\s*({.+?});', webpage, 'flashvars', default='{}'),
+            video_id)
+        if flashvars:
+            video_title = flashvars.get('video_title')
+            thumbnail = flashvars.get('image_url')
+            duration = int_or_none(flashvars.get('video_duration'))
+        else:
+            video_title, thumbnail, duration = [None] * 3
+
+        if not video_title:
+            video_title = self._html_search_regex(r'<h1 [^>]+>([^<]+)', webpage, 'title')
+
         video_uploader = self._html_search_regex(
             r'(?s)From:&nbsp;.+?<(?:a href="/users/|a href="/channels/|span class="username)[^>]+>(.+?)<',
             webpage, 'uploader', fatal=False)
-        thumbnail = self._html_search_regex(r'"image_url":"([^"]+)', webpage, 'thumbnail', fatal=False)
-        if thumbnail:
-            thumbnail = compat_urllib_parse_unquote(thumbnail)
 
         view_count = self._extract_count(
             r'<span class="count">([\d,\.]+)</span> views', webpage, 'view')
@@ -120,6 +136,7 @@ class PornHubIE(InfoExtractor):
             'uploader': video_uploader,
             'title': video_title,
             'thumbnail': thumbnail,
+            'duration': duration,
             'view_count': view_count,
             'like_count': like_count,
             'dislike_count': dislike_count,

From 8ec64ac68354cdf9428cd58506481ef4476c47c9 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Fri, 19 Feb 2016 22:18:21 +0600
Subject: [PATCH 0734/1105] [README.md] Clarify verbose log

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 65e0ee963..db49ab6d5 100644
--- a/README.md
+++ b/README.md
@@ -935,9 +935,9 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
 
 Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues>. Unless you were prompted so or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the IRC channel [#youtube-dl](irc://chat.freenode.net/#youtube-dl) on freenode ([webchat](http://webchat.freenode.net/?randomnick=1&channels=youtube-dl)).
 
-**Please include the full output of youtube-dl when run with `-v`**, i.e. add `-v` flag to your command line, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this:
+**Please include the full output of youtube-dl when run with `-v`**, i.e. **add** `-v` flag to **your command line**, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this:
 ```
-$ youtube-dl -v http://www.youtube.com/watch?v=BaW_jenozKcj
+$ youtube-dl -v <your command line>
 [debug] System config: []
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']

From 12b84ac8c13754baeeead907d8c9d239141f8706 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 19 Feb 2016 19:29:24 +0100
Subject: [PATCH 0735/1105] [downloader/external] Add FFmpegFD(fixes #622)

- replace HlsFD and RtspFD
- add basic support for downloading part of the video or audio
---
 youtube_dl/downloader/__init__.py | 20 +++++----
 youtube_dl/downloader/external.py | 65 +++++++++++++++++++++++++++
 youtube_dl/downloader/hls.py      | 74 ++-----------------------------
 youtube_dl/downloader/rtsp.py     | 45 -------------------
 4 files changed, 80 insertions(+), 124 deletions(-)
 delete mode 100644 youtube_dl/downloader/rtsp.py

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index dccc59212..bb6afb1f8 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -1,14 +1,15 @@
 from __future__ import unicode_literals
 
 from .common import FileDownloader
-from .external import get_external_downloader
 from .f4m import F4mFD
 from .hls import HlsFD
-from .hls import NativeHlsFD
 from .http import HttpFD
-from .rtsp import RtspFD
 from .rtmp import RtmpFD
 from .dash import DashSegmentsFD
+from .external import (
+    get_external_downloader,
+    FFmpegFD,
+)
 
 from ..utils import (
     determine_protocol,
@@ -16,10 +17,10 @@ from ..utils import (
 
 PROTOCOL_MAP = {
     'rtmp': RtmpFD,
-    'm3u8_native': NativeHlsFD,
-    'm3u8': HlsFD,
-    'mms': RtspFD,
-    'rtsp': RtspFD,
+    'm3u8_native': HlsFD,
+    'm3u8': FFmpegFD,
+    'mms': FFmpegFD,
+    'rtsp': FFmpegFD,
     'f4m': F4mFD,
     'http_dash_segments': DashSegmentsFD,
 }
@@ -30,6 +31,9 @@ def get_suitable_downloader(info_dict, params={}):
     protocol = determine_protocol(info_dict)
     info_dict['protocol'] = protocol
 
+    if (info_dict.get('start_time') or info_dict.get('end_time')) and FFmpegFD.supports(info_dict):
+        return FFmpegFD
+
     external_downloader = params.get('external_downloader')
     if external_downloader is not None:
         ed = get_external_downloader(external_downloader)
@@ -37,7 +41,7 @@ def get_suitable_downloader(info_dict, params={}):
             return ed
 
     if protocol == 'm3u8' and params.get('hls_prefer_native'):
-        return NativeHlsFD
+        return HlsFD
 
     return PROTOCOL_MAP.get(protocol, HttpFD)
 
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 2bc011266..bb43677b7 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -2,8 +2,12 @@ from __future__ import unicode_literals
 
 import os.path
 import subprocess
+import sys
+import re
 
 from .common import FileDownloader
+from ..postprocessor.ffmpeg import FFmpegPostProcessor
+from ..compat import compat_str
 from ..utils import (
     cli_option,
     cli_valueless_option,
@@ -11,6 +15,7 @@ from ..utils import (
     cli_configuration_args,
     encodeFilename,
     encodeArgument,
+    handle_youtubedl_headers,
 )
 
 
@@ -136,6 +141,66 @@ class HttpieFD(ExternalFD):
             cmd += ['%s:%s' % (key, val)]
         return cmd
 
+
+class FFmpegFD(ExternalFD):
+    @classmethod
+    def supports(cls, info_dict):
+        return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms')
+
+    def _call_downloader(self, tmpfilename, info_dict):
+        url = info_dict['url']
+        ffpp = FFmpegPostProcessor(downloader=self)
+        ffpp.check_version()
+
+        args = [ffpp.executable, '-y']
+
+        start_time = info_dict.get('start_time', 0)
+        if start_time:
+            args += ['-ss', compat_str(start_time)]
+        end_time = info_dict.get('end_time')
+        if end_time:
+            args += ['-t', compat_str(end_time - start_time)]
+
+        if info_dict['http_headers'] and re.match(r'^https?://', url):
+            # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
+            # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
+            headers = handle_youtubedl_headers(info_dict['http_headers'])
+            args += [
+                '-headers',
+                ''.join('%s: %s\r\n' % (key, val) for key, val in headers.items())]
+
+        args += ['-i', url, '-c', 'copy']
+        if info_dict.get('protocol') == 'm3u8':
+            if self.params.get('hls_use_mpegts', False):
+                args += ['-f', 'mpegts']
+            else:
+                args += ['-f', 'mp4', '-bsf:a', 'aac_adtstoasc']
+        else:
+            args += ['-f', info_dict['ext']]
+
+        args = [encodeArgument(opt) for opt in args]
+        args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True))
+
+        self._debug_cmd(args)
+
+        proc = subprocess.Popen(args, stdin=subprocess.PIPE)
+        try:
+            retval = proc.wait()
+        except KeyboardInterrupt:
+            # subprocces.run would send the SIGKILL signal to ffmpeg and the
+            # mp4 file couldn't be played, but if we ask ffmpeg to quit it
+            # produces a file that is playable (this is mostly useful for live
+            # streams). Note that Windows is not affected and produces playable
+            # files (see https://github.com/rg3/youtube-dl/issues/8300).
+            if sys.platform != 'win32':
+                proc.communicate(b'q')
+            raise
+        return retval
+
+
+class AVconvFD(FFmpegFD):
+    pass
+
 _BY_NAME = dict(
     (klass.get_basename(), klass)
     for name, klass in globals().items()
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 2a775bf00..a01dac031 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -1,87 +1,19 @@
 from __future__ import unicode_literals
 
-import os
+import os.path
 import re
-import subprocess
-import sys
 
-from .common import FileDownloader
 from .fragment import FragmentFD
 
 from ..compat import compat_urlparse
-from ..postprocessor.ffmpeg import FFmpegPostProcessor
 from ..utils import (
-    encodeArgument,
     encodeFilename,
     sanitize_open,
-    handle_youtubedl_headers,
 )
 
 
-class HlsFD(FileDownloader):
-    def real_download(self, filename, info_dict):
-        url = info_dict['url']
-        self.report_destination(filename)
-        tmpfilename = self.temp_name(filename)
-
-        ffpp = FFmpegPostProcessor(downloader=self)
-        if not ffpp.available:
-            self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
-            return False
-        ffpp.check_version()
-
-        args = [ffpp.executable, '-y']
-
-        if info_dict['http_headers'] and re.match(r'^https?://', url):
-            # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
-            # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
-            headers = handle_youtubedl_headers(info_dict['http_headers'])
-            args += [
-                '-headers',
-                ''.join('%s: %s\r\n' % (key, val) for key, val in headers.items())]
-
-        args += ['-i', url, '-c', 'copy']
-        if self.params.get('hls_use_mpegts', False):
-            args += ['-f', 'mpegts']
-        else:
-            args += ['-f', 'mp4', '-bsf:a', 'aac_adtstoasc']
-
-        args = [encodeArgument(opt) for opt in args]
-        args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True))
-
-        self._debug_cmd(args)
-
-        proc = subprocess.Popen(args, stdin=subprocess.PIPE)
-        try:
-            retval = proc.wait()
-        except KeyboardInterrupt:
-            # subprocces.run would send the SIGKILL signal to ffmpeg and the
-            # mp4 file couldn't be played, but if we ask ffmpeg to quit it
-            # produces a file that is playable (this is mostly useful for live
-            # streams). Note that Windows is not affected and produces playable
-            # files (see https://github.com/rg3/youtube-dl/issues/8300).
-            if sys.platform != 'win32':
-                proc.communicate(b'q')
-            raise
-        if retval == 0:
-            fsize = os.path.getsize(encodeFilename(tmpfilename))
-            self.to_screen('\r[%s] %s bytes' % (args[0], fsize))
-            self.try_rename(tmpfilename, filename)
-            self._hook_progress({
-                'downloaded_bytes': fsize,
-                'total_bytes': fsize,
-                'filename': filename,
-                'status': 'finished',
-            })
-            return True
-        else:
-            self.to_stderr('\n')
-            self.report_error('%s exited with code %d' % (ffpp.basename, retval))
-            return False
-
-
-class NativeHlsFD(FragmentFD):
-    """ A more limited implementation that does not require ffmpeg """
+class HlsFD(FragmentFD):
+    """ A limited implementation that does not require ffmpeg """
 
     FD_NAME = 'hlsnative'
 
diff --git a/youtube_dl/downloader/rtsp.py b/youtube_dl/downloader/rtsp.py
deleted file mode 100644
index 3eb29526c..000000000
--- a/youtube_dl/downloader/rtsp.py
+++ /dev/null
@@ -1,45 +0,0 @@
-from __future__ import unicode_literals
-
-import os
-import subprocess
-
-from .common import FileDownloader
-from ..utils import (
-    check_executable,
-    encodeFilename,
-)
-
-
-class RtspFD(FileDownloader):
-    def real_download(self, filename, info_dict):
-        url = info_dict['url']
-        self.report_destination(filename)
-        tmpfilename = self.temp_name(filename)
-
-        if check_executable('mplayer', ['-h']):
-            args = [
-                'mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy',
-                '-dumpstream', '-dumpfile', tmpfilename, url]
-        elif check_executable('mpv', ['-h']):
-            args = [
-                'mpv', '-really-quiet', '--vo=null', '--stream-dump=' + tmpfilename, url]
-        else:
-            self.report_error('MMS or RTSP download detected but neither "mplayer" nor "mpv" could be run. Please install any.')
-            return False
-
-        retval = subprocess.call(args)
-        if retval == 0:
-            fsize = os.path.getsize(encodeFilename(tmpfilename))
-            self.to_screen('\r[%s] %s bytes' % (args[0], fsize))
-            self.try_rename(tmpfilename, filename)
-            self._hook_progress({
-                'downloaded_bytes': fsize,
-                'total_bytes': fsize,
-                'filename': filename,
-                'status': 'finished',
-            })
-            return True
-        else:
-            self.to_stderr('\n')
-            self.report_error('%s exited with code %d' % (args[0], retval))
-            return False

From 8765151c8a026f1c57ed239bae15a443339572e3 Mon Sep 17 00:00:00 2001
From: RiCON <RiCON@nowhere>
Date: Fri, 19 Feb 2016 17:52:56 +0000
Subject: [PATCH 0736/1105] [mtv] Extract duration from each playlist item

RSS used instead of manifest files because it's exact to the millisecond
with the video I tested while in manifest it's only exact to the second.
---
 youtube_dl/extractor/mtv.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index e8bb527b8..56ddcbbf3 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -11,6 +11,7 @@ from ..utils import (
     ExtractorError,
     find_xpath_attr,
     fix_xml_ampersands,
+    float_or_none,
     HEADRequest,
     sanitized_Request,
     unescapeHTML,
@@ -158,6 +159,9 @@ class MTVServicesInfoExtractor(InfoExtractor):
         if mtvn_id_node is not None:
             mtvn_id = mtvn_id_node.text
 
+        content_el = find_xpath_attr(itemdoc, self._xpath_ns('.//content', 'http://search.yahoo.com/mrss/'), 'duration')
+        duration = float_or_none(content_el.attrib.get('duration')) if content_el is not None else None
+
         return {
             'title': title,
             'formats': self._extract_video_formats(mediagen_doc, mtvn_id),
@@ -165,6 +169,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
             'id': video_id,
             'thumbnail': self._get_thumbnail_url(uri, itemdoc),
             'description': description,
+            'duration': duration,
         }
 
     def _get_feed_query(self, uri):

From 04cbc4980dcc35ce19c5b75222481f0839b74e98 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 19 Feb 2016 20:56:45 +0100
Subject: [PATCH 0737/1105] [mtv] imporove duration extraction

---
 youtube_dl/extractor/mtv.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 56ddcbbf3..ed068365d 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -111,7 +111,8 @@ class MTVServicesInfoExtractor(InfoExtractor):
         uri = itemdoc.find('guid').text
         video_id = self._id_from_uri(uri)
         self.report_extraction(video_id)
-        mediagen_url = itemdoc.find('%s/%s' % (_media_xml_tag('group'), _media_xml_tag('content'))).attrib['url']
+        content_el = itemdoc.find('%s/%s' % (_media_xml_tag('group'), _media_xml_tag('content')))
+        mediagen_url = content_el.attrib['url']
         # Remove the templates, like &device={device}
         mediagen_url = re.sub(r'&[^=]*?={.*?}(?=(&|$))', '', mediagen_url)
         if 'acceptMethods' not in mediagen_url:
@@ -159,9 +160,6 @@ class MTVServicesInfoExtractor(InfoExtractor):
         if mtvn_id_node is not None:
             mtvn_id = mtvn_id_node.text
 
-        content_el = find_xpath_attr(itemdoc, self._xpath_ns('.//content', 'http://search.yahoo.com/mrss/'), 'duration')
-        duration = float_or_none(content_el.attrib.get('duration')) if content_el is not None else None
-
         return {
             'title': title,
             'formats': self._extract_video_formats(mediagen_doc, mtvn_id),
@@ -169,7 +167,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
             'id': video_id,
             'thumbnail': self._get_thumbnail_url(uri, itemdoc),
             'description': description,
-            'duration': duration,
+            'duration': float_or_none(content_el.attrib.get('duration')),
         }
 
     def _get_feed_query(self, uri):

From 86bf29050e29d14dad894ee4f7ca68a05ebd29bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 20 Feb 2016 03:36:03 +0600
Subject: [PATCH 0738/1105] [test_YoutubeDL] Make test pass until more
 intelligent sort formats (Closes #8462)

---
 test/test_YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 02caf5908..59f7ab49d 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -234,7 +234,7 @@ class TestFormatSelection(unittest.TestCase):
 
     def test_youtube_format_selection(self):
         order = [
-            '38', '37', '46', '22', '45', '35', '44', '18', '34', '43', '6', '5', '36', '17', '13',
+            '38', '37', '46', '22', '45', '35', '44', '18', '34', '43', '6', '5', '17', '36', '13',
             # Apple HTTP Live Streaming
             '96', '95', '94', '93', '92', '132', '151',
             # 3D

From 99cbe98ce8617c119c2fb6a567b0e6ef7eae8859 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 20 Feb 2016 07:58:25 +0100
Subject: [PATCH 0739/1105] [downloader/external] check for external
 downloaders availability

---
 youtube_dl/downloader/__init__.py |  4 ++--
 youtube_dl/downloader/external.py | 21 +++++++++++++++++++++
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index bb6afb1f8..67c2840a5 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -31,13 +31,13 @@ def get_suitable_downloader(info_dict, params={}):
     protocol = determine_protocol(info_dict)
     info_dict['protocol'] = protocol
 
-    if (info_dict.get('start_time') or info_dict.get('end_time')) and FFmpegFD.supports(info_dict):
+    if (info_dict.get('start_time') or info_dict.get('end_time')) and FFmpegFD.available() and FFmpegFD.supports(info_dict):
         return FFmpegFD
 
     external_downloader = params.get('external_downloader')
     if external_downloader is not None:
         ed = get_external_downloader(external_downloader)
-        if ed.supports(info_dict):
+        if ed.available() and ed.supports(info_dict):
             return ed
 
     if protocol == 'm3u8' and params.get('hls_prefer_native'):
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index bb43677b7..edf85483b 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -16,6 +16,7 @@ from ..utils import (
     encodeFilename,
     encodeArgument,
     handle_youtubedl_headers,
+    check_executable,
 )
 
 
@@ -50,6 +51,10 @@ class ExternalFD(FileDownloader):
     def exe(self):
         return self.params.get('external_downloader')
 
+    @classmethod
+    def available(cls):
+        return check_executable(cls.get_basename(), cls.available_opt)
+
     @classmethod
     def supports(cls, info_dict):
         return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
@@ -81,6 +86,8 @@ class ExternalFD(FileDownloader):
 
 
 class CurlFD(ExternalFD):
+    available_opt = ['-V']
+
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '--location', '-o', tmpfilename]
         for key, val in info_dict['http_headers'].items():
@@ -94,6 +101,8 @@ class CurlFD(ExternalFD):
 
 
 class AxelFD(ExternalFD):
+    available_opt = ['-V']
+
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-o', tmpfilename]
         for key, val in info_dict['http_headers'].items():
@@ -104,6 +113,8 @@ class AxelFD(ExternalFD):
 
 
 class WgetFD(ExternalFD):
+    available_opt = ['--version']
+
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
         for key, val in info_dict['http_headers'].items():
@@ -117,6 +128,8 @@ class WgetFD(ExternalFD):
 
 
 class Aria2cFD(ExternalFD):
+    available_opt = ['-v']
+
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-c']
         cmd += self._configuration_args([
@@ -135,6 +148,10 @@ class Aria2cFD(ExternalFD):
 
 
 class HttpieFD(ExternalFD):
+    @classmethod
+    def available(cls):
+        return check_executable('http', ['--version'])
+
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = ['http', '--download', '--output', tmpfilename, info_dict['url']]
         for key, val in info_dict['http_headers'].items():
@@ -147,6 +164,10 @@ class FFmpegFD(ExternalFD):
     def supports(cls, info_dict):
         return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms')
 
+    @classmethod
+    def available(cls):
+        return FFmpegPostProcessor().available
+
     def _call_downloader(self, tmpfilename, info_dict):
         url = info_dict['url']
         ffpp = FFmpegPostProcessor(downloader=self)

From f34294fa0c0097cea7f6388d5d691d5a54950491 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 20 Feb 2016 08:06:12 +0100
Subject: [PATCH 0740/1105] [downloader/external:ffmpegfd] check for None value
 of start_time

---
 youtube_dl/downloader/external.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index edf85483b..a4fdf1af8 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -175,7 +175,7 @@ class FFmpegFD(ExternalFD):
 
         args = [ffpp.executable, '-y']
 
-        start_time = info_dict.get('start_time', 0)
+        start_time = info_dict.get('start_time') or 0
         if start_time:
             args += ['-ss', compat_str(start_time)]
         end_time = info_dict.get('end_time')

From 958759f44b3e7c37509394568d9764e2794f713a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 20 Feb 2016 15:54:00 +0800
Subject: [PATCH 0741/1105] [appletrailers] Extend _VALID_URL (#8524)

---
 youtube_dl/extractor/appletrailers.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 62ed0c918..be40f85b4 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class AppleTrailersIE(InfoExtractor):
     IE_NAME = 'appletrailers'
-    _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/(?:trailers|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.|movie)?trailers\.apple\.com/(?:trailers|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'
     _TESTS = [{
         'url': 'http://trailers.apple.com/trailers/wb/manofsteel/',
         'info_dict': {
@@ -73,6 +73,9 @@ class AppleTrailersIE(InfoExtractor):
     }, {
         'url': 'http://trailers.apple.com/ca/metropole/autrui/',
         'only_matching': True,
+    }, {
+        'url': 'http://movietrailers.apple.com/trailers/focus_features/kuboandthetwostrings/',
+        'only_matching': True,
     }]
 
     _JSON_RE = r'iTunes.playURL\((.*?)\);'

From 5bc880b9887c9689fef171683eae279adbbfc186 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 17 Feb 2016 06:01:44 +0800
Subject: [PATCH 0742/1105] [utils] Add OHDave's RSA encryption function

---
 test/test_utils.py  |  8 ++++++++
 youtube_dl/utils.py | 18 ++++++++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index a1e416dd5..e6887be9f 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -35,6 +35,7 @@ from youtube_dl.utils import (
     is_html,
     js_to_json,
     limit_length,
+    ohdave_rsa_encrypt,
     OnDemandPagedList,
     orderedSet,
     parse_duration,
@@ -792,6 +793,13 @@ The first line
                 {'nocheckcertificate': False}, '--check-certificate', 'nocheckcertificate', 'false', 'true', '='),
             ['--check-certificate=true'])
 
+    def test_ohdave_rsa_encrypt(self):
+        N = 0xab86b6371b5318aaa1d3c9e612a9f1264f372323c8c0f19875b5fc3b3fd3afcc1e5bec527aa94bfa85bffc157e4245aebda05389a5357b75115ac94f074aefcd
+        e = 65537
+
+        self.assertEqual(
+            ohdave_rsa_encrypt(b'aa111222', e, N),
+            '726664bd9a23fd0c70f9f1b84aab5e3905ce1e45a584e9cbcf9bcc7510338fc1986d6c599ff990d923aa43c51c0d9013cd572e13bc58f4ae48f2ed8c0b0ba881')
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 672ce05ea..7ce661b09 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -4,6 +4,7 @@
 from __future__ import unicode_literals
 
 import base64
+import binascii
 import calendar
 import codecs
 import contextlib
@@ -2582,3 +2583,20 @@ class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
             return None  # No Proxy
         return compat_urllib_request.ProxyHandler.proxy_open(
             self, req, proxy, type)
+
+
+def ohdave_rsa_encrypt(data, exponent, modulus):
+    '''
+    Implement OHDave's RSA algorithm. See http://www.ohdave.com/rsa/
+
+    Input:
+        data: data to encrypt, bytes-like object
+        exponent, modulus: parameter e and N of RSA algorithm, both integer
+    Output: hex string of encrypted data
+
+    Limitation: supports one block encryption only
+    '''
+
+    payload = int(binascii.hexlify(data[::-1]), 16)
+    encrypted = pow(payload, exponent, modulus)
+    return '%x' % encrypted

From 99709cc3f121010421360ba94d75e16ce15f0be5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 17 Feb 2016 06:26:27 +0800
Subject: [PATCH 0743/1105] [iqiyi] Implement _login()

Currently only email login supported
---
 youtube_dl/extractor/iqiyi.py | 236 +++++++++++++++++++++++++++++++++-
 1 file changed, 230 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 9046705a5..541fe7ae3 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -5,29 +5,199 @@ import hashlib
 import math
 import os
 import random
+import re
 import time
 import uuid
 
 from .common import InfoExtractor
 from ..compat import (
     compat_parse_qs,
+    compat_str,
     compat_urllib_parse,
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
     ExtractorError,
+    ohdave_rsa_encrypt,
     sanitized_Request,
     urlencode_postdata,
     url_basename,
 )
 
 
+def md5_text(text):
+    return hashlib.md5(text.encode('utf-8')).hexdigest()
+
+
+class IqiyiSDK(object):
+    def __init__(self, target, ip, timestamp):
+        self.target = target
+        self.ip = ip
+        self.timestamp = timestamp
+
+    @staticmethod
+    def split_sum(data):
+        return compat_str(sum(map(lambda p: int(p, 16), list(data))))
+
+    @staticmethod
+    def digit_sum(num):
+        if isinstance(num, int):
+            num = compat_str(num)
+        return compat_str(sum(map(int, num)))
+
+    def even_odd(self):
+        even = self.digit_sum(compat_str(self.timestamp)[::2])
+        odd = self.digit_sum(compat_str(self.timestamp)[1::2])
+        return even, odd
+
+    def preprocess(self, chunksize):
+        self.target = md5_text(self.target)
+        chunks = []
+        for i in range(32 // chunksize):
+            chunks.append(self.target[chunksize * i:chunksize * (i + 1)])
+        if 32 % chunksize:
+            chunks.append(self.target[32 - 32 % chunksize:])
+        return chunks, list(map(int, self.ip.split('.')))
+
+    def mod(self, modulus):
+        chunks, ip = self.preprocess(32)
+        self.target = chunks[0] + ''.join(map(lambda p: compat_str(p % modulus), ip))
+
+    def split(self, chunksize):
+        modulus_map = {
+            4: 256,
+            5: 10,
+            8: 100,
+        }
+
+        chunks, ip = self.preprocess(chunksize)
+        ret = ''
+        for i in range(len(chunks)):
+            ip_part = compat_str(ip[i] % modulus_map[chunksize]) if i < 4 else ''
+            if chunksize == 8:
+                ret += ip_part + chunks[i]
+            else:
+                ret += chunks[i] + ip_part
+        self.target = ret
+
+    def handle_input16(self):
+        self.target = md5_text(self.target)
+        self.target = self.split_sum(self.target[:16]) + self.target + self.split_sum(self.target[16:])
+
+    def handle_input8(self):
+        self.target = md5_text(self.target)
+        ret = ''
+        for i in range(4):
+            part = self.target[8 * i:8 * (i + 1)]
+            ret += self.split_sum(part) + part
+        self.target = ret
+
+    def handleSum(self):
+        self.target = md5_text(self.target)
+        self.target = self.split_sum(self.target) + self.target
+
+    def date(self, scheme):
+        self.target = md5_text(self.target)
+        d = time.localtime(self.timestamp)
+        strings = {
+            'y': compat_str(d.tm_year),
+            'm': '%02d' % d.tm_mon,
+            'd': '%02d' % d.tm_mday,
+        }
+        self.target += ''.join(map(lambda c: strings[c], list(scheme)))
+
+    def split_time_even_odd(self):
+        even, odd = self.even_odd()
+        self.target = odd + md5_text(self.target) + even
+
+    def split_time_odd_even(self):
+        even, odd = self.even_odd()
+        self.target = even + md5_text(self.target) + odd
+
+    def split_ip_time_sum(self):
+        chunks, ip = self.preprocess(32)
+        self.target = compat_str(sum(ip)) + chunks[0] + self.digit_sum(self.timestamp)
+
+    def split_time_ip_sum(self):
+        chunks, ip = self.preprocess(32)
+        self.target = self.digit_sum(self.timestamp) + chunks[0] + compat_str(sum(ip))
+
+
+class IqiyiSDKInterpreter(object):
+    BASE62_TABLE = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
+
+    def __init__(self, sdk_code):
+        self.sdk_code = sdk_code
+
+    @classmethod
+    def base62(cls, num):
+        if num == 0:
+            return '0'
+        ret = ''
+        while num:
+            ret = cls.BASE62_TABLE[num % 62] + ret
+            num = num // 62
+        return ret
+
+    def decode_eval_codes(self):
+        self.sdk_code = self.sdk_code[5:-3]
+
+        mobj = re.search(
+            r"'([^']+)',62,(\d+),'([^']+)'\.split\('\|'\),[^,]+,{}",
+            self.sdk_code)
+        obfucasted_code, count, symbols = mobj.groups()
+        count = int(count)
+        symbols = symbols.split('|')
+        symbol_table = {}
+
+        while count:
+            count -= 1
+            b62count = self.base62(count)
+            symbol_table[b62count] = symbols[count] or b62count
+
+        self.sdk_code = re.sub(
+            r'\b(\w+)\b', lambda mobj: symbol_table[mobj.group(0)],
+            obfucasted_code)
+
+    def run(self, target, ip, timestamp):
+        self.decode_eval_codes()
+
+        functions = re.findall(r'input=([a-zA-Z0-9]+)\(input', self.sdk_code)
+
+        sdk = IqiyiSDK(target, ip, timestamp)
+
+        other_functions = {
+            'handleSum': sdk.handleSum,
+            'handleInput8': sdk.handle_input8,
+            'handleInput16': sdk.handle_input16,
+            'splitTimeEvenOdd': sdk.split_time_even_odd,
+            'splitTimeOddEven': sdk.split_time_odd_even,
+            'splitIpTimeSum': sdk.split_ip_time_sum,
+            'splitTimeIpSum': sdk.split_time_ip_sum,
+        }
+        for function in functions:
+            if re.match(r'mod\d+', function):
+                sdk.mod(int(function[3:]))
+            elif re.match(r'date[ymd]{3}', function):
+                sdk.date(function[4:])
+            elif re.match(r'split\d+', function):
+                sdk.split(int(function[5:]))
+            elif function in other_functions:
+                other_functions[function]()
+            else:
+                raise ExtractorError('Unknown funcion %s' % function)
+
+        return sdk.target
+
+
 class IqiyiIE(InfoExtractor):
     IE_NAME = 'iqiyi'
     IE_DESC = '爱奇艺'
 
     _VALID_URL = r'http://(?:[^.]+\.)?iqiyi\.com/.+\.html'
 
+    _NETRC_MACHINE = 'iqiyi'
+
     _TESTS = [{
         'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
         'md5': '2cb594dc2781e6c941a110d8f358118b',
@@ -136,9 +306,63 @@ class IqiyiIE(InfoExtractor):
         ('10', 'h1'),
     ]
 
+    def _real_initialize(self):
+        self._login()
+
     @staticmethod
-    def md5_text(text):
-        return hashlib.md5(text.encode('utf-8')).hexdigest()
+    def _rsa_fun(data):
+        # public key extracted from http://static.iqiyi.com/js/qiyiV2/20160129180840/jobs/i18n/i18nIndex.js
+        N = 0xab86b6371b5318aaa1d3c9e612a9f1264f372323c8c0f19875b5fc3b3fd3afcc1e5bec527aa94bfa85bffc157e4245aebda05389a5357b75115ac94f074aefcd
+        e = 65537
+
+        return ohdave_rsa_encrypt(data, e, N)
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+
+        # No authentication to be performed
+        if not username:
+            return True
+
+        data = self._download_json(
+            'http://kylin.iqiyi.com/get_token', None,
+            note='Get token for logging', errnote='Unable to get token for logging')
+        sdk = data['sdk']
+        timestamp = int(time.time())
+        target = '/apis/reglogin/login.action?lang=zh_TW&area_code=null&email=%s&passwd=%s&agenttype=1&from=undefined&keeplogin=0&piccode=&fromurl=&_pos=1' % (
+            username, self._rsa_fun(password.encode('utf-8')))
+
+        interp = IqiyiSDKInterpreter(sdk)
+        sign = interp.run(target, data['ip'], timestamp)
+
+        validation_params = {
+            'target': target,
+            'server': 'BEA3AA1908656AABCCFF76582C4C6660',
+            'token': data['token'],
+            'bird_src': 'f8d91d57af224da7893dd397d52d811a',
+            'sign': sign,
+            'bird_t': timestamp,
+        }
+        validation_result = self._download_json(
+            'http://kylin.iqiyi.com/validate?' + compat_urllib_parse.urlencode(validation_params), None,
+            note='Validate credentials', errnote='Unable to validate credentials')
+
+        MSG_MAP = {
+            'P00107': 'please login via the web interface and enter the CAPTCHA code',
+            'P00117': 'bad username or password',
+        }
+
+        code = validation_result['code']
+        if code != 'A00000':
+            msg = MSG_MAP.get(code)
+            if not msg:
+                msg = 'error %s' % code
+                if validation_result.get('msg'):
+                    msg += ': ' + validation_result['msg']
+            self._downloader.report_warning('unable to log in: ' + msg)
+            return False
+
+        return True
 
     def _authenticate_vip_video(self, api_video_url, video_id, tvid, _uuid, do_report_warning):
         auth_params = {
@@ -199,7 +423,7 @@ class IqiyiIE(InfoExtractor):
                 note='Download path key of segment %d for format %s' % (segment_index + 1, format_id)
             )['t']
             t = str(int(math.floor(int(tm) / (600.0))))
-            return self.md5_text(t + mg + x)
+            return md5_text(t + mg + x)
 
         video_urls_dict = {}
         need_vip_warning_report = True
@@ -278,16 +502,16 @@ class IqiyiIE(InfoExtractor):
         tail = tm + tvid
         param = {
             'key': 'fvip',
-            'src': self.md5_text('youtube-dl'),
+            'src': md5_text('youtube-dl'),
             'tvId': tvid,
             'vid': video_id,
             'vinfo': 1,
             'tm': tm,
-            'enc': self.md5_text(enc_key + tail),
+            'enc': md5_text(enc_key + tail),
             'qyid': _uuid,
             'tn': random.random(),
             'um': 0,
-            'authkey': self.md5_text(self.md5_text('') + tail),
+            'authkey': md5_text(md5_text('') + tail),
             'k_tag': 1,
         }
 

From 1d14c75f55094484127dda641465b5333296d497 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 20 Feb 2016 20:49:30 +0800
Subject: [PATCH 0744/1105] [Makefile] iQiyi login test requires network

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index f826c1685..cb449b7e6 100644
--- a/Makefile
+++ b/Makefile
@@ -44,7 +44,7 @@ test:
 ot: offlinetest
 
 offlinetest: codetest
-	nosetests --verbose test --exclude test_download.py --exclude test_age_restriction.py --exclude test_subtitles.py --exclude test_write_annotations.py --exclude test_youtube_lists.py
+	nosetests --verbose test --exclude test_download.py --exclude test_age_restriction.py --exclude test_subtitles.py --exclude test_write_annotations.py --exclude test_youtube_lists.py --exclude test_iqiyi_sdk_interpreter.py
 
 tar: youtube-dl.tar.gz
 

From 9c86d509162fcf55eb328acb1c8d54b0de97e894 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 20 Feb 2016 14:11:44 +0100
Subject: [PATCH 0745/1105] [faz] Future-proof XML element check

---
 youtube_dl/extractor/faz.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/faz.py b/youtube_dl/extractor/faz.py
index 6f9b003c2..fd535457d 100644
--- a/youtube_dl/extractor/faz.py
+++ b/youtube_dl/extractor/faz.py
@@ -52,7 +52,7 @@ class FazIE(InfoExtractor):
         formats = []
         for pref, code in enumerate(['LOW', 'HIGH', 'HQ']):
             encoding = xpath_element(encodings, code)
-            if encoding:
+            if encoding is not None:
                 encoding_url = xpath_text(encoding, 'FILENAME')
                 if encoding_url:
                     formats.append({

From 73f9c2867df5048e78c6f60d2fc1d9691edb4b2d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 20 Feb 2016 22:44:04 +0800
Subject: [PATCH 0746/1105] [iqiyi] Support playlists (closes #8019)

---
 youtube_dl/extractor/iqiyi.py | 49 +++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 541fe7ae3..c3e33009a 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import hashlib
+import itertools
 import math
 import os
 import random
@@ -19,6 +20,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     ohdave_rsa_encrypt,
+    remove_start,
     sanitized_Request,
     urlencode_postdata,
     url_basename,
@@ -295,6 +297,13 @@ class IqiyiIE(InfoExtractor):
             },
         }],
         'expected_warnings': ['Needs a VIP account for full video'],
+    }, {
+        'url': 'http://www.iqiyi.com/a_19rrhb8ce1.html',
+        'info_dict': {
+            'id': '202918101',
+            'title': '灌篮高手 国语版',
+        },
+        'playlist_count': 101,
     }]
 
     _FORMATS_MAP = [
@@ -526,9 +535,49 @@ class IqiyiIE(InfoExtractor):
         enc_key = '6ab6d0280511493ba85594779759d4ed'
         return enc_key
 
+    def _extract_playlist(self, webpage):
+        PAGE_SIZE = 50
+
+        links = re.findall(
+            r'<a[^>]+class="site-piclist_pic_link"[^>]+href="(http://www\.iqiyi\.com/.+\.html)"',
+            webpage)
+        if not links:
+            return
+
+        album_id = self._search_regex(
+            r'albumId\s*:\s*(\d+),', webpage, 'album ID')
+        album_title = self._search_regex(
+            r'data-share-title="([^"]+)"', webpage, 'album title', fatal=False)
+
+        entries = list(map(self.url_result, links))
+
+        # Start from 2 because links in the first page are already on webpage
+        for page_num in itertools.count(2):
+            pagelist_page = self._download_webpage(
+                'http://cache.video.qiyi.com/jp/avlist/%s/%d/%d/' % (album_id, page_num, PAGE_SIZE),
+                album_id,
+                note='Download playlist page %d' % page_num,
+                errnote='Failed to download playlist page %d' % page_num)
+            pagelist = self._parse_json(
+                remove_start(pagelist_page, 'var tvInfoJs='), album_id)
+            vlist = pagelist['data']['vlist']
+            for item in vlist:
+                entries.append(self.url_result(item['vurl']))
+            if len(vlist) < PAGE_SIZE:
+                break
+
+        return self.playlist_result(entries, album_id, album_title)
+
     def _real_extract(self, url):
         webpage = self._download_webpage(
             url, 'temp_id', note='download video page')
+
+        # There's no simple way to determine whether an URL is a playlist or not
+        # So detect it
+        playlist_result = self._extract_playlist(webpage)
+        if playlist_result:
+            return playlist_result
+
         tvid = self._search_regex(
             r'data-player-tvid\s*=\s*[\'"](\d+)', webpage, 'tvid')
         video_id = self._search_regex(

From a4e4d7dfcd03598c8b0924d4973b84ad30bc944e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 20 Feb 2016 23:10:39 +0800
Subject: [PATCH 0747/1105] [test_iqiyi_sdk_interpreter] Add test for iQiyi
 login

---
 test/test_iqiyi_sdk_interpreter.py | 47 ++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)
 create mode 100644 test/test_iqiyi_sdk_interpreter.py

diff --git a/test/test_iqiyi_sdk_interpreter.py b/test/test_iqiyi_sdk_interpreter.py
new file mode 100644
index 000000000..9d95cb606
--- /dev/null
+++ b/test/test_iqiyi_sdk_interpreter.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from test.helper import FakeYDL
+from youtube_dl.extractor import IqiyiIE
+
+
+class IqiyiIEWithCredentials(IqiyiIE):
+    def _get_login_info(self):
+        return 'foo', 'bar'
+
+
+class WarningLogger(object):
+    def __init__(self):
+        self.messages = []
+
+    def warning(self, msg):
+        self.messages.append(msg)
+
+    def debug(self, msg):
+        pass
+
+    def error(self, msg):
+        pass
+
+
+class TestIqiyiSDKInterpreter(unittest.TestCase):
+    def test_iqiyi_sdk_interpreter(self):
+        '''
+        Test the functionality of IqiyiSDKInterpreter by trying to log in
+
+        If `sign` is incorrect, /validate call throws an HTTP 556 error
+        '''
+        logger = WarningLogger()
+        ie = IqiyiIEWithCredentials(FakeYDL({'logger': logger}))
+        ie._login()
+        self.assertTrue('unable to log in:' in logger.messages[0])
+
+if __name__ == '__main__':
+    unittest.main()

From 765ac263dbf17442f41715a1cfce13fc61a4b315 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 20 Feb 2016 19:55:10 +0100
Subject: [PATCH 0748/1105] [utils] mimetype2ext: return 'm4a' for 'audio/mp4'
 (fixes #8620)

The youtube extractor was using 'mp4' for them, therefore filters like 'bestaudio[ext=m4a]' stopped working (94278f720272c5ad2cd5900f59f8e71f31d46633 broke it).
---
 youtube_dl/utils.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7ce661b09..a2c6780ca 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1836,6 +1836,12 @@ def error_to_compat_str(err):
 
 
 def mimetype2ext(mt):
+    ext = {
+        'audio/mp4': 'm4a',
+    }.get(mt)
+    if ext is not None:
+        return ext
+
     _, _, res = mt.rpartition('/')
 
     return {

From 399a76e67bca0beb4849ea90c4f40803fbd06ed3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Feb 2016 03:28:25 +0800
Subject: [PATCH 0749/1105] [utils] Jython support: tolerate missing fcntl
 module

---
 youtube_dl/utils.py | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a2c6780ca..17747be26 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1217,13 +1217,23 @@ if sys.platform == 'win32':
             raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
 
 else:
-    import fcntl
+    # Some platforms, such as Jython, is missing fcntl
+    try:
+        import fcntl
 
-    def _lock_file(f, exclusive):
-        fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
+        def _lock_file(f, exclusive):
+            fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
 
-    def _unlock_file(f):
-        fcntl.flock(f, fcntl.LOCK_UN)
+        def _unlock_file(f):
+            fcntl.flock(f, fcntl.LOCK_UN)
+    except ImportError:
+        UNSUPPORTED_MSG = 'file locking is not supported on this platform'
+
+        def _lock_file(f, exclusive):
+            raise IOError(UNSUPPORTED_MSG)
+
+        def _unlock_file(f):
+            raise IOError(UNSUPPORTED_MSG)
 
 
 class locked_file(object):

From c1c05c67ea6087c3b0190c9f16cb9fdd8160e398 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Feb 2016 03:29:02 +0800
Subject: [PATCH 0750/1105] [utils] Jython support - disable setproctitle()
 until ctypes is complete

---
 youtube_dl/utils.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 17747be26..16b4324a4 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1397,6 +1397,12 @@ def fix_xml_ampersands(xml_str):
 
 def setproctitle(title):
     assert isinstance(title, compat_str)
+
+    # ctypes in Jython is not complete
+    # http://bugs.jython.org/issue2148
+    if sys.platform.startswith('java'):
+        return
+
     try:
         libc = ctypes.cdll.LoadLibrary('libc.so.6')
     except OSError:

From 7360db05b43741c2dfa1fd024e9c2f013ed97c9e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Feb 2016 03:25:44 +0800
Subject: [PATCH 0751/1105] [postprocessor/embedthumbnail] Allow mkv to embed
 thumbnails

Fixes #6046
---
 youtube_dl/postprocessor/embedthumbnail.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index e19dbf73d..3bad5a266 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -40,7 +40,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
                 'Skipping embedding the thumbnail because the file is missing.')
             return [], info
 
-        if info['ext'] == 'mp3':
+        if info['ext'] in ('mp3', 'mkv'):
             options = [
                 '-c', 'copy', '-map', '0', '-map', '1',
                 '-metadata:s:v', 'title="Album cover"', '-metadata:s:v', 'comment="Cover (Front)"']

From 101067de12e193c8ad42e1f474c9018eedadec9d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Feb 2016 03:29:49 +0800
Subject: [PATCH 0752/1105] Jython support - handle *.class files

---
 .gitignore | 3 ++-
 Makefile   | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 0422adf44..26dbde73d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 *.pyc
 *.pyo
+*.class
 *~
 *.DS_Store
 wine-py2exe/
@@ -32,4 +33,4 @@ test/testdata
 .tox
 youtube-dl.zsh
 .idea
-.idea/*
\ No newline at end of file
+.idea/*
diff --git a/Makefile b/Makefile
index cb449b7e6..afd7f4032 100644
--- a/Makefile
+++ b/Makefile
@@ -3,6 +3,7 @@ all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bas
 clean:
 	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe
 	find . -name "*.pyc" -delete
+	find . -name "*.class" -delete
 
 PREFIX ?= /usr/local
 BINDIR ?= $(PREFIX)/bin

From cafcf657a4e16f0bcf90195f8e35bfcce7faad8e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 20 Feb 2016 22:02:03 +0100
Subject: [PATCH 0753/1105] add more subtitles mime types to mimetype2ext and
 fix the platform subtitle extraction

---
 youtube_dl/extractor/common.py      | 12 ++----------
 youtube_dl/extractor/theplatform.py |  3 ++-
 youtube_dl/utils.py                 |  4 ++++
 3 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f411ea763..c85dcda0c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -46,6 +46,7 @@ from ..utils import (
     xpath_with_ns,
     determine_protocol,
     parse_duration,
+    mimetype2ext,
 )
 
 
@@ -1277,16 +1278,7 @@ class InfoExtractor(object):
             if not src or src in urls:
                 continue
             urls.append(src)
-            ext = textstream.get('ext') or determine_ext(src)
-            if not ext:
-                type_ = textstream.get('type')
-                SUBTITLES_TYPES = {
-                    'text/vtt': 'vtt',
-                    'text/srt': 'srt',
-                    'application/smptett+xml': 'tt',
-                }
-                if type_ in SUBTITLES_TYPES:
-                    ext = SUBTITLES_TYPES[type_]
+            ext = textstream.get('ext') or determine_ext(src) or mimetype2ext(textstream.get('type'))
             lang = textstream.get('systemLanguage') or textstream.get('systemLanguageName') or textstream.get('lang') or subtitles_lang
             subtitles.setdefault(lang, []).append({
                 'url': src,
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 755f816ff..93d871571 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -21,6 +21,7 @@ from ..utils import (
     sanitized_Request,
     unsmuggle_url,
     xpath_with_ns,
+    mimetype2ext,
 )
 
 default_ns = 'http://www.w3.org/2005/SMIL21/Language'
@@ -68,7 +69,7 @@ class ThePlatformBaseIE(InfoExtractor):
             for caption in captions:
                 lang, src, mime = caption.get('lang', 'en'), caption.get('src'), caption.get('type')
                 subtitles[lang] = [{
-                    'ext': 'srt' if mime == 'text/srt' else 'ttml',
+                    'ext': mimetype2ext(mime),
                     'url': src,
                 }]
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a2c6780ca..6978a10e4 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1846,7 +1846,11 @@ def mimetype2ext(mt):
 
     return {
         '3gpp': '3gp',
+        'smptett+xml': 'tt',
+        'srt': 'srt',
+        'ttaf+xml': 'dfxp',
         'ttml+xml': 'ttml',
+        'vtt': 'vtt',
         'x-flv': 'flv',
         'x-mp4-fragmented': 'mp4',
         'x-ms-wmv': 'wmv',

From 33d4fdabfa0a2e008fe2765ae7b81f9f736c2163 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Feb 2016 09:51:54 +0600
Subject: [PATCH 0754/1105] [extractor/generic] Add support for ok embeds
 (#8619)

---
 youtube_dl/extractor/generic.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 45adbb7a3..c6bf8d270 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1573,6 +1573,11 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'VK')
 
+        # Look for embedded Odnoklassniki player
+        mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:odnoklassniki|ok)\.ru/videoembed/.+?)\1', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'), 'Odnoklassniki')
+
         # Look for embedded ivi player
         mobj = re.search(r'<embed[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?ivi\.ru/video/player.+?)\1', webpage)
         if mobj is not None:

From b72f6163dc264218c4abefebefd1971875fff7d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Feb 2016 13:37:31 +0600
Subject: [PATCH 0755/1105] [arte:+7] Improve _VALID_URL

---
 youtube_dl/extractor/arte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 793da2ee1..f4d033e70 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -63,7 +63,7 @@ class ArteTvIE(InfoExtractor):
 
 class ArteTVPlus7IE(InfoExtractor):
     IE_NAME = 'arte.tv:+7'
-    _VALID_URL = r'https?://(?:www\.)?arte\.tv/guide/(?P<lang>fr|de|en|es)/(?:(?:sendungen|emissions)/)?(?P<id>[^/]+)/(?P<name>[^/?#&+])'
+    _VALID_URL = r'https?://(?:www\.)?arte\.tv/guide/(?P<lang>fr|de|en|es)/(?:(?:sendungen|emissions|embed)/)?(?P<id>[^/]+)/(?P<name>[^/?#&+])'
 
     @classmethod
     def _extract_url_info(cls, url):

From 305168ca3ee869aed5e95212ed529bf0e6e2f37c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Feb 2016 13:55:25 +0600
Subject: [PATCH 0756/1105] [arte:+7] Detect more embeds (Closes #8613)

---
 youtube_dl/extractor/arte.py | 40 ++++++++++++++++++++++++++++--------
 1 file changed, 31 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index f4d033e70..a4c786d78 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -110,15 +110,25 @@ class ArteTVPlus7IE(InfoExtractor):
             # en and es URLs produce react-based pages with different layout (e.g.
             # http://www.arte.tv/guide/en/053330-002-A/carnival-italy?zone=world)
             if not iframe_url:
-                embed_html = self._parse_json(
-                    self._search_regex(
-                        r'program\s*:\s*({.+?["\']embed_html["\'].+?}),?\s*\n',
-                        webpage, 'program'),
-                    video_id)['embed_html']
-                iframe_url = find_iframe_url(embed_html)
-            json_url = compat_parse_qs(
-                compat_urllib_parse_urlparse(iframe_url).query)['json_url'][0]
-        return self._extract_from_json_url(json_url, video_id, lang)
+                program = self._search_regex(
+                    r'program\s*:\s*({.+?["\']embed_html["\'].+?}),?\s*\n',
+                    webpage, 'program', default=None)
+                if program:
+                    embed_html = self._parse_json(program,video_id)
+                    if embed_html:
+                        iframe_url = find_iframe_url(embed_html['embed_html'])
+            if iframe_url:
+                json_url = compat_parse_qs(
+                    compat_urllib_parse_urlparse(iframe_url).query)['json_url'][0]
+        if json_url:
+            return self._extract_from_json_url(json_url, video_id, lang)
+        # Differend kind of embed URL (e.g.
+        # http://www.arte.tv/magazine/trepalium/fr/episode-0406-replay-trepalium)
+        embed_url = self._search_regex(
+            r'<iframe[^>]+src=(["\'])(?P<url>.+?)\1',
+            webpage, 'embed url', group='url')
+        return self.url_result(embed_url)
+
 
     def _extract_from_json_url(self, json_url, video_id, lang):
         info = self._download_json(json_url, video_id)
@@ -294,6 +304,7 @@ class ArteTVMagazineIE(ArteTVPlus7IE):
     _VALID_URL = r'https?://(?:www\.)?arte\.tv/magazine/[^/]+/(?P<lang>fr|de|en|es)/(?P<id>[^/?#&]+)'
 
     _TESTS = [{
+        # Embedded via <iframe src="http://www.arte.tv/arte_vp/index.php?json_url=..."
         'url': 'http://www.arte.tv/magazine/trepalium/fr/entretien-avec-le-realisateur-vincent-lannoo-trepalium',
         'md5': '66a093339c1278bb3719157ef07107b2',
         'info_dict': {
@@ -301,6 +312,17 @@ class ArteTVMagazineIE(ArteTVPlus7IE):
             'ext': 'mp4',
             'title': 'Trepalium - Extrait Ep.01',
         },
+    }, {
+        # Embedded via <iframe src="http://www.arte.tv/guide/fr/embed/054813-004-A/medium"
+        'url': 'http://www.arte.tv/magazine/trepalium/fr/episode-0406-replay-trepalium',
+        'md5': 'fedc64fc7a946110fe311634e79782ca',
+        'info_dict': {
+            'id': '054813-004_PLUS7-F',
+            'ext': 'mp4',
+            'title': 'Trepalium (4/6)',
+            'description': 'md5:10057003c34d54e95350be4f9b05cb40',
+            'upload_date': '20160218',
+        },
     }, {
         'url': 'http://www.arte.tv/magazine/metropolis/de/frank-woeste-german-paris-metropolis',
         'only_matching': True,

From 0dd046c16c6829a545177fe4bb10828372c91699 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Feb 2016 13:57:30 +0600
Subject: [PATCH 0757/1105] [arte:magazine] Fix test

---
 youtube_dl/extractor/arte.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index a4c786d78..9a6f49329 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -306,11 +306,12 @@ class ArteTVMagazineIE(ArteTVPlus7IE):
     _TESTS = [{
         # Embedded via <iframe src="http://www.arte.tv/arte_vp/index.php?json_url=..."
         'url': 'http://www.arte.tv/magazine/trepalium/fr/entretien-avec-le-realisateur-vincent-lannoo-trepalium',
-        'md5': '66a093339c1278bb3719157ef07107b2',
+        'md5': '2a9369bcccf847d1c741e51416299f25',
         'info_dict': {
             'id': '065965-000-A',
             'ext': 'mp4',
             'title': 'Trepalium - Extrait Ep.01',
+            'upload_date': '20160121',
         },
     }, {
         # Embedded via <iframe src="http://www.arte.tv/guide/fr/embed/054813-004-A/medium"

From 92c5e11b40c235caee69e92806001aa7899e10eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Feb 2016 14:23:58 +0600
Subject: [PATCH 0758/1105] [arte:future] Fix test

---
 youtube_dl/extractor/arte.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 9a6f49329..47840cad0 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -240,6 +240,7 @@ class ArteTVFutureIE(ArteTVPlus7IE):
             'id': '050940-028-A',
             'ext': 'mp4',
             'title': 'Les écrevisses aussi peuvent être anxieuses',
+            'upload_date': '20140902',
         },
     }, {
         'url': 'http://future.arte.tv/fr/la-science-est-elle-responsable',

From 445d72b8b5cdce331833646ba856efe946253056 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Feb 2016 16:41:24 +0800
Subject: [PATCH 0759/1105] [twitter:amplify] Add TwitterAmplifyIE for handling
 Twitter smart URLs

Closes #8075
---
 youtube_dl/extractor/__init__.py |  6 ++++-
 youtube_dl/extractor/twitter.py  | 41 ++++++++++++++++++++++++++++----
 2 files changed, 42 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1edbfbd28..1ae606f1e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -798,7 +798,11 @@ from .twitch import (
     TwitchBookmarksIE,
     TwitchStreamIE,
 )
-from .twitter import TwitterCardIE, TwitterIE
+from .twitter import (
+    TwitterCardIE,
+    TwitterIE,
+    TwitterAmplifyIE,
+)
 from .ubu import UbuIE
 from .udemy import (
     UdemyIE,
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index a161f046b..77c9176e4 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -14,7 +14,13 @@ from ..utils import (
 )
 
 
-class TwitterCardIE(InfoExtractor):
+class TwitterBaseIE(InfoExtractor):
+    def _get_vmap_video_url(self, vmap_url, video_id):
+        vmap_data = self._download_xml(vmap_url, video_id)
+        return xpath_text(vmap_data, './/MediaFile').strip()
+
+
+class TwitterCardIE(TwitterBaseIE):
     IE_NAME = 'twitter:card'
     _VALID_URL = r'https?://(?:www\.)?twitter\.com/i/cards/tfw/v1/(?P<id>\d+)'
     _TESTS = [
@@ -96,10 +102,8 @@ class TwitterCardIE(InfoExtractor):
                 video_id)
             if 'playlist' not in config:
                 if 'vmapUrl' in config:
-                    vmap_data = self._download_xml(config['vmapUrl'], video_id)
-                    video_url = xpath_text(vmap_data, './/MediaFile').strip()
                     formats.append({
-                        'url': video_url,
+                        'url': self._get_vmap_video_url(config['vmapUrl'], video_id),
                     })
                     break   # same video regardless of UA
                 continue
@@ -226,3 +230,32 @@ class TwitterIE(InfoExtractor):
             return info
 
         raise ExtractorError('There\'s not video in this tweet.')
+
+
+class TwitterAmplifyIE(TwitterBaseIE):
+    IE_NAME = 'twitter:amplify'
+    _VALID_URL = 'https?://amp\.twimg\.com/v/(?P<id>[0-9a-f\-]{36})'
+
+    _TEST = {
+        'url': 'https://amp.twimg.com/v/0ba0c3c7-0af3-4c0a-bed5-7efd1ffa2951',
+        'md5': '7df102d0b9fd7066b86f3159f8e81bf6',
+        'info_dict': {
+            'id': '0ba0c3c7-0af3-4c0a-bed5-7efd1ffa2951',
+            'ext': 'mp4',
+            'title': 'Twitter Video',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        vmap_url = self._html_search_meta(
+            'twitter:amplify:vmap', webpage, 'vmap url')
+        video_url = self._get_vmap_video_url(vmap_url, video_id)
+
+        return {
+            'id': video_id,
+            'title': 'Twitter Video',
+            'url': video_url,
+        }

From acb6e97e6a039b7bcce952e9fcf71a5ccf92aec3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Feb 2016 16:57:56 +0800
Subject: [PATCH 0760/1105] [twitter] Fix several failed tests

---
 youtube_dl/extractor/twitter.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 77c9176e4..64c429f02 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -26,7 +26,7 @@ class TwitterCardIE(TwitterBaseIE):
     _TESTS = [
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
-            'md5': '4fa26a35f9d1bf4b646590ba8e84be19',
+            # MD5 checksums are different in different places
             'info_dict': {
                 'id': '560070183650213889',
                 'ext': 'mp4',
@@ -48,7 +48,7 @@ class TwitterCardIE(TwitterBaseIE):
         },
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/654001591733886977',
-            'md5': 'b6f35e8b08a0bec6c8af77a2f4b3a814',
+            'md5': 'd4724ffe6d2437886d004fa5de1043b3',
             'info_dict': {
                 'id': 'dq4Oj5quskI',
                 'ext': 'mp4',
@@ -68,8 +68,8 @@ class TwitterCardIE(TwitterBaseIE):
                 'ext': 'mp4',
                 'upload_date': '20151113',
                 'uploader_id': '1189339351084113920',
-                'uploader': '@ArsenalTerje',
-                'title': 'Vine by @ArsenalTerje',
+                'uploader': 'ArsenalTerje',
+                'title': 'Vine by ArsenalTerje',
             },
             'add_ie': ['Vine'],
         }
@@ -142,7 +142,7 @@ class TwitterIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'https://twitter.com/freethenipple/status/643211948184596480',
-        'md5': 'db6612ec5d03355953c3ca9250c97e5e',
+        # MD5 checksums are different in different places
         'info_dict': {
             'id': '643211948184596480',
             'ext': 'mp4',

From bdbf4ba40ebae986ddf65fd1da4741af9ddf3ec9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Feb 2016 17:16:35 +0800
Subject: [PATCH 0761/1105] [twitter:amplify] Extract more metadata

---
 youtube_dl/extractor/twitter.py | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 64c429f02..29ce84f5e 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -243,6 +243,7 @@ class TwitterAmplifyIE(TwitterBaseIE):
             'id': '0ba0c3c7-0af3-4c0a-bed5-7efd1ffa2951',
             'ext': 'mp4',
             'title': 'Twitter Video',
+            'thumbnail': 're:^https?://.*',
         },
     }
 
@@ -254,8 +255,35 @@ class TwitterAmplifyIE(TwitterBaseIE):
             'twitter:amplify:vmap', webpage, 'vmap url')
         video_url = self._get_vmap_video_url(vmap_url, video_id)
 
+        thumbnails = []
+        thumbnail = self._html_search_meta(
+            'twitter:image:src', webpage, 'thumbnail', fatal=False)
+
+        def _find_dimension(target):
+            w = int_or_none(self._html_search_meta(
+                'twitter:%s:width' % target, webpage, fatal=False))
+            h = int_or_none(self._html_search_meta(
+                'twitter:%s:height' % target, webpage, fatal=False))
+            return w, h
+
+        if thumbnail:
+            thumbnail_w, thumbnail_h = _find_dimension('image')
+            thumbnails.append({
+                'url': thumbnail,
+                'width': thumbnail_w,
+                'height': thumbnail_h,
+            })
+
+        video_w, video_h = _find_dimension('player')
+        formats = [{
+            'url': video_url,
+            'width': video_w,
+            'height': video_h,
+        }]
+
         return {
             'id': video_id,
             'title': 'Twitter Video',
-            'url': video_url,
+            'formats': formats,
+            'thumbnails': thumbnails,
         }

From 132e3b74bdcb73ccacbfeecdae5713dd65829af9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Feb 2016 17:21:37 +0800
Subject: [PATCH 0762/1105] [twitter] Fix a typo

---
 youtube_dl/extractor/twitter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 29ce84f5e..8acd09f6a 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -229,7 +229,7 @@ class TwitterIE(InfoExtractor):
             })
             return info
 
-        raise ExtractorError('There\'s not video in this tweet.')
+        raise ExtractorError('There\'s no video in this tweet.')
 
 
 class TwitterAmplifyIE(TwitterBaseIE):

From 7efc1c2b493aa1d68e5f1b248980144bbacf943a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Feb 2016 17:29:28 +0800
Subject: [PATCH 0763/1105] [twitter] Fix metadata extraction and
 test_Twitter_1

---
 youtube_dl/extractor/twitter.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 8acd09f6a..5d2b5ec35 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -165,6 +165,7 @@ class TwitterIE(InfoExtractor):
             'uploader': 'Gifs',
             'uploader_id': 'giphz',
         },
+        'expected_warnings': ['height', 'width'],
     }, {
         'url': 'https://twitter.com/starwars/status/665052190608723968',
         'md5': '39b7199856dee6cd4432e72c74bc69d4',
@@ -212,20 +213,24 @@ class TwitterIE(InfoExtractor):
             return info
 
         mobj = re.search(r'''(?x)
-            <video[^>]+class="animated-gif"[^>]+
-                (?:data-height="(?P<height>\d+)")?[^>]+
-                (?:data-width="(?P<width>\d+)")?[^>]+
-                (?:poster="(?P<poster>[^"]+)")?[^>]*>\s*
+            <video[^>]+class="animated-gif"(?P<more_info>[^>]+)>\s*
                 <source[^>]+video-src="(?P<url>[^"]+)"
         ''', webpage)
 
         if mobj:
+            more_info = mobj.group('more_info')
+            height = int_or_none(self._search_regex(
+                r'data-height="(\d+)"', more_info, 'height', fatal=False))
+            width = int_or_none(self._search_regex(
+                r'data-width="(\d+)"', more_info, 'width', fatal=False))
+            thumbnail = self._search_regex(
+                r'poster="([^"]+)"', more_info, 'poster', fatal=False)
             info.update({
                 'id': twid,
                 'url': mobj.group('url'),
-                'height': int_or_none(mobj.group('height')),
-                'width': int_or_none(mobj.group('width')),
-                'thumbnail': mobj.group('poster'),
+                'height': height,
+                'width': width,
+                'thumbnail': thumbnail,
             })
             return info
 

From f5bdb4444338f1462c2bc7f79b4ad7e260fe56a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Feb 2016 01:19:39 +0600
Subject: [PATCH 0764/1105] [extractor/common] Add _remove_duplicate_formats

---
 youtube_dl/extractor/common.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index c85dcda0c..14f575635 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -900,6 +900,16 @@ class InfoExtractor(object):
                     item='%s video format' % f.get('format_id') if f.get('format_id') else 'video'),
                 formats)
 
+    @staticmethod
+    def _remove_duplicate_formats(formats):
+        format_urls = set()
+        unique_formats = []
+        for f in formats:
+            if f['url'] not in format_urls:
+                format_urls.add(f['url'])
+                unique_formats.append(f)
+        formats[:] = unique_formats
+
     def _is_valid_url(self, url, video_id, item='video'):
         url = self._proto_relative_url(url, scheme='http:')
         # For now assume non HTTP(S) URLs always valid

From 5d6e1011df826b9ee0f440f800f2a69e42023b86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Feb 2016 01:23:27 +0600
Subject: [PATCH 0765/1105] [pbs] Extract all formats (Closes #8538)

---
 youtube_dl/extractor/pbs.py | 72 +++++++++++++++++++++++++++----------
 1 file changed, 54 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index cca012953..f43e3a146 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -337,6 +337,21 @@ class PBSIE(InfoExtractor):
                 'skip_download': True,  # requires ffmpeg
             },
         },
+        {
+            # Serves hd only via wigget/partnerplayer page
+            'url': 'http://www.pbs.org/video/2365641075/',
+            'info_dict': {
+                'id': '2365641075',
+                'ext': 'mp4',
+                'title': 'FRONTLINE - Netanyahu at War',
+                'duration': 6852,
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'formats': 'mincount:8',
+            },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            },
+        },
         {
             'url': 'http://player.pbs.org/widget/partnerplayer/2365297708/?start=0&end=0&chapterbar=false&endscreen=false&topbar=true',
             'only_matching': True,
@@ -437,34 +452,54 @@ class PBSIE(InfoExtractor):
                 for vid_id in video_id]
             return self.playlist_result(entries, display_id)
 
+        info = None
+        redirects = []
+        redirect_urls = set()
+
+        def extract_redirect_urls(info):
+            for encoding_name in ('recommended_encoding', 'alternate_encoding'):
+                redirect = info.get(encoding_name)
+                if not redirect:
+                    continue
+                redirect_url = redirect.get('url')
+                if redirect_url and redirect_url not in redirect_urls:
+                    redirects.append(redirect)
+                    redirect_urls.add(redirect_url)
+
         try:
-            info = self._download_json(
+            video_info = self._download_json(
                 'http://player.pbs.org/videoInfo/%s?format=json&type=partner' % video_id,
                 display_id, 'Downloading video info JSON')
+            extract_redirect_urls(video_info)
+            info = video_info
         except ExtractorError as e:
+            # videoInfo API may not work for some videos
             if not isinstance(e.cause, compat_HTTPError) or e.cause.code != 404:
                 raise
-            # videoInfo API may not work for some videos, fallback to portalplayer API
+
+        # Player pages may also serve different qualities
+        for page in ('widget/partnerplayer', 'portalplayer'):
             player = self._download_webpage(
-                'http://player.pbs.org/portalplayer/%s' % video_id, display_id)
-            info = self._parse_json(
-                self._search_regex(
-                    r'(?s)PBS\.videoData\s*=\s*({.+?});\n',
-                    player, 'video data', default='{}'),
-                display_id, transform_source=js_to_json, fatal=False)
+                'http://player.pbs.org/%s/%s' % (page, video_id),
+                display_id, 'Downloading %s page' % page, fatal=False)
+            if player:
+                video_info = self._parse_json(
+                    self._search_regex(
+                        r'(?s)PBS\.videoData\s*=\s*({.+?});\n',
+                        player, '%s video data' % page, default='{}'),
+                    display_id, transform_source=js_to_json, fatal=False)
+                if video_info:
+                    extract_redirect_urls(video_info)
+                    if not info:
+                        info = video_info
 
         formats = []
-        for encoding_name in ('recommended_encoding', 'alternate_encoding'):
-            redirect = info.get(encoding_name)
-            if not redirect:
-                continue
-            redirect_url = redirect.get('url')
-            if not redirect_url:
-                continue
+        for num, redirect in enumerate(redirects):
+            redirect_id = redirect.get('eeid')
 
             redirect_info = self._download_json(
-                redirect_url + '?format=json', display_id,
-                'Downloading %s video url info' % encoding_name)
+                '%s?format=json' % redirect['url'], display_id,
+                'Downloading %s video url info' % (redirect_id or num))
 
             if redirect_info['status'] == 'error':
                 raise ExtractorError(
@@ -483,8 +518,9 @@ class PBSIE(InfoExtractor):
             else:
                 formats.append({
                     'url': format_url,
-                    'format_id': redirect.get('eeid'),
+                    'format_id': redirect_id,
                 })
+        self._remove_duplicate_formats(formats)
         self._sort_formats(formats)
 
         rating_str = info.get('rating')

From 1d00a8823e930bfb092c65ec65aa37e32a1ecf5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Feb 2016 01:32:23 +0600
Subject: [PATCH 0766/1105] [arte] PEP 8

---
 youtube_dl/extractor/arte.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 47840cad0..efde7e207 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -114,7 +114,7 @@ class ArteTVPlus7IE(InfoExtractor):
                     r'program\s*:\s*({.+?["\']embed_html["\'].+?}),?\s*\n',
                     webpage, 'program', default=None)
                 if program:
-                    embed_html = self._parse_json(program,video_id)
+                    embed_html = self._parse_json(program, video_id)
                     if embed_html:
                         iframe_url = find_iframe_url(embed_html['embed_html'])
             if iframe_url:
@@ -129,7 +129,6 @@ class ArteTVPlus7IE(InfoExtractor):
             webpage, 'embed url', group='url')
         return self.url_result(embed_url)
 
-
     def _extract_from_json_url(self, json_url, video_id, lang):
         info = self._download_json(json_url, video_id)
         player_info = info['videoJsonPlayer']

From d69abbd3f04b5f3fced2c6a53b4fce788e31906d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Feb 2016 03:13:18 +0600
Subject: [PATCH 0767/1105] [googledrive] Make thumbnail optional (Closes
 #8629)

---
 youtube_dl/extractor/googledrive.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index f354c9c7a..37be34091 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -82,7 +82,7 @@ class GoogleDriveIE(InfoExtractor):
         return {
             'id': video_id,
             'title': title,
-            'thumbnail': self._og_search_thumbnail(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage, default=None),
             'duration': duration,
             'formats': formats,
         }

From 6212bcb19140da77d25fe17830930ebbd8a22c1d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 22 Feb 2016 09:57:12 +0100
Subject: [PATCH 0768/1105] [tf1] fix info extraction(fixes #8599)

---
 youtube_dl/extractor/tf1.py |  4 +---
 youtube_dl/extractor/wat.py | 10 ++++++----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 6890021cf..e1a64e284 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -50,6 +50,4 @@ class TF1IE(InfoExtractor):
         wat_id = self._html_search_regex(
             r'(["\'])(?:https?:)?//www\.wat\.tv/embedframe/.*?(?P<id>\d{8})\1',
             webpage, 'wat id', group='id')
-        wat_info = self._download_json(
-            'http://www.wat.tv/interface/contentv3/%s' % wat_id, video_id)
-        return self.url_result(wat_info['media']['url'], 'Wat')
+        return self.url_result('wat:%s' % wat_id, 'Wat')
diff --git a/youtube_dl/extractor/wat.py b/youtube_dl/extractor/wat.py
index affcc52f6..37cf3d309 100644
--- a/youtube_dl/extractor/wat.py
+++ b/youtube_dl/extractor/wat.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class WatIE(InfoExtractor):
-    _VALID_URL = r'http://www\.wat\.tv/video/(?P<display_id>.*)-(?P<short_id>.*?)_.*?\.html'
+    _VALID_URL = r'(?:wat:(?P<real_id>\d{8})|http://www\.wat\.tv/video/(?P<display_id>.*)-(?P<short_id>.*?)_.*?\.html)'
     IE_NAME = 'wat.tv'
     _TESTS = [
         {
@@ -54,10 +54,12 @@ class WatIE(InfoExtractor):
         def real_id_for_chapter(chapter):
             return chapter['tc_start'].split('-')[0]
         mobj = re.match(self._VALID_URL, url)
-        short_id = mobj.group('short_id')
         display_id = mobj.group('display_id')
-        webpage = self._download_webpage(url, display_id or short_id)
-        real_id = self._search_regex(r'xtpage = ".*-(.*?)";', webpage, 'real id')
+        real_id = mobj.group('real_id')
+        if not real_id:
+            short_id = mobj.group('short_id')
+            webpage = self._download_webpage(url, display_id or short_id)
+            real_id = self._search_regex(r'xtpage = ".*-(.*?)";', webpage, 'real id')
 
         video_info = self.download_video_info(real_id)
 

From 92da3cd84800bf285b5d2125421d0c8d6a4aff09 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 22 Feb 2016 11:57:31 +0100
Subject: [PATCH 0769/1105] release 2016.02.22

---
 CONTRIBUTING.md        | 4 ++--
 docs/supportedsites.md | 8 +++++++-
 youtube_dl/version.py  | 2 +-
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index d15267d7e..39472c554 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,6 +1,6 @@
-**Please include the full output of youtube-dl when run with `-v`**, i.e. add `-v` flag to your command line, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this:
+**Please include the full output of youtube-dl when run with `-v`**, i.e. **add** `-v` flag to **your command line**, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this:
 ```
-$ youtube-dl -v http://www.youtube.com/watch?v=BaW_jenozKcj
+$ youtube-dl -v <your command line>
 [debug] System config: []
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index b384a3165..74596155c 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -30,6 +30,7 @@
  - **AlJazeera**
  - **Allocine**
  - **AlphaPorno**
+ - **AnimeOnDemand**
  - **anitube.se**
  - **AnySex**
  - **Aparat**
@@ -49,6 +50,7 @@
  - **arte.tv:ddc**
  - **arte.tv:embed**
  - **arte.tv:future**
+ - **arte.tv:magazine**
  - **AtresPlayer**
  - **ATTTechChannel**
  - **AudiMedia**
@@ -360,7 +362,7 @@
  - **MySpace:album**
  - **MySpass**
  - **Myvi**
- - **myvideo**
+ - **myvideo** (Currently broken)
  - **MyVidster**
  - **n-tv.de**
  - **NationalGeographic**
@@ -410,6 +412,7 @@
  - **NowTV** (Currently broken)
  - **NowTVList**
  - **nowvideo**: NowVideo
+ - **Noz**
  - **npo**: npo.nl and ntr.nl
  - **npo.nl:live**
  - **npo.nl:radio**
@@ -460,6 +463,7 @@
  - **PornHd**
  - **PornHub**
  - **PornHubPlaylist**
+ - **PornHubUserVideos**
  - **Pornotube**
  - **PornoVoisines**
  - **PornoXO**
@@ -522,6 +526,7 @@
  - **screen.yahoo:search**: Yahoo screen search
  - **Screencast**
  - **ScreencastOMatic**
+ - **ScreenJunkies**
  - **ScreenwaveMedia**
  - **SenateISVP**
  - **ServingSys**
@@ -655,6 +660,7 @@
  - **twitch:video**
  - **twitch:vod**
  - **twitter**
+ - **twitter:amplify**
  - **twitter:card**
  - **Ubu**
  - **udemy**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9aca8001a..7a3df6a26 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.02.13'
+__version__ = '2016.02.22'

From 1ad61430615b8552661ede21f2c3ce76c2e88d2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Feb 2016 17:37:00 +0600
Subject: [PATCH 0770/1105] [xfileshare] Add support for powerwatch (Closes
 #8628)

---
 youtube_dl/extractor/xfileshare.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index a3236e66c..94abdb4f3 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -17,7 +17,7 @@ class XFileShareIE(InfoExtractor):
     IE_DESC = 'XFileShare based sites: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me'
     _VALID_URL = r'''(?x)
         https?://(?P<host>(?:www\.)?
-            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com|vidto\.me))/
+            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com|vidto\.me|powerwatch\.pw))/
         (?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:-[0-9]+x[0-9]+\.html)?
     '''
 
@@ -81,6 +81,13 @@ class XFileShareIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'test'
         }
+    }, {
+        'url': 'http://powerwatch.pw/duecjibvicbu',
+        'info_dict': {
+            'id': 'duecjibvicbu',
+            'ext': 'mp4',
+            'title': 'Big Buck Bunny trailer',
+        },
     }]
 
     def _real_extract(self, url):
@@ -112,6 +119,7 @@ class XFileShareIE(InfoExtractor):
         title = (self._search_regex(
             [r'style="z-index: [0-9]+;">([^<]+)</span>',
              r'<td nowrap>([^<]+)</td>',
+             r'h4-fine[^>]*>([^<]+)<',
              r'>Watch (.+) ',
              r'<h2 class="video-page-head">([^<]+)</h2>'],
             webpage, 'title', default=None) or self._og_search_title(webpage)).strip()

From 0fc871d2f0ae57b88f78c32e3f76bfc1ca609ef0 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 23 Feb 2016 04:16:47 +0600
Subject: [PATCH 0771/1105] [README.md:output_template] Add example for
 channel/user playlists download

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index db49ab6d5..2fbf0219b 100644
--- a/README.md
+++ b/README.md
@@ -525,6 +525,9 @@ youtube-dl_test_video_.mp4          # A simple file name
 # Download YouTube playlist videos in separate directory indexed by video order in a playlist
 $ youtube-dl -o '%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s' https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re
 
+# Download all playlists of YouTube channel/user keeping each playlist in separate directory:
+$ youtube-dl -o '%(uploader)s/%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s' https://www.youtube.com/user/TheLinuxFoundation/playlists
+
 # Download Udemy course keeping each chapter in separate directory under MyVideos directory in your home
 $ youtube-dl -u user -p password -o '~/MyVideos/%(playlist)s/%(chapter_number)s - %(chapter)s/%(title)s.%(ext)s' https://www.udemy.com/java-tutorial/
 

From 1610f770d7912bead235b2bc0fa0cc84a746be69 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 23 Feb 2016 04:29:39 +0600
Subject: [PATCH 0772/1105] [README.md] Extract example subsections

---
 README.md | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 2fbf0219b..309de7782 100644
--- a/README.md
+++ b/README.md
@@ -513,7 +513,9 @@ The current default template is `%(title)s-%(id)s.%(ext)s`.
 
 In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title:
 
-Examples (note on Windows you may need to use double quotes instead of single):
+#### Examples
+
+Note on Windows you may need to use double quotes instead of single.
 
 ```bash
 $ youtube-dl --get-filename -o '%(title)s.%(ext)s' BaW_jenozKc
@@ -595,7 +597,10 @@ Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo
 
 If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download the best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 
-Examples (note on Windows you may need to use double quotes instead of single):
+#### Examples
+
+Note on Windows you may need to use double quotes instead of single.
+
 ```bash
 # Download best mp4 format available or any other best if no mp4 available
 $ youtube-dl -f 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best'

From 621bd0cda9b1d95e0b595d181b3576967abaa66a Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 23 Feb 2016 04:43:45 +0600
Subject: [PATCH 0773/1105] [README.md] Add tl;dr links to examples

---
 README.md | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 309de7782..dba580390 100644
--- a/README.md
+++ b/README.md
@@ -440,7 +440,11 @@ On Windows you may also need to setup the `%HOME%` environment variable manually
 
 # OUTPUT TEMPLATE
 
-The `-o` option allows users to indicate a template for the output file names. The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "http://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences have the format `%(NAME)s`. To clarify, that is a percent symbol followed by a name in parentheses, followed by a lowercase S. Allowed names are:
+The `-o` option allows users to indicate a template for the output file names.
+
+**tl;dr:** [navigate me to examples](#output-template-examples).
+
+The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "http://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences have the format `%(NAME)s`. To clarify, that is a percent symbol followed by a name in parentheses, followed by a lowercase S. Allowed names are:
 
  - `id`: Video identifier
  - `title`: Video title
@@ -513,7 +517,7 @@ The current default template is `%(title)s-%(id)s.%(ext)s`.
 
 In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title:
 
-#### Examples
+#### Output template examples
 
 Note on Windows you may need to use double quotes instead of single.
 
@@ -548,6 +552,8 @@ But sometimes you may want to download in a different format, for example when y
 
 The general syntax for format selection is `--format FORMAT` or shorter `-f FORMAT` where `FORMAT` is a *selector expression*, i.e. an expression that describes format or formats you would like to download.
 
+**tl;dr:** [navigate me to examples](#format-selection-examples).
+
 The simplest case is requesting a specific format, for example with `-f 22` you can download the format with format code equal to 22. You can get the list of available format codes for particular video using `--list-formats` or `-F`. Note that these format codes are extractor specific. 
 
 You can also use a file extension (currently `3gp`, `aac`, `flv`, `m4a`, `mp3`, `mp4`, `ogg`, `wav`, `webm` are supported) to download best quality format of particular file extension served as a single file, e.g. `-f webm` will download best quality format with `webm` extension served as a single file.
@@ -597,7 +603,7 @@ Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo
 
 If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download the best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 
-#### Examples
+#### Format selection examples
 
 Note on Windows you may need to use double quotes instead of single.
 

From 58525c94d547be1c8167d16c298bdd75506db328 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 23 Feb 2016 04:58:51 +0600
Subject: [PATCH 0774/1105] [README.md] Emphasize copyright infringement
 aspects in add-new-site-support tutorial

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index dba580390..60635d5df 100644
--- a/README.md
+++ b/README.md
@@ -830,7 +830,9 @@ If you want to create a build of youtube-dl yourself, you'll need
 
 ### Adding support for a new site
 
-If you want to add support for a new site, you can follow this quick list (assuming your service is called `yourextractor`):
+If you want to add support for a new site, first of all **make sure** this site is **not dedicated to [copyright infringement](#can-you-add-support-for-this-anime-video-site-or-site-which-shows-current-movies-for-free)**. youtube-dl does **not support** such sites thus pull requests adding support for them **will be rejected**.
+
+After you have ensured this site is distributing it's content legally, you can follow this quick list (assuming your service is called `yourextractor`):
 
 1. [Fork this repository](https://github.com/rg3/youtube-dl/fork)
 2. Check out the source code with `git clone git@github.com:YOUR_GITHUB_USERNAME/youtube-dl.git`

From 9b012728324737977e875ddd623fa4c6852e86a9 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 23 Feb 2016 05:03:57 +0600
Subject: [PATCH 0775/1105] [README.md] Update link to extractor metafields

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 60635d5df..48b76b1ec 100644
--- a/README.md
+++ b/README.md
@@ -880,7 +880,7 @@ After you have ensured this site is distributing it's content legally, you can f
     ```
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
-7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62-L200). Add tests and code for as many as you want.
+7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L68-L226). Add tests and code for as many as you want.
 8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 

From cf89ba9eff2f8d76d3b4a54d07c7b4bb3e2cb76f Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 23 Feb 2016 05:35:19 +0600
Subject: [PATCH 0776/1105] [README.md] Clarify robustness and future-proof
 requirements for new extractors

---
 README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 48b76b1ec..244ed2b59 100644
--- a/README.md
+++ b/README.md
@@ -881,8 +881,9 @@ After you have ensured this site is distributing it's content legally, you can f
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L68-L226). Add tests and code for as many as you want.
-8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
-9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
+8. Keep in mind that the only mandatory fields in info dict for successful extraction process are `id`, `title` and either `url` or `formats`, e.g. these are the critical data the extraction does not make any sense without. This means that [any field](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L138-L226) apart from aforementioned mandatory ones should be treated **as optional** and extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields. For example, if you have some intermediate dict `meta` that is a source of metadata and it has a key `summary` that you want to extract and put into resulting info dict as `description`, you should be ready that this key may be missing from the `meta` dict, i.e. you should extract it as `meta.get('summary')` and not `meta['summary']`. Similarly, you should pass `fatal=False` when extracting data from a webpage with `_search_regex/_html_search_regex`.
+9. Check the code with [flake8](https://pypi.python.org/pypi/flake8).
+10. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
         $ git add youtube_dl/extractor/__init__.py
         $ git add youtube_dl/extractor/yourextractor.py

From 6563d3171011312e4aa872e9e9fc66bb0a84b77f Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 23 Feb 2016 05:37:49 +0600
Subject: [PATCH 0777/1105] [README.md] Fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 244ed2b59..36923b15e 100644
--- a/README.md
+++ b/README.md
@@ -881,7 +881,7 @@ After you have ensured this site is distributing it's content legally, you can f
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L68-L226). Add tests and code for as many as you want.
-8. Keep in mind that the only mandatory fields in info dict for successful extraction process are `id`, `title` and either `url` or `formats`, e.g. these are the critical data the extraction does not make any sense without. This means that [any field](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L138-L226) apart from aforementioned mandatory ones should be treated **as optional** and extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields. For example, if you have some intermediate dict `meta` that is a source of metadata and it has a key `summary` that you want to extract and put into resulting info dict as `description`, you should be ready that this key may be missing from the `meta` dict, i.e. you should extract it as `meta.get('summary')` and not `meta['summary']`. Similarly, you should pass `fatal=False` when extracting data from a webpage with `_search_regex/_html_search_regex`.
+8. Keep in mind that the only mandatory fields in info dict for successful extraction process are `id`, `title` and either `url` or `formats`, i.e. these are the critical data the extraction does not make any sense without. This means that [any field](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L138-L226) apart from aforementioned mandatory ones should be treated **as optional** and extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields. For example, if you have some intermediate dict `meta` that is a source of metadata and it has a key `summary` that you want to extract and put into resulting info dict as `description`, you should be ready that this key may be missing from the `meta` dict, i.e. you should extract it as `meta.get('summary')` and not `meta['summary']`. Similarly, you should pass `fatal=False` when extracting data from a webpage with `_search_regex/_html_search_regex`.
 9. Check the code with [flake8](https://pypi.python.org/pypi/flake8).
 10. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 

From b43a7a92cdd7e85021f4e4e177b26d982195da31 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 23 Feb 2016 05:41:09 +0600
Subject: [PATCH 0778/1105] [README.md] Fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 36923b15e..cf03e0d54 100644
--- a/README.md
+++ b/README.md
@@ -890,7 +890,7 @@ After you have ensured this site is distributing it's content legally, you can f
         $ git commit -m '[yourextractor] Add new extractor'
         $ git push origin yourextractor
 
-10. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it.
+11. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it.
 
 In any case, thank you very much for your contributions!
 

From 86a7dbe66e9aed051daa8c19a1ba8b860959a30a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Feb 2016 18:20:55 +0800
Subject: [PATCH 0779/1105] [nba] Support non-video/ pages

Fixes #8589
---
 youtube_dl/extractor/nba.py | 30 +++++++++++++++++++++++++++---
 1 file changed, 27 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index a071378b6..87b38c4c7 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -1,18 +1,20 @@
 from __future__ import unicode_literals
 
+import os.path
 import re
 
 from .common import InfoExtractor
 from ..utils import (
-    parse_duration,
     int_or_none,
+    parse_duration,
+    remove_start,
     xpath_text,
     xpath_attr,
 )
 
 
 class NBAIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?P<path>(?:[^/]+/)?video/(?P<id>[^?]*?))/?(?:/index\.html)?(?:\?.*)?$'
+    _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?P<path>(?:[^/]+/)+(?P<id>[^?]*?))/?(?:/index\.html)?(?:\?.*)?$'
     _TESTS = [{
         'url': 'http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html',
         'md5': '9e7729d3010a9c71506fd1248f74e4f4',
@@ -44,14 +46,36 @@ class NBAIE(InfoExtractor):
             'timestamp': 1432134543,
             'upload_date': '20150520',
         }
+    }, {
+        'url': 'http://www.nba.com/clippers/news/doc-rivers-were-not-trading-blake',
+        'info_dict': {
+            'id': '1455672027478-Doc_Feb16_720',
+            'ext': 'mp4',
+            'title': 'Practice: Doc Rivers - 2/16/16',
+            'description': 'Head Coach Doc Rivers addresses the media following practice.',
+            'upload_date': '20160217',
+            'timestamp': 1455672000,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
         path, video_id = re.match(self._VALID_URL, url).groups()
         if path.startswith('nba/'):
             path = path[3:]
+
+        if 'video/' not in path:
+            webpage = self._download_webpage(url, video_id)
+            path = remove_start(self._search_regex(r'data-videoid="([^"]+)"', webpage, 'video id'), '/')
+            # See prepareContentId() of pkgCvp.js
+            if path.startswith('video/teams'):
+                path = 'video/channels/proxy/' + path[6:]
+
         video_info = self._download_xml('http://www.nba.com/%s.xml' % path, video_id)
-        video_id = xpath_text(video_info, 'slug')
+        video_id = os.path.splitext(xpath_text(video_info, 'slug'))[0]
         title = xpath_text(video_info, 'headline')
         description = xpath_text(video_info, 'description')
         duration = parse_duration(xpath_text(video_info, 'length'))

From b95dc034ca96fbb19995c54a8c75b5d46116a1e1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 23 Feb 2016 12:17:02 +0800
Subject: [PATCH 0780/1105] [utils] Implement cache for OnDemandPagedList

---
 youtube_dl/utils.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 6978a10e4..8ec70f93c 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1570,9 +1570,12 @@ class PagedList(object):
 
 
 class OnDemandPagedList(PagedList):
-    def __init__(self, pagefunc, pagesize):
+    def __init__(self, pagefunc, pagesize, use_cache=False):
         self._pagefunc = pagefunc
         self._pagesize = pagesize
+        self._use_cache = use_cache
+        if use_cache:
+            self._cache = {}
 
     def getslice(self, start=0, end=None):
         res = []
@@ -1582,7 +1585,13 @@ class OnDemandPagedList(PagedList):
             if start >= nextfirstid:
                 continue
 
-            page_results = list(self._pagefunc(pagenum))
+            page_results = None
+            if self._use_cache:
+                page_results = self._cache.get(pagenum)
+            if page_results is None:
+                page_results = list(self._pagefunc(pagenum))
+            if self._use_cache:
+                self._cache[pagenum] = page_results
 
             startv = (
                 start % self._pagesize

From 3cfeb1624a28b60cec24dbc0d21565f4afd924b3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 23 Feb 2016 12:17:41 +0800
Subject: [PATCH 0781/1105] [nba] Support channels (#5362, #4167)

---
 youtube_dl/extractor/nba.py | 71 +++++++++++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 87b38c4c7..3e2b3e599 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -1,11 +1,17 @@
 from __future__ import unicode_literals
 
+import functools
 import os.path
 import re
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse,
+    compat_urlparse,
+)
 from ..utils import (
     int_or_none,
+    OnDemandPagedList,
     parse_duration,
     remove_start,
     xpath_text,
@@ -60,16 +66,81 @@ class NBAIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.nba.com/timberwolves/wiggins-shootaround#',
+        'info_dict': {
+            'id': 'timberwolves',
+            'title': 'Shootaround Access - Dec. 12 | Andrew Wiggins',
+        },
+        'playlist_count': 30,
+        'params': {
+            # Download the whole playlist takes too long time
+            'playlist_items': '1-30',
+        },
+    }, {
+        'url': 'http://www.nba.com/timberwolves/wiggins-shootaround#',
+        'info_dict': {
+            'id': 'Wigginsmp4',
+            'ext': 'mp4',
+            'title': 'Shootaround Access - Dec. 12 | Andrew Wiggins',
+            'description': 'Wolves rookie Andrew Wiggins addresses the media after Friday\'s shootaround.',
+            'upload_date': '20141212',
+            'timestamp': 1418418600,
+        },
+        'params': {
+            'noplaylist': True,
+            # m3u8 download
+            'skip_download': True,
+        },
     }]
 
+    _PAGE_SIZE = 30
+
+    def _fetch_page(self, team, video_id, page):
+        search_url = 'http://searchapp2.nba.com/nba-search/query.jsp?' + compat_urllib_parse.urlencode({
+            'type': 'teamvideo',
+            'start': page * self._PAGE_SIZE + 1,
+            'npp': (page + 1) * self._PAGE_SIZE + 1,
+            'sort': 'recent',
+            'output': 'json',
+            'site': team,
+        })
+        results = self._download_json(
+            search_url, video_id, note='Download page %d of playlist data' % page)['results'][0]
+        for item in results:
+            yield self.url_result(compat_urlparse.urljoin('http://www.nba.com/', item['url']))
+
+    def _extract_playlist(self, orig_path, video_id, webpage):
+        team = orig_path.split('/')[0]
+
+        if self._downloader.params.get('noplaylist'):
+            self.to_screen('Downloading just video because of --no-playlist')
+            video_path = self._search_regex(
+                r'nbaVideoCore\.firstVideo\s*=\s*\'([^\']+)\';', webpage, 'video path')
+            video_url = 'http://www.nba.com/%s/video/%s' % (team, video_path)
+            return self.url_result(video_url)
+
+        self.to_screen('Downloading playlist - add --no-playlist to just download video')
+        playlist_title = self._og_search_title(webpage, fatal=False)
+        entries = OnDemandPagedList(
+            functools.partial(self._fetch_page, team, video_id),
+            self._PAGE_SIZE, use_cache=True)
+
+        return self.playlist_result(entries, team, playlist_title)
+
     def _real_extract(self, url):
         path, video_id = re.match(self._VALID_URL, url).groups()
+        orig_path = path
         if path.startswith('nba/'):
             path = path[3:]
 
         if 'video/' not in path:
             webpage = self._download_webpage(url, video_id)
             path = remove_start(self._search_regex(r'data-videoid="([^"]+)"', webpage, 'video id'), '/')
+
+            if path == '{{id}}':
+                return self._extract_playlist(orig_path, video_id, webpage)
+
             # See prepareContentId() of pkgCvp.js
             if path.startswith('video/teams'):
                 path = 'video/channels/proxy/' + path[6:]

From efbeddead35582126c622a74972af02db9680357 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 23 Feb 2016 13:17:24 +0800
Subject: [PATCH 0782/1105] [facebook] Support mobile URLs (closes #8638)

---
 youtube_dl/extractor/facebook.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 0a9a5ca71..d2ff38140 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -34,8 +34,9 @@ class FacebookIE(InfoExtractor):
                                 video/video\.php|
                                 photo\.php|
                                 video\.php|
-                                video/embed
-                            )\?(?:.*?)(?:v|video_id)=|
+                                video/embed|
+                                story\.php
+                            )\?(?:.*?)(?:v|video_id|story_fbid)=|
                             [^/]+/videos/(?:[^/]+/)?
                         )|
                     facebook:
@@ -92,6 +93,9 @@ class FacebookIE(InfoExtractor):
     }, {
         'url': 'facebook:544765982287235',
         'only_matching': True,
+    }, {
+        'url': 'https://m.facebook.com/story.php?story_fbid=1035862816472149&id=116132035111903',
+        'only_matching': True,
     }]
 
     def _login(self):

From bf4b3b6bd91137b94d5ae8ac6dc421e64a647355 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 Feb 2016 18:47:13 +0600
Subject: [PATCH 0783/1105] [vk] Extract video URL from extra_data (Closes
 #8646)

---
 youtube_dl/extractor/vk.py | 31 +++++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 0805e3c08..670a438af 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -11,6 +11,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    int_or_none,
     orderedSet,
     sanitized_Request,
     str_to_int,
@@ -151,6 +152,19 @@ class VKIE(InfoExtractor):
                 'view_count': int,
             },
         },
+        {
+            # video key is extra_data not url\d+
+            'url': 'http://vk.com/video-110305615_171782105',
+            'md5': 'e13fcda136f99764872e739d13fac1d1',
+            'info_dict': {
+                'id': '171782105',
+                'ext': 'mp4',
+                'title': 'S-Dance, репетиции к The way show',
+                'uploader': 'THE WAY SHOW | 17 апреля',
+                'upload_date': '20160207',
+                'view_count': int,
+            },
+        },
         {
             # removed video, just testing that we match the pattern
             'url': 'http://vk.com/feed?z=video-43215063_166094326%2Fbb50cacd3177146d7a',
@@ -298,12 +312,17 @@ class VKIE(InfoExtractor):
             view_count = str_to_int(self._search_regex(
                 r'([\d,.]+)', views, 'view count', fatal=False))
 
-        formats = [{
-            'format_id': k,
-            'url': v,
-            'width': int(k[len('url'):]),
-        } for k, v in data.items()
-            if k.startswith('url')]
+        formats = []
+        for k, v in data.items():
+            if not k.startswith('url') and k != 'extra_data' or not v:
+                continue
+            height = int_or_none(self._search_regex(
+                r'^url(\d+)', k, 'height', default=None))
+            formats.append({
+                'format_id': k,
+                'url': v,
+                'height': height,
+            })
         self._sort_formats(formats)
 
         return {

From 1b77ee6248b2a9967d3d3e809e416bd26fd7342c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 24 Feb 2016 03:32:12 +0800
Subject: [PATCH 0784/1105] [c56] Support videos hosted on Sohu (closes #8073)

---
 youtube_dl/extractor/c56.py | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/c56.py b/youtube_dl/extractor/c56.py
index cb96c3876..cac8fdcba 100644
--- a/youtube_dl/extractor/c56.py
+++ b/youtube_dl/extractor/c56.py
@@ -4,12 +4,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import js_to_json
 
 
 class C56IE(InfoExtractor):
     _VALID_URL = r'https?://(?:(?:www|player)\.)?56\.com/(?:.+?/)?(?:v_|(?:play_album.+-))(?P<textid>.+?)\.(?:html|swf)'
     IE_NAME = '56.com'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.56.com/u39/v_OTM0NDA3MTY.html',
         'md5': 'e59995ac63d0457783ea05f93f12a866',
         'info_dict': {
@@ -18,12 +19,29 @@ class C56IE(InfoExtractor):
             'title': '网事知多少 第32期：车怒',
             'duration': 283.813,
         },
-    }
+    }, {
+        'url': 'http://www.56.com/u47/v_MTM5NjQ5ODc2.html',
+        'md5': '',
+        'info_dict': {
+            'id': '82247482',
+            'title': '爱的诅咒之杜鹃花开',
+        },
+        'playlist_count': 7,
+        'add_ie': ['Sohu'],
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE)
         text_id = mobj.group('textid')
 
+        webpage = self._download_webpage(url, text_id)
+        sohu_video_info_str = self._search_regex(
+            r'var\s+sohuVideoInfo\s*=\s*({[^}]+});', webpage, 'Sohu video info', default=None)
+        if sohu_video_info_str:
+            sohu_video_info = self._parse_json(
+                sohu_video_info_str, text_id, transform_source=js_to_json)
+            return self.url_result(sohu_video_info['url'], 'Sohu')
+
         page = self._download_json(
             'http://vxml.56.com/json/%s/' % text_id, text_id, 'Downloading video info')
 

From c24883a1c0e650a41b54e7cdec8427b046c27bc1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 24 Feb 2016 03:43:24 +0800
Subject: [PATCH 0785/1105] [facebook] Fix format sorting

'hd' formats should have higher priorities
---
 youtube_dl/extractor/facebook.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index d2ff38140..6c6c3b1bd 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -212,10 +212,13 @@ class FacebookIE(InfoExtractor):
                 for src_type in ('src', 'src_no_ratelimit'):
                     src = f[0].get('%s_%s' % (quality, src_type))
                     if src:
+                        preference = -10 if format_id == 'progressive' else 0
+                        if quality == 'hd':
+                            preference += 5
                         formats.append({
                             'format_id': '%s_%s_%s' % (format_id, quality, src_type),
                             'url': src,
-                            'preference': -10 if format_id == 'progressive' else 0,
+                            'preference': preference,
                         })
             dash_manifest = f[0].get('dash_manifest')
             if dash_manifest:

From c0da50d2b2ba75e5d497caa8805ebac49845f168 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Wed, 24 Feb 2016 23:05:23 +0600
Subject: [PATCH 0786/1105] [README.md] Turn references to issues to links

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index cf03e0d54..d50d4b950 100644
--- a/README.md
+++ b/README.md
@@ -599,7 +599,7 @@ You can merge the video and audio of two formats into a single file using `-f <v
 
 Format selectors can also be grouped using parentheses, for example if you want to download the best mp4 and webm formats with a height lower than 480 you can use `-f '(mp4,webm)[height<480]'`.
 
-Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some DASH formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
+Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see [#5447](https://github.com/rg3/youtube-dl/issues/5447), [#5456](https://github.com/rg3/youtube-dl/issues/5456)). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some DASH formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
 
 If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download the best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 
@@ -747,7 +747,7 @@ means you're using an outdated version of Python. Please update to Python 2.6 or
 
 ### What is this binary file? Where has the code gone?
 
-Since June 2012 (#342) youtube-dl is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repository, as laid out above. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make youtube-dl`.
+Since June 2012 ([#342](https://github.com/rg3/youtube-dl/issues/342)) youtube-dl is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repository, as laid out above. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make youtube-dl`.
 
 ### The exe throws a *Runtime error from Visual C++*
 

From 43479d9e9df3aa4a0c919bd1b5ff74d53095cac7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Feb 2016 00:36:14 +0600
Subject: [PATCH 0787/1105] [motherless] Make categories optional (Closes
 #8654)

---
 youtube_dl/extractor/motherless.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index 97d5da626..b3bfcac9e 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -54,6 +54,11 @@ class MotherlessIE(InfoExtractor):
                 'thumbnail': 're:http://.*\.jpg',
                 'age_limit': 18,
             }
+        },
+        {
+            # no keywords
+            'url': 'http://motherless.com/8B4BBC1',
+            'only_matching': True,
         }
     ]
 
@@ -86,7 +91,7 @@ class MotherlessIE(InfoExtractor):
             r'"thumb-member-username">\s+<a href="/m/([^"]+)"',
             webpage, 'uploader_id')
 
-        categories = self._html_search_meta('keywords', webpage)
+        categories = self._html_search_meta('keywords', webpage, default=None)
         if categories:
             categories = [cat.strip() for cat in categories.split(',')]
 

From 5c0a57185cf925e22b59004150d6b00cd7fd0417 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Feb 2016 00:42:19 +0600
Subject: [PATCH 0788/1105] [motherless] Detect non-existing videos

---
 youtube_dl/extractor/motherless.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index b3bfcac9e..30e686a4e 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     str_to_int,
     unified_strdate,
 )
@@ -66,6 +67,11 @@ class MotherlessIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
+        if any(p in webpage for p in (
+                '<title>404 - MOTHERLESS.COM<',
+                ">The page you're looking for cannot be found.<")):
+            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+
         title = self._html_search_regex(
             r'id="view-upload-title">\s+([^<]+)<', webpage, 'title')
         video_url = self._html_search_regex(

From f160785c5c5a99a2fdc4724f1f66f423cf8f6bf5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Feb 2016 00:52:49 +0600
Subject: [PATCH 0789/1105] [utils] Remove AM/PM from unified_strdate patterns

---
 test/test_utils.py  | 1 +
 youtube_dl/utils.py | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index e6887be9f..d0736f435 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -249,6 +249,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(
             unified_strdate('2/2/2015 6:47:40 PM', day_first=False),
             '20150202')
+        self.assertEqual(unified_strdate('Feb 14th 2016 5:45PM'), '20160214')
         self.assertEqual(unified_strdate('25-09-2014'), '20140925')
         self.assertEqual(unified_strdate('UNKNOWN DATE FORMAT'), None)
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 8ec70f93c..a3df90fad 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -905,9 +905,9 @@ def unified_strdate(date_str, day_first=True):
         '%d %b %Y',
         '%B %d %Y',
         '%b %d %Y',
-        '%b %dst %Y %I:%M%p',
-        '%b %dnd %Y %I:%M%p',
-        '%b %dth %Y %I:%M%p',
+        '%b %dst %Y %I:%M',
+        '%b %dnd %Y %I:%M',
+        '%b %dth %Y %I:%M',
         '%Y %m %d',
         '%Y-%m-%d',
         '%Y/%m/%d',

From d0459c530d3ce145fe89cb30eba813be6b8ed66a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Feb 2016 00:54:41 +0600
Subject: [PATCH 0790/1105] [motherless] Update tests

---
 youtube_dl/extractor/motherless.py | 93 +++++++++++++++---------------
 1 file changed, 45 insertions(+), 48 deletions(-)

diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index 30e686a4e..0b4787c1d 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -13,55 +13,52 @@ from ..utils import (
 
 class MotherlessIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?motherless\.com/(?:g/[a-z0-9_]+/)?(?P<id>[A-Z0-9]+)'
-    _TESTS = [
-        {
-            'url': 'http://motherless.com/AC3FFE1',
-            'md5': '310f62e325a9fafe64f68c0bccb6e75f',
-            'info_dict': {
-                'id': 'AC3FFE1',
-                'ext': 'mp4',
-                'title': 'Fucked in the ass while playing PS3',
-                'categories': ['Gaming', 'anal', 'reluctant', 'rough', 'Wife'],
-                'upload_date': '20100913',
-                'uploader_id': 'famouslyfuckedup',
-                'thumbnail': 're:http://.*\.jpg',
-                'age_limit': 18,
-            }
-        },
-        {
-            'url': 'http://motherless.com/532291B',
-            'md5': 'bc59a6b47d1f958e61fbd38a4d31b131',
-            'info_dict': {
-                'id': '532291B',
-                'ext': 'mp4',
-                'title': 'Amazing girl playing the omegle game, PERFECT!',
-                'categories': ['Amateur', 'webcam', 'omegle', 'pink', 'young', 'masturbate', 'teen', 'game', 'hairy'],
-                'upload_date': '20140622',
-                'uploader_id': 'Sulivana7x',
-                'thumbnail': 're:http://.*\.jpg',
-                'age_limit': 18,
-            }
-        },
-        {
-            'url': 'http://motherless.com/g/cosplay/633979F',
-            'md5': '0b2a43f447a49c3e649c93ad1fafa4a0',
-            'info_dict': {
-                'id': '633979F',
-                'ext': 'mp4',
-                'title': 'Turtlette',
-                'categories': ['superheroine heroine  superher'],
-                'upload_date': '20140827',
-                'uploader_id': 'shade0230',
-                'thumbnail': 're:http://.*\.jpg',
-                'age_limit': 18,
-            }
-        },
-        {
-            # no keywords
-            'url': 'http://motherless.com/8B4BBC1',
-            'only_matching': True,
+    _TESTS = [{
+        'url': 'http://motherless.com/AC3FFE1',
+        'md5': '310f62e325a9fafe64f68c0bccb6e75f',
+        'info_dict': {
+            'id': 'AC3FFE1',
+            'ext': 'mp4',
+            'title': 'Fucked in the ass while playing PS3',
+            'categories': ['Gaming', 'anal', 'reluctant', 'rough', 'Wife'],
+            'upload_date': '20100913',
+            'uploader_id': 'famouslyfuckedup',
+            'thumbnail': 're:http://.*\.jpg',
+            'age_limit': 18,
         }
-    ]
+    }, {
+        'url': 'http://motherless.com/532291B',
+        'md5': 'bc59a6b47d1f958e61fbd38a4d31b131',
+        'info_dict': {
+            'id': '532291B',
+            'ext': 'mp4',
+            'title': 'Amazing girl playing the omegle game, PERFECT!',
+            'categories': ['Amateur', 'webcam', 'omegle', 'pink', 'young', 'masturbate', 'teen',
+                           'game', 'hairy'],
+            'upload_date': '20140622',
+            'uploader_id': 'Sulivana7x',
+            'thumbnail': 're:http://.*\.jpg',
+            'age_limit': 18,
+        },
+        'skip': '404',
+    }, {
+        'url': 'http://motherless.com/g/cosplay/633979F',
+        'md5': '0b2a43f447a49c3e649c93ad1fafa4a0',
+        'info_dict': {
+            'id': '633979F',
+            'ext': 'mp4',
+            'title': 'Turtlette',
+            'categories': ['superheroine heroine  superher'],
+            'upload_date': '20140827',
+            'uploader_id': 'shade0230',
+            'thumbnail': 're:http://.*\.jpg',
+            'age_limit': 18,
+        }
+    }, {
+        # no keywords
+        'url': 'http://motherless.com/8B4BBC1',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 9195ef745aba88a08e351af6405a87b5f355c026 Mon Sep 17 00:00:00 2001
From: mutantmonkey <mutantmonkey@mutantmonkey.in>
Date: Mon, 15 Feb 2016 17:07:13 -0800
Subject: [PATCH 0791/1105] [uStudio] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ustudio.py  | 58 ++++++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+)
 create mode 100644 youtube_dl/extractor/ustudio.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1ae606f1e..5386747f1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -813,6 +813,7 @@ from .digiteka import DigitekaIE
 from .unistra import UnistraIE
 from .urort import UrortIE
 from .ustream import UstreamIE, UstreamChannelIE
+from .ustudio import UstudioIE
 from .varzesh3 import Varzesh3IE
 from .vbox7 import Vbox7IE
 from .veehd import VeeHDIE
diff --git a/youtube_dl/extractor/ustudio.py b/youtube_dl/extractor/ustudio.py
new file mode 100644
index 000000000..0f837ad73
--- /dev/null
+++ b/youtube_dl/extractor/ustudio.py
@@ -0,0 +1,58 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class UstudioIE(InfoExtractor):
+    IE_NAME = 'uStudio'
+    _VALID_URL = r'http://(?:www\.|v1\.)?ustudio.com/video/(?P<id>[\w\d]+)/.+'
+    _TESTS = [
+        {
+            'url': 'http://ustudio.com/video/Uxu2my9bgSph/san_francisco_golden_gate_bridge',
+            'md5': '58bbfca62125378742df01fc2abbdef6',
+            'info_dict': {
+                'id': 'Uxu2my9bgSph',
+                'ext': 'mp4',
+                'title': 'San Francisco: Golden Gate Bridge',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'description': 'md5:23925500697f2c6d4830e387ba51a9be',
+                'uploader': 'Tony Farley',
+            }
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        doc = self._download_xml(
+            'http://v1.ustudio.com/embed/{0}/ustudio/config.xml'.format(
+                video_id),
+            video_id,
+            note='Downloading video info',
+            errnote='Failed to download video info')
+
+        formats = [
+            {
+                'url': quality.attrib['url'],
+                'width': int_or_none(quality.attrib.get('width')),
+                'height': int_or_none(quality.attrib.get('height')),
+            } for quality in doc.findall('./qualities/quality/video')
+        ]
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'formats': formats,
+            'description': self._og_search_description(webpage),
+            'uploader': self._html_search_regex(
+                r'<a href=".*/user/.+">(.+)</a> on',
+                webpage,
+                'uploader',
+                fatal=False),
+        }

From 20108c6b9045f819580bf2d6ca687718bd7b0510 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Feb 2016 21:30:19 +0600
Subject: [PATCH 0792/1105] [ustudio] Improve (Closes #8574)

---
 youtube_dl/extractor/ustudio.py | 91 ++++++++++++++++++---------------
 1 file changed, 50 insertions(+), 41 deletions(-)

diff --git a/youtube_dl/extractor/ustudio.py b/youtube_dl/extractor/ustudio.py
index 0f837ad73..cafc082b6 100644
--- a/youtube_dl/extractor/ustudio.py
+++ b/youtube_dl/extractor/ustudio.py
@@ -1,58 +1,67 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    unified_strdate,
+)
 
 
 class UstudioIE(InfoExtractor):
-    IE_NAME = 'uStudio'
-    _VALID_URL = r'http://(?:www\.|v1\.)?ustudio.com/video/(?P<id>[\w\d]+)/.+'
-    _TESTS = [
-        {
-            'url': 'http://ustudio.com/video/Uxu2my9bgSph/san_francisco_golden_gate_bridge',
-            'md5': '58bbfca62125378742df01fc2abbdef6',
-            'info_dict': {
-                'id': 'Uxu2my9bgSph',
-                'ext': 'mp4',
-                'title': 'San Francisco: Golden Gate Bridge',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'description': 'md5:23925500697f2c6d4830e387ba51a9be',
-                'uploader': 'Tony Farley',
-            }
-        },
-    ]
+    _VALID_URL = r'https?://(?:(?:www|v1)\.)?ustudio\.com/video/(?P<id>[^/]+)/(?P<display_id>[^/?#&]+)'
+    _TEST = {
+        'url': 'http://ustudio.com/video/Uxu2my9bgSph/san_francisco_golden_gate_bridge',
+        'md5': '58bbfca62125378742df01fc2abbdef6',
+        'info_dict': {
+            'id': 'Uxu2my9bgSph',
+            'display_id': 'san_francisco_golden_gate_bridge',
+            'ext': 'mp4',
+            'title': 'San Francisco: Golden Gate Bridge',
+            'description': 'md5:23925500697f2c6d4830e387ba51a9be',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'upload_date': '20111107',
+            'uploader': 'Tony Farley',
+        }
+    }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        webpage = self._download_webpage(url, video_id)
+        config = self._download_xml(
+            'http://v1.ustudio.com/embed/%s/ustudio/config.xml' % video_id,
+            display_id)
 
-        doc = self._download_xml(
-            'http://v1.ustudio.com/embed/{0}/ustudio/config.xml'.format(
-                video_id),
-            video_id,
-            note='Downloading video info',
-            errnote='Failed to download video info')
+        def extract(kind):
+            return [{
+                'url': item.attrib['url'],
+                'width': int_or_none(item.get('width')),
+                'height': int_or_none(item.get('height')),
+            } for item in config.findall('./qualities/quality/%s' % kind) if item.get('url')]
 
-        formats = [
-            {
-                'url': quality.attrib['url'],
-                'width': int_or_none(quality.attrib.get('width')),
-                'height': int_or_none(quality.attrib.get('height')),
-            } for quality in doc.findall('./qualities/quality/video')
-        ]
+        formats = extract('video')
         self._sort_formats(formats)
 
+        webpage = self._download_webpage(url, display_id)
+
+        title = self._og_search_title(webpage)
+        upload_date = unified_strdate(self._search_regex(
+            r'(?s)Uploaded by\s*.+?\s*on\s*<span>([^<]+)</span>',
+            webpage, 'upload date', fatal=False))
+        uploader = self._search_regex(
+            r'Uploaded by\s*<a[^>]*>([^<]+)<',
+            webpage, 'uploader', fatal=False)
+
         return {
             'id': video_id,
-            'title': self._og_search_title(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'formats': formats,
+            'display_id': display_id,
+            'title': title,
             'description': self._og_search_description(webpage),
-            'uploader': self._html_search_regex(
-                r'<a href=".*/user/.+">(.+)</a> on',
-                webpage,
-                'uploader',
-                fatal=False),
+            'thumbnails': extract('image'),
+            'upload_date': upload_date,
+            'uploader': uploader,
+            'formats': formats,
         }

From e26cde092736c3356c1ab76c899a180fb3441d4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Feb 2016 21:46:43 +0600
Subject: [PATCH 0793/1105] [space] Remove extractor (Closes #8662)

Now uses ooyala embed
---
 youtube_dl/extractor/__init__.py |  1 -
 youtube_dl/extractor/space.py    | 38 --------------------------------
 2 files changed, 39 deletions(-)
 delete mode 100644 youtube_dl/extractor/space.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5386747f1..77e2ae425 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -669,7 +669,6 @@ from .southpark import (
     SouthParkEsIE,
     SouthParkNlIE
 )
-from .space import SpaceIE
 from .spankbang import SpankBangIE
 from .spankwire import SpankwireIE
 from .spiegel import SpiegelIE, SpiegelArticleIE
diff --git a/youtube_dl/extractor/space.py b/youtube_dl/extractor/space.py
deleted file mode 100644
index ebb5d6ec0..000000000
--- a/youtube_dl/extractor/space.py
+++ /dev/null
@@ -1,38 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from .brightcove import BrightcoveLegacyIE
-from ..utils import RegexNotFoundError, ExtractorError
-
-
-class SpaceIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www|m)\.)?space\.com/\d+-(?P<title>[^/\.\?]*?)-video\.html'
-    _TEST = {
-        'add_ie': ['BrightcoveLegacy'],
-        'url': 'http://www.space.com/23373-huge-martian-landforms-detail-revealed-by-european-probe-video.html',
-        'info_dict': {
-            'id': '2780937028001',
-            'ext': 'mp4',
-            'title': 'Huge Martian Landforms\' Detail Revealed By European Probe | Video',
-            'description': 'md5:db81cf7f3122f95ed234b631a6ea1e61',
-            'uploader': 'TechMedia Networks',
-        },
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        title = mobj.group('title')
-        webpage = self._download_webpage(url, title)
-        try:
-            # Some videos require the playerKey field, which isn't define in
-            # the BrightcoveExperience object
-            brightcove_url = self._og_search_video_url(webpage)
-        except RegexNotFoundError:
-            # Other videos works fine with the info from the object
-            brightcove_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
-        if brightcove_url is None:
-            raise ExtractorError(
-                'The webpage does not contain a video', expected=True)
-        return self.url_result(brightcove_url, BrightcoveLegacyIE.ie_key())

From e048d87fc9b5cfa19806649faf20fc1a7bdb82a6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 24 Feb 2016 03:49:13 +0800
Subject: [PATCH 0794/1105] [kuwo] Fix a test

---
 youtube_dl/extractor/kuwo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index f641edef8..700e44b63 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -68,6 +68,7 @@ class KuwoIE(KuwoBaseIE):
             'id': '6446136',
             'ext': 'mp3',
             'title': '心',
+            'description': 'md5:b2ab6295d014005bfc607525bfc1e38a',
             'creator': 'IU',
             'upload_date': '20150518',
         },

From 81bdc8fdf6516b05bc3a26f82eacb1889f5e46d5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 24 Feb 2016 22:08:40 +0800
Subject: [PATCH 0795/1105] [utils] Move base62 to utils

---
 youtube_dl/extractor/iqiyi.py | 15 ++-------------
 youtube_dl/utils.py           | 14 ++++++++++++++
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index c3e33009a..4f02b9f87 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -18,6 +18,7 @@ from ..compat import (
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
+    base62,
     ExtractorError,
     ohdave_rsa_encrypt,
     remove_start,
@@ -126,21 +127,9 @@ class IqiyiSDK(object):
 
 
 class IqiyiSDKInterpreter(object):
-    BASE62_TABLE = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
-
     def __init__(self, sdk_code):
         self.sdk_code = sdk_code
 
-    @classmethod
-    def base62(cls, num):
-        if num == 0:
-            return '0'
-        ret = ''
-        while num:
-            ret = cls.BASE62_TABLE[num % 62] + ret
-            num = num // 62
-        return ret
-
     def decode_eval_codes(self):
         self.sdk_code = self.sdk_code[5:-3]
 
@@ -154,7 +143,7 @@ class IqiyiSDKInterpreter(object):
 
         while count:
             count -= 1
-            b62count = self.base62(count)
+            b62count = base62(count)
             symbol_table[b62count] = symbols[count] or b62count
 
         self.sdk_code = re.sub(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a3df90fad..d7a1586c0 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2619,3 +2619,17 @@ def ohdave_rsa_encrypt(data, exponent, modulus):
     payload = int(binascii.hexlify(data[::-1]), 16)
     encrypted = pow(payload, exponent, modulus)
     return '%x' % encrypted
+
+
+def base_n(num, n, table):
+    if num == 0:
+        return '0'
+    ret = ''
+    while num:
+        ret = table[num % n] + ret
+        num = num // n
+    return ret
+
+
+def base62(num):
+    return base_n(num, 62, '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ')

From d1e440a4a18522207a1a3e624bf801c8338f9146 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 Feb 2016 14:13:00 +0800
Subject: [PATCH 0796/1105] [jwplatform] Separate codes for for parsing
 jwplayer data

---
 youtube_dl/extractor/jwplatform.py | 59 ++++++++++++++++--------------
 1 file changed, 32 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index 8e90d5986..60a09044f 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -7,33 +7,9 @@ from .common import InfoExtractor
 from ..utils import int_or_none
 
 
-class JWPlatformIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://content\.jwplatform\.com/(?:feeds|players|jw6)/|jwplatform:)(?P<id>[a-zA-Z0-9]{8})'
-    _TEST = {
-        'url': 'http://content.jwplatform.com/players/nPripu9l-ALJ3XQCI.js',
-        'md5': 'fa8899fa601eb7c83a64e9d568bdf325',
-        'info_dict': {
-            'id': 'nPripu9l',
-            'ext': 'mov',
-            'title': 'Big Buck Bunny Trailer',
-            'description': 'Big Buck Bunny is a short animated film by the Blender Institute. It is made using free and open source software.',
-            'upload_date': '20081127',
-            'timestamp': 1227796140,
-        }
-    }
-
-    @staticmethod
-    def _extract_url(webpage):
-        mobj = re.search(
-            r'<script[^>]+?src=["\'](?P<url>(?:https?:)?//content.jwplatform.com/players/[a-zA-Z0-9]{8})',
-            webpage)
-        if mobj:
-            return mobj.group('url')
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        json_data = self._download_json('http://content.jwplatform.com/feeds/%s.json' % video_id, video_id)
-        video_data = json_data['playlist'][0]
+class JWPlatformBaseIE(InfoExtractor):
+    def _parse_jwplayer_data(self, jwplayer_data, video_id):
+        video_data = jwplayer_data['playlist'][0]
         subtitles = {}
         for track in video_data['tracks']:
             if track['kind'] == 'captions':
@@ -68,3 +44,32 @@ class JWPlatformIE(InfoExtractor):
             'subtitles': subtitles,
             'formats': formats,
         }
+
+
+class JWPlatformIE(JWPlatformBaseIE):
+    _VALID_URL = r'(?:https?://content\.jwplatform\.com/(?:feeds|players|jw6)/|jwplatform:)(?P<id>[a-zA-Z0-9]{8})'
+    _TEST = {
+        'url': 'http://content.jwplatform.com/players/nPripu9l-ALJ3XQCI.js',
+        'md5': 'fa8899fa601eb7c83a64e9d568bdf325',
+        'info_dict': {
+            'id': 'nPripu9l',
+            'ext': 'mov',
+            'title': 'Big Buck Bunny Trailer',
+            'description': 'Big Buck Bunny is a short animated film by the Blender Institute. It is made using free and open source software.',
+            'upload_date': '20081127',
+            'timestamp': 1227796140,
+        }
+    }
+
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<script[^>]+?src=["\'](?P<url>(?:https?:)?//content.jwplatform.com/players/[a-zA-Z0-9]{8})',
+            webpage)
+        if mobj:
+            return mobj.group('url')
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        json_data = self._download_json('http://content.jwplatform.com/feeds/%s.json' % video_id, video_id)
+        return self._parse_jwplayer_data(json_data, video_id)

From 481888294d19ef52075e531ce26588d97b3d16d2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 Feb 2016 14:23:16 +0800
Subject: [PATCH 0797/1105] [utils] Add base36 for use in Vidzi

---
 youtube_dl/utils.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d7a1586c0..be1f3b0d7 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2631,5 +2631,9 @@ def base_n(num, n, table):
     return ret
 
 
+def base36(num):
+    return base_n(num, 36, '0123456789abcdefghijklmnopqrstuvwxyz')
+
+
 def base62(num):
     return base_n(num, 62, '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ')

From 8f4a2124a914207912bf9fc37e593210e8dd423b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 Feb 2016 14:26:07 +0800
Subject: [PATCH 0798/1105] [vidzi] Fix extraction

---
 youtube_dl/extractor/jwplatform.py |  6 ++---
 youtube_dl/extractor/vidzi.py      | 40 +++++++++++++++++++++---------
 2 files changed, 31 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index 60a09044f..6770685d7 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -8,7 +8,7 @@ from ..utils import int_or_none
 
 
 class JWPlatformBaseIE(InfoExtractor):
-    def _parse_jwplayer_data(self, jwplayer_data, video_id):
+    def _parse_jwplayer_data(self, jwplayer_data, video_id, require_title=True):
         video_data = jwplayer_data['playlist'][0]
         subtitles = {}
         for track in video_data['tracks']:
@@ -19,7 +19,7 @@ class JWPlatformBaseIE(InfoExtractor):
         for source in video_data['sources']:
             source_url = self._proto_relative_url(source['file'])
             source_type = source.get('type') or ''
-            if source_type == 'application/vnd.apple.mpegurl':
+            if source_type in ('application/vnd.apple.mpegurl', 'hls'):
                 formats.extend(self._extract_m3u8_formats(
                     source_url, video_id, 'mp4', 'm3u8_native', fatal=False))
             elif source_type.startswith('audio'):
@@ -37,7 +37,7 @@ class JWPlatformBaseIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': video_data['title'],
+            'title': video_data['title'] if require_title else video_data.get('title'),
             'description': video_data.get('description'),
             'thumbnail': self._proto_relative_url(video_data.get('image')),
             'timestamp': int_or_none(video_data.get('pubdate')),
diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index 7c6e98026..4ec07db3a 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -1,11 +1,16 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
-from ..utils import smuggle_url
+import re
+
+from .jwplatform import JWPlatformBaseIE
+from ..utils import (
+    base36,
+    js_to_json,
+)
 
 
-class VidziIE(InfoExtractor):
+class VidziIE(JWPlatformBaseIE):
     _VALID_URL = r'https?://(?:www\.)?vidzi\.tv/(?P<id>\w+)'
     _TEST = {
         'url': 'http://vidzi.tv/cghql9yq6emu.html',
@@ -14,7 +19,6 @@ class VidziIE(InfoExtractor):
             'id': 'cghql9yq6emu',
             'ext': 'mp4',
             'title': 'youtube-dl test video  1\\\\2\'3/4<5\\\\6ä7↭',
-            'uploader': 'vidzi.tv',
         },
         'params': {
             # m3u8 download
@@ -29,11 +33,23 @@ class VidziIE(InfoExtractor):
         title = self._html_search_regex(
             r'(?s)<h2 class="video-title">(.*?)</h2>', webpage, 'title')
 
-        # Vidzi now uses jwplayer, which can be handled by GenericIE
-        return {
-            '_type': 'url_transparent',
-            'id': video_id,
-            'title': title,
-            'url': smuggle_url(url, {'to_generic': True}),
-            'ie_key': 'Generic',
-        }
+        mobj = re.search(r"}\('(.+)',36,(\d+),'([^']+)'\.split\('\|'\)", webpage)
+        code, count, symbols = mobj.groups()
+
+        count = int(count)
+        symbols = symbols.split('|')
+
+        while count:
+            count -= 1
+            if symbols[count]:
+                code = re.sub(r'\b%s\b' % base36(count), symbols[count], code)
+
+        code = code.replace('\\\'', '\'')
+        jwplayer_data = self._parse_json(
+            self._search_regex(r'setup\(([^)]+)\)', code, 'jwplayer data'),
+            video_id, transform_source=js_to_json)
+
+        info_dict = self._parse_jwplayer_data(jwplayer_data, video_id, require_title=False)
+        info_dict['title'] = title
+
+        return info_dict

From 59f898b7a72284efb994a8c6baee7771046226dd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 Feb 2016 14:37:20 +0800
Subject: [PATCH 0799/1105] [utils] Merge base_n functions

---
 youtube_dl/extractor/iqiyi.py |  4 ++--
 youtube_dl/extractor/vidzi.py |  4 ++--
 youtube_dl/utils.py           | 16 +++++++---------
 3 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 4f02b9f87..76ecd55a4 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -18,7 +18,7 @@ from ..compat import (
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
-    base62,
+    base_n,
     ExtractorError,
     ohdave_rsa_encrypt,
     remove_start,
@@ -143,7 +143,7 @@ class IqiyiSDKInterpreter(object):
 
         while count:
             count -= 1
-            b62count = base62(count)
+            b62count = base_n(count, 62)
             symbol_table[b62count] = symbols[count] or b62count
 
         self.sdk_code = re.sub(
diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index 4ec07db3a..d671e27de 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -5,7 +5,7 @@ import re
 
 from .jwplatform import JWPlatformBaseIE
 from ..utils import (
-    base36,
+    base_n,
     js_to_json,
 )
 
@@ -42,7 +42,7 @@ class VidziIE(JWPlatformBaseIE):
         while count:
             count -= 1
             if symbols[count]:
-                code = re.sub(r'\b%s\b' % base36(count), symbols[count], code)
+                code = re.sub(r'\b%s\b' % base_n(count, 36), symbols[count], code)
 
         code = code.replace('\\\'', '\'')
         jwplayer_data = self._parse_json(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index be1f3b0d7..900e07a8e 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2621,19 +2621,17 @@ def ohdave_rsa_encrypt(data, exponent, modulus):
     return '%x' % encrypted
 
 
-def base_n(num, n, table):
+def base_n(num, n, table=None):
     if num == 0:
         return '0'
+
+    FULL_TABLE = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
+    assert n <= len(FULL_TABLE)
+    if not table:
+        table = FULL_TABLE[:n]
+
     ret = ''
     while num:
         ret = table[num % n] + ret
         num = num // n
     return ret
-
-
-def base36(num):
-    return base_n(num, 36, '0123456789abcdefghijklmnopqrstuvwxyz')
-
-
-def base62(num):
-    return base_n(num, 62, '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ')

From f52354a88927eb66bcb1f603d2d91162b5bd2b5f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 Feb 2016 14:58:29 +0800
Subject: [PATCH 0800/1105] [utils] Move codes for handling eval() from
 iqiyi.py

---
 youtube_dl/extractor/iqiyi.py | 24 ++----------------------
 youtube_dl/utils.py           | 20 ++++++++++++++++++++
 2 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 76ecd55a4..2b3952210 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -18,7 +18,7 @@ from ..compat import (
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
-    base_n,
+    decode_packed_codes,
     ExtractorError,
     ohdave_rsa_encrypt,
     remove_start,
@@ -130,28 +130,8 @@ class IqiyiSDKInterpreter(object):
     def __init__(self, sdk_code):
         self.sdk_code = sdk_code
 
-    def decode_eval_codes(self):
-        self.sdk_code = self.sdk_code[5:-3]
-
-        mobj = re.search(
-            r"'([^']+)',62,(\d+),'([^']+)'\.split\('\|'\),[^,]+,{}",
-            self.sdk_code)
-        obfucasted_code, count, symbols = mobj.groups()
-        count = int(count)
-        symbols = symbols.split('|')
-        symbol_table = {}
-
-        while count:
-            count -= 1
-            b62count = base_n(count, 62)
-            symbol_table[b62count] = symbols[count] or b62count
-
-        self.sdk_code = re.sub(
-            r'\b(\w+)\b', lambda mobj: symbol_table[mobj.group(0)],
-            obfucasted_code)
-
     def run(self, target, ip, timestamp):
-        self.decode_eval_codes()
+        self.sdk_code = decode_packed_codes(self.sdk_code)
 
         functions = re.findall(r'input=([a-zA-Z0-9]+)\(input', self.sdk_code)
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 900e07a8e..fc7e2fb7f 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2635,3 +2635,23 @@ def base_n(num, n, table=None):
         ret = table[num % n] + ret
         num = num // n
     return ret
+
+
+def decode_packed_codes(code):
+    mobj = re.search(
+        r"'([^']+)',(\d+),(\d+),'([^']+)'\.split\('\|'\),[^,]+,{}",
+        code)
+    obfucasted_code, base, count, symbols = mobj.groups()
+    base = int(base)
+    count = int(count)
+    symbols = symbols.split('|')
+    symbol_table = {}
+
+    while count:
+        count -= 1
+        base_n_count = base_n(count, base)
+        symbol_table[base_n_count] = symbols[count] or base_n_count
+
+    return re.sub(
+        r'\b(\w+)\b', lambda mobj: symbol_table[mobj.group(0)],
+        obfucasted_code)

From e4fc8d2ebedfc6ea7f66ff9146940c73802a2edf Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 Feb 2016 15:00:48 +0800
Subject: [PATCH 0801/1105] [videomega] Fix extraction (closes #7606)

---
 youtube_dl/extractor/videomega.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index 5e2e7cbac..4f0dcd18c 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -4,11 +4,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import sanitized_Request
+from ..utils import (
+    decode_packed_codes,
+    sanitized_Request,
+)
 
 
 class VideoMegaIE(InfoExtractor):
-    _WORKING = False
     _VALID_URL = r'(?:videomega:|https?://(?:www\.)?videomega\.tv/(?:(?:view|iframe|cdn)\.php)?\?ref=)(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA',
@@ -42,8 +44,10 @@ class VideoMegaIE(InfoExtractor):
             r'(?:^[Vv]ideo[Mm]ega\.tv\s-\s*|\s*-\svideomega\.tv$)', '', title)
         thumbnail = self._search_regex(
             r'<video[^>]+?poster="([^"]+)"', webpage, 'thumbnail', fatal=False)
+
+        real_codes = decode_packed_codes(webpage)
         video_url = self._search_regex(
-            r'<source[^>]+?src="([^"]+)"', webpage, 'video URL')
+            r'"src"\s*,\s*"([^"]+)"', real_codes, 'video URL')
 
         return {
             'id': video_id,

From 680079be39563a2ff810602413db5245d6cbf148 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 Feb 2016 15:13:03 +0800
Subject: [PATCH 0802/1105] [utils] Relaxing regex in decode_packed_codes for
 vidzi

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index fc7e2fb7f..756ad4fd1 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2639,7 +2639,7 @@ def base_n(num, n, table=None):
 
 def decode_packed_codes(code):
     mobj = re.search(
-        r"'([^']+)',(\d+),(\d+),'([^']+)'\.split\('\|'\),[^,]+,{}",
+        r"}\('(.+)',(\d+),(\d+),'([^']+)'\.split\('\|'\)",
         code)
     obfucasted_code, base, count, symbols = mobj.groups()
     base = int(base)

From efbd6fb8bb86c07e6f924a7ec2c4bd486face3a4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 Feb 2016 15:13:35 +0800
Subject: [PATCH 0803/1105] [vidzi] Use decode_packed_codes

Javascript codes found on Vidzi are slightly different from those found
in VideoMega and iQiyi. Nevertheless, the difference has no effects on
the final result.
---
 youtube_dl/extractor/vidzi.py | 17 ++---------------
 1 file changed, 2 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index d671e27de..3c78fb3d5 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -1,11 +1,9 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .jwplatform import JWPlatformBaseIE
 from ..utils import (
-    base_n,
+    decode_packed_codes,
     js_to_json,
 )
 
@@ -33,18 +31,7 @@ class VidziIE(JWPlatformBaseIE):
         title = self._html_search_regex(
             r'(?s)<h2 class="video-title">(.*?)</h2>', webpage, 'title')
 
-        mobj = re.search(r"}\('(.+)',36,(\d+),'([^']+)'\.split\('\|'\)", webpage)
-        code, count, symbols = mobj.groups()
-
-        count = int(count)
-        symbols = symbols.split('|')
-
-        while count:
-            count -= 1
-            if symbols[count]:
-                code = re.sub(r'\b%s\b' % base_n(count, 36), symbols[count], code)
-
-        code = code.replace('\\\'', '\'')
+        code = decode_packed_codes(webpage).replace('\\\'', '\'')
         jwplayer_data = self._parse_json(
             self._search_regex(r'setup\(([^)]+)\)', code, 'jwplayer data'),
             video_id, transform_source=js_to_json)

From b78b292f0c51323edaf3e18ae4f45927a55e9198 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 Feb 2016 22:21:47 +0600
Subject: [PATCH 0804/1105] [youtube] Add alternative automatic captions
 extraction approach (Closes #8667)

---
 youtube_dl/extractor/youtube.py | 83 ++++++++++++++++++++++-----------
 1 file changed, 55 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e24dd3e5b..ec90c2111 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -975,40 +975,67 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             return {}
         try:
             args = player_config['args']
-            caption_url = args['ttsurl']
-            if not caption_url:
-                self._downloader.report_warning(err_msg)
-                return {}
-            timestamp = args['timestamp']
-            # We get the available subtitles
-            list_params = compat_urllib_parse.urlencode({
-                'type': 'list',
-                'tlangs': 1,
-                'asrs': 1,
-            })
-            list_url = caption_url + '&' + list_params
-            caption_list = self._download_xml(list_url, video_id)
-            original_lang_node = caption_list.find('track')
-            if original_lang_node is None:
-                self._downloader.report_warning('Video doesn\'t have automatic captions')
-                return {}
-            original_lang = original_lang_node.attrib['lang_code']
-            caption_kind = original_lang_node.attrib.get('kind', '')
+            caption_url = args.get('ttsurl')
+            if caption_url:
+                timestamp = args['timestamp']
+                # We get the available subtitles
+                list_params = compat_urllib_parse.urlencode({
+                    'type': 'list',
+                    'tlangs': 1,
+                    'asrs': 1,
+                })
+                list_url = caption_url + '&' + list_params
+                caption_list = self._download_xml(list_url, video_id)
+                original_lang_node = caption_list.find('track')
+                if original_lang_node is None:
+                    self._downloader.report_warning('Video doesn\'t have automatic captions')
+                    return {}
+                original_lang = original_lang_node.attrib['lang_code']
+                caption_kind = original_lang_node.attrib.get('kind', '')
+
+                sub_lang_list = {}
+                for lang_node in caption_list.findall('target'):
+                    sub_lang = lang_node.attrib['lang_code']
+                    sub_formats = []
+                    for ext in self._SUBTITLE_FORMATS:
+                        params = compat_urllib_parse.urlencode({
+                            'lang': original_lang,
+                            'tlang': sub_lang,
+                            'fmt': ext,
+                            'ts': timestamp,
+                            'kind': caption_kind,
+                        })
+                        sub_formats.append({
+                            'url': caption_url + '&' + params,
+                            'ext': ext,
+                        })
+                    sub_lang_list[sub_lang] = sub_formats
+                return sub_lang_list
+
+            # Some videos don't provide ttsurl but rather caption_tracks and
+            # caption_translation_languages (e.g. 20LmZk1hakA)
+            caption_tracks = args['caption_tracks']
+            caption_translation_languages = args['caption_translation_languages']
+            caption_url = compat_parse_qs(caption_tracks.split(',')[0])['u'][0]
+            parsed_caption_url = compat_urlparse.urlparse(caption_url)
+            caption_qs = compat_parse_qs(parsed_caption_url.query)
 
             sub_lang_list = {}
-            for lang_node in caption_list.findall('target'):
-                sub_lang = lang_node.attrib['lang_code']
+            for lang in caption_translation_languages.split(','):
+                lang_qs = compat_parse_qs(compat_urllib_parse_unquote_plus(lang))
+                sub_lang = lang_qs.get('lc', [None])[0]
+                if not sub_lang:
+                    continue
                 sub_formats = []
                 for ext in self._SUBTITLE_FORMATS:
-                    params = compat_urllib_parse.urlencode({
-                        'lang': original_lang,
-                        'tlang': sub_lang,
-                        'fmt': ext,
-                        'ts': timestamp,
-                        'kind': caption_kind,
+                    caption_qs.update({
+                        'tlang': [sub_lang],
+                        'fmt': [ext],
                     })
+                    sub_url = compat_urlparse.urlunparse(parsed_caption_url._replace(
+                        query=compat_urllib_parse.urlencode(caption_qs, True)))
                     sub_formats.append({
-                        'url': caption_url + '&' + params,
+                        'url': sub_url,
                         'ext': ext,
                     })
                 sub_lang_list[sub_lang] = sub_formats

From 2ebd2eac880e21e0fe9751e6cef28ac009f69d79 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 27 Feb 2016 00:57:35 +0800
Subject: [PATCH 0805/1105] [letv] Speedup M3U8 decryption

---
 youtube_dl/extractor/letv.py | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 9665ece89..9fd494c29 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -94,17 +94,16 @@ class LetvIE(InfoExtractor):
             return encrypted_data
         encrypted_data = encrypted_data[5:]
 
-        _loc4_ = bytearray()
-        while encrypted_data:
-            b = compat_ord(encrypted_data[0])
-            _loc4_.extend([b // 16, b & 0x0f])
-            encrypted_data = encrypted_data[1:]
+        _loc4_ = bytearray(2 * len(encrypted_data))
+        for idx, val in enumerate(encrypted_data):
+            b = compat_ord(val)
+            _loc4_[2 * idx] = b // 16
+            _loc4_[2 * idx + 1] = b % 16
         idx = len(_loc4_) - 11
         _loc4_ = _loc4_[idx:] + _loc4_[:idx]
-        _loc7_ = bytearray()
-        while _loc4_:
-            _loc7_.append(_loc4_[0] * 16 + _loc4_[1])
-            _loc4_ = _loc4_[2:]
+        _loc7_ = bytearray(len(encrypted_data))
+        for i in range(len(encrypted_data)):
+            _loc7_[i] = _loc4_[2 * i] * 16 + _loc4_[2 * i + 1]
 
         return bytes(_loc7_)
 

From 4435c6e98eb44b10c306b53c85c32458fbfeac88 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 27 Feb 2016 02:54:43 +0800
Subject: [PATCH 0806/1105] [bokecc] Add new extractor (#2336)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/bokecc.py   | 60 ++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+)
 create mode 100644 youtube_dl/extractor/bokecc.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 77e2ae425..c30679736 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -74,6 +74,7 @@ from .bleacherreport import (
 )
 from .blinkx import BlinkxIE
 from .bloomberg import BloombergIE
+from .bokecc import BokeCCIE
 from .bpb import BpbIE
 from .br import BRIE
 from .breakcom import BreakIE
diff --git a/youtube_dl/extractor/bokecc.py b/youtube_dl/extractor/bokecc.py
new file mode 100644
index 000000000..122a1cbb6
--- /dev/null
+++ b/youtube_dl/extractor/bokecc.py
@@ -0,0 +1,60 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_parse_qs
+from ..utils import ExtractorError
+
+
+class BokeCCBaseIE(InfoExtractor):
+    def _extract_bokecc_formats(self, webpage, video_id, format_id=None):
+        player_params_str = self._html_search_regex(
+            r'<(?:script|embed)[^>]+src="http://p\.bokecc\.com/player\?([^"]+)',
+            webpage, 'player params')
+
+        player_params = compat_parse_qs(player_params_str)
+
+        info_xml = self._download_xml(
+            'http://p.bokecc.com/servlet/playinfo?uid=%s&vid=%s&m=1' % (
+                player_params['siteid'][0], player_params['vid'][0]), video_id)
+
+        formats = [{
+            'format_id': format_id,
+            'url': quality.find('./copy').attrib['playurl'],
+            'preference': int(quality.attrib['value']),
+        } for quality in info_xml.findall('./video/quality')]
+
+        self._sort_formats(formats)
+
+        return formats
+
+
+class BokeCCIE(BokeCCBaseIE):
+    _IE_DESC = 'CC视频'
+    _VALID_URL = r'http://union\.bokecc\.com/playvideo\.bo\?(?P<query>.*)'
+
+    _TESTS = [{
+        'url': 'http://union.bokecc.com/playvideo.bo?vid=E44D40C15E65EA30&uid=CD0C5D3C8614B28B',
+        'info_dict': {
+            'id': 'CD0C5D3C8614B28B_E44D40C15E65EA30',
+            'ext': 'flv',
+            'title': 'BokeCC Video',
+        },
+    }]
+
+    def _real_extract(self, url):
+        qs = compat_parse_qs(re.match(self._VALID_URL, url).group('query'))
+        if not qs.get('vid') or not qs.get('uid'):
+            raise ExtractorError('Invalid URL', expected=True)
+
+        video_id = '%s_%s' % (qs['uid'][0], qs['vid'][0])
+
+        webpage = self._download_webpage(url, video_id)
+
+        return {
+            'id': video_id,
+            'title': 'BokeCC Video',  # no title provided in the webpage
+            'formats': self._extract_bokecc_formats(webpage, video_id),
+        }

From 5633b4d39d178402c6d89146c8c9c34e3bf58619 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 27 Feb 2016 02:55:11 +0800
Subject: [PATCH 0807/1105] [infoq] Use BokeCC extractor function

---
 youtube_dl/extractor/infoq.py | 31 ++++---------------------------
 1 file changed, 4 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index 016af2084..cca0b8a93 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -4,15 +4,12 @@ from __future__ import unicode_literals
 
 import base64
 
-from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_unquote,
-    compat_parse_qs,
-)
+from ..compat import compat_urllib_parse_unquote
 from ..utils import determine_ext
+from .bokecc import BokeCCBaseIE
 
 
-class InfoQIE(InfoExtractor):
+class InfoQIE(BokeCCBaseIE):
     _VALID_URL = r'https?://(?:www\.)?infoq\.com/(?:[^/]+/)+(?P<id>[^/]+)'
 
     _TESTS = [{
@@ -38,26 +35,6 @@ class InfoQIE(InfoExtractor):
         },
     }]
 
-    def _extract_bokecc_videos(self, webpage, video_id):
-        # TODO: bokecc.com is a Chinese video cloud platform
-        # It should have an independent extractor but I don't have other
-        # examples using bokecc
-        player_params_str = self._html_search_regex(
-            r'<script[^>]+src="http://p\.bokecc\.com/player\?([^"]+)',
-            webpage, 'player params', default=None)
-
-        player_params = compat_parse_qs(player_params_str)
-
-        info_xml = self._download_xml(
-            'http://p.bokecc.com/servlet/playinfo?uid=%s&vid=%s&m=1' % (
-                player_params['siteid'][0], player_params['vid'][0]), video_id)
-
-        return [{
-            'format_id': 'bokecc',
-            'url': quality.find('./copy').attrib['playurl'],
-            'preference': int(quality.attrib['value']),
-        } for quality in info_xml.findall('./video/quality')]
-
     def _extract_rtmp_videos(self, webpage):
         # The server URL is hardcoded
         video_url = 'rtmpe://video.infoq.com/cfx/st/'
@@ -101,7 +78,7 @@ class InfoQIE(InfoExtractor):
 
         if '/cn/' in url:
             # for China videos, HTTP video URL exists but always fails with 403
-            formats = self._extract_bokecc_videos(webpage, video_id)
+            formats = self._extract_bokecc_formats(webpage, video_id)
         else:
             formats = self._extract_rtmp_videos(webpage) + self._extract_http_videos(webpage)
 

From 5eb6bdced4765cdeb70411c6aa93ecb4163a9ffe Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 27 Feb 2016 03:19:50 +0800
Subject: [PATCH 0808/1105] [utils] Multiple changes to base_n()

1. Renamed to encode_base_n()
2. Allow tables longer than 62 characters
3. Raise ValueError instead of AssertionError for invalid input data
4. Return the first character in the table instead of '0' for number 0
5. Add tests
---
 test/test_utils.py  | 12 ++++++++++++
 youtube_dl/utils.py | 14 ++++++++------
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index d0736f435..97587ad2f 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -18,6 +18,7 @@ import xml.etree.ElementTree
 from youtube_dl.utils import (
     age_restricted,
     args_to_str,
+    encode_base_n,
     clean_html,
     DateRange,
     detect_exe_version,
@@ -802,5 +803,16 @@ The first line
             ohdave_rsa_encrypt(b'aa111222', e, N),
             '726664bd9a23fd0c70f9f1b84aab5e3905ce1e45a584e9cbcf9bcc7510338fc1986d6c599ff990d923aa43c51c0d9013cd572e13bc58f4ae48f2ed8c0b0ba881')
 
+    def test_encode_base_n(self):
+        self.assertEqual(encode_base_n(0, 30), '0')
+        self.assertEqual(encode_base_n(80, 30), '2k')
+
+        custom_table = '9876543210ZYXWVUTSRQPONMLKJIHGFEDCBA'
+        self.assertEqual(encode_base_n(0, 30, custom_table), '9')
+        self.assertEqual(encode_base_n(80, 30, custom_table), '7P')
+
+        self.assertRaises(ValueError, encode_base_n, 0, 70)
+        self.assertRaises(ValueError, encode_base_n, 0, 60, custom_table)
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 756ad4fd1..606977c58 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2621,15 +2621,17 @@ def ohdave_rsa_encrypt(data, exponent, modulus):
     return '%x' % encrypted
 
 
-def base_n(num, n, table=None):
-    if num == 0:
-        return '0'
-
+def encode_base_n(num, n, table=None):
     FULL_TABLE = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
-    assert n <= len(FULL_TABLE)
     if not table:
         table = FULL_TABLE[:n]
 
+    if n > len(table):
+        raise ValueError('base %d exceeds table length %d' % (n, len(table)))
+
+    if num == 0:
+        return table[0]
+
     ret = ''
     while num:
         ret = table[num % n] + ret
@@ -2649,7 +2651,7 @@ def decode_packed_codes(code):
 
     while count:
         count -= 1
-        base_n_count = base_n(count, base)
+        base_n_count = encode_base_n(count, base)
         symbol_table[base_n_count] = symbols[count] or base_n_count
 
     return re.sub(

From fbb6edd298a34cf63a5fd0bd900ca2337038810c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Feb 2016 06:48:13 +0600
Subject: [PATCH 0809/1105] [extractor/common] Properly extract audio only
 formats in master m3u8 playlists

---
 youtube_dl/extractor/common.py | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 14f575635..51351fb57 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1084,19 +1084,29 @@ class InfoExtractor(object):
                     'protocol': entry_protocol,
                     'preference': preference,
                 }
-                codecs = last_info.get('CODECS')
-                if codecs:
-                    # TODO: looks like video codec is not always necessarily goes first
-                    va_codecs = codecs.split(',')
-                    if va_codecs[0]:
-                        f['vcodec'] = va_codecs[0]
-                    if len(va_codecs) > 1 and va_codecs[1]:
-                        f['acodec'] = va_codecs[1]
                 resolution = last_info.get('RESOLUTION')
                 if resolution:
                     width_str, height_str = resolution.split('x')
                     f['width'] = int(width_str)
                     f['height'] = int(height_str)
+                codecs = last_info.get('CODECS')
+                if codecs:
+                    vcodec, acodec = [None] * 2
+                    va_codecs = codecs.split(',')
+                    if len(va_codecs) == 1:
+                        # Audio only entries usually come with single codec and
+                        # no resolution. For more robustness we also check it to
+                        # be mp4 audio.
+                        if not resolution and va_codecs[0].startswith('mp4a'):
+                            vcodec, acodec = 'none', va_codecs[0]
+                        else:
+                            vcodec = va_codecs[0]
+                    else:
+                        vcodec, acodec = va_codecs[:2]
+                    f.update({
+                        'acodec': acodec,
+                        'vcodec': vcodec,
+                    })
                 if last_media is not None:
                     f['m3u8_media'] = last_media
                     last_media = None

From 9cdffeeb3fbfa28407a8a519a58b3714b7cec874 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Feb 2016 07:01:11 +0600
Subject: [PATCH 0810/1105] [extractor/common] Clarify rationale on media
 playlist detection

---
 youtube_dl/extractor/common.py | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 51351fb57..3f16b1b9e 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1033,11 +1033,21 @@ class InfoExtractor(object):
             return []
         m3u8_doc, urlh = res
         m3u8_url = urlh.geturl()
-        # A Media Playlist Tag MUST NOT appear in a Master Playlist
-        # https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3
-        # The EXT-X-TARGETDURATION tag is REQUIRED for every M3U8 Media Playlists
-        # https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.1
-        if '#EXT-X-TARGETDURATION' in m3u8_doc:
+
+        # We should try extracting formats only from master playlists [1], i.e.
+        # playlists that describe available qualities. On the other hand media
+        # playlists [2] should be returned as is since they contain just the media
+        # without qualities renditions.
+        # Fortunately, master playlist can be easily distinguished from media
+        # playlist based on particular tags availability. As of [1, 2] master
+        # playlist tags MUST NOT appear in a media playist and vice versa.
+        # As of [3] #EXT-X-TARGETDURATION tag is REQUIRED for every media playlist
+        # and MUST NOT appear in master playlist thus we can clearly detect media
+        # playlist with this criterion.
+        # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.4
+        # 2. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3
+        # 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.1
+        if '#EXT-X-TARGETDURATION' in m3u8_doc:  # media playlist, return as is
             return [{
                 'url': m3u8_url,
                 'format_id': m3u8_id,

From f7f2e53a0a25e8c99f6f1a50cb3843d8e6f5be6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Feb 2016 15:51:25 +0600
Subject: [PATCH 0811/1105] [imdb] Recognize 1080p formats (Closes #8677)

---
 youtube_dl/extractor/imdb.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index 02e1e428e..b61b2dc4e 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -42,7 +42,7 @@ class ImdbIE(InfoExtractor):
             for f_url, f_name in extra_formats]
         format_pages.append(player_page)
 
-        quality = qualities(['SD', '480p', '720p'])
+        quality = qualities(('SD', '480p', '720p', '1080p'))
         formats = []
         for format_page in format_pages:
             json_data = self._search_regex(

From d6e9c2706fd803b5963fba47b168b6bccc063667 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Feb 2016 16:58:11 +0600
Subject: [PATCH 0812/1105] [tnaflixnetwork:embed] Add extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/tnaflix.py  | 26 +++++++++++++++++++++++++-
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c30679736..5817140c0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -737,6 +737,7 @@ from .tmz import (
     TMZArticleIE,
 )
 from .tnaflix import (
+    TNAFlixNetworkEmbedIE,
     TNAFlixIE,
     EMPFlixIE,
     MovieFapIE,
diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 49516abca..547d83d09 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -71,7 +71,7 @@ class TNAFlixNetworkBaseIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-        display_id = mobj.group('display_id')
+        display_id = mobj.group('display_id') if 'display_id' in mobj.groupdict() else video_id
 
         webpage = self._download_webpage(url, display_id)
 
@@ -152,6 +152,30 @@ class TNAFlixNetworkBaseIE(InfoExtractor):
         }
 
 
+class TNAFlixNetworkEmbedIE(TNAFlixNetworkBaseIE):
+    _VALID_URL = r'https?://player\.(?:tna|emp)flix\.com/video/(?P<id>\d+)'
+
+    _TITLE_REGEX = r'<title>([^<]+)</title>'
+
+    _TESTS = [{
+        'url': 'https://player.tnaflix.com/video/6538',
+        'info_dict': {
+            'id': '6538',
+            'display_id': '6538',
+            'ext': 'mp4',
+            'title': 'Educational xxx video',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'age_limit': 18,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://player.empflix.com/video/33051',
+        'only_matching': True,
+    }]
+
+
 class TNAFlixIE(TNAFlixNetworkBaseIE):
     _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/[^/]+/(?P<display_id>[^/]+)/video(?P<id>\d+)'
 

From 8fab62482abf8e75f2dd98fedbe95d81ce767382 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Feb 2016 16:59:10 +0600
Subject: [PATCH 0813/1105] [tnaflixnetwork] Fallback age limit to 18

---
 youtube_dl/extractor/tnaflix.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 547d83d09..e8c272704 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -117,7 +117,7 @@ class TNAFlixNetworkBaseIE(InfoExtractor):
         title = self._html_search_regex(
             self._TITLE_REGEX, webpage, 'title') if self._TITLE_REGEX else self._og_search_title(webpage)
 
-        age_limit = self._rta_search(webpage)
+        age_limit = self._rta_search(webpage) or 18
 
         duration = parse_duration(self._html_search_meta(
             'duration', webpage, 'duration', default=None))

From 63719a8ac3054f702238632495e291dd6f81e6b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Feb 2016 17:15:06 +0600
Subject: [PATCH 0814/1105] [tnaflixnetwork:embed] Add _extract_urls

---
 youtube_dl/extractor/tnaflix.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index e8c272704..79f036fe4 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -175,6 +175,12 @@ class TNAFlixNetworkEmbedIE(TNAFlixNetworkBaseIE):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_urls(webpage):
+        return [url for _, url in re.findall(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.(?:tna|emp)flix\.com/video/\d+)\1',
+            webpage)]
+
 
 class TNAFlixIE(TNAFlixNetworkBaseIE):
     _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/[^/]+/(?P<display_id>[^/]+)/video(?P<id>\d+)'

From 2c9ca78281f84abe194bdf23a5e06b747961c9eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Feb 2016 17:15:49 +0600
Subject: [PATCH 0815/1105] [extractor/generic] Add support for tnaflix network
 embeds (Closes #7505)

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c6bf8d270..ca745ae41 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -47,6 +47,7 @@ from .senateisvp import SenateISVPIE
 from .svt import SVTIE
 from .pornhub import PornHubIE
 from .xhamster import XHamsterEmbedIE
+from .tnaflix import TNAFlixNetworkEmbedIE
 from .vimeo import VimeoIE
 from .dailymotion import DailymotionCloudIE
 from .onionstudios import OnionStudiosIE
@@ -1633,6 +1634,11 @@ class GenericIE(InfoExtractor):
         if xhamster_urls:
             return _playlist_from_matches(xhamster_urls, ie='XHamsterEmbed')
 
+        # Look for embedded TNAFlixNetwork player
+        tnaflix_urls = TNAFlixNetworkEmbedIE._extract_urls(webpage)
+        if tnaflix_urls:
+            return _playlist_from_matches(tnaflix_urls, ie=TNAFlixNetworkEmbedIE.ie_key())
+
         # Look for embedded Tvigle player
         mobj = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//cloud\.tvigle\.ru/video/.+?)\1', webpage)

From 950505370416e79073acdeaa73a1023950363266 Mon Sep 17 00:00:00 2001
From: Aidan Rowe <aidanrowe@gmail.com>
Date: Mon, 8 Feb 2016 20:57:07 +1000
Subject: [PATCH 0816/1105] [dplay] add support for it.dplay.com and dplay.dk

---
 youtube_dl/extractor/dplay.py | 84 +++++++++++++++++++++++++++--------
 1 file changed, 65 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index 6cda56a7f..adcfc1f0e 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -5,40 +5,83 @@ import time
 
 from .common import InfoExtractor
 from ..utils import int_or_none
-
+from ..compat import compat_urlparse
 
 class DPlayIE(InfoExtractor):
-    _VALID_URL = r'http://www\.dplay\.se/[^/]+/(?P<id>[^/?#]+)'
+    _VALID_URL = r'(?P<domain>http://(?:it|www)\.dplay\.(?:com|dk|se))/[^/]+/(?P<id>[^/?#]+)'
 
-    _TEST = {
-        'url': 'http://www.dplay.se/nugammalt-77-handelser-som-format-sverige/season-1-svensken-lar-sig-njuta-av-livet/',
-        'info_dict': {
-            'id': '3172',
-            'ext': 'mp4',
-            'display_id': 'season-1-svensken-lar-sig-njuta-av-livet',
-            'title': 'Svensken lär sig njuta av livet',
-            'duration': 2650,
+    _TESTS = [
+        {
+            'url': 'http://it.dplay.com/take-me-out/stagione-1-episodio-25/',
+            'info_dict': {
+                'id': '1255600',
+                'ext': 'mp4',
+                'display_id': 'stagione-1-episodio-25',
+                'title': 'Episodio 25',
+                'duration': 2761,
+                'description': "Gabriele Corsi conduce un nuovo provocante e divertente dating show. 30 ragazze single hanno l'opportunità di conoscere un ragazzo e decidere se tenerlo in gioco oppure no accendendo o spegnendo le luci.",
+                'season_number': 1,
+                'episode_number': 25,
+            },
         },
-    }
+        {
+            'url': 'http://www.dplay.se/nugammalt-77-handelser-som-format-sverige/season-1-svensken-lar-sig-njuta-av-livet/',
+            'info_dict': {
+                'id': '3172',
+                'ext': 'mp4',
+                'display_id': 'season-1-svensken-lar-sig-njuta-av-livet',
+                'title': 'Svensken lär sig njuta av livet',
+                'duration': 2650,
+                'description': "\"Svensken lär sig njuta av livet\". Införandet av systembolaget, industrisemestern och Skarastadgan. Med hjälp av arkivmaterial, experter och fakta ska händelserna dissekeras, analyseras och dras till sin absoluta underhållningsspets.",
+                'season_number': 1,
+                'episode_number': 1,
+            },
+        },
+        {
+            'url': 'http://www.dplay.dk/mig-og-min-mor/season-6-episode-12/',
+            'info_dict': {
+                'id': '70816',
+                'ext': 'mp4',
+                'display_id': 'season-6-episode-12',
+                'title': 'Episode 12',
+                'duration': 2563,
+                'description': " I sæsonafslutningen sker der store ting for mor og datter.\nDagen er endelig kommet for den højgravide Irina - hun skal føde! Men det bliver en lang og sej kamp for Irina, som selvfølgelig har mor Mila med som støtte hele vejen.\nMor Jette og Jessica er igen hjemme i Danmark efter deres store USA-eventyr. Og for at holde fast i den amerikanske ånd, tager pigerne i dag til gospel-undervisning. \nOg så skal Joy og mor Mia under kniven  - de skal nemlig have gjort lårene mindre og ballerne større. \n ",
+                'season_number': 6,
+                'episode_number': 12,
+            },
+        }
+    ]
 
     def _real_extract(self, url):
+        #this extrator works with it.dplay.com, www.dplay.se and www.dplay.dk
+        # so we need to determine the domain to send the requests to
+        domain = self._search_regex(self._VALID_URL, url, 'domain')
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
         video_id = self._search_regex(
             r'data-video-id="(\d+)"', webpage, 'video id')
 
-        info = self._download_json(
-            'http://www.dplay.se/api/v2/ajax/videos?video_id=' + video_id,
+        video_url = compat_urlparse.urljoin(domain, 'api/v2/ajax/videos?video_id=')
+
+        info = self._download_json(video_url + video_id,
             video_id)['data'][0]
 
-        self._set_cookie(
-            'secure.dplay.se', 'dsc-geo',
-            '{"countryCode":"NL","expiry":%d}' % ((time.time() + 20 * 60) * 1000))
         # TODO: consider adding support for 'stream_type=hds', it seems to
         # require setting some cookies
-        manifest_url = self._download_json(
-            'https://secure.dplay.se/secure/api/v2/user/authorization/stream/%s?stream_type=hls' % video_id,
-            video_id, 'Getting manifest url for hls stream')['hls']
+        # get url's TLD to determine which cookie and url to use
+        domain_tld = domain.split('.')[-1]
+        if domain_tld == 'se' or domain_tld == 'dk':
+            self._set_cookie(
+                'secure.dplay.%s' % domain_tld, 'dsc-geo',
+                '{"countryCode":"%s","expiry":%d}' % (domain_tld.upper(), ((time.time() + 20 * 60) * 1000)))
+
+            manifest_url = self._download_json(
+                'https://secure.dplay.%s/secure/api/v2/user/authorization/stream/%s?stream_type=hls' % (domain_tld, video_id),
+                video_id, 'Getting manifest url for hls stream')['hls']
+        else:
+            #.it requires no cookies at this point
+            manifest_url = info['hls']
+
         formats = self._extract_m3u8_formats(
             manifest_url, video_id, ext='mp4', entry_protocol='m3u8_native')
 
@@ -48,4 +91,7 @@ class DPlayIE(InfoExtractor):
             'title': info['title'],
             'formats': formats,
             'duration': int_or_none(info.get('video_metadata_length'), scale=1000),
+            'description': info.get('video_metadata_longDescription'),
+            'season_number': int_or_none(info.get('season')),
+            'episode_number': int_or_none(info.get('episode')),
         }

From 940b606a0743c0f23aa4313019d8af67d863f064 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Feb 2016 21:30:47 +0600
Subject: [PATCH 0817/1105] [dplay] Improve, extract all formats and metadata
 (Closes #8463)

---
 youtube_dl/extractor/dplay.py | 172 ++++++++++++++++++++--------------
 1 file changed, 103 insertions(+), 69 deletions(-)

diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index adcfc1f0e..87071c4f8 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -1,97 +1,131 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
+import json
+import re
 import time
 
 from .common import InfoExtractor
 from ..utils import int_or_none
-from ..compat import compat_urlparse
+
 
 class DPlayIE(InfoExtractor):
-    _VALID_URL = r'(?P<domain>http://(?:it|www)\.dplay\.(?:com|dk|se))/[^/]+/(?P<id>[^/?#]+)'
+    _VALID_URL = r'http://(?P<domain>it\.dplay\.com|www\.dplay\.(?:dk|se))/[^/]+/(?P<id>[^/?#]+)'
 
-    _TESTS = [
-        {
-            'url': 'http://it.dplay.com/take-me-out/stagione-1-episodio-25/',
-            'info_dict': {
-                'id': '1255600',
-                'ext': 'mp4',
-                'display_id': 'stagione-1-episodio-25',
-                'title': 'Episodio 25',
-                'duration': 2761,
-                'description': "Gabriele Corsi conduce un nuovo provocante e divertente dating show. 30 ragazze single hanno l'opportunità di conoscere un ragazzo e decidere se tenerlo in gioco oppure no accendendo o spegnendo le luci.",
-                'season_number': 1,
-                'episode_number': 25,
-            },
+    _TESTS = [{
+        'url': 'http://it.dplay.com/take-me-out/stagione-1-episodio-25/',
+        'info_dict': {
+            'id': '1255600',
+            'display_id': 'stagione-1-episodio-25',
+            'ext': 'mp4',
+            'title': 'Episodio 25',
+            'description': 'md5:cae5f40ad988811b197d2d27a53227eb',
+            'duration': 2761,
+            'timestamp': 1454701800,
+            'upload_date': '20160205',
+            'creator': 'RTIT',
+            'series': 'Take me out',
+            'season_number': 1,
+            'episode_number': 25,
+            'age_limit': 0,
         },
-        {
-            'url': 'http://www.dplay.se/nugammalt-77-handelser-som-format-sverige/season-1-svensken-lar-sig-njuta-av-livet/',
-            'info_dict': {
-                'id': '3172',
-                'ext': 'mp4',
-                'display_id': 'season-1-svensken-lar-sig-njuta-av-livet',
-                'title': 'Svensken lär sig njuta av livet',
-                'duration': 2650,
-                'description': "\"Svensken lär sig njuta av livet\". Införandet av systembolaget, industrisemestern och Skarastadgan. Med hjälp av arkivmaterial, experter och fakta ska händelserna dissekeras, analyseras och dras till sin absoluta underhållningsspets.",
-                'season_number': 1,
-                'episode_number': 1,
-            },
+        'expected_warnings': ['Unable to download f4m manifest'],
+    }, {
+        'url': 'http://www.dplay.se/nugammalt-77-handelser-som-format-sverige/season-1-svensken-lar-sig-njuta-av-livet/',
+        'info_dict': {
+            'id': '3172',
+            'display_id': 'season-1-svensken-lar-sig-njuta-av-livet',
+            'ext': 'flv',
+            'title': 'Svensken lär sig njuta av livet',
+            'description': 'md5:d3819c9bccffd0fe458ca42451dd50d8',
+            'duration': 2650,
+            'timestamp': 1365454320,
+            'upload_date': '20130408',
+            'creator': 'Kanal 5 (Home)',
+            'series': 'Nugammalt - 77 händelser som format Sverige',
+            'season_number': 1,
+            'episode_number': 1,
+            'age_limit': 0,
         },
-        {
-            'url': 'http://www.dplay.dk/mig-og-min-mor/season-6-episode-12/',
-            'info_dict': {
-                'id': '70816',
-                'ext': 'mp4',
-                'display_id': 'season-6-episode-12',
-                'title': 'Episode 12',
-                'duration': 2563,
-                'description': " I sæsonafslutningen sker der store ting for mor og datter.\nDagen er endelig kommet for den højgravide Irina - hun skal føde! Men det bliver en lang og sej kamp for Irina, som selvfølgelig har mor Mila med som støtte hele vejen.\nMor Jette og Jessica er igen hjemme i Danmark efter deres store USA-eventyr. Og for at holde fast i den amerikanske ånd, tager pigerne i dag til gospel-undervisning. \nOg så skal Joy og mor Mia under kniven  - de skal nemlig have gjort lårene mindre og ballerne større. \n ",
-                'season_number': 6,
-                'episode_number': 12,
-            },
-        }
-    ]
+    }, {
+        'url': 'http://www.dplay.dk/mig-og-min-mor/season-6-episode-12/',
+        'info_dict': {
+            'id': '70816',
+            'display_id': 'season-6-episode-12',
+            'ext': 'flv',
+            'title': 'Episode 12',
+            'description': 'md5:9c86e51a93f8a4401fc9641ef9894c90',
+            'duration': 2563,
+            'timestamp': 1429696800,
+            'upload_date': '20150422',
+            'creator': 'Kanal 4',
+            'series': 'Mig og min mor',
+            'season_number': 6,
+            'episode_number': 12,
+            'age_limit': 0,
+        },
+    }]
 
     def _real_extract(self, url):
-        #this extrator works with it.dplay.com, www.dplay.se and www.dplay.dk
-        # so we need to determine the domain to send the requests to
-        domain = self._search_regex(self._VALID_URL, url, 'domain')
-        display_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('id')
+        domain = mobj.group('domain')
+
         webpage = self._download_webpage(url, display_id)
+
         video_id = self._search_regex(
-            r'data-video-id="(\d+)"', webpage, 'video id')
+            r'data-video-id=["\'](\d+)', webpage, 'video id')
 
-        video_url = compat_urlparse.urljoin(domain, 'api/v2/ajax/videos?video_id=')
-
-        info = self._download_json(video_url + video_id,
+        info = self._download_json(
+            'http://%s/api/v2/ajax/videos?video_id=%s' % (domain, video_id),
             video_id)['data'][0]
 
-        # TODO: consider adding support for 'stream_type=hds', it seems to
-        # require setting some cookies
-        # get url's TLD to determine which cookie and url to use
+        title = info['title']
+
+        PROTOCOLS = ('hls', 'hds')
+        formats = []
+
+        def extract_formats(protocol, manifest_url):
+            if protocol == 'hls':
+                formats.extend(self._extract_m3u8_formats(
+                    manifest_url, video_id, ext='mp4',
+                    entry_protocol='m3u8_native', m3u8_id=protocol, fatal=False))
+            elif protocol == 'hds':
+                formats.extend(self._extract_f4m_formats(
+                    manifest_url + '&hdcore=3.8.0&plugin=flowplayer-3.8.0.0',
+                    video_id, f4m_id=protocol, fatal=False))
+
         domain_tld = domain.split('.')[-1]
-        if domain_tld == 'se' or domain_tld == 'dk':
-            self._set_cookie(
-                'secure.dplay.%s' % domain_tld, 'dsc-geo',
-                '{"countryCode":"%s","expiry":%d}' % (domain_tld.upper(), ((time.time() + 20 * 60) * 1000)))
-
-            manifest_url = self._download_json(
-                'https://secure.dplay.%s/secure/api/v2/user/authorization/stream/%s?stream_type=hls' % (domain_tld, video_id),
-                video_id, 'Getting manifest url for hls stream')['hls']
+        if domain_tld in ('se', 'dk'):
+            for protocol in PROTOCOLS:
+                self._set_cookie(
+                    'secure.dplay.%s' % domain_tld, 'dsc-geo',
+                    json.dumps({
+                        'countryCode': domain_tld.upper(),
+                        'expiry': (time.time() + 20 * 60) * 1000,
+                    }))
+                stream = self._download_json(
+                    'https://secure.dplay.%s/secure/api/v2/user/authorization/stream/%s?stream_type=%s'
+                    % (domain_tld, video_id, protocol), video_id,
+                    'Downloading %s stream JSON' % protocol, fatal=False)
+                if stream and stream.get(protocol):
+                    extract_formats(protocol, stream[protocol])
         else:
-            #.it requires no cookies at this point
-            manifest_url = info['hls']
-
-        formats = self._extract_m3u8_formats(
-            manifest_url, video_id, ext='mp4', entry_protocol='m3u8_native')
+            for protocol in PROTOCOLS:
+                if info.get(protocol):
+                    extract_formats(protocol, info[protocol])
 
         return {
             'id': video_id,
             'display_id': display_id,
-            'title': info['title'],
-            'formats': formats,
-            'duration': int_or_none(info.get('video_metadata_length'), scale=1000),
+            'title': title,
             'description': info.get('video_metadata_longDescription'),
+            'duration': int_or_none(info.get('video_metadata_length'), scale=1000),
+            'timestamp': int_or_none(info.get('video_publish_date')),
+            'creator': info.get('video_metadata_homeChannel'),
+            'series': info.get('video_metadata_show'),
             'season_number': int_or_none(info.get('season')),
             'episode_number': int_or_none(info.get('episode')),
+            'age_limit': int_or_none(info.get('minimum_age')),
+            'formats': formats,
         }

From 20afe8bd14c76af308d4fe2fce4cef05067b10c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Feb 2016 21:31:43 +0600
Subject: [PATCH 0818/1105] Credit @aidan- for more dplay sites support (#8463)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 0e90a3ecb..b51e23f2d 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -160,3 +160,4 @@ Erwin de Haan
 Jens Wille
 Robin Houtevelts
 Patrick Griffis
+Aidan Rowe

From 5add979d918ceb2718026bdfa66f141078b09b19 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Feb 2016 21:42:08 +0600
Subject: [PATCH 0819/1105] [dplay] Add support for dplay.no

---
 youtube_dl/extractor/dplay.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index 87071c4f8..a638c827c 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -10,7 +10,7 @@ from ..utils import int_or_none
 
 
 class DPlayIE(InfoExtractor):
-    _VALID_URL = r'http://(?P<domain>it\.dplay\.com|www\.dplay\.(?:dk|se))/[^/]+/(?P<id>[^/?#]+)'
+    _VALID_URL = r'http://(?P<domain>it\.dplay\.com|www\.dplay\.(?:dk|se|no))/[^/]+/(?P<id>[^/?#]+)'
 
     _TESTS = [{
         'url': 'http://it.dplay.com/take-me-out/stagione-1-episodio-25/',
@@ -64,6 +64,9 @@ class DPlayIE(InfoExtractor):
             'episode_number': 12,
             'age_limit': 0,
         },
+    }, {
+        'url': 'http://www.dplay.no/pga-tour/season-1-hoydepunkter-18-21-februar/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From da665ddc2551147b7b38f919c9d5c719a93f9c5d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 27 Feb 2016 21:31:21 +0100
Subject: [PATCH 0820/1105] release 2016.02.27

---
 CONTRIBUTING.md        | 13 ++++++++-----
 docs/supportedsites.md |  6 ++++--
 youtube_dl/version.py  |  2 +-
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 39472c554..c996f03ab 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -92,7 +92,9 @@ If you want to create a build of youtube-dl yourself, you'll need
 
 ### Adding support for a new site
 
-If you want to add support for a new site, you can follow this quick list (assuming your service is called `yourextractor`):
+If you want to add support for a new site, first of all **make sure** this site is **not dedicated to [copyright infringement](#can-you-add-support-for-this-anime-video-site-or-site-which-shows-current-movies-for-free)**. youtube-dl does **not support** such sites thus pull requests adding support for them **will be rejected**.
+
+After you have ensured this site is distributing it's content legally, you can follow this quick list (assuming your service is called `yourextractor`):
 
 1. [Fork this repository](https://github.com/rg3/youtube-dl/fork)
 2. Check out the source code with `git clone git@github.com:YOUR_GITHUB_USERNAME/youtube-dl.git`
@@ -140,16 +142,17 @@ If you want to add support for a new site, you can follow this quick list (assum
     ```
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
-7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62-L200). Add tests and code for as many as you want.
-8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
-9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
+7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L68-L226). Add tests and code for as many as you want.
+8. Keep in mind that the only mandatory fields in info dict for successful extraction process are `id`, `title` and either `url` or `formats`, i.e. these are the critical data the extraction does not make any sense without. This means that [any field](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L138-L226) apart from aforementioned mandatory ones should be treated **as optional** and extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields. For example, if you have some intermediate dict `meta` that is a source of metadata and it has a key `summary` that you want to extract and put into resulting info dict as `description`, you should be ready that this key may be missing from the `meta` dict, i.e. you should extract it as `meta.get('summary')` and not `meta['summary']`. Similarly, you should pass `fatal=False` when extracting data from a webpage with `_search_regex/_html_search_regex`.
+9. Check the code with [flake8](https://pypi.python.org/pypi/flake8).
+10. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
         $ git add youtube_dl/extractor/__init__.py
         $ git add youtube_dl/extractor/yourextractor.py
         $ git commit -m '[yourextractor] Add new extractor'
         $ git push origin yourextractor
 
-10. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it.
+11. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it.
 
 In any case, thank you very much for your contributions!
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 74596155c..43403233d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -77,6 +77,7 @@
  - **BleacherReportCMS**
  - **blinkx**
  - **Bloomberg**
+ - **BokeCC**
  - **Bpb**: Bundeszentrale für politische Bildung
  - **BR**: Bayerischer Rundfunk Mediathek
  - **Break**
@@ -560,7 +561,6 @@
  - **southpark.de**
  - **southpark.nl**
  - **southparkstudios.dk**
- - **Space**
  - **SpankBang**
  - **Spankwire**
  - **Spiegel**
@@ -620,6 +620,7 @@
  - **TMZ**
  - **TMZArticle**
  - **TNAFlix**
+ - **TNAFlixNetworkEmbed**
  - **toggle**
  - **tou.tv**
  - **Toypics**: Toypics user profile
@@ -670,6 +671,7 @@
  - **Urort**: NRK P3 Urørt
  - **ustream**
  - **ustream:channel**
+ - **Ustudio**
  - **Varzesh3**
  - **Vbox7**
  - **VeeHD**
@@ -685,7 +687,7 @@
  - **video.mit.edu**
  - **VideoDetective**
  - **videofy.me**
- - **VideoMega** (Currently broken)
+ - **VideoMega**
  - **videomore**
  - **videomore:season**
  - **videomore:video**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7a3df6a26..e2836357f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.02.22'
+__version__ = '2016.02.27'

From 4587915b2aed8d2a8373614e5984fe6e5ccaeae2 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 28 Feb 2016 02:56:09 +0600
Subject: [PATCH 0821/1105] [README.md] Make configuration file example more
 diverse

---
 README.md | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index d50d4b950..7d9d501d7 100644
--- a/README.md
+++ b/README.md
@@ -409,11 +409,14 @@ which means you can modify it, redistribute it or use it however you like.
 
 # CONFIGURATION
 
-You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime and use a proxy:
+You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`.
+
+For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime, use a proxy and save all videos under `Movies` directory in your `$HOME`:
 ```
---extract-audio
+-x
 --no-mtime
 --proxy 127.0.0.1:3128
+-o ~/Movies/%(title)s.%(ext)s
 ```
 
 You can use `--ignore-config` if you want to disable the configuration file for a particular youtube-dl run.

From a048ffc9b01a4c6a63603edbb7204479ddebbec6 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 28 Feb 2016 03:04:06 +0600
Subject: [PATCH 0822/1105] [README.md] Clarify configuration file options
 syntax

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 7d9d501d7..ea4ad327b 100644
--- a/README.md
+++ b/README.md
@@ -419,6 +419,8 @@ For example, with the following configuration file youtube-dl will always extrac
 -o ~/Movies/%(title)s.%(ext)s
 ```
 
+Note that options in configuration file are just the same options aka switches used in regular command line calls thus there **must be no whitespace** after `-` or `--`, e.g. `-o` or `--proxy` but not `- o` or `-- proxy`.
+
 You can use `--ignore-config` if you want to disable the configuration file for a particular youtube-dl run.
 
 ### Authentication with `.netrc` file

From 7a0e7779fe571972b72edc6aab4b8c93db4b22e8 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 28 Feb 2016 03:12:13 +0600
Subject: [PATCH 0823/1105] [README.md] Use simple wording instead of env
 variable for home

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ea4ad327b..44b65da02 100644
--- a/README.md
+++ b/README.md
@@ -411,7 +411,7 @@ which means you can modify it, redistribute it or use it however you like.
 
 You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`.
 
-For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime, use a proxy and save all videos under `Movies` directory in your `$HOME`:
+For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime, use a proxy and save all videos under `Movies` directory in your home directory:
 ```
 -x
 --no-mtime

From 8870bb4653982a81c2ff332103499e12a825099c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Feb 2016 03:37:48 +0600
Subject: [PATCH 0824/1105] [webofstories] Tolerate malforder og:title (Closes
 #8417)

---
 youtube_dl/extractor/webofstories.py | 64 +++++++++++++++++-----------
 1 file changed, 39 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/webofstories.py b/youtube_dl/extractor/webofstories.py
index 2037d9b3d..7aea47ed5 100644
--- a/youtube_dl/extractor/webofstories.py
+++ b/youtube_dl/extractor/webofstories.py
@@ -12,38 +12,52 @@ class WebOfStoriesIE(InfoExtractor):
     _VIDEO_DOMAIN = 'http://eu-mobile.webofstories.com/'
     _GREAT_LIFE_STREAMER = 'rtmp://eu-cdn1.webofstories.com/cfx/st/'
     _USER_STREAMER = 'rtmp://eu-users.webofstories.com/cfx/st/'
-    _TESTS = [
-        {
-            'url': 'http://www.webofstories.com/play/hans.bethe/71',
-            'md5': '373e4dd915f60cfe3116322642ddf364',
-            'info_dict': {
-                'id': '4536',
-                'ext': 'mp4',
-                'title': 'The temperature of the sun',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'description': 'Hans Bethe talks about calculating the temperature of the sun',
-                'duration': 238,
-            }
+    _TESTS = [{
+        'url': 'http://www.webofstories.com/play/hans.bethe/71',
+        'md5': '373e4dd915f60cfe3116322642ddf364',
+        'info_dict': {
+            'id': '4536',
+            'ext': 'mp4',
+            'title': 'The temperature of the sun',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'Hans Bethe talks about calculating the temperature of the sun',
+            'duration': 238,
+        }
+    }, {
+        'url': 'http://www.webofstories.com/play/55908',
+        'md5': '2985a698e1fe3211022422c4b5ed962c',
+        'info_dict': {
+            'id': '55908',
+            'ext': 'mp4',
+            'title': 'The story of Gemmata obscuriglobus',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'Planctomycete talks about The story of Gemmata obscuriglobus',
+            'duration': 169,
         },
-        {
-            'url': 'http://www.webofstories.com/play/55908',
-            'md5': '2985a698e1fe3211022422c4b5ed962c',
-            'info_dict': {
-                'id': '55908',
-                'ext': 'mp4',
-                'title': 'The story of Gemmata obscuriglobus',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'description': 'Planctomycete talks about The story of Gemmata obscuriglobus',
-                'duration': 169,
-            }
+        'skip': 'notfound',
+    }, {
+        # malformed og:title meta
+        'url': 'http://www.webofstories.com/play/54215?o=MS',
+        'info_dict': {
+            'id': '54215',
+            'ext': 'mp4',
+            'title': '"A Leg to Stand On"',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'Oliver Sacks talks about the death and resurrection of a limb',
+            'duration': 97,
         },
-    ]
+        'params': {
+            'skip_download': True,
+        },
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
-        title = self._og_search_title(webpage)
+        # Sometimes og:title meta is malformed
+        title = self._og_search_title(webpage, default=None) or self._html_search_regex(
+            r'(?s)<strong>Title:\s*</strong>(.+?)<', webpage, 'title')
         description = self._html_search_meta('description', webpage)
         thumbnail = self._og_search_thumbnail(webpage)
 

From 9173202b84ea303c83c30b7305a1dcec5ccbe9e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 28 Feb 2016 14:06:26 +0100
Subject: [PATCH 0825/1105] [zdf] Ignore hls manifests that use https (closes
 #8665)

The certificates are misconfigured, you get the following error mesage:

    ssl.CertificateError: hostname u'zdf-hdios-none-i.zdf.de' doesn't match either of 'a248.e.akamai.net', '*.akamaihd.net', '*.akamaihd-staging.net', '*.akamaized.net', '*.akamaized-staging.net'
---
 youtube_dl/extractor/zdf.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index c619a75e2..81c22a627 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -137,6 +137,10 @@ class ZDFIE(InfoExtractor):
                 formats.extend(self._extract_smil_formats(
                     video_url, video_id, fatal=False))
             elif ext == 'm3u8':
+                # the certificates are misconfigured (see
+                # https://github.com/rg3/youtube-dl/issues/8665)
+                if video_url.startswith('https://'):
+                    continue
                 formats.extend(self._extract_m3u8_formats(
                     video_url, video_id, 'mp4', m3u8_id=format_id, fatal=False))
             elif ext == 'f4m':

From 6c10dbeae948b491b538f70481ee5b348a636067 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Feb 2016 20:05:58 +0600
Subject: [PATCH 0826/1105] [screenwavemedia] Improve formats extraction

---
 youtube_dl/extractor/screenwavemedia.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 2cf210e0d..d2af26c1c 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -70,19 +70,19 @@ class ScreenwaveMediaIE(InfoExtractor):
 
         formats = []
         for source in sources:
-            if source['type'] == 'hls':
-                formats.extend(self._extract_m3u8_formats(source['file'], video_id, ext='mp4'))
+            file_ = source.get('file')
+            if not file_:
+                continue
+            if source.get('type') == 'hls':
+                formats.extend(self._extract_m3u8_formats(file_, video_id, ext='mp4'))
             else:
-                file_ = source.get('file')
-                if not file_:
-                    continue
                 format_label = source.get('label')
                 format_id = self._search_regex(
                     r'_(.+?)\.[^.]+$', file_, 'format id', default=None)
                 height = int_or_none(self._search_regex(
                     r'^(\d+)[pP]', format_label, 'height', default=None))
                 formats.append({
-                    'url': source['file'],
+                    'url': file_,
                     'format_id': format_id,
                     'format': format_label,
                     'ext': source.get('type'),

From 2b2dfae83ef80f86e9983ded3c4514b58b1eed60 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Feb 2016 20:16:31 +0600
Subject: [PATCH 0827/1105] [screenwavemedia] Improve formats sorting

---
 youtube_dl/extractor/screenwavemedia.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index d2af26c1c..8e77788ae 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -88,7 +88,7 @@ class ScreenwaveMediaIE(InfoExtractor):
                     'ext': source.get('type'),
                     'height': height,
                 })
-        self._sort_formats(formats)
+        self._sort_formats(formats, field_preference=('height', 'width', 'tbr', 'format_id'))
 
         return {
             'id': video_id,

From 6dae56384a3a50f832b647f798a4b4bbd770448f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Feb 2016 21:46:36 +0600
Subject: [PATCH 0828/1105] [screenwavemedia] Check formats' URLs

---
 youtube_dl/extractor/screenwavemedia.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 8e77788ae..44b0bbee6 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -76,9 +76,11 @@ class ScreenwaveMediaIE(InfoExtractor):
             if source.get('type') == 'hls':
                 formats.extend(self._extract_m3u8_formats(file_, video_id, ext='mp4'))
             else:
-                format_label = source.get('label')
                 format_id = self._search_regex(
                     r'_(.+?)\.[^.]+$', file_, 'format id', default=None)
+                if not self._is_valid_url(file_, video_id, format_id or 'video'):
+                    continue
+                format_label = source.get('label')
                 height = int_or_none(self._search_regex(
                     r'^(\d+)[pP]', format_label, 'height', default=None))
                 formats.append({

From 4b3cd7316cbb95100f7fc4dd03d86e0fd7674996 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Feb 2016 03:28:21 +0600
Subject: [PATCH 0829/1105] [tf1] Improve wat id regex (Closes #8691)

---
 youtube_dl/extractor/tf1.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index e1a64e284..9ee844684 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -48,6 +48,6 @@ class TF1IE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         wat_id = self._html_search_regex(
-            r'(["\'])(?:https?:)?//www\.wat\.tv/embedframe/.*?(?P<id>\d{8})\1',
+            r'(["\'])(?:https?:)?//www\.wat\.tv/embedframe/.*?(?P<id>\d{8})(?:#.*?)?\1',
             webpage, 'wat id', group='id')
         return self.url_result('wat:%s' % wat_id, 'Wat')

From d77ab8e255e593d8534bdd47e84c0cc03c4e6efd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Mar 2016 01:01:33 +0600
Subject: [PATCH 0830/1105] Add --mark-watched feature (Closes #5054)

---
 youtube_dl/__init__.py          |  1 +
 youtube_dl/extractor/common.py  |  9 +++++++++
 youtube_dl/extractor/youtube.py | 26 ++++++++++++++++++++++++++
 youtube_dl/options.py           |  4 ++++
 4 files changed, 40 insertions(+)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index f5f064241..79b389840 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -355,6 +355,7 @@ def _real_main(argv=None):
         'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
         'encoding': opts.encoding,
         'extract_flat': opts.extract_flat,
+        'mark_watched': opts.mark_watched,
         'merge_output_format': opts.merge_output_format,
         'postprocessors': postprocessors,
         'fixup': opts.fixup,
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 3f16b1b9e..a7c700099 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1620,6 +1620,15 @@ class InfoExtractor(object):
     def _get_automatic_captions(self, *args, **kwargs):
         raise NotImplementedError('This method must be implemented by subclasses')
 
+    def mark_watched(self, *args, **kwargs):
+        if (self._downloader.params.get('mark_watched', False) and
+                (self._get_login_info()[0] is not None or
+                    self._downloader.params.get('cookiefile') is not None)):
+            self._mark_watched(*args, **kwargs)
+
+    def _mark_watched(self, *args, **kwargs):
+        raise NotImplementedError('This method must be implemented by subclasses')
+
 
 class SearchInfoExtractor(InfoExtractor):
     """
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ec90c2111..ba339b67d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -6,6 +6,7 @@ from __future__ import unicode_literals
 import itertools
 import json
 import os.path
+import random
 import re
 import time
 import traceback
@@ -1046,6 +1047,29 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             self._downloader.report_warning(err_msg)
             return {}
 
+    def _mark_watched(self, video_id, video_info):
+        playback_url = video_info.get('videostats_playback_base_url', [None])[0]
+        if not playback_url:
+            return
+        parsed_playback_url = compat_urlparse.urlparse(playback_url)
+        qs = compat_urlparse.parse_qs(parsed_playback_url.query)
+
+        # cpn generation algorithm is reverse engineered from base.js.
+        # In fact it works even with dummy cpn.
+        CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
+        cpn = ''.join((CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16)))
+
+        qs.update({
+            'ver': ['2'],
+            'cpn': [cpn],
+        })
+        playback_url = compat_urlparse.urlunparse(
+            parsed_playback_url._replace(query=compat_urllib_parse.urlencode(qs, True)))
+
+        self._download_webpage(
+            playback_url, video_id, 'Marking watched',
+            'Unable to mark watched', fatal=False)
+
     @classmethod
     def extract_id(cls, url):
         mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
@@ -1555,6 +1579,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         self._sort_formats(formats)
 
+        self.mark_watched(video_id, video_info)
+
         return {
             'id': video_id,
             'uploader': video_uploader,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 3afa8bb6f..048dee881 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -170,6 +170,10 @@ def parseOpts(overrideArguments=None):
         action='store_const', dest='extract_flat', const='in_playlist',
         default=False,
         help='Do not extract the videos of a playlist, only list them.')
+    general.add_option(
+        '--mark-watched',
+        action='store_true', dest='mark_watched', default=False,
+        help='Mark videos watched (YouTube only)')
     general.add_option(
         '--no-color', '--no-colors',
         action='store_true', dest='no_color',

From 2812c24c167dfaeed62737f2b5ba2a5c7c0de97f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Mar 2016 01:24:26 +0600
Subject: [PATCH 0831/1105] [mdr] Fix extraction (Closes #8702)

---
 youtube_dl/extractor/mdr.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index 425fc9e2a..2338e7f96 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 class MDRIE(InfoExtractor):
     IE_DESC = 'MDR.DE and KiKA'
-    _VALID_URL = r'https?://(?:www\.)?(?:mdr|kika)\.de/(?:.*)/[a-z]+(?P<id>\d+)(?:_.+?)?\.html'
+    _VALID_URL = r'https?://(?:www\.)?(?:mdr|kika)\.de/(?:.*)/[a-z]+-?(?P<id>\d+)(?:_.+?)?\.html'
 
     _TESTS = [{
         # MDR regularly deletes its videos
@@ -60,6 +60,9 @@ class MDRIE(InfoExtractor):
     }, {
         'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/einzelsendung2534.html',
         'only_matching': True,
+    }, {
+        'url': 'http://www.mdr.de/mediathek/mdr-videos/a/video-1334.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -68,8 +71,8 @@ class MDRIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         data_url = self._search_regex(
-            r'dataURL\s*:\s*(["\'])(?P<url>/.+/(?:video|audio)[0-9]+-avCustom\.xml)\1',
-            webpage, 'data url', group='url')
+            r'(?:dataURL|playerXml(?:["\'])?)\s*:\s*(["\'])(?P<url>\\?/.+/(?:video|audio)-?[0-9]+-avCustom\.xml)\1',
+            webpage, 'data url', default=None, group='url').replace('\/', '/')
 
         doc = self._download_xml(
             compat_urlparse.urljoin(url, data_url), video_id)

From 3e7696822064c6ec7e7d62aab98093d89eeb7cad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 29 Feb 2016 20:57:26 +0100
Subject: [PATCH 0832/1105] [rtve.es:live] Fix extraction

* Update _VALID_URL to match the current URLs
* Use the m3u8 manifest since I haven't figured out how to use the rtmp stream
---
 youtube_dl/extractor/rtve.py | 30 ++++++++++++++----------------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 603d7bd00..8a8c5d2a0 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -10,6 +10,7 @@ from ..utils import (
     ExtractorError,
     float_or_none,
     remove_end,
+    remove_start,
     sanitized_Request,
     std_headers,
     struct_unpack,
@@ -178,14 +179,14 @@ class RTVEInfantilIE(InfoExtractor):
 class RTVELiveIE(InfoExtractor):
     IE_NAME = 'rtve.es:live'
     IE_DESC = 'RTVE.es live streams'
-    _VALID_URL = r'http://www\.rtve\.es/(?:deportes/directo|noticias|television)/(?P<id>[a-zA-Z0-9-]+)'
+    _VALID_URL = r'http://www\.rtve\.es/directo/(?P<id>[a-zA-Z0-9-]+)'
 
     _TESTS = [{
-        'url': 'http://www.rtve.es/noticias/directo-la-1/',
+        'url': 'http://www.rtve.es/directo/la-1/',
         'info_dict': {
-            'id': 'directo-la-1',
-            'ext': 'flv',
-            'title': 're:^La 1 de TVE [0-9]{4}-[0-9]{2}-[0-9]{2}Z[0-9]{6}$',
+            'id': 'la-1',
+            'ext': 'mp4',
+            'title': 're:^La 1 [0-9]{4}-[0-9]{2}-[0-9]{2}Z[0-9]{6}$',
         },
         'params': {
             'skip_download': 'live stream',
@@ -198,23 +199,20 @@ class RTVELiveIE(InfoExtractor):
         video_id = mobj.group('id')
 
         webpage = self._download_webpage(url, video_id)
-        player_url = self._search_regex(
-            r'<param name="movie" value="([^"]+)"/>', webpage, 'player URL')
-        title = remove_end(self._og_search_title(webpage), ' en directo')
+        title = remove_end(self._og_search_title(webpage), ' en directo en RTVE.es')
+        title = remove_start(title, 'Estoy viendo ')
         title += ' ' + time.strftime('%Y-%m-%dZ%H%M%S', start_time)
 
         vidplayer_id = self._search_regex(
-            r' id="vidplayer([0-9]+)"', webpage, 'internal video ID')
-        png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/default/videos/%s.png' % vidplayer_id
+            r'playerId=player([0-9]+)', webpage, 'internal video ID')
+        png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/amonet/videos/%s.png' % vidplayer_id
         png = self._download_webpage(png_url, video_id, 'Downloading url information')
-        video_url = _decrypt_url(png)
+        m3u8_url = _decrypt_url(png)
+        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
 
         return {
             'id': video_id,
-            'ext': 'flv',
             'title': title,
-            'url': video_url,
-            'app': 'rtve-live-live?ovpfv=2.1.2',
-            'player_url': player_url,
-            'rtmp_live': True,
+            'formats': formats,
+            'is_live': True,
         }

From e781ab63dba6968f54adb612907c76f6f9896095 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 1 Mar 2016 00:05:39 +0100
Subject: [PATCH 0833/1105] release 2016.03.01

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e2836357f..adafd601b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.02.27'
+__version__ = '2016.03.01'

From 9fb556eef0c6e83594ca4a7fe4d924a6a2a43a60 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 1 Mar 2016 08:42:33 +0800
Subject: [PATCH 0834/1105] [iqiyi] SWF URLs are not used anymore

Since automatic detection of enc_key failed

Closes #8705
---
 youtube_dl/extractor/iqiyi.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 2b3952210..d3bee3a19 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -498,7 +498,7 @@ class IqiyiIE(InfoExtractor):
         raw_data = self._download_json(api_url, video_id)
         return raw_data
 
-    def get_enc_key(self, swf_url, video_id):
+    def get_enc_key(self, video_id):
         # TODO: automatic key extraction
         # last update at 2016-01-22 for Zombie::bite
         enc_key = '6ab6d0280511493ba85594779759d4ed'
@@ -551,11 +551,9 @@ class IqiyiIE(InfoExtractor):
             r'data-player-tvid\s*=\s*[\'"](\d+)', webpage, 'tvid')
         video_id = self._search_regex(
             r'data-player-videoid\s*=\s*[\'"]([a-f\d]+)', webpage, 'video_id')
-        swf_url = self._search_regex(
-            r'(http://[^\'"]+MainPlayer[^.]+\.swf)', webpage, 'swf player URL')
         _uuid = uuid.uuid4().hex
 
-        enc_key = self.get_enc_key(swf_url, video_id)
+        enc_key = self.get_enc_key(video_id)
 
         raw_data = self.get_raw_data(tvid, video_id, enc_key, _uuid)
 

From e7998f59aa2e204ddfef2f0eea38df1075ec0191 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Mar 2016 22:59:11 +0600
Subject: [PATCH 0835/1105] [lifenews] Fix extraction and improve (Closes
 #2482, closes #8714)

---
 youtube_dl/extractor/lifenews.py | 101 +++++++++++++++++++++----------
 1 file changed, 68 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index f8cbca7b3..a8fd639cc 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -20,18 +20,18 @@ class LifeNewsIE(InfoExtractor):
     _VALID_URL = r'http://lifenews\.ru/(?:mobile/)?(?P<section>news|video)/(?P<id>\d+)'
 
     _TESTS = [{
-        'url': 'http://lifenews.ru/news/126342',
-        'md5': 'e1b50a5c5fb98a6a544250f2e0db570a',
+        # single video embedded via video/source
+        'url': 'http://lifenews.ru/news/98736',
+        'md5': '77c95eaefaca216e32a76a343ad89d23',
         'info_dict': {
-            'id': '126342',
+            'id': '98736',
             'ext': 'mp4',
-            'title': 'МВД разыскивает мужчин, оставивших в IKEA сумку с автоматом',
-            'description': 'Камеры наблюдения гипермаркета зафиксировали троих мужчин, спрятавших оружейный арсенал в камере хранения.',
-            'thumbnail': 're:http://.*\.jpg',
-            'upload_date': '20140130',
+            'title': 'Мужчина нашел дома архив оборонного завода',
+            'description': 'md5:3b06b1b39b5e2bea548e403d99b8bf26',
+            'upload_date': '20120805',
         }
     }, {
-        # video in <iframe>
+        # single video embedded via iframe
         'url': 'http://lifenews.ru/news/152125',
         'md5': '77d19a6f0886cd76bdbf44b4d971a273',
         'info_dict': {
@@ -42,15 +42,33 @@ class LifeNewsIE(InfoExtractor):
             'upload_date': '20150402',
         }
     }, {
+        # two videos embedded via iframe
         'url': 'http://lifenews.ru/news/153461',
-        'md5': '9b6ef8bc0ffa25aebc8bdb40d89ab795',
         'info_dict': {
             'id': '153461',
-            'ext': 'mp4',
             'title': 'В Москве спасли потерявшегося медвежонка, который спрятался на дереве',
             'description': 'Маленький хищник не смог найти дорогу домой и обрел временное убежище на тополе недалеко от жилого массива, пока его не нашла соседская собака.',
             'upload_date': '20150505',
-        }
+        },
+        'playlist': [{
+            'md5': '9b6ef8bc0ffa25aebc8bdb40d89ab795',
+            'info_dict': {
+                'id': '153461-video1',
+                'ext': 'mp4',
+                'title': 'В Москве спасли потерявшегося медвежонка, который спрятался на дереве (Видео 1)',
+                'description': 'Маленький хищник не смог найти дорогу домой и обрел временное убежище на тополе недалеко от жилого массива, пока его не нашла соседская собака.',
+                'upload_date': '20150505',
+            },
+        }, {
+            'md5': 'ebb3bf3b1ce40e878d0d628e93eb0322',
+            'info_dict': {
+                'id': '153461-video2',
+                'ext': 'mp4',
+                'title': 'В Москве спасли потерявшегося медвежонка, который спрятался на дереве (Видео 2)',
+                'description': 'Маленький хищник не смог найти дорогу домой и обрел временное убежище на тополе недалеко от жилого массива, пока его не нашла соседская собака.',
+                'upload_date': '20150505',
+            },
+        }],
     }, {
         'url': 'http://lifenews.ru/video/13035',
         'only_matching': True,
@@ -65,10 +83,14 @@ class LifeNewsIE(InfoExtractor):
             'http://lifenews.ru/%s/%s' % (section, video_id),
             video_id, 'Downloading page')
 
-        videos = re.findall(r'<video.*?poster="(?P<poster>[^"]+)".*?src="(?P<video>[^"]+)".*?></video>', webpage)
-        iframe_link = self._html_search_regex(
-            '<iframe[^>]+src=["\']([^"\']+)["\']', webpage, 'iframe link', default=None)
-        if not videos and not iframe_link:
+        video_urls = re.findall(
+            r'<video[^>]+><source[^>]+src=["\'](.+?)["\']', webpage)
+
+        iframe_links = re.findall(
+            r'<iframe[^>]+src=["\']((?:https?:)?//embed\.life\.ru/embed/.+?)["\']',
+            webpage)
+
+        if not video_urls and not iframe_links:
             raise ExtractorError('No media links available for %s' % video_id)
 
         title = remove_end(
@@ -95,31 +117,44 @@ class LifeNewsIE(InfoExtractor):
             'upload_date': upload_date,
         }
 
-        def make_entry(video_id, media, video_number=None):
+        def make_entry(video_id, video_url, index=None):
             cur_info = dict(common_info)
             cur_info.update({
-                'id': video_id,
-                'url': media[1],
-                'thumbnail': media[0],
-                'title': title if video_number is None else '%s-video%s' % (title, video_number),
+                'id': video_id if not index else '%s-video%s' % (video_id, index),
+                'url': video_url,
+                'title': title if not index else '%s (Видео %s)' % (title, index),
             })
             return cur_info
 
-        if iframe_link:
-            iframe_link = self._proto_relative_url(iframe_link, 'http:')
-            cur_info = dict(common_info)
-            cur_info.update({
-                '_type': 'url_transparent',
-                'id': video_id,
-                'title': title,
-                'url': iframe_link,
-            })
+        def make_video_entry(video_id, video_url, index=None):
+            video_url = compat_urlparse.urljoin(url, video_url)
+            return make_entry(video_id, video_url, index)
+
+        def make_iframe_entry(video_id, video_url, index=None):
+            video_url = self._proto_relative_url(video_url, 'http:')
+            cur_info = make_entry(video_id, video_url, index)
+            cur_info['_type'] = 'url_transparent'
             return cur_info
 
-        if len(videos) == 1:
-            return make_entry(video_id, videos[0])
-        else:
-            return [make_entry(video_id, media, video_number + 1) for video_number, media in enumerate(videos)]
+        if len(video_urls) == 1 and not iframe_links:
+            return make_video_entry(video_id, video_urls[0])
+
+        if len(iframe_links) == 1 and not video_urls:
+            return make_iframe_entry(video_id, iframe_links[0])
+
+        entries = []
+
+        if video_urls:
+            for num, video_url in enumerate(video_urls, 1):
+                entries.append(make_video_entry(video_id, video_url, num))
+
+        if iframe_links:
+            for num, iframe_link in enumerate(iframe_links, len(video_urls) + 1):
+                entries.append(make_iframe_entry(video_id, iframe_link, num))
+
+        playlist = common_info.copy()
+        playlist.update(self.playlist_result(entries, video_id, title, description))
+        return playlist
 
 
 class LifeEmbedIE(InfoExtractor):

From 66ca2cfddd840e775f0b810c393a91c55a135c20 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Mar 2016 23:26:53 +0600
Subject: [PATCH 0836/1105] [wistia] Fix extraction (Closes #8707)

---
 youtube_dl/extractor/wistia.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/wistia.py b/youtube_dl/extractor/wistia.py
index fdb16d91c..41061dd31 100644
--- a/youtube_dl/extractor/wistia.py
+++ b/youtube_dl/extractor/wistia.py
@@ -35,7 +35,8 @@ class WistiaIE(InfoExtractor):
 
         formats = []
         thumbnails = []
-        for atype, a in data['assets'].items():
+        for a in data['assets']:
+            atype = a.get('type')
             if atype == 'still':
                 thumbnails.append({
                     'url': a['url'],

From 90f794c6c3ec685ad1ec3b92fba3fc0be6322c10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Mar 2016 23:41:23 +0600
Subject: [PATCH 0837/1105] [options] Add --no-mark-watched (#5054)

---
 youtube_dl/options.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 048dee881..9dd7a8034 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -174,6 +174,10 @@ def parseOpts(overrideArguments=None):
         '--mark-watched',
         action='store_true', dest='mark_watched', default=False,
         help='Mark videos watched (YouTube only)')
+    general.add_option(
+        '--no-mark-watched',
+        action='store_false', dest='mark_watched', default=False,
+        help='Do not mark videos watched (YouTube only)')
     general.add_option(
         '--no-color', '--no-colors',
         action='store_true', dest='no_color',

From f17f86513eaa09c2945b4a48a28efe5152be7017 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 1 Mar 2016 21:08:50 +0100
Subject: [PATCH 0838/1105] Add fixup for media files produced by HlsNative
 downloader(fixes #4776)

---
 youtube_dl/YoutubeDL.py              | 17 +++++++++++++++++
 youtube_dl/postprocessor/__init__.py |  2 ++
 youtube_dl/postprocessor/ffmpeg.py   | 19 +++++++++++++++----
 3 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index f4324039c..97cf31eb2 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -87,6 +87,7 @@ from .extractor import get_info_extractor, gen_extractors
 from .downloader import get_suitable_downloader
 from .downloader.rtmp import rtmpdump_version
 from .postprocessor import (
+    FFmpegFixupM3u8PP,
     FFmpegFixupM4aPP,
     FFmpegFixupStretchedPP,
     FFmpegMergerPP,
@@ -1670,6 +1671,22 @@ class YoutubeDL(object):
                     else:
                         assert fixup_policy in ('ignore', 'never')
 
+                if info_dict.get('protocol') == 'm3u8_native' or info_dict.get('protocol') == 'm3u8' and self._downloader.params.get('hls_prefer_native', False):
+                    if fixup_policy == 'warn':
+                        self.report_warning('%s: malformated aac bitstream.' % (
+                            info_dict['id']))
+                    elif fixup_policy == 'detect_or_warn':
+                        fixup_pp = FFmpegFixupM3u8PP(self)
+                        if fixup_pp.available:
+                            info_dict.setdefault('__postprocessors', [])
+                            info_dict['__postprocessors'].append(fixup_pp)
+                        else:
+                            self.report_warning(
+                                '%s: malformated aac bitstream. Install ffmpeg or avconv to fix this automatically.' % (
+                                    info_dict['id']))
+                    else:
+                        assert fixup_policy in ('ignore', 'never')
+
                 try:
                     self.post_process(filename, info_dict)
                 except (PostProcessingError) as err:
diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py
index 0d8ef6ca2..3ea518399 100644
--- a/youtube_dl/postprocessor/__init__.py
+++ b/youtube_dl/postprocessor/__init__.py
@@ -6,6 +6,7 @@ from .ffmpeg import (
     FFmpegEmbedSubtitlePP,
     FFmpegExtractAudioPP,
     FFmpegFixupStretchedPP,
+    FFmpegFixupM3u8PP,
     FFmpegFixupM4aPP,
     FFmpegMergerPP,
     FFmpegMetadataPP,
@@ -26,6 +27,7 @@ __all__ = [
     'ExecAfterDownloadPP',
     'FFmpegEmbedSubtitlePP',
     'FFmpegExtractAudioPP',
+    'FFmpegFixupM3u8PP',
     'FFmpegFixupM4aPP',
     'FFmpegFixupStretchedPP',
     'FFmpegMergerPP',
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 380bc6f29..81102f9bb 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -391,10 +391,6 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
         for (name, value) in metadata.items():
             options.extend(['-metadata', '%s=%s' % (name, value)])
 
-        # https://github.com/rg3/youtube-dl/issues/8350
-        if info.get('protocol') == 'm3u8_native' or info.get('protocol') == 'm3u8' and self._downloader.params.get('hls_prefer_native', False):
-            options.extend(['-bsf:a', 'aac_adtstoasc'])
-
         self._downloader.to_screen('[ffmpeg] Adding metadata to \'%s\'' % filename)
         self.run_ffmpeg(filename, temp_filename, options)
         os.remove(encodeFilename(filename))
@@ -467,6 +463,21 @@ class FFmpegFixupM4aPP(FFmpegPostProcessor):
         return [], info
 
 
+class FFmpegFixupM3u8PP(FFmpegPostProcessor):
+    def run(self, info):
+        filename = info['filepath']
+        temp_filename = prepend_extension(filename, 'temp')
+
+        options = ['-c', 'copy', '-f', 'mp4', '-bsf:a', 'aac_adtstoasc']
+        self._downloader.to_screen('[ffmpeg] Fixing malformated aac bitstream in "%s"' % filename)
+        self.run_ffmpeg(filename, temp_filename, options)
+
+        os.remove(encodeFilename(filename))
+        os.rename(encodeFilename(temp_filename), encodeFilename(filename))
+
+        return [], info
+
+
 class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
     def __init__(self, downloader=None, format=None):
         super(FFmpegSubtitlesConvertorPP, self).__init__(downloader)

From 3099b3127684d1386101e86005feb18c29136a81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Mar 2016 20:52:06 +0600
Subject: [PATCH 0839/1105] [nrk:skole] Add extractor (Closes #8728)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/nrk.py      | 30 +++++++++++++++++++++++++++++-
 2 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5817140c0..b3bf73963 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -506,6 +506,7 @@ from .npr import NprIE
 from .nrk import (
     NRKIE,
     NRKPlaylistIE,
+    NRKSkoleIE,
     NRKTVIE,
 )
 from .ntvde import NTVDeIE
diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index a126f5054..4ede1a362 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -87,7 +87,7 @@ class NRKIE(InfoExtractor):
 
 
 class NRKPlaylistIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nrk\.no/(?!video)(?:[^/]+/)+(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?nrk\.no/(?!video|skole)(?:[^/]+/)+(?P<id>[^/]+)'
 
     _TESTS = [{
         'url': 'http://www.nrk.no/troms/gjenopplev-den-historiske-solformorkelsen-1.12270763',
@@ -126,6 +126,34 @@ class NRKPlaylistIE(InfoExtractor):
             entries, playlist_id, playlist_title, playlist_description)
 
 
+class NRKSkoleIE(InfoExtractor):
+    IE_DESC = 'NRK Skole'
+    _VALID_URL = r'https?://(?:www\.)?nrk\.no/skole/klippdetalj?.*\btopic=nrk(?::|%3[Aa])klipp(?:/|%2[Ff])(?P<id>\d+)'
+
+    _TESTS = [{
+        'url': 'http://nrk.no/skole/klippdetalj?topic=nrk:klipp/616532',
+        'md5': '04cd85877cc1913bce73c5d28a47e00f',
+        'info_dict': {
+            'id': '6021',
+            'ext': 'flv',
+            'title': 'Genetikk og eneggede tvillinger',
+            'description': 'md5:3aca25dcf38ec30f0363428d2b265f8d',
+            'duration': 399,
+        },
+    }, {
+        'url': 'http://www.nrk.no/skole/klippdetalj?topic=nrk%3Aklipp%2F616532#embed',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        nrk_id = self._search_regex(r'data-nrk-id=["\'](\d+)', webpage, 'nrk id')
+        return self.url_result('nrk:%s' % nrk_id)
+
+
 class NRKTVIE(InfoExtractor):
     IE_DESC = 'NRK TV and NRK Radio'
     _VALID_URL = r'(?P<baseurl>https?://(?:tv|radio)\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'

From 61140904184f3fe84eea21e698c64d6b4fe7c889 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Mar 2016 20:57:04 +0600
Subject: [PATCH 0840/1105] [nrk:skole] Relax _VALID_URL

---
 youtube_dl/extractor/nrk.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 4ede1a362..3b21fbd4d 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_urlparse,
+    compat_urllib_parse_unquote,
+)
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -128,7 +131,7 @@ class NRKPlaylistIE(InfoExtractor):
 
 class NRKSkoleIE(InfoExtractor):
     IE_DESC = 'NRK Skole'
-    _VALID_URL = r'https?://(?:www\.)?nrk\.no/skole/klippdetalj?.*\btopic=nrk(?::|%3[Aa])klipp(?:/|%2[Ff])(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?nrk\.no/skole/klippdetalj?.*\btopic=(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'http://nrk.no/skole/klippdetalj?topic=nrk:klipp/616532',
@@ -143,10 +146,13 @@ class NRKSkoleIE(InfoExtractor):
     }, {
         'url': 'http://www.nrk.no/skole/klippdetalj?topic=nrk%3Aklipp%2F616532#embed',
         'only_matching': True,
+    }, {
+        'url': 'http://www.nrk.no/skole/klippdetalj?topic=urn:x-mediadb:21379',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        video_id = compat_urllib_parse_unquote(self._match_id(url))
 
         webpage = self._download_webpage(url, video_id)
 

From 82156fdbf0913c75181484dcc813565713bf78e9 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 2 Mar 2016 14:23:17 +0100
Subject: [PATCH 0841/1105] [youtube] extract width and height from fmt_list

---
 youtube_dl/extractor/youtube.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ba339b67d..56e6bd096 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1399,6 +1399,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts', [''])[0]
             if 'rtmpe%3Dyes' in encoded_url_map:
                 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)
+            fmt_list = video_info.get('fmt_list', [''])[0]
+            if fmt_list:
+                for fmt in fmt_list.split(','):
+                    spec = fmt.split('/')
+                    width, height = spec[1].split('x')
+                    self._formats[spec[0]].update({
+                        'resolution': spec[1],
+                        'width': int_or_none(width),
+                        'height': int_or_none(height),
+                    })
             formats = []
             for url_data_str in encoded_url_map.split(','):
                 url_data = compat_parse_qs(url_data_str)

From 4a5ba28a87a1a1632e58a1de404eb1fa268118a3 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 2 Mar 2016 15:53:09 +0100
Subject: [PATCH 0842/1105] [youtube] add tbr to _formats extracted from
 watch_as3.swf

---
 youtube_dl/extractor/youtube.py | 71 +++++++++++++++++----------------
 1 file changed, 36 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 56e6bd096..5356aaeb1 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -278,62 +278,63 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                      (?(1).+)?                                                # if we found the ID, everything can follow
                      $"""
     _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
+    # tbr was extracted from com/google/youtube/model/VideoFormat.as in watch_as3.swf and converted from Bytes/S to KBits/S
     _formats = {
-        '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
-        '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
-        '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
-        '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
-        '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
-        '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
-        '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
-        '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
+        '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263', 'tbr': 320},
+        '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263', 'tbr': 896},
+        '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v', 'tbr': 60},
+        '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v', 'tbr': 80},
+        '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264', 'tbr': 736},
+        '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'tbr': 3192},
+        '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'tbr': 928},
+        '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'tbr': 1280},
         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
-        '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
-        '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
-        '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
-        '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
-        '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
-        '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
+        '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v', 'tbr': 256},
+        '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'tbr': 6192},
+        '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'tbr': 10128},
+        '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'tbr': 928},
+        '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'tbr': 1280},
+        '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'tbr': 3192},
         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
-        '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
-        '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
+        '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'tbr': 1280},
+        '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'tbr': 1280},
 
 
         # 3D videos
-        '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
-        '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
-        '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
-        '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
+        '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20, 'tbr': 800},
+        '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20, 'tbr': 1152},
+        '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20, 'tbr': 3000},
+        '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20, 'tbr': 6000},
         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 
         # Apple HTTP Live Streaming
-        '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
-        '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
-        '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
-        '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
-        '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
+        '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10, 'tbr': 186.625},
+        '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10, 'tbr': 951.5625},
+        '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10, 'tbr': 1312.5},
+        '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10, 'tbr': 3207.421875},
+        '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10, 'tbr': 6349.21875},
         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 
         # DASH mp4 video
-        '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
-        '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 261.71875},
+        '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 951.5625},
+        '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 1312.5},
+        '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 3207.421875},
+        '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 6349.21875},
+        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 10128.0},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
+        '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 91.796875},
         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
 
         # Dash mp4 audio
-        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'preference': -50, 'container': 'm4a_dash'},
-        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'preference': -50, 'container': 'm4a_dash'},
-        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
+        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'preference': -50, 'container': 'm4a_dash', 'tbr': 32},
+        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'preference': -50, 'container': 'm4a_dash', 'tbr': 128},
+        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'preference': -50, 'container': 'm4a_dash', 'tbr': 320},
 
         # Dash webm
         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},

From 85ca019d9679d75c26b7ee4106dd7d98136f3c88 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 2 Mar 2016 16:02:48 +0100
Subject: [PATCH 0843/1105] [youtube] add basic info for some unknown formats
 extracted from watch_as3.swf

---
 youtube_dl/extractor/youtube.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5356aaeb1..d42f5c17f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -310,11 +310,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 
         # Apple HTTP Live Streaming
+        '91': {'format_note': 'HLS', 'tbr': 98.4375},
         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10, 'tbr': 186.625},
         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10, 'tbr': 951.5625},
         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10, 'tbr': 1312.5},
         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10, 'tbr': 3207.421875},
         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10, 'tbr': 6349.21875},
+        '97': {'format_note': 'HLS', 'tbr': 10128},
         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 
@@ -371,6 +373,25 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         # RTMP (unnamed)
         '_rtmp': {'protocol': 'rtmp'},
+
+        # formats extracted from com/google/youtube/model/VideoFormat.as in watch_as3.swf
+        '20': {'tbr': 24000},
+        '23': {'tbr': 62.5},
+        '24': {'tbr': 125},
+        '25': {'tbr': 312.5},
+        '33': {'tbr': 132},
+        '40': {'tbr': 40},
+        '61': {'tbr': 928},
+        '62': {'tbr': 1280},
+        '63': {'tbr': 3192},
+        '64': {'tbr': 6192},
+        '65': {'tbr': 10128},
+        '81': {'ext': 'mp4', 'tbr': 928},
+        '88': {'tbr': 7192},
+        '98': {'tbr': 320},
+        '119': {'ext': 'mp4', 'tbr': 320},
+        '304': {'format_note': 'DASH'},
+        '305': {'format_note': 'DASH'},
     }
     _SUBTITLE_FORMATS = ('ttml', 'vtt')
 

From e38cafe986994d65230e6518752def8b53ad7308 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Mar 2016 21:11:18 +0600
Subject: [PATCH 0844/1105] [YoutubeDL] Skip postprocessing and archive report
 when outputting to stdout (Closes #8729)

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index f4324039c..8f3a8b9e3 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1631,7 +1631,7 @@ class YoutubeDL(object):
                 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
                 return
 
-            if success:
+            if success and filename != '-':
                 # Fixup content
                 fixup_policy = self.params.get('fixup')
                 if fixup_policy is None:

From c48dd4400f2aae391a640b13133ae2e24fb92fba Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 2 Mar 2016 17:34:56 +0100
Subject: [PATCH 0845/1105] Revert "[youtube] add basic info for some unknown
 formats extracted from watch_as3.swf"

This reverts commit 85ca019d9679d75c26b7ee4106dd7d98136f3c88.
---
 youtube_dl/extractor/youtube.py | 21 ---------------------
 1 file changed, 21 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index d42f5c17f..5356aaeb1 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -310,13 +310,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 
         # Apple HTTP Live Streaming
-        '91': {'format_note': 'HLS', 'tbr': 98.4375},
         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10, 'tbr': 186.625},
         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10, 'tbr': 951.5625},
         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10, 'tbr': 1312.5},
         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10, 'tbr': 3207.421875},
         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10, 'tbr': 6349.21875},
-        '97': {'format_note': 'HLS', 'tbr': 10128},
         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 
@@ -373,25 +371,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         # RTMP (unnamed)
         '_rtmp': {'protocol': 'rtmp'},
-
-        # formats extracted from com/google/youtube/model/VideoFormat.as in watch_as3.swf
-        '20': {'tbr': 24000},
-        '23': {'tbr': 62.5},
-        '24': {'tbr': 125},
-        '25': {'tbr': 312.5},
-        '33': {'tbr': 132},
-        '40': {'tbr': 40},
-        '61': {'tbr': 928},
-        '62': {'tbr': 1280},
-        '63': {'tbr': 3192},
-        '64': {'tbr': 6192},
-        '65': {'tbr': 10128},
-        '81': {'ext': 'mp4', 'tbr': 928},
-        '88': {'tbr': 7192},
-        '98': {'tbr': 320},
-        '119': {'ext': 'mp4', 'tbr': 320},
-        '304': {'format_note': 'DASH'},
-        '305': {'format_note': 'DASH'},
     }
     _SUBTITLE_FORMATS = ('ttml', 'vtt')
 

From c2d3cb4c63495ef9f825efe67ee61d77d6fab9c0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 2 Mar 2016 17:35:04 +0100
Subject: [PATCH 0846/1105] Revert "[youtube] add tbr to _formats extracted
 from watch_as3.swf"

This reverts commit 4a5ba28a87a1a1632e58a1de404eb1fa268118a3.
---
 youtube_dl/extractor/youtube.py | 71 ++++++++++++++++-----------------
 1 file changed, 35 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5356aaeb1..56e6bd096 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -278,63 +278,62 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                      (?(1).+)?                                                # if we found the ID, everything can follow
                      $"""
     _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
-    # tbr was extracted from com/google/youtube/model/VideoFormat.as in watch_as3.swf and converted from Bytes/S to KBits/S
     _formats = {
-        '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263', 'tbr': 320},
-        '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263', 'tbr': 896},
-        '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v', 'tbr': 60},
-        '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v', 'tbr': 80},
-        '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264', 'tbr': 736},
-        '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'tbr': 3192},
-        '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'tbr': 928},
-        '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'tbr': 1280},
+        '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
+        '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
+        '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
+        '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
+        '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
+        '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
+        '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
+        '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
-        '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v', 'tbr': 256},
-        '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'tbr': 6192},
-        '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'tbr': 10128},
-        '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'tbr': 928},
-        '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'tbr': 1280},
-        '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'tbr': 3192},
+        '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
+        '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
+        '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
+        '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
+        '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
+        '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
-        '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'tbr': 1280},
-        '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'tbr': 1280},
+        '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
+        '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 
 
         # 3D videos
-        '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20, 'tbr': 800},
-        '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20, 'tbr': 1152},
-        '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20, 'tbr': 3000},
-        '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20, 'tbr': 6000},
+        '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
+        '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
+        '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
+        '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 
         # Apple HTTP Live Streaming
-        '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10, 'tbr': 186.625},
-        '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10, 'tbr': 951.5625},
-        '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10, 'tbr': 1312.5},
-        '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10, 'tbr': 3207.421875},
-        '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10, 'tbr': 6349.21875},
+        '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
+        '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
+        '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
+        '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
+        '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 
         # DASH mp4 video
-        '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 261.71875},
-        '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 951.5625},
-        '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 1312.5},
-        '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 3207.421875},
-        '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 6349.21875},
-        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 10128.0},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
-        '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 91.796875},
+        '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
+        '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
 
         # Dash mp4 audio
-        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'preference': -50, 'container': 'm4a_dash', 'tbr': 32},
-        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'preference': -50, 'container': 'm4a_dash', 'tbr': 128},
-        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'preference': -50, 'container': 'm4a_dash', 'tbr': 320},
+        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'preference': -50, 'container': 'm4a_dash'},
+        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'preference': -50, 'container': 'm4a_dash'},
+        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
 
         # Dash webm
         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},

From 3318832e9d42e85160ff2dab01d0f52ed739fbd6 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 2 Mar 2016 17:52:13 +0100
Subject: [PATCH 0847/1105] [youtube] improve width and height extraction from
 fmt_list

---
 youtube_dl/extractor/youtube.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 56e6bd096..e4bef8f2f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1399,16 +1399,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts', [''])[0]
             if 'rtmpe%3Dyes' in encoded_url_map:
                 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)
+            formats_spec = {}
             fmt_list = video_info.get('fmt_list', [''])[0]
             if fmt_list:
                 for fmt in fmt_list.split(','):
                     spec = fmt.split('/')
-                    width, height = spec[1].split('x')
-                    self._formats[spec[0]].update({
-                        'resolution': spec[1],
-                        'width': int_or_none(width),
-                        'height': int_or_none(height),
-                    })
+                    if len(spec) > 1:
+                        width_height = spec[1].split('x')
+                        if len(width_height) == 2:
+                            formats_spec[spec[0]] = {
+                                'resolution': spec[1],
+                                'width': int_or_none(width_height[0]),
+                                'height': int_or_none(width_height[1]),
+                            }
             formats = []
             for url_data_str in encoded_url_map.split(','):
                 url_data = compat_parse_qs(url_data_str)
@@ -1477,6 +1480,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 }
                 if format_id in self._formats:
                     dct.update(self._formats[format_id])
+                if format_id in formats_spec:
+                    dct.update(formats_spec[format_id])
 
                 # Some itags are not included in DASH manifest thus corresponding formats will
                 # lack metadata (see https://github.com/rg3/youtube-dl/pull/5993).

From 2bc0c46f989ce575e22502bb51f0fdcabd8baa15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Mar 2016 23:06:39 +0600
Subject: [PATCH 0848/1105] [extractor/common] Document license metafield

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a7c700099..d84ce4525 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -157,6 +157,7 @@ class InfoExtractor(object):
     thumbnail:      Full URL to a video thumbnail image.
     description:    Full video description.
     uploader:       Full name of the video uploader.
+    license:        License name the video is licensed under.
     creator:        The main artist who created the video.
     release_date:   The date (YYYYMMDD) when the video was released.
     timestamp:      UNIX timestamp of the moment the video became available.

From 7caf9830b0ae80363d3b1817b322452474631cc1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Mar 2016 23:07:25 +0600
Subject: [PATCH 0849/1105] [youtube] Extract license (Closes #8725)

---
 youtube_dl/extractor/youtube.py | 45 ++++++++++++++++++++++++++++++++-
 1 file changed, 44 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e4bef8f2f..43fdbfab1 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -384,6 +384,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'Philipp Hagemeister',
                 'uploader_id': 'phihag',
                 'upload_date': '20121002',
+                'license': 'Standard YouTube License',
                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
                 'categories': ['Science & Technology'],
                 'tags': ['youtube-dl'],
@@ -402,12 +403,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'upload_date': '20120506',
                 'title': 'Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]',
                 'alt_title': 'I Love It (feat. Charli XCX)',
-                'description': 'md5:782e8651347686cba06e58f71ab51773',
+                'description': 'md5:f3ceb5ef83a08d95b9d146f973157cc8',
                 'tags': ['Icona Pop i love it', 'sweden', 'pop music', 'big beat records', 'big beat', 'charli',
                          'xcx', 'charli xcx', 'girls', 'hbo', 'i love it', "i don't care", 'icona', 'pop',
                          'iconic ep', 'iconic', 'love', 'it'],
                 'uploader': 'Icona Pop',
                 'uploader_id': 'IconaPop',
+                'license': 'Standard YouTube License',
                 'creator': 'Icona Pop',
             }
         },
@@ -423,6 +425,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:64249768eec3bc4276236606ea996373',
                 'uploader': 'justintimberlakeVEVO',
                 'uploader_id': 'justintimberlakeVEVO',
+                'license': 'Standard YouTube License',
                 'creator': 'Justin Timberlake',
                 'age_limit': 18,
             }
@@ -438,6 +441,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
                 'uploader': 'SET India',
                 'uploader_id': 'setindia',
+                'license': 'Standard YouTube License',
                 'age_limit': 18,
             }
         },
@@ -451,6 +455,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'Philipp Hagemeister',
                 'uploader_id': 'phihag',
                 'upload_date': '20121002',
+                'license': 'Standard YouTube License',
                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
                 'categories': ['Science & Technology'],
                 'tags': ['youtube-dl'],
@@ -471,6 +476,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_id': '8KVIDEO',
                 'description': '',
                 'uploader': '8KVIDEO',
+                'license': 'Standard YouTube License',
                 'title': 'UHDTV TEST 8K VIDEO.mp4'
             },
             'params': {
@@ -489,6 +495,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'AfrojackVEVO',
                 'uploader_id': 'AfrojackVEVO',
                 'upload_date': '20131011',
+                'license': 'Standard YouTube License',
             },
             'params': {
                 'youtube_include_dash_manifest': True,
@@ -507,6 +514,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'TaylorSwiftVEVO',
                 'uploader_id': 'TaylorSwiftVEVO',
                 'upload_date': '20140818',
+                'license': 'Standard YouTube License',
                 'creator': 'Taylor Swift',
             },
             'params': {
@@ -523,6 +531,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'upload_date': '20100909',
                 'uploader': 'The Amazing Atheist',
                 'uploader_id': 'TheAmazingAtheist',
+                'license': 'Standard YouTube License',
                 'title': 'Burning Everyone\'s Koran',
                 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms\n\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
             }
@@ -538,6 +547,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'The Witcher',
                 'uploader_id': 'WitcherGame',
                 'upload_date': '20140605',
+                'license': 'Standard YouTube License',
                 'age_limit': 18,
             },
         },
@@ -552,6 +562,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'LloydVEVO',
                 'uploader_id': 'LloydVEVO',
                 'upload_date': '20110629',
+                'license': 'Standard YouTube License',
                 'age_limit': 18,
             },
         },
@@ -566,6 +577,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'creator': 'deadmau5',
                 'description': 'md5:12c56784b8032162bb936a5f76d55360',
                 'uploader': 'deadmau5',
+                'license': 'Standard YouTube License',
                 'title': 'Deadmau5 - Some Chords (HD)',
                 'alt_title': 'Some Chords',
             },
@@ -581,6 +593,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20150827',
                 'uploader_id': 'olympic',
+                'license': 'Standard YouTube License',
                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
                 'uploader': 'Olympics',
                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
@@ -600,6 +613,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_id': 'AllenMeow',
                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
                 'uploader': '孫艾倫',
+                'license': 'Standard YouTube License',
                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
             },
         },
@@ -631,6 +645,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'upload_date': '20150625',
                 'uploader_id': 'dorappi2000',
                 'uploader': 'dorappi2000',
+                'license': 'Standard YouTube License',
                 'formats': 'mincount:33',
             },
         },
@@ -645,6 +660,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'Airtek',
                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
+                'license': 'Standard YouTube License',
                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
             },
             'params': {
@@ -669,6 +685,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'upload_date': '20150721',
                     'uploader': 'Beer Games Beer',
                     'uploader_id': 'beergamesbeer',
+                    'license': 'Standard YouTube License',
                 },
             }, {
                 'info_dict': {
@@ -679,6 +696,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'upload_date': '20150721',
                     'uploader': 'Beer Games Beer',
                     'uploader_id': 'beergamesbeer',
+                    'license': 'Standard YouTube License',
                 },
             }, {
                 'info_dict': {
@@ -689,6 +707,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'upload_date': '20150721',
                     'uploader': 'Beer Games Beer',
                     'uploader_id': 'beergamesbeer',
+                    'license': 'Standard YouTube License',
                 },
             }, {
                 'info_dict': {
@@ -699,6 +718,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'upload_date': '20150721',
                     'uploader': 'Beer Games Beer',
                     'uploader_id': 'beergamesbeer',
+                    'license': 'Standard YouTube License',
                 },
             }],
             'params': {
@@ -733,6 +753,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'upload_date': '20151119',
                 'uploader_id': 'IronSoulElf',
                 'uploader': 'IronSoulElf',
+                'license': 'Standard YouTube License',
                 'creator': 'Todd Haberman, Daniel Law Heath & Aaron Kaplan',
             },
             'params': {
@@ -760,6 +781,23 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            # Video licensed under Creative Commons
+            'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
+            'info_dict': {
+                'id': 'M4gD1WSo5mA',
+                'ext': 'mp4',
+                'title': 'md5:e41008789470fc2533a3252216f1c1d1',
+                'description': 'md5:a677553cf0840649b731a3024aeff4cc',
+                'upload_date': '20150127',
+                'uploader_id': 'BerkmanCenter',
+                'uploader': 'BerkmanCenter',
+                'license': 'Creative Commons Attribution license (reuse allowed)',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         {
             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
             'only_matching': True,
@@ -1326,6 +1364,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
         upload_date = unified_strdate(upload_date)
 
+        video_license = self._html_search_regex(
+            r'<h4[^>]+class="title"[^>]*>\s*License\s*</h4>\s*<ul[^>]*>\s*<li>(.+?)</li',
+            video_webpage, 'license', default=None)
+
         m_music = re.search(
             r'<h4[^>]+class="title"[^>]*>\s*Music\s*</h4>\s*<ul[^>]*>\s*<li>(?P<title>.+?) by (?P<creator>.+?)(?:\(.+?\))?</li',
             video_webpage)
@@ -1601,6 +1643,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'uploader': video_uploader,
             'uploader_id': video_uploader_id,
             'upload_date': upload_date,
+            'license': video_license,
             'creator': video_creator,
             'title': video_title,
             'alt_title': video_alt_title,

From 47462a125bd47cb8adc06caeaf46e5692d841395 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Wed, 2 Mar 2016 23:10:01 +0600
Subject: [PATCH 0850/1105] [README.md] Document license field for output
 template

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 44b65da02..d66804a7a 100644
--- a/README.md
+++ b/README.md
@@ -458,6 +458,7 @@ The basic usage is not to set any template arguments when downloading a single f
  - `alt_title`: A secondary title of the video
  - `display_id`: An alternative identifier for the video
  - `uploader`: Full name of the video uploader
+ - `license`: License name the video is licensed under
  - `creator`: The main artist who created the video
  - `release_date`: The date (YYYYMMDD) when the video was released
  - `timestamp`: UNIX timestamp of the moment the video became available

From 7bcd2830dd906c7a83b31d3ba8c8b10d4e7756af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Mar 2016 23:31:24 +0600
Subject: [PATCH 0851/1105] [extractor/common] Document uploader_url

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index d84ce4525..402f2f436 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -164,6 +164,7 @@ class InfoExtractor(object):
     upload_date:    Video upload date (YYYYMMDD).
                     If not explicitly set, calculated from timestamp.
     uploader_id:    Nickname or id of the video uploader.
+    uploader_url:   Full URL to a personal webpage of the video uploader.
     location:       Physical location where the video was filmed.
     subtitles:      The available subtitles as a dictionary in the format
                     {language: subformats}. "subformats" is a list sorted from

From fd050249afce1bcc9e7f4a127069375467007b55 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Mar 2016 23:49:10 +0600
Subject: [PATCH 0852/1105] [youtube] Extract uploader_url (Closes #8724)

---
 youtube_dl/extractor/youtube.py | 46 +++++++++++++++++++++++++++++++--
 1 file changed, 44 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 43fdbfab1..27e67feb4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -383,6 +383,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
                 'uploader': 'Philipp Hagemeister',
                 'uploader_id': 'phihag',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/phihag',
                 'upload_date': '20121002',
                 'license': 'Standard YouTube License',
                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
@@ -409,6 +410,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                          'iconic ep', 'iconic', 'love', 'it'],
                 'uploader': 'Icona Pop',
                 'uploader_id': 'IconaPop',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/IconaPop',
                 'license': 'Standard YouTube License',
                 'creator': 'Icona Pop',
             }
@@ -425,6 +427,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:64249768eec3bc4276236606ea996373',
                 'uploader': 'justintimberlakeVEVO',
                 'uploader_id': 'justintimberlakeVEVO',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/justintimberlakeVEVO',
                 'license': 'Standard YouTube License',
                 'creator': 'Justin Timberlake',
                 'age_limit': 18,
@@ -441,6 +444,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
                 'uploader': 'SET India',
                 'uploader_id': 'setindia',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/setindia',
                 'license': 'Standard YouTube License',
                 'age_limit': 18,
             }
@@ -454,6 +458,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
                 'uploader': 'Philipp Hagemeister',
                 'uploader_id': 'phihag',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/phihag',
                 'upload_date': '20121002',
                 'license': 'Standard YouTube License',
                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
@@ -474,6 +479,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'm4a',
                 'upload_date': '20121002',
                 'uploader_id': '8KVIDEO',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
                 'description': '',
                 'uploader': '8KVIDEO',
                 'license': 'Standard YouTube License',
@@ -531,6 +537,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'upload_date': '20100909',
                 'uploader': 'The Amazing Atheist',
                 'uploader_id': 'TheAmazingAtheist',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/TheAmazingAtheist',
                 'license': 'Standard YouTube License',
                 'title': 'Burning Everyone\'s Koran',
                 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms\n\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
@@ -546,6 +553,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
                 'uploader': 'The Witcher',
                 'uploader_id': 'WitcherGame',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
                 'upload_date': '20140605',
                 'license': 'Standard YouTube License',
                 'age_limit': 18,
@@ -561,6 +569,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:33765bb339e1b47e7e72b5490139bb41',
                 'uploader': 'LloydVEVO',
                 'uploader_id': 'LloydVEVO',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/LloydVEVO',
                 'upload_date': '20110629',
                 'license': 'Standard YouTube License',
                 'age_limit': 18,
@@ -574,6 +583,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20100430',
                 'uploader_id': 'deadmau5',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/deadmau5',
                 'creator': 'deadmau5',
                 'description': 'md5:12c56784b8032162bb936a5f76d55360',
                 'uploader': 'deadmau5',
@@ -593,6 +603,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20150827',
                 'uploader_id': 'olympic',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/olympic',
                 'license': 'Standard YouTube License',
                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
                 'uploader': 'Olympics',
@@ -611,6 +622,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'stretched_ratio': 16 / 9.,
                 'upload_date': '20110310',
                 'uploader_id': 'AllenMeow',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
                 'uploader': '孫艾倫',
                 'license': 'Standard YouTube License',
@@ -644,6 +656,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
                 'upload_date': '20150625',
                 'uploader_id': 'dorappi2000',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
                 'uploader': 'dorappi2000',
                 'license': 'Standard YouTube License',
                 'formats': 'mincount:33',
@@ -685,6 +698,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'upload_date': '20150721',
                     'uploader': 'Beer Games Beer',
                     'uploader_id': 'beergamesbeer',
+                    'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
                     'license': 'Standard YouTube License',
                 },
             }, {
@@ -696,6 +710,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'upload_date': '20150721',
                     'uploader': 'Beer Games Beer',
                     'uploader_id': 'beergamesbeer',
+                    'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
                     'license': 'Standard YouTube License',
                 },
             }, {
@@ -707,6 +722,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'upload_date': '20150721',
                     'uploader': 'Beer Games Beer',
                     'uploader_id': 'beergamesbeer',
+                    'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
                     'license': 'Standard YouTube License',
                 },
             }, {
@@ -718,6 +734,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'upload_date': '20150721',
                     'uploader': 'Beer Games Beer',
                     'uploader_id': 'beergamesbeer',
+                    'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
                     'license': 'Standard YouTube License',
                 },
             }],
@@ -752,6 +769,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
                 'upload_date': '20151119',
                 'uploader_id': 'IronSoulElf',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
                 'uploader': 'IronSoulElf',
                 'license': 'Standard YouTube License',
                 'creator': 'Todd Haberman, Daniel Law Heath & Aaron Kaplan',
@@ -791,6 +809,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
                 'upload_date': '20150127',
                 'uploader_id': 'BerkmanCenter',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
                 'uploader': 'BerkmanCenter',
                 'license': 'Creative Commons Attribution license (reuse allowed)',
             },
@@ -798,6 +817,24 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            # Channel-like uploader_url
+            'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
+            'info_dict': {
+                'id': 'eQcmzGIKrzg',
+                'ext': 'mp4',
+                'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
+                'description': 'md5:dda0d780d5a6e120758d1711d062a867',
+                'upload_date': '20151119',
+                'uploader': 'Bernie 2016',
+                'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
+                'license': 'Creative Commons Attribution license (reuse allowed)',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         {
             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
             'only_matching': True,
@@ -1334,9 +1371,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         # uploader_id
         video_uploader_id = None
-        mobj = re.search(r'<link itemprop="url" href="http://www.youtube.com/(?:user|channel)/([^"]+)">', video_webpage)
+        video_uploader_url = None
+        mobj = re.search(
+            r'<link itemprop="url" href="(?P<uploader_url>https?://www.youtube.com/(?:user|channel)/(?P<uploader_id>[^"]+))">',
+            video_webpage)
         if mobj is not None:
-            video_uploader_id = mobj.group(1)
+            video_uploader_id = mobj.group('uploader_id')
+            video_uploader_url = mobj.group('uploader_url')
         else:
             self._downloader.report_warning('unable to extract uploader nickname')
 
@@ -1642,6 +1683,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'id': video_id,
             'uploader': video_uploader,
             'uploader_id': video_uploader_id,
+            'uploader_url': video_uploader_url,
             'upload_date': upload_date,
             'license': video_license,
             'creator': video_creator,

From a2d7797cee21e54866f11ce41863a92c992a56f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 3 Mar 2016 00:00:11 +0600
Subject: [PATCH 0853/1105] [vimeo] Extract uploader_url (Closes #8727)

---
 youtube_dl/extractor/vimeo.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 3049dffb6..9f282a1da 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -93,6 +93,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'title': "youtube-dl test video - \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
                 'description': 'md5:2d3305bad981a06ff79f027f19865021',
                 'upload_date': '20121220',
+                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/user7108434',
                 'uploader_id': 'user7108434',
                 'uploader': 'Filippo Valsorda',
                 'duration': 10,
@@ -105,6 +106,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             'info_dict': {
                 'id': '68093876',
                 'ext': 'mp4',
+                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/openstreetmapus',
                 'uploader_id': 'openstreetmapus',
                 'uploader': 'OpenStreetMap US',
                 'title': 'Andy Allan - Putting the Carto into OpenStreetMap Cartography',
@@ -121,6 +123,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'ext': 'mp4',
                 'title': 'Kathy Sierra: Building the minimum Badass User, Business of Software 2012',
                 'uploader': 'The BLN & Business of Software',
+                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/theblnbusinessofsoftware',
                 'uploader_id': 'theblnbusinessofsoftware',
                 'duration': 3610,
                 'description': None,
@@ -135,6 +138,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'ext': 'mp4',
                 'title': 'youtube-dl password protected test video',
                 'upload_date': '20130614',
+                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/user18948128',
                 'uploader_id': 'user18948128',
                 'uploader': 'Jaime Marquínez Ferrándiz',
                 'duration': 10,
@@ -154,6 +158,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'ext': 'mp4',
                 'title': 'Key & Peele: Terrorist Interrogation',
                 'description': 'md5:8678b246399b070816b12313e8b4eb5c',
+                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/atencio',
                 'uploader_id': 'atencio',
                 'uploader': 'Peter Atencio',
                 'upload_date': '20130927',
@@ -169,6 +174,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'title': 'The New Vimeo Player (You Know, For Videos)',
                 'description': 'md5:2ec900bf97c3f389378a96aee11260ea',
                 'upload_date': '20131015',
+                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/staff',
                 'uploader_id': 'staff',
                 'uploader': 'Vimeo Staff',
                 'duration': 62,
@@ -183,6 +189,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'ext': 'mp4',
                 'title': 'Pier Solar OUYA Official Trailer',
                 'uploader': 'Tulio Gonçalves',
+                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/user28849593',
                 'uploader_id': 'user28849593',
             },
         },
@@ -195,6 +202,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'ext': 'mp4',
                 'title': 'FOX CLASSICS - Forever Classic ID - A Full Minute',
                 'uploader': 'The DMCI',
+                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/dmci',
                 'uploader_id': 'dmci',
                 'upload_date': '20111220',
                 'description': 'md5:ae23671e82d05415868f7ad1aec21147',
@@ -370,9 +378,10 @@ class VimeoIE(VimeoBaseInfoExtractor):
         # Extract title
         video_title = config['video']['title']
 
-        # Extract uploader and uploader_id
-        video_uploader = config['video']['owner']['name']
-        video_uploader_id = config['video']['owner']['url'].split('/')[-1] if config['video']['owner']['url'] else None
+        # Extract uploader, uploader_url and uploader_id
+        video_uploader = config['video'].get('owner', {}).get('name')
+        video_uploader_url = config['video'].get('owner', {}).get('url')
+        video_uploader_id = video_uploader_url.split('/')[-1] if video_uploader_url else None
 
         # Extract video thumbnail
         video_thumbnail = config['video'].get('thumbnail')
@@ -473,6 +482,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         return {
             'id': video_id,
             'uploader': video_uploader,
+            'uploader_url': video_uploader_url,
             'uploader_id': video_uploader_id,
             'upload_date': video_upload_date,
             'title': video_title,

From 2316dc2b9a1b31ea6845d31571d620c413884926 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 3 Mar 2016 00:41:36 +0600
Subject: [PATCH 0854/1105] [twitch:playlistbase] Mark broken

Twitch paging mechanism is completely broken on twitch side serving all videos all the time and making our travis builds stall.
---
 youtube_dl/extractor/twitch.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 69882da63..5b8586097 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -303,6 +303,7 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
 
 
 class TwitchProfileIE(TwitchPlaylistBaseIE):
+    _WORKING = False
     IE_NAME = 'twitch:profile'
     _VALID_URL = r'%s/(?P<id>[^/]+)/profile/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
     _PLAYLIST_TYPE = 'profile'
@@ -318,6 +319,7 @@ class TwitchProfileIE(TwitchPlaylistBaseIE):
 
 
 class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
+    _WORKING = False
     IE_NAME = 'twitch:past_broadcasts'
     _VALID_URL = r'%s/(?P<id>[^/]+)/profile/past_broadcasts/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
     _PLAYLIST_URL = TwitchPlaylistBaseIE._PLAYLIST_URL + '&broadcasts=true'
@@ -334,6 +336,7 @@ class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
 
 
 class TwitchBookmarksIE(TwitchPlaylistBaseIE):
+    _WORKING = False
     IE_NAME = 'twitch:bookmarks'
     _VALID_URL = r'%s/(?P<id>[^/]+)/profile/bookmarks/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
     _PLAYLIST_URL = '%s/api/bookmark/?user=%%s&offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE

From 5e73a67d44bc25bd8660704cd437d35bd1e9541e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 02:38:41 +0800
Subject: [PATCH 0855/1105] [letv] Domain name changed

---
 youtube_dl/extractor/letv.py | 33 ++++++++++++++++++---------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 9fd494c29..246556316 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -27,10 +27,12 @@ from ..utils import (
 
 class LetvIE(InfoExtractor):
     IE_DESC = '乐视网'
-    _VALID_URL = r'http://www\.letv\.com/ptv/vplay/(?P<id>\d+).html'
+    _VALID_URL = r'http://www\.le\.com/ptv/vplay/(?P<id>\d+).html'
+
+    _URL_TEMPLATE = r'http://www.le.com/ptv/vplay/%s.html'
 
     _TESTS = [{
-        'url': 'http://www.letv.com/ptv/vplay/22005890.html',
+        'url': 'http://www.le.com/ptv/vplay/22005890.html',
         'md5': 'edadcfe5406976f42f9f266057ee5e40',
         'info_dict': {
             'id': '22005890',
@@ -42,7 +44,7 @@ class LetvIE(InfoExtractor):
             'hls_prefer_native': True,
         },
     }, {
-        'url': 'http://www.letv.com/ptv/vplay/1415246.html',
+        'url': 'http://www.le.com/ptv/vplay/1415246.html',
         'info_dict': {
             'id': '1415246',
             'ext': 'mp4',
@@ -54,7 +56,7 @@ class LetvIE(InfoExtractor):
         },
     }, {
         'note': 'This video is available only in Mainland China, thus a proxy is needed',
-        'url': 'http://www.letv.com/ptv/vplay/1118082.html',
+        'url': 'http://www.le.com/ptv/vplay/1118082.html',
         'md5': '2424c74948a62e5f31988438979c5ad1',
         'info_dict': {
             'id': '1118082',
@@ -116,10 +118,10 @@ class LetvIE(InfoExtractor):
             'splatid': 101,
             'format': 1,
             'tkey': self.calc_time_key(int(time.time())),
-            'domain': 'www.letv.com'
+            'domain': 'www.le.com'
         }
         play_json_req = sanitized_Request(
-            'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params)
+            'http://api.le.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params)
         )
         cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
         if cn_verification_proxy:
@@ -193,9 +195,9 @@ class LetvIE(InfoExtractor):
 
 
 class LetvTvIE(InfoExtractor):
-    _VALID_URL = r'http://www.letv.com/tv/(?P<id>\d+).html'
+    _VALID_URL = r'http://www.le.com/tv/(?P<id>\d+).html'
     _TESTS = [{
-        'url': 'http://www.letv.com/tv/46177.html',
+        'url': 'http://www.le.com/tv/46177.html',
         'info_dict': {
             'id': '46177',
             'title': '美人天下',
@@ -208,10 +210,11 @@ class LetvTvIE(InfoExtractor):
         playlist_id = self._match_id(url)
         page = self._download_webpage(url, playlist_id)
 
-        media_urls = list(set(re.findall(
-            r'http://www.letv.com/ptv/vplay/\d+.html', page)))
-        entries = [self.url_result(media_url, ie='Letv')
-                   for media_url in media_urls]
+        # Currently old domain names are still used in playlists
+        media_ids = list(set(re.findall(
+            r'http://www.letv.com/ptv/vplay/(\d+).html', page)))
+        entries = [self.url_result(LetvIE._URL_TEMPLATE % media_id, ie='Letv')
+                   for media_id in media_ids]
 
         title = self._html_search_meta('keywords', page,
                                        fatal=False).split('，')[0]
@@ -222,9 +225,9 @@ class LetvTvIE(InfoExtractor):
 
 
 class LetvPlaylistIE(LetvTvIE):
-    _VALID_URL = r'http://tv.letv.com/[a-z]+/(?P<id>[a-z]+)/index.s?html'
+    _VALID_URL = r'http://tv.le.com/[a-z]+/(?P<id>[a-z]+)/index.s?html'
     _TESTS = [{
-        'url': 'http://tv.letv.com/izt/wuzetian/index.html',
+        'url': 'http://tv.le.com/izt/wuzetian/index.html',
         'info_dict': {
             'id': 'wuzetian',
             'title': '武媚娘传奇',
@@ -233,7 +236,7 @@ class LetvPlaylistIE(LetvTvIE):
         # This playlist contains some extra videos other than the drama itself
         'playlist_mincount': 96
     }, {
-        'url': 'http://tv.letv.com/pzt/lswjzzjc/index.shtml',
+        'url': 'http://tv.le.com/pzt/lswjzzjc/index.shtml',
         'info_dict': {
             'id': 'lswjzzjc',
             # The title should be "劲舞青春", but I can't find a simple way to

From 7465e7e42d0a02196e8d8ecd49edcadca4823897 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 02:40:51 +0800
Subject: [PATCH 0856/1105] [letv] Keep videos' order in playlists

---
 youtube_dl/extractor/letv.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 246556316..91172c096 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -22,6 +22,7 @@ from ..utils import (
     str_or_none,
     encode_data_uri,
     url_basename,
+    orderedSet,
 )
 
 
@@ -211,8 +212,8 @@ class LetvTvIE(InfoExtractor):
         page = self._download_webpage(url, playlist_id)
 
         # Currently old domain names are still used in playlists
-        media_ids = list(set(re.findall(
-            r'http://www.letv.com/ptv/vplay/(\d+).html', page)))
+        media_ids = orderedSet(re.findall(
+            r'http://www.letv.com/ptv/vplay/(\d+).html', page))
         entries = [self.url_result(LetvIE._URL_TEMPLATE % media_id, ie='Letv')
                    for media_id in media_ids]
 

From 7a8fef31730f0a9823c1379f67902dd95a9ea580 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 02:42:19 +0800
Subject: [PATCH 0857/1105] [letv] Order imports alphabetically

---
 youtube_dl/extractor/letv.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 91172c096..2ece30839 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -1,28 +1,28 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import base64
 import datetime
+import hashlib
 import re
 import time
-import base64
-import hashlib
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
     compat_ord,
     compat_str,
+    compat_urllib_parse,
 )
 from ..utils import (
     determine_ext,
+    encode_data_uri,
     ExtractorError,
+    int_or_none,
+    orderedSet,
     parse_iso8601,
     sanitized_Request,
-    int_or_none,
     str_or_none,
-    encode_data_uri,
     url_basename,
-    orderedSet,
 )
 
 

From f3bc19a989d9f6d6b21abd4271cb99a18897069f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 02:50:12 +0800
Subject: [PATCH 0858/1105] [letv] Correct regular expressions and fix a typo

---
 youtube_dl/extractor/letv.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 2ece30839..ba8126f75 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -28,9 +28,9 @@ from ..utils import (
 
 class LetvIE(InfoExtractor):
     IE_DESC = '乐视网'
-    _VALID_URL = r'http://www\.le\.com/ptv/vplay/(?P<id>\d+).html'
+    _VALID_URL = r'http://www\.le\.com/ptv/vplay/(?P<id>\d+)\.html'
 
-    _URL_TEMPLATE = r'http://www.le.com/ptv/vplay/%s.html'
+    _URL_TEMPLATE = 'http://www.le.com/ptv/vplay/%s.html'
 
     _TESTS = [{
         'url': 'http://www.le.com/ptv/vplay/22005890.html',
@@ -196,7 +196,7 @@ class LetvIE(InfoExtractor):
 
 
 class LetvTvIE(InfoExtractor):
-    _VALID_URL = r'http://www.le.com/tv/(?P<id>\d+).html'
+    _VALID_URL = r'http://www\.le\.com/tv/(?P<id>\d+)\.html'
     _TESTS = [{
         'url': 'http://www.le.com/tv/46177.html',
         'info_dict': {
@@ -213,7 +213,7 @@ class LetvTvIE(InfoExtractor):
 
         # Currently old domain names are still used in playlists
         media_ids = orderedSet(re.findall(
-            r'http://www.letv.com/ptv/vplay/(\d+).html', page))
+            r'http://www\.letv\.com/ptv/vplay/(\d+)\.html', page))
         entries = [self.url_result(LetvIE._URL_TEMPLATE % media_id, ie='Letv')
                    for media_id in media_ids]
 
@@ -226,7 +226,7 @@ class LetvTvIE(InfoExtractor):
 
 
 class LetvPlaylistIE(LetvTvIE):
-    _VALID_URL = r'http://tv.le.com/[a-z]+/(?P<id>[a-z]+)/index.s?html'
+    _VALID_URL = r'http://tv\.le\.com/[a-z]+/(?P<id>[a-z]+)/index\.s?html'
     _TESTS = [{
         'url': 'http://tv.le.com/izt/wuzetian/index.html',
         'info_dict': {

From 2eb25b256bf38db9f35696822cee399099eb4777 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 03:14:00 +0800
Subject: [PATCH 0859/1105] [letv] Merge LetvTvIE into LetvPlaylistIE

And
1. Add more URL examples
2. Improve the matching pattern
---
 youtube_dl/extractor/__init__.py |  1 -
 youtube_dl/extractor/letv.py     | 56 ++++++++++++++++++--------------
 2 files changed, 31 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b3bf73963..19c563172 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -354,7 +354,6 @@ from .lecture2go import Lecture2GoIE
 from .lemonde import LemondeIE
 from .letv import (
     LetvIE,
-    LetvTvIE,
     LetvPlaylistIE,
     LetvCloudIE,
 )
diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index ba8126f75..ce25bb4db 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -195,8 +195,9 @@ class LetvIE(InfoExtractor):
         }
 
 
-class LetvTvIE(InfoExtractor):
-    _VALID_URL = r'http://www\.le\.com/tv/(?P<id>\d+)\.html'
+class LetvPlaylistIE(InfoExtractor):
+    _VALID_URL = r'http://[a-z]+\.le\.com/[a-z]+/(?P<id>[a-z0-9_]+)'
+
     _TESTS = [{
         'url': 'http://www.le.com/tv/46177.html',
         'info_dict': {
@@ -205,29 +206,7 @@ class LetvTvIE(InfoExtractor):
             'description': 'md5:395666ff41b44080396e59570dbac01c'
         },
         'playlist_count': 35
-    }]
-
-    def _real_extract(self, url):
-        playlist_id = self._match_id(url)
-        page = self._download_webpage(url, playlist_id)
-
-        # Currently old domain names are still used in playlists
-        media_ids = orderedSet(re.findall(
-            r'http://www\.letv\.com/ptv/vplay/(\d+)\.html', page))
-        entries = [self.url_result(LetvIE._URL_TEMPLATE % media_id, ie='Letv')
-                   for media_id in media_ids]
-
-        title = self._html_search_meta('keywords', page,
-                                       fatal=False).split('，')[0]
-        description = self._html_search_meta('description', page, fatal=False)
-
-        return self.playlist_result(entries, playlist_id, playlist_title=title,
-                                    playlist_description=description)
-
-
-class LetvPlaylistIE(LetvTvIE):
-    _VALID_URL = r'http://tv\.le\.com/[a-z]+/(?P<id>[a-z]+)/index\.s?html'
-    _TESTS = [{
+    }, {
         'url': 'http://tv.le.com/izt/wuzetian/index.html',
         'info_dict': {
             'id': 'wuzetian',
@@ -246,8 +225,35 @@ class LetvPlaylistIE(LetvTvIE):
             'description': 'md5:b1eef244f45589a7b5b1af9ff25a4489'
         },
         'playlist_mincount': 7
+    }, {
+        'url': 'http://www.le.com/comic/92063.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://list.le.com/listn/c1009_sc532002_d2_p1_o1.html',
+        'only_matching': True,
     }]
 
+    @classmethod
+    def suitable(cls, url):
+        return False if LetvIE.suitable(url) else super(LetvPlaylistIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        page = self._download_webpage(url, playlist_id)
+
+        # Currently old domain names are still used in playlists
+        media_ids = orderedSet(re.findall(
+            r'<a[^>]+href="http://www\.letv\.com/ptv/vplay/(\d+)\.html', page))
+        entries = [self.url_result(LetvIE._URL_TEMPLATE % media_id, ie='Letv')
+                   for media_id in media_ids]
+
+        title = self._html_search_meta('keywords', page,
+                                       fatal=False).split('，')[0]
+        description = self._html_search_meta('description', page, fatal=False)
+
+        return self.playlist_result(entries, playlist_id, playlist_title=title,
+                                    playlist_description=description)
+
 
 class LetvCloudIE(InfoExtractor):
     IE_DESC = '乐视云'

From 8c5a9944246d459f0eac876ae358dedb32fc890a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 03:24:20 +0800
Subject: [PATCH 0860/1105] [leeco] Letv renamed to LeEco

LeEco is the company name and Le is the domain name.

For more information see the Chinese news post
http://www.techorz.com/company-news/letv-renamed-to-leeco-and-new-logo/
---
 youtube_dl/extractor/__init__.py           |  6 +++---
 youtube_dl/extractor/{letv.py => leeco.py} | 10 ++++++----
 2 files changed, 9 insertions(+), 7 deletions(-)
 rename youtube_dl/extractor/{letv.py => leeco.py} (97%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 19c563172..037654a23 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -352,9 +352,9 @@ from .la7 import LA7IE
 from .laola1tv import Laola1TvIE
 from .lecture2go import Lecture2GoIE
 from .lemonde import LemondeIE
-from .letv import (
-    LetvIE,
-    LetvPlaylistIE,
+from .leeco import (
+    LeIE,
+    LePlaylistIE,
     LetvCloudIE,
 )
 from .libsyn import LibsynIE
diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/leeco.py
similarity index 97%
rename from youtube_dl/extractor/letv.py
rename to youtube_dl/extractor/leeco.py
index ce25bb4db..d0cd3f591 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/leeco.py
@@ -26,7 +26,7 @@ from ..utils import (
 )
 
 
-class LetvIE(InfoExtractor):
+class LeIE(InfoExtractor):
     IE_DESC = '乐视网'
     _VALID_URL = r'http://www\.le\.com/ptv/vplay/(?P<id>\d+)\.html'
 
@@ -195,7 +195,7 @@ class LetvIE(InfoExtractor):
         }
 
 
-class LetvPlaylistIE(InfoExtractor):
+class LePlaylistIE(InfoExtractor):
     _VALID_URL = r'http://[a-z]+\.le\.com/[a-z]+/(?P<id>[a-z0-9_]+)'
 
     _TESTS = [{
@@ -235,7 +235,7 @@ class LetvPlaylistIE(InfoExtractor):
 
     @classmethod
     def suitable(cls, url):
-        return False if LetvIE.suitable(url) else super(LetvPlaylistIE, cls).suitable(url)
+        return False if LeIE.suitable(url) else super(LePlaylistIE, cls).suitable(url)
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
@@ -244,7 +244,7 @@ class LetvPlaylistIE(InfoExtractor):
         # Currently old domain names are still used in playlists
         media_ids = orderedSet(re.findall(
             r'<a[^>]+href="http://www\.letv\.com/ptv/vplay/(\d+)\.html', page))
-        entries = [self.url_result(LetvIE._URL_TEMPLATE % media_id, ie='Letv')
+        entries = [self.url_result(LeIE._URL_TEMPLATE % media_id, ie='Le')
                    for media_id in media_ids]
 
         title = self._html_search_meta('keywords', page,
@@ -256,6 +256,8 @@ class LetvPlaylistIE(InfoExtractor):
 
 
 class LetvCloudIE(InfoExtractor):
+    # Most of *.letv.com is changed to *.le.com on 2016/01/02
+    # but yuntv.letv.com is kept, so also keep the extractor name
     IE_DESC = '乐视云'
     _VALID_URL = r'https?://yuntv\.letv\.com/bcloud.html\?.+'
 

From e07237f6403fe2c18212b36e27bb630dd8e5cef1 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 2 Mar 2016 21:40:21 +0100
Subject: [PATCH 0861/1105] [utils] remove check for val from find_xpath_attr

---
 youtube_dl/utils.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 606977c58..210c47fce 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -160,8 +160,6 @@ if sys.version_info >= (2, 7):
     def find_xpath_attr(node, xpath, key, val=None):
         """ Find the xpath xpath[@key=val] """
         assert re.match(r'^[a-zA-Z_-]+$', key)
-        if val:
-            assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)
         expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
         return node.find(expr)
 else:

From 4459bef20369584c46dc5550fa4e49b7858754bc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 2 Mar 2016 21:41:29 +0100
Subject: [PATCH 0862/1105] [thepltform] detect other types of errors

---
 youtube_dl/extractor/theplatform.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 93d871571..9a57b49df 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -22,6 +22,7 @@ from ..utils import (
     unsmuggle_url,
     xpath_with_ns,
     mimetype2ext,
+    find_xpath_attr,
 )
 
 default_ns = 'http://www.w3.org/2005/SMIL21/Language'
@@ -31,15 +32,11 @@ _x = lambda p: xpath_with_ns(p, {'smil': default_ns})
 class ThePlatformBaseIE(InfoExtractor):
     def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL data'):
         meta = self._download_xml(smil_url, video_id, note=note)
-        try:
-            error_msg = next(
-                n.attrib['abstract']
-                for n in meta.findall(_x('.//smil:ref'))
-                if n.attrib.get('title') == 'Geographic Restriction' or n.attrib.get('title') == 'Expired')
-        except StopIteration:
-            pass
-        else:
-            raise ExtractorError(error_msg, expected=True)
+        error_element = find_xpath_attr(
+            meta, _x('.//smil:ref'), 'src',
+            'http://link.theplatform.com/s/errorFiles/Unavailable.mp4')
+        if error_element is not None:
+            raise ExtractorError(error_element.attrib['abstract'], expected=True)
 
         formats = self._parse_smil_formats(
             meta, smil_url, video_id, namespace=default_ns,

From 0ae937a798c2c9a2eae4c6dbadc68a1a4c5e101c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 13:39:04 +0800
Subject: [PATCH 0863/1105] [twitter] Support twitter.com/i/videos/tweet/ URLS

Closes #8737
---
 youtube_dl/extractor/twitter.py | 53 +++++++++++++++++++++++++++------
 1 file changed, 44 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 5d2b5ec35..5cbb536f9 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -22,7 +22,7 @@ class TwitterBaseIE(InfoExtractor):
 
 class TwitterCardIE(TwitterBaseIE):
     IE_NAME = 'twitter:card'
-    _VALID_URL = r'https?://(?:www\.)?twitter\.com/i/cards/tfw/v1/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?twitter\.com/i/(?:cards/tfw/v1|videos/tweet)/(?P<id>\d+)'
     _TESTS = [
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
@@ -30,7 +30,7 @@ class TwitterCardIE(TwitterBaseIE):
             'info_dict': {
                 'id': '560070183650213889',
                 'ext': 'mp4',
-                'title': 'TwitterCard',
+                'title': 'Twitter Card',
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'duration': 30.033,
             }
@@ -41,7 +41,7 @@ class TwitterCardIE(TwitterBaseIE):
             'info_dict': {
                 'id': '623160978427936768',
                 'ext': 'mp4',
-                'title': 'TwitterCard',
+                'title': 'Twitter Card',
                 'thumbnail': 're:^https?://.*\.jpg',
                 'duration': 80.155,
             },
@@ -72,7 +72,16 @@ class TwitterCardIE(TwitterBaseIE):
                 'title': 'Vine by ArsenalTerje',
             },
             'add_ie': ['Vine'],
-        }
+        }, {
+            'url': 'https://twitter.com/i/videos/tweet/705235433198714880',
+            'md5': '3846d0a07109b5ab622425449b59049d',
+            'info_dict': {
+                'id': '705235433198714880',
+                'ext': 'mp4',
+                'title': 'Twitter web player',
+                'thumbnail': 're:^https?://.*\.jpg',
+            },
+        },
     ]
 
     def _real_extract(self, url):
@@ -98,12 +107,13 @@ class TwitterCardIE(TwitterBaseIE):
                 return self.url_result(iframe_url)
 
             config = self._parse_json(self._html_search_regex(
-                r'data-player-config="([^"]+)"', webpage, 'data player config'),
+                r'data-(?:player-)?config="([^"]+)"', webpage, 'data player config'),
                 video_id)
             if 'playlist' not in config:
-                if 'vmapUrl' in config:
+                vmap_url = config.get('vmapUrl') or config.get('vmap_url')
+                if vmap_url:
                     formats.append({
-                        'url': self._get_vmap_video_url(config['vmapUrl'], video_id),
+                        'url': self._get_vmap_video_url(vmap_url, video_id),
                     })
                     break   # same video regardless of UA
                 continue
@@ -123,12 +133,13 @@ class TwitterCardIE(TwitterBaseIE):
             formats.append(f)
         self._sort_formats(formats)
 
-        thumbnail = config.get('posterImageUrl')
+        title = self._search_regex(r'<title>([^<]+)</title>', webpage, 'title')
+        thumbnail = config.get('posterImageUrl') or config.get('image_src')
         duration = float_or_none(config.get('duration'))
 
         return {
             'id': video_id,
-            'title': 'TwitterCard',
+            'title': title,
             'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,
@@ -177,6 +188,21 @@ class TwitterIE(InfoExtractor):
             'uploader_id': 'starwars',
             'uploader': 'Star Wars',
         },
+    }, {
+        'url': 'https://twitter.com/BTNBrentYarina/status/705235433198714880',
+        'info_dict': {
+            'id': '705235433198714880',
+            'ext': 'mp4',
+            'title': 'Brent Yarina - Khalil Iverson\'s missed highlight dunk. And made highlight dunk. In one highlight.',
+            'description': 'Brent Yarina on Twitter: "Khalil Iverson\'s missed highlight dunk. And made highlight dunk. In one highlight."',
+            'uploader_id': 'BTNBrentYarina',
+            'uploader': 'Brent Yarina',
+        },
+        'params': {
+            # The same video as https://twitter.com/i/videos/tweet/705235433198714880
+            # Test case of TwitterCardIE
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
@@ -234,6 +260,15 @@ class TwitterIE(InfoExtractor):
             })
             return info
 
+        if 'class="PlayableMedia' in webpage:
+            info.update({
+                '_type': 'url_transparent',
+                'ie_key': 'TwitterCard',
+                'url': '%s//twitter.com/i/videos/tweet/%s' % (self.http_scheme(), twid),
+            })
+
+            return info
+
         raise ExtractorError('There\'s no video in this tweet.')
 
 

From b8972bd69da48509333a8239fc0b755dccb1a681 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 14:24:24 +0800
Subject: [PATCH 0864/1105] [twitter] Fix extraction of test_Twitter and
 test_Twitter_1

---
 youtube_dl/extractor/twitter.py | 71 +++++++++++++++++++++++----------
 1 file changed, 51 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 5cbb536f9..3627d49f1 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -95,6 +95,8 @@ class TwitterCardIE(TwitterBaseIE):
 
         config = None
         formats = []
+        duration = None
+
         for user_agent in USER_AGENTS:
             request = sanitized_Request(url)
             request.add_header('User-Agent', user_agent)
@@ -109,33 +111,60 @@ class TwitterCardIE(TwitterBaseIE):
             config = self._parse_json(self._html_search_regex(
                 r'data-(?:player-)?config="([^"]+)"', webpage, 'data player config'),
                 video_id)
-            if 'playlist' not in config:
-                vmap_url = config.get('vmapUrl') or config.get('vmap_url')
-                if vmap_url:
-                    formats.append({
-                        'url': self._get_vmap_video_url(vmap_url, video_id),
+
+            playlist = config.get('playlist')
+            if playlist:
+                video_url = playlist[0]['source']
+
+                f = {
+                    'url': video_url,
+                }
+
+                m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
+                if m:
+                    f.update({
+                        'width': int(m.group('width')),
+                        'height': int(m.group('height')),
                     })
-                    break   # same video regardless of UA
+                formats.append(f)
                 continue
 
-            video_url = config['playlist'][0]['source']
-
-            f = {
-                'url': video_url,
-            }
-
-            m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
-            if m:
-                f.update({
-                    'width': int(m.group('width')),
-                    'height': int(m.group('height')),
+            vmap_url = config.get('vmapUrl') or config.get('vmap_url')
+            if vmap_url:
+                formats.append({
+                    'url': self._get_vmap_video_url(vmap_url, video_id),
                 })
-            formats.append(f)
+                break   # same video regardless of UA
+
+            media_info = config.get('status', {}).get('entities', [{}])[0].get('mediaInfo', {})
+            if media_info:
+                for media_variant in media_info['variants']:
+                    media_url = media_variant['url']
+                    if media_url.endswith('.m3u8'):
+                        formats.extend(self._extract_m3u8_formats(media_url, video_id, ext='mp4', m3u8_id='hls'))
+                    elif media_url.endswith('.mpd'):
+                        formats.extend(self._extract_mpd_formats(media_url, video_id, mpd_id='dash'))
+                    else:
+                        vbr = int_or_none(media_variant.get('bitRate'), scale=1000)
+                        a_format = {
+                            'url': media_url,
+                            'format_id': 'http-%d' % vbr if vbr else 'http',
+                            'vbr': vbr,
+                        }
+                        # Reported bitRate may be zero
+                        if not a_format['vbr']:
+                            del a_format['vbr']
+
+                        formats.append(a_format)
+
+                duration = float_or_none(media_info.get('duration', {}).get('nanos'), scale=1e9)
+                break   # same video regardless of UA
+
         self._sort_formats(formats)
 
         title = self._search_regex(r'<title>([^<]+)</title>', webpage, 'title')
         thumbnail = config.get('posterImageUrl') or config.get('image_src')
-        duration = float_or_none(config.get('duration'))
+        duration = float_or_none(config.get('duration')) or duration
 
         return {
             'id': video_id,
@@ -153,7 +182,6 @@ class TwitterIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'https://twitter.com/freethenipple/status/643211948184596480',
-        # MD5 checksums are different in different places
         'info_dict': {
             'id': '643211948184596480',
             'ext': 'mp4',
@@ -164,6 +192,9 @@ class TwitterIE(InfoExtractor):
             'uploader': 'FREE THE NIPPLE',
             'uploader_id': 'freethenipple',
         },
+        'params': {
+            'skip_download': True,  # requires ffmpeg
+        },
     }, {
         'url': 'https://twitter.com/giphz/status/657991469417025536/photo/1',
         'md5': 'f36dcd5fb92bf7057f155e7d927eeb42',

From c8398a9b87dadf4cd8464e1ab77d2874d9ae484b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 14:27:27 +0800
Subject: [PATCH 0865/1105] [twitter] Now Twitter serves the same file for
 Firefox and Chrome

---
 youtube_dl/extractor/twitter.py | 107 ++++++++++++++------------------
 1 file changed, 47 insertions(+), 60 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 3627d49f1..97a4a5948 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -10,7 +10,6 @@ from ..utils import (
     remove_end,
     int_or_none,
     ExtractorError,
-    sanitized_Request,
 )
 
 
@@ -87,78 +86,66 @@ class TwitterCardIE(TwitterBaseIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        # Different formats served for different User-Agents
-        USER_AGENTS = [
-            'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/20.0 (Chrome)',  # mp4
-            'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0',  # webm
-        ]
-
         config = None
         formats = []
         duration = None
 
-        for user_agent in USER_AGENTS:
-            request = sanitized_Request(url)
-            request.add_header('User-Agent', user_agent)
-            webpage = self._download_webpage(request, video_id)
+        webpage = self._download_webpage(url, video_id)
 
-            iframe_url = self._html_search_regex(
-                r'<iframe[^>]+src="((?:https?:)?//(?:www.youtube.com/embed/[^"]+|(?:www\.)?vine\.co/v/\w+/card))"',
-                webpage, 'video iframe', default=None)
-            if iframe_url:
-                return self.url_result(iframe_url)
+        iframe_url = self._html_search_regex(
+            r'<iframe[^>]+src="((?:https?:)?//(?:www.youtube.com/embed/[^"]+|(?:www\.)?vine\.co/v/\w+/card))"',
+            webpage, 'video iframe', default=None)
+        if iframe_url:
+            return self.url_result(iframe_url)
 
-            config = self._parse_json(self._html_search_regex(
-                r'data-(?:player-)?config="([^"]+)"', webpage, 'data player config'),
-                video_id)
+        config = self._parse_json(self._html_search_regex(
+            r'data-(?:player-)?config="([^"]+)"', webpage, 'data player config'),
+            video_id)
 
-            playlist = config.get('playlist')
-            if playlist:
-                video_url = playlist[0]['source']
+        playlist = config.get('playlist')
+        if playlist:
+            video_url = playlist[0]['source']
 
-                f = {
-                    'url': video_url,
-                }
+            f = {
+                'url': video_url,
+            }
 
-                m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
-                if m:
-                    f.update({
-                        'width': int(m.group('width')),
-                        'height': int(m.group('height')),
-                    })
-                formats.append(f)
-                continue
-
-            vmap_url = config.get('vmapUrl') or config.get('vmap_url')
-            if vmap_url:
-                formats.append({
-                    'url': self._get_vmap_video_url(vmap_url, video_id),
+            m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
+            if m:
+                f.update({
+                    'width': int(m.group('width')),
+                    'height': int(m.group('height')),
                 })
-                break   # same video regardless of UA
+            formats.append(f)
 
-            media_info = config.get('status', {}).get('entities', [{}])[0].get('mediaInfo', {})
-            if media_info:
-                for media_variant in media_info['variants']:
-                    media_url = media_variant['url']
-                    if media_url.endswith('.m3u8'):
-                        formats.extend(self._extract_m3u8_formats(media_url, video_id, ext='mp4', m3u8_id='hls'))
-                    elif media_url.endswith('.mpd'):
-                        formats.extend(self._extract_mpd_formats(media_url, video_id, mpd_id='dash'))
-                    else:
-                        vbr = int_or_none(media_variant.get('bitRate'), scale=1000)
-                        a_format = {
-                            'url': media_url,
-                            'format_id': 'http-%d' % vbr if vbr else 'http',
-                            'vbr': vbr,
-                        }
-                        # Reported bitRate may be zero
-                        if not a_format['vbr']:
-                            del a_format['vbr']
+        vmap_url = config.get('vmapUrl') or config.get('vmap_url')
+        if vmap_url:
+            formats.append({
+                'url': self._get_vmap_video_url(vmap_url, video_id),
+            })
 
-                        formats.append(a_format)
+        media_info = config.get('status', {}).get('entities', [{}])[0].get('mediaInfo', {})
+        if media_info:
+            for media_variant in media_info['variants']:
+                media_url = media_variant['url']
+                if media_url.endswith('.m3u8'):
+                    formats.extend(self._extract_m3u8_formats(media_url, video_id, ext='mp4', m3u8_id='hls'))
+                elif media_url.endswith('.mpd'):
+                    formats.extend(self._extract_mpd_formats(media_url, video_id, mpd_id='dash'))
+                else:
+                    vbr = int_or_none(media_variant.get('bitRate'), scale=1000)
+                    a_format = {
+                        'url': media_url,
+                        'format_id': 'http-%d' % vbr if vbr else 'http',
+                        'vbr': vbr,
+                    }
+                    # Reported bitRate may be zero
+                    if not a_format['vbr']:
+                        del a_format['vbr']
 
-                duration = float_or_none(media_info.get('duration', {}).get('nanos'), scale=1e9)
-                break   # same video regardless of UA
+                    formats.append(a_format)
+
+            duration = float_or_none(media_info.get('duration', {}).get('nanos'), scale=1e9)
 
         self._sort_formats(formats)
 

From 03879ff0547b6d1b96c530075cd99f99b8c74a2b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 14:42:49 +0800
Subject: [PATCH 0866/1105] [twitter] Media info is not always in the first
 entity

Fixes #8704
---
 youtube_dl/extractor/twitter.py | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 97a4a5948..67762a003 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -124,7 +124,12 @@ class TwitterCardIE(TwitterBaseIE):
                 'url': self._get_vmap_video_url(vmap_url, video_id),
             })
 
-        media_info = config.get('status', {}).get('entities', [{}])[0].get('mediaInfo', {})
+        media_info = None
+
+        for entity in config.get('status', {}).get('entities', []):
+            if 'mediaInfo' in entity:
+                media_info = entity['mediaInfo']
+
         if media_info:
             for media_variant in media_info['variants']:
                 media_url = media_variant['url']
@@ -221,6 +226,21 @@ class TwitterIE(InfoExtractor):
             # Test case of TwitterCardIE
             'skip_download': True,
         },
+    }, {
+        'url': 'https://twitter.com/jaydingeer/status/700207533655363584',
+        'md5': '',
+        'info_dict': {
+            'id': '700207533655363584',
+            'ext': 'mp4',
+            'title': 'jay - BEAT PROD: @suhmeduh #Damndaniel',
+            'description': 'jay on Twitter: "BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'uploader': 'jay',
+            'uploader_id': 'jaydingeer',
+        },
+        'params': {
+            'skip_download': True,  # requires ffmpeg
+        },
     }]
 
     def _real_extract(self, url):

From f3bdae76de7265475277c265f9fff12cdc59825b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 3 Mar 2016 10:27:22 +0100
Subject: [PATCH 0867/1105] [extractor/common] add update_url_params helper
 method to add or update query string params

---
 youtube_dl/extractor/common.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 402f2f436..282559597 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -517,6 +517,13 @@ class InfoExtractor(object):
             else:
                 self.report_warning(errmsg + str(ve))
 
+    def update_url_params(self, url, params):
+        parsed_url = compat_urlparse.urlparse(url)
+        qs = compat_urlparse.parse_qs(parsed_url.query)
+        qs.update(params)
+        return compat_urlparse.urlunparse(
+            parsed_url._replace(query=compat_urllib_parse.urlencode(qs, True)))
+
     def report_warning(self, msg, video_id=None):
         idstr = '' if video_id is None else '%s: ' % video_id
         self._downloader.report_warning(

From fa9e259fd91ce1bf31310330adc20ddef2b1d948 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 3 Mar 2016 10:54:39 +0100
Subject: [PATCH 0868/1105] [extractor/common] use compat_parse_qs in
 update_url_params

---
 youtube_dl/extractor/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 282559597..a95387cee 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -20,6 +20,7 @@ from ..compat import (
     compat_urllib_error,
     compat_urllib_parse,
     compat_urlparse,
+    compat_parse_qs,
     compat_str,
     compat_etree_fromstring,
 )
@@ -519,7 +520,7 @@ class InfoExtractor(object):
 
     def update_url_params(self, url, params):
         parsed_url = compat_urlparse.urlparse(url)
-        qs = compat_urlparse.parse_qs(parsed_url.query)
+        qs = compat_parse_qs(parsed_url.query)
         qs.update(params)
         return compat_urlparse.urlunparse(
             parsed_url._replace(query=compat_urllib_parse.urlencode(qs, True)))

From 8bb56eeeea8154f811076c0a9093203fab224003 Mon Sep 17 00:00:00 2001
From: Brian Foley <bpfoley@gmail.com>
Date: Sat, 2 Jan 2016 19:49:59 +0000
Subject: [PATCH 0869/1105] [utils] Add extract_attributes for extracting html
 tag attributes

This is much more robust than just using regexps, and handles all
the common scenarios, such as empty/no values, repeated attributes,
entity decoding, mixed case names, and the different possible value
quoting schemes.
---
 test/test_utils.py   | 40 ++++++++++++++++++++++++++++++++++++++++
 youtube_dl/compat.py |  6 ++++++
 youtube_dl/utils.py  | 30 ++++++++++++++++++++++++++++++
 3 files changed, 76 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 97587ad2f..cb85e18f0 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -28,6 +28,7 @@ from youtube_dl.utils import (
     encodeFilename,
     escape_rfc3986,
     escape_url,
+    extract_attributes,
     ExtractorError,
     find_xpath_attr,
     fix_xml_ampersands,
@@ -75,6 +76,7 @@ from youtube_dl.utils import (
     cli_bool_option,
 )
 from youtube_dl.compat import (
+    compat_chr,
     compat_etree_fromstring,
 )
 
@@ -591,6 +593,44 @@ class TestUtil(unittest.TestCase):
         on = js_to_json('{"abc": "def",}')
         self.assertEqual(json.loads(on), {'abc': 'def'})
 
+    def test_extract_attributes(self):
+        self.assertEqual(extract_attributes('<e x="y">'), {'x': 'y'})
+        self.assertEqual(extract_attributes("<e x='y'>"), {'x': 'y'})
+        self.assertEqual(extract_attributes('<e x=y>'), {'x': 'y'})
+        self.assertEqual(extract_attributes('<e x="a \'b\' c">'), {'x': "a 'b' c"})
+        self.assertEqual(extract_attributes('<e x=\'a "b" c\'>'), {'x': 'a "b" c'})
+        self.assertEqual(extract_attributes('<e x="&#121;">'), {'x': 'y'})
+        self.assertEqual(extract_attributes('<e x="&#x79;">'), {'x': 'y'})
+        self.assertEqual(extract_attributes('<e x="&amp;">'), {'x': '&'})  # XML
+        self.assertEqual(extract_attributes('<e x="&quot;">'), {'x': '"'})
+        self.assertEqual(extract_attributes('<e x="&pound;">'), {'x': '£'}) # HTML 3.2
+        self.assertEqual(extract_attributes('<e x="&lambda;">'), {'x': 'λ'}) # HTML 4.0
+        self.assertEqual(extract_attributes('<e x="&foo">'), {'x': '&foo'})
+        self.assertEqual(extract_attributes('<e x="\'">'), {'x': "'"})
+        self.assertEqual(extract_attributes('<e x=\'"\'>'), {'x': '"'})
+        self.assertEqual(extract_attributes('<e x >'), {'x': None})
+        self.assertEqual(extract_attributes('<e x=y a>'), {'x': 'y', 'a': None})
+        self.assertEqual(extract_attributes('<e x= y>'), {'x': 'y'})
+        self.assertEqual(extract_attributes('<e x=1 y=2 x=3>'), {'y': '2', 'x': '3'})
+        self.assertEqual(extract_attributes('<e \nx=\ny\n>'), {'x': 'y'})
+        self.assertEqual(extract_attributes('<e \nx=\n"y"\n>'), {'x': 'y'})
+        self.assertEqual(extract_attributes("<e \nx=\n'y'\n>"), {'x': 'y'})
+        self.assertEqual(extract_attributes('<e \nx="\ny\n">'), {'x': '\ny\n'})
+        self.assertEqual(extract_attributes('<e CAPS=x>'), {'caps': 'x'}) # Names lowercased
+        self.assertEqual(extract_attributes('<e x=1 X=2>'), {'x': '2'})
+        self.assertEqual(extract_attributes('<e X=1 x=2>'), {'x': '2'})
+        self.assertEqual(extract_attributes('<e _:funny-name1=1>'), {'_:funny-name1': '1'})
+        self.assertEqual(extract_attributes('<e x="Fáilte 世界 \U0001f600">'), {'x': 'Fáilte 世界 \U0001f600'})
+        self.assertEqual(extract_attributes('<e x="décompose&#769;">'), {'x': 'décompose\u0301'})
+        # "Narrow" Python builds don't support unicode code points outside BMP.
+        try:
+            compat_chr(0x10000)
+            supports_outside_bmp = True
+        except ValueError:
+            supports_outside_bmp = False
+        if supports_outside_bmp:
+            self.assertEqual(extract_attributes('<e x="Smile &#128512;!">'), {'x': 'Smile \U0001f600!'})
+
     def test_clean_html(self):
         self.assertEqual(clean_html('a:\nb'), 'a: b')
         self.assertEqual(clean_html('a:\n   "b"'), 'a:    "b"')
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index b497da696..7b9afc36d 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -77,6 +77,11 @@ try:
 except ImportError:  # Python 2
     from urllib import urlretrieve as compat_urlretrieve
 
+try:
+    from html.parser import HTMLParser as compat_HTMLParser
+except ImportError:  # Python 2
+    from HTMLParser import HTMLParser as compat_HTMLParser
+
 
 try:
     from subprocess import DEVNULL
@@ -540,6 +545,7 @@ else:
     from tokenize import generate_tokens as compat_tokenize_tokenize
 
 __all__ = [
+    'compat_HTMLParser',
     'compat_HTTPError',
     'compat_basestring',
     'compat_chr',
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 210c47fce..a0234a3a8 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -35,6 +35,7 @@ import xml.etree.ElementTree
 import zlib
 
 from .compat import (
+    compat_HTMLParser,
     compat_basestring,
     compat_chr,
     compat_etree_fromstring,
@@ -272,6 +273,35 @@ def get_element_by_attribute(attribute, value, html):
 
     return unescapeHTML(res)
 
+class HTMLAttributeParser(compat_HTMLParser):
+    """Trivial HTML parser to gather the attributes for a single element"""
+    def __init__(self):
+        self.attrs = { }
+        compat_HTMLParser.__init__(self)
+
+    def handle_starttag(self, tag, attrs):
+        self.attrs = dict(attrs)
+
+def extract_attributes(html_element):
+    """Given a string for an HTML element such as
+    <el
+         a="foo" B="bar" c="&98;az" d=boz
+         empty= noval entity="&amp;"
+         sq='"' dq="'"
+    >
+    Decode and return a dictionary of attributes.
+    {
+        'a': 'foo', 'b': 'bar', c: 'baz', d: 'boz',
+        'empty': '', 'noval': None, 'entity': '&',
+        'sq': '"', 'dq': '\''
+    }.
+    NB HTMLParser is stricter in Python 2.6 & 3.2 than in later versions,
+    but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
+    """
+    parser = HTMLAttributeParser()
+    parser.feed(html_element)
+    parser.close()
+    return parser.attrs
 
 def clean_html(html):
     """Clean an HTML snippet into a readable string"""

From 8ee239e9216f12eae38deb079090b677086e9de1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 18:47:54 +0800
Subject: [PATCH 0870/1105] [utils] Jython support - handle filenames correctly

Now test:youtube downloads
---
 youtube_dl/utils.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 16b4324a4..fc64a4186 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -467,6 +467,10 @@ def encodeFilename(s, for_subprocess=False):
     if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
         return s
 
+    # Jython assumes filenames are Unicode strings though reported as Python 2.x compatible
+    if sys.platform.startswith('java'):
+        return s
+
     return s.encode(get_subprocess_encoding(), 'ignore')
 
 

From e9c0cdd3895436170de33324c1762364380c6a5a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 19:24:24 +0800
Subject: [PATCH 0871/1105] [jython] Introduce compat_os_name

os.name is always 'java' on Jython
---
 test/helper.py                      |  7 +++++--
 youtube_dl/YoutubeDL.py             | 13 +++++++------
 youtube_dl/compat.py                |  8 ++++++--
 youtube_dl/downloader/common.py     |  3 ++-
 youtube_dl/extractor/common.py      |  9 +++++----
 youtube_dl/postprocessor/xattrpp.py |  5 +++--
 6 files changed, 28 insertions(+), 17 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index bdd7acca4..f2d878212 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -11,8 +11,11 @@ import sys
 
 import youtube_dl.extractor
 from youtube_dl import YoutubeDL
-from youtube_dl.utils import (
+from youtube_dl.compat import (
+    compat_os_name,
     compat_str,
+)
+from youtube_dl.utils import (
     preferredencoding,
     write_string,
 )
@@ -42,7 +45,7 @@ def report_warning(message):
     Print the message to stderr, it will be prefixed with 'WARNING:'
     If stderr is a tty file the 'WARNING:' will be colored
     '''
-    if sys.stderr.isatty() and os.name != 'nt':
+    if sys.stderr.isatty() and compat_os_name != 'nt':
         _msg_header = '\033[0;33mWARNING:\033[0m'
     else:
         _msg_header = 'WARNING:'
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8f3a8b9e3..dcc867e45 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -24,9 +24,6 @@ import time
 import tokenize
 import traceback
 
-if os.name == 'nt':
-    import ctypes
-
 from .compat import (
     compat_basestring,
     compat_cookiejar,
@@ -34,6 +31,7 @@ from .compat import (
     compat_get_terminal_size,
     compat_http_client,
     compat_kwargs,
+    compat_os_name,
     compat_str,
     compat_tokenize_tokenize,
     compat_urllib_error,
@@ -95,6 +93,9 @@ from .postprocessor import (
 )
 from .version import __version__
 
+if compat_os_name == 'nt':
+    import ctypes
+
 
 class YoutubeDL(object):
     """YoutubeDL class.
@@ -450,7 +451,7 @@ class YoutubeDL(object):
     def to_console_title(self, message):
         if not self.params.get('consoletitle', False):
             return
-        if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
+        if compat_os_name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
             # c_wchar_p() might not be necessary if `message` is
             # already of type unicode()
             ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
@@ -521,7 +522,7 @@ class YoutubeDL(object):
         else:
             if self.params.get('no_warnings'):
                 return
-            if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
+            if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
                 _msg_header = '\033[0;33mWARNING:\033[0m'
             else:
                 _msg_header = 'WARNING:'
@@ -533,7 +534,7 @@ class YoutubeDL(object):
         Do the same as trouble, but prefixes the message with 'ERROR:', colored
         in red if stderr is a tty file.
         '''
-        if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
+        if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
             _msg_header = '\033[0;31mERROR:\033[0m'
         else:
             _msg_header = 'ERROR:'
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index b497da696..2771fb5fa 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -326,6 +326,9 @@ def compat_ord(c):
         return ord(c)
 
 
+compat_os_name = os._name if os.name == 'java' else os.name
+
+
 if sys.version_info >= (3, 0):
     compat_getenv = os.getenv
     compat_expanduser = os.path.expanduser
@@ -346,7 +349,7 @@ else:
     # The following are os.path.expanduser implementations from cpython 2.7.8 stdlib
     # for different platforms with correct environment variables decoding.
 
-    if os.name == 'posix':
+    if compat_os_name == 'posix':
         def compat_expanduser(path):
             """Expand ~ and ~user constructions.  If user or $HOME is unknown,
             do nothing."""
@@ -370,7 +373,7 @@ else:
                 userhome = pwent.pw_dir
             userhome = userhome.rstrip('/')
             return (userhome + path[i:]) or '/'
-    elif os.name == 'nt' or os.name == 'ce':
+    elif compat_os_name == 'nt' or compat_os_name == 'ce':
         def compat_expanduser(path):
             """Expand ~ and ~user constructs.
 
@@ -556,6 +559,7 @@ __all__ = [
     'compat_itertools_count',
     'compat_kwargs',
     'compat_ord',
+    'compat_os_name',
     'compat_parse_qs',
     'compat_print',
     'compat_shlex_split',
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 2d5154051..f39db58f6 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -5,6 +5,7 @@ import re
 import sys
 import time
 
+from ..compat import compat_os_name
 from ..utils import (
     encodeFilename,
     error_to_compat_str,
@@ -219,7 +220,7 @@ class FileDownloader(object):
         if self.params.get('progress_with_newline', False):
             self.to_screen(fullmsg)
         else:
-            if os.name == 'nt':
+            if compat_os_name == 'nt':
                 prev_len = getattr(self, '_report_progress_prev_line_length',
                                    0)
                 if prev_len > len(fullmsg):
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 402f2f436..07bd2cbe2 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -15,13 +15,14 @@ import math
 from ..compat import (
     compat_cookiejar,
     compat_cookies,
+    compat_etree_fromstring,
     compat_getpass,
     compat_http_client,
+    compat_os_name,
+    compat_str,
     compat_urllib_error,
     compat_urllib_parse,
     compat_urlparse,
-    compat_str,
-    compat_etree_fromstring,
 )
 from ..utils import (
     NO_DEFAULT,
@@ -427,7 +428,7 @@ class InfoExtractor(object):
             self.to_screen('Saving request to ' + filename)
             # Working around MAX_PATH limitation on Windows (see
             # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)
-            if os.name == 'nt':
+            if compat_os_name == 'nt':
                 absfilepath = os.path.abspath(filename)
                 if len(absfilepath) > 259:
                     filename = '\\\\?\\' + absfilepath
@@ -596,7 +597,7 @@ class InfoExtractor(object):
                 if mobj:
                     break
 
-        if not self._downloader.params.get('no_color') and os.name != 'nt' and sys.stderr.isatty():
+        if not self._downloader.params.get('no_color') and compat_os_name != 'nt' and sys.stderr.isatty():
             _name = '\033[0;34m%s\033[0m' % name
         else:
             _name = name
diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index 480d48d05..e39ca60aa 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -6,6 +6,7 @@ import sys
 import errno
 
 from .common import PostProcessor
+from ..compat import compat_os_name
 from ..utils import (
     check_executable,
     hyphenate_date,
@@ -73,7 +74,7 @@ class XAttrMetadataPP(PostProcessor):
                     raise XAttrMetadataError(e.errno, e.strerror)
 
         except ImportError:
-            if os.name == 'nt':
+            if compat_os_name == 'nt':
                 # Write xattrs to NTFS Alternate Data Streams:
                 # http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
                 def write_xattr(path, key, value):
@@ -168,7 +169,7 @@ class XAttrMetadataPP(PostProcessor):
                     'Unable to write extended attributes due to too long values.')
             else:
                 msg = 'This filesystem doesn\'t support extended attributes. '
-                if os.name == 'nt':
+                if compat_os_name == 'nt':
                     msg += 'You need to use NTFS.'
                 else:
                     msg += '(You may have to enable them in your /etc/fstab)'

From 429491f531035f4fe9e958e6bf4108867107efec Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 21:55:17 +0800
Subject: [PATCH 0872/1105] [test/http] Fix failure in Jython

make offlinetest passed on the latest Jython hg version with patched
lib-python/2.7/urllib2.py pulled from CPython 2.7.11
---
 test/test_http.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/test/test_http.py b/test/test_http.py
index f2e305b6f..fc59b1aed 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -52,7 +52,12 @@ class TestHTTP(unittest.TestCase):
             ('localhost', 0), HTTPTestRequestHandler)
         self.httpd.socket = ssl.wrap_socket(
             self.httpd.socket, certfile=certfn, server_side=True)
-        self.port = self.httpd.socket.getsockname()[1]
+        if os.name == 'java':
+            # In Jython SSLSocket is not a subclass of socket.socket
+            sock = self.httpd.socket.sock
+        else:
+            sock = self.httpd.socket
+        self.port = sock.getsockname()[1]
         self.server_thread = threading.Thread(target=self.httpd.serve_forever)
         self.server_thread.daemon = True
         self.server_thread.start()

From 1b0635aba3ded1142ff3f2b1c4bd67fb0cb3d6ae Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 21:57:49 +0800
Subject: [PATCH 0873/1105] [Makefile] Allow specifying the Python version in
 offline tests

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index afd7f4032..e98806791 100644
--- a/Makefile
+++ b/Makefile
@@ -45,7 +45,7 @@ test:
 ot: offlinetest
 
 offlinetest: codetest
-	nosetests --verbose test --exclude test_download.py --exclude test_age_restriction.py --exclude test_subtitles.py --exclude test_write_annotations.py --exclude test_youtube_lists.py --exclude test_iqiyi_sdk_interpreter.py
+	$(PYTHON) -m nose --verbose test --exclude test_download.py --exclude test_age_restriction.py --exclude test_subtitles.py --exclude test_write_annotations.py --exclude test_youtube_lists.py --exclude test_iqiyi_sdk_interpreter.py
 
 tar: youtube-dl.tar.gz
 

From 539a1641c6effa3a5b2be2fecd2fd4a6f35cef62 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 3 Mar 2016 22:10:36 +0600
Subject: [PATCH 0874/1105] [twitch] Workaround broken paging (Closes #8740)

---
 youtube_dl/extractor/twitch.py | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 5b8586097..d92d2fd40 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -271,7 +271,7 @@ class TwitchVodIE(TwitchItemBaseIE):
 
 class TwitchPlaylistBaseIE(TwitchBaseIE):
     _PLAYLIST_URL = '%s/kraken/channels/%%s/videos/?offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
-    _PAGE_LIMIT = 100
+    _PAGE_LIMIT = 10
 
     def _extract_playlist(self, channel_id):
         info = self._download_json(
@@ -281,14 +281,34 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
         entries = []
         offset = 0
         limit = self._PAGE_LIMIT
+        broken_paging_detected = False
+        counter_override = None
         for counter in itertools.count(1):
             response = self._download_json(
                 self._PLAYLIST_URL % (channel_id, offset, limit),
-                channel_id, 'Downloading %s videos JSON page %d' % (self._PLAYLIST_TYPE, counter))
+                channel_id,
+                'Downloading %s videos JSON page %s'
+                % (self._PLAYLIST_TYPE, counter_override or counter))
             page_entries = self._extract_playlist_page(response)
             if not page_entries:
                 break
+            total = int_or_none(response.get('_total'))
+            # Since the beginning of March 2016 twitch's paging mechanism
+            # is completely broken on the twitch side. It simply ignores
+            # a limit and returns the whole offset number of videos.
+            # Working around by just requesting all videos at once.
+            if not broken_paging_detected and total and len(page_entries) > limit:
+                self.report_warning(
+                    'Twitch paging is broken on twitch side, requesting all videos at once',
+                    channel_id)
+                broken_paging_detected = True
+                limit = total
+                offset = 0
+                counter_override = '(all at once)'
+                continue
             entries.extend(page_entries)
+            if broken_paging_detected or total and len(page_entries) >= total:
+                break
             offset += limit
         return self.playlist_result(
             [self.url_result(entry) for entry in set(entries)],
@@ -303,7 +323,6 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
 
 
 class TwitchProfileIE(TwitchPlaylistBaseIE):
-    _WORKING = False
     IE_NAME = 'twitch:profile'
     _VALID_URL = r'%s/(?P<id>[^/]+)/profile/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
     _PLAYLIST_TYPE = 'profile'
@@ -319,7 +338,6 @@ class TwitchProfileIE(TwitchPlaylistBaseIE):
 
 
 class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
-    _WORKING = False
     IE_NAME = 'twitch:past_broadcasts'
     _VALID_URL = r'%s/(?P<id>[^/]+)/profile/past_broadcasts/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
     _PLAYLIST_URL = TwitchPlaylistBaseIE._PLAYLIST_URL + '&broadcasts=true'
@@ -336,7 +354,6 @@ class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
 
 
 class TwitchBookmarksIE(TwitchPlaylistBaseIE):
-    _WORKING = False
     IE_NAME = 'twitch:bookmarks'
     _VALID_URL = r'%s/(?P<id>[^/]+)/profile/bookmarks/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
     _PLAYLIST_URL = '%s/api/bookmark/?user=%%s&offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE

From 8bbb4b56ee6c582f416ca1f8951a6821fccc1548 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 3 Mar 2016 22:11:26 +0600
Subject: [PATCH 0875/1105] [twitch:playlistsbase] Use orderedSet

---
 youtube_dl/extractor/twitch.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index d92d2fd40..a9f8a5b8b 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -17,6 +17,7 @@ from ..utils import (
     encode_dict,
     ExtractorError,
     int_or_none,
+    orderedSet,
     parse_duration,
     parse_iso8601,
     sanitized_Request,
@@ -311,7 +312,7 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
                 break
             offset += limit
         return self.playlist_result(
-            [self.url_result(entry) for entry in set(entries)],
+            [self.url_result(entry) for entry in orderedSet(entries)],
             channel_id, channel_name)
 
     def _extract_playlist_page(self, response):

From ececca6cde9f42f24d1d8b74a3c56f1149f70f5d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 3 Mar 2016 22:12:55 +0600
Subject: [PATCH 0876/1105] [twitch:playlistbase] Restore original _PAGE_LIMIT

---
 youtube_dl/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index a9f8a5b8b..0cf240e1e 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -272,7 +272,7 @@ class TwitchVodIE(TwitchItemBaseIE):
 
 class TwitchPlaylistBaseIE(TwitchBaseIE):
     _PLAYLIST_URL = '%s/kraken/channels/%%s/videos/?offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
-    _PAGE_LIMIT = 10
+    _PAGE_LIMIT = 100
 
     def _extract_playlist(self, channel_id):
         info = self._download_json(

From a8276b268074eb844f0d62ff1cc9fd163a91d970 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 3 Mar 2016 22:18:32 +0600
Subject: [PATCH 0877/1105] [twitch:playlistbase] Fix all at once fetch

---
 youtube_dl/extractor/twitch.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 0cf240e1e..8639293e3 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -303,8 +303,7 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
                     'Twitch paging is broken on twitch side, requesting all videos at once',
                     channel_id)
                 broken_paging_detected = True
-                limit = total
-                offset = 0
+                offset = total
                 counter_override = '(all at once)'
                 continue
             entries.extend(page_entries)

From 38f9ef31dc434a6702686844b421085955137c55 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 3 Mar 2016 18:34:52 +0100
Subject: [PATCH 0878/1105] [utils] add update_url_query function

---
 youtube_dl/extractor/common.py | 8 --------
 youtube_dl/utils.py            | 8 ++++++++
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a95387cee..402f2f436 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -20,7 +20,6 @@ from ..compat import (
     compat_urllib_error,
     compat_urllib_parse,
     compat_urlparse,
-    compat_parse_qs,
     compat_str,
     compat_etree_fromstring,
 )
@@ -518,13 +517,6 @@ class InfoExtractor(object):
             else:
                 self.report_warning(errmsg + str(ve))
 
-    def update_url_params(self, url, params):
-        parsed_url = compat_urlparse.urlparse(url)
-        qs = compat_parse_qs(parsed_url.query)
-        qs.update(params)
-        return compat_urlparse.urlunparse(
-            parsed_url._replace(query=compat_urllib_parse.urlencode(qs, True)))
-
     def report_warning(self, msg, video_id=None):
         idstr = '' if video_id is None else '%s: ' % video_id
         self._downloader.report_warning(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 210c47fce..31d60f323 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1719,6 +1719,14 @@ def urlencode_postdata(*args, **kargs):
     return compat_urllib_parse.urlencode(*args, **kargs).encode('ascii')
 
 
+def update_url_query(url, query):
+    parsed_url = compat_urlparse.urlparse(url)
+    qs = compat_parse_qs(parsed_url.query)
+    qs.update(query)
+    return compat_urlparse.urlunparse(parsed_url._replace(
+        query=compat_urllib_parse.urlencode(qs, True)))
+
+
 def encode_dict(d, encoding='utf-8'):
     def encode(v):
         return v.encode(encoding) if isinstance(v, compat_basestring) else v

From fb640d0a3d3af77e6835d9a6a2bf80920785511a Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 3 Mar 2016 18:40:05 +0100
Subject: [PATCH 0879/1105] [test/test_utils] add tests for update_url_query

---
 test/test_utils.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 97587ad2f..19a546619 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -61,6 +61,7 @@ from youtube_dl.utils import (
     lowercase_escape,
     url_basename,
     urlencode_postdata,
+    update_url_query,
     version_tuple,
     xpath_with_ns,
     xpath_element,
@@ -76,6 +77,8 @@ from youtube_dl.utils import (
 )
 from youtube_dl.compat import (
     compat_etree_fromstring,
+    compat_urlparse,
+    compat_parse_qs,
 )
 
 
@@ -454,6 +457,31 @@ class TestUtil(unittest.TestCase):
         data = urlencode_postdata({'username': 'foo@bar.com', 'password': '1234'})
         self.assertTrue(isinstance(data, bytes))
 
+    def test_update_url_query(self):
+        def query_dict(url):
+            return compat_parse_qs(compat_urlparse.urlparse(url).query)
+        self.assertEqual(query_dict(update_url_query(
+            'http://example.com/path', {'quality': ['HD'], 'format': ['mp4']})),
+            query_dict('http://example.com/path?quality=HD&format=mp4'))
+        self.assertEqual(query_dict(update_url_query(
+            'http://example.com/path', {'system': ['LINUX', 'WINDOWS']})),
+            query_dict('http://example.com/path?system=LINUX&system=WINDOWS'))
+        self.assertEqual(query_dict(update_url_query(
+            'http://example.com/path', {'fields': 'id,formats,subtitles'})),
+            query_dict('http://example.com/path?fields=id,formats,subtitles'))
+        self.assertEqual(query_dict(update_url_query(
+            'http://example.com/path', {'fields': ('id,formats,subtitles', 'thumbnails')})),
+            query_dict('http://example.com/path?fields=id,formats,subtitles&fields=thumbnails'))
+        self.assertEqual(query_dict(update_url_query(
+            'http://example.com/path?manifest=f4m', {'manifest': []})),
+            query_dict('http://example.com/path'))
+        self.assertEqual(query_dict(update_url_query(
+            'http://example.com/path?system=LINUX&system=WINDOWS', {'system': 'LINUX'})),
+            query_dict('http://example.com/path?system=LINUX'))
+        self.assertEqual(query_dict(update_url_query(
+            'http://example.com/path', {'fields': b'id,formats,subtitles'})),
+            query_dict('http://example.com/path?fields=id,formats,subtitles'))
+
     def test_dict_get(self):
         FALSE_VALUES = {
             'none': None,

From d0ff690d68016f24d8cc90e69fac20847e7afa8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 4 Mar 2016 00:09:40 +0600
Subject: [PATCH 0880/1105] [indavideo:embed] Fix tags extraction (Closes
 #8738)

---
 youtube_dl/extractor/indavideo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/indavideo.py b/youtube_dl/extractor/indavideo.py
index 12fb5e8e1..9622f198a 100644
--- a/youtube_dl/extractor/indavideo.py
+++ b/youtube_dl/extractor/indavideo.py
@@ -73,7 +73,7 @@ class IndavideoEmbedIE(InfoExtractor):
             'url': self._proto_relative_url(thumbnail)
         } for thumbnail in video.get('thumbnails', [])]
 
-        tags = [tag['title'] for tag in video.get('tags', [])]
+        tags = [tag['title'] for tag in video.get('tags') or []]
 
         return {
             'id': video.get('id') or video_id,

From 3201a67f618f3bb7b13b4df2a9c7f68f481940a4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 3 Mar 2016 19:18:57 +0100
Subject: [PATCH 0881/1105] [test/test_utils] add more tests for
 update_url_query

---
 test/test_utils.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 19a546619..2bcf8ecf0 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -481,6 +481,15 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(query_dict(update_url_query(
             'http://example.com/path', {'fields': b'id,formats,subtitles'})),
             query_dict('http://example.com/path?fields=id,formats,subtitles'))
+        self.assertEqual(query_dict(update_url_query(
+            'http://example.com/path', {'width': 1080, 'height': 720})),
+            query_dict('http://example.com/path?width=1080&height=720'))
+        self.assertEqual(query_dict(update_url_query(
+            'http://example.com/path', {'bitrate': 5020.43})),
+            query_dict('http://example.com/path?bitrate=5020.43'))
+        self.assertEqual(query_dict(update_url_query(
+            'http://example.com/path', {'test': '第二行тест'})),
+            query_dict('http://example.com/path?test=%E7%AC%AC%E4%BA%8C%E8%A1%8C%D1%82%D0%B5%D1%81%D1%82'))
 
     def test_dict_get(self):
         FALSE_VALUES = {

From e1dd521e4964eae3aa4207c00e1673ab1ea0d4c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 4 Mar 2016 01:16:58 +0600
Subject: [PATCH 0882/1105] [livestream] Fix FutureWarning (Closes #8742)

---
 youtube_dl/extractor/livestream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 38fb3d9e4..988436226 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -64,7 +64,7 @@ class LivestreamIE(InfoExtractor):
     def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
         base_ele = find_xpath_attr(
             smil, self._xpath_ns('.//meta', namespace), 'name', 'httpBase')
-        base = base_ele.get('content') if base_ele else 'http://livestreamvod-f.akamaihd.net/'
+        base = base_ele.get('content') if base_ele is not None else 'http://livestreamvod-f.akamaihd.net/'
 
         formats = []
         video_nodes = smil.findall(self._xpath_ns('.//video', namespace))

From 24ca0e9c0b217c99a852682d9aa1357cd380eb06 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 4 Mar 2016 13:36:29 +0800
Subject: [PATCH 0883/1105] [douyutv] Fix tests

---
 youtube_dl/extractor/douyutv.py | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index 373b3b4b4..bdc768c78 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -18,7 +18,7 @@ class DouyuTVIE(InfoExtractor):
             'display_id': 'iseven',
             'ext': 'flv',
             'title': 're:^清晨醒脑！T-ara根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
-            'description': 'md5:c93d6692dde6fe33809a46edcbecca44',
+            'description': 'md5:f34981259a03e980a3c6404190a3ed61',
             'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': '7师傅',
             'uploader_id': '431925',
@@ -26,7 +26,7 @@ class DouyuTVIE(InfoExtractor):
         },
         'params': {
             'skip_download': True,
-        }
+        },
     }, {
         'url': 'http://www.douyutv.com/85982',
         'info_dict': {
@@ -42,7 +42,24 @@ class DouyuTVIE(InfoExtractor):
         },
         'params': {
             'skip_download': True,
-        }
+        },
+        'skip': 'Romm not found',
+    }, {
+        'url': 'http://www.douyutv.com/17732',
+        'info_dict': {
+            'id': '17732',
+            'display_id': '17732',
+            'ext': 'flv',
+            'title': 're:^清晨醒脑！T-ara根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 'md5:f34981259a03e980a3c6404190a3ed61',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': '7师傅',
+            'uploader_id': '431925',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):

From 8a370aedace161f97b7bff572275f3e8dd59da2e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 4 Mar 2016 13:38:45 +0800
Subject: [PATCH 0884/1105] [leeco] format_id should be strings

---
 youtube_dl/extractor/leeco.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/leeco.py b/youtube_dl/extractor/leeco.py
index d0cd3f591..9c2bce036 100644
--- a/youtube_dl/extractor/leeco.py
+++ b/youtube_dl/extractor/leeco.py
@@ -338,7 +338,7 @@ class LetvCloudIE(InfoExtractor):
             formats.append({
                 'url': url,
                 'ext': determine_ext(decoded_url),
-                'format_id': int_or_none(play_url.get('vtype')),
+                'format_id': str_or_none(play_url.get('vtype')),
                 'format_note': str_or_none(play_url.get('definition')),
                 'width': int_or_none(play_url.get('vwidth')),
                 'height': int_or_none(play_url.get('vheight')),

From d58cb3ec7e115cf07a6cf6e035cbba2b5c6861fd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 4 Mar 2016 13:46:38 +0800
Subject: [PATCH 0885/1105] [leeco] Skip an invalid test. test_LePlaylist_1 is
 sufficient

---
 youtube_dl/extractor/leeco.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/leeco.py b/youtube_dl/extractor/leeco.py
index 9c2bce036..df47e88ba 100644
--- a/youtube_dl/extractor/leeco.py
+++ b/youtube_dl/extractor/leeco.py
@@ -217,14 +217,8 @@ class LePlaylistIE(InfoExtractor):
         'playlist_mincount': 96
     }, {
         'url': 'http://tv.le.com/pzt/lswjzzjc/index.shtml',
-        'info_dict': {
-            'id': 'lswjzzjc',
-            # The title should be "劲舞青春", but I can't find a simple way to
-            # determine the playlist title
-            'title': '乐视午间自制剧场',
-            'description': 'md5:b1eef244f45589a7b5b1af9ff25a4489'
-        },
-        'playlist_mincount': 7
+        # This series is moved to http://www.le.com/tv/10005297.html
+        'only_matching': True,
     }, {
         'url': 'http://www.le.com/comic/92063.html',
         'only_matching': True,

From b6f94d81ea8d3243edf5a4378760281d9d9c2cba Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 4 Mar 2016 14:32:01 +0800
Subject: [PATCH 0886/1105] [kusi] Add a test for the alternative form of URL

---
 youtube_dl/extractor/kusi.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/kusi.py b/youtube_dl/extractor/kusi.py
index 20407411b..5e87c5f79 100644
--- a/youtube_dl/extractor/kusi.py
+++ b/youtube_dl/extractor/kusi.py
@@ -10,7 +10,7 @@ from ..utils import int_or_none
 
 class KUSIIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?kusi\.com/(?P<path>story/.+|video\?clipId=(?P<clipId>\d+))'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.kusi.com/story/31183873/turko-files-case-closed-put-on-hold',
         'md5': 'f926e7684294cf8cb7bdf8858e1b3988',
         'info_dict': {
@@ -18,8 +18,19 @@ class KUSIIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Turko Files: Case Closed! & Put On Hold!',
             'duration': 231000,
-        }
-    }
+        },
+    }, {
+        'url': 'http://kusi.com/video?clipId=12203019',
+        'info_dict': {
+            'id': '12203019',
+            'ext': 'mp4',
+            'title': 'Turko Files: Case Closed! & Put On Hold!',
+            'duration': 231000,
+        },
+        'params': {
+            'skip_download': True,  # Same as previous one
+        },
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 6d6536acb27483087a8a4ff73f501eaca313ec6a Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Mar 2016 10:25:16 +0100
Subject: [PATCH 0887/1105] [fivemin] improve extraction

- skip m3u8 formats(404 error)
- skip unavailable test
- download embed page only when it's needed
- update _VALID_URL regex(joystiq.com redirect to engadget.com)
---
 youtube_dl/extractor/fivemin.py | 51 +++++++++++++++++++--------------
 1 file changed, 30 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/fivemin.py b/youtube_dl/extractor/fivemin.py
index 2955965d9..67d50a386 100644
--- a/youtube_dl/extractor/fivemin.py
+++ b/youtube_dl/extractor/fivemin.py
@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
@@ -16,12 +18,7 @@ from ..utils import (
 
 class FiveMinIE(InfoExtractor):
     IE_NAME = '5min'
-    _VALID_URL = r'''(?x)
-        (?:https?://[^/]*?5min\.com/Scripts/PlayerSeed\.js\?(?:.*?&)?playList=|
-            https?://(?:(?:massively|www)\.)?joystiq\.com/video/|
-            5min:)
-        (?P<id>\d+)
-        '''
+    _VALID_URL = r'(?:5min:(?P<id>\d+)(?::(?P<sid>\d+))?|https?://[^/]*?5min\.com/Scripts/PlayerSeed\.js\?(?P<query>.*))'
 
     _TESTS = [
         {
@@ -45,6 +42,7 @@ class FiveMinIE(InfoExtractor):
                 'title': 'How to Make a Next-Level Fruit Salad',
                 'duration': 184,
             },
+            'skip': 'no longer available',
         },
     ]
     _ERRORS = {
@@ -91,20 +89,33 @@ class FiveMinIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        sid = mobj.group('sid')
+
+        if mobj.group('query'):
+            qs = compat_parse_qs(mobj.group('query'))
+            if not qs.get('playList'):
+                raise ExtractorError('Invalid URL', expected=True)
+            video_id = qs['playList'][0]
+            if qs.get('sid'):
+                sid = qs['sid'][0]
+
         embed_url = 'https://embed.5min.com/playerseed/?playList=%s' % video_id
-        embed_page = self._download_webpage(embed_url, video_id,
-                                            'Downloading embed page')
-        sid = self._search_regex(r'sid=(\d+)', embed_page, 'sid')
-        query = compat_urllib_parse.urlencode({
-            'func': 'GetResults',
-            'playlist': video_id,
-            'sid': sid,
-            'isPlayerSeed': 'true',
-            'url': embed_url,
-        })
+        if not sid:
+            embed_page = self._download_webpage(embed_url, video_id,
+                                                'Downloading embed page')
+            sid = self._search_regex(r'sid=(\d+)', embed_page, 'sid')
+
         response = self._download_json(
-            'https://syn.5min.com/handlers/SenseHandler.ashx?' + query,
+            'https://syn.5min.com/handlers/SenseHandler.ashx?' +
+            compat_urllib_parse.urlencode({
+                'func': 'GetResults',
+                'playlist': video_id,
+                'sid': sid,
+                'isPlayerSeed': 'true',
+                'url': embed_url,
+            }),
             video_id)
         if not response['success']:
             raise ExtractorError(
@@ -118,9 +129,7 @@ class FiveMinIE(InfoExtractor):
         parsed_video_url = compat_urllib_parse_urlparse(compat_parse_qs(
             compat_urllib_parse_urlparse(info['EmbededURL']).query)['videoUrl'][0])
         for rendition in info['Renditions']:
-            if rendition['RenditionType'] == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(rendition['Url'], video_id, m3u8_id='hls'))
-            elif rendition['RenditionType'] == 'aac':
+            if rendition['RenditionType'] == 'aac' or rendition['RenditionType'] == 'm3u8':
                 continue
             else:
                 rendition_url = compat_urlparse.urlunparse(parsed_video_url._replace(path=replace_extension(parsed_video_url.path.replace('//', '/%s/' % rendition['ID']), rendition['RenditionType'])))

From d48352fb5d08208475c0eecf8d88d05d04747041 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Mar 2016 10:40:39 +0100
Subject: [PATCH 0888/1105] [engadget] remove support for legacy urls

---
 youtube_dl/extractor/engadget.py | 25 +++----------------------
 1 file changed, 3 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/engadget.py b/youtube_dl/extractor/engadget.py
index e4180701d..e5e57d485 100644
--- a/youtube_dl/extractor/engadget.py
+++ b/youtube_dl/extractor/engadget.py
@@ -1,21 +1,13 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
-    url_basename,
-)
 
 
 class EngadgetIE(InfoExtractor):
-    _VALID_URL = r'''(?x)https?://www.engadget.com/
-        (?:video(?:/5min)?/(?P<id>\d+)|
-            [\d/]+/.*?)
-        '''
+    _VALID_URL = r'https?://www.engadget.com/video/(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://www.engadget.com/video/5min/518153925/',
+        'url': 'http://www.engadget.com/video/518153925/',
         'md5': 'c6820d4828a5064447a4d9fc73f312c9',
         'info_dict': {
             'id': '518153925',
@@ -27,15 +19,4 @@ class EngadgetIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
-        if video_id is not None:
-            return self.url_result('5min:%s' % video_id)
-        else:
-            title = url_basename(url)
-            webpage = self._download_webpage(url, title)
-            ids = re.findall(r'<iframe[^>]+?playList=(\d+)', webpage)
-            return {
-                '_type': 'playlist',
-                'title': title,
-                'entries': [self.url_result('5min:%s' % vid) for vid in ids]
-            }
+        return self.url_result('5min:%s' % video_id)

From 1d4c9ed90ca8c01fcfdb838bafaa201c04bd599e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Mar 2016 10:42:58 +0100
Subject: [PATCH 0889/1105] [aol] imporve extraction

- add support for aol features
- remove support for legacy urls
---
 youtube_dl/extractor/__init__.py |  5 ++-
 youtube_dl/extractor/aol.py      | 70 +++++++++++---------------------
 2 files changed, 27 insertions(+), 48 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 037654a23..db1827666 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -23,7 +23,10 @@ from .alphaporno import AlphaPornoIE
 from .animeondemand import AnimeOnDemandIE
 from .anitube import AnitubeIE
 from .anysex import AnySexIE
-from .aol import AolIE
+from .aol import (
+    AolIE,
+    AolFeaturesIE,
+)
 from .allocine import AllocineIE
 from .aparat import AparatIE
 from .appleconnect import AppleConnectIE
diff --git a/youtube_dl/extractor/aol.py b/youtube_dl/extractor/aol.py
index b51eafc45..b761b2cc4 100644
--- a/youtube_dl/extractor/aol.py
+++ b/youtube_dl/extractor/aol.py
@@ -1,24 +1,11 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
 class AolIE(InfoExtractor):
     IE_NAME = 'on.aol.com'
-    _VALID_URL = r'''(?x)
-        (?:
-            aol-video:|
-            http://on\.aol\.com/
-            (?:
-                video/.*-|
-                playlist/(?P<playlist_display_id>[^/?#]+?)-(?P<playlist_id>[0-9]+)[?#].*_videoid=
-            )
-        )
-        (?P<id>[0-9]+)
-        (?:$|\?)
-    '''
+    _VALID_URL = r'(?:aol-video:|http://on\.aol\.com/video/.*-)(?P<id>[0-9]+)(?:$|\?)'
 
     _TESTS = [{
         'url': 'http://on.aol.com/video/u-s--official-warns-of-largest-ever-irs-phone-scam-518167793?icid=OnHomepageC2Wide_MustSee_Img',
@@ -29,42 +16,31 @@ class AolIE(InfoExtractor):
             'title': 'U.S. Official Warns Of \'Largest Ever\' IRS Phone Scam',
         },
         'add_ie': ['FiveMin'],
-    }, {
-        'url': 'http://on.aol.com/playlist/brace-yourself---todays-weirdest-news-152147?icid=OnHomepageC4_Omg_Img#_videoid=518184316',
-        'info_dict': {
-            'id': '152147',
-            'title': 'Brace Yourself - Today\'s Weirdest News',
-        },
-        'playlist_mincount': 10,
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        playlist_id = mobj.group('playlist_id')
-        if not playlist_id or self._downloader.params.get('noplaylist'):
-            return self.url_result('5min:%s' % video_id)
+        video_id = self._match_id(url)
+        return self.url_result('5min:%s' % video_id)
 
-        self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
 
-        webpage = self._download_webpage(url, playlist_id)
-        title = self._html_search_regex(
-            r'<h1 class="video-title[^"]*">(.+?)</h1>', webpage, 'title')
-        playlist_html = self._search_regex(
-            r"(?s)<ul\s+class='video-related[^']*'>(.*?)</ul>", webpage,
-            'playlist HTML')
-        entries = [{
-            '_type': 'url',
-            'url': 'aol-video:%s' % m.group('id'),
-            'ie_key': 'Aol',
-        } for m in re.finditer(
-            r"<a\s+href='.*videoid=(?P<id>[0-9]+)'\s+class='video-thumb'>",
-            playlist_html)]
+class AolFeaturesIE(InfoExtractor):
+    IE_NAME = 'features.aol.com'
+    _VALID_URL = r'http://features\.aol\.com/video/(?P<id>[^/?#]+)'
 
-        return {
-            '_type': 'playlist',
-            'id': playlist_id,
-            'display_id': mobj.group('playlist_display_id'),
-            'title': title,
-            'entries': entries,
-        }
+    _TESTS = [{
+        'url': 'http://features.aol.com/video/behind-secret-second-careers-late-night-talk-show-hosts',
+        'md5': '7db483bb0c09c85e241f84a34238cc75',
+        'info_dict': {
+            'id': '519507715',
+            'ext': 'mp4',
+            'title': 'What To Watch - February 17, 2016',
+        },
+        'add_ie': ['FiveMin'],
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        return self.url_result(self._search_regex(
+            r'<script type="text/javascript" src="(https?://[^/]*?5min\.com/Scripts/PlayerSeed\.js[^"]+)"',
+            webpage, '5min embed url'), 'FiveMin')

From 5f1688f27106b6501a2a97d7e6fc1d9f079f1f9f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 4 Mar 2016 23:08:47 +0800
Subject: [PATCH 0890/1105] [kusi] Simplify and improve

---
 youtube_dl/extractor/kusi.py | 72 ++++++++++++++++++++++++------------
 1 file changed, 49 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/kusi.py b/youtube_dl/extractor/kusi.py
index 5e87c5f79..5d458e31e 100644
--- a/youtube_dl/extractor/kusi.py
+++ b/youtube_dl/extractor/kusi.py
@@ -1,11 +1,18 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import random
 import re
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse_unquote_plus
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    float_or_none,
+    timeconvert,
+    update_url_query,
+    xpath_text,
+)
 
 
 class KUSIIE(InfoExtractor):
@@ -17,7 +24,7 @@ class KUSIIE(InfoExtractor):
             'id': '12203019',
             'ext': 'mp4',
             'title': 'Turko Files: Case Closed! & Put On Hold!',
-            'duration': 231000,
+            'duration': 231,
         },
     }, {
         'url': 'http://kusi.com/video?clipId=12203019',
@@ -25,7 +32,10 @@ class KUSIIE(InfoExtractor):
             'id': '12203019',
             'ext': 'mp4',
             'title': 'Turko Files: Case Closed! & Put On Hold!',
-            'duration': 231000,
+            'duration': 231.0,
+            'upload_date': '20160210',
+            'timestamp': 1455087571,
+            'thumbnail': 're:^https?://.*\.jpg$'
         },
         'params': {
             'skip_download': True,  # Same as previous one
@@ -34,33 +44,47 @@ class KUSIIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
+        clip_id = mobj.group('clipId')
+        video_id = clip_id or mobj.group('path')
 
-        if mobj.group('clipId') is not None:
-            video_id = mobj.group('clipId')
-        else:
-            webpage = self._download_webpage(url, mobj.group('path'))
-            video_id = self._html_search_regex(r'"clipId", "(\d+)"', webpage,
-                                               'clipId')
+        webpage = self._download_webpage(url, video_id)
 
-        xml_url = 'http://www.kusi.com/build.asp?buildtype=buildfeaturexml'\
-                  'request&featureType=Clip&featureid={0}&affiliateno=956&'\
-                  'clientgroupid=1&rnd=562461'.format(video_id)
-        doc = self._download_xml(xml_url, video_id,
-                                 note='Downloading video info',
-                                 errnote='Failed to download video info')
+        if clip_id is None:
+            video_id = clip_id = self._html_search_regex(
+                r'"clipId"\s*,\s*"(\d+)"', webpage, 'clip id')
 
-        video_title = doc.find('HEADLINE').text
-        duration = int_or_none(doc.find('DURATION'), get_attr='text')
-        description = doc.find('ABSTRACT')
+        affiliate_id = self._search_regex(
+            r'affiliateId\s*:\s*\'([^\']+)\'', webpage, 'affiliate id')
+
+        # See __Packages/worldnow/model/GalleryModel.as of WNGallery.swf
+        xml_url = update_url_query('http://www.kusi.com/build.asp', {
+            'buildtype': 'buildfeaturexmlrequest',
+            'featureType': 'Clip',
+            'featureid': clip_id,
+            'affiliateno': affiliate_id,
+            'clientgroupid': '1',
+            'rnd': int(round(random.random() * 1000000)),
+        })
+
+        doc = self._download_xml(xml_url, video_id)
+
+        video_title = xpath_text(doc, 'HEADLINE')
+        duration = float_or_none(
+            xpath_text(doc, 'DURATION', fatal=False), scale=1000)
+        description = xpath_text(doc, 'ABSTRACT', fatal=False)
+        thumbnail = xpath_text(doc, './THUMBNAILIMAGE/FILENAME', fatal=False)
+        createtion_time = timeconvert(
+            xpath_text(doc, 'rfc822creationdate', fatal=False))
 
         quality_options = doc.find('{http://search.yahoo.com/mrss/}group').findall('{http://search.yahoo.com/mrss/}content')
         formats = []
         for quality in quality_options:
-            if 'height' in quality.attrib:
-                formats.append({
-                    'url': compat_urllib_parse_unquote_plus(quality.attrib['url']),
-                    'height': quality.attrib['height'],
-                })
+            formats.append({
+                'url': compat_urllib_parse_unquote_plus(quality.attrib['url']),
+                'height': int_or_none(quality.attrib.get('height')),
+                'width': int_or_none(quality.attrib.get('width')),
+                'vbr': float_or_none(quality.attrib.get('bitratebits'), scale=1024),
+            })
         self._sort_formats(formats)
 
         return {
@@ -69,4 +93,6 @@ class KUSIIE(InfoExtractor):
             'description': description,
             'duration': duration,
             'formats': formats,
+            'thumbnail': thumbnail,
+            'timestamp': createtion_time,
         }

From ae7d31af1cd1b9c2f1cae86acb8a4156556dc748 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 4 Mar 2016 21:32:54 +0600
Subject: [PATCH 0891/1105] [yandexmusic] Capture and output API errors

---
 youtube_dl/extractor/yandexmusic.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index d3cc1a29f..e699e663f 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -10,13 +10,27 @@ from ..compat import (
     compat_urllib_parse,
 )
 from ..utils import (
+    ExtractorError,
     int_or_none,
     float_or_none,
     sanitized_Request,
 )
 
 
-class YandexMusicTrackIE(InfoExtractor):
+class YandexMusicBaseIE(InfoExtractor):
+    @staticmethod
+    def _handle_error(response):
+        error = response.get('error')
+        if error:
+            raise ExtractorError(error, expected=True)
+
+    def _download_json(self, *args, **kwargs):
+        response = super(YandexMusicBaseIE, self)._download_json(*args, **kwargs)
+        self._handle_error(response)
+        return response
+
+
+class YandexMusicTrackIE(YandexMusicBaseIE):
     IE_NAME = 'yandexmusic:track'
     IE_DESC = 'Яндекс.Музыка - Трек'
     _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/album/(?P<album_id>\d+)/track/(?P<id>\d+)'
@@ -73,7 +87,7 @@ class YandexMusicTrackIE(InfoExtractor):
         return self._get_track_info(track)
 
 
-class YandexMusicPlaylistBaseIE(InfoExtractor):
+class YandexMusicPlaylistBaseIE(YandexMusicBaseIE):
     def _build_playlist(self, tracks):
         return [
             self.url_result(

From fa880d20ad2a2185d9ea691f8a2fa5cfbfbbb1b2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 4 Mar 2016 23:57:05 +0800
Subject: [PATCH 0892/1105] [kusi] Two fixes

Thanks @dstftw for pointing out those
---
 youtube_dl/extractor/kusi.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/kusi.py b/youtube_dl/extractor/kusi.py
index 5d458e31e..f74dbbf76 100644
--- a/youtube_dl/extractor/kusi.py
+++ b/youtube_dl/extractor/kusi.py
@@ -68,13 +68,11 @@ class KUSIIE(InfoExtractor):
 
         doc = self._download_xml(xml_url, video_id)
 
-        video_title = xpath_text(doc, 'HEADLINE')
-        duration = float_or_none(
-            xpath_text(doc, 'DURATION', fatal=False), scale=1000)
-        description = xpath_text(doc, 'ABSTRACT', fatal=False)
-        thumbnail = xpath_text(doc, './THUMBNAILIMAGE/FILENAME', fatal=False)
-        createtion_time = timeconvert(
-            xpath_text(doc, 'rfc822creationdate', fatal=False))
+        video_title = xpath_text(doc, 'HEADLINE', fatal=True)
+        duration = float_or_none(xpath_text(doc, 'DURATION'), scale=1000)
+        description = xpath_text(doc, 'ABSTRACT')
+        thumbnail = xpath_text(doc, './THUMBNAILIMAGE/FILENAME')
+        createtion_time = timeconvert(xpath_text(doc, 'rfc822creationdate'))
 
         quality_options = doc.find('{http://search.yahoo.com/mrss/}group').findall('{http://search.yahoo.com/mrss/}content')
         formats = []
@@ -83,7 +81,7 @@ class KUSIIE(InfoExtractor):
                 'url': compat_urllib_parse_unquote_plus(quality.attrib['url']),
                 'height': int_or_none(quality.attrib.get('height')),
                 'width': int_or_none(quality.attrib.get('width')),
-                'vbr': float_or_none(quality.attrib.get('bitratebits'), scale=1024),
+                'vbr': float_or_none(quality.attrib.get('bitratebits'), scale=1000),
             })
         self._sort_formats(formats)
 

From 28559564b20813c3e3b7206f493cc789a4f1c03d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Mar 2016 00:04:29 +0800
Subject: [PATCH 0893/1105] [kusi] Correct test_KUSI

---
 youtube_dl/extractor/kusi.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kusi.py b/youtube_dl/extractor/kusi.py
index f74dbbf76..931f34c9b 100644
--- a/youtube_dl/extractor/kusi.py
+++ b/youtube_dl/extractor/kusi.py
@@ -24,7 +24,10 @@ class KUSIIE(InfoExtractor):
             'id': '12203019',
             'ext': 'mp4',
             'title': 'Turko Files: Case Closed! & Put On Hold!',
-            'duration': 231,
+            'duration': 231.0,
+            'upload_date': '20160210',
+            'timestamp': 1455087571,
+            'thumbnail': 're:^https?://.*\.jpg$'
         },
     }, {
         'url': 'http://kusi.com/video?clipId=12203019',

From 7760b9ff4d5e864121811750c0bba78b1d9efdd5 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Mar 2016 17:55:50 +0100
Subject: [PATCH 0894/1105] [audimedia] update _VALID_URL and video_id regex
 and improve http format_id

---
 youtube_dl/extractor/audimedia.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/audimedia.py b/youtube_dl/extractor/audimedia.py
index 3b2effa15..aa6925623 100644
--- a/youtube_dl/extractor/audimedia.py
+++ b/youtube_dl/extractor/audimedia.py
@@ -10,9 +10,9 @@ from ..utils import (
 
 
 class AudiMediaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?audimedia\.tv/(?:en|de)/vid/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?audi-mediacenter\.com/(?:en|de)/audimediatv/(?P<id>[^/?#]+)'
     _TEST = {
-        'url': 'https://audimedia.tv/en/vid/60-seconds-of-audi-sport-104-2015-wec-bahrain-rookie-test',
+        'url': 'https://www.audi-mediacenter.com/en/audimediatv/60-seconds-of-audi-sport-104-2015-wec-bahrain-rookie-test-1467',
         'md5': '79a8b71c46d49042609795ab59779b66',
         'info_dict': {
             'id': '1565',
@@ -32,7 +32,10 @@ class AudiMediaIE(InfoExtractor):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
 
-        raw_payload = self._search_regex(r'<script[^>]+class="amtv-embed"[^>]+id="([^"]+)"', webpage, 'raw payload')
+        raw_payload = self._search_regex([
+            r'class="amtv-embed"[^>]+id="([^"]+)"',
+            r'class=\\"amtv-embed\\"[^>]+id=\\"([^"]+)\\"',
+        ], webpage, 'raw payload')
         _, stage_mode, video_id, lang = raw_payload.split('-')
 
         # TODO: handle s and e stage_mode (live streams and ended live streams)
@@ -59,13 +62,19 @@ class AudiMediaIE(InfoExtractor):
                 video_version_url = video_version.get('download_url') or video_version.get('stream_url')
                 if not video_version_url:
                     continue
-                formats.append({
+                f = {
                     'url': video_version_url,
                     'width': int_or_none(video_version.get('width')),
                     'height': int_or_none(video_version.get('height')),
                     'abr': int_or_none(video_version.get('audio_bitrate')),
                     'vbr': int_or_none(video_version.get('video_bitrate')),
-                })
+                }
+                bitrate = self._search_regex(r'(\d+)k', video_version_url, 'bitrate', default=None)
+                if bitrate:
+                    f.update({
+                        'format_id': 'http-%s' % bitrate,
+                    })
+                formats.append(f)
             self._sort_formats(formats)
 
             return {

From 1315224cbbf2dded21a53e880d58bd32f17ed3f0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Mar 2016 20:14:09 +0100
Subject: [PATCH 0895/1105] [bleacherreport] update tests

---
 youtube_dl/extractor/bleacherreport.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bleacherreport.py b/youtube_dl/extractor/bleacherreport.py
index 38bda3af5..7a8e1f60b 100644
--- a/youtube_dl/extractor/bleacherreport.py
+++ b/youtube_dl/extractor/bleacherreport.py
@@ -28,10 +28,10 @@ class BleacherReportIE(InfoExtractor):
         'add_ie': ['Ooyala'],
     }, {
         'url': 'http://bleacherreport.com/articles/2586817-aussie-golfers-get-fright-of-their-lives-after-being-chased-by-angry-kangaroo',
-        'md5': 'af5f90dc9c7ba1c19d0a3eac806bbf50',
+        'md5': '6a5cd403418c7b01719248ca97fb0692',
         'info_dict': {
             'id': '2586817',
-            'ext': 'mp4',
+            'ext': 'webm',
             'title': 'Aussie Golfers Get Fright of Their Lives After Being Chased by Angry Kangaroo',
             'timestamp': 1446839961,
             'uploader': 'Sean Fay',
@@ -93,10 +93,14 @@ class BleacherReportCMSIE(AMPIE):
         'md5': '8c2c12e3af7805152675446c905d159b',
         'info_dict': {
             'id': '8fd44c2f-3dc5-4821-9118-2c825a98c0e1',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Cena vs. Rollins Would Expose the Heavyweight Division',
             'description': 'md5:984afb4ade2f9c0db35f3267ed88b36e',
         },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):

From d627cec60849d313ad7d39204ef7f59f695b6bca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Mar 2016 01:50:12 +0600
Subject: [PATCH 0896/1105] [youporn] Fix quality extraction (Closes #8758)

---
 youtube_dl/extractor/youporn.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index b29baafc4..567a8c5e6 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -101,8 +101,9 @@ class YouPornIE(InfoExtractor):
             }
             # Video URL's path looks like this:
             #  /201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
+            #  /201012/17/505835/vl_240p_240k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
             # We will benefit from it by extracting some metadata
-            mobj = re.search(r'/(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+/', video_url)
+            mobj = re.search(r'(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+/', video_url)
             if mobj:
                 height = int(mobj.group('height'))
                 bitrate = int(mobj.group('bitrate'))

From 002c75524854033f02d03fa854a45d0fcd3ac74c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Mar 2016 01:51:27 +0600
Subject: [PATCH 0897/1105] [youporn] Fix sources regex

---
 youtube_dl/extractor/youporn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 567a8c5e6..1124fe6c2 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -75,7 +75,7 @@ class YouPornIE(InfoExtractor):
         links = []
 
         sources = self._search_regex(
-            r'sources\s*:\s*({.+?})', webpage, 'sources', default=None)
+            r'(?s)sources\s*:\s*({.+?})', webpage, 'sources', default=None)
         if sources:
             for _, link in re.findall(r'[^:]+\s*:\s*(["\'])(http.+?)\1', sources):
                 links.append(link)

From cf074e5ddd3840071df78ec509bcd74520a8f425 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Mar 2016 21:42:04 +0100
Subject: [PATCH 0898/1105] [foxnews] update test

---
 youtube_dl/extractor/foxnews.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/foxnews.py b/youtube_dl/extractor/foxnews.py
index 318ac013d..1dc50318c 100644
--- a/youtube_dl/extractor/foxnews.py
+++ b/youtube_dl/extractor/foxnews.py
@@ -36,6 +36,10 @@ class FoxNewsIE(AMPIE):
                 # 'upload_date': '20141204',
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
         },
         {
             'url': 'http://video.foxnews.com/v/video-embed.html?video_id=3937480&d=video.foxnews.com',

From 3233a68fbb1cb608534db43f358f7203e59cd2de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 4 Mar 2016 22:18:40 +0100
Subject: [PATCH 0899/1105] [utils] update_url_query: Encode the strings in the
 query dict
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The test case with {'test': '第二行тест'} was failing on python 2 (the non-ascii characters were replaced with '?').
---
 youtube_dl/utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d431aa6b7..22a39a0ab 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1743,6 +1743,7 @@ def update_url_query(url, query):
     parsed_url = compat_urlparse.urlparse(url)
     qs = compat_parse_qs(parsed_url.query)
     qs.update(query)
+    qs = encode_dict(qs)
     return compat_urlparse.urlunparse(parsed_url._replace(
         query=compat_urllib_parse.urlencode(qs, True)))
 

From 845817aadfa5f811afe5834c0b34f50f6c3a2a67 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Mar 2016 18:14:58 +0800
Subject: [PATCH 0900/1105] [twitter] Provide more metadata

---
 youtube_dl/extractor/twitter.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 67762a003..e70b2ab3c 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -102,6 +102,14 @@ class TwitterCardIE(TwitterBaseIE):
             r'data-(?:player-)?config="([^"]+)"', webpage, 'data player config'),
             video_id)
 
+        def _search_dimensions_in_video_url(a_format, video_url):
+            m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
+            if m:
+                a_format.update({
+                    'width': int(m.group('width')),
+                    'height': int(m.group('height')),
+                })
+
         playlist = config.get('playlist')
         if playlist:
             video_url = playlist[0]['source']
@@ -110,12 +118,8 @@ class TwitterCardIE(TwitterBaseIE):
                 'url': video_url,
             }
 
-            m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
-            if m:
-                f.update({
-                    'width': int(m.group('width')),
-                    'height': int(m.group('height')),
-                })
+            _search_dimensions_in_video_url(f, video_url)
+
             formats.append(f)
 
         vmap_url = config.get('vmapUrl') or config.get('vmap_url')
@@ -148,6 +152,8 @@ class TwitterCardIE(TwitterBaseIE):
                     if not a_format['vbr']:
                         del a_format['vbr']
 
+                    _search_dimensions_in_video_url(a_format, media_url)
+
                     formats.append(a_format)
 
             duration = float_or_none(media_info.get('duration', {}).get('nanos'), scale=1e9)

From 19a17d462388b5d10196352e5f9d1491d608783a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Mar 2016 18:18:28 +0800
Subject: [PATCH 0901/1105] [utils] Add codec2ext

---
 youtube_dl/utils.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 22a39a0ab..dc0bf5627 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1893,6 +1893,19 @@ def mimetype2ext(mt):
     }.get(res, res)
 
 
+def codec2ext(codec):
+    codec_type = codec.split('.')[0]
+
+    # Leave the return value None for unknown values as codec_type
+    # is not a good fallback for file extensions
+    return {
+        'avc1': 'mp4',
+        'mp4a': 'm4a',
+        'vorbis': 'webm',
+        'vp9': 'webm',
+    }.get(codec_type)
+
+
 def urlhandle_detect_ext(url_handle):
     try:
         url_handle.headers

From 2def60c5f3426e4f6b7e5f0bc8a92811e5a592b9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Mar 2016 18:18:39 +0800
Subject: [PATCH 0902/1105] [common] Use codec2ext for DASH formats (#8764)

---
 youtube_dl/extractor/common.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 07bd2cbe2..f337caf20 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -29,6 +29,7 @@ from ..utils import (
     age_restricted,
     bug_reports_message,
     clean_html,
+    codec2ext,
     compiled_regex_type,
     determine_ext,
     error_to_compat_str,
@@ -1471,6 +1472,7 @@ class InfoExtractor(object):
                         f = {
                             'format_id': '%s-%s' % (mpd_id, representation_id) if mpd_id else representation_id,
                             'url': base_url,
+                            'ext': codec2ext(representation_attrib.get('codecs')),
                             'width': int_or_none(representation_attrib.get('width')),
                             'height': int_or_none(representation_attrib.get('height')),
                             'tbr': int_or_none(representation_attrib.get('bandwidth'), 1000),

From d1b29d13421fd91093fdf3699aea17d8ba4b636b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Mar 2016 16:43:29 +0600
Subject: [PATCH 0903/1105] [elpais] Add support for alternative layout (Closes
 #8744)

---
 youtube_dl/extractor/elpais.py | 31 +++++++++++++++++++++----------
 1 file changed, 21 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/elpais.py b/youtube_dl/extractor/elpais.py
index 00a69e631..8c725a4e6 100644
--- a/youtube_dl/extractor/elpais.py
+++ b/youtube_dl/extractor/elpais.py
@@ -9,7 +9,7 @@ class ElPaisIE(InfoExtractor):
     _VALID_URL = r'https?://(?:[^.]+\.)?elpais\.com/.*/(?P<id>[^/#?]+)\.html(?:$|[?#])'
     IE_DESC = 'El País'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://blogs.elpais.com/la-voz-de-inaki/2014/02/tiempo-nuevo-recetas-viejas.html',
         'md5': '98406f301f19562170ec071b83433d55',
         'info_dict': {
@@ -19,30 +19,41 @@ class ElPaisIE(InfoExtractor):
             'description': 'De lunes a viernes, a partir de las ocho de la mañana, Iñaki Gabilondo nos cuenta su visión de la actualidad nacional e internacional.',
             'upload_date': '20140206',
         }
-    }
+    }, {
+        'url': 'http://elcomidista.elpais.com/elcomidista/2016/02/24/articulo/1456340311_668921.html#?id_externo_nwl=newsletter_diaria20160303t',
+        'md5': '3bd5b09509f3519d7d9e763179b013de',
+        'info_dict': {
+            'id': '1456340311_668921',
+            'ext': 'mp4',
+            'title': 'Cómo hacer el mejor café con cafetera italiana',
+            'description': 'Que sí, que las cápsulas son cómodas. Pero si le pides algo más a la vida, quizá deberías aprender a usar bien la cafetera italiana. No tienes más que ver este vídeo y seguir sus siete normas básicas.',
+            'upload_date': '20160303',
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         prefix = self._html_search_regex(
-            r'var url_cache = "([^"]+)";', webpage, 'URL prefix')
+            r'var\s+url_cache\s*=\s*"([^"]+)";', webpage, 'URL prefix')
         video_suffix = self._search_regex(
-            r"URLMediaFile = url_cache \+ '([^']+)'", webpage, 'video URL')
+            r"(?:URLMediaFile|urlVideo_\d+)\s*=\s*url_cache\s*\+\s*'([^']+)'", webpage, 'video URL')
         video_url = prefix + video_suffix
         thumbnail_suffix = self._search_regex(
-            r"URLMediaStill = url_cache \+ '([^']+)'", webpage, 'thumbnail URL',
-            fatal=False)
+            r"(?:URLMediaStill|urlFotogramaFijo_\d+)\s*=\s*url_cache\s*\+\s*'([^']+)'",
+            webpage, 'thumbnail URL', fatal=False)
         thumbnail = (
             None if thumbnail_suffix is None
             else prefix + thumbnail_suffix)
         title = self._html_search_regex(
-            '<h2 class="entry-header entry-title.*?>(.*?)</h2>',
+            (r"tituloVideo\s*=\s*'([^']+)'", webpage, 'title',
+             r'<h2 class="entry-header entry-title.*?>(.*?)</h2>'),
             webpage, 'title')
-        date_str = self._search_regex(
+        upload_date = unified_strdate(self._search_regex(
             r'<p class="date-header date-int updated"\s+title="([^"]+)">',
-            webpage, 'upload date', fatal=False)
-        upload_date = (None if date_str is None else unified_strdate(date_str))
+            webpage, 'upload date', default=None) or self._html_search_meta(
+            'datePublished', webpage, 'timestamp'))
 
         return {
             'id': video_id,

From 18b0b23992b5ee4042967b63e366c937484e434e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Mar 2016 12:14:48 +0100
Subject: [PATCH 0904/1105] [revision3] add support pages of type embed

---
 youtube_dl/extractor/revision3.py | 66 +++++++++++++++++++++++--------
 1 file changed, 49 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/revision3.py b/youtube_dl/extractor/revision3.py
index b1b8800b9..fb2196ad6 100644
--- a/youtube_dl/extractor/revision3.py
+++ b/youtube_dl/extractor/revision3.py
@@ -52,6 +52,20 @@ class Revision3IE(InfoExtractor):
             'uploader': 'DNews',
             'uploader_id': 'dnews',
         },
+    }, {
+        'url': 'http://testtube.com/tt-editors-picks/the-israel-palestine-conflict-explained-in-ten-min',
+        'info_dict': {
+            'id': '73573',
+            'ext': 'mp4',
+            'display_id': 'tt-editors-picks/the-israel-palestine-conflict-explained-in-ten-min',
+            'title': 'The Israel-Palestine Conflict Explained in Ten Minutes',
+            'description': 'If you\'d like to learn about the struggle between Israelis and Palestinians, this video is a great place to start',
+            'uploader': 'Editors\' Picks',
+            'uploader_id': 'tt-editors-picks',
+            'timestamp': 1453309200,
+            'upload_date': '20160120',
+        },
+        'add_ie': ['Youtube'],
     }]
     _PAGE_DATA_TEMPLATE = 'http://www.%s/apiProxy/ddn/%s?domain=%s'
     _API_KEY = 'ba9c741bce1b9d8e3defcc22193f3651b8867e62'
@@ -61,9 +75,38 @@ class Revision3IE(InfoExtractor):
         page_info = self._download_json(
             self._PAGE_DATA_TEMPLATE % (domain, display_id, domain), display_id)
 
-        if page_info['data']['type'] == 'episode':
-            episode_data = page_info['data']
-            video_id = compat_str(episode_data['video']['data']['id'])
+        page_data = page_info['data']
+        page_type = page_data['type']
+        if page_type == 'episode' or page_type == 'embed':
+            show_data = page_data['show']['data']
+            video_id = compat_str(page_data['video']['data']['id'])
+
+            preference = qualities(['mini', 'small', 'medium', 'large'])
+            thumbnails = [{
+                'url': image_url,
+                'id': image_id,
+                'preference': preference(image_id)
+            } for image_id, image_url in page_data.get('images', {}).items()]
+
+            info = {
+                'id': video_id,
+                'display_id': display_id,
+                'title': unescapeHTML(page_data['name']),
+                'description': unescapeHTML(page_data.get('summary')),
+                'timestamp': parse_iso8601(page_data.get('publishTime'), ' '),
+                'author': page_data.get('author'),
+                'uploader': show_data.get('name'),
+                'uploader_id': show_data.get('slug'),
+                'thumbnails': thumbnails,
+            }
+
+            if page_type == 'embed':
+                info.update({
+                    '_type': 'url_transparent',
+                    'url': page_data['video']['data']['embed'],
+                })
+                return info
+
             video_data = self._download_json(
                 'http://revision3.com/api/getPlaylist.json?api_key=%s&codecs=h264,vp8,theora&video_id=%s' % (self._API_KEY, video_id),
                 video_id)['items'][0]
@@ -84,26 +127,15 @@ class Revision3IE(InfoExtractor):
                         })
             self._sort_formats(formats)
 
-            preference = qualities(['mini', 'small', 'medium', 'large'])
-            thumbnails = [{
-                'url': image_url,
-                'id': image_id,
-                'preference': preference(image_id)
-            } for image_id, image_url in video_data.get('images', {}).items()]
-
-            return {
-                'id': video_id,
-                'display_id': display_id,
+            info.update({
                 'title': unescapeHTML(video_data['title']),
                 'description': unescapeHTML(video_data.get('summary')),
-                'timestamp': parse_iso8601(episode_data.get('publishTime'), ' '),
-                'author': episode_data.get('author'),
                 'uploader': video_data.get('show', {}).get('name'),
                 'uploader_id': video_data.get('show', {}).get('slug'),
                 'duration': int_or_none(video_data.get('duration')),
-                'thumbnails': thumbnails,
                 'formats': formats,
-            }
+            })
+            return info
         else:
             show_data = page_info['show']['data']
             episodes_data = page_info['episodes']['data']

From 107d0c421a35083fd2412de5dce558fc16f337e7 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Mar 2016 13:43:29 +0100
Subject: [PATCH 0905/1105] [revision3] add support for pages of type tag

---
 youtube_dl/extractor/revision3.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/revision3.py b/youtube_dl/extractor/revision3.py
index fb2196ad6..ffb84628e 100644
--- a/youtube_dl/extractor/revision3.py
+++ b/youtube_dl/extractor/revision3.py
@@ -31,6 +31,7 @@ class Revision3IE(InfoExtractor):
             'uploader_id': 'technobuffalo',
         }
     }, {
+        # Show
         'url': 'http://testtube.com/brainstuff',
         'info_dict': {
             'id': '251',
@@ -66,6 +67,14 @@ class Revision3IE(InfoExtractor):
             'upload_date': '20160120',
         },
         'add_ie': ['Youtube'],
+    }, {
+        # Tag
+        'url': 'http://testtube.com/tech-news',
+        'info_dict': {
+            'id': '21018',
+            'title': 'tech news',
+        },
+        'playlist_mincount': 9,
     }]
     _PAGE_DATA_TEMPLATE = 'http://www.%s/apiProxy/ddn/%s?domain=%s'
     _API_KEY = 'ba9c741bce1b9d8e3defcc22193f3651b8867e62'
@@ -77,7 +86,7 @@ class Revision3IE(InfoExtractor):
 
         page_data = page_info['data']
         page_type = page_data['type']
-        if page_type == 'episode' or page_type == 'embed':
+        if page_type in ('episode', 'embed'):
             show_data = page_data['show']['data']
             video_id = compat_str(page_data['video']['data']['id'])
 
@@ -137,7 +146,7 @@ class Revision3IE(InfoExtractor):
             })
             return info
         else:
-            show_data = page_info['show']['data']
+            list_data = page_info[page_type]['data']
             episodes_data = page_info['episodes']['data']
             num_episodes = page_info['meta']['totalEpisodes']
             processed_episodes = 0
@@ -145,7 +154,7 @@ class Revision3IE(InfoExtractor):
             page_num = 1
             while True:
                 entries.extend([self.url_result(
-                    'http://%s/%s/%s' % (domain, display_id, episode['slug'])) for episode in episodes_data])
+                    'http://%s%s' % (domain, episode['path'])) for episode in episodes_data])
                 processed_episodes += len(episodes_data)
                 if processed_episodes == num_episodes:
                     break
@@ -155,5 +164,5 @@ class Revision3IE(InfoExtractor):
                     display_id)['episodes']['data']
 
             return self.playlist_result(
-                entries, compat_str(show_data['id']),
-                show_data.get('name'), show_data.get('summary'))
+                entries, compat_str(list_data['id']),
+                list_data.get('name'), list_data.get('summary'))

From 51ce91174b9aefa5cb6b98bb0d7b03de09c30e00 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Mar 2016 19:38:58 +0600
Subject: [PATCH 0906/1105] [YoutubeDL] Fix resolution with missing height in
 output template dict

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index dcc867e45..f91851df9 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -567,7 +567,7 @@ class YoutubeDL(object):
                 elif template_dict.get('height'):
                     template_dict['resolution'] = '%sp' % template_dict['height']
                 elif template_dict.get('width'):
-                    template_dict['resolution'] = '?x%d' % template_dict['width']
+                    template_dict['resolution'] = '%dx?' % template_dict['width']
 
             sanitize = lambda k, v: sanitize_filename(
                 compat_str(v),

From 2629a3802c019168340b9345b21b656f48e360fc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Mar 2016 15:42:15 +0100
Subject: [PATCH 0907/1105] [revison3] fix video_id for --download-archive

---
 youtube_dl/extractor/revision3.py | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/revision3.py b/youtube_dl/extractor/revision3.py
index ffb84628e..99979ebe1 100644
--- a/youtube_dl/extractor/revision3.py
+++ b/youtube_dl/extractor/revision3.py
@@ -19,7 +19,7 @@ class Revision3IE(InfoExtractor):
         'url': 'http://www.revision3.com/technobuffalo/5-google-predictions-for-2016',
         'md5': 'd94a72d85d0a829766de4deb8daaf7df',
         'info_dict': {
-            'id': '73034',
+            'id': '71089',
             'display_id': 'technobuffalo/5-google-predictions-for-2016',
             'ext': 'webm',
             'title': '5 Google Predictions for 2016',
@@ -42,7 +42,7 @@ class Revision3IE(InfoExtractor):
     }, {
         'url': 'https://testtube.com/dnews/5-weird-ways-plants-can-eat-animals?utm_source=FB&utm_medium=DNews&utm_campaign=DNewsSocial',
         'info_dict': {
-            'id': '60163',
+            'id': '58227',
             'display_id': 'dnews/5-weird-ways-plants-can-eat-animals',
             'duration': 275,
             'ext': 'webm',
@@ -56,7 +56,7 @@ class Revision3IE(InfoExtractor):
     }, {
         'url': 'http://testtube.com/tt-editors-picks/the-israel-palestine-conflict-explained-in-ten-min',
         'info_dict': {
-            'id': '73573',
+            'id': '71618',
             'ext': 'mp4',
             'display_id': 'tt-editors-picks/the-israel-palestine-conflict-explained-in-ten-min',
             'title': 'The Israel-Palestine Conflict Explained in Ten Minutes',
@@ -81,6 +81,7 @@ class Revision3IE(InfoExtractor):
 
     def _real_extract(self, url):
         domain, display_id = re.match(self._VALID_URL, url).groups()
+        site = domain.split('.')[0]
         page_info = self._download_json(
             self._PAGE_DATA_TEMPLATE % (domain, display_id, domain), display_id)
 
@@ -88,6 +89,7 @@ class Revision3IE(InfoExtractor):
         page_type = page_data['type']
         if page_type in ('episode', 'embed'):
             show_data = page_data['show']['data']
+            page_id = compat_str(page_data['id'])
             video_id = compat_str(page_data['video']['data']['id'])
 
             preference = qualities(['mini', 'small', 'medium', 'large'])
@@ -98,7 +100,7 @@ class Revision3IE(InfoExtractor):
             } for image_id, image_url in page_data.get('images', {}).items()]
 
             info = {
-                'id': video_id,
+                'id': page_id,
                 'display_id': display_id,
                 'title': unescapeHTML(page_data['name']),
                 'description': unescapeHTML(page_data.get('summary')),
@@ -107,6 +109,7 @@ class Revision3IE(InfoExtractor):
                 'uploader': show_data.get('name'),
                 'uploader_id': show_data.get('slug'),
                 'thumbnails': thumbnails,
+                'extractor_key': site,
             }
 
             if page_type == 'embed':
@@ -153,8 +156,13 @@ class Revision3IE(InfoExtractor):
             entries = []
             page_num = 1
             while True:
-                entries.extend([self.url_result(
-                    'http://%s%s' % (domain, episode['path'])) for episode in episodes_data])
+                entries.extend([{
+                    '_type': 'url',
+                    'url': 'http://%s%s' % (domain, episode['path']),
+                    'id': compat_str(episode['id']),
+                    'ie_key': 'Revision3',
+                    'extractor_key': site,
+                } for episode in episodes_data])
                 processed_episodes += len(episodes_data)
                 if processed_episodes == num_episodes:
                     break

From 1e501f6c40cc5e1bd84c0432a323f87f49e18ad0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Mar 2016 21:01:43 +0600
Subject: [PATCH 0908/1105] [jeuxvideo] Fix config URL extraction (Closes
 #8774)

---
 youtube_dl/extractor/jeuxvideo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/jeuxvideo.py b/youtube_dl/extractor/jeuxvideo.py
index eef7daa29..137db873c 100644
--- a/youtube_dl/extractor/jeuxvideo.py
+++ b/youtube_dl/extractor/jeuxvideo.py
@@ -30,7 +30,7 @@ class JeuxVideoIE(InfoExtractor):
         webpage = self._download_webpage(url, title)
         title = self._html_search_meta('name', webpage) or self._og_search_title(webpage)
         config_url = self._html_search_regex(
-            r'data-src="(/contenu/medias/video.php.*?)"',
+            r'data-src(?:set-video)?="(/contenu/medias/video.php.*?)"',
             webpage, 'config URL')
         config_url = 'http://www.jeuxvideo.com' + config_url
 

From 7177fd24f8ed262024501c1020e5791c811fbfe4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Mar 2016 17:51:46 +0100
Subject: [PATCH 0909/1105] [vgtv] support ap.vgtv.no and fix old videos
 extraction(fixes #8719)

---
 youtube_dl/extractor/vgtv.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 14e945d49..e148b1ef5 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -20,6 +20,7 @@ class VGTVIE(XstreamIE):
         'aftenbladet.no/tv': 'satv',
         'fvn.no/fvntv': 'fvntv',
         'aftenposten.no/webtv': 'aptv',
+        'ap.vgtv.no/webtv': 'aptv',
     }
 
     _APP_NAME_TO_VENDOR = {
@@ -35,7 +36,7 @@ class VGTVIE(XstreamIE):
                     (?P<host>
                         %s
                     )
-                    /
+                    /?
                     (?:
                         \#!/(?:video|live)/|
                         embed?.*id=
@@ -107,19 +108,27 @@ class VGTVIE(XstreamIE):
             'md5': 'fd828cd29774a729bf4d4425fe192972',
             'info_dict': {
                 'id': '21039',
-                'ext': 'mov',
+                'ext': 'mp4',
                 'title': 'TRAILER: «SWEATSHOP» - I can´t take any more',
                 'description': 'md5:21891f2b0dd7ec2f78d84a50e54f8238',
                 'duration': 66,
                 'timestamp': 1417002452,
                 'upload_date': '20141126',
                 'view_count': int,
-            }
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
         },
         {
             'url': 'http://www.bt.no/tv/#!/video/100250/norling-dette-er-forskjellen-paa-1-divisjon-og-eliteserien',
             'only_matching': True,
         },
+        {
+            'url': 'http://ap.vgtv.no/webtv#!/video/111084/de-nye-bysyklene-lettere-bedre-gir-stoerre-hjul-og-feste-til-mobil',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):
@@ -144,8 +153,6 @@ class VGTVIE(XstreamIE):
         if len(video_id) == 5:
             if appname == 'bttv':
                 info = self._extract_video_info('btno', video_id)
-            elif appname == 'aptv':
-                info = self._extract_video_info('ap', video_id)
 
         streams = data['streamUrls']
         stream_type = data.get('streamType')

From 75313f2baaec17062d60255976a8afd0fcf995d4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Mar 2016 21:10:00 +0100
Subject: [PATCH 0910/1105] [cnet] fix info extraction

---
 youtube_dl/extractor/cnet.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/cnet.py b/youtube_dl/extractor/cnet.py
index 5c3908f72..3cf0bf95b 100644
--- a/youtube_dl/extractor/cnet.py
+++ b/youtube_dl/extractor/cnet.py
@@ -51,9 +51,7 @@ class CNETIE(ThePlatformIE):
             uploader = None
             uploader_id = None
 
-        mpx_account = data['config']['uvpConfig']['default']['mpx_account']
-
-        metadata = self.get_metadata('%s/%s' % (mpx_account, list(vdata['files'].values())[0]), video_id)
+        metadata = self.get_metadata('kYEXFC/%s' % list(vdata['files'].values())[0], video_id)
         description = vdata.get('description') or metadata.get('description')
         duration = int_or_none(vdata.get('duration')) or metadata.get('duration')
 
@@ -62,7 +60,7 @@ class CNETIE(ThePlatformIE):
         for (fkey, vid) in vdata['files'].items():
             if fkey == 'hls_phone' and 'hls_tablet' in vdata['files']:
                 continue
-            release_url = 'http://link.theplatform.com/s/%s/%s?format=SMIL&mbr=true' % (mpx_account, vid)
+            release_url = 'http://link.theplatform.com/s/kYEXFC/%s?format=SMIL&mbr=true' % vid
             if fkey == 'hds':
                 release_url += '&manifest=f4m'
             tp_formats, tp_subtitles = self._extract_theplatform_smil(release_url, video_id, 'Downloading %s SMIL data' % fkey)

From 1db772673e6345d830e1db3470ade0d234515509 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Mar 2016 21:34:34 +0100
Subject: [PATCH 0911/1105] [cinemassacre] update tests

---
 youtube_dl/extractor/cinemassacre.py | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
index 6d9cd8abd..042c4f2f1 100644
--- a/youtube_dl/extractor/cinemassacre.py
+++ b/youtube_dl/extractor/cinemassacre.py
@@ -21,6 +21,10 @@ class CinemassacreIE(InfoExtractor):
                 'title': '“Angry Video Game Nerd: The Movie” – Trailer',
                 'description': 'md5:fb87405fcb42a331742a0dce2708560b',
             },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
         },
         {
             'url': 'http://cinemassacre.com/2013/10/02/the-mummys-hand-1940',
@@ -31,14 +35,18 @@ class CinemassacreIE(InfoExtractor):
                 'upload_date': '20131002',
                 'title': 'The Mummy’s Hand (1940)',
             },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
         },
         {
             # Youtube embedded video
             'url': 'http://cinemassacre.com/2006/12/07/chronologically-confused-about-bad-movie-and-video-game-sequel-titles/',
-            'md5': 'df4cf8a1dcedaec79a73d96d83b99023',
+            'md5': 'ec9838a5520ef5409b3e4e42fcb0a3b9',
             'info_dict': {
                 'id': 'OEVzPCY2T-g',
-                'ext': 'mp4',
+                'ext': 'webm',
                 'title': 'AVGN: Chronologically Confused about Bad Movie and Video Game Sequel Titles',
                 'upload_date': '20061207',
                 'uploader': 'Cinemassacre',
@@ -49,12 +57,12 @@ class CinemassacreIE(InfoExtractor):
         {
             # Youtube embedded video
             'url': 'http://cinemassacre.com/2006/09/01/mckids/',
-            'md5': '6eb30961fa795fedc750eac4881ad2e1',
+            'md5': '7393c4e0f54602ad110c793eb7a6513a',
             'info_dict': {
                 'id': 'FnxsNhuikpo',
-                'ext': 'mp4',
+                'ext': 'webm',
                 'upload_date': '20060901',
-                'uploader': 'Cinemassacre Extras',
+                'uploader': 'Cinemassacre Extra',
                 'description': 'md5:de9b751efa9e45fbaafd9c8a1123ed53',
                 'uploader_id': 'Cinemassacre',
                 'title': 'AVGN: McKids',
@@ -69,7 +77,11 @@ class CinemassacreIE(InfoExtractor):
                 'description': 'Let’s Play Mario Kart 64 !! Mario Kart 64 is a classic go-kart racing game released for the Nintendo 64 (N64). Today James & Mike do 4 player Battle Mode with Kyle and Bootsy!',
                 'title': 'Mario Kart 64 (Nintendo 64) James & Mike Mondays',
                 'upload_date': '20150525',
-            }
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
         }
     ]
 

From f22ba4bd60e9b4deeb1ca3e1f8162e607dc9648e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Mar 2016 21:52:24 +0100
Subject: [PATCH 0912/1105] update tests related to the change in youtube http
 format sorting

the change was done in 82156fdbf0913c75181484dcc813565713bf78e9
---
 youtube_dl/extractor/freespeech.py  | 2 +-
 youtube_dl/extractor/khanacademy.py | 4 ++--
 youtube_dl/extractor/mit.py         | 2 +-
 youtube_dl/extractor/pyvideo.py     | 6 +++---
 youtube_dl/extractor/ted.py         | 2 +-
 youtube_dl/extractor/viki.py        | 6 +++---
 youtube_dl/extractor/vk.py          | 4 ++--
 youtube_dl/extractor/wimp.py        | 2 +-
 8 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/freespeech.py b/youtube_dl/extractor/freespeech.py
index c210177f7..1477708bb 100644
--- a/youtube_dl/extractor/freespeech.py
+++ b/youtube_dl/extractor/freespeech.py
@@ -14,7 +14,7 @@ class FreespeechIE(InfoExtractor):
         'url': 'https://www.freespeech.org/video/obama-romney-campaign-colorado-ahead-debate-0',
         'info_dict': {
             'id': 'poKsVCZ64uU',
-            'ext': 'mp4',
+            'ext': 'webm',
             'title': 'Obama, Romney Campaign in Colorado Ahead of Debate',
             'description': 'Obama, Romney Campaign in Colorado Ahead of Debate',
             'uploader': 'freespeechtv',
diff --git a/youtube_dl/extractor/khanacademy.py b/youtube_dl/extractor/khanacademy.py
index 08a671fa8..61739efa7 100644
--- a/youtube_dl/extractor/khanacademy.py
+++ b/youtube_dl/extractor/khanacademy.py
@@ -14,10 +14,10 @@ class KhanAcademyIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.khanacademy.org/video/one-time-pad',
-        'md5': '7021db7f2d47d4fff89b13177cb1e8f4',
+        'md5': '7b391cce85e758fb94f763ddc1bbb979',
         'info_dict': {
             'id': 'one-time-pad',
-            'ext': 'mp4',
+            'ext': 'webm',
             'title': 'The one-time pad',
             'description': 'The perfect cipher',
             'duration': 176,
diff --git a/youtube_dl/extractor/mit.py b/youtube_dl/extractor/mit.py
index 29ca45778..819c1b90b 100644
--- a/youtube_dl/extractor/mit.py
+++ b/youtube_dl/extractor/mit.py
@@ -99,7 +99,7 @@ class OCWMITIE(InfoExtractor):
             'url': 'http://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-041-probabilistic-systems-analysis-and-applied-probability-fall-2010/video-lectures/lecture-7-multiple-variables-expectations-independence/',
             'info_dict': {
                 'id': 'EObHWIEKGjA',
-                'ext': 'mp4',
+                'ext': 'webm',
                 'title': 'Lecture 7: Multiple Discrete Random Variables: Expectations, Conditioning, Independence',
                 'description': 'In this lecture, the professor discussed multiple random variables, expectations, and binomial distribution.',
                 'upload_date': '20121109',
diff --git a/youtube_dl/extractor/pyvideo.py b/youtube_dl/extractor/pyvideo.py
index 6d5732d45..30a5f2de4 100644
--- a/youtube_dl/extractor/pyvideo.py
+++ b/youtube_dl/extractor/pyvideo.py
@@ -12,14 +12,14 @@ class PyvideoIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://pyvideo.org/video/1737/become-a-logging-expert-in-30-minutes',
-            'md5': 'de317418c8bc76b1fd8633e4f32acbc6',
+            'md5': '520915673e53a5c5d487c36e0c4d85b5',
             'info_dict': {
                 'id': '24_4WWkSmNo',
-                'ext': 'mp4',
+                'ext': 'webm',
                 'title': 'Become a logging expert in 30 minutes',
                 'description': 'md5:9665350d466c67fb5b1598de379021f7',
                 'upload_date': '20130320',
-                'uploader': 'NextDayVideo',
+                'uploader': 'Next Day Video',
                 'uploader_id': 'NextDayVideo',
             },
             'add_ie': ['Youtube'],
diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index a48d77c30..cf8851438 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -73,7 +73,7 @@ class TEDIE(InfoExtractor):
         'add_ie': ['Youtube'],
         'info_dict': {
             'id': '_ZG8HBuDjgc',
-            'ext': 'mp4',
+            'ext': 'webm',
             'title': 'Douglas Adams: Parrots the Universe and Everything',
             'description': 'md5:01ad1e199c49ac640cb1196c0e9016af',
             'uploader': 'University of California Television (UCTV)',
diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 433fc9914..e04b814c8 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -176,13 +176,13 @@ class VikiIE(VikiBaseIE):
     }, {
         # youtube external
         'url': 'http://www.viki.com/videos/50562v-poor-nastya-complete-episode-1',
-        'md5': '216d1afdc0c64d1febc1e9f2bd4b864b',
+        'md5': '63f8600c1da6f01b7640eee7eca4f1da',
         'info_dict': {
             'id': '50562v',
-            'ext': 'mp4',
+            'ext': 'webm',
             'title': 'Poor Nastya [COMPLETE] - Episode 1',
             'description': '',
-            'duration': 607,
+            'duration': 606,
             'timestamp': 1274949505,
             'upload_date': '20101213',
             'uploader': 'ad14065n',
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 670a438af..d560a4b5e 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -142,10 +142,10 @@ class VKIE(InfoExtractor):
             'url': 'https://vk.com/video276849682_170681728',
             'info_dict': {
                 'id': 'V3K4mi0SYkc',
-                'ext': 'mp4',
+                'ext': 'webm',
                 'title': "DSWD Awards 'Children's Joy Foundation, Inc.' Certificate of Registration and License to Operate",
                 'description': 'md5:bf9c26cfa4acdfb146362682edd3827a',
-                'duration': 179,
+                'duration': 178,
                 'upload_date': '20130116',
                 'uploader': "Children's Joy Foundation",
                 'uploader_id': 'thecjf',
diff --git a/youtube_dl/extractor/wimp.py b/youtube_dl/extractor/wimp.py
index 041ff6c55..fb0accac7 100644
--- a/youtube_dl/extractor/wimp.py
+++ b/youtube_dl/extractor/wimp.py
@@ -20,7 +20,7 @@ class WimpIE(InfoExtractor):
         'md5': '4e2986c793694b55b37cf92521d12bb4',
         'info_dict': {
             'id': 'clowncar',
-            'ext': 'mp4',
+            'ext': 'webm',
             'title': 'It\'s like a clown car.',
             'description': 'md5:0e56db1370a6e49c5c1d19124c0d2fb2',
         },

From ff059017c0650e9635bbbc0565a7ec5f27f16615 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Mar 2016 04:30:19 +0600
Subject: [PATCH 0913/1105] [YoutubeDL] Fix typo in m3u8_native fixup

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 94e4ea432..72d7f0379 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1672,7 +1672,7 @@ class YoutubeDL(object):
                     else:
                         assert fixup_policy in ('ignore', 'never')
 
-                if info_dict.get('protocol') == 'm3u8_native' or info_dict.get('protocol') == 'm3u8' and self._downloader.params.get('hls_prefer_native', False):
+                if info_dict.get('protocol') == 'm3u8_native' or info_dict.get('protocol') == 'm3u8' and self.params.get('hls_prefer_native', False):
                     if fixup_policy == 'warn':
                         self.report_warning('%s: malformated aac bitstream.' % (
                             info_dict['id']))

From d1e4a464cd9f0efd6a9020c8577eb319847c0a68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Mar 2016 04:32:18 +0600
Subject: [PATCH 0914/1105] [YoutubeDL] Carry long lines and improve
 readability

---
 youtube_dl/YoutubeDL.py | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 72d7f0379..2dfdea032 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1639,6 +1639,8 @@ class YoutubeDL(object):
                 if fixup_policy is None:
                     fixup_policy = 'detect_or_warn'
 
+                INSTALL_FFMPEG_MESSAGE = 'Install ffmpeg or avconv to fix this automatically.'
+
                 stretched_ratio = info_dict.get('stretched_ratio')
                 if stretched_ratio is not None and stretched_ratio != 1:
                     if fixup_policy == 'warn':
@@ -1651,15 +1653,18 @@ class YoutubeDL(object):
                             info_dict['__postprocessors'].append(stretched_pp)
                         else:
                             self.report_warning(
-                                '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
-                                    info_dict['id'], stretched_ratio))
+                                '%s: Non-uniform pixel ratio (%s). %s'
+                                % (info_dict['id'], stretched_ratio, INSTALL_FFMPEG_MESSAGE))
                     else:
                         assert fixup_policy in ('ignore', 'never')
 
-                if info_dict.get('requested_formats') is None and info_dict.get('container') == 'm4a_dash':
+                if (info_dict.get('requested_formats') is None and
+                        info_dict.get('container') == 'm4a_dash'):
                     if fixup_policy == 'warn':
-                        self.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
-                            info_dict['id']))
+                        self.report_warning(
+                            '%s: writing DASH m4a. '
+                            'Only some players support this container.'
+                            % info_dict['id'])
                     elif fixup_policy == 'detect_or_warn':
                         fixup_pp = FFmpegFixupM4aPP(self)
                         if fixup_pp.available:
@@ -1667,12 +1672,15 @@ class YoutubeDL(object):
                             info_dict['__postprocessors'].append(fixup_pp)
                         else:
                             self.report_warning(
-                                '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
-                                    info_dict['id']))
+                                '%s: writing DASH m4a. '
+                                'Only some players support this container. %s'
+                                % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
                     else:
                         assert fixup_policy in ('ignore', 'never')
 
-                if info_dict.get('protocol') == 'm3u8_native' or info_dict.get('protocol') == 'm3u8' and self.params.get('hls_prefer_native', False):
+                if (info_dict.get('protocol') == 'm3u8_native' or
+                        info_dict.get('protocol') == 'm3u8' and
+                        self.params.get('hls_prefer_native')):
                     if fixup_policy == 'warn':
                         self.report_warning('%s: malformated aac bitstream.' % (
                             info_dict['id']))
@@ -1683,8 +1691,8 @@ class YoutubeDL(object):
                             info_dict['__postprocessors'].append(fixup_pp)
                         else:
                             self.report_warning(
-                                '%s: malformated aac bitstream. Install ffmpeg or avconv to fix this automatically.' % (
-                                    info_dict['id']))
+                                '%s: malformated aac bitstream. %s'
+                                % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
                     else:
                         assert fixup_policy in ('ignore', 'never')
 

From 1b5284b13f1b579b2cbac5ce6ab9faa5b95800fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Mar 2016 05:36:52 +0600
Subject: [PATCH 0915/1105] [downloader/fragment] Make speed more smooth

At the beginning of every segment there was a drop to Unknown speed due to timeslice being too small to calculate speed.
Now last speed from the previous fragment is used.
---
 youtube_dl/downloader/fragment.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 5bc99492b..a5bae9669 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -99,7 +99,8 @@ class FragmentFD(FileDownloader):
                     state['eta'] = self.calc_eta(
                         start, time_now, estimated_size,
                         state['downloaded_bytes'])
-                state['speed'] = s.get('speed')
+                state['speed'] = s.get('speed') or ctx.get('speed')
+                ctx['speed'] = state['speed']
                 ctx['prev_frag_downloaded_bytes'] = frag_downloaded_bytes
             self._hook_progress(state)
 

From 0f56a4b44366234dafbd4b56559c610d6a0d5b4c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 6 Mar 2016 17:01:05 +0800
Subject: [PATCH 0916/1105] [vimeo] Don't pollute std_headers

Fixes #8778
---
 youtube_dl/extractor/vimeo.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 9f282a1da..560a80efd 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -277,9 +277,8 @@ class VimeoIE(VimeoBaseInfoExtractor):
 
     def _real_extract(self, url):
         url, data = unsmuggle_url(url, {})
-        headers = std_headers
+        headers = std_headers.copy()
         if 'http_headers' in data:
-            headers = headers.copy()
             headers.update(data['http_headers'])
         if 'Referer' not in headers:
             headers['Referer'] = url
@@ -294,7 +293,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             url = 'https://vimeo.com/' + video_id
 
         # Retrieve video webpage to extract further information
-        request = sanitized_Request(url, None, headers)
+        request = sanitized_Request(url, headers=headers)
         try:
             webpage = self._download_webpage(request, video_id)
         except ExtractorError as ee:

From bdf7f13954f50b55cc38fa5f877ae261affbdc32 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 6 Mar 2016 10:08:02 +0100
Subject: [PATCH 0917/1105] release 2016.03.06

---
 README.md              | 2 ++
 docs/supportedsites.md | 8 +++++---
 youtube_dl/version.py  | 2 +-
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index d66804a7a..a774e73e7 100644
--- a/README.md
+++ b/README.md
@@ -80,6 +80,8 @@ which means you can modify it, redistribute it or use it however you like.
                                      on Windows)
     --flat-playlist                  Do not extract the videos of a playlist,
                                      only list them.
+    --mark-watched                   Mark videos watched (YouTube only)
+    --no-mark-watched                Do not mark videos watched (YouTube only)
     --no-color                       Do not emit color codes in output
 
 ## Network Options:
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 43403233d..0682e4af0 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -194,6 +194,7 @@
  - **faz.net**
  - **fc2**
  - **Fczenit**
+ - **features.aol.com**
  - **fernsehkritik.tv**
  - **Firstpost**
  - **FiveTV**
@@ -293,6 +294,7 @@
  - **kontrtube**: KontrTube.ru - Труба зовёт
  - **KrasView**: Красвью
  - **Ku6**
+ - **KUSI**
  - **kuwo:album**: 酷我音乐 - 专辑
  - **kuwo:category**: 酷我音乐 - 分类
  - **kuwo:chart**: 酷我音乐 - 排行榜
@@ -301,12 +303,11 @@
  - **kuwo:song**: 酷我音乐
  - **la7.tv**
  - **Laola1Tv**
+ - **Le**: 乐视网
  - **Lecture2Go**
  - **Lemonde**
- - **Letv**: 乐视网
+ - **LePlaylist**
  - **LetvCloud**: 乐视云
- - **LetvPlaylist**
- - **LetvTv**
  - **Libsyn**
  - **life:embed**
  - **lifenews**: LIFE | NEWS
@@ -421,6 +422,7 @@
  - **Npr**
  - **NRK**
  - **NRKPlaylist**
+ - **NRKSkole**: NRK Skole
  - **NRKTV**: NRK TV and NRK Radio
  - **ntv.ru**
  - **Nuvid**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index adafd601b..246f5740d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.03.01'
+__version__ = '2016.03.06'

From e41acb63647e4aad52dff25b505bdc0bb2a6b9e6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 6 Mar 2016 17:38:39 +0800
Subject: [PATCH 0918/1105] [safari] Don't pollute std_headers (#8778)

---
 youtube_dl/extractor/safari.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 7de7b7273..a65fc8ed7 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -36,12 +36,13 @@ class SafariBaseIE(InfoExtractor):
         if username is None:
             self.raise_login_required('safaribooksonline.com account is required')
 
-        headers = std_headers
+        headers = std_headers.copy()
         if 'Referer' not in headers:
             headers['Referer'] = self._LOGIN_URL
+        login_page_request = sanitized_Request(self._LOGIN_URL, headers=headers)
 
         login_page = self._download_webpage(
-            self._LOGIN_URL, None,
+            login_page_request, None,
             'Downloading login form')
 
         csrf = self._html_search_regex(

From af7d5a63b2f127dda0923a68dfe05e8572a733c9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 6 Mar 2016 17:47:07 +0800
Subject: [PATCH 0919/1105] [common] Document protocol http_dash_segments

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 07bd2cbe2..bfa9c82f6 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -105,7 +105,7 @@ class InfoExtractor(object):
                     * protocol   The protocol that will be used for the actual
                                  download, lower-case.
                                  "http", "https", "rtsp", "rtmp", "rtmpe",
-                                 "m3u8", or "m3u8_native".
+                                 "m3u8", "m3u8_native" or "http_dash_segments".
                     * preference Order number of this format. If this field is
                                  present and not None, the formats get sorted
                                  by this field, regardless of all other values.

From 6d210f20905a5c31f72eff16f9f4dbbed7f52c70 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 6 Mar 2016 13:32:03 +0800
Subject: [PATCH 0920/1105] [utils] Add more codecs to codec2ext

BBC uses avc3. Here's an example (thanks to @remitamine for this example)

http://rdmedia.bbc.co.uk/dash/ondemand/bbb/2/client_manifest-common_init.mpd

See also https://trac.ffmpeg.org/ticket/5217
---
 youtube_dl/utils.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index dc0bf5627..0e04e91a4 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1900,6 +1900,9 @@ def codec2ext(codec):
     # is not a good fallback for file extensions
     return {
         'avc1': 'mp4',
+        'avc2': 'mp4',
+        'avc3': 'mp4',
+        'avc4': 'mp4',
         'mp4a': 'm4a',
         'vorbis': 'webm',
         'vp9': 'webm',

From c58e74062f58219a3843b795648eaa2fc160b9b4 Mon Sep 17 00:00:00 2001
From: "aystroganov@gmail.com" <aystroganov@gmail.com>
Date: Sun, 6 Mar 2016 21:57:41 +1100
Subject: [PATCH 0921/1105] [Sexu] fix extractor

---
 youtube_dl/extractor/sexu.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/sexu.py b/youtube_dl/extractor/sexu.py
index 6365a8779..474e9fdfa 100644
--- a/youtube_dl/extractor/sexu.py
+++ b/youtube_dl/extractor/sexu.py
@@ -14,7 +14,7 @@ class SexuIE(InfoExtractor):
             'id': '961791',
             'ext': 'mp4',
             'title': 'md5:4d05a19a5fc049a63dbbaf05fb71d91b',
-            'description': 'md5:c5ed8625eb386855d5a7967bd7b77a54',
+            'description': 'md5:2b75327061310a3afb3fbd7d09e2e403',
             'categories': list,  # NSFW
             'thumbnail': 're:https?://.*\.jpg$',
             'age_limit': 18,
@@ -26,7 +26,7 @@ class SexuIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         quality_arr = self._search_regex(
-            r'sources:\s*\[([^\]]+)\]', webpage, 'forrmat string')
+            r'"sources":\s*\[([^\]]+)\]', webpage, 'format string')
         formats = [{
             'url': fmt[0].replace('\\', ''),
             'format_id': fmt[1],
@@ -41,7 +41,7 @@ class SexuIE(InfoExtractor):
             'description', webpage, 'description')
 
         thumbnail = self._html_search_regex(
-            r'image:\s*"([^"]+)"',
+            r'"image":\s*"([^"]+)"',
             webpage, 'thumbnail', fatal=False)
 
         categories_str = self._html_search_meta(

From e7bd17373d90073792e6ca0dd7c5a650f14bf2e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Mar 2016 18:08:53 +0600
Subject: [PATCH 0922/1105] [sexu] Improve extraction (Closes #8782)

---
 youtube_dl/extractor/sexu.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/sexu.py b/youtube_dl/extractor/sexu.py
index 474e9fdfa..a99b2a8e7 100644
--- a/youtube_dl/extractor/sexu.py
+++ b/youtube_dl/extractor/sexu.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -25,13 +23,18 @@ class SexuIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        quality_arr = self._search_regex(
-            r'"sources":\s*\[([^\]]+)\]', webpage, 'format string')
+        jwvideo = self._parse_json(
+            self._search_regex(r'\.setup\(\s*({.+?})\s*\);', webpage, 'jwvideo'),
+            video_id)
+
+        sources = jwvideo['sources']
+
         formats = [{
-            'url': fmt[0].replace('\\', ''),
-            'format_id': fmt[1],
-            'height': int(fmt[1][:3]),
-        } for fmt in re.findall(r'"file":"([^"]+)","label":"([^"]+)"', quality_arr)]
+            'url': source['file'].replace('\\', ''),
+            'format_id': source.get('label'),
+            'height': self._search_regex(
+                r'^(\d+)[pP]', source.get('label', ''), 'height', default=None),
+        } for source in sources if source.get('file')]
         self._sort_formats(formats)
 
         title = self._html_search_regex(
@@ -40,9 +43,7 @@ class SexuIE(InfoExtractor):
         description = self._html_search_meta(
             'description', webpage, 'description')
 
-        thumbnail = self._html_search_regex(
-            r'"image":\s*"([^"]+)"',
-            webpage, 'thumbnail', fatal=False)
+        thumbnail = jwvideo.get('image')
 
         categories_str = self._html_search_meta(
             'keywords', webpage, 'categories')

From 1818e4c2b4e0f55169d65131f257bb87d1ce52d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Mar 2016 02:10:16 +0600
Subject: [PATCH 0923/1105] [arte] Fix typo

---
 youtube_dl/extractor/arte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index efde7e207..7f99effa9 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -122,7 +122,7 @@ class ArteTVPlus7IE(InfoExtractor):
                     compat_urllib_parse_urlparse(iframe_url).query)['json_url'][0]
         if json_url:
             return self._extract_from_json_url(json_url, video_id, lang)
-        # Differend kind of embed URL (e.g.
+        # Different kind of embed URL (e.g.
         # http://www.arte.tv/magazine/trepalium/fr/episode-0406-replay-trepalium)
         embed_url = self._search_regex(
             r'<iframe[^>]+src=(["\'])(?P<url>.+?)\1',

From 46c329d6f66368a7129e136ffe3fdf40dbb67b91 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Mar 2016 02:19:54 +0600
Subject: [PATCH 0924/1105] [arte] Improve extraction (Closes #8768)

---
 youtube_dl/extractor/arte.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 7f99effa9..3e119e21b 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -121,7 +121,10 @@ class ArteTVPlus7IE(InfoExtractor):
                 json_url = compat_parse_qs(
                     compat_urllib_parse_urlparse(iframe_url).query)['json_url'][0]
         if json_url:
-            return self._extract_from_json_url(json_url, video_id, lang)
+            title = self._search_regex(
+                r'<h3[^>]+title=(["\'])(?P<title>.+?)\1',
+                webpage, 'title', default=None, group='title')
+            return self._extract_from_json_url(json_url, video_id, lang, title=title)
         # Different kind of embed URL (e.g.
         # http://www.arte.tv/magazine/trepalium/fr/episode-0406-replay-trepalium)
         embed_url = self._search_regex(
@@ -129,7 +132,7 @@ class ArteTVPlus7IE(InfoExtractor):
             webpage, 'embed url', group='url')
         return self.url_result(embed_url)
 
-    def _extract_from_json_url(self, json_url, video_id, lang):
+    def _extract_from_json_url(self, json_url, video_id, lang, title=None):
         info = self._download_json(json_url, video_id)
         player_info = info['videoJsonPlayer']
 
@@ -137,7 +140,7 @@ class ArteTVPlus7IE(InfoExtractor):
         if not upload_date_str:
             upload_date_str = (player_info.get('VRA') or player_info.get('VDA') or '').split(' ')[0]
 
-        title = player_info['VTI'].strip()
+        title = (player_info.get('VTI') or title or player_info['VID']).strip()
         subtitle = player_info.get('VSU', '').strip()
         if subtitle:
             title += ' - %s' % subtitle

From 7404061141147d0346d6c27e7a76ff7285658219 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Mar 2016 02:30:47 +0600
Subject: [PATCH 0925/1105] Credit @mutantmonkey for ustudio (#8574) and kusi
 (#8575)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index b51e23f2d..b6b47ac57 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -161,3 +161,4 @@ Jens Wille
 Robin Houtevelts
 Patrick Griffis
 Aidan Rowe
+mutantmonkey

From ac45505528a71e729a47153ce60053f861e644a9 Mon Sep 17 00:00:00 2001
From: Benjamin Congdon <bencongdon96@gmail.com>
Date: Mon, 7 Mar 2016 21:03:24 -0600
Subject: [PATCH 0926/1105] Added flag for 'allow_audio_only' format in Twitch
 queries

---
 youtube_dl/extractor/twitch.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 8639293e3..958bf8fff 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -252,6 +252,7 @@ class TwitchVodIE(TwitchItemBaseIE):
                 self._USHER_BASE, item_id,
                 compat_urllib_parse.urlencode({
                     'allow_source': 'true',
+                    'allow_audio_only': 'true',
                     'allow_spectre': 'true',
                     'player': 'twitchweb',
                     'nauth': access_token['token'],
@@ -431,6 +432,7 @@ class TwitchStreamIE(TwitchBaseIE):
 
         query = {
             'allow_source': 'true',
+            'allow_audio_only': 'true',
             'p': random.randint(1000000, 10000000),
             'player': 'twitchweb',
             'segment_preference': '4',

From e375a149e16139d0bd2d677d90d4ffe62c6116df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Mar 2016 23:11:09 +0600
Subject: [PATCH 0927/1105] [livestream] Properly build smil URLs (#8794)

---
 youtube_dl/extractor/livestream.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 988436226..eada7c299 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -14,6 +14,7 @@ from ..utils import (
     xpath_with_ns,
     xpath_text,
     orderedSet,
+    update_url_query,
     int_or_none,
     float_or_none,
     parse_iso8601,
@@ -72,7 +73,10 @@ class LivestreamIE(InfoExtractor):
         for vn in video_nodes:
             tbr = int_or_none(vn.attrib.get('system-bitrate'), 1000)
             furl = (
-                '%s%s?v=3.0.3&fp=WIN%%2014,0,0,145' % (base, vn.attrib['src']))
+                update_url_query(compat_urlparse.urljoin(base, vn.attrib['src']), {
+                    'v': '3.0.3',
+                    'fp': 'WIN% 14,0,0,145',
+                }))
             if 'clipBegin' in vn.attrib:
                 furl += '&ssek=' + vn.attrib['clipBegin']
             formats.append({

From 74278def2e69aacd022e00e2aea5f42ffcd9633a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Mar 2016 00:51:07 +0600
Subject: [PATCH 0928/1105] [vimeo:ondemand] Separate ondemand extractor
 (Closes #8330, closes #8801)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/vimeo.py    | 60 +++++++++++++++++++++++++++-----
 2 files changed, 52 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 899bf8114..e07f724bf 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -859,6 +859,7 @@ from .vimeo import (
     VimeoChannelIE,
     VimeoGroupsIE,
     VimeoLikesIE,
+    VimeoOndemandIE,
     VimeoReviewIE,
     VimeoUserIE,
     VimeoWatchLaterIE,
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 560a80efd..4bbb20c65 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -73,15 +73,26 @@ class VimeoIE(VimeoBaseInfoExtractor):
 
     # _VALID_URL matches Vimeo URLs
     _VALID_URL = r'''(?x)
-        https?://
-        (?:(?:www|(?P<player>player))\.)?
-        vimeo(?P<pro>pro)?\.com/
-        (?!channels/[^/?#]+/?(?:$|[?#])|album/)
-        (?:.*?/)?
-        (?:(?:play_redirect_hls|moogaloop\.swf)\?clip_id=)?
-        (?:videos?/)?
-        (?P<id>[0-9]+)
-        /?(?:[?&].*)?(?:[#].*)?$'''
+                    https?://
+                        (?:
+                            (?:
+                                www|
+                                (?P<player>player)
+                            )
+                            \.
+                        )?
+                        vimeo(?P<pro>pro)?\.com/
+                        (?!channels/[^/?#]+/?(?:$|[?#])|(?:album|ondemand)/)
+                        (?:.*?/)?
+                        (?:
+                            (?:
+                                play_redirect_hls|
+                                moogaloop\.swf)\?clip_id=
+                            )?
+                        (?:videos?/)?
+                        (?P<id>[0-9]+)
+                        /?(?:[?&].*)?(?:[#].*)?$
+                    '''
     IE_NAME = 'vimeo'
     _TESTS = [
         {
@@ -497,6 +508,37 @@ class VimeoIE(VimeoBaseInfoExtractor):
         }
 
 
+class VimeoOndemandIE(VimeoBaseInfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?vimeo\.com/ondemand/(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        # ondemand video not available via https://vimeo.com/id
+        'url': 'https://vimeo.com/ondemand/20704',
+        'md5': 'c424deda8c7f73c1dfb3edd7630e2f35',
+        'info_dict': {
+            'id': '105442900',
+            'ext': 'mp4',
+            'title': 'המעבדה - במאי יותם פלדמן',
+            'uploader': 'גם סרטים',
+            'uploader_url': 're:https?://(?:www\.)?vimeo\.com/gumfilms',
+            'uploader_id': 'gumfilms',
+        },
+    }, {
+        'url': 'https://vimeo.com/ondemand/nazmaalik',
+        'only_matching': True,
+    }, {
+        'url': 'https://vimeo.com/ondemand/141692381',
+        'only_matching': True,
+    }, {
+        'url': 'https://vimeo.com/ondemand/thelastcolony/150274832',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        return self.url_result(self._og_search_video_url(webpage), VimeoIE.ie_key())
+
+
 class VimeoChannelIE(VimeoBaseInfoExtractor):
     IE_NAME = 'vimeo:channel'
     _VALID_URL = r'https://vimeo\.com/channels/(?P<id>[^/?#]+)/?(?:$|[?#])'

From 1e501364d5a7f5c99037e6791616ee48989bbaea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Mar 2016 00:52:52 +0600
Subject: [PATCH 0929/1105] [vimeo:ondemand] Clarify IE_NAME

---
 youtube_dl/extractor/vimeo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 4bbb20c65..71c30d2cd 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -509,6 +509,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
 
 
 class VimeoOndemandIE(VimeoBaseInfoExtractor):
+    IE_NAME = 'vimeo:ondemand'
     _VALID_URL = r'https?://(?:www\.)?vimeo\.com/ondemand/(?P<id>[^/?#&]+)'
     _TESTS = [{
         # ondemand video not available via https://vimeo.com/id

From 5d583bdf6cff8161e2df87ba1a0c97a943596eab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Mar 2016 01:03:18 +0600
Subject: [PATCH 0930/1105] [YoutubeDL] Improve _format_note

---
 test/test_YoutubeDL.py  | 4 ++++
 youtube_dl/YoutubeDL.py | 4 +++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 59f7ab49d..bf78b005f 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -502,6 +502,10 @@ class TestYoutubeDL(unittest.TestCase):
         assertRegexpMatches(self, ydl._format_note({
             'vbr': 10,
         }), '^\s*10k$')
+        assertRegexpMatches(self, ydl._format_note({
+            'fps': 30,
+        }), '^30fps$')
+
 
     def test_postprocessors(self):
         filename = 'post-processor-testfile.mp4'
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 2dfdea032..b3391088f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1856,7 +1856,9 @@ class YoutubeDL(object):
         if fdict.get('vbr') is not None:
             res += '%4dk' % fdict['vbr']
         if fdict.get('fps') is not None:
-            res += ', %sfps' % fdict['fps']
+            if res:
+                res += ', '
+            res += '%sfps' % fdict['fps']
         if fdict.get('acodec') is not None:
             if res:
                 res += ', '

From e0dccdd39886c8cda1dddde0d5f5f5a2dcd33746 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Mar 2016 09:04:48 +0600
Subject: [PATCH 0931/1105] [test_YoutubeDL] PEP 8

---
 test/test_YoutubeDL.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index bf78b005f..efbee3b71 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -506,7 +506,6 @@ class TestYoutubeDL(unittest.TestCase):
             'fps': 30,
         }), '^30fps$')
 
-
     def test_postprocessors(self):
         filename = 'post-processor-testfile.mp4'
         audiofile = filename + '.mp3'

From 9c7b509b2a3da1e23e81a2b9039185e09180328a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 10 Mar 2016 14:33:48 +0800
Subject: [PATCH 0932/1105] [facebook] Merge FacebookPostIE into FacebookIE

Fixes #8713
---
 youtube_dl/extractor/__init__.py |  5 +--
 youtube_dl/extractor/facebook.py | 68 ++++++++++++++++++--------------
 2 files changed, 40 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e07f724bf..98de5ddff 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -212,10 +212,7 @@ from .everyonesmixtape import EveryonesMixtapeIE
 from .exfm import ExfmIE
 from .expotv import ExpoTVIE
 from .extremetube import ExtremeTubeIE
-from .facebook import (
-    FacebookIE,
-    FacebookPostIE,
-)
+from .facebook import FacebookIE
 from .faz import FazIE
 from .fc2 import FC2IE
 from .fczenit import FczenitIE
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 6c6c3b1bd..23a6fcd88 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -37,7 +37,8 @@ class FacebookIE(InfoExtractor):
                                 video/embed|
                                 story\.php
                             )\?(?:.*?)(?:v|video_id|story_fbid)=|
-                            [^/]+/videos/(?:[^/]+/)?
+                            [^/]+/videos/(?:[^/]+/)?|
+                            [^/]+/posts/
                         )|
                     facebook:
                 )
@@ -50,6 +51,8 @@ class FacebookIE(InfoExtractor):
 
     _CHROME_USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.97 Safari/537.36'
 
+    _VIDEO_PAGE_TEMPLATE = 'https://www.facebook.com/video/video.php?v=%s'
+
     _TESTS = [{
         'url': 'https://www.facebook.com/video.php?v=637842556329505&fref=nf',
         'md5': '6a40d33c0eccbb1af76cf0485a052659',
@@ -81,6 +84,15 @@ class FacebookIE(InfoExtractor):
             'title': 'When you post epic content on instagram.com/433 8 million followers, this is ...',
             'uploader': 'Demy de Zeeuw',
         },
+    }, {
+        'url': 'https://www.facebook.com/maxlayn/posts/10153807558977570',
+        'md5': '037b1fa7f3c2d02b7a0d7bc16031ecc6',
+        'info_dict': {
+            'id': '544765982287235',
+            'ext': 'mp4',
+            'title': '"What are you doing running in the snow?"',
+            'uploader': 'FailArmy',
+        }
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',
         'only_matching': True,
@@ -164,9 +176,8 @@ class FacebookIE(InfoExtractor):
     def _real_initialize(self):
         self._login()
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        req = sanitized_Request('https://www.facebook.com/video/video.php?v=%s' % video_id)
+    def _extract_from_url(self, url, video_id, fatal_if_no_video=True):
+        req = sanitized_Request(url)
         req.add_header('User-Agent', self._CHROME_USER_AGENT)
         webpage = self._download_webpage(req, video_id)
 
@@ -196,6 +207,8 @@ class FacebookIE(InfoExtractor):
                     break
 
         if not video_data:
+            if not fatal_if_no_video:
+                return webpage, False
             m_msg = re.search(r'class="[^"]*uiInterstitialContent[^"]*"><div>(.*?)</div>', webpage)
             if m_msg is not None:
                 raise ExtractorError(
@@ -241,39 +254,36 @@ class FacebookIE(InfoExtractor):
             video_title = 'Facebook video #%s' % video_id
         uploader = clean_html(get_element_by_id('fbPhotoPageAuthorName', webpage))
 
-        return {
+        info_dict = {
             'id': video_id,
             'title': video_title,
             'formats': formats,
             'uploader': uploader,
         }
 
-
-class FacebookPostIE(InfoExtractor):
-    IE_NAME = 'facebook:post'
-    _VALID_URL = r'https?://(?:\w+\.)?facebook\.com/[^/]+/posts/(?P<id>\d+)'
-    _TEST = {
-        'url': 'https://www.facebook.com/maxlayn/posts/10153807558977570',
-        'md5': '037b1fa7f3c2d02b7a0d7bc16031ecc6',
-        'info_dict': {
-            'id': '544765982287235',
-            'ext': 'mp4',
-            'title': '"What are you doing running in the snow?"',
-            'uploader': 'FailArmy',
-        }
-    }
+        return webpage, info_dict
 
     def _real_extract(self, url):
-        post_id = self._match_id(url)
+        video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, post_id)
+        real_url = self._VIDEO_PAGE_TEMPLATE % video_id if url.startswith('facebook:') else url
+        webpage, info_dict = self._extract_from_url(real_url, video_id, fatal_if_no_video=False)
 
-        entries = [
-            self.url_result('facebook:%s' % video_id, FacebookIE.ie_key())
-            for video_id in self._parse_json(
-                self._search_regex(
-                    r'(["\'])video_ids\1\s*:\s*(?P<ids>\[.+?\])',
-                    webpage, 'video ids', group='ids'),
-                post_id)]
+        if info_dict:
+            return info_dict
 
-        return self.playlist_result(entries, post_id)
+        if '/posts/' in url:
+            entries = [
+                self.url_result('facebook:%s' % video_id, FacebookIE.ie_key())
+                for video_id in self._parse_json(
+                    self._search_regex(
+                        r'(["\'])video_ids\1\s*:\s*(?P<ids>\[.+?\])',
+                        webpage, 'video ids', group='ids'),
+                    video_id)]
+
+            return self.playlist_result(entries, video_id)
+        else:
+            _, info_dict = self._extract_from_url(
+                self._VIDEO_PAGE_TEMPLATE % video_id,
+                video_id, fatal_if_no_video=True)
+            return info_dict

From 98801241965ff81afd68392df52e38a54d49ded2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 10 Mar 2016 14:58:05 +0800
Subject: [PATCH 0933/1105] [facebook] Fix for m.facebook.com URLs

---
 youtube_dl/extractor/facebook.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 23a6fcd88..b354c5521 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -93,6 +93,15 @@ class FacebookIE(InfoExtractor):
             'title': '"What are you doing running in the snow?"',
             'uploader': 'FailArmy',
         }
+    }, {
+        'url': 'https://m.facebook.com/story.php?story_fbid=1035862816472149&id=116132035111903',
+        'md5': '1deb90b6ac27f7efcf6d747c8a27f5e3',
+        'info_dict': {
+            'id': '1035862816472149',
+            'ext': 'mp4',
+            'title': 'What the Flock Is Going On In New Zealand  Credit: ViralHog',
+            'uploader': 'S. Saint',
+        },
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',
         'only_matching': True,
@@ -105,9 +114,6 @@ class FacebookIE(InfoExtractor):
     }, {
         'url': 'facebook:544765982287235',
         'only_matching': True,
-    }, {
-        'url': 'https://m.facebook.com/story.php?story_fbid=1035862816472149&id=116132035111903',
-        'only_matching': True,
     }]
 
     def _login(self):
@@ -200,7 +206,7 @@ class FacebookIE(InfoExtractor):
 
         if not video_data:
             server_js_data = self._parse_json(self._search_regex(
-                r'handleServerJS\(({.+})\);', webpage, 'server js data'), video_id)
+                r'handleServerJS\(({.+})\);', webpage, 'server js data', default='{}'), video_id)
             for item in server_js_data.get('instances', []):
                 if item[1][0] == 'VideoConfig':
                     video_data = video_data_list2dict(item[2][0]['videoData'])

From 5080cbf9fd26441256390f169d24ab3259c740f7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 10 Mar 2016 15:26:32 +0800
Subject: [PATCH 0934/1105] [facebook] Handle escaped swf params

Fixes #8713
---
 youtube_dl/extractor/facebook.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index b354c5521..49231d3e9 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -102,6 +102,15 @@ class FacebookIE(InfoExtractor):
             'title': 'What the Flock Is Going On In New Zealand  Credit: ViralHog',
             'uploader': 'S. Saint',
         },
+    }, {
+        'note': 'swf params escaped',
+        'url': 'https://www.facebook.com/barackobama/posts/10153664894881749',
+        'md5': '97ba073838964d12c70566e0085c2b91',
+        'info_dict': {
+            'id': '10153664894881749',
+            'ext': 'mp4',
+            'title': 'Facebook video #10153664894881749',
+        },
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',
         'only_matching': True,
@@ -189,11 +198,12 @@ class FacebookIE(InfoExtractor):
 
         video_data = None
 
-        BEFORE = '{swf.addParam(param[0], param[1]);});\n'
+        BEFORE = '{swf.addParam(param[0], param[1]);});'
         AFTER = '.forEach(function(variable) {swf.addVariable(variable[0], variable[1]);});'
-        m = re.search(re.escape(BEFORE) + '(.*?)' + re.escape(AFTER), webpage)
+        m = re.search(re.escape(BEFORE) + '(?:\n|\\\\n)(.*?)' + re.escape(AFTER), webpage)
         if m:
-            data = dict(json.loads(m.group(1)))
+            swf_params = m.group(1).replace('\\\\', '\\').replace('\\"', '"')
+            data = dict(json.loads(swf_params))
             params_raw = compat_urllib_parse_unquote(data['params'])
             video_data = json.loads(params_raw)['video_data']
 

From 31db8709bff7209d215c00eb93bdb2cf178d7f58 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 10 Mar 2016 21:37:26 +0800
Subject: [PATCH 0935/1105] [iqiyi] Update enc_key

---
 youtube_dl/extractor/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index d3bee3a19..e7c0cb3f6 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -501,7 +501,7 @@ class IqiyiIE(InfoExtractor):
     def get_enc_key(self, video_id):
         # TODO: automatic key extraction
         # last update at 2016-01-22 for Zombie::bite
-        enc_key = '6ab6d0280511493ba85594779759d4ed'
+        enc_key = '8ed797d224d043e7ac23d95b70227d32'
         return enc_key
 
     def _extract_playlist(self, webpage):

From f6f6217a98adc07c41f7a45a9068aeaf706ad862 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 10 Mar 2016 15:17:04 +0100
Subject: [PATCH 0936/1105] [facebook] Don't override variable in list
 comprehension

---
 youtube_dl/extractor/facebook.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 49231d3e9..5e8589479 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -290,8 +290,8 @@ class FacebookIE(InfoExtractor):
 
         if '/posts/' in url:
             entries = [
-                self.url_result('facebook:%s' % video_id, FacebookIE.ie_key())
-                for video_id in self._parse_json(
+                self.url_result('facebook:%s' % vid, FacebookIE.ie_key())
+                for vid in self._parse_json(
                     self._search_regex(
                         r'(["\'])video_ids\1\s*:\s*(?P<ids>\[.+?\])',
                         webpage, 'video ids', group='ids'),

From e738e43358a7f34dbbcb73b45b228d7b9b3360a7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 11 Mar 2016 16:20:27 +0800
Subject: [PATCH 0937/1105] [facebook] Support videos in groups

Viewing/Downloading videos in groups requires logging in, even for
those in public groups.

Fixes #6951.
---
 youtube_dl/extractor/facebook.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 5e8589479..f5bbd39d2 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -38,7 +38,8 @@ class FacebookIE(InfoExtractor):
                                 story\.php
                             )\?(?:.*?)(?:v|video_id|story_fbid)=|
                             [^/]+/videos/(?:[^/]+/)?|
-                            [^/]+/posts/
+                            [^/]+/posts/|
+                            groups/[^/]+/permalink/
                         )|
                     facebook:
                 )
@@ -123,6 +124,9 @@ class FacebookIE(InfoExtractor):
     }, {
         'url': 'facebook:544765982287235',
         'only_matching': True,
+    }, {
+        'url': 'https://www.facebook.com/groups/164828000315060/permalink/764967300301124/',
+        'only_matching': True,
     }]
 
     def _login(self):

From 61f317c24c040d051ec6652cb0d4c650c1fc0361 Mon Sep 17 00:00:00 2001
From: Benjamin Congdon <bencongdon96@gmail.com>
Date: Wed, 9 Mar 2016 21:16:21 -0600
Subject: [PATCH 0938/1105] Added extractor for AudioBoom.com

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/audioboom.py | 39 +++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100644 youtube_dl/extractor/audioboom.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 98de5ddff..1a7d689df 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -54,6 +54,7 @@ from .arte import (
 from .atresplayer import AtresPlayerIE
 from .atttechchannel import ATTTechChannelIE
 from .audimedia import AudiMediaIE
+from .audioboom import AudioBoomIE
 from .audiomack import AudiomackIE, AudiomackAlbumIE
 from .azubu import AzubuIE, AzubuLiveIE
 from .baidu import BaiduVideoIE
diff --git a/youtube_dl/extractor/audioboom.py b/youtube_dl/extractor/audioboom.py
new file mode 100644
index 000000000..84d4755f5
--- /dev/null
+++ b/youtube_dl/extractor/audioboom.py
@@ -0,0 +1,39 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import float_or_none
+
+
+class AudioBoomIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?audioboom\.com/boos/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://audioboom.com/boos/4279833-3-09-2016-czaban-hour-3?t=0',
+        'md5': '63a8d73a055c6ed0f1e51921a10a5a76',
+        'info_dict': {
+            'id': '4279833',
+            'ext': 'mp3',
+            'title': '3/09/2016 Czaban Hour 3',
+            'description': 'Guest:   Nate Davis - NFL free agency,   Guest:   Stan Gans',
+            'duration': 2245.72
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._og_search_title(webpage)
+
+        download_url = self._og_search_property('audio', webpage, 'url')
+
+        duration = float_or_none(self._html_search_meta(
+            'weibo:audio:duration', webpage, fatal=False))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': download_url,
+            'description': self._og_search_description(webpage),
+            'duration': duration,
+        }

From 883c05237869ceea3eabdf1e4ca53f4b45d9a04b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 11 Mar 2016 19:44:17 +0600
Subject: [PATCH 0939/1105] [audioboom] Improve robustness and extract uploader
 (Closes #8812)

---
 youtube_dl/extractor/audioboom.py | 41 +++++++++++++++++++++++++------
 1 file changed, 34 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/audioboom.py b/youtube_dl/extractor/audioboom.py
index 84d4755f5..2ec2d7092 100644
--- a/youtube_dl/extractor/audioboom.py
+++ b/youtube_dl/extractor/audioboom.py
@@ -15,25 +15,52 @@ class AudioBoomIE(InfoExtractor):
             'ext': 'mp3',
             'title': '3/09/2016 Czaban Hour 3',
             'description': 'Guest:   Nate Davis - NFL free agency,   Guest:   Stan Gans',
-            'duration': 2245.72
+            'duration': 2245.72,
+            'uploader': 'Steve Czaban',
+            'uploader_url': 're:https?://(?:www\.)?audioboom\.com/channel/steveczabanyahoosportsradio',
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
-        title = self._og_search_title(webpage)
+        clip = None
 
-        download_url = self._og_search_property('audio', webpage, 'url')
+        clip_store = self._parse_json(
+            self._search_regex(
+                r'data-new-clip-store=(["\'])(?P<json>{.*?"clipId"\s*:\s*%s.*?})\1' % video_id,
+                webpage, 'clip store', default='{}', group='json'),
+            video_id, fatal=False)
+        if clip_store:
+            clips = clip_store.get('clips')
+            if clips and isinstance(clips, list) and isinstance(clips[0], dict):
+                clip = clips[0]
 
-        duration = float_or_none(self._html_search_meta(
-            'weibo:audio:duration', webpage, fatal=False))
+        def from_clip(field):
+            if clip:
+                clip.get(field)
+
+        audio_url = from_clip('clipURLPriorToLoading') or self._og_search_property(
+            'audio', webpage, 'audio url')
+        title = from_clip('title') or self._og_search_title(webpage)
+        description = from_clip('description') or self._og_search_description(webpage)
+
+        duration = float_or_none(from_clip('duration') or self._html_search_meta(
+            'weibo:audio:duration', webpage))
+
+        uploader = from_clip('author') or self._og_search_property(
+            'audio:artist', webpage, 'uploader', fatal=False)
+        uploader_url = from_clip('author_url') or self._html_search_meta(
+            'audioboo:channel', webpage, 'uploader url')
 
         return {
             'id': video_id,
+            'url': audio_url,
             'title': title,
-            'url': download_url,
-            'description': self._og_search_description(webpage),
+            'description': description,
             'duration': duration,
+            'uploader': uploader,
+            'uploader_url': uploader_url,
         }

From 6fa6d385490e913af48538a493384e7288d30c8f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 11 Mar 2016 19:46:06 +0600
Subject: [PATCH 0940/1105] Credit @benjamincongdon for audioboom (#8812)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index b6b47ac57..aa48cd5a6 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -162,3 +162,4 @@ Robin Houtevelts
 Patrick Griffis
 Aidan Rowe
 mutantmonkey
+Ben Congdon

From 25cb05bda9c8609c5e2f329f199735837b4d8c60 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 11 Mar 2016 23:48:05 +0800
Subject: [PATCH 0941/1105] [utils] Remove codec2ext

This function is orignally used for determining file extensions of DASH
formats. Now in DASH, ext is determined by mime_type. See #8766 for more
information.
---
 youtube_dl/utils.py | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 0e04e91a4..22a39a0ab 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1893,22 +1893,6 @@ def mimetype2ext(mt):
     }.get(res, res)
 
 
-def codec2ext(codec):
-    codec_type = codec.split('.')[0]
-
-    # Leave the return value None for unknown values as codec_type
-    # is not a good fallback for file extensions
-    return {
-        'avc1': 'mp4',
-        'avc2': 'mp4',
-        'avc3': 'mp4',
-        'avc4': 'mp4',
-        'mp4a': 'm4a',
-        'vorbis': 'webm',
-        'vp9': 'webm',
-    }.get(codec_type)
-
-
 def urlhandle_detect_ext(url_handle):
     try:
         url_handle.headers

From a6c8b75904f8c67c08708efe3a8d3dce94af9ee2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 11 Mar 2016 23:49:55 +0800
Subject: [PATCH 0942/1105] [common] Use mimeType to determine file extensions
 (#8766)

---
 youtube_dl/extractor/common.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 393677251..52b4c125a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -29,7 +29,6 @@ from ..utils import (
     age_restricted,
     bug_reports_message,
     clean_html,
-    codec2ext,
     compiled_regex_type,
     determine_ext,
     error_to_compat_str,
@@ -1448,8 +1447,9 @@ class InfoExtractor(object):
                         continue
                     representation_attrib = adaptation_set.attrib.copy()
                     representation_attrib.update(representation.attrib)
-                    mime_type = representation_attrib.get('mimeType')
-                    content_type = mime_type.split('/')[0] if mime_type else representation_attrib.get('contentType')
+                    # According to page 41 of ISO/IEC 29001-1:2014, @mimeType is mandatory
+                    mime_type = representation_attrib['mimeType']
+                    content_type = mime_type.split('/')[0]
                     if content_type == 'text':
                         # TODO implement WebVTT downloading
                         pass
@@ -1472,7 +1472,7 @@ class InfoExtractor(object):
                         f = {
                             'format_id': '%s-%s' % (mpd_id, representation_id) if mpd_id else representation_id,
                             'url': base_url,
-                            'ext': codec2ext(representation_attrib.get('codecs')),
+                            'ext': mimetype2ext(mime_type),
                             'width': int_or_none(representation_attrib.get('width')),
                             'height': int_or_none(representation_attrib.get('height')),
                             'tbr': int_or_none(representation_attrib.get('bandwidth'), 1000),

From fac7e79277942a182ba68d1ba01b8b3f7a9935b3 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 11 Mar 2016 16:52:07 +0100
Subject: [PATCH 0943/1105] [kaltura] add support for videos with reference id

---
 youtube_dl/extractor/kaltura.py | 54 +++++++++++++++++++++++++--------
 1 file changed, 42 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index ccbc39c66..78e29f481 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -8,6 +8,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
     compat_urlparse,
+    compat_parse_qs,
 )
 from ..utils import (
     clean_html,
@@ -20,21 +21,17 @@ from ..utils import (
 class KalturaIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                 (?:
-                    kaltura:(?P<partner_id_s>\d+):(?P<id_s>[0-9a-z_]+)|
+                    kaltura:(?P<partner_id>\d+):(?P<id>[0-9a-z_]+)|
                     https?://
                         (:?(?:www|cdnapi(?:sec)?)\.)?kaltura\.com/
                         (?:
                             (?:
                                 # flash player
-                                index\.php/kwidget/
-                                (?:[^/]+/)*?wid/_(?P<partner_id>\d+)/
-                                (?:[^/]+/)*?entry_id/(?P<id>[0-9a-z_]+)|
+                                index\.php/kwidget|
                                 # html5 player
-                                html5/html5lib/
-                                (?:[^/]+/)*?entry_id/(?P<id_html5>[0-9a-z_]+)
-                                .*\?.*\bwid=_(?P<partner_id_html5>\d+)
+                                html5/html5lib/[^/]+/mwEmbedFrame\.php
                             )
-                        )
+                        )(?:/(?P<path>[^?]+))?(?:\?(?P<query>.*))?
                 )
                 '''
     _API_BASE = 'http://cdnapi.kaltura.com/api_v3/index.php?'
@@ -127,10 +124,43 @@ class KalturaIE(InfoExtractor):
         url, smuggled_data = unsmuggle_url(url, {})
 
         mobj = re.match(self._VALID_URL, url)
-        partner_id = mobj.group('partner_id_s') or mobj.group('partner_id') or mobj.group('partner_id_html5')
-        entry_id = mobj.group('id_s') or mobj.group('id') or mobj.group('id_html5')
-
-        info, flavor_assets = self._get_video_info(entry_id, partner_id)
+        partner_id, entry_id = mobj.group('partner_id', 'id')
+        info, flavor_assets = None, None
+        if partner_id and entry_id:
+            info, flavor_assets = self._get_video_info(entry_id, partner_id)
+        else:
+            path, query = mobj.group('path', 'query')
+            if not path and not query:
+                raise ExtractorError('Invalid URL', expected=True)
+            params = {}
+            if query:
+                params = compat_parse_qs(query)
+            if path:
+                splitted_path = path.split('/')
+                if not splitted_path[-1]:
+                    splitted_path = splitted_path[:-1]
+                for i in range(0, len(splitted_path), 2):
+                    params[splitted_path[i]] = [splitted_path[i + 1]]
+            if 'wid' in params:
+                partner_id = params['wid'][0][1:]
+            elif 'p' in params:
+                partner_id = params['p'][0]
+            else:
+                raise ExtractorError('Invalid URL', expected=True)
+            if 'entry_id' in params:
+                entry_id = params['entry_id'][0]
+                info, flavor_assets = self._get_video_info(entry_id, partner_id)
+            elif 'uiconf_id' in params and 'flashvars[referenceId]' in params:
+                reference_id = params['flashvars[referenceId]'][0]
+                webpage = self._download_webpage(url, reference_id)
+                entry_data = self._parse_json(self._search_regex(
+                    r'window\.kalturaIframePackageData\s*=\s*({.*});',
+                    webpage, 'kalturaIframePackageData'),
+                    reference_id)['entryResult']
+                info, flavor_assets = entry_data['meta'], entry_data['contextData']['flavorAssets']
+                entry_id = info['id']
+            else:
+                raise ExtractorError('Invalid URL', expected=True)
 
         source_url = smuggled_data.get('source_url')
         if source_url:

From bcb668de189b92bd1dcad661d50ae981f25deca4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 11 Mar 2016 16:57:06 +0100
Subject: [PATCH 0944/1105] [safari] extract free and preview videos(#7491)

---
 youtube_dl/extractor/safari.py | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index a65fc8ed7..ae619636c 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -12,6 +12,7 @@ from ..utils import (
     smuggle_url,
     std_headers,
     urlencode_postdata,
+    update_url_query,
 )
 
 
@@ -26,6 +27,7 @@ class SafariBaseIE(InfoExtractor):
     LOGGED_IN = False
 
     def _real_initialize(self):
+        return
         # We only need to log in once for courses or individual videos
         if not self.LOGGED_IN:
             self._login()
@@ -86,13 +88,15 @@ class SafariIE(SafariBaseIE):
 
     _TESTS = [{
         'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/part00.html',
-        'md5': '5b0c4cc1b3c1ba15dda7344085aa5592',
+        'md5': 'dcc5a425e79f2564148652616af1f2a3',
         'info_dict': {
-            'id': '2842601850001',
+            'id': '0_qbqx90ic',
             'ext': 'mp4',
-            'title': 'Introduction',
+            'title': 'Introduction to Hadoop Fundamentals LiveLessons',
+            'timestamp': 1437758058,
+            'upload_date': '20150724',
+            'uploader_id': 'stork',
         },
-        'skip': 'Requires safaribooksonline account credentials',
     }, {
         'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
         'only_matching': True,
@@ -107,15 +111,16 @@ class SafariIE(SafariBaseIE):
         course_id = mobj.group('course_id')
         part = mobj.group('part')
 
-        webpage = self._download_webpage(
-            '%s/%s/chapter-content/%s.html' % (self._API_BASE, course_id, part),
-            part)
+        webpage = self._download_webpage(url, '%s/%s' % (course_id, part))
+        reference_id = self._search_regex(r'data-reference-id="([^"]+)"', webpage, 'kaltura reference id')
+        partner_id = self._search_regex(r'data-partner-id="([^"]+)"', webpage, 'kaltura widget id')
+        ui_id = self._search_regex(r'data-ui-id="([^"]+)"', webpage, 'kaltura uiconf id')
 
-        bc_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
-        if not bc_url:
-            raise ExtractorError('Could not extract Brightcove URL from %s' % url, expected=True)
-
-        return self.url_result(smuggle_url(bc_url, {'Referer': url}), 'BrightcoveLegacy')
+        return self.url_result(update_url_query('https://cdnapisec.kaltura.com/html5/html5lib/v2.37.1/mwEmbedFrame.php', {
+            'wid': '_%s' % partner_id,
+            'uiconf_id': ui_id,
+            'flashvars[referenceId]': reference_id,
+        }), 'Kaltura')
 
 
 class SafariCourseIE(SafariBaseIE):

From cdfee16818088d5442e9ea9fd9a2f85273bb59ed Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 10 Mar 2016 19:49:13 +0100
Subject: [PATCH 0945/1105] [extractor/common] add data, headers and query
 params to _request_webpage

---
 youtube_dl/extractor/common.py | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index bfa9c82f6..d2443c93c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -48,6 +48,7 @@ from ..utils import (
     determine_protocol,
     parse_duration,
     mimetype2ext,
+    update_url_query,
 )
 
 
@@ -345,7 +346,7 @@ class InfoExtractor(object):
     def IE_NAME(self):
         return compat_str(type(self).__name__[:-2])
 
-    def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
+    def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, data=None, headers=None, query=None):
         """ Returns the response handle """
         if note is None:
             self.report_download_webpage(video_id)
@@ -354,6 +355,12 @@ class InfoExtractor(object):
                 self.to_screen('%s' % (note,))
             else:
                 self.to_screen('%s: %s' % (video_id, note))
+        # data, headers and query params will be ignored for `Request` objects
+        if isinstance(url_or_request, compat_str):
+            if query:
+                url_or_request = update_url_query(url_or_request, query)
+            if data or headers:
+                url_or_request = sanitized_Request(url_or_request, data, headers or {})
         try:
             return self._downloader.urlopen(url_or_request)
         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
@@ -369,13 +376,13 @@ class InfoExtractor(object):
                 self._downloader.report_warning(errmsg)
                 return False
 
-    def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None):
+    def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None, data=None, headers=None, query=None):
         """ Returns a tuple (page content as string, URL handle) """
         # Strip hashes from the URL (#1038)
         if isinstance(url_or_request, (compat_str, str)):
             url_or_request = url_or_request.partition('#')[0]
 
-        urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal)
+        urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal, data=data, headers=headers, query=query)
         if urlh is False:
             assert not fatal
             return False
@@ -462,13 +469,13 @@ class InfoExtractor(object):
 
         return content
 
-    def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None):
+    def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None, data=None, headers=None, query=None):
         """ Returns the data of the page as a string """
         success = False
         try_count = 0
         while success is False:
             try:
-                res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal, encoding=encoding)
+                res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal, encoding=encoding, data=data, headers=headers, query=query)
                 success = True
             except compat_http_client.IncompleteRead as e:
                 try_count += 1
@@ -483,10 +490,10 @@ class InfoExtractor(object):
 
     def _download_xml(self, url_or_request, video_id,
                       note='Downloading XML', errnote='Unable to download XML',
-                      transform_source=None, fatal=True, encoding=None):
+                      transform_source=None, fatal=True, encoding=None, data=None, headers=None, query=None):
         """Return the xml as an xml.etree.ElementTree.Element"""
         xml_string = self._download_webpage(
-            url_or_request, video_id, note, errnote, fatal=fatal, encoding=encoding)
+            url_or_request, video_id, note, errnote, fatal=fatal, encoding=encoding, data=data, headers=headers, query=query)
         if xml_string is False:
             return xml_string
         if transform_source:
@@ -497,10 +504,10 @@ class InfoExtractor(object):
                        note='Downloading JSON metadata',
                        errnote='Unable to download JSON metadata',
                        transform_source=None,
-                       fatal=True, encoding=None):
+                       fatal=True, encoding=None, data=None, headers=None, query=None):
         json_string = self._download_webpage(
             url_or_request, video_id, note, errnote, fatal=fatal,
-            encoding=encoding)
+            encoding=encoding, data=data, headers=headers, query=query)
         if (not fatal) and json_string is False:
             return None
         return self._parse_json(

From 58e6d097d894416c0d5d1f19a059b3614f823a9d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Mar 2016 00:36:39 +0600
Subject: [PATCH 0946/1105] [googledrive] Relax _VALID_URL (Closes #8829)

---
 youtube_dl/extractor/googledrive.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index 37be34091..766fc26d0 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -10,8 +10,8 @@ from ..utils import (
 
 
 class GoogleDriveIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/)|video\.google\.com/get_player\?.*?docid=)(?P<id>[a-zA-Z0-9_-]{28})'
-    _TEST = {
+    _VALID_URL = r'https?://(?:(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/)|video\.google\.com/get_player\?.*?docid=)(?P<id>[a-zA-Z0-9_-]{28,})'
+    _TESTS = [{
         'url': 'https://drive.google.com/file/d/0ByeS4oOUV-49Zzh4R1J6R09zazQ/edit?pli=1',
         'md5': '881f7700aec4f538571fa1e0eed4a7b6',
         'info_dict': {
@@ -20,7 +20,11 @@ class GoogleDriveIE(InfoExtractor):
             'title': 'Big Buck Bunny.mp4',
             'duration': 46,
         }
-    }
+    }, {
+        # video id is longer than 28 characters
+        'url': 'https://drive.google.com/file/d/1ENcQ_jeCuj7y19s66_Ou9dRP4GKGsodiDQ/edit',
+        'only_matching': True,
+    }]
     _FORMATS_EXT = {
         '5': 'flv',
         '6': 'flv',
@@ -43,7 +47,7 @@ class GoogleDriveIE(InfoExtractor):
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<iframe[^>]+src="https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9_-]{28})',
+            r'<iframe[^>]+src="https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9_-]{28,})',
             webpage)
         if mobj:
             return 'https://drive.google.com/file/d/%s' % mobj.group('id')

From 09f572fbc07e79cc3f62e65149237ee1952aa12b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 11 Mar 2016 22:37:07 +0100
Subject: [PATCH 0947/1105] [extractor/common] add transform_source to
 _download_smil and _extract_smil_formats

---
 youtube_dl/extractor/common.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index aaca25a12..0b8b906ab 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1147,8 +1147,8 @@ class InfoExtractor(object):
                 out.append('{%s}%s' % (namespace, c))
         return '/'.join(out)
 
-    def _extract_smil_formats(self, smil_url, video_id, fatal=True, f4m_params=None):
-        smil = self._download_smil(smil_url, video_id, fatal=fatal)
+    def _extract_smil_formats(self, smil_url, video_id, fatal=True, f4m_params=None, transform_source=None):
+        smil = self._download_smil(smil_url, video_id, fatal=fatal, transform_source=transform_source)
 
         if smil is False:
             assert not fatal
@@ -1165,10 +1165,10 @@ class InfoExtractor(object):
             return {}
         return self._parse_smil(smil, smil_url, video_id, f4m_params=f4m_params)
 
-    def _download_smil(self, smil_url, video_id, fatal=True):
+    def _download_smil(self, smil_url, video_id, fatal=True, transform_source=None):
         return self._download_xml(
             smil_url, video_id, 'Downloading SMIL file',
-            'Unable to download SMIL file', fatal=fatal)
+            'Unable to download SMIL file', fatal=fatal, transform_source=transform_source)
 
     def _parse_smil(self, smil, smil_url, video_id, f4m_params=None):
         namespace = self._parse_smil_namespace(smil)

From c8868a9d83291c4502126ca375300026cc90acf9 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 11 Mar 2016 22:44:18 +0100
Subject: [PATCH 0948/1105] [dw] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/dw.py       | 43 ++++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+)
 create mode 100644 youtube_dl/extractor/dw.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1a7d689df..4bc7fbab0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -189,6 +189,7 @@ from .dumpert import DumpertIE
 from .defense import DefenseGouvFrIE
 from .discovery import DiscoveryIE
 from .dropbox import DropboxIE
+from .dw import DWIE
 from .eagleplatform import EaglePlatformIE
 from .ebaumsworld import EbaumsWorldIE
 from .echomsk import EchoMskIE
diff --git a/youtube_dl/extractor/dw.py b/youtube_dl/extractor/dw.py
new file mode 100644
index 000000000..df901e1b2
--- /dev/null
+++ b/youtube_dl/extractor/dw.py
@@ -0,0 +1,43 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class DWIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?dw\.com/(?:[^/]+/)+av-(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.dw.com/en/intelligent-light/av-19112290',
+        'md5': '7372046e1815c5a534b43f3c3c36e6e9',
+        'info_dict': {
+            'id': '19112290',
+            'ext': 'mp4',
+            'title': 'Intelligent light',
+            'description': 'md5:90e00d5881719f2a6a5827cb74985af1',
+            'upload_date': '20160311',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        formats = self._extract_smil_formats(
+            'http://www.dw.com/smil/v-%s' % video_id, video_id,
+            transform_source=lambda s: s.replace(
+                'rtmp://tv-od.dw.de/flash/',
+                'http://tv-download.dw.de/dwtv_video/flv/'))
+
+        webpage = self._download_webpage(url, video_id)
+        hidden_inputs = self._hidden_inputs(webpage)
+        title = hidden_inputs['media_title']
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': self._og_search_description(webpage),
+            'thumbnail': hidden_inputs.get('preview_image'),
+            'duration': int_or_none(hidden_inputs.get('file_duration')),
+            'upload_date': hidden_inputs.get('display_date'),
+            'formats': formats,
+        }

From 91d6aafb483b15c819c587357f7fd626f7a7d380 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 11 Mar 2016 23:55:26 +0100
Subject: [PATCH 0949/1105] [dw] add support for audio pages

---
 youtube_dl/extractor/dw.py | 32 ++++++++++++++++++++++++--------
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/dw.py b/youtube_dl/extractor/dw.py
index df901e1b2..dde66b41d 100644
--- a/youtube_dl/extractor/dw.py
+++ b/youtube_dl/extractor/dw.py
@@ -7,7 +7,8 @@ from ..utils import int_or_none
 
 class DWIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dw\.com/(?:[^/]+/)+av-(?P<id>\d+)'
-    _TEST = {
+    _TESTS = [{
+        # video
         'url': 'http://www.dw.com/en/intelligent-light/av-19112290',
         'md5': '7372046e1815c5a534b43f3c3c36e6e9',
         'info_dict': {
@@ -17,21 +18,36 @@ class DWIE(InfoExtractor):
             'description': 'md5:90e00d5881719f2a6a5827cb74985af1',
             'upload_date': '20160311',
         }
-    }
+    }, {
+        # audio
+        'url': 'http://www.dw.com/en/worldlink-my-business/av-19111941',
+        'md5': '2814c9a1321c3a51f8a7aeb067a360dd',
+        'info_dict': {
+            'id': '19111941',
+            'ext': 'mp3',
+            'title': 'WorldLink: My business',
+            'description': 'md5:bc9ca6e4e063361e21c920c53af12405',
+            'upload_date': '20160311',
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        formats = self._extract_smil_formats(
-            'http://www.dw.com/smil/v-%s' % video_id, video_id,
-            transform_source=lambda s: s.replace(
-                'rtmp://tv-od.dw.de/flash/',
-                'http://tv-download.dw.de/dwtv_video/flv/'))
-
         webpage = self._download_webpage(url, video_id)
         hidden_inputs = self._hidden_inputs(webpage)
         title = hidden_inputs['media_title']
 
+        formats = []
+        if hidden_inputs.get('player_type') == 'video' and hidden_inputs.get('stream_file') == '1':
+            formats = self._extract_smil_formats(
+                'http://www.dw.com/smil/v-%s' % video_id, video_id,
+                transform_source=lambda s: s.replace(
+                    'rtmp://tv-od.dw.de/flash/',
+                    'http://tv-download.dw.de/dwtv_video/flv/'))
+        else:
+            formats = [{'url': hidden_inputs['file_name']}]
+
         return {
             'id': video_id,
             'title': title,

From 36bb63e0841f1d4fc19e76805739fff60abc283b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 12 Mar 2016 08:33:22 +0100
Subject: [PATCH 0950/1105] [dw] add support for article pages(closes #8790)

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/dw.py       | 36 +++++++++++++++++++++++++++-----
 2 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 4bc7fbab0..c5ca01ee7 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -189,7 +189,10 @@ from .dumpert import DumpertIE
 from .defense import DefenseGouvFrIE
 from .discovery import DiscoveryIE
 from .dropbox import DropboxIE
-from .dw import DWIE
+from .dw import (
+    DWIE,
+    DWArticleIE,
+)
 from .eagleplatform import EaglePlatformIE
 from .ebaumsworld import EbaumsWorldIE
 from .echomsk import EchoMskIE
diff --git a/youtube_dl/extractor/dw.py b/youtube_dl/extractor/dw.py
index dde66b41d..b6c985547 100644
--- a/youtube_dl/extractor/dw.py
+++ b/youtube_dl/extractor/dw.py
@@ -3,9 +3,11 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import int_or_none
+from ..compat import compat_urlparse
 
 
 class DWIE(InfoExtractor):
+    IE_NAME = 'dw'
     _VALID_URL = r'https?://(?:www\.)?dw\.com/(?:[^/]+/)+av-(?P<id>\d+)'
     _TESTS = [{
         # video
@@ -32,16 +34,15 @@ class DWIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
+        media_id = self._match_id(url)
+        webpage = self._download_webpage(url, media_id)
         hidden_inputs = self._hidden_inputs(webpage)
         title = hidden_inputs['media_title']
 
         formats = []
         if hidden_inputs.get('player_type') == 'video' and hidden_inputs.get('stream_file') == '1':
             formats = self._extract_smil_formats(
-                'http://www.dw.com/smil/v-%s' % video_id, video_id,
+                'http://www.dw.com/smil/v-%s' % media_id, media_id,
                 transform_source=lambda s: s.replace(
                     'rtmp://tv-od.dw.de/flash/',
                     'http://tv-download.dw.de/dwtv_video/flv/'))
@@ -49,7 +50,7 @@ class DWIE(InfoExtractor):
             formats = [{'url': hidden_inputs['file_name']}]
 
         return {
-            'id': video_id,
+            'id': media_id,
             'title': title,
             'description': self._og_search_description(webpage),
             'thumbnail': hidden_inputs.get('preview_image'),
@@ -57,3 +58,28 @@ class DWIE(InfoExtractor):
             'upload_date': hidden_inputs.get('display_date'),
             'formats': formats,
         }
+
+
+class DWArticleIE(InfoExtractor):
+    IE_NAME = 'dw:article'
+    _VALID_URL = r'https?://(?:www\.)?dw\.com/(?:[^/]+/)+a-(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.dw.com/en/no-hope-limited-options-for-refugees-in-idomeni/a-19111009',
+        'md5': '8ca657f9d068bbef74d6fc38b97fc869',
+        'info_dict': {
+            'id': '19105868',
+            'ext': 'mp4',
+            'title': 'The harsh life of refugees in Idomeni',
+            'description': 'md5:196015cc7e48ebf474db9399420043c7',
+            'upload_date': '20160310',
+        }
+    }
+
+    def _real_extract(self, url):
+        article_id = self._match_id(url)
+        webpage = self._download_webpage(url, article_id)
+        hidden_inputs = self._hidden_inputs(webpage)
+        media_id = hidden_inputs['media_id']
+        media_path = self._search_regex(r'href="([^"]+av-%s)"\s+class="overlayLink"' % media_id, webpage, 'media url')
+        media_url = compat_urlparse.urljoin(url, media_path)
+        return self.url_result(media_url, 'DW', media_id)

From e5a2e17a9c65dd481cfbda55a778d6536a832a15 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 12 Mar 2016 18:43:45 +0100
Subject: [PATCH 0951/1105] [kaltura] optimize url info extraction

---
 youtube_dl/extractor/kaltura.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 78e29f481..79a72d45f 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -137,10 +137,7 @@ class KalturaIE(InfoExtractor):
                 params = compat_parse_qs(query)
             if path:
                 splitted_path = path.split('/')
-                if not splitted_path[-1]:
-                    splitted_path = splitted_path[:-1]
-                for i in range(0, len(splitted_path), 2):
-                    params[splitted_path[i]] = [splitted_path[i + 1]]
+                params.update(dict((zip(splitted_path[::2], [[v] for v in splitted_path[1::2]]))))
             if 'wid' in params:
                 partner_id = params['wid'][0][1:]
             elif 'p' in params:

From 9dce3c095b377104c0a05f6907d73a5b6fae5245 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 02:01:10 +0600
Subject: [PATCH 0952/1105] [kaltura] Respect kaltura session

---
 youtube_dl/extractor/kaltura.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 79a72d45f..44d7c84a1 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -125,7 +125,7 @@ class KalturaIE(InfoExtractor):
 
         mobj = re.match(self._VALID_URL, url)
         partner_id, entry_id = mobj.group('partner_id', 'id')
-        info, flavor_assets = None, None
+        ks = None
         if partner_id and entry_id:
             info, flavor_assets = self._get_video_info(entry_id, partner_id)
         else:
@@ -158,6 +158,7 @@ class KalturaIE(InfoExtractor):
                 entry_id = info['id']
             else:
                 raise ExtractorError('Invalid URL', expected=True)
+            ks = params.get('flashvars[ks]', [None])[0]
 
         source_url = smuggled_data.get('source_url')
         if source_url:
@@ -167,14 +168,19 @@ class KalturaIE(InfoExtractor):
         else:
             referrer = None
 
+        def sign_url(unsigned_url):
+            if ks:
+                unsigned_url += '/ks/%s' % ks
+            if referrer:
+                unsigned_url += '?referrer=%s' % referrer
+            return unsigned_url
+
         formats = []
         for f in flavor_assets:
             # Continue if asset is not ready
             if f['status'] != 2:
                 continue
-            video_url = '%s/flavorId/%s' % (info['dataUrl'], f['id'])
-            if referrer:
-                video_url += '?referrer=%s' % referrer
+            video_url = sign_url('%s/flavorId/%s' % (info['dataUrl'], f['id']))
             formats.append({
                 'format_id': '%(fileExt)s-%(bitrate)s' % f,
                 'ext': f.get('fileExt'),
@@ -187,9 +193,7 @@ class KalturaIE(InfoExtractor):
                 'width': int_or_none(f.get('width')),
                 'url': video_url,
             })
-        m3u8_url = info['dataUrl'].replace('format/url', 'format/applehttp')
-        if referrer:
-            m3u8_url += '?referrer=%s' % referrer
+        m3u8_url = sign_url(info['dataUrl'].replace('format/url', 'format/applehttp'))
         formats.extend(self._extract_m3u8_formats(
             m3u8_url, entry_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
 

From 73cbd709f9fae8e27bd019bce4f5764db05634a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 02:03:07 +0600
Subject: [PATCH 0953/1105] [safari] Respect kaltura session (Closes #7491)

---
 youtube_dl/extractor/safari.py | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index ae619636c..56a7dfb92 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -21,13 +21,12 @@ class SafariBaseIE(InfoExtractor):
     _SUCCESSFUL_LOGIN_REGEX = r'<a href="/accounts/logout/"[^>]*>Sign Out</a>'
     _NETRC_MACHINE = 'safari'
 
-    _API_BASE = 'https://www.safaribooksonline.com/api/v1/book'
+    _API_BASE = 'https://www.safaribooksonline.com/api/v1'
     _API_FORMAT = 'json'
 
     LOGGED_IN = False
 
     def _real_initialize(self):
-        return
         # We only need to log in once for courses or individual videos
         if not self.LOGGED_IN:
             self._login()
@@ -36,7 +35,7 @@ class SafariBaseIE(InfoExtractor):
     def _login(self):
         (username, password) = self._get_login_info()
         if username is None:
-            self.raise_login_required('safaribooksonline.com account is required')
+            return
 
         headers = std_headers.copy()
         if 'Referer' not in headers:
@@ -116,11 +115,25 @@ class SafariIE(SafariBaseIE):
         partner_id = self._search_regex(r'data-partner-id="([^"]+)"', webpage, 'kaltura widget id')
         ui_id = self._search_regex(r'data-ui-id="([^"]+)"', webpage, 'kaltura uiconf id')
 
-        return self.url_result(update_url_query('https://cdnapisec.kaltura.com/html5/html5lib/v2.37.1/mwEmbedFrame.php', {
+        query = {
             'wid': '_%s' % partner_id,
             'uiconf_id': ui_id,
             'flashvars[referenceId]': reference_id,
-        }), 'Kaltura')
+        }
+
+        if self.LOGGED_IN:
+            kaltura_session = self._download_json(
+                '%s/player/kaltura_session/?reference_id=%s' % (self._API_BASE, reference_id),
+                course_id, 'Downloading kaltura session JSON',
+                'Unable to download kaltura session JSON', fatal=False)
+            if kaltura_session:
+                session = kaltura_session.get('session')
+                if session:
+                    query['flashvars[ks]'] = session
+
+        return self.url_result(update_url_query(
+            'https://cdnapisec.kaltura.com/html5/html5lib/v2.37.1/mwEmbedFrame.php', query),
+            'Kaltura')
 
 
 class SafariCourseIE(SafariBaseIE):
@@ -146,7 +159,7 @@ class SafariCourseIE(SafariBaseIE):
         course_id = self._match_id(url)
 
         course_json = self._download_json(
-            '%s/%s/?override_format=%s' % (self._API_BASE, course_id, self._API_FORMAT),
+            '%s/book/%s/?override_format=%s' % (self._API_BASE, course_id, self._API_FORMAT),
             course_id, 'Downloading course JSON')
 
         if 'chapters' not in course_json:

From e9c8999ede2e07c64f2b592b12158d763dc933c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 02:08:36 +0600
Subject: [PATCH 0954/1105] [safari] Fix authentication

---
 youtube_dl/extractor/safari.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 56a7dfb92..10c7f39b0 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -27,12 +27,13 @@ class SafariBaseIE(InfoExtractor):
     LOGGED_IN = False
 
     def _real_initialize(self):
-        # We only need to log in once for courses or individual videos
-        if not self.LOGGED_IN:
-            self._login()
-            SafariBaseIE.LOGGED_IN = True
+        self._login()
 
     def _login(self):
+        # We only need to log in once for courses or individual videos
+        if self.LOGGED_IN:
+            return
+
         (username, password) = self._get_login_info()
         if username is None:
             return
@@ -68,6 +69,8 @@ class SafariBaseIE(InfoExtractor):
                 'Login failed; make sure your credentials are correct and try again.',
                 expected=True)
 
+        SafariBaseIE.LOGGED_IN = True
+
         self.to_screen('Login successful')
 
 

From 0fdbb3322b263aeb66aa9a774fa59a9bad33321c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 03:16:08 +0600
Subject: [PATCH 0955/1105] [extractor/common] Add _parse_f4m_formats routine

---
 youtube_dl/extractor/common.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 0b8b906ab..ecd7da767 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -973,6 +973,13 @@ class InfoExtractor(object):
         if manifest is False:
             return []
 
+        return self._parse_f4m_formats(
+            manifest, manifest_url, video_id, preference=preference, f4m_id=f4m_id,
+            transform_source=transform_source, fatal=fatal)
+
+    def _parse_f4m_formats(self, manifest, manifest_url, video_id, preference=None, f4m_id=None,
+                           transform_source=lambda s: fix_xml_ampersands(s).strip(),
+                           fatal=True):
         formats = []
         manifest_version = '1.0'
         media_nodes = manifest.findall('{http://ns.adobe.com/f4m/1.0}media')
@@ -998,7 +1005,8 @@ class InfoExtractor(object):
                 # bitrate in f4m downloader
                 if determine_ext(manifest_url) == 'f4m':
                     formats.extend(self._extract_f4m_formats(
-                        manifest_url, video_id, preference, f4m_id, fatal=fatal))
+                        manifest_url, video_id, preference=preference, f4m_id=f4m_id,
+                        transform_source=transform_source, fatal=fatal))
                     continue
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({

From f930e0c76e4c0dea36bf9af3aef77f08888fcdf9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 03:17:25 +0600
Subject: [PATCH 0956/1105] [extractor/generic] Extract f4m formats and
 refactor common info

---
 youtube_dl/extractor/generic.py | 37 +++++++++++++++++++--------------
 1 file changed, 21 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ca745ae41..36d3a7433 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1242,28 +1242,34 @@ class GenericIE(InfoExtractor):
             full_response = self._request_webpage(request, video_id)
             head_response = full_response
 
+        info_dict = {
+            'id': video_id,
+            'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
+        }
+
         # Check for direct link to a video
         content_type = head_response.headers.get('Content-Type', '')
         m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>.+)$', content_type)
         if m:
             upload_date = unified_strdate(
                 head_response.headers.get('Last-Modified'))
-            formats = []
-            if m.group('format_id').endswith('mpegurl'):
+            format_id = m.group('format_id')
+            if format_id.endswith('mpegurl'):
                 formats = self._extract_m3u8_formats(url, video_id, 'mp4')
+            elif format_id == 'f4m':
+                formats = self._extract_f4m_formats(url, video_id)
             else:
                 formats = [{
                     'format_id': m.group('format_id'),
                     'url': url,
                     'vcodec': 'none' if m.group('type') == 'audio' else None
                 }]
-            return {
-                'id': video_id,
-                'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
+            info_dict.update({
                 'direct': True,
                 'formats': formats,
                 'upload_date': upload_date,
-            }
+            })
+            return info_dict
 
         if not self._downloader.params.get('test', False) and not is_intentional:
             force = self._downloader.params.get('force_generic_extractor', False)
@@ -1291,13 +1297,12 @@ class GenericIE(InfoExtractor):
                 'URL could be a direct video link, returning it as such.')
             upload_date = unified_strdate(
                 head_response.headers.get('Last-Modified'))
-            return {
-                'id': video_id,
-                'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
+            info_dict.update({
                 'direct': True,
                 'url': url,
                 'upload_date': upload_date,
-            }
+            })
+            return info_dict
 
         webpage = self._webpage_read_content(
             full_response, url, video_id, prefix=first_bytes)
@@ -1314,12 +1319,12 @@ class GenericIE(InfoExtractor):
             elif doc.tag == '{http://xspf.org/ns/0/}playlist':
                 return self.playlist_result(self._parse_xspf(doc, video_id), video_id)
             elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag):
-                return {
-                    'id': video_id,
-                    'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
-                    'formats': self._parse_mpd_formats(
-                        doc, video_id, mpd_base_url=url.rpartition('/')[0]),
-                }
+                info_dict['formats'] = self._parse_mpd_formats(
+                    doc, video_id, mpd_base_url=url.rpartition('/')[0])
+                return info_dict
+            elif re.match(r'^{http://ns\.adobe\.com/f4m/[12]\.0}manifest$', doc.tag):
+                info_dict['formats'] = self._parse_f4m_formats(doc, url, video_id)
+                return info_dict
         except compat_xml_parse_error:
             pass
 

From 3f2f4a94aa63e156e8ad19ea493f23e2afec6b62 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 03:38:20 +0600
Subject: [PATCH 0957/1105] [extractor/generic] Extract f4m formats from final
 URLs

---
 youtube_dl/extractor/generic.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 36d3a7433..8121f04a5 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1990,6 +1990,8 @@ class GenericIE(InfoExtractor):
                 entry_info_dict['formats'] = self._extract_m3u8_formats(video_url, video_id, ext='mp4')
             elif ext == 'mpd':
                 entry_info_dict['formats'] = self._extract_mpd_formats(video_url, video_id)
+            elif ext == 'f4m':
+                entry_info_dict['formats'] = self._extract_f4m_formats(video_url, video_id)
             else:
                 entry_info_dict['url'] = video_url
 

From 46a4ea827633c6590727f4688e8f4fbb2a545922 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 03:48:38 +0600
Subject: [PATCH 0958/1105] [safari] Remove unused imports

---
 youtube_dl/extractor/safari.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 10c7f39b0..256396bb8 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -4,12 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from .brightcove import BrightcoveLegacyIE
 
 from ..utils import (
     ExtractorError,
     sanitized_Request,
-    smuggle_url,
     std_headers,
     urlencode_postdata,
     update_url_query,

From b970dfddaf6a086b62733c21550447ef4933364f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 12 Mar 2016 22:13:25 +0100
Subject: [PATCH 0959/1105] [minoto] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/minoto.py   | 56 ++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100644 youtube_dl/extractor/minoto.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c5ca01ee7..ba9c8b7a5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -398,6 +398,7 @@ from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
 from .minhateca import MinhatecaIE
 from .ministrygrid import MinistryGridIE
+from .minoto import MinotoIE
 from .miomio import MioMioIE
 from .mit import TechTVMITIE, MITIE, OCWMITIE
 from .mitele import MiTeleIE
diff --git a/youtube_dl/extractor/minoto.py b/youtube_dl/extractor/minoto.py
new file mode 100644
index 000000000..959a10589
--- /dev/null
+++ b/youtube_dl/extractor/minoto.py
@@ -0,0 +1,56 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class MinotoIE(InfoExtractor):
+    _VALID_URL = r'(?:minoto:|https?://(?:play|iframe|embed)\.minoto-video\.com/(?P<player_id>[0-9]+)/)(?P<id>[a-zA-Z0-9]+)'
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        player_id = mobj.group('player_id') or '1'
+        video_id = mobj.group('id')
+        video_data = self._download_json('http://play.minoto-video.com/%s/%s.js' % (player_id, video_id), video_id)
+        video_metadata = video_data['video-metadata']
+        formats = []
+        for fmt in video_data['video-files']:
+            fmt_url = fmt.get('url')
+            if not fmt_url:
+                continue
+            container = fmt.get('container')
+            if container == 'hls':
+                formats.extend(fmt_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+            else:
+                fmt_profile = fmt.get('profile') or {}
+                f = {
+                    'format_id': fmt_profile.get('name-short'),
+                    'format_note': fmt_profile.get('name'),
+                    'url': fmt_url,
+                    'container': container,
+                    'tbr': int_or_none(fmt.get('bitrate')),
+                    'filesize': int_or_none(fmt.get('filesize')),
+                    'width': int_or_none(fmt.get('width')),
+                    'height': int_or_none(fmt.get('height')),
+                }
+                codecs = fmt.get('codecs')
+                if codecs:
+                    codecs = codecs.split(',')
+                    if len(codecs) == 2:
+                        f.update({
+                            'vcodec': codecs[0],
+                            'acodec': codecs[1],
+                        })
+                formats.append(f)
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': video_metadata['title'],
+            'description': video_metadata.get('description'),
+            'thumbnail': video_metadata.get('video-poster', {}).get('url'),
+            'formats': formats,
+        }

From a42dfa629eccf4fc3c2a8d74b8d753a607c0fb25 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 12 Mar 2016 22:51:36 +0100
Subject: [PATCH 0960/1105] [makerschannel] Add new extractor(closes #8839)

---
 youtube_dl/extractor/__init__.py      |  1 +
 youtube_dl/extractor/makerschannel.py | 40 +++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)
 create mode 100644 youtube_dl/extractor/makerschannel.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ba9c8b7a5..dc5ecea30 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -389,6 +389,7 @@ from .lynda import (
 from .m6 import M6IE
 from .macgamestore import MacGameStoreIE
 from .mailru import MailRuIE
+from .makerschannel import MakersChannelIE
 from .makertv import MakerTVIE
 from .malemotion import MalemotionIE
 from .matchtv import MatchTVIE
diff --git a/youtube_dl/extractor/makerschannel.py b/youtube_dl/extractor/makerschannel.py
new file mode 100644
index 000000000..f5d00e61d
--- /dev/null
+++ b/youtube_dl/extractor/makerschannel.py
@@ -0,0 +1,40 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class MakersChannelIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?makerschannel\.com/.*(?P<id_type>video|production)_id=(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://makerschannel.com/en/zoomin/community-highlights?video_id=849',
+        'md5': '624a512c6969236b5967bf9286345ad1',
+        'info_dict': {
+            'id': '849',
+            'ext': 'mp4',
+            'title': 'Landing a bus on a plane is an epic win',
+            'uploader': 'ZoomIn',
+            'description': 'md5:cd9cca2ea7b69b78be81d07020c97139',
+        }
+    }
+
+    def _real_extract(self, url):
+        id_type, url_id = re.match(self._VALID_URL, url).groups()
+        webpage = self._download_webpage(url, url_id)
+        video_data = self._html_search_regex(r'<div([^>]+data-%s-id="%s"[^>]+)>' % (id_type, url_id), webpage, 'video data')
+
+        def extract_data_val(attr, fatal=False):
+            return self._html_search_regex(r'data-%s\s*=\s*"([^"]+)"' % attr, video_data, attr, fatal=fatal)
+        minoto_id = self._search_regex(r'/id/([a-zA-Z0-9]+)', extract_data_val('video-src', True), 'minoto id')
+
+        return {
+            '_type': 'url_transparent',
+            'url': 'minoto:%s' % minoto_id,
+            'id': extract_data_val('video-id', True),
+            'title': extract_data_val('title', True),
+            'description': extract_data_val('description'),
+            'thumbnail': extract_data_val('image'),
+            'uploader': extract_data_val('channel'),
+        }

From 8e4aa7bf18af4403bf98742270483f3b9cfbdeb6 Mon Sep 17 00:00:00 2001
From: Benjamin Congdon <bencongdon96@gmail.com>
Date: Fri, 11 Mar 2016 09:31:35 -0600
Subject: [PATCH 0961/1105] [bbc] Fix BBC Extractor to work with 'School
 Report'

---
 youtube_dl/extractor/bbc.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 9d0dfb961..f4d8b4a2f 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -10,7 +10,6 @@ from ..utils import (
     int_or_none,
     parse_duration,
     parse_iso8601,
-    remove_end,
     unescapeHTML,
 )
 from ..compat import (
@@ -796,9 +795,15 @@ class BBCIE(BBCCoUkIE):
                             entries.append(self._extract_from_playlist_sxml(
                                 playlist.get('progressiveDownloadUrl'), playlist_id, timestamp))
 
+        playlist_title = self._og_search_title(webpage, default=None)
+        playlist_title = playlist_title or self._html_search_regex(
+            r'<title>(.*?)</title>', webpage, 'playlist title')
+
+        playlist_title = self._search_regex(r'(.+)\s*-\s*BBC', playlist_title, 'title', default=playlist_title)
+
+        playlist_description = self._og_search_description(webpage, default=None)
+
         if entries:
-            playlist_title = playlist_title or remove_end(self._og_search_title(webpage), ' - BBC News')
-            playlist_description = playlist_description or self._og_search_description(webpage, default=None)
             return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
 
         # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
@@ -829,10 +834,6 @@ class BBCIE(BBCCoUkIE):
                 'subtitles': subtitles,
             }
 
-        playlist_title = self._html_search_regex(
-            r'<title>(.*?)(?:\s*-\s*BBC [^ ]+)?</title>', webpage, 'playlist title')
-        playlist_description = self._og_search_description(webpage, default=None)
-
         def extract_all(pattern):
             return list(filter(None, map(
                 lambda s: self._parse_json(s, playlist_id, fatal=False),

From 0e832c2c97c62f67593ad356ea6d507778c56759 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 15:54:56 +0600
Subject: [PATCH 0962/1105] [bbc] Improve title and description extraction
 (Closes #8826, closes #8822)

---
 youtube_dl/extractor/bbc.py | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index f4d8b4a2f..497ebfd72 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -563,6 +563,14 @@ class BBCIE(BBCCoUkIE):
             'title': 'BBC Blogs - Adam Curtis - BUGGER',
         },
         'playlist_count': 18,
+    }, {
+        # school report playlist with single video
+        'url': 'http://www.bbc.co.uk/schoolreport/35744779',
+        'info_dict': {
+            'id': '35744779',
+            'title': 'School which breaks down barriers in Jerusalem',
+        },
+        'playlist_count': 1,
     }, {
         # single video embedded with data-playable containing vpid
         'url': 'http://www.bbc.com/news/world-europe-32041533',
@@ -734,8 +742,17 @@ class BBCIE(BBCCoUkIE):
 
         json_ld_info = self._search_json_ld(webpage, playlist_id, default=None)
         timestamp = json_ld_info.get('timestamp')
+
         playlist_title = json_ld_info.get('title')
-        playlist_description = json_ld_info.get('description')
+        if not playlist_title:
+            playlist_title = self._og_search_title(
+                webpage, default=None) or self._html_search_regex(
+                r'<title>(.+?)</title>', webpage, 'playlist title', default=None)
+            if playlist_title:
+                playlist_title = re.sub(r'(.+)\s*-\s*BBC.*?$', r'\1', playlist_title).strip()
+
+        playlist_description = json_ld_info.get(
+            'description') or self._og_search_description(webpage, default=None)
 
         if not timestamp:
             timestamp = parse_iso8601(self._search_regex(
@@ -795,14 +812,6 @@ class BBCIE(BBCCoUkIE):
                             entries.append(self._extract_from_playlist_sxml(
                                 playlist.get('progressiveDownloadUrl'), playlist_id, timestamp))
 
-        playlist_title = self._og_search_title(webpage, default=None)
-        playlist_title = playlist_title or self._html_search_regex(
-            r'<title>(.*?)</title>', webpage, 'playlist title')
-
-        playlist_title = self._search_regex(r'(.+)\s*-\s*BBC', playlist_title, 'title', default=playlist_title)
-
-        playlist_description = self._og_search_description(webpage, default=None)
-
         if entries:
             return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
 

From b7d7674f1ead233bcf9d4237ab893e31ec9d5ec8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 15:56:34 +0600
Subject: [PATCH 0963/1105] [bbc] Update test

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 497ebfd72..b46bc4856 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -560,7 +560,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460',
         'info_dict': {
             'id': '3662a707-0af9-3149-963f-47bea720b460',
-            'title': 'BBC Blogs - Adam Curtis - BUGGER',
+            'title': 'BUGGER',
         },
         'playlist_count': 18,
     }, {

From 450b233cc293e9a1fb63daf0ee0deb48a809dfd9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 15:59:54 +0600
Subject: [PATCH 0964/1105] [bbc] Update test

---
 youtube_dl/extractor/bbc.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index b46bc4856..e62b3860e 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -563,14 +563,6 @@ class BBCIE(BBCCoUkIE):
             'title': 'BUGGER',
         },
         'playlist_count': 18,
-    }, {
-        # school report playlist with single video
-        'url': 'http://www.bbc.co.uk/schoolreport/35744779',
-        'info_dict': {
-            'id': '35744779',
-            'title': 'School which breaks down barriers in Jerusalem',
-        },
-        'playlist_count': 1,
     }, {
         # single video embedded with data-playable containing vpid
         'url': 'http://www.bbc.com/news/world-europe-32041533',
@@ -677,9 +669,17 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/sport/0/football/34475836',
         'info_dict': {
             'id': '34475836',
-            'title': 'What Liverpool can expect from Klopp',
+            'title': 'Jurgen Klopp: Furious football from a witty and winning coach',
         },
         'playlist_count': 3,
+    }, {
+        # school report article with single video
+        'url': 'http://www.bbc.co.uk/schoolreport/35744779',
+        'info_dict': {
+            'id': '35744779',
+            'title': 'School which breaks down barriers in Jerusalem',
+        },
+        'playlist_count': 1,
     }, {
         # single video with playlist URL from weather section
         'url': 'http://www.bbc.com/weather/features/33601775',

From fb47597b09cf8474931a4509bebc510403b30e25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 16:27:20 +0600
Subject: [PATCH 0965/1105] [bbc] Generalize unit table lookup and add
 parse_count

---
 test/test_utils.py  | 10 ++++++++++
 youtube_dl/utils.py | 37 ++++++++++++++++++++++++++++++-------
 2 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 2bcf8ecf0..bc28ceb34 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -41,6 +41,7 @@ from youtube_dl.utils import (
     orderedSet,
     parse_duration,
     parse_filesize,
+    parse_count,
     parse_iso8601,
     read_batch_urls,
     sanitize_filename,
@@ -653,6 +654,15 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_filesize('1.2Tb'), 1200000000000)
         self.assertEqual(parse_filesize('1,24 KB'), 1240)
 
+    def test_parse_count(self):
+        self.assertEqual(parse_count(None), None)
+        self.assertEqual(parse_count(''), None)
+        self.assertEqual(parse_count('0'), 0)
+        self.assertEqual(parse_count('1000'), 1000)
+        self.assertEqual(parse_count('1.000'), 1000)
+        self.assertEqual(parse_count('1.1k'), 1100)
+        self.assertEqual(parse_count('1.1kk'), 1100000)
+
     def test_version_tuple(self):
         self.assertEqual(version_tuple('1'), (1,))
         self.assertEqual(version_tuple('10.23.344'), (10, 23, 344))
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 22a39a0ab..1d6ee9964 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1316,6 +1316,17 @@ def format_bytes(bytes):
     return '%.2f%s' % (converted, suffix)
 
 
+def lookup_unit_table(unit_table, s):
+    units_re = '|'.join(re.escape(u) for u in unit_table)
+    m = re.match(
+        r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
+    if not m:
+        return None
+    num_str = m.group('num').replace(',', '.')
+    mult = unit_table[m.group('unit')]
+    return int(float(num_str) * mult)
+
+
 def parse_filesize(s):
     if s is None:
         return None
@@ -1359,16 +1370,28 @@ def parse_filesize(s):
         'Yb': 1000 ** 8,
     }
 
-    units_re = '|'.join(re.escape(u) for u in _UNIT_TABLE)
-    m = re.match(
-        r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
-    if not m:
+    return lookup_unit_table(_UNIT_TABLE, s)
+
+
+def parse_count(s):
+    if s is None:
         return None
 
-    num_str = m.group('num').replace(',', '.')
-    mult = _UNIT_TABLE[m.group('unit')]
-    return int(float(num_str) * mult)
+    s = s.strip()
 
+    if re.match(r'^[\d,.]+$', s):
+        return str_to_int(s)
+
+    _UNIT_TABLE = {
+        'k': 1000,
+        'K': 1000,
+        'm': 1000 ** 2,
+        'M': 1000 ** 2,
+        'kk': 1000 ** 2,
+        'KK': 1000 ** 2,
+    }
+
+    return lookup_unit_table(_UNIT_TABLE, s)
 
 def month_by_name(name):
     """ Return the number of a month by (locale-independently) English name """

From 7f4173ae7c0dae870250018cd4915df36ccf0f54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 16:27:58 +0600
Subject: [PATCH 0966/1105] [mixcloud] Fix view count extraction (Closes #8831,
 closes #8845)

---
 youtube_dl/extractor/mixcloud.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index c2b7ed9ab..101497118 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -7,6 +7,7 @@ from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     ExtractorError,
     HEADRequest,
+    parse_count,
     str_to_int,
 )
 
@@ -85,8 +86,8 @@ class MixcloudIE(InfoExtractor):
         uploader_id = self._search_regex(
             r'\s+"profile": "([^"]+)",', webpage, 'uploader id', fatal=False)
         description = self._og_search_description(webpage)
-        like_count = str_to_int(self._search_regex(
-            r'\bbutton-favorite\b[^>]+m-ajax-toggle-count="([^"]+)"',
+        like_count = parse_count(self._search_regex(
+            r'\bbutton-favorite[^>]+>.*?<span[^>]+class=["\']toggle-number[^>]+>\s*([^<]+)',
             webpage, 'like count', fatal=False))
         view_count = str_to_int(self._search_regex(
             [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',

From a755f82549cb6bffd8ff9545e0b0a4883b7422d2 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 13 Mar 2016 12:15:29 +0100
Subject: [PATCH 0967/1105] [ffmpeg] convert format ext to ffmpeg output
 formats codes

---
 youtube_dl/downloader/external.py  |  4 ++--
 youtube_dl/postprocessor/ffmpeg.py | 13 +++++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index a4fdf1af8..fcb956e6c 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -6,7 +6,7 @@ import sys
 import re
 
 from .common import FileDownloader
-from ..postprocessor.ffmpeg import FFmpegPostProcessor
+from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
 from ..compat import compat_str
 from ..utils import (
     cli_option,
@@ -197,7 +197,7 @@ class FFmpegFD(ExternalFD):
             else:
                 args += ['-f', 'mp4', '-bsf:a', 'aac_adtstoasc']
         else:
-            args += ['-f', info_dict['ext']]
+            args += ['-f', EXT_TO_OUT_FORMATS.get(info_dict['ext'], info_dict['ext'])]
 
         args = [encodeArgument(opt) for opt in args]
         args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True))
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 380bc6f29..cc7aaeda3 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -25,6 +25,19 @@ from ..utils import (
 )
 
 
+EXT_TO_OUT_FORMATS = {
+    "aac": "adts",
+    "m4a": "ipod",
+    "mka": "matroska",
+    "mkv": "matroska",
+    "mpg": "mpeg",
+    "ogv": "ogg",
+    "ts": "mpegts",
+    "wma": "asf",
+    "wmv": "asf",
+}
+
+
 class FFmpegPostProcessorError(PostProcessingError):
     pass
 

From 0a477f873115a11dbaadae674c2aaabe15ec4249 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 17:22:23 +0600
Subject: [PATCH 0968/1105] [vice:show] Add extractor (Closes #8847)

---
 youtube_dl/extractor/__init__.py |  5 ++-
 youtube_dl/extractor/vice.py     | 76 ++++++++++++++++++++++++--------
 2 files changed, 61 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index dc5ecea30..c5b80f4aa 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -836,7 +836,10 @@ from .vgtv import (
     VGTVIE,
 )
 from .vh1 import VH1IE
-from .vice import ViceIE
+from .vice import (
+    ViceIE,
+    ViceShowIE,
+)
 from .viddler import ViddlerIE
 from .videodetective import VideoDetectiveIE
 from .videofyme import VideofyMeIE
diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index 3db6286e4..46c785ae1 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -1,31 +1,37 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from .ooyala import OoyalaIE
 from ..utils import ExtractorError
 
 
 class ViceIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?:[^/]+/)+(?P<id>.+)'
+    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?:[^/]+/)?videos?/(?P<id>[^/?#&]+)'
 
-    _TESTS = [
-        {
-            'url': 'http://www.vice.com/Fringes/cowboy-capitalists-part-1',
-            'info_dict': {
-                'id': '43cW1mYzpia9IlestBjVpd23Yu3afAfp',
-                'ext': 'mp4',
-                'title': 'VICE_COWBOYCAPITALISTS_PART01_v1_VICE_WM_1080p.mov',
-                'duration': 725.983,
-            },
-            'params': {
-                # Requires ffmpeg (m3u8 manifest)
-                'skip_download': True,
-            },
-        }, {
-            'url': 'https://news.vice.com/video/experimenting-on-animals-inside-the-monkey-lab',
-            'only_matching': True,
-        }
-    ]
+    _TESTS = [{
+        'url': 'http://www.vice.com/video/cowboy-capitalists-part-1',
+        'info_dict': {
+            'id': '43cW1mYzpia9IlestBjVpd23Yu3afAfp',
+            'ext': 'mp4',
+            'title': 'VICE_COWBOYCAPITALISTS_PART01_v1_VICE_WM_1080p.mov',
+            'duration': 725.983,
+        },
+        'params': {
+            # Requires ffmpeg (m3u8 manifest)
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://news.vice.com/video/experimenting-on-animals-inside-the-monkey-lab',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.vice.com/ru/video/big-night-out-ibiza-clive-martin-229',
+        'only_matching': True,
+    }, {
+        'url': 'https://munchies.vice.com/en/videos/watch-the-trailer-for-our-new-series-the-pizza-show',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -38,3 +44,35 @@ class ViceIE(InfoExtractor):
         except ExtractorError:
             raise ExtractorError('The page doesn\'t contain a video', expected=True)
         return self.url_result(ooyala_url, ie='Ooyala')
+
+
+class ViceShowIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?:[^/]+/)?show/(?P<id>[^/?#&]+)'
+
+    _TEST = {
+        'url': 'https://munchies.vice.com/en/show/fuck-thats-delicious-2',
+        'info_dict': {
+            'id': 'fuck-thats-delicious-2',
+            'title': "Fuck, That's Delicious",
+            'description': 'Follow the culinary adventures of rapper Action Bronson during his ongoing world tour.',
+        },
+        'playlist_count': 17,
+    }
+
+    def _real_extract(self, url):
+        show_id = self._match_id(url)
+        webpage = self._download_webpage(url, show_id)
+
+        entries = [
+            self.url_result(video_url, ViceIE.ie_key())
+            for video_url, _ in re.findall(
+                r'<h2[^>]+class="article-title"[^>]+data-id="\d+"[^>]*>\s*<a[^>]+href="(%s.*?)"'
+                % ViceIE._VALID_URL, webpage)]
+
+        title = self._search_regex(
+            r'<title>(.+?)</title>', webpage, 'title', default=None)
+        if title:
+            title = re.sub(r'(.+)\s*\|\s*.+$', r'\1', title).strip()
+        description = self._html_search_meta('description', webpage, 'description')
+
+        return self.playlist_result(entries, show_id, title, description)

From 2f7ae819ac844f4052dcd4ef031fdae01daff3b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 17:23:08 +0600
Subject: [PATCH 0969/1105] [utils] PEP 8

---
 youtube_dl/utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 1d6ee9964..9fd0ec8d5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1393,6 +1393,7 @@ def parse_count(s):
 
     return lookup_unit_table(_UNIT_TABLE, s)
 
+
 def month_by_name(name):
     """ Return the number of a month by (locale-independently) English name """
 

From 634415ca1791b22ddb8a272266084d8b22199d09 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 13 Mar 2016 12:23:10 +0100
Subject: [PATCH 0970/1105] [downloader/external] skip FFmpegFD when requesting
 multiple formats

---
 youtube_dl/downloader/external.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index fcb956e6c..4e299b4ee 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -162,7 +162,7 @@ class HttpieFD(ExternalFD):
 class FFmpegFD(ExternalFD):
     @classmethod
     def supports(cls, info_dict):
-        return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms')
+        return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms') and not info_dict.get('requested_formats')
 
     @classmethod
     def available(cls):

From b7b72db9ad248e16a7e26489fee93e52b70948d7 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 10 Mar 2016 20:17:35 +0100
Subject: [PATCH 0971/1105] [YoutubeDL] check for --list-thumbnails immediately
 after processing them

---
 youtube_dl/YoutubeDL.py | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b3391088f..8c651cd52 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1234,6 +1234,10 @@ class YoutubeDL(object):
                 if t.get('id') is None:
                     t['id'] = '%d' % i
 
+        if self.params.get('list_thumbnails'):
+            self.list_thumbnails(info_dict)
+            return
+
         if thumbnails and 'thumbnail' not in info_dict:
             info_dict['thumbnail'] = thumbnails[-1]['url']
 
@@ -1335,9 +1339,6 @@ class YoutubeDL(object):
         if self.params.get('listformats'):
             self.list_formats(info_dict)
             return
-        if self.params.get('list_thumbnails'):
-            self.list_thumbnails(info_dict)
-            return
 
         req_format = self.params.get('format')
         if req_format is None:
@@ -1901,13 +1902,8 @@ class YoutubeDL(object):
     def list_thumbnails(self, info_dict):
         thumbnails = info_dict.get('thumbnails')
         if not thumbnails:
-            tn_url = info_dict.get('thumbnail')
-            if tn_url:
-                thumbnails = [{'id': '0', 'url': tn_url}]
-            else:
-                self.to_screen(
-                    '[info] No thumbnails present for %s' % info_dict['id'])
-                return
+            self.to_screen('[info] No thumbnails present for %s' % info_dict['id'])
+            return
 
         self.to_screen(
             '[info] Thumbnails for %s:' % info_dict['id'])

From 91ee320bfaa279571b27bf44730c8677ff2b3980 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 13 Mar 2016 14:37:45 +0100
Subject: [PATCH 0972/1105] [downloader/external] wrap available_opt in a list

---
 youtube_dl/downloader/external.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 4e299b4ee..daedf66de 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -53,7 +53,7 @@ class ExternalFD(FileDownloader):
 
     @classmethod
     def available(cls):
-        return check_executable(cls.get_basename(), cls.available_opt)
+        return check_executable(cls.get_basename(), [cls.AVAILABLE_OPT])
 
     @classmethod
     def supports(cls, info_dict):
@@ -86,7 +86,7 @@ class ExternalFD(FileDownloader):
 
 
 class CurlFD(ExternalFD):
-    available_opt = ['-V']
+    AVAILABLE_OPT = '-V'
 
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '--location', '-o', tmpfilename]
@@ -101,7 +101,7 @@ class CurlFD(ExternalFD):
 
 
 class AxelFD(ExternalFD):
-    available_opt = ['-V']
+    AVAILABLE_OPT = '-V'
 
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-o', tmpfilename]
@@ -113,7 +113,7 @@ class AxelFD(ExternalFD):
 
 
 class WgetFD(ExternalFD):
-    available_opt = ['--version']
+    AVAILABLE_OPT = '--version'
 
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
@@ -128,7 +128,7 @@ class WgetFD(ExternalFD):
 
 
 class Aria2cFD(ExternalFD):
-    available_opt = ['-v']
+    AVAILABLE_OPT = '-v'
 
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-c']

From 2cb99ebbd0284f0de4bafd03179653bb9599b080 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 13 Mar 2016 14:53:17 +0100
Subject: [PATCH 0973/1105] [downloader/external] add can_download mathod for
 checking downloader availibilty and support

---
 youtube_dl/downloader/__init__.py | 4 ++--
 youtube_dl/downloader/external.py | 4 ++++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index 67c2840a5..daa5498d4 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -31,13 +31,13 @@ def get_suitable_downloader(info_dict, params={}):
     protocol = determine_protocol(info_dict)
     info_dict['protocol'] = protocol
 
-    if (info_dict.get('start_time') or info_dict.get('end_time')) and FFmpegFD.available() and FFmpegFD.supports(info_dict):
+    if (info_dict.get('start_time') or info_dict.get('end_time')) and FFmpegFD.can_download(info_dict):
         return FFmpegFD
 
     external_downloader = params.get('external_downloader')
     if external_downloader is not None:
         ed = get_external_downloader(external_downloader)
-        if ed.available() and ed.supports(info_dict):
+        if ed.can_download(info_dict):
             return ed
 
     if protocol == 'm3u8' and params.get('hls_prefer_native'):
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index daedf66de..fe2a0198c 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -59,6 +59,10 @@ class ExternalFD(FileDownloader):
     def supports(cls, info_dict):
         return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
 
+    @classmethod
+    def can_download(cls, info_dict):
+        return cls.available() and cls.supports(info_dict)
+
     def _option(self, command_option, param):
         return cli_option(self.params, command_option, param)
 

From be24916a7f6af9540076a6bc39bc78a71d6264bb Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 13 Mar 2016 15:24:02 +0100
Subject: [PATCH 0974/1105] [downloader/rtsp] Add rtsp and mms downloader

---
 youtube_dl/downloader/__init__.py |  5 ++--
 youtube_dl/downloader/rtsp.py     | 45 +++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 2 deletions(-)
 create mode 100644 youtube_dl/downloader/rtsp.py

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index daa5498d4..0b65aa3e2 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -6,6 +6,7 @@ from .hls import HlsFD
 from .http import HttpFD
 from .rtmp import RtmpFD
 from .dash import DashSegmentsFD
+from .rtsp import RtspFD
 from .external import (
     get_external_downloader,
     FFmpegFD,
@@ -19,8 +20,8 @@ PROTOCOL_MAP = {
     'rtmp': RtmpFD,
     'm3u8_native': HlsFD,
     'm3u8': FFmpegFD,
-    'mms': FFmpegFD,
-    'rtsp': FFmpegFD,
+    'mms': RtspFD,
+    'rtsp': RtspFD,
     'f4m': F4mFD,
     'http_dash_segments': DashSegmentsFD,
 }
diff --git a/youtube_dl/downloader/rtsp.py b/youtube_dl/downloader/rtsp.py
new file mode 100644
index 000000000..3eb29526c
--- /dev/null
+++ b/youtube_dl/downloader/rtsp.py
@@ -0,0 +1,45 @@
+from __future__ import unicode_literals
+
+import os
+import subprocess
+
+from .common import FileDownloader
+from ..utils import (
+    check_executable,
+    encodeFilename,
+)
+
+
+class RtspFD(FileDownloader):
+    def real_download(self, filename, info_dict):
+        url = info_dict['url']
+        self.report_destination(filename)
+        tmpfilename = self.temp_name(filename)
+
+        if check_executable('mplayer', ['-h']):
+            args = [
+                'mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy',
+                '-dumpstream', '-dumpfile', tmpfilename, url]
+        elif check_executable('mpv', ['-h']):
+            args = [
+                'mpv', '-really-quiet', '--vo=null', '--stream-dump=' + tmpfilename, url]
+        else:
+            self.report_error('MMS or RTSP download detected but neither "mplayer" nor "mpv" could be run. Please install any.')
+            return False
+
+        retval = subprocess.call(args)
+        if retval == 0:
+            fsize = os.path.getsize(encodeFilename(tmpfilename))
+            self.to_screen('\r[%s] %s bytes' % (args[0], fsize))
+            self.try_rename(tmpfilename, filename)
+            self._hook_progress({
+                'downloaded_bytes': fsize,
+                'total_bytes': fsize,
+                'filename': filename,
+                'status': 'finished',
+            })
+            return True
+        else:
+            self.to_stderr('\n')
+            self.report_error('%s exited with code %d' % (args[0], retval))
+            return False

From da1973a0383ddc8125702e74a88d3a35bbfcf06d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 13 Mar 2016 16:16:26 +0100
Subject: [PATCH 0975/1105] [extractor/__init__] disable time range downloading

---
 youtube_dl/downloader/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index 0b65aa3e2..19471a250 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -32,8 +32,8 @@ def get_suitable_downloader(info_dict, params={}):
     protocol = determine_protocol(info_dict)
     info_dict['protocol'] = protocol
 
-    if (info_dict.get('start_time') or info_dict.get('end_time')) and FFmpegFD.can_download(info_dict):
-        return FFmpegFD
+    # if (info_dict.get('start_time') or info_dict.get('end_time')) and FFmpegFD.can_download(info_dict):
+    #     return FFmpegFD
 
     external_downloader = params.get('external_downloader')
     if external_downloader is not None:

From 6ae27bed01111ddd3c533fb94ae187306e7cf940 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 13 Mar 2016 20:25:39 +0100
Subject: [PATCH 0976/1105] [download/external] move the check for multiple
 selected formats to get_suitable_downloader

---
 youtube_dl/downloader/__init__.py | 2 +-
 youtube_dl/downloader/external.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index 19471a250..73b34fdae 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -32,7 +32,7 @@ def get_suitable_downloader(info_dict, params={}):
     protocol = determine_protocol(info_dict)
     info_dict['protocol'] = protocol
 
-    # if (info_dict.get('start_time') or info_dict.get('end_time')) and FFmpegFD.can_download(info_dict):
+    # if (info_dict.get('start_time') or info_dict.get('end_time')) and not info_dict.get('requested_formats') and FFmpegFD.can_download(info_dict):
     #     return FFmpegFD
 
     external_downloader = params.get('external_downloader')
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index fe2a0198c..85cf834c7 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -166,7 +166,7 @@ class HttpieFD(ExternalFD):
 class FFmpegFD(ExternalFD):
     @classmethod
     def supports(cls, info_dict):
-        return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms') and not info_dict.get('requested_formats')
+        return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms')
 
     @classmethod
     def available(cls):

From 77dea16ac8bb38d735ac6f002eaba7570a67d361 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 13 Mar 2016 20:30:23 +0100
Subject: [PATCH 0977/1105] [downloader/external] check for ffmpeg availablity
 when it used for m3u8 download

---
 youtube_dl/downloader/external.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 85cf834c7..845a63e93 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -175,6 +175,9 @@ class FFmpegFD(ExternalFD):
     def _call_downloader(self, tmpfilename, info_dict):
         url = info_dict['url']
         ffpp = FFmpegPostProcessor(downloader=self)
+        if not ffpp.available:
+            self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
+            return False
         ffpp.check_version()
 
         args = [ffpp.executable, '-y']

From 694c47b2619b9d3698e0bc7bba45bb937f97bd3c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 13 Mar 2016 21:11:19 +0100
Subject: [PATCH 0978/1105] [external/downloader] don't pass -t and -ss to
 ffmpeg

---
 youtube_dl/downloader/external.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 845a63e93..2ddd76882 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -7,7 +7,6 @@ import re
 
 from .common import FileDownloader
 from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
-from ..compat import compat_str
 from ..utils import (
     cli_option,
     cli_valueless_option,
@@ -182,12 +181,12 @@ class FFmpegFD(ExternalFD):
 
         args = [ffpp.executable, '-y']
 
-        start_time = info_dict.get('start_time') or 0
-        if start_time:
-            args += ['-ss', compat_str(start_time)]
-        end_time = info_dict.get('end_time')
-        if end_time:
-            args += ['-t', compat_str(end_time - start_time)]
+        # start_time = info_dict.get('start_time') or 0
+        # if start_time:
+        #     args += ['-ss', compat_str(start_time)]
+        # end_time = info_dict.get('end_time')
+        # if end_time:
+        #     args += ['-t', compat_str(end_time - start_time)]
 
         if info_dict['http_headers'] and re.match(r'^https?://', url):
             # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:

From d8515fd41c504baea129de26d60ca55dd69ae3f8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 13 Mar 2016 21:13:50 +0100
Subject: [PATCH 0979/1105] [downloader/external] pass configuration args to
 ffmpeg

---
 youtube_dl/downloader/external.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 2ddd76882..697f81e3f 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -181,6 +181,8 @@ class FFmpegFD(ExternalFD):
 
         args = [ffpp.executable, '-y']
 
+        args += self._configuration_args()
+
         # start_time = info_dict.get('start_time') or 0
         # if start_time:
         #     args += ['-ss', compat_str(start_time)]

From 5d53c327015f3370c3af454d18d51f2f4401fad4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 13 Mar 2016 22:36:15 +0100
Subject: [PATCH 0980/1105] [usatoday] Add new extractor(closes #8655)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/usatoday.py | 48 ++++++++++++++++++++++++++++++++
 2 files changed, 49 insertions(+)
 create mode 100644 youtube_dl/extractor/usatoday.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c5b80f4aa..7874e7699 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -821,6 +821,7 @@ from .udn import UDNEmbedIE
 from .digiteka import DigitekaIE
 from .unistra import UnistraIE
 from .urort import UrortIE
+from .usatoday import USATodayIE
 from .ustream import UstreamIE, UstreamChannelIE
 from .ustudio import UstudioIE
 from .varzesh3 import Varzesh3IE
diff --git a/youtube_dl/extractor/usatoday.py b/youtube_dl/extractor/usatoday.py
new file mode 100644
index 000000000..e5678dc78
--- /dev/null
+++ b/youtube_dl/extractor/usatoday.py
@@ -0,0 +1,48 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    get_element_by_attribute,
+    parse_duration,
+    update_url_query,
+    ExtractorError,
+)
+from ..compat import compat_str
+
+
+class USATodayIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?usatoday\.com/(?:[^/]+/)*(?P<id>[^?/#]+)'
+    _TEST = {
+        'url': 'http://www.usatoday.com/media/cinematic/video/81729424/us-france-warn-syrian-regime-ahead-of-new-peace-talks/',
+        'md5': '4d40974481fa3475f8bccfd20c5361f8',
+        'info_dict': {
+            'id': '81729424',
+            'ext': 'mp4',
+            'title': 'US, France warn Syrian regime ahead of new peace talks',
+            'timestamp': 1457891045,
+            'description': 'md5:7e50464fdf2126b0f533748d3c78d58f',
+            'uploader_id': '29906170001',
+            'upload_date': '20160313',
+        }
+    }
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/29906170001/38a9eecc-bdd8-42a3-ba14-95397e48b3f8_default/index.html?videoId=%s'
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(update_url_query(url, {'ajax': 'true'}), display_id)
+        ui_video_data = get_element_by_attribute('class', 'ui-video-data', webpage)
+        if not ui_video_data:
+            raise ExtractorError('no video on the webpage', expected=True)
+        video_data = self._parse_json(ui_video_data, display_id)
+
+        return {
+            '_type': 'url_transparent',
+            'url': self.BRIGHTCOVE_URL_TEMPLATE % video_data['brightcove_id'],
+            'id': compat_str(video_data['id']),
+            'title': video_data['title'],
+            'thumbnail': video_data.get('thumbnail'),
+            'description': video_data.get('description'),
+            'duration': parse_duration(video_data.get('length')),
+            'ie_key': 'BrightcoveNew',
+        }

From 6bb266693f4b8cd92e27ea0efd46356dc254115b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 14 Mar 2016 10:25:20 +0100
Subject: [PATCH 0981/1105] release 2016.03.14

---
 README.md              | 2 +-
 docs/supportedsites.md | 9 ++++++++-
 youtube_dl/version.py  | 2 +-
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index a774e73e7..68db546ef 100644
--- a/README.md
+++ b/README.md
@@ -181,7 +181,7 @@ which means you can modify it, redistribute it or use it however you like.
                                      to play it)
     --external-downloader COMMAND    Use the specified external downloader.
                                      Currently supports
-                                     aria2c,axel,curl,httpie,wget
+                                     aria2c,avconv,axel,curl,ffmpeg,httpie,wget
     --external-downloader-args ARGS  Give these arguments to the external
                                      downloader
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 0682e4af0..a6dcc2576 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -54,6 +54,7 @@
  - **AtresPlayer**
  - **ATTTechChannel**
  - **AudiMedia**
+ - **AudioBoom**
  - **audiomack**
  - **audiomack:album**
  - **Azubu**
@@ -167,6 +168,8 @@
  - **Dump**
  - **Dumpert**
  - **dvtv**: http://video.aktualne.cz/
+ - **dw**
+ - **dw:article**
  - **EaglePlatform**
  - **EbaumsWorld**
  - **EchoMsk**
@@ -190,7 +193,6 @@
  - **ExpoTV**
  - **ExtremeTube**
  - **facebook**
- - **facebook:post**
  - **faz.net**
  - **fc2**
  - **Fczenit**
@@ -325,6 +327,7 @@
  - **m6**
  - **macgamestore**: MacGameStore trailers
  - **mailru**: Видео@Mail.Ru
+ - **MakersChannel**
  - **MakerTV**
  - **Malemotion**
  - **MatchTV**
@@ -335,6 +338,7 @@
  - **Mgoon**
  - **Minhateca**
  - **MinistryGrid**
+ - **Minoto**
  - **miomio.tv**
  - **MiTele**: mitele.es
  - **mixcloud**
@@ -671,6 +675,7 @@
  - **UDNEmbed**: 聯合影音
  - **Unistra**
  - **Urort**: NRK P3 Urørt
+ - **USAToday**
  - **ustream**
  - **ustream:channel**
  - **Ustudio**
@@ -684,6 +689,7 @@
  - **VGTV**: VGTV, BTTV, FTV, Aftenposten and Aftonbladet
  - **vh1.com**
  - **Vice**
+ - **ViceShow**
  - **Viddler**
  - **video.google:search**: Google Video search
  - **video.mit.edu**
@@ -711,6 +717,7 @@
  - **vimeo:channel**
  - **vimeo:group**
  - **vimeo:likes**: Vimeo user likes
+ - **vimeo:ondemand**
  - **vimeo:review**: Review pages on vimeo
  - **vimeo:user**
  - **vimeo:watchlater**: Vimeo watch later list, "vimeowatchlater" keyword (requires authentication)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 246f5740d..9216fa547 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.03.06'
+__version__ = '2016.03.14'

From 4230c4894d0439387bcd09dea4aba697b16975ce Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 14 Mar 2016 16:49:16 +0100
Subject: [PATCH 0982/1105] [external/downloader] fix rtmp downloading using
 FFmpegFD

---
 youtube_dl/downloader/external.py | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 697f81e3f..30277dc20 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -198,12 +198,39 @@ class FFmpegFD(ExternalFD):
                 '-headers',
                 ''.join('%s: %s\r\n' % (key, val) for key, val in headers.items())]
 
+        protocol = info_dict.get('protocol')
+
+        if protocol == 'rtmp':
+            player_url = info_dict.get('player_url')
+            page_url = info_dict.get('page_url')
+            app = info_dict.get('app')
+            play_path = info_dict.get('play_path')
+            tc_url = info_dict.get('tc_url')
+            flash_version = info_dict.get('flash_version')
+            live = info_dict.get('rtmp_live', False)
+            if player_url is not None:
+                args += ['-rtmp_swfverify', player_url]
+            if page_url is not None:
+                args += ['-rtmp_pageurl', page_url]
+            if app is not None:
+                args += ['-rtmp_app', app]
+            if play_path is not None:
+                args += ['-rtmp_playpath', play_path]
+            if tc_url is not None:
+                args += ['-rtmp_tcurl', tc_url]
+            if flash_version is not None:
+                args += ['-rtmp_flashver', flash_version]
+            if live:
+                args += ['-rtmp_live', 'live']
+
         args += ['-i', url, '-c', 'copy']
-        if info_dict.get('protocol') == 'm3u8':
+        if protocol == 'm3u8':
             if self.params.get('hls_use_mpegts', False):
                 args += ['-f', 'mpegts']
             else:
                 args += ['-f', 'mp4', '-bsf:a', 'aac_adtstoasc']
+        elif protocol == 'rtmp':
+            args += ['-f', 'flv']
         else:
             args += ['-f', EXT_TO_OUT_FORMATS.get(info_dict['ext'], info_dict['ext'])]
 

From 203f3d779aad18927962d01c8d1e9927835bd84f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 14 Mar 2016 18:32:29 +0100
Subject: [PATCH 0983/1105] [gameinformer] update the extractor to use
 BrightcoveNewIE

---
 youtube_dl/extractor/gameinformer.py | 29 ++++++++--------------------
 1 file changed, 8 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/gameinformer.py b/youtube_dl/extractor/gameinformer.py
index 25870c131..1a99e35ed 100644
--- a/youtube_dl/extractor/gameinformer.py
+++ b/youtube_dl/extractor/gameinformer.py
@@ -10,34 +10,21 @@ class GameInformerIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?gameinformer\.com/(?:[^/]+/)*(?P<id>.+)\.aspx'
     _TEST = {
         'url': 'http://www.gameinformer.com/b/features/archive/2015/09/26/replay-animal-crossing.aspx',
+        'md5': '292f26da1ab4beb4c9099f1304d2b071',
         'info_dict': {
             'id': '4515472681001',
-            'ext': 'm3u8',
+            'ext': 'mp4',
             'title': 'Replay - Animal Crossing',
             'description': 'md5:2e211891b215c85d061adc7a4dd2d930',
-            'timestamp': 1443457610706,
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
+            'timestamp': 1443457610,
+            'upload_date': '20150928',
+            'uploader_id': '694940074001',
         },
     }
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/694940074001/default_default/index.html?videoId=%s'
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-
-        bc_api_url = self._search_regex(r"getVideo\('([^']+)'", webpage, 'brightcove api url')
-        json_data = self._download_json(
-            bc_api_url + '&video_fields=id,name,shortDescription,publishedDate,videoStillURL,length,IOSRenditions',
-            display_id)
-
-        return {
-            'id': compat_str(json_data['id']),
-            'display_id': display_id,
-            'url': json_data['IOSRenditions'][0]['url'],
-            'title': json_data['name'],
-            'description': json_data.get('shortDescription'),
-            'timestamp': int_or_none(json_data.get('publishedDate')),
-            'duration': int_or_none(json_data.get('length')),
-        }
+        brightcove_id = self._search_regex(r"getVideo\('[^']+video_id=(\d+)", webpage, 'brightcove id')
+        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)

From bf475e1990f4f4e52cf36ee10a39106c25999b26 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 14 Mar 2016 21:53:00 +0100
Subject: [PATCH 0984/1105] [tlc] fix extraction and update extractor to use
 BrightcoveNewIE

---
 youtube_dl/extractor/tlc.py | 35 +++++++++++++----------------------
 1 file changed, 13 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/tlc.py b/youtube_dl/extractor/tlc.py
index adc05ed5f..17add9543 100644
--- a/youtube_dl/extractor/tlc.py
+++ b/youtube_dl/extractor/tlc.py
@@ -4,12 +4,12 @@ import re
 
 from .common import InfoExtractor
 from .brightcove import BrightcoveLegacyIE
-from ..compat import compat_urlparse
+from ..compat import compat_parse_qs
 
 
 class TlcDeIE(InfoExtractor):
     IE_NAME = 'tlc.de'
-    _VALID_URL = r'http://www\.tlc\.de/sendungen/[^/]+/videos/(?P<title>[^/?]+)'
+    _VALID_URL = r'http://www\.tlc\.de/(?:[^/]+/)*videos/(?P<title>[^/?#]+)?(?:.*#(?P<id>\d+))?'
 
     _TEST = {
         'url': 'http://www.tlc.de/sendungen/breaking-amish/videos/#3235167922001',
@@ -17,32 +17,23 @@ class TlcDeIE(InfoExtractor):
             'id': '3235167922001',
             'ext': 'mp4',
             'title': 'Breaking Amish: Die Welt da draußen',
-            'uploader': 'Discovery Networks - Germany',
             'description': (
                 'Vier Amische und eine Mennonitin wagen in New York'
                 '  den Sprung in ein komplett anderes Leben. Begleitet sie auf'
                 ' ihrem spannenden Weg.'),
+            'timestamp': 1396598084,
+            'upload_date': '20140404',
+            'uploader_id': '1659832546',
         },
     }
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1659832546/default_default/index.html?videoId=%s'
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        title = mobj.group('title')
-        webpage = self._download_webpage(url, title)
-        iframe_url = self._search_regex(
-            '<iframe src="(http://www\.tlc\.de/wp-content/.+?)"', webpage,
-            'iframe url')
-        # Otherwise we don't get the correct 'BrightcoveExperience' element,
-        # example: http://www.tlc.de/sendungen/cake-boss/videos/cake-boss-cannoli-drama/
-        iframe_url = iframe_url.replace('.htm?', '.php?')
-        url_fragment = compat_urlparse.urlparse(url).fragment
-        if url_fragment:
-            # Since the fragment is not send to the server, we always get the same iframe
-            iframe_url = re.sub(r'playlist=(\d+)', 'playlist=%s' % url_fragment, iframe_url)
-        iframe = self._download_webpage(iframe_url, title)
-
-        return {
-            '_type': 'url',
-            'url': BrightcoveLegacyIE._extract_brightcove_url(iframe),
-            'ie': BrightcoveLegacyIE.ie_key(),
-        }
+        brightcove_id = mobj.group('id')
+        if not brightcove_id:
+            title = mobj.group('title')
+            webpage = self._download_webpage(url, title)
+            brightcove_legacy_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
+            brightcove_id = compat_parse_qs(brightcove_legacy_url)['@videoPlayer'][0]
+        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)

From 744daf941843bba00e47c37e85df91637861b596 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 14 Mar 2016 21:57:26 +0100
Subject: [PATCH 0985/1105] [gameinformer] remove unused imports

---
 youtube_dl/extractor/gameinformer.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/gameinformer.py b/youtube_dl/extractor/gameinformer.py
index 1a99e35ed..a66e309de 100644
--- a/youtube_dl/extractor/gameinformer.py
+++ b/youtube_dl/extractor/gameinformer.py
@@ -2,8 +2,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_str
-from ..utils import int_or_none
 
 
 class GameInformerIE(InfoExtractor):

From d13bdc38241b52ff942c02ac2c15def4520b39a5 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 26 Nov 2015 18:38:57 +0100
Subject: [PATCH 0986/1105] [brightcove] raise ExtractorError on 403 errors and
 fix regex to work with tenplay

---
 youtube_dl/extractor/brightcove.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index c947337f9..5258b907c 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -13,6 +13,7 @@ from ..compat import (
     compat_urllib_parse_urlparse,
     compat_urlparse,
     compat_xml_parse_error,
+    compat_HTTPError,
 )
 from ..utils import (
     determine_ext,
@@ -424,7 +425,7 @@ class BrightcoveNewIE(InfoExtractor):
                     </video>.*?
                     <script[^>]+
                         src=["\'](?:https?:)?//players\.brightcove\.net/
-                        (\d+)/([\da-f-]+)_([^/]+)/index\.min\.js
+                        (\d+)/([\da-f-]+)_([^/]+)/index(?:\.min)?\.js
                 ''', webpage):
             entries.append(
                 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
@@ -458,7 +459,13 @@ class BrightcoveNewIE(InfoExtractor):
             'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s'
             % (account_id, video_id),
             headers={'Accept': 'application/json;pk=%s' % policy_key})
-        json_data = self._download_json(req, video_id)
+        try:
+            json_data = self._download_json(req, video_id)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                json_data = self._parse_json(e.cause.read().decode(), video_id)
+                raise ExtractorError(json_data[0]['message'], expected=True)
+            raise
 
         title = json_data['name']
 

From 877f440f7be1a4f83c230d6ec06aeb5242e41084 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 15 Mar 2016 00:49:23 +0100
Subject: [PATCH 0987/1105] [rice] Add new extractor(closes #1736)

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/rice.py     | 116 +++++++++++++++++++++++++++++++
 2 files changed, 117 insertions(+)
 create mode 100644 youtube_dl/extractor/rice.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7874e7699..179c11ffa 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -598,6 +598,7 @@ from .regiotv import RegioTVIE
 from .restudy import RestudyIE
 from .reverbnation import ReverbNationIE
 from .revision3 import Revision3IE
+from .rice import RICEIE
 from .ringtv import RingTVIE
 from .ro220 import Ro220IE
 from .rottentomatoes import RottenTomatoesIE
diff --git a/youtube_dl/extractor/rice.py b/youtube_dl/extractor/rice.py
new file mode 100644
index 000000000..f855719ac
--- /dev/null
+++ b/youtube_dl/extractor/rice.py
@@ -0,0 +1,116 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_parse_qs
+from ..utils import (
+    xpath_text,
+    xpath_element,
+    int_or_none,
+    parse_iso8601,
+    ExtractorError,
+)
+
+
+class RICEIE(InfoExtractor):
+    _VALID_URL = r'https?://mediahub\.rice\.edu/app/[Pp]ortal/video\.aspx\?(?P<query>.+)'
+    _TEST = {
+        'url': 'https://mediahub.rice.edu/app/Portal/video.aspx?PortalID=25ffd62c-3d01-4b29-8c70-7c94270efb3e&DestinationID=66bc9434-03bd-4725-b47e-c659d8d809db&ContentID=YEWIvbhb40aqdjMD1ALSqw',
+        'md5': '9b83b4a2eead4912dc3b7fac7c449b6a',
+        'info_dict': {
+            'id': 'YEWIvbhb40aqdjMD1ALSqw',
+            'ext': 'mp4',
+            'title': 'Active Learning in Archeology',
+            'upload_date': '20140616',
+            'timestamp': 1402926346,
+        }
+    }
+    _NS = 'http://schemas.datacontract.org/2004/07/ensembleVideo.Data.Service.Contracts.Models.Player.Config'
+
+    def _real_extract(self, url):
+        qs = compat_parse_qs(re.match(self._VALID_URL, url).group('query'))
+        if not qs.get('PortalID') or not qs.get('DestinationID') or not qs.get('ContentID'):
+            raise ExtractorError('Invalid URL', expected=True)
+
+        portal_id = qs['PortalID'][0]
+        playlist_id = qs['DestinationID'][0]
+        content_id = qs['ContentID'][0]
+
+        content_data = self._download_xml('https://mediahub.rice.edu/api/portal/GetContentTitle', content_id, query={
+            'portalId': portal_id,
+            'playlistId': playlist_id,
+            'contentId': content_id
+        })
+        metadata = xpath_element(content_data, './/metaData', fatal=True)
+        title = xpath_text(metadata, 'primaryTitle', fatal=True)
+        encodings = xpath_element(content_data, './/encodings', fatal=True)
+        player_data = self._download_xml('https://mediahub.rice.edu/api/player/GetPlayerConfig', content_id, query={
+            'temporaryLinkId': xpath_text(encodings, 'temporaryLinkId', fatal=True),
+            'contentId': content_id,
+        })
+
+        common_fmt = {}
+        dimensions = xpath_text(encodings, 'dimensions')
+        if dimensions:
+            wh = dimensions.split('x')
+            if len(wh) == 2:
+                common_fmt.update({
+                    'width': int_or_none(wh[0]),
+                    'height': int_or_none(wh[1]),
+                })
+
+        formats = []
+        rtsp_path = xpath_text(player_data, self._xpath_ns('RtspPath', self._NS))
+        if rtsp_path:
+            fmt = {
+                'url': rtsp_path,
+                'format_id': 'rtsp',
+            }
+            fmt.update(common_fmt)
+            formats.append(fmt)
+        for source in player_data.findall(self._xpath_ns('.//Source', self._NS)):
+            video_url = xpath_text(source, self._xpath_ns('File', self._NS))
+            if not video_url:
+                continue
+            if '.m3u8' in video_url:
+                formats.extend(self._extract_m3u8_formats(video_url, content_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+            else:
+                fmt = {
+                    'url': video_url,
+                    'format_id': video_url.split(':')[0],
+                }
+                fmt.update(common_fmt)
+                rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+))/(?P<playpath>mp4:.+)$', video_url)
+                if rtmp:
+                    fmt.update({
+                        'url': rtmp.group('url'),
+                        'play_path': rtmp.group('playpath'),
+                        'app': rtmp.group('app'),
+                        'ext': 'flv',
+                    })
+                formats.append(fmt)
+        self._sort_formats(formats)
+
+        thumbnails = []
+        for content_asset in content_data.findall('.//contentAssets'):
+            asset_type = xpath_text(content_asset, 'type')
+            if asset_type == 'image':
+                image_url = xpath_text(content_asset, 'httpPath')
+                if not image_url:
+                    continue
+                thumbnails.append({
+                    'id': xpath_text(content_asset, 'ID'),
+                    'url': image_url,
+                })
+
+        return {
+            'id': content_id,
+            'title': title,
+            'description': xpath_text(metadata, 'abstract'),
+            'duration': int_or_none(xpath_text(metadata, 'duration')),
+            'timestamp': parse_iso8601(xpath_text(metadata, 'dateUpdated')),
+            'thumbnails': thumbnails,
+            'formats': formats,
+        }

From 8ac73bdbe44e14007fb3e630eb70e2a395133480 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 15 Mar 2016 10:03:08 +0100
Subject: [PATCH 0988/1105] [brightcove:new] Add support for non numeric ref:
 preffixed video ids

---
 youtube_dl/extractor/brightcove.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 5258b907c..9760e606f 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -356,7 +356,7 @@ class BrightcoveLegacyIE(InfoExtractor):
 
 class BrightcoveNewIE(InfoExtractor):
     IE_NAME = 'brightcove:new'
-    _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>(?:ref:)?\d+)'
+    _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>\d+|ref:[^&]+)'
     _TESTS = [{
         'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
         'md5': 'c8100925723840d4b0d243f7025703be',
@@ -392,6 +392,10 @@ class BrightcoveNewIE(InfoExtractor):
         # ref: prefixed video id
         'url': 'http://players.brightcove.net/3910869709001/21519b5c-4b3b-4363-accb-bdc8f358f823_default/index.html?videoId=ref:7069442',
         'only_matching': True,
+    }, {
+        # non numeric ref: prefixed video id
+        'url': 'http://players.brightcove.net/710858724001/default_default/index.html?videoId=ref:event-stream-356',
+        'only_matching': True,
     }]
 
     @staticmethod

From 975b9c9ab09306054d159d57085ed6ea1582b4de Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 15 Mar 2016 10:06:53 +0100
Subject: [PATCH 0989/1105] [brightcove:new] detect m3u8 manifests by M2TS
 container

---
 youtube_dl/extractor/brightcove.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 9760e606f..f8413d5f2 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -475,9 +475,10 @@ class BrightcoveNewIE(InfoExtractor):
 
         formats = []
         for source in json_data.get('sources', []):
+            container = source.get('container')
             source_type = source.get('type')
             src = source.get('src')
-            if source_type == 'application/x-mpegURL':
+            if source_type == 'application/x-mpegURL' or container == 'M2TS':
                 if not src:
                     continue
                 formats.extend(self._extract_m3u8_formats(
@@ -495,7 +496,7 @@ class BrightcoveNewIE(InfoExtractor):
                     'width': int_or_none(source.get('width')),
                     'height': height,
                     'filesize': int_or_none(source.get('size')),
-                    'container': source.get('container'),
+                    'container': container,
                     'vcodec': source.get('codec'),
                     'ext': source.get('container').lower(),
                 }

From cf6cd07396479940bbf9ca71778a72807208fabb Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 15 Mar 2016 15:24:12 +0100
Subject: [PATCH 0990/1105] [noz] extract f4m and m3u8 formats

---
 youtube_dl/extractor/noz.py | 35 ++++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/noz.py b/youtube_dl/extractor/noz.py
index 0ffb44b47..4e60b13a5 100644
--- a/youtube_dl/extractor/noz.py
+++ b/youtube_dl/extractor/noz.py
@@ -6,6 +6,7 @@ from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     int_or_none,
     xpath_text,
+    update_url_query,
 )
 
 
@@ -46,17 +47,29 @@ class NozIE(InfoExtractor):
             doc, './/article/movie/file/duration'))
         formats = []
         for qnode in doc.findall('.//article/movie/file/qualities/qual'):
-            video_node = qnode.find('./html_urls/video_url[@format="video/mp4"]')
-            if video_node is None:
-                continue  # auto
-            formats.append({
-                'url': video_node.text,
-                'format_name': xpath_text(qnode, './name'),
-                'format_id': xpath_text(qnode, './id'),
-                'height': int_or_none(xpath_text(qnode, './height')),
-                'width': int_or_none(xpath_text(qnode, './width')),
-                'tbr': int_or_none(xpath_text(qnode, './bitrate'), scale=1000),
-            })
+            http_url = xpath_text(
+                qnode, './html_urls/video_url[@format="video/mp4"]')
+            if http_url:
+                formats.append({
+                    'url': http_url,
+                    'format_name': xpath_text(qnode, './name'),
+                    'format_id': '%s-%s' % ('http', xpath_text(qnode, './id')),
+                    'height': int_or_none(xpath_text(qnode, './height')),
+                    'width': int_or_none(xpath_text(qnode, './width')),
+                    'tbr': int_or_none(xpath_text(qnode, './bitrate'), scale=1000),
+                })
+            else:
+                f4m_url = xpath_text(qnode, 'url_hd2')
+                if f4m_url:
+                    formats.extend(self._extract_f4m_formats(
+                        update_url_query(f4m_url, {'hdcore': '3.4.0'}),
+                        video_id, f4m_id='hds', fatal=False))
+                m3u8_url = xpath_text(
+                    qnode, './html_urls/video_url[@format="application/vnd.apple.mpegurl"]')
+                if m3u8_url:
+                    formats.extend(self._extract_m3u8_formats(
+                        m3u8_url, video_id, 'mp4', 'm3u8_native',
+                        m3u8_id='hls', fatal=False))
         self._sort_formats(formats)
 
         return {

From dec2cae0a768d2eb1f7d28cfd267d0bf5383bcdd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 15 Mar 2016 21:45:43 +0600
Subject: [PATCH 0991/1105] [twitch:playlistbase] Clarify pagination bug

Pagination bug has been fixed by twitch on 15.03.2016.
---
 youtube_dl/extractor/twitch.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 958bf8fff..d4169ec6d 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -299,9 +299,10 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
             # is completely broken on the twitch side. It simply ignores
             # a limit and returns the whole offset number of videos.
             # Working around by just requesting all videos at once.
+            # Upd: pagination bug was fixed by twitch on 15.03.2016.
             if not broken_paging_detected and total and len(page_entries) > limit:
                 self.report_warning(
-                    'Twitch paging is broken on twitch side, requesting all videos at once',
+                    'Twitch pagination is broken on twitch side, requesting all videos at once',
                     channel_id)
                 broken_paging_detected = True
                 offset = total

From 3222b0dc7d98d27557a36933a58b9d5ee1ce4394 Mon Sep 17 00:00:00 2001
From: Mark Lee <youtube-dl@lazymalevolence.com>
Date: Tue, 15 Mar 2016 08:48:55 -0700
Subject: [PATCH 0992/1105] Fix protocol and join-parts interaction

---
 youtube_dl/YoutubeDL.py        | 8 ++++----
 youtube_dl/extractor/common.py | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 2f242e87b..36ce66e23 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1327,11 +1327,11 @@ class YoutubeDL(object):
             # Automatically determine file extension if missing
             if 'ext' not in format:
                 format['ext'] = determine_ext(format['url']).lower()
-            # Automatically determine protocol if missing (useful for format
-            # selection purposes)
-            if 'protocol' not in format:
-                format['protocol'] = determine_protocol(format)
             if 'parts' not in format:
+                # Automatically determine protocol if missing (useful for format
+                # selection purposes)
+                if 'protocol' not in format:
+                    format['protocol'] = determine_protocol(format)
                 # Add HTTP headers, so that external programs can use them from the
                 # json output
                 full_format_info = info_dict.copy()
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b689e5bf5..efd078b69 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -863,7 +863,7 @@ class InfoExtractor(object):
                 if f.get('ext') in ['f4f', 'f4m']:  # Not yet supported
                     preference -= 0.5
 
-            proto_preference = 0 if determine_protocol(f) in ['http', 'https'] else -0.1
+            proto_preference = 0 if 'url' in f and determine_protocol(f) in ['http', 'https'] else -0.1
 
             if f.get('vcodec') == 'none':  # audio only
                 if self._downloader.params.get('prefer_free_formats'):

From 89807d6a8293d7b028a204628aec32b6f39148dd Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 15 Mar 2016 18:48:21 +0100
Subject: [PATCH 0993/1105] [brightcove] extract dash formats and detect audio
 formats

---
 youtube_dl/extractor/brightcove.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index f8413d5f2..86de5b9ee 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -484,6 +484,10 @@ class BrightcoveNewIE(InfoExtractor):
                 formats.extend(self._extract_m3u8_formats(
                     src, video_id, 'mp4', entry_protocol='m3u8_native',
                     m3u8_id='hls', fatal=False))
+            elif source_type == 'application/dash+xml':
+                if not src:
+                    continue
+                formats.extend(self._extract_mpd_formats(src, video_id, 'dash', fatal=False))
             else:
                 streaming_src = source.get('streaming_src')
                 stream_name, app_name = source.get('stream_name'), source.get('app_name')
@@ -491,15 +495,23 @@ class BrightcoveNewIE(InfoExtractor):
                     continue
                 tbr = float_or_none(source.get('avg_bitrate'), 1000)
                 height = int_or_none(source.get('height'))
+                width = int_or_none(source.get('width'))
                 f = {
                     'tbr': tbr,
-                    'width': int_or_none(source.get('width')),
-                    'height': height,
                     'filesize': int_or_none(source.get('size')),
                     'container': container,
-                    'vcodec': source.get('codec'),
-                    'ext': source.get('container').lower(),
+                    'ext': container.lower(),
                 }
+                if width == 0 and height == 0:
+                    f.update({
+                        'vcodec': 'none',
+                    })
+                else:
+                    f.update({
+                        'width': width,
+                        'height': height,
+                        'vcodec': source.get('codec'),
+                    })
 
                 def build_format_id(kind):
                     format_id = kind

From edfd93518ea0a929dbdd51cc3472bddceaf96514 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 15 Mar 2016 19:33:09 +0100
Subject: [PATCH 0994/1105] [svt] extract dashhbbtv formats(#8867)

---
 youtube_dl/extractor/svt.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index 399c3b8ee..2ab30e45f 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -19,20 +19,25 @@ class SVTBaseIE(InfoExtractor):
         video_info = info['video']
         formats = []
         for vr in video_info['videoReferences']:
+            player_type = vr.get('playerType')
             vurl = vr['url']
             ext = determine_ext(vurl)
             if ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
                     vurl, video_id,
                     ext='mp4', entry_protocol='m3u8_native',
-                    m3u8_id=vr.get('playerType')))
+                    m3u8_id=player_type, fatal=False))
             elif ext == 'f4m':
                 formats.extend(self._extract_f4m_formats(
                     vurl + '?hdcore=3.3.0', video_id,
-                    f4m_id=vr.get('playerType')))
+                    f4m_id=player_type, fatal=False))
+            elif ext == 'mpd':
+                if player_type == 'dashhbbtv':
+                    formats.extend(self._extract_mpd_formats(
+                        vurl, video_id, mpd_id=player_type, fatal=False))
             else:
                 formats.append({
-                    'format_id': vr.get('playerType'),
+                    'format_id': player_type,
                     'url': vurl,
                 })
         self._sort_formats(formats)

From cb6e477dfe09b68a810e587269958b1e56077b00 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 15 Mar 2016 19:38:10 +0100
Subject: [PATCH 0995/1105] [aljazeera] update the extractor to use
 BrightcoveNewIE

---
 youtube_dl/extractor/aljazeera.py | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/aljazeera.py b/youtube_dl/extractor/aljazeera.py
index 5b2c0dc9a..cddcaa489 100644
--- a/youtube_dl/extractor/aljazeera.py
+++ b/youtube_dl/extractor/aljazeera.py
@@ -13,24 +13,18 @@ class AlJazeeraIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'The Slum - Episode 1: Deliverance',
             'description': 'As a birth attendant advocating for family planning, Remy is on the frontline of Tondo\'s battle with overcrowding.',
-            'uploader': 'Al Jazeera English',
+            'uploader_id': '665003303001',
+            'timestamp': 1411116829,
+            'upload_date': '20140919',
         },
-        'add_ie': ['BrightcoveLegacy'],
+        'add_ie': ['BrightcoveNew'],
         'skip': 'Not accessible from Travis CI server',
     }
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/665003303001/default_default/index.html?videoId=%s'
 
     def _real_extract(self, url):
         program_name = self._match_id(url)
         webpage = self._download_webpage(url, program_name)
         brightcove_id = self._search_regex(
             r'RenderPagesVideo\(\'(.+?)\'', webpage, 'brightcove id')
-
-        return {
-            '_type': 'url',
-            'url': (
-                'brightcove:'
-                'playerKey=AQ~~%2CAAAAmtVJIFk~%2CTVGOQ5ZTwJbeMWnq5d_H4MOM57xfzApc'
-                '&%40videoPlayer={0}'.format(brightcove_id)
-            ),
-            'ie_key': 'BrightcoveLegacy',
-        }
+        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)

From 3ff8279e80b4c057f12998dcca5164209bb71078 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 15 Mar 2016 02:20:37 +0800
Subject: [PATCH 0996/1105] [kuwo:mv] Fix the test and extraction of
 georestricted MVs

---
 youtube_dl/extractor/kuwo.py | 33 +++++++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 700e44b63..f94804d06 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -23,7 +23,7 @@ class KuwoBaseIE(InfoExtractor):
         {'format': 'aac', 'ext': 'aac', 'abr': 48, 'preference': 10}
     ]
 
-    def _get_formats(self, song_id):
+    def _get_formats(self, song_id, tolerate_ip_deny=False):
         formats = []
         for file_format in self._FORMATS:
             song_url = self._download_webpage(
@@ -32,7 +32,7 @@ class KuwoBaseIE(InfoExtractor):
                 song_id, note='Download %s url info' % file_format['format'],
             )
 
-            if song_url == 'IPDeny':
+            if song_url == 'IPDeny' and not tolerate_ip_deny:
                 raise ExtractorError('This song is blocked in this region', expected=True)
 
             if song_url.startswith('http://') or song_url.startswith('https://'):
@@ -43,7 +43,12 @@ class KuwoBaseIE(InfoExtractor):
                     'preference': file_format['preference'],
                     'abr': file_format.get('abr'),
                 })
-        self._sort_formats(formats)
+
+        # XXX _sort_formats fails if there are not formats, while it's not the
+        # desired behavior if 'IPDeny' is ignored
+        # This check can be removed if https://github.com/rg3/youtube-dl/pull/8051 is merged
+        if not tolerate_ip_deny:
+            self._sort_formats(formats)
         return formats
 
 
@@ -288,10 +293,16 @@ class KuwoMvIE(KuwoBaseIE):
         'url': 'http://www.kuwo.cn/mv/6480076/',
         'info_dict': {
             'id': '6480076',
-            'ext': 'mkv',
-            'title': '我们家MV',
+            'ext': 'mp4',
+            'title': 'My HouseMV',
             'creator': '2PM',
         },
+        # In this video, music URLs (anti.s) are blocked outside China and
+        # USA, while the MV URL (mvurl) is available globally, so force the MV
+        # URL for consistent results in different countries
+        'params': {
+            'format': 'mv',
+        },
     }
     _FORMATS = KuwoBaseIE._FORMATS + [
         {'format': 'mkv', 'ext': 'mkv', 'preference': 250},
@@ -313,7 +324,17 @@ class KuwoMvIE(KuwoBaseIE):
         else:
             raise ExtractorError('Unable to find song or singer names')
 
-        formats = self._get_formats(song_id)
+        formats = self._get_formats(song_id, tolerate_ip_deny=True)
+
+        mv_url = self._download_webpage(
+            'http://www.kuwo.cn/yy/st/mvurl?rid=MUSIC_%s' % song_id,
+            song_id, note='Download %s MV URL' % song_id)
+        formats.append({
+            'url': mv_url,
+            'format_id': 'mv',
+        })
+
+        self._sort_formats(formats)
 
         return {
             'id': song_id,

From 0c9ff2404119bc93fe9755fa5ecdf8e8dd53b146 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 15 Mar 2016 20:54:55 +0100
Subject: [PATCH 0997/1105] [noz] fix extraction in python 2.6

---
 youtube_dl/extractor/noz.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/noz.py b/youtube_dl/extractor/noz.py
index 4e60b13a5..656443c49 100644
--- a/youtube_dl/extractor/noz.py
+++ b/youtube_dl/extractor/noz.py
@@ -5,6 +5,7 @@ from .common import InfoExtractor
 from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     int_or_none,
+    find_xpath_attr,
     xpath_text,
     update_url_query,
 )
@@ -47,8 +48,9 @@ class NozIE(InfoExtractor):
             doc, './/article/movie/file/duration'))
         formats = []
         for qnode in doc.findall('.//article/movie/file/qualities/qual'):
-            http_url = xpath_text(
-                qnode, './html_urls/video_url[@format="video/mp4"]')
+            http_url_ele = find_xpath_attr(
+                qnode, './html_urls/video_url', 'format', 'video/mp4')
+            http_url = http_url_ele.text if http_url_ele is not None else None
             if http_url:
                 formats.append({
                     'url': http_url,
@@ -64,8 +66,10 @@ class NozIE(InfoExtractor):
                     formats.extend(self._extract_f4m_formats(
                         update_url_query(f4m_url, {'hdcore': '3.4.0'}),
                         video_id, f4m_id='hds', fatal=False))
-                m3u8_url = xpath_text(
-                    qnode, './html_urls/video_url[@format="application/vnd.apple.mpegurl"]')
+                m3u8_url_ele = find_xpath_attr(
+                    qnode, './html_urls/video_url',
+                    'format', 'application/vnd.apple.mpegurl')
+                m3u8_url = m3u8_url_ele.text if m3u8_url_ele is not None else None
                 if m3u8_url:
                     formats.extend(self._extract_m3u8_formats(
                         m3u8_url, video_id, 'mp4', 'm3u8_native',

From 64d4f31d78d9555d7f79b8bebeabb7535a842090 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 15 Mar 2016 22:50:43 +0100
Subject: [PATCH 0998/1105] [brightcove:new] update embed_in_page embeds regex
 to match non numeric ref id

---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 86de5b9ee..b182edb26 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -425,7 +425,7 @@ class BrightcoveNewIE(InfoExtractor):
                 # According to [4] data-video-id may be prefixed with ref:
                 r'''(?sx)
                     <video[^>]+
-                        data-video-id=["\']((?:ref:)?\d+)["\'][^>]*>.*?
+                        data-video-id=["\'](\d+|ref:[^"\']+)["\'][^>]*>.*?
                     </video>.*?
                     <script[^>]+
                         src=["\'](?:https?:)?//players\.brightcove\.net/

From 2cab48704ced0d703f2b6de8575dc06e83207462 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 15 Mar 2016 22:57:59 +0100
Subject: [PATCH 0999/1105] [thestar] Add new extractor(closes #5955)

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/brightcove.py |  2 +-
 youtube_dl/extractor/thestar.py    | 31 ++++++++++++++++++++++++++++++
 3 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 youtube_dl/extractor/thestar.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 179c11ffa..bf9fa17c9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -737,6 +737,7 @@ from .theplatform import (
     ThePlatformFeedIE,
 )
 from .thesixtyone import TheSixtyOneIE
+from .thestar import TheStarIE
 from .thisamericanlife import ThisAmericanLifeIE
 from .thisav import ThisAVIE
 from .tinypic import TinyPicIE
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index b182edb26..f56b642ab 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -157,7 +157,7 @@ class BrightcoveLegacyIE(InfoExtractor):
         if playerKey is not None:
             params['playerKey'] = playerKey
         # The three fields hold the id of the video
-        videoPlayer = find_param('@videoPlayer') or find_param('videoId') or find_param('videoID')
+        videoPlayer = find_param('@videoPlayer') or find_param('videoId') or find_param('videoID') or find_param('@videoList')
         if videoPlayer is not None:
             params['@videoPlayer'] = videoPlayer
         linkBase = find_param('linkBaseURL')
diff --git a/youtube_dl/extractor/thestar.py b/youtube_dl/extractor/thestar.py
new file mode 100644
index 000000000..b7e9af2af
--- /dev/null
+++ b/youtube_dl/extractor/thestar.py
@@ -0,0 +1,31 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from .brightcove import BrightcoveLegacyIE
+from ..compat import compat_parse_qs
+
+
+class TheStarIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?thestar\.com/(?:[^/]+/)*(?P<id>.+)\.html'
+    _TEST = {
+        'url': 'http://www.thestar.com/life/2016/02/01/mankind-why-this-woman-started-a-men-s-skincare-line.html',
+        'md5': '2c62dd4db2027e35579fefb97a8b6554',
+        'info_dict': {
+            'id': '4732393888001',
+            'ext': 'mp4',
+            'title': 'Mankind: Why this woman started a men\'s skin care line',
+            'description': 'Robert Cribb talks to Young Lee, the founder of Uncle Peter\'s MAN.',
+            'uploader_id': '794267642001',
+            'timestamp': 1454353482,
+            'upload_date': '20160201',
+        }
+    }
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/794267642001/default_default/index.html?videoId=%s'
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        brightcove_legacy_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
+        brightcove_id = compat_parse_qs(brightcove_legacy_url)['@videoPlayer'][0]
+        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)

From 48254c3f2cb315c4b9d2b679a6126f1e1208fbf8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 16 Mar 2016 09:14:37 +0100
Subject: [PATCH 1000/1105] [brightcove] some improvements and fixes

- use FFmpeg downloader to download m3u8 formats extracted
from BrightcoveNew(some of the m3u8 media playlists use AES-128)
- update comment and update_url_query to handle url query
---
 youtube_dl/extractor/brightcove.py | 32 +++++++++++++-----------------
 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index f56b642ab..304fb89e3 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -24,16 +24,16 @@ from ..utils import (
     js_to_json,
     int_or_none,
     parse_iso8601,
-    sanitized_Request,
     unescapeHTML,
     unsmuggle_url,
+    update_url_query,
 )
 
 
 class BrightcoveLegacyIE(InfoExtractor):
     IE_NAME = 'brightcove:legacy'
     _VALID_URL = r'(?:https?://.*brightcove\.com/(services|viewer).*?\?|brightcove:)(?P<query>.*)'
-    _FEDERATED_URL_TEMPLATE = 'http://c.brightcove.com/services/viewer/htmlFederated?%s'
+    _FEDERATED_URL = 'http://c.brightcove.com/services/viewer/htmlFederated'
 
     _TESTS = [
         {
@@ -156,7 +156,7 @@ class BrightcoveLegacyIE(InfoExtractor):
         # Not all pages define this value
         if playerKey is not None:
             params['playerKey'] = playerKey
-        # The three fields hold the id of the video
+        # These fields hold the id of the video
         videoPlayer = find_param('@videoPlayer') or find_param('videoId') or find_param('videoID') or find_param('@videoList')
         if videoPlayer is not None:
             params['@videoPlayer'] = videoPlayer
@@ -185,8 +185,7 @@ class BrightcoveLegacyIE(InfoExtractor):
 
     @classmethod
     def _make_brightcove_url(cls, params):
-        data = compat_urllib_parse.urlencode(params)
-        return cls._FEDERATED_URL_TEMPLATE % data
+        return update_url_query(cls._FEDERATED_URL, params)
 
     @classmethod
     def _extract_brightcove_url(cls, webpage):
@@ -240,7 +239,7 @@ class BrightcoveLegacyIE(InfoExtractor):
             # We set the original url as the default 'Referer' header
             referer = smuggled_data.get('Referer', url)
             return self._get_video_info(
-                videoPlayer[0], query_str, query, referer=referer)
+                videoPlayer[0], query, referer=referer)
         elif 'playerKey' in query:
             player_key = query['playerKey']
             return self._get_playlist_info(player_key[0])
@@ -249,15 +248,14 @@ class BrightcoveLegacyIE(InfoExtractor):
                 'Cannot find playerKey= variable. Did you forget quotes in a shell invocation?',
                 expected=True)
 
-    def _get_video_info(self, video_id, query_str, query, referer=None):
-        request_url = self._FEDERATED_URL_TEMPLATE % query_str
-        req = sanitized_Request(request_url)
+    def _get_video_info(self, video_id, query, referer=None):
+        headers = {}
         linkBase = query.get('linkBaseURL')
         if linkBase is not None:
             referer = linkBase[0]
         if referer is not None:
-            req.add_header('Referer', referer)
-        webpage = self._download_webpage(req, video_id)
+            headers['Referer'] = referer
+        webpage = self._download_webpage(self._FEDERATED_URL, video_id, headers=headers, query=query)
 
         error_msg = self._html_search_regex(
             r"<h1>We're sorry.</h1>([\s\n]*<p>.*?</p>)+", webpage,
@@ -459,12 +457,11 @@ class BrightcoveNewIE(InfoExtractor):
                 r'policyKey\s*:\s*(["\'])(?P<pk>.+?)\1',
                 webpage, 'policy key', group='pk')
 
-        req = sanitized_Request(
-            'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s'
-            % (account_id, video_id),
-            headers={'Accept': 'application/json;pk=%s' % policy_key})
+        api_url = 'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s' % (account_id, video_id)
         try:
-            json_data = self._download_json(req, video_id)
+            json_data = self._download_json(api_url, video_id, headers={
+                'Accept': 'application/json;pk=%s' % policy_key
+            })
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
                 json_data = self._parse_json(e.cause.read().decode(), video_id)
@@ -482,8 +479,7 @@ class BrightcoveNewIE(InfoExtractor):
                 if not src:
                     continue
                 formats.extend(self._extract_m3u8_formats(
-                    src, video_id, 'mp4', entry_protocol='m3u8_native',
-                    m3u8_id='hls', fatal=False))
+                    src, video_id, 'mp4', m3u8_id='hls', fatal=False))
             elif source_type == 'application/dash+xml':
                 if not src:
                     continue

From 23edc49509052e06afe7032802a0f4deb6710b47 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 16 Mar 2016 10:47:39 +0100
Subject: [PATCH 1001/1105] [tv3] Add new extractor(closes #8059)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/tv3.py      | 33 ++++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+)
 create mode 100644 youtube_dl/extractor/tv3.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bf9fa17c9..9502d07a4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -784,6 +784,7 @@ from .tv2 import (
     TV2IE,
     TV2ArticleIE,
 )
+from .tv3 import TV3IE
 from .tv4 import TV4IE
 from .tvc import (
     TVCIE,
diff --git a/youtube_dl/extractor/tv3.py b/youtube_dl/extractor/tv3.py
new file mode 100644
index 000000000..d3f690dc7
--- /dev/null
+++ b/youtube_dl/extractor/tv3.py
@@ -0,0 +1,33 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class TV3IE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?tv3\.co\.nz/(?P<id>[^/]+)/tabid/\d+/articleID/\d+/MCat/\d+/Default\.aspx'
+    _TEST = {
+        'url': 'http://www.tv3.co.nz/MOTORSPORT-SRS-SsangYong-Hampton-Downs-Round-3/tabid/3692/articleID/121615/MCat/2915/Default.aspx',
+        'info_dict': {
+            'id': '4659127992001',
+            'ext': 'mp4',
+            'title': 'CRC Motorsport: SRS SsangYong Hampton Downs Round 3 - S2015 Ep3',
+            'description': 'SsangYong Racing Series returns for Round 3 with drivers from New Zealand and Australia taking to the grid at Hampton Downs raceway.',
+            'uploader_id': '3812193411001',
+            'upload_date': '20151213',
+            'timestamp': 1449975272,
+        },
+        'expected_warnings': [
+            'Failed to download MPD manifest'
+        ],
+        'params': {
+            'skip_download': True,
+        },
+    }
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/3812193411001/default_default/index.html?videoId=%s'
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        brightcove_id = self._search_regex(r'<param\s*name="@videoPlayer"\s*value="(\d+)"', webpage, 'brightcove id')
+        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)

From 354dbbd8808dc5e835c7042f84c175eb56e0bcfc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 16 Mar 2016 11:46:53 +0100
Subject: [PATCH 1002/1105] [brightcove:new] extract protocol-less embed
 URLs(closes #2914)

---
 youtube_dl/extractor/brightcove.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 304fb89e3..3ab383461 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -413,8 +413,8 @@ class BrightcoveNewIE(InfoExtractor):
 
         # Look for iframe embeds [1]
         for _, url in re.findall(
-                r'<iframe[^>]+src=(["\'])((?:https?:)//players\.brightcove\.net/\d+/[^/]+/index\.html.+?)\1', webpage):
-            entries.append(url)
+                r'<iframe[^>]+src=(["\'])((?:https?:)?//players\.brightcove\.net/\d+/[^/]+/index\.html.+?)\1', webpage):
+            entries.append(url if url.startswith('http') else 'http:' + url)
 
         # Look for embed_in_page embeds [2]
         for video_id, account_id, player_id, embed in re.findall(

From a7ba57dc176efaa50b5121a1f63963f4fc0111e7 Mon Sep 17 00:00:00 2001
From: Quan Hua <quanhua92@gmail.com>
Date: Wed, 16 Mar 2016 10:15:39 +0700
Subject: [PATCH 1003/1105] [udemy] Update course id regex to cover v4 layout
 (Closes #8753, closes #8868, closes #8870)

---
 youtube_dl/extractor/udemy.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index f5b5e7fd6..0fd2a0a0a 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -144,7 +144,8 @@ class UdemyIE(InfoExtractor):
         webpage = self._download_webpage(url, lecture_id)
 
         course_id = self._search_regex(
-            r'data-course-id=["\'](\d+)', webpage, 'course id')
+            (r'data-course-id=["\'](\d+)', r'&quot;id&quot;: (\d+)'),
+            webpage, 'course id')
 
         try:
             lecture = self._download_lecture(course_id, lecture_id)

From 70cab344c48598904fde657620156be62b70ee0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Mar 2016 21:46:09 +0600
Subject: [PATCH 1004/1105] [udemy] Improve course id v4 regex

---
 youtube_dl/extractor/udemy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 0fd2a0a0a..74cc36ece 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -144,7 +144,7 @@ class UdemyIE(InfoExtractor):
         webpage = self._download_webpage(url, lecture_id)
 
         course_id = self._search_regex(
-            (r'data-course-id=["\'](\d+)', r'&quot;id&quot;: (\d+)'),
+            (r'data-course-id=["\'](\d+)', r'&quot;id&quot;\s*:\s*(\d+)'),
             webpage, 'course id')
 
         try:

From 96f4f796fb02e3ef13fa6584b8f77ebafaabf59f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Mar 2016 21:47:51 +0600
Subject: [PATCH 1005/1105] [brightcover] Remove unused import

---
 youtube_dl/extractor/brightcove.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 3ab383461..59e8008f9 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -9,7 +9,6 @@ from ..compat import (
     compat_etree_fromstring,
     compat_parse_qs,
     compat_str,
-    compat_urllib_parse,
     compat_urllib_parse_urlparse,
     compat_urlparse,
     compat_xml_parse_error,

From c5229f3926d64bce101d328fc5acf25bda83e0d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Mar 2016 21:50:04 +0600
Subject: [PATCH 1006/1105] [utils] PEP 8

---
 test/test_utils.py  | 6 +++---
 youtube_dl/utils.py | 5 ++++-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 5a0109977..9a3a8ddff 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -641,8 +641,8 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(extract_attributes('<e x="&#x79;">'), {'x': 'y'})
         self.assertEqual(extract_attributes('<e x="&amp;">'), {'x': '&'})  # XML
         self.assertEqual(extract_attributes('<e x="&quot;">'), {'x': '"'})
-        self.assertEqual(extract_attributes('<e x="&pound;">'), {'x': '£'}) # HTML 3.2
-        self.assertEqual(extract_attributes('<e x="&lambda;">'), {'x': 'λ'}) # HTML 4.0
+        self.assertEqual(extract_attributes('<e x="&pound;">'), {'x': '£'})  # HTML 3.2
+        self.assertEqual(extract_attributes('<e x="&lambda;">'), {'x': 'λ'})  # HTML 4.0
         self.assertEqual(extract_attributes('<e x="&foo">'), {'x': '&foo'})
         self.assertEqual(extract_attributes('<e x="\'">'), {'x': "'"})
         self.assertEqual(extract_attributes('<e x=\'"\'>'), {'x': '"'})
@@ -654,7 +654,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(extract_attributes('<e \nx=\n"y"\n>'), {'x': 'y'})
         self.assertEqual(extract_attributes("<e \nx=\n'y'\n>"), {'x': 'y'})
         self.assertEqual(extract_attributes('<e \nx="\ny\n">'), {'x': '\ny\n'})
-        self.assertEqual(extract_attributes('<e CAPS=x>'), {'caps': 'x'}) # Names lowercased
+        self.assertEqual(extract_attributes('<e CAPS=x>'), {'caps': 'x'})  # Names lowercased
         self.assertEqual(extract_attributes('<e x=1 X=2>'), {'x': '2'})
         self.assertEqual(extract_attributes('<e X=1 x=2>'), {'x': '2'})
         self.assertEqual(extract_attributes('<e _:funny-name1=1>'), {'_:funny-name1': '1'})
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index ec186918c..8ec1bd469 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -273,15 +273,17 @@ def get_element_by_attribute(attribute, value, html):
 
     return unescapeHTML(res)
 
+
 class HTMLAttributeParser(compat_HTMLParser):
     """Trivial HTML parser to gather the attributes for a single element"""
     def __init__(self):
-        self.attrs = { }
+        self.attrs = {}
         compat_HTMLParser.__init__(self)
 
     def handle_starttag(self, tag, attrs):
         self.attrs = dict(attrs)
 
+
 def extract_attributes(html_element):
     """Given a string for an HTML element such as
     <el
@@ -303,6 +305,7 @@ def extract_attributes(html_element):
     parser.close()
     return parser.attrs
 
+
 def clean_html(html):
     """Clean an HTML snippet into a readable string"""
 

From 63f41d3821f28c7565618e725e6c1af618aa1602 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 16 Mar 2016 21:26:25 +0100
Subject: [PATCH 1007/1105] [bravotv] Add new extractor(#4657)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/bravotv.py  | 28 ++++++++++++++++++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100644 youtube_dl/extractor/bravotv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9502d07a4..725ebec04 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -81,6 +81,7 @@ from .bloomberg import BloombergIE
 from .bokecc import BokeCCIE
 from .bpb import BpbIE
 from .br import BRIE
+from .bravotv import BravoTVIE
 from .breakcom import BreakIE
 from .brightcove import (
     BrightcoveLegacyIE,
diff --git a/youtube_dl/extractor/bravotv.py b/youtube_dl/extractor/bravotv.py
new file mode 100644
index 000000000..69d00b466
--- /dev/null
+++ b/youtube_dl/extractor/bravotv.py
@@ -0,0 +1,28 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import smuggle_url
+
+
+class BravoTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?bravotv\.com/(?:[^/]+/)+videos/(?P<id>[^/?]+)'
+    _TEST = {
+        'url': 'http://www.bravotv.com/last-chance-kitchen/season-5/videos/lck-ep-12-fishy-finale',
+        'md5': 'd60cdf68904e854fac669bd26cccf801',
+        'info_dict': {
+            'id': 'LitrBdX64qLn',
+            'ext': 'mp4',
+            'title': 'Last Chance Kitchen Returns',
+            'description': 'S13: Last Chance Kitchen Returns for Top Chef Season 13',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        account_pid = self._search_regex(r'"account_pid"\s*:\s*"([^"]+)"', webpage, 'account pid')
+        release_pid = self._search_regex(r'"release_pid"\s*:\s*"([^"]+)"', webpage, 'release pid')
+        return self.url_result(smuggle_url(
+            'http://link.theplatform.com/s/%s/%s?format=SMIL&mbr=true&switch=progressive' % (account_pid, release_pid),
+            {'force_smil_url': True}), 'ThePlatform', release_pid)

From a646a8cf980a946cfc15d2286fcec6ee3987886f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 17 Mar 2016 02:02:18 +0100
Subject: [PATCH 1008/1105] [sbs] improve extraction(fixes #3811)

- extract error messages
- force the platform smil url(previously the manifest param
in the query is not respected which make theplatform return non working
mp4 files for some videos)
---
 youtube_dl/extractor/sbs.py | 31 +++++++++++++++++++++----------
 1 file changed, 21 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py
index d6ee2d9e2..2f96477ca 100644
--- a/youtube_dl/extractor/sbs.py
+++ b/youtube_dl/extractor/sbs.py
@@ -2,6 +2,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import (
+    smuggle_url,
+    ExtractorError,
+)
 
 
 class SBSIE(InfoExtractor):
@@ -31,21 +35,28 @@ class SBSIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        player_params = self._download_json(
+            'http://www.sbs.com.au/api/video_pdkvars/id/%s?form=json' % video_id, video_id)
 
-        webpage = self._download_webpage(
-            'http://www.sbs.com.au/ondemand/video/single/%s?context=web' % video_id, video_id)
-
-        player_params = self._parse_json(
-            self._search_regex(
-                r'(?s)var\s+playerParams\s*=\s*({.+?});', webpage, 'playerParams'),
-            video_id)
+        error = player_params.get('error')
+        if error:
+            error_message = 'Sorry, The video you are looking for does not exist.'
+            video_data = error.get('results') or {}
+            error_code = error.get('errorCode')
+            if error_code == 'ComingSoon':
+                error_message = '%s is not yet available.' % video_data.get('title', '')
+            elif error_code in ('Forbidden', 'intranetAccessOnly'):
+                error_message = 'Sorry, This video cannot be accessed via this website'
+            elif error_code == 'Expired':
+                error_message = 'Sorry, %s is no longer available.' % video_data.get('title', '')
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, error_message), expected=True)
 
         urls = player_params['releaseUrls']
-        theplatform_url = (urls.get('progressive') or urls.get('standard') or
-                           urls.get('html') or player_params['relatedItemsURL'])
+        theplatform_url = (urls.get('progressive') or urls.get('html') or
+                           urls.get('standard') or player_params['relatedItemsURL'])
 
         return {
             '_type': 'url_transparent',
             'id': video_id,
-            'url': theplatform_url,
+            'url': smuggle_url(theplatform_url, {'force_smil_url': True}),
         }

From 11f12195af73a2b0a09de928247cb87aed6dd693 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 17 Mar 2016 19:25:37 +0800
Subject: [PATCH 1009/1105] [youtube] Added itag 91

Seen in https://www.youtube.com/watch?v=jMN4cxyhJjk
---
 youtube_dl/extractor/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 27e67feb4..466f5da2e 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -309,6 +309,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 
         # Apple HTTP Live Streaming
+        '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},

From 0436ec0e7a4683539bc7844511ba76fbcab03f7b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 17 Mar 2016 16:05:31 +0100
Subject: [PATCH 1010/1105] [once] Add new format extractor

---
 youtube_dl/extractor/once.py | 37 ++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100644 youtube_dl/extractor/once.py

diff --git a/youtube_dl/extractor/once.py b/youtube_dl/extractor/once.py
new file mode 100644
index 000000000..403f8c0af
--- /dev/null
+++ b/youtube_dl/extractor/once.py
@@ -0,0 +1,37 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class OnceIE(InfoExtractor):
+    _VALID_URL = r'https?://once\.unicornmedia\.com/now/[^/]+/[^/]+/(?P<domain_id>[^/]+)/(?P<application_id>[^/]+)/(?:[^/]+/)?(?P<media_item_id>[^/]+)/content\.(?:once|m3u8|mp4)'
+    ADAPTIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/master/playlist/%s/%s/%s/content.m3u8'
+    PROGRESSIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/media/progressive/%s/%s/%s/%s/content.mp4'
+
+    def _extract_once_formats(self, url):
+        domain_id, application_id, media_item_id = re.match(
+            OnceIE._VALID_URL, url).groups()
+        adaptive_formats = self._extract_m3u8_formats(
+            self.ADAPTIVE_URL_TEMPLATE % (
+                domain_id, application_id, media_item_id),
+            media_item_id, 'mp4', m3u8_id='hls', fatal=False)
+        formats = []
+        formats.extend(adaptive_formats)
+        for adaptive_format in adaptive_formats:
+            rendition_id = self._search_regex(
+                r'/now/media/playlist/[^/]+/[^/]+/([^/]+)',
+                adaptive_format['url'], 'redition id', default=None)
+            if rendition_id:
+                progressive_format = adaptive_format.copy()
+                progressive_format.update({
+                    'url': self.PROGRESSIVE_URL_TEMPLATE % (
+                        domain_id, application_id, rendition_id, media_item_id),
+                    'format_id': adaptive_format['format_id'].replace(
+                        'hls', 'http'),
+                    'protocol': 'http',
+                })
+                formats.append(progressive_format)
+        return formats

From 9f02ff537c6ddfd3f1ea3586f3e44f0ec07a2aea Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 17 Mar 2016 16:06:25 +0100
Subject: [PATCH 1011/1105] [theplatform] extract brightcove once formats

---
 youtube_dl/extractor/theplatform.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 9a57b49df..ffe7c57ad 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -8,13 +8,12 @@ import binascii
 import hashlib
 
 
-from .common import InfoExtractor
+from .once import OnceIE
 from ..compat import (
     compat_parse_qs,
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
-    determine_ext,
     ExtractorError,
     float_or_none,
     int_or_none,
@@ -29,7 +28,7 @@ default_ns = 'http://www.w3.org/2005/SMIL21/Language'
 _x = lambda p: xpath_with_ns(p, {'smil': default_ns})
 
 
-class ThePlatformBaseIE(InfoExtractor):
+class ThePlatformBaseIE(OnceIE):
     def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL data'):
         meta = self._download_xml(smil_url, video_id, note=note)
         error_element = find_xpath_attr(
@@ -38,17 +37,19 @@ class ThePlatformBaseIE(InfoExtractor):
         if error_element is not None:
             raise ExtractorError(error_element.attrib['abstract'], expected=True)
 
-        formats = self._parse_smil_formats(
+        smil_formats = self._parse_smil_formats(
             meta, smil_url, video_id, namespace=default_ns,
             # the parameters are from syfy.com, other sites may use others,
             # they also work for nbc.com
             f4m_params={'g': 'UXWGVKRWHFSP', 'hdcore': '3.0.3'},
             transform_rtmp_url=lambda streamer, src: (streamer, 'mp4:' + src))
 
-        for _format in formats:
-            ext = determine_ext(_format['url'])
-            if ext == 'once':
-                _format['ext'] = 'mp4'
+        formats = []
+        for _format in smil_formats:
+            if OnceIE.suitable(_format['url']):
+                formats.extend(self._extract_once_formats(_format['url']))
+            else:
+                formats.append(_format)
 
         self._sort_formats(formats)
 
@@ -125,7 +126,7 @@ class ThePlatformIE(ThePlatformBaseIE):
         'only_matching': True,
     }, {
         'url': 'http://player.theplatform.com/p/2E2eJC/nbcNewsOffsite?guid=tdy_or_siri_150701',
-        'md5': '734f3790fb5fc4903da391beeebc4836',
+        'md5': 'fb96bb3d85118930a5b055783a3bd992',
         'info_dict': {
             'id': 'tdy_or_siri_150701',
             'ext': 'mp4',
@@ -135,7 +136,6 @@ class ThePlatformIE(ThePlatformBaseIE):
             'thumbnail': 're:^https?://.*\.jpg$',
             'timestamp': 1435752600,
             'upload_date': '20150701',
-            'categories': ['Today/Shows/Orange Room', 'Today/Sections/Money', 'Today/Topics/Tech', "Today/Topics/Editor's picks"],
         },
     }, {
         # From http://www.nbc.com/the-blacklist/video/sir-crispin-crandall/2928790?onid=137781#vc137781=1
@@ -250,7 +250,7 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
     _TEST = {
         # From http://player.theplatform.com/p/7wvmTC/MSNBCEmbeddedOffSite?guid=n_hardball_5biden_140207
         'url': 'http://feed.theplatform.com/f/7wvmTC/msnbc_video-p-test?form=json&pretty=true&range=-40&byGuid=n_hardball_5biden_140207',
-        'md5': '22d2b84f058d3586efcd99e57d59d314',
+        'md5': '6e32495b5073ab414471b615c5ded394',
         'info_dict': {
             'id': 'n_hardball_5biden_140207',
             'ext': 'mp4',

From 574b2a7393ef389792d5010704e505ef0eaaa5e8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 17 Mar 2016 16:07:36 +0100
Subject: [PATCH 1012/1105] [nbc:nbcnews] improve extraction(fixes #6922)

- extract more metadata and formats
- relax regex
---
 youtube_dl/extractor/nbc.py | 137 ++++++++++++++++++++++++------------
 1 file changed, 91 insertions(+), 46 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 2202cfa33..bb0817e34 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -3,13 +3,16 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from .theplatform import ThePlatformIE
 from ..utils import (
-    ExtractorError,
     find_xpath_attr,
     lowercase_escape,
     smuggle_url,
     unescapeHTML,
+    update_url_query,
+    int_or_none,
+    HEADRequest,
+    parse_iso8601,
 )
 
 
@@ -131,10 +134,10 @@ class NBCSportsIE(InfoExtractor):
             NBCSportsVPlayerIE._extract_url(webpage), 'NBCSportsVPlayer')
 
 
-class NBCNewsIE(InfoExtractor):
+class NBCNewsIE(ThePlatformIE):
     _VALID_URL = r'''(?x)https?://(?:www\.)?nbcnews\.com/
         (?:video/.+?/(?P<id>\d+)|
-        (?:watch|feature|nightly-news)/[^/]+/(?P<title>.+))
+        ([^/]+/)*(?P<display_id>[^/?]+))
         '''
 
     _TESTS = [
@@ -149,15 +152,14 @@ class NBCNewsIE(InfoExtractor):
             },
         },
         {
-            'url': 'http://www.nbcnews.com/feature/edward-snowden-interview/how-twitter-reacted-snowden-interview-n117236',
-            'md5': 'b2421750c9f260783721d898f4c42063',
+            'url': 'http://www.nbcnews.com/watch/nbcnews-com/how-twitter-reacted-to-the-snowden-interview-269389891880',
+            'md5': 'af1adfa51312291a017720403826bb64',
             'info_dict': {
-                'id': 'I1wpAI_zmhsQ',
+                'id': '269389891880',
                 'ext': 'mp4',
                 'title': 'How Twitter Reacted To The Snowden Interview',
                 'description': 'md5:65a0bd5d76fe114f3c2727aa3a81fe64',
             },
-            'add_ie': ['ThePlatform'],
         },
         {
             'url': 'http://www.nbcnews.com/feature/dateline-full-episodes/full-episode-family-business-n285156',
@@ -168,17 +170,29 @@ class NBCNewsIE(InfoExtractor):
                 'title': 'FULL EPISODE: Family Business',
                 'description': 'md5:757988edbaae9d7be1d585eb5d55cc04',
             },
+            'skip': 'This page is unavailable.',
         },
         {
             'url': 'http://www.nbcnews.com/nightly-news/video/nightly-news-with-brian-williams-full-broadcast-february-4-394064451844',
-            'md5': 'b5dda8cddd8650baa0dcb616dd2cf60d',
+            'md5': '73135a2e0ef819107bbb55a5a9b2a802',
             'info_dict': {
-                'id': 'sekXqyTVnmN3',
+                'id': '394064451844',
                 'ext': 'mp4',
                 'title': 'Nightly News with Brian Williams Full Broadcast (February 4)',
                 'description': 'md5:1c10c1eccbe84a26e5debb4381e2d3c5',
             },
         },
+        {
+            'url': 'http://www.nbcnews.com/business/autos/volkswagen-11-million-vehicles-could-have-suspect-software-emissions-scandal-n431456',
+            'md5': 'a49e173825e5fcd15c13fc297fced39d',
+            'info_dict': {
+                'id': '529953347624',
+                'ext': 'mp4',
+                'title': 'Volkswagen U.S. Chief: We \'Totally Screwed Up\'',
+                'description': 'md5:d22d1281a24f22ea0880741bb4dd6301',
+            },
+            'expected_warnings': ['http-6000 is not available']
+        },
         {
             'url': 'http://www.nbcnews.com/watch/dateline/full-episode--deadly-betrayal-386250819952',
             'only_matching': True,
@@ -202,49 +216,80 @@ class NBCNewsIE(InfoExtractor):
             }
         else:
             # "feature" and "nightly-news" pages use theplatform.com
-            title = mobj.group('title')
-            webpage = self._download_webpage(url, title)
+            display_id = mobj.group('display_id')
+            webpage = self._download_webpage(url, display_id)
+            info = None
             bootstrap_json = self._search_regex(
-                r'var\s+(?:bootstrapJson|playlistData)\s*=\s*({.+});?\s*$',
-                webpage, 'bootstrap json', flags=re.MULTILINE)
-            bootstrap = self._parse_json(bootstrap_json, video_id)
-            info = bootstrap['results'][0]['video']
-            mpxid = info['mpxId']
+                r'(?m)var\s+(?:bootstrapJson|playlistData)\s*=\s*({.+});?\s*$',
+                webpage, 'bootstrap json', default=None)
+            if bootstrap_json:
+                bootstrap = self._parse_json(bootstrap_json, display_id)
+                info = bootstrap['results'][0]['video']
+            else:
+                player_instance_json = self._search_regex(
+                    r'videoObj\s*:\s*({.+})', webpage, 'player instance')
+                info = self._parse_json(player_instance_json, display_id)
+            video_id = info['mpxId']
+            title = info['title']
 
-            base_urls = [
-                info['fallbackPlaylistUrl'],
-                info['associatedPlaylistUrl'],
-            ]
+            subtitles = {}
+            caption_links = info.get('captionLinks')
+            if caption_links:
+                for (sub_key, sub_ext) in (('smpte-tt', 'ttml'), ('web-vtt', 'vtt'), ('srt', 'srt')):
+                    sub_url = caption_links.get(sub_key)
+                    if sub_url:
+                        subtitles.setdefault('en', []).append({
+                            'url': sub_url,
+                            'ext': sub_ext,
+                        })
 
-            for base_url in base_urls:
-                if not base_url:
+            formats = []
+            for video_asset in info['videoAssets']:
+                video_url = video_asset.get('publicUrl')
+                if not video_url:
                     continue
-                playlist_url = base_url + '?form=MPXNBCNewsAPI'
-
-                try:
-                    all_videos = self._download_json(playlist_url, title)
-                except ExtractorError as ee:
-                    if isinstance(ee.cause, compat_HTTPError):
-                        continue
-                    raise
-
-                if not all_videos or 'videos' not in all_videos:
+                container = video_asset.get('format')
+                asset_type = video_asset.get('assetType') or ''
+                if container == 'ISM' or asset_type == 'FireTV-Once':
                     continue
-
-                try:
-                    info = next(v for v in all_videos['videos'] if v['mpxId'] == mpxid)
-                    break
-                except StopIteration:
-                    continue
-
-            if info is None:
-                raise ExtractorError('Could not find video in playlists')
+                elif asset_type == 'OnceURL':
+                    tp_formats, tp_subtitles = self._extract_theplatform_smil(
+                        video_url, video_id)
+                    formats.extend(tp_formats)
+                    subtitles = self._merge_subtitles(subtitles, tp_subtitles)
+                else:
+                    tbr = int_or_none(video_asset.get('bitRate'), 1000)
+                    format_id = 'http%s' % ('-%d' % tbr if tbr else '')
+                    video_url = update_url_query(
+                        video_url, {'format': 'redirect'})
+                    # resolve the url so that we can check availability and detect the correct extension
+                    head = self._request_webpage(
+                        HEADRequest(video_url), video_id,
+                        'Checking %s url' % format_id,
+                        '%s is not available' % format_id,
+                        fatal=False)
+                    if head:
+                        video_url = head.geturl()
+                        formats.append({
+                            'format_id': format_id,
+                            'url': video_url,
+                            'width': int_or_none(video_asset.get('width')),
+                            'height': int_or_none(video_asset.get('height')),
+                            'tbr': tbr,
+                            'container': video_asset.get('format'),
+                        })
+            self._sort_formats(formats)
 
             return {
-                '_type': 'url',
-                # We get the best quality video
-                'url': info['videoAssets'][-1]['publicUrl'],
-                'ie_key': 'ThePlatform',
+                'id': video_id,
+                'title': title,
+                'description': info.get('description'),
+                'thumbnail': info.get('description'),
+                'thumbnail': info.get('thumbnail'),
+                'duration': int_or_none(info.get('duration')),
+                'timestamp': parse_iso8601(info.get('pubDate')),
+                'formats': formats,
+                'subtitles': subtitles,
             }
 
 

From cf45ed786e580999afe864724c3b7d16abadb4e1 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 17 Mar 2016 17:48:17 +0100
Subject: [PATCH 1013/1105] [wistia] extract more metadata

---
 youtube_dl/extractor/wistia.py | 42 ++++++++++++++++++++++------------
 1 file changed, 27 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/wistia.py b/youtube_dl/extractor/wistia.py
index 41061dd31..8b14840a2 100644
--- a/youtube_dl/extractor/wistia.py
+++ b/youtube_dl/extractor/wistia.py
@@ -4,6 +4,7 @@ from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     sanitized_Request,
+    int_or_none,
 )
 
 
@@ -18,6 +19,9 @@ class WistiaIE(InfoExtractor):
             'id': 'sh7fpupwlt',
             'ext': 'mov',
             'title': 'Being Resourceful',
+            'description': 'a Clients From Hell Video Series video from worldwidewebhosting',
+            'upload_date': '20131204',
+            'timestamp': 1386185018,
             'duration': 117,
         },
     }
@@ -32,35 +36,43 @@ class WistiaIE(InfoExtractor):
             raise ExtractorError('Error while getting the playlist',
                                  expected=True)
         data = data_json['media']
+        title = data['name']
 
         formats = []
         thumbnails = []
         for a in data['assets']:
+            astatus = a.get('status')
             atype = a.get('type')
-            if atype == 'still':
+            if (astatus is not None and astatus != 2) or atype == 'preview':
+                continue
+            elif atype in ('still', 'still_image'):
                 thumbnails.append({
                     'url': a['url'],
                     'resolution': '%dx%d' % (a['width'], a['height']),
                 })
-                continue
-            if atype == 'preview':
-                continue
-            formats.append({
-                'format_id': atype,
-                'url': a['url'],
-                'width': a['width'],
-                'height': a['height'],
-                'filesize': a['size'],
-                'ext': a['ext'],
-                'preference': 1 if atype == 'original' else None,
-            })
+            else:
+                formats.append({
+                    'format_id': atype,
+                    'url': a['url'],
+                    'tbr': int_or_none(a.get('bitrate')),
+                    'vbr': int_or_none(a.get('opt_vbitrate')),
+                    'width': int_or_none(a.get('width')),
+                    'height': int_or_none(a.get('height')),
+                    'filesize': int_or_none(a.get('size')),
+                    'vcodec': a.get('codec'),
+                    'container': a.get('container'),
+                    'ext': a.get('ext'),
+                    'preference': 1 if atype == 'original' else None,
+                })
 
         self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': data['name'],
+            'title': title,
+            'description': data.get('seoDescription'),
             'formats': formats,
             'thumbnails': thumbnails,
-            'duration': data.get('duration'),
+            'duration': int_or_none(data.get('duration')),
+            'timestamp': int_or_none(data.get('createdAt')),
         }

From cc162f6a0aa63a3e050c55cec9da728aa2cb9100 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Mar 2016 22:55:04 +0600
Subject: [PATCH 1014/1105] [crunchyroll] Fix custom _download_webpage (Closes
 #8883)

---
 youtube_dl/extractor/crunchyroll.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index c7032ffa2..85fa7a725 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -54,7 +54,7 @@ class CrunchyrollBaseIE(InfoExtractor):
     def _real_initialize(self):
         self._login()
 
-    def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None):
+    def _download_webpage(self, url_or_request, *args, **kwargs):
         request = (url_or_request if isinstance(url_or_request, compat_urllib_request.Request)
                    else sanitized_Request(url_or_request))
         # Accept-Language must be set explicitly to accept any language to avoid issues
@@ -65,8 +65,7 @@ class CrunchyrollBaseIE(InfoExtractor):
         # Crunchyroll to not work in georestriction cases in some browsers that don't place
         # the locale lang first in header. However allowing any language seems to workaround the issue.
         request.add_header('Accept-Language', '*')
-        return super(CrunchyrollBaseIE, self)._download_webpage(
-            request, video_id, note, errnote, fatal, tries, timeout, encoding)
+        return super(CrunchyrollBaseIE, self)._download_webpage(request, *args, **kwargs)
 
     @staticmethod
     def _add_skip_wall(url):

From 8c97e7efb6ab273f0b7c91f0aa9ac6869c911bf1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Mar 2016 23:43:14 +0600
Subject: [PATCH 1015/1105] [animeondemand] Expand episode title regex (Closes
 #8875)

---
 youtube_dl/extractor/animeondemand.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index a7d8daf7b..3dbbe2a62 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -93,7 +93,7 @@ class AnimeOnDemandIE(InfoExtractor):
 
         for episode_html in re.findall(r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', webpage):
             m = re.search(
-                r'class="episodebox-title"[^>]+title="Episode (?P<number>\d+) - (?P<title>.+?)"', episode_html)
+                r'class="episodebox-title"[^>]+title="(?:Episode|Film)\s*(?P<number>\d+)\s*-\s*(?P<title>.+?)"', episode_html)
             if not m:
                 continue
 

From b57fecfdddc78b5ef5cfd1c3302f7b79ab1bf64f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Mar 2016 23:50:10 +0600
Subject: [PATCH 1016/1105] [animeondemand] Add test

---
 youtube_dl/extractor/animeondemand.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 3dbbe2a62..6cb3a84f9 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -18,7 +18,7 @@ class AnimeOnDemandIE(InfoExtractor):
     _LOGIN_URL = 'https://www.anime-on-demand.de/users/sign_in'
     _APPLY_HTML5_URL = 'https://www.anime-on-demand.de/html5apply'
     _NETRC_MACHINE = 'animeondemand'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://www.anime-on-demand.de/anime/161',
         'info_dict': {
             'id': '161',
@@ -26,7 +26,11 @@ class AnimeOnDemandIE(InfoExtractor):
             'description': 'md5:6681ce3c07c7189d255ac6ab23812d31',
         },
         'playlist_mincount': 4,
-    }
+    }, {
+        # Film wording is used instead of Episode
+        'url': 'https://www.anime-on-demand.de/anime/39',
+        'only_matching': True,
+    }]
 
     def _login(self):
         (username, password) = self._get_login_info()

From 85e8f26b827e77cbed6a83268787d450ab2bea3b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 00:02:34 +0600
Subject: [PATCH 1017/1105] [animeondemand] Improve extraction

---
 youtube_dl/extractor/animeondemand.py | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 6cb3a84f9..3631c2451 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -30,6 +30,10 @@ class AnimeOnDemandIE(InfoExtractor):
         # Film wording is used instead of Episode
         'url': 'https://www.anime-on-demand.de/anime/39',
         'only_matching': True,
+    }, {
+        # Episodes without titles
+        'url': 'https://www.anime-on-demand.de/anime/162',
+        'only_matching': True,
     }]
 
     def _login(self):
@@ -95,14 +99,22 @@ class AnimeOnDemandIE(InfoExtractor):
 
         entries = []
 
-        for episode_html in re.findall(r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', webpage):
-            m = re.search(
-                r'class="episodebox-title"[^>]+title="(?:Episode|Film)\s*(?P<number>\d+)\s*-\s*(?P<title>.+?)"', episode_html)
-            if not m:
+        for num, episode_html in enumerate(re.findall(
+                r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', webpage)):
+            episodebox_title = self._search_regex(
+                (r'class="episodebox-title"[^>]+title="(.+?)"',
+                 r'class="episodebox-title"[^>]+>(.+?)<'),
+                webpage, 'episodebox title', default=None)
+            if not episodebox_title:
                 continue
 
-            episode_number = int(m.group('number'))
-            episode_title = m.group('title')
+            episode_number = int(self._search_regex(
+                r'^(?:Episode|Film)\s*(\d+)',
+                episodebox_title, 'episode number', default=num))
+            episode_title = self._search_regex(
+                r'(?:Episode|Film)\s*\d+\s*-\s*(?P<title>.+?)',
+                episodebox_title, 'episode title', default=None)
+
             video_id = 'episode-%d' % episode_number
 
             common_info = {

From 0d0e282912a7ade43a148518c742557c310a41a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 00:12:34 +0600
Subject: [PATCH 1018/1105] [animeondemand] Fix typo and improve

---
 youtube_dl/extractor/animeondemand.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 3631c2451..0158407f6 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -100,19 +100,19 @@ class AnimeOnDemandIE(InfoExtractor):
         entries = []
 
         for num, episode_html in enumerate(re.findall(
-                r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', webpage)):
+                r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', webpage), 1):
             episodebox_title = self._search_regex(
-                (r'class="episodebox-title"[^>]+title="(.+?)"',
-                 r'class="episodebox-title"[^>]+>(.+?)<'),
-                webpage, 'episodebox title', default=None)
+                (r'class="episodebox-title"[^>]+title=(["\'])(?P<title>.+?)\1',
+                 r'class="episodebox-title"[^>]+>(?P<title>.+?)<'),
+                episode_html, 'episodebox title', default=None, group='title')
             if not episodebox_title:
                 continue
 
             episode_number = int(self._search_regex(
-                r'^(?:Episode|Film)\s*(\d+)',
+                r'(?:Episode|Film)\s*(\d+)',
                 episodebox_title, 'episode number', default=num))
             episode_title = self._search_regex(
-                r'(?:Episode|Film)\s*\d+\s*-\s*(?P<title>.+?)',
+                r'(?:Episode|Film)\s*\d+\s*-\s*(.+)',
                 episodebox_title, 'episode title', default=None)
 
             video_id = 'episode-%d' % episode_number

From 57f7e3c62df187457a057be88fca43136f4c507f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 02:51:38 +0600
Subject: [PATCH 1019/1105] [compat] Add compat_xpath

---
 youtube_dl/compat.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 74702786a..dbb91a6ef 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -256,6 +256,16 @@ else:
                 el.text = el.text.decode('utf-8')
         return doc
 
+if sys.version_info < (2, 7):
+    # Here comes the crazy part: In 2.6, if the xpath is a unicode,
+    # .//node does not match if a node is a direct child of . !
+    def compat_xpath(xpath):
+        if isinstance(xpath, compat_str):
+            xpath = xpath.encode('ascii')
+        return xpath
+else:
+    compat_xpath = lambda xpath: xpath
+
 try:
     from urllib.parse import parse_qs as compat_parse_qs
 except ImportError:  # Python 2
@@ -585,6 +595,7 @@ __all__ = [
     'compat_urlparse',
     'compat_urlretrieve',
     'compat_xml_parse_error',
+    'compat_xpath',
     'shlex_quote',
     'subprocess_check_output',
     'workaround_optparse_bug9161',

From 810c10baa1e0177a6a0ef39496f7e972db02d806 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 02:52:23 +0600
Subject: [PATCH 1020/1105] [utils] Use compat_xpath

---
 youtube_dl/utils.py | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 8ec1bd469..ef6e7c7cb 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -50,6 +50,7 @@ from .compat import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urlparse,
+    compat_xpath,
     shlex_quote,
 )
 
@@ -165,12 +166,7 @@ if sys.version_info >= (2, 7):
         return node.find(expr)
 else:
     def find_xpath_attr(node, xpath, key, val=None):
-        # Here comes the crazy part: In 2.6, if the xpath is a unicode,
-        # .//node does not match if a node is a direct child of . !
-        if isinstance(xpath, compat_str):
-            xpath = xpath.encode('ascii')
-
-        for f in node.findall(xpath):
+        for f in node.findall(compat_xpath(xpath)):
             if key not in f.attrib:
                 continue
             if val is None or f.attrib.get(key) == val:
@@ -195,9 +191,7 @@ def xpath_with_ns(path, ns_map):
 
 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
     def _find_xpath(xpath):
-        if sys.version_info < (2, 7):  # Crazy 2.6
-            xpath = xpath.encode('ascii')
-        return node.find(xpath)
+        return node.find(compat_xpath(xpath))
 
     if isinstance(xpath, (str, compat_str)):
         n = _find_xpath(xpath)

From e3d17b3c07c6d8bc7fd45af1e45523e8fde5fb58 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 02:54:27 +0600
Subject: [PATCH 1021/1105] [noz] Fix extraction on python 2.6 by means of
 using compat_xpath

---
 youtube_dl/extractor/noz.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/noz.py b/youtube_dl/extractor/noz.py
index 656443c49..c47a33d15 100644
--- a/youtube_dl/extractor/noz.py
+++ b/youtube_dl/extractor/noz.py
@@ -2,7 +2,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_unquote
+from ..compat import (
+    compat_urllib_parse_unquote,
+    compat_xpath,
+)
 from ..utils import (
     int_or_none,
     find_xpath_attr,
@@ -47,7 +50,7 @@ class NozIE(InfoExtractor):
         duration = int_or_none(xpath_text(
             doc, './/article/movie/file/duration'))
         formats = []
-        for qnode in doc.findall('.//article/movie/file/qualities/qual'):
+        for qnode in doc.findall(compat_xpath('.//article/movie/file/qualities/qual')):
             http_url_ele = find_xpath_attr(
                 qnode, './html_urls/video_url', 'format', 'video/mp4')
             http_url = http_url_ele.text if http_url_ele is not None else None

From 4c92fd2e835cde89866d3dfb1fc05d23196b19db Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 18 Mar 2016 09:21:21 +0100
Subject: [PATCH 1022/1105] [theplatform] always force theplatform to return a
 smil for _extract_theplatform_smil

---
 youtube_dl/extractor/bravotv.py            |  2 +-
 youtube_dl/extractor/cbsnews.py            |  2 +-
 youtube_dl/extractor/cnet.py               |  2 +-
 youtube_dl/extractor/nationalgeographic.py |  2 +-
 youtube_dl/extractor/theplatform.py        | 10 ++++++----
 5 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/bravotv.py b/youtube_dl/extractor/bravotv.py
index 69d00b466..34d451f38 100644
--- a/youtube_dl/extractor/bravotv.py
+++ b/youtube_dl/extractor/bravotv.py
@@ -24,5 +24,5 @@ class BravoTVIE(InfoExtractor):
         account_pid = self._search_regex(r'"account_pid"\s*:\s*"([^"]+)"', webpage, 'account pid')
         release_pid = self._search_regex(r'"release_pid"\s*:\s*"([^"]+)"', webpage, 'release pid')
         return self.url_result(smuggle_url(
-            'http://link.theplatform.com/s/%s/%s?format=SMIL&mbr=true&switch=progressive' % (account_pid, release_pid),
+            'http://link.theplatform.com/s/%s/%s?mbr=true&switch=progressive' % (account_pid, release_pid),
             {'force_smil_url': True}), 'ThePlatform', release_pid)
diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 7319ee1b7..8ddcc5097 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -78,7 +78,7 @@ class CBSNewsIE(ThePlatformIE):
             pid = item.get('media' + format_id)
             if not pid:
                 continue
-            release_url = 'http://link.theplatform.com/s/dJ5BDC/%s?format=SMIL&mbr=true' % pid
+            release_url = 'http://link.theplatform.com/s/dJ5BDC/%s?mbr=true' % pid
             tp_formats, tp_subtitles = self._extract_theplatform_smil(release_url, video_id, 'Downloading %s SMIL data' % pid)
             formats.extend(tp_formats)
             subtitles = self._merge_subtitles(subtitles, tp_subtitles)
diff --git a/youtube_dl/extractor/cnet.py b/youtube_dl/extractor/cnet.py
index 3cf0bf95b..c154b3e19 100644
--- a/youtube_dl/extractor/cnet.py
+++ b/youtube_dl/extractor/cnet.py
@@ -60,7 +60,7 @@ class CNETIE(ThePlatformIE):
         for (fkey, vid) in vdata['files'].items():
             if fkey == 'hls_phone' and 'hls_tablet' in vdata['files']:
                 continue
-            release_url = 'http://link.theplatform.com/s/kYEXFC/%s?format=SMIL&mbr=true' % vid
+            release_url = 'http://link.theplatform.com/s/kYEXFC/%s?mbr=true' % vid
             if fkey == 'hds':
                 release_url += '&manifest=f4m'
             tp_formats, tp_subtitles = self._extract_theplatform_smil(release_url, video_id, 'Downloading %s SMIL data' % fkey)
diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index 6fc9e7b05..7ce8d9b18 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -48,7 +48,7 @@ class NationalGeographicIE(InfoExtractor):
         theplatform_id = url_basename(content.attrib.get('url'))
 
         return self.url_result(smuggle_url(
-            'http://link.theplatform.com/s/ngs/%s?format=SMIL&formats=MPEG4&manifest=f4m' % theplatform_id,
+            'http://link.theplatform.com/s/ngs/%s?formats=MPEG4&manifest=f4m' % theplatform_id,
             # For some reason, the normal links don't work and we must force
             # the use of f4m
             {'force_smil_url': True}))
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index ffe7c57ad..a148f78ce 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -22,6 +22,7 @@ from ..utils import (
     xpath_with_ns,
     mimetype2ext,
     find_xpath_attr,
+    update_url_query,
 )
 
 default_ns = 'http://www.w3.org/2005/SMIL21/Language'
@@ -30,6 +31,7 @@ _x = lambda p: xpath_with_ns(p, {'smil': default_ns})
 
 class ThePlatformBaseIE(OnceIE):
     def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL data'):
+        smil_url = update_url_query(smil_url, {'format': 'SMIL'})
         meta = self._download_xml(smil_url, video_id, note=note)
         error_element = find_xpath_attr(
             meta, _x('.//smil:ref'), 'src',
@@ -213,7 +215,7 @@ class ThePlatformIE(ThePlatformBaseIE):
                 webpage, 'smil url', group='url')
             path = self._search_regex(
                 r'link\.theplatform\.com/s/((?:[^/?#&]+/)+[^/?#&]+)', smil_url, 'path')
-            smil_url += '?' if '?' not in smil_url else '&' + 'formats=m3u,mpeg4&format=SMIL'
+            smil_url += '?' if '?' not in smil_url else '&' + 'formats=m3u,mpeg4'
         elif mobj.group('config'):
             config_url = url + '&form=json'
             config_url = config_url.replace('swf/', 'config/')
@@ -223,9 +225,9 @@ class ThePlatformIE(ThePlatformBaseIE):
                 release_url = config['releaseUrl']
             else:
                 release_url = 'http://link.theplatform.com/s/%s?mbr=true' % path
-            smil_url = release_url + '&format=SMIL&formats=MPEG4&manifest=f4m'
+            smil_url = release_url + '&formats=MPEG4&manifest=f4m'
         else:
-            smil_url = 'http://link.theplatform.com/s/%s/meta.smil?format=smil&mbr=true' % path
+            smil_url = 'http://link.theplatform.com/s/%s?mbr=true' % path
 
         sig = smuggled_data.get('sig')
         if sig:
@@ -280,7 +282,7 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
         first_video_id = None
         duration = None
         for item in entry['media$content']:
-            smil_url = item['plfile$url'] + '&format=SMIL&mbr=true'
+            smil_url = item['plfile$url'] + '&mbr=true'
             cur_video_id = ThePlatformIE._match_id(smil_url)
             if first_video_id is None:
                 first_video_id = cur_video_id

From 87c03c6bd22e99d6410c907128ab872e79df1560 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 18 Mar 2016 09:43:28 +0100
Subject: [PATCH 1023/1105] [theplatform] remove unnecessary import

---
 youtube_dl/extractor/theplatform.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index a148f78ce..2230dfe02 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -22,7 +22,6 @@ from ..utils import (
     xpath_with_ns,
     mimetype2ext,
     find_xpath_attr,
-    update_url_query,
 )
 
 default_ns = 'http://www.w3.org/2005/SMIL21/Language'
@@ -31,8 +30,7 @@ _x = lambda p: xpath_with_ns(p, {'smil': default_ns})
 
 class ThePlatformBaseIE(OnceIE):
     def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL data'):
-        smil_url = update_url_query(smil_url, {'format': 'SMIL'})
-        meta = self._download_xml(smil_url, video_id, note=note)
+        meta = self._download_xml(smil_url, video_id, note=note, query={'format': 'SMIL'})
         error_element = find_xpath_attr(
             meta, _x('.//smil:ref'), 'src',
             'http://link.theplatform.com/s/errorFiles/Unavailable.mp4')

From 0d33166ec586b9c75e20835adca927e923cb36e3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 18 Mar 2016 11:43:48 +0100
Subject: [PATCH 1024/1105] release 2016.03.18

---
 docs/supportedsites.md | 4 ++++
 youtube_dl/version.py  | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a6dcc2576..3415efc45 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -81,6 +81,7 @@
  - **BokeCC**
  - **Bpb**: Bundeszentrale für politische Bildung
  - **BR**: Bayerischer Rundfunk Mediathek
+ - **BravoTV**
  - **Break**
  - **brightcove:legacy**
  - **brightcove:new**
@@ -499,6 +500,7 @@
  - **Restudy**
  - **ReverbNation**
  - **Revision3**
+ - **RICE**
  - **RingTV**
  - **RottenTomatoes**
  - **Roxwel**
@@ -617,6 +619,7 @@
  - **ThePlatform**
  - **ThePlatformFeed**
  - **TheSixtyOne**
+ - **TheStar**
  - **ThisAmericanLife**
  - **ThisAV**
  - **THVideo**
@@ -650,6 +653,7 @@
  - **tv.dfb.de**
  - **TV2**
  - **TV2Article**
+ - **TV3**
  - **TV4**: tv4.se and tv4play.se
  - **TVC**
  - **TVCArticle**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9216fa547..6b2c5fac9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.03.14'
+__version__ = '2016.03.18'

From 61870915323abd126f5440282b1fd5734ee1ce6f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 18 Mar 2016 11:50:04 +0100
Subject: [PATCH 1025/1105] [once] check http formats availability

---
 youtube_dl/extractor/once.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/once.py b/youtube_dl/extractor/once.py
index 403f8c0af..080045d4c 100644
--- a/youtube_dl/extractor/once.py
+++ b/youtube_dl/extractor/once.py
@@ -14,13 +14,12 @@ class OnceIE(InfoExtractor):
     def _extract_once_formats(self, url):
         domain_id, application_id, media_item_id = re.match(
             OnceIE._VALID_URL, url).groups()
-        adaptive_formats = self._extract_m3u8_formats(
+        formats = self._extract_m3u8_formats(
             self.ADAPTIVE_URL_TEMPLATE % (
                 domain_id, application_id, media_item_id),
             media_item_id, 'mp4', m3u8_id='hls', fatal=False)
-        formats = []
-        formats.extend(adaptive_formats)
-        for adaptive_format in adaptive_formats:
+        progressive_formats = []
+        for adaptive_format in formats:
             rendition_id = self._search_regex(
                 r'/now/media/playlist/[^/]+/[^/]+/([^/]+)',
                 adaptive_format['url'], 'redition id', default=None)
@@ -33,5 +32,7 @@ class OnceIE(InfoExtractor):
                         'hls', 'http'),
                     'protocol': 'http',
                 })
-                formats.append(progressive_format)
+                progressive_formats.append(progressive_format)
+        self._check_formats(progressive_formats, media_item_id)
+        formats.extend(progressive_formats)
         return formats

From cae21032ab38f404a9959e6b28984b960e579fb5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 21:08:25 +0600
Subject: [PATCH 1026/1105] [theplatform] Improve geo restriction detection

---
 youtube_dl/extractor/theplatform.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 2230dfe02..863914299 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -31,10 +31,9 @@ _x = lambda p: xpath_with_ns(p, {'smil': default_ns})
 class ThePlatformBaseIE(OnceIE):
     def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL data'):
         meta = self._download_xml(smil_url, video_id, note=note, query={'format': 'SMIL'})
-        error_element = find_xpath_attr(
-            meta, _x('.//smil:ref'), 'src',
-            'http://link.theplatform.com/s/errorFiles/Unavailable.mp4')
-        if error_element is not None:
+        error_element = find_xpath_attr(meta, _x('.//smil:ref'), 'src')
+        if error_element is not None and error_element.attrib['src'].startswith(
+                'http://link.theplatform.com/s/errorFiles/Unavailable.'):
             raise ExtractorError(error_element.attrib['abstract'], expected=True)
 
         smil_formats = self._parse_smil_formats(

From 263eff9537c73caa9bff42b1e675043eaa124f9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 21:50:10 +0600
Subject: [PATCH 1027/1105] [extractor/generic] Properly extract format id from
 Content-Type

Fixes extraction for cases like: audio/x-mpegURL; charset=utf-8
---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8121f04a5..b75db1252 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1249,7 +1249,7 @@ class GenericIE(InfoExtractor):
 
         # Check for direct link to a video
         content_type = head_response.headers.get('Content-Type', '')
-        m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>.+)$', content_type)
+        m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>[^;\s]+)', content_type)
         if m:
             upload_date = unified_strdate(
                 head_response.headers.get('Last-Modified'))

From 955737b2d40c0ce947c13659a27aae0c41077c65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 21:50:44 +0600
Subject: [PATCH 1028/1105] [extractor/generic] Force Content-Type to lowecase

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b75db1252..cce7799e2 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1248,7 +1248,7 @@ class GenericIE(InfoExtractor):
         }
 
         # Check for direct link to a video
-        content_type = head_response.headers.get('Content-Type', '')
+        content_type = head_response.headers.get('Content-Type', '').lower()
         m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>[^;\s]+)', content_type)
         if m:
             upload_date = unified_strdate(

From 20938f768b16c945c6041ba3c0a7ae1a4e790881 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 21:54:33 +0600
Subject: [PATCH 1029/1105] [extractor/generic] Add another test for generic
 m3u8

---
 youtube_dl/extractor/generic.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index cce7799e2..62b51e84e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -239,6 +239,20 @@ class GenericIE(InfoExtractor):
                 'format': 'bestvideo',
             },
         },
+        # m3u8 served with Content-Type: audio/x-mpegURL; charset=utf-8
+        {
+            'url': 'http://once.unicornmedia.com/now/master/playlist/bb0b18ba-64f5-4b1b-a29f-0ac252f06b68/77a785f3-5188-4806-b788-0893a61634ed/93677179-2d99-4ef4-9e17-fe70d49abfbf/content.m3u8',
+            'info_dict': {
+                'id': 'content',
+                'ext': 'mp4',
+                'title': 'content',
+                'formats': 'mincount:8',
+            },
+            'params': {
+                # m3u8 downloads
+                'skip_download': True,
+            }
+        },
         # google redirect
         {
             'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',

From 303dcdb99505b29ef4c499cc395ab9ec90c07ec1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 22:41:16 +0600
Subject: [PATCH 1030/1105] [extractor/generic] Simplify upload_date extraction

---
 youtube_dl/extractor/generic.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 62b51e84e..a2e7ba5ad 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1259,14 +1259,13 @@ class GenericIE(InfoExtractor):
         info_dict = {
             'id': video_id,
             'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
+            'upload_date': unified_strdate(head_response.headers.get('Last-Modified'))
         }
 
         # Check for direct link to a video
         content_type = head_response.headers.get('Content-Type', '').lower()
         m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>[^;\s]+)', content_type)
         if m:
-            upload_date = unified_strdate(
-                head_response.headers.get('Last-Modified'))
             format_id = m.group('format_id')
             if format_id.endswith('mpegurl'):
                 formats = self._extract_m3u8_formats(url, video_id, 'mp4')
@@ -1281,7 +1280,6 @@ class GenericIE(InfoExtractor):
             info_dict.update({
                 'direct': True,
                 'formats': formats,
-                'upload_date': upload_date,
             })
             return info_dict
 
@@ -1309,12 +1307,9 @@ class GenericIE(InfoExtractor):
         if not is_html(first_bytes):
             self._downloader.report_warning(
                 'URL could be a direct video link, returning it as such.')
-            upload_date = unified_strdate(
-                head_response.headers.get('Last-Modified'))
             info_dict.update({
                 'direct': True,
                 'url': url,
-                'upload_date': upload_date,
             })
             return info_dict
 

From de6c51e88eb61b49a95ccfcfa82547c2172eb52b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 22:43:07 +0600
Subject: [PATCH 1031/1105] [extractor/generic] Fix direct link semantics

---
 youtube_dl/extractor/generic.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a2e7ba5ad..5649e26da 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1277,10 +1277,8 @@ class GenericIE(InfoExtractor):
                     'url': url,
                     'vcodec': 'none' if m.group('type') == 'audio' else None
                 }]
-            info_dict.update({
-                'direct': True,
-                'formats': formats,
-            })
+                info_dict['direct'] = True
+            info_dict['formats'] = formats
             return info_dict
 
         if not self._downloader.params.get('test', False) and not is_intentional:

From 5940862d5a75ae45a640e0ce3104dd18c9864e26 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 22:45:28 +0600
Subject: [PATCH 1032/1105] [extractor/generic] Detect m3u playlists served
 without proper Content-Type

---
 youtube_dl/extractor/generic.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 5649e26da..24d43a247 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1299,9 +1299,15 @@ class GenericIE(InfoExtractor):
             request.add_header('Accept-Encoding', '*')
             full_response = self._request_webpage(request, video_id)
 
+        first_bytes = full_response.read(512)
+
+        # Is it an M3U playlist?
+        if first_bytes.startswith('#EXTM3U'):
+            info_dict['formats'] = self._extract_m3u8_formats(url, video_id, 'mp4')
+            return info_dict
+
         # Maybe it's a direct link to a video?
         # Be careful not to download the whole thing!
-        first_bytes = full_response.read(512)
         if not is_html(first_bytes):
             self._downloader.report_warning(
                 'URL could be a direct video link, returning it as such.')

From edd9b71c2cca7e5a0df8799710d9ad410ec77d29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 22:49:11 +0600
Subject: [PATCH 1033/1105] [extractor/generic] Add a test for m3u playlist
 served without proper Content-Type

---
 youtube_dl/extractor/generic.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 24d43a247..f28a65d9b 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -253,6 +253,21 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             }
         },
+        # m3u8 served with Content-Type: text/plain
+        {
+            'url': 'http://www.nacentapps.com/m3u8/index.m3u8',
+            'info_dict': {
+                'id': 'index',
+                'ext': 'mp4',
+                'title': 'index',
+                'upload_date': '20140720',
+                'formats': 'mincount:11',
+            },
+            'params': {
+                # m3u8 downloads
+                'skip_download': True,
+            }
+        },
         # google redirect
         {
             'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',

From 19e2617a6fb614a84340757dacb2ea918c097a84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 23:42:15 +0600
Subject: [PATCH 1034/1105] [commonprotocols] Add generic support for rtmp URLs
 (Closes #8488)

---
 youtube_dl/extractor/__init__.py        |  1 +
 youtube_dl/extractor/commonprotocols.py | 36 +++++++++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 youtube_dl/extractor/commonprotocols.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 725ebec04..acc0b03bd 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -136,6 +136,7 @@ from .collegerama import CollegeRamaIE
 from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
 from .comcarcoff import ComCarCoffIE
 from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
+from .commonprotocols import RtmpIE
 from .condenast import CondeNastIE
 from .cracked import CrackedIE
 from .crackle import CrackleIE
diff --git a/youtube_dl/extractor/commonprotocols.py b/youtube_dl/extractor/commonprotocols.py
new file mode 100644
index 000000000..5d130a170
--- /dev/null
+++ b/youtube_dl/extractor/commonprotocols.py
@@ -0,0 +1,36 @@
+from __future__ import unicode_literals
+
+import os
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse_unquote,
+    compat_urlparse,
+)
+from ..utils import url_basename
+
+
+class RtmpIE(InfoExtractor):
+    IE_DESC = False  # Do not list
+    _VALID_URL = r'(?i)rtmp[est]?://.+'
+
+    _TESTS = [{
+        'url': 'rtmp://cp44293.edgefcs.net/ondemand?auth=daEcTdydfdqcsb8cZcDbAaCbhamacbbawaS-bw7dBb-bWG-GqpGFqCpNCnGoyL&aifp=v001&slist=public/unsecure/audio/2c97899446428e4301471a8cb72b4b97--audio--pmg-20110908-0900a_flv_aac_med_int.mp4',
+        'only_matching': True,
+    }, {
+        'url': 'rtmp://edge.live.hitbox.tv/live/dimak',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = compat_urllib_parse_unquote(os.path.splitext(url.rstrip('/').split('/')[-1])[0])
+        title = compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0])
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': [{
+                'url': url,
+                'ext': 'flv',
+                'format_id': compat_urlparse.urlparse(url).scheme,
+            }],
+        }

From d5aacf9a90e0855976401b6085ac56b66ca09d12 Mon Sep 17 00:00:00 2001
From: John Peel <john@dgby.org>
Date: Fri, 18 Mar 2016 00:33:03 -0700
Subject: [PATCH 1035/1105] Added format_id to the filers on -f.

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8c651cd52..93b6ca54d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -905,7 +905,7 @@ class YoutubeDL(object):
                 '*=': lambda attr, value: value in attr,
             }
             str_operator_rex = re.compile(r'''(?x)
-                \s*(?P<key>ext|acodec|vcodec|container|protocol)
+                \s*(?P<key>ext|acodec|vcodec|container|protocol|format_id)
                 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
                 \s*(?P<value>[a-zA-Z0-9._-]+)
                 \s*$

From 4c3b16d5d1bf4806693d2895928ac1b03585b2b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Mar 2016 00:04:26 +0600
Subject: [PATCH 1036/1105] [test_YoutubeDL] Add test for format_id format
 selection

---
 test/test_YoutubeDL.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index efbee3b71..ca25025e2 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -222,6 +222,11 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['format_id'], 'dash-video-low')
 
+        ydl = YDL({'format': 'bestvideo[format_id^=dash][format_id$=low]'})
+        ydl.process_ie_result(info_dict.copy())
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'dash-video-low')
+
         formats = [
             {'format_id': 'vid-vcodec-dot', 'ext': 'mp4', 'preference': 1, 'vcodec': 'avc1.123456', 'acodec': 'none', 'url': TEST_URL},
         ]

From 09fc33198a4cfc93a98ce1ba7d51d41c487e5f56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 18 Mar 2016 19:18:55 +0100
Subject: [PATCH 1037/1105] utils: lookup_unit_table: Use a stricter regex

In parse_count multiple units start with the same letter, so it would match different units depending on the order they were sorted when iterating over them.
---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index ef6e7c7cb..bad1c4ea8 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1346,7 +1346,7 @@ def format_bytes(bytes):
 def lookup_unit_table(unit_table, s):
     units_re = '|'.join(re.escape(u) for u in unit_table)
     m = re.match(
-        r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
+        r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)$' % units_re, s)
     if not m:
         return None
     num_str = m.group('num').replace(',', '.')

From 4cd70099ea79a4a82b26694937ca46d31f7436ca Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 18 Mar 2016 21:17:45 +0100
Subject: [PATCH 1038/1105] [hbo] Add new extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/hbo.py      | 122 +++++++++++++++++++++++++++++++
 2 files changed, 123 insertions(+)
 create mode 100644 youtube_dl/extractor/hbo.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index acc0b03bd..529051a93 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -284,6 +284,7 @@ from .goshgay import GoshgayIE
 from .gputechconf import GPUTechConfIE
 from .groupon import GrouponIE
 from .hark import HarkIE
+from .hbo import HBOIE
 from .hearthisat import HearThisAtIE
 from .heise import HeiseIE
 from .hellporno import HellPornoIE
diff --git a/youtube_dl/extractor/hbo.py b/youtube_dl/extractor/hbo.py
new file mode 100644
index 000000000..dad0f3994
--- /dev/null
+++ b/youtube_dl/extractor/hbo.py
@@ -0,0 +1,122 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    xpath_text,
+    xpath_element,
+    int_or_none,
+    parse_duration,
+)
+
+
+class HBOIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?hbo\.com/video/video\.html\?.*vid=(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.hbo.com/video/video.html?autoplay=true&g=u&vid=1437839',
+        'md5': '1c33253f0c7782142c993c0ba62a8753',
+        'info_dict': {
+            'id': '1437839',
+            'ext': 'mp4',
+            'title': 'Ep. 64 Clip: Encryption',
+        }
+    }
+    _FORMATS_INFO = {
+        '1920': {
+            'width': 1280,
+            'height': 720,
+        },
+        '640': {
+            'width': 768,
+            'height': 432,
+        },
+        'highwifi': {
+            'width': 640,
+            'height': 360,
+        },
+        'high3g': {
+            'width': 640,
+            'height': 360,
+        },
+        'medwifi': {
+            'width': 400,
+            'height': 224,
+        },
+        'med3g': {
+            'width': 400,
+            'height': 224,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_data = self._download_xml(
+            'http://render.lv3.hbo.com/data/content/global/videos/data/%s.xml' % video_id, video_id)
+        title = xpath_text(video_data, 'title', 'title', True)
+
+        formats = []
+        for source in xpath_element(video_data, 'videos', 'sources', True):
+            if source.tag == 'size':
+                path = xpath_text(source, './/path')
+                if not path:
+                    continue
+                width = source.attrib.get('width')
+                format_info = self._FORMATS_INFO.get(width, {})
+                height = format_info.get('height')
+                fmt = {
+                    'url': path,
+                    'format_id': 'http%s' % ('-%dp' % height if height else ''),
+                    'width': format_info.get('width'),
+                    'height': height,
+                }
+                rtmp = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>.+))/(?P<playpath>mp4:.+)$', path)
+                if rtmp:
+                    fmt.update({
+                        'url': rtmp.group('url'),
+                        'play_path': rtmp.group('playpath'),
+                        'app': rtmp.group('app'),
+                        'ext': 'flv',
+                        'format_id': fmt['format_id'].replace('http', 'rtmp'),
+                    })
+                formats.append(fmt)
+            else:
+                video_url = source.text
+                if not video_url:
+                    continue
+                if source.tag == 'tarball':
+                    formats.extend(self._extract_m3u8_formats(
+                        video_url.replace('.tar', '/base_index_w8.m3u8'),
+                        video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+                else:
+                    format_info = self._FORMATS_INFO.get(source.tag, {})
+                    formats.append({
+                        'format_id': 'http-%s' % source.tag,
+                        'url': video_url,
+                        'width': format_info.get('width'),
+                        'height': format_info.get('height'),
+                    })
+        self._sort_formats(formats, ('width', 'height', 'tbr', 'format_id'))
+
+        thumbnails = []
+        card_sizes = xpath_element(video_data, 'titleCardSizes')
+        if card_sizes is not None:
+            for size in card_sizes:
+                path = xpath_text(size, 'path')
+                if not path:
+                    continue
+                width = int_or_none(size.get('width'))
+                thumbnails.append({
+                    'id': width,
+                    'url': path,
+                    'width': width,
+                })
+
+        return {
+            'id': video_id,
+            'title': title,
+            'duration': parse_duration(xpath_element(video_data, 'duration/tv14')),
+            'formats': formats,
+            'thumbnails': thumbnails,
+        }

From 0d769bcb781b46a00ddf958d6ea945560f2d6cd5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Mar 2016 05:43:43 +0600
Subject: [PATCH 1039/1105] [extractor/generic] Fix missing byte literal prefix

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index f28a65d9b..26de27a7e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1317,7 +1317,7 @@ class GenericIE(InfoExtractor):
         first_bytes = full_response.read(512)
 
         # Is it an M3U playlist?
-        if first_bytes.startswith('#EXTM3U'):
+        if first_bytes.startswith(b'#EXTM3U'):
             info_dict['formats'] = self._extract_m3u8_formats(url, video_id, 'mp4')
             return info_dict
 

From 782b1b5bd1cdaaead6865dee5d300486e7dd8348 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 19 Mar 2016 11:42:35 +0100
Subject: [PATCH 1040/1105] [utils] lookup_unit_table: Match word boundary
 instead of end of string

---
 test/test_utils.py  | 2 ++
 youtube_dl/utils.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 9a3a8ddff..325b870cc 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -702,6 +702,8 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_count('1.000'), 1000)
         self.assertEqual(parse_count('1.1k'), 1100)
         self.assertEqual(parse_count('1.1kk'), 1100000)
+        self.assertEqual(parse_count('1.1kk '), 1100000)
+        self.assertEqual(parse_count('1.1kk views'), 1100000)
 
     def test_version_tuple(self):
         self.assertEqual(version_tuple('1'), (1,))
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index bad1c4ea8..067b8a184 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1346,7 +1346,7 @@ def format_bytes(bytes):
 def lookup_unit_table(unit_table, s):
     units_re = '|'.join(re.escape(u) for u in unit_table)
     m = re.match(
-        r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)$' % units_re, s)
+        r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)\b' % units_re, s)
     if not m:
         return None
     num_str = m.group('num').replace(',', '.')

From 52bb437e412726a37d585cf782c88bc8c8a042a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Mar 2016 20:40:36 +0600
Subject: [PATCH 1041/1105] [options] Add --fragment-retries option

---
 youtube_dl/__init__.py | 17 ++++++++++++-----
 youtube_dl/options.py  |  4 ++++
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 79b389840..737f6545d 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -144,14 +144,20 @@ def _real_main(argv=None):
         if numeric_limit is None:
             parser.error('invalid max_filesize specified')
         opts.max_filesize = numeric_limit
-    if opts.retries is not None:
-        if opts.retries in ('inf', 'infinite'):
-            opts_retries = float('inf')
+
+    def parse_retries(retries):
+        if retries in ('inf', 'infinite'):
+            parsed_retries = float('inf')
         else:
             try:
-                opts_retries = int(opts.retries)
+                parsed_retries = int(retries)
             except (TypeError, ValueError):
                 parser.error('invalid retry count specified')
+        return parsed_retries
+    if opts.retries is not None:
+        opts.retries = parse_retries(opts.retries)
+    if opts.fragment_retries is not None:
+        opts.fragment_retries = parse_retries(opts.fragment_retries)
     if opts.buffersize is not None:
         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
         if numeric_buffersize is None:
@@ -299,7 +305,8 @@ def _real_main(argv=None):
         'force_generic_extractor': opts.force_generic_extractor,
         'ratelimit': opts.ratelimit,
         'nooverwrites': opts.nooverwrites,
-        'retries': opts_retries,
+        'retries': opts.retries,
+        'fragment_retries': opts.fragment_retries,
         'buffersize': opts.buffersize,
         'noresizebuffer': opts.noresizebuffer,
         'continuedl': opts.continue_dl,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 9dd7a8034..822728afc 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -399,6 +399,10 @@ def parseOpts(overrideArguments=None):
         '-R', '--retries',
         dest='retries', metavar='RETRIES', default=10,
         help='Number of retries (default is %default), or "infinite".')
+    downloader.add_option(
+        '--fragment-retries',
+        dest='fragment_retries', metavar='RETRIES', default=10,
+        help='Number of retries for a fragment (default is %default), or "infinite" (DASH only)')
     downloader.add_option(
         '--buffer-size',
         dest='buffersize', metavar='SIZE', default='1024',

From 721f26b8211a22648523f51c80f9b81d1eaa3b32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Mar 2016 20:41:24 +0600
Subject: [PATCH 1042/1105] [downloader/fragment] Add report_retry_fragment

---
 youtube_dl/downloader/fragment.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index a5bae9669..df66c35f0 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -21,6 +21,11 @@ class FragmentFD(FileDownloader):
     A base file downloader class for fragmented media (e.g. f4m/m3u8 manifests).
     """
 
+    def report_retry_fragment(self, fragment_name, count, retries):
+        self.to_screen(
+            '[download] Got server HTTP error. Retrying fragment %s (attempt %d of %.0f)...'
+            % (fragment_name, count, retries))
+
     def _prepare_and_start_frag_download(self, ctx):
         self._prepare_frag_download(ctx)
         self._start_frag_download(ctx)

From e33baba0dd6584475f75badec2186a7d86b88a5d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Mar 2016 20:42:23 +0600
Subject: [PATCH 1043/1105] [downloader/dash] Add fragment retry capability

YouTube may often return 404 HTTP error for a fragment causing the
whole download to fail. However if the same fragment is immediately
retried with the same request data this usually succeeds (1-2 attemps
is usually enough) thus allowing to download the whole file successfully.
So, we will retry all fragments that fail with 404 HTTP error for now.
---
 youtube_dl/downloader/dash.py | 42 ++++++++++++++++++++++++++---------
 1 file changed, 32 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 8b1b17c6e..8bbab9dbc 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -4,6 +4,7 @@ import os
 import re
 
 from .fragment import FragmentFD
+from ..compat import compat_urllib_error
 from ..utils import (
     sanitize_open,
     encodeFilename,
@@ -36,20 +37,41 @@ class DashSegmentsFD(FragmentFD):
 
         segments_filenames = []
 
-        def append_url_to_file(target_url, target_filename):
-            success = ctx['dl'].download(target_filename, {'url': combine_url(base_url, target_url)})
-            if not success:
+        fragment_retries = self.params.get('fragment_retries', 0)
+
+        def append_url_to_file(target_url, tmp_filename, segment_name):
+            target_filename = '%s-%s' % (tmp_filename, segment_name)
+            count = 0
+            while count <= fragment_retries:
+                try:
+                    success = ctx['dl'].download(target_filename, {'url': combine_url(base_url, target_url)})
+                    if not success:
+                        return False
+                    down, target_sanitized = sanitize_open(target_filename, 'rb')
+                    ctx['dest_stream'].write(down.read())
+                    down.close()
+                    segments_filenames.append(target_sanitized)
+                    break
+                except (compat_urllib_error.HTTPError, ) as err:
+                    # YouTube may often return 404 HTTP error for a fragment causing the
+                    # whole download to fail. However if the same fragment is immediately
+                    # retried with the same request data this usually succeeds (1-2 attemps
+                    # is usually enough) thus allowing to download the whole file successfully.
+                    # So, we will retry all fragments that fail with 404 HTTP error for now.
+                    if err.code != 404:
+                        raise
+                    # Retry fragment
+                    count += 1
+                    if count <= fragment_retries:
+                        self.report_retry_fragment(segment_name, count, fragment_retries)
+            if count > fragment_retries:
+                self.report_error('giving up after %s fragment retries' % fragment_retries)
                 return False
-            down, target_sanitized = sanitize_open(target_filename, 'rb')
-            ctx['dest_stream'].write(down.read())
-            down.close()
-            segments_filenames.append(target_sanitized)
 
         if initialization_url:
-            append_url_to_file(initialization_url, ctx['tmpfilename'] + '-Init')
+            append_url_to_file(initialization_url, ctx['tmpfilename'], 'Init')
         for i, segment_url in enumerate(segment_urls):
-            segment_filename = '%s-Seg%d' % (ctx['tmpfilename'], i)
-            append_url_to_file(segment_url, segment_filename)
+            append_url_to_file(segment_url, ctx['tmpfilename'], 'Seg%d' % i)
 
         self._finish_frag_download(ctx)
 

From 617e58d85063b68fb9736355e8354b05e82b1147 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Mar 2016 20:51:30 +0600
Subject: [PATCH 1044/1105] [downloader/{common,fragment}] Fix total retries
 reporting on python 2.6

---
 youtube_dl/downloader/common.py   | 8 +++++++-
 youtube_dl/downloader/fragment.py | 4 ++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index f39db58f6..1dba9f49a 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -115,6 +115,10 @@ class FileDownloader(object):
             return '%10s' % '---b/s'
         return '%10s' % ('%s/s' % format_bytes(speed))
 
+    @staticmethod
+    def format_retries(retries):
+        return 'inf' if retries == float('inf') else '%.0f' % retries
+
     @staticmethod
     def best_block_size(elapsed_time, bytes):
         new_min = max(bytes / 2.0, 1.0)
@@ -297,7 +301,9 @@ class FileDownloader(object):
 
     def report_retry(self, count, retries):
         """Report retry in case of HTTP error 5xx"""
-        self.to_screen('[download] Got server HTTP error. Retrying (attempt %d of %.0f)...' % (count, retries))
+        self.to_screen(
+            '[download] Got server HTTP error. Retrying (attempt %d of %s)...'
+            % (count, self.format_retries(retries)))
 
     def report_file_already_downloaded(self, file_name):
         """Report file has already been fully downloaded."""
diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index df66c35f0..c2671e6d2 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -23,8 +23,8 @@ class FragmentFD(FileDownloader):
 
     def report_retry_fragment(self, fragment_name, count, retries):
         self.to_screen(
-            '[download] Got server HTTP error. Retrying fragment %s (attempt %d of %.0f)...'
-            % (fragment_name, count, retries))
+            '[download] Got server HTTP error. Retrying fragment %s (attempt %d of %s)...'
+            % (fragment_name, count, self.format_retries(retries)))
 
     def _prepare_and_start_frag_download(self, ctx):
         self._prepare_frag_download(ctx)

From 16a8b7986b88572aea12c0f80c499e6e8085f1cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Mar 2016 20:54:21 +0600
Subject: [PATCH 1045/1105] [downloader/fragment] Document fragment_retries

---
 youtube_dl/downloader/fragment.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index c2671e6d2..ba903ae10 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -19,6 +19,10 @@ class HttpQuietDownloader(HttpFD):
 class FragmentFD(FileDownloader):
     """
     A base file downloader class for fragmented media (e.g. f4m/m3u8 manifests).
+
+    Available options:
+
+    fragment_retries:   Number of times to retry a fragment for HTTP error (DASH only)
     """
 
     def report_retry_fragment(self, fragment_name, count, retries):

From 3aec71766da38478740437c901514e666a39dbb0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Mar 2016 22:30:48 +0600
Subject: [PATCH 1046/1105] [safari:api] Separate extractor (Closes #8871)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/safari.py   | 52 +++++++++++++++++++-------------
 2 files changed, 32 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 529051a93..b3bc38916 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -628,6 +628,7 @@ from .ruutu import RuutuIE
 from .sandia import SandiaIE
 from .safari import (
     SafariIE,
+    SafariApiIE,
     SafariCourseIE,
 )
 from .sapo import SapoIE
diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 256396bb8..6ba91f202 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -75,16 +75,7 @@ class SafariBaseIE(InfoExtractor):
 class SafariIE(SafariBaseIE):
     IE_NAME = 'safari'
     IE_DESC = 'safaribooksonline.com online video'
-    _VALID_URL = r'''(?x)https?://
-                            (?:www\.)?safaribooksonline\.com/
-                                (?:
-                                    library/view/[^/]+|
-                                    api/v1/book
-                                )/
-                                (?P<course_id>[^/]+)/
-                                    (?:chapter(?:-content)?/)?
-                                (?P<part>part\d+)\.html
-    '''
+    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/library/view/[^/]+/(?P<course_id>[^/]+)/(?P<part>part\d+)\.html'
 
     _TESTS = [{
         'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/part00.html',
@@ -97,9 +88,6 @@ class SafariIE(SafariBaseIE):
             'upload_date': '20150724',
             'uploader_id': 'stork',
         },
-    }, {
-        'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
-        'only_matching': True,
     }, {
         # non-digits in course id
         'url': 'https://www.safaribooksonline.com/library/view/create-a-nodejs/100000006A0210/part00.html',
@@ -108,13 +96,18 @@ class SafariIE(SafariBaseIE):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        course_id = mobj.group('course_id')
-        part = mobj.group('part')
+        video_id = '%s/%s' % (mobj.group('course_id'), mobj.group('part'))
 
-        webpage = self._download_webpage(url, '%s/%s' % (course_id, part))
-        reference_id = self._search_regex(r'data-reference-id="([^"]+)"', webpage, 'kaltura reference id')
-        partner_id = self._search_regex(r'data-partner-id="([^"]+)"', webpage, 'kaltura widget id')
-        ui_id = self._search_regex(r'data-ui-id="([^"]+)"', webpage, 'kaltura uiconf id')
+        webpage = self._download_webpage(url, video_id)
+        reference_id = self._search_regex(
+            r'data-reference-id=(["\'])(?P<id>.+?)\1',
+            webpage, 'kaltura reference id', group='id')
+        partner_id = self._search_regex(
+            r'data-partner-id=(["\'])(?P<id>.+?)\1',
+            webpage, 'kaltura widget id', group='id')
+        ui_id = self._search_regex(
+            r'data-ui-id=(["\'])(?P<id>.+?)\1',
+            webpage, 'kaltura uiconf id', group='id')
 
         query = {
             'wid': '_%s' % partner_id,
@@ -125,7 +118,7 @@ class SafariIE(SafariBaseIE):
         if self.LOGGED_IN:
             kaltura_session = self._download_json(
                 '%s/player/kaltura_session/?reference_id=%s' % (self._API_BASE, reference_id),
-                course_id, 'Downloading kaltura session JSON',
+                video_id, 'Downloading kaltura session JSON',
                 'Unable to download kaltura session JSON', fatal=False)
             if kaltura_session:
                 session = kaltura_session.get('session')
@@ -137,6 +130,23 @@ class SafariIE(SafariBaseIE):
             'Kaltura')
 
 
+class SafariApiIE(SafariBaseIE):
+    IE_NAME = 'safari:api'
+    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/api/v1/book/(?P<course_id>[^/]+)/chapter(?:-content)?/(?P<part>part\d+)\.html'
+
+    _TEST = {
+        'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
+        'only_matching': True,
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        part = self._download_json(
+            url, '%s/%s' % (mobj.group('course_id'), mobj.group('part')),
+            'Downloading part JSON')
+        return self.url_result(part['web_url'], SafariIE.ie_key())
+
+
 class SafariCourseIE(SafariBaseIE):
     IE_NAME = 'safari:course'
     IE_DESC = 'safaribooksonline.com online courses'
@@ -168,7 +178,7 @@ class SafariCourseIE(SafariBaseIE):
                 'No chapters found for course %s' % course_id, expected=True)
 
         entries = [
-            self.url_result(chapter, 'Safari')
+            self.url_result(chapter, SafariApiIE.ie_key())
             for chapter in course_json['chapters']]
 
         course_title = course_json['title']

From 8b0d7a66ef5451556bb8ae5b085c7bef4c992f8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kacper=20Michaj=C5=82ow?= <kasper93@gmail.com>
Date: Wed, 9 Mar 2016 20:55:27 +0100
Subject: [PATCH 1047/1105] [cda] Add new extractor for cda.pl

Fixes #8760
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/cda.py      | 96 ++++++++++++++++++++++++++++++++
 2 files changed, 97 insertions(+)
 create mode 100755 youtube_dl/extractor/cda.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b3bc38916..5f5eca42b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -108,6 +108,7 @@ from .cbsnews import (
 )
 from .cbssports import CBSSportsIE
 from .ccc import CCCIE
+from .cda import CDAIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
 from .chaturbate import ChaturbateIE
diff --git a/youtube_dl/extractor/cda.py b/youtube_dl/extractor/cda.py
new file mode 100755
index 000000000..4c53b8dda
--- /dev/null
+++ b/youtube_dl/extractor/cda.py
@@ -0,0 +1,96 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    decode_packed_codes,
+    ExtractorError,
+    parse_duration
+)
+
+
+class CDAIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:(?:www|ebd)\.)?cda\.pl/(?:video|[0-9]+x[0-9]+)/(?P<id>[0-9a-z]+)'
+    _TESTS = [
+        {
+            'url': 'http://www.cda.pl/video/5749950c',
+            'md5': '6f844bf51b15f31fae165365707ae970',
+            'info_dict': {
+                'id': '5749950c',
+                'ext': 'mp4',
+                'height': 720,
+                'title': 'Oto dlaczego przed zakrętem należy zwolnić.',
+                'duration': 39
+            }
+        },
+        {
+            'url': 'http://www.cda.pl/video/57413289',
+            'md5': 'a88828770a8310fc00be6c95faf7f4d5',
+            'info_dict': {
+                'id': '57413289',
+                'ext': 'mp4',
+                'title': 'Lądowanie na lotnisku na Maderze',
+                'duration': 137
+            }
+        }
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage('http://ebd.cda.pl/0x0/' + video_id, video_id)
+
+        if 'Ten film jest dostępny dla użytkowników premium' in webpage:
+            raise ExtractorError('This video is only available for premium users.', expected=True)
+
+        title = self._html_search_regex(r'<title>(.+?)</title>', webpage, 'title', fatal=False)
+
+        def _get_format(page, version=''):
+            unpacked = decode_packed_codes(page)
+            duration = self._search_regex(r"duration:\\'(.+?)\\'", unpacked, 'duration', fatal=False)
+            format_id = None
+            height = None
+
+            m = re.search(r'<a data-quality="(?P<format_id>[^"]+)" href="[^"]+" class="quality-btn quality-btn-active">(?P<height>[0-9]+)p<\/a>', page)
+            if m:
+                format_id = m.group('format_id')
+                height = int(m.group('height'))
+
+            url = self._search_regex(r"url:\\'(.+?)\\'", unpacked, version + ' url', fatal=False)
+            if url is None:
+                return None
+
+            return {
+                'format_id': format_id,
+                'height': height,
+                'url': url
+            }, parse_duration(duration)
+
+        formats = []
+
+        format_desc, duration = _get_format(webpage) or (None, None)
+        if format_desc is not None:
+            formats.append(format_desc)
+
+        pattern = re.compile(r'<a data-quality="[^"]+" href="([^"]+)" class="quality-btn">([0-9]+p)<\/a>')
+        for version in re.findall(pattern, webpage):
+            webpage = self._download_webpage(version[0], video_id, 'Downloading %s version information' % version[1], fatal=False)
+            if not webpage:
+                # Manually report warning because empty page is returned when invalid version is requested.
+                self.report_warning('Unable to download %s version information' % version[1])
+                continue
+
+            format_desc, duration_ = _get_format(webpage, version[1]) or (None, None)
+            duration = duration or duration_
+            if format_desc is not None:
+                formats.append(format_desc)
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'duration': duration
+        }

From f1ced6df51e4d81523e9051cadb6e4f5ceac19f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Mar 2016 23:17:14 +0600
Subject: [PATCH 1048/1105] [cda] Improve and simplify (Closes #8805)

---
 youtube_dl/extractor/cda.py | 132 ++++++++++++++++++------------------
 1 file changed, 66 insertions(+), 66 deletions(-)

diff --git a/youtube_dl/extractor/cda.py b/youtube_dl/extractor/cda.py
index 4c53b8dda..498d2c0d8 100755
--- a/youtube_dl/extractor/cda.py
+++ b/youtube_dl/extractor/cda.py
@@ -12,30 +12,30 @@ from ..utils import (
 
 
 class CDAIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www|ebd)\.)?cda\.pl/(?:video|[0-9]+x[0-9]+)/(?P<id>[0-9a-z]+)'
-    _TESTS = [
-        {
-            'url': 'http://www.cda.pl/video/5749950c',
-            'md5': '6f844bf51b15f31fae165365707ae970',
-            'info_dict': {
-                'id': '5749950c',
-                'ext': 'mp4',
-                'height': 720,
-                'title': 'Oto dlaczego przed zakrętem należy zwolnić.',
-                'duration': 39
-            }
-        },
-        {
-            'url': 'http://www.cda.pl/video/57413289',
-            'md5': 'a88828770a8310fc00be6c95faf7f4d5',
-            'info_dict': {
-                'id': '57413289',
-                'ext': 'mp4',
-                'title': 'Lądowanie na lotnisku na Maderze',
-                'duration': 137
-            }
+    _VALID_URL = r'https?://(?:(?:www\.)?cda\.pl/video|ebd\.cda\.pl/[0-9]+x[0-9]+)/(?P<id>[0-9a-z]+)'
+    _TESTS = [{
+        'url': 'http://www.cda.pl/video/5749950c',
+        'md5': '6f844bf51b15f31fae165365707ae970',
+        'info_dict': {
+            'id': '5749950c',
+            'ext': 'mp4',
+            'height': 720,
+            'title': 'Oto dlaczego przed zakrętem należy zwolnić.',
+            'duration': 39
         }
-    ]
+    }, {
+        'url': 'http://www.cda.pl/video/57413289',
+        'md5': 'a88828770a8310fc00be6c95faf7f4d5',
+        'info_dict': {
+            'id': '57413289',
+            'ext': 'mp4',
+            'title': 'Lądowanie na lotnisku na Maderze',
+            'duration': 137
+        }
+    }, {
+        'url': 'http://ebd.cda.pl/0x0/5749950c',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -44,53 +44,53 @@ class CDAIE(InfoExtractor):
         if 'Ten film jest dostępny dla użytkowników premium' in webpage:
             raise ExtractorError('This video is only available for premium users.', expected=True)
 
-        title = self._html_search_regex(r'<title>(.+?)</title>', webpage, 'title', fatal=False)
-
-        def _get_format(page, version=''):
-            unpacked = decode_packed_codes(page)
-            duration = self._search_regex(r"duration:\\'(.+?)\\'", unpacked, 'duration', fatal=False)
-            format_id = None
-            height = None
-
-            m = re.search(r'<a data-quality="(?P<format_id>[^"]+)" href="[^"]+" class="quality-btn quality-btn-active">(?P<height>[0-9]+)p<\/a>', page)
-            if m:
-                format_id = m.group('format_id')
-                height = int(m.group('height'))
-
-            url = self._search_regex(r"url:\\'(.+?)\\'", unpacked, version + ' url', fatal=False)
-            if url is None:
-                return None
-
-            return {
-                'format_id': format_id,
-                'height': height,
-                'url': url
-            }, parse_duration(duration)
+        title = self._html_search_regex(r'<title>(.+?)</title>', webpage, 'title')
 
         formats = []
 
-        format_desc, duration = _get_format(webpage) or (None, None)
-        if format_desc is not None:
-            formats.append(format_desc)
-
-        pattern = re.compile(r'<a data-quality="[^"]+" href="([^"]+)" class="quality-btn">([0-9]+p)<\/a>')
-        for version in re.findall(pattern, webpage):
-            webpage = self._download_webpage(version[0], video_id, 'Downloading %s version information' % version[1], fatal=False)
-            if not webpage:
-                # Manually report warning because empty page is returned when invalid version is requested.
-                self.report_warning('Unable to download %s version information' % version[1])
-                continue
-
-            format_desc, duration_ = _get_format(webpage, version[1]) or (None, None)
-            duration = duration or duration_
-            if format_desc is not None:
-                formats.append(format_desc)
-
-        self._sort_formats(formats)
-
-        return {
+        info_dict = {
             'id': video_id,
             'title': title,
             'formats': formats,
-            'duration': duration
+            'duration': None,
         }
+
+        def extract_format(page, version):
+            unpacked = decode_packed_codes(page)
+            format_url = self._search_regex(
+                r"url:\\'(.+?)\\'", unpacked, '%s url' % version, fatal=False)
+            if not format_url:
+                return
+            f = {
+                'url': format_url,
+            }
+            m = re.search(
+                r'<a[^>]+data-quality="(?P<format_id>[^"]+)"[^>]+href="[^"]+"[^>]+class="[^"]*quality-btn-active[^"]*">(?P<height>[0-9]+)p',
+                page)
+            if m:
+                f.update({
+                    'format_id': m.group('format_id'),
+                    'height': int(m.group('height')),
+                })
+            info_dict['formats'].append(f)
+            if not info_dict['duration']:
+                info_dict['duration'] = parse_duration(self._search_regex(
+                    r"duration:\\'(.+?)\\'", unpacked, 'duration', fatal=False))
+
+        extract_format(webpage, 'default')
+
+        for href, resolution in re.findall(
+                r'<a[^>]+data-quality="[^"]+"[^>]+href="([^"]+)"[^>]+class="quality-btn"[^>]*>([0-9]+p)',
+                webpage):
+            webpage = self._download_webpage(
+                href, video_id, 'Downloading %s version information' % resolution, fatal=False)
+            if not webpage:
+                # Manually report warning because empty page is returned when
+                # invalid version is requested.
+                self.report_warning('Unable to download %s version information' % resolution)
+                continue
+            extract_format(webpage, resolution)
+
+        self._sort_formats(formats)
+
+        return info_dict

From 9261e347ccf63b31bd2035996279b0ad1a45247a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Mar 2016 23:18:04 +0600
Subject: [PATCH 1049/1105] Credit @kasper93 for cda (#8805)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index aa48cd5a6..6c9747913 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -163,3 +163,4 @@ Patrick Griffis
 Aidan Rowe
 mutantmonkey
 Ben Congdon
+Kacper Michajłow

From e36f4aa72b01b3f6a322edc094cdf1c20b071367 Mon Sep 17 00:00:00 2001
From: jjatria <jjatria@gmail.com>
Date: Wed, 28 Oct 2015 18:31:52 +0000
Subject: [PATCH 1050/1105] [biobiotv] Add extractor

---
 youtube_dl/extractor/biobiotv.py | 75 ++++++++++++++++++++++++++++++++
 1 file changed, 75 insertions(+)
 create mode 100644 youtube_dl/extractor/biobiotv.py

diff --git a/youtube_dl/extractor/biobiotv.py b/youtube_dl/extractor/biobiotv.py
new file mode 100644
index 000000000..aae0588ef
--- /dev/null
+++ b/youtube_dl/extractor/biobiotv.py
@@ -0,0 +1,75 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class BioBioTVIE(InfoExtractor):
+    _VALID_URL = r'https?://tv\.biobiochile\.cl/notas/(?P<year>\d{4})/\d{2}/\d{2}/(?P<id>[\w-]+)(?:\.shtml)?'
+
+    _TESTS = [{
+        'url': 'http://tv.biobiochile.cl/notas/2015/10/21/sobre-camaras-y-camarillas-parlamentarias.shtml',
+        'md5': '26f51f03cf580265defefb4518faec09',
+        'info_dict': {
+            'id': 'col_c266',
+            'display_id': 'sobre-camaras-y-camarillas-parlamentarias',
+            'ext': 'mp4',
+            'title': 'Sobre Cámaras y camarillas parlamentarias - BioBioChile TV',
+            'thumbnail': 'http://media.biobiochile.cl/wp-content/uploads/2015/10/atria-2010-730x350.jpg',
+            'url': 'http://unlimited2-cl.digitalproserver.com/bbtv/2015/col_c266.mp4',
+            'uploader': 'Fernando Atria',
+        }
+    }, {
+        'url': 'http://tv.biobiochile.cl/notas/2015/10/22/ninos-transexuales-de-quien-es-la-decision.shtml',
+        'md5': 'a8c868e6b5f6c17d56873d5633204f84',
+        'info_dict': {
+            'id': 'col_c270',
+            'display_id': 'ninos-transexuales-de-quien-es-la-decision',
+            'ext': 'mp4',
+            'title': 'Niños transexuales: ¿De quién es la decisión? - BioBioChile TV',
+            'thumbnail': 'http://media.biobiochile.cl/wp-content/uploads/2015/10/samantha-2210-730x350.jpg',
+            'url': 'http://unlimited2-cl.digitalproserver.com/bbtv/2015/col_c270.mp4',
+            'uploader': 'Samantha Morán',
+        }
+    }, {
+        'url': 'http://tv.biobiochile.cl/notas/2015/10/21/exclusivo-hector-pinto-formador-de-chupete-revela-version-del-ex-delantero-albo.shtml',
+        'md5': 'c8369b50d42ff0a4f6b969fbd1a7c32d',
+        'info_dict': {
+            'id': 'Keno_Pinto',
+            'display_id': 'exclusivo-hector-pinto-formador-de-chupete-revela-version-del-ex-delantero-albo',
+            'ext': 'mp4',
+            'title': 'Exclusivo: Héctor Pinto, formador de “Chupete”, revela versión del ex delantero albo - BioBioChile TV',
+            'thumbnail': 'http://media.biobiochile.cl/wp-content/uploads/2015/10/pinto-730x350.jpg',
+            'url': 'http://unlimited2-cl.digitalproserver.com/bbtv/2015/Keno_Pinto.mp4',
+            'uploader': 'Juan Pablo Echenique',
+        }
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('id')
+        year = mobj.group('year')
+
+        webpage = self._download_webpage(url, display_id)
+
+        title = self._html_search_meta(
+            'og:title', webpage, 'title', fatal=True)
+
+        thumbnail = self._html_search_meta(
+            'og:image', webpage, 'thumbnail', fatal=True)
+
+        video_id = self._html_search_regex(
+            r'loadFWPlayerVideo\(\"player_0\", \"\d{4}/(.+)\.mp4\"\)', webpage, 'title')
+
+        url = 'http://unlimited2-cl.digitalproserver.com/bbtv/' + year + '/' + video_id + '.mp4'
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': url,
+            'display_id': display_id,
+            'thumbnail': thumbnail,
+            'uploader': self._search_regex(r'biobiochile\.cl/author[^"]+"[^>]*>([^<]+)<', webpage, 'uploader', fatal=False),
+        }

From fa023ccb2c00f393c78ae4cbbabec7a8ec7b3ac6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Mar 2016 01:31:55 +0600
Subject: [PATCH 1051/1105] [biobiochiletv] Fix extraction, extract m3u8
 formats and overall improve (Closes #7314)

---
 youtube_dl/extractor/__init__.py      |  1 +
 youtube_dl/extractor/biobiochiletv.py | 86 +++++++++++++++++++++++++++
 youtube_dl/extractor/biobiotv.py      | 75 -----------------------
 3 files changed, 87 insertions(+), 75 deletions(-)
 create mode 100644 youtube_dl/extractor/biobiochiletv.py
 delete mode 100644 youtube_dl/extractor/biobiotv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5f5eca42b..b773edb3d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -72,6 +72,7 @@ from .bet import BetIE
 from .bigflix import BigflixIE
 from .bild import BildIE
 from .bilibili import BiliBiliIE
+from .biobiochiletv import BioBioChileTVIE
 from .bleacherreport import (
     BleacherReportIE,
     BleacherReportCMSIE,
diff --git a/youtube_dl/extractor/biobiochiletv.py b/youtube_dl/extractor/biobiochiletv.py
new file mode 100644
index 000000000..133228133
--- /dev/null
+++ b/youtube_dl/extractor/biobiochiletv.py
@@ -0,0 +1,86 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import remove_end
+
+
+class BioBioChileTVIE(InfoExtractor):
+    _VALID_URL = r'https?://tv\.biobiochile\.cl/notas/(?:[^/]+/)+(?P<id>[^/]+)\.shtml'
+
+    _TESTS = [{
+        'url': 'http://tv.biobiochile.cl/notas/2015/10/21/sobre-camaras-y-camarillas-parlamentarias.shtml',
+        'md5': '26f51f03cf580265defefb4518faec09',
+        'info_dict': {
+            'id': 'sobre-camaras-y-camarillas-parlamentarias',
+            'ext': 'mp4',
+            'title': 'Sobre Cámaras y camarillas parlamentarias',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'Fernando Atria',
+        },
+    }, {
+        # different uploader layout
+        'url': 'http://tv.biobiochile.cl/notas/2016/03/18/natalia-valdebenito-repasa-a-diputado-hasbun-paso-a-la-categoria-de-hablar-brutalidades.shtml',
+        'md5': 'edc2e6b58974c46d5b047dea3c539ff3',
+        'info_dict': {
+            'id': 'natalia-valdebenito-repasa-a-diputado-hasbun-paso-a-la-categoria-de-hablar-brutalidades',
+            'ext': 'mp4',
+            'title': 'Natalia Valdebenito repasa a diputado Hasbún: Pasó a la categoría de hablar brutalidades',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'Piangella Obrador',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://tv.biobiochile.cl/notas/2015/10/22/ninos-transexuales-de-quien-es-la-decision.shtml',
+        'only_matching': True,
+    }, {
+        'url': 'http://tv.biobiochile.cl/notas/2015/10/21/exclusivo-hector-pinto-formador-de-chupete-revela-version-del-ex-delantero-albo.shtml',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        title = remove_end(self._og_search_title(webpage), ' - BioBioChile TV')
+
+        file_url = self._search_regex(
+            r'loadFWPlayerVideo\([^,]+,\s*(["\'])(?P<url>.+?)\1',
+            webpage, 'file url', group='url')
+
+        base_url = self._search_regex(
+            r'file\s*:\s*(["\'])(?P<url>.+?)\1\s*\+\s*fileURL', webpage,
+            'base url', default='http://unlimited2-cl.digitalproserver.com/bbtv/',
+            group='url')
+
+        formats = self._extract_m3u8_formats(
+            '%s%s/playlist.m3u8' % (base_url, file_url), video_id, 'mp4',
+            entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)
+        f = {
+            'url': '%s%s' % (base_url, file_url),
+            'format_id': 'http',
+            'protocol': 'http',
+            'preference': 1,
+        }
+        if formats:
+            f_copy = formats[-1].copy()
+            f_copy.update(f)
+            f = f_copy
+        formats.append(f)
+        self._sort_formats(formats)
+
+        thumbnail = self._og_search_thumbnail(webpage)
+        uploader = self._html_search_regex(
+            r'<a[^>]+href=["\']https?://busca\.biobiochile\.cl/author[^>]+>(.+?)</a>',
+            webpage, 'uploader', fatal=False)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/biobiotv.py b/youtube_dl/extractor/biobiotv.py
deleted file mode 100644
index aae0588ef..000000000
--- a/youtube_dl/extractor/biobiotv.py
+++ /dev/null
@@ -1,75 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-
-
-class BioBioTVIE(InfoExtractor):
-    _VALID_URL = r'https?://tv\.biobiochile\.cl/notas/(?P<year>\d{4})/\d{2}/\d{2}/(?P<id>[\w-]+)(?:\.shtml)?'
-
-    _TESTS = [{
-        'url': 'http://tv.biobiochile.cl/notas/2015/10/21/sobre-camaras-y-camarillas-parlamentarias.shtml',
-        'md5': '26f51f03cf580265defefb4518faec09',
-        'info_dict': {
-            'id': 'col_c266',
-            'display_id': 'sobre-camaras-y-camarillas-parlamentarias',
-            'ext': 'mp4',
-            'title': 'Sobre Cámaras y camarillas parlamentarias - BioBioChile TV',
-            'thumbnail': 'http://media.biobiochile.cl/wp-content/uploads/2015/10/atria-2010-730x350.jpg',
-            'url': 'http://unlimited2-cl.digitalproserver.com/bbtv/2015/col_c266.mp4',
-            'uploader': 'Fernando Atria',
-        }
-    }, {
-        'url': 'http://tv.biobiochile.cl/notas/2015/10/22/ninos-transexuales-de-quien-es-la-decision.shtml',
-        'md5': 'a8c868e6b5f6c17d56873d5633204f84',
-        'info_dict': {
-            'id': 'col_c270',
-            'display_id': 'ninos-transexuales-de-quien-es-la-decision',
-            'ext': 'mp4',
-            'title': 'Niños transexuales: ¿De quién es la decisión? - BioBioChile TV',
-            'thumbnail': 'http://media.biobiochile.cl/wp-content/uploads/2015/10/samantha-2210-730x350.jpg',
-            'url': 'http://unlimited2-cl.digitalproserver.com/bbtv/2015/col_c270.mp4',
-            'uploader': 'Samantha Morán',
-        }
-    }, {
-        'url': 'http://tv.biobiochile.cl/notas/2015/10/21/exclusivo-hector-pinto-formador-de-chupete-revela-version-del-ex-delantero-albo.shtml',
-        'md5': 'c8369b50d42ff0a4f6b969fbd1a7c32d',
-        'info_dict': {
-            'id': 'Keno_Pinto',
-            'display_id': 'exclusivo-hector-pinto-formador-de-chupete-revela-version-del-ex-delantero-albo',
-            'ext': 'mp4',
-            'title': 'Exclusivo: Héctor Pinto, formador de “Chupete”, revela versión del ex delantero albo - BioBioChile TV',
-            'thumbnail': 'http://media.biobiochile.cl/wp-content/uploads/2015/10/pinto-730x350.jpg',
-            'url': 'http://unlimited2-cl.digitalproserver.com/bbtv/2015/Keno_Pinto.mp4',
-            'uploader': 'Juan Pablo Echenique',
-        }
-    }]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('id')
-        year = mobj.group('year')
-
-        webpage = self._download_webpage(url, display_id)
-
-        title = self._html_search_meta(
-            'og:title', webpage, 'title', fatal=True)
-
-        thumbnail = self._html_search_meta(
-            'og:image', webpage, 'thumbnail', fatal=True)
-
-        video_id = self._html_search_regex(
-            r'loadFWPlayerVideo\(\"player_0\", \"\d{4}/(.+)\.mp4\"\)', webpage, 'title')
-
-        url = 'http://unlimited2-cl.digitalproserver.com/bbtv/' + year + '/' + video_id + '.mp4'
-
-        return {
-            'id': video_id,
-            'title': title,
-            'url': url,
-            'display_id': display_id,
-            'thumbnail': thumbnail,
-            'uploader': self._search_regex(r'biobiochile\.cl/author[^"]+"[^>]*>([^<]+)<', webpage, 'uploader', fatal=False),
-        }

From 94dcade8f892f27f1cdbee29be2e06c08442976e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Mar 2016 01:36:20 +0600
Subject: [PATCH 1052/1105] Credit @jjatria for biobiochiletv (#7314)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 6c9747913..e507686f2 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -164,3 +164,4 @@ Aidan Rowe
 mutantmonkey
 Ben Congdon
 Kacper Michajłow
+José Joaquín Atria

From d95114dd8359d10c6a0ca5eaddbbd94806173957 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Mar 2016 02:34:02 +0600
Subject: [PATCH 1053/1105] [91porn] Unquote final URL (Closes #8881)

---
 youtube_dl/extractor/porn91.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index 3e15533e9..a6dd2694c 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -1,7 +1,10 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-from ..compat import compat_urllib_parse
+from ..compat import (
+    compat_urllib_parse,
+    compat_urllib_parse_unquote,
+)
 from .common import InfoExtractor
 from ..utils import (
     parse_duration,
@@ -55,7 +58,8 @@ class Porn91IE(InfoExtractor):
         info_cn = self._download_webpage(
             'http://91porn.com/getfile.php?' + url_params, video_id,
             'get real video url')
-        video_url = self._search_regex(r'file=([^&]+)&', info_cn, 'url')
+        video_url = compat_urllib_parse_unquote(self._search_regex(
+            r'file=([^&]+)&', info_cn, 'url'))
 
         duration = parse_duration(self._search_regex(
             r'时长:\s*</span>\s*(\d+:\d+)', webpage, 'duration', fatal=False))

From 298c04b46497b924b1cbb2f031c5d73d09d2933d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Mar 2016 02:35:48 +0600
Subject: [PATCH 1054/1105] [91porn] Use common messages' wording

---
 youtube_dl/extractor/porn91.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index a6dd2694c..63ce87ee3 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -31,9 +31,10 @@ class Porn91IE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        url = 'http://91porn.com/view_video.php?viewkey=%s' % video_id
         self._set_cookie('91porn.com', 'language', 'cn_CN')
-        webpage = self._download_webpage(url, video_id, 'get HTML content')
+
+        webpage = self._download_webpage(
+            'http://91porn.com/view_video.php?viewkey=%s' % video_id, video_id)
 
         if '作为游客，你每天只可观看10个视频' in webpage:
             raise ExtractorError('91 Porn says: Daily limit 10 videos exceeded', expected=True)
@@ -57,7 +58,7 @@ class Porn91IE(InfoExtractor):
         })
         info_cn = self._download_webpage(
             'http://91porn.com/getfile.php?' + url_params, video_id,
-            'get real video url')
+            'Downloading real video url')
         video_url = compat_urllib_parse_unquote(self._search_regex(
             r'file=([^&]+)&', info_cn, 'url'))
 

From 5c7cd37ebd6dfb7d5809d2798d0188decce42914 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 19 Mar 2016 21:50:16 +0100
Subject: [PATCH 1055/1105] tox.ini: Exclude test_iqiyi_sdk_interpreter.py

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 48504329f..2d7134005 100644
--- a/tox.ini
+++ b/tox.ini
@@ -8,6 +8,6 @@ deps =
 passenv = HOME
 defaultargs = test --exclude test_download.py --exclude test_age_restriction.py
     --exclude test_subtitles.py --exclude test_write_annotations.py
-    --exclude test_youtube_lists.py
+    --exclude test_youtube_lists.py --exclude test_iqiyi_sdk_interpreter.py
 commands = nosetests --verbose {posargs:{[testenv]defaultargs}}  # --with-coverage --cover-package=youtube_dl --cover-html
                                                # test.test_download:TestDownload.test_NowVideo

From 3ff63fb3657fee9e0c2df9d5bb96ae5827f257cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 19 Mar 2016 21:51:13 +0100
Subject: [PATCH 1056/1105] Makefile: make it compatible with bmake

It's the portable version of BSD make: http://crufty.net/help/sjg/bmake.html
The syntax for conditionals is different in GNU make and BSD make, so we use the shell
---
 Makefile | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/Makefile b/Makefile
index e98806791..6689ec06f 100644
--- a/Makefile
+++ b/Makefile
@@ -12,15 +12,7 @@ SHAREDIR ?= $(PREFIX)/share
 PYTHON ?= /usr/bin/env python
 
 # set SYSCONFDIR to /etc if PREFIX=/usr or PREFIX=/usr/local
-ifeq ($(PREFIX),/usr)
-	SYSCONFDIR=/etc
-else
-	ifeq ($(PREFIX),/usr/local)
-		SYSCONFDIR=/etc
-	else
-		SYSCONFDIR=$(PREFIX)/etc
-	endif
-endif
+SYSCONFDIR != if [ $(PREFIX) = /usr -o $(PREFIX) = /usr/local ]; then echo /etc; else echo $(PREFIX)/etc; fi
 
 install: youtube-dl youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish
 	install -d $(DESTDIR)$(BINDIR)

From 40025ee2a339d1a357869fd8d8718a737d250d9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Mar 2016 04:12:34 +0600
Subject: [PATCH 1057/1105] [postprocessort/ffmpeg] Allow embedding webvtt into
 webm (Closes #8874)

---
 youtube_dl/options.py              |  2 +-
 youtube_dl/postprocessor/ffmpeg.py | 25 +++++++++++++++++++++----
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 9dd7a8034..755ed6540 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -720,7 +720,7 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '--embed-subs',
         action='store_true', dest='embedsubtitles', default=False,
-        help='Embed subtitles in the video (only for mkv and mp4 videos)')
+        help='Embed subtitles in the video (only for mp4, webm and mkv videos)')
     postproc.add_option(
         '--embed-thumbnail',
         action='store_true', dest='embedthumbnail', default=False,
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index a8819f258..06b8c0548 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -331,17 +331,34 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor):
 
 class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
     def run(self, information):
-        if information['ext'] not in ['mp4', 'mkv']:
-            self._downloader.to_screen('[ffmpeg] Subtitles can only be embedded in mp4 or mkv files')
+        if information['ext'] not in ('mp4', 'webm', 'mkv'):
+            self._downloader.to_screen('[ffmpeg] Subtitles can only be embedded in mp4, webm or mkv files')
             return [], information
         subtitles = information.get('requested_subtitles')
         if not subtitles:
             self._downloader.to_screen('[ffmpeg] There aren\'t any subtitles to embed')
             return [], information
 
-        sub_langs = list(subtitles.keys())
         filename = information['filepath']
-        sub_filenames = [subtitles_filename(filename, lang, sub_info['ext']) for lang, sub_info in subtitles.items()]
+
+        ext = information['ext']
+        sub_langs = []
+        sub_filenames = []
+        webm_vtt_warn = False
+
+        for lang, sub_info in subtitles.items():
+            sub_ext = sub_info['ext']
+            if ext != 'webm' or ext == 'webm' and sub_ext == 'vtt':
+                sub_langs.append(lang)
+                sub_filenames.append(subtitles_filename(filename, lang, sub_ext))
+            else:
+                if not webm_vtt_warn and ext == 'webm' and sub_ext != 'vtt':
+                    webm_vtt_warn = True
+                    self._downloader.to_screen('[ffmpeg] Only WebVTT subtitles can be embedded in webm files')
+
+        if not sub_langs:
+            return [], information
+
         input_files = [filename] + sub_filenames
 
         opts = [

From 96a9f22d983e414583f52eafece7902b1248377b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Mar 2016 10:26:58 +0600
Subject: [PATCH 1058/1105] [discovery] Relax _VALID_URL (Closes #8903)

---
 youtube_dl/extractor/discovery.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index ce680a9f3..fdce1429a 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -9,7 +9,7 @@ from ..compat import compat_str
 
 
 class DiscoveryIE(InfoExtractor):
-    _VALID_URL = r'''(?x)http://(?:www\.)?(?:
+    _VALID_URL = r'''(?x)https?://(?:www\.)?(?:
             discovery|
             investigationdiscovery|
             discoverylife|

From db264e3cc3bbab191972bbe1c4efb526ff8bfc26 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Mar 2016 12:44:04 +0600
Subject: [PATCH 1059/1105] [francetvinfo] Add support for france3-regions and
 strip title (Closes #7673)

---
 youtube_dl/extractor/francetv.py | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 3f4ac3093..7db5fb418 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -82,6 +82,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
         subtitle = info.get('sous_titre')
         if subtitle:
             title += ' - %s' % subtitle
+        title = title.strip()
 
         subtitles = {}
         subtitles_list = [{
@@ -125,7 +126,7 @@ class PluzzIE(FranceTVBaseInfoExtractor):
 
 class FranceTvInfoIE(FranceTVBaseInfoExtractor):
     IE_NAME = 'francetvinfo.fr'
-    _VALID_URL = r'https?://(?:www|mobile)\.francetvinfo\.fr/.*/(?P<title>.+)\.html'
+    _VALID_URL = r'https?://(?:www|mobile|france3-regions)\.francetvinfo\.fr/.*/(?P<title>.+)\.html'
 
     _TESTS = [{
         'url': 'http://www.francetvinfo.fr/replay-jt/france-3/soir-3/jt-grand-soir-3-lundi-26-aout-2013_393427.html',
@@ -160,6 +161,21 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
             'title': 'Les entreprises familiales : le secret de la réussite',
             'thumbnail': 're:^https?://.*\.jpe?g$',
         }
+    }, {
+        'url': 'http://france3-regions.francetvinfo.fr/bretagne/cotes-d-armor/thalassa-echappee-breizh-ce-venredi-dans-les-cotes-d-armor-954961.html',
+        'md5': 'f485bda6e185e7d15dbc69b72bae993e',
+        'info_dict': {
+            'id': 'NI_657393',
+            'ext': 'flv',
+            'title': 'Olivier Monthus, réalisateur de "Bretagne, le choix de l’Armor"',
+            'description': 'md5:a3264114c9d29aeca11ced113c37b16c',
+            'thumbnail': 're:^https?://.*\.jpe?g$',
+            'timestamp': 1458300695,
+            'upload_date': '20160318',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
@@ -172,7 +188,9 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
             return self.url_result(dmcloud_url, 'DailymotionCloud')
 
         video_id, catalogue = self._search_regex(
-            r'id-video=([^@]+@[^"]+)', webpage, 'video id').split('@')
+            (r'id-video=([^@]+@[^"]+)',
+             r'<a[^>]+href="(?:https?:)?//videos\.francetv\.fr/video/([^@]+@[^"]+)"'),
+            webpage, 'video id').split('@')
         return self._extract_video(video_id, catalogue)
 
 

From 3c20208eff29fcd3e5d4b13f3d4ffa1be7c56309 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Mar 2016 13:00:46 +0600
Subject: [PATCH 1060/1105] [francetv] Improve formats extraction

---
 youtube_dl/extractor/francetv.py | 36 +++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 7db5fb418..ad94e31f3 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -60,22 +60,24 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
                     video_id, 'Downloading f4m manifest token', fatal=False)
                 if f4m_url:
                     formats.extend(self._extract_f4m_formats(
-                        f4m_url + '&hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id, 1, format_id))
+                        f4m_url + '&hdcore=3.7.0&plugin=aasp-3.7.0.39.44',
+                        video_id, f4m_id=format_id, fatal=False))
             elif ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(video_url, video_id, 'mp4', m3u8_id=format_id))
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id=format_id, fatal=False))
             elif video_url.startswith('rtmp'):
                 formats.append({
                     'url': video_url,
                     'format_id': 'rtmp-%s' % format_id,
                     'ext': 'flv',
-                    'preference': 1,
                 })
             else:
-                formats.append({
-                    'url': video_url,
-                    'format_id': format_id,
-                    'preference': -1,
-                })
+                if self._is_valid_url(video_url, video_id, format_id):
+                    formats.append({
+                        'url': video_url,
+                        'format_id': format_id,
+                    })
         self._sort_formats(formats)
 
         title = info['titre']
@@ -132,7 +134,7 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
         'url': 'http://www.francetvinfo.fr/replay-jt/france-3/soir-3/jt-grand-soir-3-lundi-26-aout-2013_393427.html',
         'info_dict': {
             'id': '84981923',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Soir 3',
             'upload_date': '20130826',
             'timestamp': 1377548400,
@@ -140,6 +142,10 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
                 'fr': 'mincount:2',
             },
         },
+        'params': {
+            # m3u8 downloads
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.francetvinfo.fr/elections/europeennes/direct-europeennes-regardez-le-debat-entre-les-candidats-a-la-presidence-de-la-commission_600639.html',
         'info_dict': {
@@ -156,17 +162,23 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
         'url': 'http://www.francetvinfo.fr/economie/entreprises/les-entreprises-familiales-le-secret-de-la-reussite_933271.html',
         'md5': 'f485bda6e185e7d15dbc69b72bae993e',
         'info_dict': {
-            'id': '556e03339473995ee145930c',
+            'id': 'NI_173343',
             'ext': 'mp4',
             'title': 'Les entreprises familiales : le secret de la réussite',
             'thumbnail': 're:^https?://.*\.jpe?g$',
-        }
+            'timestamp': 1433273139,
+            'upload_date': '20150602',
+        },
+        'params': {
+            # m3u8 downloads
+            'skip_download': True,
+        },
     }, {
         'url': 'http://france3-regions.francetvinfo.fr/bretagne/cotes-d-armor/thalassa-echappee-breizh-ce-venredi-dans-les-cotes-d-armor-954961.html',
         'md5': 'f485bda6e185e7d15dbc69b72bae993e',
         'info_dict': {
             'id': 'NI_657393',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Olivier Monthus, réalisateur de "Bretagne, le choix de l’Armor"',
             'description': 'md5:a3264114c9d29aeca11ced113c37b16c',
             'thumbnail': 're:^https?://.*\.jpe?g$',

From 664bcd80b99ac84c3cc7a08e8284abc024a1e58c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 20 Mar 2016 15:45:31 +0800
Subject: [PATCH 1061/1105] [tudou] Use InAdvancePagedList (closes #8884)

---
 youtube_dl/extractor/tudou.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index f56b66d06..9892e8a62 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
     int_or_none,
+    InAdvancePagedList,
     float_or_none,
     unescapeHTML,
 )
@@ -75,15 +76,16 @@ class TudouIE(InfoExtractor):
         quality = sorted(filter(lambda k: k.isdigit(), segments.keys()),
                          key=lambda k: int(k))[-1]
         parts = segments[quality]
-        result = []
         len_parts = len(parts)
         if len_parts > 1:
             self.to_screen('%s: found %s parts' % (video_id, len_parts))
-        for part in parts:
+
+        def part_func(partnum):
+            part = parts[partnum]
             part_id = part['k']
             final_url = self._url_for_id(part_id, quality)
             ext = (final_url.split('?')[0]).split('.')[-1]
-            part_info = {
+            return [{
                 'id': '%s' % part_id,
                 'url': final_url,
                 'ext': ext,
@@ -97,12 +99,13 @@ class TudouIE(InfoExtractor):
                 'http_headers': {
                     'Referer': self._PLAYER_URL,
                 },
-            }
-            result.append(part_info)
+            }]
+
+        entries = InAdvancePagedList(part_func, len_parts, 1)
 
         return {
             '_type': 'multi_video',
-            'entries': result,
+            'entries': entries,
             'id': video_id,
             'title': title,
         }

From 2bfeee69b976fe049761dd3012e30b637ee05a58 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 20 Mar 2016 15:54:58 +0800
Subject: [PATCH 1062/1105] [openload] Add new extractor (closes #8489)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/openload.py | 92 ++++++++++++++++++++++++++++++++
 2 files changed, 93 insertions(+)
 create mode 100644 youtube_dl/extractor/openload.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b773edb3d..ee792bbe0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -535,6 +535,7 @@ from .ooyala import (
     OoyalaIE,
     OoyalaExternalIE,
 )
+from .openload import OpenloadIE
 from .ora import OraTVIE
 from .orf import (
     ORFTVthekIE,
diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
new file mode 100644
index 000000000..71021d573
--- /dev/null
+++ b/youtube_dl/extractor/openload.py
@@ -0,0 +1,92 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_chr
+from ..utils import encode_base_n
+
+
+class OpenloadIE(InfoExtractor):
+    _VALID_URL = r'https://openload.co/f/(?P<id>[a-zA-Z0-9]+)'
+
+    _TEST = {
+        'url': 'https://openload.co/f/kUEfGclsU9o',
+        'md5': 'bf1c059b004ebc7a256f89408e65c36e',
+        'info_dict': {
+            'id': 'kUEfGclsU9o',
+            'ext': 'mp4',
+            'title': 'skyrim_no-audio_1080.mp4',
+        },
+    }
+
+    @staticmethod
+    def openload_level2_debase(m):
+        radix, num = int(m.group(1)) + 27, int(m.group(2))
+        return '"' + encode_base_n(num, radix) + '"'
+
+    @classmethod
+    def openload_level2(cls, txt):
+        # The function name is ǃ \u01c3
+        # Using escaped unicode literals does not work in Python 3.2
+        return re.sub(r'ǃ\((\d+),(\d+)\)', cls.openload_level2_debase, txt, re.UNICODE).replace('"+"', '')
+
+    # Openload uses a variant of aadecode
+    # openload_decode and related functions are originally written by
+    # vitas@matfyz.cz and released with public domain
+    # See https://github.com/rg3/youtube-dl/issues/8489
+    @classmethod
+    def openload_decode(cls, txt):
+        symbol_table = [
+            ('_', '(ﾟДﾟ) [ﾟΘﾟ]'),
+            ('a', '(ﾟДﾟ) [ﾟωﾟﾉ]'),
+            ('b', '(ﾟДﾟ) [ﾟΘﾟﾉ]'),
+            ('c', '(ﾟДﾟ) [\'c\']'),
+            ('d', '(ﾟДﾟ) [ﾟｰﾟﾉ]'),
+            ('e', '(ﾟДﾟ) [ﾟДﾟﾉ]'),
+            ('f', '(ﾟДﾟ) [1]'),
+
+            ('o', '(ﾟДﾟ) [\'o\']'),
+            ('u', '(oﾟｰﾟo)'),
+            ('c', '(ﾟДﾟ) [\'c\']'),
+
+            ('7', '((ﾟｰﾟ) + (o^_^o))'),
+            ('6', '((o^_^o) +(o^_^o) +(c^_^o))'),
+            ('5', '((ﾟｰﾟ) + (ﾟΘﾟ))'),
+            ('4', '(-~3)'),
+            ('3', '(-~-~1)'),
+            ('2', '(-~1)'),
+            ('1', '(-~0)'),
+            ('0', '((c^_^o)-(c^_^o))'),
+        ]
+        delim = '(ﾟДﾟ)[ﾟεﾟ]+'
+        ret = ''
+        for aachar in txt.split(delim):
+            for val, pat in symbol_table:
+                aachar = aachar.replace(pat, val)
+            aachar = aachar.replace('+ ', '')
+            m = re.match(r'^\d+', aachar)
+            if m:
+                ret += compat_chr(int(m.group(0), 8))
+            else:
+                m = re.match(r'^u([\da-f]+)', aachar)
+                if m:
+                    ret += compat_chr(int(m.group(1), 16))
+        return cls.openload_level2(ret)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        code = self._search_regex(
+            r'<video[^>]+>\s*<script[^>]+>([^<]+)</script>',
+            webpage, 'JS code')
+
+        video_url = self._search_regex(
+            r'return\s+"(https?://[^"]+)"', self.openload_decode(code), 'video URL')
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'url': video_url,
+        }

From 9e3c2f1d741acc4dd576f77c185e99cfd6bb2ea4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 20 Mar 2016 16:49:44 +0800
Subject: [PATCH 1063/1105] [openload] Misc improvements

* Add thumbnail
* Detect errors (#6469)
* Match more (#6469, #8489)
---
 youtube_dl/extractor/openload.py | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 71021d573..4468f31fc 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -5,21 +5,31 @@ import re
 
 from .common import InfoExtractor
 from ..compat import compat_chr
-from ..utils import encode_base_n
+from ..utils import (
+    encode_base_n,
+    ExtractorError,
+)
 
 
 class OpenloadIE(InfoExtractor):
-    _VALID_URL = r'https://openload.co/f/(?P<id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'https://openload.(?:co|io)/(?:f|embed)/(?P<id>[a-zA-Z0-9-]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'https://openload.co/f/kUEfGclsU9o',
         'md5': 'bf1c059b004ebc7a256f89408e65c36e',
         'info_dict': {
             'id': 'kUEfGclsU9o',
             'ext': 'mp4',
             'title': 'skyrim_no-audio_1080.mp4',
+            'thumbnail': 're:^https?://.*\.jpg$',
         },
-    }
+    }, {
+        'url': 'https://openload.co/embed/kUEfGclsU9o/skyrim_no-audio_1080.mp4',
+        'only_matching': True,
+    }, {
+        'url': 'https://openload.io/f/ZAn6oz-VZGE/',
+        'only_matching': True,
+    }]
 
     @staticmethod
     def openload_level2_debase(m):
@@ -78,6 +88,10 @@ class OpenloadIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+
+        if 'File not found' in webpage:
+            raise ExtractorError('File not found', expected=True)
+
         code = self._search_regex(
             r'<video[^>]+>\s*<script[^>]+>([^<]+)</script>',
             webpage, 'JS code')
@@ -88,5 +102,6 @@ class OpenloadIE(InfoExtractor):
         return {
             'id': video_id,
             'title': self._og_search_title(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
             'url': video_url,
         }

From 920d318d3cf70ee0b80cc67e7d3f85b5d45a20a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 20 Mar 2016 10:55:14 +0100
Subject: [PATCH 1064/1105] README: document that BSD make is also supported
 (#8902)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 68db546ef..fcc12d2b3 100644
--- a/README.md
+++ b/README.md
@@ -831,7 +831,7 @@ To run the test, simply invoke your favorite test runner, or execute a test file
 If you want to create a build of youtube-dl yourself, you'll need
 
 * python
-* make
+* make (both GNU make and BSD make are supported)
 * pandoc
 * zip
 * nosetests

From 2648918c814773e746c6d26da834d32eac952ffa Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 20 Mar 2016 18:14:02 +0800
Subject: [PATCH 1065/1105] [vlive] Fix creator extraction (closes #8814)

---
 youtube_dl/extractor/vlive.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index 9e2aa58bd..bd5545173 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -64,7 +64,7 @@ class VLiveIE(InfoExtractor):
 
         thumbnail = self._og_search_thumbnail(webpage)
         creator = self._html_search_regex(
-            r'<div[^>]+class="info_area"[^>]*>\s*<strong[^>]+class="name"[^>]*>([^<]+)</strong>',
+            r'<div[^>]+class="info_area"[^>]*>\s*<a\s+[^>]*>([^<]+)',
             webpage, 'creator', fatal=False)
 
         view_count = int_or_none(playinfo.get('meta', {}).get('count'))

From 7caae128a72596e22cdfb538272c38a37d095db2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 20 Mar 2016 19:11:02 +0800
Subject: [PATCH 1066/1105] Credit @vitstradal for the key algorithm in
 OpenloadIE (#8489)

[ci skip]
---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index e507686f2..51dfc8ddd 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -165,3 +165,4 @@ mutantmonkey
 Ben Congdon
 Kacper Michajłow
 José Joaquín Atria
+Viťas Strádal

From daef04a4e75ccd2ff5e2d2495baa0ac9bcf75724 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 20 Mar 2016 20:17:56 +0800
Subject: [PATCH 1067/1105] [kwuo] Fix KuwoChartIE and KuwoSingerIE and accept
 new URL forms

---
 youtube_dl/extractor/kuwo.py | 52 ++++++++++++++++++------------------
 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index f94804d06..45d65e61f 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -2,13 +2,13 @@
 from __future__ import unicode_literals
 
 import re
-import itertools
 
 from .common import InfoExtractor
 from ..utils import (
     get_element_by_id,
     clean_html,
     ExtractorError,
+    InAdvancePagedList,
     remove_start,
 )
 
@@ -55,7 +55,7 @@ class KuwoBaseIE(InfoExtractor):
 class KuwoIE(KuwoBaseIE):
     IE_NAME = 'kuwo:song'
     IE_DESC = '酷我音乐'
-    _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>\d+?)/'
+    _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>\d+?)'
     _TESTS = [{
         'url': 'http://www.kuwo.cn/yinyue/635632/',
         'info_dict': {
@@ -80,6 +80,9 @@ class KuwoIE(KuwoBaseIE):
         'params': {
             'format': 'mp3-320'
         },
+    }, {
+        'url': 'http://www.kuwo.cn/yinyue/3197154?catalog=yueku2016',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -172,8 +175,6 @@ class KuwoChartIE(InfoExtractor):
         'url': 'http://yinyue.kuwo.cn/billboard_香港中文龙虎榜.htm',
         'info_dict': {
             'id': '香港中文龙虎榜',
-            'title': '香港中文龙虎榜',
-            'description': 're:\d{4}第\d{2}期',
         },
         'playlist_mincount': 10,
     }
@@ -184,17 +185,11 @@ class KuwoChartIE(InfoExtractor):
             url, chart_id, note='Download chart info',
             errnote='Unable to get chart info')
 
-        chart_name = self._html_search_regex(
-            r'<h1[^>]+class="unDis">([^<]+)</h1>', webpage, 'chart name')
-
-        chart_desc = self._html_search_regex(
-            r'<p[^>]+class="tabDef">(\d{4}第\d{2}期)</p>', webpage, 'chart desc')
-
         entries = [
             self.url_result(song_url, 'Kuwo') for song_url in re.findall(
-                r'<a[^>]+href="(http://www\.kuwo\.cn/yinyue/\d+)/"', webpage)
+                r'<a[^>]+href="(http://www\.kuwo\.cn/yinyue/\d+)', webpage)
         ]
-        return self.playlist_result(entries, chart_id, chart_name, chart_desc)
+        return self.playlist_result(entries, chart_id)
 
 
 class KuwoSingerIE(InfoExtractor):
@@ -207,7 +202,7 @@ class KuwoSingerIE(InfoExtractor):
             'id': 'bruno+mars',
             'title': 'Bruno Mars',
         },
-        'playlist_count': 10,
+        'playlist_mincount': 329,
     }, {
         'url': 'http://www.kuwo.cn/mingxing/Ali/music.htm',
         'info_dict': {
@@ -218,6 +213,8 @@ class KuwoSingerIE(InfoExtractor):
         'skip': 'Regularly stalls travis build',  # See https://travis-ci.org/rg3/youtube-dl/jobs/78878540
     }]
 
+    PAGE_SIZE = 15
+
     def _real_extract(self, url):
         singer_id = self._match_id(url)
         webpage = self._download_webpage(
@@ -225,25 +222,28 @@ class KuwoSingerIE(InfoExtractor):
             errnote='Unable to get singer info')
 
         singer_name = self._html_search_regex(
-            r'<div class="title clearfix">\s*<h1>([^<]+)<span', webpage, 'singer name'
-        )
+            r'<h1>([^<]+)</h1>', webpage, 'singer name')
 
-        entries = []
-        first_page_only = False if re.search(r'/music(?:_\d+)?\.htm', url) else True
-        for page_num in itertools.count(1):
+        artist_id = self._html_search_regex(
+            r'data-artistid="(\d+)"', webpage, 'artist id')
+
+        page_count = int(self._html_search_regex(
+            r'data-page="(\d+)"', webpage, 'page count'))
+
+        def page_func(page_num):
             webpage = self._download_webpage(
-                'http://www.kuwo.cn/mingxing/%s/music_%d.htm' % (singer_id, page_num),
-                singer_id, note='Download song list page #%d' % page_num,
-                errnote='Unable to get song list page #%d' % page_num)
+                'http://www.kuwo.cn/artist/contentMusicsAjax',
+                singer_id, note='Download song list page #%d' % (page_num + 1),
+                errnote='Unable to get song list page #%d' % (page_num + 1),
+                query={'artistId': artist_id, 'pn': page_num, 'rn': self.PAGE_SIZE})
 
-            entries.extend([
+            return [
                 self.url_result(song_url, 'Kuwo') for song_url in re.findall(
-                    r'<p[^>]+class="m_name"><a[^>]+href="(http://www\.kuwo\.cn/yinyue/\d+)/',
+                    r'<div[^>]+class="name"><a[^>]+href="(http://www\.kuwo\.cn/yinyue/\d+)',
                     webpage)
-            ][:10 if first_page_only else None])
+            ]
 
-            if first_page_only or not re.search(r'<a[^>]+href="[^"]+">下一页</a>', webpage):
-                break
+        entries = InAdvancePagedList(page_func, page_count, self.PAGE_SIZE)
 
         return self.playlist_result(entries, singer_id, singer_name)
 

From 3e8bb9a972a377442f5f433123ea70b332248f70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Mar 2016 20:39:00 +0600
Subject: [PATCH 1068/1105] [animeondemand] Detect geo restriction

---
 youtube_dl/extractor/animeondemand.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 0158407f6..85306a69c 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -44,6 +44,10 @@ class AnimeOnDemandIE(InfoExtractor):
         login_page = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login page')
 
+        if '>Our licensing terms allow the distribution of animes only to German-speaking countries of Europe' in login_page:
+            self.raise_geo_restricted(
+                '%s is only available in German-speaking countries of Europe' % self.IE_NAME)
+
         login_form = self._form_hidden_inputs('new_user', login_page)
 
         login_form.update({

From 3c5d183c19f29c1f52fe913ce7e7d47f6eebff2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Mar 2016 21:51:22 +0600
Subject: [PATCH 1069/1105] [animeondemand] Extract all formats (Closes #8906)

---
 youtube_dl/extractor/animeondemand.py | 85 ++++++++++++++++++++-------
 1 file changed, 65 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 85306a69c..4352525e2 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -3,10 +3,14 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_urlparse,
+    compat_str,
+)
 from ..utils import (
     determine_ext,
     encode_dict,
+    extract_attributes,
     ExtractorError,
     sanitized_Request,
     urlencode_postdata,
@@ -34,6 +38,10 @@ class AnimeOnDemandIE(InfoExtractor):
         # Episodes without titles
         'url': 'https://www.anime-on-demand.de/anime/162',
         'only_matching': True,
+    }, {
+        # ger/jap, Dub/OmU, account required
+        'url': 'https://www.anime-on-demand.de/anime/169',
+        'only_matching': True,
     }]
 
     def _login(self):
@@ -130,33 +138,70 @@ class AnimeOnDemandIE(InfoExtractor):
 
             formats = []
 
-            playlist_url = self._search_regex(
-                r'data-playlist=(["\'])(?P<url>.+?)\1',
-                episode_html, 'data playlist', default=None, group='url')
-            if playlist_url:
-                request = sanitized_Request(
-                    compat_urlparse.urljoin(url, playlist_url),
-                    headers={
-                        'X-Requested-With': 'XMLHttpRequest',
-                        'X-CSRF-Token': csrf_token,
-                        'Referer': url,
-                        'Accept': 'application/json, text/javascript, */*; q=0.01',
-                    })
+            for input_ in re.findall(
+                    r'<input[^>]+class=["\'].*?streamstarter_html5[^>]+>', episode_html):
+                attributes = extract_attributes(input_)
+                playlist_urls = []
+                for playlist_key in ('data-playlist', 'data-otherplaylist'):
+                    playlist_url = attributes.get(playlist_key)
+                    if isinstance(playlist_url, compat_str) and re.match(
+                            r'/?[\da-zA-Z]+', playlist_url):
+                        playlist_urls.append(attributes[playlist_key])
+                if not playlist_urls:
+                    continue
 
-                playlist = self._download_json(
-                    request, video_id, 'Downloading playlist JSON', fatal=False)
-                if playlist:
-                    playlist = playlist['playlist'][0]
-                    title = playlist['title']
+                lang = attributes.get('data-lang')
+                lang_note = attributes.get('value')
+
+                for playlist_url in playlist_urls:
+                    kind = self._search_regex(
+                        r'videomaterialurl/\d+/([^/]+)/',
+                        playlist_url, 'media kind', default=None)
+                    format_id_list = []
+                    if lang:
+                        format_id_list.append(lang)
+                    if kind:
+                        format_id_list.append(kind)
+                    if not format_id_list:
+                        format_id_list.append('hls')
+                    format_id = '-'.join(format_id_list)
+                    format_note = ', '.join(filter(None, (kind, lang_note)))
+                    request = sanitized_Request(
+                        compat_urlparse.urljoin(url, playlist_url),
+                        headers={
+                            'X-Requested-With': 'XMLHttpRequest',
+                            'X-CSRF-Token': csrf_token,
+                            'Referer': url,
+                            'Accept': 'application/json, text/javascript, */*; q=0.01',
+                        })
+                    playlist = self._download_json(
+                        request, video_id, 'Downloading %s playlist JSON' % format_id,
+                        fatal=False)
+                    if not playlist:
+                        continue
+                    playlist = playlist.get('playlist')
+                    if not playlist or not isinstance(playlist, list):
+                        continue
+                    playlist = playlist[0]
+                    title = playlist.get('title')
+                    if not title:
+                        continue
                     description = playlist.get('description')
                     for source in playlist.get('sources', []):
                         file_ = source.get('file')
                         if file_ and determine_ext(file_) == 'm3u8':
-                            formats = self._extract_m3u8_formats(
+                            m3u8_formats = self._extract_m3u8_formats(
                                 file_, video_id, 'mp4',
-                                entry_protocol='m3u8_native', m3u8_id='hls')
+                                entry_protocol='m3u8_native', m3u8_id=format_id)
+                            for f in m3u8_formats:
+                                f.update({
+                                    'language': lang,
+                                    'format_note': format_note,
+                                })
+                            formats.extend(m3u8_formats)
 
             if formats:
+                self._sort_formats(formats)
                 f = common_info.copy()
                 f.update({
                     'title': title,

From 9016d76f71b30bd61d69f80dc88fa53f978cf99c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Mar 2016 22:01:45 +0600
Subject: [PATCH 1070/1105] [YoutubeDL] Improve _format_note

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 93b6ca54d..29d7a3106 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1836,7 +1836,7 @@ class YoutubeDL(object):
         if fdict.get('language'):
             if res:
                 res += ' '
-            res += '[%s]' % fdict['language']
+            res += '[%s] ' % fdict['language']
         if fdict.get('format_note') is not None:
             res += fdict['format_note'] + ' '
         if fdict.get('tbr') is not None:

From 12af4beb3e28f986170ed00488b48e2e8bcd4e13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Mar 2016 21:17:29 +0600
Subject: [PATCH 1071/1105] [mailru] Add support for https (Closes #8920)

---
 youtube_dl/extractor/mailru.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mailru.py b/youtube_dl/extractor/mailru.py
index 71085f279..46eb00492 100644
--- a/youtube_dl/extractor/mailru.py
+++ b/youtube_dl/extractor/mailru.py
@@ -13,7 +13,7 @@ from ..utils import (
 class MailRuIE(InfoExtractor):
     IE_NAME = 'mailru'
     IE_DESC = 'Видео@Mail.Ru'
-    _VALID_URL = r'http://(?:www\.)?my\.mail\.ru/(?:video/.*#video=/?(?P<idv1>(?:[^/]+/){3}\d+)|(?:(?P<idv2prefix>(?:[^/]+/){2})video/(?P<idv2suffix>[^/]+/\d+))\.html)'
+    _VALID_URL = r'https?://(?:www\.)?my\.mail\.ru/(?:video/.*#video=/?(?P<idv1>(?:[^/]+/){3}\d+)|(?:(?P<idv2prefix>(?:[^/]+/){2})video/(?P<idv2suffix>[^/]+/\d+))\.html)'
 
     _TESTS = [
         {

From 0cef27ad255b5cb994b1fa0e80a04bd09514925a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Mar 2016 21:22:37 +0600
Subject: [PATCH 1072/1105] Add missing r prefix for _VALID_URLs

---
 youtube_dl/extractor/bbc.py  | 2 +-
 youtube_dl/extractor/nova.py | 2 +-
 youtube_dl/extractor/tv2.py  | 4 ++--
 youtube_dl/extractor/vgtv.py | 4 ++--
 youtube_dl/extractor/wdr.py  | 2 +-
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index e62b3860e..c3176700a 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -942,7 +942,7 @@ class BBCIE(BBCCoUkIE):
 
 
 class BBCCoUkArticleIE(InfoExtractor):
-    _VALID_URL = 'http://www.bbc.co.uk/programmes/articles/(?P<id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'http://www.bbc.co.uk/programmes/articles/(?P<id>[a-zA-Z0-9]+)'
     IE_NAME = 'bbc.co.uk:article'
     IE_DESC = 'BBC articles'
 
diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 3f9c776ef..72f72b803 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class NovaIE(InfoExtractor):
     IE_DESC = 'TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz'
-    _VALID_URL = 'http://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+?)(?:\.html|/|$)'
+    _VALID_URL = r'http://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+?)(?:\.html|/|$)'
     _TESTS = [{
         'url': 'http://tvnoviny.nova.cz/clanek/novinky/co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou.html?utm_source=tvnoviny&utm_medium=cpfooter&utm_campaign=novaplus',
         'info_dict': {
diff --git a/youtube_dl/extractor/tv2.py b/youtube_dl/extractor/tv2.py
index 1457e524e..535d0d361 100644
--- a/youtube_dl/extractor/tv2.py
+++ b/youtube_dl/extractor/tv2.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class TV2IE(InfoExtractor):
-    _VALID_URL = 'http://(?:www\.)?tv2\.no/v/(?P<id>\d+)'
+    _VALID_URL = r'http://(?:www\.)?tv2\.no/v/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.tv2.no/v/916509/',
         'info_dict': {
@@ -100,7 +100,7 @@ class TV2IE(InfoExtractor):
 
 
 class TV2ArticleIE(InfoExtractor):
-    _VALID_URL = 'http://(?:www\.)?tv2\.no/(?:a|\d{4}/\d{2}/\d{2}(/[^/]+)+)/(?P<id>\d+)'
+    _VALID_URL = r'http://(?:www\.)?tv2\.no/(?:a|\d{4}/\d{2}/\d{2}(/[^/]+)+)/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.tv2.no/2015/05/16/nyheter/alesund/krim/pingvin/6930542',
         'info_dict': {
diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index e148b1ef5..77d8978d4 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -214,7 +214,7 @@ class VGTVIE(XstreamIE):
 class BTArticleIE(InfoExtractor):
     IE_NAME = 'bt:article'
     IE_DESC = 'Bergens Tidende Articles'
-    _VALID_URL = 'http://(?:www\.)?bt\.no/(?:[^/]+/)+(?P<id>[^/]+)-\d+\.html'
+    _VALID_URL = r'http://(?:www\.)?bt\.no/(?:[^/]+/)+(?P<id>[^/]+)-\d+\.html'
     _TEST = {
         'url': 'http://www.bt.no/nyheter/lokalt/Kjemper-for-internatet-1788214.html',
         'md5': '2acbe8ad129b3469d5ae51b1158878df',
@@ -241,7 +241,7 @@ class BTArticleIE(InfoExtractor):
 class BTVestlendingenIE(InfoExtractor):
     IE_NAME = 'bt:vestlendingen'
     IE_DESC = 'Bergens Tidende - Vestlendingen'
-    _VALID_URL = 'http://(?:www\.)?bt\.no/spesial/vestlendingen/#!/(?P<id>\d+)'
+    _VALID_URL = r'http://(?:www\.)?bt\.no/spesial/vestlendingen/#!/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.bt.no/spesial/vestlendingen/#!/86588',
         'md5': 'd7d17e3337dc80de6d3a540aefbe441b',
diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index a851578e0..65cab4069 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -244,7 +244,7 @@ class WDRMobileIE(InfoExtractor):
 
 
 class WDRMausIE(InfoExtractor):
-    _VALID_URL = 'http://(?:www\.)?wdrmaus\.de/(?:[^/]+/){,2}(?P<id>[^/?#]+)(?:/index\.php5|(?<!index)\.php5|/(?:$|[?#]))'
+    _VALID_URL = r'http://(?:www\.)?wdrmaus\.de/(?:[^/]+/){,2}(?P<id>[^/?#]+)(?:/index\.php5|(?<!index)\.php5|/(?:$|[?#]))'
     IE_DESC = 'Sendung mit der Maus'
     _TESTS = [{
         'url': 'http://www.wdrmaus.de/aktuelle-sendung/index.php5',

From 5886b38d73c54239c85c3e0d8e7c1585d1bbb7da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Mar 2016 21:36:32 +0600
Subject: [PATCH 1073/1105] Add support for https for all extractors as
 preventive and future-proof measure

---
 youtube_dl/extractor/abc.py                 |  2 +-
 youtube_dl/extractor/addanime.py            |  2 +-
 youtube_dl/extractor/aftonbladet.py         |  2 +-
 youtube_dl/extractor/aljazeera.py           |  2 +-
 youtube_dl/extractor/aol.py                 |  4 ++--
 youtube_dl/extractor/arte.py                |  2 +-
 youtube_dl/extractor/azubu.py               |  2 +-
 youtube_dl/extractor/baidu.py               |  2 +-
 youtube_dl/extractor/bbc.py                 |  2 +-
 youtube_dl/extractor/behindkink.py          |  2 +-
 youtube_dl/extractor/bilibili.py            |  2 +-
 youtube_dl/extractor/bokecc.py              |  2 +-
 youtube_dl/extractor/bpb.py                 |  2 +-
 youtube_dl/extractor/breakcom.py            |  2 +-
 youtube_dl/extractor/camdemy.py             |  4 ++--
 youtube_dl/extractor/cbsnews.py             |  4 ++--
 youtube_dl/extractor/cbssports.py           |  2 +-
 youtube_dl/extractor/cliphunter.py          |  2 +-
 youtube_dl/extractor/clipsyndicate.py       |  2 +-
 youtube_dl/extractor/clubic.py              |  2 +-
 youtube_dl/extractor/comcarcoff.py          |  2 +-
 youtube_dl/extractor/condenast.py           |  2 +-
 youtube_dl/extractor/cspan.py               |  2 +-
 youtube_dl/extractor/ctsnews.py             |  2 +-
 youtube_dl/extractor/dctp.py                |  2 +-
 youtube_dl/extractor/defense.py             |  2 +-
 youtube_dl/extractor/douyutv.py             |  2 +-
 youtube_dl/extractor/dplay.py               |  2 +-
 youtube_dl/extractor/dreisat.py             |  2 +-
 youtube_dl/extractor/dvtv.py                |  2 +-
 youtube_dl/extractor/echomsk.py             |  2 +-
 youtube_dl/extractor/exfm.py                |  2 +-
 youtube_dl/extractor/fc2.py                 |  2 +-
 youtube_dl/extractor/firstpost.py           |  2 +-
 youtube_dl/extractor/firsttv.py             |  2 +-
 youtube_dl/extractor/fktv.py                |  2 +-
 youtube_dl/extractor/footyroom.py           |  2 +-
 youtube_dl/extractor/foxgay.py              |  2 +-
 youtube_dl/extractor/franceinter.py         |  2 +-
 youtube_dl/extractor/freevideo.py           |  2 +-
 youtube_dl/extractor/gamekings.py           |  2 +-
 youtube_dl/extractor/gamespot.py            |  2 +-
 youtube_dl/extractor/gamestar.py            |  2 +-
 youtube_dl/extractor/gametrailers.py        |  2 +-
 youtube_dl/extractor/hotnewhiphop.py        |  2 +-
 youtube_dl/extractor/hypem.py               |  2 +-
 youtube_dl/extractor/imdb.py                |  4 ++--
 youtube_dl/extractor/iqiyi.py               |  2 +-
 youtube_dl/extractor/jadorecettepub.py      |  2 +-
 youtube_dl/extractor/jeuxvideo.py           |  2 +-
 youtube_dl/extractor/karaoketv.py           |  2 +-
 youtube_dl/extractor/karrierevideos.py      |  2 +-
 youtube_dl/extractor/kontrtube.py           |  2 +-
 youtube_dl/extractor/ku6.py                 |  2 +-
 youtube_dl/extractor/kusi.py                |  2 +-
 youtube_dl/extractor/kuwo.py                | 12 ++++++------
 youtube_dl/extractor/leeco.py               |  4 ++--
 youtube_dl/extractor/lifenews.py            |  4 ++--
 youtube_dl/extractor/limelight.py           |  6 +++---
 youtube_dl/extractor/m6.py                  |  2 +-
 youtube_dl/extractor/metacafe.py            |  2 +-
 youtube_dl/extractor/mit.py                 |  2 +-
 youtube_dl/extractor/mitele.py              |  2 +-
 youtube_dl/extractor/mooshare.py            |  2 +-
 youtube_dl/extractor/motherless.py          |  2 +-
 youtube_dl/extractor/motorsport.py          |  2 +-
 youtube_dl/extractor/myspass.py             |  2 +-
 youtube_dl/extractor/myvideo.py             |  2 +-
 youtube_dl/extractor/myvidster.py           |  2 +-
 youtube_dl/extractor/nationalgeographic.py  |  2 +-
 youtube_dl/extractor/nbc.py                 |  4 ++--
 youtube_dl/extractor/nextmedia.py           |  6 +++---
 youtube_dl/extractor/noco.py                |  2 +-
 youtube_dl/extractor/normalboots.py         |  2 +-
 youtube_dl/extractor/nova.py                |  2 +-
 youtube_dl/extractor/npr.py                 |  2 +-
 youtube_dl/extractor/ntvru.py               |  2 +-
 youtube_dl/extractor/orf.py                 |  6 +++---
 youtube_dl/extractor/philharmoniedeparis.py |  2 +-
 youtube_dl/extractor/photobucket.py         |  2 +-
 youtube_dl/extractor/pornhd.py              |  2 +-
 youtube_dl/extractor/pornovoisines.py       |  2 +-
 youtube_dl/extractor/pyvideo.py             |  2 +-
 youtube_dl/extractor/qqmusic.py             | 10 +++++-----
 youtube_dl/extractor/rai.py                 |  4 ++--
 youtube_dl/extractor/redtube.py             |  2 +-
 youtube_dl/extractor/ringtv.py              |  2 +-
 youtube_dl/extractor/rtve.py                |  4 ++--
 youtube_dl/extractor/ruhd.py                |  2 +-
 youtube_dl/extractor/rutube.py              |  6 +++---
 youtube_dl/extractor/screenjunkies.py       |  2 +-
 youtube_dl/extractor/senateisvp.py          |  2 +-
 youtube_dl/extractor/shared.py              |  2 +-
 youtube_dl/extractor/sport5.py              |  2 +-
 youtube_dl/extractor/ssa.py                 |  2 +-
 youtube_dl/extractor/sztvhu.py              |  2 +-
 youtube_dl/extractor/teamcoco.py            |  2 +-
 youtube_dl/extractor/tele13.py              |  2 +-
 youtube_dl/extractor/tf1.py                 |  2 +-
 youtube_dl/extractor/thvideo.py             |  2 +-
 youtube_dl/extractor/tinypic.py             |  2 +-
 youtube_dl/extractor/tlc.py                 |  2 +-
 youtube_dl/extractor/toypics.py             |  2 +-
 youtube_dl/extractor/traileraddict.py       |  2 +-
 youtube_dl/extractor/trollvids.py           |  2 +-
 youtube_dl/extractor/tumblr.py              |  2 +-
 youtube_dl/extractor/tv2.py                 |  4 ++--
 youtube_dl/extractor/tvc.py                 |  4 ++--
 youtube_dl/extractor/tvplay.py              |  2 +-
 youtube_dl/extractor/ubu.py                 |  2 +-
 youtube_dl/extractor/unistra.py             |  2 +-
 youtube_dl/extractor/vbox7.py               |  2 +-
 youtube_dl/extractor/veoh.py                |  2 +-
 youtube_dl/extractor/vesti.py               |  2 +-
 youtube_dl/extractor/vgtv.py                |  4 ++--
 youtube_dl/extractor/videott.py             |  2 +-
 youtube_dl/extractor/viidea.py              |  2 +-
 youtube_dl/extractor/vube.py                |  2 +-
 youtube_dl/extractor/vuclip.py              |  2 +-
 youtube_dl/extractor/walla.py               |  2 +-
 youtube_dl/extractor/wat.py                 |  2 +-
 youtube_dl/extractor/wdr.py                 |  2 +-
 youtube_dl/extractor/weiqitv.py             |  2 +-
 youtube_dl/extractor/wimp.py                |  2 +-
 youtube_dl/extractor/xbef.py                |  2 +-
 youtube_dl/extractor/yam.py                 |  2 +-
 youtube_dl/extractor/ynet.py                |  2 +-
 127 files changed, 156 insertions(+), 156 deletions(-)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index 6a29e587f..b584277be 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class ABCIE(InfoExtractor):
     IE_NAME = 'abc.net.au'
-    _VALID_URL = r'http://www\.abc\.net\.au/news/(?:[^/]+/){1,2}(?P<id>\d+)'
+    _VALID_URL = r'https?://www\.abc\.net\.au/news/(?:[^/]+/){1,2}(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.abc.net.au/news/2014-11-05/australia-to-staff-ebola-treatment-centre-in-sierra-leone/5868334',
diff --git a/youtube_dl/extractor/addanime.py b/youtube_dl/extractor/addanime.py
index e3e6d2113..fb1cc02e1 100644
--- a/youtube_dl/extractor/addanime.py
+++ b/youtube_dl/extractor/addanime.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class AddAnimeIE(InfoExtractor):
-    _VALID_URL = r'http://(?:\w+\.)?add-anime\.net/(?:watch_video\.php\?(?:.*?)v=|video/)(?P<id>[\w_]+)'
+    _VALID_URL = r'https?://(?:\w+\.)?add-anime\.net/(?:watch_video\.php\?(?:.*?)v=|video/)(?P<id>[\w_]+)'
     _TESTS = [{
         'url': 'http://www.add-anime.net/watch_video.php?v=24MR3YO5SAS9',
         'md5': '72954ea10bc979ab5e2eb288b21425a0',
diff --git a/youtube_dl/extractor/aftonbladet.py b/youtube_dl/extractor/aftonbladet.py
index e0518cf26..d548592fe 100644
--- a/youtube_dl/extractor/aftonbladet.py
+++ b/youtube_dl/extractor/aftonbladet.py
@@ -6,7 +6,7 @@ from ..utils import int_or_none
 
 
 class AftonbladetIE(InfoExtractor):
-    _VALID_URL = r'http://tv\.aftonbladet\.se/abtv/articles/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://tv\.aftonbladet\.se/abtv/articles/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://tv.aftonbladet.se/abtv/articles/36015',
         'info_dict': {
diff --git a/youtube_dl/extractor/aljazeera.py b/youtube_dl/extractor/aljazeera.py
index cddcaa489..b081695d8 100644
--- a/youtube_dl/extractor/aljazeera.py
+++ b/youtube_dl/extractor/aljazeera.py
@@ -4,7 +4,7 @@ from .common import InfoExtractor
 
 
 class AlJazeeraIE(InfoExtractor):
-    _VALID_URL = r'http://www\.aljazeera\.com/programmes/.*?/(?P<id>[^/]+)\.html'
+    _VALID_URL = r'https?://www\.aljazeera\.com/programmes/.*?/(?P<id>[^/]+)\.html'
 
     _TEST = {
         'url': 'http://www.aljazeera.com/programmes/the-slum/2014/08/deliverance-201482883754237240.html',
diff --git a/youtube_dl/extractor/aol.py b/youtube_dl/extractor/aol.py
index b761b2cc4..95a99c6b0 100644
--- a/youtube_dl/extractor/aol.py
+++ b/youtube_dl/extractor/aol.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 class AolIE(InfoExtractor):
     IE_NAME = 'on.aol.com'
-    _VALID_URL = r'(?:aol-video:|http://on\.aol\.com/video/.*-)(?P<id>[0-9]+)(?:$|\?)'
+    _VALID_URL = r'(?:aol-video:|https?://on\.aol\.com/video/.*-)(?P<id>[0-9]+)(?:$|\?)'
 
     _TESTS = [{
         'url': 'http://on.aol.com/video/u-s--official-warns-of-largest-ever-irs-phone-scam-518167793?icid=OnHomepageC2Wide_MustSee_Img',
@@ -25,7 +25,7 @@ class AolIE(InfoExtractor):
 
 class AolFeaturesIE(InfoExtractor):
     IE_NAME = 'features.aol.com'
-    _VALID_URL = r'http://features\.aol\.com/video/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://features\.aol\.com/video/(?P<id>[^/?#]+)'
 
     _TESTS = [{
         'url': 'http://features.aol.com/video/behind-secret-second-careers-late-night-talk-show-hosts',
diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 3e119e21b..ae0f27dcb 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -23,7 +23,7 @@ from ..utils import (
 
 
 class ArteTvIE(InfoExtractor):
-    _VALID_URL = r'http://videos\.arte\.tv/(?P<lang>fr|de|en|es)/.*-(?P<id>.*?)\.html'
+    _VALID_URL = r'https?://videos\.arte\.tv/(?P<lang>fr|de|en|es)/.*-(?P<id>.*?)\.html'
     IE_NAME = 'arte.tv'
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/azubu.py b/youtube_dl/extractor/azubu.py
index 011edf128..1805b7312 100644
--- a/youtube_dl/extractor/azubu.py
+++ b/youtube_dl/extractor/azubu.py
@@ -98,7 +98,7 @@ class AzubuIE(InfoExtractor):
 
 
 class AzubuLiveIE(InfoExtractor):
-    _VALID_URL = r'http://www.azubu.tv/(?P<id>[^/]+)$'
+    _VALID_URL = r'https?://www.azubu.tv/(?P<id>[^/]+)$'
 
     _TEST = {
         'url': 'http://www.azubu.tv/MarsTVMDLen',
diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py
index 76b21e596..234a661d3 100644
--- a/youtube_dl/extractor/baidu.py
+++ b/youtube_dl/extractor/baidu.py
@@ -9,7 +9,7 @@ from ..utils import unescapeHTML
 
 class BaiduVideoIE(InfoExtractor):
     IE_DESC = '百度视频'
-    _VALID_URL = r'http://v\.baidu\.com/(?P<type>[a-z]+)/(?P<id>\d+)\.htm'
+    _VALID_URL = r'https?://v\.baidu\.com/(?P<type>[a-z]+)/(?P<id>\d+)\.htm'
     _TESTS = [{
         'url': 'http://v.baidu.com/comic/1069.htm?frp=bdbrand&q=%E4%B8%AD%E5%8D%8E%E5%B0%8F%E5%BD%93%E5%AE%B6',
         'info_dict': {
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index c3176700a..2dfcee98d 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -942,7 +942,7 @@ class BBCIE(BBCCoUkIE):
 
 
 class BBCCoUkArticleIE(InfoExtractor):
-    _VALID_URL = r'http://www.bbc.co.uk/programmes/articles/(?P<id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'https?://www.bbc.co.uk/programmes/articles/(?P<id>[a-zA-Z0-9]+)'
     IE_NAME = 'bbc.co.uk:article'
     IE_DESC = 'BBC articles'
 
diff --git a/youtube_dl/extractor/behindkink.py b/youtube_dl/extractor/behindkink.py
index 1bdc25812..9bca853b3 100644
--- a/youtube_dl/extractor/behindkink.py
+++ b/youtube_dl/extractor/behindkink.py
@@ -8,7 +8,7 @@ from ..utils import url_basename
 
 
 class BehindKinkIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?behindkink\.com/(?P<year>[0-9]{4})/(?P<month>[0-9]{2})/(?P<day>[0-9]{2})/(?P<id>[^/#?_]+)'
+    _VALID_URL = r'https?://(?:www\.)?behindkink\.com/(?P<year>[0-9]{4})/(?P<month>[0-9]{2})/(?P<day>[0-9]{2})/(?P<id>[^/#?_]+)'
     _TEST = {
         'url': 'http://www.behindkink.com/2014/12/05/what-are-you-passionate-about-marley-blaze/',
         'md5': '507b57d8fdcd75a41a9a7bdb7989c762',
diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 59beb11bc..8baff2041 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class BiliBiliIE(InfoExtractor):
-    _VALID_URL = r'http://www\.bilibili\.(?:tv|com)/video/av(?P<id>\d+)(?:/index_(?P<page_num>\d+).html)?'
+    _VALID_URL = r'https?://www\.bilibili\.(?:tv|com)/video/av(?P<id>\d+)(?:/index_(?P<page_num>\d+).html)?'
 
     _TESTS = [{
         'url': 'http://www.bilibili.tv/video/av1074402/',
diff --git a/youtube_dl/extractor/bokecc.py b/youtube_dl/extractor/bokecc.py
index 122a1cbb6..86a7f4d7d 100644
--- a/youtube_dl/extractor/bokecc.py
+++ b/youtube_dl/extractor/bokecc.py
@@ -33,7 +33,7 @@ class BokeCCBaseIE(InfoExtractor):
 
 class BokeCCIE(BokeCCBaseIE):
     _IE_DESC = 'CC视频'
-    _VALID_URL = r'http://union\.bokecc\.com/playvideo\.bo\?(?P<query>.*)'
+    _VALID_URL = r'https?://union\.bokecc\.com/playvideo\.bo\?(?P<query>.*)'
 
     _TESTS = [{
         'url': 'http://union.bokecc.com/playvideo.bo?vid=E44D40C15E65EA30&uid=CD0C5D3C8614B28B',
diff --git a/youtube_dl/extractor/bpb.py b/youtube_dl/extractor/bpb.py
index c28e72927..6ad45a1e6 100644
--- a/youtube_dl/extractor/bpb.py
+++ b/youtube_dl/extractor/bpb.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class BpbIE(InfoExtractor):
     IE_DESC = 'Bundeszentrale für politische Bildung'
-    _VALID_URL = r'http://www\.bpb\.de/mediathek/(?P<id>[0-9]+)/'
+    _VALID_URL = r'https?://www\.bpb\.de/mediathek/(?P<id>[0-9]+)/'
 
     _TEST = {
         'url': 'http://www.bpb.de/mediathek/297/joachim-gauck-zu-1989-und-die-erinnerung-an-die-ddr',
diff --git a/youtube_dl/extractor/breakcom.py b/youtube_dl/extractor/breakcom.py
index aa08051b1..725859b4d 100644
--- a/youtube_dl/extractor/breakcom.py
+++ b/youtube_dl/extractor/breakcom.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class BreakIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?break\.com/video/(?:[^/]+/)*.+-(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?break\.com/video/(?:[^/]+/)*.+-(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.break.com/video/when-girls-act-like-guys-2468056',
         'info_dict': {
diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
index 897f3a104..dd4d96cec 100644
--- a/youtube_dl/extractor/camdemy.py
+++ b/youtube_dl/extractor/camdemy.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class CamdemyIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?camdemy\.com/media/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?camdemy\.com/media/(?P<id>\d+)'
     _TESTS = [{
         # single file
         'url': 'http://www.camdemy.com/media/5181/',
@@ -104,7 +104,7 @@ class CamdemyIE(InfoExtractor):
 
 
 class CamdemyFolderIE(InfoExtractor):
-    _VALID_URL = r'http://www.camdemy.com/folder/(?P<id>\d+)'
+    _VALID_URL = r'https?://www.camdemy.com/folder/(?P<id>\d+)'
     _TESTS = [{
         # links with trailing slash
         'url': 'http://www.camdemy.com/folder/450',
diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 8ddcc5097..f23bac9a1 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 class CBSNewsIE(ThePlatformIE):
     IE_DESC = 'CBS News'
-    _VALID_URL = r'http://(?:www\.)?cbsnews\.com/(?:news|videos)/(?P<id>[\da-z_-]+)'
+    _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/(?:news|videos)/(?P<id>[\da-z_-]+)'
 
     _TESTS = [
         {
@@ -96,7 +96,7 @@ class CBSNewsIE(ThePlatformIE):
 
 class CBSNewsLiveVideoIE(InfoExtractor):
     IE_DESC = 'CBS News Live Videos'
-    _VALID_URL = r'http://(?:www\.)?cbsnews\.com/live/video/(?P<id>[\da-z_-]+)'
+    _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/live/video/(?P<id>[\da-z_-]+)'
 
     _TEST = {
         'url': 'http://www.cbsnews.com/live/video/clinton-sanders-prepare-to-face-off-in-nh/',
diff --git a/youtube_dl/extractor/cbssports.py b/youtube_dl/extractor/cbssports.py
index ae47e74cc..549ae32f3 100644
--- a/youtube_dl/extractor/cbssports.py
+++ b/youtube_dl/extractor/cbssports.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 
 
 class CBSSportsIE(InfoExtractor):
-    _VALID_URL = r'http://www\.cbssports\.com/video/player/(?P<section>[^/]+)/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://www\.cbssports\.com/video/player/(?P<section>[^/]+)/(?P<id>[^/]+)'
 
     _TEST = {
         'url': 'http://www.cbssports.com/video/player/tennis/318462531970/0/us-open-flashbacks-1990s',
diff --git a/youtube_dl/extractor/cliphunter.py b/youtube_dl/extractor/cliphunter.py
index 2996b6b09..19f8b397e 100644
--- a/youtube_dl/extractor/cliphunter.py
+++ b/youtube_dl/extractor/cliphunter.py
@@ -19,7 +19,7 @@ def _decode(s):
 class CliphunterIE(InfoExtractor):
     IE_NAME = 'cliphunter'
 
-    _VALID_URL = r'''(?x)http://(?:www\.)?cliphunter\.com/w/
+    _VALID_URL = r'''(?x)https?://(?:www\.)?cliphunter\.com/w/
         (?P<id>[0-9]+)/
         (?P<seo>.+?)(?:$|[#\?])
     '''
diff --git a/youtube_dl/extractor/clipsyndicate.py b/youtube_dl/extractor/clipsyndicate.py
index 8306d6fb7..0b6ad895f 100644
--- a/youtube_dl/extractor/clipsyndicate.py
+++ b/youtube_dl/extractor/clipsyndicate.py
@@ -8,7 +8,7 @@ from ..utils import (
 
 
 class ClipsyndicateIE(InfoExtractor):
-    _VALID_URL = r'http://(?:chic|www)\.clipsyndicate\.com/video/play(list/\d+)?/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:chic|www)\.clipsyndicate\.com/video/play(list/\d+)?/(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.clipsyndicate.com/video/play/4629301/brick_briscoe',
diff --git a/youtube_dl/extractor/clubic.py b/youtube_dl/extractor/clubic.py
index 1dfa7c12e..2fba93543 100644
--- a/youtube_dl/extractor/clubic.py
+++ b/youtube_dl/extractor/clubic.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class ClubicIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?clubic\.com/video/(?:[^/]+/)*video.*-(?P<id>[0-9]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?clubic\.com/video/(?:[^/]+/)*video.*-(?P<id>[0-9]+)\.html'
 
     _TESTS = [{
         'url': 'http://www.clubic.com/video/clubic-week/video-clubic-week-2-0-le-fbi-se-lance-dans-la-photo-d-identite-448474.html',
diff --git a/youtube_dl/extractor/comcarcoff.py b/youtube_dl/extractor/comcarcoff.py
index 7dff68492..e697d1410 100644
--- a/youtube_dl/extractor/comcarcoff.py
+++ b/youtube_dl/extractor/comcarcoff.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class ComCarCoffIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?comediansincarsgettingcoffee\.com/(?P<id>[a-z0-9\-]*)'
+    _VALID_URL = r'https?://(?:www\.)?comediansincarsgettingcoffee\.com/(?P<id>[a-z0-9\-]*)'
     _TESTS = [{
         'url': 'http://comediansincarsgettingcoffee.com/miranda-sings-happy-thanksgiving-miranda/',
         'info_dict': {
diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index 6f92ae2ed..054978ff2 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -45,7 +45,7 @@ class CondeNastIE(InfoExtractor):
         'wmagazine': 'W Magazine',
     }
 
-    _VALID_URL = r'http://(?:video|www|player)\.(?P<site>%s)\.com/(?P<type>watch|series|video|embed(?:js)?)/(?P<id>[^/?#]+)' % '|'.join(_SITES.keys())
+    _VALID_URL = r'https?://(?:video|www|player)\.(?P<site>%s)\.com/(?P<type>watch|series|video|embed(?:js)?)/(?P<id>[^/?#]+)' % '|'.join(_SITES.keys())
     IE_DESC = 'Condé Nast media group: %s' % ', '.join(sorted(_SITES.values()))
 
     EMBED_URL = r'(?:https?:)?//player\.(?P<site>%s)\.com/(?P<type>embed(?:js)?)/.+?' % '|'.join(_SITES.keys())
diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index b8b9d058d..84b36f44c 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -15,7 +15,7 @@ from .senateisvp import SenateISVPIE
 
 
 class CSpanIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?c-span\.org/video/\?(?P<id>[0-9a-f]+)'
+    _VALID_URL = r'https?://(?:www\.)?c-span\.org/video/\?(?P<id>[0-9a-f]+)'
     IE_DESC = 'C-SPAN'
     _TESTS = [{
         'url': 'http://www.c-span.org/video/?313572-1/HolderonV',
diff --git a/youtube_dl/extractor/ctsnews.py b/youtube_dl/extractor/ctsnews.py
index 45049bf37..1622fc844 100644
--- a/youtube_dl/extractor/ctsnews.py
+++ b/youtube_dl/extractor/ctsnews.py
@@ -8,7 +8,7 @@ from ..utils import parse_iso8601, ExtractorError
 class CtsNewsIE(InfoExtractor):
     IE_DESC = '華視新聞'
     # https connection failed (Connection reset)
-    _VALID_URL = r'http://news\.cts\.com\.tw/[a-z]+/[a-z]+/\d+/(?P<id>\d+)\.html'
+    _VALID_URL = r'https?://news\.cts\.com\.tw/[a-z]+/[a-z]+/\d+/(?P<id>\d+)\.html'
     _TESTS = [{
         'url': 'http://news.cts.com.tw/cts/international/201501/201501291578109.html',
         'md5': 'a9875cb790252b08431186d741beaabe',
diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index aa2c09eb6..9099f5046 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -6,7 +6,7 @@ from ..compat import compat_str
 
 
 class DctpTvIE(InfoExtractor):
-    _VALID_URL = r'http://www.dctp.tv/(#/)?filme/(?P<id>.+?)/$'
+    _VALID_URL = r'https?://www.dctp.tv/(#/)?filme/(?P<id>.+?)/$'
     _TEST = {
         'url': 'http://www.dctp.tv/filme/videoinstallation-fuer-eine-kaufhausfassade/',
         'info_dict': {
diff --git a/youtube_dl/extractor/defense.py b/youtube_dl/extractor/defense.py
index 98e3aedfd..9fe144e14 100644
--- a/youtube_dl/extractor/defense.py
+++ b/youtube_dl/extractor/defense.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 class DefenseGouvFrIE(InfoExtractor):
     IE_NAME = 'defense.gouv.fr'
-    _VALID_URL = r'http://.*?\.defense\.gouv\.fr/layout/set/ligthboxvideo/base-de-medias/webtv/(?P<id>[^/?#]*)'
+    _VALID_URL = r'https?://.*?\.defense\.gouv\.fr/layout/set/ligthboxvideo/base-de-medias/webtv/(?P<id>[^/?#]*)'
 
     _TEST = {
         'url': 'http://www.defense.gouv.fr/layout/set/ligthboxvideo/base-de-medias/webtv/attaque-chimique-syrienne-du-21-aout-2013-1',
diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index bdc768c78..bcb670945 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -10,7 +10,7 @@ from ..compat import (compat_str, compat_basestring)
 
 class DouyuTVIE(InfoExtractor):
     IE_DESC = '斗鱼'
-    _VALID_URL = r'http://(?:www\.)?douyutv\.com/(?P<id>[A-Za-z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?douyutv\.com/(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'http://www.douyutv.com/iseven',
         'info_dict': {
diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index a638c827c..1e7dcada6 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -10,7 +10,7 @@ from ..utils import int_or_none
 
 
 class DPlayIE(InfoExtractor):
-    _VALID_URL = r'http://(?P<domain>it\.dplay\.com|www\.dplay\.(?:dk|se|no))/[^/]+/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?P<domain>it\.dplay\.com|www\.dplay\.(?:dk|se|no))/[^/]+/(?P<id>[^/?#]+)'
 
     _TESTS = [{
         'url': 'http://it.dplay.com/take-me-out/stagione-1-episodio-25/',
diff --git a/youtube_dl/extractor/dreisat.py b/youtube_dl/extractor/dreisat.py
index 028144f20..0040e70d4 100644
--- a/youtube_dl/extractor/dreisat.py
+++ b/youtube_dl/extractor/dreisat.py
@@ -7,7 +7,7 @@ from .zdf import ZDFIE
 
 class DreiSatIE(ZDFIE):
     IE_NAME = '3sat'
-    _VALID_URL = r'(?:http://)?(?:www\.)?3sat\.de/mediathek/(?:index\.php|mediathek\.php)?\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)$'
+    _VALID_URL = r'(?:https?://)?(?:www\.)?3sat\.de/mediathek/(?:index\.php|mediathek\.php)?\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)$'
     _TESTS = [
         {
             'url': 'http://www.3sat.de/mediathek/index.php?mode=play&obj=45918',
diff --git a/youtube_dl/extractor/dvtv.py b/youtube_dl/extractor/dvtv.py
index c1a4bc757..974c69dbc 100644
--- a/youtube_dl/extractor/dvtv.py
+++ b/youtube_dl/extractor/dvtv.py
@@ -15,7 +15,7 @@ class DVTVIE(InfoExtractor):
     IE_NAME = 'dvtv'
     IE_DESC = 'http://video.aktualne.cz/'
 
-    _VALID_URL = r'http://video\.aktualne\.cz/(?:[^/]+/)+r~(?P<id>[0-9a-f]{32})'
+    _VALID_URL = r'https?://video\.aktualne\.cz/(?:[^/]+/)+r~(?P<id>[0-9a-f]{32})'
 
     _TESTS = [{
         'url': 'http://video.aktualne.cz/dvtv/vondra-o-ceskem-stoleti-pri-pohledu-na-havla-mi-bylo-trapne/r~e5efe9ca855511e4833a0025900fea04/',
diff --git a/youtube_dl/extractor/echomsk.py b/youtube_dl/extractor/echomsk.py
index d2d94049d..6b7cc652f 100644
--- a/youtube_dl/extractor/echomsk.py
+++ b/youtube_dl/extractor/echomsk.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 
 
 class EchoMskIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?echo\.msk\.ru/sounds/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?echo\.msk\.ru/sounds/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.echo.msk.ru/sounds/1464134.html',
         'md5': '2e44b3b78daff5b458e4dbc37f191f7c',
diff --git a/youtube_dl/extractor/exfm.py b/youtube_dl/extractor/exfm.py
index 0c0fe6d65..09ed4f2b5 100644
--- a/youtube_dl/extractor/exfm.py
+++ b/youtube_dl/extractor/exfm.py
@@ -8,7 +8,7 @@ from .common import InfoExtractor
 class ExfmIE(InfoExtractor):
     IE_NAME = 'exfm'
     IE_DESC = 'ex.fm'
-    _VALID_URL = r'http://(?:www\.)?ex\.fm/song/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?ex\.fm/song/(?P<id>[^/]+)'
     _SOUNDCLOUD_URL = r'http://(?:www\.)?api\.soundcloud\.com/tracks/([^/]+)/stream'
     _TESTS = [
         {
diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index 9580f5c0c..508684d2e 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 
 class FC2IE(InfoExtractor):
-    _VALID_URL = r'^http://video\.fc2\.com/(?:[^/]+/)*content/(?P<id>[^/]+)'
+    _VALID_URL = r'^https?://video\.fc2\.com/(?:[^/]+/)*content/(?P<id>[^/]+)'
     IE_NAME = 'fc2'
     _NETRC_MACHINE = 'fc2'
     _TESTS = [{
diff --git a/youtube_dl/extractor/firstpost.py b/youtube_dl/extractor/firstpost.py
index 298227d57..e8936cb24 100644
--- a/youtube_dl/extractor/firstpost.py
+++ b/youtube_dl/extractor/firstpost.py
@@ -4,7 +4,7 @@ from .common import InfoExtractor
 
 
 class FirstpostIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?firstpost\.com/[^/]+/.*-(?P<id>[0-9]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?firstpost\.com/[^/]+/.*-(?P<id>[0-9]+)\.html'
 
     _TEST = {
         'url': 'http://www.firstpost.com/india/india-to-launch-indigenous-aircraft-carrier-monday-1025403.html',
diff --git a/youtube_dl/extractor/firsttv.py b/youtube_dl/extractor/firsttv.py
index 510d4b108..98b165143 100644
--- a/youtube_dl/extractor/firsttv.py
+++ b/youtube_dl/extractor/firsttv.py
@@ -8,7 +8,7 @@ from ..utils import int_or_none
 class FirstTVIE(InfoExtractor):
     IE_NAME = '1tv'
     IE_DESC = 'Первый канал'
-    _VALID_URL = r'http://(?:www\.)?1tv\.ru/(?:[^/]+/)+(?P<id>.+)'
+    _VALID_URL = r'https?://(?:www\.)?1tv\.ru/(?:[^/]+/)+(?P<id>.+)'
 
     _TESTS = [{
         'url': 'http://www.1tv.ru/videoarchive/73390',
diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index 5f6e65dae..a3a291599 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 class FKTVIE(InfoExtractor):
     IE_NAME = 'fernsehkritik.tv'
-    _VALID_URL = r'http://(?:www\.)?fernsehkritik\.tv/folge-(?P<id>[0-9]+)(?:/.*)?'
+    _VALID_URL = r'https?://(?:www\.)?fernsehkritik\.tv/folge-(?P<id>[0-9]+)(?:/.*)?'
 
     _TEST = {
         'url': 'http://fernsehkritik.tv/folge-1',
diff --git a/youtube_dl/extractor/footyroom.py b/youtube_dl/extractor/footyroom.py
index 370fd006f..d2503ae2e 100644
--- a/youtube_dl/extractor/footyroom.py
+++ b/youtube_dl/extractor/footyroom.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 
 class FootyRoomIE(InfoExtractor):
-    _VALID_URL = r'http://footyroom\.com/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://footyroom\.com/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://footyroom.com/schalke-04-0-2-real-madrid-2015-02/',
         'info_dict': {
diff --git a/youtube_dl/extractor/foxgay.py b/youtube_dl/extractor/foxgay.py
index 08b8ea362..70c1a815d 100644
--- a/youtube_dl/extractor/foxgay.py
+++ b/youtube_dl/extractor/foxgay.py
@@ -4,7 +4,7 @@ from .common import InfoExtractor
 
 
 class FoxgayIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?foxgay\.com/videos/(?:\S+-)?(?P<id>\d+)\.shtml'
+    _VALID_URL = r'https?://(?:www\.)?foxgay\.com/videos/(?:\S+-)?(?P<id>\d+)\.shtml'
     _TEST = {
         'url': 'http://foxgay.com/videos/fuck-turkish-style-2582.shtml',
         'md5': '80d72beab5d04e1655a56ad37afe6841',
diff --git a/youtube_dl/extractor/franceinter.py b/youtube_dl/extractor/franceinter.py
index 0388ba00c..2369f868d 100644
--- a/youtube_dl/extractor/franceinter.py
+++ b/youtube_dl/extractor/franceinter.py
@@ -6,7 +6,7 @@ from ..utils import int_or_none
 
 
 class FranceInterIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?franceinter\.fr/player/reecouter\?play=(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?franceinter\.fr/player/reecouter\?play=(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.franceinter.fr/player/reecouter?play=793962',
         'md5': '4764932e466e6f6c79c317d2e74f6884',
diff --git a/youtube_dl/extractor/freevideo.py b/youtube_dl/extractor/freevideo.py
index c7bec027b..cd8423a6f 100644
--- a/youtube_dl/extractor/freevideo.py
+++ b/youtube_dl/extractor/freevideo.py
@@ -5,7 +5,7 @@ from ..utils import ExtractorError
 
 
 class FreeVideoIE(InfoExtractor):
-    _VALID_URL = r'^http://www.freevideo.cz/vase-videa/(?P<id>[^.]+)\.html(?:$|[?#])'
+    _VALID_URL = r'^https?://www.freevideo.cz/vase-videa/(?P<id>[^.]+)\.html(?:$|[?#])'
 
     _TEST = {
         'url': 'http://www.freevideo.cz/vase-videa/vysukany-zadecek-22033.html',
diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index f6b9046f9..cbcddcb7c 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -10,7 +10,7 @@ from .youtube import YoutubeIE
 
 
 class GamekingsIE(InfoExtractor):
-    _VALID_URL = r'http://www\.gamekings\.nl/(?:videos|nieuws)/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://www\.gamekings\.nl/(?:videos|nieuws)/(?P<id>[^/]+)'
     _TESTS = [{
         # YouTube embed video
         'url': 'http://www.gamekings.nl/videos/phoenix-wright-ace-attorney-dual-destinies-review/',
diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index b3f1bafcc..4ffdd7515 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class GameSpotIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?gamespot\.com/.*-(?P<id>\d+)/?'
+    _VALID_URL = r'https?://(?:www\.)?gamespot\.com/.*-(?P<id>\d+)/?'
     _TESTS = [{
         'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',
         'md5': 'b2a30deaa8654fcccd43713a6b6a4825',
diff --git a/youtube_dl/extractor/gamestar.py b/youtube_dl/extractor/gamestar.py
index 590ccf526..69058a583 100644
--- a/youtube_dl/extractor/gamestar.py
+++ b/youtube_dl/extractor/gamestar.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class GameStarIE(InfoExtractor):
-    _VALID_URL = r'http://www\.gamestar\.de/videos/.*,(?P<id>[0-9]+)\.html'
+    _VALID_URL = r'https?://www\.gamestar\.de/videos/.*,(?P<id>[0-9]+)\.html'
     _TEST = {
         'url': 'http://www.gamestar.de/videos/trailer,3/hobbit-3-die-schlacht-der-fuenf-heere,76110.html',
         'md5': '96974ecbb7fd8d0d20fca5a00810cea7',
diff --git a/youtube_dl/extractor/gametrailers.py b/youtube_dl/extractor/gametrailers.py
index c3f031d9c..1e7948ab8 100644
--- a/youtube_dl/extractor/gametrailers.py
+++ b/youtube_dl/extractor/gametrailers.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class GametrailersIE(InfoExtractor):
-    _VALID_URL = r'http://www\.gametrailers\.com/videos/view/[^/]+/(?P<id>.+)'
+    _VALID_URL = r'https?://www\.gametrailers\.com/videos/view/[^/]+/(?P<id>.+)'
 
     _TEST = {
         'url': 'http://www.gametrailers.com/videos/view/gametrailers-com/116437-Just-Cause-3-Review',
diff --git a/youtube_dl/extractor/hotnewhiphop.py b/youtube_dl/extractor/hotnewhiphop.py
index 31e219945..efc3e8429 100644
--- a/youtube_dl/extractor/hotnewhiphop.py
+++ b/youtube_dl/extractor/hotnewhiphop.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class HotNewHipHopIE(InfoExtractor):
-    _VALID_URL = r'http://www\.hotnewhiphop\.com/.*\.(?P<id>.*)\.html'
+    _VALID_URL = r'https?://www\.hotnewhiphop\.com/.*\.(?P<id>.*)\.html'
     _TEST = {
         'url': 'http://www.hotnewhiphop.com/freddie-gibbs-lay-it-down-song.1435540.html',
         'md5': '2c2cd2f76ef11a9b3b581e8b232f3d96',
diff --git a/youtube_dl/extractor/hypem.py b/youtube_dl/extractor/hypem.py
index b3706fe6d..e0ab31802 100644
--- a/youtube_dl/extractor/hypem.py
+++ b/youtube_dl/extractor/hypem.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class HypemIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?hypem\.com/track/(?P<id>[^/]+)/'
+    _VALID_URL = r'https?://(?:www\.)?hypem\.com/track/(?P<id>[^/]+)/'
     _TEST = {
         'url': 'http://hypem.com/track/1v6ga/BODYWORK+-+TAME',
         'md5': 'b9cc91b5af8995e9f0c1cee04c575828',
diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index b61b2dc4e..8bed8ccd0 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -12,7 +12,7 @@ from ..utils import (
 class ImdbIE(InfoExtractor):
     IE_NAME = 'imdb'
     IE_DESC = 'Internet Movie Database trailers'
-    _VALID_URL = r'http://(?:www|m)\.imdb\.com/video/imdb/vi(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www|m)\.imdb\.com/video/imdb/vi(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://www.imdb.com/video/imdb/vi2524815897',
@@ -70,7 +70,7 @@ class ImdbIE(InfoExtractor):
 class ImdbListIE(InfoExtractor):
     IE_NAME = 'imdb:list'
     IE_DESC = 'Internet Movie Database lists'
-    _VALID_URL = r'http://www\.imdb\.com/list/(?P<id>[\da-zA-Z_-]{11})'
+    _VALID_URL = r'https?://www\.imdb\.com/list/(?P<id>[\da-zA-Z_-]{11})'
     _TEST = {
         'url': 'http://www.imdb.com/list/JFs9NWw6XI0',
         'info_dict': {
diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index e7c0cb3f6..1a4c64713 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -165,7 +165,7 @@ class IqiyiIE(InfoExtractor):
     IE_NAME = 'iqiyi'
     IE_DESC = '爱奇艺'
 
-    _VALID_URL = r'http://(?:[^.]+\.)?iqiyi\.com/.+\.html'
+    _VALID_URL = r'https?://(?:[^.]+\.)?iqiyi\.com/.+\.html'
 
     _NETRC_MACHINE = 'iqiyi'
 
diff --git a/youtube_dl/extractor/jadorecettepub.py b/youtube_dl/extractor/jadorecettepub.py
index 063e86de4..158c09a33 100644
--- a/youtube_dl/extractor/jadorecettepub.py
+++ b/youtube_dl/extractor/jadorecettepub.py
@@ -9,7 +9,7 @@ from .youtube import YoutubeIE
 
 
 class JadoreCettePubIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?jadorecettepub\.com/[0-9]{4}/[0-9]{2}/(?P<id>.*?)\.html'
+    _VALID_URL = r'https?://(?:www\.)?jadorecettepub\.com/[0-9]{4}/[0-9]{2}/(?P<id>.*?)\.html'
 
     _TEST = {
         'url': 'http://www.jadorecettepub.com/2010/12/star-wars-massacre-par-les-japonais.html',
diff --git a/youtube_dl/extractor/jeuxvideo.py b/youtube_dl/extractor/jeuxvideo.py
index 137db873c..1a4227f6b 100644
--- a/youtube_dl/extractor/jeuxvideo.py
+++ b/youtube_dl/extractor/jeuxvideo.py
@@ -8,7 +8,7 @@ from .common import InfoExtractor
 
 
 class JeuxVideoIE(InfoExtractor):
-    _VALID_URL = r'http://.*?\.jeuxvideo\.com/.*/(.*?)\.htm'
+    _VALID_URL = r'https?://.*?\.jeuxvideo\.com/.*/(.*?)\.htm'
 
     _TESTS = [{
         'url': 'http://www.jeuxvideo.com/reportages-videos-jeux/0004/00046170/tearaway-playstation-vita-gc-2013-tearaway-nous-presente-ses-papiers-d-identite-00115182.htm',
diff --git a/youtube_dl/extractor/karaoketv.py b/youtube_dl/extractor/karaoketv.py
index 06daf5a89..b4c30b7f3 100644
--- a/youtube_dl/extractor/karaoketv.py
+++ b/youtube_dl/extractor/karaoketv.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class KaraoketvIE(InfoExtractor):
-    _VALID_URL = r'http://karaoketv\.co\.il/\?container=songs&id=(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://karaoketv\.co\.il/\?container=songs&id=(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://karaoketv.co.il/?container=songs&id=171568',
         'info_dict': {
diff --git a/youtube_dl/extractor/karrierevideos.py b/youtube_dl/extractor/karrierevideos.py
index bed94bc93..2cb04e533 100644
--- a/youtube_dl/extractor/karrierevideos.py
+++ b/youtube_dl/extractor/karrierevideos.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class KarriereVideosIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?karrierevideos\.at(?:/[^/]+)+/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?karrierevideos\.at(?:/[^/]+)+/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://www.karrierevideos.at/berufsvideos/mittlere-hoehere-schulen/altenpflegerin',
         'info_dict': {
diff --git a/youtube_dl/extractor/kontrtube.py b/youtube_dl/extractor/kontrtube.py
index a59c529f4..704bd7b34 100644
--- a/youtube_dl/extractor/kontrtube.py
+++ b/youtube_dl/extractor/kontrtube.py
@@ -13,7 +13,7 @@ from ..utils import (
 class KontrTubeIE(InfoExtractor):
     IE_NAME = 'kontrtube'
     IE_DESC = 'KontrTube.ru - Труба зовёт'
-    _VALID_URL = r'http://(?:www\.)?kontrtube\.ru/videos/(?P<id>\d+)/(?P<display_id>[^/]+)/'
+    _VALID_URL = r'https?://(?:www\.)?kontrtube\.ru/videos/(?P<id>\d+)/(?P<display_id>[^/]+)/'
 
     _TEST = {
         'url': 'http://www.kontrtube.ru/videos/2678/nad-olimpiyskoy-derevney-v-sochi-podnyat-rossiyskiy-flag/',
diff --git a/youtube_dl/extractor/ku6.py b/youtube_dl/extractor/ku6.py
index a602980a1..a574408e5 100644
--- a/youtube_dl/extractor/ku6.py
+++ b/youtube_dl/extractor/ku6.py
@@ -4,7 +4,7 @@ from .common import InfoExtractor
 
 
 class Ku6IE(InfoExtractor):
-    _VALID_URL = r'http://v\.ku6\.com/show/(?P<id>[a-zA-Z0-9\-\_]+)(?:\.)*html'
+    _VALID_URL = r'https?://v\.ku6\.com/show/(?P<id>[a-zA-Z0-9\-\_]+)(?:\.)*html'
     _TEST = {
         'url': 'http://v.ku6.com/show/JG-8yS14xzBr4bCn1pu0xw...html',
         'md5': '01203549b9efbb45f4b87d55bdea1ed1',
diff --git a/youtube_dl/extractor/kusi.py b/youtube_dl/extractor/kusi.py
index 931f34c9b..12cc56e44 100644
--- a/youtube_dl/extractor/kusi.py
+++ b/youtube_dl/extractor/kusi.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class KUSIIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?kusi\.com/(?P<path>story/.+|video\?clipId=(?P<clipId>\d+))'
+    _VALID_URL = r'https?://(?:www\.)?kusi\.com/(?P<path>story/.+|video\?clipId=(?P<clipId>\d+))'
     _TESTS = [{
         'url': 'http://www.kusi.com/story/31183873/turko-files-case-closed-put-on-hold',
         'md5': 'f926e7684294cf8cb7bdf8858e1b3988',
diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 45d65e61f..a586308b2 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -55,7 +55,7 @@ class KuwoBaseIE(InfoExtractor):
 class KuwoIE(KuwoBaseIE):
     IE_NAME = 'kuwo:song'
     IE_DESC = '酷我音乐'
-    _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>\d+?)'
+    _VALID_URL = r'https?://www\.kuwo\.cn/yinyue/(?P<id>\d+?)'
     _TESTS = [{
         'url': 'http://www.kuwo.cn/yinyue/635632/',
         'info_dict': {
@@ -134,7 +134,7 @@ class KuwoIE(KuwoBaseIE):
 class KuwoAlbumIE(InfoExtractor):
     IE_NAME = 'kuwo:album'
     IE_DESC = '酷我音乐 - 专辑'
-    _VALID_URL = r'http://www\.kuwo\.cn/album/(?P<id>\d+?)/'
+    _VALID_URL = r'https?://www\.kuwo\.cn/album/(?P<id>\d+?)/'
     _TEST = {
         'url': 'http://www.kuwo.cn/album/502294/',
         'info_dict': {
@@ -170,7 +170,7 @@ class KuwoAlbumIE(InfoExtractor):
 class KuwoChartIE(InfoExtractor):
     IE_NAME = 'kuwo:chart'
     IE_DESC = '酷我音乐 - 排行榜'
-    _VALID_URL = r'http://yinyue\.kuwo\.cn/billboard_(?P<id>[^.]+).htm'
+    _VALID_URL = r'https?://yinyue\.kuwo\.cn/billboard_(?P<id>[^.]+).htm'
     _TEST = {
         'url': 'http://yinyue.kuwo.cn/billboard_香港中文龙虎榜.htm',
         'info_dict': {
@@ -195,7 +195,7 @@ class KuwoChartIE(InfoExtractor):
 class KuwoSingerIE(InfoExtractor):
     IE_NAME = 'kuwo:singer'
     IE_DESC = '酷我音乐 - 歌手'
-    _VALID_URL = r'http://www\.kuwo\.cn/mingxing/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://www\.kuwo\.cn/mingxing/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://www.kuwo.cn/mingxing/bruno+mars/',
         'info_dict': {
@@ -251,7 +251,7 @@ class KuwoSingerIE(InfoExtractor):
 class KuwoCategoryIE(InfoExtractor):
     IE_NAME = 'kuwo:category'
     IE_DESC = '酷我音乐 - 分类'
-    _VALID_URL = r'http://yinyue\.kuwo\.cn/yy/cinfo_(?P<id>\d+?).htm'
+    _VALID_URL = r'https?://yinyue\.kuwo\.cn/yy/cinfo_(?P<id>\d+?).htm'
     _TEST = {
         'url': 'http://yinyue.kuwo.cn/yy/cinfo_86375.htm',
         'info_dict': {
@@ -288,7 +288,7 @@ class KuwoCategoryIE(InfoExtractor):
 class KuwoMvIE(KuwoBaseIE):
     IE_NAME = 'kuwo:mv'
     IE_DESC = '酷我音乐 - MV'
-    _VALID_URL = r'http://www\.kuwo\.cn/mv/(?P<id>\d+?)/'
+    _VALID_URL = r'https?://www\.kuwo\.cn/mv/(?P<id>\d+?)/'
     _TEST = {
         'url': 'http://www.kuwo.cn/mv/6480076/',
         'info_dict': {
diff --git a/youtube_dl/extractor/leeco.py b/youtube_dl/extractor/leeco.py
index df47e88ba..462b752dd 100644
--- a/youtube_dl/extractor/leeco.py
+++ b/youtube_dl/extractor/leeco.py
@@ -28,7 +28,7 @@ from ..utils import (
 
 class LeIE(InfoExtractor):
     IE_DESC = '乐视网'
-    _VALID_URL = r'http://www\.le\.com/ptv/vplay/(?P<id>\d+)\.html'
+    _VALID_URL = r'https?://www\.le\.com/ptv/vplay/(?P<id>\d+)\.html'
 
     _URL_TEMPLATE = 'http://www.le.com/ptv/vplay/%s.html'
 
@@ -196,7 +196,7 @@ class LeIE(InfoExtractor):
 
 
 class LePlaylistIE(InfoExtractor):
-    _VALID_URL = r'http://[a-z]+\.le\.com/[a-z]+/(?P<id>[a-z0-9_]+)'
+    _VALID_URL = r'https?://[a-z]+\.le\.com/[a-z]+/(?P<id>[a-z0-9_]+)'
 
     _TESTS = [{
         'url': 'http://www.le.com/tv/46177.html',
diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index a8fd639cc..ba2f80a75 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -17,7 +17,7 @@ from ..utils import (
 class LifeNewsIE(InfoExtractor):
     IE_NAME = 'lifenews'
     IE_DESC = 'LIFE | NEWS'
-    _VALID_URL = r'http://lifenews\.ru/(?:mobile/)?(?P<section>news|video)/(?P<id>\d+)'
+    _VALID_URL = r'https?://lifenews\.ru/(?:mobile/)?(?P<section>news|video)/(?P<id>\d+)'
 
     _TESTS = [{
         # single video embedded via video/source
@@ -159,7 +159,7 @@ class LifeNewsIE(InfoExtractor):
 
 class LifeEmbedIE(InfoExtractor):
     IE_NAME = 'life:embed'
-    _VALID_URL = r'http://embed\.life\.ru/embed/(?P<id>[\da-f]{32})'
+    _VALID_URL = r'https?://embed\.life\.ru/embed/(?P<id>[\da-f]{32})'
 
     _TEST = {
         'url': 'http://embed.life.ru/embed/e50c2dec2867350528e2574c899b8291',
diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index 1a0625ac3..2599d45c3 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -123,7 +123,7 @@ class LimelightBaseIE(InfoExtractor):
 
 class LimelightMediaIE(LimelightBaseIE):
     IE_NAME = 'limelight'
-    _VALID_URL = r'(?:limelight:media:|http://link\.videoplatform\.limelight\.com/media/\??\bmediaId=)(?P<id>[a-z0-9]{32})'
+    _VALID_URL = r'(?:limelight:media:|https?://link\.videoplatform\.limelight\.com/media/\??\bmediaId=)(?P<id>[a-z0-9]{32})'
     _TESTS = [{
         'url': 'http://link.videoplatform.limelight.com/media/?mediaId=3ffd040b522b4485b6d84effc750cd86',
         'info_dict': {
@@ -176,7 +176,7 @@ class LimelightMediaIE(LimelightBaseIE):
 
 class LimelightChannelIE(LimelightBaseIE):
     IE_NAME = 'limelight:channel'
-    _VALID_URL = r'(?:limelight:channel:|http://link\.videoplatform\.limelight\.com/media/\??\bchannelId=)(?P<id>[a-z0-9]{32})'
+    _VALID_URL = r'(?:limelight:channel:|https?://link\.videoplatform\.limelight\.com/media/\??\bchannelId=)(?P<id>[a-z0-9]{32})'
     _TEST = {
         'url': 'http://link.videoplatform.limelight.com/media/?channelId=ab6a524c379342f9b23642917020c082',
         'info_dict': {
@@ -207,7 +207,7 @@ class LimelightChannelIE(LimelightBaseIE):
 
 class LimelightChannelListIE(LimelightBaseIE):
     IE_NAME = 'limelight:channel_list'
-    _VALID_URL = r'(?:limelight:channel_list:|http://link\.videoplatform\.limelight\.com/media/\?.*?\bchannelListId=)(?P<id>[a-z0-9]{32})'
+    _VALID_URL = r'(?:limelight:channel_list:|https?://link\.videoplatform\.limelight\.com/media/\?.*?\bchannelListId=)(?P<id>[a-z0-9]{32})'
     _TEST = {
         'url': 'http://link.videoplatform.limelight.com/media/?channelListId=301b117890c4465c8179ede21fd92e2b',
         'info_dict': {
diff --git a/youtube_dl/extractor/m6.py b/youtube_dl/extractor/m6.py
index 7e025831b..d5945ad66 100644
--- a/youtube_dl/extractor/m6.py
+++ b/youtube_dl/extractor/m6.py
@@ -8,7 +8,7 @@ from .common import InfoExtractor
 
 class M6IE(InfoExtractor):
     IE_NAME = 'm6'
-    _VALID_URL = r'http://(?:www\.)?m6\.fr/[^/]+/videos/(?P<id>\d+)-[^\.]+\.html'
+    _VALID_URL = r'https?://(?:www\.)?m6\.fr/[^/]+/videos/(?P<id>\d+)-[^\.]+\.html'
 
     _TEST = {
         'url': 'http://www.m6.fr/emission-les_reines_du_shopping/videos/11323908-emeline_est_la_reine_du_shopping_sur_le_theme_ma_fete_d_8217_anniversaire.html',
diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index 67d6271e1..c31e8798a 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 
 class MetacafeIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
+    _VALID_URL = r'https?://(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
     _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
     _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
     IE_NAME = 'metacafe'
diff --git a/youtube_dl/extractor/mit.py b/youtube_dl/extractor/mit.py
index 819c1b90b..1aea78d11 100644
--- a/youtube_dl/extractor/mit.py
+++ b/youtube_dl/extractor/mit.py
@@ -91,7 +91,7 @@ class MITIE(TechTVMITIE):
 
 class OCWMITIE(InfoExtractor):
     IE_NAME = 'ocw.mit.edu'
-    _VALID_URL = r'^http://ocw\.mit\.edu/courses/(?P<topic>[a-z0-9\-]+)'
+    _VALID_URL = r'^https?://ocw\.mit\.edu/courses/(?P<topic>[a-z0-9\-]+)'
     _BASE_URL = 'http://ocw.mit.edu/'
 
     _TESTS = [
diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index c595f2077..9e584860a 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 class MiTeleIE(InfoExtractor):
     IE_DESC = 'mitele.es'
-    _VALID_URL = r'http://www\.mitele\.es/[^/]+/[^/]+/[^/]+/(?P<id>[^/]+)/'
+    _VALID_URL = r'https?://www\.mitele\.es/[^/]+/[^/]+/[^/]+/(?P<id>[^/]+)/'
 
     _TESTS = [{
         'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
diff --git a/youtube_dl/extractor/mooshare.py b/youtube_dl/extractor/mooshare.py
index 7cc7f054f..f010f52d5 100644
--- a/youtube_dl/extractor/mooshare.py
+++ b/youtube_dl/extractor/mooshare.py
@@ -13,7 +13,7 @@ from ..utils import (
 class MooshareIE(InfoExtractor):
     IE_NAME = 'mooshare'
     IE_DESC = 'Mooshare.biz'
-    _VALID_URL = r'http://(?:www\.)?mooshare\.biz/(?P<id>[\da-z]{12})'
+    _VALID_URL = r'https?://(?:www\.)?mooshare\.biz/(?P<id>[\da-z]{12})'
 
     _TESTS = [
         {
diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index 0b4787c1d..ad04b12cd 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class MotherlessIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?motherless\.com/(?:g/[a-z0-9_]+/)?(?P<id>[A-Z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?motherless\.com/(?:g/[a-z0-9_]+/)?(?P<id>[A-Z0-9]+)'
     _TESTS = [{
         'url': 'http://motherless.com/AC3FFE1',
         'md5': '310f62e325a9fafe64f68c0bccb6e75f',
diff --git a/youtube_dl/extractor/motorsport.py b/youtube_dl/extractor/motorsport.py
index c1a482dba..370328b36 100644
--- a/youtube_dl/extractor/motorsport.py
+++ b/youtube_dl/extractor/motorsport.py
@@ -9,7 +9,7 @@ from ..compat import (
 
 class MotorsportIE(InfoExtractor):
     IE_DESC = 'motorsport.com'
-    _VALID_URL = r'http://www\.motorsport\.com/[^/?#]+/video/(?:[^/?#]+/)(?P<id>[^/]+)/?(?:$|[?#])'
+    _VALID_URL = r'https?://www\.motorsport\.com/[^/?#]+/video/(?:[^/?#]+/)(?P<id>[^/]+)/?(?:$|[?#])'
     _TEST = {
         'url': 'http://www.motorsport.com/f1/video/main-gallery/red-bull-racing-2014-rules-explained/',
         'info_dict': {
diff --git a/youtube_dl/extractor/myspass.py b/youtube_dl/extractor/myspass.py
index f936b92bb..1ca7b1a9e 100644
--- a/youtube_dl/extractor/myspass.py
+++ b/youtube_dl/extractor/myspass.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class MySpassIE(InfoExtractor):
-    _VALID_URL = r'http://www\.myspass\.de/.*'
+    _VALID_URL = r'https?://www\.myspass\.de/.*'
     _TEST = {
         'url': 'http://www.myspass.de/myspass/shows/tvshows/absolute-mehrheit/Absolute-Mehrheit-vom-17022013-Die-Highlights-Teil-2--/11741/',
         'md5': '0b49f4844a068f8b33f4b7c88405862b',
diff --git a/youtube_dl/extractor/myvideo.py b/youtube_dl/extractor/myvideo.py
index 1e21cf98a..c83a1eab5 100644
--- a/youtube_dl/extractor/myvideo.py
+++ b/youtube_dl/extractor/myvideo.py
@@ -20,7 +20,7 @@ from ..utils import (
 
 class MyVideoIE(InfoExtractor):
     _WORKING = False
-    _VALID_URL = r'http://(?:www\.)?myvideo\.de/(?:[^/]+/)?watch/(?P<id>[0-9]+)/[^?/]+.*'
+    _VALID_URL = r'https?://(?:www\.)?myvideo\.de/(?:[^/]+/)?watch/(?P<id>[0-9]+)/[^?/]+.*'
     IE_NAME = 'myvideo'
     _TEST = {
         'url': 'http://www.myvideo.de/watch/8229274/bowling_fail_or_win',
diff --git a/youtube_dl/extractor/myvidster.py b/youtube_dl/extractor/myvidster.py
index a94ab8358..731c24542 100644
--- a/youtube_dl/extractor/myvidster.py
+++ b/youtube_dl/extractor/myvidster.py
@@ -4,7 +4,7 @@ from .common import InfoExtractor
 
 
 class MyVidsterIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?myvidster\.com/video/(?P<id>\d+)/'
+    _VALID_URL = r'https?://(?:www\.)?myvidster\.com/video/(?P<id>\d+)/'
 
     _TEST = {
         'url': 'http://www.myvidster.com/video/32059805/Hot_chemistry_with_raw_love_making',
diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index 7ce8d9b18..d5e53365c 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -8,7 +8,7 @@ from ..utils import (
 
 
 class NationalGeographicIE(InfoExtractor):
-    _VALID_URL = r'http://video\.nationalgeographic\.com/.*?'
+    _VALID_URL = r'https?://video\.nationalgeographic\.com/.*?'
 
     _TESTS = [
         {
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index bb0817e34..a622f2212 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -115,7 +115,7 @@ class NBCSportsVPlayerIE(InfoExtractor):
 
 class NBCSportsIE(InfoExtractor):
     # Does not include https because its certificate is invalid
-    _VALID_URL = r'http://www\.nbcsports\.com//?(?:[^/]+/)+(?P<id>[0-9a-z-]+)'
+    _VALID_URL = r'https?://www\.nbcsports\.com//?(?:[^/]+/)+(?P<id>[0-9a-z-]+)'
 
     _TEST = {
         'url': 'http://www.nbcsports.com//college-basketball/ncaab/tom-izzo-michigan-st-has-so-much-respect-duke',
@@ -295,7 +295,7 @@ class NBCNewsIE(ThePlatformIE):
 
 class MSNBCIE(InfoExtractor):
     # https URLs redirect to corresponding http ones
-    _VALID_URL = r'http://www\.msnbc\.com/[^/]+/watch/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://www\.msnbc\.com/[^/]+/watch/(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://www.msnbc.com/all-in-with-chris-hayes/watch/the-chaotic-gop-immigration-vote-314487875924',
         'md5': '6d236bf4f3dddc226633ce6e2c3f814d',
diff --git a/youtube_dl/extractor/nextmedia.py b/youtube_dl/extractor/nextmedia.py
index d1688457f..aae7aeeeb 100644
--- a/youtube_dl/extractor/nextmedia.py
+++ b/youtube_dl/extractor/nextmedia.py
@@ -7,7 +7,7 @@ from ..utils import parse_iso8601
 
 class NextMediaIE(InfoExtractor):
     IE_DESC = '蘋果日報'
-    _VALID_URL = r'http://hk.apple.nextmedia.com/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)'
+    _VALID_URL = r'https?://hk.apple.nextmedia.com/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://hk.apple.nextmedia.com/realtime/news/20141108/53109199',
         'md5': 'dff9fad7009311c421176d1ac90bfe4f',
@@ -68,7 +68,7 @@ class NextMediaIE(InfoExtractor):
 
 class NextMediaActionNewsIE(NextMediaIE):
     IE_DESC = '蘋果日報 - 動新聞'
-    _VALID_URL = r'http://hk.dv.nextmedia.com/actionnews/[^/]+/(?P<date>\d+)/(?P<id>\d+)/\d+'
+    _VALID_URL = r'https?://hk.dv.nextmedia.com/actionnews/[^/]+/(?P<date>\d+)/(?P<id>\d+)/\d+'
     _TESTS = [{
         'url': 'http://hk.dv.nextmedia.com/actionnews/hit/20150121/19009428/20061460',
         'md5': '05fce8ffeed7a5e00665d4b7cf0f9201',
@@ -93,7 +93,7 @@ class NextMediaActionNewsIE(NextMediaIE):
 
 class AppleDailyIE(NextMediaIE):
     IE_DESC = '臺灣蘋果日報'
-    _VALID_URL = r'http://(www|ent).appledaily.com.tw/(?:animation|appledaily|enews|realtimenews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
+    _VALID_URL = r'https?://(www|ent).appledaily.com.tw/(?:animation|appledaily|enews|realtimenews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
     _TESTS = [{
         'url': 'http://ent.appledaily.com.tw/enews/article/entertainment/20150128/36354694',
         'md5': 'a843ab23d150977cc55ef94f1e2c1e4d',
diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index d440313d5..ec7317a2f 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -22,7 +22,7 @@ from ..utils import (
 
 
 class NocoIE(InfoExtractor):
-    _VALID_URL = r'http://(?:(?:www\.)?noco\.tv/emission/|player\.noco\.tv/\?idvideo=)(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:(?:www\.)?noco\.tv/emission/|player\.noco\.tv/\?idvideo=)(?P<id>\d+)'
     _LOGIN_URL = 'http://noco.tv/do.php'
     _API_URL_TEMPLATE = 'https://api.noco.tv/1.1/%s?ts=%s&tk=%s'
     _SUB_LANG_TEMPLATE = '&sub_lang=%s'
diff --git a/youtube_dl/extractor/normalboots.py b/youtube_dl/extractor/normalboots.py
index 5952d136f..77e091072 100644
--- a/youtube_dl/extractor/normalboots.py
+++ b/youtube_dl/extractor/normalboots.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class NormalbootsIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?normalboots\.com/video/(?P<id>[0-9a-z-]*)/?$'
+    _VALID_URL = r'https?://(?:www\.)?normalboots\.com/video/(?P<id>[0-9a-z-]*)/?$'
     _TEST = {
         'url': 'http://normalboots.com/video/home-alone-games-jontron/',
         'md5': '8bf6de238915dd501105b44ef5f1e0f6',
diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 72f72b803..17671ad39 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class NovaIE(InfoExtractor):
     IE_DESC = 'TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz'
-    _VALID_URL = r'http://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+?)(?:\.html|/|$)'
+    _VALID_URL = r'https?://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+?)(?:\.html|/|$)'
     _TESTS = [{
         'url': 'http://tvnoviny.nova.cz/clanek/novinky/co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou.html?utm_source=tvnoviny&utm_medium=cpfooter&utm_campaign=novaplus',
         'info_dict': {
diff --git a/youtube_dl/extractor/npr.py b/youtube_dl/extractor/npr.py
index 125c7010b..a3f0abb4e 100644
--- a/youtube_dl/extractor/npr.py
+++ b/youtube_dl/extractor/npr.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class NprIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?npr\.org/player/v2/mediaPlayer\.html\?.*\bid=(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?npr\.org/player/v2/mediaPlayer\.html\?.*\bid=(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.npr.org/player/v2/mediaPlayer.html?id=449974205',
         'info_dict': {
diff --git a/youtube_dl/extractor/ntvru.py b/youtube_dl/extractor/ntvru.py
index 2cd924d05..0895d7ea4 100644
--- a/youtube_dl/extractor/ntvru.py
+++ b/youtube_dl/extractor/ntvru.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 class NTVRuIE(InfoExtractor):
     IE_NAME = 'ntv.ru'
-    _VALID_URL = r'http://(?:www\.)?ntv\.ru/(?P<id>.+)'
+    _VALID_URL = r'https?://(?:www\.)?ntv\.ru/(?P<id>.+)'
 
     _TESTS = [
         {
diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 958eb398b..66c75f8b3 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -137,7 +137,7 @@ class ORFTVthekIE(InfoExtractor):
 class ORFOE1IE(InfoExtractor):
     IE_NAME = 'orf:oe1'
     IE_DESC = 'Radio Österreich 1'
-    _VALID_URL = r'http://oe1\.orf\.at/(?:programm/|konsole.*?#\?track_id=)(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://oe1\.orf\.at/(?:programm/|konsole.*?#\?track_id=)(?P<id>[0-9]+)'
 
     # Audios on ORF radio are only available for 7 days, so we can't add tests.
     _TEST = {
@@ -171,7 +171,7 @@ class ORFOE1IE(InfoExtractor):
 class ORFFM4IE(InfoExtractor):
     IE_NAME = 'orf:fm4'
     IE_DESC = 'radio FM4'
-    _VALID_URL = r'http://fm4\.orf\.at/(?:7tage/?#|player/)(?P<date>[0-9]+)/(?P<show>\w+)'
+    _VALID_URL = r'https?://fm4\.orf\.at/(?:7tage/?#|player/)(?P<date>[0-9]+)/(?P<show>\w+)'
 
     _TEST = {
         'url': 'http://fm4.orf.at/player/20160110/IS/',
@@ -222,7 +222,7 @@ class ORFFM4IE(InfoExtractor):
 class ORFIPTVIE(InfoExtractor):
     IE_NAME = 'orf:iptv'
     IE_DESC = 'iptv.ORF.at'
-    _VALID_URL = r'http://iptv\.orf\.at/(?:#/)?stories/(?P<id>\d+)'
+    _VALID_URL = r'https?://iptv\.orf\.at/(?:#/)?stories/(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://iptv.orf.at/stories/2275236/',
diff --git a/youtube_dl/extractor/philharmoniedeparis.py b/youtube_dl/extractor/philharmoniedeparis.py
index 6e60e5fe9..f1008ae51 100644
--- a/youtube_dl/extractor/philharmoniedeparis.py
+++ b/youtube_dl/extractor/philharmoniedeparis.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class PhilharmonieDeParisIE(InfoExtractor):
     IE_DESC = 'Philharmonie de Paris'
-    _VALID_URL = r'http://live\.philharmoniedeparis\.fr/(?:[Cc]oncert/|misc/Playlist\.ashx\?id=)(?P<id>\d+)'
+    _VALID_URL = r'https?://live\.philharmoniedeparis\.fr/(?:[Cc]oncert/|misc/Playlist\.ashx\?id=)(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://live.philharmoniedeparis.fr/concert/1032066.html',
         'info_dict': {
diff --git a/youtube_dl/extractor/photobucket.py b/youtube_dl/extractor/photobucket.py
index 788411ccc..6c8bbe1d9 100644
--- a/youtube_dl/extractor/photobucket.py
+++ b/youtube_dl/extractor/photobucket.py
@@ -8,7 +8,7 @@ from ..compat import compat_urllib_parse_unquote
 
 
 class PhotobucketIE(InfoExtractor):
-    _VALID_URL = r'http://(?:[a-z0-9]+\.)?photobucket\.com/.*(([\?\&]current=)|_)(?P<id>.*)\.(?P<ext>(flv)|(mp4))'
+    _VALID_URL = r'https?://(?:[a-z0-9]+\.)?photobucket\.com/.*(([\?\&]current=)|_)(?P<id>.*)\.(?P<ext>(flv)|(mp4))'
     _TEST = {
         'url': 'http://media.photobucket.com/user/rachaneronas/media/TiredofLinkBuildingTryBacklinkMyDomaincom_zpsc0c3b9fa.mp4.html?filters[term]=search&filters[primary]=videos&filters[secondary]=images&sort=1&o=0',
         'md5': '7dabfb92b0a31f6c16cebc0f8e60ff99',
diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py
index 57c78ba52..39b53ecf6 100644
--- a/youtube_dl/extractor/pornhd.py
+++ b/youtube_dl/extractor/pornhd.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class PornHdIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?pornhd\.com/(?:[a-z]{2,4}/)?videos/(?P<id>\d+)(?:/(?P<display_id>.+))?'
+    _VALID_URL = r'https?://(?:www\.)?pornhd\.com/(?:[a-z]{2,4}/)?videos/(?P<id>\d+)(?:/(?P<display_id>.+))?'
     _TEST = {
         'url': 'http://www.pornhd.com/videos/1962/sierra-day-gets-his-cum-all-over-herself-hd-porn-video',
         'md5': '956b8ca569f7f4d8ec563e2c41598441',
diff --git a/youtube_dl/extractor/pornovoisines.py b/youtube_dl/extractor/pornovoisines.py
index 1a53fd71c..6b51e5c54 100644
--- a/youtube_dl/extractor/pornovoisines.py
+++ b/youtube_dl/extractor/pornovoisines.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class PornoVoisinesIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?pornovoisines\.com/showvideo/(?P<id>\d+)/(?P<display_id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?pornovoisines\.com/showvideo/(?P<id>\d+)/(?P<display_id>[^/]+)'
 
     _VIDEO_URL_TEMPLATE = 'http://stream%d.pornovoisines.com' \
         '/static/media/video/transcoded/%s-640x360-1000-trscded.mp4'
diff --git a/youtube_dl/extractor/pyvideo.py b/youtube_dl/extractor/pyvideo.py
index 30a5f2de4..cc0416cb8 100644
--- a/youtube_dl/extractor/pyvideo.py
+++ b/youtube_dl/extractor/pyvideo.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 
 
 class PyvideoIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?pyvideo\.org/video/(?P<id>\d+)/(.*)'
+    _VALID_URL = r'https?://(?:www\.)?pyvideo\.org/video/(?P<id>\d+)/(.*)'
 
     _TESTS = [
         {
diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 45a3c41c5..ff0af9543 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -18,7 +18,7 @@ from ..utils import (
 class QQMusicIE(InfoExtractor):
     IE_NAME = 'qqmusic'
     IE_DESC = 'QQ音乐'
-    _VALID_URL = r'http://y.qq.com/#type=song&mid=(?P<id>[0-9A-Za-z]+)'
+    _VALID_URL = r'https?://y.qq.com/#type=song&mid=(?P<id>[0-9A-Za-z]+)'
     _TESTS = [{
         'url': 'http://y.qq.com/#type=song&mid=004295Et37taLD',
         'md5': '9ce1c1c8445f561506d2e3cfb0255705',
@@ -172,7 +172,7 @@ class QQPlaylistBaseIE(InfoExtractor):
 class QQMusicSingerIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:singer'
     IE_DESC = 'QQ音乐 - 歌手'
-    _VALID_URL = r'http://y.qq.com/#type=singer&mid=(?P<id>[0-9A-Za-z]+)'
+    _VALID_URL = r'https?://y.qq.com/#type=singer&mid=(?P<id>[0-9A-Za-z]+)'
     _TEST = {
         'url': 'http://y.qq.com/#type=singer&mid=001BLpXF2DyJe2',
         'info_dict': {
@@ -217,7 +217,7 @@ class QQMusicSingerIE(QQPlaylistBaseIE):
 class QQMusicAlbumIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:album'
     IE_DESC = 'QQ音乐 - 专辑'
-    _VALID_URL = r'http://y.qq.com/#type=album&mid=(?P<id>[0-9A-Za-z]+)'
+    _VALID_URL = r'https?://y.qq.com/#type=album&mid=(?P<id>[0-9A-Za-z]+)'
 
     _TESTS = [{
         'url': 'http://y.qq.com/#type=album&mid=000gXCTb2AhRR1',
@@ -260,7 +260,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
 class QQMusicToplistIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:toplist'
     IE_DESC = 'QQ音乐 - 排行榜'
-    _VALID_URL = r'http://y\.qq\.com/#type=toplist&p=(?P<id>(top|global)_[0-9]+)'
+    _VALID_URL = r'https?://y\.qq\.com/#type=toplist&p=(?P<id>(top|global)_[0-9]+)'
 
     _TESTS = [{
         'url': 'http://y.qq.com/#type=toplist&p=global_123',
@@ -314,7 +314,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
 class QQMusicPlaylistIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:playlist'
     IE_DESC = 'QQ音乐 - 歌单'
-    _VALID_URL = r'http://y\.qq\.com/#type=taoge&id=(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://y\.qq\.com/#type=taoge&id=(?P<id>[0-9]+)'
 
     _TESTS = [{
         'url': 'http://y.qq.com/#type=taoge&id=3462654915',
diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index a4dc5c335..e36ce1aa1 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -18,7 +18,7 @@ from ..utils import (
 
 
 class RaiTVIE(InfoExtractor):
-    _VALID_URL = r'http://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/(?:[^/]+/)+media/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
+    _VALID_URL = r'https?://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/(?:[^/]+/)+media/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
     _TESTS = [
         {
             'url': 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
@@ -175,7 +175,7 @@ class RaiTVIE(InfoExtractor):
 
 
 class RaiIE(InfoExtractor):
-    _VALID_URL = r'http://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
+    _VALID_URL = r'https?://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
     _TESTS = [
         {
             'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html',
diff --git a/youtube_dl/extractor/redtube.py b/youtube_dl/extractor/redtube.py
index d6054d717..7ba41ba59 100644
--- a/youtube_dl/extractor/redtube.py
+++ b/youtube_dl/extractor/redtube.py
@@ -5,7 +5,7 @@ from ..utils import ExtractorError
 
 
 class RedTubeIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?redtube\.com/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?redtube\.com/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.redtube.com/66418',
         'md5': '7b8c22b5e7098a3e1c09709df1126d2d',
diff --git a/youtube_dl/extractor/ringtv.py b/youtube_dl/extractor/ringtv.py
index 508758075..2c2c707bd 100644
--- a/youtube_dl/extractor/ringtv.py
+++ b/youtube_dl/extractor/ringtv.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 
 
 class RingTVIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?ringtv\.craveonline\.com/(?P<type>news|videos/video)/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?ringtv\.craveonline\.com/(?P<type>news|videos/video)/(?P<id>[^/?#]+)'
     _TEST = {
         'url': 'http://ringtv.craveonline.com/news/310833-luis-collazo-says-victor-ortiz-better-not-quit-on-jan-30',
         'md5': 'd25945f5df41cdca2d2587165ac28720',
diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 8a8c5d2a0..08cd1ae6c 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -62,7 +62,7 @@ def _decrypt_url(png):
 class RTVEALaCartaIE(InfoExtractor):
     IE_NAME = 'rtve.es:alacarta'
     IE_DESC = 'RTVE a la carta'
-    _VALID_URL = r'http://www\.rtve\.es/(m/)?alacarta/videos/[^/]+/[^/]+/(?P<id>\d+)'
+    _VALID_URL = r'https?://www\.rtve\.es/(m/)?alacarta/videos/[^/]+/[^/]+/(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.rtve.es/alacarta/videos/balonmano/o-swiss-cup-masculina-final-espana-suecia/2491869/',
@@ -179,7 +179,7 @@ class RTVEInfantilIE(InfoExtractor):
 class RTVELiveIE(InfoExtractor):
     IE_NAME = 'rtve.es:live'
     IE_DESC = 'RTVE.es live streams'
-    _VALID_URL = r'http://www\.rtve\.es/directo/(?P<id>[a-zA-Z0-9-]+)'
+    _VALID_URL = r'https?://www\.rtve\.es/directo/(?P<id>[a-zA-Z0-9-]+)'
 
     _TESTS = [{
         'url': 'http://www.rtve.es/directo/la-1/',
diff --git a/youtube_dl/extractor/ruhd.py b/youtube_dl/extractor/ruhd.py
index 0e470e73f..1f7c26299 100644
--- a/youtube_dl/extractor/ruhd.py
+++ b/youtube_dl/extractor/ruhd.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 
 class RUHDIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?ruhd\.ru/play\.php\?vid=(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?ruhd\.ru/play\.php\?vid=(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.ruhd.ru/play.php?vid=207',
         'md5': 'd1a9ec4edf8598e3fbd92bb16072ba83',
diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index c5c47d01e..9ca4ae147 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -122,7 +122,7 @@ class RutubeEmbedIE(InfoExtractor):
 class RutubeChannelIE(InfoExtractor):
     IE_NAME = 'rutube:channel'
     IE_DESC = 'Rutube channels'
-    _VALID_URL = r'http://rutube\.ru/tags/video/(?P<id>\d+)'
+    _VALID_URL = r'https?://rutube\.ru/tags/video/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://rutube.ru/tags/video/1800/',
         'info_dict': {
@@ -156,7 +156,7 @@ class RutubeChannelIE(InfoExtractor):
 class RutubeMovieIE(RutubeChannelIE):
     IE_NAME = 'rutube:movie'
     IE_DESC = 'Rutube movies'
-    _VALID_URL = r'http://rutube\.ru/metainfo/tv/(?P<id>\d+)'
+    _VALID_URL = r'https?://rutube\.ru/metainfo/tv/(?P<id>\d+)'
     _TESTS = []
 
     _MOVIE_TEMPLATE = 'http://rutube.ru/api/metainfo/tv/%s/?format=json'
@@ -174,7 +174,7 @@ class RutubeMovieIE(RutubeChannelIE):
 class RutubePersonIE(RutubeChannelIE):
     IE_NAME = 'rutube:person'
     IE_DESC = 'Rutube person videos'
-    _VALID_URL = r'http://rutube\.ru/video/person/(?P<id>\d+)'
+    _VALID_URL = r'https?://rutube\.ru/video/person/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://rutube.ru/video/person/313878/',
         'info_dict': {
diff --git a/youtube_dl/extractor/screenjunkies.py b/youtube_dl/extractor/screenjunkies.py
index f2af15f6b..dd0a6ba19 100644
--- a/youtube_dl/extractor/screenjunkies.py
+++ b/youtube_dl/extractor/screenjunkies.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class ScreenJunkiesIE(InfoExtractor):
-    _VALID_URL = r'http://www.screenjunkies.com/video/(?P<display_id>[^/]+?)(?:-(?P<id>\d+))?(?:[/?#&]|$)'
+    _VALID_URL = r'https?://www.screenjunkies.com/video/(?P<display_id>[^/]+?)(?:-(?P<id>\d+))?(?:[/?#&]|$)'
     _TESTS = [{
         'url': 'http://www.screenjunkies.com/video/best-quentin-tarantino-movie-2841915',
         'md5': '5c2b686bec3d43de42bde9ec047536b0',
diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
index 4d3b58522..c5f474dd1 100644
--- a/youtube_dl/extractor/senateisvp.py
+++ b/youtube_dl/extractor/senateisvp.py
@@ -48,7 +48,7 @@ class SenateISVPIE(InfoExtractor):
         ['arch', '', 'http://ussenate-f.akamaihd.net/']
     ]
     _IE_NAME = 'senate.gov'
-    _VALID_URL = r'http://www\.senate\.gov/isvp/?\?(?P<qs>.+)'
+    _VALID_URL = r'https?://www\.senate\.gov/isvp/?\?(?P<qs>.+)'
     _TESTS = [{
         'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png',
         'info_dict': {
diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index 8eda3c864..96fe0b90d 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 class SharedIE(InfoExtractor):
     IE_DESC = 'shared.sx and vivo.sx'
-    _VALID_URL = r'http://(?:shared|vivo)\.sx/(?P<id>[\da-z]{10})'
+    _VALID_URL = r'https?://(?:shared|vivo)\.sx/(?P<id>[\da-z]{10})'
 
     _TESTS = [{
         'url': 'http://shared.sx/0060718775',
diff --git a/youtube_dl/extractor/sport5.py b/youtube_dl/extractor/sport5.py
index dfe50ed45..7e6783306 100644
--- a/youtube_dl/extractor/sport5.py
+++ b/youtube_dl/extractor/sport5.py
@@ -8,7 +8,7 @@ from ..utils import ExtractorError
 
 
 class Sport5IE(InfoExtractor):
-    _VALID_URL = r'http://(?:www|vod)?\.sport5\.co\.il/.*\b(?:Vi|docID)=(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www|vod)?\.sport5\.co\.il/.*\b(?:Vi|docID)=(?P<id>\d+)'
     _TESTS = [
         {
             'url': 'http://vod.sport5.co.il/?Vc=147&Vi=176331&Page=1',
diff --git a/youtube_dl/extractor/ssa.py b/youtube_dl/extractor/ssa.py
index 13101c714..54d1843f2 100644
--- a/youtube_dl/extractor/ssa.py
+++ b/youtube_dl/extractor/ssa.py
@@ -8,7 +8,7 @@ from ..utils import (
 
 
 class SSAIE(InfoExtractor):
-    _VALID_URL = r'http://ssa\.nls\.uk/film/(?P<id>\d+)'
+    _VALID_URL = r'https?://ssa\.nls\.uk/film/(?P<id>\d+)'
     _TEST = {
         'url': 'http://ssa.nls.uk/film/3561',
         'info_dict': {
diff --git a/youtube_dl/extractor/sztvhu.py b/youtube_dl/extractor/sztvhu.py
index aa5964acb..f562aa6d3 100644
--- a/youtube_dl/extractor/sztvhu.py
+++ b/youtube_dl/extractor/sztvhu.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 
 class SztvHuIE(InfoExtractor):
-    _VALID_URL = r'http://(?:(?:www\.)?sztv\.hu|www\.tvszombathely\.hu)/(?:[^/]+)/.+-(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:(?:www\.)?sztv\.hu|www\.tvszombathely\.hu)/(?:[^/]+)/.+-(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://sztv.hu/hirek/cserkeszek-nepszerusitettek-a-kornyezettudatos-eletmodot-a-savaria-teren-20130909',
         'md5': 'a6df607b11fb07d0e9f2ad94613375cb',
diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index d1b7264b4..b49ab5f5b 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -16,7 +16,7 @@ from ..compat import compat_ord
 
 
 class TeamcocoIE(InfoExtractor):
-    _VALID_URL = r'http://teamcoco\.com/video/(?P<video_id>[0-9]+)?/?(?P<display_id>.*)'
+    _VALID_URL = r'https?://teamcoco\.com/video/(?P<video_id>[0-9]+)?/?(?P<display_id>.*)'
     _TESTS = [
         {
             'url': 'http://teamcoco.com/video/80187/conan-becomes-a-mary-kay-beauty-consultant',
diff --git a/youtube_dl/extractor/tele13.py b/youtube_dl/extractor/tele13.py
index 4e860db0a..a29a64b6d 100644
--- a/youtube_dl/extractor/tele13.py
+++ b/youtube_dl/extractor/tele13.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class Tele13IE(InfoExtractor):
-    _VALID_URL = r'^http://(?:www\.)?t13\.cl/videos(?:/[^/]+)+/(?P<id>[\w-]+)'
+    _VALID_URL = r'^https?://(?:www\.)?t13\.cl/videos(?:/[^/]+)+/(?P<id>[\w-]+)'
     _TESTS = [
         {
             'url': 'http://www.t13.cl/videos/actualidad/el-circulo-de-hierro-de-michelle-bachelet-en-su-regreso-a-la-moneda',
diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 9ee844684..3f54b2744 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 
 class TF1IE(InfoExtractor):
     """TF1 uses the wat.tv player."""
-    _VALID_URL = r'http://(?:(?:videos|www|lci)\.tf1|www\.tfou)\.fr/(?:[^/]+/)*(?P<id>.+?)\.html'
+    _VALID_URL = r'https?://(?:(?:videos|www|lci)\.tf1|www\.tfou)\.fr/(?:[^/]+/)*(?P<id>.+?)\.html'
     _TESTS = [{
         'url': 'http://videos.tf1.fr/auto-moto/citroen-grand-c4-picasso-2013-presentation-officielle-8062060.html',
         'info_dict': {
diff --git a/youtube_dl/extractor/thvideo.py b/youtube_dl/extractor/thvideo.py
index 496f15d80..406f4a826 100644
--- a/youtube_dl/extractor/thvideo.py
+++ b/youtube_dl/extractor/thvideo.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class THVideoIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?thvideo\.tv/(?:v/th|mobile\.php\?cid=)(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?thvideo\.tv/(?:v/th|mobile\.php\?cid=)(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://thvideo.tv/v/th1987/',
         'md5': 'fa107b1f73817e325e9433505a70db50',
diff --git a/youtube_dl/extractor/tinypic.py b/youtube_dl/extractor/tinypic.py
index e036b8cdf..c43cace24 100644
--- a/youtube_dl/extractor/tinypic.py
+++ b/youtube_dl/extractor/tinypic.py
@@ -9,7 +9,7 @@ from ..utils import ExtractorError
 class TinyPicIE(InfoExtractor):
     IE_NAME = 'tinypic'
     IE_DESC = 'tinypic.com videos'
-    _VALID_URL = r'http://(?:.+?\.)?tinypic\.com/player\.php\?v=(?P<id>[^&]+)&s=\d+'
+    _VALID_URL = r'https?://(?:.+?\.)?tinypic\.com/player\.php\?v=(?P<id>[^&]+)&s=\d+'
 
     _TESTS = [
         {
diff --git a/youtube_dl/extractor/tlc.py b/youtube_dl/extractor/tlc.py
index 17add9543..abad3ff64 100644
--- a/youtube_dl/extractor/tlc.py
+++ b/youtube_dl/extractor/tlc.py
@@ -9,7 +9,7 @@ from ..compat import compat_parse_qs
 
 class TlcDeIE(InfoExtractor):
     IE_NAME = 'tlc.de'
-    _VALID_URL = r'http://www\.tlc\.de/(?:[^/]+/)*videos/(?P<title>[^/?#]+)?(?:.*#(?P<id>\d+))?'
+    _VALID_URL = r'https?://www\.tlc\.de/(?:[^/]+/)*videos/(?P<title>[^/?#]+)?(?:.*#(?P<id>\d+))?'
 
     _TEST = {
         'url': 'http://www.tlc.de/sendungen/breaking-amish/videos/#3235167922001',
diff --git a/youtube_dl/extractor/toypics.py b/youtube_dl/extractor/toypics.py
index 2756f56d3..2579ba8c6 100644
--- a/youtube_dl/extractor/toypics.py
+++ b/youtube_dl/extractor/toypics.py
@@ -41,7 +41,7 @@ class ToypicsIE(InfoExtractor):
 
 class ToypicsUserIE(InfoExtractor):
     IE_DESC = 'Toypics user profile'
-    _VALID_URL = r'http://videos\.toypics\.net/(?P<username>[^/?]+)(?:$|[?#])'
+    _VALID_URL = r'https?://videos\.toypics\.net/(?P<username>[^/?]+)(?:$|[?#])'
     _TEST = {
         'url': 'http://videos.toypics.net/Mikey',
         'info_dict': {
diff --git a/youtube_dl/extractor/traileraddict.py b/youtube_dl/extractor/traileraddict.py
index 0e01b15fc..747370d12 100644
--- a/youtube_dl/extractor/traileraddict.py
+++ b/youtube_dl/extractor/traileraddict.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 
 class TrailerAddictIE(InfoExtractor):
     _WORKING = False
-    _VALID_URL = r'(?:http://)?(?:www\.)?traileraddict\.com/(?:trailer|clip)/(?P<movie>.+?)/(?P<trailer_name>.+)'
+    _VALID_URL = r'(?:https?://)?(?:www\.)?traileraddict\.com/(?:trailer|clip)/(?P<movie>.+?)/(?P<trailer_name>.+)'
     _TEST = {
         'url': 'http://www.traileraddict.com/trailer/prince-avalanche/trailer',
         'md5': '41365557f3c8c397d091da510e73ceb4',
diff --git a/youtube_dl/extractor/trollvids.py b/youtube_dl/extractor/trollvids.py
index d239949a6..657705623 100644
--- a/youtube_dl/extractor/trollvids.py
+++ b/youtube_dl/extractor/trollvids.py
@@ -7,7 +7,7 @@ from .nuevo import NuevoBaseIE
 
 
 class TrollvidsIE(NuevoBaseIE):
-    _VALID_URL = r'http://(?:www\.)?trollvids\.com/video/(?P<id>\d+)/(?P<display_id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?trollvids\.com/video/(?P<id>\d+)/(?P<display_id>[^/?#&]+)'
     IE_NAME = 'trollvids'
     _TEST = {
         'url': 'http://trollvids.com/video/2349002/%E3%80%90MMD-R-18%E3%80%91%E3%82%AC%E3%83%BC%E3%83%AB%E3%83%95%E3%83%AC%E3%83%B3%E3%83%89-carrymeoff',
diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 4f844706d..cea117c79 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -8,7 +8,7 @@ from ..utils import int_or_none
 
 
 class TumblrIE(InfoExtractor):
-    _VALID_URL = r'http://(?P<blog_name>.*?)\.tumblr\.com/(?:post|video)/(?P<id>[0-9]+)(?:$|[/?#])'
+    _VALID_URL = r'https?://(?P<blog_name>.*?)\.tumblr\.com/(?:post|video)/(?P<id>[0-9]+)(?:$|[/?#])'
     _TESTS = [{
         'url': 'http://tatianamaslanydaily.tumblr.com/post/54196191430/orphan-black-dvd-extra-behind-the-scenes',
         'md5': '479bb068e5b16462f5176a6828829767',
diff --git a/youtube_dl/extractor/tv2.py b/youtube_dl/extractor/tv2.py
index 535d0d361..86bb7915d 100644
--- a/youtube_dl/extractor/tv2.py
+++ b/youtube_dl/extractor/tv2.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class TV2IE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?tv2\.no/v/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?tv2\.no/v/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.tv2.no/v/916509/',
         'info_dict': {
@@ -100,7 +100,7 @@ class TV2IE(InfoExtractor):
 
 
 class TV2ArticleIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?tv2\.no/(?:a|\d{4}/\d{2}/\d{2}(/[^/]+)+)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?tv2\.no/(?:a|\d{4}/\d{2}/\d{2}(/[^/]+)+)/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.tv2.no/2015/05/16/nyheter/alesund/krim/pingvin/6930542',
         'info_dict': {
diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py
index 3a4f393fc..4065354dd 100644
--- a/youtube_dl/extractor/tvc.py
+++ b/youtube_dl/extractor/tvc.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class TVCIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?tvc\.ru/video/iframe/id/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?tvc\.ru/video/iframe/id/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.tvc.ru/video/iframe/id/74622/isPlay/false/id_stat/channel/?acc_video_id=/channel/brand/id/17/show/episodes/episode_id/39702',
         'md5': 'bbc5ff531d1e90e856f60fc4b3afd708',
@@ -64,7 +64,7 @@ class TVCIE(InfoExtractor):
 
 
 class TVCArticleIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?tvc\.ru/(?!video/iframe/id/)(?P<id>[^?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?tvc\.ru/(?!video/iframe/id/)(?P<id>[^?#]+)'
     _TESTS = [{
         'url': 'http://www.tvc.ru/channel/brand/id/29/show/episodes/episode_id/39702/',
         'info_dict': {
diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index b4683de54..df70a6b23 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 class TVPlayIE(InfoExtractor):
     IE_DESC = 'TV3Play and related services'
-    _VALID_URL = r'''(?x)http://(?:www\.)?
+    _VALID_URL = r'''(?x)https?://(?:www\.)?
         (?:tvplay\.lv/parraides|
            tv3play\.lt/programos|
            play\.tv3\.lt/programos|
diff --git a/youtube_dl/extractor/ubu.py b/youtube_dl/extractor/ubu.py
index d50237758..1d52cbc98 100644
--- a/youtube_dl/extractor/ubu.py
+++ b/youtube_dl/extractor/ubu.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class UbuIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?ubu\.com/film/(?P<id>[\da-z_-]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?ubu\.com/film/(?P<id>[\da-z_-]+)\.html'
     _TEST = {
         'url': 'http://ubu.com/film/her_noise.html',
         'md5': '138d5652618bf0f03878978db9bef1ee',
diff --git a/youtube_dl/extractor/unistra.py b/youtube_dl/extractor/unistra.py
index 594bee4f9..66d9f1bf3 100644
--- a/youtube_dl/extractor/unistra.py
+++ b/youtube_dl/extractor/unistra.py
@@ -7,7 +7,7 @@ from ..utils import qualities
 
 
 class UnistraIE(InfoExtractor):
-    _VALID_URL = r'http://utv\.unistra\.fr/(?:index|video)\.php\?id_video\=(?P<id>\d+)'
+    _VALID_URL = r'https?://utv\.unistra\.fr/(?:index|video)\.php\?id_video\=(?P<id>\d+)'
 
     _TESTS = [
         {
diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index 3794bcded..b755dda90 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class Vbox7IE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?vbox7\.com/play:(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?vbox7\.com/play:(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://vbox7.com/play:249bb972c2',
         'md5': '99f65c0c9ef9b682b97313e052734c3f',
diff --git a/youtube_dl/extractor/veoh.py b/youtube_dl/extractor/veoh.py
index 9633f7ffe..23ce0a0d1 100644
--- a/youtube_dl/extractor/veoh.py
+++ b/youtube_dl/extractor/veoh.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class VeohIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?veoh\.com/(?:watch|iphone/#_Watch)/(?P<id>(?:v|yapi-)[\da-zA-Z]+)'
+    _VALID_URL = r'https?://(?:www\.)?veoh\.com/(?:watch|iphone/#_Watch)/(?P<id>(?:v|yapi-)[\da-zA-Z]+)'
 
     _TESTS = [
         {
diff --git a/youtube_dl/extractor/vesti.py b/youtube_dl/extractor/vesti.py
index a0c59a2e0..cb64ae0bd 100644
--- a/youtube_dl/extractor/vesti.py
+++ b/youtube_dl/extractor/vesti.py
@@ -10,7 +10,7 @@ from .rutv import RUTVIE
 
 class VestiIE(InfoExtractor):
     IE_DESC = 'Вести.Ru'
-    _VALID_URL = r'http://(?:.+?\.)?vesti\.ru/(?P<id>.+)'
+    _VALID_URL = r'https?://(?:.+?\.)?vesti\.ru/(?P<id>.+)'
 
     _TESTS = [
         {
diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 77d8978d4..b11cd254c 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -214,7 +214,7 @@ class VGTVIE(XstreamIE):
 class BTArticleIE(InfoExtractor):
     IE_NAME = 'bt:article'
     IE_DESC = 'Bergens Tidende Articles'
-    _VALID_URL = r'http://(?:www\.)?bt\.no/(?:[^/]+/)+(?P<id>[^/]+)-\d+\.html'
+    _VALID_URL = r'https?://(?:www\.)?bt\.no/(?:[^/]+/)+(?P<id>[^/]+)-\d+\.html'
     _TEST = {
         'url': 'http://www.bt.no/nyheter/lokalt/Kjemper-for-internatet-1788214.html',
         'md5': '2acbe8ad129b3469d5ae51b1158878df',
@@ -241,7 +241,7 @@ class BTArticleIE(InfoExtractor):
 class BTVestlendingenIE(InfoExtractor):
     IE_NAME = 'bt:vestlendingen'
     IE_DESC = 'Bergens Tidende - Vestlendingen'
-    _VALID_URL = r'http://(?:www\.)?bt\.no/spesial/vestlendingen/#!/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?bt\.no/spesial/vestlendingen/#!/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.bt.no/spesial/vestlendingen/#!/86588',
         'md5': 'd7d17e3337dc80de6d3a540aefbe441b',
diff --git a/youtube_dl/extractor/videott.py b/youtube_dl/extractor/videott.py
index 2cd36508a..0f798711b 100644
--- a/youtube_dl/extractor/videott.py
+++ b/youtube_dl/extractor/videott.py
@@ -14,7 +14,7 @@ class VideoTtIE(InfoExtractor):
     _WORKING = False
     ID_NAME = 'video.tt'
     IE_DESC = 'video.tt - Your True Tube'
-    _VALID_URL = r'http://(?:www\.)?video\.tt/(?:(?:video|embed)/|watch_video\.php\?v=)(?P<id>[\da-zA-Z]{9})'
+    _VALID_URL = r'https?://(?:www\.)?video\.tt/(?:(?:video|embed)/|watch_video\.php\?v=)(?P<id>[\da-zA-Z]{9})'
 
     _TESTS = [{
         'url': 'http://www.video.tt/watch_video.php?v=amd5YujV8',
diff --git a/youtube_dl/extractor/viidea.py b/youtube_dl/extractor/viidea.py
index 315984bf9..03b9f1353 100644
--- a/youtube_dl/extractor/viidea.py
+++ b/youtube_dl/extractor/viidea.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class ViideaIE(InfoExtractor):
-    _VALID_URL = r'''(?x)http://(?:www\.)?(?:
+    _VALID_URL = r'''(?x)https?://(?:www\.)?(?:
             videolectures\.net|
             flexilearn\.viidea\.net|
             presentations\.ocwconsortium\.org|
diff --git a/youtube_dl/extractor/vube.py b/youtube_dl/extractor/vube.py
index 149e36467..10ca6acb1 100644
--- a/youtube_dl/extractor/vube.py
+++ b/youtube_dl/extractor/vube.py
@@ -15,7 +15,7 @@ from ..utils import (
 class VubeIE(InfoExtractor):
     IE_NAME = 'vube'
     IE_DESC = 'Vube.com'
-    _VALID_URL = r'http://vube\.com/(?:[^/]+/)+(?P<id>[\da-zA-Z]{10})\b'
+    _VALID_URL = r'https?://vube\.com/(?:[^/]+/)+(?P<id>[\da-zA-Z]{10})\b'
 
     _TESTS = [
         {
diff --git a/youtube_dl/extractor/vuclip.py b/youtube_dl/extractor/vuclip.py
index a6d9b5fee..eaa888f00 100644
--- a/youtube_dl/extractor/vuclip.py
+++ b/youtube_dl/extractor/vuclip.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class VuClipIE(InfoExtractor):
-    _VALID_URL = r'http://(?:m\.)?vuclip\.com/w\?.*?cid=(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:m\.)?vuclip\.com/w\?.*?cid=(?P<id>[0-9]+)'
 
     _TEST = {
         'url': 'http://m.vuclip.com/w?cid=922692425&fid=70295&z=1010&nvar&frm=index.html',
diff --git a/youtube_dl/extractor/walla.py b/youtube_dl/extractor/walla.py
index 24efbd6e6..8b9488340 100644
--- a/youtube_dl/extractor/walla.py
+++ b/youtube_dl/extractor/walla.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class WallaIE(InfoExtractor):
-    _VALID_URL = r'http://vod\.walla\.co\.il/[^/]+/(?P<id>\d+)/(?P<display_id>.+)'
+    _VALID_URL = r'https?://vod\.walla\.co\.il/[^/]+/(?P<id>\d+)/(?P<display_id>.+)'
     _TEST = {
         'url': 'http://vod.walla.co.il/movie/2642630/one-direction-all-for-one',
         'info_dict': {
diff --git a/youtube_dl/extractor/wat.py b/youtube_dl/extractor/wat.py
index 37cf3d309..5227bb5ad 100644
--- a/youtube_dl/extractor/wat.py
+++ b/youtube_dl/extractor/wat.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class WatIE(InfoExtractor):
-    _VALID_URL = r'(?:wat:(?P<real_id>\d{8})|http://www\.wat\.tv/video/(?P<display_id>.*)-(?P<short_id>.*?)_.*?\.html)'
+    _VALID_URL = r'(?:wat:(?P<real_id>\d{8})|https?://www\.wat\.tv/video/(?P<display_id>.*)-(?P<short_id>.*?)_.*?\.html)'
     IE_NAME = 'wat.tv'
     _TESTS = [
         {
diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 65cab4069..31c904303 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -244,7 +244,7 @@ class WDRMobileIE(InfoExtractor):
 
 
 class WDRMausIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?wdrmaus\.de/(?:[^/]+/){,2}(?P<id>[^/?#]+)(?:/index\.php5|(?<!index)\.php5|/(?:$|[?#]))'
+    _VALID_URL = r'https?://(?:www\.)?wdrmaus\.de/(?:[^/]+/){,2}(?P<id>[^/?#]+)(?:/index\.php5|(?<!index)\.php5|/(?:$|[?#]))'
     IE_DESC = 'Sendung mit der Maus'
     _TESTS = [{
         'url': 'http://www.wdrmaus.de/aktuelle-sendung/index.php5',
diff --git a/youtube_dl/extractor/weiqitv.py b/youtube_dl/extractor/weiqitv.py
index e333ae345..3dafbeec2 100644
--- a/youtube_dl/extractor/weiqitv.py
+++ b/youtube_dl/extractor/weiqitv.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 
 class WeiqiTVIE(InfoExtractor):
     IE_DESC = 'WQTV'
-    _VALID_URL = r'http://www\.weiqitv\.com/index/video_play\?videoId=(?P<id>[A-Za-z0-9]+)'
+    _VALID_URL = r'https?://www\.weiqitv\.com/index/video_play\?videoId=(?P<id>[A-Za-z0-9]+)'
 
     _TESTS = [{
         'url': 'http://www.weiqitv.com/index/video_play?videoId=53c744f09874f0e76a8b46f3',
diff --git a/youtube_dl/extractor/wimp.py b/youtube_dl/extractor/wimp.py
index fb0accac7..828c03dc3 100644
--- a/youtube_dl/extractor/wimp.py
+++ b/youtube_dl/extractor/wimp.py
@@ -5,7 +5,7 @@ from .youtube import YoutubeIE
 
 
 class WimpIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?wimp\.com/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?wimp\.com/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://www.wimp.com/maruexhausted/',
         'md5': 'ee21217ffd66d058e8b16be340b74883',
diff --git a/youtube_dl/extractor/xbef.py b/youtube_dl/extractor/xbef.py
index 4ff99e5ca..e4a2baad2 100644
--- a/youtube_dl/extractor/xbef.py
+++ b/youtube_dl/extractor/xbef.py
@@ -5,7 +5,7 @@ from ..compat import compat_urllib_parse_unquote
 
 
 class XBefIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?xbef\.com/video/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?xbef\.com/video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://xbef.com/video/5119-glamourous-lesbians-smoking-drinking-and-fucking',
         'md5': 'a478b565baff61634a98f5e5338be995',
diff --git a/youtube_dl/extractor/yam.py b/youtube_dl/extractor/yam.py
index 001ee17b6..63bbc0634 100644
--- a/youtube_dl/extractor/yam.py
+++ b/youtube_dl/extractor/yam.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 class YamIE(InfoExtractor):
     IE_DESC = '蕃薯藤yam天空部落'
-    _VALID_URL = r'http://mymedia.yam.com/m/(?P<id>\d+)'
+    _VALID_URL = r'https?://mymedia.yam.com/m/(?P<id>\d+)'
 
     _TESTS = [{
         # An audio hosted on Yam
diff --git a/youtube_dl/extractor/ynet.py b/youtube_dl/extractor/ynet.py
index 869f3e819..2522551dc 100644
--- a/youtube_dl/extractor/ynet.py
+++ b/youtube_dl/extractor/ynet.py
@@ -9,7 +9,7 @@ from ..compat import compat_urllib_parse_unquote_plus
 
 
 class YnetIE(InfoExtractor):
-    _VALID_URL = r'http://(?:.+?\.)?ynet\.co\.il/(?:.+?/)?0,7340,(?P<id>L(?:-[0-9]+)+),00\.html'
+    _VALID_URL = r'https?://(?:.+?\.)?ynet\.co\.il/(?:.+?/)?0,7340,(?P<id>L(?:-[0-9]+)+),00\.html'
     _TESTS = [
         {
             'url': 'http://hot.ynet.co.il/home/0,7340,L-11659-99244,00.html',

From 1600ed1ff9edb33ef901bda2c4b2732df7e0e4e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Mar 2016 21:46:49 +0600
Subject: [PATCH 1074/1105] [rutv] Improve flash version pattern (Closes #8911)

---
 youtube_dl/extractor/rutv.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index f7fe1fece..a2379eb04 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -14,7 +14,7 @@ class RUTVIE(InfoExtractor):
     IE_DESC = 'RUTV.RU'
     _VALID_URL = r'''(?x)
         https?://player\.(?:rutv\.ru|vgtrk\.com)/
-            (?P<path>flash2v/container\.swf\?id=
+            (?P<path>flash\d+v/container\.swf\?id=
             |iframe/(?P<type>swf|video|live)/id/
             |index/iframe/cast_id/)
             (?P<id>\d+)'''
@@ -109,7 +109,7 @@ class RUTVIE(InfoExtractor):
             return mobj.group('url')
 
         mobj = re.search(
-            r'<meta[^>]+?property=(["\'])og:video\1[^>]+?content=(["\'])(?P<url>https?://player\.(?:rutv\.ru|vgtrk\.com)/flash2v/container\.swf\?id=.+?\2)',
+            r'<meta[^>]+?property=(["\'])og:video\1[^>]+?content=(["\'])(?P<url>https?://player\.(?:rutv\.ru|vgtrk\.com)/flash\d+v/container\.swf\?id=.+?\2)',
             webpage)
         if mobj:
             return mobj.group('url')
@@ -119,7 +119,7 @@ class RUTVIE(InfoExtractor):
         video_id = mobj.group('id')
         video_path = mobj.group('path')
 
-        if video_path.startswith('flash2v'):
+        if re.match(r'flash\d+v', video_path):
             video_type = 'video'
         elif video_path.startswith('iframe'):
             video_type = mobj.group('type')
@@ -168,7 +168,7 @@ class RUTVIE(InfoExtractor):
                         'play_path': mobj.group('playpath'),
                         'app': mobj.group('app'),
                         'page_url': 'http://player.rutv.ru',
-                        'player_url': 'http://player.rutv.ru/flash2v/osmf.swf?i=22',
+                        'player_url': 'http://player.rutv.ru/flash3v/osmf.swf?i=22',
                         'rtmp_live': True,
                         'ext': 'flv',
                         'vbr': int(quality),

From 065c4b27bfc4488758b357e023e9b6b1679c9641 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Mar 2016 22:07:34 +0600
Subject: [PATCH 1075/1105] [xhamster:embed] Extract vars (Closes #8912)

---
 youtube_dl/extractor/xhamster.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index fd43e8854..b3547174d 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -4,6 +4,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    dict_get,
     float_or_none,
     int_or_none,
     unified_strdate,
@@ -170,6 +171,12 @@ class XHamsterEmbedIE(InfoExtractor):
 
         video_url = self._search_regex(
             r'href="(https?://xhamster\.com/movies/%s/[^"]+\.html[^"]*)"' % video_id,
-            webpage, 'xhamster url')
+            webpage, 'xhamster url', default=None)
+
+        if not video_url:
+            vars = self._parse_json(
+                self._search_regex(r'vars\s*:\s*({.+?})\s*,\s*\n', webpage, 'vars'),
+                video_id)
+            video_url = dict_get(vars, ('downloadLink', 'homepageLink', 'commentsLink', 'shareUrl'))
 
         return self.url_result(video_url, 'XHamster')

From ff5873b72de16854ae8d506d5648148a54828243 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Mar 2016 22:24:42 +0600
Subject: [PATCH 1076/1105] [motherless] Detect friends only videos

---
 youtube_dl/extractor/motherless.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index ad04b12cd..5e1a8a71a 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -69,6 +69,9 @@ class MotherlessIE(InfoExtractor):
                 ">The page you're looking for cannot be found.<")):
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
+        if '>The content you are trying to view is for friends only.' in webpage:
+            raise ExtractorError('Video %s is for friends only' % video_id, expected=True)
+
         title = self._html_search_regex(
             r'id="view-upload-title">\s+([^<]+)<', webpage, 'title')
         video_url = self._html_search_regex(

From 5c69f7a479936a8fc429228c8259dc5fcbf428b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Mar 2016 23:31:40 +0600
Subject: [PATCH 1077/1105] [animeondemand] Respect startvideo (Closes #8923)

---
 youtube_dl/extractor/animeondemand.py | 37 +++++++++++++++++++--------
 1 file changed, 26 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 4352525e2..81a843035 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -163,7 +163,7 @@ class AnimeOnDemandIE(InfoExtractor):
                     if kind:
                         format_id_list.append(kind)
                     if not format_id_list:
-                        format_id_list.append('hls')
+                        format_id_list.append(compat_str(num))
                     format_id = '-'.join(format_id_list)
                     format_note = ', '.join(filter(None, (kind, lang_note)))
                     request = sanitized_Request(
@@ -179,26 +179,41 @@ class AnimeOnDemandIE(InfoExtractor):
                         fatal=False)
                     if not playlist:
                         continue
+                    start_video = playlist.get('startvideo', 0)
                     playlist = playlist.get('playlist')
                     if not playlist or not isinstance(playlist, list):
                         continue
-                    playlist = playlist[0]
+                    playlist = playlist[start_video]
                     title = playlist.get('title')
                     if not title:
                         continue
                     description = playlist.get('description')
                     for source in playlist.get('sources', []):
                         file_ = source.get('file')
-                        if file_ and determine_ext(file_) == 'm3u8':
-                            m3u8_formats = self._extract_m3u8_formats(
+                        if not file_:
+                            continue
+                        ext = determine_ext(file_)
+                        format_id_list = [lang, kind]
+                        if ext == 'm3u8':
+                            format_id_list.append('hls')
+                        elif source.get('type') == 'video/dash' or ext == 'mpd':
+                            format_id_list.append('dash')
+                        format_id = '-'.join(filter(None, format_id_list))
+                        if ext == 'm3u8':
+                            file_formats = self._extract_m3u8_formats(
                                 file_, video_id, 'mp4',
-                                entry_protocol='m3u8_native', m3u8_id=format_id)
-                            for f in m3u8_formats:
-                                f.update({
-                                    'language': lang,
-                                    'format_note': format_note,
-                                })
-                            formats.extend(m3u8_formats)
+                                entry_protocol='m3u8_native', m3u8_id=format_id, fatal=False)
+                        elif source.get('type') == 'video/dash' or ext == 'mpd':
+                            file_formats = self._extract_mpd_formats(
+                                file_, video_id, mpd_id=format_id, fatal=False)
+                        else:
+                            continue
+                        for f in file_formats:
+                            f.update({
+                                'language': lang,
+                                'format_note': format_note,
+                            })
+                        formats.extend(file_formats)
 
             if formats:
                 self._sort_formats(formats)

From 85c637b7376f0426e5e0a6812da2a72b2ca28680 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Mar 2016 23:35:50 +0600
Subject: [PATCH 1078/1105] [animeondemand] Extract teaser when no full episode
 available (#8923)

---
 youtube_dl/extractor/animeondemand.py | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 81a843035..a47697738 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -225,16 +225,18 @@ class AnimeOnDemandIE(InfoExtractor):
                 })
                 entries.append(f)
 
-            m = re.search(
-                r'data-dialog-header=(["\'])(?P<title>.+?)\1[^>]+href=(["\'])(?P<href>.+?)\3[^>]*>Teaser<',
-                episode_html)
-            if m:
-                f = common_info.copy()
-                f.update({
-                    'id': '%s-teaser' % f['id'],
-                    'title': m.group('title'),
-                    'url': compat_urlparse.urljoin(url, m.group('href')),
-                })
-                entries.append(f)
+            # Extract teaser only when full episode is not available
+            if not formats:
+                m = re.search(
+                    r'data-dialog-header=(["\'])(?P<title>.+?)\1[^>]+href=(["\'])(?P<href>.+?)\3[^>]*>Teaser<',
+                    episode_html)
+                if m:
+                    f = common_info.copy()
+                    f.update({
+                        'id': '%s-teaser' % f['id'],
+                        'title': m.group('title'),
+                        'url': compat_urlparse.urljoin(url, m.group('href')),
+                    })
+                    entries.append(f)
 
         return self.playlist_result(entries, anime_id, anime_title, anime_description)

From bc5d16b3023ab29216d868bbfe8838b5e92e72f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Mar 2016 23:37:39 +0600
Subject: [PATCH 1079/1105] [animeondemand] Skip dash for now

---
 youtube_dl/extractor/animeondemand.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index a47697738..2cede55a7 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -204,6 +204,7 @@ class AnimeOnDemandIE(InfoExtractor):
                                 file_, video_id, 'mp4',
                                 entry_protocol='m3u8_native', m3u8_id=format_id, fatal=False)
                         elif source.get('type') == 'video/dash' or ext == 'mpd':
+                            continue
                             file_formats = self._extract_mpd_formats(
                                 file_, video_id, mpd_id=format_id, fatal=False)
                         else:

From cc7397b04d4a21b5ac680858ee2600e3b3bfb569 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 22 Mar 2016 21:12:29 +0600
Subject: [PATCH 1080/1105] [ceskatelevize] Make m3u8 formats extraction non
 fatal (Closes #8933)

---
 youtube_dl/extractor/ceskatelevize.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index b27b4e670..b355111cb 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -129,7 +129,8 @@ class CeskaTelevizeIE(InfoExtractor):
             formats = []
             for format_id, stream_url in item['streamUrls'].items():
                 formats.extend(self._extract_m3u8_formats(
-                    stream_url, playlist_id, 'mp4', entry_protocol='m3u8_native'))
+                    stream_url, playlist_id, 'mp4',
+                    entry_protocol='m3u8_native', fatal=False))
             self._sort_formats(formats)
 
             item_id = item.get('id') or item['assetId']

From 2beeb286e179a00bc0c76fc55ca5c8d19e74ca41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 22 Mar 2016 22:32:59 +0600
Subject: [PATCH 1081/1105] [laola1tv] Add support for livestreams (Closes
 #8934)

---
 youtube_dl/extractor/laola1tv.py | 31 ++++++++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index 5d8ebbeb3..41d80bc12 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -19,7 +19,7 @@ from ..utils import (
 
 
 class Laola1TvIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?laola1\.tv/(?P<lang>[a-z]+)-(?P<portal>[a-z]+)/[^/]+/(?P<slug>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?laola1\.tv/(?P<lang>[a-z]+)-(?P<portal>[a-z]+)/(?P<kind>[^/]+)/(?P<slug>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://www.laola1.tv/de-de/video/straubing-tigers-koelner-haie/227883.html',
         'info_dict': {
@@ -33,7 +33,7 @@ class Laola1TvIE(InfoExtractor):
         },
         'params': {
             'skip_download': True,
-        }
+        },
     }, {
         'url': 'http://www.laola1.tv/de-de/video/straubing-tigers-koelner-haie',
         'info_dict': {
@@ -47,12 +47,28 @@ class Laola1TvIE(InfoExtractor):
         },
         'params': {
             'skip_download': True,
-        }
+        },
+    }, {
+        'url': 'http://www.laola1.tv/de-de/livestream/2016-03-22-belogorie-belgorod-trentino-diatec-lde',
+        'info_dict': {
+            'id': '487850',
+            'display_id': '2016-03-22-belogorie-belgorod-trentino-diatec-lde',
+            'ext': 'flv',
+            'title': 'Belogorie BELGOROD - TRENTINO Diatec',
+            'upload_date': '20160322',
+            'uploader': 'CEV - Europäischer Volleyball Verband',
+            'is_live': True,
+            'categories': ['Volleyball'],
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         display_id = mobj.group('slug')
+        kind = mobj.group('kind')
         lang = mobj.group('lang')
         portal = mobj.group('portal')
 
@@ -85,12 +101,17 @@ class Laola1TvIE(InfoExtractor):
         _v = lambda x, **k: xpath_text(hd_doc, './/video/' + x, **k)
         title = _v('title', fatal=True)
 
+        VS_TARGETS = {
+            'video': '2',
+            'livestream': '17',
+        }
+
         req = sanitized_Request(
             'https://club.laola1.tv/sp/laola1/api/v3/user/session/premium/player/stream-access?%s' %
             compat_urllib_parse.urlencode({
                 'videoId': video_id,
-                'target': '2',
-                'label': 'laola1tv',
+                'target': VS_TARGETS.get(kind, '2'),
+                'label': _v('label'),
                 'area': _v('area'),
             }),
             urlencode_postdata(

From c6ca11f1b31a292413cab03012654fd0021814a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 22 Mar 2016 23:48:05 +0600
Subject: [PATCH 1082/1105] [once] Prevent ads from embedding into m3u8
 playlists (Closes #8893)

---
 youtube_dl/extractor/once.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/once.py b/youtube_dl/extractor/once.py
index 080045d4c..5db949b17 100644
--- a/youtube_dl/extractor/once.py
+++ b/youtube_dl/extractor/once.py
@@ -20,6 +20,10 @@ class OnceIE(InfoExtractor):
             media_item_id, 'mp4', m3u8_id='hls', fatal=False)
         progressive_formats = []
         for adaptive_format in formats:
+            # Prevent advertisement from embedding into m3u8 playlist (see
+            # https://github.com/rg3/youtube-dl/issues/8893#issuecomment-199912684)
+            adaptive_format['url'] = re.sub(
+                r'\badsegmentlength=\d+', r'adsegmentlength=0', adaptive_format['url'])
             rendition_id = self._search_regex(
                 r'/now/media/playlist/[^/]+/[^/]+/([^/]+)',
                 adaptive_format['url'], 'redition id', default=None)

From 7da2c87119db8beda1bdc979fad38c08fc1252e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 22 Mar 2016 22:17:59 +0100
Subject: [PATCH 1083/1105] Add extractor for thescene.com (closes #8929)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/thescene.py | 48 ++++++++++++++++++++++++++++++++
 2 files changed, 49 insertions(+)
 create mode 100644 youtube_dl/extractor/thescene.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ee792bbe0..8f7df4d12 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -743,6 +743,7 @@ from .theplatform import (
     ThePlatformIE,
     ThePlatformFeedIE,
 )
+from .thescene import TheSceneIE
 from .thesixtyone import TheSixtyOneIE
 from .thestar import TheStarIE
 from .thisamericanlife import ThisAmericanLifeIE
diff --git a/youtube_dl/extractor/thescene.py b/youtube_dl/extractor/thescene.py
new file mode 100644
index 000000000..08d666eaf
--- /dev/null
+++ b/youtube_dl/extractor/thescene.py
@@ -0,0 +1,48 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..compat import compat_urllib_parse
+from ..utils import qualities
+
+
+class TheSceneIE(InfoExtractor):
+    _VALID_URL = r'https://thescene\.com/watch/[^/]+/(?P<id>[^/#?]+)'
+
+    _TEST = {
+        'url': 'https://thescene.com/watch/vogue/narciso-rodriguez-spring-2013-ready-to-wear',
+        'info_dict': {
+            'id': '520e8faac2b4c00e3c6e5f43',
+            'ext': 'mp4',
+            'title': 'Narciso Rodriguez: Spring 2013 Ready-to-Wear',
+            'display_id': 'narciso-rodriguez-spring-2013-ready-to-wear',
+        },
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        player_url = compat_urllib_parse.urljoin(
+            url,
+            self._html_search_regex(
+                r'id=\'js-player-script\'[^>]+src=\'(.+?)\'', webpage, 'player url'))
+
+        self.to_screen(player_url)
+        player = self._download_webpage(player_url, player_url)
+        info = self._parse_json(self._search_regex(r'(?m)var\s+video\s+=\s+({.+?});$', player, 'info json'), display_id)
+
+        qualities_order = qualities(['low', 'high'])
+        formats = [{
+            'format_id': '{0}-{1}'.format(f['type'].split('/')[0], f['quality']),
+            'url': f['src'],
+            'quality': qualities_order(f['quality']),
+        } for f in info['sources'][0]]
+        self._sort_formats(formats)
+
+        return {
+            'id': info['id'],
+            'title': info['title'],
+            'formats': formats,
+            'thumbnail': info.get('poster_frame'),
+            'display_id': display_id,
+        }

From efbed08dc20c530fe428256e4dcbea4dc4423d0d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 23 Mar 2016 22:24:52 +0800
Subject: [PATCH 1084/1105] [utils] Encode hostnames before passing to urllib

With IDN (Internationalized Domain Name) and a proxy, non-ascii URLs
are passed down to urllib/urllib2, causing UnicodeEncodeError

Fixes #8890
---
 test/test_http.py   | 10 ++++++++++
 youtube_dl/utils.py |  1 +
 2 files changed, 11 insertions(+)

diff --git a/test/test_http.py b/test/test_http.py
index fc59b1aed..15e0ad369 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# coding: utf-8
 from __future__ import unicode_literals
 
 # Allow direct execution
@@ -120,5 +121,14 @@ class TestProxy(unittest.TestCase):
         response = ydl.urlopen(req).read().decode('utf-8')
         self.assertEqual(response, 'cn: {0}'.format(url))
 
+    def test_proxy_with_idn(self):
+        ydl = YoutubeDL({
+            'proxy': 'localhost:{0}'.format(self.port),
+        })
+        url = 'http://中文.tw/'
+        response = ydl.urlopen(url).read().decode('utf-8')
+        # b'xn--fiq228c' is '中文'.encode('idna')
+        self.assertEqual(response, 'normal: http://xn--fiq228c.tw/')
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 067b8a184..03bb7782f 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1746,6 +1746,7 @@ def escape_url(url):
     """Escape URL as suggested by RFC 3986"""
     url_parsed = compat_urllib_parse_urlparse(url)
     return url_parsed._replace(
+        netloc=url_parsed.netloc.encode('idna').decode('ascii'),
         path=escape_rfc3986(url_parsed.path),
         params=escape_rfc3986(url_parsed.params),
         query=escape_rfc3986(url_parsed.query),

From 882c6992967914c245e086ddaacde9d595cd6ed9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Mar 2016 20:45:39 +0600
Subject: [PATCH 1085/1105] [tunein] Fix stream data extraction (Closes #8899,
 closes #8924)

---
 youtube_dl/extractor/tunein.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/tunein.py b/youtube_dl/extractor/tunein.py
index 8322cc14d..ae4cfaec2 100644
--- a/youtube_dl/extractor/tunein.py
+++ b/youtube_dl/extractor/tunein.py
@@ -1,7 +1,7 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
+import re
 
 from .common import InfoExtractor
 from ..utils import ExtractorError
@@ -27,10 +27,9 @@ class TuneInBaseIE(InfoExtractor):
         if not streams_url.startswith('http://'):
             streams_url = compat_urlparse.urljoin(url, streams_url)
 
-        stream_data = self._download_webpage(
-            streams_url, content_id, note='Downloading stream data')
-        streams = json.loads(self._search_regex(
-            r'\((.*)\);', stream_data, 'stream info'))['Streams']
+        streams = self._download_json(
+            streams_url, content_id, note='Downloading stream data',
+            transform_source=lambda s: re.sub(r'^\s*\((.*)\);\s*$', r'\1', s))['Streams']
 
         is_live = None
         formats = []

From 2d60465e44c3290fa1ee4239fe18eb9d0c69d9e9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 23 Mar 2016 23:20:28 +0800
Subject: [PATCH 1086/1105] [test/test_utils] Update for escape_url change

---
 test/test_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 325b870cc..8ba531542 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -577,7 +577,7 @@ class TestUtil(unittest.TestCase):
         )
         self.assertEqual(
             escape_url('http://тест.рф/фрагмент'),
-            'http://тест.рф/%D1%84%D1%80%D0%B0%D0%B3%D0%BC%D0%B5%D0%BD%D1%82'
+            'http://xn--e1aybc.xn--p1ai/%D1%84%D1%80%D0%B0%D0%B3%D0%BC%D0%B5%D0%BD%D1%82'
         )
         self.assertEqual(
             escape_url('http://тест.рф/абв?абв=абв#абв'),

From 81f36eba886349475235dbacef29d6b23b40538f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 23 Mar 2016 23:23:26 +0800
Subject: [PATCH 1087/1105] [test/test_utils] Update for escape_url change
 (again)

---
 test/test_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 8ba531542..a35debfe1 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -581,7 +581,7 @@ class TestUtil(unittest.TestCase):
         )
         self.assertEqual(
             escape_url('http://тест.рф/абв?абв=абв#абв'),
-            'http://тест.рф/%D0%B0%D0%B1%D0%B2?%D0%B0%D0%B1%D0%B2=%D0%B0%D0%B1%D0%B2#%D0%B0%D0%B1%D0%B2'
+            'http://xn--e1aybc.xn--p1ai/%D0%B0%D0%B1%D0%B2?%D0%B0%D0%B1%D0%B2=%D0%B0%D0%B1%D0%B2#%D0%B0%D0%B1%D0%B2'
         )
         self.assertEqual(escape_url('http://vimeo.com/56015672#at=0'), 'http://vimeo.com/56015672#at=0')
 

From 088e1aac5970ea2c24fa902873a5e0b984b37595 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 23 Mar 2016 23:55:08 +0800
Subject: [PATCH 1088/1105] [generic] Support Vine embeds (#8817)

---
 youtube_dl/extractor/generic.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 26de27a7e..93e0563b5 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1909,6 +1909,14 @@ class GenericIE(InfoExtractor):
                 self._proto_relative_url(unescapeHTML(mobj.group(1))),
                 'AdobeTVVideo')
 
+        # Look for Vine embeds
+        mobj = re.search(
+            r'<iframe[^>]+src=[\'"]((?:https?:)?//(?:www\.)?vine\.co/v/[^/]+/embed/(?:simple|postcard))',
+            webpage)
+        if mobj is not None:
+            return self.url_result(
+                self._proto_relative_url(unescapeHTML(mobj.group(1))), 'Vine')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From fc27ea94642a8e2e9b0fcfdcc0c370ec7484c971 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 23 Mar 2016 23:55:52 +0800
Subject: [PATCH 1089/1105] [tumblr] Support Vine embeds (#8817)

---
 youtube_dl/extractor/tumblr.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index cea117c79..584716986 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -67,6 +67,22 @@ class TumblrIE(InfoExtractor):
             'uploader_id': 'user32021558',
         },
         'add_ie': ['Vimeo'],
+    }, {
+        'url': 'http://sutiblr.tumblr.com/post/139638707273',
+        'md5': '2dd184b3669e049ba40563a7d423f95c',
+        'info_dict': {
+            'id': 'ir7qBEIKqvq',
+            'ext': 'mp4',
+            'title': 'Vine by sutiblr',
+            'alt_title': 'Vine by sutiblr',
+            'uploader': 'sutiblr',
+            'uploader_id': '1198993975374495744',
+            'upload_date': '20160220',
+            'like_count': int,
+            'comment_count': int,
+            'repost_count': int,
+        },
+        'add_ie': ['Vine'],
     }]
 
     def _real_extract(self, url):

From c4096e8aeaa373159e350a3674b0ce18b6c519e2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 24 Mar 2016 16:29:33 +0800
Subject: [PATCH 1090/1105] [instagram] Extract embed videos (#8817)

---
 youtube_dl/extractor/instagram.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index ed3e07118..e8b27b379 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -4,6 +4,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    get_element_by_attribute,
     int_or_none,
     limit_length,
 )
@@ -38,6 +39,18 @@ class InstagramIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_embed_url(webpage):
+        blockquote_el = get_element_by_attribute(
+            'class', 'instagram-media', webpage)
+        if blockquote_el is None:
+            return
+
+        mobj = re.search(
+            r'<a[^>]+href=([\'"])(?P<link>[^\'"]+)\1', blockquote_el)
+        if mobj:
+            return mobj.group('link')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From 87696e78d7203cf47bdd27773d06ba15af7c819d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 24 Mar 2016 16:30:01 +0800
Subject: [PATCH 1091/1105] [instagram] Unescape description (#8817)

---
 youtube_dl/extractor/instagram.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index e8b27b379..4e62098b0 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -7,6 +7,7 @@ from ..utils import (
     get_element_by_attribute,
     int_or_none,
     limit_length,
+    lowercase_escape,
 )
 
 
@@ -59,6 +60,8 @@ class InstagramIE(InfoExtractor):
                                          webpage, 'uploader id', fatal=False)
         desc = self._search_regex(
             r'"caption":"(.+?)"', webpage, 'description', default=None)
+        if desc is not None:
+            desc = lowercase_escape(desc)
 
         return {
             'id': video_id,

From 5a51775a58b901c63bb0b59e44a113ad16ceb236 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 24 Mar 2016 16:32:27 +0800
Subject: [PATCH 1092/1105] [generic] Extract Instagram embeds (#8817)

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 93e0563b5..12f2309fc 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -59,6 +59,7 @@ from .videomore import VideomoreIE
 from .googledrive import GoogleDriveIE
 from .jwplatform import JWPlatformIE
 from .digiteka import DigitekaIE
+from .instagram import InstagramIE
 
 
 class GenericIE(InfoExtractor):
@@ -1917,6 +1918,11 @@ class GenericIE(InfoExtractor):
             return self.url_result(
                 self._proto_relative_url(unescapeHTML(mobj.group(1))), 'Vine')
 
+        # Look for Instagram embeds
+        instagram_embed_url = InstagramIE._extract_embed_url(webpage)
+        if instagram_embed_url is not None:
+            return self.url_result(instagram_embed_url, InstagramIE.ie_key())
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From 32d88410eb2ef0ca25ab770af8f2ca2326c0aca7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 24 Mar 2016 16:32:53 +0800
Subject: [PATCH 1093/1105] [tumblr] Add a test with Instagram embed

Closes #8817
---
 youtube_dl/extractor/tumblr.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 584716986..e5bcf7798 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -83,6 +83,18 @@ class TumblrIE(InfoExtractor):
             'repost_count': int,
         },
         'add_ie': ['Vine'],
+    }, {
+        'url': 'http://vitasidorkina.tumblr.com/post/134652425014/joskriver-victoriassecret-invisibility-or',
+        'md5': '01c12ceb82cbf6b2fe0703aa56b3ad72',
+        'info_dict': {
+            'id': '-7LnUPGlSo',
+            'ext': 'mp4',
+            'title': 'Video by victoriassecret',
+            'description': 'Invisibility or flight…which superpower would YOU choose? #VSFashionShow #ThisOrThat',
+            'uploader_id': 'victoriassecret',
+            'thumbnail': 're:^https?://.*\.jpg'
+        },
+        'add_ie': ['Instagram'],
     }]
 
     def _real_extract(self, url):

From 622d19160bd81161e18c6ce868c359549a4d0413 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 24 Mar 2016 18:06:15 +0800
Subject: [PATCH 1094/1105] [utils] Clarify Python versions affected by buggy
 struct module

---
 youtube_dl/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 03bb7782f..b6e1dc809 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1756,7 +1756,8 @@ def escape_url(url):
 try:
     struct.pack('!I', 0)
 except TypeError:
-    # In Python 2.6 (and some 2.7 versions), struct requires a bytes argument
+    # In Python 2.6 and 2.7.x < 2.7.7, struct requires a bytes argument
+    # See https://bugs.python.org/issue19099
     def struct_pack(spec, *args):
         if isinstance(spec, compat_str):
             spec = spec.encode('ascii')

From 5767b4eeae997a4ef75e348b46489cbb55126414 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Mar 2016 22:23:31 +0600
Subject: [PATCH 1095/1105] [mtv] Fix description extraction (Closes #8962)

---
 youtube_dl/extractor/mtv.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index ed068365d..824bbcb4e 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -17,6 +17,7 @@ from ..utils import (
     unescapeHTML,
     url_basename,
     RegexNotFoundError,
+    xpath_text,
 )
 
 
@@ -130,11 +131,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
             message += item.text
             raise ExtractorError(message, expected=True)
 
-        description_node = itemdoc.find('description')
-        if description_node is not None:
-            description = description_node.text.strip()
-        else:
-            description = None
+        description = xpath_text(itemdoc, 'description')
 
         title_el = None
         if title_el is None:

From 369e7e3ff02201210864b4e20af2893c40894ddf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Mar 2016 22:54:26 +0600
Subject: [PATCH 1096/1105] [iprima] Fix extraction (Closes #8953)

---
 youtube_dl/extractor/iprima.py | 44 ++++++++++++++++++++++++++++++----
 1 file changed, 39 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 61a0de472..788bbe0d5 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -6,6 +6,8 @@ import time
 
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
+    js_to_json,
     sanitized_Request,
 )
 
@@ -30,8 +32,7 @@ class IPrimaIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
@@ -43,9 +44,42 @@ class IPrimaIE(InfoExtractor):
         req.add_header('Referer', url)
         playerpage = self._download_webpage(req, video_id, note='Downloading player')
 
-        m3u8_url = self._search_regex(r"'src': '([^']+\.m3u8)'", playerpage, 'm3u8 url')
+        formats = []
 
-        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
+        def extract_formats(format_url, format_key=None, lang=None):
+            ext = determine_ext(format_url)
+            new_formats = []
+            if format_key == 'hls' or ext == 'm3u8':
+                new_formats = self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False)
+            elif format_key == 'dash' or ext == 'mpd':
+                return
+                new_formats = self._extract_mpd_formats(
+                    format_url, video_id, mpd_id='dash', fatal=False)
+            if lang:
+                for f in new_formats:
+                    if not f.get('language'):
+                        f['language'] = lang
+            formats.extend(new_formats)
+
+        options = self._parse_json(
+            self._search_regex(
+                r'(?s)var\s+playerOptions\s*=\s*({.+?});',
+                playerpage, 'player options', default='{}'),
+            video_id, transform_source=js_to_json, fatal=False)
+        if options:
+            for key, tracks in options.get('tracks', {}).items():
+                if not isinstance(tracks, list):
+                    continue
+                for track in tracks:
+                    src = track.get('src')
+                    if src:
+                        extract_formats(src, key.lower(), track.get('lang'))
+
+        if not formats:
+            for _, src in re.findall(r'src["\']\s*:\s*(["\'])(.+?)\1', playerpage):
+                extract_formats(src)
 
         self._sort_formats(formats)
 

From 993271da0a70d6d5c194a10e48d43f3aa2abc956 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Mar 2016 23:28:24 +0600
Subject: [PATCH 1097/1105] [nytimes] Tolerate missing metadata (Closes #8952)

---
 youtube_dl/extractor/nytimes.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/nytimes.py b/youtube_dl/extractor/nytimes.py
index 7f254b867..681683e86 100644
--- a/youtube_dl/extractor/nytimes.py
+++ b/youtube_dl/extractor/nytimes.py
@@ -18,8 +18,9 @@ class NYTimesBaseIE(InfoExtractor):
         description = video_data.get('summary')
         duration = float_or_none(video_data.get('duration'), 1000)
 
-        uploader = video_data['byline']
-        timestamp = parse_iso8601(video_data['publication_date'][:-8])
+        uploader = video_data.get('byline')
+        publication_date = video_data.get('publication_date')
+        timestamp = parse_iso8601(publication_date[:-8]) if publication_date else None
 
         def get_file_size(file_size):
             if isinstance(file_size, int):
@@ -37,7 +38,7 @@ class NYTimesBaseIE(InfoExtractor):
                 'width': int_or_none(video.get('width')),
                 'height': int_or_none(video.get('height')),
                 'filesize': get_file_size(video.get('fileSize')),
-            } for video in video_data['renditions']
+            } for video in video_data['renditions'] if video.get('url')
         ]
         self._sort_formats(formats)
 
@@ -46,7 +47,7 @@ class NYTimesBaseIE(InfoExtractor):
                 'url': 'http://www.nytimes.com/%s' % image['url'],
                 'width': int_or_none(image.get('width')),
                 'height': int_or_none(image.get('height')),
-            } for image in video_data['images']
+            } for image in video_data.get('images', []) if image.get('url')
         ]
 
         return {

From f07e276a04292c3fa87f703931bad9b716e7ccdf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Mar 2016 01:18:14 +0600
Subject: [PATCH 1098/1105] [youtube:live] Add extractor (Closes #8959)

---
 youtube_dl/extractor/youtube.py | 48 ++++++++++++++++++++++++++++++++-
 1 file changed, 47 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 466f5da2e..96fa3b5aa 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1911,7 +1911,8 @@ class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
 
     @classmethod
     def suitable(cls, url):
-        return False if YoutubePlaylistsIE.suitable(url) else super(YoutubeChannelIE, cls).suitable(url)
+        return (False if YoutubePlaylistsIE.suitable(url) or YoutubeLiveIE.suitable(url)
+                else super(YoutubeChannelIE, cls).suitable(url))
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)
@@ -1986,6 +1987,51 @@ class YoutubeUserIE(YoutubeChannelIE):
             return super(YoutubeUserIE, cls).suitable(url)
 
 
+class YoutubeLiveIE(YoutubeBaseInfoExtractor):
+    IE_DESC = 'YouTube.com live streams'
+    _VALID_URL = r'(?P<base_url>https?://(?:\w+\.)?youtube\.com/(?:user|channel)/(?P<id>[^/]+))/live'
+    IE_NAME = 'youtube:live'
+
+    _TESTS = [{
+        'url': 'http://www.youtube.com/user/TheYoungTurks/live',
+        'info_dict': {
+            'id': 'a48o2S1cPoo',
+            'ext': 'mp4',
+            'title': 'The Young Turks - Live Main Show',
+            'uploader': 'The Young Turks',
+            'uploader_id': 'TheYoungTurks',
+            'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
+            'upload_date': '20150715',
+            'license': 'Standard YouTube License',
+            'description': 'md5:438179573adcdff3c97ebb1ee632b891',
+            'categories': ['News & Politics'],
+            'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
+            'like_count': int,
+            'dislike_count': int,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        channel_id = mobj.group('id')
+        base_url = mobj.group('base_url')
+        webpage = self._download_webpage(url, channel_id, fatal=False)
+        if webpage:
+            page_type = self._og_search_property(
+                'type', webpage, 'page type', default=None)
+            video_id = self._html_search_meta(
+                'videoId', webpage, 'video id', default=None)
+            if page_type == 'video' and video_id and re.match(r'^[0-9A-Za-z_-]{11}$', video_id):
+                return self.url_result(video_id, YoutubeIE.ie_key())
+        return self.url_result(base_url)
+
+
 class YoutubePlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
     IE_DESC = 'YouTube.com user/channel playlists'
     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/(?:user|channel)/(?P<id>[^/]+)/playlists'

From d041a736741e37e75e94cddf9c8258de8f08b501 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Mar 2016 01:39:25 +0600
Subject: [PATCH 1099/1105] [extractor/__init__] Add youtube:live and sort
 youtube extractors alphabetically

---
 youtube_dl/extractor/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8f7df4d12..d99873419 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -960,7 +960,9 @@ from .youtube import (
     YoutubeChannelIE,
     YoutubeFavouritesIE,
     YoutubeHistoryIE,
+    YoutubeLiveIE,
     YoutubePlaylistIE,
+    YoutubePlaylistsIE,
     YoutubeRecommendedIE,
     YoutubeSearchDateIE,
     YoutubeSearchIE,
@@ -970,7 +972,6 @@ from .youtube import (
     YoutubeTruncatedIDIE,
     YoutubeTruncatedURLIE,
     YoutubeUserIE,
-    YoutubePlaylistsIE,
     YoutubeWatchLaterIE,
 )
 from .zapiks import ZapiksIE

From ff9d5d093854a974afdd0191d331d817e7e4c2a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Mar 2016 02:26:46 +0600
Subject: [PATCH 1100/1105] [udemy] Improve course enrolling

---
 youtube_dl/extractor/udemy.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 74cc36ece..a5634ece9 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -5,6 +5,7 @@ from ..compat import (
     compat_HTTPError,
     compat_urllib_parse,
     compat_urllib_request,
+    compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
@@ -35,7 +36,7 @@ class UdemyIE(InfoExtractor):
         'skip': 'Requires udemy account credentials',
     }]
 
-    def _enroll_course(self, webpage, course_id):
+    def _enroll_course(self, base_url, webpage, course_id):
         checkout_url = unescapeHTML(self._search_regex(
             r'href=(["\'])(?P<url>https?://(?:www\.)?udemy\.com/payment/checkout/.+?)\1',
             webpage, 'checkout url', group='url', default=None))
@@ -45,9 +46,11 @@ class UdemyIE(InfoExtractor):
                 'Use this URL to confirm purchase: %s' % (course_id, checkout_url), expected=True)
 
         enroll_url = unescapeHTML(self._search_regex(
-            r'href=(["\'])(?P<url>https?://(?:www\.)?udemy\.com/course/subscribe/.+?)\1',
+            r'href=(["\'])(?P<url>(?:https?://(?:www\.)?udemy\.com)?/course/subscribe/.+?)\1',
             webpage, 'enroll url', group='url', default=None))
         if enroll_url:
+            if not enroll_url.startswith('http'):
+                enroll_url = compat_urlparse.urljoin(base_url, enroll_url)
             webpage = self._download_webpage(enroll_url, course_id, 'Enrolling in the course')
             if '>You have enrolled in' in webpage:
                 self.to_screen('%s: Successfully enrolled in the course' % course_id)
@@ -152,7 +155,7 @@ class UdemyIE(InfoExtractor):
         except ExtractorError as e:
             # Error could possibly mean we are not enrolled in the course
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                self._enroll_course(webpage, course_id)
+                self._enroll_course(url, webpage, course_id)
                 lecture = self._download_lecture(course_id, lecture_id)
             else:
                 raise
@@ -244,7 +247,7 @@ class UdemyCourseIE(UdemyIE):
         course_id = response['id']
         course_title = response.get('title')
 
-        self._enroll_course(webpage, course_id)
+        self._enroll_course(url, webpage, course_id)
 
         response = self._download_json(
             'https://www.udemy.com/api-1.1/courses/%s/curriculum' % course_id,

From f0e83681d97db52af9dc73d1c8e51d671503a222 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Mar 2016 02:27:13 +0600
Subject: [PATCH 1101/1105] [udemy] Extract formats from outputs

---
 youtube_dl/extractor/udemy.py | 78 +++++++++++++++++++++--------------
 1 file changed, 48 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index a5634ece9..2b886d6c8 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -180,39 +180,57 @@ class UdemyIE(InfoExtractor):
         video_id = asset['id']
         thumbnail = asset.get('thumbnailUrl') or asset.get('thumbnail_url')
         duration = float_or_none(asset.get('data', {}).get('duration'))
-        outputs = asset.get('data', {}).get('outputs', {})
 
         formats = []
-        for format_ in asset.get('download_urls', {}).get('Video', []):
-            video_url = format_.get('file')
-            if not video_url:
-                continue
-            format_id = format_.get('label')
-            f = {
-                'url': format_['file'],
-                'height': int_or_none(format_id),
+
+        def extract_output_format(src):
+            return {
+                'url': src['url'],
+                'format_id': '%sp' % (src.get('label') or format_id),
+                'width': int_or_none(src.get('width')),
+                'height': int_or_none(src.get('height')),
+                'vbr': int_or_none(src.get('video_bitrate_in_kbps')),
+                'vcodec': src.get('video_codec'),
+                'fps': int_or_none(src.get('frame_rate')),
+                'abr': int_or_none(src.get('audio_bitrate_in_kbps')),
+                'acodec': src.get('audio_codec'),
+                'asr': int_or_none(src.get('audio_sample_rate')),
+                'tbr': int_or_none(src.get('total_bitrate_in_kbps')),
+                'filesize': int_or_none(src.get('file_size_in_bytes')),
             }
-            if format_id:
-                # Some videos contain additional metadata (e.g.
-                # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
-                output = outputs.get(format_id)
-                if isinstance(output, dict):
-                    f.update({
-                        'format_id': '%sp' % (output.get('label') or format_id),
-                        'width': int_or_none(output.get('width')),
-                        'height': int_or_none(output.get('height')),
-                        'vbr': int_or_none(output.get('video_bitrate_in_kbps')),
-                        'vcodec': output.get('video_codec'),
-                        'fps': int_or_none(output.get('frame_rate')),
-                        'abr': int_or_none(output.get('audio_bitrate_in_kbps')),
-                        'acodec': output.get('audio_codec'),
-                        'asr': int_or_none(output.get('audio_sample_rate')),
-                        'tbr': int_or_none(output.get('total_bitrate_in_kbps')),
-                        'filesize': int_or_none(output.get('file_size_in_bytes')),
-                    })
-                else:
-                    f['format_id'] = '%sp' % format_id
-            formats.append(f)
+
+        outputs = asset.get('data', {}).get('outputs')
+        if not isinstance(outputs, dict):
+            outputs = {}
+
+        for format_id, output in outputs.items():
+            if isinstance(output, dict) and output.get('url'):
+                formats.append(extract_output_format(output))
+
+        download_urls = asset.get('download_urls')
+        if isinstance(download_urls, dict):
+            video = download_urls.get('Video')
+            if isinstance(video, list):
+                for format_ in video:
+                    video_url = format_.get('file')
+                    if not video_url:
+                        continue
+                    format_id = format_.get('label')
+                    f = {
+                        'url': format_['file'],
+                        'height': int_or_none(format_id),
+                    }
+                    if format_id:
+                        # Some videos contain additional metadata (e.g.
+                        # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
+                        output = outputs.get(format_id)
+                        if isinstance(output, dict):
+                            output_format = extract_output_format(output)
+                            output_format.update(f)
+                            f = output_format
+                        else:
+                            f['format_id'] = '%sp' % format_id
+                    formats.append(f)
 
         self._sort_formats(formats)
 

From 5eb7db4ee96c97d98a87d8d7df46e8c6a607b682 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Mar 2016 02:28:39 +0600
Subject: [PATCH 1102/1105] [udemy] Add support for new URL schema

---
 youtube_dl/extractor/udemy.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 2b886d6c8..a9046b865 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -18,7 +18,16 @@ from ..utils import (
 
 class UdemyIE(InfoExtractor):
     IE_NAME = 'udemy'
-    _VALID_URL = r'https?://www\.udemy\.com/(?:[^#]+#/lecture/|lecture/view/?\?lectureId=)(?P<id>\d+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        www\.udemy\.com/
+                        (?:
+                            [^#]+\#/lecture/|
+                            lecture/view/?\?lectureId=|
+                            [^/]+/learn/v4/t/lecture/
+                        )
+                        (?P<id>\d+)
+                    '''
     _LOGIN_URL = 'https://www.udemy.com/join/login-popup/?displayType=ajax&showSkipButton=1'
     _ORIGIN_URL = 'https://www.udemy.com'
     _NETRC_MACHINE = 'udemy'
@@ -34,6 +43,10 @@ class UdemyIE(InfoExtractor):
             'duration': 579.29,
         },
         'skip': 'Requires udemy account credentials',
+    }, {
+        # new URL schema
+        'url': 'https://www.udemy.com/electric-bass-right-from-the-start/learn/v4/t/lecture/4580906',
+        'only_matching': True,
     }]
 
     def _enroll_course(self, base_url, webpage, course_id):

From e0317686666f9de4a6eca3fc26ede32e664f2bec Mon Sep 17 00:00:00 2001
From: Kagami Hiiragi <kagami@genshiken.org>
Date: Thu, 24 Mar 2016 13:55:02 +0300
Subject: [PATCH 1103/1105] [mnet] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/mnet.py     | 76 ++++++++++++++++++++++++++++++++
 2 files changed, 77 insertions(+)
 create mode 100644 youtube_dl/extractor/mnet.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d99873419..1e4b078a4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -410,6 +410,7 @@ from .mit import TechTVMITIE, MITIE, OCWMITIE
 from .mitele import MiTeleIE
 from .mixcloud import MixcloudIE
 from .mlb import MLBIE
+from .mnet import MnetIE
 from .mpora import MporaIE
 from .moevideo import MoeVideoIE
 from .mofosex import MofosexIE
diff --git a/youtube_dl/extractor/mnet.py b/youtube_dl/extractor/mnet.py
new file mode 100644
index 000000000..8e83b1fc3
--- /dev/null
+++ b/youtube_dl/extractor/mnet.py
@@ -0,0 +1,76 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_duration,
+    parse_iso8601,
+)
+
+
+class MnetIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?mnet\.(?:com|interest\.me)/tv/vod/(?:.*?\bclip_id=)?(?P<id>[0-9]+)'
+    _TESTS = [
+        {
+            'url': 'http://www.mnet.com/tv/vod/171008',
+            'md5': '6abd7a837fa9fe56d22709a60b19bffb',
+            'info_dict': {
+                'id': '171008',
+                'title': 'SS_이해인@히든박스',
+                'description': 'md5:b9efa592c3918b615ba69fe9f8a05c55',
+                'duration': 88,
+                'upload_date': '20151231',
+                'timestamp': 1451564040,
+                'age_limit': 0,
+                'thumbnails': 'mincount:5',
+                'ext': 'flv',
+            },
+        },
+        {
+            'url': 'http://mnet.interest.me/tv/vod/172790',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://www.mnet.com/tv/vod/vod_view.asp?clip_id=172790&tabMenu=',
+            'only_matching': True,
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        info_url = 'http://content.api.mnet.com/player/vodConfig?id=%s' % video_id
+        info = self._download_json(info_url, video_id)
+        info = info['data']['info']
+
+        title = info['title']
+        rtmp_info_url = info['cdn'] + 'CLIP'
+        rtmp_info = self._download_json(rtmp_info_url, video_id)
+        file_url = rtmp_info['serverurl'] + rtmp_info['fileurl']
+        description = info.get('ment')
+        duration = parse_duration(info.get('time'))
+        timestamp = parse_iso8601(info.get('date'), delimiter=' ')
+        age_limit = info.get('adult')
+        if age_limit is not None:
+            age_limit = 0 if age_limit == 'N' else 18
+        thumbnails = [
+            {
+                'id': thumb_format,
+                'url': thumb['url'],
+                'width': int_or_none(thumb.get('width')),
+                'height': int_or_none(thumb.get('height')),
+            }
+            for (thumb_format, thumb) in info.get('cover', {}).items()
+        ]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': file_url,
+            'description': description,
+            'duration': duration,
+            'timestamp': timestamp,
+            'age_limit': age_limit,
+            'thumbnails': thumbnails,
+            'ext': 'flv',
+        }

From 98e68806fb8cfe2a81ee8a6ac6705bb3d61ed2d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Mar 2016 03:26:29 +0600
Subject: [PATCH 1104/1105] [mnet] Improve (Closes #8958)

---
 youtube_dl/extractor/mnet.py | 85 +++++++++++++++++++-----------------
 1 file changed, 45 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/extractor/mnet.py b/youtube_dl/extractor/mnet.py
index 8e83b1fc3..e3f42e7bd 100644
--- a/youtube_dl/extractor/mnet.py
+++ b/youtube_dl/extractor/mnet.py
@@ -11,66 +11,71 @@ from ..utils import (
 
 class MnetIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?mnet\.(?:com|interest\.me)/tv/vod/(?:.*?\bclip_id=)?(?P<id>[0-9]+)'
-    _TESTS = [
-        {
-            'url': 'http://www.mnet.com/tv/vod/171008',
-            'md5': '6abd7a837fa9fe56d22709a60b19bffb',
-            'info_dict': {
-                'id': '171008',
-                'title': 'SS_이해인@히든박스',
-                'description': 'md5:b9efa592c3918b615ba69fe9f8a05c55',
-                'duration': 88,
-                'upload_date': '20151231',
-                'timestamp': 1451564040,
-                'age_limit': 0,
-                'thumbnails': 'mincount:5',
-                'ext': 'flv',
-            },
+    _TESTS = [{
+        'url': 'http://www.mnet.com/tv/vod/171008',
+        'info_dict': {
+            'id': '171008',
+            'title': 'SS_이해인@히든박스',
+            'description': 'md5:b9efa592c3918b615ba69fe9f8a05c55',
+            'duration': 88,
+            'upload_date': '20151231',
+            'timestamp': 1451564040,
+            'age_limit': 0,
+            'thumbnails': 'mincount:5',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'ext': 'flv',
         },
-        {
-            'url': 'http://mnet.interest.me/tv/vod/172790',
-            'only_matching': True,
+        'params': {
+            # rtmp download
+            'skip_download': True,
         },
-        {
-            'url': 'http://www.mnet.com/tv/vod/vod_view.asp?clip_id=172790&tabMenu=',
-            'only_matching': True,
-        },
-    ]
+    }, {
+        'url': 'http://mnet.interest.me/tv/vod/172790',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.mnet.com/tv/vod/vod_view.asp?clip_id=172790&tabMenu=',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        info_url = 'http://content.api.mnet.com/player/vodConfig?id=%s' % video_id
-        info = self._download_json(info_url, video_id)
-        info = info['data']['info']
+
+        info = self._download_json(
+            'http://content.api.mnet.com/player/vodConfig?id=%s&ctype=CLIP' % video_id,
+            video_id, 'Downloading vod config JSON')['data']['info']
 
         title = info['title']
-        rtmp_info_url = info['cdn'] + 'CLIP'
-        rtmp_info = self._download_json(rtmp_info_url, video_id)
-        file_url = rtmp_info['serverurl'] + rtmp_info['fileurl']
+
+        rtmp_info = self._download_json(
+            info['cdn'], video_id, 'Downloading vod cdn JSON')
+
+        formats = [{
+            'url': rtmp_info['serverurl'] + rtmp_info['fileurl'],
+            'ext': 'flv',
+            'page_url': url,
+            'player_url': 'http://flvfile.mnet.com/service/player/201602/cjem_player_tv.swf?v=201602191318',
+        }]
+
         description = info.get('ment')
         duration = parse_duration(info.get('time'))
         timestamp = parse_iso8601(info.get('date'), delimiter=' ')
         age_limit = info.get('adult')
         if age_limit is not None:
             age_limit = 0 if age_limit == 'N' else 18
-        thumbnails = [
-            {
-                'id': thumb_format,
-                'url': thumb['url'],
-                'width': int_or_none(thumb.get('width')),
-                'height': int_or_none(thumb.get('height')),
-            }
-            for (thumb_format, thumb) in info.get('cover', {}).items()
-        ]
+        thumbnails = [{
+            'id': thumb_format,
+            'url': thumb['url'],
+            'width': int_or_none(thumb.get('width')),
+            'height': int_or_none(thumb.get('height')),
+        } for thumb_format, thumb in info.get('cover', {}).items() if thumb.get('url')]
 
         return {
             'id': video_id,
             'title': title,
-            'url': file_url,
             'description': description,
             'duration': duration,
             'timestamp': timestamp,
             'age_limit': age_limit,
             'thumbnails': thumbnails,
-            'ext': 'flv',
+            'formats': formats,
         }

From 3f15fec1d13cf4b18c093271bee7ad7586c97f9c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Mar 2016 03:56:27 +0600
Subject: [PATCH 1105/1105] Credit @Kagami for mnet (#8958)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 51dfc8ddd..ea8d39978 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -166,3 +166,4 @@ Ben Congdon
 Kacper Michajłow
 José Joaquín Atria
 Viťas Strádal
+Kagami Hiiragi