From 1f9fb20fcda76f165ce39b01fe907fc74c8054d3 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 25 Sep 2015 07:39:22 +0100
Subject: [PATCH 001/150] [nextmedia] update AppleDailyIE tests

---
 youtube_dl/extractor/nextmedia.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nextmedia.py b/youtube_dl/extractor/nextmedia.py
index c10784f6b..d1688457f 100644
--- a/youtube_dl/extractor/nextmedia.py
+++ b/youtube_dl/extractor/nextmedia.py
@@ -126,7 +126,8 @@ class AppleDailyIE(NextMediaIE):
             'thumbnail': 're:^https?://.*\.jpg$',
             'description': 'md5:23c0aac567dc08c9c16a3161a2c2e3cd',
             'upload_date': '20150128',
-        }
+        },
+        'skip': 'redirect to http://www.appledaily.com.tw/animation/',
     }, {
         # No thumbnail
         'url': 'http://www.appledaily.com.tw/animation/realtimenews/new/20150128/5003673/',
@@ -140,10 +141,19 @@ class AppleDailyIE(NextMediaIE):
         },
         'expected_warnings': [
             'video thumbnail',
-        ]
+        ],
+        'skip': 'redirect to http://www.appledaily.com.tw/animation/',
     }, {
         'url': 'http://www.appledaily.com.tw/appledaily/article/supplement/20140417/35770334/',
-        'only_matching': True,
+        'md5': 'eaa20e6b9df418c912d7f5dec2ba734d',
+        'info_dict': {
+            'id': '35770334',
+            'ext': 'mp4',
+            'title': '咖啡占卜測 XU裝熟指數',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:7b859991a6a4fedbdf3dd3b66545c748',
+            'upload_date': '20140417',
+        },
     }]
 
     _URL_PATTERN = r'\{url: \'(.+)\'\}'

From 8a64969404ecbbac9cb8b970f7ca36a9e6c03cf2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Sep 2015 21:33:21 +0600
Subject: [PATCH 002/150] [adultswim] Prefer stream (Closes #7015)

---
 youtube_dl/extractor/adultswim.py | 37 ++++++++++++++++++++++---------
 1 file changed, 26 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 4327c2f61..27de07587 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
     ExtractorError,
     float_or_none,
     xpath_text,
@@ -123,7 +124,6 @@ class AdultSwimIE(InfoExtractor):
         else:
             collections = bootstrapped_data['show']['collections']
             collection, video_info = self.find_collection_containing_video(collections, episode_path)
-
             # Video wasn't found in the collections, let's try `slugged_video`.
             if video_info is None:
                 if bootstrapped_data.get('slugged_video', {}).get('slug') == episode_path:
@@ -133,7 +133,9 @@ class AdultSwimIE(InfoExtractor):
 
             show = bootstrapped_data['show']
             show_title = show['title']
-            segment_ids = [clip['videoPlaybackID'] for clip in video_info['clips']]
+            stream = video_info.get('stream')
+            clips = [stream] if stream else video_info['clips']
+            segment_ids = [clip['videoPlaybackID'] for clip in clips]
 
         episode_id = video_info['id']
         episode_title = video_info['title']
@@ -142,7 +144,7 @@ class AdultSwimIE(InfoExtractor):
 
         entries = []
         for part_num, segment_id in enumerate(segment_ids):
-            segment_url = 'http://www.adultswim.com/videos/api/v0/assets?id=%s&platform=mobile' % segment_id
+            segment_url = 'http://www.adultswim.com/videos/api/v0/assets?id=%s&platform=desktop' % segment_id
 
             segment_title = '%s - %s' % (show_title, episode_title)
             if len(segment_ids) > 1:
@@ -158,17 +160,30 @@ class AdultSwimIE(InfoExtractor):
             formats = []
             file_els = idoc.findall('.//files/file') or idoc.findall('./files/file')
 
+            unique_urls = []
+            unique_file_els = []
             for file_el in file_els:
+                media_url = file_el.text
+                if not media_url or determine_ext(media_url) == 'f4m':
+                    continue
+                if file_el.text not in unique_urls:
+                    unique_urls.append(file_el.text)
+                    unique_file_els.append(file_el)
+
+            for file_el in unique_file_els:
                 bitrate = file_el.attrib.get('bitrate')
                 ftype = file_el.attrib.get('type')
-
-                formats.append({
-                    'format_id': '%s_%s' % (bitrate, ftype),
-                    'url': file_el.text.strip(),
-                    # The bitrate may not be a number (for example: 'iphone')
-                    'tbr': int(bitrate) if bitrate.isdigit() else None,
-                    'quality': 1 if ftype == 'hd' else -1
-                })
+                media_url = file_el.text
+                if determine_ext(media_url) == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        media_url, segment_title, 'mp4', 'm3u8_native', preference=0, m3u8_id='hls'))
+                else:
+                    formats.append({
+                        'format_id': '%s_%s' % (bitrate, ftype),
+                        'url': file_el.text.strip(),
+                        # The bitrate may not be a number (for example: 'iphone')
+                        'tbr': int(bitrate) if bitrate.isdigit() else None,
+                    })
 
             self._sort_formats(formats)
 

From c596ce91cd22914e031b9ac94ec38d1137e50d60 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 30 Sep 2015 15:39:47 +0200
Subject: [PATCH 003/150] [comedycentral] Fix youtube-dl :thedailyshow

We'll let the generic IE follow the redirect and call back to us with the episode URL
---
 youtube_dl/extractor/comedycentral.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 91ebb0ce5..3e4bd10b6 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -151,12 +151,7 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
         mobj = re.match(self._VALID_URL, url)
 
         if mobj.group('shortname'):
-            if mobj.group('shortname') in ('tds', 'thedailyshow'):
-                url = 'http://thedailyshow.cc.com/full-episodes/'
-            else:
-                url = 'http://thecolbertreport.cc.com/full-episodes/'
-            mobj = re.match(self._VALID_URL, url, re.VERBOSE)
-            assert mobj is not None
+            return self.url_result('http://www.cc.com/shows/the-daily-show-with-trevor-noah/full-episodes')
 
         if mobj.group('clip'):
             if mobj.group('videotitle'):

From aedb930cfcd068600d3b5b0b3aef32df1a28f0c0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 30 Sep 2015 13:30:48 +0100
Subject: [PATCH 004/150] [nfl] fix content id regex(fixes #7012)

---
 youtube_dl/extractor/nfl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
index 55dc6107d..4c35c561b 100644
--- a/youtube_dl/extractor/nfl.py
+++ b/youtube_dl/extractor/nfl.py
@@ -151,7 +151,7 @@ class NFLIE(InfoExtractor):
             group='config'))
         # For articles, the id in the url is not the video id
         video_id = self._search_regex(
-            r'(?:<nflcs:avplayer[^>]+data-contentId\s*=\s*|contentId\s*:\s*)(["\'])(?P<id>.+?)\1',
+            r'(?:<nflcs:avplayer[^>]+data-content[Ii]d\s*=\s*|content[Ii]d\s*:\s*)(["\'])(?P<id>.+?)\1',
             webpage, 'video id', default=video_id, group='id')
         config = self._download_json(config_url, video_id, 'Downloading player config')
         url_template = NFLIE.prepend_host(

From ee2d190253ae3e0a8254029dc6e8d7e42c194042 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Sep 2015 20:06:21 +0600
Subject: [PATCH 005/150] [nfl] Add test for #7012

---
 youtube_dl/extractor/nfl.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
index 4c35c561b..200874d68 100644
--- a/youtube_dl/extractor/nfl.py
+++ b/youtube_dl/extractor/nfl.py
@@ -107,6 +107,20 @@ class NFLIE(InfoExtractor):
             'timestamp': 1442618809,
             'upload_date': '20150918',
         },
+    }, {
+        # lowercase data-contentid
+        'url': 'http://www.steelers.com/news/article-1/Tomlin-on-Ben-getting-Vick-ready/56399c96-4160-48cf-a7ad-1d17d4a3aef7',
+        'info_dict': {
+            'id': '12693586-6ea9-4743-9c1c-02c59e4a5ef2',
+            'ext': 'mp4',
+            'title': 'Tomlin looks ahead to Ravens on a short week',
+            'description': 'md5:32f3f7b139f43913181d5cbb24ecad75',
+            'timestamp': 1443459651,
+            'upload_date': '20150928',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.nfl.com/videos/nfl-network-top-ten/09000d5d810a6bd4/Top-10-Gutsiest-Performances-Jack-Youngblood',
         'only_matching': True,

From 93bc7ef165731c729fd6fbbb3e95ecbbb16b4c21 Mon Sep 17 00:00:00 2001
From: Qijiang Fan <fqj1994@gmail.com>
Date: Sun, 30 Aug 2015 14:33:12 +0800
Subject: [PATCH 006/150] [test] recursively check dict and list in
 expect_info_dict

This allows to use md5:, re:, etc within the str inside a list
or dict.
---
 test/helper.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index cb6eec8d9..6612970ef 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -89,7 +89,7 @@ def gettestcases(include_onlymatching=False):
 md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
 
 
-def expect_info_dict(self, got_dict, expected_dict):
+def expect_dict(self, got_dict, expected_dict):
     for info_field, expected in expected_dict.items():
         if isinstance(expected, compat_str) and expected.startswith('re:'):
             got = got_dict.get(info_field)
@@ -127,6 +127,22 @@ def expect_info_dict(self, got_dict, expected_dict):
             got = got_dict.get(info_field)
             self.assertTrue(isinstance(got, expected),
                             'Expected type %r for field %s, but got value %r of type %r' % (expected, info_field, got, type(got)))
+        elif isinstance(expected, dict) and isinstance(got_dict.get(info_field, None), dict):
+            expect_dict(self, got_dict.get(info_field), expected)
+        elif isinstance(expected, list) and isinstance(got_dict.get(info_field, None), list):
+            got = got_dict.get(info_field, None)
+            self.assertEqual(len(expected), len(got),
+                             'Expect a list of length %d, but got a list of length %d' % (
+                             len(expected), len(got)))
+            _id = 0
+            for i, j in zip(got, expected):
+                _type_i = type(i)
+                _type_j = type(j)
+                self.assertEqual(_type_j, _type_i,
+                                 'Type doesn\'t match at element %d of the list in field %s, expect %s, got %s' % (
+                                 _id, info_field, _type_j, _type_i))
+                expect_dict(self, {'_': i}, {'_': j})
+                _id += 1
         else:
             if isinstance(expected, compat_str) and expected.startswith('md5:'):
                 got = 'md5:' + md5(got_dict.get(info_field))
@@ -149,6 +165,9 @@ def expect_info_dict(self, got_dict, expected_dict):
             self.assertEqual(expected, got,
                              'invalid value for field %s, expected %r, got %r' % (info_field, expected, got))
 
+
+def expect_info_dict(self, got_dict, expected_dict):
+    expect_dict(self, got_dict, expected_dict)
     # Check for the presence of mandatory fields
     if got_dict.get('_type') not in ('playlist', 'multi_video'):
         for key in ('id', 'url', 'title', 'ext'):

From 40c931de4b3c06c1959362c2380f4bf243ef5702 Mon Sep 17 00:00:00 2001
From: Qijiang Fan <fqj1994@gmail.com>
Date: Sat, 26 Sep 2015 23:10:38 +0800
Subject: [PATCH 007/150] [test] split expect_dict to two functions

---
 test/helper.py | 143 ++++++++++++++++++++++++-------------------------
 1 file changed, 70 insertions(+), 73 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 6612970ef..e50d7ff28 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -89,81 +89,78 @@ def gettestcases(include_onlymatching=False):
 md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
 
 
+def expect_value(self, got, expected, field):
+    if isinstance(expected, compat_str) and expected.startswith('re:'):
+        match_str = expected[len('re:'):]
+        match_rex = re.compile(match_str)
+
+        self.assertTrue(
+            isinstance(got, compat_str),
+            'Expected a %s object, but got %s for field %s' % (
+                compat_str.__name__, type(got).__name__, field))
+        self.assertTrue(
+            match_rex.match(got),
+            'field %s (value: %r) should match %r' % (field, got, match_str))
+    elif isinstance(expected, compat_str) and expected.startswith('startswith:'):
+        start_str = expected[len('startswith:'):]
+        self.assertTrue(
+            isinstance(got, compat_str),
+            'Expected a %s object, but got %s for field %s' % (
+                compat_str.__name__, type(got).__name__, field))
+        self.assertTrue(
+            got.startswith(start_str),
+            'field %s (value: %r) should start with %r' % (field, got, start_str))
+    elif isinstance(expected, compat_str) and expected.startswith('contains:'):
+        contains_str = expected[len('contains:'):]
+        self.assertTrue(
+            isinstance(got, compat_str),
+            'Expected a %s object, but got %s for field %s' % (
+                compat_str.__name__, type(got).__name__, field))
+        self.assertTrue(
+            contains_str in got,
+            'field %s (value: %r) should contain %r' % (field, got, contains_str))
+    elif isinstance(expected, type):
+        self.assertTrue(isinstance(got, expected),
+                        'Expected type %r for field %s, but got value %r of type %r' % (expected, field, got, type(got)))
+    elif isinstance(expected, dict) and isinstance(got, dict):
+        expect_dict(self, got, expected)
+    elif isinstance(expected, list) and isinstance(got, list):
+        self.assertEqual(len(expected), len(got),
+                         'Expect a list of length %d, but got a list of length %d' % (
+                         len(expected), len(got)))
+        _id = 0
+        for i, j in zip(got, expected):
+            _type_i = type(i)
+            _type_j = type(j)
+            self.assertEqual(_type_j, _type_i,
+                             'Type doesn\'t match at element %d of the list in field %s, expect %s, got %s' % (
+                             _id, field, _type_j, _type_i))
+            expect_value(self, i, j, field)
+            _id += 1
+    else:
+        if isinstance(expected, compat_str) and expected.startswith('md5:'):
+            got = 'md5:' + md5(got)
+        elif isinstance(expected, compat_str) and expected.startswith('mincount:'):
+            self.assertTrue(
+                isinstance(got, (list, dict)),
+                'Expected field %s to be a list or a dict, but it is of type %s' % (
+                    field, type(got).__name__))
+            expected_num = int(expected.partition(':')[2])
+            assertGreaterEqual(
+                self, len(got), expected_num,
+                'Expected %d items in field %s, but only got %d' % (
+                    expected_num, field, len(got)
+                )
+            )
+            return
+        self.assertEqual(expected, got,
+                         'invalid value for field %s, expected %r, got %r' % (field, expected, got))
+
+
 def expect_dict(self, got_dict, expected_dict):
     for info_field, expected in expected_dict.items():
-        if isinstance(expected, compat_str) and expected.startswith('re:'):
-            got = got_dict.get(info_field)
-            match_str = expected[len('re:'):]
-            match_rex = re.compile(match_str)
-
-            self.assertTrue(
-                isinstance(got, compat_str),
-                'Expected a %s object, but got %s for field %s' % (
-                    compat_str.__name__, type(got).__name__, info_field))
-            self.assertTrue(
-                match_rex.match(got),
-                'field %s (value: %r) should match %r' % (info_field, got, match_str))
-        elif isinstance(expected, compat_str) and expected.startswith('startswith:'):
-            got = got_dict.get(info_field)
-            start_str = expected[len('startswith:'):]
-            self.assertTrue(
-                isinstance(got, compat_str),
-                'Expected a %s object, but got %s for field %s' % (
-                    compat_str.__name__, type(got).__name__, info_field))
-            self.assertTrue(
-                got.startswith(start_str),
-                'field %s (value: %r) should start with %r' % (info_field, got, start_str))
-        elif isinstance(expected, compat_str) and expected.startswith('contains:'):
-            got = got_dict.get(info_field)
-            contains_str = expected[len('contains:'):]
-            self.assertTrue(
-                isinstance(got, compat_str),
-                'Expected a %s object, but got %s for field %s' % (
-                    compat_str.__name__, type(got).__name__, info_field))
-            self.assertTrue(
-                contains_str in got,
-                'field %s (value: %r) should contain %r' % (info_field, got, contains_str))
-        elif isinstance(expected, type):
-            got = got_dict.get(info_field)
-            self.assertTrue(isinstance(got, expected),
-                            'Expected type %r for field %s, but got value %r of type %r' % (expected, info_field, got, type(got)))
-        elif isinstance(expected, dict) and isinstance(got_dict.get(info_field, None), dict):
-            expect_dict(self, got_dict.get(info_field), expected)
-        elif isinstance(expected, list) and isinstance(got_dict.get(info_field, None), list):
-            got = got_dict.get(info_field, None)
-            self.assertEqual(len(expected), len(got),
-                             'Expect a list of length %d, but got a list of length %d' % (
-                             len(expected), len(got)))
-            _id = 0
-            for i, j in zip(got, expected):
-                _type_i = type(i)
-                _type_j = type(j)
-                self.assertEqual(_type_j, _type_i,
-                                 'Type doesn\'t match at element %d of the list in field %s, expect %s, got %s' % (
-                                 _id, info_field, _type_j, _type_i))
-                expect_dict(self, {'_': i}, {'_': j})
-                _id += 1
-        else:
-            if isinstance(expected, compat_str) and expected.startswith('md5:'):
-                got = 'md5:' + md5(got_dict.get(info_field))
-            elif isinstance(expected, compat_str) and expected.startswith('mincount:'):
-                got = got_dict.get(info_field)
-                self.assertTrue(
-                    isinstance(got, (list, dict)),
-                    'Expected field %s to be a list or a dict, but it is of type %s' % (
-                        info_field, type(got).__name__))
-                expected_num = int(expected.partition(':')[2])
-                assertGreaterEqual(
-                    self, len(got), expected_num,
-                    'Expected %d items in field %s, but only got %d' % (
-                        expected_num, info_field, len(got)
-                    )
-                )
-                continue
-            else:
-                got = got_dict.get(info_field)
-            self.assertEqual(expected, got,
-                             'invalid value for field %s, expected %r, got %r' % (info_field, expected, got))
+        got = got_dict.get(info_field)
+        expect_value(self, got, expected, info_field)
 
 
 def expect_info_dict(self, got_dict, expected_dict):

From 687c04cbb8d80f76b45caf9377b2d3b64462c203 Mon Sep 17 00:00:00 2001
From: Qijiang Fan <fqj1994@gmail.com>
Date: Wed, 30 Sep 2015 10:30:04 +0200
Subject: [PATCH 008/150] [test] use descriptive variable name

---
 test/helper.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index e50d7ff28..5c44105d4 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -128,15 +128,13 @@ def expect_value(self, got, expected, field):
         self.assertEqual(len(expected), len(got),
                          'Expect a list of length %d, but got a list of length %d' % (
                          len(expected), len(got)))
-        _id = 0
-        for i, j in zip(got, expected):
-            _type_i = type(i)
-            _type_j = type(j)
-            self.assertEqual(_type_j, _type_i,
+        for index, (item_got, item_expected) in enumerate(zip(got, expected)):
+            type_got = type(item_got)
+            type_expected = type(item_expected)
+            self.assertEqual(type_expected, type_got,
                              'Type doesn\'t match at element %d of the list in field %s, expect %s, got %s' % (
-                             _id, field, _type_j, _type_i))
-            expect_value(self, i, j, field)
-            _id += 1
+                             index, field, type_expected, type_got))
+            expect_value(self, item_got, item_expected, field)
     else:
         if isinstance(expected, compat_str) and expected.startswith('md5:'):
             got = 'md5:' + md5(got)

From 2e885de796e211300fd70709da17325b84c86dd6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Sep 2015 20:21:01 +0600
Subject: [PATCH 009/150] [test/helper] Formatting

---
 test/helper.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 5c44105d4..288ed237d 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -120,20 +120,22 @@ def expect_value(self, got, expected, field):
             contains_str in got,
             'field %s (value: %r) should contain %r' % (field, got, contains_str))
     elif isinstance(expected, type):
-        self.assertTrue(isinstance(got, expected),
-                        'Expected type %r for field %s, but got value %r of type %r' % (expected, field, got, type(got)))
+        self.assertTrue(
+            isinstance(got, expected),
+            'Expected type %r for field %s, but got value %r of type %r' % (expected, field, got, type(got)))
     elif isinstance(expected, dict) and isinstance(got, dict):
         expect_dict(self, got, expected)
     elif isinstance(expected, list) and isinstance(got, list):
-        self.assertEqual(len(expected), len(got),
-                         'Expect a list of length %d, but got a list of length %d' % (
-                         len(expected), len(got)))
+        self.assertEqual(
+            len(expected), len(got),
+            'Expect a list of length %d, but got a list of length %d' % (len(expected), len(got)))
         for index, (item_got, item_expected) in enumerate(zip(got, expected)):
             type_got = type(item_got)
             type_expected = type(item_expected)
-            self.assertEqual(type_expected, type_got,
-                             'Type doesn\'t match at element %d of the list in field %s, expect %s, got %s' % (
-                             index, field, type_expected, type_got))
+            self.assertEqual(
+                type_expected, type_got,
+                'Type doesn\'t match at element %d of the list in field %s, expect %s, got %s' % (
+                index, field, type_expected, type_got))
             expect_value(self, item_got, item_expected, field)
     else:
         if isinstance(expected, compat_str) and expected.startswith('md5:'):
@@ -146,13 +148,11 @@ def expect_value(self, got, expected, field):
             expected_num = int(expected.partition(':')[2])
             assertGreaterEqual(
                 self, len(got), expected_num,
-                'Expected %d items in field %s, but only got %d' % (
-                    expected_num, field, len(got)
-                )
-            )
+                'Expected %d items in field %s, but only got %d' % (expected_num, field, len(got)))
             return
-        self.assertEqual(expected, got,
-                         'invalid value for field %s, expected %r, got %r' % (field, expected, got))
+        self.assertEqual(
+            expected, got,
+            'invalid value for field %s, expected %r, got %r' % (field, expected, got))
 
 
 def expect_dict(self, got_dict, expected_dict):

From 386a7b52d548da0aaf11e5805d67d5ab30d9dfab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Sep 2015 20:26:42 +0600
Subject: [PATCH 010/150] [test/helper] Spelling

---
 test/helper.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 288ed237d..0ce9a0fb3 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -134,7 +134,7 @@ def expect_value(self, got, expected, field):
             type_expected = type(item_expected)
             self.assertEqual(
                 type_expected, type_got,
-                'Type doesn\'t match at element %d of the list in field %s, expect %s, got %s' % (
+                'Type mismatch for list item at index %d for field %s, expected %r, got %r' % (
                 index, field, type_expected, type_got))
             expect_value(self, item_got, item_expected, field)
     else:
@@ -152,7 +152,7 @@ def expect_value(self, got, expected, field):
             return
         self.assertEqual(
             expected, got,
-            'invalid value for field %s, expected %r, got %r' % (field, expected, got))
+            'Invalid value for field %s, expected %r, got %r' % (field, expected, got))
 
 
 def expect_dict(self, got_dict, expected_dict):

From f88f1b40ce7f70a3a510259154364a25d7dceff6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Sep 2015 20:31:29 +0600
Subject: [PATCH 011/150] [test/helper] Clarify field for list length mismatch

---
 test/helper.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index 0ce9a0fb3..28fd135b2 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -128,7 +128,8 @@ def expect_value(self, got, expected, field):
     elif isinstance(expected, list) and isinstance(got, list):
         self.assertEqual(
             len(expected), len(got),
-            'Expect a list of length %d, but got a list of length %d' % (len(expected), len(got)))
+            'Expect a list of length %d, but got a list of length %d for field %s' % (
+                len(expected), len(got), field))
         for index, (item_got, item_expected) in enumerate(zip(got, expected)):
             type_got = type(item_got)
             type_expected = type(item_expected)

From 8466336104164b37b3f32c32278e2426590db42f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Sep 2015 22:12:52 +0600
Subject: [PATCH 012/150] [vk] Detect vimeo embeds (Closes #7021)

---
 youtube_dl/extractor/vk.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index c30c5a8e5..765e9e6fd 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -17,6 +17,7 @@ from ..utils import (
     unescapeHTML,
     unified_strdate,
 )
+from .vimeo import VimeoIE
 
 
 class VKIE(InfoExtractor):
@@ -249,6 +250,10 @@ class VKIE(InfoExtractor):
         if youtube_url:
             return self.url_result(youtube_url, 'Youtube')
 
+        vimeo_url = VimeoIE._extract_vimeo_url(url, info_page)
+        if vimeo_url is not None:
+            return self.url_result(vimeo_url)
+
         m_rutube = re.search(
             r'\ssrc="((?:https?:)?//rutube\.ru\\?/video\\?/embed(?:.*?))\\?"', info_page)
         if m_rutube is not None:

From f540b937062a4de2cd0a097245ec82dd9fd23277 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 02:33:48 +0600
Subject: [PATCH 013/150] [naver] Improve error regex

---
 youtube_dl/extractor/naver.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index 925967753..35cbb3e6d 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -46,11 +46,11 @@ class NaverIE(InfoExtractor):
         m_id = re.search(r'var rmcPlayer = new nhn.rmcnmv.RMCVideoPlayer\("(.+?)", "(.+?)"',
                          webpage)
         if m_id is None:
-            m_error = re.search(
-                r'(?s)<div class="(?:nation_error|nation_box)">\s*(?:<!--.*?-->)?\s*<p class="[^"]+">(?P<msg>.+?)</p>\s*</div>',
-                webpage)
-            if m_error:
-                raise ExtractorError(clean_html(m_error.group('msg')), expected=True)
+            error = self._html_search_regex(
+                r'(?s)<div class="(?:nation_error|nation_box|error_box)">\s*(?:<!--.*?-->)?\s*<p class="[^"]+">(?P<msg>.+?)</p>\s*</div>',
+                webpage, 'error', default=None)
+            if error:
+                raise ExtractorError(error, expected=True)
             raise ExtractorError('couldn\'t extract vid and key')
         vid = m_id.group(1)
         key = m_id.group(2)

From b83831df1fb0dd241210a2c992ac9753c5df59cd Mon Sep 17 00:00:00 2001
From: Joakim Fremstad <jfremstad@users.noreply.github.com>
Date: Thu, 1 Oct 2015 14:58:49 +0200
Subject: [PATCH 014/150] [nrk] Spelling

---
 youtube_dl/extractor/nrk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index d066a96db..5ed235f1e 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -49,7 +49,7 @@ class NRKIE(InfoExtractor):
 
         if data['usageRights']['isGeoBlocked']:
             raise ExtractorError(
-                'NRK har ikke rettig-heter til å vise dette programmet utenfor Norge',
+                'NRK har ikke rettigheter til å vise dette programmet utenfor Norge',
                 expected=True)
 
         video_url = data['mediaUrl'] + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81'

From 2b3f951a2e2e01adfb068eac05b976ff445db073 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 20:33:17 +0600
Subject: [PATCH 015/150] [nrktv] Rework subtitles and eliminate downloading
 twice

---
 youtube_dl/extractor/nrk.py | 28 +++++++++-------------------
 1 file changed, 9 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 5ed235f1e..8ac38a174 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
     ExtractorError,
     float_or_none,
@@ -196,20 +197,6 @@ class NRKTVIE(InfoExtractor):
         }
     ]
 
-    def _debug_print(self, txt):
-        if self._downloader.params.get('verbose', False):
-            self.to_screen('[debug] %s' % txt)
-
-    def _get_subtitles(self, subtitlesurl, video_id, baseurl):
-        url = "%s%s" % (baseurl, subtitlesurl)
-        self._debug_print('%s: Subtitle url: %s' % (video_id, url))
-        captions = self._download_xml(
-            url, video_id, 'Downloading subtitles')
-        lang = captions.get('lang', 'no')
-        return {lang: [
-            {'ext': 'ttml', 'url': url},
-        ]}
-
     def _extract_f4m(self, manifest_url, video_id):
         return self._extract_f4m_formats(
             manifest_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124', video_id, f4m_id='hds')
@@ -218,7 +205,7 @@ class NRKTVIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         part_id = mobj.group('part_id')
-        baseurl = mobj.group('baseurl')
+        base_url = mobj.group('baseurl')
 
         webpage = self._download_webpage(url, video_id)
 
@@ -278,11 +265,14 @@ class NRKTVIE(InfoExtractor):
         self._sort_formats(formats)
 
         subtitles_url = self._html_search_regex(
-            r'data-subtitlesurl[ ]*=[ ]*"([^"]+)"',
-            webpage, 'subtitle URL', default=None)
-        subtitles = None
+            r'data-subtitlesurl\s*=\s*(["\'])(?P<url>.+?)\1',
+            webpage, 'subtitle URL', default=None, group='url')
+        subtitles = {}
         if subtitles_url:
-            subtitles = self.extract_subtitles(subtitles_url, video_id, baseurl)
+            subtitles['no'] = [{
+                'ext': 'ttml',
+                'url': compat_urlparse.urljoin(base_url, subtitles_url),
+            }]
 
         return {
             'id': video_id,

From e7d8e98a9ffdec2502bedb21a4f043df6da225a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 21:51:04 +0600
Subject: [PATCH 016/150] [extractor/common] Allow float bitrates

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2fe0d5d37..9c40d56a9 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1084,7 +1084,7 @@ class InfoExtractor(object):
             if not src:
                 continue
 
-            bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
+            bitrate = float_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
             filesize = int_or_none(video.get('size') or video.get('fileSize'))
             width = int_or_none(video.get('width'))
             height = int_or_none(video.get('height'))

From 1e5bcdec0264190ed2a05ee49c1f9f5b20ba3aa6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:08:16 +0600
Subject: [PATCH 017/150] [extractor/common] Extract images from SMIL

---
 youtube_dl/extractor/common.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9c40d56a9..5684227dc 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1054,10 +1054,18 @@ class InfoExtractor(object):
             elif not description and name in ('description', 'abstract'):
                 description = content
 
+        thumbnails = [{
+            'id': image.get('type'),
+            'url': image.get('src'),
+            'width': int_or_none(image.get('width')),
+            'height': int_or_none(image.get('height')),
+        } for image in smil.findall(self._xpath_ns('.//image', namespace)) if image.get('src')]
+
         return {
             'id': video_id,
             'title': title or video_id,
             'description': description,
+            'thumbnails': thumbnails,
             'formats': formats,
             'subtitles': subtitles,
         }

From 647eab4541d1f55dea6aa20af9b35c2726dda48a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:18:59 +0600
Subject: [PATCH 018/150] [extractor/common] Extract upload date from SMIL

---
 youtube_dl/extractor/common.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5684227dc..b86d06523 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -39,6 +39,7 @@ from ..utils import (
     RegexNotFoundError,
     sanitize_filename,
     unescapeHTML,
+    unified_strdate,
     url_basename,
     xpath_text,
     xpath_with_ns,
@@ -1044,6 +1045,7 @@ class InfoExtractor(object):
         video_id = os.path.splitext(url_basename(smil_url))[0]
         title = None
         description = None
+        upload_date = None
         for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
             name = meta.attrib.get('name')
             content = meta.attrib.get('content')
@@ -1053,6 +1055,8 @@ class InfoExtractor(object):
                 title = content
             elif not description and name in ('description', 'abstract'):
                 description = content
+            elif not upload_date and name == 'date':
+                upload_date = unified_strdate(content)
 
         thumbnails = [{
             'id': image.get('type'),
@@ -1065,6 +1069,7 @@ class InfoExtractor(object):
             'id': video_id,
             'title': title or video_id,
             'description': description,
+            'upload_date': upload_date,
             'thumbnails': thumbnails,
             'formats': formats,
             'subtitles': subtitles,

From acfb717a18cfa9c0f377372068e16862bba345b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:19:39 +0600
Subject: [PATCH 019/150] [videolecturesnet] Use generic SMIL extraction

---
 youtube_dl/extractor/videolecturesnet.py | 62 +++---------------------
 1 file changed, 8 insertions(+), 54 deletions(-)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index ef2da5632..160dbb590 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -1,13 +1,8 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
-    find_xpath_attr,
-    int_or_none,
     parse_duration,
-    unified_strdate,
 )
 
 
@@ -29,58 +24,17 @@ class VideoLecturesNetIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         smil_url = 'http://videolectures.net/%s/video/1/smil.xml' % video_id
-        smil = self._download_xml(smil_url, video_id)
+        smil = self._download_smil(smil_url, video_id)
 
-        title = find_xpath_attr(smil, './/meta', 'name', 'title').attrib['content']
-        description_el = find_xpath_attr(smil, './/meta', 'name', 'abstract')
-        description = (
-            None if description_el is None
-            else description_el.attrib['content'])
-        upload_date = unified_strdate(
-            find_xpath_attr(smil, './/meta', 'name', 'date').attrib['content'])
+        info = self._parse_smil(smil, smil_url, video_id)
+
+        info['id'] = video_id
 
         switch = smil.find('.//switch')
-        duration = parse_duration(switch.attrib.get('dur'))
-        thumbnail_el = find_xpath_attr(switch, './image', 'type', 'thumbnail')
-        thumbnail = (
-            None if thumbnail_el is None else thumbnail_el.attrib.get('src'))
+        if switch is not None:
+            info['duration'] = parse_duration(switch.attrib.get('dur'))
 
-        formats = []
-        for v in switch.findall('./video'):
-            proto = v.attrib.get('proto')
-            if proto not in ['http', 'rtmp']:
-                continue
-            f = {
-                'width': int_or_none(v.attrib.get('width')),
-                'height': int_or_none(v.attrib.get('height')),
-                'filesize': int_or_none(v.attrib.get('size')),
-                'tbr': int_or_none(v.attrib.get('systemBitrate')) / 1000.0,
-                'ext': v.attrib.get('ext'),
-            }
-            src = v.attrib['src']
-            if proto == 'http':
-                if self._is_valid_url(src, video_id):
-                    f['url'] = src
-                    formats.append(f)
-            elif proto == 'rtmp':
-                f.update({
-                    'url': v.attrib['streamer'],
-                    'play_path': src,
-                    'rtmp_real_time': True,
-                })
-                formats.append(f)
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'upload_date': upload_date,
-            'duration': duration,
-            'thumbnail': thumbnail,
-            'formats': formats,
-        }
+        return info

From 0c996b9f488bfaa74d79e94739af80a0be38e125 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:39:38 +0600
Subject: [PATCH 020/150] [videolecturesnet] Add support for playlists (Closes
 #7031)

---
 youtube_dl/extractor/videolecturesnet.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index 160dbb590..eadff8d18 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -1,9 +1,10 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
-from ..utils import (
-    parse_duration,
-)
+from ..compat import compat_urlparse
+from ..utils import parse_duration
 
 
 class VideoLecturesNetIE(InfoExtractor):
@@ -27,7 +28,17 @@ class VideoLecturesNetIE(InfoExtractor):
         video_id = self._match_id(url)
 
         smil_url = 'http://videolectures.net/%s/video/1/smil.xml' % video_id
-        smil = self._download_smil(smil_url, video_id)
+        smil = self._download_smil(smil_url, video_id, fatal=False)
+
+        # Probably a playlist
+        if smil is False:
+            webpage = self._download_webpage(url, video_id)
+            entries = [
+                self.url_result(compat_urlparse.urljoin(url, video_url), 'VideoLecturesNet')
+                for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', webpage)]
+            playlist_title = self._html_search_meta('title', webpage, 'title', fatal=True)
+            playlist_description = self._html_search_meta('description', webpage, 'description')
+            return self.playlist_result(entries, video_id, playlist_title, playlist_description)
 
         info = self._parse_smil(smil, smil_url, video_id)
 

From fb97809e64a145768346345ff2b5e5539880c014 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:44:51 +0600
Subject: [PATCH 021/150] [videolecturesnet] Improve playlist extraction

---
 youtube_dl/extractor/videolecturesnet.py | 32 +++++++++++++++---------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index eadff8d18..704165406 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -3,8 +3,14 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
-from ..utils import parse_duration
+from ..compat import (
+    compat_HTTPError,
+    compat_urlparse,
+)
+from ..utils import (
+    ExtractorError,
+    parse_duration,
+)
 
 
 class VideoLecturesNetIE(InfoExtractor):
@@ -28,17 +34,19 @@ class VideoLecturesNetIE(InfoExtractor):
         video_id = self._match_id(url)
 
         smil_url = 'http://videolectures.net/%s/video/1/smil.xml' % video_id
-        smil = self._download_smil(smil_url, video_id, fatal=False)
 
-        # Probably a playlist
-        if smil is False:
-            webpage = self._download_webpage(url, video_id)
-            entries = [
-                self.url_result(compat_urlparse.urljoin(url, video_url), 'VideoLecturesNet')
-                for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', webpage)]
-            playlist_title = self._html_search_meta('title', webpage, 'title', fatal=True)
-            playlist_description = self._html_search_meta('description', webpage, 'description')
-            return self.playlist_result(entries, video_id, playlist_title, playlist_description)
+        try:
+            smil = self._download_smil(smil_url, video_id)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
+                # Probably a playlist
+                webpage = self._download_webpage(url, video_id)
+                entries = [
+                    self.url_result(compat_urlparse.urljoin(url, video_url), 'VideoLecturesNet')
+                    for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', webpage)]
+                playlist_title = self._html_search_meta('title', webpage, 'title', fatal=True)
+                playlist_description = self._html_search_meta('description', webpage, 'description')
+                return self.playlist_result(entries, video_id, playlist_title, playlist_description)
 
         info = self._parse_smil(smil, smil_url, video_id)
 

From 6edaa0e25b880be5d1ea513d9d907c91e18a8347 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:45:10 +0600
Subject: [PATCH 022/150] [videolecturesnet] Add playlist test

---
 youtube_dl/extractor/videolecturesnet.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index 704165406..113a2289b 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -17,7 +17,7 @@ class VideoLecturesNetIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/#?]+)/*(?:[#?].*)?$'
     IE_NAME = 'videolectures.net'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/',
         'info_dict': {
             'id': 'promogram_igor_mekjavic_eng',
@@ -28,7 +28,15 @@ class VideoLecturesNetIE(InfoExtractor):
             'duration': 565,
             'thumbnail': 're:http://.*\.jpg',
         },
-    }
+    }, {
+        'url': 'http://videolectures.net/deeplearning2015_montreal/',
+        'info_dict': {
+            'id': 'deeplearning2015_montreal',
+            'title': 'Deep Learning Summer School, Montreal 2015',
+            'description': 'md5:90121a40cc6926df1bf04dcd8563ed3b',
+        },
+        'playlist_count': 30,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From c78e48177c72baa0c985c4d47f95e6edc6ec33dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:54:54 +0600
Subject: [PATCH 023/150] [extractor/common] Check validity of direct URLs

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b86d06523..b928e24be 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1145,7 +1145,7 @@ class InfoExtractor(object):
                 formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds'))
                 continue
 
-            if src_url.startswith('http'):
+            if src_url.startswith('http') and self._is_valid_url(src, video_id):
                 http_count += 1
                 formats.append({
                     'url': src_url,

From 3a1341a7bc9f11d972c7649a33e143fab72f8bc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:59:20 +0600
Subject: [PATCH 024/150] [extractor/common] Make m3u8 extraction for SMIL non
 fatal

---
 youtube_dl/extractor/common.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b928e24be..9a5a7cc2c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1129,8 +1129,10 @@ class InfoExtractor(object):
             src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
 
             if proto == 'm3u8' or src_ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    src_url, video_id, ext or 'mp4', m3u8_id='hls'))
+                m3u8_formats = self._extract_m3u8_formats(
+                    src_url, video_id, ext or 'mp4', m3u8_id='hls', fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
                 continue
 
             if src_ext == 'f4m':

From 4de6131090e0232c7cc99bcaafe6a3e71269b7af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 23:03:31 +0600
Subject: [PATCH 025/150] [extractor/common] Add fatal to _extract_f4m_formats

---
 youtube_dl/extractor/common.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9a5a7cc2c..c2aa3a749 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -870,13 +870,18 @@ class InfoExtractor(object):
         time.sleep(timeout)
 
     def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None,
-                             transform_source=lambda s: fix_xml_ampersands(s).strip()):
+                             transform_source=lambda s: fix_xml_ampersands(s).strip(),
+                             fatal=True):
         manifest = self._download_xml(
             manifest_url, video_id, 'Downloading f4m manifest',
             'Unable to download f4m manifest',
             # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
             # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244)
-            transform_source=transform_source)
+            transform_source=transform_source,
+            fatal=fatal)
+
+        if manifest is False:
+            return manifest
 
         formats = []
         manifest_version = '1.0'
@@ -897,7 +902,10 @@ class InfoExtractor(object):
                 # may differ leading to inability to resolve the format by requested
                 # bitrate in f4m downloader
                 if determine_ext(manifest_url) == 'f4m':
-                    formats.extend(self._extract_f4m_formats(manifest_url, video_id, preference, f4m_id))
+                    f4m_formats = self._extract_f4m_formats(
+                        manifest_url, video_id, preference, f4m_id, fatal=fatal)
+                    if f4m_formats:
+                        formats.extend(f4m_formats)
                     continue
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({

From e5851b963ab0872f27f2db1cff3c89548b6bf95e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 23:04:56 +0600
Subject: [PATCH 026/150] [extractor/common] Make f4m extraction for SMIL non
 fatal

---
 youtube_dl/extractor/common.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index c2aa3a749..4fe2307cd 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1152,7 +1152,9 @@ class InfoExtractor(object):
                     }
                 f4m_url += '&' if '?' in f4m_url else '?'
                 f4m_url += compat_urllib_parse.urlencode(f4m_params)
-                formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds'))
+                f4m_formats = self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False)
+                if f4m_formats:
+                    formats.extend(f4m_formats)
                 continue
 
             if src_url.startswith('http') and self._is_valid_url(src, video_id):

From 06c6efa9701d0527147875adf0caf437ffbb0397 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 23:10:36 +0600
Subject: [PATCH 027/150] [videolecturesnet] Add test video with broken direct
 format links

---
 youtube_dl/extractor/videolecturesnet.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index 113a2289b..649ac9433 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -28,6 +28,21 @@ class VideoLecturesNetIE(InfoExtractor):
             'duration': 565,
             'thumbnail': 're:http://.*\.jpg',
         },
+    }, {
+        # video with invalid direct format links (HTTP 403)
+        'url': 'http://videolectures.net/russir2010_filippova_nlp/',
+        'info_dict': {
+            'id': 'russir2010_filippova_nlp',
+            'ext': 'flv',
+            'title': 'NLP at Google',
+            'description': 'md5:fc7a6d9bf0302d7cc0e53f7ca23747b3',
+            'duration': 5352,
+            'thumbnail': 're:http://.*\.jpg',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
     }, {
         'url': 'http://videolectures.net/deeplearning2015_montreal/',
         'info_dict': {

From 44451f22d5d1a5bed5f5851b27a963860813ecd6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 2 Oct 2015 13:41:52 +0200
Subject: [PATCH 028/150] [naver] Remove unused import

---
 youtube_dl/extractor/naver.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index 35cbb3e6d..1f5fc2145 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -10,7 +10,6 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
-    clean_html,
 )
 
 

From 7d0ada5ff907824c66c466ee9b83008210250d5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 2 Oct 2015 13:42:11 +0200
Subject: [PATCH 029/150] [test/helper] Fix style

Use the correct indentation to please flake8
---
 test/helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index 28fd135b2..bdd7acca4 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -136,7 +136,7 @@ def expect_value(self, got, expected, field):
             self.assertEqual(
                 type_expected, type_got,
                 'Type mismatch for list item at index %d for field %s, expected %r, got %r' % (
-                index, field, type_expected, type_got))
+                    index, field, type_expected, type_got))
             expect_value(self, item_got, item_expected, field)
     else:
         if isinstance(expected, compat_str) and expected.startswith('md5:'):

From 0facd2af3ebfda68b79c7e2e1c575d73f9680802 Mon Sep 17 00:00:00 2001
From: fluks <fluks.github@gmail.com>
Date: Fri, 2 Oct 2015 04:08:13 +0300
Subject: [PATCH 030/150] Fix ruutu extractor bug

If there's no resolution attribute in xml, only width gets a
value, height doesn't and ValueError is raised.
---
 youtube_dl/extractor/ruutu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index c67ad25ce..7720f1383 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -74,7 +74,7 @@ class RuutuIE(InfoExtractor):
                         preference = -1 if proto == 'rtmp' else 1
                         label = child.get('label')
                         tbr = int_or_none(child.get('bitrate'))
-                        width, height = [int_or_none(x) for x in child.get('resolution', '').split('x')]
+                        width, height = [int_or_none(x) for x in child.get('resolution', 'x').split('x')]
                         formats.append({
                             'format_id': '%s-%s' % (proto, label if label else tbr),
                             'url': video_url,

From 59a9efe85b15e53c5928b7fdb810c150f5bf4b78 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Oct 2015 20:48:39 +0600
Subject: [PATCH 031/150] [ruutu] Limit resolution split to 2 pieces (Closes
 #7037, closes #7042)

---
 youtube_dl/extractor/ruutu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index 7720f1383..a16b73ff4 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -74,7 +74,7 @@ class RuutuIE(InfoExtractor):
                         preference = -1 if proto == 'rtmp' else 1
                         label = child.get('label')
                         tbr = int_or_none(child.get('bitrate'))
-                        width, height = [int_or_none(x) for x in child.get('resolution', 'x').split('x')]
+                        width, height = [int_or_none(x) for x in child.get('resolution', 'x').split('x')[:2]]
                         formats.append({
                             'format_id': '%s-%s' % (proto, label if label else tbr),
                             'url': video_url,

From 3bb3f0410822d3d21c6199bb8915b598990628e6 Mon Sep 17 00:00:00 2001
From: ngld <ngld@tproxy.de>
Date: Wed, 12 Aug 2015 16:59:04 +0200
Subject: [PATCH 032/150] [europa] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/europa.py   | 60 ++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+)
 create mode 100644 youtube_dl/extractor/europa.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a73a1317e..495a18c17 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -158,6 +158,7 @@ from .eroprofile import EroProfileIE
 from .escapist import EscapistIE
 from .espn import ESPNIE
 from .esri import EsriVideoIE
+from .europa import EuropaIE
 from .everyonesmixtape import EveryonesMixtapeIE
 from .exfm import ExfmIE
 from .expotv import ExpoTVIE
diff --git a/youtube_dl/extractor/europa.py b/youtube_dl/extractor/europa.py
new file mode 100644
index 000000000..c437c4886
--- /dev/null
+++ b/youtube_dl/extractor/europa.py
@@ -0,0 +1,60 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    compat_urlparse,
+    xpath_text
+)
+
+
+class EuropaIE(InfoExtractor):
+    _VALID_URL = r'https?://ec\.europa\.eu/avservices/video/player\.cfm\?(?:[^&]|&(?!ref))*ref=(?P<id>[A-Za-z0-9]+)'
+    _TEST = {
+        'url': 'http://ec.europa.eu/avservices/video/player.cfm?ref=I107758',
+        'md5': '728cca2fd41d5aa7350cec1141fbe620',
+        'info_dict': {
+            'id': 'I107758',
+            'ext': 'mp4',
+            'title': 'TRADE - Wikileaks on TTIP',
+            'description': 'NEW  LIVE EC Midday press briefing of 11/08/2015',
+            'thumbnail': 're:^http://defiris\.ec\.streamcloud\.be/findmedia/18/107758/THUMB_[0-9A-Z]+\.jpg$'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+        lang = query.get('sitelang', ['en'])[0]
+
+        playlist = self._download_xml('http://ec.europa.eu/avservices/video/player/playlist.cfm?ID=' + video_id, video_id)
+        videos = {}
+        formats = []
+
+        for item in playlist.findall('info/title/item'):
+            videos[xpath_text(item, 'lg')] = {'title': xpath_text(item, 'label').strip()}
+
+        for item in playlist.findall('info/description/item'):
+            videos[xpath_text(item, 'lg')]['description'] = xpath_text(item, 'label').strip()
+
+        for item in playlist.findall('files/file'):
+            lg = xpath_text(item, 'lg')
+            vid = videos[lg]
+            vid['format_note'] = xpath_text(item, 'lglabel')
+            vid['url'] = xpath_text(item, 'url')
+
+            if lg == lang:
+                vid['language_preference'] = 10
+
+            formats.append(vid)
+
+        formats.reverse()
+        def_video = videos.get(lang, videos['int'])
+
+        return {
+            'id': video_id,
+            'title': def_video['title'],
+            'description': def_video['description'],
+            'thumbnail': xpath_text(playlist, 'info/thumburl', 'thumburl'),
+            'formats': formats
+        }

From af17794c654bd24bbd5f47997596430b201ea08e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Oct 2015 22:29:15 +0600
Subject: [PATCH 033/150] [europa] Improve extraction

---
 youtube_dl/extractor/europa.py | 94 ++++++++++++++++++++++------------
 1 file changed, 62 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/europa.py b/youtube_dl/extractor/europa.py
index c437c4886..02ba8d63c 100644
--- a/youtube_dl/extractor/europa.py
+++ b/youtube_dl/extractor/europa.py
@@ -2,59 +2,89 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
-    compat_urlparse,
+    int_or_none,
+    orderedSet,
+    parse_duration,
+    qualities,
+    unified_strdate,
     xpath_text
 )
 
 
 class EuropaIE(InfoExtractor):
-    _VALID_URL = r'https?://ec\.europa\.eu/avservices/video/player\.cfm\?(?:[^&]|&(?!ref))*ref=(?P<id>[A-Za-z0-9]+)'
-    _TEST = {
+    _VALID_URL = r'https?://ec\.europa\.eu/avservices/video/player\.cfm\?.*?\bref=(?P<id>[A-Za-z0-9]+)'
+    _TESTS = [{
         'url': 'http://ec.europa.eu/avservices/video/player.cfm?ref=I107758',
-        'md5': '728cca2fd41d5aa7350cec1141fbe620',
+        'md5': '574f080699ddd1e19a675b0ddf010371',
         'info_dict': {
             'id': 'I107758',
             'ext': 'mp4',
             'title': 'TRADE - Wikileaks on TTIP',
             'description': 'NEW  LIVE EC Midday press briefing of 11/08/2015',
-            'thumbnail': 're:^http://defiris\.ec\.streamcloud\.be/findmedia/18/107758/THUMB_[0-9A-Z]+\.jpg$'
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'upload_date': '20150811',
+            'duration': 34,
+            'view_count': int,
+            'formats': 'mincount:3',
         }
-    }
+    }, {
+        'url': 'http://ec.europa.eu/avservices/video/player.cfm?sitelang=en&ref=I107786',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
+        playlist = self._download_xml(
+            'http://ec.europa.eu/avservices/video/player/playlist.cfm?ID=%s' % video_id, video_id)
+
+        def get_item(type_, preference):
+            items = {}
+            for item in playlist.findall('./info/%s/item' % type_):
+                lang, label = xpath_text(item, 'lg', default=None), xpath_text(item, 'label', default=None)
+                if lang and label:
+                    items[lang] = label.strip()
+            for p in preference:
+                if items.get(p):
+                    return items[p]
+
         query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
-        lang = query.get('sitelang', ['en'])[0]
+        preferred_lang = query.get('sitelang', ('en', ))[0]
+
+        preferred_langs = orderedSet((preferred_lang, 'en', 'int'))
+
+        title = get_item('title', preferred_langs) or video_id
+        description = get_item('description', preferred_langs)
+        thumbnmail = xpath_text(playlist, './info/thumburl', 'thumbnail')
+        upload_date = unified_strdate(xpath_text(playlist, './info/date', 'upload date'))
+        duration = parse_duration(xpath_text(playlist, './info/duration', 'duration'))
+        view_count = int_or_none(xpath_text(playlist,'./info/views', 'views'))
+
+        language_preference = qualities(preferred_langs[::-1])
 
-        playlist = self._download_xml('http://ec.europa.eu/avservices/video/player/playlist.cfm?ID=' + video_id, video_id)
-        videos = {}
         formats = []
-
-        for item in playlist.findall('info/title/item'):
-            videos[xpath_text(item, 'lg')] = {'title': xpath_text(item, 'label').strip()}
-
-        for item in playlist.findall('info/description/item'):
-            videos[xpath_text(item, 'lg')]['description'] = xpath_text(item, 'label').strip()
-
-        for item in playlist.findall('files/file'):
-            lg = xpath_text(item, 'lg')
-            vid = videos[lg]
-            vid['format_note'] = xpath_text(item, 'lglabel')
-            vid['url'] = xpath_text(item, 'url')
-
-            if lg == lang:
-                vid['language_preference'] = 10
-
-            formats.append(vid)
-
-        formats.reverse()
-        def_video = videos.get(lang, videos['int'])
+        for file_ in playlist.findall('./files/file'):
+            video_url = xpath_text(file_, './url')
+            if not video_url:
+                continue
+            lang = xpath_text(file_, './lg')
+            formats.append({
+                'url': video_url,
+                'format_id': lang,
+                'format_note': xpath_text(file_, './lglabel'),
+                'language_preference': language_preference(lang)
+            })
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': def_video['title'],
-            'description': def_video['description'],
-            'thumbnail': xpath_text(playlist, 'info/thumburl', 'thumburl'),
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnmail,
+            'upload_date': upload_date,
+            'duration': duration,
+            'view_count': view_count,
             'formats': formats
         }

From f3b098fb90b985484d800fcdbfe18add2360e4df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Oct 2015 23:22:53 +0600
Subject: [PATCH 034/150] [europa] Add support for audio URLs

---
 youtube_dl/extractor/europa.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/europa.py b/youtube_dl/extractor/europa.py
index 02ba8d63c..9e33cacff 100644
--- a/youtube_dl/extractor/europa.py
+++ b/youtube_dl/extractor/europa.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class EuropaIE(InfoExtractor):
-    _VALID_URL = r'https?://ec\.europa\.eu/avservices/video/player\.cfm\?.*?\bref=(?P<id>[A-Za-z0-9]+)'
+    _VALID_URL = r'https?://ec\.europa\.eu/avservices/(?:video/player|audio/audioDetails)\.cfm\?.*?\bref=(?P<id>[A-Za-z0-9-]+)'
     _TESTS = [{
         'url': 'http://ec.europa.eu/avservices/video/player.cfm?ref=I107758',
         'md5': '574f080699ddd1e19a675b0ddf010371',
@@ -32,6 +32,9 @@ class EuropaIE(InfoExtractor):
     }, {
         'url': 'http://ec.europa.eu/avservices/video/player.cfm?sitelang=en&ref=I107786',
         'only_matching': True,
+    }, {
+        'url': 'http://ec.europa.eu/avservices/audio/audioDetails.cfm?ref=I-109295&sitelang=en',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From b203095d4c95d471bc2ac7045693c6938ee914d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 2 Oct 2015 22:40:35 +0200
Subject: [PATCH 035/150] [europa] Style fix: add whitespace after comma

---
 youtube_dl/extractor/europa.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/europa.py b/youtube_dl/extractor/europa.py
index 9e33cacff..adc43919e 100644
--- a/youtube_dl/extractor/europa.py
+++ b/youtube_dl/extractor/europa.py
@@ -63,7 +63,7 @@ class EuropaIE(InfoExtractor):
         thumbnmail = xpath_text(playlist, './info/thumburl', 'thumbnail')
         upload_date = unified_strdate(xpath_text(playlist, './info/date', 'upload date'))
         duration = parse_duration(xpath_text(playlist, './info/duration', 'duration'))
-        view_count = int_or_none(xpath_text(playlist,'./info/views', 'views'))
+        view_count = int_or_none(xpath_text(playlist, './info/views', 'views'))
 
         language_preference = qualities(preferred_langs[::-1])
 

From 5495937f461268a850a6a54d3fe19ed1f0f01eef Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Sep 2015 20:00:10 +0800
Subject: [PATCH 036/150] [options] Cleanup double spaces in help texts

---
 youtube_dl/options.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 5eccc0a70..3dd6d290b 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -276,7 +276,7 @@ def parseOpts(overrideArguments=None):
             'For example, to only match videos that have been liked more than '
             '100 times and disliked less than 50 times (or the dislike '
             'functionality is not available at the given service), but who '
-            'also have a description, use  --match-filter '
+            'also have a description, use --match-filter '
             '"like_count > 100 & dislike_count <? 50 & description" .'
         ))
     selection.add_option(
@@ -602,7 +602,7 @@ def parseOpts(overrideArguments=None):
     filesystem.add_option(
         '-A', '--auto-number',
         action='store_true', dest='autonumber', default=False,
-        help='[deprecated; use  -o "%(autonumber)s-%(title)s.%(ext)s" ] Number downloaded files starting from 00000')
+        help='[deprecated; use -o "%(autonumber)s-%(title)s.%(ext)s" ] Number downloaded files starting from 00000')
     filesystem.add_option(
         '-t', '--title',
         action='store_true', dest='usetitle', default=False,

From 13118a50b88c26fe5d932eccf77e297249a78634 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Sep 2015 20:04:27 +0800
Subject: [PATCH 037/150] [compat] Allow overriding by only COLUMNS or LINES in
 compat_get_terminal_size

Now the semantic of this function is identical to
shutil.get_terminal_size() in Python 3.3+. The new behavior also
corresponds to the old get_term_width(), which is removed in
003c69a84b68cadb46aeb8e03115848a722fd675
---
 youtube_dl/compat.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 1ff42d94b..c36c9c23f 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -416,7 +416,7 @@ if hasattr(shutil, 'get_terminal_size'):  # Python >= 3.3
 else:
     _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines'])
 
-    def compat_get_terminal_size():
+    def compat_get_terminal_size(fallback=(80, 24)):
         columns = compat_getenv('COLUMNS', None)
         if columns:
             columns = int(columns)
@@ -428,14 +428,20 @@ else:
         else:
             lines = None
 
-        try:
-            sp = subprocess.Popen(
-                ['stty', 'size'],
-                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            out, err = sp.communicate()
-            lines, columns = map(int, out.split())
-        except Exception:
-            pass
+        if columns <= 0 or lines <= 0:
+            try:
+                sp = subprocess.Popen(
+                    ['stty', 'size'],
+                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                out, err = sp.communicate()
+                _columns, _lines = map(int, out.split())
+            except Exception:
+                _columns, _lines = _terminal_size(*fallback)
+
+            if columns <= 0:
+                columns = _columns
+            if lines <= 0:
+                lines = _lines
         return _terminal_size(columns, lines)
 
 try:

From bad84757eb135b85d5a1b29524a064d23ab4e1e9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Sep 2015 20:10:23 +0800
Subject: [PATCH 038/150] [doc] Better formatting of youtube-dl.1 (closes
 #6510)

---
 devscripts/prepare_manpage.py | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/devscripts/prepare_manpage.py b/devscripts/prepare_manpage.py
index 7ece37754..776e6556e 100644
--- a/devscripts/prepare_manpage.py
+++ b/devscripts/prepare_manpage.py
@@ -8,6 +8,35 @@ import re
 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 README_FILE = os.path.join(ROOT_DIR, 'README.md')
 
+
+def filter_options(readme):
+    ret = ''
+    in_options = False
+    for line in readme.split('\n'):
+        if line.startswith('# '):
+            if line[2:].startswith('OPTIONS'):
+                in_options = True
+            else:
+                in_options = False
+
+        if in_options:
+            if line.lstrip().startswith('-'):
+                option, description = re.split(r'\s{2,}', line.lstrip())
+                split_option = option.split(' ')
+
+                if not split_option[-1].startswith('-'):  # metavar
+                    option = ' '.join(split_option[:-1] + ['*%s*' % split_option[-1]])
+
+                # Pandoc's definition_lists. See http://pandoc.org/README.html
+                # for more information.
+                ret += '\n%s\n:   %s\n' % (option, description)
+            else:
+                ret += line.lstrip() + '\n'
+        else:
+            ret += line + '\n'
+
+    return ret
+
 with io.open(README_FILE, encoding='utf-8') as f:
     readme = f.read()
 
@@ -26,6 +55,8 @@ readme = re.sub(r'(?s)^.*?(?=# DESCRIPTION)', '', readme)
 readme = re.sub(r'\s+youtube-dl \[OPTIONS\] URL \[URL\.\.\.\]', '', readme)
 readme = PREFIX + readme
 
+readme = filter_options(readme)
+
 if sys.version_info < (3, 0):
     print(readme.encode('utf-8'))
 else:

From 97d5bfcba65c8575ab06a34e91fae30a5fda3161 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 3 Oct 2015 14:17:17 +0100
Subject: [PATCH 039/150] [engadget] accept short video urls

---
 youtube_dl/extractor/engadget.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/engadget.py b/youtube_dl/extractor/engadget.py
index 4ea37ebd9..e4180701d 100644
--- a/youtube_dl/extractor/engadget.py
+++ b/youtube_dl/extractor/engadget.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 class EngadgetIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://www.engadget.com/
-        (?:video/5min/(?P<id>\d+)|
+        (?:video(?:/5min)?/(?P<id>\d+)|
             [\d/]+/.*?)
         '''
 

From 60d23e5e592aebe4a77dfb4ab70e87337967721c Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 3 Oct 2015 16:25:33 +0300
Subject: [PATCH 040/150] [tapely] Improve _VALID_URL

---
 youtube_dl/extractor/tapely.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tapely.py b/youtube_dl/extractor/tapely.py
index f1f43d0a7..744f9db38 100644
--- a/youtube_dl/extractor/tapely.py
+++ b/youtube_dl/extractor/tapely.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class TapelyIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tape\.ly/(?P<id>[A-Za-z0-9\-_]+)(?:/(?P<songnr>\d+))?'
+    _VALID_URL = r'https?://(?:www\.)?(?:tape\.ly|tapely\.com)/(?P<id>[A-Za-z0-9\-_]+)(?:/(?P<songnr>\d+))?'
     _API_URL = 'http://tape.ly/showtape?id={0:}'
     _S3_SONG_URL = 'http://mytape.s3.amazonaws.com/{0:}'
     _SOUNDCLOUD_SONG_URL = 'http://api.soundcloud.com{0:}'
@@ -42,6 +42,10 @@ class TapelyIE(InfoExtractor):
                 'ext': 'm4a',
             },
         },
+        {
+            'url': 'https://tapely.com/my-grief-as-told-by-water',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):

From ef5acfe32de4c995625f9800cfe0776237961436 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 1 Sep 2015 23:05:19 +0100
Subject: [PATCH 041/150] [limelight] Add new extractor

---
 youtube_dl/extractor/__init__.py  |   5 +
 youtube_dl/extractor/limelight.py | 176 ++++++++++++++++++++++++++++++
 2 files changed, 181 insertions(+)
 create mode 100644 youtube_dl/extractor/limelight.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 495a18c17..20cc3660c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -295,6 +295,11 @@ from .lifenews import (
     LifeNewsIE,
     LifeEmbedIE,
 )
+from .limelight import (
+    LimeLightMediaIE,
+    LimeLightChannelIE,
+    LimeLightChannelListIE,
+)
 from .liveleak import LiveLeakIE
 from .livestream import (
     LivestreamIE,
diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
new file mode 100644
index 000000000..dcfc215c7
--- /dev/null
+++ b/youtube_dl/extractor/limelight.py
@@ -0,0 +1,176 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    determine_ext,
+)
+
+
+class LimeLightBaseIE(InfoExtractor):
+
+    def get_playlist_service(self, id, method):
+        return self._download_json(self.PLAYLIST_SERVICE_URL % (id, method), id)
+
+    def get_api(self, orgId, id, method):
+        return self._download_json(self.API_URL % (orgId, id, method), id)
+
+    def process_data(self, mobileUrls, streams, properties):
+        video_id = properties['media_id']
+        formats = []
+
+        for mobileUrl in mobileUrls:
+            if '.m3u8' in mobileUrl['mobileUrl']:
+                formats.extend(self._extract_m3u8_formats(mobileUrl['mobileUrl'], video_id))
+            else:
+                formats.append({'url': mobileUrl['mobileUrl']})
+
+        for stream in streams:
+            if '.f4m' in stream['url']:
+                formats.extend(self._extract_f4m_formats(stream['url'], video_id))
+            else:
+                fmt = {
+                    'url': stream.get('url'),
+                    'abr': stream.get('audioBitRate'),
+                    'vbr': stream.get('videoBitRate'),
+                    'fps': stream.get('videoFrameRate'),
+                    'width': stream.get('videoWidthInPixels'),
+                    'height': stream.get('videoHeightInPixels'),
+                    'ext': determine_ext(stream.get('url'))
+                }
+                rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+))/(?P<playpath>mp4:.+)$', stream['url'])
+                if rtmp:
+                    fmt.update({
+                        'url': rtmp.group('url'),
+                        'play_path': rtmp.group('playpath'),
+                        'app': rtmp.group('app'),
+                    })
+                formats.append(fmt)
+
+        self._sort_formats(formats)
+
+        title = properties['title']
+        description = properties.get('description')
+        timestamp = properties.get('create_date')
+        duration = int_or_none(properties.get('duration_in_milliseconds'))
+        filesize = properties.get('total_storage_in_bytes')
+        categories = [properties.get('category')]
+        thumbnails = [{
+            'url': thumbnail.get('url'),
+            'width': int_or_none(thumbnail.get('width')),
+            'height': int_or_none(thumbnail.get('height')),
+        } for thumbnail in properties.get('thumbnails')]
+        subtitles = {caption.get('language_code'): [{'url': caption.get('url')}] for caption in properties.get('captions')}
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'formats': formats,
+            'timestamp': timestamp,
+            'duration': duration,
+            'filesize': filesize,
+            'categories': categories,
+            'thumbnails': thumbnails,
+            'subtitles': subtitles,
+        }
+
+
+class LimeLightMediaIE(LimeLightBaseIE):
+    IE_NAME = 'limelight'
+    _VALID_URL = r'http://link\.videoplatform\.limelight\.com/media/?.*mediaId=(?P<id>[a-z0-9]{32})'
+    _TEST = {
+        'url': 'http://link.videoplatform.limelight.com/media/?mediaId=3ffd040b522b4485b6d84effc750cd86',
+        'md5': '3213605088be599705677ef785db6972',
+        'info_dict': {
+            'id': '3ffd040b522b4485b6d84effc750cd86',
+            'ext': 'mp4',
+            'title': 'HaP and the HB Prince Trailer',
+            'description': 'As Harry Potter begins his 6th year at Hogwarts School of Witchcraft and Wizardry, he discovers an old book marked mysteriously "This book is the property of the Half-Blood Prince" and begins to learn more about Lord Voldemort\'s dark past.',
+            'thumbnail': 're:^https?://.*\.jpeg$',
+            'duration': 144230,
+            'timestamp': 1244136834,
+            "upload_date": "20090604",
+        }
+    }
+    PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/media/%s/%s'
+    API_URL = 'http://api.video.limelight.com/rest/organizations/%s/media/%s/%s.json'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        mobile_json_data = self.get_playlist_service(video_id, 'getMobilePlaylistByMediaId')
+        pc_json_data = self.get_playlist_service(video_id, 'getPlaylistByMediaId')
+        properties = self.get_api(pc_json_data['orgId'], video_id, 'properties')
+
+        return self.process_data(mobile_json_data['mediaList'][0]['mobileUrls'], pc_json_data['playlistItems'][0]['streams'], properties)
+
+
+class LimeLightChannelIE(LimeLightBaseIE):
+    IE_NAME = 'limelight:channel'
+    _VALID_URL = r'http://link\.videoplatform\.limelight\.com/media/?.*channelId=(?P<id>[a-z0-9]{32})'
+    _TEST = {
+        'url': 'http://link.videoplatform.limelight.com/media/?channelId=ab6a524c379342f9b23642917020c082',
+        'info_dict': {
+            'id': 'ab6a524c379342f9b23642917020c082',
+            'title': 'Javascript Sample Code',
+        },
+        'playlist_mincount': 3,
+    }
+    PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/channel/%s/%s'
+    API_URL = 'http://api.video.limelight.com/rest/organizations/%s/channels/%s/%s.json'
+
+    def _real_extract(self, url):
+        channel_id = self._match_id(url)
+
+        mobile_json_data = self.get_playlist_service(channel_id, 'getMobilePlaylistWithNItemsByChannelId?begin=0&count=-1')
+        pc_json_data = self.get_playlist_service(channel_id, 'getPlaylistByChannelId')
+        medias = self.get_api(pc_json_data['orgId'], channel_id, 'media')
+
+        entries = []
+        for i in range(len(medias['media_list'])):
+            entries.append(self.process_data(mobile_json_data['mediaList'][i]['mobileUrls'], pc_json_data['playlistItems'][i]['streams'], medias['media_list'][i]))
+
+        return {
+            'id': channel_id,
+            'title': pc_json_data['title'],
+            'entries': entries,
+            '_type': 'playlist',
+        }
+
+
+class LimeLightChannelListIE(LimeLightBaseIE):
+    IE_NAME = 'limelight:channel_list'
+    _VALID_URL = r'http://link\.videoplatform\.limelight\.com/media/?.*channelListId=(?P<id>[a-z0-9]{32})'
+    _TEST = {
+        'url': 'http://link.videoplatform.limelight.com/media/?channelListId=301b117890c4465c8179ede21fd92e2b',
+        'info_dict': {
+            'id': '301b117890c4465c8179ede21fd92e2b',
+            'title': 'Website - Hero Player',
+        },
+        'playlist_mincount': 2,
+    }
+    PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/channel_list/%s/%s'
+
+    def _real_extract(self, url):
+        channel_list_id = self._match_id(url)
+
+        json_data = self.get_playlist_service(channel_list_id, 'getMobileChannelListById')
+
+        entries = []
+        for channel in json_data['channelList']:
+            entries.append({
+                'url': 'http://link.videoplatform.limelight.com/media/?channelId=%s' % channel['id'],
+                '_type': 'url',
+                'ie_key': 'LimeLightChannel',
+            })
+
+        return {
+            'id': channel_list_id,
+            'title': json_data['title'],
+            'entries': entries,
+            '_type': 'playlist',
+        }

From 4bba371644818d79b5f8481b5b31c53ea8ecbcc5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Oct 2015 20:33:42 +0600
Subject: [PATCH 042/150] [YoutubeDL] Autocalculate ext for subtitles when
 missing

---
 youtube_dl/YoutubeDL.py        | 11 +++++++++--
 youtube_dl/extractor/common.py |  1 +
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d65253882..adf70d658 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1232,13 +1232,20 @@ class YoutubeDL(object):
             except (ValueError, OverflowError, OSError):
                 pass
 
+        subtitles = info_dict.get('subtitles')
+        if subtitles:
+            for _, subtitle in subtitles.items():
+                for subtitle_format in subtitle:
+                    if 'ext' not in subtitle_format:
+                        subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
+
         if self.params.get('listsubtitles', False):
             if 'automatic_captions' in info_dict:
                 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
-            self.list_subtitles(info_dict['id'], info_dict.get('subtitles'), 'subtitles')
+            self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
             return
         info_dict['requested_subtitles'] = self.process_subtitles(
-            info_dict['id'], info_dict.get('subtitles'),
+            info_dict['id'], subtitles,
             info_dict.get('automatic_captions'))
 
         # We now pick which formats have to be downloaded
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 4fe2307cd..dbae75406 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -165,6 +165,7 @@ class InfoExtractor(object):
                     with the "ext" entry and one of:
                         * "data": The subtitles file contents
                         * "url": A URL pointing to the subtitles file
+                    "ext" will be calculated from URL if missing
     automatic_captions: Like 'subtitles', used by the YoutubeIE for
                     automatically generated captions
     duration:       Length of the video in seconds, as an integer.

From d7fc56318b72607758e7484c22076ec2999f10b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Oct 2015 20:41:57 +0600
Subject: [PATCH 043/150] [limelight] Fix python 2.6, simplify, make more
 robust (Closes #6734)

---
 youtube_dl/extractor/__init__.py  |   6 +-
 youtube_dl/extractor/limelight.py | 185 ++++++++++++++++++------------
 2 files changed, 113 insertions(+), 78 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 20cc3660c..3ace1cc2c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -296,9 +296,9 @@ from .lifenews import (
     LifeEmbedIE,
 )
 from .limelight import (
-    LimeLightMediaIE,
-    LimeLightChannelIE,
-    LimeLightChannelListIE,
+    LimelightMediaIE,
+    LimelightChannelIE,
+    LimelightChannelListIE,
 )
 from .liveleak import LiveLeakIE
 from .livestream import (
diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index dcfc215c7..599d8413d 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -5,65 +5,105 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    int_or_none,
     determine_ext,
+    float_or_none,
+    int_or_none,
 )
 
 
-class LimeLightBaseIE(InfoExtractor):
+class LimelightBaseIE(InfoExtractor):
+    _PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/%s/%s/%s'
+    _API_URL = 'http://api.video.limelight.com/rest/organizations/%s/%s/%s/%s.json'
 
-    def get_playlist_service(self, id, method):
-        return self._download_json(self.PLAYLIST_SERVICE_URL % (id, method), id)
+    def _call_playlist_service(self, item_id, method, fatal=True):
+        return self._download_json(
+            self._PLAYLIST_SERVICE_URL % (self._PLAYLIST_SERVICE_PATH, item_id, method),
+            item_id, 'Downloading PlaylistService %s JSON' % method, fatal=fatal)
 
-    def get_api(self, orgId, id, method):
-        return self._download_json(self.API_URL % (orgId, id, method), id)
+    def _call_api(self, organization_id, item_id, method):
+        return self._download_json(
+            self._API_URL % (organization_id, self._API_PATH, item_id, method),
+            item_id, 'Downloading API %s JSON' % method)
 
-    def process_data(self, mobileUrls, streams, properties):
+    def _extract(self, item_id, pc_method, mobile_method, meta_method):
+        pc = self._call_playlist_service(item_id, pc_method)
+        metadata = self._call_api(pc['orgId'], item_id, meta_method)
+        mobile = self._call_playlist_service(item_id, mobile_method, fatal=False)
+        return pc, mobile, metadata
+
+    def _extract_info(self, streams, mobile_urls, properties):
         video_id = properties['media_id']
         formats = []
 
-        for mobileUrl in mobileUrls:
-            if '.m3u8' in mobileUrl['mobileUrl']:
-                formats.extend(self._extract_m3u8_formats(mobileUrl['mobileUrl'], video_id))
-            else:
-                formats.append({'url': mobileUrl['mobileUrl']})
-
         for stream in streams:
-            if '.f4m' in stream['url']:
-                formats.extend(self._extract_f4m_formats(stream['url'], video_id))
+            stream_url = stream.get('url')
+            if not stream_url:
+                continue
+            if '.f4m' in stream_url:
+                formats.extend(self._extract_f4m_formats(stream_url, video_id))
             else:
                 fmt = {
-                    'url': stream.get('url'),
-                    'abr': stream.get('audioBitRate'),
-                    'vbr': stream.get('videoBitRate'),
-                    'fps': stream.get('videoFrameRate'),
-                    'width': stream.get('videoWidthInPixels'),
-                    'height': stream.get('videoHeightInPixels'),
-                    'ext': determine_ext(stream.get('url'))
+                    'url': stream_url,
+                    'abr': float_or_none(stream.get('audioBitRate')),
+                    'vbr': float_or_none(stream.get('videoBitRate')),
+                    'fps': float_or_none(stream.get('videoFrameRate')),
+                    'width': int_or_none(stream.get('videoWidthInPixels')),
+                    'height': int_or_none(stream.get('videoHeightInPixels')),
+                    'ext': determine_ext(stream_url)
                 }
-                rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+))/(?P<playpath>mp4:.+)$', stream['url'])
+                rtmp = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>.+))/(?P<playpath>mp4:.+)$', stream_url)
                 if rtmp:
+                    format_id = 'rtmp'
+                    if stream.get('videoBitRate'):
+                        format_id += '-%d' % int_or_none(stream['videoBitRate'])
                     fmt.update({
                         'url': rtmp.group('url'),
                         'play_path': rtmp.group('playpath'),
                         'app': rtmp.group('app'),
+                        'ext': 'flv',
+                        'format_id': format_id,
                     })
                 formats.append(fmt)
 
+        for mobile_url in mobile_urls:
+            media_url = mobile_url.get('mobileUrl')
+            if not media_url:
+                continue
+            format_id = mobile_url.get('targetMediaPlatform')
+            if determine_ext(media_url) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    media_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    preference=-1, m3u8_id=format_id))
+            else:
+                formats.append({
+                    'url': media_url,
+                    'format_id': format_id,
+                    'preference': -1,
+                })
+
         self._sort_formats(formats)
 
         title = properties['title']
         description = properties.get('description')
-        timestamp = properties.get('create_date')
-        duration = int_or_none(properties.get('duration_in_milliseconds'))
-        filesize = properties.get('total_storage_in_bytes')
+        timestamp = int_or_none(properties.get('publish_date') or properties.get('create_date'))
+        duration = float_or_none(properties.get('duration_in_milliseconds'), 1000)
+        filesize = int_or_none(properties.get('total_storage_in_bytes'))
         categories = [properties.get('category')]
+        tags = properties.get('tags', [])
         thumbnails = [{
-            'url': thumbnail.get('url'),
+            'url': thumbnail['url'],
             'width': int_or_none(thumbnail.get('width')),
             'height': int_or_none(thumbnail.get('height')),
-        } for thumbnail in properties.get('thumbnails')]
-        subtitles = {caption.get('language_code'): [{'url': caption.get('url')}] for caption in properties.get('captions')}
+        } for thumbnail in properties.get('thumbnails', []) if thumbnail.get('url')]
+
+        subtitles = {}
+        for caption in properties.get('captions', {}):
+            lang = caption.get('language_code')
+            subtitles_url = caption.get('url')
+            if lang and subtitles_url:
+                subtitles[lang] = [{
+                    'url': subtitles_url,
+                }]
 
         return {
             'id': video_id,
@@ -74,44 +114,50 @@ class LimeLightBaseIE(InfoExtractor):
             'duration': duration,
             'filesize': filesize,
             'categories': categories,
+            'tags': tags,
             'thumbnails': thumbnails,
             'subtitles': subtitles,
         }
 
 
-class LimeLightMediaIE(LimeLightBaseIE):
+class LimelightMediaIE(LimelightBaseIE):
     IE_NAME = 'limelight'
-    _VALID_URL = r'http://link\.videoplatform\.limelight\.com/media/?.*mediaId=(?P<id>[a-z0-9]{32})'
+    _VALID_URL = r'(?:limelight:media:|http://link\.videoplatform\.limelight\.com/media/\??\bmediaId=)(?P<id>[a-z0-9]{32})'
     _TEST = {
         'url': 'http://link.videoplatform.limelight.com/media/?mediaId=3ffd040b522b4485b6d84effc750cd86',
-        'md5': '3213605088be599705677ef785db6972',
         'info_dict': {
             'id': '3ffd040b522b4485b6d84effc750cd86',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'HaP and the HB Prince Trailer',
             'description': 'As Harry Potter begins his 6th year at Hogwarts School of Witchcraft and Wizardry, he discovers an old book marked mysteriously "This book is the property of the Half-Blood Prince" and begins to learn more about Lord Voldemort\'s dark past.',
             'thumbnail': 're:^https?://.*\.jpeg$',
-            'duration': 144230,
+            'duration': 144.23,
             'timestamp': 1244136834,
-            "upload_date": "20090604",
-        }
+            'upload_date': '20090604',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
     }
-    PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/media/%s/%s'
-    API_URL = 'http://api.video.limelight.com/rest/organizations/%s/media/%s/%s.json'
+    _PLAYLIST_SERVICE_PATH = 'media'
+    _API_PATH = 'media'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        mobile_json_data = self.get_playlist_service(video_id, 'getMobilePlaylistByMediaId')
-        pc_json_data = self.get_playlist_service(video_id, 'getPlaylistByMediaId')
-        properties = self.get_api(pc_json_data['orgId'], video_id, 'properties')
+        pc, mobile, metadata = self._extract(
+            video_id, 'getPlaylistByMediaId', 'getMobilePlaylistByMediaId', 'properties')
 
-        return self.process_data(mobile_json_data['mediaList'][0]['mobileUrls'], pc_json_data['playlistItems'][0]['streams'], properties)
+        return self._extract_info(
+            pc['playlistItems'][0].get('streams', []),
+            mobile['mediaList'][0].get('mobileUrls', []) if mobile else [],
+            metadata)
 
 
-class LimeLightChannelIE(LimeLightBaseIE):
+class LimelightChannelIE(LimelightBaseIE):
     IE_NAME = 'limelight:channel'
-    _VALID_URL = r'http://link\.videoplatform\.limelight\.com/media/?.*channelId=(?P<id>[a-z0-9]{32})'
+    _VALID_URL = r'(?:limelight:channel:|http://link\.videoplatform\.limelight\.com/media/\??\bchannelId=)(?P<id>[a-z0-9]{32})'
     _TEST = {
         'url': 'http://link.videoplatform.limelight.com/media/?channelId=ab6a524c379342f9b23642917020c082',
         'info_dict': {
@@ -120,31 +166,29 @@ class LimeLightChannelIE(LimeLightBaseIE):
         },
         'playlist_mincount': 3,
     }
-    PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/channel/%s/%s'
-    API_URL = 'http://api.video.limelight.com/rest/organizations/%s/channels/%s/%s.json'
+    _PLAYLIST_SERVICE_PATH = 'channel'
+    _API_PATH = 'channels'
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
-        mobile_json_data = self.get_playlist_service(channel_id, 'getMobilePlaylistWithNItemsByChannelId?begin=0&count=-1')
-        pc_json_data = self.get_playlist_service(channel_id, 'getPlaylistByChannelId')
-        medias = self.get_api(pc_json_data['orgId'], channel_id, 'media')
+        pc, mobile, medias = self._extract(
+            channel_id, 'getPlaylistByChannelId',
+            'getMobilePlaylistWithNItemsByChannelId?begin=0&count=-1', 'media')
 
-        entries = []
-        for i in range(len(medias['media_list'])):
-            entries.append(self.process_data(mobile_json_data['mediaList'][i]['mobileUrls'], pc_json_data['playlistItems'][i]['streams'], medias['media_list'][i]))
+        entries = [
+            self._extract_info(
+                pc['playlistItems'][i].get('streams', []),
+                mobile['mediaList'][i].get('mobileUrls', []) if mobile else [],
+                medias['media_list'][i])
+            for i in range(len(medias['media_list']))]
 
-        return {
-            'id': channel_id,
-            'title': pc_json_data['title'],
-            'entries': entries,
-            '_type': 'playlist',
-        }
+        return self.playlist_result(entries, channel_id, pc['title'])
 
 
-class LimeLightChannelListIE(LimeLightBaseIE):
+class LimelightChannelListIE(LimelightBaseIE):
     IE_NAME = 'limelight:channel_list'
-    _VALID_URL = r'http://link\.videoplatform\.limelight\.com/media/?.*channelListId=(?P<id>[a-z0-9]{32})'
+    _VALID_URL = r'(?:limelight:channel_list:|http://link\.videoplatform\.limelight\.com/media/\?.*?\bchannelListId=)(?P<id>[a-z0-9]{32})'
     _TEST = {
         'url': 'http://link.videoplatform.limelight.com/media/?channelListId=301b117890c4465c8179ede21fd92e2b',
         'info_dict': {
@@ -153,24 +197,15 @@ class LimeLightChannelListIE(LimeLightBaseIE):
         },
         'playlist_mincount': 2,
     }
-    PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/channel_list/%s/%s'
+    _PLAYLIST_SERVICE_PATH = 'channel_list'
 
     def _real_extract(self, url):
         channel_list_id = self._match_id(url)
 
-        json_data = self.get_playlist_service(channel_list_id, 'getMobileChannelListById')
+        channel_list = self._call_playlist_service(channel_list_id, 'getMobileChannelListById')
 
-        entries = []
-        for channel in json_data['channelList']:
-            entries.append({
-                'url': 'http://link.videoplatform.limelight.com/media/?channelId=%s' % channel['id'],
-                '_type': 'url',
-                'ie_key': 'LimeLightChannel',
-            })
+        entries = [
+            self.url_result('limelight:channel:%s' % channel['id'], 'LimelightChannel')
+            for channel in channel_list['channelList']]
 
-        return {
-            'id': channel_list_id,
-            'title': json_data['title'],
-            'entries': entries,
-            '_type': 'playlist',
-        }
+        return self.playlist_result(entries, channel_list_id, channel_list['title'])

From 9c544e2537abda1d65e96f2b33a79984f3ab7c10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Oct 2015 20:48:44 +0600
Subject: [PATCH 044/150] [limelight] Add test video with subtitles

---
 youtube_dl/extractor/limelight.py | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index 599d8413d..fb03dd527 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -123,13 +123,13 @@ class LimelightBaseIE(InfoExtractor):
 class LimelightMediaIE(LimelightBaseIE):
     IE_NAME = 'limelight'
     _VALID_URL = r'(?:limelight:media:|http://link\.videoplatform\.limelight\.com/media/\??\bmediaId=)(?P<id>[a-z0-9]{32})'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://link.videoplatform.limelight.com/media/?mediaId=3ffd040b522b4485b6d84effc750cd86',
         'info_dict': {
             'id': '3ffd040b522b4485b6d84effc750cd86',
             'ext': 'flv',
             'title': 'HaP and the HB Prince Trailer',
-            'description': 'As Harry Potter begins his 6th year at Hogwarts School of Witchcraft and Wizardry, he discovers an old book marked mysteriously "This book is the property of the Half-Blood Prince" and begins to learn more about Lord Voldemort\'s dark past.',
+            'description': 'md5:8005b944181778e313d95c1237ddb640',
             'thumbnail': 're:^https?://.*\.jpeg$',
             'duration': 144.23,
             'timestamp': 1244136834,
@@ -139,7 +139,25 @@ class LimelightMediaIE(LimelightBaseIE):
             # rtmp download
             'skip_download': True,
         },
-    }
+    }, {
+        # video with subtitles
+        'url': 'limelight:media:a3e00274d4564ec4a9b29b9466432335',
+        'info_dict': {
+            'id': 'a3e00274d4564ec4a9b29b9466432335',
+            'ext': 'flv',
+            'title': '3Play Media Overview Video',
+            'description': '',
+            'thumbnail': 're:^https?://.*\.jpeg$',
+            'duration': 78.101,
+            'timestamp': 1338929955,
+            'upload_date': '20120605',
+            'subtitles': 'mincount:9',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }]
     _PLAYLIST_SERVICE_PATH = 'media'
     _API_PATH = 'media'
 

From 0659dfccfea9df3206c476e83a2b090456c25a83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Oct 2015 21:13:13 +0600
Subject: [PATCH 045/150] [pbs] Improve player regex (Closes #7059)

---
 youtube_dl/extractor/pbs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 683c81de3..0bca3152b 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -167,7 +167,7 @@ class PBSIE(InfoExtractor):
                 return media_id, presumptive_id, upload_date
 
             url = self._search_regex(
-                r'<iframe\s+[^>]*\s+src=["\']([^\'"]+partnerplayer[^\'"]+)["\']',
+                r'(?s)<iframe[^>]+?(?:[a-z-]+?=["\'].+?["\'][^>]+?)*?\bsrc=["\']([^\'"]+partnerplayer[^\'"]+)["\']',
                 webpage, 'player URL')
             mobj = re.match(self._VALID_URL, url)
 

From 96229998c29705c8ee4230915ec7ff050bcfecf8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Oct 2015 21:19:47 +0600
Subject: [PATCH 046/150] [pbs] Allow empty attribute in player regex

---
 youtube_dl/extractor/pbs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 0bca3152b..66b3dda47 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -167,7 +167,7 @@ class PBSIE(InfoExtractor):
                 return media_id, presumptive_id, upload_date
 
             url = self._search_regex(
-                r'(?s)<iframe[^>]+?(?:[a-z-]+?=["\'].+?["\'][^>]+?)*?\bsrc=["\']([^\'"]+partnerplayer[^\'"]+)["\']',
+                r'(?s)<iframe[^>]+?(?:[a-z-]+?=["\'].*?["\'][^>]+?)*?\bsrc=["\']([^\'"]+partnerplayer[^\'"]+)["\']',
                 webpage, 'player URL')
             mobj = re.match(self._VALID_URL, url)
 

From 90ab741e909c949039e31805da04f5e546a1a8c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Oct 2015 21:37:49 +0600
Subject: [PATCH 047/150] [pbs] Add test for #7059

---
 youtube_dl/extractor/pbs.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 66b3dda47..6923c6094 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -134,6 +134,24 @@ class PBSIE(InfoExtractor):
             'params': {
                 'skip_download': True,  # requires ffmpeg
             },
+        },
+        {
+            # Video embedded in iframe containing angle brackets as attribute's value (e.g.
+            # "<iframe style='position: absolute;<br />\ntop: 0; left: 0;' ...", see
+            # https://github.com/rg3/youtube-dl/issues/7059)
+            'url': 'http://www.pbs.org/food/features/a-chefs-life-season-3-episode-5-prickly-business/',
+            'info_dict': {
+                'id': '2365546844',
+                'display_id': 'a-chefs-life-season-3-episode-5-prickly-business',
+                'ext': 'mp4',
+                'title': "A Chef's Life - Season 3, Ep. 5: Prickly Business",
+                'description': 'md5:61db2ddf27c9912f09c241014b118ed1',
+                'duration': 1480,
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            },
         }
     ]
 

From e759a00119768862e63dbda33522f2399f1f43a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 20:21:53 +0600
Subject: [PATCH 048/150] [appletrailers] Quotes consistency

---
 youtube_dl/extractor/appletrailers.py | 68 +++++++++++++--------------
 1 file changed, 34 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 576f03b5b..f68dc3236 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -13,53 +13,53 @@ from ..utils import (
 class AppleTrailersIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/(?:trailers|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'
     _TESTS = [{
-        "url": "http://trailers.apple.com/trailers/wb/manofsteel/",
+        'url': 'http://trailers.apple.com/trailers/wb/manofsteel/',
         'info_dict': {
             'id': 'manofsteel',
         },
-        "playlist": [
+        'playlist': [
             {
-                "md5": "d97a8e575432dbcb81b7c3acb741f8a8",
-                "info_dict": {
-                    "id": "manofsteel-trailer4",
-                    "ext": "mov",
-                    "duration": 111,
-                    "title": "Trailer 4",
-                    "upload_date": "20130523",
-                    "uploader_id": "wb",
+                'md5': 'd97a8e575432dbcb81b7c3acb741f8a8',
+                'info_dict': {
+                    'id': 'manofsteel-trailer4',
+                    'ext': 'mov',
+                    'duration': 111,
+                    'title': 'Trailer 4',
+                    'upload_date': '20130523',
+                    'uploader_id': 'wb',
                 },
             },
             {
-                "md5": "b8017b7131b721fb4e8d6f49e1df908c",
-                "info_dict": {
-                    "id": "manofsteel-trailer3",
-                    "ext": "mov",
-                    "duration": 182,
-                    "title": "Trailer 3",
-                    "upload_date": "20130417",
-                    "uploader_id": "wb",
+                'md5': 'b8017b7131b721fb4e8d6f49e1df908c',
+                'info_dict': {
+                    'id': 'manofsteel-trailer3',
+                    'ext': 'mov',
+                    'duration': 182,
+                    'title': 'Trailer 3',
+                    'upload_date': '20130417',
+                    'uploader_id': 'wb',
                 },
             },
             {
-                "md5": "d0f1e1150989b9924679b441f3404d48",
-                "info_dict": {
-                    "id": "manofsteel-trailer",
-                    "ext": "mov",
-                    "duration": 148,
-                    "title": "Trailer",
-                    "upload_date": "20121212",
-                    "uploader_id": "wb",
+                'md5': 'd0f1e1150989b9924679b441f3404d48',
+                'info_dict': {
+                    'id': 'manofsteel-trailer',
+                    'ext': 'mov',
+                    'duration': 148,
+                    'title': 'Trailer',
+                    'upload_date': '20121212',
+                    'uploader_id': 'wb',
                 },
             },
             {
-                "md5": "5fe08795b943eb2e757fa95cb6def1cb",
-                "info_dict": {
-                    "id": "manofsteel-teaser",
-                    "ext": "mov",
-                    "duration": 93,
-                    "title": "Teaser",
-                    "upload_date": "20120721",
-                    "uploader_id": "wb",
+                'md5': '5fe08795b943eb2e757fa95cb6def1cb',
+                'info_dict': {
+                    'id': 'manofsteel-teaser',
+                    'ext': 'mov',
+                    'duration': 93,
+                    'title': 'Teaser',
+                    'upload_date': '20120721',
+                    'uploader_id': 'wb',
                 },
             },
         ]

From 2a27e66234ec0030d2ca6f1ddf229db7f7eb8ded Mon Sep 17 00:00:00 2001
From: Oli Allen <oli@oliallen.com>
Date: Thu, 1 Oct 2015 22:50:12 +0100
Subject: [PATCH 049/150] [tumblr] Added support for HD video where available
 (#7036)

[tumblr] Replaced test URL for HD video as old one lead to 404

[tumblr] Don't make assumptions about video resolution, cleaner handling of no HD version available

[tumblr] Removed extraneous resolution key in HD video tests
---
 youtube_dl/extractor/tumblr.py | 58 ++++++++++++++++++++++++++++++++--
 1 file changed, 55 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 3d3b635e4..71cced562 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -28,6 +28,32 @@ class TumblrIE(InfoExtractor):
             'description': 'md5:dba62ac8639482759c8eb10ce474586a',
             'thumbnail': 're:http://.*\.jpg',
         }
+    }, {
+        'url': 'http://hdvideotest.tumblr.com/post/130323439814/test-description-for-my-hd-video',
+        'md5': '99a84522f60972bf064a0b80f87bcbb5',
+        'info_dict': {
+            'id': '130323439814',
+            'ext': 'mp4',
+            'title': 'HD Video Testing \u2014 Test description for my HD video',
+            'description': 'md5:97cc3ab5fcd27ee4af6356701541319c',
+            'thumbnail': 're:http://.*\.jpg',
+        },
+        'params': {
+            'format': 'sd',
+        },
+    }, {
+        'url': 'http://hdvideotest.tumblr.com/post/130323439814/test-description-for-my-hd-video',
+        'md5': '7ae503065ad150122dc3089f8cf1546c',
+        'info_dict': {
+            'id': '130323439814',
+            'ext': 'mp4',
+            'title': 'HD Video Testing \u2014 Test description for my HD video',
+            'description': 'md5:97cc3ab5fcd27ee4af6356701541319c',
+            'thumbnail': 're:http://.*\.jpg',
+        },
+        'params': {
+            'format': 'hd',
+        },
     }, {
         'url': 'http://naked-yogi.tumblr.com/post/118312946248/naked-smoking-stretching',
         'md5': 'de07e5211d60d4f3a2c3df757ea9f6ab',
@@ -57,6 +83,8 @@ class TumblrIE(InfoExtractor):
         video_id = m_url.group('id')
         blog = m_url.group('blog_name')
 
+        video_urls = []
+
         url = 'http://%s.tumblr.com/post/%s/' % (blog, video_id)
         webpage, urlh = self._download_webpage_handle(url, video_id)
 
@@ -68,8 +96,32 @@ class TumblrIE(InfoExtractor):
 
         iframe = self._download_webpage(iframe_url, video_id,
                                         'Downloading iframe page')
-        video_url = self._search_regex(r'<source src="([^"]+)"',
-                                       iframe, 'video url')
+
+        sd_video_url = self._search_regex(r'<source src="([^"]+)"',
+                                          iframe, 'sd video url')
+        resolution_id = sd_video_url.split("/")[-1] + 'p'
+        if len(resolution_id) != 4:
+            resolution_id = None
+        video_urls.append({
+            'ext': 'mp4',
+            'format_id': 'sd',
+            'url': sd_video_url,
+            'resolution': resolution_id,
+        })
+
+        hd_video_url = self._search_regex(r'hdUrl":"([^"]+)"', iframe,
+                                          'hd video url', default=None)
+        if hd_video_url:
+            hd_video_url = hd_video_url.replace("\\", "")
+            resolution_id = hd_video_url.split("/")[-1] + 'p'
+            if len(resolution_id) != 4:
+                resolution_id = None
+            video_urls.append({
+                'ext': 'mp4',
+                'format_id': 'hd',
+                'url': hd_video_url,
+                'resolution': resolution_id,
+            })
 
         # The only place where you can get a title, it's not complete,
         # but searching in other places doesn't work for all videos
@@ -79,7 +131,7 @@ class TumblrIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'url': video_url,
+            'formats': video_urls,
             'ext': 'mp4',
             'title': video_title,
             'description': self._og_search_description(webpage, default=None),

From 140ac7396542d92a8ddd53be6c35c7a79db16180 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 21:53:01 +0600
Subject: [PATCH 050/150] [tumblr] Simplify and extract duration

---
 youtube_dl/extractor/tumblr.py | 60 ++++++++++++++++++----------------
 1 file changed, 32 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 71cced562..449ba29fa 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import int_or_none
 
 
 class TumblrIE(InfoExtractor):
@@ -83,8 +84,6 @@ class TumblrIE(InfoExtractor):
         video_id = m_url.group('id')
         blog = m_url.group('blog_name')
 
-        video_urls = []
-
         url = 'http://%s.tumblr.com/post/%s/' % (blog, video_id)
         webpage, urlh = self._download_webpage_handle(url, video_id)
 
@@ -94,34 +93,38 @@ class TumblrIE(InfoExtractor):
         if iframe_url is None:
             return self.url_result(urlh.geturl(), 'Generic')
 
-        iframe = self._download_webpage(iframe_url, video_id,
-                                        'Downloading iframe page')
+        iframe = self._download_webpage(iframe_url, video_id, 'Downloading iframe page')
 
-        sd_video_url = self._search_regex(r'<source src="([^"]+)"',
-                                          iframe, 'sd video url')
-        resolution_id = sd_video_url.split("/")[-1] + 'p'
-        if len(resolution_id) != 4:
-            resolution_id = None
-        video_urls.append({
+        duration = None
+        sources = []
+
+        sd_url = self._search_regex(
+            r'<source[^>]+src=(["\'])(?P<url>.+?)\1', iframe,
+            'sd video url', default=None, group='url')
+        if sd_url:
+            sources.append((sd_url, 'sd'))
+
+        options = self._parse_json(
+            self._search_regex(
+                r'data-crt-options=(["\'])(?P<options>.+?)\1', iframe,
+                'hd video url', default='', group='options'),
+            video_id, fatal=False)
+        if options:
+            duration = int_or_none(options.get('duration'))
+            hd_url = options.get('hdUrl')
+            if hd_url:
+                sources.append((hd_url, 'hd'))
+
+        formats = [{
+            'url': video_url,
             'ext': 'mp4',
-            'format_id': 'sd',
-            'url': sd_video_url,
-            'resolution': resolution_id,
-        })
+            'format_id': format_id,
+            'height': int_or_none(self._search_regex(
+                r'/(\d{3,4})$', video_url, 'height', default=None)),
+            'quality': quality,
+        } for quality, (video_url, format_id) in enumerate(sources)]
 
-        hd_video_url = self._search_regex(r'hdUrl":"([^"]+)"', iframe,
-                                          'hd video url', default=None)
-        if hd_video_url:
-            hd_video_url = hd_video_url.replace("\\", "")
-            resolution_id = hd_video_url.split("/")[-1] + 'p'
-            if len(resolution_id) != 4:
-                resolution_id = None
-            video_urls.append({
-                'ext': 'mp4',
-                'format_id': 'hd',
-                'url': hd_video_url,
-                'resolution': resolution_id,
-            })
+        self._sort_formats(formats)
 
         # The only place where you can get a title, it's not complete,
         # but searching in other places doesn't work for all videos
@@ -131,9 +134,10 @@ class TumblrIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'formats': video_urls,
             'ext': 'mp4',
             'title': video_title,
             'description': self._og_search_description(webpage, default=None),
             'thumbnail': self._og_search_thumbnail(webpage, default=None),
+            'duration': duration,
+            'formats': formats,
         }

From 5d84b79a3002925084690e17669738bf1ea711f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 21:53:59 +0600
Subject: [PATCH 051/150] [tumblr] Remove redundant test

---
 youtube_dl/extractor/tumblr.py | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 449ba29fa..9f270318b 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -29,19 +29,6 @@ class TumblrIE(InfoExtractor):
             'description': 'md5:dba62ac8639482759c8eb10ce474586a',
             'thumbnail': 're:http://.*\.jpg',
         }
-    }, {
-        'url': 'http://hdvideotest.tumblr.com/post/130323439814/test-description-for-my-hd-video',
-        'md5': '99a84522f60972bf064a0b80f87bcbb5',
-        'info_dict': {
-            'id': '130323439814',
-            'ext': 'mp4',
-            'title': 'HD Video Testing \u2014 Test description for my HD video',
-            'description': 'md5:97cc3ab5fcd27ee4af6356701541319c',
-            'thumbnail': 're:http://.*\.jpg',
-        },
-        'params': {
-            'format': 'sd',
-        },
     }, {
         'url': 'http://hdvideotest.tumblr.com/post/130323439814/test-description-for-my-hd-video',
         'md5': '7ae503065ad150122dc3089f8cf1546c',

From 88c86d211bd2f542625d3b91ac4b15e62995345d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 21:54:54 +0600
Subject: [PATCH 052/150] [tumblr] Add missing fields for vidme test

---
 youtube_dl/extractor/tumblr.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 9f270318b..cb91d08eb 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -51,6 +51,9 @@ class TumblrIE(InfoExtractor):
             'title': 'naked smoking & stretching',
             'upload_date': '20150506',
             'timestamp': 1430931613,
+            'age_limit': 18,
+            'uploader_id': '1638622',
+            'uploader': 'naked-yogi',
         },
         'add_ie': ['Vidme'],
     }, {

From 7fd4ed9939d7c467c61663cad7189ee8ad27c89b Mon Sep 17 00:00:00 2001
From: David Rabinowitz <drabinowitz@agtinternational.com>
Date: Thu, 1 Oct 2015 12:07:27 +0300
Subject: [PATCH 053/150] Fixed the ustream extractor to use the current
 ustream API

---
 youtube_dl/extractor/ustream.py | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index c39c278ab..18add908e 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -52,17 +52,12 @@ class UstreamIE(InfoExtractor):
             desktop_url = 'http://www.ustream.tv/recorded/' + desktop_video_id
             return self.url_result(desktop_url, 'Ustream')
 
-        params = self._download_json(
-            'http://cdngw.ustream.tv/rgwjson/Viewer.getVideo/' + json.dumps({
-                'brandId': 1,
-                'videoId': int(video_id),
-                'autoplay': False,
-            }), video_id)
+        params = self._download_json('https://api.ustream.tv/videos/' + video_id + '.json', video_id)
 
         if 'error' in params:
             raise ExtractorError(params['error']['message'], expected=True)
 
-        video_url = params['flv']
+        video_url = params['video']['media_urls']['flv']
 
         webpage = self._download_webpage(url, video_id)
 

From 5820c4a29ea3781c47f6249c179a081e28eeeb64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 22:30:38 +0600
Subject: [PATCH 054/150] [ustream] Switch extraction to api

---
 youtube_dl/extractor/ustream.py | 69 ++++++++++++++++++---------------
 1 file changed, 37 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 18add908e..3065c9f31 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -7,7 +7,11 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urlparse,
 )
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    float_or_none,
+)
 
 
 class UstreamIE(InfoExtractor):
@@ -54,46 +58,47 @@ class UstreamIE(InfoExtractor):
 
         params = self._download_json('https://api.ustream.tv/videos/' + video_id + '.json', video_id)
 
-        if 'error' in params:
-            raise ExtractorError(params['error']['message'], expected=True)
+        error = params.get('error')
+        if error:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error), expected=True)
 
-        video_url = params['video']['media_urls']['flv']
+        video = params['video']
 
-        webpage = self._download_webpage(url, video_id)
+        formats = [{
+            'id': format_id,
+            'url': video_url,
+            'ext': format_id,
+        } for format_id, video_url in video['media_urls'].items()]
+        self._sort_formats(formats)
 
-        self.report_extraction(video_id)
+        title = video['title']
+        description = video.get('description')
+        timestamp = int_or_none(video.get('created_at'))
+        duration = float_or_none(video.get('length'))
+        filesize = float_or_none(video.get('file_size'))
+        view_count = int_or_none(video.get('views'))
 
-        video_title = self._html_search_regex(r'data-title="(?P<title>.+)"',
-                                              webpage, 'title', default=None)
+        uploader = video.get('owner', {}).get('username')
+        uploader_id = video.get('owner', {}).get('id')
 
-        if not video_title:
-            try:
-                video_title = params['moduleConfig']['meta']['title']
-            except KeyError:
-                pass
-
-        if not video_title:
-            video_title = 'Ustream video ' + video_id
-
-        uploader = self._html_search_regex(r'data-content-type="channel".*?>(?P<uploader>.*?)</a>',
-                                           webpage, 'uploader', fatal=False, flags=re.DOTALL, default=None)
-
-        if not uploader:
-            try:
-                uploader = params['moduleConfig']['meta']['userName']
-            except KeyError:
-                uploader = None
-
-        thumbnail = self._html_search_regex(r'<link rel="image_src" href="(?P<thumb>.*?)"',
-                                            webpage, 'thumbnail', fatal=False)
+        thumbnails = [{
+            'id': thumbnail_id,
+            'url': thumbnail_url,
+        } for thumbnail_id, thumbnail_url in video.get('thumbnail', {}).items()]
 
         return {
             'id': video_id,
-            'url': video_url,
-            'ext': 'flv',
-            'title': video_title,
+            'title': title,
+            'description': description,
+            'thumbnails': thumbnails,
+            'timestamp': timestamp,
+            'duration': duration,
+            'filesize': filesize,
+            'view_count': view_count,
             'uploader': uploader,
-            'thumbnail': thumbnail,
+            'uploader_id': uploader_id,
+            'formats': formats,
         }
 
 

From 4853eb63fec766c7154e9353e91e531a3f3b1d1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 22:40:20 +0600
Subject: [PATCH 055/150] [ustream] Modernize

---
 youtube_dl/extractor/ustream.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 3065c9f31..9fdeb064d 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class UstreamIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.ustream\.tv/(?P<type>recorded|embed|embed/recorded)/(?P<videoID>\d+)'
+    _VALID_URL = r'https?://www\.ustream\.tv/(?P<type>recorded|embed|embed/recorded)/(?P<id>\d+)'
     IE_NAME = 'ustream'
     _TESTS = [{
         'url': 'http://www.ustream.tv/recorded/20274954',
@@ -41,22 +41,23 @@ class UstreamIE(InfoExtractor):
 
     def _real_extract(self, url):
         m = re.match(self._VALID_URL, url)
-        video_id = m.group('videoID')
+        video_id = m.group('id')
 
         # some sites use this embed format (see: http://github.com/rg3/youtube-dl/issues/2990)
         if m.group('type') == 'embed/recorded':
-            video_id = m.group('videoID')
+            video_id = m.group('id')
             desktop_url = 'http://www.ustream.tv/recorded/' + video_id
             return self.url_result(desktop_url, 'Ustream')
         if m.group('type') == 'embed':
-            video_id = m.group('videoID')
+            video_id = m.group('id')
             webpage = self._download_webpage(url, video_id)
             desktop_video_id = self._html_search_regex(
                 r'ContentVideoIds=\["([^"]*?)"\]', webpage, 'desktop_video_id')
             desktop_url = 'http://www.ustream.tv/recorded/' + desktop_video_id
             return self.url_result(desktop_url, 'Ustream')
 
-        params = self._download_json('https://api.ustream.tv/videos/' + video_id + '.json', video_id)
+        params = self._download_json(
+            'https://api.ustream.tv/videos/%s.json' % video_id, video_id)
 
         error = params.get('error')
         if error:

From f2a7ed77ef76b8d564326f564afc880473fba7ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 22:44:36 +0600
Subject: [PATCH 056/150] [tumblr] Remove redundant field

---
 youtube_dl/extractor/tumblr.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index cb91d08eb..4f844706d 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -124,7 +124,6 @@ class TumblrIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'ext': 'mp4',
             'title': video_title,
             'description': self._og_search_description(webpage, default=None),
             'thumbnail': self._og_search_thumbnail(webpage, default=None),

From 0bf219889e39d4d7e75fdb59d7452b8e09f4ddab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 22:44:59 +0600
Subject: [PATCH 057/150] [ustream] Remove unused import

---
 youtube_dl/extractor/ustream.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 9fdeb064d..7243d0eca 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import json
 import re
 
 from .common import InfoExtractor

From 41db73330835d6e8fe29ce18f869247a739467ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 22:48:47 +0600
Subject: [PATCH 058/150] [ustream] Move filesize

---
 youtube_dl/extractor/ustream.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 7243d0eca..888f39f7a 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -65,18 +65,20 @@ class UstreamIE(InfoExtractor):
 
         video = params['video']
 
+        title = video['title']
+        filesize = float_or_none(video.get('file_size'))
+
         formats = [{
             'id': format_id,
             'url': video_url,
             'ext': format_id,
+            'filesize': filesize,
         } for format_id, video_url in video['media_urls'].items()]
         self._sort_formats(formats)
 
-        title = video['title']
         description = video.get('description')
         timestamp = int_or_none(video.get('created_at'))
         duration = float_or_none(video.get('length'))
-        filesize = float_or_none(video.get('file_size'))
         view_count = int_or_none(video.get('views'))
 
         uploader = video.get('owner', {}).get('username')
@@ -94,7 +96,6 @@ class UstreamIE(InfoExtractor):
             'thumbnails': thumbnails,
             'timestamp': timestamp,
             'duration': duration,
-            'filesize': filesize,
             'view_count': view_count,
             'uploader': uploader,
             'uploader_id': uploader_id,

From dc5756fd7729f616185508fba296b330f0892c24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 22:51:04 +0600
Subject: [PATCH 059/150] [ustream] Fix typo

---
 youtube_dl/extractor/ustream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 888f39f7a..a29d67e9f 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -69,7 +69,7 @@ class UstreamIE(InfoExtractor):
         filesize = float_or_none(video.get('file_size'))
 
         formats = [{
-            'id': format_id,
+            'id': video_id,
             'url': video_url,
             'ext': format_id,
             'filesize': filesize,

From 40fbb05e1c58625349160ec134343af789b803ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 22:52:51 +0600
Subject: [PATCH 060/150] [ustream] Fix tests

---
 youtube_dl/extractor/ustream.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index a29d67e9f..73b05ecab 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -22,8 +22,12 @@ class UstreamIE(InfoExtractor):
         'info_dict': {
             'id': '20274954',
             'ext': 'flv',
-            'uploader': 'Young Americans for Liberty',
             'title': 'Young Americans for Liberty February 7, 2012 2:28 AM',
+            'description': 'Young Americans for Liberty February 7, 2012 2:28 AM',
+            'timestamp': 1328577035,
+            'upload_date': '20120207',
+            'uploader': 'yaliberty',
+            'uploader_id': '6780869',
         },
     }, {
         # From http://sportscanada.tv/canadagames/index.php/week2/figure-skating/444
@@ -35,7 +39,8 @@ class UstreamIE(InfoExtractor):
             'ext': 'flv',
             'title': '-CG11- Canada Games Figure Skating',
             'uploader': 'sportscanadatv',
-        }
+        },
+        'skip': 'This Pro Broadcaster has chosen to remove this video from the ustream.tv site.',
     }]
 
     def _real_extract(self, url):

From fcc25462693fb0468d66b28664ba622f8f3bdb3d Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 6 Oct 2015 02:30:05 +0600
Subject: [PATCH 061/150] [README.md] Markdown improvements

---
 README.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 80152071d..e7004c9fc 100644
--- a/README.md
+++ b/README.md
@@ -359,7 +359,7 @@ If you have installed youtube-dl with a package manager, pip, setup.py or a tarb
 
 By default, youtube-dl intends to have the best options (incidentally, if you have a convincing case that these should be different, [please file an issue where you explain that](https://yt-dl.org/bug)). Therefore, it is unnecessary and sometimes harmful to copy long option strings from webpages. In particular, the only option out of `-citw` that is regularly useful is `-i`.
 
-### Can you please put the -b option back?
+### Can you please put the `-b` option back?
 
 Most people asking this question are not aware that youtube-dl now defaults to downloading the highest available quality as reported by YouTube, which will be 1080p or 720p in some cases, so you no longer need the `-b` option. For some specific videos, maybe YouTube does not report them to be available in a specific high quality format you're interested in. In that case, simply request it with the `-f` option and youtube-dl will try to download it.
 
@@ -371,13 +371,13 @@ Apparently YouTube requires you to pass a CAPTCHA test if you download too much.
 
 Once the video is fully downloaded, use any video player, such as [vlc](http://www.videolan.org) or [mplayer](http://www.mplayerhq.hu/).
 
-### I extracted a video URL with -g, but it does not play on another machine / in my webbrowser.
+### I extracted a video URL with `-g`, but it does not play on another machine / in my webbrowser.
 
 It depends a lot on the service. In many cases, requests for the video (to download/play it) must come from the same IP address and with the same cookies.  Use the `--cookies` option to write the required cookies into a file, and advise your downloader to read cookies from that file. Some sites also require a common user agent to be used, use `--dump-user-agent` to see the one in use by youtube-dl.
 
 It may be beneficial to use IPv6; in some cases, the restrictions are only applied to IPv4. Some services (sometimes only for a subset of videos) do not restrict the video URL by IP address, cookie, or user-agent, but these are the exception rather than the rule.
 
-Please bear in mind that some URL protocols are **not** supported by browsers out of the box, including RTMP. If you are using -g, your own downloader must support these as well.
+Please bear in mind that some URL protocols are **not** supported by browsers out of the box, including RTMP. If you are using `-g`, your own downloader must support these as well.
 
 If you want to play the video on a machine that is not running youtube-dl, you can relay the video content from the machine that runs youtube-dl. You can use `-o -` to let youtube-dl stream a video to stdout, or simply allow the player to download the files written by youtube-dl in turn.
 
@@ -643,15 +643,15 @@ So please elaborate on what feature you are requesting, or what bug you want to
 
 If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a commiter myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over.
 
-For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the -v flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
+For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the `-v` flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
 
-If your server has multiple IPs or you suspect censorship, adding --call-home may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
+If your server has multiple IPs or you suspect censorship, adding `--call-home` may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
 
 **Site support requests must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL.
 
 ###  Are you using the latest version?
 
-Before reporting any issue, type youtube-dl -U. This should report that you're up-to-date. About 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well.
+Before reporting any issue, type `youtube-dl -U`. This should report that you're up-to-date. About 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well.
 
 ###  Is the issue already documented?
 

From c4af7684d85a17441c5f6f0b6e9f8c470644fec0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 6 Oct 2015 09:08:10 +0200
Subject: [PATCH 062/150] release 2015.10.06

---
 CONTRIBUTING.md        |   6 +-
 README.md              | 395 +++++++++++++++++++++++++++++------------
 docs/supportedsites.md |   4 +
 youtube_dl/version.py  |   2 +-
 4 files changed, 289 insertions(+), 118 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index f8ab29631..57a94231d 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -16,15 +16,15 @@ So please elaborate on what feature you are requesting, or what bug you want to
 
 If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a commiter myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over.
 
-For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the -v flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
+For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the `-v` flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
 
-If your server has multiple IPs or you suspect censorship, adding --call-home may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
+If your server has multiple IPs or you suspect censorship, adding `--call-home` may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
 
 **Site support requests must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL.
 
 ###  Are you using the latest version?
 
-Before reporting any issue, type youtube-dl -U. This should report that you're up-to-date. About 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well.
+Before reporting any issue, type `youtube-dl -U`. This should report that you're up-to-date. About 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well.
 
 ###  Is the issue already documented?
 
diff --git a/README.md b/README.md
index e7004c9fc..5ff8ca85e 100644
--- a/README.md
+++ b/README.md
@@ -49,110 +49,220 @@ which means you can modify it, redistribute it or use it however you like.
 # OPTIONS
     -h, --help                       Print this help text and exit
     --version                        Print program version and exit
-    -U, --update                     Update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)
-    -i, --ignore-errors              Continue on download errors, for example to skip unavailable videos in a playlist
-    --abort-on-error                 Abort downloading of further videos (in the playlist or the command line) if an error occurs
+    -U, --update                     Update this program to latest version. Make
+                                     sure that you have sufficient permissions
+                                     (run with sudo if needed)
+    -i, --ignore-errors              Continue on download errors, for example to
+                                     skip unavailable videos in a playlist
+    --abort-on-error                 Abort downloading of further videos (in the
+                                     playlist or the command line) if an error
+                                     occurs
     --dump-user-agent                Display the current browser identification
     --list-extractors                List all supported extractors
-    --extractor-descriptions         Output descriptions of all supported extractors
-    --force-generic-extractor        Force extraction to use the generic extractor
-    --default-search PREFIX          Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple".
-                                     Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The
-                                     default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.
-    --ignore-config                  Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: Do not read the user configuration
-                                     in ~/.config/youtube-dl/config (%APPDATA%/youtube-dl/config.txt on Windows)
-    --flat-playlist                  Do not extract the videos of a playlist, only list them.
+    --extractor-descriptions         Output descriptions of all supported
+                                     extractors
+    --force-generic-extractor        Force extraction to use the generic
+                                     extractor
+    --default-search PREFIX          Use this prefix for unqualified URLs. For
+                                     example "gvsearch2:" downloads two videos
+                                     from google videos for youtube-dl "large
+                                     apple". Use the value "auto" to let
+                                     youtube-dl guess ("auto_warning" to emit a
+                                     warning when guessing). "error" just throws
+                                     an error. The default value "fixup_error"
+                                     repairs broken URLs, but emits an error if
+                                     this is not possible instead of searching.
+    --ignore-config                  Do not read configuration files. When given
+                                     in the global configuration file /etc
+                                     /youtube-dl.conf: Do not read the user
+                                     configuration in ~/.config/youtube-
+                                     dl/config (%APPDATA%/youtube-dl/config.txt
+                                     on Windows)
+    --flat-playlist                  Do not extract the videos of a playlist,
+                                     only list them.
     --no-color                       Do not emit color codes in output
 
 ## Network Options:
-    --proxy URL                      Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection
+    --proxy URL                      Use the specified HTTP/HTTPS proxy. Pass in
+                                     an empty string (--proxy "") for direct
+                                     connection
     --socket-timeout SECONDS         Time to wait before giving up, in seconds
-    --source-address IP              Client-side IP address to bind to (experimental)
-    -4, --force-ipv4                 Make all connections via IPv4 (experimental)
-    -6, --force-ipv6                 Make all connections via IPv6 (experimental)
-    --cn-verification-proxy URL      Use this proxy to verify the IP address for some Chinese sites. The default proxy specified by --proxy (or none, if the options is
-                                     not present) is used for the actual downloading. (experimental)
+    --source-address IP              Client-side IP address to bind to
+                                     (experimental)
+    -4, --force-ipv4                 Make all connections via IPv4
+                                     (experimental)
+    -6, --force-ipv6                 Make all connections via IPv6
+                                     (experimental)
+    --cn-verification-proxy URL      Use this proxy to verify the IP address for
+                                     some Chinese sites. The default proxy
+                                     specified by --proxy (or none, if the
+                                     options is not present) is used for the
+                                     actual downloading. (experimental)
 
 ## Video Selection:
     --playlist-start NUMBER          Playlist video to start at (default is 1)
     --playlist-end NUMBER            Playlist video to end at (default is last)
-    --playlist-items ITEM_SPEC       Playlist video items to download. Specify indices of the videos in the playlist separated by commas like: "--playlist-items 1,2,5,8"
-                                     if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will
-                                     download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.
-    --match-title REGEX              Download only matching titles (regex or caseless sub-string)
-    --reject-title REGEX             Skip download for matching titles (regex or caseless sub-string)
+    --playlist-items ITEM_SPEC       Playlist video items to download. Specify
+                                     indices of the videos in the playlist
+                                     separated by commas like: "--playlist-items
+                                     1,2,5,8" if you want to download videos
+                                     indexed 1, 2, 5, 8 in the playlist. You can
+                                     specify range: "--playlist-items
+                                     1-3,7,10-13", it will download the videos
+                                     at index 1, 2, 3, 7, 10, 11, 12 and 13.
+    --match-title REGEX              Download only matching titles (regex or
+                                     caseless sub-string)
+    --reject-title REGEX             Skip download for matching titles (regex or
+                                     caseless sub-string)
     --max-downloads NUMBER           Abort after downloading NUMBER files
-    --min-filesize SIZE              Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)
-    --max-filesize SIZE              Do not download any videos larger than SIZE (e.g. 50k or 44.6m)
+    --min-filesize SIZE              Do not download any videos smaller than
+                                     SIZE (e.g. 50k or 44.6m)
+    --max-filesize SIZE              Do not download any videos larger than SIZE
+                                     (e.g. 50k or 44.6m)
     --date DATE                      Download only videos uploaded in this date
-    --datebefore DATE                Download only videos uploaded on or before this date (i.e. inclusive)
-    --dateafter DATE                 Download only videos uploaded on or after this date (i.e. inclusive)
-    --min-views COUNT                Do not download any videos with less than COUNT views
-    --max-views COUNT                Do not download any videos with more than COUNT views
-    --match-filter FILTER            Generic video filter (experimental). Specify any key (see help for -o for a list of available keys) to match if the key is present,
-                                     !key to check if the key is not present,key > NUMBER (like "comment_count > 12", also works with >=, <, <=, !=, =) to compare against
-                                     a number, and & to require multiple matches. Values which are not known are excluded unless you put a question mark (?) after the
-                                     operator.For example, to only match videos that have been liked more than 100 times and disliked less than 50 times (or the dislike
-                                     functionality is not available at the given service), but who also have a description, use  --match-filter "like_count > 100 &
+    --datebefore DATE                Download only videos uploaded on or before
+                                     this date (i.e. inclusive)
+    --dateafter DATE                 Download only videos uploaded on or after
+                                     this date (i.e. inclusive)
+    --min-views COUNT                Do not download any videos with less than
+                                     COUNT views
+    --max-views COUNT                Do not download any videos with more than
+                                     COUNT views
+    --match-filter FILTER            Generic video filter (experimental).
+                                     Specify any key (see help for -o for a list
+                                     of available keys) to match if the key is
+                                     present, !key to check if the key is not
+                                     present,key > NUMBER (like "comment_count >
+                                     12", also works with >=, <, <=, !=, =) to
+                                     compare against a number, and & to require
+                                     multiple matches. Values which are not
+                                     known are excluded unless you put a
+                                     question mark (?) after the operator.For
+                                     example, to only match videos that have
+                                     been liked more than 100 times and disliked
+                                     less than 50 times (or the dislike
+                                     functionality is not available at the given
+                                     service), but who also have a description,
+                                     use --match-filter "like_count > 100 &
                                      dislike_count <? 50 & description" .
-    --no-playlist                    Download only the video, if the URL refers to a video and a playlist.
-    --yes-playlist                   Download the playlist, if the URL refers to a video and a playlist.
-    --age-limit YEARS                Download only videos suitable for the given age
-    --download-archive FILE          Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.
-    --include-ads                    Download advertisements as well (experimental)
+    --no-playlist                    Download only the video, if the URL refers
+                                     to a video and a playlist.
+    --yes-playlist                   Download the playlist, if the URL refers to
+                                     a video and a playlist.
+    --age-limit YEARS                Download only videos suitable for the given
+                                     age
+    --download-archive FILE          Download only videos not listed in the
+                                     archive file. Record the IDs of all
+                                     downloaded videos in it.
+    --include-ads                    Download advertisements as well
+                                     (experimental)
 
 ## Download Options:
-    -r, --rate-limit LIMIT           Maximum download rate in bytes per second (e.g. 50K or 4.2M)
-    -R, --retries RETRIES            Number of retries (default is 10), or "infinite".
-    --buffer-size SIZE               Size of download buffer (e.g. 1024 or 16K) (default is 1024)
-    --no-resize-buffer               Do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.
+    -r, --rate-limit LIMIT           Maximum download rate in bytes per second
+                                     (e.g. 50K or 4.2M)
+    -R, --retries RETRIES            Number of retries (default is 10), or
+                                     "infinite".
+    --buffer-size SIZE               Size of download buffer (e.g. 1024 or 16K)
+                                     (default is 1024)
+    --no-resize-buffer               Do not automatically adjust the buffer
+                                     size. By default, the buffer size is
+                                     automatically resized from an initial value
+                                     of SIZE.
     --playlist-reverse               Download playlist videos in reverse order
-    --xattr-set-filesize             Set file xattribute ytdl.filesize with expected filesize (experimental)
-    --hls-prefer-native              Use the native HLS downloader instead of ffmpeg (experimental)
-    --external-downloader COMMAND    Use the specified external downloader. Currently supports aria2c,axel,curl,httpie,wget
-    --external-downloader-args ARGS  Give these arguments to the external downloader
+    --xattr-set-filesize             Set file xattribute ytdl.filesize with
+                                     expected filesize (experimental)
+    --hls-prefer-native              Use the native HLS downloader instead of
+                                     ffmpeg (experimental)
+    --external-downloader COMMAND    Use the specified external downloader.
+                                     Currently supports
+                                     aria2c,axel,curl,httpie,wget
+    --external-downloader-args ARGS  Give these arguments to the external
+                                     downloader
 
 ## Filesystem Options:
-    -a, --batch-file FILE            File containing URLs to download ('-' for stdin)
+    -a, --batch-file FILE            File containing URLs to download ('-' for
+                                     stdin)
     --id                             Use only video ID in file name
-    -o, --output TEMPLATE            Output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(uploader_id)s for the uploader
-                                     nickname if different, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(format)s for
-                                     the format description (like "22 - 1280x720" or "HD"), %(format_id)s for the unique id of the format (like YouTube's itags: "137"),
-                                     %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id,
-                                     %(playlist_title)s, %(playlist_id)s, or %(playlist)s (=title if present, ID otherwise) for the playlist the video is in,
-                                     %(playlist_index)s for the position in the playlist. %(height)s and %(width)s for the width and height of the video format.
-                                     %(resolution)s for a textual description of the resolution of the video format. %% for a literal percent. Use - to output to stdout.
-                                     Can also be used to download to a different directory, for example with -o '/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s' .
-    --autonumber-size NUMBER         Specify the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given
-    --restrict-filenames             Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames
-    -A, --auto-number                [deprecated; use  -o "%(autonumber)s-%(title)s.%(ext)s" ] Number downloaded files starting from 00000
-    -t, --title                      [deprecated] Use title in file name (default)
+    -o, --output TEMPLATE            Output filename template. Use %(title)s to
+                                     get the title, %(uploader)s for the
+                                     uploader name, %(uploader_id)s for the
+                                     uploader nickname if different,
+                                     %(autonumber)s to get an automatically
+                                     incremented number, %(ext)s for the
+                                     filename extension, %(format)s for the
+                                     format description (like "22 - 1280x720" or
+                                     "HD"), %(format_id)s for the unique id of
+                                     the format (like YouTube's itags: "137"),
+                                     %(upload_date)s for the upload date
+                                     (YYYYMMDD), %(extractor)s for the provider
+                                     (youtube, metacafe, etc), %(id)s for the
+                                     video id, %(playlist_title)s,
+                                     %(playlist_id)s, or %(playlist)s (=title if
+                                     present, ID otherwise) for the playlist the
+                                     video is in, %(playlist_index)s for the
+                                     position in the playlist. %(height)s and
+                                     %(width)s for the width and height of the
+                                     video format. %(resolution)s for a textual
+                                     description of the resolution of the video
+                                     format. %% for a literal percent. Use - to
+                                     output to stdout. Can also be used to
+                                     download to a different directory, for
+                                     example with -o '/my/downloads/%(uploader)s
+                                     /%(title)s-%(id)s.%(ext)s' .
+    --autonumber-size NUMBER         Specify the number of digits in
+                                     %(autonumber)s when it is present in output
+                                     filename template or --auto-number option
+                                     is given
+    --restrict-filenames             Restrict filenames to only ASCII
+                                     characters, and avoid "&" and spaces in
+                                     filenames
+    -A, --auto-number                [deprecated; use -o
+                                     "%(autonumber)s-%(title)s.%(ext)s" ] Number
+                                     downloaded files starting from 00000
+    -t, --title                      [deprecated] Use title in file name
+                                     (default)
     -l, --literal                    [deprecated] Alias of --title
     -w, --no-overwrites              Do not overwrite files
-    -c, --continue                   Force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.
-    --no-continue                    Do not resume partially downloaded files (restart from beginning)
-    --no-part                        Do not use .part files - write directly into output file
-    --no-mtime                       Do not use the Last-modified header to set the file modification time
-    --write-description              Write video description to a .description file
+    -c, --continue                   Force resume of partially downloaded files.
+                                     By default, youtube-dl will resume
+                                     downloads if possible.
+    --no-continue                    Do not resume partially downloaded files
+                                     (restart from beginning)
+    --no-part                        Do not use .part files - write directly
+                                     into output file
+    --no-mtime                       Do not use the Last-modified header to set
+                                     the file modification time
+    --write-description              Write video description to a .description
+                                     file
     --write-info-json                Write video metadata to a .info.json file
-    --write-annotations              Write video annotations to a .annotations.xml file
-    --load-info FILE                 JSON file containing the video information (created with the "--write-info-json" option)
-    --cookies FILE                   File to read cookies from and dump cookie jar in
-    --cache-dir DIR                  Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl
-                                     or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may
-                                     change.
+    --write-annotations              Write video annotations to a
+                                     .annotations.xml file
+    --load-info FILE                 JSON file containing the video information
+                                     (created with the "--write-info-json"
+                                     option)
+    --cookies FILE                   File to read cookies from and dump cookie
+                                     jar in
+    --cache-dir DIR                  Location in the filesystem where youtube-dl
+                                     can store some downloaded information
+                                     permanently. By default $XDG_CACHE_HOME
+                                     /youtube-dl or ~/.cache/youtube-dl . At the
+                                     moment, only YouTube player files (for
+                                     videos with obfuscated signatures) are
+                                     cached, but that may change.
     --no-cache-dir                   Disable filesystem caching
     --rm-cache-dir                   Delete all filesystem cache files
 
 ## Thumbnail images:
     --write-thumbnail                Write thumbnail image to disk
     --write-all-thumbnails           Write all thumbnail image formats to disk
-    --list-thumbnails                Simulate and list all available thumbnail formats
+    --list-thumbnails                Simulate and list all available thumbnail
+                                     formats
 
 ## Verbosity / Simulation Options:
     -q, --quiet                      Activate quiet mode
     --no-warnings                    Ignore warnings
-    -s, --simulate                   Do not download the video and do not write anything to disk
+    -s, --simulate                   Do not download the video and do not write
+                                     anything to disk
     --skip-download                  Do not download the video
     -g, --get-url                    Simulate, quiet but print URL
     -e, --get-title                  Simulate, quiet but print title
@@ -162,78 +272,135 @@ which means you can modify it, redistribute it or use it however you like.
     --get-duration                   Simulate, quiet but print video length
     --get-filename                   Simulate, quiet but print output filename
     --get-format                     Simulate, quiet but print output format
-    -j, --dump-json                  Simulate, quiet but print JSON information. See --output for a description of available keys.
-    -J, --dump-single-json           Simulate, quiet but print JSON information for each command-line argument. If the URL refers to a playlist, dump the whole playlist
-                                     information in a single line.
-    --print-json                     Be quiet and print the video information as JSON (video is still being downloaded).
+    -j, --dump-json                  Simulate, quiet but print JSON information.
+                                     See --output for a description of available
+                                     keys.
+    -J, --dump-single-json           Simulate, quiet but print JSON information
+                                     for each command-line argument. If the URL
+                                     refers to a playlist, dump the whole
+                                     playlist information in a single line.
+    --print-json                     Be quiet and print the video information as
+                                     JSON (video is still being downloaded).
     --newline                        Output progress bar as new lines
     --no-progress                    Do not print progress bar
     --console-title                  Display progress in console titlebar
     -v, --verbose                    Print various debugging information
-    --dump-pages                     Print downloaded pages encoded using base64 to debug problems (very verbose)
-    --write-pages                    Write downloaded intermediary pages to files in the current directory to debug problems
+    --dump-pages                     Print downloaded pages encoded using base64
+                                     to debug problems (very verbose)
+    --write-pages                    Write downloaded intermediary pages to
+                                     files in the current directory to debug
+                                     problems
     --print-traffic                  Display sent and read HTTP traffic
     -C, --call-home                  Contact the youtube-dl server for debugging
-    --no-call-home                   Do NOT contact the youtube-dl server for debugging
+    --no-call-home                   Do NOT contact the youtube-dl server for
+                                     debugging
 
 ## Workarounds:
     --encoding ENCODING              Force the specified encoding (experimental)
     --no-check-certificate           Suppress HTTPS certificate validation
-    --prefer-insecure                Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)
+    --prefer-insecure                Use an unencrypted connection to retrieve
+                                     information about the video. (Currently
+                                     supported only for YouTube)
     --user-agent UA                  Specify a custom user agent
-    --referer URL                    Specify a custom referer, use if the video access is restricted to one domain
-    --add-header FIELD:VALUE         Specify a custom HTTP header and its value, separated by a colon ':'. You can use this option multiple times
-    --bidi-workaround                Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH
-    --sleep-interval SECONDS         Number of seconds to sleep before each download.
+    --referer URL                    Specify a custom referer, use if the video
+                                     access is restricted to one domain
+    --add-header FIELD:VALUE         Specify a custom HTTP header and its value,
+                                     separated by a colon ':'. You can use this
+                                     option multiple times
+    --bidi-workaround                Work around terminals that lack
+                                     bidirectional text support. Requires bidiv
+                                     or fribidi executable in PATH
+    --sleep-interval SECONDS         Number of seconds to sleep before each
+                                     download.
 
 ## Video Format Options:
-    -f, --format FORMAT              Video format code, see the "FORMAT SELECTION" for all the info
+    -f, --format FORMAT              Video format code, see the "FORMAT
+                                     SELECTION" for all the info
     --all-formats                    Download all available video formats
-    --prefer-free-formats            Prefer free video formats unless a specific one is requested
+    --prefer-free-formats            Prefer free video formats unless a specific
+                                     one is requested
     -F, --list-formats               List all available formats
-    --youtube-skip-dash-manifest     Do not download the DASH manifests and related data on YouTube videos
-    --merge-output-format FORMAT     If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, webm, flv. Ignored if no
-                                     merge is required
+    --youtube-skip-dash-manifest     Do not download the DASH manifests and
+                                     related data on YouTube videos
+    --merge-output-format FORMAT     If a merge is required (e.g.
+                                     bestvideo+bestaudio), output to given
+                                     container format. One of mkv, mp4, ogg,
+                                     webm, flv. Ignored if no merge is required
 
 ## Subtitle Options:
     --write-sub                      Write subtitle file
-    --write-auto-sub                 Write automatic subtitle file (YouTube only)
-    --all-subs                       Download all the available subtitles of the video
+    --write-auto-sub                 Write automatic subtitle file (YouTube
+                                     only)
+    --all-subs                       Download all the available subtitles of the
+                                     video
     --list-subs                      List all available subtitles for the video
-    --sub-format FORMAT              Subtitle format, accepts formats preference, for example: "srt" or "ass/srt/best"
-    --sub-lang LANGS                 Languages of the subtitles to download (optional) separated by commas, use IETF language tags like 'en,pt'
+    --sub-format FORMAT              Subtitle format, accepts formats
+                                     preference, for example: "srt" or
+                                     "ass/srt/best"
+    --sub-lang LANGS                 Languages of the subtitles to download
+                                     (optional) separated by commas, use IETF
+                                     language tags like 'en,pt'
 
 ## Authentication Options:
     -u, --username USERNAME          Login with this account ID
-    -p, --password PASSWORD          Account password. If this option is left out, youtube-dl will ask interactively.
+    -p, --password PASSWORD          Account password. If this option is left
+                                     out, youtube-dl will ask interactively.
     -2, --twofactor TWOFACTOR        Two-factor auth code
     -n, --netrc                      Use .netrc authentication data
     --video-password PASSWORD        Video password (vimeo, smotri, youku)
 
 ## Post-processing Options:
-    -x, --extract-audio              Convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)
-    --audio-format FORMAT            Specify audio format: "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "best" by default
-    --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default
-                                     5)
-    --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|avi)
+    -x, --extract-audio              Convert video files to audio-only files
+                                     (requires ffmpeg or avconv and ffprobe or
+                                     avprobe)
+    --audio-format FORMAT            Specify audio format: "best", "aac",
+                                     "vorbis", "mp3", "m4a", "opus", or "wav";
+                                     "best" by default
+    --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert
+                                     a value between 0 (better) and 9 (worse)
+                                     for VBR or a specific bitrate like 128K
+                                     (default 5)
+    --recode-video FORMAT            Encode the video to another format if
+                                     necessary (currently supported:
+                                     mp4|flv|ogg|webm|mkv|avi)
     --postprocessor-args ARGS        Give these arguments to the postprocessor
-    -k, --keep-video                 Keep the video file on disk after the post-processing; the video is erased by default
-    --no-post-overwrites             Do not overwrite post-processed files; the post-processed files are overwritten by default
-    --embed-subs                     Embed subtitles in the video (only for mkv and mp4 videos)
+    -k, --keep-video                 Keep the video file on disk after the post-
+                                     processing; the video is erased by default
+    --no-post-overwrites             Do not overwrite post-processed files; the
+                                     post-processed files are overwritten by
+                                     default
+    --embed-subs                     Embed subtitles in the video (only for mkv
+                                     and mp4 videos)
     --embed-thumbnail                Embed thumbnail in the audio as cover art
     --add-metadata                   Write metadata to the video file
-    --metadata-from-title FORMAT     Parse additional metadata like song title / artist from the video title. The format syntax is the same as --output, the parsed
-                                     parameters replace existing values. Additional templates: %(album)s, %(artist)s. Example: --metadata-from-title "%(artist)s -
-                                     %(title)s" matches a title like "Coldplay - Paradise"
-    --xattrs                         Write metadata to the video file's xattrs (using dublin core and xdg standards)
-    --fixup POLICY                   Automatically correct known faults of the file. One of never (do nothing), warn (only emit a warning), detect_or_warn (the default;
-                                     fix file if we can, warn otherwise)
-    --prefer-avconv                  Prefer avconv over ffmpeg for running the postprocessors (default)
-    --prefer-ffmpeg                  Prefer ffmpeg over avconv for running the postprocessors
-    --ffmpeg-location PATH           Location of the ffmpeg/avconv binary; either the path to the binary or its containing directory.
-    --exec CMD                       Execute a command on the file after downloading, similar to find's -exec syntax. Example: --exec 'adb push {} /sdcard/Music/ && rm
-                                     {}'
-    --convert-subtitles FORMAT       Convert the subtitles to other format (currently supported: srt|ass|vtt)
+    --metadata-from-title FORMAT     Parse additional metadata like song title /
+                                     artist from the video title. The format
+                                     syntax is the same as --output, the parsed
+                                     parameters replace existing values.
+                                     Additional templates: %(album)s,
+                                     %(artist)s. Example: --metadata-from-title
+                                     "%(artist)s - %(title)s" matches a title
+                                     like "Coldplay - Paradise"
+    --xattrs                         Write metadata to the video file's xattrs
+                                     (using dublin core and xdg standards)
+    --fixup POLICY                   Automatically correct known faults of the
+                                     file. One of never (do nothing), warn (only
+                                     emit a warning), detect_or_warn (the
+                                     default; fix file if we can, warn
+                                     otherwise)
+    --prefer-avconv                  Prefer avconv over ffmpeg for running the
+                                     postprocessors (default)
+    --prefer-ffmpeg                  Prefer ffmpeg over avconv for running the
+                                     postprocessors
+    --ffmpeg-location PATH           Location of the ffmpeg/avconv binary;
+                                     either the path to the binary or its
+                                     containing directory.
+    --exec CMD                       Execute a command on the file after
+                                     downloading, similar to find's -exec
+                                     syntax. Example: --exec 'adb push {}
+                                     /sdcard/Music/ && rm {}'
+    --convert-subtitles FORMAT       Convert the subtitles to other format
+                                     (currently supported: srt|ass|vtt)
 
 # CONFIGURATION
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index fa83b68ad..5beb2ecd4 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -150,6 +150,7 @@
  - **Escapist**
  - **ESPN** (Currently broken)
  - **EsriVideo**
+ - **Europa**
  - **EveryonesMixtape**
  - **exfm**: ex.fm
  - **ExpoTV**
@@ -263,6 +264,9 @@
  - **Libsyn**
  - **life:embed**
  - **lifenews**: LIFE | NEWS
+ - **limelight**
+ - **limelight:channel**
+ - **limelight:channel_list**
  - **LiveLeak**
  - **livestream**
  - **livestream:original**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8f0977849..46c3356e9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.09.28'
+__version__ = '2015.10.06'

From 4810c48d6d7e950c2cb1203f4d07bea1ba02c1e1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 6 Oct 2015 14:28:14 +0200
Subject: [PATCH 063/150] [compat] Do not compare None <= 0

The result is meaningless (and it emits a warning in cpython2 when called with -3), so handle None before making integer comparisons.
---
 youtube_dl/compat.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index c36c9c23f..1ba4ab78c 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -417,18 +417,18 @@ else:
     _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines'])
 
     def compat_get_terminal_size(fallback=(80, 24)):
-        columns = compat_getenv('COLUMNS', None)
+        columns = compat_getenv('COLUMNS')
         if columns:
             columns = int(columns)
         else:
             columns = None
-        lines = compat_getenv('LINES', None)
+        lines = compat_getenv('LINES')
         if lines:
             lines = int(lines)
         else:
             lines = None
 
-        if columns <= 0 or lines <= 0:
+        if columns is None or lines is None or columns <= 0 or lines <= 0:
             try:
                 sp = subprocess.Popen(
                     ['stty', 'size'],
@@ -438,9 +438,9 @@ else:
             except Exception:
                 _columns, _lines = _terminal_size(*fallback)
 
-            if columns <= 0:
+            if columns is None or columns <= 0:
                 columns = _columns
-            if lines <= 0:
+            if lines is None or lines <= 0:
                 lines = _lines
         return _terminal_size(columns, lines)
 

From 86be82610c35a684bee97b22c8d9a2a83bab1bba Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 6 Oct 2015 17:43:50 +0200
Subject: [PATCH 064/150] release 2015.10.06.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 46c3356e9..4fce70fa6 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.06'
+__version__ = '2015.10.06.1'

From f2dbc54066f56a98c689099b920e6a596d4ffdfc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 6 Oct 2015 22:02:28 +0600
Subject: [PATCH 065/150] [compat] Fix wrong lines/columns order

stty size is rows x columns
---
 youtube_dl/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 1ba4ab78c..192e1c515 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -434,7 +434,7 @@ else:
                     ['stty', 'size'],
                     stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                 out, err = sp.communicate()
-                _columns, _lines = map(int, out.split())
+                _lines, _columns = map(int, out.split())
             except Exception:
                 _columns, _lines = _terminal_size(*fallback)
 

From f648e682a7a82f71a278ec92159ac5d343a4b3eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 6 Oct 2015 22:58:18 +0600
Subject: [PATCH 066/150] [bandcamp] Prepend download URL with scheme when
 necessary (Closes #7077)

---
 youtube_dl/extractor/bandcamp.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 505877b77..a27f3e748 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -93,8 +93,8 @@ class BandcampIE(InfoExtractor):
         final_url_webpage = self._download_webpage(request_url, video_id, 'Requesting download url')
         # If we could correctly generate the .rand field the url would be
         # in the "download_url" key
-        final_url = self._search_regex(
-            r'"retry_url":"(.*?)"', final_url_webpage, 'final video URL')
+        final_url = self._proto_relative_url(self._search_regex(
+            r'"retry_url":"(.+?)"', final_url_webpage, 'final video URL'), 'http:')
 
         return {
             'id': video_id,

From 83a56686944225137ce646748b8e940a21584941 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 6 Oct 2015 23:08:28 +0600
Subject: [PATCH 067/150] [canalplus] Extend video id regex (Closes #7076)

---
 youtube_dl/extractor/canalplus.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 57e0cda2c..c0ca3a0fa 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -78,7 +78,8 @@ class CanalplusIE(InfoExtractor):
         if video_id is None:
             webpage = self._download_webpage(url, display_id)
             video_id = self._search_regex(
-                r'<canal:player[^>]+?videoId="(\d+)"', webpage, 'video id')
+                [r'<canal:player[^>]+?videoId=(["\'])(?P<id>\d+)', r'id=["\']canal_video_player(?P<id>\d+)'],
+                 webpage, 'video id', group='id')
 
         info_url = self._VIDEO_INFO_TEMPLATE % (site_id, video_id)
         doc = self._download_xml(info_url, video_id, 'Downloading video XML')

From fc10824cb67feb837ea57d60decc293b7b719cfb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 7 Oct 2015 02:43:12 +0600
Subject: [PATCH 068/150] [canalplus] PEP 8

---
 youtube_dl/extractor/canalplus.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index c0ca3a0fa..004372f8d 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -79,7 +79,7 @@ class CanalplusIE(InfoExtractor):
             webpage = self._download_webpage(url, display_id)
             video_id = self._search_regex(
                 [r'<canal:player[^>]+?videoId=(["\'])(?P<id>\d+)', r'id=["\']canal_video_player(?P<id>\d+)'],
-                 webpage, 'video id', group='id')
+                webpage, 'video id', group='id')
 
         info_url = self._VIDEO_INFO_TEMPLATE % (site_id, video_id)
         doc = self._download_xml(info_url, video_id, 'Downloading video XML')

From 945e5c56e3ccd45cdcc3ee45b9ffcbad7e614f90 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 6 Oct 2015 23:46:16 +0200
Subject: [PATCH 069/150] release 2015.10.06.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4fce70fa6..112c78835 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.06.1'
+__version__ = '2015.10.06.2'

From bd5376c182bd5ee2103aec21144286dbefbeb797 Mon Sep 17 00:00:00 2001
From: Tom Gijselinck <tomgijselinck@gmail.com>
Date: Wed, 7 Oct 2015 10:32:44 +0200
Subject: [PATCH 070/150] Fix typos and improve grammar and spelling

---
 README.md | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/README.md b/README.md
index 5ff8ca85e..cf4aebf3d 100644
--- a/README.md
+++ b/README.md
@@ -404,18 +404,18 @@ which means you can modify it, redistribute it or use it however you like.
 
 # CONFIGURATION
 
-You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux, system wide configuration file is located at `/etc/youtube-dl.conf` and user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. For example, with the following configration file youtube-dl will always extract the audio, not copy the mtime and use proxy:
+You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime and use a proxy:
 ```
 --extract-audio
 --no-mtime
 --proxy 127.0.0.1:3128
 ```
 
-You can use `--ignore-config` if you want to disable configuration file for a particular youtube-dl run.
+You can use `--ignore-config` if you want to disable the configuration file for a particular youtube-dl run.
 
 ### Authentication with `.netrc` file ###
 
-You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every youtube-dl execution and prevent tracking plain text passwords in shell command history. You can achieve this using [`.netrc` file](http://stackoverflow.com/tags/.netrc/info) on per extractor basis. For that you will need to create `.netrc` file in your `$HOME` and restrict permissions to read/write by you only:
+You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every youtube-dl execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](http://stackoverflow.com/tags/.netrc/info) on per extractor basis. For that you will need to create a`.netrc` file in your `$HOME` and restrict permissions to read/write by you only:
 ```
 touch $HOME/.netrc
 chmod a-rwx,u+rw $HOME/.netrc
@@ -429,13 +429,13 @@ For example:
 machine youtube login myaccount@gmail.com password my_youtube_password
 machine twitch login my_twitch_account_name password my_twitch_password
 ```
-To activate authentication with `.netrc` file you should pass `--netrc` to youtube-dl or place it in [configuration file](#configuration).
+To activate authentication with the `.netrc` file you should pass `--netrc` to youtube-dl or place it in the [configuration file](#configuration).
 
-On Windows you may also need to setup `%HOME%` environment variable manually.
+On Windows you may also need to setup the `%HOME%` environment variable manually.
 
 # OUTPUT TEMPLATE
 
-The `-o` option allows users to indicate a template for the output file names. The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "http://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences have the format `%(NAME)s`. To clarify, that is a percent symbol followed by a name in parenthesis, followed by a lowercase S. Allowed names are:
+The `-o` option allows users to indicate a template for the output file names. The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "http://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences have the format `%(NAME)s`. To clarify, that is a percent symbol followed by a name in parentheses, followed by a lowercase S. Allowed names are:
 
  - `id`: The sequence will be replaced by the video identifier.
  - `url`: The sequence will be replaced by the video URL.
@@ -463,18 +463,18 @@ youtube-dl_test_video_.mp4          # A simple file name
 
 # FORMAT SELECTION
 
-By default youtube-dl tries to download the best quality, but sometimes you may want to download other format.
+By default youtube-dl tries to download the best quality, but sometimes you may want to download in a different format.
 The simplest case is requesting a specific format, for example `-f 22`. You can get the list of available formats using `--list-formats`, you can also use a file extension (currently it supports aac, m4a, mp3, mp4, ogg, wav, webm) or the special names `best`, `bestvideo`, `bestaudio` and `worst`.
 
 If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`. Format selectors can also be grouped using parentheses, for example if you want to download the best mp4 and webm formats with a height lower than 480 you can use `-f '(mp4,webm)[height<480]'`.
 
-Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
+Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
 
-If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
+If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download the best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 
 # VIDEO SELECTION
 
-Videos can be filtered by their upload date using the options `--date`, `--datebefore` or `--dateafter`, they accept dates in two formats:
+Videos can be filtered by their upload date using the options `--date`, `--datebefore` or `--dateafter`. They accept dates in two formats:
 
  - Absolute dates: Dates in the format `YYYYMMDD`.
  - Relative dates: Dates in the format `(now|today)[+-][0-9](day|week|month|year)(s)?`
@@ -488,7 +488,7 @@ $ youtube-dl --dateafter now-6months
 # Download only the videos uploaded on January 1, 1970
 $ youtube-dl --date 19700101
 
-$ # will only download the videos uploaded in the 200x decade
+$ # Download only the videos uploaded in the 200x decade
 $ youtube-dl --dateafter 20000101 --datebefore 20091231
 ```
 
@@ -500,7 +500,7 @@ If you've followed [our manual installation instructions](http://rg3.github.io/y
 
 If you have used pip, a simple `sudo pip install -U youtube-dl` is sufficient to update.
 
-If you have installed youtube-dl using a package manager like *apt-get* or *yum*, use the standard system update mechanism to update. Note that distribution packages are often outdated. As a rule of thumb, youtube-dl releases at least once a month, and often weekly or even daily. Simply go to http://yt-dl.org/ to find out the current version. Unfortunately, there is nothing we youtube-dl developers can do if your distributions serves a really outdated version. You can (and should) complain to your distribution in their bugtracker or support forum.
+If you have installed youtube-dl using a package manager like *apt-get* or *yum*, use the standard system update mechanism to update. Note that distribution packages are often outdated. As a rule of thumb, youtube-dl releases at least once a month, and often weekly or even daily. Simply go to http://yt-dl.org/ to find out the current version. Unfortunately, there is nothing we youtube-dl developers can do if your distribution serves a really outdated version. You can (and should) complain to your distribution in their bugtracker or support forum.
 
 As a last resort, you can also uninstall the version installed by your package manager and follow our manual installation instructions. For that, remove the distribution's package, with a line like
 
@@ -558,7 +558,7 @@ YouTube requires an additional signature since September 2012 which is not suppo
 
 ### Video URL contains an ampersand and I'm getting some strange output `[1] 2839` or `'v' is not recognized as an internal or external command` ###
 
-That's actually the output from your shell. Since ampersand is one of the special shell characters it's interpreted by shell preventing you from passing the whole URL to youtube-dl. To disable your shell from interpreting the ampersands (or any other special characters) you have to either put the whole URL in quotes or escape them with a backslash (which approach will work depends on your shell).
+That's actually the output from your shell. Since ampersand is one of the special shell characters it's interpreted by the shell preventing you from passing the whole URL to youtube-dl. To disable your shell from interpreting the ampersands (or any other special characters) you have to either put the whole URL in quotes or escape them with a backslash (which approach will work depends on your shell).
 
 For example if your URL is https://www.youtube.com/watch?t=4&v=BaW_jenozKc you should end up with following command:
 
@@ -618,9 +618,9 @@ Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the opt
 
 ### How do I pass cookies to youtube-dl?
 
-Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`. Note that cookies file must be in Mozilla/Netscape format and the first line of cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in cookies file and convert newlines if necessary to correspond your OS, namely `CRLF` (`\r\n`) for Windows, `LF` (`\n`) for Linux and `CR` (`\r`) for Mac OS. `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
+Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`. Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows, `LF` (`\n`) for Linux and `CR` (`\r`) for Mac OS. `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
 
-Passing cookies to youtube-dl is a good way to workaround login when particular extractor does not implement it explicitly.
+Passing cookies to youtube-dl is a good way to workaround login when a particular extractor does not implement it explicitly.
 
 ### Can you add support for this anime video site, or site which shows current movies for free?
 
@@ -720,7 +720,7 @@ If you want to add support for a new site, you can follow this quick list (assum
             }
     ```
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
-6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
+6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62-L200). Add tests and code for as many as you want.
 8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:

From db7a28eccb743db4e546a12fb78ae5dc40ef44d9 Mon Sep 17 00:00:00 2001
From: AndroKev <AndroKev@users.noreply.github.com>
Date: Thu, 8 Oct 2015 09:56:39 +0200
Subject: [PATCH 071/150] FIX: nowtv

now the download works for me
thx to http://board.gulli.com/thread/1251646-videos-von-webseiten-runterladen-sammelthread-alle-fragen-hier-rein-/?p=14980081#post14980081
---
 youtube_dl/extractor/nowtv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index c8257719f..b0bdffc4e 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -167,8 +167,8 @@ class NowTVIE(InfoExtractor):
                 'app': app,
                 'play_path': 'mp4:%s' % play_path,
                 'ext': 'flv',
-                'page_url': url,
-                'player_url': 'http://rtl-now.rtl.de/includes/nc_player.swf',
+                'page_url': 'http://rtlnow.rtl.de',
+                'player_url': 'http://cdn.static-fra.de/now/vodplayer.swf',
                 'tbr': int_or_none(item.get('bitrate')),
             })
         self._sort_formats(formats)

From b90b0c4ffa6f84a2ea5556d4df99de5f8ef2c7dd Mon Sep 17 00:00:00 2001
From: kitty <magicvidyakitty@gmail.com>
Date: Thu, 8 Oct 2015 06:31:23 -0700
Subject: [PATCH 072/150] Fixed 4tube.com extractor to pull metadata from
 associated Javascript and not the HTML of the desired page.

---
 youtube_dl/extractor/fourtube.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index 3bb4f6239..226ee67f0 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -45,11 +45,9 @@ class FourTubeIE(InfoExtractor):
             'uploadDate', webpage))
         thumbnail = self._html_search_meta('thumbnailUrl', webpage)
         uploader_id = self._html_search_regex(
-            r'<a class="img-avatar" href="[^"]+/channels/([^/"]+)" title="Go to [^"]+ page">',
-            webpage, 'uploader id')
+            r'<a class="img-avatar" href="[^"]+/users/([^/"]+)" title="Go to [^"]+ page">', webpage, 'uploader id')
         uploader = self._html_search_regex(
-            r'<a class="img-avatar" href="[^"]+/channels/[^/"]+" title="Go to ([^"]+) page">',
-            webpage, 'uploader')
+            r'<a class="img-avatar" href="[^"]+/users/[^/"]+" title="Go to ([^"]+) page">', webpage, 'uploader')
 
         categories_html = self._search_regex(
             r'(?s)><i class="icon icon-tag"></i>\s*Categories / Tags\s*.*?<ul class="list">(.*?)</ul>',
@@ -68,9 +66,12 @@ class FourTubeIE(InfoExtractor):
             webpage, 'like count', fatal=False))
         duration = parse_duration(self._html_search_meta('duration', webpage))
 
+        player_url = self._search_regex(r'<script id="playerembed" src="([^"]+)">',webpage,'player javascript')
+        player_js = self._download_webpage(player_url,video_id,'Downloading player Javascript')
+
         params_js = self._search_regex(
             r'\$\.ajax\(url,\ opts\);\s*\}\s*\}\)\(([0-9,\[\] ]+)\)',
-            webpage, 'initialization parameters'
+            player_js, 'initialization parameters'
         )
         params = self._parse_json('[%s]' % params_js, video_id)
         media_id = params[0]

From 96c48553317cf28ea92d7feaa5701e3ebd93ea25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Oct 2015 20:21:07 +0600
Subject: [PATCH 073/150] [4tube] Style and make more robust

---
 youtube_dl/extractor/fourtube.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index 226ee67f0..406387e57 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -66,8 +66,11 @@ class FourTubeIE(InfoExtractor):
             webpage, 'like count', fatal=False))
         duration = parse_duration(self._html_search_meta('duration', webpage))
 
-        player_url = self._search_regex(r'<script id="playerembed" src="([^"]+)">',webpage,'player javascript')
-        player_js = self._download_webpage(player_url,video_id,'Downloading player Javascript')
+        player_js = self._download_webpage(
+            self._search_regex(
+                r'<script[^>]id=(["\'])playerembed\1[^>]+src=(["\'])(?P<url>.+?)\2',
+                webpage, 'player JS', group='url'),
+            video_id, 'Downloading player JS')
 
         params_js = self._search_regex(
             r'\$\.ajax\(url,\ opts\);\s*\}\s*\}\)\(([0-9,\[\] ]+)\)',

From c677e49bd11c3dfada05de9fc1d4de04a577d70e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Oct 2015 20:22:08 +0600
Subject: [PATCH 074/150] [4tube] Revert uploader regexes and make non fatal

---
 youtube_dl/extractor/fourtube.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index 406387e57..cd7668f18 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -45,9 +45,11 @@ class FourTubeIE(InfoExtractor):
             'uploadDate', webpage))
         thumbnail = self._html_search_meta('thumbnailUrl', webpage)
         uploader_id = self._html_search_regex(
-            r'<a class="img-avatar" href="[^"]+/users/([^/"]+)" title="Go to [^"]+ page">', webpage, 'uploader id')
+            r'<a class="img-avatar" href="[^"]+/channels/([^/"]+)" title="Go to [^"]+ page">',
+            webpage, 'uploader id', fatal=False)
         uploader = self._html_search_regex(
-            r'<a class="img-avatar" href="[^"]+/users/[^/"]+" title="Go to ([^"]+) page">', webpage, 'uploader')
+            r'<a class="img-avatar" href="[^"]+/channels/[^/"]+" title="Go to ([^"]+) page">',
+            webpage, 'uploader', fatal=False)
 
         categories_html = self._search_regex(
             r'(?s)><i class="icon icon-tag"></i>\s*Categories / Tags\s*.*?<ul class="list">(.*?)</ul>',

From 4356d907c1afae09ff67a17c90b53be466b13623 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Oct 2015 20:49:56 +0600
Subject: [PATCH 075/150] [4tube] Try extracting sources from the webpage
 before fetching player.js (Closes #7103)

---
 youtube_dl/extractor/fourtube.py | 31 ++++++++++++++++++-------------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index cd7668f18..fb6d108c0 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -68,19 +68,24 @@ class FourTubeIE(InfoExtractor):
             webpage, 'like count', fatal=False))
         duration = parse_duration(self._html_search_meta('duration', webpage))
 
-        player_js = self._download_webpage(
-            self._search_regex(
-                r'<script[^>]id=(["\'])playerembed\1[^>]+src=(["\'])(?P<url>.+?)\2',
-                webpage, 'player JS', group='url'),
-            video_id, 'Downloading player JS')
-
-        params_js = self._search_regex(
-            r'\$\.ajax\(url,\ opts\);\s*\}\s*\}\)\(([0-9,\[\] ]+)\)',
-            player_js, 'initialization parameters'
-        )
-        params = self._parse_json('[%s]' % params_js, video_id)
-        media_id = params[0]
-        sources = ['%s' % p for p in params[2]]
+        media_id = self._search_regex(
+            r'<button[^>]+data-id=(["\'])(?P<id>\d+)\1[^>]+data-quality=', webpage,
+            'media id', default=None, group='id')
+        sources = [
+            quality
+            for _, quality in re.findall(r'<button[^>]+data-quality=(["\'])(.+?)\1', webpage)]
+        if not (media_id and sources):
+            player_js = self._download_webpage(
+                self._search_regex(
+                    r'<script[^>]id=(["\'])playerembed\1[^>]+src=(["\'])(?P<url>.+?)\2',
+                    webpage, 'player JS', group='url'),
+                video_id, 'Downloading player JS')
+            params_js = self._search_regex(
+                r'\$\.ajax\(url,\ opts\);\s*\}\s*\}\)\(([0-9,\[\] ]+)\)',
+                player_js, 'initialization parameters')
+            params = self._parse_json('[%s]' % params_js, video_id)
+            media_id = params[0]
+            sources = ['%s' % p for p in params[2]]
 
         token_url = 'http://tkn.4tube.com/{0}/desktop/{1}'.format(
             media_id, '+'.join(sources))

From 7faf7e752342b21772c8647902bbf4f13cc57956 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 8 Oct 2015 18:39:24 +0100
Subject: [PATCH 076/150] [pbs] detect errors from http error code

---
 youtube_dl/extractor/pbs.py | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 6923c6094..82218b933 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -39,6 +39,7 @@ class PBSIE(InfoExtractor):
             'params': {
                 'skip_download': True,  # requires ffmpeg
             },
+            'skip': 'Expired',
         },
         {
             'url': 'http://www.pbs.org/wgbh/pages/frontline/losing-iraq/',
@@ -108,12 +109,12 @@ class PBSIE(InfoExtractor):
         {
             'url': 'http://www.pbs.org/wgbh/americanexperience/films/death/player/',
             'info_dict': {
-                'id': '2280706814',
+                'id': '2276541483',
                 'display_id': 'player',
                 'ext': 'mp4',
-                'title': 'American Experience - Death and the Civil War',
+                'title': 'American Experience - Death and the Civil War, Chapter 1',
                 'description': 'American Experience, TV’s most-watched history series, brings to life the compelling stories from our past that inform our understanding of the world today.',
-                'duration': 6705,
+                'duration': 682,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
             'params': {
@@ -134,6 +135,7 @@ class PBSIE(InfoExtractor):
             'params': {
                 'skip_download': True,  # requires ffmpeg
             },
+            'skip': 'Expired',
         },
         {
             # Video embedded in iframe containing angle brackets as attribute's value (e.g.
@@ -231,13 +233,18 @@ class PBSIE(InfoExtractor):
                 'Downloading %s video url info' % encoding_name)
 
             if redirect_info['status'] == 'error':
-                if redirect_info['http_code'] == 403:
-                    message = (
-                        'The video is not available in your region due to '
-                        'right restrictions')
+                http_code = redirect_info['http_code']
+                if http_code == 403:
+                    message = 'We\'re sorry, but this video is not available in your region due to right restrictions.'
+                elif http_code == 101:
+                    message = 'We\'re sorry, but this video is not yet available.'
+                elif http_code == 404:
+                    message = 'We are experiencing technical difficulties that are preventing us from playing the video at this time. Please check back again soon.'
+                elif http_code == 410:
+                    message = 'This video has expired and is no longer available for online streaming.'
                 else:
                     message = redirect_info['message']
-                raise ExtractorError(message, expected=True)
+                raise ExtractorError('PBS said: %s' % message, expected=True)
 
             format_url = redirect_info.get('url')
             if not format_url:

From 0553d0ee40b6ecd62ed25ba8999aa2e4f7641590 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 8 Oct 2015 18:57:57 +0100
Subject: [PATCH 077/150] [pbs] place errors into a dict

---
 youtube_dl/extractor/pbs.py | 20 +++++++-------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 82218b933..814e97f48 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -39,7 +39,6 @@ class PBSIE(InfoExtractor):
             'params': {
                 'skip_download': True,  # requires ffmpeg
             },
-            'skip': 'Expired',
         },
         {
             'url': 'http://www.pbs.org/wgbh/pages/frontline/losing-iraq/',
@@ -156,6 +155,12 @@ class PBSIE(InfoExtractor):
             },
         }
     ]
+    _ERRORS = {
+        101: 'We\'re sorry, but this video is not yet available.',
+        403: 'We\'re sorry, but this video is not available in your region due to right restrictions.',
+        404: 'We are experiencing technical difficulties that are preventing us from playing the video at this time. Please check back again soon.',
+        410: 'This video has expired and is no longer available for online streaming.',
+    }
 
     def _extract_webpage(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -233,18 +238,7 @@ class PBSIE(InfoExtractor):
                 'Downloading %s video url info' % encoding_name)
 
             if redirect_info['status'] == 'error':
-                http_code = redirect_info['http_code']
-                if http_code == 403:
-                    message = 'We\'re sorry, but this video is not available in your region due to right restrictions.'
-                elif http_code == 101:
-                    message = 'We\'re sorry, but this video is not yet available.'
-                elif http_code == 404:
-                    message = 'We are experiencing technical difficulties that are preventing us from playing the video at this time. Please check back again soon.'
-                elif http_code == 410:
-                    message = 'This video has expired and is no longer available for online streaming.'
-                else:
-                    message = redirect_info['message']
-                raise ExtractorError('PBS said: %s' % message, expected=True)
+                raise ExtractorError('PBS said: %s' % self._ERRORS.get(redirect_info['http_code'], redirect_info['message']), expected=True)
 
             format_url = redirect_info.get('url')
             if not format_url:

From 9d5fb3b58d95fbd8c28d9c0eaf4a652660324d9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Oct 2015 00:09:10 +0600
Subject: [PATCH 078/150] [pbs] Carry long line

---
 youtube_dl/extractor/pbs.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 814e97f48..3448736a2 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -238,7 +238,11 @@ class PBSIE(InfoExtractor):
                 'Downloading %s video url info' % encoding_name)
 
             if redirect_info['status'] == 'error':
-                raise ExtractorError('PBS said: %s' % self._ERRORS.get(redirect_info['http_code'], redirect_info['message']), expected=True)
+                raise ExtractorError(
+                    '%s said: %s' % (
+                        self.IE_NAME,
+                        self._ERRORS.get(redirect_info['http_code'], redirect_info['message'])),
+                    expected=True)
 
             format_url = redirect_info.get('url')
             if not format_url:

From 54a5428518adc4ddca085bdac471bc4f286024e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Oct 2015 03:54:49 +0600
Subject: [PATCH 079/150] [dailymotion] Update player v5 regex (Closes #7107)

---
 youtube_dl/extractor/dailymotion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 2d90b2224..80a05cfee 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -119,7 +119,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             webpage, 'comment count', fatal=False))
 
         player_v5 = self._search_regex(
-            r'playerV5\s*=\s*dmp\.create\([^,]+?,\s*({.+?})\);',
+            [r'buildPlayer\(({.+?})\);', r'playerV5\s*=\s*dmp\.create\([^,]+?,\s*({.+?})\);'],
             webpage, 'player v5', default=None)
         if player_v5:
             player = self._parse_json(player_v5, video_id)

From 57935b2564c082b90a60468d4c844b219118886a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 9 Oct 2015 14:11:00 +0800
Subject: [PATCH 080/150] [extractor/common] Allow HTML5 unquoted attribute
 values

Fixes #7108

HTML5 allows unquoted attribute values. See the "Unquoted attribute value
syntax" section [1] for more information

[1] http://www.w3.org/TR/html5/syntax.html
---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index dbae75406..242618c58 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -646,7 +646,7 @@ class InfoExtractor(object):
     @staticmethod
     def _og_regexes(prop):
         content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\')'
-        property_re = r'(?:name|property)=[\'"]og:%s[\'"]' % re.escape(prop)
+        property_re = r'(?:name|property)=[\'"]?og:%s[\'"]?' % re.escape(prop)
         template = r'<meta[^>]+?%s[^>]+?%s'
         return [
             template % (property_re, content_re),

From ef47b2c15f3fa9d9d491090f2ea46d2bd9967d21 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Oct 2015 09:09:22 +0200
Subject: [PATCH 081/150] release 2015.10.09

---
 CONTRIBUTING.md       | 2 +-
 youtube_dl/version.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 57a94231d..32c2fd84c 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -124,7 +124,7 @@ If you want to add support for a new site, you can follow this quick list (assum
             }
     ```
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
-6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
+6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62-L200). Add tests and code for as many as you want.
 8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 112c78835..faae8a2d7 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.06.2'
+__version__ = '2015.10.09'

From 4180a3d8b7c40792b5371ca8804c1dad8fabd56d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 01:44:33 +0600
Subject: [PATCH 082/150] [extractor/common] Allow quoteless content attribute
 in og regexes (Closes #7115)

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 242618c58..0082a4c84 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -645,7 +645,7 @@ class InfoExtractor(object):
     # Helper functions for extracting OpenGraph info
     @staticmethod
     def _og_regexes(prop):
-        content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\')'
+        content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\'|\s*([^\s"\'=<>`]+?))'
         property_re = r'(?:name|property)=[\'"]?og:%s[\'"]?' % re.escape(prop)
         template = r'<meta[^>]+?%s[^>]+?%s'
         return [

From 47c165c3a9f40dd9a175b11f29f9e2002fdda8a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 05:56:01 +0600
Subject: [PATCH 083/150] [vimeo] Fix authentication (Closes #7110)

---
 youtube_dl/extractor/vimeo.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 50df79ca1..7dd52627d 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -40,6 +40,9 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         self.report_login()
         webpage = self._download_webpage(self._LOGIN_URL, None, False)
         token = self._extract_xsrft(webpage)
+        vuid = self._search_regex(
+            r'["\']vuid["\']\s*:\s*(["\'])(?P<vuid>.+?)\1',
+            webpage, 'vuid', group='vuid')
         data = urlencode_postdata({
             'action': 'login',
             'email': username,
@@ -49,6 +52,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         })
         login_request = compat_urllib_request.Request(self._LOGIN_URL, data)
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        login_request.add_header('Cookie', 'vuid=%s' % vuid)
         login_request.add_header('Referer', self._LOGIN_URL)
         self._download_webpage(login_request, None, False, 'Wrong login info')
 

From 6a959f2e5266dcc6037a33589eb34fad9190c63e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 10 Oct 2015 15:03:01 +0800
Subject: [PATCH 084/150] [iqiyi] Update enc_key

---
 youtube_dl/extractor/iqiyi.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index ce1ab3820..0e53cb154 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -191,7 +191,7 @@ class IqiyiIE(InfoExtractor):
             'vid': video_id,
             'vinfo': 1,
             'tm': tm,
-            'enc': self.md5_text((enc_key + tail)[1:64:2] + tail),
+            'enc': self.md5_text(enc_key + tail),
             'qyid': _uuid,
             'tn': random.random(),
             'um': 0,
@@ -205,7 +205,9 @@ class IqiyiIE(InfoExtractor):
 
     def get_enc_key(self, swf_url, video_id):
         # TODO: automatic key extraction
-        enc_key = 'eac64f22daf001da6ba9aa8da4d501508bbe90a4d4091fea3b0582a85b38c2cc'  # last update at 2015-09-23-23 for Zombie::bite
+        # last update at 2015-10-10 for Zombie::bite
+        # '7239670519b6ac209a0bee4ef0446a6b24894b8ac2751506e42116212a0d0272e505'[2:66][1::2]
+        enc_key = '97596c0abee04ab49ba25564161ad225'
         return enc_key
 
     def _real_extract(self, url):

From dab062fb6ecd48e0c243a6d030d89b44cd44bd84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 20:34:06 +0600
Subject: [PATCH 085/150] [bbc] Add support for videos in news articles
 embedded with data-playable

---
 youtube_dl/extractor/bbc.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index cc2f6fed2..b2b39ff21 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -11,6 +11,7 @@ from ..utils import (
     int_or_none,
     parse_duration,
     parse_iso8601,
+    unescapeHTML,
 )
 from ..compat import compat_HTTPError
 
@@ -682,6 +683,21 @@ class BBCIE(BBCCoUkIE):
             [r'data-video-player-vpid="([\da-z]{8})"',
              r'<param[^>]+name="externalIdentifier"[^>]+value="([\da-z]{8})"'],
             webpage, 'vpid', default=None)
+
+        duration = None
+        if not programme_id:
+            # single video in news article embedded with data-playable (e.g.
+            # http://www.bbc.com/news/world-us-canada-34473351)
+            data_playable = self._parse_json(
+                unescapeHTML(self._search_regex(
+                    r'data-playable="({.+?})"', webpage, 'data playable', default='{}')),
+                programme_id, fatal=False)
+            if data_playable:
+                items = data_playable.get('settings', {}).get('playlistObject', {}).get('items')
+                if items and isinstance(items, list):
+                    duration = int_or_none(items[0].get('duration'))
+                    programme_id = items[0].get('vpid')
+
         if programme_id:
             formats, subtitles = self._download_media_selector(programme_id)
             self._sort_formats(formats)
@@ -699,6 +715,7 @@ class BBCIE(BBCCoUkIE):
                 'title': title,
                 'description': description,
                 'timestamp': timestamp,
+                'duration': duration,
                 'formats': formats,
                 'subtitles': subtitles,
             }

From de66571371e2a9705ecd3aed903880c4846d04ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 20:40:56 +0600
Subject: [PATCH 086/150] [bbc] Support multiple videos in articles embedded
 with playlist.sxml

---
 youtube_dl/extractor/bbc.py | 38 ++++++++++++++++++++-----------------
 1 file changed, 21 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index b2b39ff21..930637cd7 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -660,23 +660,27 @@ class BBCIE(BBCCoUkIE):
              r'itemprop="datePublished"[^>]+datetime="([^"]+)"'],
             webpage, 'date', default=None))
 
-        # single video with playlist.sxml URL (e.g. http://www.bbc.com/sport/0/football/3365340ng)
-        playlist = self._search_regex(
-            r'<param[^>]+name="playlist"[^>]+value="([^"]+)"',
-            webpage, 'playlist', default=None)
-        if playlist:
-            programme_id, title, description, duration, formats, subtitles = \
-                self._process_legacy_playlist_url(playlist, playlist_id)
-            self._sort_formats(formats)
-            return {
-                'id': programme_id,
-                'title': title,
-                'description': description,
-                'duration': duration,
-                'timestamp': timestamp,
-                'formats': formats,
-                'subtitles': subtitles,
-            }
+        # article with multiple videos embedded with playlist.sxml (e.g.
+        # http://www.bbc.com/sport/0/football/34475836)
+        playlists = re.findall(r'<param[^>]+name="playlist"[^>]+value="([^"]+)"', webpage)
+        if playlists:
+            entries = []
+            for playlist in playlists:
+                programme_id, title, description, duration, formats, subtitles = \
+                    self._process_legacy_playlist_url(playlist, playlist_id)
+                self._sort_formats(formats)
+                entries.append({
+                    'id': programme_id,
+                    'title': title,
+                    'description': description,
+                    'duration': duration,
+                    'timestamp': timestamp,
+                    'formats': formats,
+                    'subtitles': subtitles,
+                })
+            playlist_title = self._og_search_title(webpage)
+            playlist_description = self._og_search_description(webpage)
+            return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
 
         # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
         programme_id = self._search_regex(

From b5d48cb1ef741ef999d038a3bc80943b1ae37a50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 20:55:46 +0600
Subject: [PATCH 087/150] [bbc] Add test for atricle with multiple videos
 embedded with playlist.sxml

---
 youtube_dl/extractor/bbc.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 930637cd7..ffe2afa96 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -608,6 +608,14 @@ class BBCIE(BBCCoUkIE):
             # rtmp download
             'skip_download': True,
         }
+    }, {
+        # article with multiple videos embedded with playlist.sxml
+        'url': 'http://www.bbc.com/sport/0/football/34475836',
+        'info_dict': {
+            'id': '34475836',
+            'title': 'What Liverpool can expect from Klopp',
+        },
+        'playlist_count': 3,
     }, {
         # single video with playlist URL from weather section
         'url': 'http://www.bbc.com/weather/features/33601775',

From 975977860d1880bbf5e9996ad77fb7f67622ccbe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 20:56:07 +0600
Subject: [PATCH 088/150] [bbc] Make playlist title optional

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index ffe2afa96..f943b5fd1 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -687,7 +687,7 @@ class BBCIE(BBCCoUkIE):
                     'subtitles': subtitles,
                 })
             playlist_title = self._og_search_title(webpage)
-            playlist_description = self._og_search_description(webpage)
+            playlist_description = self._og_search_description(webpage, default=None)
             return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
 
         # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)

From baf39a1aa8af38ca295e5efdad277ddee5b11eb5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 21:32:27 +0600
Subject: [PATCH 089/150] [bbc] Add one more scenario for data-playable embeds

---
 youtube_dl/extractor/bbc.py | 53 ++++++++++++++++++++++++-------------
 1 file changed, 35 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index f943b5fd1..df61f0157 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -657,6 +657,20 @@ class BBCIE(BBCCoUkIE):
 
         return [], []
 
+    def _extract_from_playlist_sxml(self, url, playlist_id, timestamp):
+        programme_id, title, description, duration, formats, subtitles = \
+            self._process_legacy_playlist_url(url, playlist_id)
+        self._sort_formats(formats)
+        return {
+            'id': programme_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'timestamp': timestamp,
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
 
@@ -672,20 +686,9 @@ class BBCIE(BBCCoUkIE):
         # http://www.bbc.com/sport/0/football/34475836)
         playlists = re.findall(r'<param[^>]+name="playlist"[^>]+value="([^"]+)"', webpage)
         if playlists:
-            entries = []
-            for playlist in playlists:
-                programme_id, title, description, duration, formats, subtitles = \
-                    self._process_legacy_playlist_url(playlist, playlist_id)
-                self._sort_formats(formats)
-                entries.append({
-                    'id': programme_id,
-                    'title': title,
-                    'description': description,
-                    'duration': duration,
-                    'timestamp': timestamp,
-                    'formats': formats,
-                    'subtitles': subtitles,
-                })
+            entries = [
+                self._extract_from_playlist_sxml(playlist_url, playlist_id, timestamp)
+                for playlist_url in playlists]
             playlist_title = self._og_search_title(webpage)
             playlist_description = self._og_search_description(webpage, default=None)
             return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
@@ -705,10 +708,24 @@ class BBCIE(BBCCoUkIE):
                     r'data-playable="({.+?})"', webpage, 'data playable', default='{}')),
                 programme_id, fatal=False)
             if data_playable:
-                items = data_playable.get('settings', {}).get('playlistObject', {}).get('items')
-                if items and isinstance(items, list):
-                    duration = int_or_none(items[0].get('duration'))
-                    programme_id = items[0].get('vpid')
+                # data-playable has video vpid in settings.playlistObject.items (e.g.
+                # http://www.bbc.com/news/world-us-canada-34473351)
+                settings = data_playable.get('settings', {})
+                if settings:
+                    playlist_object = settings.get('playlistObject', {})
+                    if playlist_object:
+                        items = playlist_object.get('items')
+                        if items and isinstance(items, list):
+                            duration = int_or_none(items[0].get('duration'))
+                            programme_id = items[0].get('vpid')
+                if not programme_id:
+                    # data-playable has no vpid but has a playlist.sxml URLs
+                    # in otherSettings.playlist (e.g.
+                    # http://www.bbc.com/turkce/multimedya/2015/10/151010_vid_ankara_patlama_ani)
+                    playlist = data_playable.get('otherSettings', {}).get('playlist', {})
+                    if playlist:
+                        return self._extract_from_playlist_sxml(
+                            playlist.get('progressiveDownloadUrl'), playlist_id, timestamp)
 
         if programme_id:
             formats, subtitles = self._download_media_selector(programme_id)

From 87cc0fbd18345117926d27db7312cba663d0f5ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 21:35:20 +0600
Subject: [PATCH 090/150] [bbc] Make summary optional in legacy playlist

---
 youtube_dl/extractor/bbc.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index df61f0157..911ccd757 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -401,7 +401,8 @@ class BBCCoUkIE(InfoExtractor):
             if kind != 'programme' and kind != 'radioProgramme':
                 continue
             title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
-            description = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary').text
+            description_el = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary')
+            description = description_el.text if description_el else None
 
             def get_programme_id(item):
                 def get_from_attributes(item):

From 97ae4d166c43b7e5068a7563bc5fbce385d86c2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ph=E1=BA=A1m=20Ng=E1=BB=8Dc=20Quang=20Nam?=
 <nampnq@gmail.com>
Date: Sat, 10 Oct 2015 22:07:06 +0700
Subject: [PATCH 091/150] [zingmp3:album] Add support for playlists

Update for work with playlist, it same album but different url
Ex:
http://mp3.zing.vn/playlist/Duong-Hong-Loan-apollobee/IWCAACCB.html
http://mp3.zing.vn/album/Duong-Hong-Loan-apollobee/IWCAACCB.html
---
 youtube_dl/extractor/zingmp3.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/zingmp3.py b/youtube_dl/extractor/zingmp3.py
index 7dc1e2f2b..42ac124f0 100644
--- a/youtube_dl/extractor/zingmp3.py
+++ b/youtube_dl/extractor/zingmp3.py
@@ -85,16 +85,22 @@ class ZingMp3SongIE(ZingMp3BaseInfoExtractor):
 
 
 class ZingMp3AlbumIE(ZingMp3BaseInfoExtractor):
-    _VALID_URL = r'https?://mp3\.zing\.vn/album/(?P<slug>[^/]+)/(?P<album_id>\w+)\.html'
-    _TESTS = [{
-        'url': 'http://mp3.zing.vn/album/Lau-Dai-Tinh-Ai-Bang-Kieu-Minh-Tuyet/ZWZBWDAF.html',
-        'info_dict': {
-            '_type': 'playlist',
-            'id': 'ZWZBWDAF',
-            'title': 'Lâu Đài Tình Ái - Bằng Kiều ft. Minh Tuyết | Album 320 lossless',
+    _VALID_URL = r'https?://mp3\.zing\.vn/(?:album|playlist)/(?P<slug>[^/]+)/(?P<album_id>\w+)\.html'
+    _TESTS = [
+        {
+            'url': 'http://mp3.zing.vn/album/Lau-Dai-Tinh-Ai-Bang-Kieu-Minh-Tuyet/ZWZBWDAF.html',
+            'info_dict': {
+                '_type': 'playlist',
+                'id': 'ZWZBWDAF',
+                'title': 'Lâu Đài Tình Ái - Bằng Kiều ft. Minh Tuyết | Album 320 lossless',
+            },
+            'playlist_count': 10,
         },
-        'playlist_count': 10,
-    }]
+        {
+            'url': 'http://mp3.zing.vn/playlist/Duong-Hong-Loan-apollobee/IWCAACCB.html',
+            'only_matching': True,
+        }
+    ]
     IE_NAME = 'zingmp3:album'
     IE_DESC = 'mp3.zing.vn albums'
 

From 43abd79950b7ba0c3b6e8a26b624e439036b1c7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 22:05:01 +0600
Subject: [PATCH 092/150] [zingmp3:album] Style

---
 youtube_dl/extractor/zingmp3.py | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/zingmp3.py b/youtube_dl/extractor/zingmp3.py
index 42ac124f0..1059dd75f 100644
--- a/youtube_dl/extractor/zingmp3.py
+++ b/youtube_dl/extractor/zingmp3.py
@@ -86,21 +86,18 @@ class ZingMp3SongIE(ZingMp3BaseInfoExtractor):
 
 class ZingMp3AlbumIE(ZingMp3BaseInfoExtractor):
     _VALID_URL = r'https?://mp3\.zing\.vn/(?:album|playlist)/(?P<slug>[^/]+)/(?P<album_id>\w+)\.html'
-    _TESTS = [
-        {
-            'url': 'http://mp3.zing.vn/album/Lau-Dai-Tinh-Ai-Bang-Kieu-Minh-Tuyet/ZWZBWDAF.html',
-            'info_dict': {
-                '_type': 'playlist',
-                'id': 'ZWZBWDAF',
-                'title': 'Lâu Đài Tình Ái - Bằng Kiều ft. Minh Tuyết | Album 320 lossless',
-            },
-            'playlist_count': 10,
+    _TESTS = [{
+        'url': 'http://mp3.zing.vn/album/Lau-Dai-Tinh-Ai-Bang-Kieu-Minh-Tuyet/ZWZBWDAF.html',
+        'info_dict': {
+            '_type': 'playlist',
+            'id': 'ZWZBWDAF',
+            'title': 'Lâu Đài Tình Ái - Bằng Kiều ft. Minh Tuyết | Album 320 lossless',
         },
-        {
-            'url': 'http://mp3.zing.vn/playlist/Duong-Hong-Loan-apollobee/IWCAACCB.html',
-            'only_matching': True,
-        }
-    ]
+        'playlist_count': 10,
+    }, {
+        'url': 'http://mp3.zing.vn/playlist/Duong-Hong-Loan-apollobee/IWCAACCB.html',
+        'only_matching': True,
+    }]
     IE_NAME = 'zingmp3:album'
     IE_DESC = 'mp3.zing.vn albums'
 

From 8119597d6ff52a671fa359378dfbdee13f24f882 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 22:08:38 +0600
Subject: [PATCH 093/150] [zingmp3] Add fatal flag

---
 youtube_dl/extractor/zingmp3.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/zingmp3.py b/youtube_dl/extractor/zingmp3.py
index 1059dd75f..3f46f9049 100644
--- a/youtube_dl/extractor/zingmp3.py
+++ b/youtube_dl/extractor/zingmp3.py
@@ -9,9 +9,11 @@ from ..utils import ExtractorError
 
 class ZingMp3BaseInfoExtractor(InfoExtractor):
 
-    def _extract_item(self, item):
+    def _extract_item(self, item, fatal=True):
         error_message = item.find('./errormessage').text
         if error_message:
+            if not fatal:
+                return
             raise ExtractorError(
                 '%s returned error: %s' % (self.IE_NAME, error_message),
                 expected=True)

From f790c43f6e33782c6a6eaaff68bbd30aad875efe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 22:09:21 +0600
Subject: [PATCH 094/150] [zingmp3:album] Skip broken items

---
 youtube_dl/extractor/zingmp3.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/zingmp3.py b/youtube_dl/extractor/zingmp3.py
index 3f46f9049..437eecb67 100644
--- a/youtube_dl/extractor/zingmp3.py
+++ b/youtube_dl/extractor/zingmp3.py
@@ -45,7 +45,9 @@ class ZingMp3BaseInfoExtractor(InfoExtractor):
             entries = []
 
             for i, item in enumerate(items, 1):
-                entry = self._extract_item(item)
+                entry = self._extract_item(item, fatal=False)
+                if not entry:
+                    continue
                 entry['id'] = '%s-%d' % (id, i)
                 entries.append(entry)
 

From e6174ee9753e044bdeac22c653f5762376cadbac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 23:01:20 +0600
Subject: [PATCH 095/150] [bbc] Extract legacy playlist embedded media

---
 youtube_dl/extractor/bbc.py | 48 ++++++++++++++++++++++++++-----------
 1 file changed, 34 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 911ccd757..7bbbc0b90 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -29,6 +29,14 @@ class BBCCoUkIE(InfoExtractor):
         'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s',
     ]
 
+    _MEDIASELECTION_NS = 'http://bbc.co.uk/2008/mp/mediaselection'
+    _EMP_PLAYLIST_NS = 'http://bbc.co.uk/2008/emp/playlist'
+
+    _NAMESPACES = (
+        _MEDIASELECTION_NS,
+        _EMP_PLAYLIST_NS,
+    )
+
     _TESTS = [
         {
             'url': 'http://www.bbc.co.uk/programmes/b039g8p7',
@@ -194,6 +202,7 @@ class BBCCoUkIE(InfoExtractor):
 
     def _extract_connection(self, connection, programme_id):
         formats = []
+        kind = connection.get('kind')
         protocol = connection.get('protocol')
         supplier = connection.get('supplier')
         if protocol == 'http':
@@ -219,7 +228,7 @@ class BBCCoUkIE(InfoExtractor):
             else:
                 formats.append({
                     'url': href,
-                    'format_id': supplier,
+                    'format_id': supplier or kind or protocol,
                 })
         elif protocol == 'rtmp':
             application = connection.get('application', 'ondemand')
@@ -239,16 +248,24 @@ class BBCCoUkIE(InfoExtractor):
         return formats
 
     def _extract_items(self, playlist):
-        return playlist.findall('./{http://bbc.co.uk/2008/emp/playlist}item')
+        return playlist.findall('./{%s}item' % self._EMP_PLAYLIST_NS)
+
+    def _findall_ns(self, element, xpath):
+        elements = []
+        for ns in self._NAMESPACES:
+            elements.extend(element.findall(xpath % ns))
+        return elements
 
     def _extract_medias(self, media_selection):
-        error = media_selection.find('./{http://bbc.co.uk/2008/mp/mediaselection}error')
+        error = media_selection.find('./{%s}error' % self._MEDIASELECTION_NS)
+        if error is None:
+            media_selection.find('./{%s}error' % self._EMP_PLAYLIST_NS)
         if error is not None:
             raise BBCCoUkIE.MediaSelectionError(error.get('id'))
-        return media_selection.findall('./{http://bbc.co.uk/2008/mp/mediaselection}media')
+        return self._findall_ns(media_selection, './{%s}media')
 
     def _extract_connections(self, media):
-        return media.findall('./{http://bbc.co.uk/2008/mp/mediaselection}connection')
+        return self._findall_ns(media, './{%s}connection')
 
     def _extract_video(self, media, programme_id):
         formats = []
@@ -262,13 +279,14 @@ class BBCCoUkIE(InfoExtractor):
             conn_formats = self._extract_connection(connection, programme_id)
             for format in conn_formats:
                 format.update({
-                    'format_id': '%s_%s' % (service, format['format_id']),
                     'width': width,
                     'height': height,
                     'vbr': vbr,
                     'vcodec': vcodec,
                     'filesize': file_size,
                 })
+                if service:
+                    format['format_id'] = '%s_%s' % (service, format['format_id'])
             formats.extend(conn_formats)
         return formats
 
@@ -383,7 +401,7 @@ class BBCCoUkIE(InfoExtractor):
             url, playlist_id, 'Downloading legacy playlist XML')
 
     def _extract_from_legacy_playlist(self, playlist, playlist_id):
-        no_items = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}noItems')
+        no_items = playlist.find('./{%s}noItems' % self._EMP_PLAYLIST_NS)
         if no_items is not None:
             reason = no_items.get('reason')
             if reason == 'preAvailability':
@@ -400,8 +418,8 @@ class BBCCoUkIE(InfoExtractor):
             kind = item.get('kind')
             if kind != 'programme' and kind != 'radioProgramme':
                 continue
-            title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
-            description_el = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary')
+            title = playlist.find('./{%s}title' % self._EMP_PLAYLIST_NS).text
+            description_el = playlist.find('./{%s}summary' % self._EMP_PLAYLIST_NS)
             description = description_el.text if description_el else None
 
             def get_programme_id(item):
@@ -411,16 +429,18 @@ class BBCCoUkIE(InfoExtractor):
                         if value and re.match(r'^[pb][\da-z]{7}$', value):
                             return value
                 get_from_attributes(item)
-                mediator = item.find('./{http://bbc.co.uk/2008/emp/playlist}mediator')
+                mediator = item.find('./{%s}mediator' % self._EMP_PLAYLIST_NS)
                 if mediator is not None:
                     return get_from_attributes(mediator)
 
             programme_id = get_programme_id(item)
             duration = int_or_none(item.get('duration'))
-            # TODO: programme_id can be None and media items can be incorporated right inside
-            # playlist's item (e.g. http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
-            # as f4m and m3u8
-            formats, subtitles = self._download_media_selector(programme_id)
+
+            if programme_id:
+                formats, subtitles = self._download_media_selector(programme_id)
+            else:
+                formats, subtitles = self._process_media_selector(item, playlist_id)
+                programme_id = playlist_id
 
         return programme_id, title, description, duration, formats, subtitles
 

From c936d8cc7b247f79dce2f5749dea5c18a08858fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 23:14:25 +0600
Subject: [PATCH 096/150] [bbc] Add another test

---
 youtube_dl/extractor/bbc.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 7bbbc0b90..0a08a57bd 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -559,6 +559,19 @@ class BBCIE(BBCCoUkIE):
         'params': {
             'skip_download': True,
         }
+    }, {
+        # single video embedded with playlist.sxml in data-playable
+        'url': 'http://www.bbc.com/turkce/multimedya/2015/10/151010_vid_ankara_patlama_ani',
+        'info_dict': {
+            'id': '151010_vid_ankara_patlama_ani',
+            'ext': 'mp4',
+            'title': "Ankara'da patlama anı",
+            'timestamp': 1444480325,
+            'upload_date': '20151010',
+        },
+        'params': {
+            'skip_download': True,
+        }
     }, {
         # single video embedded with mediaAssetPage.init() (regional section)
         'url': 'http://www.bbc.com/mundo/video_fotos/2015/06/150619_video_honduras_militares_hospitales_corrupcion_aw',

From b1ec70e4a9fc53b0ec583f48a5262c9f864db40b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 23:20:30 +0600
Subject: [PATCH 097/150] [bbc] Improve data-playable regex

---
 youtube_dl/extractor/bbc.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 0a08a57bd..972abd0d4 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -739,7 +739,8 @@ class BBCIE(BBCCoUkIE):
             # http://www.bbc.com/news/world-us-canada-34473351)
             data_playable = self._parse_json(
                 unescapeHTML(self._search_regex(
-                    r'data-playable="({.+?})"', webpage, 'data playable', default='{}')),
+                    r'data-playable=(["\'])(?P<json>{.+?})\1', webpage,
+                    'data playable', default='{}', group='json')),
                 programme_id, fatal=False)
             if data_playable:
                 # data-playable has video vpid in settings.playlistObject.items (e.g.

From 78f9d843186977c614c5a0f6004732f5d410cd0c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 23:40:20 +0600
Subject: [PATCH 098/150] [bbc] Support playlists of data-playable

---
 youtube_dl/extractor/bbc.py | 73 ++++++++++++++++++++++---------------
 1 file changed, 44 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 972abd0d4..a15e67114 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -716,6 +716,8 @@ class BBCIE(BBCCoUkIE):
              r'itemprop="datePublished"[^>]+datetime="([^"]+)"'],
             webpage, 'date', default=None))
 
+        entries = []
+
         # article with multiple videos embedded with playlist.sxml (e.g.
         # http://www.bbc.com/sport/0/football/34475836)
         playlists = re.findall(r'<param[^>]+name="playlist"[^>]+value="([^"]+)"', webpage)
@@ -723,6 +725,48 @@ class BBCIE(BBCCoUkIE):
             entries = [
                 self._extract_from_playlist_sxml(playlist_url, playlist_id, timestamp)
                 for playlist_url in playlists]
+
+        # news article with multiple videos embedded with data-playable
+        data_playables = re.findall(r'data-playable=(["\'])({.+?})\1', webpage)
+        if data_playables:
+            for _, data_playable_json in data_playables:
+                data_playable = self._parse_json(
+                    unescapeHTML(data_playable_json), playlist_id, fatal=False)
+                if not data_playable:
+                    continue
+                settings = data_playable.get('settings', {})
+                if settings:
+                    # data-playable with video vpid in settings.playlistObject.items (e.g.
+                    # http://www.bbc.com/news/world-us-canada-34473351)
+                    playlist_object = settings.get('playlistObject', {})
+                    if playlist_object:
+                        items = playlist_object.get('items')
+                        if items and isinstance(items, list):
+                            title = playlist_object['title']
+                            description = playlist_object.get('summary')
+                            duration = int_or_none(items[0].get('duration'))
+                            programme_id = items[0].get('vpid')
+                            formats, subtitles = self._download_media_selector(programme_id)
+                            self._sort_formats(formats)
+                            entries.append({
+                                'id': programme_id,
+                                'title': title,
+                                'description': description,
+                                'timestamp': timestamp,
+                                'duration': duration,
+                                'formats': formats,
+                                'subtitles': subtitles,
+                            })
+                    else:
+                        # data-playable without vpid but with a playlist.sxml URLs
+                        # in otherSettings.playlist (e.g.
+                        # http://www.bbc.com/turkce/multimedya/2015/10/151010_vid_ankara_patlama_ani)
+                        playlist = data_playable.get('otherSettings', {}).get('playlist', {})
+                        if playlist:
+                            entries.append(self._extract_from_playlist_sxml(
+                                playlist.get('progressiveDownloadUrl'), playlist_id, timestamp))
+
+        if entries:
             playlist_title = self._og_search_title(webpage)
             playlist_description = self._og_search_description(webpage, default=None)
             return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
@@ -733,35 +777,6 @@ class BBCIE(BBCCoUkIE):
              r'<param[^>]+name="externalIdentifier"[^>]+value="([\da-z]{8})"'],
             webpage, 'vpid', default=None)
 
-        duration = None
-        if not programme_id:
-            # single video in news article embedded with data-playable (e.g.
-            # http://www.bbc.com/news/world-us-canada-34473351)
-            data_playable = self._parse_json(
-                unescapeHTML(self._search_regex(
-                    r'data-playable=(["\'])(?P<json>{.+?})\1', webpage,
-                    'data playable', default='{}', group='json')),
-                programme_id, fatal=False)
-            if data_playable:
-                # data-playable has video vpid in settings.playlistObject.items (e.g.
-                # http://www.bbc.com/news/world-us-canada-34473351)
-                settings = data_playable.get('settings', {})
-                if settings:
-                    playlist_object = settings.get('playlistObject', {})
-                    if playlist_object:
-                        items = playlist_object.get('items')
-                        if items and isinstance(items, list):
-                            duration = int_or_none(items[0].get('duration'))
-                            programme_id = items[0].get('vpid')
-                if not programme_id:
-                    # data-playable has no vpid but has a playlist.sxml URLs
-                    # in otherSettings.playlist (e.g.
-                    # http://www.bbc.com/turkce/multimedya/2015/10/151010_vid_ankara_patlama_ani)
-                    playlist = data_playable.get('otherSettings', {}).get('playlist', {})
-                    if playlist:
-                        return self._extract_from_playlist_sxml(
-                            playlist.get('progressiveDownloadUrl'), playlist_id, timestamp)
-
         if programme_id:
             formats, subtitles = self._download_media_selector(programme_id)
             self._sort_formats(formats)

From f6295bcb04c0e518369c31a44b691ad276fcc22f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 23:45:15 +0600
Subject: [PATCH 099/150] [bbc] Remove duration

---
 youtube_dl/extractor/bbc.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index a15e67114..fad1f9ca7 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -794,7 +794,6 @@ class BBCIE(BBCCoUkIE):
                 'title': title,
                 'description': description,
                 'timestamp': timestamp,
-                'duration': duration,
                 'formats': formats,
                 'subtitles': subtitles,
             }

From 6a747190605229e9cfba5450cf0ecaf435b7a85e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 23:56:55 +0600
Subject: [PATCH 100/150] [bbc] Update tests

---
 youtube_dl/extractor/bbc.py | 33 ++++++++++-----------------------
 1 file changed, 10 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index fad1f9ca7..4eae4f52e 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -501,8 +501,7 @@ class BBCIE(BBCCoUkIE):
     ]
 
     _TESTS = [{
-        # article with multiple videos embedded with data-media-meta containing
-        # playlist.sxml, externalId and no direct video links
+        # article with multiple videos embedded with data-playable containing vpids
         'url': 'http://www.bbc.com/news/world-europe-32668511',
         'info_dict': {
             'id': 'world-europe-32668511',
@@ -511,7 +510,7 @@ class BBCIE(BBCCoUkIE):
         },
         'playlist_count': 2,
     }, {
-        # article with multiple videos embedded with data-media-meta (more videos)
+        # article with multiple videos embedded with data-playable (more videos)
         'url': 'http://www.bbc.com/news/business-28299555',
         'info_dict': {
             'id': 'business-28299555',
@@ -522,6 +521,7 @@ class BBCIE(BBCCoUkIE):
         'skip': 'Save time',
     }, {
         # article with multiple videos embedded with `new SMP()`
+        # broken
         'url': 'http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460',
         'info_dict': {
             'id': '3662a707-0af9-3149-963f-47bea720b460',
@@ -529,7 +529,7 @@ class BBCIE(BBCCoUkIE):
         },
         'playlist_count': 18,
     }, {
-        # single video embedded with mediaAssetPage.init()
+        # single video embedded with data-playable containing vpid
         'url': 'http://www.bbc.com/news/world-europe-32041533',
         'info_dict': {
             'id': 'p02mprgb',
@@ -544,9 +544,9 @@ class BBCIE(BBCCoUkIE):
             'skip_download': True,
         }
     }, {
-        # article with single video embedded with data-media-meta containing
-        # direct video links (for now these are extracted) and playlist.xml (with
-        # media items as f4m and m3u8 - currently unsupported)
+        # article with single video embedded with data-playable containing XML playlist
+        # with direct video links as progressiveDownloadUrl (for now these are extracted)
+        # and playlist with f4m and m3u8 as streamingUrl
         'url': 'http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu',
         'info_dict': {
             'id': '150615_telabyad_kentin_cogu',
@@ -560,20 +560,7 @@ class BBCIE(BBCCoUkIE):
             'skip_download': True,
         }
     }, {
-        # single video embedded with playlist.sxml in data-playable
-        'url': 'http://www.bbc.com/turkce/multimedya/2015/10/151010_vid_ankara_patlama_ani',
-        'info_dict': {
-            'id': '151010_vid_ankara_patlama_ani',
-            'ext': 'mp4',
-            'title': "Ankara'da patlama anı",
-            'timestamp': 1444480325,
-            'upload_date': '20151010',
-        },
-        'params': {
-            'skip_download': True,
-        }
-    }, {
-        # single video embedded with mediaAssetPage.init() (regional section)
+        # single video embedded with data-playable containing XML playlists (regional section)
         'url': 'http://www.bbc.com/mundo/video_fotos/2015/06/150619_video_honduras_militares_hospitales_corrupcion_aw',
         'info_dict': {
             'id': '150619_video_honduras_militares_hospitales_corrupcion_aw',
@@ -629,7 +616,7 @@ class BBCIE(BBCCoUkIE):
             'skip_download': True,
         }
     }, {
-        # single video with playlist.sxml URL
+        # single video with playlist.sxml URL in playlist param
         'url': 'http://www.bbc.com/sport/0/football/33653409',
         'info_dict': {
             'id': 'p02xycnp',
@@ -643,7 +630,7 @@ class BBCIE(BBCCoUkIE):
             'skip_download': True,
         }
     }, {
-        # article with multiple videos embedded with playlist.sxml
+        # article with multiple videos embedded with playlist.sxml in playlist param
         'url': 'http://www.bbc.com/sport/0/football/34475836',
         'info_dict': {
             'id': '34475836',

From ae8bdfd1a1548c83ab7df378096da927b5374a29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 00:25:09 +0600
Subject: [PATCH 101/150] [bbc] Extract article JSON and actualize tests

---
 youtube_dl/extractor/bbc.py | 45 ++++++++++++++++++++++++-------------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 4eae4f52e..b98db95b9 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -11,6 +11,7 @@ from ..utils import (
     int_or_none,
     parse_duration,
     parse_iso8601,
+    remove_end,
     unescapeHTML,
 )
 from ..compat import compat_HTTPError
@@ -533,7 +534,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/news/world-europe-32041533',
         'info_dict': {
             'id': 'p02mprgb',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
             'duration': 47,
             'timestamp': 1427219242,
@@ -552,7 +553,6 @@ class BBCIE(BBCCoUkIE):
             'id': '150615_telabyad_kentin_cogu',
             'ext': 'mp4',
             'title': "YPG: Tel Abyad'ın tamamı kontrolümüzde",
-            'duration': 47,
             'timestamp': 1434397334,
             'upload_date': '20150615',
         },
@@ -566,7 +566,6 @@ class BBCIE(BBCCoUkIE):
             'id': '150619_video_honduras_militares_hospitales_corrupcion_aw',
             'ext': 'mp4',
             'title': 'Honduras militariza sus hospitales por nuevo escándalo de corrupción',
-            'duration': 87,
             'timestamp': 1434713142,
             'upload_date': '20150619',
         },
@@ -578,7 +577,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/news/video_and_audio/must_see/33376376',
         'info_dict': {
             'id': 'p02w6qjc',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': '''Judge Mindy Glazer: "I'm sorry to see you here... I always wondered what happened to you"''',
             'duration': 56,
         },
@@ -605,11 +604,11 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/autos/story/20130513-hyundais-rock-star',
         'info_dict': {
             'id': 'p018zqqg',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Hyundai Santa Fe Sport: Rock star',
             'description': 'md5:b042a26142c4154a6e472933cf20793d',
-            'timestamp': 1368473503,
-            'upload_date': '20130513',
+            'timestamp': 1415867444,
+            'upload_date': '20141113',
         },
         'params': {
             # rtmp download
@@ -620,9 +619,8 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/sport/0/football/33653409',
         'info_dict': {
             'id': 'p02xycnp',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?',
-            'description': 'md5:398fca0e2e701c609d726e034fa1fc89',
             'duration': 140,
         },
         'params': {
@@ -697,11 +695,26 @@ class BBCIE(BBCCoUkIE):
 
         webpage = self._download_webpage(url, playlist_id)
 
-        timestamp = parse_iso8601(self._search_regex(
-            [r'"datePublished":\s*"([^"]+)',
-             r'<meta[^>]+property="article:published_time"[^>]+content="([^"]+)"',
-             r'itemprop="datePublished"[^>]+datetime="([^"]+)"'],
-            webpage, 'date', default=None))
+        timestamp = None
+        playlist_title = None
+        playlist_description = None
+
+        ld = self._parse_json(
+            self._search_regex(
+                r'(?s)<script type="application/ld\+json">(.+?)</script>',
+                webpage, 'ld json', default='{}'),
+            playlist_id, fatal=False)
+        if ld:
+            timestamp = parse_iso8601(ld.get('datePublished'))
+            playlist_title = ld.get('headline')
+            playlist_description = ld.get('articleBody')
+
+        if not timestamp:
+            timestamp = parse_iso8601(self._search_regex(
+                [r'<meta[^>]+property="article:published_time"[^>]+content="([^"]+)"',
+                 r'itemprop="datePublished"[^>]+datetime="([^"]+)"',
+                 r'"datePublished":\s*"([^"]+)',],
+                webpage, 'date', default=None))
 
         entries = []
 
@@ -754,8 +767,8 @@ class BBCIE(BBCCoUkIE):
                                 playlist.get('progressiveDownloadUrl'), playlist_id, timestamp))
 
         if entries:
-            playlist_title = self._og_search_title(webpage)
-            playlist_description = self._og_search_description(webpage, default=None)
+            playlist_title = playlist_title or remove_end(self._og_search_title(webpage), ' - BBC News')
+            playlist_description = playlist_description or self._og_search_description(webpage, default=None)
             return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
 
         # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)

From 55ebae26f937a6a3a1fcb78d2f797972080f8b88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 00:37:39 +0600
Subject: [PATCH 102/150] [bbc] Prefer iptv-all mediaset

---
 youtube_dl/extractor/bbc.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index b98db95b9..f3ded3f9c 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -493,6 +493,9 @@ class BBCIE(BBCCoUkIE):
     _VALID_URL = r'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P<id>[^/#?]+)'
 
     _MEDIASELECTOR_URLS = [
+        # Provides HQ HLS streams but fails with geolocation in some cases when it's
+        # even not geo restricted at all
+        'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/iptv-all/vpid/%s',
         # Provides more formats, namely direct mp4 links, but fails on some videos with
         # notukerror for non UK (?) users (e.g.
         # http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
@@ -534,8 +537,9 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/news/world-europe-32041533',
         'info_dict': {
             'id': 'p02mprgb',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
+            'description': 'md5:2868290467291b37feda7863f7a83f54',
             'duration': 47,
             'timestamp': 1427219242,
             'upload_date': '20150324',
@@ -577,7 +581,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/news/video_and_audio/must_see/33376376',
         'info_dict': {
             'id': 'p02w6qjc',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': '''Judge Mindy Glazer: "I'm sorry to see you here... I always wondered what happened to you"''',
             'duration': 56,
         },
@@ -604,7 +608,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/autos/story/20130513-hyundais-rock-star',
         'info_dict': {
             'id': 'p018zqqg',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Hyundai Santa Fe Sport: Rock star',
             'description': 'md5:b042a26142c4154a6e472933cf20793d',
             'timestamp': 1415867444,
@@ -619,7 +623,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/sport/0/football/33653409',
         'info_dict': {
             'id': 'p02xycnp',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?',
             'duration': 140,
         },

From 6f7893653c86c620099d7bf0e3bd4951be8b4ad1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 00:39:28 +0600
Subject: [PATCH 103/150] [bbc] PEP 8

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index f3ded3f9c..68995f81e 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -717,7 +717,7 @@ class BBCIE(BBCCoUkIE):
             timestamp = parse_iso8601(self._search_regex(
                 [r'<meta[^>]+property="article:published_time"[^>]+content="([^"]+)"',
                  r'itemprop="datePublished"[^>]+datetime="([^"]+)"',
-                 r'"datePublished":\s*"([^"]+)',],
+                 r'"datePublished":\s*"([^"]+)'],
                 webpage, 'date', default=None))
 
         entries = []

From f4076bb73686484310a3a2b7c0c4c20db6e186bc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 19:56:55 +0100
Subject: [PATCH 104/150] [vimeo] extract m3u8 manifest and bitrate

---
 youtube_dl/extractor/vimeo.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 7dd52627d..93638d6b2 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -397,8 +397,12 @@ class VimeoIE(VimeoBaseInfoExtractor):
                     'format_id': format_id,
                     'width': file_info.get('width'),
                     'height': file_info.get('height'),
+                    'tbr': file_info.get('bitrate'),
                 })
         formats = []
+        hls = config_files.get("hls")
+        if hls:
+            formats = self._extract_m3u8_formats(hls['all'], video_id, m3u8_id='hls')
         for key in ('other', 'sd', 'hd'):
             formats += files[key]
         if len(formats) == 0:

From 58cd7e173e70ae40a79bb10e08b2c2ea02bc8248 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 12:28:12 +0100
Subject: [PATCH 105/150] [adultswim] detect when video needs authentication

---
 youtube_dl/extractor/adultswim.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 27de07587..0eb21b16d 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -41,7 +41,8 @@ class AdultSwimIE(InfoExtractor):
             'id': 'rQxZvXQ4ROaSOqq-or2Mow',
             'title': 'Rick and Morty - Pilot',
             'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
-        }
+        },
+        'skip': 'This video is only available for registered users',
     }, {
         'url': 'http://www.adultswim.com/videos/playlists/american-parenting/putting-francine-out-of-business/',
         'playlist': [
@@ -84,7 +85,10 @@ class AdultSwimIE(InfoExtractor):
     def find_video_info(collection, slug):
         for video in collection.get('videos'):
             if video.get('slug') == slug:
-                return video
+                if video.get('auth'):
+                    raise ExtractorError('This video is only available for registered users', expected=True)
+                else:
+                    return video
 
     @staticmethod
     def find_collection_by_linkURL(collections, linkURL):
@@ -97,7 +101,10 @@ class AdultSwimIE(InfoExtractor):
         for collection in collections:
             for video in collection.get('videos'):
                 if video.get('slug') == slug:
-                    return collection, video
+                    if video.get('auth'):
+                        raise ExtractorError('This video is only available for registered users', expected=True)
+                    else:
+                        return collection, video
         return None, None
 
     def _real_extract(self, url):
@@ -128,6 +135,8 @@ class AdultSwimIE(InfoExtractor):
             if video_info is None:
                 if bootstrapped_data.get('slugged_video', {}).get('slug') == episode_path:
                     video_info = bootstrapped_data['slugged_video']
+                    if video_info.get('auth'):
+                        raise ExtractorError('This video is only available for registered users', expected=True)
                 else:
                     raise ExtractorError('Unable to find video info')
 

From 00cde0b8dc3a534a929b88da086cc540dc212ea4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 17:57:05 +0100
Subject: [PATCH 106/150] [adultswim] raise ExtractorError if no clips in
 video_info

---
 youtube_dl/extractor/adultswim.py | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 0eb21b16d..35606878d 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -85,10 +85,7 @@ class AdultSwimIE(InfoExtractor):
     def find_video_info(collection, slug):
         for video in collection.get('videos'):
             if video.get('slug') == slug:
-                if video.get('auth'):
-                    raise ExtractorError('This video is only available for registered users', expected=True)
-                else:
-                    return video
+                return video
 
     @staticmethod
     def find_collection_by_linkURL(collections, linkURL):
@@ -101,10 +98,7 @@ class AdultSwimIE(InfoExtractor):
         for collection in collections:
             for video in collection.get('videos'):
                 if video.get('slug') == slug:
-                    if video.get('auth'):
-                        raise ExtractorError('This video is only available for registered users', expected=True)
-                    else:
-                        return collection, video
+                    return collection, video
         return None, None
 
     def _real_extract(self, url):
@@ -135,15 +129,18 @@ class AdultSwimIE(InfoExtractor):
             if video_info is None:
                 if bootstrapped_data.get('slugged_video', {}).get('slug') == episode_path:
                     video_info = bootstrapped_data['slugged_video']
-                    if video_info.get('auth'):
-                        raise ExtractorError('This video is only available for registered users', expected=True)
                 else:
                     raise ExtractorError('Unable to find video info')
 
             show = bootstrapped_data['show']
             show_title = show['title']
             stream = video_info.get('stream')
-            clips = [stream] if stream else video_info['clips']
+            clips = [stream] if stream else video_info.get('clips')
+            if not clips:
+                if video_info.get('auth'):
+                    raise ExtractorError('This video is only available for registered users', expected=True)
+                else:
+                    raise ExtractorError('Unable to find clips')
             segment_ids = [clip['videoPlaybackID'] for clip in clips]
 
         episode_id = video_info['id']

From 75f105d455bd0a2226ce4fa78b56c7a344ad6bd8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 01:04:41 +0600
Subject: [PATCH 107/150] [adultswim] Clarify no media message

---
 youtube_dl/extractor/adultswim.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 35606878d..130afe791 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -137,10 +137,11 @@ class AdultSwimIE(InfoExtractor):
             stream = video_info.get('stream')
             clips = [stream] if stream else video_info.get('clips')
             if not clips:
-                if video_info.get('auth'):
-                    raise ExtractorError('This video is only available for registered users', expected=True)
-                else:
-                    raise ExtractorError('Unable to find clips')
+                raise ExtractorError(
+                    'This video is only available via cable service provider subscription that'
+                    ' is not currently supported. You may want to use --cookies.'
+                    if video_info.get('auth') is True else 'Unable to find stream or clips',
+                    expected=True)
             segment_ids = [clip['videoPlaybackID'] for clip in clips]
 
         episode_id = video_info['id']

From e5c209a1bcea206bee684914599c84acf886487c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 20:34:10 +0100
Subject: [PATCH 108/150] [vimeo] add parameters to _extract_m3u8_formats and
 sort formats

---
 youtube_dl/extractor/vimeo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 93638d6b2..2ea5f0b79 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -402,11 +402,12 @@ class VimeoIE(VimeoBaseInfoExtractor):
         formats = []
         hls = config_files.get("hls")
         if hls:
-            formats = self._extract_m3u8_formats(hls['all'], video_id, m3u8_id='hls')
+            formats = self._extract_m3u8_formats(hls['all'], video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
         for key in ('other', 'sd', 'hd'):
             formats += files[key]
         if len(formats) == 0:
             raise ExtractorError('No known codec found')
+        self._sort_formats(formats)
 
         subtitles = {}
         text_tracks = config['request'].get('text_tracks')

From fff496c689e44ac96909cf55c9ae746fb6b14e07 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 20:45:34 +0100
Subject: [PATCH 109/150] [vimeo] remove check for empty formats

---
 youtube_dl/extractor/vimeo.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 2ea5f0b79..2051ac9de 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -405,8 +405,6 @@ class VimeoIE(VimeoBaseInfoExtractor):
             formats = self._extract_m3u8_formats(hls['all'], video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
         for key in ('other', 'sd', 'hd'):
             formats += files[key]
-        if len(formats) == 0:
-            raise ExtractorError('No known codec found')
         self._sort_formats(formats)
 
         subtitles = {}

From 35a3ff1d337edd527c73db133d87ed23ca4469f5 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 21:05:29 +0100
Subject: [PATCH 110/150] [vimeo] always convert width, height and bitrate to
 int

---
 youtube_dl/extractor/vimeo.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 2051ac9de..97590d220 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -395,14 +395,14 @@ class VimeoIE(VimeoBaseInfoExtractor):
                     'ext': codec_extension,
                     'url': video_url,
                     'format_id': format_id,
-                    'width': file_info.get('width'),
-                    'height': file_info.get('height'),
-                    'tbr': file_info.get('bitrate'),
+                    'width': int_or_none(file_info.get('width')),
+                    'height': int_or_none(file_info.get('height')),
+                    'tbr': int_or_none(file_info.get('bitrate')),
                 })
         formats = []
-        hls = config_files.get("hls")
-        if hls:
-            formats = self._extract_m3u8_formats(hls['all'], video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
+        m3u8_url = config_files.get('hls', {}).get('all')
+        if m3u8_url:
+            formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
         for key in ('other', 'sd', 'hd'):
             formats += files[key]
         self._sort_formats(formats)

From 68f3b61f0e278891b5a0d6557d297dcfd3fb53cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 02:22:42 +0600
Subject: [PATCH 111/150] [vimeo] Update tests

---
 youtube_dl/extractor/vimeo.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 97590d220..9c173ce07 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -84,12 +84,12 @@ class VimeoIE(VimeoBaseInfoExtractor):
             'info_dict': {
                 'id': '56015672',
                 'ext': 'mp4',
-                "upload_date": "20121220",
-                "description": "This is a test case for youtube-dl.\nFor more information, see github.com/rg3/youtube-dl\nTest chars: \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
-                "uploader_id": "user7108434",
-                "uploader": "Filippo Valsorda",
-                "title": "youtube-dl test video - \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
-                "duration": 10,
+                'title': "youtube-dl test video - \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
+                'description': 'md5:2d3305bad981a06ff79f027f19865021',
+                'upload_date': '20121220',
+                'uploader_id': 'user7108434',
+                'uploader': 'Filippo Valsorda',
+                'duration': 10,
             },
         },
         {
@@ -102,7 +102,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'uploader_id': 'openstreetmapus',
                 'uploader': 'OpenStreetMap US',
                 'title': 'Andy Allan - Putting the Carto into OpenStreetMap Cartography',
-                'description': 'md5:380943ec71b89736ff4bf27183233d09',
+                'description': 'md5:fd69a7b8d8c34a4e1d2ec2e4afd6ec30',
                 'duration': 1595,
             },
         },
@@ -156,7 +156,6 @@ class VimeoIE(VimeoBaseInfoExtractor):
         },
         {
             'url': 'http://vimeo.com/76979871',
-            'md5': '3363dd6ffebe3784d56f4132317fd446',
             'note': 'Video with subtitles',
             'info_dict': {
                 'id': '76979871',

From 7c84562945cc5c37979b0d8d786118b1fe31f731 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 02:31:37 +0600
Subject: [PATCH 112/150] [vimeo] Fix password protected videos

---
 youtube_dl/extractor/vimeo.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 9c173ce07..346eb60b4 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -39,10 +39,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
             return
         self.report_login()
         webpage = self._download_webpage(self._LOGIN_URL, None, False)
-        token = self._extract_xsrft(webpage)
-        vuid = self._search_regex(
-            r'["\']vuid["\']\s*:\s*(["\'])(?P<vuid>.+?)\1',
-            webpage, 'vuid', group='vuid')
+        token, vuid = self._extract_xsrft_and_vuid(webpage)
         data = urlencode_postdata({
             'action': 'login',
             'email': username,
@@ -56,10 +53,14 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         login_request.add_header('Referer', self._LOGIN_URL)
         self._download_webpage(login_request, None, False, 'Wrong login info')
 
-    def _extract_xsrft(self, webpage):
-        return self._search_regex(
+    def _extract_xsrft_and_vuid(self, webpage):
+        xsrft = self._search_regex(
             r'xsrft\s*[=:]\s*(?P<q>["\'])(?P<xsrft>.+?)(?P=q)',
             webpage, 'login token', group='xsrft')
+        vuid = self._search_regex(
+            r'["\']vuid["\']\s*:\s*(["\'])(?P<vuid>.+?)\1',
+            webpage, 'vuid', group='vuid')
+        return xsrft, vuid
 
 
 class VimeoIE(VimeoBaseInfoExtractor):
@@ -201,7 +202,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
-        token = self._extract_xsrft(webpage)
+        token, vuid = self._extract_xsrft_and_vuid(webpage)
         data = urlencode_postdata({
             'password': password,
             'token': token,
@@ -211,6 +212,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             url = url.replace('http://', 'https://')
         password_request = compat_urllib_request.Request(url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        password_request.add_header('Cookie', 'clip_v=1; vuid=%s' % vuid)
         password_request.add_header('Referer', url)
         return self._download_webpage(
             password_request, video_id,
@@ -465,7 +467,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
         if password is None:
             raise ExtractorError('This album is protected by a password, use the --video-password option', expected=True)
         fields = self._hidden_inputs(login_form)
-        token = self._extract_xsrft(webpage)
+        token, vuid = self._extract_xsrft_and_vuid(webpage)
         fields['token'] = token
         fields['password'] = password
         post = urlencode_postdata(fields)
@@ -474,6 +476,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
         password_url = compat_urlparse.urljoin(page_url, password_path)
         password_request = compat_urllib_request.Request(password_url, post)
         password_request.add_header('Content-type', 'application/x-www-form-urlencoded')
+        password_request.add_header('Cookie', 'vuid=%s' % vuid)
         self._set_cookie('vimeo.com', 'xsrft', token)
 
         return self._download_webpage(

From 70cb4d51c91c5b6c72b8564258cb161bcc68626c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 00:45:23 +0100
Subject: [PATCH 113/150] [bild] extract info from json request

---
 youtube_dl/extractor/bild.py | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/bild.py b/youtube_dl/extractor/bild.py
index 4d8cce1ef..ea84f20f2 100644
--- a/youtube_dl/extractor/bild.py
+++ b/youtube_dl/extractor/bild.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
-    fix_xml_ampersands,
+    unescapeHTML,
 )
 
 
@@ -17,7 +17,7 @@ class BildIE(InfoExtractor):
         'info_dict': {
             'id': '38184146',
             'ext': 'mp4',
-            'title': 'BILD hat sie getestet',
+            'title': 'Das können die  neuen iPads ',
             'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 196,
             'description': 'Mit dem iPad Air 2 und dem iPad Mini 3 hat Apple zwei neue Tablet-Modelle präsentiert. BILD-Reporter Sven Stein durfte die Geräte bereits testen. ',
@@ -27,16 +27,13 @@ class BildIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        xml_url = url.split(".bild.html")[0] + ",view=xml.bild.xml"
-        doc = self._download_xml(xml_url, video_id, transform_source=fix_xml_ampersands)
-
-        duration = int_or_none(doc.attrib.get('duration'), scale=1000)
+        video_data = self._download_json(url.split(".bild.html")[0] + ",view=json.bild.html", video_id)
 
         return {
             'id': video_id,
-            'title': doc.attrib['ueberschrift'],
-            'description': doc.attrib.get('text'),
-            'url': doc.attrib['src'],
-            'thumbnail': doc.attrib.get('img'),
-            'duration': duration,
+            'title': unescapeHTML(video_data['title']),
+            'description': unescapeHTML(video_data.get('description')),
+            'url': video_data['clipList'][0]['srces'][0]['src'],
+            'thumbnail': video_data.get('poster'),
+            'duration': int_or_none(video_data.get('durationSec')),
         }

From d8348c351d5ed4bb820b190e42b9a02e2dfc27a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 19:16:51 +0600
Subject: [PATCH 114/150] [bild] Strip title

---
 youtube_dl/extractor/bild.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/bild.py b/youtube_dl/extractor/bild.py
index ea84f20f2..1a0184861 100644
--- a/youtube_dl/extractor/bild.py
+++ b/youtube_dl/extractor/bild.py
@@ -17,21 +17,22 @@ class BildIE(InfoExtractor):
         'info_dict': {
             'id': '38184146',
             'ext': 'mp4',
-            'title': 'Das können die  neuen iPads ',
+            'title': 'Das können die  neuen iPads',
+            'description': 'md5:a4058c4fa2a804ab59c00d7244bbf62f',
             'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 196,
-            'description': 'Mit dem iPad Air 2 und dem iPad Mini 3 hat Apple zwei neue Tablet-Modelle präsentiert. BILD-Reporter Sven Stein durfte die Geräte bereits testen. ',
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        video_data = self._download_json(url.split(".bild.html")[0] + ",view=json.bild.html", video_id)
+        video_data = self._download_json(
+            url.split('.bild.html')[0] + ',view=json.bild.html', video_id)
 
         return {
             'id': video_id,
-            'title': unescapeHTML(video_data['title']),
+            'title': unescapeHTML(video_data['title']).strip(),
             'description': unescapeHTML(video_data.get('description')),
             'url': video_data['clipList'][0]['srces'][0]['src'],
             'thumbnail': video_data.get('poster'),

From 03e3b4e1198631b30914e8669b01bf1825a4385c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 9 Oct 2015 23:36:31 +0100
Subject: [PATCH 115/150] [expotv] parse m3u8 manifest

---
 youtube_dl/extractor/expotv.py | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/expotv.py b/youtube_dl/extractor/expotv.py
index a38b773e8..23a38c7c1 100644
--- a/youtube_dl/extractor/expotv.py
+++ b/youtube_dl/extractor/expotv.py
@@ -33,20 +33,24 @@ class ExpoTVIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
         player_key = self._search_regex(
             r'<param name="playerKey" value="([^"]+)"', webpage, 'player key')
-        config_url = 'http://client.expotv.com/video/config/%s/%s' % (
-            video_id, player_key)
         config = self._download_json(
-            config_url, video_id,
-            note='Downloading video configuration')
+                'http://client.expotv.com/video/config/%s/%s' % (video_id, player_key),
+                video_id,
+                note='Downloading video configuration')
 
-        formats = [{
-            'url': fcfg['file'],
-            'height': int_or_none(fcfg.get('height')),
-            'format_note': fcfg.get('label'),
-            'ext': self._search_regex(
-                r'filename=.*\.([a-z0-9_A-Z]+)&', fcfg['file'],
-                'file extension', default=None),
-        } for fcfg in config['sources']]
+        formats = []
+        for fcfg in config['sources']:
+            if fcfg['type'] == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(fcfg['file'], video_id))
+            else:
+                formats.append({
+                    'url': fcfg['file'],
+                    'height': int_or_none(fcfg.get('height')),
+                    'format_id': fcfg.get('label'),
+                    'ext': self._search_regex(
+                        r'filename=.*\.([a-z0-9_A-Z]+)&', fcfg['file'],
+                        'file extension', default=None),
+                })
         self._sort_formats(formats)
 
         title = self._og_search_title(webpage)

From 7d49502ab0990972bdb479e6bf32dbc8ffdb3e1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 19:28:00 +0600
Subject: [PATCH 116/150] [bild] Make more robust and improve hls extraction

---
 youtube_dl/extractor/expotv.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/expotv.py b/youtube_dl/extractor/expotv.py
index 23a38c7c1..1585a03bb 100644
--- a/youtube_dl/extractor/expotv.py
+++ b/youtube_dl/extractor/expotv.py
@@ -34,22 +34,25 @@ class ExpoTVIE(InfoExtractor):
         player_key = self._search_regex(
             r'<param name="playerKey" value="([^"]+)"', webpage, 'player key')
         config = self._download_json(
-                'http://client.expotv.com/video/config/%s/%s' % (video_id, player_key),
-                video_id,
-                note='Downloading video configuration')
+            'http://client.expotv.com/video/config/%s/%s' % (video_id, player_key),
+            video_id, 'Downloading video configuration')
 
         formats = []
         for fcfg in config['sources']:
-            if fcfg['type'] == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(fcfg['file'], video_id))
+            media_url = fcfg.get('file')
+            if not media_url:
+                continue
+            if fcfg.get('type') == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    media_url, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls'))
             else:
                 formats.append({
-                    'url': fcfg['file'],
+                    'url': media_url,
                     'height': int_or_none(fcfg.get('height')),
                     'format_id': fcfg.get('label'),
                     'ext': self._search_regex(
-                        r'filename=.*\.([a-z0-9_A-Z]+)&', fcfg['file'],
-                        'file extension', default=None),
+                        r'filename=.*\.([a-z0-9_A-Z]+)&', media_url,
+                        'file extension', default=None) or fcfg.get('type'),
                 })
         self._sort_formats(formats)
 

From da4daed5ef77b4a7219b1786978065a3606b85bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 19:36:23 +0600
Subject: [PATCH 117/150] [vimeo] Do not fail when no hls formats

---
 youtube_dl/extractor/vimeo.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 346eb60b4..6ee3069a8 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -403,7 +403,10 @@ class VimeoIE(VimeoBaseInfoExtractor):
         formats = []
         m3u8_url = config_files.get('hls', {}).get('all')
         if m3u8_url:
-            formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
+            m3u8_formats = self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
+            if m3u8_formats:
+                formats.append(m3u8_formats)
         for key in ('other', 'sd', 'hd'):
             formats += files[key]
         self._sort_formats(formats)

From 1bd390358205728823ff38a51b12ae35f9468929 Mon Sep 17 00:00:00 2001
From: PC <tioocnt@yandex.com>
Date: Tue, 6 Oct 2015 22:28:58 +0100
Subject: [PATCH 118/150] chaturbate streams

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/chaturbate.py | 24 ++++++++++++++++++++++++
 2 files changed, 25 insertions(+)
 create mode 100644 youtube_dl/extractor/chaturbate.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3ace1cc2c..75720843c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -76,6 +76,7 @@ from .cbssports import CBSSportsIE
 from .ccc import CCCIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
+from .chaturbate import ChaturbateIE
 from .chilloutzone import ChilloutzoneIE
 from .chirbit import (
     ChirbitIE,
diff --git a/youtube_dl/extractor/chaturbate.py b/youtube_dl/extractor/chaturbate.py
new file mode 100644
index 000000000..5e24e1e4f
--- /dev/null
+++ b/youtube_dl/extractor/chaturbate.py
@@ -0,0 +1,24 @@
+# encoding: utf-8
+
+from .common import InfoExtractor
+
+
+class ChaturbateIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?chaturbate\.com/(?P<id>[^/]+)/?$'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        m3u8_url = self._search_regex(r"'(https?://.*?\.m3u8)'", webpage, 'playlist')
+
+        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
+
+        return {
+            'id': video_id,
+            'title': self._live_title(video_id),
+            'description': self._html_search_meta('description', webpage, 'description'),
+            'is_live': True,
+            'thumbnail': 'https://cdn-s.highwebmedia.com/uHK3McUtGCG3SMFcd4ZJsRv8/roomimage/%s.jpg' % (video_id,),
+            'formats': formats,
+        }

From 0f61db4469db4c17712b8a3f9e527e845ec96ed4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 20:35:22 +0600
Subject: [PATCH 119/150] [chaturbate] Improve and capture error message

---
 youtube_dl/extractor/chaturbate.py | 36 +++++++++++++++++++++++++-----
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/chaturbate.py b/youtube_dl/extractor/chaturbate.py
index 5e24e1e4f..0b67ba67d 100644
--- a/youtube_dl/extractor/chaturbate.py
+++ b/youtube_dl/extractor/chaturbate.py
@@ -1,24 +1,50 @@
-# encoding: utf-8
+from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import ExtractorError
 
 
 class ChaturbateIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?chaturbate\.com/(?P<id>[^/]+)/?$'
+    _VALID_URL = r'https?://(?:[^/]+\.)?chaturbate\.com/(?P<id>[^/?#]+)'
+    _TESTS = [{
+        'url': 'https://www.chaturbate.com/siswet19/',
+        'info_dict': {
+            'id': 'siswet19',
+            'ext': 'mp4',
+            'title': 're:^siswet19 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'age_limit': 18,
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }, {
+        'url': 'https://en.chaturbate.com/siswet19/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
-        m3u8_url = self._search_regex(r"'(https?://.*?\.m3u8)'", webpage, 'playlist')
+        m3u8_url = self._search_regex(
+            r'src=(["\'])(?P<url>http.+?\.m3u8.*?)\1', webpage,
+            'playlist', default=None, group='url')
+
+        if not m3u8_url:
+            error = self._search_regex(
+                r'<span[^>]+class=(["\'])desc_span\1[^>]*>(?P<error>[^<]+)</span>',
+                webpage, 'error', group='error')
+            raise ExtractorError(error, expected=True)
 
         formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
 
         return {
             'id': video_id,
             'title': self._live_title(video_id),
-            'description': self._html_search_meta('description', webpage, 'description'),
+            'thumbnail': 'https://cdn-s.highwebmedia.com/uHK3McUtGCG3SMFcd4ZJsRv8/roomimage/%s.jpg' % video_id,
+            'age_limit': self._rta_search(webpage),
             'is_live': True,
-            'thumbnail': 'https://cdn-s.highwebmedia.com/uHK3McUtGCG3SMFcd4ZJsRv8/roomimage/%s.jpg' % (video_id,),
             'formats': formats,
         }

From f101079ae02560fee2df1ea0cacb1989eefc8e6c Mon Sep 17 00:00:00 2001
From: AndroKev <AndroKev@users.noreply.github.com>
Date: Sun, 11 Oct 2015 19:17:24 +0200
Subject: [PATCH 120/150] [downloader/rtmp] Respect --no-continue option

now when downloading a "rtmp-file" the --no-continue option works!
---
 youtube_dl/downloader/rtmp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 7d19bb808..f1d219ba9 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -105,7 +105,7 @@ class RtmpFD(FileDownloader):
         protocol = info_dict.get('rtmp_protocol', None)
         real_time = info_dict.get('rtmp_real_time', False)
         no_resume = info_dict.get('no_resume', False)
-        continue_dl = info_dict.get('continuedl', True)
+        continue_dl = self.params.get('continuedl', True)
 
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)

From 964e7b2dd09099bd021a770522684716f341db43 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Oct 2015 00:43:54 +0600
Subject: [PATCH 121/150] [downloader/common] Always skip "already downloaded"
 check when outputting to stdout

---
 youtube_dl/downloader/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 97e755d4b..29a4500d3 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -325,7 +325,7 @@ class FileDownloader(object):
         )
 
         # Check file already present
-        if filename != '-' and nooverwrites_and_exists or continuedl_and_exists:
+        if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
             self.report_file_already_downloaded(filename)
             self._hook_progress({
                 'filename': filename,

From 57d1db8dd00b833c08b3ac74fbd589a0216775f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 11 Oct 2015 22:45:13 +0200
Subject: [PATCH 122/150] [rtbf] Fix extraction (closes #7133)

---
 youtube_dl/extractor/rtbf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtbf.py b/youtube_dl/extractor/rtbf.py
index e4215d546..04a66df90 100644
--- a/youtube_dl/extractor/rtbf.py
+++ b/youtube_dl/extractor/rtbf.py
@@ -36,7 +36,7 @@ class RTBFIE(InfoExtractor):
 
         data = self._parse_json(
             unescapeHTML(self._search_regex(
-                r'data-video="([^"]+)"', webpage, 'data video')),
+                r'data-media="([^"]+)"', webpage, 'data video')),
             video_id)
 
         if data.get('provider').lower() == 'youtube':

From cd7364a89c69c4c19e4b9d4e54d08fd1eff9c51d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 12 Oct 2015 06:37:20 +0200
Subject: [PATCH 123/150] release 2015.10.12

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 5beb2ecd4..dc0354095 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -81,6 +81,7 @@
  - **CBSSports**
  - **CeskaTelevize**
  - **channel9**: Channel 9
+ - **Chaturbate**
  - **Chilloutzone**
  - **chirbit**
  - **chirbit:profile**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index faae8a2d7..0908e963d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.09'
+__version__ = '2015.10.12'

From 73e732eb6b5fb2edd61701ae110cc52fe6836364 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 12 Oct 2015 16:34:55 +0800
Subject: [PATCH 124/150] [vimeo] Fix m3u8 formats

Reported at
https://github.com/rg3/youtube-dl/pull/7126#issuecomment-147327584
---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 6ee3069a8..027f47ee3 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -406,7 +406,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             m3u8_formats = self._extract_m3u8_formats(
                 m3u8_url, video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
             if m3u8_formats:
-                formats.append(m3u8_formats)
+                formats.extend(m3u8_formats)
         for key in ('other', 'sd', 'hd'):
             formats += files[key]
         self._sort_formats(formats)

From ee2edd838a1e8770488e695c380943ded44d0983 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 13 Oct 2015 00:53:05 +0200
Subject: [PATCH 125/150] release 2015.10.13

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0908e963d..aaa43d315 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.12'
+__version__ = '2015.10.13'

From 5946cda7c6f2e4a7eb90fff6f10c66af0ff2a0d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 13 Oct 2015 21:04:39 +0600
Subject: [PATCH 126/150] [beeg] Fix extraction (Closes #7155)

---
 youtube_dl/extractor/beeg.py | 68 +++++++++++++++++++-----------------
 1 file changed, 35 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index b38057f2f..e6c928699 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -1,65 +1,67 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+)
 
 
 class BeegIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?beeg\.com/(?P<id>\d+)'
     _TEST = {
         'url': 'http://beeg.com/5416503',
-        'md5': '1bff67111adb785c51d1b42959ec10e5',
+        'md5': '46c384def73b33dbc581262e5ee67cef',
         'info_dict': {
             'id': '5416503',
             'ext': 'mp4',
             'title': 'Sultry Striptease',
-            'description': 'md5:6db3c6177972822aaba18652ff59c773',
-            'categories': list,  # NSFW
-            'thumbnail': 're:https?://.*\.jpg$',
+            'description': 'md5:d22219c09da287c14bed3d6c37ce4bc2',
+            'timestamp': 1391813355,
+            'upload_date': '20140207',
+            'duration': 383,
+            'tags': list,
             'age_limit': 18,
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
-
-        quality_arr = self._search_regex(
-            r'(?s)var\s+qualityArr\s*=\s*{\s*(.+?)\s*}', webpage, 'quality formats')
-
-        formats = [{
-            'url': fmt[1],
-            'format_id': fmt[0],
-            'height': int(fmt[0][:-1]),
-        } for fmt in re.findall(r"'([^']+)'\s*:\s*'([^']+)'", quality_arr)]
+        video = self._download_json(
+            'http://beeg.com/api/v1/video/%s' % video_id, video_id)
 
+        formats = []
+        for format_id, video_url in video.items():
+            height = self._search_regex(
+                r'^(\d+)[pP]$', format_id, 'height', default=None)
+            if not height:
+                continue
+            formats.append({
+                'url': self._proto_relative_url(video_url.replace('{DATA_MARKERS}', ''), 'http:'),
+                'format_id': format_id,
+                'height': int(height),
+            })
         self._sort_formats(formats)
 
-        title = self._html_search_regex(
-            r'<title>([^<]+)\s*-\s*beeg\.?</title>', webpage, 'title')
+        title = video['title']
+        video_id = video.get('id') or video_id
+        display_id = video.get('code')
+        description = video.get('desc')
 
-        description = self._html_search_regex(
-            r'<meta name="description" content="([^"]*)"',
-            webpage, 'description', fatal=False)
-        thumbnail = self._html_search_regex(
-            r'\'previewer.url\'\s*:\s*"([^"]*)"',
-            webpage, 'thumbnail', fatal=False)
+        timestamp = parse_iso8601(video.get('date'), ' ')
+        duration = int_or_none(video.get('duration'))
 
-        categories_str = self._html_search_regex(
-            r'<meta name="keywords" content="([^"]+)"', webpage, 'categories', fatal=False)
-        categories = (
-            None if categories_str is None
-            else categories_str.split(','))
+        tags = [tag.strip() for tag in video['tags'].split(',')] if video.get('tags') else None
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': title,
             'description': description,
-            'thumbnail': thumbnail,
-            'categories': categories,
+            'timestamp': timestamp,
+            'duration': duration,
+            'tags': tags,
             'formats': formats,
             'age_limit': 18,
         }

From 3eeff489e80838cd2d1b3f55fc96db747386fd50 Mon Sep 17 00:00:00 2001
From: DesweR <egomail@inbox.ru>
Date: Tue, 13 Oct 2015 16:29:16 +0700
Subject: [PATCH 127/150] Extract thumbnail url

---
 youtube_dl/extractor/yandexmusic.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 4098e4629..6842f834f 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -46,6 +46,14 @@ class YandexMusicTrackIE(InfoExtractor):
                 % (data['host'], key, data['ts'] + data['path'], storage[1]))
 
     def _get_track_info(self, track):
+        album = track['albums'][0]
+        a_thumb = None
+
+        if 'coverUri' in album:
+            a_thumb = album['coverUri']
+            if a_thumb:
+                a_thumb = 'http://' + a_thumb.replace('%%', '1000x1000')
+
         return {
             'id': track['id'],
             'ext': 'mp3',
@@ -53,6 +61,7 @@ class YandexMusicTrackIE(InfoExtractor):
             'title': '%s - %s' % (track['artists'][0]['name'], track['title']),
             'filesize': int_or_none(track.get('fileSize')),
             'duration': float_or_none(track.get('durationMs'), 1000),
+            'thumbnail': a_thumb,
         }
 
     def _real_extract(self, url):

From b30c4992a93d411f4f89faf2af153fc580138a90 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 13 Oct 2015 21:14:33 +0200
Subject: [PATCH 128/150] [channel9] Return a single dictionary for single
 videos (closes #7086)

Returning a list is deprecated.
---
 youtube_dl/extractor/channel9.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index 3dfc24f5b..79fd0a30e 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -224,12 +224,12 @@ class Channel9IE(InfoExtractor):
         if contents is None:
             return contents
 
-        authors = self._extract_authors(html)
+        if len(contents) > 1:
+            raise ExtractorError('Got more than one entry')
+        result = contents[0]
+        result['authors'] = self._extract_authors(html)
 
-        for content in contents:
-            content['authors'] = authors
-
-        return contents
+        return result
 
     def _extract_session(self, html, content_path):
         contents = self._extract_content(html, content_path)

From 506e261d2073d8c00d5b43d272e8173cb0d63728 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 13 Oct 2015 21:18:30 +0200
Subject: [PATCH 129/150] [channel9] strip 'session_day'

---
 youtube_dl/extractor/channel9.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index 79fd0a30e..1ce004932 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -158,7 +158,7 @@ class Channel9IE(InfoExtractor):
 
     def _extract_session_day(self, html):
         m = re.search(r'<li class="day">\s*<a href="/Events/[^"]+">(?P<day>[^<]+)</a>\s*</li>', html)
-        return m.group('day') if m is not None else None
+        return m.group('day').strip() if m is not None else None
 
     def _extract_session_room(self, html):
         m = re.search(r'<li class="room">\s*(?P<room>.+?)\s*</li>', html)

From 3dc582e5ea69af4ad7f51d30c1d87cf93aa6b72b Mon Sep 17 00:00:00 2001
From: kaspi <je326@hotmail.com>
Date: Mon, 12 Oct 2015 01:25:57 -0400
Subject: [PATCH 130/150] [fczenit] Add extractor

Closes #7143.
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/fczenit.py  | 41 ++++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+)
 create mode 100644 youtube_dl/extractor/fczenit.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 75720843c..f6d185818 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -167,6 +167,7 @@ from .extremetube import ExtremeTubeIE
 from .facebook import FacebookIE
 from .faz import FazIE
 from .fc2 import FC2IE
+from .fczenit import FczenitIE
 from .firstpost import FirstpostIE
 from .firsttv import FirstTVIE
 from .fivemin import FiveMinIE
diff --git a/youtube_dl/extractor/fczenit.py b/youtube_dl/extractor/fczenit.py
new file mode 100644
index 000000000..f1f150ef2
--- /dev/null
+++ b/youtube_dl/extractor/fczenit.py
@@ -0,0 +1,41 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class FczenitIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?fc-zenit\.ru/video/gl(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://fc-zenit.ru/video/gl6785/',
+        'md5': '458bacc24549173fe5a5aa29174a5606',
+        'info_dict': {
+            'id': '6785',
+            'ext': 'mp4',
+            'title': '«Зенит-ТВ»: как Олег Шатов играл против «Урала»',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        video_title = self._html_search_regex(r'<div class=\"photoalbum__title\">([^<]+)', webpage, 'title')
+
+        bitrates_raw = self._html_search_regex(r'bitrates:.*\n(.*)\]', webpage, 'video URL')
+        bitrates = re.findall(r'url:.?\'(.+?)\'.*?bitrate:.?([0-9]{3}?)', bitrates_raw)
+
+        formats = [{
+            'url': furl,
+            'tbr': tbr,
+        } for furl, tbr in bitrates]
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': video_title,
+            'formats': formats,
+        }

From 26669ea3cf596f2ea4bce9e21ce73c1d8fc3ff72 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 16:51:37 +0100
Subject: [PATCH 131/150] [5min] extract more video info and formats

Closes #7124.
---
 youtube_dl/extractor/fivemin.py | 84 ++++++++++++++++++++++++++++-----
 1 file changed, 71 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/fivemin.py b/youtube_dl/extractor/fivemin.py
index 157094e8c..2955965d9 100644
--- a/youtube_dl/extractor/fivemin.py
+++ b/youtube_dl/extractor/fivemin.py
@@ -2,11 +2,15 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_str,
     compat_urllib_parse,
+    compat_parse_qs,
+    compat_urllib_parse_urlparse,
+    compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
+    parse_duration,
+    replace_extension,
 )
 
 
@@ -28,6 +32,7 @@ class FiveMinIE(InfoExtractor):
                 'id': '518013791',
                 'ext': 'mp4',
                 'title': 'iPad Mini with Retina Display Review',
+                'duration': 177,
             },
         },
         {
@@ -38,9 +43,52 @@ class FiveMinIE(InfoExtractor):
                 'id': '518086247',
                 'ext': 'mp4',
                 'title': 'How to Make a Next-Level Fruit Salad',
+                'duration': 184,
             },
         },
     ]
+    _ERRORS = {
+        'ErrorVideoNotExist': 'We\'re sorry, but the video you are trying to watch does not exist.',
+        'ErrorVideoNoLongerAvailable': 'We\'re sorry, but the video you are trying to watch is no longer available.',
+        'ErrorVideoRejected': 'We\'re sorry, but the video you are trying to watch has been removed.',
+        'ErrorVideoUserNotGeo': 'We\'re sorry, but the video you are trying to watch cannot be viewed from your current location.',
+        'ErrorVideoLibraryRestriction': 'We\'re sorry, but the video you are trying to watch is currently unavailable for viewing at this domain.',
+        'ErrorExposurePermission': 'We\'re sorry, but the video you are trying to watch is currently unavailable for viewing at this domain.',
+    }
+    _QUALITIES = {
+        1: {
+            'width': 640,
+            'height': 360,
+        },
+        2: {
+            'width': 854,
+            'height': 480,
+        },
+        4: {
+            'width': 1280,
+            'height': 720,
+        },
+        8: {
+            'width': 1920,
+            'height': 1080,
+        },
+        16: {
+            'width': 640,
+            'height': 360,
+        },
+        32: {
+            'width': 854,
+            'height': 480,
+        },
+        64: {
+            'width': 1280,
+            'height': 720,
+        },
+        128: {
+            'width': 640,
+            'height': 360,
+        },
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -59,26 +107,36 @@ class FiveMinIE(InfoExtractor):
             'https://syn.5min.com/handlers/SenseHandler.ashx?' + query,
             video_id)
         if not response['success']:
-            err_msg = response['errorMessage']
-            if err_msg == 'ErrorVideoUserNotGeo':
-                msg = 'Video not available from your location'
-            else:
-                msg = 'Aol said: %s' % err_msg
-            raise ExtractorError(msg, expected=True, video_id=video_id)
+            raise ExtractorError(
+                '%s said: %s' % (
+                    self.IE_NAME,
+                    self._ERRORS.get(response['errorMessage'], response['errorMessage'])),
+                expected=True)
         info = response['binding'][0]
 
-        second_id = compat_str(int(video_id[:-2]) + 1)
         formats = []
-        for quality, height in [(1, 320), (2, 480), (4, 720), (8, 1080)]:
-            if any(r['ID'] == quality for r in info['Renditions']):
+        parsed_video_url = compat_urllib_parse_urlparse(compat_parse_qs(
+            compat_urllib_parse_urlparse(info['EmbededURL']).query)['videoUrl'][0])
+        for rendition in info['Renditions']:
+            if rendition['RenditionType'] == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(rendition['Url'], video_id, m3u8_id='hls'))
+            elif rendition['RenditionType'] == 'aac':
+                continue
+            else:
+                rendition_url = compat_urlparse.urlunparse(parsed_video_url._replace(path=replace_extension(parsed_video_url.path.replace('//', '/%s/' % rendition['ID']), rendition['RenditionType'])))
+                quality = self._QUALITIES.get(rendition['ID'], {})
                 formats.append({
-                    'format_id': compat_str(quality),
-                    'url': 'http://avideos.5min.com/%s/%s/%s_%s.mp4' % (second_id[-3:], second_id, video_id, quality),
-                    'height': height,
+                    'format_id': '%s-%d' % (rendition['RenditionType'], rendition['ID']),
+                    'url': rendition_url,
+                    'width': quality.get('width'),
+                    'height': quality.get('height'),
                 })
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': info['Title'],
+            'thumbnail': info.get('ThumbURL'),
+            'duration': parse_duration(info.get('Duration')),
             'formats': formats,
         }

From 1f36085df94c2addd1175e7e299f6235aca3ac68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 14 Oct 2015 13:41:39 +0200
Subject: [PATCH 132/150] [vimeo] Fix extraction of password protected videos
 (fixes #7169)

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 027f47ee3..fa1b22049 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -212,7 +212,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             url = url.replace('http://', 'https://')
         password_request = compat_urllib_request.Request(url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        password_request.add_header('Cookie', 'clip_v=1; vuid=%s' % vuid)
+        password_request.add_header('Cookie', 'clip_test2=1; vuid=%s' % vuid)
         password_request.add_header('Referer', url)
         return self._download_webpage(
             password_request, video_id,

From 36bb63fad19df5ee419979f875e2265936511644 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 14 Oct 2015 14:13:53 +0100
Subject: [PATCH 133/150] [criterion] fix description extraction

---
 youtube_dl/extractor/criterion.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/criterion.py b/youtube_dl/extractor/criterion.py
index 4fb178165..dedb810a0 100644
--- a/youtube_dl/extractor/criterion.py
+++ b/youtube_dl/extractor/criterion.py
@@ -27,9 +27,7 @@ class CriterionIE(InfoExtractor):
         final_url = self._search_regex(
             r'so.addVariable\("videoURL", "(.+?)"\)\;', webpage, 'video url')
         title = self._og_search_title(webpage)
-        description = self._html_search_regex(
-            r'<meta name="description" content="(.+?)" />',
-            webpage, 'video description')
+        description = self._html_search_meta('description', webpage)
         thumbnail = self._search_regex(
             r'so.addVariable\("thumbnailURL", "(.+?)"\)\;',
             webpage, 'thumbnail url')

From 7a6d76a64d8a89a08bb79791506fc18b993c4580 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Oct 2015 20:49:39 +0600
Subject: [PATCH 134/150] [extractor/common] Require closing quote in
 _og_regexes (Closes #7174)

E.g. do not match `property='og:video:type'` when `og:video` is requested.
---
 youtube_dl/extractor/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 0082a4c84..a0c4af92f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -646,7 +646,8 @@ class InfoExtractor(object):
     @staticmethod
     def _og_regexes(prop):
         content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\'|\s*([^\s"\'=<>`]+?))'
-        property_re = r'(?:name|property)=[\'"]?og:%s[\'"]?' % re.escape(prop)
+        property_re = (r'(?:name|property)=(?:\'og:%(prop)s\'|"og:%(prop)s"|\s*og:%(prop)s\b)'
+                       % {'prop': re.escape(prop)})
         template = r'<meta[^>]+?%s[^>]+?%s'
         return [
             template % (property_re, content_re),

From 1c29e81e620241b9013b23e7acd9d6ab06587fb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Oct 2015 20:58:52 +0600
Subject: [PATCH 135/150] [test_InfoExtractor] Add test for
 7a6d76a64d8a89a08bb79791506fc18b993c4580

---
 test/test_InfoExtractor.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index be8d12997..4ce5b5a35 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -35,10 +35,12 @@ class TestInfoExtractor(unittest.TestCase):
             <meta name="og:title" content='Foo'/>
             <meta content="Some video's description " name="og:description"/>
             <meta property='og:image' content='http://domain.com/pic.jpg?key1=val1&amp;key2=val2'/>
+            <meta content='application/x-shockwave-flash' property='og:video:type'>
             '''
         self.assertEqual(ie._og_search_title(html), 'Foo')
         self.assertEqual(ie._og_search_description(html), 'Some video\'s description ')
         self.assertEqual(ie._og_search_thumbnail(html), 'http://domain.com/pic.jpg?key1=val1&key2=val2')
+        self.assertEqual(ie._og_search_video_url(html, default=None), None)
 
     def test_html_search_meta(self):
         ie = self.ie

From db0a8ad97993cb3f0c398d3a5dc55389565e0ffd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Oct 2015 21:11:06 +0600
Subject: [PATCH 136/150] [test_InfoExtractor] Add test for unquoted attribute

---
 test/test_InfoExtractor.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 4ce5b5a35..2a00d09a5 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -36,11 +36,13 @@ class TestInfoExtractor(unittest.TestCase):
             <meta content="Some video's description " name="og:description"/>
             <meta property='og:image' content='http://domain.com/pic.jpg?key1=val1&amp;key2=val2'/>
             <meta content='application/x-shockwave-flash' property='og:video:type'>
+            <meta content='Foo' property=og:foobar>
             '''
         self.assertEqual(ie._og_search_title(html), 'Foo')
         self.assertEqual(ie._og_search_description(html), 'Some video\'s description ')
         self.assertEqual(ie._og_search_thumbnail(html), 'http://domain.com/pic.jpg?key1=val1&key2=val2')
         self.assertEqual(ie._og_search_video_url(html, default=None), None)
+        self.assertEqual(ie._og_search_property('foobar', html), 'Foo')
 
     def test_html_search_meta(self):
         ie = self.ie

From ab953c64a0e8b8558e95d0318110c0885a4eec3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Oct 2015 21:15:29 +0600
Subject: [PATCH 137/150] [yandexmusic:track] Extract original size thumbnail
 (Closes #7160)

---
 youtube_dl/extractor/yandexmusic.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 6842f834f..08dc81f3a 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -46,14 +46,12 @@ class YandexMusicTrackIE(InfoExtractor):
                 % (data['host'], key, data['ts'] + data['path'], storage[1]))
 
     def _get_track_info(self, track):
-        album = track['albums'][0]
-        a_thumb = None
-
-        if 'coverUri' in album:
-            a_thumb = album['coverUri']
-            if a_thumb:
-                a_thumb = 'http://' + a_thumb.replace('%%', '1000x1000')
-
+        thumbnail = None
+        cover_uri = track.get('albums', [{}])[0].get('coverUri')
+        if cover_uri:
+            thumbnail = cover_uri.replace('%%', 'orig')
+            if not thumbnail.startswith('http'):
+                thumbnail = 'http://' + thumbnail
         return {
             'id': track['id'],
             'ext': 'mp3',
@@ -61,7 +59,7 @@ class YandexMusicTrackIE(InfoExtractor):
             'title': '%s - %s' % (track['artists'][0]['name'], track['title']),
             'filesize': int_or_none(track.get('fileSize')),
             'duration': float_or_none(track.get('durationMs'), 1000),
-            'thumbnail': a_thumb,
+            'thumbnail': thumbnail,
         }
 
     def _real_extract(self, url):

From 9fb66c780cee8668b1bb07f70e70ae1161e13320 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 14 Oct 2015 17:25:07 +0100
Subject: [PATCH 138/150] [megavideoz] remove extractor

---
 youtube_dl/extractor/__init__.py   |  1 -
 youtube_dl/extractor/megavideoz.py | 56 ------------------------------
 2 files changed, 57 deletions(-)
 delete mode 100644 youtube_dl/extractor/megavideoz.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f6d185818..462717b1e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -319,7 +319,6 @@ from .macgamestore import MacGameStoreIE
 from .mailru import MailRuIE
 from .malemotion import MalemotionIE
 from .mdr import MDRIE
-from .megavideoz import MegaVideozIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
diff --git a/youtube_dl/extractor/megavideoz.py b/youtube_dl/extractor/megavideoz.py
deleted file mode 100644
index af7ff07ea..000000000
--- a/youtube_dl/extractor/megavideoz.py
+++ /dev/null
@@ -1,56 +0,0 @@
-# encoding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    float_or_none,
-    xpath_text,
-)
-
-
-class MegaVideozIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?megavideoz\.eu/video/(?P<id>[^/]+)(?:/(?P<display_id>[^/]+))?'
-    _TEST = {
-        'url': 'http://megavideoz.eu/video/WM6UB919XMXH/SMPTE-Universal-Film-Leader',
-        'info_dict': {
-            'id': '48723',
-            'display_id': 'SMPTE-Universal-Film-Leader',
-            'ext': 'mp4',
-            'title': 'SMPTE Universal Film Leader',
-            'thumbnail': 're:https?://.*?\.jpg',
-            'duration': 10.93,
-        }
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        display_id = mobj.group('display_id') or video_id
-
-        webpage = self._download_webpage(url, display_id)
-
-        if any(p in webpage for p in ('>Video Not Found<', '>404 Error<')):
-            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
-
-        config = self._download_xml(
-            self._search_regex(
-                r"var\s+cnf\s*=\s*'([^']+)'", webpage, 'cnf url'),
-            display_id)
-
-        video_url = xpath_text(config, './file', 'video url', fatal=True)
-        title = xpath_text(config, './title', 'title', fatal=True)
-        thumbnail = xpath_text(config, './image', 'thumbnail')
-        duration = float_or_none(xpath_text(config, './duration', 'duration'))
-        video_id = xpath_text(config, './mediaid', 'video id') or video_id
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'url': video_url,
-            'title': title,
-            'thumbnail': thumbnail,
-            'duration': duration
-        }

From 1812afb7b396f4954d5d1ca1cec1c3f2d67550c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Oct 2015 22:35:01 +0600
Subject: [PATCH 139/150] [utils] Do not fail in int_or_none on non-numeric
 data (Closes #7175)

---
 youtube_dl/utils.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 1dc3153fd..86c693358 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1371,7 +1371,12 @@ def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
             v = getattr(v, get_attr, None)
     if v == '':
         v = None
-    return default if v is None else (int(v) * invscale // scale)
+    if v is None:
+        return default
+    try:
+        return int(v) * invscale // scale
+    except ValueError:
+        pass
 
 
 def str_or_none(v, default=None):

From caf80631f0c57b29187e2aa909fa1a3a6325d6e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Oct 2015 22:36:37 +0600
Subject: [PATCH 140/150] [utils] Do not fail in float_or_none on non-numeric
 data

---
 youtube_dl/utils.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 86c693358..83b44caaa 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1392,7 +1392,12 @@ def str_to_int(int_str):
 
 
 def float_or_none(v, scale=1, invscale=1, default=None):
-    return default if v is None else (float(v) * invscale / scale)
+    if v is None:
+        return default
+    try:
+        return float(v) * invscale / scale
+    except ValueError:
+        return default
 
 
 def parse_duration(s):

From af98f8ff37b3a0d9d1f743f4fc6c646333501eb6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Oct 2015 22:37:03 +0600
Subject: [PATCH 141/150] [utils] Return default on fail in int_or_none

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 83b44caaa..7dbe25661 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1376,7 +1376,7 @@ def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
     try:
         return int(v) * invscale // scale
     except ValueError:
-        pass
+        return default
 
 
 def str_or_none(v, default=None):

From 1db82381e38181aafbd78c65c58f005ad84cc08a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Oct 2015 01:52:25 +0600
Subject: [PATCH 142/150] [channel9] Add low quality formats and modernize

---
 youtube_dl/extractor/channel9.py | 35 ++++++++++++--------------------
 1 file changed, 13 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index 1ce004932..3a88181d8 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -3,7 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    parse_filesize,
+    qualities,
+)
 
 
 class Channel9IE(InfoExtractor):
@@ -52,23 +56,6 @@ class Channel9IE(InfoExtractor):
 
     _RSS_URL = 'http://channel9.msdn.com/%s/RSS'
 
-    # Sorted by quality
-    _known_formats = ['MP3', 'MP4', 'Mid Quality WMV', 'Mid Quality MP4', 'High Quality WMV', 'High Quality MP4']
-
-    def _restore_bytes(self, formatted_size):
-        if not formatted_size:
-            return 0
-        m = re.match(r'^(?P<size>\d+(?:\.\d+)?)\s+(?P<units>[a-zA-Z]+)', formatted_size)
-        if not m:
-            return 0
-        units = m.group('units')
-        try:
-            exponent = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'].index(units.upper())
-        except ValueError:
-            return 0
-        size = float(m.group('size'))
-        return int(size * (1024 ** exponent))
-
     def _formats_from_html(self, html):
         FORMAT_REGEX = r'''
             (?x)
@@ -78,16 +65,20 @@ class Channel9IE(InfoExtractor):
             <h3>File\s+size</h3>\s*(?P<filesize>.*?)\s*
             </div>)?                                                # File size part may be missing
         '''
-        # Extract known formats
+        quality = qualities((
+            'MP3', 'MP4',
+            'Low Quality WMV', 'Low Quality MP4',
+            'Mid Quality WMV', 'Mid Quality MP4',
+            'High Quality WMV', 'High Quality MP4'))
         formats = [{
             'url': x.group('url'),
             'format_id': x.group('quality'),
             'format_note': x.group('note'),
             'format': '%s (%s)' % (x.group('quality'), x.group('note')),
-            'filesize': self._restore_bytes(x.group('filesize')),  # File size is approximate
-            'preference': self._known_formats.index(x.group('quality')),
+            'filesize_approx': parse_filesize(x.group('filesize')),
+            'quality': quality(x.group('quality')),
             'vcodec': 'none' if x.group('note') == 'Audio only' else None,
-        } for x in list(re.finditer(FORMAT_REGEX, html)) if x.group('quality') in self._known_formats]
+        } for x in list(re.finditer(FORMAT_REGEX, html))]
 
         self._sort_formats(formats)
 

From a13d06de420f6968425d48030c37e1150ff9ed6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Oct 2015 01:57:59 +0600
Subject: [PATCH 143/150] [channel9] Add test for low quality mp4

---
 youtube_dl/extractor/channel9.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index 3a88181d8..554399787 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -51,6 +51,21 @@ class Channel9IE(InfoExtractor):
                 'thumbnail': 'http://video.ch9.ms/ch9/87e1/0300391f-a455-4c72-bec3-4422f19287e1/selfservicenuk_512.jpg',
                 'authors': ['Mike Wilmot'],
             },
+        },
+        {
+            # low quality mp4 is best
+            'url': 'https://channel9.msdn.com/Events/CPP/CppCon-2015/Ranges-for-the-Standard-Library',
+            'info_dict': {
+                'id': 'Events/CPP/CppCon-2015/Ranges-for-the-Standard-Library',
+                'ext': 'mp4',
+                'title': 'Ranges for the Standard Library',
+                'description': 'md5:2e6b4917677af3728c5f6d63784c4c5d',
+                'duration': 5646,
+                'thumbnail': 're:http://.*\.jpg',
+            },
+            'params': {
+                'skip_download': True,
+            },
         }
     ]
 

From fafc7950e2230bf25ac7c7563f1704cf8f134f64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Oct 2015 01:59:11 +0600
Subject: [PATCH 144/150] [channel9] Update tests' thumbnails

---
 youtube_dl/extractor/channel9.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index 554399787..c74553dcf 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -32,7 +32,7 @@ class Channel9IE(InfoExtractor):
                 'title': 'Developer Kick-Off Session: Stuff We Love',
                 'description': 'md5:c08d72240b7c87fcecafe2692f80e35f',
                 'duration': 4576,
-                'thumbnail': 'http://video.ch9.ms/ch9/9d51/03902f2d-fc97-4d3c-b195-0bfe15a19d51/KOS002_220.jpg',
+                'thumbnail': 're:http://.*\.jpg',
                 'session_code': 'KOS002',
                 'session_day': 'Day 1',
                 'session_room': 'Arena 1A',
@@ -48,7 +48,7 @@ class Channel9IE(InfoExtractor):
                 'title': 'Self-service BI with Power BI - nuclear testing',
                 'description': 'md5:d1e6ecaafa7fb52a2cacdf9599829f5b',
                 'duration': 1540,
-                'thumbnail': 'http://video.ch9.ms/ch9/87e1/0300391f-a455-4c72-bec3-4422f19287e1/selfservicenuk_512.jpg',
+                'thumbnail': 're:http://.*\.jpg',
                 'authors': ['Mike Wilmot'],
             },
         },

From 6744f36db710eebe2ccc633e7f4f6132b968b0ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 16 Oct 2015 08:44:19 +0600
Subject: [PATCH 145/150] [jeuxvideo] Fallback on og:title (Closes #7186,
 closes #7190)

---
 youtube_dl/extractor/jeuxvideo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/jeuxvideo.py b/youtube_dl/extractor/jeuxvideo.py
index 1df084d87..eef7daa29 100644
--- a/youtube_dl/extractor/jeuxvideo.py
+++ b/youtube_dl/extractor/jeuxvideo.py
@@ -28,7 +28,7 @@ class JeuxVideoIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         title = mobj.group(1)
         webpage = self._download_webpage(url, title)
-        title = self._html_search_meta('name', webpage)
+        title = self._html_search_meta('name', webpage) or self._og_search_title(webpage)
         config_url = self._html_search_regex(
             r'data-src="(/contenu/medias/video.php.*?)"',
             webpage, 'config URL')

From 8daeeedc06f420e2a87ba4755b56e721391cedba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 00:26:45 +0600
Subject: [PATCH 146/150] [bbc] Fix FutureWarning

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 68995f81e..1b3a33e4e 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -421,7 +421,7 @@ class BBCCoUkIE(InfoExtractor):
                 continue
             title = playlist.find('./{%s}title' % self._EMP_PLAYLIST_NS).text
             description_el = playlist.find('./{%s}summary' % self._EMP_PLAYLIST_NS)
-            description = description_el.text if description_el else None
+            description = description_el.text if description_el is not None else None
 
             def get_programme_id(item):
                 def get_from_attributes(item):

From 1e52776ac3ebbafc2ec4697f3bc6ba05b7e5a9f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 00:46:38 +0600
Subject: [PATCH 147/150] [bandcamp] Prepend download URL with scheme when
 necessary (2) (#7077)

---
 youtube_dl/extractor/bandcamp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index a27f3e748..f19e19001 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -52,7 +52,7 @@ class BandcampIE(InfoExtractor):
                     ext, abr_str = format_id.split('-', 1)
                     formats.append({
                         'format_id': format_id,
-                        'url': format_url,
+                        'url': self._proto_relative_url(format_url, 'http:'),
                         'ext': ext,
                         'vcodec': 'none',
                         'acodec': ext,

From ba717dca97925a21870fedcb46358d06cd5485ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 00:51:35 +0600
Subject: [PATCH 148/150] [bandcamp] Modernize

---
 youtube_dl/extractor/bandcamp.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index f19e19001..c1ef8051d 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -10,6 +10,8 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    float_or_none,
+    int_or_none,
 )
 
 
@@ -56,7 +58,7 @@ class BandcampIE(InfoExtractor):
                         'ext': ext,
                         'vcodec': 'none',
                         'acodec': ext,
-                        'abr': int(abr_str),
+                        'abr': int_or_none(abr_str),
                     })
 
                 self._sort_formats(formats)
@@ -65,7 +67,7 @@ class BandcampIE(InfoExtractor):
                     'id': compat_str(data['id']),
                     'title': data['title'],
                     'formats': formats,
-                    'duration': float(data['duration']),
+                    'duration': float_or_none(data.get('duration')),
                 }
             else:
                 raise ExtractorError('No free songs found')

From 246ce1085804ead9126328cb1ec761d308f561c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 01:08:23 +0600
Subject: [PATCH 149/150] [rte] Update _VALID_URL (Closes #7198)

---
 youtube_dl/extractor/rte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index 04158b993..427c70866 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class RteIE(InfoExtractor):
-    _VALID_URL = r'http?://(?:www\.)?rte\.ie/player/[^/]{2,3}/show/(?P<id>[0-9]+)/'
+    _VALID_URL = r'http?://(?:www\.)?rte\.ie/player/[^/]{2,3}/show/[^/]+/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.rte.ie/player/de/show/10363114/',
         'info_dict': {

From 2ccb37beb9e35ebbf2cdf65a4c1641e5286de1e8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 16 Oct 2015 21:40:38 +0200
Subject: [PATCH 150/150] release 2015.10.16

---
 docs/supportedsites.md | 2 +-
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index dc0354095..47f7da86d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -159,6 +159,7 @@
  - **facebook**
  - **faz.net**
  - **fc2**
+ - **Fczenit**
  - **fernsehkritik.tv**
  - **Firstpost**
  - **FiveTV**
@@ -281,7 +282,6 @@
  - **Malemotion**
  - **MDR**
  - **media.ccc.de**
- - **MegaVideoz**
  - **metacafe**
  - **Metacritic**
  - **Mgoon**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index aaa43d315..31d2a9dc0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.13'
+__version__ = '2015.10.16'