From 03486dbb0133e42074c272f60e24f18c856fdf0d Mon Sep 17 00:00:00 2001
From: Random User <rndusr@posteo.de>
Date: Sat, 25 Mar 2017 19:37:45 +0100
Subject: [PATCH 001/611] Add test for JWPlayer where config is passed as
 variable

---
 youtube_dl/extractor/generic.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 9868ca6d0..c8c103ae3 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -972,6 +972,20 @@ class GenericIE(InfoExtractor):
                 'thumbnail': r're:^https?://.*\.jpg$',
             },
         },
+        {
+            # JWPlayer config passed as variable
+            'url': 'http://www.txxx.com/videos/3326530/ariele/',
+            'info_dict': {
+                'id': '3326530_hq',
+                'ext': 'mp4',
+                'title': 'ARIELE | Tube Cup',
+                'uploader': 'www.txxx.com',
+                'age_limit': 18,
+            },
+            'params': {
+                'skip_download': True,
+            }
+        },
         # rtl.nl embed
         {
             'url': 'http://www.rtlnieuws.nl/nieuws/buitenland/aanslagen-kopenhagen',

From c73e330e7adc9c0c15ac51aeea8fbb7dad95351a Mon Sep 17 00:00:00 2001
From: Random User <rndusr@posteo.de>
Date: Sat, 25 Mar 2017 19:38:30 +0100
Subject: [PATCH 002/611] _find_jwplayer_data() returns dict or None

This simplifies code for callers of `_find_jwplayer_data()` which no longer have
to run `_parse_json()` on the return value.

It also makes sure that `_find_jwplayer_data()` returns either a `dict` or
`None` and nothing else.
---
 youtube_dl/extractor/common.py  | 18 ++++++++++++------
 youtube_dl/extractor/generic.py | 12 ++++--------
 youtube_dl/extractor/tvnoe.py   |  5 ++---
 3 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index eb3c091aa..c2ca73ee1 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2161,18 +2161,24 @@ class InfoExtractor(object):
                     })
         return formats
 
-    @staticmethod
-    def _find_jwplayer_data(webpage):
+    def _find_jwplayer_data(self, webpage, video_id=None, transform_source=js_to_json):
         mobj = re.search(
             r'jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\)\.setup\s*\((?P<options>[^)]+)\)',
             webpage)
         if mobj:
-            return mobj.group('options')
+            try:
+                jwplayer_data = self._parse_json(mobj.group('options'),
+                                                 video_id=video_id,
+                                                 transform_source=transform_source)
+            except ExtractorError:
+                pass
+            else:
+                if isinstance(jwplayer_data, dict):
+                    return jwplayer_data
 
     def _extract_jwplayer_data(self, webpage, video_id, *args, **kwargs):
-        jwplayer_data = self._parse_json(
-            self._find_jwplayer_data(webpage), video_id,
-            transform_source=js_to_json)
+        jwplayer_data = self._find_jwplayer_data(
+            webpage, video_id, transform_source=js_to_json)
         return self._parse_jwplayer_data(
             jwplayer_data, video_id, *args, **kwargs)
 
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c8c103ae3..3fe0237b6 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2518,14 +2518,10 @@ class GenericIE(InfoExtractor):
                 self._sort_formats(entry['formats'])
             return self.playlist_result(entries)
 
-        jwplayer_data_str = self._find_jwplayer_data(webpage)
-        if jwplayer_data_str:
-            try:
-                jwplayer_data = self._parse_json(
-                    jwplayer_data_str, video_id, transform_source=js_to_json)
-                return self._parse_jwplayer_data(jwplayer_data, video_id)
-            except ExtractorError:
-                pass
+        jwplayer_data = self._find_jwplayer_data(
+            webpage, video_id, transform_source=js_to_json)
+        if jwplayer_data:
+            return self._parse_jwplayer_data(jwplayer_data, video_id)
 
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
diff --git a/youtube_dl/extractor/tvnoe.py b/youtube_dl/extractor/tvnoe.py
index 1a5b76bf2..26a5aeae4 100644
--- a/youtube_dl/extractor/tvnoe.py
+++ b/youtube_dl/extractor/tvnoe.py
@@ -31,9 +31,8 @@ class TVNoeIE(InfoExtractor):
             r'<iframe[^>]+src="([^"]+)"', webpage, 'iframe URL')
 
         ifs_page = self._download_webpage(iframe_url, video_id)
-        jwplayer_data = self._parse_json(
-            self._find_jwplayer_data(ifs_page),
-            video_id, transform_source=js_to_json)
+        jwplayer_data = self._find_jwplayer_data(
+            ifs_page, video_id, transform_source=js_to_json)
         info_dict = self._parse_jwplayer_data(
             jwplayer_data, video_id, require_title=False, base_url=iframe_url)
 

From f7923a4c399e0ce8e6cd230db92aefbfcff297c3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 26 Mar 2017 22:07:12 +0800
Subject: [PATCH 003/611] [ChangeLog] Update after #12307

---
 ChangeLog | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 45d6f244d..adc64053b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,10 @@
 version <unreleased>
 
+Core
+* Don't raise an error if JWPlayer config data is not a Javascript object
+  literal. _find_jwplayer_data() now returns a dict rather than an str.
+  (#12307)
+
 Extractors
 * [afreecatv] Fix extraction (#12179)
 

From 82eefd0be00b7557782ae75602b463e226dd964f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Mar 2017 23:39:12 +0700
Subject: [PATCH 004/611] [ChangeLog] Actualize

---
 ChangeLog | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index adc64053b..e79067cff 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -2,11 +2,21 @@ version <unreleased>
 
 Core
 * Don't raise an error if JWPlayer config data is not a Javascript object
-  literal. _find_jwplayer_data() now returns a dict rather than an str.
-  (#12307)
+  literal. _find_jwplayer_data now returns a dict rather than an str. (#12307)
+* Expand environment variables for options representing paths (#12556)
++ [utils] Introduce expand_path
+* [downloader/hls] Delegate downloading to ffmpeg immediately for live streams
 
 Extractors
 * [afreecatv] Fix extraction (#12179)
++ [atvat] Add support for atv.at (#5325)
++ [fox] Add metadata extraction (#12391)
++ [atresplayer] Extract DASH formats
++ [atresplayer] Extract HD manifest (#12548)
+* [atresplayer] Fix login error detection (#12548)
+* [franceculture] Fix extraction (#12547)
+* [youtube] Improve URL regular expression (#12538)
+* [generic] Do not follow redirects to the same URL
 
 
 version 2017.03.24

From 9e691da06791a0a617ed69ef21e272536e247ed1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Mar 2017 08:11:40 +0700
Subject: [PATCH 005/611] release 2017.03.26

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 1 +
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index dfff41d2d..2f717926c 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.24*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.24**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.26*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.26**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.03.24
+[debug] youtube-dl version 2017.03.26
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index e79067cff..07725b12a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.03.26
 
 Core
 * Don't raise an error if JWPlayer config data is not a Javascript object
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 7c99ba3c2..e9dbc021b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -67,6 +67,7 @@
  - **arte.tv:playlist**
  - **AtresPlayer**
  - **ATTTechChannel**
+ - **ATVAt**
  - **AudiMedia**
  - **AudioBoom**
  - **audiomack**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 13904c724..94e8198ec 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.03.24'
+__version__ = '2017.03.26'

From aea1dccbd07b073ef36b325a9a21eb3f642322d9 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Tue, 28 Mar 2017 15:42:03 +0200
Subject: [PATCH 006/611] [openload] fix extractor

---
 youtube_dl/extractor/openload.py | 73 +++++++++++++-------------------
 1 file changed, 30 insertions(+), 43 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 58ffde541..d8036b54a 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -75,51 +75,38 @@ class OpenloadIE(InfoExtractor):
             '<span[^>]+id="[^"]+"[^>]*>([0-9A-Za-z]+)</span>',
             webpage, 'openload ID')
 
-        video_url_chars = []
-
-        first_char = ord(ol_id[0])
-        key = first_char - 55
-        maxKey = max(2, key)
-        key = min(maxKey, len(ol_id) - 38)
-        t = ol_id[key:key + 36]
-
-        hashMap = {}
-        v = ol_id.replace(t, '')
-        h = 0
-
-        while h < len(t):
-            f = t[h:h + 3]
-            i = int(f, 8)
-            hashMap[h / 3] = i
-            h += 3
-
-        h = 0
-        H = 0
-        while h < len(v):
-            B = ''
-            C = ''
-            if len(v) >= h + 2:
-                B = v[h:h + 2]
-            if len(v) >= h + 3:
-                C = v[h:h + 3]
-            i = int(B, 16)
-            h += 2
-            if H % 3 == 0:
-                i = int(C, 8)
-                h += 1
-            elif H % 2 == 0 and H != 0 and ord(v[H - 1]) < 60:
-                i = int(C, 10)
-                h += 1
-            index = H % 7
-
-            A = hashMap[index]
-            i ^= 213
-            i ^= A
-            video_url_chars.append(compat_chr(i))
-            H += 1
+        decoded = ''
+        a = ol_id[0:24]
+        b = []
+        for i in range(0, len(a), 8):
+            b.append(int(a[i:i + 8] or '0', 16))
+        ol_id = ol_id[24:]
+        j = 0
+        k = 0
+        while j < len(ol_id):
+            c = 128
+            d = 0
+            e = 0
+            f = 0
+            _more = True
+            while _more:
+                if j + 1 >= len(ol_id):
+                    c = 143
+                f = int(ol_id[j:j + 2] or '0', 16)
+                j += 2
+                d += (f & 127) << e
+                e += 7
+                _more = f >= c
+            g = d ^ b[k % 3]
+            for i in range(4):
+                char_dec = (g >> 8 * i) & (c + 127)
+                char = compat_chr(char_dec)
+                if char != '#':
+                    decoded += char
+            k += 1
 
         video_url = 'https://openload.co/stream/%s?mime=true'
-        video_url = video_url % (''.join(video_url_chars))
+        video_url = video_url % decoded
 
         title = self._og_search_title(webpage, default=None) or self._search_regex(
             r'<span[^>]+class=["\']title["\'][^>]*>([^<]+)', webpage,

From 12ee65ea0d09c6ac42ad06b3d561b4a26db00cfe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 28 Mar 2017 23:35:48 +0700
Subject: [PATCH 007/611] [options] Mention ISM for --fragment-retries and
 --skip-unavailable-fragments

---
 youtube_dl/options.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 6b811535f..2d2f5e47b 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -459,11 +459,11 @@ def parseOpts(overrideArguments=None):
     downloader.add_option(
         '--fragment-retries',
         dest='fragment_retries', metavar='RETRIES', default=10,
-        help='Number of retries for a fragment (default is %default), or "infinite" (DASH and hlsnative only)')
+        help='Number of retries for a fragment (default is %default), or "infinite" (DASH, hlsnative and ISM)')
     downloader.add_option(
         '--skip-unavailable-fragments',
         action='store_true', dest='skip_unavailable_fragments', default=True,
-        help='Skip unavailable fragments (DASH and hlsnative only)')
+        help='Skip unavailable fragments (DASH, hlsnative and ISM)')
     downloader.add_option(
         '--abort-on-unavailable-fragment',
         action='store_false', dest='skip_unavailable_fragments',

From 128244657b92582f7f4793c2d1be86b04032ac7f Mon Sep 17 00:00:00 2001
From: plroman <dev@plr.pm>
Date: Tue, 28 Mar 2017 23:23:20 +0200
Subject: [PATCH 008/611] [allocine] Fix extraction

---
 youtube_dl/extractor/allocine.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/allocine.py b/youtube_dl/extractor/allocine.py
index 90f11d39f..0463a070b 100644
--- a/youtube_dl/extractor/allocine.py
+++ b/youtube_dl/extractor/allocine.py
@@ -70,7 +70,7 @@ class AllocineIE(InfoExtractor):
         if model:
             model_data = self._parse_json(model, display_id)
 
-            for video_url in model_data['sources'].values():
+            for video_url in model_data['videos'][0]['sources'].values():
                 video_id, format_id = url_basename(video_url).split('_')[:2]
                 formats.append({
                     'format_id': format_id,
@@ -78,7 +78,7 @@ class AllocineIE(InfoExtractor):
                     'url': video_url,
                 })
 
-            title = model_data['title']
+            title = model_data['videos'][0]['title']
         else:
             video_id = display_id
             media_data = self._download_json(

From 639e5b2a848c0a73e8525472dd8bb4b14a8c4746 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Mar 2017 04:43:12 +0700
Subject: [PATCH 009/611] [allocine] Extract more metadata

---
 youtube_dl/extractor/allocine.py | 46 +++++++++++++++++++++++---------
 1 file changed, 34 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/allocine.py b/youtube_dl/extractor/allocine.py
index 0463a070b..cd533acfc 100644
--- a/youtube_dl/extractor/allocine.py
+++ b/youtube_dl/extractor/allocine.py
@@ -2,9 +2,13 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
-    remove_end,
+    int_or_none,
     qualities,
+    remove_end,
+    try_get,
+    unified_timestamp,
     url_basename,
 )
 
@@ -22,6 +26,10 @@ class AllocineIE(InfoExtractor):
             'title': 'Astérix - Le Domaine des Dieux Teaser VF',
             'description': 'md5:4a754271d9c6f16c72629a8a993ee884',
             'thumbnail': r're:http://.*\.jpg',
+            'duration': 39,
+            'timestamp': 1404273600,
+            'upload_date': '20140702',
+            'view_count': int,
         },
     }, {
         'url': 'http://www.allocine.fr/video/player_gen_cmedia=19540403&cfilm=222257.html',
@@ -33,6 +41,10 @@ class AllocineIE(InfoExtractor):
             'title': 'Planes 2 Bande-annonce VF',
             'description': 'Regardez la bande annonce du film Planes 2 (Planes 2 Bande-annonce VF). Planes 2, un film de Roberts Gannaway',
             'thumbnail': r're:http://.*\.jpg',
+            'duration': 69,
+            'timestamp': 1385659800,
+            'upload_date': '20131128',
+            'view_count': int,
         },
     }, {
         'url': 'http://www.allocine.fr/video/player_gen_cmedia=19544709&cfilm=181290.html',
@@ -44,6 +56,10 @@ class AllocineIE(InfoExtractor):
             'title': 'Dragons 2 - Bande annonce finale VF',
             'description': 'md5:6cdd2d7c2687d4c6aafe80a35e17267a',
             'thumbnail': r're:http://.*\.jpg',
+            'duration': 144,
+            'timestamp': 1397589900,
+            'upload_date': '20140415',
+            'view_count': int,
         },
     }, {
         'url': 'http://www.allocine.fr/video/video-19550147/',
@@ -69,34 +85,37 @@ class AllocineIE(InfoExtractor):
             r'data-model="([^"]+)"', webpage, 'data model', default=None)
         if model:
             model_data = self._parse_json(model, display_id)
-
-            for video_url in model_data['videos'][0]['sources'].values():
+            video = model_data['videos'][0]
+            title = video['title']
+            for video_url in video['sources'].values():
                 video_id, format_id = url_basename(video_url).split('_')[:2]
                 formats.append({
                     'format_id': format_id,
                     'quality': quality(format_id),
                     'url': video_url,
                 })
-
-            title = model_data['videos'][0]['title']
+            duration = int_or_none(video.get('duration'))
+            view_count = int_or_none(video.get('view_count'))
+            timestamp = unified_timestamp(try_get(
+                video, lambda x: x['added_at']['date'], compat_str))
         else:
             video_id = display_id
             media_data = self._download_json(
                 'http://www.allocine.fr/ws/AcVisiondataV5.ashx?media=%s' % video_id, display_id)
+            title = remove_end(
+                self._html_search_regex(
+                    r'(?s)<title>(.+?)</title>', webpage, 'title').strip(),
+                ' - AlloCiné')
             for key, value in media_data['video'].items():
                 if not key.endswith('Path'):
                     continue
-
                 format_id = key[:-len('Path')]
                 formats.append({
                     'format_id': format_id,
                     'quality': quality(format_id),
                     'url': value,
                 })
-
-            title = remove_end(self._html_search_regex(
-                r'(?s)<title>(.+?)</title>', webpage, 'title'
-            ).strip(), ' - AlloCiné')
+            duration, view_count, timestamp = [None] * 3
 
         self._sort_formats(formats)
 
@@ -104,7 +123,10 @@ class AllocineIE(InfoExtractor):
             'id': video_id,
             'display_id': display_id,
             'title': title,
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'formats': formats,
             'description': self._og_search_description(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'duration': duration,
+            'timestamp': timestamp,
+            'view_count': view_count,
+            'formats': formats,
         }

From 82be732b174ea8e9984e7b0582c69e41b266d1da Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 31 Mar 2017 12:24:23 +0100
Subject: [PATCH 010/611] [adn] Add new extractor

---
 youtube_dl/extractor/adn.py        | 136 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |   1 +
 2 files changed, 137 insertions(+)
 create mode 100644 youtube_dl/extractor/adn.py

diff --git a/youtube_dl/extractor/adn.py b/youtube_dl/extractor/adn.py
new file mode 100644
index 000000000..e44caa00b
--- /dev/null
+++ b/youtube_dl/extractor/adn.py
@@ -0,0 +1,136 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import base64
+import json
+import os
+
+from .common import InfoExtractor
+from ..aes import aes_cbc_decrypt
+from ..compat import compat_ord
+from ..utils import (
+    bytes_to_intlist,
+    ExtractorError,
+    float_or_none,
+    intlist_to_bytes,
+    srt_subtitles_timecode,
+    strip_or_none,
+)
+
+
+class ADNIE(InfoExtractor):
+    IE_DESC = 'Anime Digital Network'
+    _VALID_URL = r'https?://(?:www\.)?animedigitalnetwork\.fr/video/[^/]+/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://animedigitalnetwork.fr/video/blue-exorcist-kyoto-saga/7778-episode-1-debut-des-hostilites',
+        'md5': 'e497370d847fd79d9d4c74be55575c7a',
+        'info_dict': {
+            'id': '7778',
+            'ext': 'mp4',
+            'title': 'Blue Exorcist - Kyôto Saga - Épisode 1',
+            'description': 'md5:2f7b5aa76edbc1a7a92cedcda8a528d5',
+        }
+    }
+
+    def _get_subtitles(self, sub_path, video_id):
+        if not sub_path:
+            return None
+
+        enc_subtitles = self._download_webpage(
+            'http://animedigitalnetwork.fr/' + sub_path,
+            video_id, fatal=False)
+        if not enc_subtitles:
+            return None
+
+        # http://animedigitalnetwork.fr/components/com_vodvideo/videojs/adn-vjs.min.js
+        dec_subtitles = intlist_to_bytes(aes_cbc_decrypt(
+            bytes_to_intlist(base64.b64decode(enc_subtitles[24:])),
+            bytes_to_intlist(b'\xb5@\xcfq\xa3\x98"N\xe4\xf3\x12\x98}}\x16\xd8'),
+            bytes_to_intlist(base64.b64decode(enc_subtitles[:24]))
+        ))
+        subtitles_json = self._parse_json(
+            dec_subtitles[:-compat_ord(dec_subtitles[-1])],
+            None, fatal=False)
+        if not subtitles_json:
+            return None
+
+        subtitles = {}
+        for sub_lang, sub in subtitles_json.items():
+            srt = ''
+            for num, current in enumerate(sub):
+                start, end, text = (
+                    float_or_none(current.get('startTime')),
+                    float_or_none(current.get('endTime')),
+                    current.get('text'))
+                if start is None or end is None or text is None:
+                    continue
+                srt += os.linesep.join(
+                    (
+                        '%d' % num,
+                        '%s --> %s' % (
+                            srt_subtitles_timecode(start),
+                            srt_subtitles_timecode(end)),
+                        text,
+                        os.linesep,
+                    ))
+
+            if sub_lang == 'vostf':
+                sub_lang = 'fr'
+            subtitles.setdefault(sub_lang, []).extend([{
+                'ext': 'json',
+                'data': json.dumps(sub),
+            }, {
+                'ext': 'srt',
+                'data': srt,
+            }])
+        return subtitles
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        player_config = self._parse_json(self._search_regex(
+            r'playerConfig\s*=\s*({.+});', webpage, 'player config'), video_id)
+
+        video_info = {}
+        video_info_str = self._search_regex(
+            r'videoInfo\s*=\s*({.+});', webpage,
+            'video info', fatal=False)
+        if video_info_str:
+            video_info = self._parse_json(
+                video_info_str, video_id, fatal=False) or {}
+
+        options = player_config.get('options') or {}
+        metas = options.get('metas') or {}
+        title = metas.get('title') or video_info['title']
+        links = player_config.get('links') or {}
+
+        formats = []
+        for format_id, qualities in links.items():
+            for load_balancer_url in qualities.values():
+                load_balancer_data = self._download_json(
+                    load_balancer_url, video_id, fatal=False) or {}
+                m3u8_url = load_balancer_data.get('location')
+                if not m3u8_url:
+                    continue
+                m3u8_formats = self._extract_m3u8_formats(
+                    m3u8_url, video_id, 'mp4', 'm3u8_native',
+                    m3u8_id=format_id, fatal=False)
+                if format_id == 'vf':
+                    for f in m3u8_formats:
+                        f['language'] = 'fr'
+                formats.extend(m3u8_formats)
+        error = options.get('error')
+        if not formats and error:
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, error), expected=True)
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': strip_or_none(metas.get('summary') or video_info.get('resume')),
+            'thumbnail': video_info.get('image'),
+            'formats': formats,
+            'subtitles': self.extract_subtitles(player_config.get('subtitles'), video_id),
+            'episode': metas.get('subtitle') or video_info.get('videoTitle'),
+            'series': video_info.get('playlistTitle'),
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6a7028a4d..43933ad5b 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -19,6 +19,7 @@ from .acast import (
     ACastChannelIE,
 )
 from .addanime import AddAnimeIE
+from .adn import ADNIE
 from .adobetv import (
     AdobeTVIE,
     AdobeTVShowIE,

From 3e943cfe09eda6ef9b0fa419fdd22155fbaa047f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 31 Mar 2017 14:54:06 +0100
Subject: [PATCH 011/611] [generic] pass base_url to _parse_jwplayer_data

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 274f81738..73911940c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2568,7 +2568,7 @@ class GenericIE(InfoExtractor):
             webpage, video_id, transform_source=js_to_json)
         if jwplayer_data:
             info = self._parse_jwplayer_data(
-                jwplayer_data, video_id, require_title=False)
+                jwplayer_data, video_id, require_title=False, base_url=url)
             if not info.get('title'):
                 info['title'] = video_title
             return info

From 1640eb096166c81918125a0a7462eb2edb063167 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 31 Mar 2017 23:57:35 +0700
Subject: [PATCH 012/611] [YoutubeDL] Return early when extraction of
 url_transparent fails

---
 youtube_dl/YoutubeDL.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 21586f0f4..54bc8b06d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -837,6 +837,12 @@ class YoutubeDL(object):
                 ie_result['url'], ie_key=ie_result.get('ie_key'),
                 extra_info=extra_info, download=False, process=False)
 
+            # extract_info may return None when ignoreerrors is enabled and
+            # extraction failed with an error, don't crash and return early
+            # in this case
+            if not info:
+                return info
+
             force_properties = dict(
                 (k, v) for k, v in ie_result.items() if v is not None)
             for f in ('_type', 'url', 'ie_key'):

From 7453999580f2809153a84420d3ca72b24186c02b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Apr 2017 00:25:27 +0700
Subject: [PATCH 013/611] [packtpub] Add extractor (closes #12610)

---
 youtube_dl/extractor/extractors.py |   4 +
 youtube_dl/extractor/packtpub.py   | 138 +++++++++++++++++++++++++++++
 2 files changed, 142 insertions(+)
 create mode 100644 youtube_dl/extractor/packtpub.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 43933ad5b..6ad7444fe 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -729,6 +729,10 @@ from .orf import (
     ORFFM4IE,
     ORFIPTVIE,
 )
+from .packtpub import (
+    PacktPubIE,
+    PacktPubCourseIE,
+)
 from .pandatv import PandaTVIE
 from .pandoratv import PandoraTVIE
 from .parliamentliveuk import ParliamentLiveUKIE
diff --git a/youtube_dl/extractor/packtpub.py b/youtube_dl/extractor/packtpub.py
new file mode 100644
index 000000000..881f3bcc7
--- /dev/null
+++ b/youtube_dl/extractor/packtpub.py
@@ -0,0 +1,138 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    clean_html,
+    ExtractorError,
+    remove_end,
+    strip_or_none,
+    unified_timestamp,
+    urljoin,
+)
+
+
+class PacktPubBaseIE(InfoExtractor):
+    _PACKT_BASE = 'https://www.packtpub.com'
+    _MAPT_REST = '%s/mapt-rest' % _PACKT_BASE
+
+
+class PacktPubIE(PacktPubBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?packtpub\.com/mapt/video/[^/]+/(?P<course_id>\d+)/(?P<chapter_id>\d+)/(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'https://www.packtpub.com/mapt/video/web-development/9781787122215/20528/20530/Project+Intro',
+        'md5': '1e74bd6cfd45d7d07666f4684ef58f70',
+        'info_dict': {
+            'id': '20530',
+            'ext': 'mp4',
+            'title': 'Project Intro',
+            'thumbnail': r're:(?i)^https?://.*\.jpg',
+            'timestamp': 1490918400,
+            'upload_date': '20170331',
+        },
+    }
+
+    def _handle_error(self, response):
+        if response.get('status') != 'success':
+            raise ExtractorError(
+                '% said: %s' % (self.IE_NAME, response['message']),
+                expected=True)
+
+    def _download_json(self, *args, **kwargs):
+        response = super(PacktPubIE, self)._download_json(*args, **kwargs)
+        self._handle_error(response)
+        return response
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        course_id, chapter_id, video_id = mobj.group(
+            'course_id', 'chapter_id', 'id')
+
+        video = self._download_json(
+            '%s/users/me/products/%s/chapters/%s/sections/%s'
+            % (self._MAPT_REST, course_id, chapter_id, video_id), video_id,
+            'Downloading JSON video')['data']
+
+        content = video.get('content')
+        if not content:
+            raise ExtractorError('This video is locked', expected=True)
+
+        video_url = content['file']
+
+        metadata = self._download_json(
+            '%s/products/%s/chapters/%s/sections/%s/metadata'
+            % (self._MAPT_REST, course_id, chapter_id, video_id),
+            video_id)['data']
+
+        title = metadata['pageTitle']
+        course_title = metadata.get('title')
+        if course_title:
+            title = remove_end(title, ' - %s' % course_title)
+        timestamp = unified_timestamp(metadata.get('publicationDate'))
+        thumbnail = urljoin(self._PACKT_BASE, metadata.get('filepath'))
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+        }
+
+
+class PacktPubCourseIE(PacktPubBaseIE):
+    _VALID_URL = r'(?P<url>https?://(?:www\.)?packtpub\.com/mapt/video/[^/]+/(?P<id>\d+))'
+    _TEST = {
+        'url': 'https://www.packtpub.com/mapt/video/web-development/9781787122215',
+        'info_dict': {
+            'id': '9781787122215',
+            'title': 'Learn Nodejs by building 12 projects [Video]',
+        },
+        'playlist_count': 90,
+    }
+
+    @classmethod
+    def suitable(cls, url):
+        return False if PacktPubIE.suitable(url) else super(
+            PacktPubCourseIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        url, course_id = mobj.group('url', 'id')
+
+        course = self._download_json(
+            '%s/products/%s/metadata' % (self._MAPT_REST, course_id),
+            course_id)['data']
+
+        entries = []
+        for chapter_num, chapter in enumerate(course['tableOfContents'], 1):
+            if chapter.get('type') != 'chapter':
+                continue
+            children = chapter.get('children')
+            if not isinstance(children, list):
+                continue
+            chapter_info = {
+                'chapter': chapter.get('title'),
+                'chapter_number': chapter_num,
+                'chapter_id': chapter.get('id'),
+            }
+            for section in children:
+                if section.get('type') != 'section':
+                    continue
+                section_url = section.get('seoUrl')
+                if not isinstance(section_url, compat_str):
+                    continue
+                entry = {
+                    '_type': 'url_transparent',
+                    'url': urljoin(url + '/', section_url),
+                    'title': strip_or_none(section.get('title')),
+                    'description': clean_html(section.get('summary')),
+                    'ie_key': PacktPubIE.ie_key(),
+                }
+                entry.update(chapter_info)
+                entries.append(entry)
+
+        return self.playlist_result(entries, course_id, course.get('title'))

From 77c8ebe6318055cc34eaedca63f4866c4c47437a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 31 Mar 2017 23:28:24 +0100
Subject: [PATCH 014/611] [vrv] Add new extractor

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/vrv.py        | 151 +++++++++++++++++++++++++++++
 2 files changed, 152 insertions(+)
 create mode 100644 youtube_dl/extractor/vrv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6ad7444fe..1b427e256 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1182,6 +1182,7 @@ from .voxmedia import VoxMediaIE
 from .vporn import VpornIE
 from .vrt import VRTIE
 from .vrak import VrakIE
+from .vrv import VRVIE
 from .medialaan import MedialaanIE
 from .vube import VubeIE
 from .vuclip import VuClipIE
diff --git a/youtube_dl/extractor/vrv.py b/youtube_dl/extractor/vrv.py
new file mode 100644
index 000000000..33618c951
--- /dev/null
+++ b/youtube_dl/extractor/vrv.py
@@ -0,0 +1,151 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import base64
+import json
+import hashlib
+import hmac
+import random
+import string
+import time
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse_urlencode,
+    compat_urlparse,
+)
+from ..utils import (
+    float_or_none,
+    int_or_none,
+)
+
+
+class VRVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?vrv\.co/watch/(?P<id>[A-Z0-9]+)'
+    _TEST = {
+        'url': 'https://vrv.co/watch/GR9PNZ396/Hidden-America-with-Jonah-Ray:BOSTON-WHERE-THE-PAST-IS-THE-PRESENT',
+        'info_dict': {
+            'id': 'GR9PNZ396',
+            'ext': 'mp4',
+            'title': 'BOSTON: WHERE THE PAST IS THE PRESENT',
+            'description': 'md5:4ec8844ac262ca2df9e67c0983c6b83f',
+            'uploader_id': 'seeso',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+    _API_DOMAIN = None
+    _API_PARAMS = {}
+    _CMS_SIGNING = {}
+
+    def _call_api(self, path, video_id, note, data=None):
+        base_url = self._API_DOMAIN + '/core/' + path
+        encoded_query = compat_urllib_parse_urlencode({
+            'oauth_consumer_key': self._API_PARAMS['oAuthKey'],
+            'oauth_nonce': ''.join([random.choice(string.ascii_letters) for _ in range(32)]),
+            'oauth_signature_method': 'HMAC-SHA1',
+            'oauth_timestamp': int(time.time()),
+            'oauth_version': '1.0',
+        })
+        headers = self.geo_verification_headers()
+        if data:
+            data = json.dumps(data).encode()
+            headers['Content-Type'] = 'application/json'
+        method = 'POST' if data else 'GET'
+        base_string = '&'.join([method, compat_urlparse.quote(base_url, ''), compat_urlparse.quote(encoded_query, '')])
+        oauth_signature = base64.b64encode(hmac.new(
+            (self._API_PARAMS['oAuthSecret'] + '&').encode('ascii'),
+            base_string.encode(), hashlib.sha1).digest()).decode()
+        encoded_query += '&oauth_signature=' + compat_urlparse.quote(oauth_signature, '')
+        return self._download_json(
+            '?'.join([base_url, encoded_query]), video_id,
+            note='Downloading %s JSON metadata' % note, headers=headers, data=data)
+
+    def _call_cms(self, path, video_id, note):
+        return self._download_json(
+            self._API_DOMAIN + path, video_id, query=self._CMS_SIGNING,
+            note='Downloading %s JSON metadata' % note, headers=self.geo_verification_headers())
+
+    def _set_api_params(self, webpage, video_id):
+        if not self._API_PARAMS:
+            self._API_PARAMS = self._parse_json(self._search_regex(
+                r'window\.__APP_CONFIG__\s*=\s*({.+?})</script>',
+                webpage, 'api config'), video_id)['cxApiParams']
+            self._API_DOMAIN = self._API_PARAMS.get('apiDomain', 'https://api.vrv.co')
+
+    def _set_cms_signing(self, video_id):
+        if not self._CMS_SIGNING:
+            self._CMS_SIGNING = self._call_api('index', video_id, 'CMS Signing')['cms_signing']
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, video_id,
+            headers=self.geo_verification_headers())
+        media_resource = self._parse_json(self._search_regex(
+            r'window\.__INITIAL_STATE__\s*=\s*({.+?})</script>',
+            webpage, 'inital state'), video_id).get('watch', {}).get('mediaResource') or {}
+
+        video_data = media_resource.get('json')
+        if not video_data:
+            self._set_api_params(webpage, video_id)
+            episode_path = self._call_api('cms_resource', video_id, 'episode resource path', data={
+                'resource_key': 'cms:/episodes/' + video_id,
+            })['__links__']['cms_resource']['href']
+            self._set_cms_signing(video_id)
+            video_data = self._call_cms(episode_path, video_id, 'video')
+        title = video_data['title']
+
+        streams_json = media_resource.get('streams', {}).get('json', {})
+        if not streams_json:
+            self._set_api_params(webpage, video_id)
+            streams_path = video_data['__links__']['streams']['href']
+            self._set_cms_signing(video_id)
+            streams_json = self._call_cms(streams_path, video_id, 'streams')
+
+        audio_locale = streams_json.get('audio_locale')
+        formats = []
+        for stream_id, stream in streams_json.get('streams', {}).get('adaptive_hls', {}).items():
+            stream_url = stream.get('url')
+            if not stream_url:
+                continue
+            stream_id = stream_id or audio_locale
+            m3u8_formats = self._extract_m3u8_formats(
+                stream_url, video_id, 'mp4', m3u8_id=stream_id,
+                note='Downloading %s m3u8 information' % stream_id,
+                fatal=False)
+            if audio_locale:
+                for f in m3u8_formats:
+                    f['language'] = audio_locale
+            formats.extend(m3u8_formats)
+        self._sort_formats(formats)
+
+        thumbnails = []
+        for thumbnail in video_data.get('images', {}).get('thumbnails', []):
+            thumbnail_url = thumbnail.get('source')
+            if not thumbnail_url:
+                continue
+            thumbnails.append({
+                'url': thumbnail_url,
+                'width': int_or_none(thumbnail.get('width')),
+                'height': int_or_none(thumbnail.get('height')),
+            })
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnails': thumbnails,
+            'description': video_data.get('description'),
+            'duration': float_or_none(video_data.get('duration_ms'), 1000),
+            'uploader_id': video_data.get('channel_id'),
+            'series': video_data.get('series_title'),
+            'season': video_data.get('season_title'),
+            'season_number': int_or_none(video_data.get('season_number')),
+            'season_id': video_data.get('season_id'),
+            'episode': title,
+            'episode_number': int_or_none(video_data.get('episode_number')),
+            'episode_id': video_data.get('production_episode_id'),
+        }

From be61efdf1754d026f270f6d87446040231d56954 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 1 Apr 2017 07:26:40 +0100
Subject: [PATCH 015/611] [tvplay] Bypass geo restriction

---
 youtube_dl/extractor/tvplay.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index 3eda0a399..99ff82a5d 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -225,7 +225,11 @@ class TVPlayIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
+        geo_country = self._search_regex(
+            r'https?://[^/]+\.([a-z]{2})', url,
+            'geo country', default=None)
+        if geo_country:
+            self._initialize_geo_bypass([geo_country.upper()])
         video = self._download_json(
             'http://playapi.mtgx.tv/v3/videos/%s' % video_id, video_id, 'Downloading video JSON')
 

From e97fc8d6b837921ea8429727f026238b857e1b31 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 1 Apr 2017 07:50:24 +0100
Subject: [PATCH 016/611] [cwtv] extract ISM formats

---
 youtube_dl/extractor/cwtv.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/cwtv.py b/youtube_dl/extractor/cwtv.py
index 1ab9333b2..f4cf0f1c5 100644
--- a/youtube_dl/extractor/cwtv.py
+++ b/youtube_dl/extractor/cwtv.py
@@ -82,6 +82,11 @@ class CWTVIE(InfoExtractor):
                             'url': quality_url,
                             'tbr': tbr,
                         })
+        video_metadata = video_data['assetFields']
+        ism_url = video_metadata.get('smoothStreamingUrl')
+        if ism_url:
+            formats.extend(self._extract_ism_formats(
+                ism_url, video_id, ism_id='mss', fatal=False))
         self._sort_formats(formats)
 
         thumbnails = [{
@@ -90,8 +95,6 @@ class CWTVIE(InfoExtractor):
             'height': image.get('height'),
         } for image_id, image in video_data['images'].items() if image.get('uri')] if video_data.get('images') else None
 
-        video_metadata = video_data['assetFields']
-
         subtitles = {
             'en': [{
                 'url': video_metadata['UnicornCcUrl'],

From ca77b92f94010bdf2d44de44cb23e32075b7dcaa Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 1 Apr 2017 09:33:23 +0100
Subject: [PATCH 017/611] [crunchyroll] pass geo verifcation proxy

---
 youtube_dl/extractor/crunchyroll.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index d15fd3744..2ed8b30bb 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -390,7 +390,9 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         else:
             webpage_url = 'http://www.' + mobj.group('url')
 
-        webpage = self._download_webpage(self._add_skip_wall(webpage_url), video_id, 'Downloading webpage')
+        webpage = self._download_webpage(
+            self._add_skip_wall(webpage_url), video_id,
+            headers=self.geo_verification_headers())
         note_m = self._html_search_regex(
             r'<div class="showmedia-trailer-notice">(.+?)</div>',
             webpage, 'trailer-notice', default='')
@@ -565,7 +567,9 @@ class CrunchyrollShowPlaylistIE(CrunchyrollBaseIE):
     def _real_extract(self, url):
         show_id = self._match_id(url)
 
-        webpage = self._download_webpage(self._add_skip_wall(url), show_id)
+        webpage = self._download_webpage(
+            self._add_skip_wall(url), show_id,
+            headers=self.geo_verification_headers())
         title = self._html_search_regex(
             r'(?s)<h1[^>]*>\s*<span itemprop="name">(.*?)</span>',
             webpage, 'title')

From 2cd668ee591df4f271ed4394ba9b38262ae3c40e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Apr 2017 18:55:48 +0700
Subject: [PATCH 018/611] [xfileshare] Improve extraction and extract hls
 formats

---
 youtube_dl/extractor/xfileshare.py | 57 +++++++++++++++++++++---------
 1 file changed, 41 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index e616adce3..6de5b26d7 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     decode_packed_codes,
+    determine_ext,
     ExtractorError,
     int_or_none,
     NO_DEFAULT,
@@ -95,6 +96,16 @@ class XFileShareIE(InfoExtractor):
         # removed by administrator
         'url': 'http://xvidstage.com/amfy7atlkx25',
         'only_matching': True,
+    }, {
+        'url': 'http://vidabc.com/i8ybqscrphfv',
+        'info_dict': {
+            'id': 'i8ybqscrphfv',
+            'ext': 'mp4',
+            'title': 're:Beauty and the Beast 2017',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
@@ -133,31 +144,45 @@ class XFileShareIE(InfoExtractor):
             webpage, 'title', default=None) or self._og_search_title(
             webpage, default=None) or video_id).strip()
 
-        def extract_video_url(default=NO_DEFAULT):
-            return self._search_regex(
-                (r'file\s*:\s*(["\'])(?P<url>http.+?)\1,',
-                 r'file_link\s*=\s*(["\'])(?P<url>http.+?)\1',
-                 r'addVariable\((\\?["\'])file\1\s*,\s*(\\?["\'])(?P<url>http.+?)\2\)',
-                 r'<embed[^>]+src=(["\'])(?P<url>http.+?)\1'),
-                webpage, 'file url', default=default, group='url')
+        def extract_formats(default=NO_DEFAULT):
+            urls = []
+            for regex in (
+                    r'file\s*:\s*(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1',
+                    r'file_link\s*=\s*(["\'])(?P<url>http(?:(?!\1).)+)\1',
+                    r'addVariable\((\\?["\'])file\1\s*,\s*(\\?["\'])(?P<url>http(?:(?!\2).)+)\2\)',
+                    r'<embed[^>]+src=(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1'):
+                for mobj in re.finditer(regex, webpage):
+                    video_url = mobj.group('url')
+                    if video_url not in urls:
+                        urls.append(video_url)
+            formats = []
+            for video_url in urls:
+                if determine_ext(video_url) == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        video_url, video_id, 'mp4',
+                        entry_protocol='m3u8_native', m3u8_id='hls',
+                        fatal=False))
+                else:
+                    formats.append({
+                        'url': video_url,
+                        'format_id': 'sd',
+                    })
+            if not formats and default is not NO_DEFAULT:
+                return default
+            self._sort_formats(formats)
+            return formats
 
-        video_url = extract_video_url(default=None)
+        formats = extract_formats(default=None)
 
-        if not video_url:
+        if not formats:
             webpage = decode_packed_codes(self._search_regex(
                 r"(}\('(.+)',(\d+),(\d+),'[^']*\b(?:file|embed)\b[^']*'\.split\('\|'\))",
                 webpage, 'packed code'))
-            video_url = extract_video_url()
+            formats = extract_formats()
 
         thumbnail = self._search_regex(
             r'image\s*:\s*["\'](http[^"\']+)["\'],', webpage, 'thumbnail', default=None)
 
-        formats = [{
-            'format_id': 'sd',
-            'url': video_url,
-            'quality': 1,
-        }]
-
         return {
             'id': video_id,
             'title': title,

From eecea00d36f29f3b22e5936ed48fa91456ab066a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Apr 2017 18:56:35 +0700
Subject: [PATCH 019/611] [xfileshare] Add support for vidabc.com (closes
 #12589)

---
 youtube_dl/extractor/xfileshare.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 6de5b26d7..6856fb3bf 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -27,6 +27,7 @@ class XFileShareIE(InfoExtractor):
         ('vidto.me', 'Vidto'),
         ('streamin.to', 'Streamin.To'),
         ('xvidstage.com', 'XVIDSTAGE'),
+        ('vidabc.com', 'Vid ABC'),
     )
 
     IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1])

From 91399b2fcc95e72f052ee9eab8e12b68d1815c9e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 1 Apr 2017 13:32:38 +0100
Subject: [PATCH 020/611] [funimation] fix extraction(closes #10696)(#11773)

---
 youtube_dl/extractor/funimation.py | 211 ++++++++++-------------------
 1 file changed, 73 insertions(+), 138 deletions(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index eba00cd5a..e44a2a87f 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -7,9 +7,9 @@ from ..compat import (
     compat_urllib_parse_unquote_plus,
 )
 from ..utils import (
-    clean_html,
     determine_ext,
     int_or_none,
+    js_to_json,
     sanitized_Request,
     ExtractorError,
     urlencode_postdata
@@ -17,34 +17,26 @@ from ..utils import (
 
 
 class FunimationIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/[^/]+/videos/(?:official|promotional)/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?funimation(?:\.com|now\.uk)/shows/[^/]+/(?P<id>[^/?#&]+)'
 
     _NETRC_MACHINE = 'funimation'
 
     _TESTS = [{
-        'url': 'http://www.funimation.com/shows/air/videos/official/breeze',
+        'url': 'https://www.funimation.com/shows/hacksign/role-play/',
         'info_dict': {
-            'id': '658',
-            'display_id': 'breeze',
-            'ext': 'mp4',
-            'title': 'Air - 1 - Breeze',
-            'description': 'md5:1769f43cd5fc130ace8fd87232207892',
-            'thumbnail': r're:https?://.*\.jpg',
-        },
-        'skip': 'Access without user interaction is forbidden by CloudFlare, and video removed',
-    }, {
-        'url': 'http://www.funimation.com/shows/hacksign/videos/official/role-play',
-        'info_dict': {
-            'id': '31128',
+            'id': '91144',
             'display_id': 'role-play',
             'ext': 'mp4',
-            'title': '.hack//SIGN - 1 - Role Play',
+            'title': '.hack//SIGN - Role Play',
             'description': 'md5:b602bdc15eef4c9bbb201bb6e6a4a2dd',
             'thumbnail': r're:https?://.*\.jpg',
         },
-        'skip': 'Access without user interaction is forbidden by CloudFlare',
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
-        'url': 'http://www.funimation.com/shows/attack-on-titan-junior-high/videos/promotional/broadcast-dub-preview',
+        'url': 'https://www.funimation.com/shows/attack-on-titan-junior-high/broadcast-dub-preview/',
         'info_dict': {
             'id': '9635',
             'display_id': 'broadcast-dub-preview',
@@ -54,25 +46,13 @@ class FunimationIE(InfoExtractor):
             'thumbnail': r're:https?://.*\.(?:jpg|png)',
         },
         'skip': 'Access without user interaction is forbidden by CloudFlare',
+    }, {
+        'url': 'https://www.funimationnow.uk/shows/puzzle-dragons-x/drop-impact/simulcast/',
+        'only_matching': True,
     }]
 
     _LOGIN_URL = 'http://www.funimation.com/login'
 
-    def _download_webpage(self, *args, **kwargs):
-        try:
-            return super(FunimationIE, self)._download_webpage(*args, **kwargs)
-        except ExtractorError as ee:
-            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
-                response = ee.cause.read()
-                if b'>Please complete the security check to access<' in response:
-                    raise ExtractorError(
-                        'Access to funimation.com is blocked by CloudFlare. '
-                        'Please browse to http://www.funimation.com/, solve '
-                        'the reCAPTCHA, export browser cookies to a text file,'
-                        ' and then try again with --cookies YOUR_COOKIE_FILE.',
-                        expected=True)
-            raise
-
     def _extract_cloudflare_session_ua(self, url):
         ci_session_cookie = self._get_cookies(url).get('ci_session')
         if ci_session_cookie:
@@ -114,119 +94,74 @@ class FunimationIE(InfoExtractor):
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        def _search_kane(name):
+            return self._search_regex(
+                r"KANE_customdimensions\.%s\s*=\s*'([^']+)';" % name,
+                webpage, name, default=None)
+
+        title_data = self._parse_json(self._search_regex(
+            r'TITLE_DATA\s*=\s*({[^}]+})',
+            webpage, 'title data', default=''),
+            display_id, js_to_json, fatal=False) or {}
+
+        video_id = title_data.get('id') or self._search_regex([
+            r"KANE_customdimensions.videoID\s*=\s*'(\d+)';",
+            r'<iframe[^>]+src="/player/(\d+)"',
+        ], webpage, 'video_id', default=None)
+        if not video_id:
+            player_url = self._html_search_meta([
+                'al:web:url',
+                'og:video:url',
+                'og:video:secure_url',
+            ], webpage, fatal=True)
+            video_id = self._search_regex(r'/player/(\d+)', player_url, 'video id')
+
+        title = episode = title_data.get('title') or _search_kane('videoTitle') or self._og_search_title(webpage)
+        series = _search_kane('showName')
+        if series:
+            title = '%s - %s' % (series, title)
+        description = self._html_search_meta(['description', 'og:description'], webpage, fatal=True)
+
+        try:
+            sources = self._download_json(
+                'https://prod-api-funimationnow.dadcdigital.com/api/source/catalog/video/%s/signed/' % video_id,
+                video_id)['items']
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                error = self._parse_json(e.cause.read(), video_id)['errors'][0]
+                raise ExtractorError('%s said: %s' % (
+                    self.IE_NAME, error.get('detail') or error.get('title')), expected=True)
+            raise
 
-        errors = []
         formats = []
-
-        ERRORS_MAP = {
-            'ERROR_MATURE_CONTENT_LOGGED_IN': 'matureContentLoggedIn',
-            'ERROR_MATURE_CONTENT_LOGGED_OUT': 'matureContentLoggedOut',
-            'ERROR_SUBSCRIPTION_LOGGED_OUT': 'subscriptionLoggedOut',
-            'ERROR_VIDEO_EXPIRED': 'videoExpired',
-            'ERROR_TERRITORY_UNAVAILABLE': 'territoryUnavailable',
-            'SVODBASIC_SUBSCRIPTION_IN_PLAYER': 'basicSubscription',
-            'SVODNON_SUBSCRIPTION_IN_PLAYER': 'nonSubscription',
-            'ERROR_PLAYER_NOT_RESPONDING': 'playerNotResponding',
-            'ERROR_UNABLE_TO_CONNECT_TO_CDN': 'unableToConnectToCDN',
-            'ERROR_STREAM_NOT_FOUND': 'streamNotFound',
-        }
-
-        USER_AGENTS = (
-            # PC UA is served with m3u8 that provides some bonus lower quality formats
-            ('pc', 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0'),
-            # Mobile UA allows to extract direct links and also does not fail when
-            # PC UA fails with hulu error (e.g.
-            # http://www.funimation.com/shows/hacksign/videos/official/role-play)
-            ('mobile', 'Mozilla/5.0 (Linux; Android 4.4.2; Nexus 4 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.114 Mobile Safari/537.36'),
-        )
-
-        user_agent = self._extract_cloudflare_session_ua(url)
-        if user_agent:
-            USER_AGENTS = ((None, user_agent),)
-
-        for kind, user_agent in USER_AGENTS:
-            request = sanitized_Request(url)
-            request.add_header('User-Agent', user_agent)
-            webpage = self._download_webpage(
-                request, display_id,
-                'Downloading %s webpage' % kind if kind else 'Downloading webpage')
-
-            playlist = self._parse_json(
-                self._search_regex(
-                    r'var\s+playersData\s*=\s*(\[.+?\]);\n',
-                    webpage, 'players data'),
-                display_id)[0]['playlist']
-
-            items = next(item['items'] for item in playlist if item.get('items'))
-            item = next(item for item in items if item.get('itemAK') == display_id)
-
-            error_messages = {}
-            video_error_messages = self._search_regex(
-                r'var\s+videoErrorMessages\s*=\s*({.+?});\n',
-                webpage, 'error messages', default=None)
-            if video_error_messages:
-                error_messages_json = self._parse_json(video_error_messages, display_id, fatal=False)
-                if error_messages_json:
-                    for _, error in error_messages_json.items():
-                        type_ = error.get('type')
-                        description = error.get('description')
-                        content = error.get('content')
-                        if type_ == 'text' and description and content:
-                            error_message = ERRORS_MAP.get(description)
-                            if error_message:
-                                error_messages[error_message] = content
-
-            for video in item.get('videoSet', []):
-                auth_token = video.get('authToken')
-                if not auth_token:
-                    continue
-                funimation_id = video.get('FUNImationID') or video.get('videoId')
-                preference = 1 if video.get('languageMode') == 'dub' else 0
-                if not auth_token.startswith('?'):
-                    auth_token = '?%s' % auth_token
-                for quality, height in (('sd', 480), ('hd', 720), ('hd1080', 1080)):
-                    format_url = video.get('%sUrl' % quality)
-                    if not format_url:
-                        continue
-                    if not format_url.startswith(('http', '//')):
-                        errors.append(format_url)
-                        continue
-                    if determine_ext(format_url) == 'm3u8':
-                        formats.extend(self._extract_m3u8_formats(
-                            format_url + auth_token, display_id, 'mp4', entry_protocol='m3u8_native',
-                            preference=preference, m3u8_id='%s-hls' % funimation_id, fatal=False))
-                    else:
-                        tbr = int_or_none(self._search_regex(
-                            r'-(\d+)[Kk]', format_url, 'tbr', default=None))
-                        formats.append({
-                            'url': format_url + auth_token,
-                            'format_id': '%s-http-%dp' % (funimation_id, height),
-                            'height': height,
-                            'tbr': tbr,
-                            'preference': preference,
-                        })
-
-        if not formats and errors:
-            raise ExtractorError(
-                '%s returned error: %s'
-                % (self.IE_NAME, clean_html(error_messages.get(errors[0], errors[0]))),
-                expected=True)
-
+        for source in sources:
+            source_url = source.get('src')
+            if not source_url:
+                continue
+            source_type = source.get('videoType') or determine_ext(source_url)
+            if source_type == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    source_url, video_id, 'mp4',
+                    m3u8_id='hls', fatal=False))
+            else:
+                formats.append({
+                    'format_id': source_type,
+                    'url': source_url,
+                })
         self._sort_formats(formats)
 
-        title = item['title']
-        artist = item.get('artist')
-        if artist:
-            title = '%s - %s' % (artist, title)
-        description = self._og_search_description(webpage) or item.get('description')
-        thumbnail = self._og_search_thumbnail(webpage) or item.get('posterUrl')
-        video_id = item.get('itemId') or display_id
-
         return {
             'id': video_id,
             'display_id': display_id,
             'title': title,
             'description': description,
-            'thumbnail': thumbnail,
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'series': series,
+            'season_number': int_or_none(title_data.get('seasonNum') or _search_kane('season')),
+            'episode_number': int_or_none(title_data.get('episodeNum')),
+            'episode': episode,
+            'season_id': title_data.get('seriesId'),
             'formats': formats,
         }

From a6f3a162f35cc05ac5a34773b438dd4c5f0d164a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 1 Apr 2017 15:35:39 +0100
Subject: [PATCH 021/611] [limelight] improve extraction for audio only formats

---
 youtube_dl/extractor/limelight.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index 422be2528..f52c2e169 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -62,13 +62,21 @@ class LimelightBaseIE(InfoExtractor):
                 fmt = {
                     'url': stream_url,
                     'abr': float_or_none(stream.get('audioBitRate')),
-                    'vbr': float_or_none(stream.get('videoBitRate')),
                     'fps': float_or_none(stream.get('videoFrameRate')),
-                    'width': int_or_none(stream.get('videoWidthInPixels')),
-                    'height': int_or_none(stream.get('videoHeightInPixels')),
                     'ext': ext,
                 }
-                rtmp = re.search(r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+))/(?P<playpath>mp4:.+)$', stream_url)
+                width = int_or_none(stream.get('videoWidthInPixels'))
+                height = int_or_none(stream.get('videoHeightInPixels'))
+                vbr = float_or_none(stream.get('videoBitRate'))
+                if width or height or vbr:
+                    fmt.update({
+                        'width': width,
+                        'height': height,
+                        'vbr': vbr,
+                    })
+                else:
+                    fmt['vcodec'] = 'none'
+                rtmp = re.search(r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+))/(?P<playpath>mp[34]:.+)$', stream_url)
                 if rtmp:
                     format_id = 'rtmp'
                     if stream.get('videoBitRate'):

From 48ab554feb9c6d3e0f13e1357e04f4c89089e2d3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 1 Apr 2017 18:09:36 +0100
Subject: [PATCH 022/611] [vrv] add support for series pages

---
 youtube_dl/extractor/extractors.py |  5 +-
 youtube_dl/extractor/vrv.py        | 88 ++++++++++++++++++++++--------
 2 files changed, 68 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 1b427e256..980333a11 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1182,7 +1182,10 @@ from .voxmedia import VoxMediaIE
 from .vporn import VpornIE
 from .vrt import VRTIE
 from .vrak import VrakIE
-from .vrv import VRVIE
+from .vrv import (
+    VRVIE,
+    VRVSeriesIE,
+)
 from .medialaan import MedialaanIE
 from .vube import VubeIE
 from .vuclip import VuClipIE
diff --git a/youtube_dl/extractor/vrv.py b/youtube_dl/extractor/vrv.py
index 33618c951..487047fd7 100644
--- a/youtube_dl/extractor/vrv.py
+++ b/youtube_dl/extractor/vrv.py
@@ -20,22 +20,7 @@ from ..utils import (
 )
 
 
-class VRVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vrv\.co/watch/(?P<id>[A-Z0-9]+)'
-    _TEST = {
-        'url': 'https://vrv.co/watch/GR9PNZ396/Hidden-America-with-Jonah-Ray:BOSTON-WHERE-THE-PAST-IS-THE-PRESENT',
-        'info_dict': {
-            'id': 'GR9PNZ396',
-            'ext': 'mp4',
-            'title': 'BOSTON: WHERE THE PAST IS THE PRESENT',
-            'description': 'md5:4ec8844ac262ca2df9e67c0983c6b83f',
-            'uploader_id': 'seeso',
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }
+class VRVBaseIE(InfoExtractor):
     _API_DOMAIN = None
     _API_PARAMS = {}
     _CMS_SIGNING = {}
@@ -64,6 +49,8 @@ class VRVIE(InfoExtractor):
             note='Downloading %s JSON metadata' % note, headers=headers, data=data)
 
     def _call_cms(self, path, video_id, note):
+        if not self._CMS_SIGNING:
+            self._CMS_SIGNING = self._call_api('index', video_id, 'CMS Signing')['cms_signing']
         return self._download_json(
             self._API_DOMAIN + path, video_id, query=self._CMS_SIGNING,
             note='Downloading %s JSON metadata' % note, headers=self.geo_verification_headers())
@@ -75,9 +62,30 @@ class VRVIE(InfoExtractor):
                 webpage, 'api config'), video_id)['cxApiParams']
             self._API_DOMAIN = self._API_PARAMS.get('apiDomain', 'https://api.vrv.co')
 
-    def _set_cms_signing(self, video_id):
-        if not self._CMS_SIGNING:
-            self._CMS_SIGNING = self._call_api('index', video_id, 'CMS Signing')['cms_signing']
+    def _get_cms_resource(self, resource_key, video_id):
+        return self._call_api(
+            'cms_resource', video_id, 'resource path', data={
+                'resource_key': resource_key,
+            })['__links__']['cms_resource']['href']
+
+
+class VRVIE(VRVBaseIE):
+    IE_NAME = 'vrv'
+    _VALID_URL = r'https?://(?:www\.)?vrv\.co/watch/(?P<id>[A-Z0-9]+)'
+    _TEST = {
+        'url': 'https://vrv.co/watch/GR9PNZ396/Hidden-America-with-Jonah-Ray:BOSTON-WHERE-THE-PAST-IS-THE-PRESENT',
+        'info_dict': {
+            'id': 'GR9PNZ396',
+            'ext': 'mp4',
+            'title': 'BOSTON: WHERE THE PAST IS THE PRESENT',
+            'description': 'md5:4ec8844ac262ca2df9e67c0983c6b83f',
+            'uploader_id': 'seeso',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -91,10 +99,8 @@ class VRVIE(InfoExtractor):
         video_data = media_resource.get('json')
         if not video_data:
             self._set_api_params(webpage, video_id)
-            episode_path = self._call_api('cms_resource', video_id, 'episode resource path', data={
-                'resource_key': 'cms:/episodes/' + video_id,
-            })['__links__']['cms_resource']['href']
-            self._set_cms_signing(video_id)
+            episode_path = self._get_cms_resource(
+                'cms:/episodes/' + video_id, video_id)
             video_data = self._call_cms(episode_path, video_id, 'video')
         title = video_data['title']
 
@@ -102,7 +108,6 @@ class VRVIE(InfoExtractor):
         if not streams_json:
             self._set_api_params(webpage, video_id)
             streams_path = video_data['__links__']['streams']['href']
-            self._set_cms_signing(video_id)
             streams_json = self._call_cms(streams_path, video_id, 'streams')
 
         audio_locale = streams_json.get('audio_locale')
@@ -149,3 +154,38 @@ class VRVIE(InfoExtractor):
             'episode_number': int_or_none(video_data.get('episode_number')),
             'episode_id': video_data.get('production_episode_id'),
         }
+
+
+class VRVSeriesIE(VRVBaseIE):
+    IE_NAME = 'vrv:series'
+    _VALID_URL = r'https?://(?:www\.)?vrv\.co/series/(?P<id>[A-Z0-9]+)'
+    _TEST = {
+        'url': 'https://vrv.co/series/G68VXG3G6/The-Perfect-Insider',
+        'info_dict': {
+            'id': 'G68VXG3G6',
+        },
+        'playlist_mincount': 11,
+    }
+
+    def _real_extract(self, url):
+        series_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, series_id,
+            headers=self.geo_verification_headers())
+
+        self._set_api_params(webpage, series_id)
+        seasons_path = self._get_cms_resource(
+            'cms:/seasons?series_id=' + series_id, series_id)
+        seasons_data = self._call_cms(seasons_path, series_id, 'seasons')
+
+        entries = []
+        for season in seasons_data.get('items', []):
+            episodes_path = season['__links__']['season/episodes']['href']
+            episodes = self._call_cms(episodes_path, series_id, 'episodes')
+            for episode in episodes.get('items', []):
+                episode_id = episode['id']
+                entries.append(self.url_result(
+                    'https://vrv.co/watch/' + episode_id,
+                    'VRV', episode_id, episode.get('title')))
+
+        return self.playlist_result(entries, series_id)

From 51342717cddafde83dbf39f2212be40a196a577a Mon Sep 17 00:00:00 2001
From: Timendum <timedum@gmail.com>
Date: Tue, 14 Mar 2017 16:11:09 +0100
Subject: [PATCH 023/611] [rai] Fix extraction

---
 youtube_dl/extractor/extractors.py |   2 +-
 youtube_dl/extractor/rai.py        | 359 ++++++++++++++++-------------
 2 files changed, 197 insertions(+), 164 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 980333a11..d9e8d53ac 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -802,7 +802,7 @@ from .radiojavan import RadioJavanIE
 from .radiobremen import RadioBremenIE
 from .radiofrance import RadioFranceIE
 from .rai import (
-    RaiTVIE,
+    RaiPlayIE,
     RaiIE,
 )
 from .rbmaradio import RBMARadioIE
diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 41afbd9af..b67e94f88 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -3,8 +3,8 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
-    determine_ext,
     ExtractorError,
+    determine_ext,
     find_xpath_attr,
     fix_xml_ampersands,
     int_or_none,
@@ -55,7 +55,200 @@ class RaiBaseIE(InfoExtractor):
 
         return formats
 
-    def _extract_from_content_id(self, content_id, base_url):
+
+class RaiPlayIE(RaiBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?raiplay\.it/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})\.html'
+    _TESTS = [{
+        'url': 'http://www.raiplay.it/video/2016/10/La-Casa-Bianca-e06118bb-59a9-4636-b914-498e4cfd2c66.html?source=twitter',
+        'md5': '340aa3b7afb54bfd14a8c11786450d76',
+        'info_dict': {
+            'id': 'e06118bb-59a9-4636-b914-498e4cfd2c66',
+            'ext': 'mp4',
+            'title': 'La Casa Bianca',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'uploader': r're:^Rai.+',
+            'description': 're:^[A-Za-z]+'
+        }
+    }, {
+        'url': 'http://www.raiplay.it/video/2016/11/gazebotraindesi-efebe701-969c-4593-92f3-285f0d1ce750.html?',
+        'md5': 'ed4da3d70ccf8129a33ab16b34d20ab8',
+        'info_dict': {
+            'id': 'efebe701-969c-4593-92f3-285f0d1ce750',
+            'ext': 'mp4',
+            'title': 'Gazebo - #gazebotraindesi',
+            'thumbnail': r're:^https?://.*\.png$',
+            'uploader': r're:^Rai.+',
+            'description': r're:^[A-Za-z]+'
+        }
+    }, {
+        'url': 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
+        'md5': '8970abf8caf8aef4696e7b1f2adfc696',
+        'info_dict': {
+            'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391',
+            'ext': 'mp4',
+            'title': 'Report - Report del 07/04/2014',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'uploader': r're:^Rai.+',
+            'description': r're:^[A-Za-z]+'
+        }
+    }]
+    _RESOLUTION = '600x400'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        # remove query and fragment part from url
+        canonical_url = compat_urlparse.urljoin(url, compat_urlparse.urlparse(url).path)
+        webpage = self._download_webpage(canonical_url, video_id)
+
+        media = self._download_json('%s?json' % canonical_url,
+                                    video_id, 'Downloading video JSON')
+
+        thumbnails = []
+        if 'images' in media:
+            for _, value in media.get('images').items():
+                if value:
+                    thumbnails.append({
+                        'url': value.replace('[RESOLUTION]', self._RESOLUTION)
+                    })
+
+        if 'video' not in media:
+            raise ExtractorError('No video found')
+
+        video = media.get('video')
+        duration = parse_duration(video.get('duration')),
+        formats = self._extract_relinker_formats(video.get('contentUrl'), video_id)
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage).replace(' - video - RaiPlay', ''),
+            'description': self._og_search_description(webpage),
+            'uploader': media.get('channel'),
+            'duration': duration,
+            'thumbnails': thumbnails,
+            'formats': formats
+        }
+
+
+class RaiIE(RaiBaseIE):
+    _VALID_URL = r'https?://.+\.(?:rai|rainews)\.it/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
+    _TESTS = [{
+        # subdomain test case
+        'url': 'http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html',
+        'info_dict': {
+            'id': '04a9f4bd-b563-40cf-82a6-aad3529cb4a9',
+            'ext': 'mp4',
+            'title': 'TG PRIMO TEMPO',
+            'upload_date': '20140612',
+            'duration': 1758,
+            'thumbnail': r're:^https?://.*\.jpg$'
+        }
+    }, {
+        # rainews test case
+        'url': 'http://www.rainews.it/dl/rainews/media/Weekend-al-cinema-da-Hollywood-arriva-il-thriller-di-Tate-Taylor-La-ragazza-del-treno-1632c009-c843-4836-bb65-80c33084a64b.html',
+        'info_dict': {
+            'id': '1632c009-c843-4836-bb65-80c33084a64b',
+            'ext': 'mp4',
+            'title': 'Weekend al cinema, da Hollywood arriva il thriller di Tate Taylor \"La ragazza del treno\" ',
+            'upload_date': '20161103',
+            'thumbnail': r're:^https?://.*\.png$',
+            'description': r're:^[A-Za-z]+'
+        }
+    }, {
+        # with media information
+        'url': 'http://www.rai.it/dl/RaiTV/programmi/media/ContentItem-efb17665-691c-45d5-a60c-5301333cbb0c.html',
+        'md5': '11959b4e44fa74de47011b5799490adf',
+        'info_dict': {
+            'id': 'efb17665-691c-45d5-a60c-5301333cbb0c',
+            'ext': 'mp4',
+            'title': 'TG1 ore 20:00 del 03/11/2016',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'upload_date': '20161103',
+            'description': r're:^[A-Za-z]+'
+        }
+    }, {
+        # drawMediaRaiTV test case
+        'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html',
+        'md5': '2dd727e61114e1ee9c47f0da6914e178',
+        'info_dict': {
+            'id': '59d69d28-6bb6-409d-a4b5-ed44096560af',
+            'ext': 'mp4',
+            'title': 'Il pacco',
+            'description': 'md5:4b1afae1364115ce5d78ed83cd2e5b3a',
+            'upload_date': '20141221',
+        },
+    }, {
+        # Direct relinker URL
+        'url': 'http://www.rai.tv/dl/RaiTV/dirette/PublishingBlock-1912dbbf-3f96-44c3-b4cf-523681fbacbc.html?channel=EuroNews',
+        # HDS live stream, MD5 is unstable
+        'info_dict': {
+            'id': '1912dbbf-3f96-44c3-b4cf-523681fbacbc',
+            'ext': 'flv',
+            'title': 'EuroNews',
+        },
+    }, {
+        # Embedded content item ID
+        'url': 'http://www.tg1.rai.it/dl/tg1/2010/edizioni/ContentSet-9b6e0cba-4bef-4aef-8cf0-9f7f665b7dfb-tg1.html?item=undefined',
+        'info_dict': {
+            'id': 'd80d4b70-3812-4501-a888-92edec729f00',
+            'ext': 'mp4',
+            'title': r're:TG1 ore \d{2}:\d{2} del \d{2}/\d{2}/\d{4}',
+            'upload_date': r're:\d{8}',
+            'description': r're:.+',
+        },
+    }, {
+        'url': 'http://www.rainews.it/dl/rainews/live/ContentItem-3156f2f2-dc70-4953-8e2f-70d7489d4ce9.html',
+        # HDS live stream, MD5 is unstable
+        'info_dict': {
+            'id': '3156f2f2-dc70-4953-8e2f-70d7489d4ce9',
+            'ext': 'mp4',
+            'title': 'La diretta di Rainews24',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        iframe_url = self._search_regex(
+            [r'<iframe[^>]+src="([^"]*/dl/[^"]+\?iframe\b[^"]*)"',
+             r'drawMediaRaiTV\(["\'](.+?)["\']'],
+            webpage, 'iframe', default=None)
+        if iframe_url:
+            if not iframe_url.startswith('http'):
+                iframe_url = compat_urlparse.urljoin(url, iframe_url)
+            return self.url_result(iframe_url)
+
+        content_item_id = self._search_regex(
+            r'initEdizione\((?P<q1>[\'"])ContentItem-(?P<content_id>[^\'"]+)(?P=q1)',
+            webpage, 'content item ID', group='content_id', default=None)
+        if content_item_id:
+            return self._extract_from_content_id(content_item_id, url)
+
+        try:
+            return self._extract_from_content_id(video_id, url)
+        except ExtractorError:
+            # no media data, only direct relinker
+            pass
+
+        relinker_url = compat_urlparse.urljoin(url, self._search_regex(
+            r'(?:var\s+videoURL|mediaInfo\.mediaUri)\s*=\s*(?P<q1>[\'"])(?P<url>(https?:)?//mediapolis\.rai\.it/relinker/relinkerServlet\.htm\?cont=\d+)(?P=q1)',
+            webpage, 'relinker URL', group='url'))
+        formats = self._extract_relinker_formats(relinker_url, video_id)
+        self._sort_formats(formats)
+
+        title = self._search_regex(
+            r'var\s+videoTitolo\s*=\s*([\'"])(?P<title>[^\'"]+)\1',
+            webpage, 'title', group='title', default=None) or self._og_search_title(webpage)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+        }
+
+    def _extract_from_content_id(self, content_id, url):
         media = self._download_json(
             'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-%s.html?json' % content_id,
             content_id, 'Downloading video JSON')
@@ -65,7 +258,7 @@ class RaiBaseIE(InfoExtractor):
             thumbnail_url = media.get(image_type)
             if thumbnail_url:
                 thumbnails.append({
-                    'url': compat_urlparse.urljoin(base_url, thumbnail_url),
+                    'url': compat_urlparse.urljoin(url, thumbnail_url),
                 })
 
         formats = []
@@ -105,163 +298,3 @@ class RaiBaseIE(InfoExtractor):
             'formats': formats,
             'subtitles': subtitles,
         }
-
-
-class RaiTVIE(RaiBaseIE):
-    _VALID_URL = r'https?://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/(?:[^/]+/)+(?:media|ondemand)/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
-    _TESTS = [
-        {
-            'url': 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
-            'md5': '8970abf8caf8aef4696e7b1f2adfc696',
-            'info_dict': {
-                'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391',
-                'ext': 'mp4',
-                'title': 'Report del 07/04/2014',
-                'description': 'md5:f27c544694cacb46a078db84ec35d2d9',
-                'upload_date': '20140407',
-                'duration': 6160,
-                'thumbnail': r're:^https?://.*\.jpg$',
-            }
-        },
-        {
-            # no m3u8 stream
-            'url': 'http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html',
-            # HDS download, MD5 is unstable
-            'info_dict': {
-                'id': '04a9f4bd-b563-40cf-82a6-aad3529cb4a9',
-                'ext': 'flv',
-                'title': 'TG PRIMO TEMPO',
-                'upload_date': '20140612',
-                'duration': 1758,
-                'thumbnail': r're:^https?://.*\.jpg$',
-            },
-            'skip': 'Geo-restricted to Italy',
-        },
-        {
-            'url': 'http://www.rainews.it/dl/rainews/media/state-of-the-net-Antonella-La-Carpia-regole-virali-7aafdea9-0e5d-49d5-88a6-7e65da67ae13.html',
-            'md5': '35cf7c229f22eeef43e48b5cf923bef0',
-            'info_dict': {
-                'id': '7aafdea9-0e5d-49d5-88a6-7e65da67ae13',
-                'ext': 'mp4',
-                'title': 'State of the Net, Antonella La Carpia: regole virali',
-                'description': 'md5:b0ba04a324126903e3da7763272ae63c',
-                'upload_date': '20140613',
-            },
-            'skip': 'Error 404',
-        },
-        {
-            'url': 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-b4a49761-e0cc-4b14-8736-2729f6f73132-tg2.html',
-            'info_dict': {
-                'id': 'b4a49761-e0cc-4b14-8736-2729f6f73132',
-                'ext': 'mp4',
-                'title': 'Alluvione in Sardegna e dissesto idrogeologico',
-                'description': 'Edizione delle ore 20:30 ',
-            },
-            'skip': 'invalid urls',
-        },
-        {
-            'url': 'http://www.ilcandidato.rai.it/dl/ray/media/Il-Candidato---Primo-episodio-Le-Primarie-28e5525a-b495-45e8-a7c3-bc48ba45d2b6.html',
-            'md5': 'e57493e1cb8bc7c564663f363b171847',
-            'info_dict': {
-                'id': '28e5525a-b495-45e8-a7c3-bc48ba45d2b6',
-                'ext': 'mp4',
-                'title': 'Il Candidato - Primo episodio: "Le Primarie"',
-                'description': 'md5:364b604f7db50594678f483353164fb8',
-                'upload_date': '20140923',
-                'duration': 386,
-                'thumbnail': r're:^https?://.*\.jpg$',
-            }
-        },
-    ]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        return self._extract_from_content_id(video_id, url)
-
-
-class RaiIE(RaiBaseIE):
-    _VALID_URL = r'https?://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
-    _TESTS = [
-        {
-            'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html',
-            'md5': '2dd727e61114e1ee9c47f0da6914e178',
-            'info_dict': {
-                'id': '59d69d28-6bb6-409d-a4b5-ed44096560af',
-                'ext': 'mp4',
-                'title': 'Il pacco',
-                'description': 'md5:4b1afae1364115ce5d78ed83cd2e5b3a',
-                'upload_date': '20141221',
-            },
-        },
-        {
-            # Direct relinker URL
-            'url': 'http://www.rai.tv/dl/RaiTV/dirette/PublishingBlock-1912dbbf-3f96-44c3-b4cf-523681fbacbc.html?channel=EuroNews',
-            # HDS live stream, MD5 is unstable
-            'info_dict': {
-                'id': '1912dbbf-3f96-44c3-b4cf-523681fbacbc',
-                'ext': 'flv',
-                'title': 'EuroNews',
-            },
-            'skip': 'Geo-restricted to Italy',
-        },
-        {
-            # Embedded content item ID
-            'url': 'http://www.tg1.rai.it/dl/tg1/2010/edizioni/ContentSet-9b6e0cba-4bef-4aef-8cf0-9f7f665b7dfb-tg1.html?item=undefined',
-            'md5': '84c1135ce960e8822ae63cec34441d63',
-            'info_dict': {
-                'id': '0960e765-62c8-474a-ac4b-7eb3e2be39c8',
-                'ext': 'mp4',
-                'title': 'TG1 ore 20:00 del 02/07/2016',
-                'upload_date': '20160702',
-            },
-        },
-        {
-            'url': 'http://www.rainews.it/dl/rainews/live/ContentItem-3156f2f2-dc70-4953-8e2f-70d7489d4ce9.html',
-            # HDS live stream, MD5 is unstable
-            'info_dict': {
-                'id': '3156f2f2-dc70-4953-8e2f-70d7489d4ce9',
-                'ext': 'flv',
-                'title': 'La diretta di Rainews24',
-            },
-        },
-    ]
-
-    @classmethod
-    def suitable(cls, url):
-        return False if RaiTVIE.suitable(url) else super(RaiIE, cls).suitable(url)
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        iframe_url = self._search_regex(
-            [r'<iframe[^>]+src="([^"]*/dl/[^"]+\?iframe\b[^"]*)"',
-             r'drawMediaRaiTV\(["\'](.+?)["\']'],
-            webpage, 'iframe', default=None)
-        if iframe_url:
-            if not iframe_url.startswith('http'):
-                iframe_url = compat_urlparse.urljoin(url, iframe_url)
-            return self.url_result(iframe_url)
-
-        content_item_id = self._search_regex(
-            r'initEdizione\((?P<q1>[\'"])ContentItem-(?P<content_id>[^\'"]+)(?P=q1)',
-            webpage, 'content item ID', group='content_id', default=None)
-        if content_item_id:
-            return self._extract_from_content_id(content_item_id, url)
-
-        relinker_url = compat_urlparse.urljoin(url, self._search_regex(
-            r'(?:var\s+videoURL|mediaInfo\.mediaUri)\s*=\s*(?P<q1>[\'"])(?P<url>(https?:)?//mediapolis\.rai\.it/relinker/relinkerServlet\.htm\?cont=\d+)(?P=q1)',
-            webpage, 'relinker URL', group='url'))
-        formats = self._extract_relinker_formats(relinker_url, video_id)
-        self._sort_formats(formats)
-
-        title = self._search_regex(
-            r'var\s+videoTitolo\s*=\s*([\'"])(?P<title>[^\'"]+)\1',
-            webpage, 'title', group='title', default=None) or self._og_search_title(webpage)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-        }

From b8d8cced9b55c57f3b09e83972be9d6318a459ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Apr 2017 02:14:42 +0700
Subject: [PATCH 024/611] [rai] Improve extraction (closes #11790) * Fix
 georestriction detection * Detect live streams + Extract relinker metadata *
 Improve ContentItem detection + Extract series metadata * Fix tests

---
 youtube_dl/extractor/rai.py | 359 +++++++++++++++++++++++-------------
 1 file changed, 233 insertions(+), 126 deletions(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index b67e94f88..b77b0a08e 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -1,23 +1,40 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_urlparse,
+    compat_str,
+)
 from ..utils import (
     ExtractorError,
     determine_ext,
     find_xpath_attr,
     fix_xml_ampersands,
+    GeoRestrictedError,
     int_or_none,
     parse_duration,
+    strip_or_none,
+    try_get,
     unified_strdate,
+    unified_timestamp,
     update_url_query,
+    urljoin,
     xpath_text,
 )
 
 
 class RaiBaseIE(InfoExtractor):
-    def _extract_relinker_formats(self, relinker_url, video_id):
+    _UUID_RE = r'[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}'
+    _GEO_COUNTRIES = ['IT']
+    _GEO_BYPASS = False
+
+    def _extract_relinker_info(self, relinker_url, video_id):
         formats = []
+        geoprotection = None
+        is_live = None
+        duration = None
 
         for platform in ('mon', 'flash', 'native'):
             relinker = self._download_xml(
@@ -27,9 +44,27 @@ class RaiBaseIE(InfoExtractor):
                 query={'output': 45, 'pl': platform},
                 headers=self.geo_verification_headers())
 
-            media_url = find_xpath_attr(relinker, './url', 'type', 'content').text
+            if not geoprotection:
+                geoprotection = xpath_text(
+                    relinker, './geoprotection', default=None) == 'Y'
+
+            if not is_live:
+                is_live = xpath_text(
+                    relinker, './is_live', default=None) == 'Y'
+            if not duration:
+                duration = parse_duration(xpath_text(
+                    relinker, './duration', default=None))
+
+            url_elem = find_xpath_attr(relinker, './url', 'type', 'content')
+            if url_elem is None:
+                continue
+
+            media_url = url_elem.text
+
+            # This does not imply geo restriction (e.g.
+            # http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html)
             if media_url == 'http://download.rai.it/video_no_available.mp4':
-                self.raise_geo_restricted()
+                continue
 
             ext = determine_ext(media_url)
             if (ext == 'm3u8' and platform != 'mon') or (ext == 'f4m' and platform != 'flash'):
@@ -53,11 +88,18 @@ class RaiBaseIE(InfoExtractor):
                     'format_id': 'http-%d' % bitrate if bitrate > 0 else 'http',
                 })
 
-        return formats
+        if not formats and geoprotection is True:
+            self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
+
+        return dict((k, v) for k, v in {
+            'is_live': is_live,
+            'duration': duration,
+            'formats': formats,
+        }.items() if v is not None)
 
 
 class RaiPlayIE(RaiBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?raiplay\.it/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})\.html'
+    _VALID_URL = r'(?P<url>https?://(?:www\.)?raiplay\.it/.+?-(?P<id>%s)\.html)' % RaiBaseIE._UUID_RE
     _TESTS = [{
         'url': 'http://www.raiplay.it/video/2016/10/La-Casa-Bianca-e06118bb-59a9-4636-b914-498e4cfd2c66.html?source=twitter',
         'md5': '340aa3b7afb54bfd14a8c11786450d76',
@@ -65,110 +107,130 @@ class RaiPlayIE(RaiBaseIE):
             'id': 'e06118bb-59a9-4636-b914-498e4cfd2c66',
             'ext': 'mp4',
             'title': 'La Casa Bianca',
+            'alt_title': 'S2016 - Puntata del 23/10/2016',
+            'description': 'md5:a09d45890850458077d1f68bb036e0a5',
             'thumbnail': r're:^https?://.*\.jpg$',
-            'uploader': r're:^Rai.+',
-            'description': 're:^[A-Za-z]+'
-        }
-    }, {
-        'url': 'http://www.raiplay.it/video/2016/11/gazebotraindesi-efebe701-969c-4593-92f3-285f0d1ce750.html?',
-        'md5': 'ed4da3d70ccf8129a33ab16b34d20ab8',
-        'info_dict': {
-            'id': 'efebe701-969c-4593-92f3-285f0d1ce750',
-            'ext': 'mp4',
-            'title': 'Gazebo - #gazebotraindesi',
-            'thumbnail': r're:^https?://.*\.png$',
-            'uploader': r're:^Rai.+',
-            'description': r're:^[A-Za-z]+'
-        }
+            'uploader': 'Rai 3',
+            'creator': 'Rai 3',
+            'duration': 3278,
+            'timestamp': 1477764300,
+            'upload_date': '20161029',
+            'series': 'La Casa Bianca',
+            'season': '2016',
+        },
     }, {
         'url': 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
         'md5': '8970abf8caf8aef4696e7b1f2adfc696',
         'info_dict': {
             'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391',
             'ext': 'mp4',
-            'title': 'Report - Report del 07/04/2014',
+            'title': 'Report del 07/04/2014',
+            'alt_title': 'S2013/14 - Puntata del 07/04/2014',
+            'description': 'md5:f27c544694cacb46a078db84ec35d2d9',
             'thumbnail': r're:^https?://.*\.jpg$',
-            'uploader': r're:^Rai.+',
-            'description': r're:^[A-Za-z]+'
-        }
+            'uploader': 'Rai 5',
+            'creator': 'Rai 5',
+            'duration': 6160,
+            'series': 'Report',
+            'season_number': 5,
+            'season': '2013/14',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.raiplay.it/video/2016/11/gazebotraindesi-efebe701-969c-4593-92f3-285f0d1ce750.html?',
+        'only_matching': True,
     }]
-    _RESOLUTION = '600x400'
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        url, video_id = mobj.group('url', 'id')
 
-        # remove query and fragment part from url
-        canonical_url = compat_urlparse.urljoin(url, compat_urlparse.urlparse(url).path)
-        webpage = self._download_webpage(canonical_url, video_id)
+        media = self._download_json(
+            '%s?json' % url, video_id, 'Downloading video JSON')
 
-        media = self._download_json('%s?json' % canonical_url,
-                                    video_id, 'Downloading video JSON')
+        title = media['name']
+
+        video = media['video']
+
+        relinker_info = self._extract_relinker_info(video['contentUrl'], video_id)
+        self._sort_formats(relinker_info['formats'])
 
         thumbnails = []
         if 'images' in media:
             for _, value in media.get('images').items():
                 if value:
                     thumbnails.append({
-                        'url': value.replace('[RESOLUTION]', self._RESOLUTION)
+                        'url': value.replace('[RESOLUTION]', '600x400')
                     })
 
-        if 'video' not in media:
-            raise ExtractorError('No video found')
+        timestamp = unified_timestamp(try_get(
+            media, lambda x: x['availabilities'][0]['start'], compat_str))
 
-        video = media.get('video')
-        duration = parse_duration(video.get('duration')),
-        formats = self._extract_relinker_formats(video.get('contentUrl'), video_id)
-        self._sort_formats(formats)
-
-        return {
+        info = {
             'id': video_id,
-            'title': self._og_search_title(webpage).replace(' - video - RaiPlay', ''),
-            'description': self._og_search_description(webpage),
+            'title': title,
+            'alt_title': media.get('subtitle'),
+            'description': media.get('description'),
             'uploader': media.get('channel'),
-            'duration': duration,
+            'creator': media.get('editor'),
+            'duration': parse_duration(video.get('duration')),
+            'timestamp': timestamp,
             'thumbnails': thumbnails,
-            'formats': formats
+            'series': try_get(
+                media, lambda x: x['isPartOf']['name'], compat_str),
+            'season_number': int_or_none(try_get(
+                media, lambda x: x['isPartOf']['numeroStagioni'])),
+            'season': media.get('stagione') or None,
         }
 
+        info.update(relinker_info)
+
+        return info
+
 
 class RaiIE(RaiBaseIE):
-    _VALID_URL = r'https?://.+\.(?:rai|rainews)\.it/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
+    _VALID_URL = r'https?://[^/]+\.(?:rai\.(?:it|tv)|rainews\.it)/dl/.+?-(?P<id>%s)(?:-.+?)?\.html' % RaiBaseIE._UUID_RE
     _TESTS = [{
-        # subdomain test case
+        # var uniquename = "ContentItem-..."
+        # data-id="ContentItem-..."
         'url': 'http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html',
         'info_dict': {
             'id': '04a9f4bd-b563-40cf-82a6-aad3529cb4a9',
             'ext': 'mp4',
             'title': 'TG PRIMO TEMPO',
-            'upload_date': '20140612',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 1758,
-            'thumbnail': r're:^https?://.*\.jpg$'
+            'upload_date': '20140612',
         }
     }, {
-        # rainews test case
+        # with ContentItem in many metas
         'url': 'http://www.rainews.it/dl/rainews/media/Weekend-al-cinema-da-Hollywood-arriva-il-thriller-di-Tate-Taylor-La-ragazza-del-treno-1632c009-c843-4836-bb65-80c33084a64b.html',
         'info_dict': {
             'id': '1632c009-c843-4836-bb65-80c33084a64b',
             'ext': 'mp4',
-            'title': 'Weekend al cinema, da Hollywood arriva il thriller di Tate Taylor \"La ragazza del treno\" ',
-            'upload_date': '20161103',
+            'title': 'Weekend al cinema, da Hollywood arriva il thriller di Tate Taylor "La ragazza del treno"',
+            'description': 'I film in uscita questa settimana.',
             'thumbnail': r're:^https?://.*\.png$',
-            'description': r're:^[A-Za-z]+'
+            'duration': 833,
+            'upload_date': '20161103',
         }
     }, {
-        # with media information
+        # with ContentItem in og:url
         'url': 'http://www.rai.it/dl/RaiTV/programmi/media/ContentItem-efb17665-691c-45d5-a60c-5301333cbb0c.html',
         'md5': '11959b4e44fa74de47011b5799490adf',
         'info_dict': {
             'id': 'efb17665-691c-45d5-a60c-5301333cbb0c',
             'ext': 'mp4',
             'title': 'TG1 ore 20:00 del 03/11/2016',
+            'description': 'TG1 edizione integrale ore 20:00 del giorno 03/11/2016',
             'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 2214,
             'upload_date': '20161103',
-            'description': r're:^[A-Za-z]+'
         }
     }, {
-        # drawMediaRaiTV test case
+        # drawMediaRaiTV(...)
         'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html',
         'md5': '2dd727e61114e1ee9c47f0da6914e178',
         'info_dict': {
@@ -176,83 +238,67 @@ class RaiIE(RaiBaseIE):
             'ext': 'mp4',
             'title': 'Il pacco',
             'description': 'md5:4b1afae1364115ce5d78ed83cd2e5b3a',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'upload_date': '20141221',
         },
     }, {
-        # Direct relinker URL
+        # initEdizione('ContentItem-...'
+        'url': 'http://www.tg1.rai.it/dl/tg1/2010/edizioni/ContentSet-9b6e0cba-4bef-4aef-8cf0-9f7f665b7dfb-tg1.html?item=undefined',
+        'info_dict': {
+            'id': 'c2187016-8484-4e3a-8ac8-35e475b07303',
+            'ext': 'mp4',
+            'title': r're:TG1 ore \d{2}:\d{2} del \d{2}/\d{2}/\d{4}',
+            'duration': 2274,
+            'upload_date': '20170401',
+        },
+        'skip': 'Changes daily',
+    }, {
+        # HDS live stream with only relinker URL
         'url': 'http://www.rai.tv/dl/RaiTV/dirette/PublishingBlock-1912dbbf-3f96-44c3-b4cf-523681fbacbc.html?channel=EuroNews',
-        # HDS live stream, MD5 is unstable
         'info_dict': {
             'id': '1912dbbf-3f96-44c3-b4cf-523681fbacbc',
             'ext': 'flv',
             'title': 'EuroNews',
         },
-    }, {
-        # Embedded content item ID
-        'url': 'http://www.tg1.rai.it/dl/tg1/2010/edizioni/ContentSet-9b6e0cba-4bef-4aef-8cf0-9f7f665b7dfb-tg1.html?item=undefined',
-        'info_dict': {
-            'id': 'd80d4b70-3812-4501-a888-92edec729f00',
-            'ext': 'mp4',
-            'title': r're:TG1 ore \d{2}:\d{2} del \d{2}/\d{2}/\d{4}',
-            'upload_date': r're:\d{8}',
-            'description': r're:.+',
+        'params': {
+            'skip_download': True,
         },
     }, {
+        # HLS live stream with ContentItem in og:url
         'url': 'http://www.rainews.it/dl/rainews/live/ContentItem-3156f2f2-dc70-4953-8e2f-70d7489d4ce9.html',
-        # HDS live stream, MD5 is unstable
         'info_dict': {
             'id': '3156f2f2-dc70-4953-8e2f-70d7489d4ce9',
             'ext': 'mp4',
             'title': 'La diretta di Rainews24',
         },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        iframe_url = self._search_regex(
-            [r'<iframe[^>]+src="([^"]*/dl/[^"]+\?iframe\b[^"]*)"',
-             r'drawMediaRaiTV\(["\'](.+?)["\']'],
-            webpage, 'iframe', default=None)
-        if iframe_url:
-            if not iframe_url.startswith('http'):
-                iframe_url = compat_urlparse.urljoin(url, iframe_url)
-            return self.url_result(iframe_url)
-
-        content_item_id = self._search_regex(
-            r'initEdizione\((?P<q1>[\'"])ContentItem-(?P<content_id>[^\'"]+)(?P=q1)',
-            webpage, 'content item ID', group='content_id', default=None)
-        if content_item_id:
-            return self._extract_from_content_id(content_item_id, url)
-
-        try:
-            return self._extract_from_content_id(video_id, url)
-        except ExtractorError:
-            # no media data, only direct relinker
-            pass
-
-        relinker_url = compat_urlparse.urljoin(url, self._search_regex(
-            r'(?:var\s+videoURL|mediaInfo\.mediaUri)\s*=\s*(?P<q1>[\'"])(?P<url>(https?:)?//mediapolis\.rai\.it/relinker/relinkerServlet\.htm\?cont=\d+)(?P=q1)',
-            webpage, 'relinker URL', group='url'))
-        formats = self._extract_relinker_formats(relinker_url, video_id)
-        self._sort_formats(formats)
-
-        title = self._search_regex(
-            r'var\s+videoTitolo\s*=\s*([\'"])(?P<title>[^\'"]+)\1',
-            webpage, 'title', group='title', default=None) or self._og_search_title(webpage)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-        }
-
     def _extract_from_content_id(self, content_id, url):
         media = self._download_json(
             'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-%s.html?json' % content_id,
             content_id, 'Downloading video JSON')
 
+        title = media['name'].strip()
+
+        media_type = media['type']
+        if 'Audio' in media_type:
+            relinker_info = {
+                'formats': {
+                    'format_id': media.get('formatoAudio'),
+                    'url': media['audioUrl'],
+                    'ext': media.get('formatoAudio'),
+                }
+            }
+        elif 'Video' in media_type:
+            relinker_info = self._extract_relinker_info(media['mediaUri'], content_id)
+        else:
+            raise ExtractorError('not a media file')
+
+        self._sort_formats(relinker_info['formats'])
+
         thumbnails = []
         for image_type in ('image', 'image_medium', 'image_300'):
             thumbnail_url = media.get(image_type)
@@ -261,20 +307,6 @@ class RaiIE(RaiBaseIE):
                     'url': compat_urlparse.urljoin(url, thumbnail_url),
                 })
 
-        formats = []
-        media_type = media['type']
-        if 'Audio' in media_type:
-            formats.append({
-                'format_id': media.get('formatoAudio'),
-                'url': media['audioUrl'],
-                'ext': media.get('formatoAudio'),
-            })
-        elif 'Video' in media_type:
-            formats.extend(self._extract_relinker_formats(media['mediaUri'], content_id))
-            self._sort_formats(formats)
-        else:
-            raise ExtractorError('not a media file')
-
         subtitles = {}
         captions = media.get('subtitlesUrl')
         if captions:
@@ -287,14 +319,89 @@ class RaiIE(RaiBaseIE):
                 'url': captions,
             }]
 
-        return {
+        info = {
             'id': content_id,
-            'title': media['name'],
-            'description': media.get('desc'),
+            'title': title,
+            'description': strip_or_none(media.get('desc')),
             'thumbnails': thumbnails,
             'uploader': media.get('author'),
             'upload_date': unified_strdate(media.get('date')),
             'duration': parse_duration(media.get('length')),
-            'formats': formats,
             'subtitles': subtitles,
         }
+
+        info.update(relinker_info)
+
+        return info
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        content_item_id = None
+
+        content_item_url = self._html_search_meta(
+            ('og:url', 'og:video', 'og:video:secure_url', 'twitter:url',
+             'twitter:player', 'jsonlink'), webpage, default=None)
+        if content_item_url:
+            content_item_id = self._search_regex(
+                r'ContentItem-(%s)' % self._UUID_RE, content_item_url,
+                'content item id', default=None)
+
+        if not content_item_id:
+            content_item_id = self._search_regex(
+                r'''(?x)
+                    (?:
+                        (?:initEdizione|drawMediaRaiTV)\(|
+                        <(?:[^>]+\bdata-id|var\s+uniquename)=
+                    )
+                    (["\'])
+                    (?:(?!\1).)*\bContentItem-(?P<id>%s)
+                ''' % self._UUID_RE,
+                webpage, 'content item id', default=None, group='id')
+
+        content_item_ids = set()
+        content_item_ids.add(content_item_id)
+        if video_id not in content_item_ids:
+            content_item_ids.add(video_id)
+
+        for content_item_id in content_item_ids:
+            try:
+                return self._extract_from_content_id(content_item_id, url)
+            except GeoRestrictedError:
+                raise
+            except ExtractorError:
+                pass
+
+        relinker_url = self._search_regex(
+            r'''(?x)
+                (?:
+                    var\s+videoURL|
+                    mediaInfo\.mediaUri
+                )\s*=\s*
+                ([\'"])
+                (?P<url>
+                    (?:https?:)?
+                    //mediapolis(?:vod)?\.rai\.it/relinker/relinkerServlet\.htm\?
+                    (?:(?!\1).)*\bcont=(?:(?!\1).)+)\1
+            ''',
+            webpage, 'relinker URL', group='url')
+
+        relinker_info = self._extract_relinker_info(
+            urljoin(url, relinker_url), video_id)
+        self._sort_formats(relinker_info['formats'])
+
+        title = self._search_regex(
+            r'var\s+videoTitolo\s*=\s*([\'"])(?P<title>[^\'"]+)\1',
+            webpage, 'title', group='title',
+            default=None) or self._og_search_title(webpage)
+
+        info = {
+            'id': video_id,
+            'title': title,
+        }
+
+        info.update(relinker_info)
+
+        return info

From 361f293ab85c29ab62cb91577d2be34814d5c552 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Apr 2017 02:24:13 +0700
Subject: [PATCH 025/611] [rai] Skip not found content item id

---
 youtube_dl/extractor/rai.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index b77b0a08e..077546a73 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -362,7 +362,8 @@ class RaiIE(RaiBaseIE):
                 webpage, 'content item id', default=None, group='id')
 
         content_item_ids = set()
-        content_item_ids.add(content_item_id)
+        if content_item_id:
+            content_item_ids.add(content_item_id)
         if video_id not in content_item_ids:
             content_item_ids.add(video_id)
 

From a76c25146a93052f367a0fb8cdd9a08ba9cef491 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Apr 2017 02:37:18 +0700
Subject: [PATCH 026/611] [ChangeLog] Actualize

---
 ChangeLog | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 07725b12a..3ffc647f1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,27 @@
+version <unreleased>
+
+Core
+[YoutubeDL] Return early when extraction of url_transparent fails
+
+Extractors
+* [rai] Fix and improve extraction (#11790)
++ [vrv] Add support for series pages
+* [limelight] Improve extraction for audio only formats
+* [funimation] Fix extraction (#10696, #11773)
++ [xfileshare] Add support for vidabc.com (#12589)
++ [xfileshare] Improve extraction and extract hls formats
++ [crunchyroll] Pass geo verifcation proxy
++ [cwtv] Extract ISM formats
++ [tvplay] Bypass geo restriction
++ [vrv] Add support for vrv.co
++ [packtpub] Add support for packtpub.com (#12610)
++ [generic] Pass base_url to _parse_jwplayer_data
++ [adn] Add support for animedigitalnetwork.fr (#4866)
++ [allocine] Extract more metadata
+* [allocine] Fix extraction (#12592)
+* [openload] Fix extraction
+
+
 version 2017.03.26
 
 Core

From b56e41a701d73072b7d62a151b7aafd87955dfe8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Apr 2017 02:39:15 +0700
Subject: [PATCH 027/611] release 2017.04.02

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 README.md                 | 8 ++++----
 docs/supportedsites.md    | 9 +++++++--
 youtube_dl/version.py     | 2 +-
 5 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 2f717926c..c1b737619 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.26*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.26**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.02*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.02**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.03.26
+[debug] youtube-dl version 2017.04.02
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 3ffc647f1..0199bdf1f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.04.02
 
 Core
 [YoutubeDL] Return early when extraction of url_transparent fails
diff --git a/README.md b/README.md
index 86b44781c..41f647aaa 100644
--- a/README.md
+++ b/README.md
@@ -181,10 +181,10 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     -R, --retries RETRIES            Number of retries (default is 10), or
                                      "infinite".
     --fragment-retries RETRIES       Number of retries for a fragment (default
-                                     is 10), or "infinite" (DASH and hlsnative
-                                     only)
-    --skip-unavailable-fragments     Skip unavailable fragments (DASH and
-                                     hlsnative only)
+                                     is 10), or "infinite" (DASH, hlsnative and
+                                     ISM)
+    --skip-unavailable-fragments     Skip unavailable fragments (DASH, hlsnative
+                                     and ISM)
     --abort-on-unavailable-fragment  Abort downloading when some fragment is not
                                      available
     --buffer-size SIZE               Size of download buffer (e.g. 1024 or 16K)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index e9dbc021b..5c1855111 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -28,6 +28,7 @@
  - **acast**
  - **acast:channel**
  - **AddAnime**
+ - **ADN**: Anime Digital Network
  - **AdobeTV**
  - **AdobeTVChannel**
  - **AdobeTVShow**
@@ -572,6 +573,8 @@
  - **orf:iptv**: iptv.ORF.at
  - **orf:oe1**: Radio Österreich 1
  - **orf:tvthek**: ORF TVthek
+ - **PacktPub**
+ - **PacktPubCourse**
  - **PandaTV**: 熊猫TV
  - **pandora.tv**: 판도라TV
  - **parliamentlive.tv**: UK parliament videos
@@ -629,7 +632,7 @@
  - **radiofrance**
  - **RadioJavan**
  - **Rai**
- - **RaiTV**
+ - **RaiPlay**
  - **RBMARadio**
  - **RDS**: RDS.ca
  - **RedBullTV**
@@ -926,6 +929,8 @@
  - **vpro**: npo.nl and ntr.nl
  - **Vrak**
  - **VRT**
+ - **vrv**
+ - **vrv:series**
  - **vube**: Vube.com
  - **VuClip**
  - **VVVVID**
@@ -953,7 +958,7 @@
  - **WSJ**: Wall Street Journal
  - **XBef**
  - **XboxClips**
- - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To, XVIDSTAGE
+ - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To, XVIDSTAGE, Vid ABC
  - **XHamster**
  - **XHamsterEmbed**
  - **xiami:album**: 虾米音乐 - 专辑
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 94e8198ec..f612d03ca 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.03.26'
+__version__ = '2017.04.02'

From b3633fa0ce0f98801582f8e4e348436b0f361eb7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Apr 2017 03:20:28 +0700
Subject: [PATCH 028/611] [pericope] Add support for pscp.tv URLs

---
 youtube_dl/extractor/periscope.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 0e3623024..1add6b840 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -20,7 +20,7 @@ class PeriscopeBaseIE(InfoExtractor):
 class PeriscopeIE(PeriscopeBaseIE):
     IE_DESC = 'Periscope'
     IE_NAME = 'periscope'
-    _VALID_URL = r'https?://(?:www\.)?periscope\.tv/[^/]+/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:periscope|pscp)\.tv/[^/]+/(?P<id>[^/?#]+)'
     # Alive example URLs can be found here http://onperiscope.com/
     _TESTS = [{
         'url': 'https://www.periscope.tv/w/aJUQnjY3MjA3ODF8NTYxMDIyMDl2zCg2pECBgwTqRpQuQD352EMPTKQjT4uqlM3cgWFA-g==',
@@ -41,6 +41,9 @@ class PeriscopeIE(PeriscopeBaseIE):
     }, {
         'url': 'https://www.periscope.tv/bastaakanoggano/1OdKrlkZZjOJX',
         'only_matching': True,
+    }, {
+        'url': 'https://www.periscope.tv/w/1ZkKzPbMVggJv',
+        'only_matching': True,
     }]
 
     @staticmethod
@@ -103,7 +106,7 @@ class PeriscopeIE(PeriscopeBaseIE):
 
 
 class PeriscopeUserIE(PeriscopeBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?periscope\.tv/(?P<id>[^/]+)/?$'
+    _VALID_URL = r'https?://(?:www\.)?(?:periscope|pscp)\.tv/(?P<id>[^/]+)/?$'
     IE_DESC = 'Periscope user videos'
     IE_NAME = 'periscope:user'
 

From 4457823dda410c5406f5ab5474b9b1f9325fa7ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Apr 2017 03:56:49 +0700
Subject: [PATCH 029/611] [extractor/common] Move censorship checks to a
 separate method and add check for just another ISP

---
 youtube_dl/extractor/common.py | 48 ++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6c3c095f7..cdfa7000b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -547,6 +547,34 @@ class InfoExtractor(object):
 
         return encoding
 
+    def __check_blocked(self, content):
+        first_block = content[:512]
+        if ('<title>Access to this site is blocked</title>' in content and
+                'Websense' in first_block):
+            msg = 'Access to this webpage has been blocked by Websense filtering software in your network.'
+            blocked_iframe = self._html_search_regex(
+                r'<iframe src="([^"]+)"', content,
+                'Websense information URL', default=None)
+            if blocked_iframe:
+                msg += ' Visit %s for more details' % blocked_iframe
+            raise ExtractorError(msg, expected=True)
+        if '<title>The URL you requested has been blocked</title>' in first_block:
+            msg = (
+                'Access to this webpage has been blocked by Indian censorship. '
+                'Use a VPN or proxy server (with --proxy) to route around it.')
+            block_msg = self._html_search_regex(
+                r'</h1><p>(.*?)</p>',
+                content, 'block message', default=None)
+            if block_msg:
+                msg += ' (Message: "%s")' % block_msg.replace('\n', ' ')
+            raise ExtractorError(msg, expected=True)
+        if ('<title>TTK :: Доступ к ресурсу ограничен</title>' in content and
+                'blocklist.rkn.gov.ru' in content):
+            raise ExtractorError(
+                'Access to this webpage has been blocked by decision of the Russian government. '
+                'Visit http://blocklist.rkn.gov.ru/ for a block reason.',
+                expected=True)
+
     def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True, prefix=None, encoding=None):
         content_type = urlh.headers.get('Content-Type', '')
         webpage_bytes = urlh.read()
@@ -588,25 +616,7 @@ class InfoExtractor(object):
         except LookupError:
             content = webpage_bytes.decode('utf-8', 'replace')
 
-        if ('<title>Access to this site is blocked</title>' in content and
-                'Websense' in content[:512]):
-            msg = 'Access to this webpage has been blocked by Websense filtering software in your network.'
-            blocked_iframe = self._html_search_regex(
-                r'<iframe src="([^"]+)"', content,
-                'Websense information URL', default=None)
-            if blocked_iframe:
-                msg += ' Visit %s for more details' % blocked_iframe
-            raise ExtractorError(msg, expected=True)
-        if '<title>The URL you requested has been blocked</title>' in content[:512]:
-            msg = (
-                'Access to this webpage has been blocked by Indian censorship. '
-                'Use a VPN or proxy server (with --proxy) to route around it.')
-            block_msg = self._html_search_regex(
-                r'</h1><p>(.*?)</p>',
-                content, 'block message', default=None)
-            if block_msg:
-                msg += ' (Message: "%s")' % block_msg.replace('\n', ' ')
-            raise ExtractorError(msg, expected=True)
+        self.__check_blocked(content)
 
         return content
 

From fd47550885fc0abcdddbfdec70e00bc0d6a9865e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Apr 2017 04:42:10 +0700
Subject: [PATCH 030/611] [extractor/common] Add coding cookie

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index cdfa7000b..ae8af61de 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import base64

From 61e2331ad83569b3256e32f415c22cc439c000ed Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 1 Apr 2017 23:47:35 +0100
Subject: [PATCH 031/611] [tv5mondeplus] Add new extractor(closes #11386)

---
 youtube_dl/extractor/extractors.py   |  1 +
 youtube_dl/extractor/tv5mondeplus.py | 78 ++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)
 create mode 100644 youtube_dl/extractor/tv5mondeplus.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d9e8d53ac..46e5e3a1d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1023,6 +1023,7 @@ from .tv2 import (
 )
 from .tv3 import TV3IE
 from .tv4 import TV4IE
+from .tv5mondeplus import TV5MondePlusIE
 from .tva import TVAIE
 from .tvanouvelles import (
     TVANouvellesIE,
diff --git a/youtube_dl/extractor/tv5mondeplus.py b/youtube_dl/extractor/tv5mondeplus.py
new file mode 100644
index 000000000..8b2d7812a
--- /dev/null
+++ b/youtube_dl/extractor/tv5mondeplus.py
@@ -0,0 +1,78 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    extract_attributes,
+    get_element_by_class,
+    int_or_none,
+    parse_duration,
+    parse_iso8601,
+)
+
+
+class TV5MondePlusIE(InfoExtractor):
+    IE_DESC = 'TV5MONDE+'
+    _VALID_URL = r'https?://(?:www\.)?tv5mondeplus\.com/toutes-les-videos/[^/]+/(?P<id>[^/?#]+)'
+    _TEST = {
+        'url': 'http://www.tv5mondeplus.com/toutes-les-videos/documentaire/tdah-mon-amour-tele-quebec-tdah-mon-amour-ep001-enfants',
+        'md5': '12130fc199f020673138a83466542ec6',
+        'info_dict': {
+            'id': '0a774110-dc60-4037-f769-996439514f1f',
+            'ext': 'mp4',
+            'title': 'Tdah, mon amour - Enfants',
+            'description': 'md5:b65f0cc50e46947e62e5d352e9916cc4',
+            'upload_date': '20170401',
+            'timestamp': 1491022860,
+        }
+    }
+    _GEO_BYPASS = False
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        if ">Ce programme n'est malheureusement pas disponible pour votre zone géographique.<" in webpage:
+            self.raise_geo_restricted(countries=['FR'])
+
+        series = get_element_by_class('video-detail__title', webpage)
+        title = episode = get_element_by_class(
+            'video-detail__subtitle', webpage) or series
+        if series and series != title:
+            title = '%s - %s' % (series, title)
+        vpl_data = extract_attributes(self._search_regex(
+            r'(<[^>]+class="video_player_loader"[^>]+>)',
+            webpage, 'video player loader'))
+
+        video_files = self._parse_json(
+            vpl_data['data-broadcast'], display_id).get('files', [])
+        formats = []
+        for video_file in video_files:
+            v_url = video_file.get('url')
+            if not v_url:
+                continue
+            video_format = video_file.get('format') or determine_ext(v_url)
+            if video_format == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    v_url, display_id, 'mp4', 'm3u8_native',
+                    m3u8_id='hls', fatal=False))
+            else:
+                formats.append({
+                    'url': v_url,
+                    'format_id': video_format,
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': vpl_data.get('data-guid') or display_id,
+            'display_id': display_id,
+            'title': title,
+            'description': get_element_by_class('video-detail__description', webpage),
+            'thumbnail': vpl_data.get('data-image'),
+            'duration': int_or_none(vpl_data.get('data-duration')) or parse_duration(self._html_search_meta('duration', webpage)),
+            'timestamp': parse_iso8601(self._html_search_meta('uploadDate', webpage)),
+            'formats': formats,
+            'episode': episode,
+            'series': series,
+        }

From dbf70c489f1bcf36d44afb8384d7abdb8a34d960 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 2 Apr 2017 00:26:48 +0100
Subject: [PATCH 032/611] [tv5mondeplus] clean description and use stable id

---
 youtube_dl/extractor/tv5mondeplus.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/tv5mondeplus.py b/youtube_dl/extractor/tv5mondeplus.py
index 8b2d7812a..88b6baa31 100644
--- a/youtube_dl/extractor/tv5mondeplus.py
+++ b/youtube_dl/extractor/tv5mondeplus.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    clean_html,
     determine_ext,
     extract_attributes,
     get_element_by_class,
@@ -19,10 +20,10 @@ class TV5MondePlusIE(InfoExtractor):
         'url': 'http://www.tv5mondeplus.com/toutes-les-videos/documentaire/tdah-mon-amour-tele-quebec-tdah-mon-amour-ep001-enfants',
         'md5': '12130fc199f020673138a83466542ec6',
         'info_dict': {
-            'id': '0a774110-dc60-4037-f769-996439514f1f',
+            'id': 'tdah-mon-amour-tele-quebec-tdah-mon-amour-ep001-enfants',
             'ext': 'mp4',
             'title': 'Tdah, mon amour - Enfants',
-            'description': 'md5:b65f0cc50e46947e62e5d352e9916cc4',
+            'description': 'md5:230e3aca23115afcf8006d1bece6df74',
             'upload_date': '20170401',
             'timestamp': 1491022860,
         }
@@ -65,10 +66,10 @@ class TV5MondePlusIE(InfoExtractor):
         self._sort_formats(formats)
 
         return {
-            'id': vpl_data.get('data-guid') or display_id,
+            'id': display_id,
             'display_id': display_id,
             'title': title,
-            'description': get_element_by_class('video-detail__description', webpage),
+            'description': clean_html(get_element_by_class('video-detail__description', webpage)),
             'thumbnail': vpl_data.get('data-image'),
             'duration': int_or_none(vpl_data.get('data-duration')) or parse_duration(self._html_search_meta('duration', webpage)),
             'timestamp': parse_iso8601(self._html_search_meta('uploadDate', webpage)),

From a9bb61a4259972378dec5ef45d181fec150146b8 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 2 Apr 2017 09:22:09 +0100
Subject: [PATCH 033/611] [discoveryvr] Add new extractor(closes #12578)

---
 youtube_dl/extractor/discoveryvr.py | 59 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py  |  1 +
 2 files changed, 60 insertions(+)
 create mode 100644 youtube_dl/extractor/discoveryvr.py

diff --git a/youtube_dl/extractor/discoveryvr.py b/youtube_dl/extractor/discoveryvr.py
new file mode 100644
index 000000000..cb63c2649
--- /dev/null
+++ b/youtube_dl/extractor/discoveryvr.py
@@ -0,0 +1,59 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import parse_duration
+
+
+class DiscoveryVRIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?discoveryvr\.com/watch/(?P<id>[^/?#]+)'
+    _TEST = {
+        'url': 'http://www.discoveryvr.com/watch/discovery-vr-an-introduction',
+        'md5': '32b1929798c464a54356378b7912eca4',
+        'info_dict': {
+            'id': 'discovery-vr-an-introduction',
+            'ext': 'mp4',
+            'title': 'Discovery VR - An Introduction',
+            'description': 'md5:80d418a10efb8899d9403e61d8790f06',
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        bootstrap_data = self._search_regex(
+            r'root\.DVR\.bootstrapData\s+=\s+"({.+?})";',
+            webpage, 'bootstrap data')
+        bootstrap_data = self._parse_json(
+            bootstrap_data.encode('utf-8').decode('unicode_escape'),
+            display_id)
+        videos = self._parse_json(bootstrap_data['videos'], display_id)['allVideos']
+        video_data = next(video for video in videos if video.get('slug') == display_id)
+
+        series = video_data.get('showTitle')
+        title = episode = video_data.get('title') or series
+        if series and series != title:
+            title = '%s - %s' % (series, title)
+
+        formats = []
+        for f, format_id in (('cdnUriM3U8', 'mobi'), ('webVideoUrlSd', 'sd'), ('webVideoUrlHd', 'hd')):
+            f_url = video_data.get(f)
+            if not f_url:
+                continue
+            formats.append({
+                'format_id': format_id,
+                'url': f_url,
+            })
+
+        return {
+            'id': display_id,
+            'display_id': display_id,
+            'title': title,
+            'description': video_data.get('description'),
+            'thumbnail': video_data.get('thumbnail'),
+            'duration': parse_duration(video_data.get('runTime')),
+            'formats': formats,
+            'episode': episode,
+            'series': series,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 46e5e3a1d..13ca1d2cd 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -273,6 +273,7 @@ from .discoverygo import (
     DiscoveryGoPlaylistIE,
 )
 from .discoverynetworks import DiscoveryNetworksDeIE
+from .discoveryvr import DiscoveryVRIE
 from .disney import DisneyIE
 from .dispeak import DigitallySpeakingIE
 from .dropbox import DropboxIE

From e2435ba5f3c03f737bb3d2f209c3b47ef87fd41e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Apr 2017 03:52:44 +0700
Subject: [PATCH 034/611] [ChangeLog] Actualize

---
 ChangeLog | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 0199bdf1f..cb58b1687 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,19 @@
+version <unreleased>
+
+Core
++ [extractor/common] Add censorship check for TransTelekom ISP
+* [extractor/common] Move censorship checks to a separate method
+
+Extractors
++ [discoveryvr] Add support for discoveryvr.com (#12578)
++ [tv5mondeplus] Add support for tv5mondeplus.com (#11386)
++ [periscope] Add support for pscp.tv URLs (#12618, #12625)
+
+
 version 2017.04.02
 
 Core
-[YoutubeDL] Return early when extraction of url_transparent fails
+* [YoutubeDL] Return early when extraction of url_transparent fails
 
 Extractors
 * [rai] Fix and improve extraction (#11790)

From b022f4f600a2d73c65e569a9f0a19eed47272938 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Apr 2017 03:53:55 +0700
Subject: [PATCH 035/611] release 2017.04.03

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index c1b737619..8461a6173 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.02*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.02**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.03*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.03**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.04.02
+[debug] youtube-dl version 2017.04.03
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index cb58b1687..06a03bb9a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.04.03
 
 Core
 + [extractor/common] Add censorship check for TransTelekom ISP
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 5c1855111..a075c7012 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -213,6 +213,7 @@
  - **DiscoveryGo**
  - **DiscoveryGoPlaylist**
  - **DiscoveryNetworksDe**
+ - **DiscoveryVR**
  - **Disney**
  - **Dotsub**
  - **DouyuTV**: 斗鱼
@@ -815,6 +816,7 @@
  - **TV2Article**
  - **TV3**
  - **TV4**: tv4.se and tv4play.se
+ - **TV5MondePlus**: TV5MONDE+
  - **TVA**
  - **TVANouvelles**
  - **TVANouvellesArticle**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f612d03ca..1b260064b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.04.02'
+__version__ = '2017.04.03'

From 2ab0bfcd810b521bcf4de6dd555c0a47e8a69f70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 4 Apr 2017 03:05:18 +0700
Subject: [PATCH 036/611] [vshare] Add extractor (closes #12278)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/vshare.py     | 38 ++++++++++++++++++++++++++++++
 2 files changed, 39 insertions(+)
 create mode 100644 youtube_dl/extractor/vshare.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 13ca1d2cd..7398dceef 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1188,6 +1188,7 @@ from .vrv import (
     VRVIE,
     VRVSeriesIE,
 )
+from .vshare import VShareIE
 from .medialaan import MedialaanIE
 from .vube import VubeIE
 from .vuclip import VuClipIE
diff --git a/youtube_dl/extractor/vshare.py b/youtube_dl/extractor/vshare.py
new file mode 100644
index 000000000..5addbc280
--- /dev/null
+++ b/youtube_dl/extractor/vshare.py
@@ -0,0 +1,38 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class VShareIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?vshare\.io/[dv]/(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://vshare.io/d/0f64ce6',
+        'md5': '16d7b8fef58846db47419199ff1ab3e7',
+        'info_dict': {
+            'id': '0f64ce6',
+            'title': 'vl14062007715967',
+            'ext': 'mp4',
+        }
+    }, {
+        'url': 'https://vshare.io/v/0f64ce6/width-650/height-430/1',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            'https://vshare.io/d/%s' % video_id, video_id)
+
+        title = self._html_search_regex(
+            r'(?s)<div id="root-container">(.+?)<br/>', webpage, 'title')
+        video_url = self._search_regex(
+            r'<a[^>]+href=(["\'])(?P<url>(?:https?:)?//.+?)\1[^>]*>[Cc]lick\s+here',
+            webpage, 'video url', group='url')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': video_url,
+        }

From b68e00b08ae60e60d9f5c2654884f3c58595075f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 3 Apr 2017 21:35:54 +0100
Subject: [PATCH 037/611] [rtl2] add support for you.rtl2.de(closes #10257)

---
 youtube_dl/extractor/extractors.py |   6 +-
 youtube_dl/extractor/rtl2.py       | 110 ++++++++++++++++++++++++++++-
 2 files changed, 114 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 7398dceef..2904dd4d1 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -834,7 +834,11 @@ from .rozhlas import RozhlasIE
 from .rtbf import RTBFIE
 from .rte import RteIE, RteRadioIE
 from .rtlnl import RtlNlIE
-from .rtl2 import RTL2IE
+from .rtl2 import (
+    RTL2IE,
+    RTL2YouIE,
+    RTL2YouSeriesIE,
+)
 from .rtp import RTPIE
 from .rts import RTSIE
 from .rtve import RTVEALaCartaIE, RTVELiveIE, RTVEInfantilIE, RTVELiveIE, RTVETelevisionIE
diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
index 721ee733c..666e90e90 100644
--- a/youtube_dl/extractor/rtl2.py
+++ b/youtube_dl/extractor/rtl2.py
@@ -1,13 +1,26 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import base64
 import re
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..aes import aes_cbc_decrypt
+from ..compat import (
+    compat_ord,
+    compat_str,
+)
+from ..utils import (
+    bytes_to_intlist,
+    ExtractorError,
+    intlist_to_bytes,
+    int_or_none,
+    strip_or_none,
+)
 
 
 class RTL2IE(InfoExtractor):
+    IE_NAME = 'rtl2'
     _VALID_URL = r'http?://(?:www\.)?rtl2\.de/[^?#]*?/(?P<id>[^?#/]*?)(?:$|/(?:$|[?#]))'
     _TESTS = [{
         'url': 'http://www.rtl2.de/sendung/grip-das-motormagazin/folge/folge-203-0',
@@ -98,3 +111,98 @@ class RTL2IE(InfoExtractor):
             'duration': int_or_none(video_info.get('duration')),
             'formats': formats,
         }
+
+
+class RTL2YouBaseIE(InfoExtractor):
+    _BACKWERK_BASE_URL = 'https://p-you-backwerk.rtl2apps.de/'
+
+
+class RTL2YouIE(RTL2YouBaseIE):
+    IE_NAME = 'rtl2:you'
+    _VALID_URL = r'http?://you\.rtl2\.de/(?:video/\d+/|youplayer/index\.html\?.*?\bvid=)(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://you.rtl2.de/video/3002/15740/MJUNIK%20%E2%80%93%20Home%20of%20YOU/307-hirn-wo-bist-du',
+        'info_dict': {
+            'id': '15740',
+            'ext': 'mp4',
+            'title': 'MJUNIK – Home of YOU - #307 Hirn, wo bist du?!',
+            'description': 'md5:ddaa95c61b372b12b66e115b2772fe01',
+            'age_limit': 12,
+        },
+    }, {
+        'url': 'http://you.rtl2.de/youplayer/index.html?vid=15712',
+        'only_matching': True,
+    }]
+    _AES_KEY = b'\xe9W\xe4.<*\xb8\x1a\xd2\xb6\x92\xf3C\xd3\xefL\x1b\x03*\xbbbH\xc0\x03\xffo\xc2\xf2(\xaa\xaa!'
+    _GEO_COUNTRIES = ['DE']
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        stream_data = self._download_json(
+            self._BACKWERK_BASE_URL + 'stream/video/' + video_id, video_id)
+
+        data, iv = base64.b64decode(stream_data['streamUrl']).decode().split(':')
+        stream_url = intlist_to_bytes(aes_cbc_decrypt(
+            bytes_to_intlist(base64.b64decode(data)),
+            bytes_to_intlist(self._AES_KEY),
+            bytes_to_intlist(base64.b64decode(iv))
+        ))
+        if b'rtl2_you_video_not_found' in stream_url:
+            raise ExtractorError('video not found', expected=True)
+
+        formats = self._extract_m3u8_formats(
+            stream_url[:-compat_ord(stream_url[-1])].decode(),
+            video_id, 'mp4', 'm3u8_native')
+        self._sort_formats(formats)
+
+        video_data = self._download_json(
+            self._BACKWERK_BASE_URL + 'video/' + video_id, video_id)
+
+        series = video_data.get('formatTitle')
+        title = episode = video_data.get('title') or series
+        if series and series != title:
+            title = '%s - %s' % (series, title)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'description': strip_or_none(video_data.get('description')),
+            'thumbnail': video_data.get('image'),
+            'duration': int_or_none(stream_data.get('duration') or video_data.get('duration'), 1000),
+            'series': series,
+            'episode': episode,
+            'age_limit': int_or_none(video_data.get('minimumAge')),
+        }
+
+
+class RTL2YouSeriesIE(RTL2YouBaseIE):
+    IE_NAME = 'rtl2:you:series'
+    _VALID_URL = r'http?://you\.rtl2\.de/videos/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://you.rtl2.de/videos/115/dragon-ball',
+        'info_dict': {
+            'id': '115',
+        },
+        'playlist_mincount': 5,
+    }
+
+    def _real_extract(self, url):
+        series_id = self._match_id(url)
+        stream_data = self._download_json(
+            self._BACKWERK_BASE_URL + 'videos',
+            series_id, query={
+                'formatId': series_id,
+                'limit': 1000000000,
+            })
+
+        entries = []
+        for video in stream_data.get('videos', []):
+            video_id = compat_str(video['videoId'])
+            if not video_id:
+                continue
+            entries.append(self.url_result(
+                'http://you.rtl2.de/video/%s/%s' % (series_id, video_id),
+                'RTL2You', video_id))
+        return self.playlist_result(entries, series_id)

From fac39cccd47555af5e9ba2c9f5f36f8ff3b628ad Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 3 Apr 2017 23:39:56 +0100
Subject: [PATCH 038/611] [odnoklassniki] fix format sorting for 1080p quality

---
 youtube_dl/extractor/odnoklassniki.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 986708e75..0ee56a45b 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -176,7 +176,7 @@ class OdnoklassnikiIE(InfoExtractor):
             })
             return info
 
-        quality = qualities(('mobile', 'lowest', 'low', 'sd', 'hd'))
+        quality = qualities(('mobile', 'lowest', 'low', 'sd', 'hd', 'full'))
 
         formats = [{
             'url': f['url'],

From 7f04386b89ced3235ea53c6c041dbfff3cbe667b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Apr 2017 02:28:09 +0700
Subject: [PATCH 039/611] [aenetworks] Add more video URL regexes (closes
 #12657)

---
 youtube_dl/extractor/aenetworks.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index dd96a47ce..9a5074037 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -107,7 +107,10 @@ class AENetworksIE(AENetworksBaseIE):
         }
         video_id = self._html_search_meta('aetn:VideoID', webpage)
         media_url = self._search_regex(
-            r"media_url\s*=\s*'([^']+)'", webpage, 'video url')
+            [r"media_url\s*=\s*'(?P<url>[^']+)'",
+             r'data-media-url=(?P<url>(?:https?:)?//[^\s>]+)',
+             r'data-media-url=(["\'])(?P<url>(?:(?!\1).)+?)\1'],
+            webpage, 'video url', group='url')
         theplatform_metadata = self._download_theplatform_metadata(self._search_regex(
             r'https?://link.theplatform.com/s/([^?]+)', media_url, 'theplatform_path'), video_id)
         info = self._parse_theplatform_metadata(theplatform_metadata)

From 0378b8b91713a3ff7960ad92391e5994648f18fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Apr 2017 23:38:16 +0700
Subject: [PATCH 040/611] [xfileshare] Add support for vidbom.com (closes
 #12661)

---
 youtube_dl/extractor/xfileshare.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 6856fb3bf..a2fcb14c9 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -28,6 +28,7 @@ class XFileShareIE(InfoExtractor):
         ('streamin.to', 'Streamin.To'),
         ('xvidstage.com', 'XVIDSTAGE'),
         ('vidabc.com', 'Vid ABC'),
+        ('vidbom.com', 'VidBom'),
     )
 
     IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1])

From 3461f5db0615fa46c2caa393f648f1ba538d1b66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Apr 2017 23:39:09 +0700
Subject: [PATCH 041/611] [xfileshare] Add support for vidlo.us (closes #12660)

---
 youtube_dl/extractor/xfileshare.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index a2fcb14c9..1b5cd122a 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -29,6 +29,7 @@ class XFileShareIE(InfoExtractor):
         ('xvidstage.com', 'XVIDSTAGE'),
         ('vidabc.com', 'Vid ABC'),
         ('vidbom.com', 'VidBom'),
+        ('vidlo.us', 'vidlo'),
     )
 
     IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1])

From a84da06f49c04203b9dadd0b4d4091228ca5afcd Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sat, 8 Apr 2017 00:21:11 -0400
Subject: [PATCH 042/611] [test_download] Improve diagnostic on wrong 'id'

---
 test/test_download.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/test_download.py b/test/test_download.py
index 01a8bcb89..bd9dd86e8 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -11,6 +11,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from test.helper import (
     assertGreaterEqual,
     expect_warnings,
+    expect_value,
     get_params,
     gettestcases,
     expect_info_dict,
@@ -200,6 +201,7 @@ def generator(test_case, tname):
                     test_case['playlist_duration_sum'], got_duration)
 
             for tc in test_cases:
+                expect_value(self, res_dict['id'], tc['info_dict']['id'], 'id')
                 tc_filename = get_tc_filename(tc)
                 if not test_case.get('params', {}).get('skip_download', False):
                     self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename)

From 3bef10a50ca61e4d6e06b0990b8ccf5f739c0601 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sat, 8 Apr 2017 00:27:52 -0400
Subject: [PATCH 043/611] [test_download] typo in comment

---
 test/test_download.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_download.py b/test/test_download.py
index bd9dd86e8..61015562d 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -152,7 +152,7 @@ def generator(test_case, tname):
             try_num = 1
             while True:
                 try:
-                    # We're not using .download here sine that is just a shim
+                    # We're not using .download here since that is just a shim
                     # for outside error handling, and returns the exit code
                     # instead of the result dict.
                     res_dict = ydl.extract_info(

From 80b2fdf9ac99ebcbd6eec87670aa6d646da7dac4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 14:10:12 +0700
Subject: [PATCH 044/611] [test_download] Match info dicts against tests before
 matching info file

---
 test/test_download.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/test_download.py b/test/test_download.py
index 61015562d..5c5824d65 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -200,8 +200,9 @@ def generator(test_case, tname):
                 self.assertEqual(
                     test_case['playlist_duration_sum'], got_duration)
 
-            for tc in test_cases:
-                expect_value(self, res_dict['id'], tc['info_dict']['id'], 'id')
+            for tc_num, tc in enumerate(test_cases):
+                tc_res_dict = res_dict['entries'][tc_num] if is_playlist else res_dict
+                expect_info_dict(self, tc_res_dict, tc.get('info_dict', {}))
                 tc_filename = get_tc_filename(tc)
                 if not test_case.get('params', {}).get('skip_download', False):
                     self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename)

From 1b3feca0a700c4d4d7c4d8b01fc5f033cf48f421 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 14:11:03 +0700
Subject: [PATCH 045/611] [raiplay] Extract subtitles

---
 youtube_dl/extractor/rai.py | 34 +++++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 077546a73..81eb9db85 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -97,6 +97,25 @@ class RaiBaseIE(InfoExtractor):
             'formats': formats,
         }.items() if v is not None)
 
+    @staticmethod
+    def _extract_subtitles(url, subtitle_url):
+        subtitles = {}
+        if subtitle_url and isinstance(subtitle_url, compat_str):
+            subtitle_url = urljoin(url, subtitle_url)
+            STL_EXT = '.stl'
+            SRT_EXT = '.srt'
+            subtitles['it'] = [{
+                'ext': 'stl',
+                'url': subtitle_url,
+            }]
+            if subtitle_url.endswith(STL_EXT):
+                srt_url = subtitle_url[:-len(STL_EXT)] + SRT_EXT
+                subtitles['it'].append({
+                    'ext': 'srt',
+                    'url': srt_url,
+                })
+        return subtitles
+
 
 class RaiPlayIE(RaiBaseIE):
     _VALID_URL = r'(?P<url>https?://(?:www\.)?raiplay\.it/.+?-(?P<id>%s)\.html)' % RaiBaseIE._UUID_RE
@@ -168,6 +187,8 @@ class RaiPlayIE(RaiBaseIE):
         timestamp = unified_timestamp(try_get(
             media, lambda x: x['availabilities'][0]['start'], compat_str))
 
+        subtitles = self._extract_subtitles(url, video.get('subtitles'))
+
         info = {
             'id': video_id,
             'title': title,
@@ -183,6 +204,7 @@ class RaiPlayIE(RaiBaseIE):
             'season_number': int_or_none(try_get(
                 media, lambda x: x['isPartOf']['numeroStagioni'])),
             'season': media.get('stagione') or None,
+            'subtitles': subtitles,
         }
 
         info.update(relinker_info)
@@ -307,17 +329,7 @@ class RaiIE(RaiBaseIE):
                     'url': compat_urlparse.urljoin(url, thumbnail_url),
                 })
 
-        subtitles = {}
-        captions = media.get('subtitlesUrl')
-        if captions:
-            STL_EXT = '.stl'
-            SRT_EXT = '.srt'
-            if captions.endswith(STL_EXT):
-                captions = captions[:-len(STL_EXT)] + SRT_EXT
-            subtitles['it'] = [{
-                'ext': 'srt',
-                'url': captions,
-            }]
+        subtitles = self._extract_subtitles(url, media.get('subtitlesUrl'))
 
         info = {
             'id': content_id,

From c4d6fc6d659ba74aab3f59aca02ff0e2e0763eb4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 14:11:20 +0700
Subject: [PATCH 046/611] [test_subtitles] Fix raiplay test

---
 test/test_subtitles.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 27e763edd..1b8de822a 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -21,7 +21,7 @@ from youtube_dl.extractor import (
     NPOIE,
     ComedyCentralIE,
     NRKTVIE,
-    RaiTVIE,
+    RaiPlayIE,
     VikiIE,
     ThePlatformIE,
     ThePlatformFeedIE,
@@ -258,9 +258,9 @@ class TestNRKSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['no']), '544fa917d3197fcbee64634559221cc2')
 
 
-class TestRaiSubtitles(BaseTestSubtitles):
-    url = 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-cb27157f-9dd0-4aee-b788-b1f67643a391.html'
-    IE = RaiTVIE
+class TestRaiPlaySubtitles(BaseTestSubtitles):
+    url = 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html'
+    IE = RaiPlayIE
 
     def test_allsubtitles(self):
         self.DL.params['writesubtitles'] = True

From 60e5016199320333ad3f3584b5b0c0147cfaeba3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 14:19:01 +0700
Subject: [PATCH 047/611] [test_download] Remove unused import

---
 test/test_download.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/test_download.py b/test/test_download.py
index 5c5824d65..fd7d2e8be 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -11,7 +11,6 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from test.helper import (
     assertGreaterEqual,
     expect_warnings,
-    expect_value,
     get_params,
     gettestcases,
     expect_info_dict,

From 4c0397329688144b8c0ec1d34227772fdfd03527 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 8 Apr 2017 15:39:58 +0800
Subject: [PATCH 048/611] [airmozilla] Fix extraction (closes #12670)

---
 ChangeLog                          |  6 ++++++
 youtube_dl/extractor/airmozilla.py | 28 ++++++++++------------------
 2 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 06a03bb9a..47804428f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [airmozilla] Fix extraction (#12670)
+
+
 version 2017.04.03
 
 Core
diff --git a/youtube_dl/extractor/airmozilla.py b/youtube_dl/extractor/airmozilla.py
index 0e0691879..9e38136b4 100644
--- a/youtube_dl/extractor/airmozilla.py
+++ b/youtube_dl/extractor/airmozilla.py
@@ -15,12 +15,12 @@ class AirMozillaIE(InfoExtractor):
     _VALID_URL = r'https?://air\.mozilla\.org/(?P<id>[0-9a-z-]+)/?'
     _TEST = {
         'url': 'https://air.mozilla.org/privacy-lab-a-meetup-for-privacy-minded-people-in-san-francisco/',
-        'md5': '2e3e7486ba5d180e829d453875b9b8bf',
+        'md5': '8d02f53ee39cf006009180e21df1f3ba',
         'info_dict': {
             'id': '6x4q2w',
             'ext': 'mp4',
             'title': 'Privacy Lab - a meetup for privacy minded people in San Francisco',
-            'thumbnail': r're:https?://vid\.ly/(?P<id>[0-9a-z-]+)/poster',
+            'thumbnail': r're:https?://.*/poster\.jpg',
             'description': 'Brings together privacy professionals and others interested in privacy at for-profits, non-profits, and NGOs in an effort to contribute to the state of the ecosystem...',
             'timestamp': 1422487800,
             'upload_date': '20150128',
@@ -34,21 +34,13 @@ class AirMozillaIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-        video_id = self._html_search_regex(r'//vid.ly/(.*?)/embed', webpage, 'id')
+        video_id = self._html_search_regex(r'//vid\.ly/(.*?)/embed', webpage, 'id')
 
         embed_script = self._download_webpage('https://vid.ly/{0}/embed'.format(video_id), video_id)
-        jwconfig = self._search_regex(r'\svar jwconfig = (\{.*?\});\s', embed_script, 'metadata')
-        metadata = self._parse_json(jwconfig, video_id)
-
-        formats = [{
-            'url': source['file'],
-            'ext': source['type'],
-            'format_id': self._search_regex(r'&format=(.*)$', source['file'], 'video format'),
-            'format': source['label'],
-            'height': int(source['label'].rstrip('p')),
-        } for source in metadata['playlist'][0]['sources']]
-        self._sort_formats(formats)
+        jwconfig = self._parse_json(self._search_regex(
+            r'initCallback\((.*)\);', embed_script, 'metadata'), video_id)['config']
 
+        info_dict = self._parse_jwplayer_data(jwconfig, video_id)
         view_count = int_or_none(self._html_search_regex(
             r'Views since archived: ([0-9]+)',
             webpage, 'view count', fatal=False))
@@ -58,17 +50,17 @@ class AirMozillaIE(InfoExtractor):
             r'Duration:\s*(\d+\s*hours?\s*\d+\s*minutes?)',
             webpage, 'duration', fatal=False))
 
-        return {
+        info_dict.update({
             'id': video_id,
             'title': self._og_search_title(webpage),
-            'formats': formats,
             'url': self._og_search_url(webpage),
             'display_id': display_id,
-            'thumbnail': metadata['playlist'][0].get('image'),
             'description': self._og_search_description(webpage),
             'timestamp': timestamp,
             'location': self._html_search_regex(r'Location: (.*)', webpage, 'location', default=None),
             'duration': duration,
             'view_count': view_count,
             'categories': re.findall(r'<a href=".*?" class="channel">(.*?)</a>', webpage),
-        }
+        })
+
+        return info_dict

From f8f2da25ab83779f5a9521f327987d5e7bace85f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 16:01:56 +0700
Subject: [PATCH 049/611] [wshh] Extract html5 entries and delegate to generic
 extractor (closes #12676)

---
 youtube_dl/extractor/worldstarhiphop.py | 46 ++++++-------------------
 1 file changed, 11 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/worldstarhiphop.py b/youtube_dl/extractor/worldstarhiphop.py
index 09415b589..82587b4ce 100644
--- a/youtube_dl/extractor/worldstarhiphop.py
+++ b/youtube_dl/extractor/worldstarhiphop.py
@@ -1,12 +1,10 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
 class WorldStarHipHopIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www|m)\.worldstar(?:candy|hiphop)\.com/(?:videos|android)/video\.php\?v=(?P<id>.*)'
+    _VALID_URL = r'https?://(?:www|m)\.worldstar(?:candy|hiphop)\.com/(?:videos|android)/video\.php\?.*?\bv=(?P<id>[^&]+)'
     _TESTS = [{
         'url': 'http://www.worldstarhiphop.com/videos/video.php?v=wshh6a7q1ny0G34ZwuIO',
         'md5': '9d04de741161603bf7071bbf4e883186',
@@ -17,48 +15,26 @@ class WorldStarHipHopIE(InfoExtractor):
         }
     }, {
         'url': 'http://m.worldstarhiphop.com/android/video.php?v=wshh6a7q1ny0G34ZwuIO',
-        'md5': 'dc1c76c83ecc4190bb1eb143899b87d3',
-        'info_dict': {
-            'id': 'wshh6a7q1ny0G34ZwuIO',
-            'ext': 'mp4',
-            'title': 'KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick!'
-        }
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        m_vevo_id = re.search(r'videoId=(.*?)&amp?', webpage)
-        if m_vevo_id is not None:
-            return self.url_result('vevo:%s' % m_vevo_id.group(1), ie='Vevo')
+        entries = self._parse_html5_media_entries(url, webpage, video_id)
 
-        video_url = self._search_regex(
-            [r'so\.addVariable\("file","(.*?)"\)',
-             r'<div class="artlist">\s*<a[^>]+href="([^"]+)">'],
-            webpage, 'video URL')
+        if not entries:
+            return self.url_result(url, 'Generic')
 
-        if 'youtube' in video_url:
-            return self.url_result(video_url, ie='Youtube')
-
-        video_title = self._html_search_regex(
+        title = self._html_search_regex(
             [r'(?s)<div class="content-heading">\s*<h1>(.*?)</h1>',
              r'<span[^>]+class="tc-sp-pinned-title">(.*)</span>'],
             webpage, 'title')
 
-        # Getting thumbnail and if not thumbnail sets correct title for WSHH candy video.
-        thumbnail = self._html_search_regex(
-            r'rel="image_src" href="(.*)" />', webpage, 'thumbnail',
-            default=None)
-        if not thumbnail:
-            _title = r'candytitles.*>(.*)</span>'
-            mobj = re.search(_title, webpage)
-            if mobj is not None:
-                video_title = mobj.group(1)
-
-        return {
+        info = entries[0]
+        info.update({
             'id': video_id,
-            'url': video_url,
-            'title': video_title,
-            'thumbnail': thumbnail,
-        }
+            'title': title,
+        })
+        return info

From a01825a5416174de56625a6ef202311e2d6c8cb1 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sat, 8 Apr 2017 05:40:57 -0400
Subject: [PATCH 050/611] [kaltura] Add support for iframe embeds

---
 youtube_dl/extractor/generic.py | 15 +++++++++++++++
 youtube_dl/extractor/kaltura.py | 13 ++++++++++++-
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 73911940c..658533cf6 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1080,6 +1080,21 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Kaltura'],
         },
+        {
+            # Kaltura iframe embed
+            'url': 'http://www.gsd.harvard.edu/event/i-m-pei-a-centennial-celebration/',
+            'md5': 'ae5ace8eb09dc1a35d03b579a9c2cc44',
+            'info_dict': {
+                'id': '0_f2cfbpwy',
+                'ext': 'mp4',
+                'title': 'I. M. Pei: A Centennial Celebration',
+                'description': 'md5:1db8f40c69edc46ca180ba30c567f37c',
+                'upload_date': '20170403',
+                'uploader_id': 'batchUser',
+                'timestamp': 1491232186,
+            },
+            'add_ie': ['Kaltura'],
+        },
         # Eagle.Platform embed (generic URL)
         {
             'url': 'http://lenta.ru/news/2015/03/06/navalny/',
diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 54374ea76..c6efdea93 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -91,6 +91,7 @@ class KalturaIE(InfoExtractor):
                     }],
                 },
             },
+            'skip': 'Gone. Maybe https://www.safaribooksonline.com/library/tutorials/introduction-to-python-anon/3469/',
             'params': {
                 'skip_download': True,
             },
@@ -108,6 +109,7 @@ class KalturaIE(InfoExtractor):
     @staticmethod
     def _extract_url(webpage):
         mobj = (
+            # Embed codes: https://knowledge.kaltura.com/embedding-kaltura-media-players-your-site
             re.search(
                 r"""(?xs)
                     kWidget\.(?:thumb)?[Ee]mbed\(
@@ -127,7 +129,16 @@ class KalturaIE(InfoExtractor):
                         (?P<q2>["\'])entry_?[Ii]d(?P=q2)
                     )\s*:\s*
                     (?P<q3>["\'])(?P<id>(?:(?!(?P=q3)).)+)(?P=q3)
-                ''', webpage))
+                ''', webpage) or
+            re.search(
+                r'''(?xs)
+                    <iframe[^>]+src=(?P<q1>["\'])
+                      (?:https?:)?//(?:www\.)?kaltura\.com/p/(?P<partner_id>\d+)/
+                      (?:(?!(?P=q1)).)*
+                      [\?&]entry_id=(?P<id>(?:(?!(?P=q1))[^&])+)
+                    (?P=q1)
+                ''', webpage)
+        )
         if mobj:
             embed_info = mobj.groupdict()
             url = 'kaltura:%(partner_id)s:%(id)s' % embed_info

From 78280352ca8e4af4e82fc92228200aef3f06ce53 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 16:48:27 +0700
Subject: [PATCH 051/611] [kaltura] Cleanup regexes from redundant escaping

---
 youtube_dl/extractor/kaltura.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index c6efdea93..41c1f3d96 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -108,34 +108,34 @@ class KalturaIE(InfoExtractor):
 
     @staticmethod
     def _extract_url(webpage):
+        # Embed codes: https://knowledge.kaltura.com/embedding-kaltura-media-players-your-site
         mobj = (
-            # Embed codes: https://knowledge.kaltura.com/embedding-kaltura-media-players-your-site
             re.search(
                 r"""(?xs)
                     kWidget\.(?:thumb)?[Ee]mbed\(
                     \{.*?
-                        (?P<q1>['\"])wid(?P=q1)\s*:\s*
-                        (?P<q2>['\"])_?(?P<partner_id>(?:(?!(?P=q2)).)+)(?P=q2),.*?
-                        (?P<q3>['\"])entry_?[Ii]d(?P=q3)\s*:\s*
-                        (?P<q4>['\"])(?P<id>(?:(?!(?P=q4)).)+)(?P=q4)(?:,|\s*\})
+                        (?P<q1>['"])wid(?P=q1)\s*:\s*
+                        (?P<q2>['"])_?(?P<partner_id>(?:(?!(?P=q2)).)+)(?P=q2),.*?
+                        (?P<q3>['"])entry_?[Ii]d(?P=q3)\s*:\s*
+                        (?P<q4>['"])(?P<id>(?:(?!(?P=q4)).)+)(?P=q4)(?:,|\s*\})
                 """, webpage) or
             re.search(
                 r'''(?xs)
-                    (?P<q1>["\'])
+                    (?P<q1>["'])
                         (?:https?:)?//cdnapi(?:sec)?\.kaltura\.com(?::\d+)?/(?:(?!(?P=q1)).)*\b(?:p|partner_id)/(?P<partner_id>\d+)(?:(?!(?P=q1)).)*
                     (?P=q1).*?
                     (?:
                         entry_?[Ii]d|
-                        (?P<q2>["\'])entry_?[Ii]d(?P=q2)
+                        (?P<q2>["'])entry_?[Ii]d(?P=q2)
                     )\s*:\s*
-                    (?P<q3>["\'])(?P<id>(?:(?!(?P=q3)).)+)(?P=q3)
+                    (?P<q3>["'])(?P<id>(?:(?!(?P=q3)).)+)(?P=q3)
                 ''', webpage) or
             re.search(
                 r'''(?xs)
-                    <iframe[^>]+src=(?P<q1>["\'])
-                      (?:https?:)?//(?:www\.)?kaltura\.com/p/(?P<partner_id>\d+)/
+                    <iframe[^>]+src=(?P<q1>["'])
+                      (?:https?:)?//(?:www\.)?kaltura\.com/(?:(?!(?P=q1)).)*\b(?:p|partner_id)/(?P<partner_id>\d+)
                       (?:(?!(?P=q1)).)*
-                      [\?&]entry_id=(?P<id>(?:(?!(?P=q1))[^&])+)
+                      [?&]entry_id=(?P<id>(?:(?!(?P=q1))[^&])+)
                     (?P=q1)
                 ''', webpage)
         )

From e18f1da97a8840522b0cefac3af0995751733d67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 19:41:14 +0700
Subject: [PATCH 052/611] [ceskateleveize] Improve extraction and remove URL
 replacement hacks

---
 youtube_dl/extractor/ceskatelevize.py | 61 +++++++++++----------------
 1 file changed, 24 insertions(+), 37 deletions(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index dd2529a6d..0daee313f 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -18,7 +18,7 @@ from ..utils import (
 
 
 class CeskaTelevizeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/(porady|ivysilani)/(?:[^/]+/)*(?P<id>[^/#?]+)/*(?:[#?].*)?$'
+    _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/ivysilani/(?:[^/?#&]+/)*(?P<id>[^/#?]+)'
     _TESTS = [{
         'url': 'http://www.ceskatelevize.cz/ivysilani/ivysilani/10441294653-hyde-park-civilizace/214411058091220',
         'info_dict': {
@@ -62,40 +62,12 @@ class CeskaTelevizeIE(InfoExtractor):
         },
         'skip': 'Georestricted to Czech Republic',
     }, {
-        # video with 18+ caution trailer
-        'url': 'http://www.ceskatelevize.cz/porady/10520528904-queer/215562210900007-bogotart/',
-        'info_dict': {
-            'id': '215562210900007-bogotart',
-            'title': 'Queer: Bogotart',
-            'description': 'Alternativní průvodce současným queer světem',
-        },
-        'playlist': [{
-            'info_dict': {
-                'id': '61924494876844842',
-                'ext': 'mp4',
-                'title': 'Queer: Bogotart (Varování 18+)',
-                'duration': 10.2,
-            },
-        }, {
-            'info_dict': {
-                'id': '61924494877068022',
-                'ext': 'mp4',
-                'title': 'Queer: Bogotart (Queer)',
-                'thumbnail': r're:^https?://.*\.jpg',
-                'duration': 1558.3,
-            },
-        }],
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
+        'url': 'http://www.ceskatelevize.cz/ivysilani/embed/iFramePlayer.php?hash=d6a3e1370d2e4fa76296b90bad4dfc19673b641e&IDEC=217 562 22150/0004&channelID=1&width=100%25',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        url = url.replace('/porady/', '/ivysilani/').replace('/video/', '')
-
-        mobj = re.match(self._VALID_URL, url)
-        playlist_id = mobj.group('id')
+        playlist_id = self._match_id(url)
 
         webpage = self._download_webpage(url, playlist_id)
 
@@ -103,13 +75,28 @@ class CeskaTelevizeIE(InfoExtractor):
         if '%s</p>' % NOT_AVAILABLE_STRING in webpage:
             raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
 
-        typ = self._html_search_regex(
-            r'getPlaylistUrl\(\[\{"type":"(.+?)","id":".+?"\}\],', webpage, 'type')
-        episode_id = self._html_search_regex(
-            r'getPlaylistUrl\(\[\{"type":".+?","id":"(.+?)"\}\],', webpage, 'episode_id')
+        type_ = None
+        episode_id = None
+
+        playlist = self._parse_json(
+            self._search_regex(
+                r'getPlaylistUrl\(\[({.+?})\]', webpage, 'playlist',
+                default='{}'), playlist_id)
+        if playlist:
+            type_ = playlist.get('type')
+            episode_id = playlist.get('id')
+
+        if not type_:
+            type_ = self._html_search_regex(
+                r'getPlaylistUrl\(\[\{"type":"(.+?)","id":".+?"\}\],',
+                webpage, 'type')
+        if not episode_id:
+            episode_id = self._html_search_regex(
+                r'getPlaylistUrl\(\[\{"type":".+?","id":"(.+?)"\}\],',
+                webpage, 'episode_id')
 
         data = {
-            'playlist[0][type]': typ,
+            'playlist[0][type]': type_,
             'playlist[0][id]': episode_id,
             'requestUrl': compat_urllib_parse_urlparse(url).path,
             'requestSource': 'iVysilani',

From 28b674ca238fbae29f2ee7bc85202fa2f115a876 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 19:42:09 +0700
Subject: [PATCH 053/611] [ceskateleveize:porady] Add extractor (closes #7411,
 closes #12645)

---
 youtube_dl/extractor/ceskatelevize.py | 45 +++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py    |  5 ++-
 2 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 0daee313f..e250de18c 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -12,6 +12,7 @@ from ..utils import (
     ExtractorError,
     float_or_none,
     sanitized_Request,
+    unescapeHTML,
     urlencode_postdata,
     USER_AGENTS,
 )
@@ -232,3 +233,47 @@ class CeskaTelevizeIE(InfoExtractor):
                     yield line
 
         return '\r\n'.join(_fix_subtitle(subtitles))
+
+
+class CeskaTelevizePoradyIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/porady/(?:[^/?#&]+/)*(?P<id>[^/#?]+)'
+    _TESTS = [{
+        # video with 18+ caution trailer
+        'url': 'http://www.ceskatelevize.cz/porady/10520528904-queer/215562210900007-bogotart/',
+        'info_dict': {
+            'id': '215562210900007-bogotart',
+            'title': 'Queer: Bogotart',
+            'description': 'Alternativní průvodce současným queer světem',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': '61924494876844842',
+                'ext': 'mp4',
+                'title': 'Queer: Bogotart (Varování 18+)',
+                'duration': 10.2,
+            },
+        }, {
+            'info_dict': {
+                'id': '61924494877068022',
+                'ext': 'mp4',
+                'title': 'Queer: Bogotart (Queer)',
+                'thumbnail': r're:^https?://.*\.jpg',
+                'duration': 1558.3,
+            },
+        }],
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        data_url = unescapeHTML(self._search_regex(
+            r'<span[^>]*\bdata-url=(["\'])(?P<url>(?:(?!\1).)+)\1',
+            webpage, 'iframe player url', group='url'))
+
+        return self.url_result(data_url, ie=CeskaTelevizeIE.ie_key())
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 2904dd4d1..72728d919 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -165,7 +165,10 @@ from .ccc import CCCIE
 from .ccma import CCMAIE
 from .cctv import CCTVIE
 from .cda import CDAIE
-from .ceskatelevize import CeskaTelevizeIE
+from .ceskatelevize import (
+    CeskaTelevizeIE,
+    CeskaTelevizePoradyIE,
+)
 from .channel9 import Channel9IE
 from .charlierose import CharlieRoseIE
 from .chaturbate import ChaturbateIE

From 5f3e0b69efa8ae80f536fb54ace4bb4d9c667d2e Mon Sep 17 00:00:00 2001
From: Entropy <0fiscalentropy@users.noreply.github.com>
Date: Fri, 7 Apr 2017 21:52:48 +0200
Subject: [PATCH 054/611] [TheSun] Add new extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/thesun.py     | 27 +++++++++++++++++++++++++++
 2 files changed, 28 insertions(+)
 create mode 100644 youtube_dl/extractor/thesun.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 72728d919..01865479c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -983,6 +983,7 @@ from .theplatform import (
 from .thescene import TheSceneIE
 from .thesixtyone import TheSixtyOneIE
 from .thestar import TheStarIE
+from .thesun import TheSunIE
 from .theweatherchannel import TheWeatherChannelIE
 from .thisamericanlife import ThisAmericanLifeIE
 from .thisav import ThisAVIE
diff --git a/youtube_dl/extractor/thesun.py b/youtube_dl/extractor/thesun.py
new file mode 100644
index 000000000..7f96bf8c9
--- /dev/null
+++ b/youtube_dl/extractor/thesun.py
@@ -0,0 +1,27 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from .ooyala import OoyalaIE
+
+
+class TheSunIE(InfoExtractor):
+    _VALID_URL = r'https://(?:www\.)?thesun\.co\.uk/\w+/(?P<id>\d+)/[\w-]'
+    _TEST = {
+        'url': 'https://www.thesun.co.uk/tvandshowbiz/2261604/orlando-bloom-and-katy-perry-post-adorable-instagram-video-together-celebrating-thanksgiving-after-split-rumours/',
+        'md5': '5667123b24f25f43f4c4f381ef34c5c2',
+        'info_dict': {
+            'id': 'h4OXN0NzE6rv6ObkEifKcNA-gYUw4xFf',
+            'ext': 'mp4',
+            'title': 'Katy Perry and Orlando Bloom shut down split rumours with cute Thanksgiving video',
+            'description': 'Still going strong',
+            'duration': 31.28,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        ooyala_id = self._search_regex(r'id\s*=\s*"thesun-ooyala-player-([^"]+)"', webpage, 'ooyala id')
+
+        return OoyalaIE._build_url_result(ooyala_id)

From 90e3f18fc1cff2c12361cbbd85303774bf7d59f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 20:08:31 +0700
Subject: [PATCH 055/611] [thesun] Extract playlists (closes #11298, closes
 #12674)

---
 youtube_dl/extractor/thesun.py | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/thesun.py b/youtube_dl/extractor/thesun.py
index 7f96bf8c9..22d003776 100644
--- a/youtube_dl/extractor/thesun.py
+++ b/youtube_dl/extractor/thesun.py
@@ -1,27 +1,32 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from .ooyala import OoyalaIE
 
 
 class TheSunIE(InfoExtractor):
-    _VALID_URL = r'https://(?:www\.)?thesun\.co\.uk/\w+/(?P<id>\d+)/[\w-]'
+    _VALID_URL = r'https://(?:www\.)?thesun\.co\.uk/[^/]+/(?P<id>\d+)'
     _TEST = {
         'url': 'https://www.thesun.co.uk/tvandshowbiz/2261604/orlando-bloom-and-katy-perry-post-adorable-instagram-video-together-celebrating-thanksgiving-after-split-rumours/',
-        'md5': '5667123b24f25f43f4c4f381ef34c5c2',
         'info_dict': {
-            'id': 'h4OXN0NzE6rv6ObkEifKcNA-gYUw4xFf',
-            'ext': 'mp4',
-            'title': 'Katy Perry and Orlando Bloom shut down split rumours with cute Thanksgiving video',
-            'description': 'Still going strong',
-            'duration': 31.28,
-        }
+            'id': '2261604',
+            'title': 'md5:cba22f48bad9218b64d5bbe0e16afddf',
+        },
+        'playlist_count': 2,
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        article_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
-        ooyala_id = self._search_regex(r'id\s*=\s*"thesun-ooyala-player-([^"]+)"', webpage, 'ooyala id')
+        webpage = self._download_webpage(url, article_id)
 
-        return OoyalaIE._build_url_result(ooyala_id)
+        entries = []
+        for ooyala_id in re.findall(
+                r'<[^>]+\b(?:id\s*=\s*"thesun-ooyala-player-|data-content-id\s*=\s*")([^"]+)',
+                webpage):
+            entries.append(OoyalaIE._build_url_result(ooyala_id))
+
+        return self.playlist_result(
+            entries, article_id, self._og_search_title(webpage, fatal=False))

From c93c0fc2fd4162c06fac7b263484f50f14ac60b8 Mon Sep 17 00:00:00 2001
From: Kfir Breger <kfir@cubicalwave.com>
Date: Mon, 27 Mar 2017 20:56:37 +0200
Subject: [PATCH 056/611] [mixcloud:playlist] Fix title extraction

---
 youtube_dl/extractor/mixcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index a24b3165a..ba0bb190f 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -250,7 +250,7 @@ class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
 
         description = self._get_user_description(profile)
         playlist_title = self._html_search_regex(
-            r'<span[^>]+class="[^"]*list-playlist-title[^"]*"[^>]*>(.*?)</span>',
+            r'<a class="parent active" href="[^"]*"><b>\d+</b><span title="[^"]*">([^</]*?)</span></a>',
             profile, 'playlist title')
 
         entries = OnDemandPagedList(

From a66e25859a0f163446c009f9a71abab4083745f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 20:51:38 +0700
Subject: [PATCH 057/611] [mixcloud:playlist] Relax title extraction and fix
 description extraction (closes #12582)

---
 youtube_dl/extractor/mixcloud.py | 30 +++++++++++++-----------------
 1 file changed, 13 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index ba0bb190f..bc38a8663 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -138,7 +138,7 @@ class MixcloudPlaylistBaseIE(InfoExtractor):
 
     def _get_user_description(self, page_content):
         return self._html_search_regex(
-            r'<div[^>]+class="description-text"[^>]*>(.+?)</div>',
+            r'<div[^>]+class="profile-bio"[^>]*>(.+?)</div>',
             page_content, 'user description', fatal=False)
 
 
@@ -151,7 +151,7 @@ class MixcloudUserIE(MixcloudPlaylistBaseIE):
         'info_dict': {
             'id': 'dholbach_uploads',
             'title': 'Daniel Holbach (uploads)',
-            'description': 'md5:327af72d1efeb404a8216c27240d1370',
+            'description': 'md5:def36060ac8747b3aabca54924897e47',
         },
         'playlist_mincount': 11,
     }, {
@@ -159,7 +159,7 @@ class MixcloudUserIE(MixcloudPlaylistBaseIE):
         'info_dict': {
             'id': 'dholbach_uploads',
             'title': 'Daniel Holbach (uploads)',
-            'description': 'md5:327af72d1efeb404a8216c27240d1370',
+            'description': 'md5:def36060ac8747b3aabca54924897e47',
         },
         'playlist_mincount': 11,
     }, {
@@ -167,7 +167,7 @@ class MixcloudUserIE(MixcloudPlaylistBaseIE):
         'info_dict': {
             'id': 'dholbach_favorites',
             'title': 'Daniel Holbach (favorites)',
-            'description': 'md5:327af72d1efeb404a8216c27240d1370',
+            'description': 'md5:def36060ac8747b3aabca54924897e47',
         },
         'params': {
             'playlist_items': '1-100',
@@ -178,7 +178,7 @@ class MixcloudUserIE(MixcloudPlaylistBaseIE):
         'info_dict': {
             'id': 'dholbach_listens',
             'title': 'Daniel Holbach (listens)',
-            'description': 'md5:327af72d1efeb404a8216c27240d1370',
+            'description': 'md5:def36060ac8747b3aabca54924897e47',
         },
         'params': {
             'playlist_items': '1-100',
@@ -229,12 +229,7 @@ class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
         'playlist_mincount': 16,
     }, {
         'url': 'https://www.mixcloud.com/maxvibes/playlists/jazzcat-on-ness-radio/',
-        'info_dict': {
-            'id': 'maxvibes_jazzcat-on-ness-radio',
-            'title': 'Jazzcat on Ness Radio',
-            'description': 'md5:7bbbf0d6359a0b8cda85224be0f8f263',
-        },
-        'playlist_mincount': 23
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -243,15 +238,16 @@ class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
         playlist_id = mobj.group('playlist')
         video_id = '%s_%s' % (user_id, playlist_id)
 
-        profile = self._download_webpage(
+        webpage = self._download_webpage(
             url, user_id,
             note='Downloading playlist page',
             errnote='Unable to download playlist page')
 
-        description = self._get_user_description(profile)
-        playlist_title = self._html_search_regex(
-            r'<a class="parent active" href="[^"]*"><b>\d+</b><span title="[^"]*">([^</]*?)</span></a>',
-            profile, 'playlist title')
+        title = self._html_search_regex(
+            r'<a[^>]+class="parent active"[^>]*><b>\d+</b><span[^>]*>([^<]+)',
+            webpage, 'playlist title',
+            default=None) or self._og_search_title(webpage, fatal=False)
+        description = self._get_user_description(webpage)
 
         entries = OnDemandPagedList(
             functools.partial(
@@ -259,7 +255,7 @@ class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
                 '%s/playlists/%s' % (user_id, playlist_id), video_id, 'tracklist'),
             self._PAGE_SIZE)
 
-        return self.playlist_result(entries, video_id, playlist_title, description)
+        return self.playlist_result(entries, video_id, title, description)
 
 
 class MixcloudStreamIE(MixcloudPlaylistBaseIE):

From 29c6726646326fdbf17fca2f0e5600b6ee3b9f63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 21:11:08 +0700
Subject: [PATCH 058/611] [mixcloud] Fix view count extraction and modernize

---
 youtube_dl/extractor/mixcloud.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index bc38a8663..0efbe660a 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -97,7 +97,7 @@ class MixcloudIE(InfoExtractor):
         view_count = str_to_int(self._search_regex(
             [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',
              r'/listeners/?">([0-9,.]+)</a>',
-             r'm-tooltip=["\']([\d,.]+) plays'],
+             r'(?:m|data)-tooltip=["\']([\d,.]+) plays'],
             webpage, 'play count', default=None))
 
         return {
@@ -143,7 +143,7 @@ class MixcloudPlaylistBaseIE(InfoExtractor):
 
 
 class MixcloudUserIE(MixcloudPlaylistBaseIE):
-    _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/(?P<type>uploads|favorites|listens)?/?$'
+    _VALID_URL = r'https?://(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/(?P<type>uploads|favorites|listens)?/?$'
     IE_NAME = 'mixcloud:user'
 
     _TESTS = [{
@@ -216,7 +216,7 @@ class MixcloudUserIE(MixcloudPlaylistBaseIE):
 
 
 class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
-    _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/playlists/(?P<playlist>[^/]+)/?$'
+    _VALID_URL = r'https?://(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/playlists/(?P<playlist>[^/]+)/?$'
     IE_NAME = 'mixcloud:playlist'
 
     _TESTS = [{
@@ -259,7 +259,7 @@ class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
 
 
 class MixcloudStreamIE(MixcloudPlaylistBaseIE):
-    _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/(?P<id>[^/]+)/stream/?$'
+    _VALID_URL = r'https?://(?:www\.)?mixcloud\.com/(?P<id>[^/]+)/stream/?$'
     IE_NAME = 'mixcloud:stream'
 
     _TEST = {

From 1591ba258a49b682124e6e545c13a5da3bc660e0 Mon Sep 17 00:00:00 2001
From: Aldo Gunsing <dev@aldogunsing.nl>
Date: Sat, 25 Mar 2017 16:51:36 +0100
Subject: [PATCH 059/611] [npo:live] Add support for default url

---
 youtube_dl/extractor/npo.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 38fefe492..d1c9bc20e 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -313,9 +313,9 @@ class NPOIE(NPOBaseIE):
 
 class NPOLiveIE(NPOBaseIE):
     IE_NAME = 'npo.nl:live'
-    _VALID_URL = r'https?://(?:www\.)?npo\.nl/live/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?npo\.nl/live(/(?P<id>[^/?#&]+))?'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.npo.nl/live/npo-1',
         'info_dict': {
             'id': 'LI_NL1_4188102',
@@ -327,10 +327,22 @@ class NPOLiveIE(NPOBaseIE):
         'params': {
             'skip_download': True,
         }
-    }
+    }, {
+        'url': 'http://www.npo.nl/live',
+        'info_dict': {
+            'id': 'LI_NL1_4188102',
+            'display_id': 'npo-1',
+            'ext': 'mp4',
+            'title': 're:^NPO 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }]
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        display_id = self._match_id(url) or 'npo-1'
 
         webpage = self._download_webpage(url, display_id)
 

From 04e431cf97e8b73190e8db77de0e3d2b5655c195 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 21:31:22 +0700
Subject: [PATCH 060/611] [npo:live] Improve (closes #12555)

---
 youtube_dl/extractor/npo.py | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index d1c9bc20e..79296f0ef 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -313,7 +313,7 @@ class NPOIE(NPOBaseIE):
 
 class NPOLiveIE(NPOBaseIE):
     IE_NAME = 'npo.nl:live'
-    _VALID_URL = r'https?://(?:www\.)?npo\.nl/live(/(?P<id>[^/?#&]+))?'
+    _VALID_URL = r'https?://(?:www\.)?npo\.nl/live(?:/(?P<id>[^/?#&]+))?'
 
     _TESTS = [{
         'url': 'http://www.npo.nl/live/npo-1',
@@ -329,16 +329,7 @@ class NPOLiveIE(NPOBaseIE):
         }
     }, {
         'url': 'http://www.npo.nl/live',
-        'info_dict': {
-            'id': 'LI_NL1_4188102',
-            'display_id': 'npo-1',
-            'ext': 'mp4',
-            'title': 're:^NPO 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
-            'is_live': True,
-        },
-        'params': {
-            'skip_download': True,
-        }
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 95152630db22a648afefc7dc447da17f8989513c Mon Sep 17 00:00:00 2001
From: Steven Maude <StevenMaude@users.noreply.github.com>
Date: Sat, 8 Apr 2017 15:39:07 +0100
Subject: [PATCH 061/611] [rbmaradio] Add support for redbullradio.com URLs

---
 youtube_dl/extractor/rbmaradio.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rbmaradio.py b/youtube_dl/extractor/rbmaradio.py
index 53b82fba3..afa7b9161 100644
--- a/youtube_dl/extractor/rbmaradio.py
+++ b/youtube_dl/extractor/rbmaradio.py
@@ -13,15 +13,15 @@ from ..utils import (
 
 
 class RBMARadioIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?rbmaradio\.com/shows/(?P<show_id>[^/]+)/episodes/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:rbmaradio|redbullradio)\.com/shows/(?P<show_id>[^/]+)/episodes/(?P<id>[^/?#&]+)'
     _TEST = {
         'url': 'https://www.rbmaradio.com/shows/main-stage/episodes/ford-lopatin-live-at-primavera-sound-2011',
         'md5': '6bc6f9bcb18994b4c983bc3bf4384d95',
         'info_dict': {
             'id': 'ford-lopatin-live-at-primavera-sound-2011',
             'ext': 'mp3',
-            'title': 'Main Stage - Ford & Lopatin',
-            'description': 'md5:4f340fb48426423530af5a9d87bd7b91',
+            'title': 'Main Stage - Ford & Lopatin at Primavera Sound',
+            'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
             'thumbnail': r're:^https?://.*\.jpg',
             'duration': 2452,
             'timestamp': 1307103164,

From 2bd875edfe674e72ae935bacdbf0a055572d1d1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 22:38:37 +0700
Subject: [PATCH 062/611] [medici] Add extractor (closes #3406)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/medici.py     | 70 ++++++++++++++++++++++++++++++
 2 files changed, 71 insertions(+)
 create mode 100644 youtube_dl/extractor/medici.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 01865479c..68e1a5cfc 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -541,6 +541,7 @@ from .mangomolo import (
 )
 from .matchtv import MatchTVIE
 from .mdr import MDRIE
+from .medici import MediciIE
 from .meipai import MeipaiIE
 from .melonvod import MelonVODIE
 from .meta import METAIE
diff --git a/youtube_dl/extractor/medici.py b/youtube_dl/extractor/medici.py
new file mode 100644
index 000000000..cd910238e
--- /dev/null
+++ b/youtube_dl/extractor/medici.py
@@ -0,0 +1,70 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    unified_strdate,
+    update_url_query,
+    urlencode_postdata,
+)
+
+
+class MediciIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?medici\.tv/#!/(?P<id>[^?#&]+)'
+    _TEST = {
+        'url': 'http://www.medici.tv/#!/daniel-harding-frans-helmerson-verbier-festival-music-camp',
+        'md5': '004c21bb0a57248085b6ff3fec72719d',
+        'info_dict': {
+            'id': '3059',
+            'ext': 'flv',
+            'title': 'Daniel Harding conducts the Verbier Festival Music Camp \u2013 With Frans Helmerson',
+            'description': 'md5:322a1e952bafb725174fd8c1a8212f58',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'upload_date': '20170408',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        # Sets csrftoken cookie
+        self._download_webpage(url, video_id)
+
+        MEDICI_URL = 'http://www.medici.tv/'
+
+        data = self._download_json(
+            MEDICI_URL, video_id,
+            data=urlencode_postdata({
+                'json': 'true',
+                'page': '/%s' % video_id,
+                'timezone_offset': -420,
+            }), headers={
+                'X-CSRFToken': self._get_cookies(url)['csrftoken'].value,
+                'X-Requested-With': 'XMLHttpRequest',
+                'Referer': MEDICI_URL,
+                'Content-Type': 'application/x-www-form-urlencoded',
+            })
+
+        video = data['video']['videos']['video1']
+
+        title = video.get('nom') or data['title']
+
+        video_id = video.get('id') or video_id
+        formats = self._extract_f4m_formats(
+            update_url_query(video['url_akamai'], {
+                'hdcore': '3.1.0',
+                'plugin=aasp': '3.1.0.43.124',
+            }), video_id, f4m_id='hds')
+
+        description = data.get('meta_description')
+        thumbnail = video.get('url_thumbnail') or data.get('main_image')
+        upload_date = unified_strdate(data['video'].get('date'))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+            'formats': formats,
+        }

From e129fa0846734da1ea5814934c0c4b8fa4dc21b8 Mon Sep 17 00:00:00 2001
From: midas02 <midas02@users.noreply.github.com>
Date: Sat, 8 Apr 2017 17:43:29 +0200
Subject: [PATCH 063/611] [vier] Add IE_DESC

---
 youtube_dl/extractor/vier.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index 5086f591e..5ef7635b6 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -9,6 +9,7 @@ from .common import InfoExtractor
 
 class VierIE(InfoExtractor):
     IE_NAME = 'vier'
+    IE_DESC = 'vier.be and vijf.be'
     _VALID_URL = r'https?://(?:www\.)?(?P<site>vier|vijf)\.be/(?:[^/]+/videos/(?P<display_id>[^/]+)(?:/(?P<id>\d+))?|video/v3/embed/(?P<embed_id>\d+))'
     _TESTS = [{
         'url': 'http://www.vier.be/planb/videos/het-wordt-warm-de-moestuin/16129',

From ca8fca9d9d8b3773762a950a029e0a5354779e2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 22:44:31 +0700
Subject: [PATCH 064/611] [vrt] Add IE_DESC (closes #12477)

---
 youtube_dl/extractor/vrt.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/vrt.py b/youtube_dl/extractor/vrt.py
index 00c72e346..444295d68 100644
--- a/youtube_dl/extractor/vrt.py
+++ b/youtube_dl/extractor/vrt.py
@@ -10,6 +10,7 @@ from ..utils import (
 
 
 class VRTIE(InfoExtractor):
+    IE_DESC = 'deredactie.be, sporza.be, cobra.be and cobra.canvas.be'
     _VALID_URL = r'https?://(?:deredactie|sporza|cobra(?:\.canvas)?)\.be/cm/(?:[^/]+/)+(?P<id>[^/]+)/*'
     _TESTS = [
         # deredactie.be

From 41b263ac8a2721a324ad21d489de7d1ea9b41164 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 22:45:45 +0700
Subject: [PATCH 065/611] [canvas] Add IE_DESC (closes #12478)

---
 youtube_dl/extractor/canvas.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
index 544c6657c..aada02917 100644
--- a/youtube_dl/extractor/canvas.py
+++ b/youtube_dl/extractor/canvas.py
@@ -7,6 +7,7 @@ from ..utils import float_or_none
 
 
 class CanvasIE(InfoExtractor):
+    IE_DESC = 'canvas.be and een.be'
     _VALID_URL = r'https?://(?:www\.)?(?P<site_id>canvas|een)\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',

From eb8cc8ea3bde565b48b7eecfb81c7c7a95ea2480 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 23:59:17 +0700
Subject: [PATCH 066/611] [ChangeLog] Actualize

---
 ChangeLog | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 47804428f..93e260606 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,23 @@
 version <unreleased>
 
 Extractors
++ [medici] Add support for medici.tv (#3406)
++ [rbmaradio] Add support for redbullradio.com URLs (#12687)
++ [npo:live] Add support for default URL (#12555)
+* [mixcloud:playlist] Fix title, description and view count extraction (#12582)
++ [thesun] Add suport for thesun.co.uk (#11298, #12674)
++ [ceskateleveize:porady] Add support for porady (#7411, #12645)
+* [ceskateleveize] Improve extraction and remove URL replacement hacks
++ [kaltura] Add support for iframe embeds (#12679)
 * [airmozilla] Fix extraction (#12670)
+* [wshh] Extract html5 entries and delegate to generic extractor (12676)
++ [raiplay] Extract subtitles
++ [xfileshare] Add support for vidlo.us (#12660)
++ [xfileshare] Add support for vidbom.com (#12661)
++ [aenetworks] Add more video URL regular expressions (#12657)
++ [odnoklassniki] Fix format sorting for 1080p quality
++ [rtl2] Add support for you.rtl2.de (#10257)
++ [vshare] Add support for vshare.io (#12278)
 
 
 version 2017.04.03

From 43724365048755adbea8bbff05708903ea52e1a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Apr 2017 00:01:28 +0700
Subject: [PATCH 067/611] release 2017.04.09

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 ChangeLog                 |  2 +-
 docs/supportedsites.md    | 16 +++++++++++-----
 youtube_dl/version.py     |  2 +-
 4 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 8461a6173..898d96b78 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.03*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.03**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.09*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.09**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.04.03
+[debug] youtube-dl version 2017.04.09
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 93e260606..143c688f5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.04.09
 
 Extractors
 + [medici] Add support for medici.tv (#3406)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a075c7012..0157ce3ef 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -127,7 +127,7 @@
  - **CamWithHer**
  - **canalc2.tv**
  - **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv
- - **Canvas**
+ - **Canvas**: canvas.be and een.be
  - **CarambaTV**
  - **CarambaTVPage**
  - **CartoonNetwork**
@@ -145,6 +145,7 @@
  - **CCTV**: 央视网
  - **CDA**
  - **CeskaTelevize**
+ - **CeskaTelevizePorady**
  - **channel9**: Channel 9
  - **CharlieRose**
  - **Chaturbate**
@@ -431,6 +432,7 @@
  - **MDR**: MDR.DE and KiKA
  - **media.ccc.de**
  - **Medialaan**
+ - **Medici**
  - **Meipai**: 美拍
  - **MelonVOD**
  - **META**
@@ -658,7 +660,9 @@
  - **rte**: Raidió Teilifís Éireann TV
  - **rte:radio**: Raidió Teilifís Éireann radio
  - **rtl.nl**: rtl.nl and rtlxl.nl
- - **RTL2**
+ - **rtl2**
+ - **rtl2:you**
+ - **rtl2:you:series**
  - **RTP**
  - **RTS**: RTS.ch
  - **rtve.es:alacarta**: RTVE a la carta
@@ -780,6 +784,7 @@
  - **TheScene**
  - **TheSixtyOne**
  - **TheStar**
+ - **TheSun**
  - **TheWeatherChannel**
  - **ThisAmericanLife**
  - **ThisAV**
@@ -893,7 +898,7 @@
  - **vidme:user**
  - **vidme:user:likes**
  - **Vidzi**
- - **vier**
+ - **vier**: vier.be and vijf.be
  - **vier:videos**
  - **ViewLift**
  - **ViewLiftEmbed**
@@ -930,9 +935,10 @@
  - **Vporn**
  - **vpro**: npo.nl and ntr.nl
  - **Vrak**
- - **VRT**
+ - **VRT**: deredactie.be, sporza.be, cobra.be and cobra.canvas.be
  - **vrv**
  - **vrv:series**
+ - **VShare**
  - **vube**: Vube.com
  - **VuClip**
  - **VVVVID**
@@ -960,7 +966,7 @@
  - **WSJ**: Wall Street Journal
  - **XBef**
  - **XboxClips**
- - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To, XVIDSTAGE, Vid ABC
+ - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To, XVIDSTAGE, Vid ABC, VidBom, vidlo
  - **XHamster**
  - **XHamsterEmbed**
  - **xiami:album**: 虾米音乐 - 专辑
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1b260064b..39268f734 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.04.03'
+__version__ = '2017.04.09'

From a6f7263cf421d9156048596cca0552e027e70788 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Apr 2017 01:39:40 +0700
Subject: [PATCH 068/611] [chaturbate] Fix extraction (closes #12665)

---
 youtube_dl/extractor/chaturbate.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/chaturbate.py b/youtube_dl/extractor/chaturbate.py
index 8fbc91c1f..e3eba4be9 100644
--- a/youtube_dl/extractor/chaturbate.py
+++ b/youtube_dl/extractor/chaturbate.py
@@ -33,10 +33,17 @@ class ChaturbateIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        m3u8_formats = [(m.group('id').lower(), m.group('url')) for m in re.finditer(
-            r'hlsSource(?P<id>.+?)\s*=\s*(?P<q>["\'])(?P<url>http.+?)(?P=q)', webpage)]
+        m3u8_urls = []
 
-        if not m3u8_formats:
+        for m in re.finditer(
+                r'(["\'])(?P<url>http.+?\.m3u8.*?)\1', webpage):
+            m3u8_fast_url, m3u8_no_fast_url = m.group('url'), m.group(
+                'url').replace('_fast', '')
+            for m3u8_url in (m3u8_fast_url, m3u8_no_fast_url):
+                if m3u8_url not in m3u8_urls:
+                    m3u8_urls.append(m3u8_url)
+
+        if not m3u8_urls:
             error = self._search_regex(
                 [r'<span[^>]+class=(["\'])desc_span\1[^>]*>(?P<error>[^<]+)</span>',
                  r'<div[^>]+id=(["\'])defchat\1[^>]*>\s*<p><strong>(?P<error>[^<]+)<'],
@@ -50,7 +57,8 @@ class ChaturbateIE(InfoExtractor):
             raise ExtractorError('Unable to find stream URL')
 
         formats = []
-        for m3u8_id, m3u8_url in m3u8_formats:
+        for m3u8_url in m3u8_urls:
+            m3u8_id = 'fast' if '_fast' in m3u8_url else 'slow'
             formats.extend(self._extract_m3u8_formats(
                 m3u8_url, video_id, ext='mp4',
                 # ffmpeg skips segments for fast m3u8

From 3f2ce6896acfd487ce915788255ddee1e04794b3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 9 Apr 2017 12:32:26 +0100
Subject: [PATCH 069/611] [adn] update subtitle decryption key

---
 youtube_dl/extractor/adn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/adn.py b/youtube_dl/extractor/adn.py
index e44caa00b..66caf6a81 100644
--- a/youtube_dl/extractor/adn.py
+++ b/youtube_dl/extractor/adn.py
@@ -45,7 +45,7 @@ class ADNIE(InfoExtractor):
         # http://animedigitalnetwork.fr/components/com_vodvideo/videojs/adn-vjs.min.js
         dec_subtitles = intlist_to_bytes(aes_cbc_decrypt(
             bytes_to_intlist(base64.b64decode(enc_subtitles[24:])),
-            bytes_to_intlist(b'\xb5@\xcfq\xa3\x98"N\xe4\xf3\x12\x98}}\x16\xd8'),
+            bytes_to_intlist(b'\nd\xaf\xd2J\xd0\xfc\xe1\xfc\xdf\xb61\xe8\xe1\xf0\xcc'),
             bytes_to_intlist(base64.b64decode(enc_subtitles[:24]))
         ))
         subtitles_json = self._parse_json(

From a206ef62df7e45992ebb26bf41f86c01d904fe75 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 10 Apr 2017 13:50:08 +0100
Subject: [PATCH 070/611] [curiositystream] fix extraction(closes #12638)

---
 youtube_dl/extractor/curiositystream.py | 54 +++++++++++++++++++++----
 1 file changed, 47 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/curiositystream.py b/youtube_dl/extractor/curiositystream.py
index e3c99468c..145d9397c 100644
--- a/youtube_dl/extractor/curiositystream.py
+++ b/youtube_dl/extractor/curiositystream.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
@@ -49,6 +51,48 @@ class CuriosityStreamBaseIE(InfoExtractor):
         limelight_media_id = media['limelight_media_id']
         title = media['title']
 
+        formats = []
+        for encoding in media.get('encodings', []):
+            m3u8_url = encoding.get('master_playlist_url')
+            if m3u8_url:
+                formats.extend(self._extract_m3u8_formats(
+                    m3u8_url, video_id, 'mp4', 'm3u8_native',
+                    m3u8_id='hls', fatal=False))
+            encoding_url = encoding.get('url')
+            file_url = encoding.get('file_url')
+            if not encoding_url and not file_url:
+                continue
+            f = {
+                'width': int_or_none(encoding.get('width')),
+                'height': int_or_none(encoding.get('height')),
+                'vbr': int_or_none(encoding.get('video_bitrate')),
+                'abr': int_or_none(encoding.get('audio_bitrate')),
+                'filesize': int_or_none(encoding.get('size_in_bytes')),
+                'vcodec': encoding.get('video_codec'),
+                'acodec': encoding.get('audio_codec'),
+                'container': encoding.get('container_type'),
+            }
+            for f_url in (encoding_url, file_url):
+                if not f_url:
+                    continue
+                fmt = f.copy()
+                rtmp = re.search(r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+))/(?P<playpath>mp[34]:.+)$', f_url)
+                if rtmp:
+                    fmt.update({
+                        'url': rtmp.group('url'),
+                        'play_path': rtmp.group('playpath'),
+                        'app': rtmp.group('app'),
+                        'ext': 'flv',
+                        'format_id': 'rtmp',
+                    })
+                else:
+                    fmt.update({
+                        'url': f_url,
+                        'format_id': 'http',
+                    })
+                formats.append(fmt)
+        self._sort_formats(formats)
+
         subtitles = {}
         for closed_caption in media.get('closed_captions', []):
             sub_url = closed_caption.get('file')
@@ -60,16 +104,14 @@ class CuriosityStreamBaseIE(InfoExtractor):
             })
 
         return {
-            '_type': 'url_transparent',
             'id': video_id,
-            'url': 'limelight:media:' + limelight_media_id,
+            'formats': formats,
             'title': title,
             'description': media.get('description'),
             'thumbnail': media.get('image_large') or media.get('image_medium') or media.get('image_small'),
             'duration': int_or_none(media.get('duration')),
             'tags': media.get('tags'),
             'subtitles': subtitles,
-            'ie_key': 'LimelightMedia',
         }
 
 
@@ -78,14 +120,12 @@ class CuriosityStreamIE(CuriosityStreamBaseIE):
     _VALID_URL = r'https?://app\.curiositystream\.com/video/(?P<id>\d+)'
     _TEST = {
         'url': 'https://app.curiositystream.com/video/2',
-        'md5': 'a0074c190e6cddaf86900b28d3e9ee7a',
+        'md5': '262bb2f257ff301115f1973540de8983',
         'info_dict': {
             'id': '2',
             'ext': 'mp4',
             'title': 'How Did You Develop The Internet?',
             'description': 'Vint Cerf, Google\'s Chief Internet Evangelist, describes how he and Bob Kahn created the internet.',
-            'timestamp': 1448388615,
-            'upload_date': '20151124',
         }
     }
 
@@ -105,7 +145,7 @@ class CuriosityStreamCollectionIE(CuriosityStreamBaseIE):
             'title': 'Curious Minds: The Internet',
             'description': 'How is the internet shaping our lives in the 21st Century?',
         },
-        'playlist_mincount': 17,
+        'playlist_mincount': 12,
     }
 
     def _real_extract(self, url):

From 1af959ef9f8936a8e211f692a2da1a69e0076962 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Apr 2017 22:53:06 +0700
Subject: [PATCH 071/611] [bbccouk] Add support for https protocol in media
 selector (closes #12701)

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 8a2ed0ab6..f0de07619 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -370,7 +370,7 @@ class BBCCoUkIE(InfoExtractor):
                                 'acodec': encoding,
                                 'vcodec': 'none',
                             })
-                        if protocol == 'http':
+                        if protocol in ('http', 'https'):
                             # Direct link
                             fmt.update({
                                 'url': href,

From 964744af952fc372e961a4b6b4b0f3ed6172a83f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Apr 2017 22:53:51 +0700
Subject: [PATCH 072/611] [bbccouk] Skip unrecognized formats in media selector
 (#12701)

---
 youtube_dl/extractor/bbc.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index f0de07619..a0c77b0b0 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -389,6 +389,8 @@ class BBCCoUkIE(InfoExtractor):
                                 'rtmp_live': False,
                                 'ext': 'flv',
                             })
+                        else:
+                            continue
                         formats.append(fmt)
             elif kind == 'captions':
                 subtitles = self.extract_subtitles(media, programme_id)

From 6240925b4021a755a9deb9c4707ee5cabcb63c2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Apr 2017 22:56:22 +0700
Subject: [PATCH 073/611] [bbccouk] Treat bitrate as audio+video bitrate in
 media selector

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index a0c77b0b0..0e05b782b 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -361,7 +361,7 @@ class BBCCoUkIE(InfoExtractor):
                             fmt.update({
                                 'width': width,
                                 'height': height,
-                                'vbr': bitrate,
+                                'tbr': bitrate,
                                 'vcodec': encoding,
                             })
                         else:

From 364a69e8c66f2259f21d2d67a0e766f0180cd217 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Apr 2017 23:59:38 +0700
Subject: [PATCH 074/611] [test_download] Fix testing playlists with single
 video and add comments

---
 test/test_download.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/test/test_download.py b/test/test_download.py
index fd7d2e8be..0e9f293b5 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -199,9 +199,16 @@ def generator(test_case, tname):
                 self.assertEqual(
                     test_case['playlist_duration_sum'], got_duration)
 
+            # Generalize both playlists and single videos to unified format for
+            # simplicity
+            if 'entries' not in res_dict:
+                res_dict['entries'] = [res_dict]
+
             for tc_num, tc in enumerate(test_cases):
-                tc_res_dict = res_dict['entries'][tc_num] if is_playlist else res_dict
+                tc_res_dict = res_dict['entries'][tc_num]
+                # First, check test cases' data against extracted data alone
                 expect_info_dict(self, tc_res_dict, tc.get('info_dict', {}))
+                # Now, check downloaded file consistency
                 tc_filename = get_tc_filename(tc)
                 if not test_case.get('params', {}).get('skip_download', False):
                     self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename)
@@ -219,13 +226,14 @@ def generator(test_case, tname):
                     if 'md5' in tc:
                         md5_for_file = _file_md5(tc_filename)
                         self.assertEqual(md5_for_file, tc['md5'])
+                # Finally, check test cases' data again but this time against
+                # extracted data from info JSON file written during processing
                 info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json'
                 self.assertTrue(
                     os.path.exists(info_json_fn),
                     'Missing info file %s' % info_json_fn)
                 with io.open(info_json_fn, encoding='utf-8') as infof:
                     info_dict = json.load(infof)
-
                 expect_info_dict(self, info_dict, tc.get('info_dict', {}))
         finally:
             try_rm_tcs_files()

From 61568e50cf999e334336ad9c2c6bbd62864aa3fc Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Mon, 10 Apr 2017 13:04:32 -0400
Subject: [PATCH 075/611] [generic] Add support for <object> youtube embeds
 (closes #12637)

---
 youtube_dl/extractor/generic.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 658533cf6..4690816cb 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -730,6 +730,21 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             }
         },
+        # YouTube <object> embed
+        {
+            'url': 'http://www.improbable.com/2017/04/03/untrained-modern-youths-and-ancient-masters-in-selfie-portraits/',
+            'md5': '516718101ec834f74318df76259fb3cc',
+            'info_dict': {
+                'id': 'msN87y-iEx0',
+                'ext': 'webm',
+                'title': 'Feynman: Mirrors FUN TO IMAGINE 6',
+                'upload_date': '20080526',
+                'description': 'md5:0ffc78ea3f01b2e2c247d5f8d1d3c18d',
+                'uploader': 'Christopher Sykes',
+                'uploader_id': 'ChristopherJSykes',
+            },
+            'add_ie': ['Youtube'],
+        },
         # Camtasia studio
         {
             'url': 'http://www.ll.mit.edu/workshops/education/videocourses/antennas/lecture1/video/',
@@ -1938,6 +1953,7 @@ class GenericIE(InfoExtractor):
                 data-video-url=|
                 <embed[^>]+?src=|
                 embedSWF\(?:\s*|
+                <object[^>]+data=|
                 new\s+SWFObject\(
             )
             (["\'])

From 6b9466de2f3f8bcaaf360a60625221c4bd7a50cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Apr 2017 02:05:53 +0700
Subject: [PATCH 076/611] [afreecatv] Fix extraction (closes #12706)

---
 youtube_dl/extractor/afreecatv.py | 126 ++++++++++++++++++++++++++----
 1 file changed, 112 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index b774d6db8..8ac88c2dc 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..compat import compat_xpath
 from ..utils import (
+    determine_ext,
     ExtractorError,
     int_or_none,
     xpath_text,
@@ -72,13 +73,54 @@ class AfreecaTVIE(InfoExtractor):
         'url': 'http://vod.afreecatv.com/PLAYER/STATION/18650793',
         'info_dict': {
             'id': '18650793',
-            'ext': 'flv',
+            'ext': 'mp4',
+            'title': '오늘은 다르다! 쏘님의 우월한 위아래~ 댄스리액션!',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': '윈아디',
             'uploader_id': 'badkids',
-            'title': '오늘은 다르다! 쏘님의 우월한 위아래~ 댄스리액션!',
+            'duration': 107,
         },
         'params': {
-            'skip_download': True,  # requires rtmpdump
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://vod.afreecatv.com/PLAYER/STATION/10481652',
+        'info_dict': {
+            'id': '10481652',
+            'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'",
+            'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
+            'uploader': 'dailyapril',
+            'uploader_id': 'dailyapril',
+            'duration': 6492,
+        },
+        'playlist_count': 2,
+        'playlist': [{
+            'md5': 'd8b7c174568da61d774ef0203159bf97',
+            'info_dict': {
+                'id': '10481652_1',
+                'ext': 'mp4',
+                'title': "BJ유트루와 함께하는 '팅커벨 메이크업!' (part 1)",
+                'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
+                'uploader': 'dailyapril',
+                'uploader_id': 'dailyapril',
+                'upload_date': '20160502',
+                'duration': 3601,
+            },
+        }, {
+            'md5': '58f2ce7f6044e34439ab2d50612ab02b',
+            'info_dict': {
+                'id': '10481652_2',
+                'ext': 'mp4',
+                'title': "BJ유트루와 함께하는 '팅커벨 메이크업!' (part 2)",
+                'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
+                'uploader': 'dailyapril',
+                'uploader_id': 'dailyapril',
+                'upload_date': '20160502',
+                'duration': 2891,
+            },
+        }],
+        'params': {
+            'skip_download': True,
         },
     }, {
         'url': 'http://www.afreecatv.com/player/Player.swf?szType=szBjId=djleegoon&nStationNo=11273158&nBbsNo=13161095&nTitleNo=36327652',
@@ -94,7 +136,7 @@ class AfreecaTVIE(InfoExtractor):
         m = re.match(r'^(?P<upload_date>\d{8})_\w+_(?P<part>\d+)$', key)
         if m:
             video_key['upload_date'] = m.group('upload_date')
-            video_key['part'] = m.group('part')
+            video_key['part'] = int(m.group('part'))
         return video_key
 
     def _real_extract(self, url):
@@ -109,23 +151,64 @@ class AfreecaTVIE(InfoExtractor):
             raise ExtractorError('Specified AfreecaTV video does not exist',
                                  expected=True)
 
-        video_url_raw = video_element.text
-
-        app, playpath = video_url_raw.split('mp4:')
+        video_url = video_element.text.strip()
 
         title = xpath_text(video_xml, './track/title', 'title', fatal=True)
+
         uploader = xpath_text(video_xml, './track/nickname', 'uploader')
         uploader_id = xpath_text(video_xml, './track/bj_id', 'uploader id')
-        duration = int_or_none(xpath_text(video_xml, './track/duration',
-                                          'duration'))
+        duration = int_or_none(xpath_text(
+            video_xml, './track/duration', 'duration'))
         thumbnail = xpath_text(video_xml, './track/titleImage', 'thumbnail')
 
-        return {
+        common_entry = {
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'thumbnail': thumbnail,
+        }
+
+        info = common_entry.copy()
+        info.update({
+            'id': video_id,
+            'title': title,
+            'duration': duration,
+        })
+
+        if not video_url:
+            entries = []
+            for file_num, file_element in enumerate(
+                    video_element.findall(compat_xpath('./file')), start=1):
+                file_url = file_element.text
+                if not file_url:
+                    continue
+                video_key = self.parse_video_key(file_element.get('key', ''))
+                if not video_key:
+                    continue
+                file_duration = int_or_none(file_element.get('duration'))
+                part = video_key.get('part', file_num)
+                format_id = '%s_%s' % (video_id, part)
+                formats = self._extract_m3u8_formats(
+                    file_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls',
+                    note='Downloading part %d m3u8 information' % file_num)
+                file_info = common_entry.copy()
+                file_info.update({
+                    'id': format_id,
+                    'title': '%s (part %d)' % (title, part),
+                    'upload_date': video_key.get('upload_date'),
+                    'duration': file_duration,
+                    'formats': formats,
+                })
+                entries.append(file_info)
+            entries_info = info.copy()
+            entries_info.update({
+                '_type': 'multi_video',
+                'entries': entries,
+            })
+            return entries_info
+
+        info = {
             'id': video_id,
-            'url': app,
-            'ext': 'flv',
-            'play_path': 'mp4:' + playpath,
-            'rtmp_live': True,  # downloading won't end without this
             'title': title,
             'uploader': uploader,
             'uploader_id': uploader_id,
@@ -133,6 +216,21 @@ class AfreecaTVIE(InfoExtractor):
             'thumbnail': thumbnail,
         }
 
+        if determine_ext(video_url) == 'm3u8':
+            info['formats'] = self._extract_m3u8_formats(
+                video_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                m3u8_id='hls')
+        else:
+            app, playpath = video_url.split('mp4:')
+            info.update({
+                'url': app,
+                'ext': 'flv',
+                'play_path': 'mp4:' + playpath,
+                'rtmp_live': True,  # downloading won't end without this
+            })
+
+        return info
+
 
 class AfreecaTVGlobalIE(AfreecaTVIE):
     IE_NAME = 'afreecatv:global'

From 689cd458a67f87c9c9537191f017a0082fa17bc9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Apr 2017 02:16:20 +0700
Subject: [PATCH 077/611] [ChangeLog] Actualize

---
 ChangeLog | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 143c688f5..bfb812742 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,16 @@
+version <unreleased>
+
+Extractors
+* [afreecatv] Fix extraction (#12706)
++ [generic] Add support for <object> YouTube embeds (#12637)
+* [bbccouk] Treat bitrate as audio+video bitrate in media selector
++ [bbccouk] Skip unrecognized formats in media selector (#12701)
++ [bbccouk] Add support for https protocol in media selector (#12701)
+* [curiositystream] Fix extraction (#12638)
+* [adn] Update subtitle decryption key
+* [chaturbate] Fix extraction (#12665, #12688, #12690)
+
+
 version 2017.04.09
 
 Extractors

From 173087816710dc59a60a4cc81c316bb04dfa7d85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Apr 2017 02:17:53 +0700
Subject: [PATCH 078/611] release 2017.04.11

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 898d96b78..61af50e5c 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.09*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.09**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.11*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.11**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.04.09
+[debug] youtube-dl version 2017.04.11
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index bfb812742..f26372bc1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.04.11
 
 Extractors
 * [afreecatv] Fix extraction (#12706)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 39268f734..5918c10bc 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.04.09'
+__version__ = '2017.04.11'

From 6214611a4a01674d1f3cd5434d5ee4494110bd91 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Apr 2017 22:30:15 +0700
Subject: [PATCH 079/611] [canalplus] Bypass geo restriction

---
 youtube_dl/extractor/canalplus.py | 39 +++++++++++++++++--------------
 1 file changed, 22 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 4b9fa2ddf..90b454115 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -45,6 +45,9 @@ class CanalplusIE(InfoExtractor):
         'itele': 'itele',
     }
 
+    # Only works for direct mp4 URLs
+    _GEO_COUNTRIES = ['FR']
+
     _TESTS = [{
         'url': 'http://www.canalplus.fr/c-emissions/pid1830-c-zapping.html?vid=1192814',
         'info_dict': {
@@ -56,6 +59,7 @@ class CanalplusIE(InfoExtractor):
             'upload_date': '20160702',
         },
     }, {
+        # geo restricted, bypassed
         'url': 'http://www.piwiplus.fr/videos-piwi/pid1405-le-labyrinthe-boing-super-ranger.html?vid=1108190',
         'info_dict': {
             'id': '1108190',
@@ -65,19 +69,20 @@ class CanalplusIE(InfoExtractor):
             'description': 'md5:4cea7a37153be42c1ba2c1d3064376ff',
             'upload_date': '20140724',
         },
-        'skip': 'Only works from France',
+        'expected_warnings': ['HTTP Error 403: Forbidden'],
     }, {
-        'url': 'http://www.c8.fr/c8-divertissement/ms-touche-pas-a-mon-poste/pid6318-videos-integrales.html',
-        'md5': '4b47b12b4ee43002626b97fad8fb1de5',
+        # geo restricted, bypassed
+        'url': 'http://www.c8.fr/c8-divertissement/ms-touche-pas-a-mon-poste/pid6318-videos-integrales.html?vid=1443684',
+        'md5': 'bb6f9f343296ab7ebd88c97b660ecf8d',
         'info_dict': {
-            'id': '1420213',
+            'id': '1443684',
             'display_id': 'pid6318-videos-integrales',
             'ext': 'mp4',
-            'title': 'TPMP ! Même le matin - Les 35H de Baba - 14/10/2016',
-            'description': 'md5:f96736c1b0ffaa96fd5b9e60ad871799',
-            'upload_date': '20161014',
+            'title': 'Guess my iep ! - TPMP - 07/04/2017',
+            'description': 'md5:6f005933f6e06760a9236d9b3b5f17fa',
+            'upload_date': '20170407',
         },
-        'skip': 'Only works from France',
+        'expected_warnings': ['HTTP Error 403: Forbidden'],
     }, {
         'url': 'http://www.itele.fr/chroniques/invite-michael-darmon/rachida-dati-nicolas-sarkozy-est-le-plus-en-phase-avec-les-inquietudes-des-francais-171510',
         'info_dict': {
@@ -134,15 +139,15 @@ class CanalplusIE(InfoExtractor):
 
         preference = qualities(['MOBILE', 'BAS_DEBIT', 'HAUT_DEBIT', 'HD'])
 
-        fmt_url = next(iter(media.get('VIDEOS')))
-        if '/geo' in fmt_url.lower():
-            response = self._request_webpage(
-                HEADRequest(fmt_url), video_id,
-                'Checking if the video is georestricted')
-            if '/blocage' in response.geturl():
-                raise ExtractorError(
-                    'The video is not available in your country',
-                    expected=True)
+        # _, fmt_url = next(iter(media['VIDEOS'].items()))
+        # if '/geo' in fmt_url.lower():
+        #     response = self._request_webpage(
+        #         HEADRequest(fmt_url), video_id,
+        #         'Checking if the video is georestricted')
+        #     if '/blocage' in response.geturl():
+        #         raise ExtractorError(
+        #             'The video is not available in your country',
+        #             expected=True)
 
         formats = []
         for format_id, format_url in media['VIDEOS'].items():

From d23028a8fb3784735c11853f0d4144d8bad37ba0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Apr 2017 22:41:48 +0700
Subject: [PATCH 080/611] [youtube] Remove explicit preference for audio-only
 and video-only formats In order not to break sorting when new formats appear

---
 youtube_dl/extractor/youtube.py | 90 ++++++++++++++++-----------------
 1 file changed, 45 insertions(+), 45 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ca40de522..2506cd27c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -317,60 +317,60 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 
         # DASH mp4 video
-        '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
-        '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
-        '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
-        '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
+        '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
+        '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
+        '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
 
         # Dash mp4 audio
-        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'preference': -50, 'container': 'm4a_dash'},
-        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'preference': -50, 'container': 'm4a_dash'},
-        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
-        '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'preference': -50, 'container': 'm4a_dash'},
-        '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'preference': -50, 'container': 'm4a_dash'},
-        '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'preference': -50, 'container': 'm4a_dash'},
-        '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'preference': -50, 'container': 'm4a_dash'},
+        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
+        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
+        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
+        '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
+        '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
+        '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
+        '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
 
         # Dash webm
-        '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9', 'preference': -40},
-        '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
-        '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
-        '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
-        '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
-        '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
-        '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
-        '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
-        '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
+        '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
+        '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
+        '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
+        '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
+        '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
+        '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
+        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
+        '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
-        '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
-        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
-        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
-        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
-        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
-        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
+        '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
+        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
+        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
+        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
 
         # Dash webm audio
-        '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},
-        '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50},
+        '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
+        '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
 
         # Dash webm audio with opus inside
-        '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50, 'preference': -50},
-        '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70, 'preference': -50},
-        '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160, 'preference': -50},
+        '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
+        '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
+        '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
 
         # RTMP (unnamed)
         '_rtmp': {'protocol': 'rtmp'},

From e109f1ff438f82305d91bb49944a616c53abc9bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 12 Apr 2017 02:15:37 +0700
Subject: [PATCH 081/611] [afreecatv] Fix extraction for videos with different
 key layout (closes #12718)

---
 youtube_dl/extractor/afreecatv.py | 39 ++++++++++++++++++++++---------
 1 file changed, 28 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index 8ac88c2dc..78d29c861 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -97,7 +97,7 @@ class AfreecaTVIE(InfoExtractor):
         'playlist': [{
             'md5': 'd8b7c174568da61d774ef0203159bf97',
             'info_dict': {
-                'id': '10481652_1',
+                'id': '20160502_c4c62b9d_174361386_1',
                 'ext': 'mp4',
                 'title': "BJ유트루와 함께하는 '팅커벨 메이크업!' (part 1)",
                 'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
@@ -109,7 +109,7 @@ class AfreecaTVIE(InfoExtractor):
         }, {
             'md5': '58f2ce7f6044e34439ab2d50612ab02b',
             'info_dict': {
-                'id': '10481652_2',
+                'id': '20160502_39e739bb_174361386_2',
                 'ext': 'mp4',
                 'title': "BJ유트루와 함께하는 '팅커벨 메이크업!' (part 2)",
                 'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
@@ -122,6 +122,22 @@ class AfreecaTVIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        # non standard key
+        'url': 'http://vod.afreecatv.com/PLAYER/STATION/20515605',
+        'info_dict': {
+            'id': '20170411_BE689A0E_190960999_1_2_h',
+            'ext': 'mp4',
+            'title': '혼자사는여자집',
+            'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
+            'uploader': '♥이슬이',
+            'uploader_id': 'dasl8121',
+            'upload_date': '20170411',
+            'duration': 213,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.afreecatv.com/player/Player.swf?szType=szBjId=djleegoon&nStationNo=11273158&nBbsNo=13161095&nTitleNo=36327652',
         'only_matching': True,
@@ -176,26 +192,27 @@ class AfreecaTVIE(InfoExtractor):
 
         if not video_url:
             entries = []
-            for file_num, file_element in enumerate(
-                    video_element.findall(compat_xpath('./file')), start=1):
+            file_elements = video_element.findall(compat_xpath('./file'))
+            one = len(file_elements) == 1
+            for file_num, file_element in enumerate(file_elements, start=1):
                 file_url = file_element.text
                 if not file_url:
                     continue
-                video_key = self.parse_video_key(file_element.get('key', ''))
-                if not video_key:
-                    continue
+                key = file_element.get('key', '')
+                upload_date = self._search_regex(
+                    r'^(\d{8})_', key, 'upload date', default=None)
                 file_duration = int_or_none(file_element.get('duration'))
-                part = video_key.get('part', file_num)
-                format_id = '%s_%s' % (video_id, part)
+                format_id = key if key else '%s_%s' % (video_id, file_num)
                 formats = self._extract_m3u8_formats(
                     file_url, video_id, 'mp4', entry_protocol='m3u8_native',
                     m3u8_id='hls',
                     note='Downloading part %d m3u8 information' % file_num)
+                title = title if one else '%s (part %d)' % (title, file_num)
                 file_info = common_entry.copy()
                 file_info.update({
                     'id': format_id,
-                    'title': '%s (part %d)' % (title, part),
-                    'upload_date': video_key.get('upload_date'),
+                    'title': title,
+                    'upload_date': upload_date,
                     'duration': file_duration,
                     'formats': formats,
                 })

From 89beedd31f8a00a811787b651a506001833da0e1 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 11 Apr 2017 21:55:59 +0100
Subject: [PATCH 082/611] [youtube] improve m3u8 format extraction

---
 youtube_dl/extractor/youtube.py | 47 +++++++++++----------------------
 1 file changed, 15 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2506cd27c..9e2b9115c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1253,21 +1253,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         video_id = mobj.group(2)
         return video_id
 
-    def _extract_from_m3u8(self, manifest_url, video_id):
-        url_map = {}
-
-        def _get_urls(_manifest):
-            lines = _manifest.split('\n')
-            urls = filter(lambda l: l and not l.startswith('#'),
-                          lines)
-            return urls
-        manifest = self._download_webpage(manifest_url, video_id, 'Downloading formats manifest')
-        formats_urls = _get_urls(manifest)
-        for format_url in formats_urls:
-            itag = self._search_regex(r'itag/(\d+?)/', format_url, 'itag')
-            url_map[itag] = format_url
-        return url_map
-
     def _extract_annotations(self, video_id):
         url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
         return self._download_webpage(url, video_id, note='Searching for annotations.', errnote='Unable to download video annotations.')
@@ -1573,19 +1558,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         if self._downloader.params.get('writeannotations', False):
             video_annotations = self._extract_annotations(video_id)
 
-        def _map_to_format_list(urlmap):
-            formats = []
-            for itag, video_real_url in urlmap.items():
-                dct = {
-                    'format_id': itag,
-                    'url': video_real_url,
-                    'player_url': player_url,
-                }
-                if itag in self._formats:
-                    dct.update(self._formats[itag])
-                formats.append(dct)
-            return formats
-
         if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
             self.report_rtmp_download()
             formats = [{
@@ -1718,11 +1690,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 formats.append(dct)
         elif video_info.get('hlsvp'):
             manifest_url = video_info['hlsvp'][0]
-            url_map = self._extract_from_m3u8(manifest_url, video_id)
-            formats = _map_to_format_list(url_map)
-            # Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming
-            for a_format in formats:
+            formats = []
+            m3u8_formats = self._extract_m3u8_formats(
+                manifest_url, video_id, 'mp4', fatal=False)
+            for a_format in m3u8_formats:
+                itag = self._search_regex(
+                    r'/itag/(\d+)/', a_format['url'], 'itag', default=None)
+                if itag:
+                    a_format['format_id'] = itag
+                    if itag in self._formats:
+                        dct = self._formats[itag].copy()
+                        dct.update(a_format)
+                        a_format = dct
+                a_format['player_url'] = player_url
+                # Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming
                 a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = 'True'
+                formats.append(a_format)
         else:
             unavailable_message = self._html_search_regex(
                 r'(?s)<h1[^>]+id="unavailable-message"[^>]*>(.+?)</h1>',

From a9a346535bd855f4dd01aa32e54345cc741769c4 Mon Sep 17 00:00:00 2001
From: Daniel Twardowski <daniel.j.twardowski@gmail.com>
Date: Wed, 12 Apr 2017 01:18:57 -0400
Subject: [PATCH 083/611] [adobepass] Add Verizon support (#10803)

---
 youtube_dl/extractor/adobepass.py | 71 +++++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index 1b2d364ca..a15843315 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -41,6 +41,11 @@ MSO_INFO = {
         'username_field': 'IDToken1',
         'password_field': 'IDToken2',
     },
+    'Verizon': {
+        'name': 'Verizon FiOS',
+        'username_field': 'IDToken1',
+        'password_field': 'IDToken2',
+    },
     'thr030': {
         'name': '3 Rivers Communications'
     },
@@ -1416,6 +1421,72 @@ class AdobePassIE(InfoExtractor):
                         if '<button class="submit" value="Resume">Resume</button>' in mvpd_confirm_page:
                             post_form(mvpd_confirm_page_res, 'Confirming Login')
 
+                elif mso_id == 'Verizon':
+                    # In general, if you're connecting from a Verizon-assigned IP,
+                    # you will not actually pass your credentials.
+                    provider_redirect_page, urlh = provider_redirect_page_res
+                    # print('provider_redirect_page_url', urlh.geturl())
+                    # https://signin.verizon.com/sso/VOLPortalLogin?SAMLRequest=<snip>
+                    if 'Please wait ...' in provider_redirect_page:
+                        # print('[debug] SAML Auto-Login is TRUE') # GO,
+                        saml_redirect_url = self._html_search_regex(
+                            r'self\.parent\.location=(["\'])(?P<url>.+?)\1',
+                            provider_redirect_page, 'Extracting SAML Redirect URL', group='url'
+                        )
+                        # print('saml_redirect_url', saml_redirect_url)
+                        # https://signin.verizon.com/sso/choice/tvpHandler.jsp?loginType=vzRedirect&partner=<snip>
+                        saml_login_page = self._download_webpage(
+                            saml_redirect_url, video_id, 'Downloading SAML Login Page'
+                        )
+                    else:
+                        if 'Please try again.' in provider_redirect_page:
+                            # print('[debug] SAML Auto-Login is FALSE') # NBC,
+                            provider_login_page_res = provider_redirect_page_res
+                        else:
+                            print('[error] SAML Auto-Login is UNKNOWN')
+                            import sys
+                            sys.exit(1)
+                        provider_login_page, urlh = provider_login_page_res
+                        # print('provider_login_page_url', urlh.geturl())
+                        # https://signin.verizon.com/sso/VOLPortalLogin?SAMLRequest=<snip>
+                        saml_login_page_res = post_form(provider_login_page_res, 'Logging in', {
+                            mso_info.get('username_field', 'username'): username,
+                            mso_info.get('password_field', 'password'): password,
+                        })
+                        saml_login_page, urlh = saml_login_page_res
+                        if 'Please try again.' in saml_login_page:
+                            print("[error] We're sorry, but either the User ID or Password entered is not correct.")
+                            import sys
+                            sys.exit(1)
+                        # elif 'Please wait - we are verifying your account...' in saml_login_page:
+                        # print('saml_login_page_url', urlh.geturl())
+                        # https://signin.verizon.com/sso/choice/tvpHandler.jsp?loginType=vzRedirect&partner=<snip>
+                    saml_login_url = self._html_search_regex(
+                        r'xmlHttp\.open\(\"POST\",\s(["\'])(?P<url>.+?)\1',
+                        saml_login_page, 'Extracting SAML Login URL', group='url')
+                    # print('saml_login_url', saml_login_url)
+                    # https://signin.verizon.com/sso/TVPHandlerServlet?loginType=vzRedirect&partner=<snip>
+                    saml_response_json = self._download_json(
+                        saml_login_url, video_id, 'Downloading SAML Response',
+                        headers={'Content-Type': 'text/xml'}
+                    )
+                    saml_target_url = saml_response_json['targetValue']
+                    saml_response = saml_response_json['SAMLResponse']
+                    saml_relay_state = saml_response_json['RelayState']
+                    # print('saml_target_url', saml_target_url)
+                    # https://sp.auth.adobe.com/sp/saml/SAMLAssertionConsumer
+                    form_data = {
+                        'SAMLResponse': saml_response,
+                        'RelayState': saml_relay_state
+                    }
+                    saml_autologin_res = self._download_webpage_handle(
+                        saml_target_url, video_id,
+                        'Auto-Login', data=urlencode_postdata(form_data),
+                        headers={'Content-Type': 'application/x-www-form-urlencoded'}
+                    )
+                    original_page, urlh = saml_autologin_res
+                    # print('original_page_url', urlh.geturl())
+
                 else:
                     # Normal, non-Comcast flow
                     provider_login_page_res = post_form(

From 79a51069e5c08afe7bf15bca7fcdf44fd0d93eb6 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 12 Apr 2017 15:17:18 +0100
Subject: [PATCH 084/611] [adobepass] improve comcast and verison login
 code(closes #10803)

---
 youtube_dl/extractor/adobepass.py | 98 ++++++++++---------------------
 1 file changed, 32 insertions(+), 66 deletions(-)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index a15843315..100cf997f 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -1389,106 +1389,72 @@ class AdobePassIE(InfoExtractor):
                     # Comcast page flow varies by video site and whether you
                     # are on Comcast's network.
                     provider_redirect_page, urlh = provider_redirect_page_res
-                    # Check for Comcast auto login
                     if 'automatically signing you in' in provider_redirect_page:
                         oauth_redirect_url = self._html_search_regex(
                             r'window\.location\s*=\s*[\'"]([^\'"]+)',
                             provider_redirect_page, 'oauth redirect')
-                        # Just need to process the request. No useful data comes back
                         self._download_webpage(
                             oauth_redirect_url, video_id, 'Confirming auto login')
                     else:
                         if '<form name="signin"' in provider_redirect_page:
-                            # already have the form, just fill it
                             provider_login_page_res = provider_redirect_page_res
                         elif 'http-equiv="refresh"' in provider_redirect_page:
-                            # redirects to the login page
                             oauth_redirect_url = self._html_search_regex(
                                 r'content="0;\s*url=([^\'"]+)',
                                 provider_redirect_page, 'meta refresh redirect')
                             provider_login_page_res = self._download_webpage_handle(
-                                oauth_redirect_url,
-                                video_id, 'Downloading Provider Login Page')
+                                oauth_redirect_url, video_id,
+                                'Downloading Provider Login Page')
                         else:
                             provider_login_page_res = post_form(
-                                provider_redirect_page_res, 'Downloading Provider Login Page')
+                                provider_redirect_page_res,
+                                'Downloading Provider Login Page')
 
-                        mvpd_confirm_page_res = post_form(provider_login_page_res, 'Logging in', {
-                            mso_info.get('username_field', 'username'): username,
-                            mso_info.get('password_field', 'password'): password,
-                        })
+                        mvpd_confirm_page_res = post_form(
+                            provider_login_page_res, 'Logging in', {
+                                mso_info['username_field']: username,
+                                mso_info['password_field']: password,
+                            })
                         mvpd_confirm_page, urlh = mvpd_confirm_page_res
                         if '<button class="submit" value="Resume">Resume</button>' in mvpd_confirm_page:
                             post_form(mvpd_confirm_page_res, 'Confirming Login')
-
                 elif mso_id == 'Verizon':
                     # In general, if you're connecting from a Verizon-assigned IP,
                     # you will not actually pass your credentials.
                     provider_redirect_page, urlh = provider_redirect_page_res
-                    # print('provider_redirect_page_url', urlh.geturl())
-                    # https://signin.verizon.com/sso/VOLPortalLogin?SAMLRequest=<snip>
                     if 'Please wait ...' in provider_redirect_page:
-                        # print('[debug] SAML Auto-Login is TRUE') # GO,
                         saml_redirect_url = self._html_search_regex(
                             r'self\.parent\.location=(["\'])(?P<url>.+?)\1',
-                            provider_redirect_page, 'Extracting SAML Redirect URL', group='url'
-                        )
-                        # print('saml_redirect_url', saml_redirect_url)
-                        # https://signin.verizon.com/sso/choice/tvpHandler.jsp?loginType=vzRedirect&partner=<snip>
+                            provider_redirect_page,
+                            'SAML Redirect URL', group='url')
                         saml_login_page = self._download_webpage(
-                            saml_redirect_url, video_id, 'Downloading SAML Login Page'
-                        )
+                            saml_redirect_url, video_id,
+                            'Downloading SAML Login Page')
                     else:
-                        if 'Please try again.' in provider_redirect_page:
-                            # print('[debug] SAML Auto-Login is FALSE') # NBC,
-                            provider_login_page_res = provider_redirect_page_res
-                        else:
-                            print('[error] SAML Auto-Login is UNKNOWN')
-                            import sys
-                            sys.exit(1)
-                        provider_login_page, urlh = provider_login_page_res
-                        # print('provider_login_page_url', urlh.geturl())
-                        # https://signin.verizon.com/sso/VOLPortalLogin?SAMLRequest=<snip>
-                        saml_login_page_res = post_form(provider_login_page_res, 'Logging in', {
-                            mso_info.get('username_field', 'username'): username,
-                            mso_info.get('password_field', 'password'): password,
-                        })
+                        saml_login_page_res = post_form(
+                            provider_redirect_page_res, 'Logging in', {
+                                mso_info['username_field']: username,
+                                mso_info['password_field']: password,
+                            })
                         saml_login_page, urlh = saml_login_page_res
                         if 'Please try again.' in saml_login_page:
-                            print("[error] We're sorry, but either the User ID or Password entered is not correct.")
-                            import sys
-                            sys.exit(1)
-                        # elif 'Please wait - we are verifying your account...' in saml_login_page:
-                        # print('saml_login_page_url', urlh.geturl())
-                        # https://signin.verizon.com/sso/choice/tvpHandler.jsp?loginType=vzRedirect&partner=<snip>
-                    saml_login_url = self._html_search_regex(
-                        r'xmlHttp\.open\(\"POST\",\s(["\'])(?P<url>.+?)\1',
-                        saml_login_page, 'Extracting SAML Login URL', group='url')
-                    # print('saml_login_url', saml_login_url)
-                    # https://signin.verizon.com/sso/TVPHandlerServlet?loginType=vzRedirect&partner=<snip>
+                            raise ExtractorError(
+                                'We\'re sorry, but either the User ID or Password entered is not correct.')
+                    saml_login_url = self._search_regex(
+                        r'xmlHttp\.open\("POST"\s*,\s*(["\'])(?P<url>.+?)\1',
+                        saml_login_page, 'SAML Login URL', group='url')
                     saml_response_json = self._download_json(
                         saml_login_url, video_id, 'Downloading SAML Response',
-                        headers={'Content-Type': 'text/xml'}
-                    )
-                    saml_target_url = saml_response_json['targetValue']
-                    saml_response = saml_response_json['SAMLResponse']
-                    saml_relay_state = saml_response_json['RelayState']
-                    # print('saml_target_url', saml_target_url)
-                    # https://sp.auth.adobe.com/sp/saml/SAMLAssertionConsumer
-                    form_data = {
-                        'SAMLResponse': saml_response,
-                        'RelayState': saml_relay_state
-                    }
-                    saml_autologin_res = self._download_webpage_handle(
-                        saml_target_url, video_id,
-                        'Auto-Login', data=urlencode_postdata(form_data),
-                        headers={'Content-Type': 'application/x-www-form-urlencoded'}
-                    )
-                    original_page, urlh = saml_autologin_res
-                    # print('original_page_url', urlh.geturl())
-
+                        headers={'Content-Type': 'text/xml'})
+                    self._download_webpage(
+                        saml_response_json['targetValue'], video_id,
+                        'Confirming Login', data=urlencode_postdata({
+                            'SAMLResponse': saml_response_json['SAMLResponse'],
+                            'RelayState': saml_response_json['RelayState']
+                        }), headers={
+                            'Content-Type': 'application/x-www-form-urlencoded'
+                        })
                 else:
-                    # Normal, non-Comcast flow
                     provider_login_page_res = post_form(
                         provider_redirect_page_res, 'Downloading Provider Login Page')
                     mvpd_confirm_page_res = post_form(provider_login_page_res, 'Logging in', {

From 0ee79a378ad9536cfb580187c9a30a0463ad5d8b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 12 Apr 2017 16:22:08 +0100
Subject: [PATCH 085/611] [generic] exclude urls with xml ext from valid video
 urls(closes #10768)(closes #11654)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 4690816cb..36d23d2f3 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2611,7 +2611,7 @@ class GenericIE(InfoExtractor):
                 return True
             vpath = compat_urlparse.urlparse(vurl).path
             vext = determine_ext(vpath)
-            return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml', 'js')
+            return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml', 'js', 'xml')
 
         def filter_video(urls):
             return list(filter(check_video, urls))

From 3962260b7d398d75ce456dbb08a85ae8fcecfd28 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 12 Apr 2017 16:54:53 +0100
Subject: [PATCH 086/611] Credit @SkiTheSlicer for Verizon adobe pass support
 (#12721)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 2d676b210..9abd5fdf4 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -210,3 +210,4 @@ Lars Vierbergen
 Juanjo Benages
 Xiao Di Guan
 Thomas Winant
+Daniel Twardowski

From 3ef1d0c73373000bf65851c7afe9c68e707e4a97 Mon Sep 17 00:00:00 2001
From: Vukkk <Vukkk@github.com>
Date: Wed, 31 Aug 2016 10:56:11 +0200
Subject: [PATCH 087/611] [tv2hu] Add new extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/tv2hu.py      | 78 ++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)
 create mode 100644 youtube_dl/extractor/tv2hu.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 68e1a5cfc..7484099f6 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1031,6 +1031,7 @@ from .tv2 import (
     TV2IE,
     TV2ArticleIE,
 )
+from .tv2hu import TV2HUIE
 from .tv3 import TV3IE
 from .tv4 import TV4IE
 from .tv5mondeplus import TV5MondePlusIE
diff --git a/youtube_dl/extractor/tv2hu.py b/youtube_dl/extractor/tv2hu.py
new file mode 100644
index 000000000..cb1ce822a
--- /dev/null
+++ b/youtube_dl/extractor/tv2hu.py
@@ -0,0 +1,78 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+class TV2HUIE(InfoExtractor):
+    IE_NAME = 'tv2.hu'
+    _VALID_URL = r'https?://(?:www\.)?tv2\.hu/(?:musoraink/)?(?P<uploader>[^/]+)/(?:teljes_adasok/)?(?P<id>[0-9]+)_(.+?)\.html'
+    _JSON_URL = r'(?P<json_url>https?://.+?\.tv2\.hu/vod/(?P<upload_date>\d+)/id_(?P<upload_id>\d+).+?&type=json)'
+
+    _TESTS = [{
+        'url': 'http://tv2.hu/ezek_megorultek/217679_ezek-megorultek---1.-adas-1.-resz.html',
+        'info_dict': {
+            'id': '217679',
+            'ext': 'mp4',
+            'title': 'Ezek megőrültek! - 1. adás 1. rész',
+            'upload_id': '220289',
+            'upload_date': '20160826',
+            'uploader': 'ezek_megorultek',
+            'thumbnail': 're:^https?://.*\.jpg$'
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://tv2.hu/ezek_megorultek/teljes_adasok/217677_ezek-megorultek---1.-adas-2.-resz.html',
+        'only_matching': True
+    }, {
+        'url': 'http://tv2.hu/musoraink/aktiv/aktiv_teljes_adas/217963_aktiv-teljes-adas---2016.08.30..html',
+        'only_matching': True
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, video_id, 'Downloading info page')
+
+        json_url = re.search(self._JSON_URL, webpage)
+
+        json_data = self._download_json(
+            json_url.group('json_url'), video_id, 'Downloading video info')
+
+        manifest_url = json_data['bitrates']['hls']
+
+        formats = self._extract_m3u8_formats(
+            manifest_url, video_id, 'mp4', entry_protocol='m3u8_native')
+
+        for i in range(len(json_data['bitrates']['mp4'])):
+            quality = json_data.get('mp4Labels')[i]
+
+            if quality.lower() == 'auto':
+                continue
+
+            formats.append({
+                'protocol': 'http',
+                'url': json_data['bitrates']['mp4'][i],
+                'height': int(quality[:-1]),
+                'width': int(quality[:-1])/9*16,
+                'ext': 'mp4',
+                'format_id': quality,
+                'format_note': 'HTTP',
+                'preference': int(quality[:-1])
+            })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage).strip(),
+            'thumbnail': self._og_search_property('image', webpage),
+            'uploader': self._search_regex(self._VALID_URL, url, 'uploader'),
+            'upload_id': json_url.group('upload_id'),
+            'upload_date': json_url.group('upload_date'),
+            'formats': formats
+        }

From e4d74e2778ca283330460d893a0923820a74df66 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 12 Apr 2017 19:31:18 +0100
Subject: [PATCH 088/611] [tv2hu] improve extraction

---
 youtube_dl/extractor/extractors.py |  2 +-
 youtube_dl/extractor/tv2hu.py      | 74 ++++++++++++------------------
 2 files changed, 30 insertions(+), 46 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 7484099f6..45591f2a7 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1031,7 +1031,7 @@ from .tv2 import (
     TV2IE,
     TV2ArticleIE,
 )
-from .tv2hu import TV2HUIE
+from .tv2hu import TV2HuIE
 from .tv3 import TV3IE
 from .tv4 import TV4IE
 from .tv5mondeplus import TV5MondePlusIE
diff --git a/youtube_dl/extractor/tv2hu.py b/youtube_dl/extractor/tv2hu.py
index cb1ce822a..ce01cc80d 100644
--- a/youtube_dl/extractor/tv2hu.py
+++ b/youtube_dl/extractor/tv2hu.py
@@ -1,29 +1,22 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..utils import int_or_none
 
-class TV2HUIE(InfoExtractor):
+
+class TV2HuIE(InfoExtractor):
     IE_NAME = 'tv2.hu'
-    _VALID_URL = r'https?://(?:www\.)?tv2\.hu/(?:musoraink/)?(?P<uploader>[^/]+)/(?:teljes_adasok/)?(?P<id>[0-9]+)_(.+?)\.html'
-    _JSON_URL = r'(?P<json_url>https?://.+?\.tv2\.hu/vod/(?P<upload_date>\d+)/id_(?P<upload_id>\d+).+?&type=json)'
-
+    _VALID_URL = r'https?://(?:www\.)?tv2\.hu/(?:[^/]+/)+(?P<id>\d+)_[^/?#]+?\.html'
     _TESTS = [{
         'url': 'http://tv2.hu/ezek_megorultek/217679_ezek-megorultek---1.-adas-1.-resz.html',
+        'md5': '585e58e2e090f34603804bb2c48e98d8',
         'info_dict': {
             'id': '217679',
             'ext': 'mp4',
             'title': 'Ezek megőrültek! - 1. adás 1. rész',
-            'upload_id': '220289',
             'upload_date': '20160826',
-            'uploader': 'ezek_megorultek',
             'thumbnail': 're:^https?://.*\.jpg$'
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
         }
     }, {
         'url': 'http://tv2.hu/ezek_megorultek/teljes_adasok/217677_ezek-megorultek---1.-adas-2.-resz.html',
@@ -35,44 +28,35 @@ class TV2HUIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(
-            url, video_id, 'Downloading info page')
+        webpage = self._download_webpage(url, video_id)
+        json_url = self._search_regex(
+            r'jsonUrl\s*=\s*"([^"]+)"', webpage, 'json url')
+        json_data = self._download_json(json_url, video_id)
 
-        json_url = re.search(self._JSON_URL, webpage)
-
-        json_data = self._download_json(
-            json_url.group('json_url'), video_id, 'Downloading video info')
-
-        manifest_url = json_data['bitrates']['hls']
-
-        formats = self._extract_m3u8_formats(
-            manifest_url, video_id, 'mp4', entry_protocol='m3u8_native')
-
-        for i in range(len(json_data['bitrates']['mp4'])):
-            quality = json_data.get('mp4Labels')[i]
-
-            if quality.lower() == 'auto':
-                continue
-
-            formats.append({
-                'protocol': 'http',
-                'url': json_data['bitrates']['mp4'][i],
-                'height': int(quality[:-1]),
-                'width': int(quality[:-1])/9*16,
-                'ext': 'mp4',
-                'format_id': quality,
-                'format_note': 'HTTP',
-                'preference': int(quality[:-1])
-            })
+        formats = []
+        for b in ('bitrates', 'backupBitrates'):
+            bitrates = json_data.get(b, {})
+            m3u8_url = bitrates.get('hls')
+            if m3u8_url:
+                formats.extend(self._extract_wowza_formats(
+                    m3u8_url, video_id, skip_protocols=['rtmp', 'rtsp']))
 
+            for mp4_url in bitrates.get('mp4', []):
+                height = int_or_none(self._search_regex(
+                    r'\.(\d+)p\.mp4', mp4_url, 'height', default=None))
+                formats.append({
+                    'format_id': 'http' + ('-%d' % height if height else ''),
+                    'url': mp4_url,
+                    'height': height,
+                    'width': int_or_none(height / 9.0 * 16.0 if height else None),
+                })
         self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': self._og_search_title(webpage).strip(),
-            'thumbnail': self._og_search_property('image', webpage),
-            'uploader': self._search_regex(self._VALID_URL, url, 'uploader'),
-            'upload_id': json_url.group('upload_id'),
-            'upload_date': json_url.group('upload_date'),
-            'formats': formats
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'upload_date': self._search_regex(
+                r'/vod/(\d{8})/', json_url, 'upload_date', default=None),
+            'formats': formats,
         }

From 40fcba5edb0f54f09e33a193a0ffefb5668ca694 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 12 Apr 2017 20:38:43 +0100
Subject: [PATCH 089/611] improve coding style

---
 youtube_dl/YoutubeDL.py                       | 2 +-
 youtube_dl/compat.py                          | 2 +-
 youtube_dl/downloader/rtmp.py                 | 2 +-
 youtube_dl/extractor/bbc.py                   | 4 ++--
 youtube_dl/extractor/common.py                | 2 +-
 youtube_dl/extractor/rudo.py                  | 2 +-
 youtube_dl/extractor/viewlift.py              | 2 +-
 youtube_dl/extractor/vlive.py                 | 4 ++--
 youtube_dl/postprocessor/ffmpeg.py            | 2 +-
 youtube_dl/postprocessor/metadatafromtitle.py | 2 +-
 10 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 54bc8b06d..7953670a7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -855,7 +855,7 @@ class YoutubeDL(object):
 
             return self.process_ie_result(
                 new_result, download=download, extra_info=extra_info)
-        elif result_type == 'playlist' or result_type == 'multi_video':
+        elif result_type in ('playlist', 'multi_video'):
             # We process each entry in the playlist
             playlist = ie_result.get('title') or ie_result.get('id')
             self.to_screen('[download] Downloading playlist: %s' % playlist)
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 0c119e417..39527117f 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2692,7 +2692,7 @@ else:
                 userhome = pwent.pw_dir
             userhome = userhome.rstrip('/')
             return (userhome + path[i:]) or '/'
-    elif compat_os_name == 'nt' or compat_os_name == 'ce':
+    elif compat_os_name in ('nt', 'ce'):
         def compat_expanduser(path):
             """Expand ~ and ~user constructs.
 
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 9de6e70bb..b823b5171 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -169,7 +169,7 @@ class RtmpFD(FileDownloader):
             self.report_error('[rtmpdump] Could not connect to RTMP server.')
             return False
 
-        while (retval == RD_INCOMPLETE or retval == RD_FAILED) and not test and not live:
+        while retval in (RD_INCOMPLETE, RD_FAILED) and not test and not live:
             prevsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen('[rtmpdump] %s bytes' % prevsize)
             time.sleep(5.0)  # This seems to be needed
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 0e05b782b..dd65b8d86 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -409,7 +409,7 @@ class BBCCoUkIE(InfoExtractor):
                 description = smp_config['summary']
                 for item in smp_config['items']:
                     kind = item['kind']
-                    if kind != 'programme' and kind != 'radioProgramme':
+                    if kind not in ('programme', 'radioProgramme'):
                         continue
                     programme_id = item.get('vpid')
                     duration = int_or_none(item.get('duration'))
@@ -450,7 +450,7 @@ class BBCCoUkIE(InfoExtractor):
 
         for item in self._extract_items(playlist):
             kind = item.get('kind')
-            if kind != 'programme' and kind != 'radioProgramme':
+            if kind not in ('programme', 'radioProgramme'):
                 continue
             title = playlist.find('./{%s}title' % self._EMP_PLAYLIST_NS).text
             description_el = playlist.find('./{%s}summary' % self._EMP_PLAYLIST_NS)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index ae8af61de..dcc9d628a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1779,7 +1779,7 @@ class InfoExtractor(object):
                     if content_type == 'text':
                         # TODO implement WebVTT downloading
                         pass
-                    elif content_type == 'video' or content_type == 'audio':
+                    elif content_type in ('video', 'audio'):
                         base_url = ''
                         for element in (representation, adaptation_set, period, mpd_doc):
                             base_url_e = element.find(_add_ns('BaseURL'))
diff --git a/youtube_dl/extractor/rudo.py b/youtube_dl/extractor/rudo.py
index 51644011e..f036f6757 100644
--- a/youtube_dl/extractor/rudo.py
+++ b/youtube_dl/extractor/rudo.py
@@ -26,7 +26,7 @@ class RudoIE(InfoExtractor):
     }
 
     @classmethod
-    def _extract_url(self, webpage):
+    def _extract_url(cls, webpage):
         mobj = re.search(
             r'<iframe[^>]+src=(?P<q1>[\'"])(?P<url>(?:https?:)?//rudo\.video/vod/[0-9a-zA-Z]+)(?P=q1)',
             webpage)
diff --git a/youtube_dl/extractor/viewlift.py b/youtube_dl/extractor/viewlift.py
index 18735cfb2..1f29c273f 100644
--- a/youtube_dl/extractor/viewlift.py
+++ b/youtube_dl/extractor/viewlift.py
@@ -68,7 +68,7 @@ class ViewLiftEmbedIE(ViewLiftBaseIE):
             type_ = source.get('type')
             ext = determine_ext(file_)
             format_id = source.get('label') or ext
-            if all(v == 'm3u8' or v == 'hls' for v in (type_, ext)):
+            if all(v in ('m3u8', 'hls') for v in (type_, ext)):
                 formats.extend(self._extract_m3u8_formats(
                     file_, video_id, 'mp4', m3u8_id='hls'))
             else:
diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index b9718901b..e58940607 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -70,9 +70,9 @@ class VLiveIE(InfoExtractor):
         status, long_video_id, key = params[2], params[5], params[6]
         status = remove_start(status, 'PRODUCT_')
 
-        if status == 'LIVE_ON_AIR' or status == 'BIG_EVENT_ON_AIR':
+        if status in ('LIVE_ON_AIR', 'BIG_EVENT_ON_AIR'):
             return self._live(video_id, webpage)
-        elif status == 'VOD_ON_AIR' or status == 'BIG_EVENT_INTRO':
+        elif status in ('VOD_ON_AIR', 'BIG_EVENT_INTRO'):
             if long_video_id and key:
                 return self._replay(video_id, webpage, long_video_id, key)
             else:
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 7c162d92a..665109558 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -552,7 +552,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
             sub_filenames.append(old_file)
             new_file = subtitles_filename(filename, lang, new_ext)
 
-            if ext == 'dfxp' or ext == 'ttml' or ext == 'tt':
+            if ext in ('dfxp', 'ttml', 'tt'):
                 self._downloader.report_warning(
                     'You have requested to convert dfxp (TTML) subtitles into another format, '
                     'which results in style information loss')
diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dl/postprocessor/metadatafromtitle.py
index 164edd3a8..a7d637a3c 100644
--- a/youtube_dl/postprocessor/metadatafromtitle.py
+++ b/youtube_dl/postprocessor/metadatafromtitle.py
@@ -26,7 +26,7 @@ class MetadataFromTitlePP(PostProcessor):
             regex += r'(?P<' + match.group(1) + '>.+)'
             lastpos = match.end()
         if lastpos < len(fmt):
-            regex += re.escape(fmt[lastpos:len(fmt)])
+            regex += re.escape(fmt[lastpos:])
         return regex
 
     def run(self, info):

From 34d98cc411d379cea02a67ba7fc9f52e74daff23 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 13 Apr 2017 16:52:52 +0800
Subject: [PATCH 090/611] [tv2hu] Fix invalid escape sequence on Py3.6

---
 youtube_dl/extractor/tv2hu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tv2hu.py b/youtube_dl/extractor/tv2hu.py
index ce01cc80d..86017b757 100644
--- a/youtube_dl/extractor/tv2hu.py
+++ b/youtube_dl/extractor/tv2hu.py
@@ -16,7 +16,7 @@ class TV2HuIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Ezek megőrültek! - 1. adás 1. rész',
             'upload_date': '20160826',
-            'thumbnail': 're:^https?://.*\.jpg$'
+            'thumbnail': r're:^https?://.*\.jpg$'
         }
     }, {
         'url': 'http://tv2.hu/ezek_megorultek/teljes_adasok/217677_ezek-megorultek---1.-adas-2.-resz.html',

From f59746372a1aae3bc80462d34a7e96200be22ab5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 13 Apr 2017 12:21:17 +0100
Subject: [PATCH 091/611] [downloader/hls] add basic support for
 EXT-X-BYTERANGE tag(#10955)

---
 youtube_dl/downloader/hls.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 4989abce1..d0a5f7ba4 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -34,7 +34,7 @@ class HlsFD(FragmentFD):
     def can_download(manifest, info_dict):
         UNSUPPORTED_FEATURES = (
             r'#EXT-X-KEY:METHOD=(?!NONE|AES-128)',  # encrypted streams [1]
-            r'#EXT-X-BYTERANGE',  # playlists composed of byte ranges of media files [2]
+            # r'#EXT-X-BYTERANGE',  # playlists composed of byte ranges of media files [2]
 
             # Live streams heuristic does not always work (e.g. geo restricted to Germany
             # http://hls-geo.daserste.de/i/videoportal/Film/c_620000/622873/format,716451,716457,716450,716458,716459,.mp4.csmil/index_4_av.m3u8?null=0)
@@ -52,7 +52,9 @@ class HlsFD(FragmentFD):
             # 4. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.5
         )
         check_results = [not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES]
-        check_results.append(can_decrypt_frag or '#EXT-X-KEY:METHOD=AES-128' not in manifest)
+        is_aes128_enc = '#EXT-X-KEY:METHOD=AES-128' in manifest
+        check_results.append(can_decrypt_frag or not is_aes128_enc)
+        check_results.append(not (is_aes128_enc and r'#EXT-X-BYTERANGE' in manifest))
         check_results.append(not info_dict.get('is_live'))
         return all(check_results)
 
@@ -100,6 +102,7 @@ class HlsFD(FragmentFD):
         i = 0
         media_sequence = 0
         decrypt_info = {'METHOD': 'NONE'}
+        byte_range = {}
         frags_filenames = []
         for line in s.splitlines():
             line = line.strip()
@@ -114,11 +117,14 @@ class HlsFD(FragmentFD):
                     if extra_query:
                         frag_url = update_url_query(frag_url, extra_query)
                     count = 0
+                    headers = info_dict.get('http_headers', {})
+                    if byte_range:
+                        headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end'])
                     while count <= fragment_retries:
                         try:
                             success = ctx['dl'].download(frag_filename, {
                                 'url': frag_url,
-                                'http_headers': info_dict.get('http_headers'),
+                                'http_headers': headers,
                             })
                             if not success:
                                 return False
@@ -167,6 +173,13 @@ class HlsFD(FragmentFD):
                         decrypt_info['KEY'] = self.ydl.urlopen(decrypt_info['URI']).read()
                 elif line.startswith('#EXT-X-MEDIA-SEQUENCE'):
                     media_sequence = int(line[22:])
+                elif line.startswith('#EXT-X-BYTERANGE'):
+                    splitted_byte_range = line[17:].split('@')
+                    sub_range_start = int(splitted_byte_range[1]) if len(splitted_byte_range) == 2 else byte_range['end']
+                    byte_range = {
+                        'start': sub_range_start,
+                        'end': sub_range_start + int(splitted_byte_range[0]),
+                    }
 
         self._finish_frag_download(ctx)
 

From b2fff30817848c1761a53d49fbe49aabd596089e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 13 Apr 2017 12:27:28 +0100
Subject: [PATCH 092/611] [go90] Add new extractor(closes #10127)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/go90.py       | 92 ++++++++++++++++++++++++++++++
 2 files changed, 93 insertions(+)
 create mode 100644 youtube_dl/extractor/go90.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 45591f2a7..1671090f4 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -384,6 +384,7 @@ from .globo import (
     GloboArticleIE,
 )
 from .go import GoIE
+from .go90 import Go90IE
 from .godtube import GodTubeIE
 from .godtv import GodTVIE
 from .golem import GolemIE
diff --git a/youtube_dl/extractor/go90.py b/youtube_dl/extractor/go90.py
new file mode 100644
index 000000000..3550eca7c
--- /dev/null
+++ b/youtube_dl/extractor/go90.py
@@ -0,0 +1,92 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+)
+
+
+class Go90IE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?go90\.com/videos/(?P<id>[0-9a-zA-Z]+)'
+    _TEST = {
+        'url': 'https://www.go90.com/videos/84BUqjLpf9D',
+        'md5': 'efa7670dbbbf21a7b07b360652b24a32',
+        'info_dict': {
+            'id': '84BUqjLpf9D',
+            'ext': 'mp4',
+            'title': 'Inside The Utah Coalition Against Pornography Convention',
+            'description': 'VICE\'s Karley Sciortino meets with activists who discuss the state\'s strong anti-porn stance. Then, VICE Sports explains NFL contracts.',
+            'timestamp': 1491868800,
+            'upload_date': '20170411',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_data = self._download_json(
+            'https://www.go90.com/api/view/items/' + video_id,
+            video_id, headers={
+                'Content-Type': 'application/json; charset=utf-8',
+            }, data=b'{"client":"web","device_type":"pc"}')
+        title = video_data['title']
+        main_video_asset = video_data['main_video_asset']
+
+        thumbnails = []
+        formats = []
+        for asset in video_data.get('assets'):
+            if asset.get('id') == main_video_asset:
+                for source in asset.get('sources', []):
+                    source_location = source.get('location')
+                    if not source_location:
+                        continue
+                    source_type = source.get('type')
+                    if source_type == 'hls':
+                        m3u8_formats = self._extract_m3u8_formats(
+                            source_location, video_id, 'mp4',
+                            'm3u8_native', m3u8_id='hls', fatal=False)
+                        for f in m3u8_formats:
+                            mobj = re.search(r'/hls-(\d+)-(\d+)K', f['url'])
+                            if mobj:
+                                height, tbr = mobj.groups()
+                                height = int_or_none(height)
+                                f.update({
+                                    'height': f.get('height') or height,
+                                    'width': f.get('width') or int_or_none(height / 9.0 * 16.0 if height else None),
+                                    'tbr': f.get('tbr') or int_or_none(tbr),
+                                })
+                        formats.extend(m3u8_formats)
+                    elif source_type == 'dash':
+                        formats.extend(self._extract_mpd_formats(
+                            source_location, video_id, mpd_id='dash', fatal=False))
+                    else:
+                        formats.append({
+                            'format_id': source.get('name'),
+                            'url': source_location,
+                            'width': int_or_none(source.get('width')),
+                            'height': int_or_none(source.get('height')),
+                            'tbr': int_or_none(source.get('bitrate')),
+                        })
+            elif asset.get('type') == 'image':
+                asset_location = asset.get('location')
+                if not asset_location:
+                    continue
+                thumbnails.append({
+                    'url': asset_location,
+                    'width': int_or_none(asset.get('width')),
+                    'height': int_or_none(asset.get('height')),
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnails': thumbnails,
+            'description': video_data.get('short_description'),
+            'like_count': int_or_none(video_data.get('like_count')),
+            'timestamp': parse_iso8601(video_data.get('released_at')),
+        }

From 3f7409f745df1dbde394643fe40d661bc109de74 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 13 Apr 2017 12:28:00 +0100
Subject: [PATCH 093/611] [hbo] extract m3u8 formats

---
 youtube_dl/extractor/hbo.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/hbo.py b/youtube_dl/extractor/hbo.py
index 931f71a5a..859ad5429 100644
--- a/youtube_dl/extractor/hbo.py
+++ b/youtube_dl/extractor/hbo.py
@@ -92,12 +92,14 @@ class HBOBaseIE(InfoExtractor):
                         video_url.replace('.tar', '/base_index_w8.m3u8'),
                         video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
                 elif source.tag == 'hls':
-                    # #EXT-X-BYTERANGE is not supported by native hls downloader
-                    # and ffmpeg (#10955)
-                    # formats.extend(self._extract_m3u8_formats(
-                    #     video_url.replace('.tar', '/base_index.m3u8'),
-                    #     video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
-                    continue
+                    m3u8_formats = self._extract_m3u8_formats(
+                        video_url.replace('.tar', '/base_index.m3u8'),
+                        video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+                    for f in m3u8_formats:
+                        if f.get('vcodec') == 'none' and not f.get('tbr'):
+                            f['tbr'] = int_or_none(self._search_regex(
+                                r'-(\d+)k/', f['url'], 'tbr', default=None))
+                    formats.extend(m3u8_formats)
                 elif source.tag == 'dash':
                     formats.extend(self._extract_mpd_formats(
                         video_url.replace('.tar', '/manifest.mpd'),
@@ -110,7 +112,7 @@ class HBOBaseIE(InfoExtractor):
                         'width': format_info.get('width'),
                         'height': format_info.get('height'),
                     })
-        self._sort_formats(formats, ('width', 'height', 'tbr', 'format_id'))
+        self._sort_formats(formats)
 
         thumbnails = []
         card_sizes = xpath_element(video_data, 'titleCardSizes')

From 95728fda7059265374e92d92004bd75e0beb630d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Apr 2017 23:39:55 +0700
Subject: [PATCH 094/611] [aenetworks] Add support for specials (closes #12723)

---
 youtube_dl/extractor/aenetworks.py | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index 9a5074037..c01c67303 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -23,7 +23,19 @@ class AENetworksBaseIE(ThePlatformIE):
 class AENetworksIE(AENetworksBaseIE):
     IE_NAME = 'aenetworks'
     IE_DESC = 'A+E Networks: A&E, Lifetime, History.com, FYI Network'
-    _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:history|aetv|mylifetime|lifetimemovieclub)\.com|fyi\.tv)/(?:shows/(?P<show_path>[^/]+(?:/[^/]+){0,2})|movies/(?P<movie_display_id>[^/]+)(?:/full-movie)?)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?
+                        (?P<domain>
+                            (?:history|aetv|mylifetime|lifetimemovieclub)\.com|
+                            fyi\.tv
+                        )/
+                        (?:
+                            shows/(?P<show_path>[^/]+(?:/[^/]+){0,2})|
+                            movies/(?P<movie_display_id>[^/]+)(?:/full-movie)?|
+                            specials/(?P<special_display_id>[^/]+)/full-special
+                        )
+                    '''
     _TESTS = [{
         'url': 'http://www.history.com/shows/mountain-men/season-1/episode-1',
         'md5': 'a97a65f7e823ae10e9244bc5433d5fe6',
@@ -65,6 +77,9 @@ class AENetworksIE(AENetworksBaseIE):
     }, {
         'url': 'https://www.lifetimemovieclub.com/movies/a-killer-among-us',
         'only_matching': True
+    }, {
+        'url': 'http://www.history.com/specials/sniper-into-the-kill-zone/full-special',
+        'only_matching': True
     }]
     _DOMAIN_TO_REQUESTOR_ID = {
         'history.com': 'HISTORY',
@@ -75,8 +90,8 @@ class AENetworksIE(AENetworksBaseIE):
     }
 
     def _real_extract(self, url):
-        domain, show_path, movie_display_id = re.match(self._VALID_URL, url).groups()
-        display_id = show_path or movie_display_id
+        domain, show_path, movie_display_id, special_display_id = re.match(self._VALID_URL, url).groups()
+        display_id = show_path or movie_display_id or special_display_id
         webpage = self._download_webpage(url, display_id)
         if show_path:
             url_parts = show_path.split('/')

From 6f4dd6667b2d7ee4d3927090c4bc3e1ee6c85ce7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 14 Apr 2017 00:29:36 +0700
Subject: [PATCH 095/611] [ChangeLog] Actualize

---
 ChangeLog | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index f26372bc1..c61b0f5e5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,23 @@
+version <unreleased>
+
+Core
++ [downloader/hls] Add basic support for EXT-X-BYTERANGE tag (#10955)
++ [adobepass] Improve Comcast and Verison login code (#10803)
++ [adobepass] Add support for Verizon (#10803)
+
+Extractors
++ [aenetworks] Add support for specials (#12723)
++ [hbo] Extract HLS formats
++ [go90] Add support for go90.com (#10127)
++ [tv2hu] Add support for tv2.hu (#10509)
++ [generic] Exclude URLs with xml ext from valid video URLs (#10768, #11654)
+* [youtube] Improve HLS formats extraction
+* [afreecatv] Fix extraction for videos with different key layout (#12718)
+- [youtube] Remove explicit preference for audio-only and video-only formats in
+  order not to break sorting when new formats appear
+* [canalplus] Bypass geo restriction
+
+
 version 2017.04.11
 
 Extractors

From 38d2f8325f8d30cdb16c4ddea277772cbca6c8d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 14 Apr 2017 00:31:49 +0700
Subject: [PATCH 096/611] release 2017.04.14

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 61af50e5c..02e9481c1 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.11*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.11**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.14*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.14**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.04.11
+[debug] youtube-dl version 2017.04.14
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index c61b0f5e5..8d5553ead 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.04.14
 
 Core
 + [downloader/hls] Add basic support for EXT-X-BYTERANGE tag (#10955)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 0157ce3ef..b29b50c8d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -308,6 +308,7 @@
  - **Globo**
  - **GloboArticle**
  - **Go**
+ - **Go90**
  - **GodTube**
  - **GodTV**
  - **Golem**
@@ -818,6 +819,7 @@
  - **Tutv**
  - **tv.dfb.de**
  - **TV2**
+ - **tv2.hu**
  - **TV2Article**
  - **TV3**
  - **TV4**: tv4.se and tv4play.se
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5918c10bc..f64ad97bb 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.04.11'
+__version__ = '2017.04.14'

From e5106ba402031f619503b3cecb3405433258eece Mon Sep 17 00:00:00 2001
From: xuhaomin <xhmgreat@sina.com>
Date: Sat, 15 Apr 2017 02:23:02 +0800
Subject: [PATCH 097/611] [youku] Fix fileid extraction

---
 youtube_dl/extractor/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index e37f237c7..73f6f0e90 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -105,7 +105,7 @@ class YoukuIE(InfoExtractor):
             if stream.get('channel_type') == 'tail':
                 continue
             format = stream.get('stream_type')
-            fileid = stream['stream_fileid']
+            fileid = stream['segs'][0]['fileid']
             fileid_dict[format] = fileid
 
         def get_fileid(format, n):

From 34c3bbe1bd1def4e4407d3170cb583ffcb2674dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Apr 2017 01:25:32 +0700
Subject: [PATCH 098/611] [youku] Keep old fileid extraction code as fallback
 (#12741)

---
 youtube_dl/extractor/youku.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 73f6f0e90..73ebe5759 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -10,12 +10,14 @@ import time
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse_urlencode,
     compat_ord,
+    compat_str,
+    compat_urllib_parse_urlencode,
 )
 from ..utils import (
     ExtractorError,
     get_element_by_attribute,
+    try_get,
 )
 
 
@@ -105,7 +107,9 @@ class YoukuIE(InfoExtractor):
             if stream.get('channel_type') == 'tail':
                 continue
             format = stream.get('stream_type')
-            fileid = stream['segs'][0]['fileid']
+            fileid = try_get(
+                stream, lambda x: x['segs'][0]['fileid'],
+                compat_str) or stream['stream_fileid']
             fileid_dict[format] = fileid
 
         def get_fileid(format, n):

From b9859e2735e754f90136117e5ac9d0645b9bc359 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Apr 2017 01:37:26 +0700
Subject: [PATCH 099/611] [ChangeLog] Actualize

---
 ChangeLog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 8d5553ead..9f958dd91 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [youku] Fix fileid extraction (#12741, #12743)
+
+
 version 2017.04.14
 
 Core

From d24bd4ffdb64b6b2c0568bd5eb79920b053e95fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Apr 2017 01:39:33 +0700
Subject: [PATCH 100/611] release 2017.04.15

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 02e9481c1..70f6b51ed 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.14*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.14**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.15*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.15**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.04.14
+[debug] youtube-dl version 2017.04.15
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 9f958dd91..cf5ee84a4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.04.15
 
 Extractors
 * [youku] Fix fileid extraction (#12741, #12743)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f64ad97bb..612b50f7b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.04.14'
+__version__ = '2017.04.15'

From 6f1b2374af37b6a290080036edae9361d2e5a66d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Apr 2017 02:33:12 +0700
Subject: [PATCH 101/611] [canalplus] PEP 8

---
 youtube_dl/extractor/canalplus.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 90b454115..d8bf073f4 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -7,8 +7,8 @@ from .common import InfoExtractor
 from ..compat import compat_urllib_parse_urlparse
 from ..utils import (
     dict_get,
-    ExtractorError,
-    HEADRequest,
+    # ExtractorError,
+    # HEADRequest,
     int_or_none,
     qualities,
     remove_end,

From 57e2cb6c3ad05720cd8e4cd707c62f6fadc18be8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Apr 2017 02:33:26 +0700
Subject: [PATCH 102/611] [curiositystream] PEP 8

---
 youtube_dl/extractor/curiositystream.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/curiositystream.py b/youtube_dl/extractor/curiositystream.py
index 145d9397c..8e45923e3 100644
--- a/youtube_dl/extractor/curiositystream.py
+++ b/youtube_dl/extractor/curiositystream.py
@@ -48,7 +48,6 @@ class CuriosityStreamBaseIE(InfoExtractor):
 
     def _extract_media_info(self, media):
         video_id = compat_str(media['id'])
-        limelight_media_id = media['limelight_media_id']
         title = media['title']
 
         formats = []

From 3dfceb286c5ffc39a0025d3728448c6d572e022b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Apr 2017 03:26:19 +0700
Subject: [PATCH 103/611] [udemy] Add support for react rendition (closes
 #12744)

---
 youtube_dl/extractor/udemy.py | 79 +++++++++++++++++++++++++----------
 1 file changed, 58 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index cce29c6e0..dae1aa3c6 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -212,12 +212,15 @@ class UdemyIE(InfoExtractor):
         thumbnail = asset.get('thumbnail_url') or asset.get('thumbnailUrl')
         duration = float_or_none(asset.get('data', {}).get('duration'))
 
+        subtitles = {}
+        automatic_captions = {}
+
         formats = []
 
-        def extract_output_format(src):
+        def extract_output_format(src, f_id):
             return {
                 'url': src['url'],
-                'format_id': '%sp' % (src.get('height') or format_id),
+                'format_id': '%sp' % (src.get('height') or f_id),
                 'width': int_or_none(src.get('width')),
                 'height': int_or_none(src.get('height')),
                 'vbr': int_or_none(src.get('video_bitrate_in_kbps')),
@@ -237,30 +240,33 @@ class UdemyIE(InfoExtractor):
         def add_output_format_meta(f, key):
             output = outputs.get(key)
             if isinstance(output, dict):
-                output_format = extract_output_format(output)
+                output_format = extract_output_format(output, key)
                 output_format.update(f)
                 return output_format
             return f
 
+        def extract_formats(source_list):
+            if not isinstance(source_list, list):
+                return
+            for source in source_list:
+                video_url = source.get('file') or source.get('src')
+                if not video_url or not isinstance(video_url, compat_str):
+                    continue
+                format_id = source.get('label')
+                f = {
+                    'url': video_url,
+                    'format_id': '%sp' % format_id,
+                    'height': int_or_none(format_id),
+                }
+                if format_id:
+                    # Some videos contain additional metadata (e.g.
+                    # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
+                    f = add_output_format_meta(f, format_id)
+                formats.append(f)
+
         download_urls = asset.get('download_urls')
         if isinstance(download_urls, dict):
-            video = download_urls.get('Video')
-            if isinstance(video, list):
-                for format_ in video:
-                    video_url = format_.get('file')
-                    if not video_url:
-                        continue
-                    format_id = format_.get('label')
-                    f = {
-                        'url': format_['file'],
-                        'format_id': '%sp' % format_id,
-                        'height': int_or_none(format_id),
-                    }
-                    if format_id:
-                        # Some videos contain additional metadata (e.g.
-                        # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
-                        f = add_output_format_meta(f, format_id)
-                    formats.append(f)
+            extract_formats(download_urls.get('Video'))
 
         view_html = lecture.get('view_html')
         if view_html:
@@ -294,6 +300,35 @@ class UdemyIE(InfoExtractor):
                         'height': height,
                     }, res))
 
+            # react rendition since 2017.04.15 (see
+            # https://github.com/rg3/youtube-dl/issues/12744)
+            data = self._parse_json(
+                self._search_regex(
+                    r'videojs-setup-data=(["\'])(?P<data>{.+?})\1', view_html,
+                    'setup data', default='{}', group='data'), video_id,
+                transform_source=unescapeHTML, fatal=False)
+            if data and isinstance(data, dict):
+                extract_formats(data.get('sources'))
+                if not duration:
+                    duration = int_or_none(data.get('duration'))
+                tracks = data.get('tracks')
+                if isinstance(tracks, list):
+                    for track in tracks:
+                        if not isinstance(track, dict):
+                            continue
+                        if track.get('kind') != 'captions':
+                            continue
+                        src = track.get('src')
+                        if not src or not isinstance(src, compat_str):
+                            continue
+                        lang = track.get('language') or track.get(
+                            'srclang') or track.get('label')
+                        sub_dict = automatic_captions if track.get(
+                            'autogenerated') is True else subtitles
+                        sub_dict.setdefault(lang, []).append({
+                            'url': src,
+                        })
+
         self._sort_formats(formats, field_preference=('height', 'width', 'tbr', 'format_id'))
 
         return {
@@ -302,7 +337,9 @@ class UdemyIE(InfoExtractor):
             'description': description,
             'thumbnail': thumbnail,
             'duration': duration,
-            'formats': formats
+            'formats': formats,
+            'subtitles': subtitles,
+            'automatic_captions': automatic_captions,
         }
 
 

From 40158f55c906720ccfb6177c0949e0e7beaa085e Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sun, 12 Feb 2017 08:26:36 -0500
Subject: [PATCH 104/611] [brightcove] Relax video tag embeds extraction

BrightcoveNewIE.extract_urls(): Handle player data when it is all in
attributes of the <video> tag, and also the id is
data-brightcove-video-id not data-video-id.

Add test to generic extractor, note HEAD gives 404, no checksum computed(?).
---
 youtube_dl/extractor/brightcove.py | 58 ++++++++++++++++++++----------
 youtube_dl/extractor/generic.py    | 54 +++++++++++++++++++++++++++-
 2 files changed, 92 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 46ef8e605..c7059a12a 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -17,6 +17,7 @@ from ..compat import (
 from ..utils import (
     determine_ext,
     ExtractorError,
+    extract_attributes,
     find_xpath_attr,
     fix_xml_ampersands,
     float_or_none,
@@ -109,6 +110,7 @@ class BrightcoveLegacyIE(InfoExtractor):
                 'upload_date': '20140827',
                 'uploader_id': '710858724001',
             },
+            'skip': 'Video gone',
         },
         {
             # playlist with 'videoList'
@@ -490,9 +492,10 @@ class BrightcoveNewIE(InfoExtractor):
     def _extract_urls(webpage):
         # Reference:
         # 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe
-        # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript
-        # 3. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/embed-in-page.html
-        # 4. https://support.brightcove.com/en/video-cloud/docs/dynamically-assigning-videos-player
+        # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#tag
+        # 3. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript
+        # 4. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/in-page-embed-player-implementation.html
+        # 5. https://support.brightcove.com/en/video-cloud/docs/dynamically-assigning-videos-player
 
         entries = []
 
@@ -501,22 +504,39 @@ class BrightcoveNewIE(InfoExtractor):
                 r'<iframe[^>]+src=(["\'])((?:https?:)?//players\.brightcove\.net/\d+/[^/]+/index\.html.+?)\1', webpage):
             entries.append(url if url.startswith('http') else 'http:' + url)
 
-        # Look for embed_in_page embeds [2]
-        for video_id, account_id, player_id, embed in re.findall(
-                # According to examples from [3] it's unclear whether video id
-                # may be optional and what to do when it is
-                # According to [4] data-video-id may be prefixed with ref:
-                r'''(?sx)
-                    <video[^>]+
-                        data-video-id=["\'](\d+|ref:[^"\']+)["\'][^>]*>.*?
-                    </video>.*?
-                    <script[^>]+
-                        src=["\'](?:https?:)?//players\.brightcove\.net/
-                        (\d+)/([^/]+)_([^/]+)/index(?:\.min)?\.js
-                ''', webpage):
-            entries.append(
-                'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
-                % (account_id, player_id, embed, video_id))
+        # Look for <video> tags [2] and embed_in_page embeds [3]
+        # [2] looks like:
+        # <video data-video-id="5320421710001" data-account="245991542" data-player="SJWAiyYWg" data-embed="default" class="video-js" controls itemscope itemtype="http://schema.org/VideoObject">
+
+        for video, script_tag, account_id, player_id, embed in re.findall(
+            r'''(?isx)
+                (<video[^>]+>)
+                (?:.*?
+                  (<script[^>]+
+                    src=["\'](?:https?:)?//players\.brightcove\.net/
+                    (\d+)/([^/]+)_([^/]+)/index(?:\.min)?\.js
+                  )
+                )?
+            ''', webpage
+        ):
+            attrs = extract_attributes(video)
+
+            # According to examples from [4] it's unclear whether video id
+            # may be optional and what to do when it is
+            video_id = attrs.get('data-video-id')
+            # See PR#12099/bostonglobe.py for 'data-brightcove-video-id' variant
+
+            if not account_id:
+                account_id = attrs.get('data-account')
+            if not player_id:
+                player_id = attrs.get('data-player')
+            if not embed:
+                embed = attrs.get('data-embed')
+
+            if video_id and account_id and player_id and embed:
+                entries.append(
+                    'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
+                    % (account_id, player_id, embed, video_id))
 
         return entries
 
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 36d23d2f3..dc60d77f5 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -449,6 +449,59 @@ class GenericIE(InfoExtractor):
                 },
             }],
         },
+        {
+            # Brightcove with UUID in videoPlayer
+            'url': 'http://www8.hp.com/cn/zh/home.html',
+            'info_dict': {
+                'id': '5255815316001',
+                'ext': 'mp4',
+                'title': 'Sprocket Video - China',
+                'description': 'Sprocket Video - China',
+                'uploader': 'HP-Video Gallery',
+                'timestamp': 1482263210,
+                'upload_date': '20161220',
+                'uploader_id': '1107601872001',
+            },
+            'params': {
+                'skip_download': True,  # m3u8 download
+            },
+            'skip': 'video rotates...weekly?',
+        },
+        {
+            # Brightcove:new type [2].
+            'url': 'http://www.delawaresportszone.com/video-st-thomas-more-earns-first-trip-to-basketball-semis',
+            'md5': '2b35148fcf48da41c9fb4591650784f3',
+            'info_dict': {
+                'id': '5348741021001',
+                'ext': 'mp4',
+                'upload_date': '20170306',
+                'uploader_id': '4191638492001',
+                'timestamp': 1488769918,
+                'title': 'VIDEO:  St. Thomas More earns first trip to basketball semis',
+
+            },
+        },
+        {
+            # Alternative brightcove <video> attributes
+            'url': 'http://www.programme-tv.net/videos/extraits/81095-guillaume-canet-evoque-les-rumeurs-d-infidelite-de-marion-cotillard-avec-brad-pitt-dans-vivement-dimanche/',
+            'info_dict': {
+                'id': '81095-guillaume-canet-evoque-les-rumeurs-d-infidelite-de-marion-cotillard-avec-brad-pitt-dans-vivement-dimanche',
+                'title': "Guillaume Canet évoque les rumeurs d'infidélité de Marion Cotillard avec Brad Pitt dans Vivement Dimanche, Extraits : toutes les vidéos avec Télé-Loisirs",
+            },
+            'playlist': [{
+                'md5': '732d22ba3d33f2f3fc253c39f8f36523',
+                'info_dict': {
+                    'id': '5311302538001',
+                    'ext': 'mp4',
+                    'title': "Guillaume Canet évoque les rumeurs d'infidélité de Marion Cotillard avec Brad Pitt dans Vivement Dimanche",
+                    'description': "Guillaume Canet évoque les rumeurs d'infidélité de Marion Cotillard avec Brad Pitt dans Vivement Dimanche (France 2, 5 février 2017)",
+                    'timestamp': 1486321708,
+                    'upload_date': '20170205',
+                    'uploader_id': '800000640001',
+                },
+                'only_matching': True,
+            }],
+        },
         {
             # Brightcove with UUID in videoPlayer
             'url': 'http://www8.hp.com/cn/zh/home.html',
@@ -1900,7 +1953,6 @@ class GenericIE(InfoExtractor):
         # Look for Brightcove Legacy Studio embeds
         bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
         if bc_urls:
-            self.to_screen('Brightcove video detected.')
             entries = [{
                 '_type': 'url',
                 'url': smuggle_url(bc_url, {'Referer': url}),

From 0254f93b08ef5f8af8ea0f1b9f7558892626900b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Apr 2017 20:30:51 +0700
Subject: [PATCH 105/611] [brightcove] Validate ambiguous embeds' URLs (closes
 #9163, closes #12005, closes #12178, closes #12480)

---
 youtube_dl/extractor/brightcove.py | 55 +++++++++++++++++-------------
 youtube_dl/extractor/generic.py    |  2 +-
 2 files changed, 33 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index c7059a12a..124497e95 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -489,7 +489,7 @@ class BrightcoveNewIE(InfoExtractor):
         return urls[0] if urls else None
 
     @staticmethod
-    def _extract_urls(webpage):
+    def _extract_urls(ie, webpage):
         # Reference:
         # 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe
         # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#tag
@@ -506,37 +506,46 @@ class BrightcoveNewIE(InfoExtractor):
 
         # Look for <video> tags [2] and embed_in_page embeds [3]
         # [2] looks like:
-        # <video data-video-id="5320421710001" data-account="245991542" data-player="SJWAiyYWg" data-embed="default" class="video-js" controls itemscope itemtype="http://schema.org/VideoObject">
-
         for video, script_tag, account_id, player_id, embed in re.findall(
-            r'''(?isx)
-                (<video[^>]+>)
-                (?:.*?
-                  (<script[^>]+
-                    src=["\'](?:https?:)?//players\.brightcove\.net/
-                    (\d+)/([^/]+)_([^/]+)/index(?:\.min)?\.js
-                  )
-                )?
-            ''', webpage
-        ):
+                r'''(?isx)
+                    (<video\s+[^>]+>)
+                    (?:.*?
+                        (<script[^>]+
+                            src=["\'](?:https?:)?//players\.brightcove\.net/
+                            (\d+)/([^/]+)_([^/]+)/index(?:\.min)?\.js
+                        )
+                    )?
+                ''', webpage):
             attrs = extract_attributes(video)
 
             # According to examples from [4] it's unclear whether video id
             # may be optional and what to do when it is
             video_id = attrs.get('data-video-id')
-            # See PR#12099/bostonglobe.py for 'data-brightcove-video-id' variant
+            if not video_id:
+                continue
 
+            account_id = account_id or attrs.get('data-account')
             if not account_id:
-                account_id = attrs.get('data-account')
-            if not player_id:
-                player_id = attrs.get('data-player')
-            if not embed:
-                embed = attrs.get('data-embed')
+                continue
 
-            if video_id and account_id and player_id and embed:
-                entries.append(
-                    'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
-                    % (account_id, player_id, embed, video_id))
+            player_id = player_id or attrs.get('data-player') or 'default'
+            embed = embed or attrs.get('data-embed') or 'default'
+
+            bc_url = 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s' % (
+                account_id, player_id, embed, video_id)
+
+            # Some brightcove videos may be embedded with video tag only and
+            # without script tag or any mentioning of brightcove at all. Such
+            # embeds are considered ambiguous since they are matched based only
+            # on data-video-id and data-account attributes and in the wild may
+            # not be brightcove embeds at all. Let's check reconstructed
+            # brightcove URLs in case of such embeds and only process valid
+            # ones. By this we ensure there is indeed a brightcove embed.
+            if not script_tag and not ie._is_valid_url(
+                    bc_url, video_id, 'possible brightcove video'):
+                continue
+
+            entries.append(bc_url)
 
         return entries
 
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index dc60d77f5..bc7c21f7a 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1967,7 +1967,7 @@ class GenericIE(InfoExtractor):
             }
 
         # Look for Brightcove New Studio embeds
-        bc_urls = BrightcoveNewIE._extract_urls(webpage)
+        bc_urls = BrightcoveNewIE._extract_urls(self, webpage)
         if bc_urls:
             return self.playlist_from_matches(bc_urls, video_id, video_title, ie='BrightcoveNew')
 

From 3266d08af29bbd6078aca172741458ddee180ab9 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sat, 25 Mar 2017 19:47:48 -0400
Subject: [PATCH 106/611] [wsj:article] Add extractor

---
 youtube_dl/extractor/extractors.py |  5 ++++-
 youtube_dl/extractor/wsj.py        | 28 +++++++++++++++++++++++++---
 2 files changed, 29 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 1671090f4..72d28a7e6 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1233,7 +1233,10 @@ from .wrzuta import (
     WrzutaIE,
     WrzutaPlaylistIE,
 )
-from .wsj import WSJIE
+from .wsj import (
+    WSJIE,
+    WSJArticleIE,
+)
 from .xbef import XBefIE
 from .xboxclips import XboxClipsIE
 from .xfileshare import XFileShareIE
diff --git a/youtube_dl/extractor/wsj.py b/youtube_dl/extractor/wsj.py
index deb7483ae..ec38a2ad8 100644
--- a/youtube_dl/extractor/wsj.py
+++ b/youtube_dl/extractor/wsj.py
@@ -10,10 +10,11 @@ from ..utils import (
 
 
 class WSJIE(InfoExtractor):
-    _VALID_URL = r'''(?x)https?://
+    _VALID_URL = r'''(?x)
         (?:
-            video-api\.wsj\.com/api-video/player/iframe\.html\?guid=|
-            (?:www\.)?wsj\.com/video/[^/]+/
+            https?://video-api\.wsj\.com/api-video/player/iframe\.html\?guid=|
+            https?://(?:www\.)?wsj\.com/video/[^/]+/|
+            wsj:
         )
         (?P<id>[a-zA-Z0-9-]+)'''
     IE_DESC = 'Wall Street Journal'
@@ -87,3 +88,24 @@ class WSJIE(InfoExtractor):
             'title': title,
             'categories': info.get('keywords'),
         }
+
+
+class WSJArticleIE(InfoExtractor):
+    _VALID_URL = r'(?i)https?://(?:www\.)?wsj\.com/articles/(?P<id>\w[^/]+)'
+    _TESTS = [{
+        'url': 'https://www.wsj.com/articles/dont-like-china-no-pandas-for-you-1490366939?',
+        'info_dict': {
+            'id': '4B13FA62-1D8C-45DB-8EA1-4105CB20B362',
+            'ext': 'mp4',
+            'upload_date': '20170221',
+            'uploader_id': 'ralcaraz',
+            'title': 'Bao Bao the Panda Leaves for China',
+        }
+    }]
+
+    def _real_extract(self, url):
+        article_id = self._match_id(url)
+        webpage = self._download_webpage(url, article_id)
+        video_id = self._search_regex(r'data-src=["\']([A-Z0-9\-]+)',
+                                      webpage, 'video id')
+        return self.url_result('wsj:%s' % video_id, WSJIE.ie_key(), video_id)

From b2a19e38293206a4ff687315baf0369c205bcd6b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Apr 2017 20:51:47 +0700
Subject: [PATCH 107/611] [wsj] Improve and modernize (closes #12558)

---
 youtube_dl/extractor/wsj.py | 40 +++++++++++++++++++++----------------
 1 file changed, 23 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/wsj.py b/youtube_dl/extractor/wsj.py
index ec38a2ad8..45cfca7c5 100644
--- a/youtube_dl/extractor/wsj.py
+++ b/youtube_dl/extractor/wsj.py
@@ -11,12 +11,13 @@ from ..utils import (
 
 class WSJIE(InfoExtractor):
     _VALID_URL = r'''(?x)
-        (?:
-            https?://video-api\.wsj\.com/api-video/player/iframe\.html\?guid=|
-            https?://(?:www\.)?wsj\.com/video/[^/]+/|
-            wsj:
-        )
-        (?P<id>[a-zA-Z0-9-]+)'''
+                        (?:
+                            https?://video-api\.wsj\.com/api-video/player/iframe\.html\?.*?\bguid=|
+                            https?://(?:www\.)?wsj\.com/video/[^/]+/|
+                            wsj:
+                        )
+                        (?P<id>[a-fA-F0-9-]{36})
+                    '''
     IE_DESC = 'Wall Street Journal'
     _TESTS = [{
         'url': 'http://video-api.wsj.com/api-video/player/iframe.html?guid=1BD01A4C-BFE8-40A5-A42F-8A8AF9898B1A',
@@ -39,12 +40,17 @@ class WSJIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        api_url = (
-            'http://video-api.wsj.com/api-video/find_all_videos.asp?'
-            'type=guid&count=1&query=%s&fields=type,hls,videoMP4List,'
-            'thumbnailList,author,description,name,duration,videoURL,'
-            'titletag,formattedCreationDate,keywords,editor' % video_id)
-        info = self._download_json(api_url, video_id)['items'][0]
+        info = self._download_json(
+            'http://video-api.wsj.com/api-video/find_all_videos.asp', video_id,
+            query={
+                'type': 'guid',
+                'count': 1,
+                'query': video_id,
+                'fields': ','.join((
+                    'type', 'hls', 'videoMP4List', 'thumbnailList', 'author',
+                    'description', 'name', 'duration', 'videoURL', 'titletag',
+                    'formattedCreationDate', 'keywords', 'editor')),
+            })['items'][0]
         title = info.get('name', info.get('titletag'))
 
         formats = []
@@ -91,8 +97,8 @@ class WSJIE(InfoExtractor):
 
 
 class WSJArticleIE(InfoExtractor):
-    _VALID_URL = r'(?i)https?://(?:www\.)?wsj\.com/articles/(?P<id>\w[^/]+)'
-    _TESTS = [{
+    _VALID_URL = r'(?i)https?://(?:www\.)?wsj\.com/articles/(?P<id>[^/?#&]+)'
+    _TEST = {
         'url': 'https://www.wsj.com/articles/dont-like-china-no-pandas-for-you-1490366939?',
         'info_dict': {
             'id': '4B13FA62-1D8C-45DB-8EA1-4105CB20B362',
@@ -101,11 +107,11 @@ class WSJArticleIE(InfoExtractor):
             'uploader_id': 'ralcaraz',
             'title': 'Bao Bao the Panda Leaves for China',
         }
-    }]
+    }
 
     def _real_extract(self, url):
         article_id = self._match_id(url)
         webpage = self._download_webpage(url, article_id)
-        video_id = self._search_regex(r'data-src=["\']([A-Z0-9\-]+)',
-                                      webpage, 'video id')
+        video_id = self._search_regex(
+            r'data-src=["\']([a-fA-F0-9-]{36})', webpage, 'video id')
         return self.url_result('wsj:%s' % video_id, WSJIE.ie_key(), video_id)

From 4db79fa1bc482ef5d97c8b73f0bf3683d5cc383c Mon Sep 17 00:00:00 2001
From: Marvin Ewald <marvin.e@protonmail.ch>
Date: Tue, 4 Apr 2017 19:18:23 +0200
Subject: [PATCH 108/611] [streamango] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/streamango.py | 54 ++++++++++++++++++++++++++++++
 2 files changed, 55 insertions(+)
 create mode 100644 youtube_dl/extractor/streamango.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 72d28a7e6..a92cbefed 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -939,6 +939,7 @@ from .srmediathek import SRMediathekIE
 from .stanfordoc import StanfordOpenClassroomIE
 from .steam import SteamIE
 from .streamable import StreamableIE
+from .streamango import StreamangoIE
 from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
 from .streetvoice import StreetVoiceIE
diff --git a/youtube_dl/extractor/streamango.py b/youtube_dl/extractor/streamango.py
new file mode 100644
index 000000000..a4ef06b66
--- /dev/null
+++ b/youtube_dl/extractor/streamango.py
@@ -0,0 +1,54 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class StreamangoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?streamango\.com/(?:f|embed)/(?P<id>.+?)/(?:.+)'
+    _TESTS = [{
+        'url': 'https://streamango.com/f/clapasobsptpkdfe/20170315_150006_mp4',
+        'md5': 'e992787515a182f55e38fc97588d802a',
+        'info_dict': {
+            'id': 'clapasobsptpkdfe',
+            'ext': 'mp4',
+            'title': '20170315_150006.mp4',
+            'url': r're:https://streamango\.com/v/d/clapasobsptpkdfe~[0-9]{10}~(?:[0-9]+\.){3}[0-9]+~.{8}/720',
+        }
+    }, {
+        'url': 'https://streamango.com/embed/clapasobsptpkdfe/20170315_150006_mp4',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        def extract_url(urltype):
+            return self._search_regex(
+                r'type\s*:\s*["\']{}["\']\s*,\s*src\s*:\s*["\'](?P<url>.+?)["\'].*'.format(urltype),
+                webpage, 'video URL', group='url')
+
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._og_search_title(webpage)
+        url = 'https:' + extract_url('video/mp4')
+        dashurl = extract_url(r'application/dash\+xml')
+
+        formats = [{
+            'url': url,
+            'ext': 'mp4',
+            'width': 1280,
+            'height': 720,
+            'format_id': 'mp4',
+        }]
+
+        formats.extend(self._extract_mpd_formats(
+            dashurl, video_id, mpd_id='dash', fatal=False))
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'url': url,
+            'title': title,
+            'formats': formats,
+        }

From 8068296276657c9d888338c2211c112d69de6fc4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Apr 2017 21:50:15 +0700
Subject: [PATCH 109/611] [streamango] Improve extraction (closes #12643)

---
 youtube_dl/extractor/streamango.py | 50 ++++++++++++++++++------------
 1 file changed, 30 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/streamango.py b/youtube_dl/extractor/streamango.py
index a4ef06b66..aa4fad162 100644
--- a/youtube_dl/extractor/streamango.py
+++ b/youtube_dl/extractor/streamango.py
@@ -1,11 +1,18 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    js_to_json,
+)
 
 
 class StreamangoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?streamango\.com/(?:f|embed)/(?P<id>.+?)/(?:.+)'
+    _VALID_URL = r'https?://(?:www\.)?streamango\.com/(?:f|embed)/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://streamango.com/f/clapasobsptpkdfe/20170315_150006_mp4',
         'md5': 'e992787515a182f55e38fc97588d802a',
@@ -13,7 +20,6 @@ class StreamangoIE(InfoExtractor):
             'id': 'clapasobsptpkdfe',
             'ext': 'mp4',
             'title': '20170315_150006.mp4',
-            'url': r're:https://streamango\.com/v/d/clapasobsptpkdfe~[0-9]{10}~(?:[0-9]+\.){3}[0-9]+~.{8}/720',
         }
     }, {
         'url': 'https://streamango.com/embed/clapasobsptpkdfe/20170315_150006_mp4',
@@ -21,29 +27,33 @@ class StreamangoIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        def extract_url(urltype):
-            return self._search_regex(
-                r'type\s*:\s*["\']{}["\']\s*,\s*src\s*:\s*["\'](?P<url>.+?)["\'].*'.format(urltype),
-                webpage, 'video URL', group='url')
-
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
         title = self._og_search_title(webpage)
-        url = 'https:' + extract_url('video/mp4')
-        dashurl = extract_url(r'application/dash\+xml')
-
-        formats = [{
-            'url': url,
-            'ext': 'mp4',
-            'width': 1280,
-            'height': 720,
-            'format_id': 'mp4',
-        }]
-
-        formats.extend(self._extract_mpd_formats(
-            dashurl, video_id, mpd_id='dash', fatal=False))
 
+        formats = []
+        for format_ in re.findall(r'({[^}]*\bsrc\s*:\s*[^}]*})', webpage):
+            video = self._parse_json(
+                format_, video_id, transform_source=js_to_json, fatal=False)
+            if not video:
+                continue
+            src = video.get('src')
+            if not src:
+                continue
+            ext = determine_ext(src, default_ext=None)
+            if video.get('type') == 'application/dash+xml' or ext == 'mpd':
+                formats.extend(self._extract_mpd_formats(
+                    src, video_id, mpd_id='dash', fatal=False))
+            else:
+                formats.append({
+                    'url': src,
+                    'ext': ext or 'mp4',
+                    'width': int_or_none(video.get('width')),
+                    'height': int_or_none(video.get('height')),
+                    'tbr': int_or_none(video.get('bitrate')),
+                })
         self._sort_formats(formats)
 
         return {

From 413c1f8e2fc9ef8a9fae2767ef79905510e8c37f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Apr 2017 00:52:15 +0700
Subject: [PATCH 110/611] [extractor/generic] Extract RSS entries as
 url_transparent (#11163)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index bc7c21f7a..6a34c2491 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1693,7 +1693,7 @@ class GenericIE(InfoExtractor):
                 continue
 
             entries.append({
-                '_type': 'url',
+                '_type': 'url_transparent',
                 'url': next_url,
                 'title': it.find('title').text,
             })

From 0563f7ac6ed49929e1b488d92439928271c403df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Apr 2017 00:56:53 +0700
Subject: [PATCH 111/611] [YoutubeDL] Propagate overridden metadata to IE
 results of type url (closes #11163)

---
 youtube_dl/YoutubeDL.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 7953670a7..3da5200d7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -851,7 +851,14 @@ class YoutubeDL(object):
             new_result = info.copy()
             new_result.update(force_properties)
 
-            assert new_result.get('_type') != 'url_transparent'
+            # Extracted info may not be a video result (i.e.
+            # info.get('_type', 'video') != video) but rather an url or
+            # url_transparent. In such cases outer metadata (from ie_result)
+            # should be propagated to inner one (info). For this to happen
+            # _type of info should be overridden with url_transparent. This
+            # fixes issue from https://github.com/rg3/youtube-dl/pull/11163.
+            if new_result.get('_type') == 'url':
+                new_result['_type'] = 'url_transparent'
 
             return self.process_ie_result(
                 new_result, download=download, extra_info=extra_info)

From 51350db5a395c45cb181b1813a71b5e1aff86993 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Apr 2017 01:14:05 +0700
Subject: [PATCH 112/611] [test_YoutubeDL] Add test for #11163

---
 test/test_YoutubeDL.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 8491a88bd..75945e38f 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -755,6 +755,7 @@ class TestYoutubeDL(unittest.TestCase):
                     '_type': 'url_transparent',
                     'url': 'foo2:',
                     'ie_key': 'Foo2',
+                    'title': 'foo1 title'
                 }
 
         class Foo2IE(InfoExtractor):
@@ -771,7 +772,7 @@ class TestYoutubeDL(unittest.TestCase):
             _VALID_URL = r'foo3:'
 
             def _real_extract(self, url):
-                return _make_result([{'url': TEST_URL}])
+                return _make_result([{'url': TEST_URL}], title='foo3 title')
 
         ydl.add_info_extractor(Foo1IE(ydl))
         ydl.add_info_extractor(Foo2IE(ydl))
@@ -779,6 +780,7 @@ class TestYoutubeDL(unittest.TestCase):
         ydl.extract_info('foo1:')
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['url'], TEST_URL)
+        self.assertEqual(downloaded['title'], 'foo1 title')
 
 
 if __name__ == '__main__':

From d35dc344af8ca0b4cc9ea738a290c04842adb9db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Apr 2017 01:39:07 +0700
Subject: [PATCH 113/611] [YoutubeDL] Apply expand_path after output template
 substitution os.path.expandvars translates '%%' into '%' making output
 template invalid. Before: '%%(ext)s' -(expand path)-> '%(ext)s' -(outtmpl
 subst.)-> 'mp4' After: '%%(ext)s' -(outtmpl subst.)-> '%(ext)s' -(expand
 path)-> '%(ext)s'

---
 youtube_dl/YoutubeDL.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 3da5200d7..819b374ef 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -672,8 +672,7 @@ class YoutubeDL(object):
                         FORMAT_RE.format(numeric_field),
                         r'%({0})s'.format(numeric_field), outtmpl)
 
-            tmpl = expand_path(outtmpl)
-            filename = tmpl % template_dict
+            filename = expand_path(outtmpl % template_dict)
             # Temporary fix for #4787
             # 'Treat' all problem characters by passing filename through preferredencoding
             # to workaround encoding issues with subprocess on python2 @ Windows

From 76b5f99617f5b1ecdf64bb6445962f4db9685282 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Apr 2017 01:47:58 +0700
Subject: [PATCH 114/611] [ChangeLog] Actualize

---
 ChangeLog | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index cf5ee84a4..34ef32c1d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,19 @@
+version <unreleased>
+
+Core
+* [YoutubeDL] Apply expand_path after output template substitution
++ [YoutubeDL] Propagate overridden meta fields to extraction results of type
+  url (#11163)
+
+Extractors
++ [generic] Extract RSS entries as url_transparent (#11163)
++ [streamango] Add support for streamango.com (#12643)
++ [wsj:article] Add support for articles (#12558)
+* [brightcove] Relax video tag embeds extraction and validate ambiguous embeds'
+  URLs (#9163, #12005, #12178, #12480)
++ [udemy] Add support for react rendition (#12744)
+
+
 version 2017.04.15
 
 Extractors

From 022250a594750cf6d2e000e9ea142146d028bfe9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Apr 2017 01:49:02 +0700
Subject: [PATCH 115/611] release 2017.04.16

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 70f6b51ed..5d5adb199 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.15*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.15**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.16*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.16**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.04.15
+[debug] youtube-dl version 2017.04.16
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 34ef32c1d..6be86a090 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.04.16
 
 Core
 * [YoutubeDL] Apply expand_path after output template substitution
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index b29b50c8d..afae82214 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -745,6 +745,7 @@
  - **Steam**
  - **Stitcher**
  - **Streamable**
+ - **Streamango**
  - **streamcloud.eu**
  - **StreamCZ**
  - **StreetVoice**
@@ -966,6 +967,7 @@
  - **wrzuta.pl**
  - **wrzuta.pl:playlist**
  - **WSJ**: Wall Street Journal
+ - **WSJArticle**
  - **XBef**
  - **XboxClips**
  - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To, XVIDSTAGE, Vid ABC, VidBom, vidlo
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 612b50f7b..8b01fbc0a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.04.15'
+__version__ = '2017.04.16'

From 334f41e0d827a361b4b8d39f55b66221281d92ae Mon Sep 17 00:00:00 2001
From: "Jeremie J. Jarosh" <jeremie@jarosh.org>
Date: Sat, 15 Apr 2017 10:12:08 -0500
Subject: [PATCH 116/611] [go90] Improve extraction

- add metadata for 'series', 'episode', 'season', 'season_id',
  'season_number', and 'episode_number'
- integrate series title into the title
- extract subtitles  (fallback to `vtt` if the subtitle file type
  detection fails as that is the most likely extension)
---
 youtube_dl/extractor/go90.py | 38 ++++++++++++++++++++++++++++++++++--
 1 file changed, 36 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/go90.py b/youtube_dl/extractor/go90.py
index 3550eca7c..9b2e1c164 100644
--- a/youtube_dl/extractor/go90.py
+++ b/youtube_dl/extractor/go90.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
     int_or_none,
     parse_iso8601,
 )
@@ -18,7 +19,7 @@ class Go90IE(InfoExtractor):
         'info_dict': {
             'id': '84BUqjLpf9D',
             'ext': 'mp4',
-            'title': 'Inside The Utah Coalition Against Pornography Convention',
+            'title': 'Daily VICE - Inside The Utah Coalition Against Pornography Convention',
             'description': 'VICE\'s Karley Sciortino meets with activists who discuss the state\'s strong anti-porn stance. Then, VICE Sports explains NFL contracts.',
             'timestamp': 1491868800,
             'upload_date': '20170411',
@@ -32,11 +33,28 @@ class Go90IE(InfoExtractor):
             video_id, headers={
                 'Content-Type': 'application/json; charset=utf-8',
             }, data=b'{"client":"web","device_type":"pc"}')
-        title = video_data['title']
         main_video_asset = video_data['main_video_asset']
 
+        episode_number = int_or_none(video_data.get('episode_number'))
+        series = None
+        season = None
+        season_id = None
+        season_number = None
+        for metadata in video_data.get('__children', {}).get('Item', {}).values():
+            if metadata.get('type') == 'show':
+                series = metadata.get('title')
+            elif metadata.get('type') == 'season':
+                season = metadata.get('title')
+                season_id = metadata.get('id')
+                season_number = int_or_none(metadata.get('season_number'))
+
+        title = episode = video_data.get('title') or series
+        if series and series != title:
+            title = '%s - %s' % (series, title)
+
         thumbnails = []
         formats = []
+        subtitles = {}
         for asset in video_data.get('assets'):
             if asset.get('id') == main_video_asset:
                 for source in asset.get('sources', []):
@@ -70,6 +88,15 @@ class Go90IE(InfoExtractor):
                             'height': int_or_none(source.get('height')),
                             'tbr': int_or_none(source.get('bitrate')),
                         })
+
+                for caption in asset.get('caption_metadata', []):
+                    caption_url = caption.get('source_url')
+                    if not caption_url:
+                        continue
+                    subtitles.setdefault(caption.get('language', 'en'), []).append({
+                        'url': caption_url,
+                        'ext': determine_ext(caption_url, 'vtt'),
+                    })
             elif asset.get('type') == 'image':
                 asset_location = asset.get('location')
                 if not asset_location:
@@ -89,4 +116,11 @@ class Go90IE(InfoExtractor):
             'description': video_data.get('short_description'),
             'like_count': int_or_none(video_data.get('like_count')),
             'timestamp': parse_iso8601(video_data.get('released_at')),
+            'series': series,
+            'episode': episode,
+            'season': season,
+            'season_id': season_id,
+            'season_number': season_number,
+            'episode_number': episode_number,
+            'subtitles': subtitles,
         }

From 1183e22c7e107938a64a3f4d920999ea00871d86 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 15 Apr 2017 23:19:23 +0100
Subject: [PATCH 117/611] Credit @triple-j for extracting more metadata from
 go90.com(#12721)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 9abd5fdf4..1bdb74285 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -211,3 +211,4 @@ Juanjo Benages
 Xiao Di Guan
 Thomas Winant
 Daniel Twardowski
+Jeremie Jarosh

From 5935ef3c5d9c70c412a0c49f03feb3d0da8dedc6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Apr 2017 21:52:07 +0700
Subject: [PATCH 118/611] [itv] Lower preference for rtmp formats (closes
 #12759)

---
 youtube_dl/extractor/itv.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index 021c6b278..7f24f14ce 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -122,6 +122,8 @@ class ITVIE(InfoExtractor):
                 'play_path': play_path,
                 'tbr': tbr,
                 'ext': 'flv',
+                # rtmp formats are now stop downloading at ~72MiB
+                'preference': -10,
             })
 
         ios_playlist_url = params.get('data-video-playlist')

From f67177cae8a2e616d265bbbafe1ceabc7df8dbd0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Apr 2017 21:52:45 +0700
Subject: [PATCH 119/611] [itv] Use native hls

---
 youtube_dl/extractor/itv.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index 7f24f14ce..5ff7e1aaa 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -174,7 +174,9 @@ class ITVIE(InfoExtractor):
                         href = ios_base_url + href
                     ext = determine_ext(href)
                     if ext == 'm3u8':
-                        formats.extend(self._extract_m3u8_formats(href, video_id, 'mp4', m3u8_id='hls', fatal=False))
+                        formats.extend(self._extract_m3u8_formats(
+                            href, video_id, 'mp4', entry_protocol='m3u8_native',
+                            m3u8_id='hls', fatal=False))
                     else:
                         formats.append({
                             'url': href,

From c2d7d76efd37c16d01100635f308f03deb9c04aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Apr 2017 23:15:24 +0700
Subject: [PATCH 120/611] [itv] Fix rtmp formats (#12759)

---
 youtube_dl/extractor/itv.py | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index 5ff7e1aaa..7442c24d9 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -116,15 +116,25 @@ class ITVIE(InfoExtractor):
             if not play_path:
                 continue
             tbr = int_or_none(media_file.get('bitrate'), 1000)
-            formats.append({
+            f = {
                 'format_id': 'rtmp' + ('-%d' % tbr if tbr else ''),
-                'url': rtmp_url,
                 'play_path': play_path,
+                # Providing this swfVfy allows to avoid truncated downloads
+                'player_url': 'http://www.itv.com/mercury/Mercury_VideoPlayer.swf',
+                'page_url': url,
                 'tbr': tbr,
                 'ext': 'flv',
-                # rtmp formats are now stop downloading at ~72MiB
-                'preference': -10,
-            })
+            }
+            app = self._search_regex(
+                'rtmpe?://[^/]+/(.+)$', rtmp_url, 'app', default=None)
+            if app:
+                f.update({
+                    'url': rtmp_url.split('?', 1)[0],
+                    'app': app,
+                })
+            else:
+                f['url'] = rtmp_url
+            formats.append(f)
 
         ios_playlist_url = params.get('data-video-playlist')
         hmac = params.get('data-video-hmac')

From 751c89a27d68c54375e96789cc90d4c8a3ce3dbc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Apr 2017 23:19:20 +0700
Subject: [PATCH 121/611] [itv] Extract series metadata

---
 youtube_dl/extractor/itv.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index 7442c24d9..f3156804d 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -203,7 +203,8 @@ class ITVIE(InfoExtractor):
                 'ext': 'ttml' if ext == 'xml' else ext,
             })
 
-        return {
+        info = self._search_json_ld(webpage, video_id, default={})
+        info.update({
             'id': video_id,
             'title': title,
             'formats': formats,
@@ -212,4 +213,5 @@ class ITVIE(InfoExtractor):
             'episode_number': int_or_none(xpath_text(playlist, 'EpisodeNumber')),
             'series': xpath_text(playlist, 'ProgrammeTitle'),
             'duartion': parse_duration(xpath_text(playlist, 'Duration')),
-        }
+        })
+        return info

From e5d39886ec8e4e40b2b7257d16cc5d8505cc1f69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Apr 2017 00:23:16 +0700
Subject: [PATCH 122/611] [limelight] Improve embeds extraction (closes #12761)
 * Move extraction code to extractor * Add extraction for
 LimelightEmbeddedPlayerFlash embeds * Extract multiple video

---
 youtube_dl/extractor/generic.py   |  6 +++++
 youtube_dl/extractor/limelight.py | 37 +++++++++++++++++++++++++++++++
 2 files changed, 43 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6a34c2491..c523abb25 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -85,6 +85,7 @@ from .ustream import UstreamIE
 from .openload import OpenloadIE
 from .videopress import VideoPressIE
 from .rutube import RutubeIE
+from .limelight import LimelightBaseIE
 
 
 class GenericIE(InfoExtractor):
@@ -2483,6 +2484,11 @@ class GenericIE(InfoExtractor):
             return self.url_result(piksel_url, PikselIE.ie_key())
 
         # Look for Limelight embeds
+        limelight_urls = LimelightBaseIE._extract_urls(webpage, url)
+        if limelight_urls:
+            return self.playlist_result(
+                limelight_urls, video_id, video_title, video_description)
+
         mobj = re.search(r'LimelightPlayer\.doLoad(Media|Channel|ChannelList)\(["\'](?P<id>[a-z0-9]{32})', webpage)
         if mobj:
             lm = {
diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index f52c2e169..0041453af 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -9,6 +9,7 @@ from ..utils import (
     determine_ext,
     float_or_none,
     int_or_none,
+    smuggle_url,
     unsmuggle_url,
     ExtractorError,
 )
@@ -18,6 +19,42 @@ class LimelightBaseIE(InfoExtractor):
     _PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/%s/%s/%s'
     _API_URL = 'http://api.video.limelight.com/rest/organizations/%s/%s/%s/%s.json'
 
+    @classmethod
+    def _extract_urls(cls, webpage, source_url):
+        lm = {
+            'Media': 'media',
+            'Channel': 'channel',
+            'ChannelList': 'channel_list',
+        }
+        entries = []
+        for kind, video_id in re.findall(
+                r'LimelightPlayer\.doLoad(Media|Channel|ChannelList)\(["\'](?P<id>[a-z0-9]{32})',
+                webpage):
+            print('video_id', video_id)
+            entries.append(cls.url_result(
+                smuggle_url(
+                    'limelight:%s:%s' % (lm[kind], video_id),
+                    {'source_url': source_url}),
+                'Limelight%s' % kind, video_id))
+        for mobj in re.finditer(
+                # As per [1] class attribute should be exactly equal to
+                # LimelightEmbeddedPlayerFlash but numerous examples seen
+                # that don't exactly match it (e.g. [2]).
+                # 1. http://support.3playmedia.com/hc/en-us/articles/227732408-Limelight-Embedding-the-Captions-Plugin-with-the-Limelight-Player-on-Your-Webpage
+                # 2. http://www.sedona.com/FacilitatorTraining2017
+                r'''(?sx)
+                    <object[^>]+class=(["\'])(?:(?!\1).)*\bLimelightEmbeddedPlayerFlash\b(?:(?!\1).)*\1[^>]*>.*?
+                        <param[^>]+
+                            name=(["\'])flashVars\2[^>]+
+                            value=(["\'])(?:(?!\3).)*mediaId=(?P<id>[a-z0-9]{32})
+                ''', webpage):
+            entries.append(cls.url_result(
+                smuggle_url(
+                    'limelight:media:%s' % mobj.group('id'),
+                    {'source_url': source_url}),
+                'LimelightMedia', mobj.group('id')))
+        return entries
+
     def _call_playlist_service(self, item_id, method, fatal=True, referer=None):
         headers = {}
         if referer:

From 91bc57e4c527f918c7735d67f4bceb3cf1e2332c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Apr 2017 00:33:47 +0700
Subject: [PATCH 123/611] [limelight] Add support for channels and channels
 lists embedded using LimelightEmbeddedPlayerFlash (#12761)

---
 youtube_dl/extractor/limelight.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index 0041453af..0a5a3956c 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -30,7 +30,6 @@ class LimelightBaseIE(InfoExtractor):
         for kind, video_id in re.findall(
                 r'LimelightPlayer\.doLoad(Media|Channel|ChannelList)\(["\'](?P<id>[a-z0-9]{32})',
                 webpage):
-            print('video_id', video_id)
             entries.append(cls.url_result(
                 smuggle_url(
                     'limelight:%s:%s' % (lm[kind], video_id),
@@ -46,13 +45,14 @@ class LimelightBaseIE(InfoExtractor):
                     <object[^>]+class=(["\'])(?:(?!\1).)*\bLimelightEmbeddedPlayerFlash\b(?:(?!\1).)*\1[^>]*>.*?
                         <param[^>]+
                             name=(["\'])flashVars\2[^>]+
-                            value=(["\'])(?:(?!\3).)*mediaId=(?P<id>[a-z0-9]{32})
+                            value=(["\'])(?:(?!\3).)*(?P<kind>media|channel(?:List)?)Id=(?P<id>[a-z0-9]{32})
                 ''', webpage):
+            kind, video_id = mobj.group('kind'), mobj.group('id')
             entries.append(cls.url_result(
                 smuggle_url(
-                    'limelight:media:%s' % mobj.group('id'),
+                    'limelight:%s:%s' % (kind, video_id),
                     {'source_url': source_url}),
-                'LimelightMedia', mobj.group('id')))
+                'Limelight%s' % kind.capitalize(), video_id))
         return entries
 
     def _call_playlist_service(self, item_id, method, fatal=True, referer=None):

From ab87c2600922bf093fb1fbb81f263c2acada8824 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Apr 2017 00:36:59 +0700
Subject: [PATCH 124/611] [extractor/generic] Add test for #12761

---
 youtube_dl/extractor/generic.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c523abb25..b22c3122a 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1652,6 +1652,15 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': [SenateISVPIE.ie_key()],
         },
+        {
+            # Limelight embeds (1 channel embed + 4 media embeds)
+            'url': 'http://www.sedona.com/FacilitatorTraining2017',
+            'info_dict': {
+                'id': 'FacilitatorTraining2017',
+                'title': 'Facilitator Training 2017',
+            },
+            'playlist_mincount': 5,
+        },
         # {
         #     # TODO: find another test
         #     # http://schema.org/VideoObject

From fa7a6e6de66a8d8ce9af09d85a678ca5d91988f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Apr 2017 00:43:56 +0700
Subject: [PATCH 125/611] [ChangeLog] Actualize

---
 ChangeLog | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 6be86a090..e923372c8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,16 @@
+version <unreleased>
+
+Extractors
+* [limelight] Improve extraction LimelightEmbeddedPlayerFlash media embeds and
+  add support for channel and channelList embeds
+* [generic] Extract multiple Limelight embeds (#12761)
++ [itv] Extract series metadata
+* [itv] Fix RTMP formats downloading (#12759)
+* [itv] Use native HLS downloader by default
++ [go90] Extract subtitles (#12752)
++ [go90] Extract series metadata (#12752)
+
+
 version 2017.04.16
 
 Core

From 16a09aefe33ba2d0122926d761f89a93c0c7e7d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Apr 2017 00:46:32 +0700
Subject: [PATCH 126/611] release 2017.04.17

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 5d5adb199..45f8db721 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.16*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.16**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.17*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.17**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.04.16
+[debug] youtube-dl version 2017.04.17
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index e923372c8..03ef0363b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.04.17
 
 Extractors
 * [limelight] Improve extraction LimelightEmbeddedPlayerFlash media embeds and
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8b01fbc0a..480e04e06 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.04.16'
+__version__ = '2017.04.17'

From 1c35b3da44a087890b7144e5e626a573f2a80bf7 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 16 Apr 2017 21:24:34 +0100
Subject: [PATCH 127/611] [odnoklassniki] extract m3u8 formats

---
 youtube_dl/extractor/odnoklassniki.py | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 0ee56a45b..b9369cea5 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_etree_fromstring,
     compat_parse_qs,
     compat_urllib_parse_unquote,
     compat_urllib_parse_urlparse,
@@ -176,14 +177,32 @@ class OdnoklassnikiIE(InfoExtractor):
             })
             return info
 
-        quality = qualities(('mobile', 'lowest', 'low', 'sd', 'hd', 'full'))
+        quality = qualities(('4', '0', '1', '2', '3', '5'))
 
         formats = [{
             'url': f['url'],
             'ext': 'mp4',
             'format_id': f['name'],
-            'quality': quality(f['name']),
         } for f in metadata['videos']]
+
+        m3u8_url = metadata.get('hlsManifestUrl')
+        if m3u8_url:
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', 'm3u8_native',
+                m3u8_id='hls', fatal=False))
+
+        dash_manifest = metadata.get('metadataEmbedded')
+        if dash_manifest:
+            formats.extend(self._parse_mpd_formats(
+                compat_etree_fromstring(dash_manifest), 'mpd'))
+
+        for fmt in formats:
+            fmt_type = self._search_regex(
+                r'\btype[/=](\d)', fmt['url'],
+                'format type', default=None)
+            if fmt_type:
+                fmt['quality'] = quality(fmt_type)
+
         self._sort_formats(formats)
 
         info['formats'] = formats

From bf1b87cd919f07b7fef204838be73981e122ee11 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 17 Apr 2017 08:48:24 +0100
Subject: [PATCH 128/611] [common] Relax JWPlayer regex and remove duplicate
 urls(#12768)

---
 youtube_dl/extractor/common.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index dcc9d628a..12e010a0d 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2182,7 +2182,7 @@ class InfoExtractor(object):
 
     def _find_jwplayer_data(self, webpage, video_id=None, transform_source=js_to_json):
         mobj = re.search(
-            r'jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\)\.setup\s*\((?P<options>[^)]+)\)',
+            r'(?s)jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\).*?\.setup\s*\((?P<options>[^)]+)\)',
             webpage)
         if mobj:
             try:
@@ -2258,11 +2258,17 @@ class InfoExtractor(object):
 
     def _parse_jwplayer_formats(self, jwplayer_sources_data, video_id=None,
                                 m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
+        urls = []
         formats = []
         for source in jwplayer_sources_data:
-            source_url = self._proto_relative_url(source['file'])
+            source_url = self._proto_relative_url(source.get('file'))
+            if not source_url:
+                continue
             if base_url:
                 source_url = compat_urlparse.urljoin(base_url, source_url)
+            if source_url in urls:
+                continue
+            urls.append(source_url)
             source_type = source.get('type') or ''
             ext = mimetype2ext(source_type) or determine_ext(source_url)
             if source_type == 'hls' or ext == 'm3u8':

From f631b557915eae6c72f2f503255903c92481e85e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 18 Apr 2017 21:46:25 +0700
Subject: [PATCH 129/611] [brightcove] Fix _extract_url (closes #12782)

---
 youtube_dl/extractor/brightcove.py | 4 ++--
 youtube_dl/extractor/nowness.py    | 2 +-
 youtube_dl/extractor/yahoo.py      | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 124497e95..c4fd96b3d 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -484,8 +484,8 @@ class BrightcoveNewIE(InfoExtractor):
     }]
 
     @staticmethod
-    def _extract_url(webpage):
-        urls = BrightcoveNewIE._extract_urls(webpage)
+    def _extract_url(ie, webpage):
+        urls = BrightcoveNewIE._extract_urls(ie, webpage)
         return urls[0] if urls else None
 
     @staticmethod
diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index b6c5ee6e4..f26dafb8f 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -28,7 +28,7 @@ class NownessBaseIE(InfoExtractor):
                         bc_url = BrightcoveLegacyIE._extract_brightcove_url(player_code)
                         if bc_url:
                             return self.url_result(bc_url, BrightcoveLegacyIE.ie_key())
-                        bc_url = BrightcoveNewIE._extract_url(player_code)
+                        bc_url = BrightcoveNewIE._extract_url(self, player_code)
                         if bc_url:
                             return self.url_result(bc_url, BrightcoveNewIE.ie_key())
                         raise ExtractorError('Could not find player definition')
diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 4951414e9..38f82bf44 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -258,7 +258,7 @@ class YahooIE(InfoExtractor):
             return self.url_result(bc_url, BrightcoveLegacyIE.ie_key())
 
         # Look for Brightcove New Studio embeds
-        bc_url = BrightcoveNewIE._extract_url(webpage)
+        bc_url = BrightcoveNewIE._extract_url(self, webpage)
         if bc_url:
             return self.url_result(bc_url, BrightcoveNewIE.ie_key())
 

From 06d0ad9a4e2266b1cc74b45a59a53fad3f23fe15 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 18 Apr 2017 23:03:03 +0800
Subject: [PATCH 130/611] [brightcove] Support URLs with bcpid instead of
 playerID

Fixes #12482
---
 ChangeLog                          |  7 +++++++
 youtube_dl/extractor/brightcove.py | 20 +++++++++++++++++---
 youtube_dl/extractor/generic.py    | 16 ++++++++++++++++
 3 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 03ef0363b..f8084fda5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,10 @@
+version <unreleased>
+
+Extractors
++ [generic] Support Brightcove videos in <iframe> (#12482)
++ [brightcove] Support URLs with bcpid instead of playerID (#12482)
+
+
 version 2017.04.17
 
 Extractors
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index c4fd96b3d..a2db705b0 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -131,6 +131,12 @@ class BrightcoveLegacyIE(InfoExtractor):
             },
             'playlist_mincount': 10,
         },
+        {
+            # playerID inferred from bcpid
+            # from http://www.un.org/chinese/News/story.asp?NewsID=27724
+            'url': 'https://link.brightcove.com/services/player/bcpid1722935254001/?bctid=5360463607001&autoStart=false&secureConnections=true&width=650&height=350',
+            'only_matching': True,  # Tested in GenericIE
+        }
     ]
     FLV_VCODECS = {
         1: 'SORENSON',
@@ -266,9 +272,13 @@ class BrightcoveLegacyIE(InfoExtractor):
         if matches:
             return list(filter(None, [cls._build_brighcove_url(m) for m in matches]))
 
-        return list(filter(None, [
-            cls._build_brighcove_url_from_js(custom_bc)
-            for custom_bc in re.findall(r'(customBC\.createVideo\(.+?\);)', webpage)]))
+        matches = re.findall(r'(customBC\.createVideo\(.+?\);)', webpage)
+        if matches:
+            return list(filter(None, [
+                cls._build_brighcove_url_from_js(custom_bc)
+                for custom_bc in matches]))
+        return [url for _, url in re.findall(
+            r'<iframe[^>]+src=([\'"])((?:https?:)?//link\.brightcove\.com/services/player/(?!\1).+)\1', webpage)]
 
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})
@@ -285,6 +295,10 @@ class BrightcoveLegacyIE(InfoExtractor):
         if videoPlayer:
             # We set the original url as the default 'Referer' header
             referer = smuggled_data.get('Referer', url)
+            if 'playerID' not in query:
+                mobj = re.search(r'/bcpid(\d+)', url)
+                if mobj is not None:
+                    query['playerID'] = [mobj.group(1)]
             return self._get_video_info(
                 videoPlayer[0], query, referer=referer)
         elif 'playerKey' in query:
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b22c3122a..67184bc5d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -430,6 +430,22 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,  # m3u8 download
             },
         },
+        {
+            # Brightcove video in <iframe>
+            'url': 'http://www.un.org/chinese/News/story.asp?NewsID=27724',
+            'md5': '36d74ef5e37c8b4a2ce92880d208b968',
+            'info_dict': {
+                'id': '5360463607001',
+                'ext': 'mp4',
+                'title': '叙利亚失明儿童在废墟上演唱《心跳》  呼吁获得正常童年生活',
+                'description': '联合国儿童基金会中东和北非区域大使、作曲家扎德·迪拉尼（Zade Dirani）在3月15日叙利亚冲突爆发7周年纪念日之际发布了为叙利亚谱写的歌曲《心跳》（HEARTBEAT），为受到六年冲突影响的叙利亚儿童发出强烈呐喊，呼吁世界做出共同努力，使叙利亚儿童重新获得享有正常童年生活的权利。',
+                'uploader': 'United Nations',
+                'uploader_id': '1362235914001',
+                'timestamp': 1489593889,
+                'upload_date': '20170315',
+            },
+            'add_ie': ['BrightcoveLegacy'],
+        },
         {
             # Brightcove with alternative playerID key
             'url': 'http://www.nature.com/nmeth/journal/v9/n7/fig_tab/nmeth.2062_SV1.html',

From bae1404893341ed89f4c9b556aa4068c13ed9f7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 18 Apr 2017 22:21:38 +0700
Subject: [PATCH 131/611] [extractor/common] Add support for video of WebPage
 context in _json_ld (closes #12778)

---
 youtube_dl/extractor/common.py | 33 +++++++++++++++++++++------------
 1 file changed, 21 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 12e010a0d..61d97ab72 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -976,6 +976,22 @@ class InfoExtractor(object):
             return info
         if isinstance(json_ld, dict):
             json_ld = [json_ld]
+
+        def extract_video_object(e):
+            assert e['@type'] == 'VideoObject'
+            info.update({
+                'url': e.get('contentUrl'),
+                'title': unescapeHTML(e.get('name')),
+                'description': unescapeHTML(e.get('description')),
+                'thumbnail': e.get('thumbnailUrl') or e.get('thumbnailURL'),
+                'duration': parse_duration(e.get('duration')),
+                'timestamp': unified_timestamp(e.get('uploadDate')),
+                'filesize': float_or_none(e.get('contentSize')),
+                'tbr': int_or_none(e.get('bitrate')),
+                'width': int_or_none(e.get('width')),
+                'height': int_or_none(e.get('height')),
+            })
+
         for e in json_ld:
             if e.get('@context') == 'http://schema.org':
                 item_type = e.get('@type')
@@ -1000,18 +1016,11 @@ class InfoExtractor(object):
                         'description': unescapeHTML(e.get('articleBody')),
                     })
                 elif item_type == 'VideoObject':
-                    info.update({
-                        'url': e.get('contentUrl'),
-                        'title': unescapeHTML(e.get('name')),
-                        'description': unescapeHTML(e.get('description')),
-                        'thumbnail': e.get('thumbnailUrl') or e.get('thumbnailURL'),
-                        'duration': parse_duration(e.get('duration')),
-                        'timestamp': unified_timestamp(e.get('uploadDate')),
-                        'filesize': float_or_none(e.get('contentSize')),
-                        'tbr': int_or_none(e.get('bitrate')),
-                        'width': int_or_none(e.get('width')),
-                        'height': int_or_none(e.get('height')),
-                    })
+                    extract_video_object(e)
+                elif item_type == 'WebPage':
+                    video = e.get('video')
+                    if isinstance(video, dict) and video.get('@type') == 'VideoObject':
+                        extract_video_object(video)
                 break
         return dict((k, v) for k, v in info.items() if v is not None)
 

From a32a9a7ef5f64bcec29ef71c8e5b6dc509232c13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 18 Apr 2017 22:39:58 +0700
Subject: [PATCH 132/611] [extractor/common] Add support multiple getters in
 try_get

---
 youtube_dl/utils.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 2340bc306..79a337df4 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2103,13 +2103,16 @@ def dict_get(d, key_or_keys, default=None, skip_false_values=True):
 
 
 def try_get(src, getter, expected_type=None):
-    try:
-        v = getter(src)
-    except (AttributeError, KeyError, TypeError, IndexError):
-        pass
-    else:
-        if expected_type is None or isinstance(v, expected_type):
-            return v
+    if not isinstance(getter, (list, tuple)):
+        getter = [getter]
+    for get in getter:
+        try:
+            v = get(src)
+        except (AttributeError, KeyError, TypeError, IndexError):
+            pass
+        else:
+            if expected_type is None or isinstance(v, expected_type):
+                return v
 
 
 def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):

From 18848d226a254a20df2b772ac3c881caccf2f260 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 18 Apr 2017 22:40:26 +0700
Subject: [PATCH 133/611] [instagram] Fix extraction (closes #12777)

---
 youtube_dl/extractor/instagram.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index c1921cbcf..4667335e0 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -112,7 +112,8 @@ class InstagramIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         (video_url, description, thumbnail, timestamp, uploader,
-         uploader_id, like_count, comment_count, height, width) = [None] * 10
+         uploader_id, like_count, comment_count, comments, height,
+         width) = [None] * 11
 
         shared_data = self._parse_json(
             self._search_regex(
@@ -121,7 +122,10 @@ class InstagramIE(InfoExtractor):
             video_id, fatal=False)
         if shared_data:
             media = try_get(
-                shared_data, lambda x: x['entry_data']['PostPage'][0]['media'], dict)
+                shared_data,
+                (lambda x: x['entry_data']['PostPage'][0]['graphql']['shortcode_media'],
+                 lambda x: x['entry_data']['PostPage'][0]['media']),
+                dict)
             if media:
                 video_url = media.get('video_url')
                 height = int_or_none(media.get('dimensions', {}).get('height'))

From 2dc48df5bc52a5c64d0fc699189fbcb162d62d9b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 18 Apr 2017 23:54:36 +0700
Subject: [PATCH 134/611] [xfileshare] Add support for gorillavid.com and
 daclips.com (closes #12776)

---
 youtube_dl/extractor/xfileshare.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 1b5cd122a..13f8be6cb 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -17,24 +17,24 @@ from ..utils import (
 
 class XFileShareIE(InfoExtractor):
     _SITES = (
-        ('daclips.in', 'DaClips'),
-        ('filehoot.com', 'FileHoot'),
-        ('gorillavid.in', 'GorillaVid'),
-        ('movpod.in', 'MovPod'),
-        ('powerwatch.pw', 'PowerWatch'),
-        ('rapidvideo.ws', 'Rapidvideo.ws'),
-        ('thevideobee.to', 'TheVideoBee'),
-        ('vidto.me', 'Vidto'),
-        ('streamin.to', 'Streamin.To'),
-        ('xvidstage.com', 'XVIDSTAGE'),
-        ('vidabc.com', 'Vid ABC'),
-        ('vidbom.com', 'VidBom'),
-        ('vidlo.us', 'vidlo'),
+        (r'daclips\.(?:in|com)', 'DaClips'),
+        (r'filehoot\.com', 'FileHoot'),
+        (r'gorillavid\.(?:in|com)', 'GorillaVid'),
+        (r'movpod\.in', 'MovPod'),
+        (r'powerwatch\.pw', 'PowerWatch'),
+        (r'rapidvideo\.ws', 'Rapidvideo.ws'),
+        (r'thevideobee\.to', 'TheVideoBee'),
+        (r'vidto\.me', 'Vidto'),
+        (r'streamin\.to', 'Streamin.To'),
+        (r'xvidstage\.com', 'XVIDSTAGE'),
+        (r'vidabc\.com', 'Vid ABC'),
+        (r'vidbom\.com', 'VidBom'),
+        (r'vidlo\.us', 'vidlo'),
     )
 
     IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1])
     _VALID_URL = (r'https?://(?P<host>(?:www\.)?(?:%s))/(?:embed-)?(?P<id>[0-9a-zA-Z]+)'
-                  % '|'.join(re.escape(site) for site in list(zip(*_SITES))[0]))
+                  % '|'.join(site for site in list(zip(*_SITES))[0]))
 
     _FILE_NOT_FOUND_REGEXES = (
         r'>(?:404 - )?File Not Found<',

From 58f6ab72ed74c2ab1b6d3885ef2c407465d8c6c2 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 19 Apr 2017 00:16:31 +0100
Subject: [PATCH 135/611] [odnoklassniki] update tests

---
 youtube_dl/extractor/odnoklassniki.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index b9369cea5..854b6800c 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -38,7 +38,7 @@ class OdnoklassnikiIE(InfoExtractor):
     }, {
         # metadataUrl
         'url': 'http://ok.ru/video/63567059965189-0?fromTime=5',
-        'md5': '9676cf86eff5391d35dea675d224e131',
+        'md5': '6ff470ea2dd51d5d18c295a355b0b6bc',
         'info_dict': {
             'id': '63567059965189-0',
             'ext': 'mp4',
@@ -54,7 +54,7 @@ class OdnoklassnikiIE(InfoExtractor):
     }, {
         # YouTube embed (metadataUrl, provider == USER_YOUTUBE)
         'url': 'http://ok.ru/video/64211978996595-1',
-        'md5': '5d7475d428845cd2e13bae6f1a992278',
+        'md5': '2f206894ffb5dbfcce2c5a14b909eea5',
         'info_dict': {
             'id': '64211978996595-1',
             'ext': 'mp4',
@@ -62,8 +62,8 @@ class OdnoklassnikiIE(InfoExtractor):
             'description': 'md5:848eb8b85e5e3471a3a803dae1343ed0',
             'duration': 440,
             'upload_date': '20150826',
-            'uploader_id': '750099571',
-            'uploader': 'Алина П',
+            'uploader_id': 'tvroscosmos',
+            'uploader': 'Телестудия Роскосмоса',
             'age_limit': 0,
         },
     }, {
@@ -82,6 +82,7 @@ class OdnoklassnikiIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+        'skip': 'Video has not been found',
     }, {
         'url': 'http://ok.ru/web-api/video/moviePlayer/20079905452',
         'only_matching': True,

From 75a24854073e590f4efc9f037b57dee348f52b61 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 28 Jun 2016 18:07:50 +0100
Subject: [PATCH 136/611] [fragment,hls,f4m,dash,ism] improve fragment
 downloading

- resume immediately
- no need to concatenate segments and decrypt them on every resume
- no need to save temp files for segments

and for hls downloader:
- no need to download keys for segments that already downloaded
---
 youtube_dl/downloader/common.py   | 29 ++++++++--------
 youtube_dl/downloader/dash.py     | 43 ++++++-----------------
 youtube_dl/downloader/f4m.py      | 33 ++++++------------
 youtube_dl/downloader/fragment.py | 57 +++++++++++++++++++++++++------
 youtube_dl/downloader/hls.py      | 34 +++++++-----------
 youtube_dl/downloader/http.py     |  8 +++--
 youtube_dl/downloader/ism.py      | 30 +++++-----------
 7 files changed, 111 insertions(+), 123 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 2c4470a95..fdb77b620 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -327,21 +327,22 @@ class FileDownloader(object):
             os.path.exists(encodeFilename(filename))
         )
 
-        continuedl_and_exists = (
-            self.params.get('continuedl', True) and
-            os.path.isfile(encodeFilename(filename)) and
-            not self.params.get('nopart', False)
-        )
+        if not hasattr(filename, 'write'):
+            continuedl_and_exists = (
+                self.params.get('continuedl', True) and
+                os.path.isfile(encodeFilename(filename)) and
+                not self.params.get('nopart', False)
+            )
 
-        # Check file already present
-        if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
-            self.report_file_already_downloaded(filename)
-            self._hook_progress({
-                'filename': filename,
-                'status': 'finished',
-                'total_bytes': os.path.getsize(encodeFilename(filename)),
-            })
-            return True
+            # Check file already present
+            if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
+                self.report_file_already_downloaded(filename)
+                self._hook_progress({
+                    'filename': filename,
+                    'status': 'finished',
+                    'total_bytes': os.path.getsize(encodeFilename(filename)),
+                })
+                return True
 
         min_sleep_interval = self.params.get('sleep_interval')
         if min_sleep_interval:
diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index e2ddc369e..94a13a543 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -1,13 +1,7 @@
 from __future__ import unicode_literals
 
-import os
-
 from .fragment import FragmentFD
 from ..compat import compat_urllib_error
-from ..utils import (
-    sanitize_open,
-    encodeFilename,
-)
 
 
 class DashSegmentsFD(FragmentFD):
@@ -28,31 +22,24 @@ class DashSegmentsFD(FragmentFD):
 
         self._prepare_and_start_frag_download(ctx)
 
-        segments_filenames = []
-
         fragment_retries = self.params.get('fragment_retries', 0)
         skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
 
-        def process_segment(segment, tmp_filename, num):
-            segment_url = segment['url']
-            segment_name = 'Frag%d' % num
-            target_filename = '%s-%s' % (tmp_filename, segment_name)
+        frag_index = 0
+        for i, segment in enumerate(segments):
+            frag_index += 1
+            if frag_index <= ctx['frag_index']:
+                continue
             # In DASH, the first segment contains necessary headers to
             # generate a valid MP4 file, so always abort for the first segment
-            fatal = num == 0 or not skip_unavailable_fragments
+            fatal = i == 0 or not skip_unavailable_fragments
             count = 0
             while count <= fragment_retries:
                 try:
-                    success = ctx['dl'].download(target_filename, {
-                        'url': segment_url,
-                        'http_headers': info_dict.get('http_headers'),
-                    })
+                    success, frag_content = self._download_fragment(ctx, segment['url'], info_dict)
                     if not success:
                         return False
-                    down, target_sanitized = sanitize_open(target_filename, 'rb')
-                    ctx['dest_stream'].write(down.read())
-                    down.close()
-                    segments_filenames.append(target_sanitized)
+                    self._append_fragment(ctx, frag_content)
                     break
                 except compat_urllib_error.HTTPError as err:
                     # YouTube may often return 404 HTTP error for a fragment causing the
@@ -63,22 +50,14 @@ class DashSegmentsFD(FragmentFD):
                     # HTTP error.
                     count += 1
                     if count <= fragment_retries:
-                        self.report_retry_fragment(err, segment_name, count, fragment_retries)
+                        self.report_retry_fragment(err, frag_index, count, fragment_retries)
             if count > fragment_retries:
                 if not fatal:
-                    self.report_skip_fragment(segment_name)
-                    return True
+                    self.report_skip_fragment(frag_index)
+                    continue
                 self.report_error('giving up after %s fragment retries' % fragment_retries)
                 return False
-            return True
-
-        for i, segment in enumerate(segments):
-            if not process_segment(segment, ctx['tmpfilename'], i):
-                return False
 
         self._finish_frag_download(ctx)
 
-        for segment_file in segments_filenames:
-            os.remove(encodeFilename(segment_file))
-
         return True
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 688e086eb..e456ed58f 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -3,7 +3,6 @@ from __future__ import division, unicode_literals
 import base64
 import io
 import itertools
-import os
 import time
 
 from .fragment import FragmentFD
@@ -16,9 +15,7 @@ from ..compat import (
     compat_struct_unpack,
 )
 from ..utils import (
-    encodeFilename,
     fix_xml_ampersands,
-    sanitize_open,
     xpath_text,
 )
 
@@ -366,17 +363,21 @@ class F4mFD(FragmentFD):
 
         dest_stream = ctx['dest_stream']
 
-        write_flv_header(dest_stream)
-        if not live:
-            write_metadata_tag(dest_stream, metadata)
+        if ctx['complete_frags_downloaded_bytes'] == 0:
+            write_flv_header(dest_stream)
+            if not live:
+                write_metadata_tag(dest_stream, metadata)
 
         base_url_parsed = compat_urllib_parse_urlparse(base_url)
 
         self._start_frag_download(ctx)
 
-        frags_filenames = []
+        frag_index = 0
         while fragments_list:
             seg_i, frag_i = fragments_list.pop(0)
+            frag_index += 1
+            if frag_index <= ctx['frag_index']:
+                continue
             name = 'Seg%d-Frag%d' % (seg_i, frag_i)
             query = []
             if base_url_parsed.query:
@@ -386,17 +387,10 @@ class F4mFD(FragmentFD):
             if info_dict.get('extra_param_to_segment_url'):
                 query.append(info_dict['extra_param_to_segment_url'])
             url_parsed = base_url_parsed._replace(path=base_url_parsed.path + name, query='&'.join(query))
-            frag_filename = '%s-%s' % (ctx['tmpfilename'], name)
             try:
-                success = ctx['dl'].download(frag_filename, {
-                    'url': url_parsed.geturl(),
-                    'http_headers': info_dict.get('http_headers'),
-                })
+                success, down_data = self._download_fragment(ctx, url_parsed.geturl(), info_dict)
                 if not success:
                     return False
-                (down, frag_sanitized) = sanitize_open(frag_filename, 'rb')
-                down_data = down.read()
-                down.close()
                 reader = FlvReader(down_data)
                 while True:
                     try:
@@ -411,12 +405,8 @@ class F4mFD(FragmentFD):
                             break
                         raise
                     if box_type == b'mdat':
-                        dest_stream.write(box_data)
+                        self._append_fragment(ctx, box_data)
                         break
-                if live:
-                    os.remove(encodeFilename(frag_sanitized))
-                else:
-                    frags_filenames.append(frag_sanitized)
             except (compat_urllib_error.HTTPError, ) as err:
                 if live and (err.code == 404 or err.code == 410):
                     # We didn't keep up with the live window. Continue
@@ -436,7 +426,4 @@ class F4mFD(FragmentFD):
 
         self._finish_frag_download(ctx)
 
-        for frag_file in frags_filenames:
-            os.remove(encodeFilename(frag_file))
-
         return True
diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 56f975266..44a3c1040 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -2,6 +2,7 @@ from __future__ import division, unicode_literals
 
 import os
 import time
+import io
 
 from .common import FileDownloader
 from .http import HttpFD
@@ -10,6 +11,7 @@ from ..utils import (
     encodeFilename,
     sanitize_open,
     sanitized_Request,
+    compat_str,
 )
 
 
@@ -30,13 +32,13 @@ class FragmentFD(FileDownloader):
                         Skip unavailable fragments (DASH and hlsnative only)
     """
 
-    def report_retry_fragment(self, err, fragment_name, count, retries):
+    def report_retry_fragment(self, err, frag_index, count, retries):
         self.to_screen(
-            '[download] Got server HTTP error: %s. Retrying fragment %s (attempt %d of %s)...'
-            % (error_to_compat_str(err), fragment_name, count, self.format_retries(retries)))
+            '[download] Got server HTTP error: %s. Retrying fragment %d (attempt %d of %s)...'
+            % (error_to_compat_str(err), frag_index, count, self.format_retries(retries)))
 
-    def report_skip_fragment(self, fragment_name):
-        self.to_screen('[download] Skipping fragment %s...' % fragment_name)
+    def report_skip_fragment(self, frag_index):
+        self.to_screen('[download] Skipping fragment %d...' % frag_index)
 
     def _prepare_url(self, info_dict, url):
         headers = info_dict.get('http_headers')
@@ -46,6 +48,25 @@ class FragmentFD(FileDownloader):
         self._prepare_frag_download(ctx)
         self._start_frag_download(ctx)
 
+    def _download_fragment(self, ctx, frag_url, info_dict, headers=None):
+        down = io.BytesIO()
+        success = ctx['dl'].download(down, {
+            'url': frag_url,
+            'http_headers': headers or info_dict.get('http_headers'),
+        })
+        if not success:
+            return False, None
+        frag_content = down.getvalue()
+        down.close()
+        return True, frag_content
+
+    def _append_fragment(self, ctx, frag_content):
+        ctx['dest_stream'].write(frag_content)
+        if not (ctx.get('live') or ctx['tmpfilename'] == '-'):
+            frag_index_stream, _ = sanitize_open(ctx['tmpfilename'] + '.fragindex', 'w')
+            frag_index_stream.write(compat_str(ctx['frag_index']))
+            frag_index_stream.close()
+
     def _prepare_frag_download(self, ctx):
         if 'live' not in ctx:
             ctx['live'] = False
@@ -66,11 +87,26 @@ class FragmentFD(FileDownloader):
             }
         )
         tmpfilename = self.temp_name(ctx['filename'])
-        dest_stream, tmpfilename = sanitize_open(tmpfilename, 'wb')
+        open_mode = 'wb'
+        resume_len = 0
+        frag_index = 0
+        # Establish possible resume length
+        if os.path.isfile(encodeFilename(tmpfilename)):
+            open_mode = 'ab'
+            resume_len = os.path.getsize(encodeFilename(tmpfilename))
+            if os.path.isfile(encodeFilename(tmpfilename + '.fragindex')):
+                frag_index_stream, _ = sanitize_open(tmpfilename + '.fragindex', 'r')
+                frag_index = int(frag_index_stream.read())
+                frag_index_stream.close()
+        dest_stream, tmpfilename = sanitize_open(tmpfilename, open_mode)
+
         ctx.update({
             'dl': dl,
             'dest_stream': dest_stream,
             'tmpfilename': tmpfilename,
+            'frag_index': frag_index,
+            # Total complete fragments downloaded so far in bytes
+            'complete_frags_downloaded_bytes': resume_len,
         })
 
     def _start_frag_download(self, ctx):
@@ -79,8 +115,8 @@ class FragmentFD(FileDownloader):
         # hook
         state = {
             'status': 'downloading',
-            'downloaded_bytes': 0,
-            'frag_index': 0,
+            'downloaded_bytes': ctx['complete_frags_downloaded_bytes'],
+            'frag_index': ctx['frag_index'],
             'frag_count': total_frags,
             'filename': ctx['filename'],
             'tmpfilename': ctx['tmpfilename'],
@@ -89,8 +125,6 @@ class FragmentFD(FileDownloader):
         start = time.time()
         ctx.update({
             'started': start,
-            # Total complete fragments downloaded so far in bytes
-            'complete_frags_downloaded_bytes': 0,
             # Amount of fragment's bytes downloaded by the time of the previous
             # frag progress hook invocation
             'prev_frag_downloaded_bytes': 0,
@@ -111,6 +145,7 @@ class FragmentFD(FileDownloader):
 
             if s['status'] == 'finished':
                 state['frag_index'] += 1
+                ctx['frag_index'] = state['frag_index']
                 state['downloaded_bytes'] += frag_total_bytes - ctx['prev_frag_downloaded_bytes']
                 ctx['complete_frags_downloaded_bytes'] = state['downloaded_bytes']
                 ctx['prev_frag_downloaded_bytes'] = 0
@@ -132,6 +167,8 @@ class FragmentFD(FileDownloader):
 
     def _finish_frag_download(self, ctx):
         ctx['dest_stream'].close()
+        if os.path.isfile(encodeFilename(ctx['tmpfilename'] + '.fragindex')):
+            os.remove(encodeFilename(ctx['tmpfilename'] + '.fragindex'))
         elapsed = time.time() - ctx['started']
         self.try_rename(ctx['tmpfilename'], ctx['filename'])
         fsize = os.path.getsize(encodeFilename(ctx['filename']))
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index d0a5f7ba4..9a87d7ca8 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import os.path
 import re
 import binascii
 try:
@@ -18,8 +17,6 @@ from ..compat import (
     compat_struct_pack,
 )
 from ..utils import (
-    encodeFilename,
-    sanitize_open,
     parse_m3u8_attributes,
     update_url_query,
 )
@@ -103,17 +100,18 @@ class HlsFD(FragmentFD):
         media_sequence = 0
         decrypt_info = {'METHOD': 'NONE'}
         byte_range = {}
-        frags_filenames = []
+        frag_index = 0
         for line in s.splitlines():
             line = line.strip()
             if line:
                 if not line.startswith('#'):
+                    frag_index += 1
+                    if frag_index <= ctx['frag_index']:
+                        continue
                     frag_url = (
                         line
                         if re.match(r'^https?://', line)
                         else compat_urlparse.urljoin(man_url, line))
-                    frag_name = 'Frag%d' % i
-                    frag_filename = '%s-%s' % (ctx['tmpfilename'], frag_name)
                     if extra_query:
                         frag_url = update_url_query(frag_url, extra_query)
                     count = 0
@@ -122,15 +120,10 @@ class HlsFD(FragmentFD):
                         headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end'])
                     while count <= fragment_retries:
                         try:
-                            success = ctx['dl'].download(frag_filename, {
-                                'url': frag_url,
-                                'http_headers': headers,
-                            })
+                            success, frag_content = self._download_fragment(
+                                ctx, frag_url, info_dict, headers)
                             if not success:
                                 return False
-                            down, frag_sanitized = sanitize_open(frag_filename, 'rb')
-                            frag_content = down.read()
-                            down.close()
                             break
                         except compat_urllib_error.HTTPError as err:
                             # Unavailable (possibly temporary) fragments may be served.
@@ -139,28 +132,29 @@ class HlsFD(FragmentFD):
                             # https://github.com/rg3/youtube-dl/issues/10448).
                             count += 1
                             if count <= fragment_retries:
-                                self.report_retry_fragment(err, frag_name, count, fragment_retries)
+                                self.report_retry_fragment(err, frag_index, count, fragment_retries)
                     if count > fragment_retries:
                         if skip_unavailable_fragments:
                             i += 1
                             media_sequence += 1
-                            self.report_skip_fragment(frag_name)
+                            self.report_skip_fragment(frag_index)
                             continue
                         self.report_error(
                             'giving up after %s fragment retries' % fragment_retries)
                         return False
                     if decrypt_info['METHOD'] == 'AES-128':
                         iv = decrypt_info.get('IV') or compat_struct_pack('>8xq', media_sequence)
+                        decrypt_info['KEY'] = decrypt_info.get('KEY') or self.ydl.urlopen(decrypt_info['URI']).read()
                         frag_content = AES.new(
                             decrypt_info['KEY'], AES.MODE_CBC, iv).decrypt(frag_content)
-                    ctx['dest_stream'].write(frag_content)
-                    frags_filenames.append(frag_sanitized)
+                    self._append_fragment(ctx, frag_content)
                     # We only download the first fragment during the test
                     if test:
                         break
                     i += 1
                     media_sequence += 1
                 elif line.startswith('#EXT-X-KEY'):
+                    decrypt_url = decrypt_info.get('URI')
                     decrypt_info = parse_m3u8_attributes(line[11:])
                     if decrypt_info['METHOD'] == 'AES-128':
                         if 'IV' in decrypt_info:
@@ -170,7 +164,8 @@ class HlsFD(FragmentFD):
                                 man_url, decrypt_info['URI'])
                         if extra_query:
                             decrypt_info['URI'] = update_url_query(decrypt_info['URI'], extra_query)
-                        decrypt_info['KEY'] = self.ydl.urlopen(decrypt_info['URI']).read()
+                        if decrypt_url != decrypt_info['URI']:
+                            decrypt_info['KEY'] = None
                 elif line.startswith('#EXT-X-MEDIA-SEQUENCE'):
                     media_sequence = int(line[22:])
                 elif line.startswith('#EXT-X-BYTERANGE'):
@@ -183,7 +178,4 @@ class HlsFD(FragmentFD):
 
         self._finish_frag_download(ctx)
 
-        for frag_file in frags_filenames:
-            os.remove(encodeFilename(frag_file))
-
         return True
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index af405b950..2896a17af 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -20,10 +20,14 @@ from ..utils import (
 
 
 class HttpFD(FileDownloader):
-    def real_download(self, filename, info_dict):
+    def real_download(self, filename_or_stream, info_dict):
         url = info_dict['url']
-        tmpfilename = self.temp_name(filename)
+        filename = filename_or_stream
         stream = None
+        if hasattr(filename_or_stream, 'write'):
+            stream = filename_or_stream
+            filename = '-'
+        tmpfilename = self.temp_name(filename)
 
         # Do not include the Accept-Encoding header
         headers = {'Youtubedl-no-compression': 'True'}
diff --git a/youtube_dl/downloader/ism.py b/youtube_dl/downloader/ism.py
index 63a636cb7..9f0fc36b3 100644
--- a/youtube_dl/downloader/ism.py
+++ b/youtube_dl/downloader/ism.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import os
 import time
 import struct
 import binascii
@@ -8,10 +7,6 @@ import io
 
 from .fragment import FragmentFD
 from ..compat import compat_urllib_error
-from ..utils import (
-    sanitize_open,
-    encodeFilename,
-)
 
 
 u8 = struct.Struct(b'>B')
@@ -225,16 +220,15 @@ class IsmFD(FragmentFD):
 
         self._prepare_and_start_frag_download(ctx)
 
-        segments_filenames = []
-
         fragment_retries = self.params.get('fragment_retries', 0)
         skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
 
         track_written = False
+        frag_index = 0
         for i, segment in enumerate(segments):
-            segment_url = segment['url']
-            segment_name = 'Frag%d' % i
-            target_filename = '%s-%s' % (ctx['tmpfilename'], segment_name)
+            frag_index += 1
+            if frag_index <= ctx['frag_index']:
+                continue
             count = 0
             while count <= fragment_retries:
                 try:
@@ -242,33 +236,27 @@ class IsmFD(FragmentFD):
                         'url': segment_url,
                         'http_headers': info_dict.get('http_headers'),
                     })
+                    success, frag_content = self._download_fragment(ctx, segment['url'], info_dict)
                     if not success:
                         return False
-                    down, target_sanitized = sanitize_open(target_filename, 'rb')
-                    down_data = down.read()
                     if not track_written:
-                        tfhd_data = extract_box_data(down_data, [b'moof', b'traf', b'tfhd'])
+                        tfhd_data = extract_box_data(frag_content, [b'moof', b'traf', b'tfhd'])
                         info_dict['_download_params']['track_id'] = u32.unpack(tfhd_data[4:8])[0]
                         write_piff_header(ctx['dest_stream'], info_dict['_download_params'])
                         track_written = True
-                    ctx['dest_stream'].write(down_data)
-                    down.close()
-                    segments_filenames.append(target_sanitized)
+                    self._append_fragment(ctx, frag_content)
                     break
                 except compat_urllib_error.HTTPError as err:
                     count += 1
                     if count <= fragment_retries:
-                        self.report_retry_fragment(err, segment_name, count, fragment_retries)
+                        self.report_retry_fragment(err, frag_index, count, fragment_retries)
             if count > fragment_retries:
                 if skip_unavailable_fragments:
-                    self.report_skip_fragment(segment_name)
+                    self.report_skip_fragment(frag_index)
                     continue
                 self.report_error('giving up after %s fragment retries' % fragment_retries)
                 return False
 
         self._finish_frag_download(ctx)
 
-        for segment_file in segments_filenames:
-            os.remove(encodeFilename(segment_file))
-
         return True

From 5b995f713bf45e9e373029e1411811dfde855088 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 23 Feb 2017 18:46:20 +0100
Subject: [PATCH 137/611] [utils] add support for ttml styles

---
 test/test_utils.py  |  41 +++++++++++++++
 youtube_dl/utils.py | 120 ++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 152 insertions(+), 9 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index aa4569b81..4cd818850 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1069,6 +1069,47 @@ The first line
 '''
         self.assertEqual(dfxp2srt(dfxp_data_no_default_namespace), srt_data)
 
+        dfxp_data_with_style = '''<?xml version="1.0" encoding="utf-8"?>
+<tt xmlns="http://www.w3.org/2006/10/ttaf1" xmlns:ttp="http://www.w3.org/2006/10/ttaf1#parameter" ttp:timeBase="media" xmlns:tts="http://www.w3.org/2006/10/ttaf1#style" xml:lang="en" xmlns:ttm="http://www.w3.org/2006/10/ttaf1#metadata">
+  <head>
+    <styling>
+      <style id="s2" style="s0" tts:color="cyan" tts:fontWeight="bold" />
+      <style id="s1" style="s0" tts:color="yellow" tts:fontStyle="italic" />
+      <style id="s3" style="s0" tts:color="lime" tts:textDecoration="underline" />
+      <style id="s0" tts:backgroundColor="black" tts:fontStyle="normal" tts:fontSize="16" tts:fontFamily="sansSerif" tts:color="white" />
+    </styling>
+  </head>
+  <body tts:textAlign="center" style="s0">
+    <div>
+      <p begin="00:00:02.08" id="p0" end="00:00:05.84">default style<span tts:color="red">custom style</span></p>
+      <p style="s2" begin="00:00:02.08" id="p0" end="00:00:05.84"><span tts:color="lime">part 1<br /></span><span tts:color="cyan">part 2</span></p>
+      <p style="s3" begin="00:00:05.84" id="p1" end="00:00:09.56">line 3<br />part 3</p>
+      <p style="s1" tts:textDecoration="underline" begin="00:00:09.56" id="p2" end="00:00:12.36"><span style="s2" tts:color="lime">inner<br /> </span>style</p>
+    </div>
+  </body>
+</tt>'''
+        srt_data = '''1
+00:00:02,080 --> 00:00:05,839
+<font color="white" face="sansSerif" size="16">default style<font color="red">custom style</font></font>
+
+2
+00:00:02,080 --> 00:00:05,839
+<b><font color="cyan" face="sansSerif" size="16"><font color="lime">part 1
+</font>part 2</font></b>
+
+3
+00:00:05,839 --> 00:00:09,560
+<u><font color="lime">line 3
+part 3</font></u>
+
+4
+00:00:09,560 --> 00:00:12,359
+<i><u><font color="yellow"><font color="lime">inner
+ </font>style</font></u></i>
+
+'''
+        self.assertEqual(dfxp2srt(dfxp_data_with_style), srt_data)
+
     def test_cli_option(self):
         self.assertEqual(cli_option({'proxy': '127.0.0.1:3128'}, '--proxy', 'proxy'), ['--proxy', '127.0.0.1:3128'])
         self.assertEqual(cli_option({'proxy': None}, '--proxy', 'proxy'), [])
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 79a337df4..91e235ff2 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2511,27 +2511,97 @@ def srt_subtitles_timecode(seconds):
 
 
 def dfxp2srt(dfxp_data):
+    LEGACY_NAMESPACES = (
+        ('http://www.w3.org/ns/ttml', [
+            'http://www.w3.org/2004/11/ttaf1',
+            'http://www.w3.org/2006/04/ttaf1',
+            'http://www.w3.org/2006/10/ttaf1',
+        ]),
+        ('http://www.w3.org/ns/ttml#styling', [
+            'http://www.w3.org/ns/ttml#style',
+        ]),
+    )
+
+    SUPPORTED_STYLING = [
+        'color',
+        'fontFamily',
+        'fontSize',
+        'fontStyle',
+        'fontWeight',
+        'textDecoration'
+    ]
+
     _x = functools.partial(xpath_with_ns, ns_map={
         'ttml': 'http://www.w3.org/ns/ttml',
-        'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
-        'ttaf1_0604': 'http://www.w3.org/2006/04/ttaf1',
+        'tts': 'http://www.w3.org/ns/ttml#styling',
     })
 
+    styles = {}
+    default_style = {}
+
     class TTMLPElementParser(object):
-        out = ''
+        _out = ''
+        _unclosed_elements = []
+        _applied_styles = []
 
         def start(self, tag, attrib):
-            if tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
-                self.out += '\n'
+            if tag in (_x('ttml:br'), 'br'):
+                self._out += '\n'
+            else:
+                unclosed_elements = []
+                style = {}
+                element_style_id = attrib.get('style')
+                if default_style:
+                    style.update(default_style)
+                if element_style_id:
+                    style.update(styles.get(element_style_id, {}))
+                for prop in SUPPORTED_STYLING:
+                    prop_val = attrib.get(_x('tts:' + prop))
+                    if prop_val:
+                        style[prop] = prop_val
+                if style:
+                    font = ''
+                    for k, v in sorted(style.items()):
+                        if self._applied_styles and self._applied_styles[-1].get(k) == v:
+                            continue
+                        if k == 'color':
+                            font += ' color="%s"' % v
+                        elif k == 'fontSize':
+                            font += ' size="%s"' % v
+                        elif k == 'fontFamily':
+                            font += ' face="%s"' % v
+                        elif k == 'fontWeight' and v == 'bold':
+                            self._out += '<b>'
+                            unclosed_elements.append('b')
+                        elif k == 'fontStyle' and v == 'italic':
+                            self._out += '<i>'
+                            unclosed_elements.append('i')
+                        elif k == 'textDecoration' and v == 'underline':
+                            self._out += '<u>'
+                            unclosed_elements.append('u')
+                    if font:
+                        self._out += '<font' + font + '>'
+                        unclosed_elements.append('font')
+                    applied_style = {}
+                    if self._applied_styles:
+                        applied_style.update(self._applied_styles[-1])
+                    applied_style.update(style)
+                    self._applied_styles.append(applied_style)
+                self._unclosed_elements.append(unclosed_elements)
 
         def end(self, tag):
-            pass
+            if tag not in (_x('ttml:br'), 'br'):
+                unclosed_elements = self._unclosed_elements.pop()
+                for element in reversed(unclosed_elements):
+                    self._out += '</%s>' % element
+                if unclosed_elements and self._applied_styles:
+                    self._applied_styles.pop()
 
         def data(self, data):
-            self.out += data
+            self._out += data
 
         def close(self):
-            return self.out.strip()
+            return self._out.strip()
 
     def parse_node(node):
         target = TTMLPElementParser()
@@ -2539,13 +2609,45 @@ def dfxp2srt(dfxp_data):
         parser.feed(xml.etree.ElementTree.tostring(node))
         return parser.close()
 
+    for k, v in LEGACY_NAMESPACES:
+        for ns in v:
+            dfxp_data = dfxp_data.replace(ns, k)
+
     dfxp = compat_etree_fromstring(dfxp_data.encode('utf-8'))
     out = []
-    paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall(_x('.//ttaf1_0604:p')) or dfxp.findall('.//p')
+    paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall('.//p')
 
     if not paras:
         raise ValueError('Invalid dfxp/TTML subtitle')
 
+    repeat = False
+    while True:
+        for style in dfxp.findall(_x('.//ttml:style')):
+            style_id = style.get('id')
+            parent_style_id = style.get('style')
+            if parent_style_id:
+                if parent_style_id not in styles:
+                    repeat = True
+                    continue
+                styles[style_id] = styles[parent_style_id].copy()
+            for prop in SUPPORTED_STYLING:
+                prop_val = style.get(_x('tts:' + prop))
+                if prop_val:
+                    styles.setdefault(style_id, {})[prop] = prop_val
+        if repeat:
+            repeat = False
+        else:
+            break
+
+    for p in ('body', 'div'):
+        ele = xpath_element(dfxp, [_x('.//ttml:' + p), './/' + p])
+        if ele is None:
+            continue
+        style = styles.get(ele.get('style'))
+        if not style:
+            continue
+        default_style.update(style)
+
     for para, index in zip(paras, itertools.count(1)):
         begin_time = parse_dfxp_time_expr(para.attrib.get('begin'))
         end_time = parse_dfxp_time_expr(para.attrib.get('end'))

From 481ef51e2345b1b34b16148eb6e5e58e9fb45cfb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 19 Apr 2017 21:47:03 +0700
Subject: [PATCH 138/611] [brightcove] PEP 8

---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index a2db705b0..183ac7dd0 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -277,7 +277,7 @@ class BrightcoveLegacyIE(InfoExtractor):
             return list(filter(None, [
                 cls._build_brighcove_url_from_js(custom_bc)
                 for custom_bc in matches]))
-        return [url for _, url in re.findall(
+        return [src for _, src in re.findall(
             r'<iframe[^>]+src=([\'"])((?:https?:)?//link\.brightcove\.com/services/player/(?!\1).+)\1', webpage)]
 
     def _real_extract(self, url):

From ea0c2f219c219de8f59f1ae82e106cec5911c56c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 19 Apr 2017 18:34:25 +0100
Subject: [PATCH 139/611] [downloader/fragment] use a general file to store
 fragment download context

---
 youtube_dl/downloader/common.py   |  3 +++
 youtube_dl/downloader/fragment.py | 19 ++++++++++++-------
 2 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index fdb77b620..5d6621147 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -187,6 +187,9 @@ class FileDownloader(object):
             return filename[:-len('.part')]
         return filename
 
+    def ytdl_filename(self, filename):
+        return filename + '.ytdl'
+
     def try_rename(self, old_filename, new_filename):
         try:
             if old_filename == new_filename:
diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 44a3c1040..80bb14d61 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -3,6 +3,7 @@ from __future__ import division, unicode_literals
 import os
 import time
 import io
+import json
 
 from .common import FileDownloader
 from .http import HttpFD
@@ -63,8 +64,10 @@ class FragmentFD(FileDownloader):
     def _append_fragment(self, ctx, frag_content):
         ctx['dest_stream'].write(frag_content)
         if not (ctx.get('live') or ctx['tmpfilename'] == '-'):
-            frag_index_stream, _ = sanitize_open(ctx['tmpfilename'] + '.fragindex', 'w')
-            frag_index_stream.write(compat_str(ctx['frag_index']))
+            frag_index_stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'w')
+            frag_index_stream.write(json.dumps({
+                'frag_index': ctx['frag_index']
+            }))
             frag_index_stream.close()
 
     def _prepare_frag_download(self, ctx):
@@ -94,9 +97,10 @@ class FragmentFD(FileDownloader):
         if os.path.isfile(encodeFilename(tmpfilename)):
             open_mode = 'ab'
             resume_len = os.path.getsize(encodeFilename(tmpfilename))
-            if os.path.isfile(encodeFilename(tmpfilename + '.fragindex')):
-                frag_index_stream, _ = sanitize_open(tmpfilename + '.fragindex', 'r')
-                frag_index = int(frag_index_stream.read())
+            ytdl_filename = encodeFilename(self.ytdl_filename(ctx['filename']))
+            if os.path.isfile(ytdl_filename):
+                frag_index_stream, _ = sanitize_open(ytdl_filename, 'r')
+                frag_index = json.loads(frag_index_stream.read())['frag_index']
                 frag_index_stream.close()
         dest_stream, tmpfilename = sanitize_open(tmpfilename, open_mode)
 
@@ -167,8 +171,9 @@ class FragmentFD(FileDownloader):
 
     def _finish_frag_download(self, ctx):
         ctx['dest_stream'].close()
-        if os.path.isfile(encodeFilename(ctx['tmpfilename'] + '.fragindex')):
-            os.remove(encodeFilename(ctx['tmpfilename'] + '.fragindex'))
+        ytdl_filename = encodeFilename(self.ytdl_filename(ctx['filename']))
+        if os.path.isfile(ytdl_filename):
+            os.remove(ytdl_filename)
         elapsed = time.time() - ctx['started']
         self.try_rename(ctx['tmpfilename'], ctx['filename'])
         fsize = os.path.getsize(encodeFilename(ctx['filename']))

From 8abc7dca39236462c0652932df23862acda92d51 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 20 Apr 2017 05:16:41 +0100
Subject: [PATCH 140/611] [amp] extract error message(closes #12795)

---
 youtube_dl/extractor/amp.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/amp.py b/youtube_dl/extractor/amp.py
index e8e40126b..98f8e69cd 100644
--- a/youtube_dl/extractor/amp.py
+++ b/youtube_dl/extractor/amp.py
@@ -7,15 +7,19 @@ from ..utils import (
     parse_iso8601,
     mimetype2ext,
     determine_ext,
+    ExtractorError,
 )
 
 
 class AMPIE(InfoExtractor):
     # parse Akamai Adaptive Media Player feed
     def _extract_feed_info(self, url):
-        item = self._download_json(
+        feed = self._download_json(
             url, None, 'Downloading Akamai AMP feed',
-            'Unable to download Akamai AMP feed')['channel']['item']
+            'Unable to download Akamai AMP feed')
+        item = feed.get('channel', {}).get('item')
+        if not item:
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, feed['error']))
 
         video_id = item['guid']
 

From f7799582500ca2b00c371c287028298f187c0477 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 21 Apr 2017 23:37:06 +0700
Subject: [PATCH 141/611] [vidzi] Fix extraction (closes #12793)

---
 youtube_dl/extractor/vidzi.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index d0556297e..e64873bce 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -42,14 +42,15 @@ class VidziIE(InfoExtractor):
         title = self._html_search_regex(
             r'(?s)<h2 class="video-title">(.*?)</h2>', webpage, 'title')
 
-        packed_codes = [mobj.group(0) for mobj in re.finditer(
-            PACKED_CODES_RE, webpage)]
-        for num, pc in enumerate(packed_codes, 1):
-            code = decode_packed_codes(pc).replace('\\\'', '\'')
+        codes = [webpage]
+        codes.extend([
+            decode_packed_codes(mobj.group(0)).replace('\\\'', '\'')
+            for mobj in re.finditer(PACKED_CODES_RE, webpage)])
+        for num, code in enumerate(codes, 1):
             jwplayer_data = self._parse_json(
                 self._search_regex(
                     r'setup\(([^)]+)\)', code, 'jwplayer data',
-                    default=NO_DEFAULT if num == len(packed_codes) else '{}'),
+                    default=NO_DEFAULT if num == len(codes) else '{}'),
                 video_id, transform_source=js_to_json)
             if jwplayer_data:
                 break

From cb2520802d7b8efdc71d6d97aeca984b5f878716 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 22 Apr 2017 07:01:00 +0700
Subject: [PATCH 142/611] [extractor/common] Improve m3u8 extraction (closes
 #12211) * Extract m3u8 parsing to separate method * Improve rendition groups
 extraction * Build stream name according stream GROUP-ID * Ignore reference
 to AUDIO group without URI when stream has no CODECS + Add test coverage for
 parsing m3u8 from #11507, #11995, #12211 and twitch vod

---
 .gitignore                                   |   1 -
 test/test_InfoExtractor.py                   | 308 ++++++++++++++++++-
 test/testdata/m3u8/pluzz_francetv_11507.m3u8 |  14 +
 test/testdata/m3u8/teamcoco_11995.m3u8       |  16 +
 test/testdata/m3u8/toggle_mobile_12211.m3u8  |  13 +
 test/testdata/m3u8/twitch_vod.m3u8           |  20 ++
 youtube_dl/extractor/common.py               | 154 ++++++----
 7 files changed, 466 insertions(+), 60 deletions(-)
 create mode 100644 test/testdata/m3u8/pluzz_francetv_11507.m3u8
 create mode 100644 test/testdata/m3u8/teamcoco_11995.m3u8
 create mode 100644 test/testdata/m3u8/toggle_mobile_12211.m3u8
 create mode 100644 test/testdata/m3u8/twitch_vod.m3u8

diff --git a/.gitignore b/.gitignore
index 9ce4b5e2d..bef084baa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -36,7 +36,6 @@ updates_key.pem
 *.swf
 *.part
 *.swp
-test/testdata
 test/local_parameters.json
 .tox
 youtube-dl.zsh
diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 881197afb..57205f6ae 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -3,12 +3,13 @@
 from __future__ import unicode_literals
 
 # Allow direct execution
+import io
 import os
 import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from test.helper import FakeYDL, expect_dict
+from test.helper import FakeYDL, expect_dict, expect_value
 from youtube_dl.extractor.common import InfoExtractor
 from youtube_dl.extractor import YoutubeIE, get_info_extractor
 from youtube_dl.utils import encode_data_uri, strip_jsonp, ExtractorError, RegexNotFoundError
@@ -175,6 +176,311 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 }]
             })
 
+    def test_parse_m3u8_formats(self):
+        _TEST_CASES = [
+            (
+                # https://github.com/rg3/youtube-dl/issues/11507
+                # http://pluzz.francetv.fr/videos/le_ministere.html
+                'pluzz_francetv_11507',
+                'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/master.m3u8?caption=2017%2F16%2F156589847-1492488987.m3u8%3Afra%3AFrancais&audiotrack=0%3Afra%3AFrancais',
+                [{
+                    'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/master.m3u8?caption=2017%2F16%2F156589847-1492488987.m3u8%3Afra%3AFrancais&audiotrack=0%3Afra%3AFrancais',
+                    'ext': 'mp4',
+                    'format_id': 'meta',
+                    'format_note': 'Quality selection URL',
+                    'protocol': 'm3u8',
+                    'preference': -100,
+                    'resolution': 'multiple'
+                }, {
+                    'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_0_av.m3u8?null=0',
+                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_0_av.m3u8?null=0',
+                    'ext': 'mp4',
+                    'format_id': '180',
+                    'protocol': 'm3u8',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc1.66.30',
+                    'tbr': 180,
+                    'width': 256,
+                    'height': 144,
+                }, {
+                    'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_1_av.m3u8?null=0',
+                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_1_av.m3u8?null=0',
+                    'ext': 'mp4',
+                    'format_id': '303',
+                    'protocol': 'm3u8',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc1.66.30',
+                    'tbr': 303,
+                    'width': 320,
+                    'height': 180,
+                }, {
+                    'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_2_av.m3u8?null=0',
+                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_2_av.m3u8?null=0',
+                    'ext': 'mp4',
+                    'format_id': '575',
+                    'protocol': 'm3u8',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc1.66.30',
+                    'tbr': 575,
+                    'width': 512,
+                    'height': 288,
+                }, {
+                    'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_3_av.m3u8?null=0',
+                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_3_av.m3u8?null=0',
+                    'ext': 'mp4',
+                    'format_id': '831',
+                    'protocol': 'm3u8',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc1.77.30',
+                    'tbr': 831,
+                    'width': 704,
+                    'height': 396,
+                }, {
+                    'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_4_av.m3u8?null=0',
+                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_4_av.m3u8?null=0',
+                    'ext': 'mp4',
+                    'protocol': 'm3u8',
+                    'format_id': '1467',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc1.77.30',
+                    'tbr': 1467,
+                    'width': 1024,
+                    'height': 576,
+                }]
+            ),
+            (
+                # https://github.com/rg3/youtube-dl/issues/11995
+                # http://teamcoco.com/video/clueless-gamer-super-bowl-for-honor
+                'teamcoco_11995',
+                'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
+                [{
+                    'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'meta',
+                    'format_note': 'Quality selection URL',
+                    'protocol': 'm3u8',
+                    'preference': -100,
+                    'resolution': 'multiple',
+                }, {
+                    'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-audio-160k_v4.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'audio-0-Default',
+                    'protocol': 'm3u8',
+                    'vcodec': 'none',
+                }, {
+                    'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-audio-64k_v4.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'audio-1-Default',
+                    'protocol': 'm3u8',
+                    'vcodec': 'none',
+                }, {
+                    'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-audio-64k_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-audio-64k_v4.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '71',
+                    'protocol': 'm3u8',
+                    'acodec': 'mp4a.40.5',
+                    'vcodec': 'none',
+                    'tbr': 71,
+                }, {
+                    'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-400k_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-400k_v4.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '413',
+                    'protocol': 'm3u8',
+                    'acodec': 'none',
+                    'vcodec': 'avc1.42001e',
+                    'tbr': 413,
+                    'width': 400,
+                    'height': 224,
+                }, {
+                    'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-400k_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-400k_v4.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '522',
+                    'protocol': 'm3u8',
+                    'acodec': 'none',
+                    'vcodec': 'avc1.42001e',
+                    'tbr': 522,
+                    'width': 400,
+                    'height': 224,
+                }, {
+                    'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-1m_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-1m_v4.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '1205',
+                    'protocol': 'm3u8',
+                    'acodec': 'none',
+                    'vcodec': 'avc1.4d001e',
+                    'tbr': 1205,
+                    'width': 640,
+                    'height': 360,
+                }, {
+                    'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-2m_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-2m_v4.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '2374',
+                    'protocol': 'm3u8',
+                    'acodec': 'none',
+                    'vcodec': 'avc1.4d001f',
+                    'tbr': 2374,
+                    'width': 1024,
+                    'height': 576,
+                }]
+            ),
+            (
+                # https://github.com/rg3/youtube-dl/issues/12211
+                # http://video.toggle.sg/en/series/whoopie-s-world/ep3/478601
+                'toggle_mobile_12211',
+                'http://cdnapi.kaltura.com/p/2082311/sp/208231100/playManifest/protocol/http/entryId/0_89q6e8ku/format/applehttp/tags/mobile_sd/f/a.m3u8',
+                [{
+                    'url': 'http://cdnapi.kaltura.com/p/2082311/sp/208231100/playManifest/protocol/http/entryId/0_89q6e8ku/format/applehttp/tags/mobile_sd/f/a.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'meta',
+                    'format_note': 'Quality selection URL',
+                    'protocol': 'm3u8',
+                    'preference': -100,
+                    'resolution': 'multiple'
+                }, {
+                    'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_sa2ntrdg/name/a.mp4/index.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'audio-English',
+                    'protocol': 'm3u8',
+                    'language': 'eng',
+                    'vcodec': 'none',
+                }, {
+                    'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_r7y0nitg/name/a.mp4/index.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'audio-Undefined',
+                    'protocol': 'm3u8',
+                    'language': 'und',
+                    'vcodec': 'none',
+                }, {
+                    'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_qlk9hlzr/name/a.mp4/index.m3u8',
+                    'manifest_url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_qlk9hlzr/name/a.mp4/index.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '155',
+                    'protocol': 'm3u8',
+                    'tbr': 155,
+                    'width': 320,
+                    'height': 180,
+                }, {
+                    'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_oefackmi/name/a.mp4/index.m3u8',
+                    'manifest_url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_oefackmi/name/a.mp4/index.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '502',
+                    'protocol': 'm3u8',
+                    'tbr': 502,
+                    'width': 480,
+                    'height': 270,
+                }, {
+                    'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/12/pv/1/flavorId/0_vyg9pj7k/name/a.mp4/index.m3u8',
+                    'manifest_url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/12/pv/1/flavorId/0_vyg9pj7k/name/a.mp4/index.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '827',
+                    'protocol': 'm3u8',
+                    'tbr': 827,
+                    'width': 640,
+                    'height': 360,
+                }, {
+                    'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/12/pv/1/flavorId/0_50n4psvx/name/a.mp4/index.m3u8',
+                    'manifest_url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/12/pv/1/flavorId/0_50n4psvx/name/a.mp4/index.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '1396',
+                    'protocol': 'm3u8',
+                    'tbr': 1396,
+                    'width': 854,
+                    'height': 480,
+                }]
+            ),
+            (
+                # http://www.twitch.tv/riotgames/v/6528877
+                'twitch_vod',
+                'https://usher.ttvnw.net/vod/6528877?allow_source=true&allow_audio_only=true&allow_spectre=true&player=twitchweb&nauth=%7B%22user_id%22%3Anull%2C%22vod_id%22%3A6528877%2C%22expires%22%3A1492887874%2C%22chansub%22%3A%7B%22restricted_bitrates%22%3A%5B%5D%7D%2C%22privileged%22%3Afalse%2C%22https_required%22%3Afalse%7D&nauthsig=3e29296a6824a0f48f9e731383f77a614fc79bee',
+                [{
+                    'url': 'https://usher.ttvnw.net/vod/6528877?allow_source=true&allow_audio_only=true&allow_spectre=true&player=twitchweb&nauth=%7B%22user_id%22%3Anull%2C%22vod_id%22%3A6528877%2C%22expires%22%3A1492887874%2C%22chansub%22%3A%7B%22restricted_bitrates%22%3A%5B%5D%7D%2C%22privileged%22%3Afalse%2C%22https_required%22%3Afalse%7D&nauthsig=3e29296a6824a0f48f9e731383f77a614fc79bee',
+                    'ext': 'mp4',
+                    'format_id': 'meta',
+                    'format_note': 'Quality selection URL',
+                    'protocol': 'm3u8',
+                    'preference': -100,
+                    'resolution': 'multiple'
+                }, {
+                    'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/audio_only/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/audio_only/index-muted-HM49I092CC.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'Audio Only',
+                    'protocol': 'm3u8',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'none',
+                    'tbr': 182,
+                }, {
+                    'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/mobile/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/mobile/index-muted-HM49I092CC.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'Mobile',
+                    'protocol': 'm3u8',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc1.42C00D',
+                    'tbr': 280,
+                    'width': 400,
+                    'height': 226,
+                }, {
+                    'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/low/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/low/index-muted-HM49I092CC.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'Low',
+                    'protocol': 'm3u8',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc1.42C01E',
+                    'tbr': 628,
+                    'width': 640,
+                    'height': 360,
+                }, {
+                    'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/medium/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/medium/index-muted-HM49I092CC.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'Medium',
+                    'protocol': 'm3u8',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc1.42C01E',
+                    'tbr': 893,
+                    'width': 852,
+                    'height': 480,
+                }, {
+                    'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/high/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/high/index-muted-HM49I092CC.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'High',
+                    'protocol': 'm3u8',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc1.42C01F',
+                    'tbr': 1603,
+                    'width': 1280,
+                    'height': 720,
+                }, {
+                    'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/chunked/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/chunked/index-muted-HM49I092CC.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'Source',
+                    'protocol': 'm3u8',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc1.100.31',
+                    'tbr': 3214,
+                    'width': 1280,
+                    'height': 720,
+                }]
+            )
+        ]
+
+        for m3u8_file, m3u8_url, expected_formats in _TEST_CASES:
+            with io.open('./test/testdata/m3u8/%s.m3u8' % m3u8_file,
+                         mode='r', encoding='utf-8') as f:
+                formats = self.ie._parse_m3u8_formats(
+                    f.read(), m3u8_url, ext='mp4')
+                self.ie._sort_formats(formats)
+                expect_value(self, formats, expected_formats, None)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/testdata/m3u8/pluzz_francetv_11507.m3u8 b/test/testdata/m3u8/pluzz_francetv_11507.m3u8
new file mode 100644
index 000000000..0809f5aa0
--- /dev/null
+++ b/test/testdata/m3u8/pluzz_francetv_11507.m3u8
@@ -0,0 +1,14 @@
+#EXTM3U
+    
#EXT-X-VERSION:5
+    
#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID="subs",NAME="Francais",DEFAULT=NO,FORCED=NO,URI="http://replayftv-pmd.francetv.fr/subtitles/2017/16/156589847-1492488987.m3u8",LANGUAGE="fra"
+    
#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="aac",LANGUAGE="fra",NAME="Francais",DEFAULT=YES, AUTOSELECT=YES
+#EXT-X-STREAM-INF:SUBTITLES="subs",AUDIO="aac",PROGRAM-ID=1,BANDWIDTH=180000,RESOLUTION=256x144,CODECS="avc1.66.30, mp4a.40.2"
+http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_0_av.m3u8?null=0
+#EXT-X-STREAM-INF:SUBTITLES="subs",AUDIO="aac",PROGRAM-ID=1,BANDWIDTH=303000,RESOLUTION=320x180,CODECS="avc1.66.30, mp4a.40.2"
+http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_1_av.m3u8?null=0
+#EXT-X-STREAM-INF:SUBTITLES="subs",AUDIO="aac",PROGRAM-ID=1,BANDWIDTH=575000,RESOLUTION=512x288,CODECS="avc1.66.30, mp4a.40.2"
+http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_2_av.m3u8?null=0
+#EXT-X-STREAM-INF:SUBTITLES="subs",AUDIO="aac",PROGRAM-ID=1,BANDWIDTH=831000,RESOLUTION=704x396,CODECS="avc1.77.30, mp4a.40.2"
+http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_3_av.m3u8?null=0
+#EXT-X-STREAM-INF:SUBTITLES="subs",AUDIO="aac",PROGRAM-ID=1,BANDWIDTH=1467000,RESOLUTION=1024x576,CODECS="avc1.77.30, mp4a.40.2"
+http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_4_av.m3u8?null=0
diff --git a/test/testdata/m3u8/teamcoco_11995.m3u8 b/test/testdata/m3u8/teamcoco_11995.m3u8
new file mode 100644
index 000000000..a6e421697
--- /dev/null
+++ b/test/testdata/m3u8/teamcoco_11995.m3u8
@@ -0,0 +1,16 @@
+#EXTM3U
+#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio-0",NAME="Default",AUTOSELECT=YES,DEFAULT=YES,URI="hls/CONAN_020217_Highlight_show-audio-160k_v4.m3u8"
+#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio-1",NAME="Default",AUTOSELECT=YES,DEFAULT=YES,URI="hls/CONAN_020217_Highlight_show-audio-64k_v4.m3u8"
+#EXT-X-I-FRAME-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=37862000,CODECS="avc1.4d001f",URI="hls/CONAN_020217_Highlight_show-2m_iframe.m3u8"
+#EXT-X-I-FRAME-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=18750000,CODECS="avc1.4d001e",URI="hls/CONAN_020217_Highlight_show-1m_iframe.m3u8"
+#EXT-X-I-FRAME-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=6535000,CODECS="avc1.42001e",URI="hls/CONAN_020217_Highlight_show-400k_iframe.m3u8"
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=2374000,RESOLUTION=1024x576,CODECS="avc1.4d001f,mp4a.40.2",AUDIO="audio-0"
+hls/CONAN_020217_Highlight_show-2m_v4.m3u8
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=1205000,RESOLUTION=640x360,CODECS="avc1.4d001e,mp4a.40.2",AUDIO="audio-0"
+hls/CONAN_020217_Highlight_show-1m_v4.m3u8
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=522000,RESOLUTION=400x224,CODECS="avc1.42001e,mp4a.40.2",AUDIO="audio-0"
+hls/CONAN_020217_Highlight_show-400k_v4.m3u8
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=413000,RESOLUTION=400x224,CODECS="avc1.42001e,mp4a.40.5",AUDIO="audio-1"
+hls/CONAN_020217_Highlight_show-400k_v4.m3u8
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=71000,CODECS="mp4a.40.5",AUDIO="audio-1"
+hls/CONAN_020217_Highlight_show-audio-64k_v4.m3u8
diff --git a/test/testdata/m3u8/toggle_mobile_12211.m3u8 b/test/testdata/m3u8/toggle_mobile_12211.m3u8
new file mode 100644
index 000000000..69604e683
--- /dev/null
+++ b/test/testdata/m3u8/toggle_mobile_12211.m3u8
@@ -0,0 +1,13 @@
+#EXTM3U
+#EXT-X-VERSION:4
+#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",LANGUAGE="eng",NAME="English",URI="http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_sa2ntrdg/name/a.mp4/index.m3u8"
+#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",LANGUAGE="und",NAME="Undefined",URI="http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_r7y0nitg/name/a.mp4/index.m3u8"
+
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=155648,RESOLUTION=320x180,AUDIO="audio"
+http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_qlk9hlzr/name/a.mp4/index.m3u8
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=502784,RESOLUTION=480x270,AUDIO="audio"
+http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_oefackmi/name/a.mp4/index.m3u8
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=827392,RESOLUTION=640x360,AUDIO="audio"
+http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/12/pv/1/flavorId/0_vyg9pj7k/name/a.mp4/index.m3u8
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=1396736,RESOLUTION=854x480,AUDIO="audio"
+http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/12/pv/1/flavorId/0_50n4psvx/name/a.mp4/index.m3u8
diff --git a/test/testdata/m3u8/twitch_vod.m3u8 b/test/testdata/m3u8/twitch_vod.m3u8
new file mode 100644
index 000000000..7617277ca
--- /dev/null
+++ b/test/testdata/m3u8/twitch_vod.m3u8
@@ -0,0 +1,20 @@
+#EXTM3U
+#EXT-X-TWITCH-INFO:ORIGIN="s3",CLUSTER="edgecast_vod",REGION="EU",MANIFEST-CLUSTER="edgecast_vod",USER-IP="109.171.17.81"
+#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID="chunked",NAME="Source",AUTOSELECT=YES,DEFAULT=YES
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=3214134,CODECS="avc1.100.31,mp4a.40.2",RESOLUTION="1280x720",VIDEO="chunked"
+https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/chunked/index-muted-HM49I092CC.m3u8
+#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID="high",NAME="High",AUTOSELECT=YES,DEFAULT=YES
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=1603789,CODECS="avc1.42C01F,mp4a.40.2",RESOLUTION="1280x720",VIDEO="high"
+https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/high/index-muted-HM49I092CC.m3u8
+#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID="medium",NAME="Medium",AUTOSELECT=YES,DEFAULT=YES
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=893387,CODECS="avc1.42C01E,mp4a.40.2",RESOLUTION="852x480",VIDEO="medium"
+https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/medium/index-muted-HM49I092CC.m3u8
+#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID="low",NAME="Low",AUTOSELECT=YES,DEFAULT=YES
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=628347,CODECS="avc1.42C01E,mp4a.40.2",RESOLUTION="640x360",VIDEO="low"
+https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/low/index-muted-HM49I092CC.m3u8
+#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID="mobile",NAME="Mobile",AUTOSELECT=YES,DEFAULT=YES
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=280474,CODECS="avc1.42C00D,mp4a.40.2",RESOLUTION="400x226",VIDEO="mobile"
+https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/mobile/index-muted-HM49I092CC.m3u8
+#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID="audio_only",NAME="Audio Only",AUTOSELECT=NO,DEFAULT=NO
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=182725,CODECS="mp4a.40.2",VIDEO="audio_only"
+https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/audio_only/index-muted-HM49I092CC.m3u8
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 61d97ab72..359c549c5 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1312,17 +1312,25 @@ class InfoExtractor(object):
                               entry_protocol='m3u8', preference=None,
                               m3u8_id=None, note=None, errnote=None,
                               fatal=True, live=False):
-
         res = self._download_webpage_handle(
             m3u8_url, video_id,
             note=note or 'Downloading m3u8 information',
             errnote=errnote or 'Failed to download m3u8 information',
             fatal=fatal)
+
         if res is False:
             return []
+
         m3u8_doc, urlh = res
         m3u8_url = urlh.geturl()
 
+        return self._parse_m3u8_formats(
+            m3u8_doc, m3u8_url, ext=ext, entry_protocol=entry_protocol,
+            preference=preference, m3u8_id=m3u8_id, live=live)
+
+    def _parse_m3u8_formats(self, m3u8_doc, m3u8_url, ext=None,
+                            entry_protocol='m3u8', preference=None,
+                            m3u8_id=None, live=False):
         if '#EXT-X-FAXS-CM:' in m3u8_doc:  # Adobe Flash Access
             return []
 
@@ -1333,19 +1341,21 @@ class InfoExtractor(object):
             if re.match(r'^https?://', u)
             else compat_urlparse.urljoin(m3u8_url, u))
 
-        # We should try extracting formats only from master playlists [1], i.e.
-        # playlists that describe available qualities. On the other hand media
-        # playlists [2] should be returned as is since they contain just the media
-        # without qualities renditions.
+        # References:
+        # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-21
+        # 2. https://github.com/rg3/youtube-dl/issues/12211
+
+        # We should try extracting formats only from master playlists [1, 4.3.4],
+        # i.e. playlists that describe available qualities. On the other hand
+        # media playlists [1, 4.3.3] should be returned as is since they contain
+        # just the media without qualities renditions.
         # Fortunately, master playlist can be easily distinguished from media
-        # playlist based on particular tags availability. As of [1, 2] master
-        # playlist tags MUST NOT appear in a media playist and vice versa.
-        # As of [3] #EXT-X-TARGETDURATION tag is REQUIRED for every media playlist
-        # and MUST NOT appear in master playlist thus we can clearly detect media
-        # playlist with this criterion.
-        # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.4
-        # 2. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3
-        # 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.1
+        # playlist based on particular tags availability. As of [1, 4.3.3, 4.3.4]
+        # master playlist tags MUST NOT appear in a media playist and vice versa.
+        # As of [1, 4.3.3.1] #EXT-X-TARGETDURATION tag is REQUIRED for every
+        # media playlist and MUST NOT appear in master playlist thus we can
+        # clearly detect media playlist with this criterion.
+
         if '#EXT-X-TARGETDURATION' in m3u8_doc:  # media playlist, return as is
             return [{
                 'url': m3u8_url,
@@ -1354,52 +1364,67 @@ class InfoExtractor(object):
                 'protocol': entry_protocol,
                 'preference': preference,
             }]
-        audio_in_video_stream = {}
-        last_info = {}
-        last_media = {}
+
+        groups = {}
+        last_stream_inf = {}
+
+        def extract_media(x_media_line):
+            media = parse_m3u8_attributes(x_media_line)
+            # As per [1, 4.3.4.1] TYPE, GROUP-ID and NAME are REQUIRED
+            media_type, group_id, name = media.get('TYPE'), media.get('GROUP-ID'), media.get('NAME')
+            if not (media_type and group_id and name):
+                return
+            groups.setdefault(group_id, []).append(media)
+            if media_type not in ('VIDEO', 'AUDIO'):
+                return
+            media_url = media.get('URI')
+            if media_url:
+                format_id = []
+                for v in (group_id, name):
+                    if v:
+                        format_id.append(v)
+                f = {
+                    'format_id': '-'.join(format_id),
+                    'url': format_url(media_url),
+                    'language': media.get('LANGUAGE'),
+                    'ext': ext,
+                    'protocol': entry_protocol,
+                    'preference': preference,
+                }
+                if media_type == 'AUDIO':
+                    f['vcodec'] = 'none'
+                formats.append(f)
+
+        def build_stream_name():
+            # Despite specification does not mention NAME attribute for
+            # EXT-X-STREAM-INF it still sometimes may be present
+            stream_name = last_stream_inf.get('NAME')
+            if stream_name:
+                return stream_name
+            # If there is no NAME in EXT-X-STREAM-INF it will be obtained
+            # from corresponding rendition group
+            stream_group_id = last_stream_inf.get('VIDEO')
+            if not stream_group_id:
+                return
+            stream_group = groups.get(stream_group_id)
+            if not stream_group:
+                return stream_group_id
+            rendition = stream_group[0]
+            return rendition.get('NAME') or stream_group_id
+
         for line in m3u8_doc.splitlines():
             if line.startswith('#EXT-X-STREAM-INF:'):
-                last_info = parse_m3u8_attributes(line)
+                last_stream_inf = parse_m3u8_attributes(line)
             elif line.startswith('#EXT-X-MEDIA:'):
-                media = parse_m3u8_attributes(line)
-                media_type = media.get('TYPE')
-                if media_type in ('VIDEO', 'AUDIO'):
-                    group_id = media.get('GROUP-ID')
-                    media_url = media.get('URI')
-                    if media_url:
-                        format_id = []
-                        for v in (group_id, media.get('NAME')):
-                            if v:
-                                format_id.append(v)
-                        f = {
-                            'format_id': '-'.join(format_id),
-                            'url': format_url(media_url),
-                            'language': media.get('LANGUAGE'),
-                            'ext': ext,
-                            'protocol': entry_protocol,
-                            'preference': preference,
-                        }
-                        if media_type == 'AUDIO':
-                            f['vcodec'] = 'none'
-                            if group_id and not audio_in_video_stream.get(group_id):
-                                audio_in_video_stream[group_id] = False
-                        formats.append(f)
-                    else:
-                        # When there is no URI in EXT-X-MEDIA let this tag's
-                        # data be used by regular URI lines below
-                        last_media = media
-                        if media_type == 'AUDIO' and group_id:
-                            audio_in_video_stream[group_id] = True
+                extract_media(line)
             elif line.startswith('#') or not line.strip():
                 continue
             else:
-                tbr = int_or_none(last_info.get('AVERAGE-BANDWIDTH') or last_info.get('BANDWIDTH'), scale=1000)
+                tbr = int_or_none(last_stream_inf.get('AVERAGE-BANDWIDTH') or last_stream_inf.get('BANDWIDTH'), scale=1000)
                 format_id = []
                 if m3u8_id:
                     format_id.append(m3u8_id)
-                # Despite specification does not mention NAME attribute for
-                # EXT-X-STREAM-INF it still sometimes may be present
-                stream_name = last_info.get('NAME') or last_media.get('NAME')
+                stream_name = build_stream_name()
                 # Bandwidth of live streams may differ over time thus making
                 # format_id unpredictable. So it's better to keep provided
                 # format_id intact.
@@ -1412,11 +1437,11 @@ class InfoExtractor(object):
                     'manifest_url': manifest_url,
                     'tbr': tbr,
                     'ext': ext,
-                    'fps': float_or_none(last_info.get('FRAME-RATE')),
+                    'fps': float_or_none(last_stream_inf.get('FRAME-RATE')),
                     'protocol': entry_protocol,
                     'preference': preference,
                 }
-                resolution = last_info.get('RESOLUTION')
+                resolution = last_stream_inf.get('RESOLUTION')
                 if resolution:
                     mobj = re.search(r'(?P<width>\d+)[xX](?P<height>\d+)', resolution)
                     if mobj:
@@ -1432,13 +1457,26 @@ class InfoExtractor(object):
                         'vbr': vbr,
                         'abr': abr,
                     })
-                f.update(parse_codecs(last_info.get('CODECS')))
-                if audio_in_video_stream.get(last_info.get('AUDIO')) is False and f['vcodec'] != 'none':
-                    # TODO: update acodec for audio only formats with the same GROUP-ID
-                    f['acodec'] = 'none'
+                codecs = parse_codecs(last_stream_inf.get('CODECS'))
+                f.update(codecs)
+                audio_group_id = last_stream_inf.get('AUDIO')
+                # As per [1, 4.3.4.1.1] any EXT-X-STREAM-INF tag which
+                # references a rendition group MUST have a CODECS attribute.
+                # However, this is not always respected, for example, [2]
+                # contains EXT-X-STREAM-INF tag which references AUDIO
+                # rendition group but does not have CODECS and despite
+                # referencing audio group an audio group, it represents
+                # a complete (with audio and video) format. So, for such cases
+                # we will ignore references to rendition groups and treat them
+                # as complete formats.
+                if audio_group_id and codecs and f.get('vcodec') != 'none':
+                    audio_group = groups.get(audio_group_id)
+                    if audio_group and audio_group[0].get('URI'):
+                        # TODO: update acodec for audio only formats with
+                        # the same GROUP-ID
+                        f['acodec'] = 'none'
                 formats.append(f)
-                last_info = {}
-                last_media = {}
+                last_stream_inf = {}
         return formats
 
     @staticmethod

From 3110bb937d7ab43bd998448f51c29a157ef0dfbd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 22 Apr 2017 21:16:36 +0800
Subject: [PATCH 143/611] [porn91] Fix extraction (closes #12814)

---
 ChangeLog                      |  1 +
 youtube_dl/extractor/porn91.py | 32 ++++++--------------------------
 2 files changed, 7 insertions(+), 26 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index f8084fda5..1d49f8c8f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [porn91] Fix extraction (#12814)
 + [generic] Support Brightcove videos in <iframe> (#12482)
 + [brightcove] Support URLs with bcpid instead of playerID (#12482)
 
diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index 073fc3e21..24c3600fe 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -1,10 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from ..compat import (
-    compat_urllib_parse_unquote,
-    compat_urllib_parse_urlencode,
-)
 from .common import InfoExtractor
 from ..utils import (
     parse_duration,
@@ -19,7 +15,7 @@ class Porn91IE(InfoExtractor):
 
     _TEST = {
         'url': 'http://91porn.com/view_video.php?viewkey=7e42283b4f5ab36da134',
-        'md5': '6df8f6d028bc8b14f5dbd73af742fb20',
+        'md5': '7fcdb5349354f40d41689bd0fa8db05a',
         'info_dict': {
             'id': '7e42283b4f5ab36da134',
             'title': '18岁大一漂亮学妹，水嫩性感，再爽一次！',
@@ -43,24 +39,7 @@ class Porn91IE(InfoExtractor):
             r'<div id="viewvideo-title">([^<]+)</div>', webpage, 'title')
         title = title.replace('\n', '')
 
-        # get real url
-        file_id = self._search_regex(
-            r'so.addVariable\(\'file\',\'(\d+)\'', webpage, 'file id')
-        sec_code = self._search_regex(
-            r'so.addVariable\(\'seccode\',\'([^\']+)\'', webpage, 'sec code')
-        max_vid = self._search_regex(
-            r'so.addVariable\(\'max_vid\',\'(\d+)\'', webpage, 'max vid')
-        url_params = compat_urllib_parse_urlencode({
-            'VID': file_id,
-            'mp4': '1',
-            'seccode': sec_code,
-            'max_vid': max_vid,
-        })
-        info_cn = self._download_webpage(
-            'http://91porn.com/getfile.php?' + url_params, video_id,
-            'Downloading real video url')
-        video_url = compat_urllib_parse_unquote(self._search_regex(
-            r'file=([^&]+)&', info_cn, 'url'))
+        info_dict = self._parse_html5_media_entries(url, webpage, video_id)[0]
 
         duration = parse_duration(self._search_regex(
             r'时长:\s*</span>\s*(\d+:\d+)', webpage, 'duration', fatal=False))
@@ -68,11 +47,12 @@ class Porn91IE(InfoExtractor):
         comment_count = int_or_none(self._search_regex(
             r'留言:\s*</span>\s*(\d+)', webpage, 'comment count', fatal=False))
 
-        return {
+        info_dict.update({
             'id': video_id,
             'title': title,
-            'url': video_url,
             'duration': duration,
             'comment_count': comment_count,
             'age_limit': self._rta_search(webpage),
-        }
+        })
+
+        return info_dict

From facfd79f9ad574db7bfcc90cbef01b3a7e7b1262 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 22 Apr 2017 21:20:25 +0800
Subject: [PATCH 144/611] [azubu] Remove extractor as the site is gone (closes
 #12813)

---
 ChangeLog                          |   1 +
 youtube_dl/extractor/azubu.py      | 140 -----------------------------
 youtube_dl/extractor/extractors.py |   1 -
 3 files changed, 1 insertion(+), 141 deletions(-)
 delete mode 100644 youtube_dl/extractor/azubu.py

diff --git a/ChangeLog b/ChangeLog
index 1d49f8c8f..d699acb31 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+- [azubu] Remove extractor (#12813)
 * [porn91] Fix extraction (#12814)
 + [generic] Support Brightcove videos in <iframe> (#12482)
 + [brightcove] Support URLs with bcpid instead of playerID (#12482)
diff --git a/youtube_dl/extractor/azubu.py b/youtube_dl/extractor/azubu.py
deleted file mode 100644
index 3ba2f00d3..000000000
--- a/youtube_dl/extractor/azubu.py
+++ /dev/null
@@ -1,140 +0,0 @@
-from __future__ import unicode_literals
-
-import json
-
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    float_or_none,
-    sanitized_Request,
-)
-
-
-class AzubuIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?azubu\.(?:tv|uol.com.br)/[^/]+#!/play/(?P<id>\d+)'
-    _TESTS = [
-        {
-            'url': 'http://www.azubu.tv/GSL#!/play/15575/2014-hot6-cup-last-big-match-ro8-day-1',
-            'md5': 'a88b42fcf844f29ad6035054bd9ecaf4',
-            'info_dict': {
-                'id': '15575',
-                'ext': 'mp4',
-                'title': '2014 HOT6 CUP LAST BIG MATCH Ro8 Day 1',
-                'description': 'md5:d06bdea27b8cc4388a90ad35b5c66c01',
-                'thumbnail': r're:^https?://.*\.jpe?g',
-                'timestamp': 1417523507.334,
-                'upload_date': '20141202',
-                'duration': 9988.7,
-                'uploader': 'GSL',
-                'uploader_id': 414310,
-                'view_count': int,
-            },
-        },
-        {
-            'url': 'http://www.azubu.tv/FnaticTV#!/play/9344/-fnatic-at-worlds-2014:-toyz---%22i-love-rekkles,-he-has-amazing-mechanics%22-',
-            'md5': 'b72a871fe1d9f70bd7673769cdb3b925',
-            'info_dict': {
-                'id': '9344',
-                'ext': 'mp4',
-                'title': 'Fnatic at Worlds 2014: Toyz - "I love Rekkles, he has amazing mechanics"',
-                'description': 'md5:4a649737b5f6c8b5c5be543e88dc62af',
-                'thumbnail': r're:^https?://.*\.jpe?g',
-                'timestamp': 1410530893.320,
-                'upload_date': '20140912',
-                'duration': 172.385,
-                'uploader': 'FnaticTV',
-                'uploader_id': 272749,
-                'view_count': int,
-            },
-            'skip': 'Channel offline',
-        },
-    ]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        data = self._download_json(
-            'http://www.azubu.tv/api/video/%s' % video_id, video_id)['data']
-
-        title = data['title'].strip()
-        description = data.get('description')
-        thumbnail = data.get('thumbnail')
-        view_count = data.get('view_count')
-        user = data.get('user', {})
-        uploader = user.get('username')
-        uploader_id = user.get('id')
-
-        stream_params = json.loads(data['stream_params'])
-
-        timestamp = float_or_none(stream_params.get('creationDate'), 1000)
-        duration = float_or_none(stream_params.get('length'), 1000)
-
-        renditions = stream_params.get('renditions') or []
-        video = stream_params.get('FLVFullLength') or stream_params.get('videoFullLength')
-        if video:
-            renditions.append(video)
-
-        if not renditions and not user.get('channel', {}).get('is_live', True):
-            raise ExtractorError('%s said: channel is offline.' % self.IE_NAME, expected=True)
-
-        formats = [{
-            'url': fmt['url'],
-            'width': fmt['frameWidth'],
-            'height': fmt['frameHeight'],
-            'vbr': float_or_none(fmt['encodingRate'], 1000),
-            'filesize': fmt['size'],
-            'vcodec': fmt['videoCodec'],
-            'container': fmt['videoContainer'],
-        } for fmt in renditions if fmt['url']]
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'timestamp': timestamp,
-            'duration': duration,
-            'uploader': uploader,
-            'uploader_id': uploader_id,
-            'view_count': view_count,
-            'formats': formats,
-        }
-
-
-class AzubuLiveIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?azubu\.(?:tv|uol.com.br)/(?P<id>[^/]+)$'
-
-    _TESTS = [{
-        'url': 'http://www.azubu.tv/MarsTVMDLen',
-        'only_matching': True,
-    }, {
-        'url': 'http://azubu.uol.com.br/adolfz',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        user = self._match_id(url)
-
-        info = self._download_json(
-            'http://api.azubu.tv/public/modules/last-video/{0}/info'.format(user),
-            user)['data']
-        if info['type'] != 'STREAM':
-            raise ExtractorError('{0} is not streaming live'.format(user), expected=True)
-
-        req = sanitized_Request(
-            'https://edge-elb.api.brightcove.com/playback/v1/accounts/3361910549001/videos/ref:' + info['reference_id'])
-        req.add_header('Accept', 'application/json;pk=BCpkADawqM1gvI0oGWg8dxQHlgT8HkdE2LnAlWAZkOlznO39bSZX726u4JqnDsK3MDXcO01JxXK2tZtJbgQChxgaFzEVdHRjaDoxaOu8hHOO8NYhwdxw9BzvgkvLUlpbDNUuDoc4E4wxDToV')
-        bc_info = self._download_json(req, user)
-        m3u8_url = next(source['src'] for source in bc_info['sources'] if source['container'] == 'M2TS')
-        formats = self._extract_m3u8_formats(m3u8_url, user, ext='mp4')
-        self._sort_formats(formats)
-
-        return {
-            'id': info['id'],
-            'title': self._live_title(info['title']),
-            'uploader_id': user,
-            'formats': formats,
-            'is_live': True,
-            'thumbnail': bc_info['poster'],
-        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index a92cbefed..355a4e56f 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -87,7 +87,6 @@ from .azmedien import (
     AZMedienPlaylistIE,
     AZMedienShowPlaylistIE,
 )
-from .azubu import AzubuIE, AzubuLiveIE
 from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE

From 54f54fcca7af7e5bbc779cfc73d58d1ed2b4b6ae Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 22 Apr 2017 21:44:01 +0800
Subject: [PATCH 145/611] [socks] Report errors elegantly when credentails are
 required but missing

In some non-standard implementations, the server may respond AUTH_USER_PASS
even if's not listed in available authentication methods. (it should
respond AUTH_NO_ACCEPTABLE per standards)
---
 ChangeLog           | 4 ++++
 youtube_dl/socks.py | 5 +++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index d699acb31..8dc6e6857 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,9 @@
 version <unreleased>
 
+Core
+
+* [socks] Handle cases where credentials are required but missing
+
 Extractors
 - [azubu] Remove extractor (#12813)
 * [porn91] Fix extraction (#12814)
diff --git a/youtube_dl/socks.py b/youtube_dl/socks.py
index 0f5d7bdb2..5d4adbe72 100644
--- a/youtube_dl/socks.py
+++ b/youtube_dl/socks.py
@@ -193,9 +193,10 @@ class sockssocket(socket.socket):
 
         self._check_response_version(SOCKS5_VERSION, version)
 
-        if method == Socks5Auth.AUTH_NO_ACCEPTABLE:
+        if method == Socks5Auth.AUTH_NO_ACCEPTABLE or (
+                method == Socks5Auth.AUTH_USER_PASS and (not self._proxy.username or not self._proxy.password)):
             self.close()
-            raise Socks5Error(method)
+            raise Socks5Error(Socks5Auth.AUTH_NO_ACCEPTABLE)
 
         if method == Socks5Auth.AUTH_USER_PASS:
             username = self._proxy.username.encode('utf-8')

From fbf56be213021669363b69c5d0866b2cf22ecf2a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 22 Apr 2017 22:11:37 +0800
Subject: [PATCH 146/611] [iqiyi] Fix playlist detection (#12504)

---
 ChangeLog                     |  1 +
 youtube_dl/extractor/iqiyi.py | 15 +++++++++------
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 8dc6e6857..50daa1a69 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -5,6 +5,7 @@ Core
 * [socks] Handle cases where credentials are required but missing
 
 Extractors
+* [iqiyi] Fix playlist detection (#12504)
 - [azubu] Remove extractor (#12813)
 * [porn91] Fix extraction (#12814)
 + [generic] Support Brightcove videos in <iframe> (#12482)
diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 2af6a6db4..8a711afb9 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -337,13 +337,16 @@ class IqiyiIE(InfoExtractor):
             url, 'temp_id', note='download video page')
 
         # There's no simple way to determine whether an URL is a playlist or not
-        # So detect it
-        playlist_result = self._extract_playlist(webpage)
-        if playlist_result:
-            return playlist_result
-
+        # Sometimes there are playlist links in individual videos, so treat it
+        # as a single video first
         tvid = self._search_regex(
-            r'data-player-tvid\s*=\s*[\'"](\d+)', webpage, 'tvid')
+            r'data-player-tvid\s*=\s*[\'"](\d+)', webpage, 'tvid', default=None)
+        if tvid is None:
+            playlist_result = self._extract_playlist(webpage)
+            if playlist_result:
+                return playlist_result
+            raise ExtractorError('Can\'t find any video')
+
         video_id = self._search_regex(
             r'data-player-videoid\s*=\s*[\'"]([a-f\d]+)', webpage, 'video_id')
 

From 3e0304fe6e3a194cfb04f21aa261effb0850da40 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 22 Apr 2017 16:42:24 +0100
Subject: [PATCH 147/611] [downloader/fragment] use the documented names for
 fragment progress_hooks fields

---
 youtube_dl/downloader/dash.py     |  2 +-
 youtube_dl/downloader/f4m.py      |  2 +-
 youtube_dl/downloader/fragment.py | 18 ++++++++++--------
 youtube_dl/downloader/hls.py      |  2 +-
 youtube_dl/downloader/ism.py      |  2 +-
 5 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 94a13a543..7491fdad8 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -28,7 +28,7 @@ class DashSegmentsFD(FragmentFD):
         frag_index = 0
         for i, segment in enumerate(segments):
             frag_index += 1
-            if frag_index <= ctx['frag_index']:
+            if frag_index <= ctx['fragment_index']:
                 continue
             # In DASH, the first segment contains necessary headers to
             # generate a valid MP4 file, so always abort for the first segment
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index e456ed58f..c8fde9a89 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -376,7 +376,7 @@ class F4mFD(FragmentFD):
         while fragments_list:
             seg_i, frag_i = fragments_list.pop(0)
             frag_index += 1
-            if frag_index <= ctx['frag_index']:
+            if frag_index <= ctx['fragment_index']:
                 continue
             name = 'Seg%d-Frag%d' % (seg_i, frag_i)
             query = []
diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 80bb14d61..6c02cfc98 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -66,7 +66,9 @@ class FragmentFD(FileDownloader):
         if not (ctx.get('live') or ctx['tmpfilename'] == '-'):
             frag_index_stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'w')
             frag_index_stream.write(json.dumps({
-                'frag_index': ctx['frag_index']
+                'download': {
+                    'last_fragment_index': ctx['fragment_index']
+                },
             }))
             frag_index_stream.close()
 
@@ -100,7 +102,7 @@ class FragmentFD(FileDownloader):
             ytdl_filename = encodeFilename(self.ytdl_filename(ctx['filename']))
             if os.path.isfile(ytdl_filename):
                 frag_index_stream, _ = sanitize_open(ytdl_filename, 'r')
-                frag_index = json.loads(frag_index_stream.read())['frag_index']
+                frag_index = json.loads(frag_index_stream.read())['download']['last_fragment_index']
                 frag_index_stream.close()
         dest_stream, tmpfilename = sanitize_open(tmpfilename, open_mode)
 
@@ -108,7 +110,7 @@ class FragmentFD(FileDownloader):
             'dl': dl,
             'dest_stream': dest_stream,
             'tmpfilename': tmpfilename,
-            'frag_index': frag_index,
+            'fragment_index': frag_index,
             # Total complete fragments downloaded so far in bytes
             'complete_frags_downloaded_bytes': resume_len,
         })
@@ -120,8 +122,8 @@ class FragmentFD(FileDownloader):
         state = {
             'status': 'downloading',
             'downloaded_bytes': ctx['complete_frags_downloaded_bytes'],
-            'frag_index': ctx['frag_index'],
-            'frag_count': total_frags,
+            'fragment_index': ctx['fragment_index'],
+            'fragment_count': total_frags,
             'filename': ctx['filename'],
             'tmpfilename': ctx['tmpfilename'],
         }
@@ -144,12 +146,12 @@ class FragmentFD(FileDownloader):
             if not ctx['live']:
                 estimated_size = (
                     (ctx['complete_frags_downloaded_bytes'] + frag_total_bytes) /
-                    (state['frag_index'] + 1) * total_frags)
+                    (state['fragment_index'] + 1) * total_frags)
                 state['total_bytes_estimate'] = estimated_size
 
             if s['status'] == 'finished':
-                state['frag_index'] += 1
-                ctx['frag_index'] = state['frag_index']
+                state['fragment_index'] += 1
+                ctx['fragment_index'] = state['fragment_index']
                 state['downloaded_bytes'] += frag_total_bytes - ctx['prev_frag_downloaded_bytes']
                 ctx['complete_frags_downloaded_bytes'] = state['downloaded_bytes']
                 ctx['prev_frag_downloaded_bytes'] = 0
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 9a87d7ca8..0e29c8a2a 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -106,7 +106,7 @@ class HlsFD(FragmentFD):
             if line:
                 if not line.startswith('#'):
                     frag_index += 1
-                    if frag_index <= ctx['frag_index']:
+                    if frag_index <= ctx['fragment_index']:
                         continue
                     frag_url = (
                         line
diff --git a/youtube_dl/downloader/ism.py b/youtube_dl/downloader/ism.py
index 9f0fc36b3..338820e71 100644
--- a/youtube_dl/downloader/ism.py
+++ b/youtube_dl/downloader/ism.py
@@ -227,7 +227,7 @@ class IsmFD(FragmentFD):
         frag_index = 0
         for i, segment in enumerate(segments):
             frag_index += 1
-            if frag_index <= ctx['frag_index']:
+            if frag_index <= ctx['fragment_index']:
                 continue
             count = 0
             while count <= fragment_retries:

From 4abdba643cc2d610fbbfe7ece9ee97a3df98ad33 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 22 Apr 2017 18:19:47 +0100
Subject: [PATCH 148/611] [downloader/fragment] remove unused code

---
 youtube_dl/downloader/fragment.py | 1 -
 youtube_dl/downloader/ism.py      | 4 ----
 2 files changed, 5 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 6c02cfc98..0a3b1ece0 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -12,7 +12,6 @@ from ..utils import (
     encodeFilename,
     sanitize_open,
     sanitized_Request,
-    compat_str,
 )
 
 
diff --git a/youtube_dl/downloader/ism.py b/youtube_dl/downloader/ism.py
index 338820e71..5f6f9faef 100644
--- a/youtube_dl/downloader/ism.py
+++ b/youtube_dl/downloader/ism.py
@@ -232,10 +232,6 @@ class IsmFD(FragmentFD):
             count = 0
             while count <= fragment_retries:
                 try:
-                    success = ctx['dl'].download(target_filename, {
-                        'url': segment_url,
-                        'http_headers': info_dict.get('http_headers'),
-                    })
                     success, frag_content = self._download_fragment(ctx, segment['url'], info_dict)
                     if not success:
                         return False

From ffbc8386b9298ef343a71bf54d7426a19755fef8 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 22 Apr 2017 22:26:20 +0100
Subject: [PATCH 149/611] [brightcove] match only video elements with
 data-video-id attribute

---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 183ac7dd0..97602ca30 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -522,7 +522,7 @@ class BrightcoveNewIE(InfoExtractor):
         # [2] looks like:
         for video, script_tag, account_id, player_id, embed in re.findall(
                 r'''(?isx)
-                    (<video\s+[^>]+>)
+                    (<video\s+[^>]*data-video-id=['"]?[^>]+>)
                     (?:.*?
                         (<script[^>]+
                             src=["\'](?:https?:)?//players\.brightcove\.net/

From 9c99bef704d185783b61a20ea1f5b58c4c55aba6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Apr 2017 11:33:19 +0700
Subject: [PATCH 150/611] [extractor/common] Use float for scaled tbr

---
 youtube_dl/extractor/common.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 359c549c5..2a099480f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1420,7 +1420,9 @@ class InfoExtractor(object):
             elif line.startswith('#') or not line.strip():
                 continue
             else:
-                tbr = int_or_none(last_stream_inf.get('AVERAGE-BANDWIDTH') or last_stream_inf.get('BANDWIDTH'), scale=1000)
+                tbr = float_or_none(
+                    last_stream_inf.get('AVERAGE-BANDWIDTH') or
+                    last_stream_inf.get('BANDWIDTH'), scale=1000)
                 format_id = []
                 if m3u8_id:
                     format_id.append(m3u8_id)
@@ -1850,7 +1852,7 @@ class InfoExtractor(object):
                             'ext': mimetype2ext(mime_type),
                             'width': int_or_none(representation_attrib.get('width')),
                             'height': int_or_none(representation_attrib.get('height')),
-                            'tbr': int_or_none(bandwidth, 1000),
+                            'tbr': float_or_none(bandwidth, 1000),
                             'asr': int_or_none(representation_attrib.get('audioSamplingRate')),
                             'fps': int_or_none(representation_attrib.get('frameRate')),
                             'language': lang if lang not in ('mul', 'und', 'zxx', 'mis') else None,

From 07ad0cf34fdeba447213c993210db1f012a812b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Apr 2017 11:48:51 +0700
Subject: [PATCH 151/611] [vidio] Improve and sort formats

---
 youtube_dl/extractor/vidio.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vidio.py b/youtube_dl/extractor/vidio.py
index 4e4b4e38c..701bb1d01 100644
--- a/youtube_dl/extractor/vidio.py
+++ b/youtube_dl/extractor/vidio.py
@@ -49,8 +49,11 @@ class VidioIE(InfoExtractor):
             thumbnail = clip.get('image')
 
         m3u8_url = m3u8_url or self._search_regex(
-            r'data(?:-vjs)?-clip-hls-url=(["\'])(?P<url>.+?)\1', webpage, 'hls url')
-        formats = self._extract_m3u8_formats(m3u8_url, display_id, 'mp4', entry_protocol='m3u8_native')
+            r'data(?:-vjs)?-clip-hls-url=(["\'])(?P<url>(?!\1).+)\1',
+            webpage, 'hls url')
+        formats = self._extract_m3u8_formats(
+            m3u8_url, display_id, 'mp4', entry_protocol='m3u8_native')
+        self._sort_formats(formats)
 
         duration = int_or_none(duration or self._search_regex(
             r'data-video-duration=(["\'])(?P<duartion>\d+)\1', webpage, 'duration'))

From ddd258f92270f48649b57ba4288027a2433af079 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Apr 2017 11:49:57 +0700
Subject: [PATCH 152/611] [test_InfoExtractor] Add m3u8 parsing test for NAME
 attribute in EXT-X-STREAM-INF tag

---
 test/test_InfoExtractor.py     | 63 ++++++++++++++++++++++++++++------
 youtube_dl/extractor/common.py |  4 ++-
 2 files changed, 56 insertions(+), 11 deletions(-)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 57205f6ae..2a3c3f7cb 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -361,7 +361,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'ext': 'mp4',
                     'format_id': '155',
                     'protocol': 'm3u8',
-                    'tbr': 155,
+                    'tbr': 155.648,
                     'width': 320,
                     'height': 180,
                 }, {
@@ -370,7 +370,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'ext': 'mp4',
                     'format_id': '502',
                     'protocol': 'm3u8',
-                    'tbr': 502,
+                    'tbr': 502.784,
                     'width': 480,
                     'height': 270,
                 }, {
@@ -379,7 +379,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'ext': 'mp4',
                     'format_id': '827',
                     'protocol': 'm3u8',
-                    'tbr': 827,
+                    'tbr': 827.392,
                     'width': 640,
                     'height': 360,
                 }, {
@@ -388,7 +388,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'ext': 'mp4',
                     'format_id': '1396',
                     'protocol': 'm3u8',
-                    'tbr': 1396,
+                    'tbr': 1396.736,
                     'width': 854,
                     'height': 480,
                 }]
@@ -413,7 +413,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'protocol': 'm3u8',
                     'acodec': 'mp4a.40.2',
                     'vcodec': 'none',
-                    'tbr': 182,
+                    'tbr': 182.725,
                 }, {
                     'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/mobile/index-muted-HM49I092CC.m3u8',
                     'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/mobile/index-muted-HM49I092CC.m3u8',
@@ -422,7 +422,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'protocol': 'm3u8',
                     'acodec': 'mp4a.40.2',
                     'vcodec': 'avc1.42C00D',
-                    'tbr': 280,
+                    'tbr': 280.474,
                     'width': 400,
                     'height': 226,
                 }, {
@@ -433,7 +433,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'protocol': 'm3u8',
                     'acodec': 'mp4a.40.2',
                     'vcodec': 'avc1.42C01E',
-                    'tbr': 628,
+                    'tbr': 628.347,
                     'width': 640,
                     'height': 360,
                 }, {
@@ -444,7 +444,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'protocol': 'm3u8',
                     'acodec': 'mp4a.40.2',
                     'vcodec': 'avc1.42C01E',
-                    'tbr': 893,
+                    'tbr': 893.387,
                     'width': 852,
                     'height': 480,
                 }, {
@@ -455,7 +455,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'protocol': 'm3u8',
                     'acodec': 'mp4a.40.2',
                     'vcodec': 'avc1.42C01F',
-                    'tbr': 1603,
+                    'tbr': 1603.789,
                     'width': 1280,
                     'height': 720,
                 }, {
@@ -466,7 +466,50 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'protocol': 'm3u8',
                     'acodec': 'mp4a.40.2',
                     'vcodec': 'avc1.100.31',
-                    'tbr': 3214,
+                    'tbr': 3214.134,
+                    'width': 1280,
+                    'height': 720,
+                }]
+            ),
+            (
+                # http://www.vidio.com/watch/165683-dj_ambred-booyah-live-2015
+                # EXT-X-STREAM-INF tag with NAME attribute that is not defined
+                # in HLS specification
+                'vidio',
+                'https://www.vidio.com/videos/165683/playlist.m3u8',
+                [{
+                    'url': 'https://www.vidio.com/videos/165683/playlist.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'meta',
+                    'format_note': 'Quality selection URL',
+                    'protocol': 'm3u8',
+                    'preference': -100,
+                    'resolution': 'multiple'
+                }, {
+                    'url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b300.mp4.m3u8',
+                    'manifest_url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b300.mp4.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '270p 3G',
+                    'protocol': 'm3u8',
+                    'tbr': 300,
+                    'width': 480,
+                    'height': 270,
+                }, {
+                    'url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b600.mp4.m3u8',
+                    'manifest_url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b600.mp4.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '360p SD',
+                    'protocol': 'm3u8',
+                    'tbr': 600,
+                    'width': 640,
+                    'height': 360,
+                }, {
+                    'url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b1200.mp4.m3u8',
+                    'manifest_url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b1200.mp4.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '720p HD',
+                    'protocol': 'm3u8',
+                    'tbr': 1200,
                     'width': 1280,
                     'height': 720,
                 }]
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2a099480f..6d023106e 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1397,7 +1397,9 @@ class InfoExtractor(object):
 
         def build_stream_name():
             # Despite specification does not mention NAME attribute for
-            # EXT-X-STREAM-INF it still sometimes may be present
+            # EXT-X-STREAM-INF tag (see [1] or vidio test in
+            # test_parse_m3u8_formats) it still sometimes may be present
+            # 1. http://www.vidio.com/watch/165683-dj_ambred-booyah-live-2015
             stream_name = last_stream_inf.get('NAME')
             if stream_name:
                 return stream_name

From 3019cb0c9976481cf2afaf0a2005d90040204fa0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Apr 2017 11:51:53 +0700
Subject: [PATCH 153/611] [extractor/common] Rephrase comment

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6d023106e..9184e53e9 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1397,8 +1397,8 @@ class InfoExtractor(object):
 
         def build_stream_name():
             # Despite specification does not mention NAME attribute for
-            # EXT-X-STREAM-INF tag (see [1] or vidio test in
-            # test_parse_m3u8_formats) it still sometimes may be present
+            # EXT-X-STREAM-INF tag it still sometimes may be present (see [1]
+            # or vidio test in TestInfoExtractor.test_parse_m3u8_formats)
             # 1. http://www.vidio.com/watch/165683-dj_ambred-booyah-live-2015
             stream_name = last_stream_inf.get('NAME')
             if stream_name:

From a4d6cf970ca17f1112465353ec1c2aa857dd2e69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 24 Apr 2017 00:50:39 +0700
Subject: [PATCH 154/611] [YoutubeDL] Fix output template for missing timestamp
 (closes #12796)

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 819b374ef..eb465c425 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -640,7 +640,7 @@ class YoutubeDL(object):
 
             NUMERIC_FIELDS = set((
                 'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
-                'upload_year', 'upload_month', 'upload_day',
+                'timestamp', 'upload_year', 'upload_month', 'upload_day',
                 'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
                 'average_rating', 'comment_count', 'age_limit',
                 'start_time', 'end_time',

From d3f0687cf7b049b976420056e02c26b5d96adeed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 24 Apr 2017 02:54:17 +0700
Subject: [PATCH 155/611] [downloader/fragment] Use temp file for current
 fragment

---
 youtube_dl/downloader/fragment.py | 59 ++++++++++++++++++++-----------
 youtube_dl/downloader/http.py     |  8 ++---
 2 files changed, 41 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 0a3b1ece0..62de0a75b 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -2,7 +2,6 @@ from __future__ import division, unicode_literals
 
 import os
 import time
-import io
 import json
 
 from .common import FileDownloader
@@ -48,28 +47,42 @@ class FragmentFD(FileDownloader):
         self._prepare_frag_download(ctx)
         self._start_frag_download(ctx)
 
+    def _read_ytdl_file(self, ctx):
+        stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'r')
+        ctx['fragment_index'] = json.loads(stream.read())['download']['current_fragment_index']
+        stream.close()
+
+    def _write_ytdl_file(self, ctx):
+        frag_index_stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'w')
+        frag_index_stream.write(json.dumps({
+            'download': {
+                'current_fragment_index': ctx['fragment_index']
+            },
+        }))
+        frag_index_stream.close()
+
     def _download_fragment(self, ctx, frag_url, info_dict, headers=None):
-        down = io.BytesIO()
-        success = ctx['dl'].download(down, {
+        fragment_filename = '%s-Frag%d' % (ctx['tmpfilename'], ctx['fragment_index'])
+        success = ctx['dl'].download(fragment_filename, {
             'url': frag_url,
             'http_headers': headers or info_dict.get('http_headers'),
         })
         if not success:
             return False, None
-        frag_content = down.getvalue()
+        down, frag_sanitized = sanitize_open(fragment_filename, 'rb')
+        ctx['fragment_filename_sanitized'] = frag_sanitized
+        frag_content = down.read()
         down.close()
         return True, frag_content
 
     def _append_fragment(self, ctx, frag_content):
-        ctx['dest_stream'].write(frag_content)
-        if not (ctx.get('live') or ctx['tmpfilename'] == '-'):
-            frag_index_stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'w')
-            frag_index_stream.write(json.dumps({
-                'download': {
-                    'last_fragment_index': ctx['fragment_index']
-                },
-            }))
-            frag_index_stream.close()
+        try:
+            ctx['dest_stream'].write(frag_content)
+        finally:
+            if not (ctx.get('live') or ctx['tmpfilename'] == '-'):
+                self._write_ytdl_file(ctx)
+            os.remove(ctx['fragment_filename_sanitized'])
+            del ctx['fragment_filename_sanitized']
 
     def _prepare_frag_download(self, ctx):
         if 'live' not in ctx:
@@ -93,23 +106,29 @@ class FragmentFD(FileDownloader):
         tmpfilename = self.temp_name(ctx['filename'])
         open_mode = 'wb'
         resume_len = 0
-        frag_index = 0
+
         # Establish possible resume length
         if os.path.isfile(encodeFilename(tmpfilename)):
             open_mode = 'ab'
             resume_len = os.path.getsize(encodeFilename(tmpfilename))
-            ytdl_filename = encodeFilename(self.ytdl_filename(ctx['filename']))
-            if os.path.isfile(ytdl_filename):
-                frag_index_stream, _ = sanitize_open(ytdl_filename, 'r')
-                frag_index = json.loads(frag_index_stream.read())['download']['last_fragment_index']
-                frag_index_stream.close()
+
+        ctx['fragment_index'] = 0
+        if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))):
+            self._read_ytdl_file(ctx)
+        else:
+            self._write_ytdl_file(ctx)
+
+        if ctx['fragment_index'] > 0:
+            assert resume_len > 0
+        else:
+            assert resume_len == 0
+
         dest_stream, tmpfilename = sanitize_open(tmpfilename, open_mode)
 
         ctx.update({
             'dl': dl,
             'dest_stream': dest_stream,
             'tmpfilename': tmpfilename,
-            'fragment_index': frag_index,
             # Total complete fragments downloaded so far in bytes
             'complete_frags_downloaded_bytes': resume_len,
         })
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 2896a17af..af405b950 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -20,14 +20,10 @@ from ..utils import (
 
 
 class HttpFD(FileDownloader):
-    def real_download(self, filename_or_stream, info_dict):
+    def real_download(self, filename, info_dict):
         url = info_dict['url']
-        filename = filename_or_stream
-        stream = None
-        if hasattr(filename_or_stream, 'write'):
-            stream = filename_or_stream
-            filename = '-'
         tmpfilename = self.temp_name(filename)
+        stream = None
 
         # Do not include the Accept-Encoding header
         headers = {'Youtubedl-no-compression': 'True'}

From 0eee52f34bfa55ba9d4ebfc4b4ba508c989f05b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 24 Apr 2017 03:09:08 +0700
Subject: [PATCH 156/611] Introduce --keep-fragments

---
 youtube_dl/__init__.py            | 1 +
 youtube_dl/downloader/fragment.py | 5 ++++-
 youtube_dl/options.py             | 4 ++++
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index f15606568..c4589411e 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -343,6 +343,7 @@ def _real_main(argv=None):
         'retries': opts.retries,
         'fragment_retries': opts.fragment_retries,
         'skip_unavailable_fragments': opts.skip_unavailable_fragments,
+        'keep_fragments': opts.keep_fragments,
         'buffersize': opts.buffersize,
         'noresizebuffer': opts.noresizebuffer,
         'continuedl': opts.continue_dl,
diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 62de0a75b..bcff94cbe 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -29,6 +29,8 @@ class FragmentFD(FileDownloader):
                         and hlsnative only)
     skip_unavailable_fragments:
                         Skip unavailable fragments (DASH and hlsnative only)
+    keep_fragments:     Keep downloaded fragments on disk after downloading is
+                        finished
     """
 
     def report_retry_fragment(self, err, frag_index, count, retries):
@@ -81,7 +83,8 @@ class FragmentFD(FileDownloader):
         finally:
             if not (ctx.get('live') or ctx['tmpfilename'] == '-'):
                 self._write_ytdl_file(ctx)
-            os.remove(ctx['fragment_filename_sanitized'])
+            if not self.params.get('keep_fragments', False):
+                os.remove(ctx['fragment_filename_sanitized'])
             del ctx['fragment_filename_sanitized']
 
     def _prepare_frag_download(self, ctx):
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 2d2f5e47b..52309fb84 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -468,6 +468,10 @@ def parseOpts(overrideArguments=None):
         '--abort-on-unavailable-fragment',
         action='store_false', dest='skip_unavailable_fragments',
         help='Abort downloading when some fragment is not available')
+    downloader.add_option(
+        '--keep-fragments',
+        action='store_true', dest='keep_fragments', default=False,
+        help='Keep downloaded fragments on disk after downloading is finished; fragments are erased by default')
     downloader.add_option(
         '--buffer-size',
         dest='buffersize', metavar='SIZE', default='1024',

From adb4b03cd583857d00fa03c050d1472bf44a0d15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 24 Apr 2017 23:05:56 +0700
Subject: [PATCH 157/611] [downloader/fragment] Don't process ytdl file when
 it's not needed yet

---
 youtube_dl/downloader/fragment.py | 36 ++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index bcff94cbe..25c8f18ec 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -49,6 +49,10 @@ class FragmentFD(FileDownloader):
         self._prepare_frag_download(ctx)
         self._start_frag_download(ctx)
 
+    @staticmethod
+    def __do_ytdl_file(ctx):
+        return not ctx['live'] and not ctx['tmpfilename'] == '-'
+
     def _read_ytdl_file(self, ctx):
         stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'r')
         ctx['fragment_index'] = json.loads(stream.read())['download']['current_fragment_index']
@@ -81,7 +85,7 @@ class FragmentFD(FileDownloader):
         try:
             ctx['dest_stream'].write(frag_content)
         finally:
-            if not (ctx.get('live') or ctx['tmpfilename'] == '-'):
+            if self.__do_ytdl_file(ctx):
                 self._write_ytdl_file(ctx)
             if not self.params.get('keep_fragments', False):
                 os.remove(ctx['fragment_filename_sanitized'])
@@ -115,16 +119,21 @@ class FragmentFD(FileDownloader):
             open_mode = 'ab'
             resume_len = os.path.getsize(encodeFilename(tmpfilename))
 
-        ctx['fragment_index'] = 0
-        if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))):
-            self._read_ytdl_file(ctx)
-        else:
-            self._write_ytdl_file(ctx)
+        # Should be initialized before ytdl file check
+        ctx.update({
+            'tmpfilename': tmpfilename,
+            'fragment_index': 0,
+        })
 
-        if ctx['fragment_index'] > 0:
-            assert resume_len > 0
-        else:
-            assert resume_len == 0
+        if self.__do_ytdl_file(ctx):
+            if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))):
+                self._read_ytdl_file(ctx)
+            else:
+                self._write_ytdl_file(ctx)
+            if ctx['fragment_index'] > 0:
+                assert resume_len > 0
+            else:
+                assert resume_len == 0
 
         dest_stream, tmpfilename = sanitize_open(tmpfilename, open_mode)
 
@@ -194,9 +203,10 @@ class FragmentFD(FileDownloader):
 
     def _finish_frag_download(self, ctx):
         ctx['dest_stream'].close()
-        ytdl_filename = encodeFilename(self.ytdl_filename(ctx['filename']))
-        if os.path.isfile(ytdl_filename):
-            os.remove(ytdl_filename)
+        if self.__do_ytdl_file(ctx):
+            ytdl_filename = encodeFilename(self.ytdl_filename(ctx['filename']))
+            if os.path.isfile(ytdl_filename):
+                os.remove(ytdl_filename)
         elapsed = time.time() - ctx['started']
         self.try_rename(ctx['tmpfilename'], ctx['filename'])
         fsize = os.path.getsize(encodeFilename(ctx['filename']))

From 290f64dbaa3a8c187d47b72b3c808bd52d15a6e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 24 Apr 2017 23:50:20 +0700
Subject: [PATCH 158/611] [downloader/fragment] Improve .ytdl format and start
 documenting

---
 youtube_dl/downloader/fragment.py | 31 ++++++++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 25c8f18ec..50c8254a4 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -31,6 +31,24 @@ class FragmentFD(FileDownloader):
                         Skip unavailable fragments (DASH and hlsnative only)
     keep_fragments:     Keep downloaded fragments on disk after downloading is
                         finished
+
+    For each incomplete fragment download youtube-dl keeps on disk a special
+    bookkeeping file with download state and metadata (in future such files will
+    be used for any incomplete download handled by youtube-dl). This file is
+    used to properly handle resuming, check download file consistency and detect
+    potential errors. The file has a .ytdl extension and represents a standard
+    JSON file of the following format:
+
+    extractor:
+        Dictionary of extractor related data. TBD.
+
+    downloader:
+        Dictionary of downloader related data. May contain following data:
+            current_fragment:
+                Dictionary with current (being downloaded) fragment data:
+                index:  Index of current fragment among all fragments
+            fragment_count:
+                Total count of fragments
     """
 
     def report_retry_fragment(self, err, frag_index, count, retries):
@@ -55,16 +73,19 @@ class FragmentFD(FileDownloader):
 
     def _read_ytdl_file(self, ctx):
         stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'r')
-        ctx['fragment_index'] = json.loads(stream.read())['download']['current_fragment_index']
+        ctx['fragment_index'] = json.loads(stream.read())['downloader']['current_fragment']['index']
         stream.close()
 
     def _write_ytdl_file(self, ctx):
         frag_index_stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'w')
-        frag_index_stream.write(json.dumps({
-            'download': {
-                'current_fragment_index': ctx['fragment_index']
+        downloader = {
+            'current_fragment': {
+                'index': ctx['fragment_index'],
             },
-        }))
+        }
+        if ctx.get('fragment_count') is not None:
+            downloader['fragment_count'] = ctx['fragment_count']
+        frag_index_stream.write(json.dumps({'downloader': downloader}))
         frag_index_stream.close()
 
     def _download_fragment(self, ctx, frag_url, info_dict, headers=None):

From 01cb57016ff747ab07c506a708093dfd633d58f6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 25 Apr 2017 22:23:57 +0800
Subject: [PATCH 159/611] [iqiyi] Fix extraction of Yule videos

---
 ChangeLog                     |  1 +
 youtube_dl/extractor/iqiyi.py | 13 +++++++++----
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 50daa1a69..d38039663 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -5,6 +5,7 @@ Core
 * [socks] Handle cases where credentials are required but missing
 
 Extractors
+* [iqiyi] Fix extraction of Yule videos
 * [iqiyi] Fix playlist detection (#12504)
 - [azubu] Remove extractor (#12813)
 * [porn91] Fix extraction (#12814)
diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 8a711afb9..fdfa7de9e 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -189,7 +189,11 @@ class IqiyiIE(InfoExtractor):
         'only_matching': True,
     }, {
         'url': 'http://yule.iqiyi.com/pcb.html',
-        'only_matching': True,
+        'info_dict': {
+            'id': '4a0af228fddb55ec96398a364248ed7f',
+            'ext': 'mp4',
+            'title': '第2017-04-21期 女艺人频遭极端粉丝骚扰',
+        },
     }, {
         # VIP-only video. The first 2 parts (6 minutes) are available without login
         # MD5 sums omitted as values are different on Travis CI and my machine
@@ -340,7 +344,7 @@ class IqiyiIE(InfoExtractor):
         # Sometimes there are playlist links in individual videos, so treat it
         # as a single video first
         tvid = self._search_regex(
-            r'data-player-tvid\s*=\s*[\'"](\d+)', webpage, 'tvid', default=None)
+            r'data-(?:player|shareplattrigger)-tvid\s*=\s*[\'"](\d+)', webpage, 'tvid', default=None)
         if tvid is None:
             playlist_result = self._extract_playlist(webpage)
             if playlist_result:
@@ -348,7 +352,7 @@ class IqiyiIE(InfoExtractor):
             raise ExtractorError('Can\'t find any video')
 
         video_id = self._search_regex(
-            r'data-player-videoid\s*=\s*[\'"]([a-f\d]+)', webpage, 'video_id')
+            r'data-(?:player|shareplattrigger)-videoid\s*=\s*[\'"]([a-f\d]+)', webpage, 'video_id')
 
         formats = []
         for _ in range(5):
@@ -380,7 +384,8 @@ class IqiyiIE(InfoExtractor):
 
         self._sort_formats(formats)
         title = (get_element_by_id('widget-videotitle', webpage) or
-                 clean_html(get_element_by_attribute('class', 'mod-play-tit', webpage)))
+                 clean_html(get_element_by_attribute('class', 'mod-play-tit', webpage)) or
+                 self._html_search_regex(r'<span[^>]+data-videochanged-title="word"[^>]*>([^<]+)</span>', webpage, 'title'))
 
         return {
             'id': video_id,

From 9f54ae2873f8091c3f2512d307b96785f40a14ca Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 25 Apr 2017 22:42:55 +0800
Subject: [PATCH 160/611] Ignore and clean *.ytdl files

---
 .gitignore | 1 +
 Makefile   | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index bef084baa..a5b585f43 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,6 +35,7 @@ updates_key.pem
 *.mkv
 *.swf
 *.part
+*.ytdl
 *.swp
 test/local_parameters.json
 .tox
diff --git a/Makefile b/Makefile
index 9d1ddc9d1..023556391 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
 
 clean:
-	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish youtube_dl/extractor/lazy_extractors.py *.dump *.part* *.info.json *.mp4 *.m4a *.flv *.mp3 *.avi *.mkv *.webm *.3gp *.wav *.ape *.swf *.jpg *.png CONTRIBUTING.md.tmp ISSUE_TEMPLATE.md.tmp youtube-dl youtube-dl.exe
+	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish youtube_dl/extractor/lazy_extractors.py *.dump *.part* *.ytdl *.info.json *.mp4 *.m4a *.flv *.mp3 *.avi *.mkv *.webm *.3gp *.wav *.ape *.swf *.jpg *.png CONTRIBUTING.md.tmp ISSUE_TEMPLATE.md.tmp youtube-dl youtube-dl.exe
 	find . -name "*.pyc" -delete
 	find . -name "*.class" -delete
 

From 538eee7b6a0399b71b8cd3f344c2e21e7f248a99 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 25 Apr 2017 22:25:40 +0700
Subject: [PATCH 161/611] Add missing test m3u8 file

---
 test/testdata/m3u8/vidio.m3u8 | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 test/testdata/m3u8/vidio.m3u8

diff --git a/test/testdata/m3u8/vidio.m3u8 b/test/testdata/m3u8/vidio.m3u8
new file mode 100644
index 000000000..89c244469
--- /dev/null
+++ b/test/testdata/m3u8/vidio.m3u8
@@ -0,0 +1,10 @@
+#EXTM3U
+
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=300000,RESOLUTION=480x270,NAME="270p 3G"
+https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b300.mp4.m3u8
+
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=600000,RESOLUTION=640x360,NAME="360p SD"
+https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b600.mp4.m3u8
+
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=1200000,RESOLUTION=1280x720,NAME="720p HD"
+https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b1200.mp4.m3u8

From 85f6de25e46562f57be4447e9109be964e6515ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 25 Apr 2017 23:33:35 +0700
Subject: [PATCH 162/611] [downloader/fragment] Clarify current_fragment's
 index and mark as experimental

---
 youtube_dl/downloader/fragment.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 50c8254a4..bb9e82578 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -46,9 +46,11 @@ class FragmentFD(FileDownloader):
         Dictionary of downloader related data. May contain following data:
             current_fragment:
                 Dictionary with current (being downloaded) fragment data:
-                index:  Index of current fragment among all fragments
+                index:  0-based index of current fragment among all fragments
             fragment_count:
                 Total count of fragments
+                
+    This feature is experimental and file format may change in future.
     """
 
     def report_retry_fragment(self, err, frag_index, count, retries):

From ac9c69ace7ee22e59a44d25c87b9b53d18762ff7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 25 Apr 2017 23:46:05 +0700
Subject: [PATCH 163/611] [extractor/common] Improve jwplayer regex

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9184e53e9..8b3f04c61 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2233,7 +2233,7 @@ class InfoExtractor(object):
 
     def _find_jwplayer_data(self, webpage, video_id=None, transform_source=js_to_json):
         mobj = re.search(
-            r'(?s)jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\).*?\.setup\s*\((?P<options>[^)]+)\)',
+            r'(?s)jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\)(?!</script>).*?\.setup\s*\((?P<options>[^)]+)\)',
             webpage)
         if mobj:
             try:

From 30a4ab191a553d4b946731864fd02dba9a6eb0c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Apr 2017 00:03:13 +0700
Subject: [PATCH 164/611] [ChangeLog] Actualize

---
 ChangeLog | 31 ++++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index d38039663..55775ba1c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,16 +1,45 @@
 version <unreleased>
 
 Core
-
+* Introduce --keep-fragments for keeping fragments of fragmented download
+  on disk after download is finished
+* [YoutubeDL] Fix output template for missing timestamp (#12796)
 * [socks] Handle cases where credentials are required but missing
+* [extractor/common] Improve HLS extraction (#12211)
+    - Extract m3u8 parsing to separate method
+    - Improve rendition groups extraction
+    - Build stream name according stream GROUP-ID
+    - Ignore reference to AUDIO group without URI when stream has no CODECS
+    - Use float for scaled tbr in _parse_m3u8_formats
+* [utils] Add support for TTML styles in dfxp2srt
+* [downloader/hls] No need to download keys for fragments that have been
+  already downloaded
+* [downloader/fragment] Improve fragment downloading
+    - Resume immediately
+    - Don't concatenate fragments and decrypt them on every resume
+    - Optimize disk storage usage, don't store intermediate fragments on disk
+    - Store bookkeeping download state file
++ [extractor/common] Add support for multiple getters in try_get
++ [extractor/common] Add support for video of WebPage context in _json_ld
+  (#12778)
++ [extractor/common] Relax JWPlayer regular expression and remove
+  duplicate URLs (#12768)
 
 Extractors
 * [iqiyi] Fix extraction of Yule videos
+* [vidio] Improve extraction and sort formats
++ [brightcove] Match only video elements with data-video-id attribute
 * [iqiyi] Fix playlist detection (#12504)
 - [azubu] Remove extractor (#12813)
 * [porn91] Fix extraction (#12814)
+* [vidzi] Fix extraction (#12793)
++ [amp] Extract error message (#12795)
++ [xfileshare] Add support for gorillavid.com and daclips.com (#12776)
+* [instagram] Fix extraction (#12777)
 + [generic] Support Brightcove videos in <iframe> (#12482)
 + [brightcove] Support URLs with bcpid instead of playerID (#12482)
+* [brightcove] Fix _extract_url (#12782)
++ [odnoklassniki] Extract HLS formats
 
 
 version 2017.04.17

From a82f41841d3268e5107cf0462fded1119229ebbf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Apr 2017 00:06:12 +0700
Subject: [PATCH 165/611] release 2017.04.26

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 README.md                 | 3 +++
 docs/supportedsites.md    | 2 --
 youtube_dl/version.py     | 2 +-
 5 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 45f8db721..dedc07604 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.17*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.17**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.26*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.26**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.04.17
+[debug] youtube-dl version 2017.04.26
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 55775ba1c..b23aa10d5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.04.26
 
 Core
 * Introduce --keep-fragments for keeping fragments of fragmented download
diff --git a/README.md b/README.md
index 41f647aaa..1ecd2005a 100644
--- a/README.md
+++ b/README.md
@@ -187,6 +187,9 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
                                      and ISM)
     --abort-on-unavailable-fragment  Abort downloading when some fragment is not
                                      available
+    --keep-fragments                 Keep downloaded fragments on disk after
+                                     downloading is finished; fragments are
+                                     erased by default
     --buffer-size SIZE               Size of download buffer (e.g. 1024 or 16K)
                                      (default is 1024)
     --no-resize-buffer               Do not automatically adjust the buffer
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index afae82214..0be7af8c4 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -81,8 +81,6 @@
  - **AZMedien**: AZ Medien videos
  - **AZMedienPlaylist**: AZ Medien playlists
  - **AZMedienShowPlaylist**: AZ Medien show playlists
- - **Azubu**
- - **AzubuLive**
  - **BaiduVideo**: 百度视频
  - **bambuser**
  - **bambuser:channel**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 480e04e06..e206501e1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.04.17'
+__version__ = '2017.04.26'

From 3dc8b61b7f5b455e4f82626b10776d669082460f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Apr 2017 01:55:29 +0700
Subject: [PATCH 166/611] [arte:+7] Relax _VALID_URL (closes #12837)

---
 youtube_dl/extractor/arte.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 69a23e88c..56baef29d 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -180,7 +180,7 @@ class ArteTVBaseIE(InfoExtractor):
 
 class ArteTVPlus7IE(ArteTVBaseIE):
     IE_NAME = 'arte.tv:+7'
-    _VALID_URL = r'https?://(?:(?:www|sites)\.)?arte\.tv/[^/]+/(?P<lang>fr|de|en|es)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:(?:www|sites)\.)?arte\.tv/(?:[^/]+/)?(?P<lang>fr|de|en|es)/(?:videos/)?(?:[^/]+/)*(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'http://www.arte.tv/guide/de/sendungen/XEN/xenius/?vid=055918-015_PLUS7-D',
@@ -188,6 +188,9 @@ class ArteTVPlus7IE(ArteTVBaseIE):
     }, {
         'url': 'http://sites.arte.tv/karambolage/de/video/karambolage-22',
         'only_matching': True,
+    }, {
+        'url': 'http://www.arte.tv/de/videos/048696-000-A/der-kluge-bauch-unser-zweites-gehirn',
+        'only_matching': True,
     }]
 
     @classmethod

From e8bfe2a946eda74f16af8734c35fba8c223e41fa Mon Sep 17 00:00:00 2001
From: Lucas M <lawlmart@users.noreply.github.com>
Date: Wed, 26 Apr 2017 12:39:53 -0400
Subject: [PATCH 167/611] [streamable] Add support for new embedded URL schema

---
 youtube_dl/extractor/streamable.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/streamable.py b/youtube_dl/extractor/streamable.py
index 9f5c237ef..34725274e 100644
--- a/youtube_dl/extractor/streamable.py
+++ b/youtube_dl/extractor/streamable.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class StreamableIE(InfoExtractor):
-    _VALID_URL = r'https?://streamable\.com/(?:e/)?(?P<id>\w+)'
+    _VALID_URL = r'https?://streamable\.com/(?:[es]/)?(?P<id>\w+)'
     _TESTS = [
         {
             'url': 'https://streamable.com/dnd1',
@@ -47,6 +47,10 @@ class StreamableIE(InfoExtractor):
         {
             'url': 'https://streamable.com/e/dnd1',
             'only_matching': True,
+        },
+        {
+            'url': 'https://streamable.com/s/okkqk/drxjds',
+            'only_matching': True,
         }
     ]
 

From 76c19510368d61acd3e1315d93ac6f16dbeb2c65 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Thu, 27 Apr 2017 10:04:21 +0200
Subject: [PATCH 168/611] [appletrailers] update test cases

---
 youtube_dl/extractor/appletrailers.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index a6801f3d4..b45b431e1 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -70,7 +70,8 @@ class AppleTrailersIE(InfoExtractor):
     }, {
         'url': 'http://trailers.apple.com/trailers/magnolia/blackthorn/',
         'info_dict': {
-            'id': 'blackthorn',
+            'id': '4489',
+            'title': 'Blackthorn',
         },
         'playlist_mincount': 2,
         'expected_warnings': ['Unable to download JSON metadata'],
@@ -261,7 +262,7 @@ class AppleTrailersSectionIE(InfoExtractor):
             'title': 'Most Popular',
             'id': 'mostpopular',
         },
-        'playlist_mincount': 80,
+        'playlist_mincount': 30,
     }, {
         'url': 'http://trailers.apple.com/#section=moviestudios',
         'info_dict': {

From d68afc5bc9f0d6cbf49751f95db6f1b8e7c1403e Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Thu, 27 Apr 2017 15:27:01 +0200
Subject: [PATCH 169/611] [cbsinteractive] fix extractor

---
 youtube_dl/extractor/cbsinteractive.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cbsinteractive.py b/youtube_dl/extractor/cbsinteractive.py
index 57b18e81d..3d48a7cb1 100644
--- a/youtube_dl/extractor/cbsinteractive.py
+++ b/youtube_dl/extractor/cbsinteractive.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .theplatform import ThePlatformIE
+from ..compat import compat_urllib_parse
 from ..utils import int_or_none
 
 
@@ -80,7 +81,7 @@ class CBSInteractiveIE(ThePlatformIE):
 
         media_guid_path = 'media/guid/%d/%s' % (self.MPX_ACCOUNTS[site], vdata['mpxRefId'])
         formats, subtitles = [], {}
-        for (fkey, vid) in vdata['files'].items():
+        for (fkey, vid) in vdata.get('files', {}).items():
             if fkey == 'hls_phone' and 'hls_tablet' in vdata['files']:
                 continue
             release_url = self.TP_RELEASE_URL_TEMPLATE % vid
@@ -89,6 +90,23 @@ class CBSInteractiveIE(ThePlatformIE):
             tp_formats, tp_subtitles = self._extract_theplatform_smil(release_url, video_id, 'Downloading %s SMIL data' % fkey)
             formats.extend(tp_formats)
             subtitles = self._merge_subtitles(subtitles, tp_subtitles)
+
+        if 'm3u8' in vdata:
+            parsed_url = compat_urllib_parse.urlparse(url)
+            m3u8_url = ('%s://%s%s'
+                % (parsed_url.scheme, parsed_url.netloc, vdata['m3u8']))
+            m3u8_formats = self._extract_m3u8_formats(m3u8_url, video_id)
+            for format in m3u8_formats:
+                format['url'] = format['url'].replace('https://', 'http://')
+            formats.extend(m3u8_formats)
+
+        if 'mp4' in vdata:
+            formats.append({
+                'url': vdata['mp4'],
+                'format_id': 'mp4',
+                'ext': 'mp4',
+            })
+
         self._sort_formats(formats)
 
         info = self._extract_theplatform_metadata('kYEXFC/%s' % media_guid_path, video_id)

From e095109da16696d7ee66823db1c2ec2d4c11c1a8 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Thu, 27 Apr 2017 15:40:17 +0200
Subject: [PATCH 170/611] [cbsinteractive] update test cases

---
 youtube_dl/extractor/cbsinteractive.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/cbsinteractive.py b/youtube_dl/extractor/cbsinteractive.py
index 3d48a7cb1..f35f686ef 100644
--- a/youtube_dl/extractor/cbsinteractive.py
+++ b/youtube_dl/extractor/cbsinteractive.py
@@ -12,9 +12,10 @@ class CBSInteractiveIE(ThePlatformIE):
     _VALID_URL = r'https?://(?:www\.)?(?P<site>cnet|zdnet)\.com/(?:videos|video/share)/(?P<id>[^/?]+)'
     _TESTS = [{
         'url': 'http://www.cnet.com/videos/hands-on-with-microsofts-windows-8-1-update/',
+        'md5': '041233212a0d06b179c87cbcca1577b8',
         'info_dict': {
             'id': '56f4ea68-bd21-4852-b08c-4de5b8354c60',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Hands-on with Microsoft Windows 8.1 Update',
             'description': 'The new update to the Windows 8 OS brings improved performance for mouse and keyboard users.',
             'uploader_id': '6085384d-619e-11e3-b231-14feb5ca9861',
@@ -23,11 +24,15 @@ class CBSInteractiveIE(ThePlatformIE):
             'timestamp': 1396479627,
             'upload_date': '20140402',
         },
+        'params': {
+            'format': 'mp4',
+        },
     }, {
         'url': 'http://www.cnet.com/videos/whiny-pothole-tweets-at-local-government-when-hit-by-cars-tomorrow-daily-187/',
+        'md5': 'f2b16d73e08d69591dd9e25564695c0c',
         'info_dict': {
             'id': '56527b93-d25d-44e3-b738-f989ce2e49ba',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Whiny potholes tweet at local government when hit by cars (Tomorrow Daily 187)',
             'description': 'Khail and Ashley wonder what other civic woes can be solved by self-tweeting objects, investigate a new kind of VR camera and watch an origami robot self-assemble, walk, climb, dig and dissolve. #TDPothole',
             'uploader_id': 'b163284d-6b73-44fc-b3e6-3da66c392d40',
@@ -36,6 +41,9 @@ class CBSInteractiveIE(ThePlatformIE):
             'timestamp': 1433289889,
             'upload_date': '20150603',
         },
+        'params': {
+            'format': 'mp4',
+        },
     }, {
         'url': 'http://www.zdnet.com/video/share/video-keeping-android-smartphones-and-tablets-secure/',
         'info_dict': {
@@ -45,8 +53,8 @@ class CBSInteractiveIE(ThePlatformIE):
             'description': 'Here\'s the best way to keep Android devices secure, and what you do when they\'ve come to the end of their lives.',
             'uploader_id': 'f2d97ea2-8175-11e2-9d12-0018fe8a00b0',
             'uploader': 'Adrian Kingsley-Hughes',
-            'timestamp': 1448961720,
-            'upload_date': '20151201',
+            'timestamp': 1449129925,
+            'upload_date': '20151203',
         },
         'params': {
             # m3u8 download

From ff99fe529e52b2465f1d973e69df01a6391568d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 25 Apr 2017 22:07:10 +0700
Subject: [PATCH 171/611] Don't list master m3u8 playlists in format list
 (closes #12832)

---
 test/test_InfoExtractor.py         | 40 ------------------------------
 youtube_dl/extractor/anvato.py     |  7 +-----
 youtube_dl/extractor/common.py     |  4 +--
 youtube_dl/extractor/funnyordie.py |  3 +--
 youtube_dl/extractor/gamespot.py   |  3 +--
 youtube_dl/extractor/lego.py       |  2 +-
 youtube_dl/extractor/pbs.py        |  2 +-
 youtube_dl/extractor/r7.py         |  3 +--
 youtube_dl/extractor/ted.py        |  2 +-
 youtube_dl/extractor/tvp.py        |  3 +--
 youtube_dl/extractor/viewster.py   |  3 +--
 11 files changed, 11 insertions(+), 61 deletions(-)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 2a3c3f7cb..14abf03c4 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -184,14 +184,6 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 'pluzz_francetv_11507',
                 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/master.m3u8?caption=2017%2F16%2F156589847-1492488987.m3u8%3Afra%3AFrancais&audiotrack=0%3Afra%3AFrancais',
                 [{
-                    'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/master.m3u8?caption=2017%2F16%2F156589847-1492488987.m3u8%3Afra%3AFrancais&audiotrack=0%3Afra%3AFrancais',
-                    'ext': 'mp4',
-                    'format_id': 'meta',
-                    'format_note': 'Quality selection URL',
-                    'protocol': 'm3u8',
-                    'preference': -100,
-                    'resolution': 'multiple'
-                }, {
                     'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_0_av.m3u8?null=0',
                     'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_0_av.m3u8?null=0',
                     'ext': 'mp4',
@@ -254,14 +246,6 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 'teamcoco_11995',
                 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
                 [{
-                    'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
-                    'ext': 'mp4',
-                    'format_id': 'meta',
-                    'format_note': 'Quality selection URL',
-                    'protocol': 'm3u8',
-                    'preference': -100,
-                    'resolution': 'multiple',
-                }, {
                     'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-audio-160k_v4.m3u8',
                     'ext': 'mp4',
                     'format_id': 'audio-0-Default',
@@ -334,14 +318,6 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 'toggle_mobile_12211',
                 'http://cdnapi.kaltura.com/p/2082311/sp/208231100/playManifest/protocol/http/entryId/0_89q6e8ku/format/applehttp/tags/mobile_sd/f/a.m3u8',
                 [{
-                    'url': 'http://cdnapi.kaltura.com/p/2082311/sp/208231100/playManifest/protocol/http/entryId/0_89q6e8ku/format/applehttp/tags/mobile_sd/f/a.m3u8',
-                    'ext': 'mp4',
-                    'format_id': 'meta',
-                    'format_note': 'Quality selection URL',
-                    'protocol': 'm3u8',
-                    'preference': -100,
-                    'resolution': 'multiple'
-                }, {
                     'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_sa2ntrdg/name/a.mp4/index.m3u8',
                     'ext': 'mp4',
                     'format_id': 'audio-English',
@@ -398,14 +374,6 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 'twitch_vod',
                 'https://usher.ttvnw.net/vod/6528877?allow_source=true&allow_audio_only=true&allow_spectre=true&player=twitchweb&nauth=%7B%22user_id%22%3Anull%2C%22vod_id%22%3A6528877%2C%22expires%22%3A1492887874%2C%22chansub%22%3A%7B%22restricted_bitrates%22%3A%5B%5D%7D%2C%22privileged%22%3Afalse%2C%22https_required%22%3Afalse%7D&nauthsig=3e29296a6824a0f48f9e731383f77a614fc79bee',
                 [{
-                    'url': 'https://usher.ttvnw.net/vod/6528877?allow_source=true&allow_audio_only=true&allow_spectre=true&player=twitchweb&nauth=%7B%22user_id%22%3Anull%2C%22vod_id%22%3A6528877%2C%22expires%22%3A1492887874%2C%22chansub%22%3A%7B%22restricted_bitrates%22%3A%5B%5D%7D%2C%22privileged%22%3Afalse%2C%22https_required%22%3Afalse%7D&nauthsig=3e29296a6824a0f48f9e731383f77a614fc79bee',
-                    'ext': 'mp4',
-                    'format_id': 'meta',
-                    'format_note': 'Quality selection URL',
-                    'protocol': 'm3u8',
-                    'preference': -100,
-                    'resolution': 'multiple'
-                }, {
                     'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/audio_only/index-muted-HM49I092CC.m3u8',
                     'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/audio_only/index-muted-HM49I092CC.m3u8',
                     'ext': 'mp4',
@@ -478,14 +446,6 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 'vidio',
                 'https://www.vidio.com/videos/165683/playlist.m3u8',
                 [{
-                    'url': 'https://www.vidio.com/videos/165683/playlist.m3u8',
-                    'ext': 'mp4',
-                    'format_id': 'meta',
-                    'format_note': 'Quality selection URL',
-                    'protocol': 'm3u8',
-                    'preference': -100,
-                    'resolution': 'multiple'
-                }, {
                     'url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b300.mp4.m3u8',
                     'manifest_url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b300.mp4.m3u8',
                     'ext': 'mp4',
diff --git a/youtube_dl/extractor/anvato.py b/youtube_dl/extractor/anvato.py
index 623f44dce..9fd91c2f6 100644
--- a/youtube_dl/extractor/anvato.py
+++ b/youtube_dl/extractor/anvato.py
@@ -178,12 +178,7 @@ class AnvatoIE(InfoExtractor):
             }
 
             if ext == 'm3u8' or media_format in ('m3u8', 'm3u8-variant'):
-                # Not using _extract_m3u8_formats here as individual media
-                # playlists are also included in published_urls.
-                if tbr is None:
-                    formats.append(self._m3u8_meta_format(video_url, ext='mp4', m3u8_id='hls'))
-                    continue
-                else:
+                if tbr is not None:
                     a_format.update({
                         'format_id': '-'.join(filter(None, ['hls', compat_str(tbr)])),
                         'ext': 'mp4',
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8b3f04c61..6d01f0800 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1334,7 +1334,7 @@ class InfoExtractor(object):
         if '#EXT-X-FAXS-CM:' in m3u8_doc:  # Adobe Flash Access
             return []
 
-        formats = [self._m3u8_meta_format(m3u8_url, ext, preference, m3u8_id)]
+        formats = []
 
         format_url = lambda u: (
             u
@@ -1438,7 +1438,7 @@ class InfoExtractor(object):
                 f = {
                     'format_id': '-'.join(format_id),
                     'url': manifest_url,
-                    'manifest_url': manifest_url,
+                    'manifest_url': m3u8_url,
                     'tbr': tbr,
                     'ext': ext,
                     'fps': float_or_none(last_stream_inf.get('FRAME-RATE')),
diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index 81c0ce9a3..49409369c 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -58,8 +58,7 @@ class FunnyOrDieIE(InfoExtractor):
             m3u8_url, video_id, 'mp4', 'm3u8_native',
             m3u8_id='hls', fatal=False)
         source_formats = list(filter(
-            lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
-            m3u8_formats))
+            lambda f: f.get('vcodec') != 'none', m3u8_formats))
 
         bitrates = [int(bitrate) for bitrate in re.findall(r'[,/]v(\d+)(?=[,/])', m3u8_url)]
         bitrates.sort()
diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index 682c49e79..00d311158 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -78,8 +78,7 @@ class GameSpotIE(OnceIE):
                     if m3u8_formats:
                         self._sort_formats(m3u8_formats)
                         m3u8_formats = list(filter(
-                            lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
-                            m3u8_formats))
+                            lambda f: f.get('vcodec') != 'none', m3u8_formats))
                     if len(qualities) == len(m3u8_formats):
                         for q, m3u8_format in zip(qualities, m3u8_formats):
                             f = m3u8_format.copy()
diff --git a/youtube_dl/extractor/lego.py b/youtube_dl/extractor/lego.py
index d3bca6435..b312e77f1 100644
--- a/youtube_dl/extractor/lego.py
+++ b/youtube_dl/extractor/lego.py
@@ -86,7 +86,7 @@ class LEGOIE(InfoExtractor):
         formats = self._extract_akamai_formats(
             '%si/s/public/%s_,%s,.mp4.csmil/master.m3u8' % (streaming_base, path, streaming_path), video_id)
         m3u8_formats = list(filter(
-            lambda f: f.get('protocol') == 'm3u8_native' and f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
+            lambda f: f.get('protocol') == 'm3u8_native' and f.get('vcodec') != 'none',
             formats))
         if len(m3u8_formats) == len(self._BITRATES):
             self._sort_formats(m3u8_formats)
diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 3e51b4dd7..6166dc2ad 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -515,7 +515,7 @@ class PBSIE(InfoExtractor):
                     http_url = format_url
         self._remove_duplicate_formats(formats)
         m3u8_formats = list(filter(
-            lambda f: f.get('protocol') == 'm3u8' and f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
+            lambda f: f.get('protocol') == 'm3u8' and f.get('vcodec') != 'none',
             formats))
         if http_url:
             for m3u8_format in m3u8_formats:
diff --git a/youtube_dl/extractor/r7.py b/youtube_dl/extractor/r7.py
index ed38c77eb..e2202d603 100644
--- a/youtube_dl/extractor/r7.py
+++ b/youtube_dl/extractor/r7.py
@@ -62,8 +62,7 @@ class R7IE(InfoExtractor):
             # m3u8 format always matches the http format, let's copy metadata from
             # one to another
             m3u8_formats = list(filter(
-                lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
-                formats))
+                lambda f: f.get('vcodec') != 'none', formats))
             if len(m3u8_formats) == 1:
                 f_copy = m3u8_formats[0].copy()
                 f_copy.update(f)
diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 1b1afab32..3f3c681ae 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -210,7 +210,7 @@ class TEDIE(InfoExtractor):
                     resources.get('stream'), video_name, 'mp4', m3u8_id=format_id, fatal=False))
 
         m3u8_formats = list(filter(
-            lambda f: f.get('protocol') == 'm3u8' and f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
+            lambda f: f.get('protocol') == 'm3u8' and f.get('vcodec') != 'none',
             formats))
         if http_url:
             for m3u8_format in m3u8_formats:
diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index 06ea2b40a..c5b3288ad 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -150,8 +150,7 @@ class TVPEmbedIE(InfoExtractor):
                 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
             self._sort_formats(m3u8_formats)
             m3u8_formats = list(filter(
-                lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
-                m3u8_formats))
+                lambda f: f.get('vcodec') != 'none', m3u8_formats))
             formats.extend(m3u8_formats)
             for i, m3u8_format in enumerate(m3u8_formats, 2):
                 http_url = '%s-%d.mp4' % (video_url_base, i)
diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index fcf0cb100..d5d5b4c69 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -176,8 +176,7 @@ class ViewsterIE(InfoExtractor):
                     if m3u8_formats:
                         self._sort_formats(m3u8_formats)
                         m3u8_formats = list(filter(
-                            lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
-                            m3u8_formats))
+                            lambda f: f.get('vcodec') != 'none', m3u8_formats))
                     if len(qualities) == len(m3u8_formats):
                         for q, m3u8_format in zip(qualities, m3u8_formats):
                             f = m3u8_format.copy()

From 374560f0181423be530660fd4b2e1a685e6648f8 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Thu, 27 Apr 2017 11:04:21 +0200
Subject: [PATCH 172/611] [test_download] Fix order when testing file's md5

---
 test/test_download.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_download.py b/test/test_download.py
index 0e9f293b5..209f5f6d6 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -225,7 +225,7 @@ def generator(test_case, tname):
                                 format_bytes(got_fsize)))
                     if 'md5' in tc:
                         md5_for_file = _file_md5(tc_filename)
-                        self.assertEqual(md5_for_file, tc['md5'])
+                        self.assertEqual(tc['md5'], md5_for_file)
                 # Finally, check test cases' data again but this time against
                 # extracted data from info JSON file written during processing
                 info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json'

From c95e2b59115bff314ae833f7d3fd08ef1768c6f8 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Thu, 27 Apr 2017 18:07:07 +0200
Subject: [PATCH 173/611] [cbc] update test cases

---
 youtube_dl/extractor/cbc.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/cbc.py b/youtube_dl/extractor/cbc.py
index cf678e7f8..87ad14e91 100644
--- a/youtube_dl/extractor/cbc.py
+++ b/youtube_dl/extractor/cbc.py
@@ -96,6 +96,7 @@ class CBCIE(InfoExtractor):
         'info_dict': {
             'title': 'Keep Rover active during the deep freeze with doggie pushups and other fun indoor tasks',
             'id': 'dog-indoor-exercise-winter-1.3928238',
+            'description': 'md5:c18552e41726ee95bd75210d1ca9194c',
         },
         'playlist_mincount': 6,
     }]
@@ -165,12 +166,11 @@ class CBCPlayerIE(InfoExtractor):
             'uploader': 'CBCC-NEW',
         },
     }, {
-        # available only when we add `formats=MPEG4,FLV,MP3` to theplatform url
         'url': 'http://www.cbc.ca/player/play/2164402062',
-        'md5': '17a61eb813539abea40618d6323a7f82',
+        'md5': '33fcd8f6719b9dd60a5e73adcb83b9f6',
         'info_dict': {
             'id': '2164402062',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Cancer survivor four times over',
             'description': 'Tim Mayer has beaten three different forms of cancer four times in five years.',
             'timestamp': 1320410746,

From 96820c1c6bd2a9613725fc17fd44fd00d3301475 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Thu, 27 Apr 2017 20:23:52 +0200
Subject: [PATCH 174/611] [cbsinteractive] extract formats with `CBSIE`

---
 youtube_dl/extractor/cbs.py            |  6 +--
 youtube_dl/extractor/cbsinteractive.py | 69 +++++++-------------------
 2 files changed, 22 insertions(+), 53 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 58f258c54..1268e38ef 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -49,13 +49,13 @@ class CBSIE(CBSBaseIE):
         'only_matching': True,
     }]
 
-    def _extract_video_info(self, content_id):
+    def _extract_video_info(self, content_id, site='cbs', mpx_acc=2198311517):
         items_data = self._download_xml(
             'http://can.cbs.com/thunder/player/videoPlayerService.php',
-            content_id, query={'partner': 'cbs', 'contentId': content_id})
+            content_id, query={'partner': site, 'contentId': content_id})
         video_data = xpath_element(items_data, './/item')
         title = xpath_text(video_data, 'videoTitle', 'title', True)
-        tp_path = 'dJ5BDC/media/guid/2198311517/%s' % content_id
+        tp_path = 'dJ5BDC/media/guid/%d/%s' % (mpx_acc, content_id)
         tp_release_url = 'http://link.theplatform.com/s/' + tp_path
 
         asset_types = []
diff --git a/youtube_dl/extractor/cbsinteractive.py b/youtube_dl/extractor/cbsinteractive.py
index f35f686ef..86f7eee20 100644
--- a/youtube_dl/extractor/cbsinteractive.py
+++ b/youtube_dl/extractor/cbsinteractive.py
@@ -3,18 +3,17 @@ from __future__ import unicode_literals
 
 import re
 
-from .theplatform import ThePlatformIE
-from ..compat import compat_urllib_parse
+from .cbs import CBSIE
 from ..utils import int_or_none
 
 
-class CBSInteractiveIE(ThePlatformIE):
+class CBSInteractiveIE(CBSIE):
     _VALID_URL = r'https?://(?:www\.)?(?P<site>cnet|zdnet)\.com/(?:videos|video/share)/(?P<id>[^/?]+)'
     _TESTS = [{
         'url': 'http://www.cnet.com/videos/hands-on-with-microsofts-windows-8-1-update/',
-        'md5': '041233212a0d06b179c87cbcca1577b8',
         'info_dict': {
-            'id': '56f4ea68-bd21-4852-b08c-4de5b8354c60',
+            'id': 'R49SYt__yAfmlXR85z4f7gNmCBDcN_00',
+            'display_id': 'hands-on-with-microsofts-windows-8-1-update',
             'ext': 'mp4',
             'title': 'Hands-on with Microsoft Windows 8.1 Update',
             'description': 'The new update to the Windows 8 OS brings improved performance for mouse and keyboard users.',
@@ -25,45 +24,46 @@ class CBSInteractiveIE(ThePlatformIE):
             'upload_date': '20140402',
         },
         'params': {
-            'format': 'mp4',
+            # m3u8 download
+            'skip_download': True,
         },
     }, {
         'url': 'http://www.cnet.com/videos/whiny-pothole-tweets-at-local-government-when-hit-by-cars-tomorrow-daily-187/',
-        'md5': 'f2b16d73e08d69591dd9e25564695c0c',
+        'md5': 'f11d27b2fa18597fbf92444d2a9ed386',
         'info_dict': {
-            'id': '56527b93-d25d-44e3-b738-f989ce2e49ba',
+            'id': 'kjOJd_OoVJqbg_ZD8MZCOk8Wekb9QccK',
+            'display_id': 'whiny-pothole-tweets-at-local-government-when-hit-by-cars-tomorrow-daily-187',
             'ext': 'mp4',
             'title': 'Whiny potholes tweet at local government when hit by cars (Tomorrow Daily 187)',
-            'description': 'Khail and Ashley wonder what other civic woes can be solved by self-tweeting objects, investigate a new kind of VR camera and watch an origami robot self-assemble, walk, climb, dig and dissolve. #TDPothole',
+            'description': 'md5:d2b9a95a5ffe978ae6fbd4cf944d618f',
             'uploader_id': 'b163284d-6b73-44fc-b3e6-3da66c392d40',
             'uploader': 'Ashley Esqueda',
             'duration': 1482,
             'timestamp': 1433289889,
             'upload_date': '20150603',
         },
-        'params': {
-            'format': 'mp4',
-        },
     }, {
         'url': 'http://www.zdnet.com/video/share/video-keeping-android-smartphones-and-tablets-secure/',
         'info_dict': {
-            'id': 'bc1af9f0-a2b5-4e54-880d-0d95525781c0',
+            'id': 'k0r4T_ehht4xW_hAOqiVQPuBDPZ8SRjt',
+            'display_id': 'video-keeping-android-smartphones-and-tablets-secure',
             'ext': 'mp4',
             'title': 'Video: Keeping Android smartphones and tablets secure',
             'description': 'Here\'s the best way to keep Android devices secure, and what you do when they\'ve come to the end of their lives.',
             'uploader_id': 'f2d97ea2-8175-11e2-9d12-0018fe8a00b0',
             'uploader': 'Adrian Kingsley-Hughes',
+            'duration': 731,
             'timestamp': 1449129925,
             'upload_date': '20151203',
         },
         'params': {
             # m3u8 download
             'skip_download': True,
-        }
+        },
     }]
-    TP_RELEASE_URL_TEMPLATE = 'http://link.theplatform.com/s/kYEXFC/%s?mbr=true'
+
     MPX_ACCOUNTS = {
-        'cnet': 2288573011,
+        'cnet': 2198311517,
         'zdnet': 2387448114,
     }
 
@@ -77,7 +77,8 @@ class CBSInteractiveIE(ThePlatformIE):
         data = self._parse_json(data_json, display_id)
         vdata = data.get('video') or data['videos'][0]
 
-        video_id = vdata['id']
+        video_id = vdata['mpxRefId']
+
         title = vdata['title']
         author = vdata.get('author')
         if author:
@@ -87,37 +88,7 @@ class CBSInteractiveIE(ThePlatformIE):
             uploader = None
             uploader_id = None
 
-        media_guid_path = 'media/guid/%d/%s' % (self.MPX_ACCOUNTS[site], vdata['mpxRefId'])
-        formats, subtitles = [], {}
-        for (fkey, vid) in vdata.get('files', {}).items():
-            if fkey == 'hls_phone' and 'hls_tablet' in vdata['files']:
-                continue
-            release_url = self.TP_RELEASE_URL_TEMPLATE % vid
-            if fkey == 'hds':
-                release_url += '&manifest=f4m'
-            tp_formats, tp_subtitles = self._extract_theplatform_smil(release_url, video_id, 'Downloading %s SMIL data' % fkey)
-            formats.extend(tp_formats)
-            subtitles = self._merge_subtitles(subtitles, tp_subtitles)
-
-        if 'm3u8' in vdata:
-            parsed_url = compat_urllib_parse.urlparse(url)
-            m3u8_url = ('%s://%s%s'
-                % (parsed_url.scheme, parsed_url.netloc, vdata['m3u8']))
-            m3u8_formats = self._extract_m3u8_formats(m3u8_url, video_id)
-            for format in m3u8_formats:
-                format['url'] = format['url'].replace('https://', 'http://')
-            formats.extend(m3u8_formats)
-
-        if 'mp4' in vdata:
-            formats.append({
-                'url': vdata['mp4'],
-                'format_id': 'mp4',
-                'ext': 'mp4',
-            })
-
-        self._sort_formats(formats)
-
-        info = self._extract_theplatform_metadata('kYEXFC/%s' % media_guid_path, video_id)
+        info = self._extract_video_info(video_id, site, self.MPX_ACCOUNTS[site])
         info.update({
             'id': video_id,
             'display_id': display_id,
@@ -125,7 +96,5 @@ class CBSInteractiveIE(ThePlatformIE):
             'duration': int_or_none(vdata.get('duration')),
             'uploader': uploader,
             'uploader_id': uploader_id,
-            'subtitles': subtitles,
-            'formats': formats,
         })
         return info

From f5edd7ae5105c5e7c58f220f326f5d18ca2c035b Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 21:51:30 +0200
Subject: [PATCH 175/611] [clipfish] Update test

---
 youtube_dl/extractor/clipfish.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/clipfish.py b/youtube_dl/extractor/clipfish.py
index bb52e0c6f..0920f6219 100644
--- a/youtube_dl/extractor/clipfish.py
+++ b/youtube_dl/extractor/clipfish.py
@@ -12,7 +12,7 @@ class ClipfishIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?clipfish\.de/(?:[^/]+/)+video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.clipfish.de/special/ugly-americans/video/4343170/s01-e01-ugly-americans-date-in-der-hoelle/',
-        'md5': '720563e467b86374c194bdead08d207d',
+        'md5': 'b9a5dc46294154c1193e2d10e0c95693',
         'info_dict': {
             'id': '4343170',
             'ext': 'mp4',

From 6f4a8884164d7df2cae638d36b2e3dc400c81ba8 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 21:53:11 +0200
Subject: [PATCH 176/611] [br] Update test

---
 youtube_dl/extractor/br.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/br.py b/youtube_dl/extractor/br.py
index ff0aa11b1..2c32b6ae2 100644
--- a/youtube_dl/extractor/br.py
+++ b/youtube_dl/extractor/br.py
@@ -77,7 +77,7 @@ class BRIE(InfoExtractor):
                 'description': 'md5:bb659990e9e59905c3d41e369db1fbe3',
                 'duration': 893,
                 'uploader': 'Eva Maria Steimle',
-                'upload_date': '20140117',
+                'upload_date': '20170208',
             }
         },
     ]

From c89b49f7432ebaed7c5032194df9240f5da4a84f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 28 Apr 2017 03:00:14 +0700
Subject: [PATCH 177/611] [extractor/common] Add manifest_url for explicit
 group rendition formats

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6d01f0800..2cb55d6af 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1386,6 +1386,7 @@ class InfoExtractor(object):
                 f = {
                     'format_id': '-'.join(format_id),
                     'url': format_url(media_url),
+                    'manifest_url': m3u8_url,
                     'language': media.get('LANGUAGE'),
                     'ext': ext,
                     'protocol': entry_protocol,

From 30bb6ce1a49f37f11afdca96fa7fd62ec56395c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 28 Apr 2017 03:01:43 +0700
Subject: [PATCH 178/611] [test_InfoExtractor] Fix test_parse_m3u8_formats

---
 test/test_InfoExtractor.py | 50 ++++++++++++++++++++------------------
 1 file changed, 27 insertions(+), 23 deletions(-)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 14abf03c4..6f52e11f7 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -185,7 +185,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/master.m3u8?caption=2017%2F16%2F156589847-1492488987.m3u8%3Afra%3AFrancais&audiotrack=0%3Afra%3AFrancais',
                 [{
                     'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_0_av.m3u8?null=0',
-                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_0_av.m3u8?null=0',
+                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/master.m3u8?caption=2017%2F16%2F156589847-1492488987.m3u8%3Afra%3AFrancais&audiotrack=0%3Afra%3AFrancais',
                     'ext': 'mp4',
                     'format_id': '180',
                     'protocol': 'm3u8',
@@ -196,7 +196,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 144,
                 }, {
                     'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_1_av.m3u8?null=0',
-                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_1_av.m3u8?null=0',
+                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/master.m3u8?caption=2017%2F16%2F156589847-1492488987.m3u8%3Afra%3AFrancais&audiotrack=0%3Afra%3AFrancais',
                     'ext': 'mp4',
                     'format_id': '303',
                     'protocol': 'm3u8',
@@ -207,7 +207,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 180,
                 }, {
                     'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_2_av.m3u8?null=0',
-                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_2_av.m3u8?null=0',
+                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/master.m3u8?caption=2017%2F16%2F156589847-1492488987.m3u8%3Afra%3AFrancais&audiotrack=0%3Afra%3AFrancais',
                     'ext': 'mp4',
                     'format_id': '575',
                     'protocol': 'm3u8',
@@ -218,7 +218,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 288,
                 }, {
                     'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_3_av.m3u8?null=0',
-                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_3_av.m3u8?null=0',
+                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/master.m3u8?caption=2017%2F16%2F156589847-1492488987.m3u8%3Afra%3AFrancais&audiotrack=0%3Afra%3AFrancais',
                     'ext': 'mp4',
                     'format_id': '831',
                     'protocol': 'm3u8',
@@ -229,7 +229,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 396,
                 }, {
                     'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_4_av.m3u8?null=0',
-                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_4_av.m3u8?null=0',
+                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/master.m3u8?caption=2017%2F16%2F156589847-1492488987.m3u8%3Afra%3AFrancais&audiotrack=0%3Afra%3AFrancais',
                     'ext': 'mp4',
                     'protocol': 'm3u8',
                     'format_id': '1467',
@@ -247,19 +247,21 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
                 [{
                     'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-audio-160k_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
                     'ext': 'mp4',
                     'format_id': 'audio-0-Default',
                     'protocol': 'm3u8',
                     'vcodec': 'none',
                 }, {
                     'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-audio-64k_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
                     'ext': 'mp4',
                     'format_id': 'audio-1-Default',
                     'protocol': 'm3u8',
                     'vcodec': 'none',
                 }, {
                     'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-audio-64k_v4.m3u8',
-                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-audio-64k_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
                     'ext': 'mp4',
                     'format_id': '71',
                     'protocol': 'm3u8',
@@ -268,7 +270,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'tbr': 71,
                 }, {
                     'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-400k_v4.m3u8',
-                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-400k_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
                     'ext': 'mp4',
                     'format_id': '413',
                     'protocol': 'm3u8',
@@ -279,7 +281,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 224,
                 }, {
                     'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-400k_v4.m3u8',
-                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-400k_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
                     'ext': 'mp4',
                     'format_id': '522',
                     'protocol': 'm3u8',
@@ -290,7 +292,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 224,
                 }, {
                     'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-1m_v4.m3u8',
-                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-1m_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
                     'ext': 'mp4',
                     'format_id': '1205',
                     'protocol': 'm3u8',
@@ -301,7 +303,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 360,
                 }, {
                     'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-2m_v4.m3u8',
-                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-2m_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
                     'ext': 'mp4',
                     'format_id': '2374',
                     'protocol': 'm3u8',
@@ -319,6 +321,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 'http://cdnapi.kaltura.com/p/2082311/sp/208231100/playManifest/protocol/http/entryId/0_89q6e8ku/format/applehttp/tags/mobile_sd/f/a.m3u8',
                 [{
                     'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_sa2ntrdg/name/a.mp4/index.m3u8',
+                    'manifest_url': 'http://cdnapi.kaltura.com/p/2082311/sp/208231100/playManifest/protocol/http/entryId/0_89q6e8ku/format/applehttp/tags/mobile_sd/f/a.m3u8',
                     'ext': 'mp4',
                     'format_id': 'audio-English',
                     'protocol': 'm3u8',
@@ -326,6 +329,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'vcodec': 'none',
                 }, {
                     'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_r7y0nitg/name/a.mp4/index.m3u8',
+                    'manifest_url': 'http://cdnapi.kaltura.com/p/2082311/sp/208231100/playManifest/protocol/http/entryId/0_89q6e8ku/format/applehttp/tags/mobile_sd/f/a.m3u8',
                     'ext': 'mp4',
                     'format_id': 'audio-Undefined',
                     'protocol': 'm3u8',
@@ -333,7 +337,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'vcodec': 'none',
                 }, {
                     'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_qlk9hlzr/name/a.mp4/index.m3u8',
-                    'manifest_url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_qlk9hlzr/name/a.mp4/index.m3u8',
+                    'manifest_url': 'http://cdnapi.kaltura.com/p/2082311/sp/208231100/playManifest/protocol/http/entryId/0_89q6e8ku/format/applehttp/tags/mobile_sd/f/a.m3u8',
                     'ext': 'mp4',
                     'format_id': '155',
                     'protocol': 'm3u8',
@@ -342,7 +346,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 180,
                 }, {
                     'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_oefackmi/name/a.mp4/index.m3u8',
-                    'manifest_url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_oefackmi/name/a.mp4/index.m3u8',
+                    'manifest_url': 'http://cdnapi.kaltura.com/p/2082311/sp/208231100/playManifest/protocol/http/entryId/0_89q6e8ku/format/applehttp/tags/mobile_sd/f/a.m3u8',
                     'ext': 'mp4',
                     'format_id': '502',
                     'protocol': 'm3u8',
@@ -351,7 +355,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 270,
                 }, {
                     'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/12/pv/1/flavorId/0_vyg9pj7k/name/a.mp4/index.m3u8',
-                    'manifest_url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/12/pv/1/flavorId/0_vyg9pj7k/name/a.mp4/index.m3u8',
+                    'manifest_url': 'http://cdnapi.kaltura.com/p/2082311/sp/208231100/playManifest/protocol/http/entryId/0_89q6e8ku/format/applehttp/tags/mobile_sd/f/a.m3u8',
                     'ext': 'mp4',
                     'format_id': '827',
                     'protocol': 'm3u8',
@@ -360,7 +364,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 360,
                 }, {
                     'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/12/pv/1/flavorId/0_50n4psvx/name/a.mp4/index.m3u8',
-                    'manifest_url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/12/pv/1/flavorId/0_50n4psvx/name/a.mp4/index.m3u8',
+                    'manifest_url': 'http://cdnapi.kaltura.com/p/2082311/sp/208231100/playManifest/protocol/http/entryId/0_89q6e8ku/format/applehttp/tags/mobile_sd/f/a.m3u8',
                     'ext': 'mp4',
                     'format_id': '1396',
                     'protocol': 'm3u8',
@@ -375,7 +379,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 'https://usher.ttvnw.net/vod/6528877?allow_source=true&allow_audio_only=true&allow_spectre=true&player=twitchweb&nauth=%7B%22user_id%22%3Anull%2C%22vod_id%22%3A6528877%2C%22expires%22%3A1492887874%2C%22chansub%22%3A%7B%22restricted_bitrates%22%3A%5B%5D%7D%2C%22privileged%22%3Afalse%2C%22https_required%22%3Afalse%7D&nauthsig=3e29296a6824a0f48f9e731383f77a614fc79bee',
                 [{
                     'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/audio_only/index-muted-HM49I092CC.m3u8',
-                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/audio_only/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://usher.ttvnw.net/vod/6528877?allow_source=true&allow_audio_only=true&allow_spectre=true&player=twitchweb&nauth=%7B%22user_id%22%3Anull%2C%22vod_id%22%3A6528877%2C%22expires%22%3A1492887874%2C%22chansub%22%3A%7B%22restricted_bitrates%22%3A%5B%5D%7D%2C%22privileged%22%3Afalse%2C%22https_required%22%3Afalse%7D&nauthsig=3e29296a6824a0f48f9e731383f77a614fc79bee',
                     'ext': 'mp4',
                     'format_id': 'Audio Only',
                     'protocol': 'm3u8',
@@ -384,7 +388,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'tbr': 182.725,
                 }, {
                     'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/mobile/index-muted-HM49I092CC.m3u8',
-                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/mobile/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://usher.ttvnw.net/vod/6528877?allow_source=true&allow_audio_only=true&allow_spectre=true&player=twitchweb&nauth=%7B%22user_id%22%3Anull%2C%22vod_id%22%3A6528877%2C%22expires%22%3A1492887874%2C%22chansub%22%3A%7B%22restricted_bitrates%22%3A%5B%5D%7D%2C%22privileged%22%3Afalse%2C%22https_required%22%3Afalse%7D&nauthsig=3e29296a6824a0f48f9e731383f77a614fc79bee',
                     'ext': 'mp4',
                     'format_id': 'Mobile',
                     'protocol': 'm3u8',
@@ -395,7 +399,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 226,
                 }, {
                     'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/low/index-muted-HM49I092CC.m3u8',
-                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/low/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://usher.ttvnw.net/vod/6528877?allow_source=true&allow_audio_only=true&allow_spectre=true&player=twitchweb&nauth=%7B%22user_id%22%3Anull%2C%22vod_id%22%3A6528877%2C%22expires%22%3A1492887874%2C%22chansub%22%3A%7B%22restricted_bitrates%22%3A%5B%5D%7D%2C%22privileged%22%3Afalse%2C%22https_required%22%3Afalse%7D&nauthsig=3e29296a6824a0f48f9e731383f77a614fc79bee',
                     'ext': 'mp4',
                     'format_id': 'Low',
                     'protocol': 'm3u8',
@@ -406,7 +410,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 360,
                 }, {
                     'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/medium/index-muted-HM49I092CC.m3u8',
-                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/medium/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://usher.ttvnw.net/vod/6528877?allow_source=true&allow_audio_only=true&allow_spectre=true&player=twitchweb&nauth=%7B%22user_id%22%3Anull%2C%22vod_id%22%3A6528877%2C%22expires%22%3A1492887874%2C%22chansub%22%3A%7B%22restricted_bitrates%22%3A%5B%5D%7D%2C%22privileged%22%3Afalse%2C%22https_required%22%3Afalse%7D&nauthsig=3e29296a6824a0f48f9e731383f77a614fc79bee',
                     'ext': 'mp4',
                     'format_id': 'Medium',
                     'protocol': 'm3u8',
@@ -417,7 +421,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 480,
                 }, {
                     'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/high/index-muted-HM49I092CC.m3u8',
-                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/high/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://usher.ttvnw.net/vod/6528877?allow_source=true&allow_audio_only=true&allow_spectre=true&player=twitchweb&nauth=%7B%22user_id%22%3Anull%2C%22vod_id%22%3A6528877%2C%22expires%22%3A1492887874%2C%22chansub%22%3A%7B%22restricted_bitrates%22%3A%5B%5D%7D%2C%22privileged%22%3Afalse%2C%22https_required%22%3Afalse%7D&nauthsig=3e29296a6824a0f48f9e731383f77a614fc79bee',
                     'ext': 'mp4',
                     'format_id': 'High',
                     'protocol': 'm3u8',
@@ -428,7 +432,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 720,
                 }, {
                     'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/chunked/index-muted-HM49I092CC.m3u8',
-                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/chunked/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://usher.ttvnw.net/vod/6528877?allow_source=true&allow_audio_only=true&allow_spectre=true&player=twitchweb&nauth=%7B%22user_id%22%3Anull%2C%22vod_id%22%3A6528877%2C%22expires%22%3A1492887874%2C%22chansub%22%3A%7B%22restricted_bitrates%22%3A%5B%5D%7D%2C%22privileged%22%3Afalse%2C%22https_required%22%3Afalse%7D&nauthsig=3e29296a6824a0f48f9e731383f77a614fc79bee',
                     'ext': 'mp4',
                     'format_id': 'Source',
                     'protocol': 'm3u8',
@@ -447,7 +451,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 'https://www.vidio.com/videos/165683/playlist.m3u8',
                 [{
                     'url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b300.mp4.m3u8',
-                    'manifest_url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b300.mp4.m3u8',
+                    'manifest_url': 'https://www.vidio.com/videos/165683/playlist.m3u8',
                     'ext': 'mp4',
                     'format_id': '270p 3G',
                     'protocol': 'm3u8',
@@ -456,7 +460,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 270,
                 }, {
                     'url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b600.mp4.m3u8',
-                    'manifest_url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b600.mp4.m3u8',
+                    'manifest_url': 'https://www.vidio.com/videos/165683/playlist.m3u8',
                     'ext': 'mp4',
                     'format_id': '360p SD',
                     'protocol': 'm3u8',
@@ -465,7 +469,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 360,
                 }, {
                     'url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b1200.mp4.m3u8',
-                    'manifest_url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b1200.mp4.m3u8',
+                    'manifest_url': 'https://www.vidio.com/videos/165683/playlist.m3u8',
                     'ext': 'mp4',
                     'format_id': '720p HD',
                     'protocol': 'm3u8',

From dde97ea8da64aae2fc328ea5e34645fad8d9b29d Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 22:07:42 +0200
Subject: [PATCH 179/611] [canalc2] Update test

---
 youtube_dl/extractor/canalc2.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index f1f128c45..acd87e371 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -16,13 +16,10 @@ class Canalc2IE(InfoExtractor):
         'md5': '060158428b650f896c542dfbb3d6487f',
         'info_dict': {
             'id': '12163',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Terrasses du Numérique',
             'duration': 122,
         },
-        'params': {
-            'skip_download': True,  # Requires rtmpdump
-        }
     }, {
         'url': 'http://archives-canalc2.u-strasbg.fr/video.asp?idVideo=11427&voir=oui',
         'only_matching': True,

From 0f63dc2402325481f087f862adefc86cb44cc888 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 22:13:12 +0200
Subject: [PATCH 180/611] [bandcamp] Update test

---
 youtube_dl/extractor/bandcamp.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 056e06376..df2972f26 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -34,12 +34,12 @@ class BandcampIE(InfoExtractor):
         '_skip': 'There is a limit of 200 free downloads / month for the test song'
     }, {
         'url': 'http://benprunty.bandcamp.com/track/lanius-battle',
-        'md5': '73d0b3171568232574e45652f8720b5c',
+        'md5': '0369ace6b939f0927e62c67a1a8d9fa7',
         'info_dict': {
             'id': '2650410135',
-            'ext': 'mp3',
-            'title': 'Lanius (Battle)',
-            'uploader': 'Ben Prunty Music',
+            'ext': 'aiff',
+            'title': 'Ben Prunty - Lanius (Battle)',
+            'uploader': 'Ben Prunty',
         },
     }]
 

From 8814ae42bc756b97aa0b341792d70ddcbab1de3b Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 22:14:11 +0200
Subject: [PATCH 181/611] [beeg] Update test

---
 youtube_dl/extractor/beeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index b0b7914d8..d5c5822f2 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -16,7 +16,7 @@ class BeegIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?beeg\.com/(?P<id>\d+)'
     _TEST = {
         'url': 'http://beeg.com/5416503',
-        'md5': '46c384def73b33dbc581262e5ee67cef',
+        'md5': 'a1a1b1a8bc70a89e49ccfd113aed0820',
         'info_dict': {
             'id': '5416503',
             'ext': 'mp4',

From c0fa4245cea1dc493e5a72e36644114a2743039c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 28 Apr 2017 03:24:59 +0700
Subject: [PATCH 182/611] [downloader/fragment] Remove assert for resume_len
 when no fragments downloaded This may be incorrect due some header (e.g. flv
 header in f4m downloader)

---
 youtube_dl/downloader/fragment.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index bb9e82578..d529ae09a 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -155,8 +155,6 @@ class FragmentFD(FileDownloader):
                 self._write_ytdl_file(ctx)
             if ctx['fragment_index'] > 0:
                 assert resume_len > 0
-            else:
-                assert resume_len == 0
 
         dest_stream, tmpfilename = sanitize_open(tmpfilename, open_mode)
 

From d8571dd6bfd62274b553e2dbc1acb67d7c0b636b Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 22:28:27 +0200
Subject: [PATCH 183/611] [bleacherreport] Update tests

---
 youtube_dl/extractor/bleacherreport.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/bleacherreport.py b/youtube_dl/extractor/bleacherreport.py
index 7a8e1f60b..e829974ff 100644
--- a/youtube_dl/extractor/bleacherreport.py
+++ b/youtube_dl/extractor/bleacherreport.py
@@ -35,7 +35,7 @@ class BleacherReportIE(InfoExtractor):
             'title': 'Aussie Golfers Get Fright of Their Lives After Being Chased by Angry Kangaroo',
             'timestamp': 1446839961,
             'uploader': 'Sean Fay',
-            'description': 'md5:825e94e0f3521df52fa83b2ed198fa20',
+            'description': 'md5:b1601e2314c4d8eec23b6eafe086a757',
             'uploader_id': 6466954,
             'upload_date': '20151011',
         },
@@ -90,17 +90,13 @@ class BleacherReportCMSIE(AMPIE):
     _VALID_URL = r'https?://(?:www\.)?bleacherreport\.com/video_embed\?id=(?P<id>[0-9a-f-]{36})'
     _TESTS = [{
         'url': 'http://bleacherreport.com/video_embed?id=8fd44c2f-3dc5-4821-9118-2c825a98c0e1',
-        'md5': '8c2c12e3af7805152675446c905d159b',
+        'md5': '2e4b0a997f9228ffa31fada5c53d1ed1',
         'info_dict': {
             'id': '8fd44c2f-3dc5-4821-9118-2c825a98c0e1',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Cena vs. Rollins Would Expose the Heavyweight Division',
             'description': 'md5:984afb4ade2f9c0db35f3267ed88b36e',
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
     }]
 
     def _real_extract(self, url):

From a15777491ad6b3bff56d8f6532ce703e0f05356a Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 22:32:25 +0200
Subject: [PATCH 184/611] [atresplayer] Update test

---
 youtube_dl/extractor/atresplayer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 99af6dc5a..01fa308ff 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -36,7 +36,7 @@ class AtresPlayerIE(InfoExtractor):
         },
         {
             'url': 'http://www.atresplayer.com/television/especial/videoencuentros/temporada-1/capitulo-112-david-bustamante_2014121600375.html',
-            'md5': '0d0e918533bbd4b263f2de4d197d4aac',
+            'md5': '6e52cbb513c405e403dbacb7aacf8747',
             'info_dict': {
                 'id': 'capitulo-112-david-bustamante',
                 'ext': 'flv',

From ac1a5b9a12f339fbed2c5b54298d8919238816cd Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 22:36:28 +0200
Subject: [PATCH 185/611] [audioboom] Update test

---
 youtube_dl/extractor/audioboom.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/audioboom.py b/youtube_dl/extractor/audioboom.py
index 8fc5f65c6..e48bb8972 100644
--- a/youtube_dl/extractor/audioboom.py
+++ b/youtube_dl/extractor/audioboom.py
@@ -16,7 +16,7 @@ class AudioBoomIE(InfoExtractor):
             'title': '3/09/2016 Czaban Hour 3',
             'description': 'Guest:   Nate Davis - NFL free agency,   Guest:   Stan Gans',
             'duration': 2245.72,
-            'uploader': 'Steve Czaban',
+            'uploader': 'SB Nation A.M.',
             'uploader_url': r're:https?://(?:www\.)?audioboom\.com/channel/steveczabanyahoosportsradio',
         }
     }, {

From 064fafe932c499c43416485c6a1fa0cc0422f818 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 22:47:25 +0200
Subject: [PATCH 186/611] [appleconnect] Update test

---
 youtube_dl/extractor/appleconnect.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/appleconnect.py b/youtube_dl/extractor/appleconnect.py
index ea7a70393..a84b8b1eb 100644
--- a/youtube_dl/extractor/appleconnect.py
+++ b/youtube_dl/extractor/appleconnect.py
@@ -12,13 +12,13 @@ class AppleConnectIE(InfoExtractor):
     _VALID_URL = r'https?://itunes\.apple\.com/\w{0,2}/?post/idsa\.(?P<id>[\w-]+)'
     _TEST = {
         'url': 'https://itunes.apple.com/us/post/idsa.4ab17a39-2720-11e5-96c5-a5b38f6c42d3',
-        'md5': '10d0f2799111df4cb1c924520ca78f98',
+        'md5': 'e7c38568a01ea45402570e6029206723',
         'info_dict': {
             'id': '4ab17a39-2720-11e5-96c5-a5b38f6c42d3',
             'ext': 'm4v',
             'title': 'Energy',
             'uploader': 'Drake',
-            'thumbnail': 'http://is5.mzstatic.com/image/thumb/Video5/v4/78/61/c5/7861c5fa-ad6d-294b-1464-cf7605b911d6/source/1920x1080sr.jpg',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'upload_date': '20150710',
             'timestamp': 1436545535,
         },

From c12b4b80f8371fd8554589ab777f5b9294a88c4a Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 22:48:32 +0200
Subject: [PATCH 187/611] [archiveorg] Update test

---
 youtube_dl/extractor/archiveorg.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/archiveorg.py b/youtube_dl/extractor/archiveorg.py
index e21045bed..3c7d7250b 100644
--- a/youtube_dl/extractor/archiveorg.py
+++ b/youtube_dl/extractor/archiveorg.py
@@ -24,12 +24,12 @@ class ArchiveOrgIE(InfoExtractor):
         }
     }, {
         'url': 'https://archive.org/details/Cops1922',
-        'md5': 'bc73c8ab3838b5a8fc6c6651fa7b58ba',
+        'md5': '0869000b4ce265e8ca62738b336b268a',
         'info_dict': {
             'id': 'Cops1922',
             'ext': 'mp4',
             'title': 'Buster Keaton\'s "Cops" (1922)',
-            'description': 'md5:b4544662605877edd99df22f9620d858',
+            'description': 'md5:89e7c77bf5d965dd5c0372cfb49470f6',
         }
     }, {
         'url': 'http://archive.org/embed/XD300-23_68HighlightsAResearchCntAugHumanIntellect',

From 6b4ddd336cf50f2ecc00b2addf4950d73b1b3087 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 23:00:15 +0200
Subject: [PATCH 188/611] [afreecatv] Fix title extraction

---
 youtube_dl/extractor/afreecatv.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index 78d29c861..c8cb91dcb 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -207,11 +207,10 @@ class AfreecaTVIE(InfoExtractor):
                     file_url, video_id, 'mp4', entry_protocol='m3u8_native',
                     m3u8_id='hls',
                     note='Downloading part %d m3u8 information' % file_num)
-                title = title if one else '%s (part %d)' % (title, file_num)
                 file_info = common_entry.copy()
                 file_info.update({
                     'id': format_id,
-                    'title': title,
+                    'title': title if one else '%s (part %d)' % (title, file_num),
                     'upload_date': upload_date,
                     'duration': file_duration,
                     'formats': formats,

From 33c62efc32be0566ec21ecba817c1b2f7a6be1ce Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 23:00:49 +0200
Subject: [PATCH 189/611] [collegerama] Update tests

---
 youtube_dl/extractor/collegerama.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/collegerama.py b/youtube_dl/extractor/collegerama.py
index 18c734766..6a41db87c 100644
--- a/youtube_dl/extractor/collegerama.py
+++ b/youtube_dl/extractor/collegerama.py
@@ -21,7 +21,7 @@ class CollegeRamaIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Een nieuwe wereld: waarden, bewustzijn en techniek van de mensheid 2.0.',
                 'description': '',
-                'thumbnail': r're:^https?://.*\.jpg$',
+                'thumbnail': r're:^https?://.*\.jpg(?:\?.*?)?$',
                 'duration': 7713.088,
                 'timestamp': 1413309600,
                 'upload_date': '20141014',
@@ -35,6 +35,7 @@ class CollegeRamaIE(InfoExtractor):
                 'ext': 'wmv',
                 'title': '64ste Vakantiecursus: Afvalwater',
                 'description': 'md5:7fd774865cc69d972f542b157c328305',
+                'thumbnail': r're:^https?://.*\.jpg(?:\?.*?)?$',
                 'duration': 10853,
                 'timestamp': 1326446400,
                 'upload_date': '20120113',

From 3fd0f70f6a1107a217f5c4c860ae180a6daf1ab6 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 23:26:59 +0200
Subject: [PATCH 190/611] [cbslocal] Update test

---
 youtube_dl/extractor/cbslocal.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/cbslocal.py b/youtube_dl/extractor/cbslocal.py
index 8d5f11dd1..7d78e3aae 100644
--- a/youtube_dl/extractor/cbslocal.py
+++ b/youtube_dl/extractor/cbslocal.py
@@ -60,8 +60,8 @@ class CBSLocalIE(AnvatoIE):
             'title': 'A Very Blue Anniversary',
             'description': 'CBS2’s Cindy Hsu has more.',
             'thumbnail': 're:^https?://.*',
-            'timestamp': 1479962220,
-            'upload_date': '20161124',
+            'timestamp': int,
+            'upload_date': r're:^\d{8}$',
             'uploader': 'CBS',
             'subtitles': {
                 'en': 'mincount:5',

From e31fed95b4b3026c34cec85c38c89b9b5c98b3d6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 28 Apr 2017 15:47:26 +0800
Subject: [PATCH 191/611] [youtube] Recognize new locale-based player URLs
 (fixes #12885)

---
 ChangeLog                       | 6 ++++++
 youtube_dl/extractor/youtube.py | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index b23aa10d5..a31f99853 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractor
+* [youtube] Recognize new locale-based player URLs (#12885)
+
+
 version 2017.04.26
 
 Core
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9e2b9115c..b59f746af 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -963,7 +963,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
     def _extract_signature_function(self, video_id, player_url, example_sig):
         id_m = re.match(
-            r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|/base)?\.(?P<ext>[a-z]+)$',
+            r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|(?:/[a-z]{2}_[A-Z]{2})?/base)?\.(?P<ext>[a-z]+)$',
             player_url)
         if not id_m:
             raise ExtractorError('Cannot identify player %r' % player_url)

From b62985a9a5d25643f1b1d6db8711013b1f123698 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 28 Apr 2017 16:25:04 +0800
Subject: [PATCH 192/611] [youtube] Recognize another HTML5 player URL (#12885)

---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b59f746af..480f403da 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1629,7 +1629,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                                 player_desc = 'flash player %s' % player_version
                             else:
                                 player_version = self._search_regex(
-                                    [r'html5player-([^/]+?)(?:/html5player(?:-new)?)?\.js', r'(?:www|player)-([^/]+)/base\.js'],
+                                    [r'html5player-([^/]+?)(?:/html5player(?:-new)?)?\.js',
+                                     r'(?:www|player)-([^/]+)(?:/[a-z]{2}_[A-Z]{2})?/base\.js'],
                                     player_url,
                                     'html5 player', fatal=False)
                                 player_desc = 'html5 player %s' % player_version

From fb8e8b2d1684e66c1d3f3587010d8ef51ebff7ce Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 28 Apr 2017 09:47:30 +0100
Subject: [PATCH 193/611] [adobepass] use geo verification headers for all
 requests

---
 youtube_dl/extractor/adobepass.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index 100cf997f..7da96c65c 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -1308,6 +1308,12 @@ class AdobePassIE(InfoExtractor):
     _USER_AGENT = 'Mozilla/5.0 (X11; Linux i686; rv:47.0) Gecko/20100101 Firefox/47.0'
     _MVPD_CACHE = 'ap-mvpd'
 
+    def _download_webpage_handle(self, *args, **kwargs):
+        headers = kwargs.get('headers', {})
+        headers.update(self.geo_verification_headers())
+        kwargs['headers'] = headers
+        return super(AdobePassIE, self)._download_webpage_handle(*args, **kwargs)
+
     @staticmethod
     def _get_mvpd_resource(provider_id, title, guid, rating):
         channel = etree.Element('channel')

From bf2a5555c0d56579484df4d2a5d65546aeb1bf39 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 28 Apr 2017 09:48:22 +0100
Subject: [PATCH 194/611] [go] add support for Disney, DisneyJunior and
 DisneyXD show pages

---
 youtube_dl/extractor/go.py | 49 +++++++++++++++++++++++++++-----------
 1 file changed, 35 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/go.py b/youtube_dl/extractor/go.py
index 4c9be47b4..9c7b1bd37 100644
--- a/youtube_dl/extractor/go.py
+++ b/youtube_dl/extractor/go.py
@@ -36,22 +36,26 @@ class GoIE(AdobePassIE):
             'requestor_id': 'DisneyXD',
         }
     }
-    _VALID_URL = r'https?://(?:(?P<sub_domain>%s)\.)?go\.com/(?:[^/]+/)*(?:vdka(?P<id>\w+)|(?:[^/]+/)*(?P<display_id>[^/?#]+))' % '|'.join(_SITE_INFO.keys())
+    _VALID_URL = r'https?://(?:(?P<sub_domain>%s)\.)?go\.com/(?:(?:[^/]+/)*(?P<id>vdka\w+)|(?:[^/]+/)*(?P<display_id>[^/?#]+))' % '|'.join(_SITE_INFO.keys())
     _TESTS = [{
-        'url': 'http://abc.go.com/shows/castle/video/most-recent/vdka0_g86w5onx',
+        'url': 'http://abc.go.com/shows/designated-survivor/video/most-recent/VDKA3807643',
         'info_dict': {
-            'id': '0_g86w5onx',
+            'id': 'VDKA3807643',
             'ext': 'mp4',
-            'title': 'Sneak Peek: Language Arts',
-            'description': 'md5:7dcdab3b2d17e5217c953256af964e9c',
+            'title': 'The Traitor in the White House',
+            'description': 'md5:05b009d2d145a1e85d25111bd37222e8',
         },
         'params': {
             # m3u8 download
             'skip_download': True,
         },
     }, {
-        'url': 'http://abc.go.com/shows/after-paradise/video/most-recent/vdka3335601',
-        'only_matching': True,
+        'url': 'http://watchdisneyxd.go.com/doraemon',
+        'info_dict': {
+            'title': 'Doraemon',
+            'id': 'SH55574025',
+        },
+        'playlist_mincount': 51,
     }, {
         'url': 'http://abc.go.com/shows/the-catch/episode-guide/season-01/10-the-wedding',
         'only_matching': True,
@@ -60,19 +64,36 @@ class GoIE(AdobePassIE):
         'only_matching': True,
     }]
 
+    def _extract_videos(self, brand, video_id='-1', show_id='-1'):
+        display_id = video_id if video_id != '-1' else show_id
+        return self._download_json(
+            'http://api.contents.watchabc.go.com/vp2/ws/contents/3000/videos/%s/001/-1/%s/-1/%s/-1/-1.json' % (brand, show_id, video_id),
+            display_id)['video']
+
     def _real_extract(self, url):
         sub_domain, video_id, display_id = re.match(self._VALID_URL, url).groups()
+        site_info = self._SITE_INFO[sub_domain]
+        brand = site_info['brand']
         if not video_id:
             webpage = self._download_webpage(url, display_id)
             video_id = self._search_regex(
                 # There may be inner quotes, e.g. data-video-id="'VDKA3609139'"
                 # from http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood
-                r'data-video-id=["\']*VDKA(\w+)', webpage, 'video id')
-        site_info = self._SITE_INFO[sub_domain]
-        brand = site_info['brand']
-        video_data = self._download_json(
-            'http://api.contents.watchabc.go.com/vp2/ws/contents/3000/videos/%s/001/-1/-1/-1/%s/-1/-1.json' % (brand, video_id),
-            video_id)['video'][0]
+                r'data-video-id=["\']*(VDKA\w+)', webpage, 'video id', default=None)
+            if not video_id:
+                # show extraction works for Disney, DisneyJunior and DisneyXD
+                # ABC and Freeform has different layout
+                show_id = self._search_regex(r'data-show-id=["\']*(SH\d+)', webpage, 'show id')
+                videos = self._extract_videos(brand, show_id=show_id)
+                show_title = self._search_regex(r'data-show-title="([^"]+)"', webpage, 'show title', fatal=False)
+                entries = []
+                for video in videos:
+                    entries.append(self.url_result(
+                        video['url'], 'Go', video.get('id'), video.get('title')))
+                entries.reverse()
+                return self.playlist_result(entries, show_id, show_title)
+        video_data = self._extract_videos(brand, video_id)[0]
+        video_id = video_data['id']
         title = video_data['title']
 
         formats = []
@@ -105,7 +126,7 @@ class GoIE(AdobePassIE):
                     self._initialize_geo_bypass(['US'])
                 entitlement = self._download_json(
                     'https://api.entitlement.watchabc.go.com/vp2/ws-secure/entitlement/2020/authorize.json',
-                    video_id, data=urlencode_postdata(data), headers=self.geo_verification_headers())
+                    video_id, data=urlencode_postdata(data))
                 errors = entitlement.get('errors', {}).get('errors', [])
                 if errors:
                     for error in errors:

From 20783b8b50e068180c62b973f6a589bd762be839 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 28 Apr 2017 12:04:56 +0100
Subject: [PATCH 195/611] [aenetworks] fix extraction for shows with single
 season

---
 youtube_dl/extractor/aenetworks.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index c01c67303..2dcdba9d2 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -101,10 +101,14 @@ class AENetworksIE(AENetworksBaseIE):
                 for season_url_path in re.findall(r'(?s)<li[^>]+data-href="(/shows/%s/season-\d+)"' % url_parts[0], webpage):
                     entries.append(self.url_result(
                         compat_urlparse.urljoin(url, season_url_path), 'AENetworks'))
-                return self.playlist_result(
-                    entries, self._html_search_meta('aetn:SeriesId', webpage),
-                    self._html_search_meta('aetn:SeriesTitle', webpage))
-            elif url_parts_len == 2:
+                if entries:
+                    return self.playlist_result(
+                        entries, self._html_search_meta('aetn:SeriesId', webpage),
+                        self._html_search_meta('aetn:SeriesTitle', webpage))
+                else:
+                    # single season
+                    url_parts_len = 2
+            if url_parts_len == 2:
                 entries = []
                 for episode_item in re.findall(r'(?s)<[^>]+class="[^"]*(?:episode|program)-item[^"]*"[^>]*>', webpage):
                     episode_attributes = extract_attributes(episode_item)
@@ -112,7 +116,7 @@ class AENetworksIE(AENetworksBaseIE):
                         url, episode_attributes['data-canonical'])
                     entries.append(self.url_result(
                         episode_url, 'AENetworks',
-                        episode_attributes['data-videoid']))
+                        episode_attributes.get('data-videoid') or episode_attributes.get('data-video-id')))
                 return self.playlist_result(
                     entries, self._html_search_meta('aetn:SeasonId', webpage))
 

From 7527923371ed5dea83ab102d882fed52ebc4672e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 28 Apr 2017 18:27:29 +0700
Subject: [PATCH 196/611] [ChangeLog] Actualize

---
 ChangeLog | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index a31f99853..0af1c6188 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,19 @@
 version <unreleased>
 
+Core
++ [adobepass] Use geo verification headers for all requests
+- [downloader/fragment] Remove assert for resume_len when no fragments
+  downloaded
++ [extractor/common] Add manifest_url for explicit group rendition formats
+* [extractor/common] Fix manifest_url for m3u8 formats
+- [extractor/common] Don't list master m3u8 playlists in format list (#12832)
+
 Extractor
+* [aenetworks] Fix extraction for shows with single season
++ [go] Add support for Disney, DisneyJunior and DisneyXD show pages
 * [youtube] Recognize new locale-based player URLs (#12885)
++ [streamable] Add support for new embedded URL schema (#12844)
+* [arte:+7] Relax URL regular expression (#12837)
 
 
 version 2017.04.26

From bc8a2ea07107c65e3561826b066888d450b98391 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 28 Apr 2017 18:30:03 +0700
Subject: [PATCH 197/611] release 2017.04.28

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index dedc07604..a84cbc801 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.26*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.26**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.28*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.28**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.04.26
+[debug] youtube-dl version 2017.04.28
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 0af1c6188..cd49ac42d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.04.28
 
 Core
 + [adobepass] Use geo verification headers for all requests
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e206501e1..8c77f1905 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.04.26'
+__version__ = '2017.04.28'

From edd9221cd22614b9d06f52b16bad278201fabd01 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Fri, 28 Apr 2017 17:34:27 +0200
Subject: [PATCH 198/611] [utils] Fix inconsistent output of clean_html

`\s` in Python 2.x doesn't match unicode whitespace characters by
default
---
 youtube_dl/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 91e235ff2..41bc20544 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -421,8 +421,8 @@ def clean_html(html):
 
     # Newline vs <br />
     html = html.replace('\n', ' ')
-    html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
-    html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
+    html = re.sub(r'(?u)\s*<\s*br\s*/?\s*>\s*', '\n', html)
+    html = re.sub(r'(?u)<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
     # Strip html tags
     html = re.sub('<.*?>', '', html)
     # Replace html entities

From 9222d945107411ba657dab965fab63de8e8781b9 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Fri, 28 Apr 2017 18:05:14 +0200
Subject: [PATCH 199/611] [test_utils] Add one more clean_html test

---
 test/test_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 4cd818850..9e1808e6b 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -899,6 +899,7 @@ class TestUtil(unittest.TestCase):
     def test_clean_html(self):
         self.assertEqual(clean_html('a:\nb'), 'a: b')
         self.assertEqual(clean_html('a:\n   "b"'), 'a:    "b"')
+        self.assertEqual(clean_html('a<br>\xa0b'), 'a\nb')
 
     def test_intlist_to_bytes(self):
         self.assertEqual(

From b364c87c423cbc031cd64566a1b283097b560d59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Apr 2017 03:39:39 +0700
Subject: [PATCH 200/611] [tvplayer] Fix extraction (closes #12908)

---
 youtube_dl/extractor/tvplayer.py | 35 +++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/tvplayer.py b/youtube_dl/extractor/tvplayer.py
index b6537141a..ebde6053f 100644
--- a/youtube_dl/extractor/tvplayer.py
+++ b/youtube_dl/extractor/tvplayer.py
@@ -2,9 +2,13 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..compat import (
+    compat_HTTPError,
+    compat_str,
+)
 from ..utils import (
     extract_attributes,
+    try_get,
     urlencode_postdata,
     ExtractorError,
 )
@@ -34,25 +38,32 @@ class TVPlayerIE(InfoExtractor):
             webpage, 'channel element'))
         title = current_channel['data-name']
 
-        resource_id = self._search_regex(
-            r'resourceId\s*=\s*"(\d+)"', webpage, 'resource id')
-        platform = self._search_regex(
-            r'platform\s*=\s*"([^"]+)"', webpage, 'platform')
+        resource_id = current_channel['data-id']
+
         token = self._search_regex(
-            r'token\s*=\s*"([^"]+)"', webpage, 'token', default='null')
-        validate = self._search_regex(
-            r'validate\s*=\s*"([^"]+)"', webpage, 'validate', default='null')
+            r'data-token=(["\'])(?P<token>(?!\1).+)\1', webpage,
+            'token', group='token')
+
+        context = self._download_json(
+            'https://tvplayer.com/watch/context', display_id,
+            'Downloading JSON context', query={
+                'resource': resource_id,
+                'nonce': token,
+            })
+
+        validate = context['validate']
+        platform = try_get(
+            context, lambda x: x['platform']['key'], compat_str) or 'firefox'
 
         try:
             response = self._download_json(
                 'http://api.tvplayer.com/api/v2/stream/live',
-                resource_id, headers={
+                display_id, 'Downloading JSON stream', headers={
                     'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
                 }, data=urlencode_postdata({
+                    'id': resource_id,
                     'service': 1,
                     'platform': platform,
-                    'id': resource_id,
-                    'token': token,
                     'validate': validate,
                 }))['tvplayer']['response']
         except ExtractorError as e:
@@ -63,7 +74,7 @@ class TVPlayerIE(InfoExtractor):
                     '%s said: %s' % (self.IE_NAME, response['error']), expected=True)
             raise
 
-        formats = self._extract_m3u8_formats(response['stream'], resource_id, 'mp4')
+        formats = self._extract_m3u8_formats(response['stream'], display_id, 'mp4')
         self._sort_formats(formats)
 
         return {

From e7db6759e4c4ca5099a997285642e60b84eee2c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Apr 2017 04:33:35 +0700
Subject: [PATCH 201/611] [downloader/external] Properly handle live stream
 downloading cancellation (closes #8932)

---
 youtube_dl/downloader/external.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index e13cf547d..e78169a0d 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -29,7 +29,17 @@ class ExternalFD(FileDownloader):
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 
-        retval = self._call_downloader(tmpfilename, info_dict)
+        try:
+            retval = self._call_downloader(tmpfilename, info_dict)
+        except KeyboardInterrupt:
+            if not info_dict.get('is_live'):
+                raise
+            # Live stream downloading cancellation should be considered as
+            # correct and expected termination thus all postprocessing
+            # should take place
+            retval = 0
+            self.to_screen('[%s] Interrupted by user' % self.get_basename())
+
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen('\r[%s] Downloaded %s bytes' % (self.get_basename(), fsize))

From 24eb7c257807aceef713f6d01824cf600a9d6232 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 29 Apr 2017 15:55:08 +0800
Subject: [PATCH 202/611] [xtube] Fix extraction with non-standard JSON
 'sources'

Closes #12734

Thanks @paulguy for the fix!
---
 ChangeLog                     |  6 ++++++
 youtube_dl/extractor/xtube.py | 23 +++++++++++++++++++++--
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index cd49ac42d..aa7543325 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [xtube] Fix extraction for older FLV videos (#12734)
+
+
 version 2017.04.28
 
 Core
diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 5584674a0..bea9b87ad 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
+    js_to_json,
     orderedSet,
     parse_duration,
     sanitized_Request,
@@ -37,6 +38,22 @@ class XTubeIE(InfoExtractor):
             'comment_count': int,
             'age_limit': 18,
         }
+    }, {
+        # FLV videos with duplicated formats
+        'url': 'http://www.xtube.com/video-watch/A-Super-Run-Part-1-YT-9299752',
+        'md5': 'a406963eb349dd43692ec54631efd88b',
+        'info_dict': {
+            'id': '9299752',
+            'display_id': 'A-Super-Run-Part-1-YT',
+            'ext': 'flv',
+            'title': 'A Super Run - Part 1 (YT)',
+            'description': 'md5:ca0d47afff4a9b2942e4b41aa970fd93',
+            'uploader': 'tshirtguy59',
+            'duration': 579,
+            'view_count': int,
+            'comment_count': int,
+            'age_limit': 18,
+        },
     }, {
         # new URL schema
         'url': 'http://www.xtube.com/video-watch/strange-erotica-625837',
@@ -68,8 +85,9 @@ class XTubeIE(InfoExtractor):
             })
 
         sources = self._parse_json(self._search_regex(
-            r'(["\'])sources\1\s*:\s*(?P<sources>{.+?}),',
-            webpage, 'sources', group='sources'), video_id)
+            r'(["\'])?sources\1?\s*:\s*(?P<sources>{.+?}),',
+            webpage, 'sources', group='sources'), video_id,
+            transform_source=js_to_json)
 
         formats = []
         for format_id, format_url in sources.items():
@@ -78,6 +96,7 @@ class XTubeIE(InfoExtractor):
                 'format_id': format_id,
                 'height': int_or_none(format_id),
             })
+        self._remove_duplicate_formats(formats)
         self._sort_formats(formats)
 
         title = self._search_regex(

From 7986c3abcdad819b61bdf0fb7111759f9fc1fc32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Apr 2017 19:49:04 +0700
Subject: [PATCH 203/611] [anvato] Improve extraction (closes #12913) * Promote
 to regular shortcut based extractor * Add mcp to access key mapping table *
 Add support for embeds extraction * Add support for anvato embeds in generic
 extractor

---
 youtube_dl/extractor/anvato.py     | 59 ++++++++++++++++++++++++++++--
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/generic.py    | 16 ++++++++
 3 files changed, 73 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/anvato.py b/youtube_dl/extractor/anvato.py
index 9fd91c2f6..ab8c237e9 100644
--- a/youtube_dl/extractor/anvato.py
+++ b/youtube_dl/extractor/anvato.py
@@ -5,6 +5,7 @@ import base64
 import hashlib
 import json
 import random
+import re
 import time
 
 from .common import InfoExtractor
@@ -16,6 +17,7 @@ from ..utils import (
     intlist_to_bytes,
     int_or_none,
     strip_jsonp,
+    unescapeHTML,
 )
 
 
@@ -26,6 +28,8 @@ def md5_text(s):
 
 
 class AnvatoIE(InfoExtractor):
+    _VALID_URL = r'anvato:(?P<access_key_or_mcp>[^:]+):(?P<id>\d+)'
+    
     # Copied from anvplayer.min.js
     _ANVACK_TABLE = {
         'nbcu_nbcd_desktop_web_prod_93d8ead38ce2024f8f544b78306fbd15895ae5e6': 'NNemUkySjxLyPTKvZRiGntBIjEyK8uqicjMakIaQ',
@@ -114,6 +118,22 @@ class AnvatoIE(InfoExtractor):
         'nbcu_nbcd_desktop_web_prod_93d8ead38ce2024f8f544b78306fbd15895ae5e6_secure': 'NNemUkySjxLyPTKvZRiGntBIjEyK8uqicjMakIaQ'
     }
 
+    _MCP_TO_ACCESS_KEY_TABLE = {
+        'qa': 'anvato_mcpqa_demo_web_stage_18b55e00db5a13faa8d03ae6e41f6f5bcb15b922',
+        'lin': 'anvato_mcp_lin_web_prod_4c36fbfd4d8d8ecae6488656e21ac6d1ac972749',
+        'univison': 'anvato_mcp_univision_web_prod_37fe34850c99a3b5cdb71dab10a417dd5cdecafa',
+        'uni': 'anvato_mcp_univision_web_prod_37fe34850c99a3b5cdb71dab10a417dd5cdecafa',
+        'dev': 'anvato_mcp_fs2go_web_prod_c7b90a93e171469cdca00a931211a2f556370d0a',
+        'sps': 'anvato_mcp_sps_web_prod_54bdc90dd6ba21710e9f7074338365bba28da336',
+        'spsstg': 'anvato_mcp_sps_web_prod_54bdc90dd6ba21710e9f7074338365bba28da336',
+        'anv': 'anvato_mcp_anv_web_prod_791407490f4c1ef2a4bcb21103e0cb1bcb3352b3',
+        'gray': 'anvato_mcp_gray_web_prod_4c10f067c393ed8fc453d3930f8ab2b159973900',
+        'hearst': 'anvato_mcp_hearst_web_prod_5356c3de0fc7c90a3727b4863ca7fec3a4524a99',
+        'cbs': 'anvato_mcp_cbs_web_prod_02f26581ff80e5bda7aad28226a8d369037f2cbe',
+        'telemundo': 'anvato_mcp_telemundo_web_prod_c5278d51ad46fda4b6ca3d0ea44a7846a054f582'
+    }
+
+    _ANVP_RE = r'<script[^>]+\bdata-anvp\s*=\s*(["\'])(?P<anvp>(?:(?!\1).)+)\1'
     _AUTH_KEY = b'\x31\xc2\x42\x84\x9e\x73\xa0\xce'
 
     def __init__(self, *args, **kwargs):
@@ -217,9 +237,42 @@ class AnvatoIE(InfoExtractor):
             'subtitles': subtitles,
         }
 
+    @staticmethod
+    def _extract_urls(ie, webpage, video_id):
+        entries = []
+        for mobj in re.finditer(AnvatoIE._ANVP_RE, webpage):
+            anvplayer_data = ie._parse_json(
+                mobj.group('anvp'), video_id, transform_source=unescapeHTML,
+                fatal=False)
+            if not anvplayer_data:
+                continue
+            video = anvplayer_data.get('video')
+            if not isinstance(video, compat_str) or not video.isdigit():
+                continue
+            access_key = anvplayer_data.get('accessKey')
+            if not access_key:
+                mcp = anvplayer_data.get('mcp')
+                if mcp:
+                    access_key = AnvatoIE._MCP_TO_ACCESS_KEY_TABLE.get(
+                        mcp.lower())
+            if not access_key:
+                continue
+            entries.append(ie.url_result(
+                'anvato:%s:%s' % (access_key, video), ie=AnvatoIE.ie_key(),
+                video_id=video))
+        return entries
+
     def _extract_anvato_videos(self, webpage, video_id):
-        anvplayer_data = self._parse_json(self._html_search_regex(
-            r'<script[^>]+data-anvp=\'([^\']+)\'', webpage,
-            'Anvato player data'), video_id)
+        anvplayer_data = self._parse_json(
+            self._html_search_regex(
+                self._ANVP_RE, webpage, 'Anvato player data', group='anvp'),
+            video_id)
         return self._get_anvato_videos(
             anvplayer_data['accessKey'], anvplayer_data['video'])
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        access_key, video_id = mobj.group('access_key_or_mcp', 'id')
+        if access_key not in self._ANVACK_TABLE:
+            access_key = self._MCP_TO_ACCESS_KEY_TABLE[access_key]
+        return self._get_anvato_videos(access_key, video_id)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 355a4e56f..39e5380b8 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -41,6 +41,7 @@ from .alphaporno import AlphaPornoIE
 from .amcnetworks import AMCNetworksIE
 from .animeondemand import AnimeOnDemandIE
 from .anitube import AnitubeIE
+from .anvato import AnvatoIE
 from .anysex import AnySexIE
 from .aol import AolIE
 from .allocine import AllocineIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 67184bc5d..7f7c1ba29 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -86,6 +86,7 @@ from .openload import OpenloadIE
 from .videopress import VideoPressIE
 from .rutube import RutubeIE
 from .limelight import LimelightBaseIE
+from .anvato import AnvatoIE
 
 
 class GenericIE(InfoExtractor):
@@ -1677,6 +1678,15 @@ class GenericIE(InfoExtractor):
             },
             'playlist_mincount': 5,
         },
+        {
+            'url': 'http://kron4.com/2017/04/28/standoff-with-walnut-creek-murder-suspect-ends-with-arrest/',
+            'info_dict': {
+                'id': 'standoff-with-walnut-creek-murder-suspect-ends-with-arrest',
+                'title': 'Standoff with Walnut Creek murder suspect ends',
+                'description': 'md5:3ccc48a60fc9441eeccfc9c469ebf788',
+            },
+            'playlist_mincount': 4,
+        },
         # {
         #     # TODO: find another test
         #     # http://schema.org/VideoObject
@@ -2537,6 +2547,12 @@ class GenericIE(InfoExtractor):
                 'limelight:media:%s' % mobj.group('id'),
                 {'source_url': url}), 'LimelightMedia', mobj.group('id'))
 
+        # Look for Anvato embeds
+        anvato_urls = AnvatoIE._extract_urls(self, webpage, video_id)
+        if anvato_urls:
+            return self.playlist_result(
+                anvato_urls, video_id, video_title, video_description)
+
         # Look for AdobeTVVideo embeds
         mobj = re.search(
             r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',

From a4c81e49688f081834656c6bbf3afc8bab9820cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Apr 2017 20:23:26 +0700
Subject: [PATCH 204/611] [yandexmusic:playlist] Fix extraction for python 3
 (closes #12888)

---
 youtube_dl/extractor/yandexmusic.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index fd6268ba4..eb1062142 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -234,7 +234,8 @@ class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
                 'overembed': 'false',
             })['playlist']
 
-        tracks, track_ids = playlist['tracks'], map(compat_str, playlist['trackIds'])
+        tracks = playlist['tracks']
+        track_ids = [compat_str(track_id) for track_id in playlist['trackIds']]
 
         # tracks dictionary shipped with playlist.jsx API is limited to 150 tracks,
         # missing tracks should be retrieved manually.

From 557194591a3afcf16f2d554aa0af34ce83ca163f Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Mon, 10 Apr 2017 01:09:10 -0400
Subject: [PATCH 205/611] [washingtonpost] Add support for embeds (closes
 #12699)

---
 youtube_dl/extractor/generic.py        | 21 +++++++++++++++++++++
 youtube_dl/extractor/washingtonpost.py |  6 ++++++
 2 files changed, 27 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7f7c1ba29..bd9d9aa13 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -87,6 +87,7 @@ from .videopress import VideoPressIE
 from .rutube import RutubeIE
 from .limelight import LimelightBaseIE
 from .anvato import AnvatoIE
+from .washingtonpost import WashingtonPostIE
 
 
 class GenericIE(InfoExtractor):
@@ -1687,6 +1688,20 @@ class GenericIE(InfoExtractor):
             },
             'playlist_mincount': 4,
         },
+        {
+            # WashingtonPost embed
+            'url': 'http://www.vanityfair.com/hollywood/2017/04/donald-trump-tv-pitches',
+            'info_dict': {
+                'id': '8caf6e88-d0ec-11e5-90d3-34c2c42653ac',
+                'ext': 'mp4',
+                'title': "No one has seen the drama series based on Trump's life \u2014 until now",
+                'description': 'Donald Trump wanted a weekly TV drama based on his life. It never aired. But The Washington Post recently obtained a scene from the pilot script — and enlisted actors.',
+                'timestamp': 1455216756,
+                'uploader': 'The Washington Post',
+                'upload_date': '20160211',
+            },
+            'add_ie': [WashingtonPostIE.ie_key()],
+        },
         # {
         #     # TODO: find another test
         #     # http://schema.org/VideoObject
@@ -2670,6 +2685,12 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 rutube_urls, ie=RutubeIE.ie_key())
 
+        # Look for WashingtonPost embeds
+        wapo_urls = WashingtonPostIE._extract_urls(webpage)
+        if wapo_urls:
+            return self.playlist_from_matches(
+                wapo_urls, video_id, video_title, ie=WashingtonPostIE.ie_key())
+
         # Looking for http://schema.org/VideoObject
         json_ld = self._search_json_ld(
             webpage, video_id, default={}, expected_type='VideoObject')
diff --git a/youtube_dl/extractor/washingtonpost.py b/youtube_dl/extractor/washingtonpost.py
index 839cad986..7e5cf0e1d 100644
--- a/youtube_dl/extractor/washingtonpost.py
+++ b/youtube_dl/extractor/washingtonpost.py
@@ -13,6 +13,7 @@ from ..utils import (
 class WashingtonPostIE(InfoExtractor):
     IE_NAME = 'washingtonpost'
     _VALID_URL = r'(?:washingtonpost:|https?://(?:www\.)?washingtonpost\.com/video/(?:[^/]+/)*)(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
+    _EMBED_URL = 'https?://(?:www\.)?washingtonpost\.com/video/c/embed/[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}'
     _TEST = {
         'url': 'https://www.washingtonpost.com/video/c/video/480ba4ee-1ec7-11e6-82c2-a7dcb313287d',
         'md5': '6f537e1334b714eb15f9563bd4b9cdfa',
@@ -27,6 +28,11 @@ class WashingtonPostIE(InfoExtractor):
         },
     }
 
+    @classmethod
+    def _extract_urls(cls, webpage):
+        return re.findall(
+            r'<iframe[^>]+\bsrc=["\'](%s)' % cls._EMBED_URL, webpage)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         video_data = self._download_json(

From 1c7c76e4fb1f5d5c05cfa22483a46e7dd870ac0f Mon Sep 17 00:00:00 2001
From: Frederic Bournival <fredbournival@gmail.com>
Date: Wed, 19 Apr 2017 17:43:32 -0400
Subject: [PATCH 206/611] [noovo] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/noovo.py      | 52 ++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+)
 create mode 100644 youtube_dl/extractor/noovo.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 39e5380b8..612007a46 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -663,6 +663,7 @@ from .nintendo import NintendoIE
 from .njpwworld import NJPWWorldIE
 from .nobelprize import NobelPrizeIE
 from .noco import NocoIE
+from .noovo import NoovoIE
 from .normalboots import NormalbootsIE
 from .nosvideo import NosVideoIE
 from .nova import NovaIE
diff --git a/youtube_dl/extractor/noovo.py b/youtube_dl/extractor/noovo.py
new file mode 100644
index 000000000..9f441c2fc
--- /dev/null
+++ b/youtube_dl/extractor/noovo.py
@@ -0,0 +1,52 @@
+# coding: utf-8
+from __future__ import unicode_literals
+from .common import InfoExtractor
+
+
+class NoovoIE(InfoExtractor):
+    IE_NAME = 'Noovo'
+    IE_DESC = 'VTele, Max, MusiquePlus'
+    _VALID_URL = r'https?://(?:[a-z0-9-]+\.)?noovo\.ca/videos/(?P<id>[a-z0-9-]+/[a-z0-9-]+)'
+    _TESTS = [{
+        'url': 'http://noovo.ca/videos/rpm-plus/chrysler-imperial',
+        'md5': '2fcc04d0a8f4a853fad91233c2fdd121',
+        'info_dict': {
+            'id': '5386045029001',
+            'description': 'Antoine présente des véhicules qu\'il aperçoit sur la rue.',
+            'ext': 'mp4',
+            'timestamp': 1491399228,
+            'title': 'Chrysler Imperial',
+            'upload_date': '20170405',
+            'uploader_id': '618566855001'
+        }
+    }, {
+        'url': 'http://noovo.ca/videos/l-amour-est-dans-le-pre/episode-13-8',
+        'md5': '1199e96fbb93f2d42717115f72097b6b',
+        'info_dict': {
+            'id': '5395865725001',
+            'description': 'md5:336d5ebc5436534e61d16e63ddfca327',
+            'ext': 'mp4',
+            'timestamp': 1492019320,
+            'title': 'md5:2895fdc124639be0ef64ea0d06f5e493',
+            'upload_date': '20170412',
+            'uploader_id': '618566855001'
+        }
+    }, {
+        'url': 'http://interventions.noovo.ca/911/video/intoxication-aux-drogues-dures/?autoplay=1',
+        'only_matching': True
+    }]
+    API_URL_TEMPLATE = 'http://api.noovo.ca/api/v1/pages/single-episode/%s'
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/618566855001/default_default/index.html?videoId=%s'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        api_url = self.API_URL_TEMPLATE % video_id
+        api_content = self._download_json(api_url, video_id)
+
+        brightcove_id = api_content.get('data').get('brightcoveId')
+        if not brightcove_id:
+            brightcove_id = api_content.get('data').get('contents')[0].get('brightcoveId')
+
+        return self.url_result(
+            self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id
+        )

From b5c39537be8b7c3405af0df30dfaea3c5ed9d649 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Apr 2017 00:23:50 +0700
Subject: [PATCH 207/611] [noovo] Improve extraction (closes #12792)

---
 youtube_dl/extractor/noovo.py | 93 ++++++++++++++++++++++++++---------
 1 file changed, 69 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/noovo.py b/youtube_dl/extractor/noovo.py
index 9f441c2fc..f7fa098a5 100644
--- a/youtube_dl/extractor/noovo.py
+++ b/youtube_dl/extractor/noovo.py
@@ -1,52 +1,97 @@
 # coding: utf-8
 from __future__ import unicode_literals
+
+from .brightcove import BrightcoveNewIE
 from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    smuggle_url,
+    try_get,
+)
 
 
 class NoovoIE(InfoExtractor):
-    IE_NAME = 'Noovo'
-    IE_DESC = 'VTele, Max, MusiquePlus'
-    _VALID_URL = r'https?://(?:[a-z0-9-]+\.)?noovo\.ca/videos/(?P<id>[a-z0-9-]+/[a-z0-9-]+)'
+    _VALID_URL = r'https?://(?:[^/]+\.)?noovo\.ca/videos/(?P<id>[^/]+/[^/?#&]+)'
     _TESTS = [{
+        # clip
         'url': 'http://noovo.ca/videos/rpm-plus/chrysler-imperial',
-        'md5': '2fcc04d0a8f4a853fad91233c2fdd121',
         'info_dict': {
             'id': '5386045029001',
-            'description': 'Antoine présente des véhicules qu\'il aperçoit sur la rue.',
             'ext': 'mp4',
-            'timestamp': 1491399228,
             'title': 'Chrysler Imperial',
+            'description': 'md5:de3c898d1eb810f3e6243e08c8b4a056',
+            'timestamp': 1491399228,
             'upload_date': '20170405',
-            'uploader_id': '618566855001'
-        }
+            'uploader_id': '618566855001',
+            'creator': 'vtele',
+            'view_count': int,
+            'series': 'RPM+',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
+        # episode
         'url': 'http://noovo.ca/videos/l-amour-est-dans-le-pre/episode-13-8',
-        'md5': '1199e96fbb93f2d42717115f72097b6b',
         'info_dict': {
             'id': '5395865725001',
+            'title': 'Épisode 13 : Les retrouvailles',
             'description': 'md5:336d5ebc5436534e61d16e63ddfca327',
             'ext': 'mp4',
             'timestamp': 1492019320,
-            'title': 'md5:2895fdc124639be0ef64ea0d06f5e493',
             'upload_date': '20170412',
-            'uploader_id': '618566855001'
-        }
-    }, {
-        'url': 'http://interventions.noovo.ca/911/video/intoxication-aux-drogues-dures/?autoplay=1',
-        'only_matching': True
+            'uploader_id': '618566855001',
+            'creator': 'vtele',
+            'view_count': int,
+            'series': "L'amour est dans le pré",
+            'season_number': 5,
+            'episode': 'Épisode 13',
+            'episode_number': 13,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
-    API_URL_TEMPLATE = 'http://api.noovo.ca/api/v1/pages/single-episode/%s'
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/618566855001/default_default/index.html?videoId=%s'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        api_url = self.API_URL_TEMPLATE % video_id
-        api_content = self._download_json(api_url, video_id)
 
-        brightcove_id = api_content.get('data').get('brightcoveId')
-        if not brightcove_id:
-            brightcove_id = api_content.get('data').get('contents')[0].get('brightcoveId')
+        data = self._download_json(
+            'http://api.noovo.ca/api/v1/pages/single-episode/%s' % video_id,
+            video_id)['data']
 
-        return self.url_result(
-            self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id
-        )
+        content = try_get(data, lambda x: x['contents'][0])
+
+        brightcove_id = data.get('brightcoveId') or content['brightcoveId']
+
+        series = try_get(
+            data, (
+                lambda x: x['show']['title'],
+                lambda x: x['season']['show']['title']),
+            compat_str)
+
+        episode = None
+        og = data.get('og')
+        if isinstance(og, dict) and og.get('type') == 'video.episode':
+            episode = og.get('title')
+
+        video = content or data
+
+        return {
+            '_type': 'url_transparent',
+            'ie_key': BrightcoveNewIE.ie_key(),
+            'url': smuggle_url(
+                self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id,
+                {'geo_countries': ['CA']}),
+            'id': brightcove_id,
+            'title': video.get('title'),
+            'creator': video.get('source'),
+            'view_count': int_or_none(video.get('viewsCount')),
+            'series': series,
+            'season_number': int_or_none(try_get(
+                data, lambda x: x['season']['seasonNumber'])),
+            'episode': episode,
+            'episode_number': int_or_none(data.get('episodeNumber')),
+        }

From ba5c3caf88417d81fda5d3b629336c9c2d6fa6e9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 30 Apr 2017 02:15:28 +0800
Subject: [PATCH 208/611] [washingtonpost] Fix invalid escape sequence on
 Python 3.6

---
 youtube_dl/extractor/washingtonpost.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/washingtonpost.py b/youtube_dl/extractor/washingtonpost.py
index 7e5cf0e1d..625d0a1cc 100644
--- a/youtube_dl/extractor/washingtonpost.py
+++ b/youtube_dl/extractor/washingtonpost.py
@@ -13,7 +13,7 @@ from ..utils import (
 class WashingtonPostIE(InfoExtractor):
     IE_NAME = 'washingtonpost'
     _VALID_URL = r'(?:washingtonpost:|https?://(?:www\.)?washingtonpost\.com/video/(?:[^/]+/)*)(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
-    _EMBED_URL = 'https?://(?:www\.)?washingtonpost\.com/video/c/embed/[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}'
+    _EMBED_URL = r'https?://(?:www\.)?washingtonpost\.com/video/c/embed/[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}'
     _TEST = {
         'url': 'https://www.washingtonpost.com/video/c/video/480ba4ee-1ec7-11e6-82c2-a7dcb313287d',
         'md5': '6f537e1334b714eb15f9563bd4b9cdfa',

From 98ce1a3fd31b11119d031ceb7ad032bea22d373a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 30 Apr 2017 09:01:05 +0100
Subject: [PATCH 209/611] [utils] add video/mp2t to mimetype2ext

---
 youtube_dl/utils.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 41bc20544..9d7798ee8 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2273,10 +2273,8 @@ def mimetype2ext(mt):
     return {
         '3gpp': '3gp',
         'smptett+xml': 'tt',
-        'srt': 'srt',
         'ttaf+xml': 'dfxp',
         'ttml+xml': 'ttml',
-        'vtt': 'vtt',
         'x-flv': 'flv',
         'x-mp4-fragmented': 'mp4',
         'x-ms-wmv': 'wmv',
@@ -2284,11 +2282,11 @@ def mimetype2ext(mt):
         'x-mpegurl': 'm3u8',
         'vnd.apple.mpegurl': 'm3u8',
         'dash+xml': 'mpd',
-        'f4m': 'f4m',
         'f4m+xml': 'f4m',
         'hds+xml': 'f4m',
         'vnd.ms-sstr+xml': 'ism',
         'quicktime': 'mov',
+        'mp2t': 'ts',
     }.get(res, res)
 
 

From 559925300959669a08456475650a42f395385372 Mon Sep 17 00:00:00 2001
From: gritstub <gritstub@users.noreply.github.com>
Date: Thu, 27 Apr 2017 11:00:02 -0700
Subject: [PATCH 210/611] [vevo] Fix extraction (config.token.key)

---
 youtube_dl/extractor/vevo.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 9aa38bc5a..9434cd585 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 import re
+import json
 
 from .common import InfoExtractor
 from ..compat import (
@@ -154,8 +155,15 @@ class VevoIE(VevoBaseIE):
     }
 
     def _initialize_api(self, video_id):
+        post_data = json.dumps({
+            'client_id': 'SPupX1tvqFEopQ1YS6SS',
+            'grant_type': 'urn:vevo:params:oauth:grant-type:anonymous',
+        }).encode('utf-8')
+        headers = {
+            'Content-Type': 'application/json',
+        }
         req = sanitized_Request(
-            'http://www.vevo.com/auth', data=b'')
+            'https://accounts.vevo.com/token', post_data, headers)
         webpage = self._download_webpage(
             req, None,
             note='Retrieving oauth token',
@@ -166,7 +174,7 @@ class VevoIE(VevoBaseIE):
                 '%s said: This page is currently unavailable in your region' % self.IE_NAME)
 
         auth_info = self._parse_json(webpage, video_id)
-        self._api_url_template = self.http_scheme() + '//apiv2.vevo.com/%s?token=' + auth_info['access_token']
+        self._api_url_template = self.http_scheme() + '//apiv2.vevo.com/%s?token=' + auth_info['legacy_token']
 
     def _call_api(self, path, *args, **kwargs):
         try:

From b07ea5eaecd6e3ab3a31c996ec815682efaf8edb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Apr 2017 17:58:22 +0700
Subject: [PATCH 211/611] [vevo] Modernize

---
 youtube_dl/extractor/vevo.py | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 9434cd585..890a149ea 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -12,7 +12,6 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     int_or_none,
-    sanitized_Request,
     parse_iso8601,
 )
 
@@ -155,19 +154,17 @@ class VevoIE(VevoBaseIE):
     }
 
     def _initialize_api(self, video_id):
-        post_data = json.dumps({
-            'client_id': 'SPupX1tvqFEopQ1YS6SS',
-            'grant_type': 'urn:vevo:params:oauth:grant-type:anonymous',
-        }).encode('utf-8')
-        headers = {
-            'Content-Type': 'application/json',
-        }
-        req = sanitized_Request(
-            'https://accounts.vevo.com/token', post_data, headers)
         webpage = self._download_webpage(
-            req, None,
+            'https://accounts.vevo.com/token', None,
             note='Retrieving oauth token',
-            errnote='Unable to retrieve oauth token')
+            errnote='Unable to retrieve oauth token',
+            data=json.dumps({
+                'client_id': 'SPupX1tvqFEopQ1YS6SS',
+                'grant_type': 'urn:vevo:params:oauth:grant-type:anonymous',
+            }).encode('utf-8'),
+            headers={
+                'Content-Type': 'application/json',
+            })
 
         if re.search(r'(?i)THIS PAGE IS CURRENTLY UNAVAILABLE IN YOUR REGION', webpage):
             self.raise_geo_restricted(

From 13081db1f5e026e8571e4f46758dc947702228f5 Mon Sep 17 00:00:00 2001
From: Sander <sander@plex.tv>
Date: Sun, 23 Apr 2017 23:49:41 +0200
Subject: [PATCH 212/611] [xvideos] Add video duration

---
 youtube_dl/extractor/xvideos.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index 30825daae..ce3a05e41 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -8,6 +8,7 @@ from ..utils import (
     clean_html,
     ExtractorError,
     determine_ext,
+    parse_duration,
 )
 
 
@@ -20,6 +21,7 @@ class XVideosIE(InfoExtractor):
             'id': '4588838',
             'ext': 'mp4',
             'title': 'Biker Takes his Girl',
+            'duration': 120,
             'age_limit': 18,
         }
     }
@@ -36,6 +38,8 @@ class XVideosIE(InfoExtractor):
             r'<title>(.*?)\s+-\s+XVID', webpage, 'title')
         video_thumbnail = self._search_regex(
             r'url_bigthumb=(.+?)&amp', webpage, 'thumbnail', fatal=False)
+        video_duration = parse_duration(self._search_regex(
+            r'<span class="duration">.*?(\d[^<]+)', webpage, 'duration', fatal=False))
 
         formats = []
 
@@ -67,6 +71,7 @@ class XVideosIE(InfoExtractor):
             'id': video_id,
             'formats': formats,
             'title': video_title,
+            'duration': video_duration,
             'thumbnail': video_thumbnail,
             'age_limit': 18,
         }

From 6ec371cd9e49eb3acf352e9211f05c3d1de72161 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Apr 2017 18:11:29 +0700
Subject: [PATCH 213/611] [xvideos] Extract og:duration (closes #12828)

---
 youtube_dl/extractor/xvideos.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index ce3a05e41..eca603028 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -6,8 +6,9 @@ from .common import InfoExtractor
 from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     clean_html,
-    ExtractorError,
     determine_ext,
+    ExtractorError,
+    int_or_none,
     parse_duration,
 )
 
@@ -21,7 +22,7 @@ class XVideosIE(InfoExtractor):
             'id': '4588838',
             'ext': 'mp4',
             'title': 'Biker Takes his Girl',
-            'duration': 120,
+            'duration': 108,
             'age_limit': 18,
         }
     }
@@ -38,8 +39,11 @@ class XVideosIE(InfoExtractor):
             r'<title>(.*?)\s+-\s+XVID', webpage, 'title')
         video_thumbnail = self._search_regex(
             r'url_bigthumb=(.+?)&amp', webpage, 'thumbnail', fatal=False)
-        video_duration = parse_duration(self._search_regex(
-            r'<span class="duration">.*?(\d[^<]+)', webpage, 'duration', fatal=False))
+        video_duration = int_or_none(self._og_search_property(
+            'duration', webpage, default=None)) or parse_duration(
+            self._search_regex(
+                r'<span[^>]+class=["\']duration["\'][^>]*>.*?(\d[^<]+)',
+                webpage, 'duration', fatal=False))
 
         formats = []
 

From 9dac2cec2d7e31b65cf063164b3a99f257a86a63 Mon Sep 17 00:00:00 2001
From: slocum <piotr5151@interia.pl>
Date: Sun, 9 Apr 2017 12:02:44 +0200
Subject: [PATCH 214/611] [zaq1] Add new extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/zaq1.py       | 67 ++++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+)
 create mode 100644 youtube_dl/extractor/zaq1.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 612007a46..c0020dd7d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1300,5 +1300,6 @@ from .youtube import (
     YoutubeWatchLaterIE,
 )
 from .zapiks import ZapiksIE
+from .zaq1 import Zaq1IE
 from .zdf import ZDFIE, ZDFChannelIE
 from .zingmp3 import ZingMp3IE
diff --git a/youtube_dl/extractor/zaq1.py b/youtube_dl/extractor/zaq1.py
new file mode 100644
index 000000000..33068f6b5
--- /dev/null
+++ b/youtube_dl/extractor/zaq1.py
@@ -0,0 +1,67 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    unified_strdate,
+    int_or_none
+)
+
+
+class Zaq1IE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?zaq1\.pl/video/(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'http://zaq1.pl/video/xev0e',
+        'md5': '24a5eb3f052e604ae597c4d0d19b351e',
+        'info_dict': {
+            'id': 'xev0e',
+            'title': 'DJ NA WESELE. TANIEC Z FIGURAMI.węgrów/sokołów podlaski/siedlce/mińsk mazowiecki/warszawa',
+            'ext': 'mp4',
+            'duration': 511,
+            'uploader': 'Anonim',
+            'upload_date': '20170330',
+        }
+    }, {
+        'url': 'http://zaq1.pl/video/x80nc',
+        'md5': '1245973520adc78139928a820959d9c5',
+        'info_dict': {
+            'id': 'x80nc',
+            'title': 'DIY Inspiration Challenge #86 | koraliki | gwiazdka na choinkę z koralików i drutu',
+            'ext': 'mp4',
+            'duration': 438,
+            'uploader': 'Anonim',
+            'upload_date': '20170404',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(
+            r'(?s)<h1>\s*<span.+class="watch-title".+title="([^"]+)">\1\s*</span>\s*</h1>', webpage, 'title')
+
+        div = self._search_regex(r'(?s)(?P<div><div.+id=(["\'])video_player\2.+</div>)', webpage, 'video url', group='div')
+        video_url = self._search_regex(r'data-video-url="(http[^"]+)"', div, 'video url')
+
+        ext = self._search_regex(r'data-file-extension="([^"]+)"', div, 'ext', None, False)
+        duration = int_or_none(self._search_regex(r'data-duration="([^"]+)"', div, 'duration', None, False))
+        thumbnail = self._search_regex(r'data-photo-url="([^"]+)"', div, 'thumbnail', None, False)
+
+        upload_date = unified_strdate(self._search_regex(r'<strong\s+class="watch-time-text">\s*Opublikowany\s+([0-9]{4}-[0-9]{2}-[0-9]{2})', webpage, 'upload date'))
+        uploader = self._search_regex(r'<div\s+id="watch7-user-header">.*Wideo dodał:\s*<a[^>]*>\s*([^<]+)\s*</a>', webpage, 'uploader')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': [{
+                'url': video_url,
+                'ext': ext,
+                'http_headers': {'Referer': url},
+            }],
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'upload_date': upload_date,
+            'duration': duration,
+        }

From deef31955bb6106939606cd2a8b677db700055e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Apr 2017 21:07:30 +0700
Subject: [PATCH 215/611] [utils] Improve unified_timestamp Seen at
 http://zaq1.pl/video/xev0e

---
 test/test_utils.py  | 1 +
 youtube_dl/utils.py | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 9e1808e6b..05fdc0e95 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -338,6 +338,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unified_timestamp('UNKNOWN DATE FORMAT'), None)
         self.assertEqual(unified_timestamp('May 16, 2016 11:15 PM'), 1463440500)
         self.assertEqual(unified_timestamp('Feb 7, 2016 at 6:35 pm'), 1454870100)
+        self.assertEqual(unified_timestamp('2017-03-30T17:52:41Q'), 1490896361)
 
     def test_determine_ext(self):
         self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 9d7798ee8..86fc5ccac 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1194,6 +1194,11 @@ def unified_timestamp(date_str, day_first=True):
     # Remove AM/PM + timezone
     date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
 
+    # Remove unrecognized timezones from ISO 8601 alike timestamps
+    m = re.search(r'\d{1,2}:\d{1,2}(?:\.\d+)?(?P<tz>\s*[A-Z]+)$', date_str)
+    if m:
+        date_str = date_str[:-len(m.group('tz'))]
+
     for expression in date_formats(day_first):
         try:
             dt = datetime.datetime.strptime(date_str, expression) - timezone + datetime.timedelta(hours=pm_delta)

From 33a81c2c6f4bd180ef69d5631862637ae0c8ec8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Apr 2017 21:11:55 +0700
Subject: [PATCH 216/611] [extractor/common] Extract view count from JSON-LD

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2cb55d6af..fba15d446 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -990,6 +990,7 @@ class InfoExtractor(object):
                 'tbr': int_or_none(e.get('bitrate')),
                 'width': int_or_none(e.get('width')),
                 'height': int_or_none(e.get('height')),
+                'view_count': int_or_none(e.get('interactionCount')),
             })
 
         for e in json_ld:

From 9cd4209724c60efe636efdec43d7df6c385d0e2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Apr 2017 21:45:00 +0700
Subject: [PATCH 217/611] [zaq1] Improve extraction (closes #12693)

---
 youtube_dl/extractor/zaq1.py | 88 +++++++++++++++++++++++++-----------
 1 file changed, 61 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/zaq1.py b/youtube_dl/extractor/zaq1.py
index 33068f6b5..889aff5d8 100644
--- a/youtube_dl/extractor/zaq1.py
+++ b/youtube_dl/extractor/zaq1.py
@@ -3,35 +3,44 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    unified_strdate,
-    int_or_none
+    int_or_none,
+    unified_timestamp,
 )
 
 
 class Zaq1IE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?zaq1\.pl/video/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?zaq1\.pl/video/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://zaq1.pl/video/xev0e',
         'md5': '24a5eb3f052e604ae597c4d0d19b351e',
         'info_dict': {
             'id': 'xev0e',
             'title': 'DJ NA WESELE. TANIEC Z FIGURAMI.węgrów/sokołów podlaski/siedlce/mińsk mazowiecki/warszawa',
+            'description': 'www.facebook.com/weseledjKontakt: 728 448 199 / 505 419 147',
             'ext': 'mp4',
             'duration': 511,
+            'timestamp': 1490896361,
             'uploader': 'Anonim',
             'upload_date': '20170330',
+            'view_count': int,
         }
     }, {
-        'url': 'http://zaq1.pl/video/x80nc',
-        'md5': '1245973520adc78139928a820959d9c5',
+        # malformed JSON-LD
+        'url': 'http://zaq1.pl/video/x81vn',
         'info_dict': {
-            'id': 'x80nc',
-            'title': 'DIY Inspiration Challenge #86 | koraliki | gwiazdka na choinkę z koralików i drutu',
+            'id': 'x81vn',
+            'title': 'SEKRETNE ŻYCIE WALTERA MITTY',
             'ext': 'mp4',
-            'duration': 438,
+            'duration': 6234,
+            'timestamp': 1493494860,
             'uploader': 'Anonim',
-            'upload_date': '20170404',
-        }
+            'upload_date': '20170429',
+            'view_count': int,
+        },
+        'params': {
+            'skip_download': True,
+        },
+        'expected_warnings': ['Failed to parse JSON'],
     }]
 
     def _real_extract(self, url):
@@ -39,29 +48,54 @@ class Zaq1IE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(
-            r'(?s)<h1>\s*<span.+class="watch-title".+title="([^"]+)">\1\s*</span>\s*</h1>', webpage, 'title')
+        video_url = self._search_regex(
+            r'data-video-url=(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
+            'video url', group='url')
 
-        div = self._search_regex(r'(?s)(?P<div><div.+id=(["\'])video_player\2.+</div>)', webpage, 'video url', group='div')
-        video_url = self._search_regex(r'data-video-url="(http[^"]+)"', div, 'video url')
+        info = self._search_json_ld(webpage, video_id, fatal=False)
 
-        ext = self._search_regex(r'data-file-extension="([^"]+)"', div, 'ext', None, False)
-        duration = int_or_none(self._search_regex(r'data-duration="([^"]+)"', div, 'duration', None, False))
-        thumbnail = self._search_regex(r'data-photo-url="([^"]+)"', div, 'thumbnail', None, False)
+        def extract_data(field, name, fatal=False):
+            return self._search_regex(
+                r'data-%s=(["\'])(?P<field>(?:(?!\1).)+)\1' % field,
+                webpage, field, fatal=fatal, group='field')
 
-        upload_date = unified_strdate(self._search_regex(r'<strong\s+class="watch-time-text">\s*Opublikowany\s+([0-9]{4}-[0-9]{2}-[0-9]{2})', webpage, 'upload date'))
-        uploader = self._search_regex(r'<div\s+id="watch7-user-header">.*Wideo dodał:\s*<a[^>]*>\s*([^<]+)\s*</a>', webpage, 'uploader')
+        if not info.get('title'):
+            info['title'] = extract_data('file-name', 'title', fatal=True)
 
-        return {
+        if not info.get('duration'):
+            info['duration'] = int_or_none(extract_data('duration', 'duration'))
+
+        if not info.get('thumbnail'):
+            info['thumbnail'] = extract_data('photo-url', 'thumbnail')
+
+        if not info.get('timestamp'):
+            info['timestamp'] = unified_timestamp(self._html_search_meta(
+                'uploadDate', webpage, 'timestamp'))
+
+        if not info.get('interactionCount'):
+            info['view_count'] = int_or_none(self._html_search_meta(
+                'interactionCount', webpage, 'view count'))
+
+        uploader = self._html_search_regex(
+            r'Wideo dodał:\s*<a[^>]*>([^<]+)</a>', webpage, 'uploader',
+            fatal=False)
+
+        width = int_or_none(self._html_search_meta(
+            'width', webpage, fatal=False))
+        height = int_or_none(self._html_search_meta(
+            'height', webpage, fatal=False))
+
+        info.update({
             'id': video_id,
-            'title': title,
             'formats': [{
                 'url': video_url,
-                'ext': ext,
-                'http_headers': {'Referer': url},
+                'width': width,
+                'height': height,
+                'http_headers': {
+                    'Referer': url,
+                },
             }],
-            'thumbnail': thumbnail,
             'uploader': uploader,
-            'upload_date': upload_date,
-            'duration': duration,
-        }
+        })
+
+        return info

From 50534b7158e0e1c842d05eae39fd6687a04a76e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Apr 2017 22:04:01 +0700
Subject: [PATCH 218/611] [downloader/fragment] PEP 8

---
 youtube_dl/downloader/fragment.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index d529ae09a..bccc8ecc1 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -49,7 +49,7 @@ class FragmentFD(FileDownloader):
                 index:  0-based index of current fragment among all fragments
             fragment_count:
                 Total count of fragments
-                
+
     This feature is experimental and file format may change in future.
     """
 

From 238cec17ae51c0f624588095d3a370a3b6964bdd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Apr 2017 22:04:21 +0700
Subject: [PATCH 219/611] [extractor/anvato] PEP 8

---
 youtube_dl/extractor/anvato.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/anvato.py b/youtube_dl/extractor/anvato.py
index ab8c237e9..8023da702 100644
--- a/youtube_dl/extractor/anvato.py
+++ b/youtube_dl/extractor/anvato.py
@@ -29,7 +29,7 @@ def md5_text(s):
 
 class AnvatoIE(InfoExtractor):
     _VALID_URL = r'anvato:(?P<access_key_or_mcp>[^:]+):(?P<id>\d+)'
-    
+
     # Copied from anvplayer.min.js
     _ANVACK_TABLE = {
         'nbcu_nbcd_desktop_web_prod_93d8ead38ce2024f8f544b78306fbd15895ae5e6': 'NNemUkySjxLyPTKvZRiGntBIjEyK8uqicjMakIaQ',

From 9edf47df7bfbcdd67bce68ea04865aa1f51df2f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 May 2017 01:03:47 +0700
Subject: [PATCH 220/611] [brightcove] Allow whitespace around attribute names
 in embedded code

---
 youtube_dl/extractor/brightcove.py |  2 +-
 youtube_dl/extractor/generic.py    | 16 ++++++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 97602ca30..3f017a2b1 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -522,7 +522,7 @@ class BrightcoveNewIE(InfoExtractor):
         # [2] looks like:
         for video, script_tag, account_id, player_id, embed in re.findall(
                 r'''(?isx)
-                    (<video\s+[^>]*data-video-id=['"]?[^>]+>)
+                    (<video\s+[^>]*\bdata-video-id\s*=\s*['"]?[^>]+>)
                     (?:.*?
                         (<script[^>]+
                             src=["\'](?:https?:)?//players\.brightcove\.net/
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index bd9d9aa13..b06f43446 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1429,6 +1429,22 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            # Brightcove embed with whitespace around attribute names
+            'url': 'http://www.stack.com/video/3167554373001/learn-to-hit-open-three-pointers-with-damian-lillard-s-baseline-drift-drill',
+            'info_dict': {
+                'id': '3167554373001',
+                'ext': 'mp4',
+                'title': "Learn to Hit Open Three-Pointers With Damian Lillard's Baseline Drift Drill",
+                'description': 'md5:57bacb0e0f29349de4972bfda3191713',
+                'uploader_id': '1079349493',
+                'upload_date': '20140207',
+                'timestamp': 1391810548,
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         # Another form of arte.tv embed
         {
             'url': 'http://www.tv-replay.fr/redirection/09-04-16/arte-reportage-arte-11508975.html',

From 74da856544177c15d95ffa78c5eda4d7089f8efd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 May 2017 01:23:05 +0700
Subject: [PATCH 221/611] [infoq] Make audio format extraction non fatal
 (closes #12938)

---
 youtube_dl/extractor/infoq.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index 9fb71e8ef..fe425e786 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -87,8 +87,8 @@ class InfoQIE(BokeCCBaseIE):
 
     def _extract_http_audio(self, webpage, video_id):
         fields = self._hidden_inputs(webpage)
-        http_audio_url = fields['filename']
-        if http_audio_url is None:
+        http_audio_url = fields.get('filename')
+        if not http_audio_url:
             return []
 
         cookies_header = {'Cookie': self._extract_cookies(webpage)}

From 086041e2f8711ab579faf5cba681eba933be1021 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 May 2017 01:34:51 +0700
Subject: [PATCH 222/611] [ChangeLog] Actualize

---
 ChangeLog | 40 +++++++++++++++++++++++++++++++---------
 1 file changed, 31 insertions(+), 9 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index aa7543325..a4a8eac8e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,29 @@
 version <unreleased>
 
+Core
++ [extractor/common] Extract view count from JSON-LD
+* [utils] Improve unified_timestamp
++ [utils] Add video/mp2t to mimetype2ext
+* [downloader/external] Properly handle live stream downloading cancellation
+  (#8932)
++ [utils] Add support for unicode whitespace in clean_html on python 2 (#12906)
+
 Extractors
+* [infoq] Make audio format extraction non fatal (#12938)
+* [brightcove] Allow whitespace around attribute names in embedded code
++ [zaq1] Add support for zaq1.pl (#12693)
++ [xvideos] Extract duration (#12828)
+* [vevo] Fix extraction (#12879)
++ [noovo] Add support for noovo.ca (#12792)
++ [washingtonpost] Add support for embeds (#12699)
+* [yandexmusic:playlist] Fix extraction for python 3 (#12888)
+* [anvato] Improve extraction (#12913)
+    * Promote to regular shortcut based extractor
+    * Add mcp to access key mapping table
+    * Add support for embeds extraction
+    * Add support for anvato embeds in generic extractor
 * [xtube] Fix extraction for older FLV videos (#12734)
+* [tvplayer] Fix extraction (#12908)
 
 
 version 2017.04.28
@@ -30,19 +52,19 @@ Core
 * [YoutubeDL] Fix output template for missing timestamp (#12796)
 * [socks] Handle cases where credentials are required but missing
 * [extractor/common] Improve HLS extraction (#12211)
-    - Extract m3u8 parsing to separate method
-    - Improve rendition groups extraction
-    - Build stream name according stream GROUP-ID
-    - Ignore reference to AUDIO group without URI when stream has no CODECS
-    - Use float for scaled tbr in _parse_m3u8_formats
+    * Extract m3u8 parsing to separate method
+    * Improve rendition groups extraction
+    * Build stream name according stream GROUP-ID
+    * Ignore reference to AUDIO group without URI when stream has no CODECS
+    * Use float for scaled tbr in _parse_m3u8_formats
 * [utils] Add support for TTML styles in dfxp2srt
 * [downloader/hls] No need to download keys for fragments that have been
   already downloaded
 * [downloader/fragment] Improve fragment downloading
-    - Resume immediately
-    - Don't concatenate fragments and decrypt them on every resume
-    - Optimize disk storage usage, don't store intermediate fragments on disk
-    - Store bookkeeping download state file
+    * Resume immediately
+    * Don't concatenate fragments and decrypt them on every resume
+    * Optimize disk storage usage, don't store intermediate fragments on disk
+    * Store bookkeeping download state file
 + [extractor/common] Add support for multiple getters in try_get
 + [extractor/common] Add support for video of WebPage context in _json_ld
   (#12778)

From e0c1e9a98c3a22c0ae559a30cff1d1ab35105beb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 May 2017 01:39:52 +0700
Subject: [PATCH 223/611] release 2017.05.01

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 +++
 youtube_dl/version.py     | 2 +-
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index a84cbc801..fc221594f 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.28*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.28**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.01*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.01**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.04.28
+[debug] youtube-dl version 2017.05.01
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index a4a8eac8e..4b27f3eff 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.05.01
 
 Core
 + [extractor/common] Extract view count from JSON-LD
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 0be7af8c4..e3c038c48 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -45,6 +45,7 @@
  - **anderetijden**: npo.nl and ntr.nl
  - **AnimeOnDemand**
  - **anitube.se**
+ - **Anvato**
  - **AnySex**
  - **Aparat**
  - **AppleConnect**
@@ -529,6 +530,7 @@
  - **NJPWWorld**: 新日本プロレスワールド
  - **NobelPrize**
  - **Noco**
+ - **Noovo**
  - **Normalboots**
  - **NosVideo**
  - **Nova**: TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz
@@ -1013,6 +1015,7 @@
  - **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword)
  - **youtube:watchlater**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)
  - **Zapiks**
+ - **Zaq1**
  - **ZDF**
  - **ZDFChannel**
  - **zingmp3**: mp3.zing.vn
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8c77f1905..c19ac49b0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.04.28'
+__version__ = '2017.05.01'

From a146fa1c68dade0fde7ef81cdb8debbec6a6fc04 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Mon, 1 May 2017 00:54:44 +0200
Subject: [PATCH 224/611] [coub] Update test and remove comment count
 extraction

---
 youtube_dl/extractor/coub.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/coub.py b/youtube_dl/extractor/coub.py
index 5fa1f006b..6ea03e65c 100644
--- a/youtube_dl/extractor/coub.py
+++ b/youtube_dl/extractor/coub.py
@@ -24,12 +24,11 @@ class CoubIE(InfoExtractor):
             'duration': 4.6,
             'timestamp': 1428527772,
             'upload_date': '20150408',
-            'uploader': 'Артём Лоскутников',
+            'uploader': 'Artyom Loskutnikov',
             'uploader_id': 'artyom.loskutnikov',
             'view_count': int,
             'like_count': int,
             'repost_count': int,
-            'comment_count': int,
             'age_limit': 0,
         },
     }, {
@@ -118,7 +117,6 @@ class CoubIE(InfoExtractor):
         view_count = int_or_none(coub.get('views_count') or coub.get('views_increase_count'))
         like_count = int_or_none(coub.get('likes_count'))
         repost_count = int_or_none(coub.get('recoubs_count'))
-        comment_count = int_or_none(coub.get('comments_count'))
 
         age_restricted = coub.get('age_restricted', coub.get('age_restricted_by_admin'))
         if age_restricted is not None:
@@ -137,7 +135,6 @@ class CoubIE(InfoExtractor):
             'view_count': view_count,
             'like_count': like_count,
             'repost_count': repost_count,
-            'comment_count': comment_count,
             'age_limit': age_limit,
             'formats': formats,
         }

From ab39a25c75be0f27f5b8ffad08e3a2d3ac51007f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 1 May 2017 09:02:15 +0100
Subject: [PATCH 225/611] [foxsports] fix extraction(closes #12945)

---
 youtube_dl/extractor/foxsports.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/foxsports.py b/youtube_dl/extractor/foxsports.py
index a3bb98377..985542727 100644
--- a/youtube_dl/extractor/foxsports.py
+++ b/youtube_dl/extractor/foxsports.py
@@ -11,10 +11,10 @@ class FoxSportsIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?foxsports\.com/(?:[^/]+/)*(?P<id>[^/]+)'
 
     _TEST = {
-        'url': 'http://www.foxsports.com/video?vid=432609859715',
+        'url': 'http://www.foxsports.com/tennessee/video/432609859715',
         'md5': 'b49050e955bebe32c301972e4012ac17',
         'info_dict': {
-            'id': 'i0qKWsk3qJaM',
+            'id': 'bwduI3X_TgUB',
             'ext': 'mp4',
             'title': 'Courtney Lee on going up 2-0 in series vs. Blazers',
             'description': 'Courtney Lee talks about Memphis being focused.',
@@ -31,8 +31,9 @@ class FoxSportsIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         config = self._parse_json(
-            self._search_regex(
-                r"data-player-config='([^']+)'", webpage, 'data player config'),
+            self._html_search_regex(
+                r"""class="[^"]*(?:fs-player|platformPlayer-wrapper)[^"]*".+?data-player-config='([^']+)'""",
+                webpage, 'data player config'),
             video_id)
 
         return self.url_result(smuggle_url(update_url_query(

From b9f9f361fa93ffaff0146a1f0ce9920770d1891c Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Mon, 1 May 2017 19:56:51 +0200
Subject: [PATCH 226/611] [crunchyroll] Update test

---
 youtube_dl/extractor/crunchyroll.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 2ed8b30bb..2ffa4a7f8 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -171,7 +171,7 @@ class CrunchyrollIE(CrunchyrollBaseIE):
         'info_dict': {
             'id': '727589',
             'ext': 'mp4',
-            'title': "KONOSUBA -God's blessing on this wonderful world! 2 Episode 1 – Give Me Deliverance from this Judicial Injustice!",
+            'title': "KONOSUBA -God's blessing on this wonderful world! 2 Episode 1 – Give Me Deliverance From This Judicial Injustice!",
             'description': 'md5:cbcf05e528124b0f3a0a419fc805ea7d',
             'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': 'Kadokawa Pictures Inc.',
@@ -179,7 +179,7 @@ class CrunchyrollIE(CrunchyrollBaseIE):
             'series': "KONOSUBA -God's blessing on this wonderful world!",
             'season': "KONOSUBA -God's blessing on this wonderful world! 2",
             'season_number': 2,
-            'episode': 'Give Me Deliverance from this Judicial Injustice!',
+            'episode': 'Give Me Deliverance From This Judicial Injustice!',
             'episode_number': 1,
         },
         'params': {

From d86d169dd5bcefc35e13aa8274ac3b30c701cd87 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Mon, 1 May 2017 20:37:23 +0200
Subject: [PATCH 227/611] [dailymotion] Add working test

---
 youtube_dl/extractor/dailymotion.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 246efde43..441114d19 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -50,6 +50,24 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
     ]
 
     _TESTS = [
+        {
+            'url': 'http://www.dailymotion.com/video/x5kesuj_office-christmas-party-review-jason-bateman-olivia-munn-t-j-miller_news',
+            'md5': '074b95bdee76b9e3654137aee9c79dfe',
+            'info_dict': {
+                'id': 'x5kesuj',
+                'ext': 'mp4',
+                'title': 'Office Christmas Party Review –  Jason Bateman, Olivia Munn, T.J. Miller',
+                'description': 'Office Christmas Party Review -  Jason Bateman, Olivia Munn, T.J. Miller',
+                'thumbnail': r're:^https?:.*\.(?:jpg|png)$',
+                'duration': 187,
+                'timestamp': 1493651285,
+                'upload_date': '20170501',
+                'uploader': 'Deadline',
+                'uploader_id': 'x1xm8ri',
+                'age_limit': 0,
+                'view_count': int,
+            },
+        },
         {
             'url': 'https://www.dailymotion.com/video/x2iuewm_steam-machine-models-pricing-listed-on-steam-store-ign-news_videogames',
             'md5': '2137c41a8e78554bb09225b8eb322406',
@@ -66,7 +84,8 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                 'uploader_id': 'xijv66',
                 'age_limit': 0,
                 'view_count': int,
-            }
+            },
+            'skip': 'video gone',
         },
         # Vevo video
         {

From 77481f138670f13eada366f2bb1c54333084d72a Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Mon, 1 May 2017 20:25:09 +0200
Subject: [PATCH 228/611] [democracynow] Update test

---
 youtube_dl/extractor/democracynow.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index bdfe638b4..5c9c0ecdc 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -21,7 +21,8 @@ class DemocracynowIE(InfoExtractor):
         'info_dict': {
             'id': '2015-0703-001',
             'ext': 'mp4',
-            'title': 'Daily Show',
+            'title': 'Daily Show for July 03, 2015',
+            'description': 'md5:80eb927244d6749900de6072c7cc2c86',
         },
     }, {
         'url': 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree',

From ae21d2fd940b6aa2e2d619c037d701848b011f5a Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Mon, 1 May 2017 20:51:51 +0200
Subject: [PATCH 229/611] [dotsub] Update test

---
 youtube_dl/extractor/dotsub.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dotsub.py b/youtube_dl/extractor/dotsub.py
index 1f75352ca..148605c0b 100644
--- a/youtube_dl/extractor/dotsub.py
+++ b/youtube_dl/extractor/dotsub.py
@@ -35,7 +35,7 @@ class DotsubIE(InfoExtractor):
             'thumbnail': 're:^https?://dotsub.com/media/747bcf58-bd59-45b7-8c8c-ac312d084ee6/p',
             'duration': 290,
             'timestamp': 1476767794.2809999,
-            'upload_date': '20160525',
+            'upload_date': '20161018',
             'uploader': 'parthivi001',
             'uploader_id': 'user52596202',
             'view_count': int,

From c6fe5a7e12a5ebbe137b889afb8aefc206d132af Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Mon, 1 May 2017 21:29:34 +0200
Subject: [PATCH 230/611] [douyutv] Update test

---
 youtube_dl/extractor/douyutv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index 82d8a042f..d22133d24 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -20,7 +20,7 @@ class DouyuTVIE(InfoExtractor):
             'id': '17732',
             'display_id': 'iseven',
             'ext': 'flv',
-            'title': 're:^清晨醒脑！T-ARA根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'title': 're:^清晨醒脑！根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'description': r're:.*m7show@163\.com.*',
             'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': '7师傅',
@@ -51,7 +51,7 @@ class DouyuTVIE(InfoExtractor):
             'id': '17732',
             'display_id': '17732',
             'ext': 'flv',
-            'title': 're:^清晨醒脑！T-ARA根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'title': 're:^清晨醒脑！根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'description': r're:.*m7show@163\.com.*',
             'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': '7师傅',

From 7fc875195fd74c235d6ca86afc642f683a485a4b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 2 May 2017 00:06:19 +0100
Subject: [PATCH 231/611] [amp] imporove thumbnail and subtitle extraction

---
 youtube_dl/extractor/amp.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/amp.py b/youtube_dl/extractor/amp.py
index 98f8e69cd..fde1a8ff7 100644
--- a/youtube_dl/extractor/amp.py
+++ b/youtube_dl/extractor/amp.py
@@ -34,9 +34,12 @@ class AMPIE(InfoExtractor):
             if isinstance(media_thumbnail, dict):
                 media_thumbnail = [media_thumbnail]
             for thumbnail_data in media_thumbnail:
-                thumbnail = thumbnail_data['@attributes']
+                thumbnail = thumbnail_data.get('@attributes', {})
+                thumbnail_url = thumbnail.get('url')
+                if not thumbnail_url:
+                    continue
                 thumbnails.append({
-                    'url': self._proto_relative_url(thumbnail['url'], 'http:'),
+                    'url': self._proto_relative_url(thumbnail_url, 'http:'),
                     'width': int_or_none(thumbnail.get('width')),
                     'height': int_or_none(thumbnail.get('height')),
                 })
@@ -47,9 +50,14 @@ class AMPIE(InfoExtractor):
             if isinstance(media_subtitle, dict):
                 media_subtitle = [media_subtitle]
             for subtitle_data in media_subtitle:
-                subtitle = subtitle_data['@attributes']
-                lang = subtitle.get('lang') or 'en'
-                subtitles[lang] = [{'url': subtitle['href']}]
+                subtitle = subtitle_data.get('@attributes', {})
+                subtitle_href = subtitle.get('href')
+                if not subtitle_href:
+                    continue
+                subtitles.setdefault(subtitle.get('lang') or 'en', []).append({
+                    'url': subtitle_href,
+                    'ext': mimetype2ext(subtitle.get('type')) or determine_ext(subtitle_href),
+                })
 
         formats = []
         media_content = get_media_node('content')

From 55949fede6da7c2d612f56863196eadcbc583de9 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 5 May 2016 21:40:19 +0100
Subject: [PATCH 232/611] [common] introduce chapters field

---
 youtube_dl/extractor/common.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index fba15d446..9541e5b42 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -245,6 +245,10 @@ class InfoExtractor(object):
                     specified in the URL.
     end_time:       Time in seconds where the reproduction should end, as
                     specified in the URL.
+    chapters:       A list of dictionaries, with the following entries:
+                        * "start_time" - The start time of the chapter in seconds
+                        * "end_time" - The end time of the chapter in seconds
+                        * "title" (optional, string)
 
     The following fields should only be used when the video belongs to some logical
     chapter or section:

From fa2a36d9bcaddf793e8bac281f5fa7859af16a46 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 5 May 2016 21:41:30 +0100
Subject: [PATCH 233/611] [ffmpeg] add support for chapters field
 postprocessing

---
 youtube_dl/postprocessor/ffmpeg.py | 31 +++++++++++++++++++++++++++---
 1 file changed, 28 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 665109558..c91ec8588 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -4,6 +4,7 @@ import io
 import os
 import subprocess
 import time
+import re
 
 
 from .common import AudioConversionError, PostProcessor
@@ -22,6 +23,7 @@ from ..utils import (
     subtitles_filename,
     dfxp2srt,
     ISO639Utils,
+    replace_extension,
 )
 
 
@@ -429,17 +431,40 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
 
         filename = info['filepath']
         temp_filename = prepend_extension(filename, 'temp')
+        in_filenames = [filename]
+        options = []
 
         if info['ext'] == 'm4a':
-            options = ['-vn', '-acodec', 'copy']
+            options.extend(['-vn', '-acodec', 'copy'])
         else:
-            options = ['-c', 'copy']
+            options.extend(['-c', 'copy'])
 
         for (name, value) in metadata.items():
             options.extend(['-metadata', '%s=%s' % (name, value)])
 
+        chapters = info.get('chapters', [])
+        if chapters:
+            metadata_filename = encodeFilename(replace_extension(filename, 'meta'))
+            with io.open(metadata_filename, 'wt', encoding='utf-8') as f:
+                def ffmpeg_escape(text):
+                    return re.sub(r'(=|;|#|\\|\n)', r'\\\1', text)
+
+                metadata_file_content = ';FFMETADATA1\n'
+                for chapter in chapters:
+                    metadata_file_content += '[CHAPTER]\nTIMEBASE=1/1000\n'
+                    metadata_file_content += 'START=%d\n' % (chapter['start_time'] * 1000)
+                    metadata_file_content += 'END=%d\n' % (chapter['end_time'] * 1000)
+                    chapter_title = chapter.get('title')
+                    if chapter_title:
+                        metadata_file_content += 'title=%s\n' % ffmpeg_escape(chapter_title)
+                f.write(metadata_file_content)
+                in_filenames.append(metadata_filename)
+                options.extend(['-map_metadata', '1'])
+
         self._downloader.to_screen('[ffmpeg] Adding metadata to \'%s\'' % filename)
-        self.run_ffmpeg(filename, temp_filename, options)
+        self.run_ffmpeg_multiple_files(in_filenames, temp_filename, options)
+        if chapters:
+            os.remove(metadata_filename)
         os.remove(encodeFilename(filename))
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
         return [], info

From 7a6d33a9a5390d0dab7e9162d6b2552cb0fe23a5 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 5 May 2016 21:42:37 +0100
Subject: [PATCH 234/611] [pbs] extract chapters information

---
 youtube_dl/extractor/pbs.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 6166dc2ad..0727e381b 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -8,6 +8,7 @@ from ..utils import (
     ExtractorError,
     determine_ext,
     int_or_none,
+    float_or_none,
     js_to_json,
     strip_jsonp,
     strip_or_none,
@@ -464,6 +465,7 @@ class PBSIE(InfoExtractor):
                     redirects.append(redirect)
                     redirect_urls.add(redirect_url)
 
+        chapters = []
         # Player pages may also serve different qualities
         for page in ('widget/partnerplayer', 'portalplayer'):
             player = self._download_webpage(
@@ -479,6 +481,20 @@ class PBSIE(InfoExtractor):
                     extract_redirect_urls(video_info)
                     if not info:
                         info = video_info
+                if not chapters:
+                    for chapter_data in re.findall(r'(?s)chapters\.push\(({.*?})\)', player):
+                        chapter = self._parse_json(chapter_data, video_id, js_to_json, fatal=False)
+                        if not chapter:
+                            continue
+                        start_time = float_or_none(chapter.get('start_time'), 1000)
+                        duration = float_or_none(chapter.get('duration'), 1000)
+                        if start_time is None or duration is None:
+                            continue
+                        chapters.append({
+                            'start_time': start_time,
+                            'end_time': start_time + duration,
+                            'title': chapter.get('title'),
+                        })
 
         formats = []
         http_url = None
@@ -588,4 +604,5 @@ class PBSIE(InfoExtractor):
             'upload_date': upload_date,
             'formats': formats,
             'subtitles': subtitles,
+            'chapters': chapters,
         }

From 5401bea27fd6bcdd030e50f3af85145807eeab27 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 4 May 2017 03:18:56 +0800
Subject: [PATCH 235/611] [leeco] Fix extraction (closes #12974)

Seems on mobile devices a similar API is used, but I always get an AD
with mimicking that API.
---
 ChangeLog                     |   6 ++
 youtube_dl/extractor/leeco.py | 103 +++++++++++-----------------------
 2 files changed, 39 insertions(+), 70 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 4b27f3eff..e31f54304 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [leeco] Fix extraction (#12974)
+
+
 version 2017.05.01
 
 Core
diff --git a/youtube_dl/extractor/leeco.py b/youtube_dl/extractor/leeco.py
index 9eda956d2..0a07c1320 100644
--- a/youtube_dl/extractor/leeco.py
+++ b/youtube_dl/extractor/leeco.py
@@ -23,7 +23,6 @@ from ..utils import (
     str_or_none,
     url_basename,
     urshift,
-    update_url_query,
 )
 
 
@@ -51,7 +50,7 @@ class LeIE(InfoExtractor):
             'id': '1415246',
             'ext': 'mp4',
             'title': '美人天下01',
-            'description': 'md5:f88573d9d7225ada1359eaf0dbf8bcda',
+            'description': 'md5:28942e650e82ed4fcc8e4de919ee854d',
         },
         'params': {
             'hls_prefer_native': True,
@@ -69,7 +68,6 @@ class LeIE(InfoExtractor):
         'params': {
             'hls_prefer_native': True,
         },
-        'skip': 'Only available in China',
     }, {
         'url': 'http://sports.le.com/video/25737697.html',
         'only_matching': True,
@@ -81,7 +79,7 @@ class LeIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    # ror() and calc_time_key() are reversed from a embedded swf file in KLetvPlayer.swf
+    # ror() and calc_time_key() are reversed from a embedded swf file in LetvPlayer.swf
     def ror(self, param1, param2):
         _loc3_ = 0
         while _loc3_ < param2:
@@ -90,15 +88,8 @@ class LeIE(InfoExtractor):
         return param1
 
     def calc_time_key(self, param1):
-        _loc2_ = 773625421
-        _loc3_ = self.ror(param1, _loc2_ % 13)
-        _loc3_ = _loc3_ ^ _loc2_
-        _loc3_ = self.ror(_loc3_, _loc2_ % 17)
-        return _loc3_
-
-    # reversed from http://jstatic.letvcdn.com/sdk/player.js
-    def get_mms_key(self, time):
-        return self.ror(time, 8) ^ 185025305
+        _loc2_ = 185025305
+        return self.ror(param1, _loc2_ % 17) ^ _loc2_
 
     # see M3U8Encryption class in KLetvPlayer.swf
     @staticmethod
@@ -122,7 +113,7 @@ class LeIE(InfoExtractor):
 
     def _check_errors(self, play_json):
         # Check for errors
-        playstatus = play_json['playstatus']
+        playstatus = play_json['msgs']['playstatus']
         if playstatus['status'] == 0:
             flag = playstatus['flag']
             if flag == 1:
@@ -134,58 +125,31 @@ class LeIE(InfoExtractor):
         media_id = self._match_id(url)
         page = self._download_webpage(url, media_id)
 
-        play_json_h5 = self._download_json(
-            'http://api.le.com/mms/out/video/playJsonH5',
-            media_id, 'Downloading html5 playJson data', query={
-                'id': media_id,
-                'platid': 3,
-                'splatid': 304,
-                'format': 1,
-                'tkey': self.get_mms_key(int(time.time())),
-                'domain': 'www.le.com',
-                'tss': 'no',
-            },
-            headers=self.geo_verification_headers())
-        self._check_errors(play_json_h5)
-
         play_json_flash = self._download_json(
-            'http://api.le.com/mms/out/video/playJson',
+            'http://player-pc.le.com/mms/out/video/playJson',
             media_id, 'Downloading flash playJson data', query={
                 'id': media_id,
                 'platid': 1,
                 'splatid': 101,
                 'format': 1,
+                'source': 1000,
                 'tkey': self.calc_time_key(int(time.time())),
                 'domain': 'www.le.com',
+                'region': 'cn',
             },
             headers=self.geo_verification_headers())
         self._check_errors(play_json_flash)
 
-        def get_h5_urls(media_url, format_id):
-            location = self._download_json(
-                media_url, media_id,
-                'Download JSON metadata for format %s' % format_id, query={
-                    'format': 1,
-                    'expect': 3,
-                    'tss': 'no',
-                })['location']
-
-            return {
-                'http': update_url_query(location, {'tss': 'no'}),
-                'hls': update_url_query(location, {'tss': 'ios'}),
-            }
-
         def get_flash_urls(media_url, format_id):
-            media_url += '&' + compat_urllib_parse_urlencode({
-                'm3v': 1,
-                'format': 1,
-                'expect': 3,
-                'rateid': format_id,
-            })
-
             nodes_data = self._download_json(
                 media_url, media_id,
-                'Download JSON metadata for format %s' % format_id)
+                'Download JSON metadata for format %s' % format_id,
+                query={
+                    'm3v': 1,
+                    'format': 1,
+                    'expect': 3,
+                    'tss': 'ios',
+                })
 
             req = self._request_webpage(
                 nodes_data['nodelist'][0]['location'], media_id,
@@ -199,29 +163,28 @@ class LeIE(InfoExtractor):
 
         extracted_formats = []
         formats = []
-        for play_json, get_urls in ((play_json_h5, get_h5_urls), (play_json_flash, get_flash_urls)):
-            playurl = play_json['playurl']
-            play_domain = playurl['domain'][0]
+        playurl = play_json_flash['msgs']['playurl']
+        play_domain = playurl['domain'][0]
 
-            for format_id, format_data in playurl.get('dispatch', []).items():
-                if format_id in extracted_formats:
-                    continue
-                extracted_formats.append(format_id)
+        for format_id, format_data in playurl.get('dispatch', []).items():
+            if format_id in extracted_formats:
+                continue
+            extracted_formats.append(format_id)
 
-                media_url = play_domain + format_data[0]
-                for protocol, format_url in get_urls(media_url, format_id).items():
-                    f = {
-                        'url': format_url,
-                        'ext': determine_ext(format_data[1]),
-                        'format_id': '%s-%s' % (protocol, format_id),
-                        'protocol': 'm3u8_native' if protocol == 'hls' else 'http',
-                        'quality': int_or_none(format_id),
-                    }
+            media_url = play_domain + format_data[0]
+            for protocol, format_url in get_flash_urls(media_url, format_id).items():
+                f = {
+                    'url': format_url,
+                    'ext': determine_ext(format_data[1]),
+                    'format_id': '%s-%s' % (protocol, format_id),
+                    'protocol': 'm3u8_native' if protocol == 'hls' else 'http',
+                    'quality': int_or_none(format_id),
+                }
 
-                    if format_id[-1:] == 'p':
-                        f['height'] = int_or_none(format_id[:-1])
+                if format_id[-1:] == 'p':
+                    f['height'] = int_or_none(format_id[:-1])
 
-                    formats.append(f)
+                formats.append(f)
         self._sort_formats(formats, ('height', 'quality', 'format_id'))
 
         publish_time = parse_iso8601(self._html_search_regex(

From 0c265486016b06342fb257966474ce591667aaff Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 May 2017 23:09:18 +0800
Subject: [PATCH 236/611] [cda] Implement birthday verification (closes #12789)

---
 ChangeLog                          |  1 +
 test/test_utils.py                 | 11 ++++++
 youtube_dl/extractor/cda.py        | 52 +++++++++++++++++++++++--
 youtube_dl/extractor/videopress.py |  9 ++---
 youtube_dl/utils.py                | 61 ++++++++++++++++++++++++++++++
 5 files changed, 125 insertions(+), 9 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index e31f54304..7324a9dd9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [cda] Support birthday verification (#12789)
 * [leeco] Fix extraction (#12974)
 
 
diff --git a/test/test_utils.py b/test/test_utils.py
index 05fdc0e95..f31559e71 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -44,6 +44,7 @@ from youtube_dl.utils import (
     limit_length,
     mimetype2ext,
     month_by_name,
+    multipart_encode,
     ohdave_rsa_encrypt,
     OnDemandPagedList,
     orderedSet,
@@ -620,6 +621,16 @@ class TestUtil(unittest.TestCase):
             'http://example.com/path', {'test': '第二行тест'})),
             query_dict('http://example.com/path?test=%E7%AC%AC%E4%BA%8C%E8%A1%8C%D1%82%D0%B5%D1%81%D1%82'))
 
+    def test_multipart_encode(self):
+        self.assertEqual(
+            multipart_encode({b'field': b'value'}, boundary='AAAAAA')[0],
+            b'--AAAAAA\r\nContent-Disposition: form-data; name="field"\r\n\r\nvalue\r\n--AAAAAA--\r\n')
+        self.assertEqual(
+            multipart_encode({'欄位'.encode('utf-8'): '值'.encode('utf-8')}, boundary='AAAAAA')[0],
+            b'--AAAAAA\r\nContent-Disposition: form-data; name="\xe6\xac\x84\xe4\xbd\x8d"\r\n\r\n\xe5\x80\xbc\r\n--AAAAAA--\r\n')
+        self.assertRaises(
+            ValueError, multipart_encode, {b'field': b'value'}, boundary='value')
+
     def test_dict_get(self):
         FALSE_VALUES = {
             'none': None,
diff --git a/youtube_dl/extractor/cda.py b/youtube_dl/extractor/cda.py
index 1ee35b501..78b7a923c 100755
--- a/youtube_dl/extractor/cda.py
+++ b/youtube_dl/extractor/cda.py
@@ -9,7 +9,10 @@ from ..utils import (
     ExtractorError,
     float_or_none,
     int_or_none,
+    multipart_encode,
     parse_duration,
+    random_birthday,
+    urljoin,
 )
 
 
@@ -27,7 +30,8 @@ class CDAIE(InfoExtractor):
             'description': 'md5:269ccd135d550da90d1662651fcb9772',
             'thumbnail': r're:^https?://.*\.jpg$',
             'average_rating': float,
-            'duration': 39
+            'duration': 39,
+            'age_limit': 0,
         }
     }, {
         'url': 'http://www.cda.pl/video/57413289',
@@ -41,13 +45,41 @@ class CDAIE(InfoExtractor):
             'uploader': 'crash404',
             'view_count': int,
             'average_rating': float,
-            'duration': 137
+            'duration': 137,
+            'age_limit': 0,
         }
+    }, {
+        # Age-restricted
+        'url': 'http://www.cda.pl/video/1273454c4',
+        'info_dict': {
+            'id': '1273454c4',
+            'ext': 'mp4',
+            'title': 'Bronson (2008) napisy HD 1080p',
+            'description': 'md5:1b6cb18508daf2dc4e0fa4db77fec24c',
+            'height': 1080,
+            'uploader': 'boniek61',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 5554,
+            'age_limit': 18,
+            'view_count': int,
+            'average_rating': float,
+        },
     }, {
         'url': 'http://ebd.cda.pl/0x0/5749950c',
         'only_matching': True,
     }]
 
+    def _download_age_confirm_page(self, url, video_id, *args, **kwargs):
+        form_data = random_birthday('rok', 'miesiac', 'dzien')
+        form_data.update({'return': url, 'module': 'video', 'module_id': video_id})
+        data, content_type = multipart_encode(form_data)
+        return self._download_webpage(
+            urljoin(url, '/a/validatebirth'), video_id, *args,
+            data=data, headers={
+                'Referer': url,
+                'Content-Type': content_type,
+            }, **kwargs)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         self._set_cookie('cda.pl', 'cda.player', 'html5')
@@ -57,6 +89,13 @@ class CDAIE(InfoExtractor):
         if 'Ten film jest dostępny dla użytkowników premium' in webpage:
             raise ExtractorError('This video is only available for premium users.', expected=True)
 
+        need_confirm_age = False
+        if self._html_search_regex(r'(<form[^>]+action="/a/validatebirth")',
+                                   webpage, 'birthday validate form', default=None):
+            webpage = self._download_age_confirm_page(
+                url, video_id, note='Confirming age')
+            need_confirm_age = True
+
         formats = []
 
         uploader = self._search_regex(r'''(?x)
@@ -81,6 +120,7 @@ class CDAIE(InfoExtractor):
             'thumbnail': self._og_search_thumbnail(webpage),
             'formats': formats,
             'duration': None,
+            'age_limit': 18 if need_confirm_age else 0,
         }
 
         def extract_format(page, version):
@@ -121,7 +161,12 @@ class CDAIE(InfoExtractor):
         for href, resolution in re.findall(
                 r'<a[^>]+data-quality="[^"]+"[^>]+href="([^"]+)"[^>]+class="quality-btn"[^>]*>([0-9]+p)',
                 webpage):
-            webpage = self._download_webpage(
+            if need_confirm_age:
+                handler = self._download_age_confirm_page
+            else:
+                handler = self._download_webpage
+
+            webpage = handler(
                 self._BASE_URL + href, video_id,
                 'Downloading %s version information' % resolution, fatal=False)
             if not webpage:
@@ -129,6 +174,7 @@ class CDAIE(InfoExtractor):
                 # invalid version is requested.
                 self.report_warning('Unable to download %s version information' % resolution)
                 continue
+
             extract_format(webpage, resolution)
 
         self._sort_formats(formats)
diff --git a/youtube_dl/extractor/videopress.py b/youtube_dl/extractor/videopress.py
index 049db25a5..e5f964d39 100644
--- a/youtube_dl/extractor/videopress.py
+++ b/youtube_dl/extractor/videopress.py
@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import random
 import re
 
 from .common import InfoExtractor
@@ -11,6 +10,7 @@ from ..utils import (
     float_or_none,
     parse_age_limit,
     qualities,
+    random_birthday,
     try_get,
     unified_timestamp,
     urljoin,
@@ -47,13 +47,10 @@ class VideoPressIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
+        query = random_birthday('birth_year', 'birth_month', 'birth_day')
         video = self._download_json(
             'https://public-api.wordpress.com/rest/v1.1/videos/%s' % video_id,
-            video_id, query={
-                'birth_month': random.randint(1, 12),
-                'birth_day': random.randint(1, 31),
-                'birth_year': random.randint(1950, 1995),
-            })
+            video_id, query=query)
 
         title = video['title']
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 86fc5ccac..25bd228ab 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -11,6 +11,7 @@ import contextlib
 import ctypes
 import datetime
 import email.utils
+import email.header
 import errno
 import functools
 import gzip
@@ -2097,6 +2098,58 @@ def update_Request(req, url=None, data=None, headers={}, query={}):
     return new_req
 
 
+def try_multipart_encode(data, boundary):
+    content_type = 'multipart/form-data; boundary=%s' % boundary
+
+    out = b''
+    for k, v in data.items():
+        out += b'--' + boundary.encode('ascii') + b'\r\n'
+        if isinstance(k, compat_str):
+            k = k.encode('utf-8')
+        if isinstance(v, compat_str):
+            v = v.encode('utf-8')
+        # RFC 2047 requires non-ASCII field names to be encoded, while RFC 7578
+        # suggests sending UTF-8 directly. Firefox sends UTF-8, too
+        content = b'Content-Disposition: form-data; name="%s"\r\n\r\n' % k + v + b'\r\n'
+        if boundary.encode('ascii') in content:
+            raise ValueError('Boundary overlaps with data')
+        out += content
+
+    out += b'--' + boundary.encode('ascii') + b'--\r\n'
+
+    return out, content_type
+
+
+def multipart_encode(data, boundary=None):
+    '''
+    Encode a dict to RFC 7578-compliant form-data
+
+    data:
+        A dict where keys and values can be either Unicode or bytes-like
+        objects.
+    boundary:
+        If specified a Unicode object, it's used as the boundary. Otherwise
+        a random boundary is generated.
+
+    Reference: https://tools.ietf.org/html/rfc7578
+    '''
+    has_specified_boundary = boundary is not None
+
+    while True:
+        if boundary is None:
+            boundary = '---------------' + str(random.randrange(0x0fffffff, 0xffffffff))
+
+        try:
+            out, content_type = try_multipart_encode(data, boundary)
+            break
+        except ValueError:
+            if has_specified_boundary:
+                raise
+            boundary = None
+
+    return out, content_type
+
+
 def dict_get(d, key_or_keys, default=None, skip_false_values=True):
     if isinstance(key_or_keys, (list, tuple)):
         for key in key_or_keys:
@@ -3760,3 +3813,11 @@ def write_xattr(path, key, value):
                         "Couldn't find a tool to set the xattrs. "
                         "Install either the python 'xattr' module, "
                         "or the 'xattr' binary.")
+
+
+def random_birthday(year_field, month_field, day_field):
+    return {
+        year_field: str(random.randint(1950, 1995)),
+        month_field: str(random.randint(1, 12)),
+        day_field: str(random.randint(1, 31)),
+    }

From ff6f9a67040c47ea1645e3f91f3153c212ebc7ca Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 4 May 2017 16:04:25 +0100
Subject: [PATCH 237/611] [extractor/common] fix typo in
 _extract_akamai_formats

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9541e5b42..b9ad8461a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2174,7 +2174,7 @@ class InfoExtractor(object):
     def _extract_akamai_formats(self, manifest_url, video_id, hosts={}):
         formats = []
         hdcore_sign = 'hdcore=3.7.0'
-        f4m_url = re.sub(r'(https?://[^/+])/i/', r'\1/z/', manifest_url).replace('/master.m3u8', '/manifest.f4m')
+        f4m_url = re.sub(r'(https?://[^/]+)/i/', r'\1/z/', manifest_url).replace('/master.m3u8', '/manifest.f4m')
         hds_host = hosts.get('hds')
         if hds_host:
             f4m_url = re.sub(r'(https?://)[^/]+', r'\1' + hds_host, f4m_url)

From 4fe14732a24214d030cdbdfc25e5906244be2dc2 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 4 May 2017 16:06:21 +0100
Subject: [PATCH 238/611] [laola1tv] fix extraction(closes #12880)

---
 youtube_dl/extractor/laola1tv.py | 92 +++++++++++++++++++++++---------
 1 file changed, 67 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index 3190b187c..e8853fb77 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import json
+
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
@@ -10,6 +12,7 @@ from ..utils import (
     xpath_text,
     urljoin,
     update_url_query,
+    js_to_json,
 )
 
 
@@ -28,6 +31,29 @@ class Laola1TvEmbedIE(InfoExtractor):
         },
     }
 
+    def _extract_token_url(self, stream_access_url, video_id, data):
+        return self._download_json(
+            stream_access_url, video_id, headers={
+                'Content-Type': 'application/json',
+            }, data=json.dumps(data).encode())['data']['stream-access'][0]
+
+    def _extract_formats(self, token_url, video_id):
+        token_doc = self._download_xml(
+            token_url, video_id, 'Downloading token',
+            headers=self.geo_verification_headers())
+
+        token_attrib = xpath_element(token_doc, './/token').attrib
+
+        if token_attrib['status'] != '0':
+            raise ExtractorError(
+                'Token error: %s' % token_attrib['comment'], expected=True)
+
+        formats = self._extract_akamai_formats(
+            '%s?hdnea=%s' % (token_attrib['url'], token_attrib['auth']),
+            video_id)
+        self._sort_formats(formats)
+        return formats
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
@@ -68,29 +94,16 @@ class Laola1TvEmbedIE(InfoExtractor):
         else:
             data_abo = urlencode_postdata(
                 dict((i, v) for i, v in enumerate(_v('req_liga_abos').split(','))))
-            token_url = self._download_json(
-                'https://club.laola1.tv/sp/laola1/api/v3/user/session/premium/player/stream-access',
-                video_id, query={
+            stream_access_url = update_url_query(
+                'https://club.laola1.tv/sp/laola1/api/v3/user/session/premium/player/stream-access', {
                     'videoId': _v('id'),
                     'target': self._search_regex(r'vs_target = (\d+);', webpage, 'vs target'),
                     'label': _v('label'),
                     'area': _v('area'),
-                }, data=data_abo)['data']['stream-access'][0]
+                })
+            token_url = self._extract_token_url(stream_access_url, video_id, data_abo)
 
-        token_doc = self._download_xml(
-            token_url, video_id, 'Downloading token',
-            headers=self.geo_verification_headers())
-
-        token_attrib = xpath_element(token_doc, './/token').attrib
-
-        if token_attrib['status'] != '0':
-            raise ExtractorError(
-                'Token error: %s' % token_attrib['comment'], expected=True)
-
-        formats = self._extract_akamai_formats(
-            '%s?hdnea=%s' % (token_attrib['url'], token_attrib['auth']),
-            video_id)
-        self._sort_formats(formats)
+        formats = self._extract_formats(token_url, video_id)
 
         categories_str = _v('meta_sports')
         categories = categories_str.split(',') if categories_str else []
@@ -107,7 +120,7 @@ class Laola1TvEmbedIE(InfoExtractor):
         }
 
 
-class Laola1TvIE(InfoExtractor):
+class Laola1TvIE(Laola1TvEmbedIE):
     IE_NAME = 'laola1tv'
     _VALID_URL = r'https?://(?:www\.)?laola1\.tv/[a-z]+-[a-z]+/[^/]+/(?P<id>[^/?#&]+)'
     _TESTS = [{
@@ -164,13 +177,42 @@ class Laola1TvIE(InfoExtractor):
         if 'Dieser Livestream ist bereits beendet.' in webpage:
             raise ExtractorError('This live stream has already finished.', expected=True)
 
-        iframe_url = urljoin(url, self._search_regex(
-            r'<iframe[^>]*?id="videoplayer"[^>]*?src="([^"]+)"',
-            webpage, 'iframe url'))
+        conf = self._parse_json(self._search_regex(
+            r'(?s)conf\s*=\s*({.+?});', webpage, 'conf'),
+            display_id, js_to_json)
+
+        video_id = conf['videoid']
+
+        config = self._download_json(conf['configUrl'], video_id, query={
+            'videoid': video_id,
+            'partnerid': conf['partnerid'],
+            'language': conf.get('language', ''),
+            'portal': conf.get('portalid', ''),
+        })
+        error = config.get('error')
+        if error:
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, error), expected=True)
+
+        video_data = config['video']
+        title = video_data['title']
+        is_live = video_data.get('isLivestream') and video_data.get('isLive')
+        meta = video_data.get('metaInformation')
+        sports = meta.get('sports')
+        categories = sports.split(',') if sports else []
+
+        token_url = self._extract_token_url(
+            video_data['streamAccess'], video_id,
+            video_data['abo']['required'])
+
+        formats = self._extract_formats(token_url, video_id)
 
         return {
-            '_type': 'url',
+            'id': video_id,
             'display_id': display_id,
-            'url': iframe_url,
-            'ie_key': 'Laola1TvEmbed',
+            'title': self._live_title(title) if is_live else title,
+            'description': video_data.get('description'),
+            'thumbnail': video_data.get('image'),
+            'categories': categories,
+            'formats': formats,
+            'is_live': is_live,
         }

From 7f09e523e87ab6fcf5713fbcd6767d00d98039e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 May 2017 22:41:47 +0700
Subject: [PATCH 239/611] [laola1tv:embed] Fix tests

---
 youtube_dl/extractor/laola1tv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index e8853fb77..d9f53c356 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -19,7 +19,7 @@ from ..utils import (
 class Laola1TvEmbedIE(InfoExtractor):
     IE_NAME = 'laola1tv:embed'
     _VALID_URL = r'https?://(?:www\.)?laola1\.tv/titanplayer\.php\?.*?\bvideoid=(?P<id>\d+)'
-    _TEST = {
+    _TESTS = [{
         # flashvars.premium = "false";
         'url': 'https://www.laola1.tv/titanplayer.php?videoid=708065&type=V&lang=en&portal=int&customer=1024',
         'info_dict': {
@@ -29,7 +29,7 @@ class Laola1TvEmbedIE(InfoExtractor):
             'uploader': 'ITTF - International Table Tennis Federation',
             'upload_date': '20161211',
         },
-    }
+    }]
 
     def _extract_token_url(self, stream_access_url, video_id, data):
         return self._download_json(

From 4947f13cd0e541bd2223187df19389c8b721c665 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 May 2017 22:42:49 +0700
Subject: [PATCH 240/611] [pbs] Improve multipart video support (closes #12981)

---
 youtube_dl/extractor/pbs.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 0727e381b..16cc667d0 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -10,6 +10,7 @@ from ..utils import (
     int_or_none,
     float_or_none,
     js_to_json,
+    orderedSet,
     strip_jsonp,
     strip_or_none,
     unified_strdate,
@@ -264,6 +265,13 @@ class PBSIE(InfoExtractor):
             },
             'playlist_count': 2,
         },
+        {
+            'url': 'http://www.pbs.org/wgbh/americanexperience/films/great-war/',
+            'info_dict': {
+                'id': 'great-war',
+            },
+            'playlist_count': 3,
+        },
         {
             'url': 'http://www.pbs.org/wgbh/americanexperience/films/death/player/',
             'info_dict': {
@@ -382,10 +390,10 @@ class PBSIE(InfoExtractor):
             # tabbed frontline videos
             MULTI_PART_REGEXES = (
                 r'<div[^>]+class="videotab[^"]*"[^>]+vid="(\d+)"',
-                r'<a[^>]+href=["\']#video-\d+["\'][^>]+data-coveid=["\'](\d+)',
+                r'<a[^>]+href=["\']#(?:video-|part)\d+["\'][^>]+data-cove[Ii]d=["\'](\d+)',
             )
             for p in MULTI_PART_REGEXES:
-                tabbed_videos = re.findall(p, webpage)
+                tabbed_videos = orderedSet(re.findall(p, webpage))
                 if tabbed_videos:
                     return tabbed_videos, presumptive_id, upload_date, description
 

From 50ad078b7bca28ec4a85caf7689e3a260def2189 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 5 May 2017 15:13:40 +0800
Subject: [PATCH 241/611] [gdcvault] Fix extraction for videos with
 gdc-player.html

Closes #12733
---
 ChangeLog                        |  1 +
 youtube_dl/extractor/gdcvault.py | 15 ++++++++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 7324a9dd9..930fdded4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [gdcvault] Fix extraction for some videos (#12733)
 + [cda] Support birthday verification (#12789)
 * [leeco] Fix extraction (#12974)
 
diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index 3136427db..f71d9092e 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -75,6 +75,19 @@ class GDCVaultIE(InfoExtractor):
                 'format': 'jp',  # The japanese audio
             }
         },
+        {
+            # gdc-player.html
+            'url': 'http://www.gdcvault.com/play/1435/An-American-engine-in-Tokyo',
+            'info_dict': {
+                'id': '1435',
+                'display_id': 'An-American-engine-in-Tokyo',
+                'ext': 'flv',
+                'title': 'An American Engine in Tokyo:/nThe collaboration of Epic Games and Square Enix/nFor THE LAST REMINANT',
+            },
+            'params': {
+                'skip_download': True,  # Requires rtmpdump
+            },
+        },
     ]
 
     def _login(self, webpage_url, display_id):
@@ -128,7 +141,7 @@ class GDCVaultIE(InfoExtractor):
                 'title': title,
             }
 
-        PLAYER_REGEX = r'<iframe src="(?P<xml_root>.+?)/player.*?\.html.*?".*?</iframe>'
+        PLAYER_REGEX = r'<iframe src="(?P<xml_root>.+?)/(?:gdc-)?player.*?\.html.*?".*?</iframe>'
 
         xml_root = self._html_search_regex(
             PLAYER_REGEX, start_page, 'xml root', default=None)

From cc7bda4fffa3913178e5ad345a8c9c6d1d8a2626 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 5 May 2017 20:01:02 +0800
Subject: [PATCH 242/611] [vice] Fix extraction for non en_us videos (closes
 #12967)

---
 ChangeLog                        |  1 +
 youtube_dl/extractor/vice.py     | 34 ++++++++++++++++++++++++++------
 youtube_dl/extractor/viceland.py | 11 ++++++++---
 3 files changed, 37 insertions(+), 9 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 930fdded4..23f83cc56 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [vice] Fix extraction for non en_us videos (#12967)
 * [gdcvault] Fix extraction for some videos (#12733)
 + [cda] Support birthday verification (#12789)
 * [leeco] Fix extraction (#12974)
diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index f0a7fd739..275fc1395 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -20,7 +20,7 @@ from ..utils import (
 
 
 class ViceBaseIE(AdobePassIE):
-    def _extract_preplay_video(self, url, webpage):
+    def _extract_preplay_video(self, url, locale, webpage):
         watch_hub_data = extract_attributes(self._search_regex(
             r'(?s)(<watch-hub\s*.+?</watch-hub>)', webpage, 'watch hub'))
         video_id = watch_hub_data['vms-id']
@@ -45,7 +45,7 @@ class ViceBaseIE(AdobePassIE):
 
         try:
             host = 'www.viceland' if is_locked else self._PREPLAY_HOST
-            preplay = self._download_json('https://%s.com/en_us/preplay/%s' % (host, video_id), video_id, query=query)
+            preplay = self._download_json('https://%s.com/%s/preplay/%s' % (host, locale, video_id), video_id, query=query)
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
                 error = json.loads(e.cause.read().decode())
@@ -88,7 +88,7 @@ class ViceBaseIE(AdobePassIE):
 
 
 class ViceIE(ViceBaseIE):
-    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?:[^/]+/)?videos?/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?P<locale>[^/]+/)(?:[^/]+/)?videos?/(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'http://www.vice.com/video/cowboy-capitalists-part-1',
@@ -115,20 +115,39 @@ class ViceIE(ViceBaseIE):
         'add_ie': ['Youtube'],
     }, {
         'url': 'https://video.vice.com/en_us/video/the-signal-from-tolva/5816510690b70e6c5fd39a56',
-        'md5': '',
         'info_dict': {
             'id': '5816510690b70e6c5fd39a56',
             'ext': 'mp4',
             'uploader': 'Waypoint',
             'title': 'The Signal From Tölva',
+            'description': 'md5:3927e3c79f9e8094606a2b3c5b5e55d5',
             'uploader_id': '57f7d621e05ca860fa9ccaf9',
-            'timestamp': 1477941983938,
+            'timestamp': 1477941983,
+            'upload_date': '20161031',
         },
         'params': {
             # m3u8 download
             'skip_download': True,
         },
         'add_ie': ['UplynkPreplay'],
+    }, {
+        'url': 'https://video.vice.com/alps/video/ulfs-wien-beruchtigste-grafitti-crew-part-1/581b12b60a0e1f4c0fb6ea2f',
+        'info_dict': {
+            'id': '581b12b60a0e1f4c0fb6ea2f',
+            'ext': 'mp4',
+            'title': 'ULFs - Wien berüchtigste Grafitti Crew - Part 1',
+            'description': '<p>Zwischen Hinterzimmer-Tattoos und U-Bahnschächten erzählen uns die Ulfs, wie es ist, "süchtig nach Sachbeschädigung" zu sein.</p>',
+            'uploader': 'VICE',
+            'uploader_id': '57a204088cb727dec794c67b',
+            'timestamp': 1485368119,
+            'upload_date': '20170125',
+            'age_limit': 14,
+        },
+        'params': {
+            # AES-encrypted m3u8
+            'skip_download': True,
+        },
+        'add_ie': ['UplynkPreplay'],
     }, {
         'url': 'https://news.vice.com/video/experimenting-on-animals-inside-the-monkey-lab',
         'only_matching': True,
@@ -142,6 +161,9 @@ class ViceIE(ViceBaseIE):
     _PREPLAY_HOST = 'video.vice'
 
     def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        locale = mobj.group('locale')
         video_id = self._match_id(url)
         webpage, urlh = self._download_webpage_handle(url, video_id)
         embed_code = self._search_regex(
@@ -153,7 +175,7 @@ class ViceIE(ViceBaseIE):
             r'data-youtube-id="([^"]+)"', webpage, 'youtube id', default=None)
         if youtube_id:
             return self.url_result(youtube_id, 'Youtube')
-        return self._extract_preplay_video(urlh.geturl(), webpage)
+        return self._extract_preplay_video(urlh.geturl(), locale, webpage)
 
 
 class ViceShowIE(InfoExtractor):
diff --git a/youtube_dl/extractor/viceland.py b/youtube_dl/extractor/viceland.py
index 87f9216b5..bd60235c8 100644
--- a/youtube_dl/extractor/viceland.py
+++ b/youtube_dl/extractor/viceland.py
@@ -1,11 +1,13 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .vice import ViceBaseIE
 
 
 class VicelandIE(ViceBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?viceland\.com/[^/]+/video/[^/]+/(?P<id>[a-f0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?viceland\.com/(?P<locale>[^/]+)/video/[^/]+/(?P<id>[a-f0-9]+)'
     _TEST = {
         'url': 'https://www.viceland.com/en_us/video/trapped/588a70d0dba8a16007de7316',
         'info_dict': {
@@ -24,10 +26,13 @@ class VicelandIE(ViceBaseIE):
             'skip_download': True,
         },
         'add_ie': ['UplynkPreplay'],
+        'skip': '404',
     }
     _PREPLAY_HOST = 'www.viceland'
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        locale = mobj.group('locale')
         webpage = self._download_webpage(url, video_id)
-        return self._extract_preplay_video(url, webpage)
+        return self._extract_preplay_video(url, locale, webpage)

From 4ac6dc3732492839b8c5e9a144ac8ef8fd1539aa Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 5 May 2017 20:26:51 +0800
Subject: [PATCH 243/611] [vice] Support Vice articles (closes #12968)

---
 ChangeLog                          |  1 +
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/vice.py       | 79 ++++++++++++++++++++++++------
 3 files changed, 67 insertions(+), 14 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 23f83cc56..0908e2e93 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [vice] Support vice articles (#12968)
 * [vice] Fix extraction for non en_us videos (#12967)
 * [gdcvault] Fix extraction for some videos (#12733)
 + [cda] Support birthday verification (#12789)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index c0020dd7d..1d7495910 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1125,6 +1125,7 @@ from .vgtv import (
 from .vh1 import VH1IE
 from .vice import (
     ViceIE,
+    ViceArticleIE,
     ViceShowIE,
 )
 from .viceland import VicelandIE
diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index 275fc1395..b2e95734b 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -88,7 +88,7 @@ class ViceBaseIE(AdobePassIE):
 
 
 class ViceIE(ViceBaseIE):
-    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?P<locale>[^/]+/)(?:[^/]+/)?videos?/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?P<locale>[^/]+)/(?:[^/]+/)?videos?/(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'http://www.vice.com/video/cowboy-capitalists-part-1',
@@ -100,19 +100,6 @@ class ViceIE(ViceBaseIE):
             'duration': 725.983,
         },
         'add_ie': ['Ooyala'],
-    }, {
-        'url': 'http://www.vice.com/video/how-to-hack-a-car',
-        'md5': 'a7ecf64ee4fa19b916c16f4b56184ae2',
-        'info_dict': {
-            'id': '3jstaBeXgAs',
-            'ext': 'mp4',
-            'title': 'How to Hack a Car: Phreaked Out (Episode 2)',
-            'description': 'md5:ee95453f7ff495db8efe14ae8bf56f30',
-            'uploader_id': 'MotherboardTV',
-            'uploader': 'Motherboard',
-            'upload_date': '20140529',
-        },
-        'add_ie': ['Youtube'],
     }, {
         'url': 'https://video.vice.com/en_us/video/the-signal-from-tolva/5816510690b70e6c5fd39a56',
         'info_dict': {
@@ -208,3 +195,67 @@ class ViceShowIE(InfoExtractor):
         description = self._html_search_meta('description', webpage, 'description')
 
         return self.playlist_result(entries, show_id, title, description)
+
+
+class ViceArticleIE(InfoExtractor):
+    _VALID_URL = r'https://www.vice.com/[^/]+/article/(?P<id>[^?#]+)'
+
+    _TESTS = [{
+        'url': 'https://www.vice.com/en_us/article/on-set-with-the-woman-making-mormon-porn-in-utah',
+        'info_dict': {
+            'id': '58dc0a3dee202d2a0ccfcbd8',
+            'ext': 'mp4',
+            'title': 'Mormon War on Porn ',
+            'description': 'md5:ad396a2481e7f8afb5ed486878421090',
+            'uploader': 'VICE',
+            'uploader_id': '57a204088cb727dec794c693',
+            'timestamp': 1489160690,
+            'upload_date': '20170310',
+        },
+        'params': {
+            # AES-encrypted m3u8
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.vice.com/video/how-to-hack-a-car',
+        'md5': 'a7ecf64ee4fa19b916c16f4b56184ae2',
+        'info_dict': {
+            'id': '3jstaBeXgAs',
+            'ext': 'mp4',
+            'title': 'How to Hack a Car: Phreaked Out (Episode 2)',
+            'description': 'md5:ee95453f7ff495db8efe14ae8bf56f30',
+            'uploader_id': 'MotherboardTV',
+            'uploader': 'Motherboard',
+            'upload_date': '20140529',
+        },
+        'add_ie': ['Youtube'],
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        prefetch_data = self._parse_json(self._search_regex(
+            r'window\.__PREFETCH_DATA\s*=\s*({.*});',
+            webpage, 'prefetch data'), display_id)
+        body = prefetch_data['body']
+        youtube_url = self._html_search_regex(
+            r'<iframe[^>]+src="(.*youtube\.com/.*)"', body, 'YouTube URL', default=None)
+        if youtube_url:
+            return {
+                '_type': 'url_transparent',
+                'url': youtube_url,
+                'display_id': display_id,
+                'ie_key': 'Youtube',
+            }
+
+        video_url = self._html_search_regex(
+            r'data-video-url="([^"]+)"', prefetch_data['embed_code'], 'video URL')
+
+        return {
+            '_type': 'url_transparent',
+            'url': video_url,
+            'display_id': display_id,
+            'ie_key': ViceIE.ie_key(),
+        }

From b2ad479d17c435b02fe4f8a02f21bb713c2833f1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 5 May 2017 20:51:59 +0800
Subject: [PATCH 244/611] [utils] Fix multipart_encode for Python < 3.5

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 25bd228ab..911cacd29 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2110,7 +2110,7 @@ def try_multipart_encode(data, boundary):
             v = v.encode('utf-8')
         # RFC 2047 requires non-ASCII field names to be encoded, while RFC 7578
         # suggests sending UTF-8 directly. Firefox sends UTF-8, too
-        content = b'Content-Disposition: form-data; name="%s"\r\n\r\n' % k + v + b'\r\n'
+        content = b'Content-Disposition: form-data; name="' + k + b'"\r\n\r\n' + v + b'\r\n'
         if boundary.encode('ascii') in content:
             raise ValueError('Boundary overlaps with data')
         out += content

From 7ad53cb7ff9afdcffb0b2d027f06f4120dbde9a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 May 2017 21:59:23 +0700
Subject: [PATCH 245/611] [laola1tv] PEP 8

---
 youtube_dl/extractor/laola1tv.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index d9f53c356..1f91ba017 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -10,7 +10,6 @@ from ..utils import (
     urlencode_postdata,
     xpath_element,
     xpath_text,
-    urljoin,
     update_url_query,
     js_to_json,
 )

From 1d9e0a4f40deaeb2f722cf964e6bf13b3835e617 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 5 May 2017 16:12:40 +0100
Subject: [PATCH 246/611] [vice] update tests and add support for ooyala embeds
 in article pages

---
 youtube_dl/extractor/vice.py | 83 +++++++++++++++++++++---------------
 1 file changed, 48 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index b2e95734b..54e207b39 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -32,7 +32,8 @@ class ViceBaseIE(AdobePassIE):
             resource = self._get_mvpd_resource(
                 'VICELAND', title, video_id,
                 watch_hub_data.get('video-rating'))
-            query['tvetoken'] = self._extract_mvpd_auth(url, video_id, 'VICELAND', resource)
+            query['tvetoken'] = self._extract_mvpd_auth(
+                url, video_id, 'VICELAND', resource)
 
         # signature generation algorithm is reverse engineered from signatureGenerator in
         # webpack:///../shared/~/vice-player/dist/js/vice-player.js in
@@ -45,11 +46,14 @@ class ViceBaseIE(AdobePassIE):
 
         try:
             host = 'www.viceland' if is_locked else self._PREPLAY_HOST
-            preplay = self._download_json('https://%s.com/%s/preplay/%s' % (host, locale, video_id), video_id, query=query)
+            preplay = self._download_json(
+                'https://%s.com/%s/preplay/%s' % (host, locale, video_id),
+                video_id, query=query)
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
                 error = json.loads(e.cause.read().decode())
-                raise ExtractorError('%s said: %s' % (self.IE_NAME, error['details']), expected=True)
+                raise ExtractorError('%s said: %s' % (
+                    self.IE_NAME, error['details']), expected=True)
             raise
 
         video_data = preplay['video']
@@ -88,16 +92,17 @@ class ViceBaseIE(AdobePassIE):
 
 
 class ViceIE(ViceBaseIE):
-    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?P<locale>[^/]+)/(?:[^/]+/)?videos?/(?P<id>[^/?#&]+)'
+    IE_NAME = 'vice'
+    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?:(?P<locale>[^/]+)/)?videos?/(?P<id>[^/?#&]+)'
 
     _TESTS = [{
-        'url': 'http://www.vice.com/video/cowboy-capitalists-part-1',
-        'md5': 'e9d77741f9e42ba583e683cd170660f7',
+        'url': 'https://news.vice.com/video/experimenting-on-animals-inside-the-monkey-lab',
+        'md5': '7d3ae2f9ba5f196cdd9f9efd43657ac2',
         'info_dict': {
-            'id': '43cW1mYzpia9IlestBjVpd23Yu3afAfp',
+            'id': 'N2bzkydjraWDGwnt8jAttCF6Y0PDv4Zj',
             'ext': 'flv',
-            'title': 'VICE_COWBOYCAPITALISTS_PART01_v1_VICE_WM_1080p.mov',
-            'duration': 725.983,
+            'title': 'Monkey Labs of Holland',
+            'description': 'md5:92b3c7dcbfe477f772dd4afa496c9149',
         },
         'add_ie': ['Ooyala'],
     }, {
@@ -136,22 +141,13 @@ class ViceIE(ViceBaseIE):
         },
         'add_ie': ['UplynkPreplay'],
     }, {
-        'url': 'https://news.vice.com/video/experimenting-on-animals-inside-the-monkey-lab',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.vice.com/ru/video/big-night-out-ibiza-clive-martin-229',
-        'only_matching': True,
-    }, {
-        'url': 'https://munchies.vice.com/en/videos/watch-the-trailer-for-our-new-series-the-pizza-show',
+        'url': 'https://video.vice.com/en_us/video/pizza-show-trailer/56d8c9a54d286ed92f7f30e4',
         'only_matching': True,
     }]
     _PREPLAY_HOST = 'video.vice'
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        locale = mobj.group('locale')
-        video_id = self._match_id(url)
+        locale, video_id = re.match(self._VALID_URL, url).groups()
         webpage, urlh = self._download_webpage_handle(url, video_id)
         embed_code = self._search_regex(
             r'embedCode=([^&\'"]+)', webpage,
@@ -166,6 +162,7 @@ class ViceIE(ViceBaseIE):
 
 
 class ViceShowIE(InfoExtractor):
+    IE_NAME = 'vice:show'
     _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?:[^/]+/)?show/(?P<id>[^/?#&]+)'
 
     _TEST = {
@@ -192,12 +189,14 @@ class ViceShowIE(InfoExtractor):
             r'<title>(.+?)</title>', webpage, 'title', default=None)
         if title:
             title = re.sub(r'(.+)\s*\|\s*.+$', r'\1', title).strip()
-        description = self._html_search_meta('description', webpage, 'description')
+        description = self._html_search_meta(
+            'description', webpage, 'description')
 
         return self.playlist_result(entries, show_id, title, description)
 
 
 class ViceArticleIE(InfoExtractor):
+    IE_NAME = 'vice:article'
     _VALID_URL = r'https://www.vice.com/[^/]+/article/(?P<id>[^?#]+)'
 
     _TESTS = [{
@@ -216,8 +215,9 @@ class ViceArticleIE(InfoExtractor):
             # AES-encrypted m3u8
             'skip_download': True,
         },
+        'add_ie': ['UplynkPreplay'],
     }, {
-        'url': 'http://www.vice.com/video/how-to-hack-a-car',
+        'url': 'https://www.vice.com/en_us/article/how-to-hack-a-car',
         'md5': 'a7ecf64ee4fa19b916c16f4b56184ae2',
         'info_dict': {
             'id': '3jstaBeXgAs',
@@ -229,6 +229,12 @@ class ViceArticleIE(InfoExtractor):
             'upload_date': '20140529',
         },
         'add_ie': ['Youtube'],
+    }, {
+        'url': 'https://www.vice.com/en_us/article/cowboy-capitalists-part-1',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.vice.com/ru/article/big-night-out-ibiza-clive-martin-229',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -240,22 +246,29 @@ class ViceArticleIE(InfoExtractor):
             r'window\.__PREFETCH_DATA\s*=\s*({.*});',
             webpage, 'prefetch data'), display_id)
         body = prefetch_data['body']
-        youtube_url = self._html_search_regex(
-            r'<iframe[^>]+src="(.*youtube\.com/.*)"', body, 'YouTube URL', default=None)
-        if youtube_url:
+
+        def _url_res(video_url, ie_key):
             return {
                 '_type': 'url_transparent',
-                'url': youtube_url,
+                'url': video_url,
                 'display_id': display_id,
-                'ie_key': 'Youtube',
+                'ie_key': ie_key,
             }
 
-        video_url = self._html_search_regex(
-            r'data-video-url="([^"]+)"', prefetch_data['embed_code'], 'video URL')
+        embed_code = self._search_regex(
+            r'embedCode=([^&\'"]+)', body,
+            'ooyala embed code', default=None)
+        if embed_code:
+            return _url_res('ooyala:%s' % embed_code, 'Ooyala')
 
-        return {
-            '_type': 'url_transparent',
-            'url': video_url,
-            'display_id': display_id,
-            'ie_key': ViceIE.ie_key(),
-        }
+        youtube_url = self._html_search_regex(
+            r'<iframe[^>]+src="(.*youtube\.com/.*)"',
+            body, 'YouTube URL', default=None)
+        if youtube_url:
+            return _url_res(youtube_url, 'Youtube')
+
+        video_url = self._html_search_regex(
+            r'data-video-url="([^"]+)"',
+            prefetch_data['embed_code'], 'video URL')
+
+        return _url_res(video_url, ViceIE.ie_key())

From 329e3dd5adf52520c87ba31395d090455114783b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 May 2017 22:59:15 +0700
Subject: [PATCH 247/611] [nrk] Extract chapters

---
 youtube_dl/extractor/nrk.py | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 7fe79cb53..3b4f51f61 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -148,13 +148,34 @@ class NRKBaseIE(InfoExtractor):
 
         vcodec = 'none' if data.get('mediaType') == 'Audio' else None
 
-        # TODO: extract chapters when https://github.com/rg3/youtube-dl/pull/9409 is merged
-
         for entry in entries:
             entry.update(common_info)
             for f in entry['formats']:
                 f['vcodec'] = vcodec
 
+        points = data.get('shortIndexPoints')
+        if isinstance(points, list):
+            chapters = []
+            for next_num, point in enumerate(points, start=1):
+                if not isinstance(point, dict):
+                    continue
+                start_time = parse_duration(point.get('startPoint'))
+                if start_time is None:
+                    continue
+                end_time = parse_duration(
+                    data.get('duration')
+                    if next_num == len(points)
+                    else points[next_num].get('startPoint'))
+                if end_time is None:
+                    continue
+                chapters.append({
+                    'start_time': start_time,
+                    'end_time': end_time,
+                    'title': point.get('title'),
+                })
+            if chapters and len(entries) == 1:
+                entries[0]['chapters'] = chapters
+
         return self.playlist_result(entries, video_id, title, description)
 
 

From 9cafc3fd8b54b9b91a145cddf9e4db0bd59e1b5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 May 2017 02:27:06 +0700
Subject: [PATCH 248/611] [youtube] Extract chapters

---
 test/test_youtube_chapters.py   | 267 ++++++++++++++++++++++++++++++++
 youtube_dl/extractor/youtube.py |  36 ++++-
 2 files changed, 301 insertions(+), 2 deletions(-)
 create mode 100644 test/test_youtube_chapters.py

diff --git a/test/test_youtube_chapters.py b/test/test_youtube_chapters.py
new file mode 100644
index 000000000..a5d05355f
--- /dev/null
+++ b/test/test_youtube_chapters.py
@@ -0,0 +1,267 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from test.helper import expect_value
+from youtube_dl.extractor import YoutubeIE
+
+
+class TestYoutubeChapters(unittest.TestCase):
+
+    _TEST_CASES = [
+        (
+            # https://www.youtube.com/watch?v=A22oy8dFjqc
+            # pattern: 00:00 - <title>
+            '''This is the absolute ULTIMATE experience of Queen's set at LIVE AID, this is the best video mixed to the absolutely superior stereo radio broadcast. This vastly superior audio mix takes a huge dump on all of the official mixes. Best viewed in 1080p. ENJOY! ***MAKE SURE TO READ THE DESCRIPTION***<br /><a href="#" onclick="yt.www.watch.player.seekTo(00*60+36);return false;">00:36</a> - Bohemian Rhapsody<br /><a href="#" onclick="yt.www.watch.player.seekTo(02*60+42);return false;">02:42</a> - Radio Ga Ga<br /><a href="#" onclick="yt.www.watch.player.seekTo(06*60+53);return false;">06:53</a> - Ay Oh!<br /><a href="#" onclick="yt.www.watch.player.seekTo(07*60+34);return false;">07:34</a> - Hammer To Fall<br /><a href="#" onclick="yt.www.watch.player.seekTo(12*60+08);return false;">12:08</a> - Crazy Little Thing Called Love<br /><a href="#" onclick="yt.www.watch.player.seekTo(16*60+03);return false;">16:03</a> - We Will Rock You<br /><a href="#" onclick="yt.www.watch.player.seekTo(17*60+18);return false;">17:18</a> - We Are The Champions<br /><a href="#" onclick="yt.www.watch.player.seekTo(21*60+12);return false;">21:12</a> - Is This The World We Created...?<br /><br />Short song analysis:<br /><br />- "Bohemian Rhapsody": Although it's a short medley version, it's one of the best performances of the ballad section, with Freddie nailing the Bb4s with the correct studio phrasing (for the first time ever!).<br /><br />- "Radio Ga Ga": Although it's missing one chorus, this is one of - if not the best - the best versions ever, Freddie nails all the Bb4s and sounds very clean! Spike Edney's Roland Jupiter 8 also really shines through on this mix, compared to the DVD releases!<br /><br />- "Audience Improv": A great improv, Freddie sounds strong and confident. You gotta love when he sustains that A4 for 4 seconds!<br /><br />- "Hammer To Fall": Despite missing a verse and a chorus, it's a strong version (possibly the best ever). Freddie sings the song amazingly, and even ad-libs a C#5 and a C5! Also notice how heavy Brian's guitar sounds compared to the thin DVD mixes - it roars!<br /><br />- "Crazy Little Thing Called Love": A great version, the crowd loves the song, the jam is great as well! Only downside to this is the slight feedback issues.<br /><br />- "We Will Rock You": Although cut down to the 1st verse and chorus, Freddie sounds strong. He nails the A4, and the solo from Dr. May is brilliant!<br /><br />- "We Are the Champions": Perhaps the high-light of the performance - Freddie is very daring on this version, he sustains the pre-chorus Bb4s, nails the 1st C5, belts great A4s, but most importantly: He nails the chorus Bb4s, in all 3 choruses! This is the only time he has ever done so! It has to be said though, the last one sounds a bit rough, but that's a side effect of belting high notes for the past 18 minutes, with nodules AND laryngitis!<br /><br />- "Is This The World We Created... ?": Freddie and Brian perform a beautiful version of this, and it is one of the best versions ever. It's both sad and hilarious that a couple of BBC engineers are talking over the song, one of them being completely oblivious of the fact that he is interrupting the performance, on live television... Which was being televised to almost 2 billion homes.<br /><br /><br />All rights go to their respective owners!<br />-----Copyright Disclaimer Under Section 107 of the Copyright Act 1976, allowance is made for fair use for purposes such as criticism, comment, news reporting, teaching, scholarship, and research. Fair use is a use permitted by copyright statute that might otherwise be infringing. Non-profit, educational or personal use tips the balance in favor of fair use''',
+            1477,
+            [{
+                'start_time': 36,
+                'end_time': 162,
+                'title': 'Bohemian Rhapsody',
+            }, {
+                'start_time': 162,
+                'end_time': 413,
+                'title': 'Radio Ga Ga',
+            }, {
+                'start_time': 413,
+                'end_time': 454,
+                'title': 'Ay Oh!',
+            }, {
+                'start_time': 454,
+                'end_time': 728,
+                'title': 'Hammer To Fall',
+            }, {
+                'start_time': 728,
+                'end_time': 963,
+                'title': 'Crazy Little Thing Called Love',
+            }, {
+                'start_time': 963,
+                'end_time': 1038,
+                'title': 'We Will Rock You',
+            }, {
+                'start_time': 1038,
+                'end_time': 1272,
+                'title': 'We Are The Champions',
+            }, {
+                'start_time': 1272,
+                'end_time': 1477,
+                'title': 'Is This The World We Created...?',
+            }]
+        ),
+        (
+            # https://www.youtube.com/watch?v=ekYlRhALiRQ
+            # pattern: <num>. <title> 0:00
+            '1.  Those Beaten Paths of Confusion <a href="#" onclick="yt.www.watch.player.seekTo(0*60+00);return false;">0:00</a><br />2.  Beyond the Shadows of Emptiness & Nothingness <a href="#" onclick="yt.www.watch.player.seekTo(11*60+47);return false;">11:47</a><br />3.  Poison Yourself...With Thought <a href="#" onclick="yt.www.watch.player.seekTo(26*60+30);return false;">26:30</a><br />4.  The Agents of Transformation <a href="#" onclick="yt.www.watch.player.seekTo(35*60+57);return false;">35:57</a><br />5.  Drowning in the Pain of Consciousness <a href="#" onclick="yt.www.watch.player.seekTo(44*60+32);return false;">44:32</a><br />6.  Deny the Disease of Life <a href="#" onclick="yt.www.watch.player.seekTo(53*60+07);return false;">53:07</a><br /><br />More info/Buy: http://crepusculonegro.storenvy.com/products/257645-cn-03-arizmenda-within-the-vacuum-of-infinity<br /><br />No copyright is intended. The rights to this video are assumed by the owner and its affiliates.',
+            4009,
+            [{
+                'start_time': 0,
+                'end_time': 707,
+                'title': '1. Those Beaten Paths of Confusion',
+            }, {
+                'start_time': 707,
+                'end_time': 1590,
+                'title': '2. Beyond the Shadows of Emptiness & Nothingness',
+            }, {
+                'start_time': 1590,
+                'end_time': 2157,
+                'title': '3. Poison Yourself...With Thought',
+            }, {
+                'start_time': 2157,
+                'end_time': 2672,
+                'title': '4. The Agents of Transformation',
+            }, {
+                'start_time': 2672,
+                'end_time': 3187,
+                'title': '5. Drowning in the Pain of Consciousness',
+            }, {
+                'start_time': 3187,
+                'end_time': 4009,
+                'title': '6. Deny the Disease of Life',
+            }]
+        ),
+        (
+            # https://www.youtube.com/watch?v=WjL4pSzog9w
+            # pattern: 00:00 <title>
+            '<a href="https://arizmenda.bandcamp.com/merch/despairs-depths-descended-cd" class="yt-uix-servicelink  " data-target-new-window="True" data-servicelink="CDAQ6TgYACITCNf1raqT2dMCFdRjGAod_o0CBSj4HQ" data-url="https://arizmenda.bandcamp.com/merch/despairs-depths-descended-cd" rel="nofollow noopener" target="_blank">https://arizmenda.bandcamp.com/merch/...</a><br /><br /><a href="#" onclick="yt.www.watch.player.seekTo(00*60+00);return false;">00:00</a> Christening Unborn Deformities <br /><a href="#" onclick="yt.www.watch.player.seekTo(07*60+08);return false;">07:08</a> Taste of Purity<br /><a href="#" onclick="yt.www.watch.player.seekTo(16*60+16);return false;">16:16</a> Sculpting Sins of a Universal Tongue<br /><a href="#" onclick="yt.www.watch.player.seekTo(24*60+45);return false;">24:45</a> Birth<br /><a href="#" onclick="yt.www.watch.player.seekTo(31*60+24);return false;">31:24</a> Neves<br /><a href="#" onclick="yt.www.watch.player.seekTo(37*60+55);return false;">37:55</a> Libations in Limbo',
+            2705,
+            [{
+                'start_time': 0,
+                'end_time': 428,
+                'title': 'Christening Unborn Deformities',
+            }, {
+                'start_time': 428,
+                'end_time': 976,
+                'title': 'Taste of Purity',
+            }, {
+                'start_time': 976,
+                'end_time': 1485,
+                'title': 'Sculpting Sins of a Universal Tongue',
+            }, {
+                'start_time': 1485,
+                'end_time': 1884,
+                'title': 'Birth',
+            }, {
+                'start_time': 1884,
+                'end_time': 2275,
+                'title': 'Neves',
+            }, {
+                'start_time': 2275,
+                'end_time': 2705,
+                'title': 'Libations in Limbo',
+            }]
+        ),
+        (
+            # https://www.youtube.com/watch?v=o3r1sn-t3is
+            # pattern: <title> 00:00 <note>
+            'Download this show in MP3: <a href="http://sh.st/njZKK" class="yt-uix-servicelink  " data-url="http://sh.st/njZKK" data-target-new-window="True" data-servicelink="CDAQ6TgYACITCK3j8_6o2dMCFVDCGAoduVAKKij4HQ" rel="nofollow noopener" target="_blank">http://sh.st/njZKK</a><br /><br />Setlist:<br />I-E-A-I-A-I-O <a href="#" onclick="yt.www.watch.player.seekTo(00*60+45);return false;">00:45</a><br />Suite-Pee <a href="#" onclick="yt.www.watch.player.seekTo(4*60+26);return false;">4:26</a>  (Incomplete)<br />Attack <a href="#" onclick="yt.www.watch.player.seekTo(5*60+31);return false;">5:31</a> (First live performance since 2011)<br />Prison Song <a href="#" onclick="yt.www.watch.player.seekTo(8*60+42);return false;">8:42</a><br />Know <a href="#" onclick="yt.www.watch.player.seekTo(12*60+32);return false;">12:32</a> (First live performance since 2011)<br />Aerials <a href="#" onclick="yt.www.watch.player.seekTo(15*60+32);return false;">15:32</a><br />Soldier Side - Intro <a href="#" onclick="yt.www.watch.player.seekTo(19*60+13);return false;">19:13</a><br />B.Y.O.B. <a href="#" onclick="yt.www.watch.player.seekTo(20*60+09);return false;">20:09</a><br />Soil <a href="#" onclick="yt.www.watch.player.seekTo(24*60+32);return false;">24:32</a><br />Darts <a href="#" onclick="yt.www.watch.player.seekTo(27*60+48);return false;">27:48</a><br />Radio/Video <a href="#" onclick="yt.www.watch.player.seekTo(30*60+38);return false;">30:38</a><br />Hypnotize <a href="#" onclick="yt.www.watch.player.seekTo(35*60+05);return false;">35:05</a><br />Temper <a href="#" onclick="yt.www.watch.player.seekTo(38*60+08);return false;">38:08</a> (First live performance since 1999)<br />CUBErt <a href="#" onclick="yt.www.watch.player.seekTo(41*60+00);return false;">41:00</a><br />Needles <a href="#" onclick="yt.www.watch.player.seekTo(42*60+57);return false;">42:57</a><br />Deer Dance <a href="#" onclick="yt.www.watch.player.seekTo(46*60+27);return false;">46:27</a><br />Bounce <a href="#" onclick="yt.www.watch.player.seekTo(49*60+38);return false;">49:38</a><br />Suggestions <a href="#" onclick="yt.www.watch.player.seekTo(51*60+25);return false;">51:25</a><br />Psycho <a href="#" onclick="yt.www.watch.player.seekTo(53*60+52);return false;">53:52</a><br />Chop Suey! <a href="#" onclick="yt.www.watch.player.seekTo(58*60+13);return false;">58:13</a><br />Lonely Day <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+01*60+15);return false;">1:01:15</a><br />Question! <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+04*60+14);return false;">1:04:14</a><br />Lost in Hollywood <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+08*60+10);return false;">1:08:10</a><br />Vicinity of Obscenity  <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+13*60+40);return false;">1:13:40</a>(First live performance since 2012)<br />Forest <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+16*60+17);return false;">1:16:17</a><br />Cigaro <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+20*60+02);return false;">1:20:02</a><br />Toxicity <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+23*60+57);return false;">1:23:57</a>(with Chino Moreno)<br />Sugar <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+27*60+53);return false;">1:27:53</a>',
+            5640,
+            [{
+                'start_time': 45,
+                'end_time': 266,
+                'title': 'I-E-A-I-A-I-O',
+            }, {
+                'start_time': 266,
+                'end_time': 331,
+                'title': 'Suite-Pee (Incomplete)',
+            }, {
+                'start_time': 331,
+                'end_time': 522,
+                'title': 'Attack (First live performance since 2011)',
+            }, {
+                'start_time': 522,
+                'end_time': 752,
+                'title': 'Prison Song',
+            }, {
+                'start_time': 752,
+                'end_time': 932,
+                'title': 'Know (First live performance since 2011)',
+            }, {
+                'start_time': 932,
+                'end_time': 1153,
+                'title': 'Aerials',
+            }, {
+                'start_time': 1153,
+                'end_time': 1209,
+                'title': 'Soldier Side - Intro',
+            }, {
+                'start_time': 1209,
+                'end_time': 1472,
+                'title': 'B.Y.O.B.',
+            }, {
+                'start_time': 1472,
+                'end_time': 1668,
+                'title': 'Soil',
+            }, {
+                'start_time': 1668,
+                'end_time': 1838,
+                'title': 'Darts',
+            }, {
+                'start_time': 1838,
+                'end_time': 2105,
+                'title': 'Radio/Video',
+            }, {
+                'start_time': 2105,
+                'end_time': 2288,
+                'title': 'Hypnotize',
+            }, {
+                'start_time': 2288,
+                'end_time': 2460,
+                'title': 'Temper (First live performance since 1999)',
+            }, {
+                'start_time': 2460,
+                'end_time': 2577,
+                'title': 'CUBErt',
+            }, {
+                'start_time': 2577,
+                'end_time': 2787,
+                'title': 'Needles',
+            }, {
+                'start_time': 2787,
+                'end_time': 2978,
+                'title': 'Deer Dance',
+            }, {
+                'start_time': 2978,
+                'end_time': 3085,
+                'title': 'Bounce',
+            }, {
+                'start_time': 3085,
+                'end_time': 3232,
+                'title': 'Suggestions',
+            }, {
+                'start_time': 3232,
+                'end_time': 3493,
+                'title': 'Psycho',
+            }, {
+                'start_time': 3493,
+                'end_time': 3675,
+                'title': 'Chop Suey!',
+            }, {
+                'start_time': 3675,
+                'end_time': 3854,
+                'title': 'Lonely Day',
+            }, {
+                'start_time': 3854,
+                'end_time': 4090,
+                'title': 'Question!',
+            }, {
+                'start_time': 4090,
+                'end_time': 4420,
+                'title': 'Lost in Hollywood',
+            }, {
+                'start_time': 4420,
+                'end_time': 4577,
+                'title': 'Vicinity of Obscenity (First live performance since 2012)',
+            }, {
+                'start_time': 4577,
+                'end_time': 4802,
+                'title': 'Forest',
+            }, {
+                'start_time': 4802,
+                'end_time': 5037,
+                'title': 'Cigaro',
+            }, {
+                'start_time': 5037,
+                'end_time': 5273,
+                'title': 'Toxicity (with Chino Moreno)',
+            }, {
+                'start_time': 5273,
+                'end_time': 5640,
+                'title': 'Sugar',
+            }]
+        ),
+        (
+            # https://www.youtube.com/watch?v=PkYLQbsqCE8
+            # pattern: <num> - <title> [<latinized title>] 0:00:00
+            '''Затемно (Zatemno) is an Obscure Black Metal Band from Russia.<br /><br />"Во прах (Vo prakh)'' Into The Ashes", Debut mini-album released may 6, 2016, by Death Knell Productions<br />Released on 6 panel digipak CD, limited to 100 copies only<br />And digital format on Bandcamp<br /><br />Tracklist<br /><br />1 - Во прах [Vo prakh] <a href="#" onclick="yt.www.watch.player.seekTo(0*3600+00*60+00);return false;">0:00:00</a><br />2 - Искупление [Iskupleniye] <a href="#" onclick="yt.www.watch.player.seekTo(0*3600+08*60+10);return false;">0:08:10</a><br />3 - Из серпов луны...[Iz serpov luny] <a href="#" onclick="yt.www.watch.player.seekTo(0*3600+14*60+30);return false;">0:14:30</a><br /><br />Links:<br /><a href="https://deathknellprod.bandcamp.com/album/--2" class="yt-uix-servicelink  " data-target-new-window="True" data-url="https://deathknellprod.bandcamp.com/album/--2" data-servicelink="CC8Q6TgYACITCNP234Kr2dMCFcNxGAodQqsIwSj4HQ" target="_blank" rel="nofollow noopener">https://deathknellprod.bandcamp.com/a...</a><br /><a href="https://www.facebook.com/DeathKnellProd/" class="yt-uix-servicelink  " data-target-new-window="True" data-url="https://www.facebook.com/DeathKnellProd/" data-servicelink="CC8Q6TgYACITCNP234Kr2dMCFcNxGAodQqsIwSj4HQ" target="_blank" rel="nofollow noopener">https://www.facebook.com/DeathKnellProd/</a><br /><br /><br />I don't have any right about this artifact, my only intention is to spread the music of the band, all rights are reserved to the Затемно (Zatemno) and his producers, Death Knell Productions.<br /><br />------------------------------------------------------------------<br /><br />Subscribe for more videos like this.<br />My link: <a href="https://web.facebook.com/AttackOfTheDragons" class="yt-uix-servicelink  " data-target-new-window="True" data-url="https://web.facebook.com/AttackOfTheDragons" data-servicelink="CC8Q6TgYACITCNP234Kr2dMCFcNxGAodQqsIwSj4HQ" target="_blank" rel="nofollow noopener">https://web.facebook.com/AttackOfTheD...</a>''',
+            1138,
+            [{
+                'start_time': 0,
+                'end_time': 490,
+                'title': '1 - Во прах [Vo prakh]',
+            }, {
+                'start_time': 490,
+                'end_time': 870,
+                'title': '2 - Искупление [Iskupleniye]',
+            }, {
+                'start_time': 870,
+                'end_time': 1138,
+                'title': '3 - Из серпов луны...[Iz serpov luny]',
+            }]
+        ),
+    ]
+
+    def test_youtube_chapters(self):
+        for description, duration, expected_chapters in self._TEST_CASES:
+            ie = YoutubeIE()
+            expect_value(
+                self, ie._extract_chapters(description, duration),
+                expected_chapters, None)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 480f403da..ec12e313c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1257,6 +1257,35 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
         return self._download_webpage(url, video_id, note='Searching for annotations.', errnote='Unable to download video annotations.')
 
+    @staticmethod
+    def _extract_chapters(description, duration):
+        if not description:
+            return None
+        chapter_lines = re.findall(
+            r'(?:^|<br\s*/>)([^<]*<a[^>]+onclick=["\']yt\.www\.watch\.player\.seekTo[^>]+>(\d{1,2}:\d{1,2}(?::\d{1,2})?)</a>[^>]*)(?=$|<br\s*/>)',
+            description)
+        if not chapter_lines:
+            return None
+        chapters = []
+        for next_num, (chapter_line, time_point) in enumerate(
+                chapter_lines, start=1):
+            start_time = parse_duration(time_point)
+            if start_time is None:
+                continue
+            end_time = (duration if next_num == len(chapter_lines)
+                        else parse_duration(chapter_lines[next_num][1]))
+            if end_time is None:
+                continue
+            chapter_title = re.sub(
+                r'<a[^>]+>[^<]+</a>', '', chapter_line).strip(' \t-')
+            chapter_title = re.sub(r'\s+', ' ', chapter_title)
+            chapters.append({
+                'start_time': start_time,
+                'end_time': end_time,
+                'title': chapter_title,
+            })
+        return chapters
+
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})
 
@@ -1399,9 +1428,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             video_title = '_'
 
         # description
-        video_description = get_element_by_id("eow-description", video_webpage)
+        description_original = video_description = get_element_by_id("eow-description", video_webpage)
         if video_description:
-            video_description = re.sub(r'''(?x)
+            description_original = video_description = re.sub(r'''(?x)
                 <a\s+
                     (?:[a-zA-Z-]+="[^"]*"\s+)*?
                     (?:title|href)="([^"]+)"\s+
@@ -1558,6 +1587,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         if self._downloader.params.get('writeannotations', False):
             video_annotations = self._extract_annotations(video_id)
 
+        chapters = self._extract_chapters(description_original, video_duration)
+
         if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
             self.report_rtmp_download()
             formats = [{
@@ -1790,6 +1821,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'duration': video_duration,
             'age_limit': 18 if age_gate else 0,
             'annotations': video_annotations,
+            'chapters': chapters,
             'webpage_url': proto + '://www.youtube.com/watch?v=%s' % video_id,
             'view_count': view_count,
             'like_count': like_count,

From a99cc4ca167e3244f068737110168716352dbbc1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 May 2017 02:46:37 +0700
Subject: [PATCH 249/611] [pornhub] Extend _VALID_URL (closes #12996)

---
 youtube_dl/extractor/pornhub.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index b25f1f193..1dcc8df00 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -33,7 +33,7 @@ class PornHubIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
-                            (?:[a-z]+\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)|
+                            (?:[a-z]+\.)?pornhub\.com/(?:(?:view_video\.php|video/show)\?viewkey=|embed/)|
                             (?:www\.)?thumbzilla\.com/video/
                         )
                         (?P<id>[\da-z]+)
@@ -97,6 +97,9 @@ class PornHubIE(InfoExtractor):
     }, {
         'url': 'https://www.thumbzilla.com/video/ph56c6114abd99a/horny-girlfriend-sex',
         'only_matching': True,
+    }, {
+        'url': 'http://www.pornhub.com/video/show?viewkey=648719015',
+        'only_matching': True,
     }]
 
     @staticmethod

From 8b4774dcac4cb82b47213126f3430ac9dc25ec28 Mon Sep 17 00:00:00 2001
From: Luca Steeb <contact@luca-steeb.com>
Date: Fri, 5 May 2017 22:35:42 +0200
Subject: [PATCH 250/611] [bandcamp] Fix thumbnail extraction

---
 youtube_dl/extractor/bandcamp.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index df2972f26..489d0ba53 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -47,6 +47,7 @@ class BandcampIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         title = mobj.group('title')
         webpage = self._download_webpage(url, title)
+        thumbnail = self._html_search_meta('og:image', webpage, default=None)
         m_download = re.search(r'freeDownloadPage: "(.*?)"', webpage)
         if not m_download:
             m_trackinfo = re.search(r'trackinfo: (.+),\s*?\n', webpage)
@@ -75,6 +76,7 @@ class BandcampIE(InfoExtractor):
                 return {
                     'id': track_id,
                     'title': data['title'],
+                    'thumbnail': thumbnail,
                     'formats': formats,
                     'duration': float_or_none(data.get('duration')),
                 }
@@ -143,7 +145,7 @@ class BandcampIE(InfoExtractor):
         return {
             'id': video_id,
             'title': title,
-            'thumbnail': info.get('thumb_url'),
+            'thumbnail': info.get('thumb_url') or thumbnail,
             'uploader': info.get('artist'),
             'artist': artist,
             'track': track,

From 1f9fefe7f578169d41d75324decbd46ec8594311 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Fri, 5 May 2017 22:39:14 +0200
Subject: [PATCH 251/611] [crackle] Update test

---
 youtube_dl/extractor/crackle.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/crackle.py b/youtube_dl/extractor/crackle.py
index f919ed208..13f425b2b 100644
--- a/youtube_dl/extractor/crackle.py
+++ b/youtube_dl/extractor/crackle.py
@@ -21,9 +21,10 @@ class CrackleIE(InfoExtractor):
             'season_number': 8,
             'episode_number': 4,
             'subtitles': {
-                'en-US': [{
-                    'ext': 'ttml',
-                }]
+                'en-US': [
+                    {'ext': 'vtt'},
+                    {'ext': 'tt'},
+                ]
             },
         },
         'params': {

From a57a8e991860fb96e6111769ab0a22455c8854fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 May 2017 05:30:56 +0700
Subject: [PATCH 252/611] [test_youtube_chapters] Add coding cookie

---
 test/test_youtube_chapters.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/test_youtube_chapters.py b/test/test_youtube_chapters.py
index a5d05355f..5435ee4a8 100644
--- a/test/test_youtube_chapters.py
+++ b/test/test_youtube_chapters.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 #!/usr/bin/env python
 from __future__ import unicode_literals
 

From fd178b8748979ab19f1cb3d7d689fe19a447ca2f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 6 May 2017 07:19:07 +0100
Subject: [PATCH 253/611] [theplatform] extract chapters

---
 youtube_dl/extractor/theplatform.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 9a424b1c6..de236bbba 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -80,14 +80,33 @@ class ThePlatformBaseIE(OnceIE):
                     'url': src,
                 })
 
+        duration = info.get('duration')
+        tp_chapters = info.get('chapters', [])
+        chapters = []
+        if tp_chapters:
+            def _add_chapter(start_time, end_time):
+                start_time = float_or_none(start_time, 1000)
+                end_time = float_or_none(end_time, 1000)
+                if start_time is None or end_time is None:
+                    return
+                chapters.append({
+                    'start_time': start_time,
+                    'end_time': end_time,
+                })
+
+            for chapter in tp_chapters[:-1]:
+                _add_chapter(chapter.get('startTime'), chapter.get('endTime'))
+            _add_chapter(tp_chapters[-1].get('startTime'), tp_chapters[-1].get('endTime') or duration)
+
         return {
             'title': info['title'],
             'subtitles': subtitles,
             'description': info['description'],
             'thumbnail': info['defaultThumbnailUrl'],
-            'duration': int_or_none(info.get('duration'), 1000),
+            'duration': float_or_none(duration, 1000),
             'timestamp': int_or_none(info.get('pubDate'), 1000) or None,
             'uploader': info.get('billingCode'),
+            'chapters': chapters,
         }
 
     def _extract_theplatform_metadata(self, path, video_id):

From 74c09c852a183813174803306176a86d449da889 Mon Sep 17 00:00:00 2001
From: midas02 <midas02@users.noreply.github.com>
Date: Sun, 30 Apr 2017 19:36:44 +0200
Subject: [PATCH 254/611] [rmcdecouverte] Fix extraction

---
 youtube_dl/extractor/rmcdecouverte.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/rmcdecouverte.py b/youtube_dl/extractor/rmcdecouverte.py
index 2340dae53..f6ef6d747 100644
--- a/youtube_dl/extractor/rmcdecouverte.py
+++ b/youtube_dl/extractor/rmcdecouverte.py
@@ -13,15 +13,15 @@ class RMCDecouverteIE(InfoExtractor):
     _VALID_URL = r'https?://rmcdecouverte\.bfmtv\.com/mediaplayer-replay.*?\bid=(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://rmcdecouverte.bfmtv.com/mediaplayer-replay/?id=1430&title=LES%20HEROS%20DU%2088e%20ETAGE',
+        'url': 'http://rmcdecouverte.bfmtv.com/mediaplayer-replay/?id=16548',
         'info_dict': {
-            'id': '5111223049001',
+            'id': '5411254766001',
             'ext': 'mp4',
-            'title': ': LES HEROS DU 88e ETAGE',
-            'description': 'Découvrez comment la bravoure de deux hommes dans la Tour Nord du World Trade Center a sauvé  la vie d\'innombrables personnes le 11 septembre 2001.',
+            'title': '39/45:LE RESEAU DES FAUX BILLETS',
+            'description': 'ic Brunet propose un nouvel \u00e9pisode des Grains de sable de l\'Histoire sur la plus grosse affaire de contrefa\u00e7on de la Seconde Guerre mondiale.',
             'uploader_id': '1969646226001',
-            'upload_date': '20160904',
-            'timestamp': 1472951103,
+            'upload_date': '20170426',
+            'timestamp': 1493166610,
         },
         'params': {
             # rtmp download
@@ -35,5 +35,8 @@ class RMCDecouverteIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         brightcove_legacy_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
-        brightcove_id = compat_parse_qs(compat_urlparse.urlparse(brightcove_legacy_url).query)['@videoPlayer'][0]
+        if brightcove_legacy_url:
+            brightcove_id = compat_parse_qs(compat_urlparse.urlparse(brightcove_legacy_url).query)['@videoPlayer'][0]
+        else:
+            brightcove_id = self._search_regex(r'data-video-id="(.*?)"', webpage, 'brightcove_id')
         return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)

From 566fbbaefd459df46ebce512cd916248e069e4f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 May 2017 17:55:32 +0700
Subject: [PATCH 255/611] [rmcdecouverte] Improve (closes #12937)

---
 youtube_dl/extractor/rmcdecouverte.py | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/rmcdecouverte.py b/youtube_dl/extractor/rmcdecouverte.py
index f6ef6d747..e921ca3e6 100644
--- a/youtube_dl/extractor/rmcdecouverte.py
+++ b/youtube_dl/extractor/rmcdecouverte.py
@@ -13,21 +13,20 @@ class RMCDecouverteIE(InfoExtractor):
     _VALID_URL = r'https?://rmcdecouverte\.bfmtv\.com/mediaplayer-replay.*?\bid=(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://rmcdecouverte.bfmtv.com/mediaplayer-replay/?id=16548',
+        'url': 'http://rmcdecouverte.bfmtv.com/mediaplayer-replay/?id=13502&title=AQUAMEN:LES%20ROIS%20DES%20AQUARIUMS%20:UN%20DELICIEUX%20PROJET',
         'info_dict': {
-            'id': '5411254766001',
+            'id': '5419055995001',
             'ext': 'mp4',
-            'title': '39/45:LE RESEAU DES FAUX BILLETS',
-            'description': 'ic Brunet propose un nouvel \u00e9pisode des Grains de sable de l\'Histoire sur la plus grosse affaire de contrefa\u00e7on de la Seconde Guerre mondiale.',
+            'title': 'UN DELICIEUX PROJET',
+            'description': 'md5:63610df7c8b1fc1698acd4d0d90ba8b5',
             'uploader_id': '1969646226001',
-            'upload_date': '20170426',
-            'timestamp': 1493166610,
+            'upload_date': '20170502',
+            'timestamp': 1493745308,
         },
         'params': {
-            # rtmp download
             'skip_download': True,
         },
-        'skip': 'Only works from France',
+        'skip': 'only available for a week',
     }
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1969646226001/default_default/index.html?videoId=%s'
 
@@ -36,7 +35,11 @@ class RMCDecouverteIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
         brightcove_legacy_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
         if brightcove_legacy_url:
-            brightcove_id = compat_parse_qs(compat_urlparse.urlparse(brightcove_legacy_url).query)['@videoPlayer'][0]
+            brightcove_id = compat_parse_qs(compat_urlparse.urlparse(
+                brightcove_legacy_url).query)['@videoPlayer'][0]
         else:
-            brightcove_id = self._search_regex(r'data-video-id="(.*?)"', webpage, 'brightcove_id')
-        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)
+            brightcove_id = self._search_regex(
+                r'data-video-id=["\'](\d+)', webpage, 'brightcove id')
+        return self.url_result(
+            self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew',
+            brightcove_id)

From 228cd9bb906eb1dae476a05dbff0796a2d9f9be5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 6 May 2017 18:57:24 +0800
Subject: [PATCH 256/611] [bilibili] Fix video downloading (closes #13001)

---
 ChangeLog                        | 1 +
 youtube_dl/extractor/bilibili.py | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 0908e2e93..7cceec10f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [bilibili] Fix video downloading (#13001)
 + [vice] Support vice articles (#12968)
 * [vice] Fix extraction for non en_us videos (#12967)
 * [gdcvault] Fix extraction for some videos (#12733)
diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 80dd8382e..1e3f25515 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -122,6 +122,11 @@ class BiliBiliIE(InfoExtractor):
                     'preference': -2 if 'hd.mp4' in backup_url else -3,
                 })
 
+            for a_format in formats:
+                a_format.setdefault('http_headers', {}).update({
+                    'Referer': url,
+                })
+
             self._sort_formats(formats)
 
             entries.append({

From 10c87c151b6f6a2f45b1a3ad1ac2d38fcf0fb0a6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 6 May 2017 19:06:18 +0800
Subject: [PATCH 257/611] [utils] Rename try_multipart_encode to
 _multipart_encode_impl

To state that this is an internal function and people should be careful
when using it outside youtube-dl.
---
 youtube_dl/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 911cacd29..aa1900436 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2098,7 +2098,7 @@ def update_Request(req, url=None, data=None, headers={}, query={}):
     return new_req
 
 
-def try_multipart_encode(data, boundary):
+def _multipart_encode_impl(data, boundary):
     content_type = 'multipart/form-data; boundary=%s' % boundary
 
     out = b''
@@ -2140,7 +2140,7 @@ def multipart_encode(data, boundary=None):
             boundary = '---------------' + str(random.randrange(0x0fffffff, 0xffffffff))
 
         try:
-            out, content_type = try_multipart_encode(data, boundary)
+            out, content_type = _multipart_encode_impl(data, boundary)
             break
         except ValueError:
             if has_specified_boundary:

From e00eb564e921b609c94f87d7db7ad7f709b704f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 May 2017 23:58:47 +0700
Subject: [PATCH 258/611] [youtube] Fix authentication (closes #12927)

---
 youtube_dl/extractor/youtube.py | 142 +++++++++++++++++---------------
 1 file changed, 77 insertions(+), 65 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ec12e313c..c2e06c3a6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -37,8 +37,7 @@ from ..utils import (
     parse_codecs,
     parse_duration,
     remove_quotes,
-    remove_start,
-    sanitized_Request,
+    # remove_start,
     smuggle_url,
     str_to_int,
     try_get,
@@ -54,7 +53,16 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
     """Provide base functions for Youtube extractors"""
     _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
     _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge'
-    _PASSWORD_CHALLENGE_URL = 'https://accounts.google.com/signin/challenge/sl/password'
+
+    _LOOKUP_URL = 'https://accounts.google.com/_/signin/sl/lookup'
+    _LOOKUP_REQ_TEMPLATE = '["{0}",null,[],null,"US",null,null,2,false,true,[null,null,[2,1,null,1,"https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn",null,[],4],1,[null,null,[]],null,null,null,true],"{0}"]'
+
+    _PASSWORD_CHALLENGE_URL = 'https://accounts.google.com/_/signin/sl/challenge'
+    _PASSWORD_CHALLENGE_REQ_TEMPLATE = '["{0}",null,1,null,[1,null,null,null,["{1}",null,true]],[null,null,[2,1,null,1,"https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn",null,[],4],1,[null,null,[]],null,null,null,true]]'
+
+    _TFA_URL = 'https://accounts.google.com/_/signin/challenge'
+    _TFA_REQ_TEMPLATE = '["{0}",null,2,null,[9,null,null,null,null,null,null,null,[null,"{1}",false,2]]]'
+
     _NETRC_MACHINE = 'youtube'
     # If True it will raise an error if no login info is provided
     _LOGIN_REQUIRED = False
@@ -96,72 +104,76 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
 
         login_form = self._hidden_inputs(login_page)
 
-        login_form.update({
-            'checkConnection': 'youtube',
-            'Email': username,
-            'Passwd': password,
-        })
-
-        login_results = self._download_webpage(
-            self._PASSWORD_CHALLENGE_URL, None,
-            note='Logging in', errnote='unable to log in', fatal=False,
-            data=urlencode_postdata(login_form))
-        if login_results is False:
-            return False
-
-        error_msg = self._html_search_regex(
-            r'<[^>]+id="errormsg_0_Passwd"[^>]*>([^<]+)<',
-            login_results, 'error message', default=None)
-        if error_msg:
-            raise ExtractorError('Unable to login: %s' % error_msg, expected=True)
-
-        if re.search(r'id="errormsg_0_Passwd"', login_results) is not None:
-            raise ExtractorError('Please use your account password and a two-factor code instead of an application-specific password.', expected=True)
-
-        # Two-Factor
-        # TODO add SMS and phone call support - these require making a request and then prompting the user
-
-        if re.search(r'(?i)<form[^>]+id="challenge"', login_results) is not None:
-            tfa_code = self._get_tfa_info('2-step verification code')
-
-            if not tfa_code:
-                self._downloader.report_warning(
-                    'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
-                    '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
-                return False
-
-            tfa_code = remove_start(tfa_code, 'G-')
-
-            tfa_form_strs = self._form_hidden_inputs('challenge', login_results)
-
-            tfa_form_strs.update({
-                'Pin': tfa_code,
-                'TrustDevice': 'on',
+        def req(url, f_req, note, errnote):
+            data = login_form.copy()
+            data.update({
+                'pstMsg': 1,
+                'checkConnection': 'youtube',
+                'checkedDomains': 'youtube',
+                'hl': 'en',
+                'deviceinfo': '[null,null,null,[],null,"US",null,null,[],"GlifWebSignIn",null,[null,null,[]]]',
+                'f.req': f_req,
+                'flowName': 'GlifWebSignIn',
+                'flowEntry': 'ServiceLogin',
             })
+            return self._download_json(
+                url, None, note=note, errnote=errnote,
+                transform_source=lambda s: re.sub(r'^[^[]*', '', s),
+                fatal=False,
+                data=urlencode_postdata(data), headers={
+                    'Content-Type': 'application/x-www-form-urlencoded;charset=utf-8',
+                    'Google-Accounts-XSRF': 1,
+                })
 
-            tfa_data = urlencode_postdata(tfa_form_strs)
+        lookup_results = req(
+            self._LOOKUP_URL, self._LOOKUP_REQ_TEMPLATE.format(username),
+            'Looking up account info', 'Unable to look up account info')
 
-            tfa_req = sanitized_Request(self._TWOFACTOR_URL, tfa_data)
-            tfa_results = self._download_webpage(
-                tfa_req, None,
-                note='Submitting TFA code', errnote='unable to submit tfa', fatal=False)
-
-            if tfa_results is False:
-                return False
-
-            if re.search(r'(?i)<form[^>]+id="challenge"', tfa_results) is not None:
-                self._downloader.report_warning('Two-factor code expired or invalid. Please try again, or use a one-use backup code instead.')
-                return False
-            if re.search(r'(?i)<form[^>]+id="gaia_loginform"', tfa_results) is not None:
-                self._downloader.report_warning('unable to log in - did the page structure change?')
-                return False
-            if re.search(r'smsauth-interstitial-reviewsettings', tfa_results) is not None:
-                self._downloader.report_warning('Your Google account has a security notice. Please log in on your web browser, resolve the notice, and try again.')
-                return False
-
-        if re.search(r'(?i)<form[^>]+id="gaia_loginform"', login_results) is not None:
-            self._downloader.report_warning('unable to log in: bad username or password')
+        if lookup_results is False:
             return False
+
+        user_hash = lookup_results[0][2]
+
+        password_challenge_results = req(
+            self._PASSWORD_CHALLENGE_URL,
+            self._PASSWORD_CHALLENGE_REQ_TEMPLATE.format(user_hash, password),
+            'Logging in', 'Unable to log in')[0]
+
+        if password_challenge_results is False:
+            return
+
+        msg = password_challenge_results[5]
+        if msg is not None and isinstance(msg, list):
+            raise ExtractorError('Unable to login: %s' % msg[5], expected=True)
+
+        password_challenge_results = password_challenge_results[-1]
+
+        # tfa = password_challenge_results[0]
+        # if isinstance(tfa, list) and tfa[0][2] == 'TWO_STEP_VERIFICATION':
+        #     tfa_code = self._get_tfa_info('2-step verification code')
+        #
+        #     if not tfa_code:
+        #         self._downloader.report_warning(
+        #             'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
+        #             '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
+        #         return False
+        #
+        #     tfa_code = remove_start(tfa_code, 'G-')
+        #     print('tfa', tfa_code)
+        #     tfa_results = req(
+        #         self._TFA_URL,
+        #         self._TFA_REQ_TEMPLATE.format(user_hash, tfa_code),
+        #         'Submitting TFA code', 'Unable to submit TFA code')
+        #
+        #     TODO
+
+        check_cookie_results = self._download_webpage(
+            password_challenge_results[2], None, 'Checking cookie')
+
+        if '>Sign out<' not in check_cookie_results:
+            self._downloader.report_warning('Unable to log in')
+            return False
+
         return True
 
     def _real_initialize(self):

From e4a75d7932013f49133178d5d84e5f33634a4879 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 May 2017 00:00:11 +0700
Subject: [PATCH 259/611] [test_youtube_chapters] PEP 8

---
 test/test_youtube_chapters.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_youtube_chapters.py b/test/test_youtube_chapters.py
index 5435ee4a8..cb12f8384 100644
--- a/test/test_youtube_chapters.py
+++ b/test/test_youtube_chapters.py
@@ -1,5 +1,5 @@
-# coding: utf-8
 #!/usr/bin/env python
+# coding: utf-8
 from __future__ import unicode_literals
 
 # Allow direct execution

From 3995d37da58ed071b54b7f81757cff4d534f5b9b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 May 2017 04:19:11 +0700
Subject: [PATCH 260/611] [youtube] Fix TFA (#12927)

---
 youtube_dl/extractor/youtube.py | 151 +++++++++++++++++++++++---------
 1 file changed, 110 insertions(+), 41 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c2e06c3a6..44a39282f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -37,7 +37,7 @@ from ..utils import (
     parse_codecs,
     parse_duration,
     remove_quotes,
-    # remove_start,
+    remove_start,
     smuggle_url,
     str_to_int,
     try_get,
@@ -55,13 +55,8 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
     _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge'
 
     _LOOKUP_URL = 'https://accounts.google.com/_/signin/sl/lookup'
-    _LOOKUP_REQ_TEMPLATE = '["{0}",null,[],null,"US",null,null,2,false,true,[null,null,[2,1,null,1,"https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn",null,[],4],1,[null,null,[]],null,null,null,true],"{0}"]'
-
-    _PASSWORD_CHALLENGE_URL = 'https://accounts.google.com/_/signin/sl/challenge'
-    _PASSWORD_CHALLENGE_REQ_TEMPLATE = '["{0}",null,1,null,[1,null,null,null,["{1}",null,true]],[null,null,[2,1,null,1,"https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn",null,[],4],1,[null,null,[]],null,null,null,true]]'
-
-    _TFA_URL = 'https://accounts.google.com/_/signin/challenge'
-    _TFA_REQ_TEMPLATE = '["{0}",null,2,null,[9,null,null,null,null,null,null,null,[null,"{1}",false,2]]]'
+    _CHALLENGE_URL = 'https://accounts.google.com/_/signin/sl/challenge'
+    _TFA_URL = 'https://accounts.google.com/_/signin/challenge?hl=en&TL={0}'
 
     _NETRC_MACHINE = 'youtube'
     # If True it will raise an error if no login info is provided
@@ -112,7 +107,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
                 'checkedDomains': 'youtube',
                 'hl': 'en',
                 'deviceinfo': '[null,null,null,[],null,"US",null,null,[],"GlifWebSignIn",null,[null,null,[]]]',
-                'f.req': f_req,
+                'f.req': json.dumps(f_req),
                 'flowName': 'GlifWebSignIn',
                 'flowEntry': 'ServiceLogin',
             })
@@ -125,53 +120,127 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
                     'Google-Accounts-XSRF': 1,
                 })
 
+        def warn(message):
+            self._downloader.report_warning(message)
+
+        lookup_req = [
+            username,
+            None, [], None, 'US', None, None, 2, False, True,
+            [
+                None, None,
+                [2, 1, None, 1,
+                 'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn',
+                 None, [], 4],
+                1, [None, None, []], None, None, None, True
+            ],
+            username,
+        ]
+
         lookup_results = req(
-            self._LOOKUP_URL, self._LOOKUP_REQ_TEMPLATE.format(username),
+            self._LOOKUP_URL, lookup_req,
             'Looking up account info', 'Unable to look up account info')
 
         if lookup_results is False:
             return False
 
-        user_hash = lookup_results[0][2]
+        user_hash = try_get(lookup_results, lambda x: x[0][2], compat_str)
+        if not user_hash:
+            warn('Unable to extract user hash')
+            return False
 
-        password_challenge_results = req(
-            self._PASSWORD_CHALLENGE_URL,
-            self._PASSWORD_CHALLENGE_REQ_TEMPLATE.format(user_hash, password),
-            'Logging in', 'Unable to log in')[0]
+        challenge_req = [
+            user_hash,
+            None, 1, None, [1, None, None, None, [password, None, True]],
+            [
+                None, None, [2, 1, None, 1, 'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn', None, [], 4],
+                1, [None, None, []], None, None, None, True
+            ]]
 
-        if password_challenge_results is False:
+        challenge_results = req(
+            self._CHALLENGE_URL, challenge_req,
+            'Logging in', 'Unable to log in')
+
+        if challenge_results is False:
             return
 
-        msg = password_challenge_results[5]
-        if msg is not None and isinstance(msg, list):
-            raise ExtractorError('Unable to login: %s' % msg[5], expected=True)
+        login_res = try_get(challenge_results, lambda x: x[0][5], list)
+        if login_res:
+            login_msg = try_get(login_res, lambda x: x[5], compat_str)
+            warn(
+                'Unable to login: %s' % 'Invalid password'
+                if login_msg == 'INCORRECT_ANSWER_ENTERED' else login_msg)
+            return False
 
-        password_challenge_results = password_challenge_results[-1]
+        res = try_get(challenge_results, lambda x: x[0][-1], list)
+        if not res:
+            warn('Unable to extract result entry')
+            return False
 
-        # tfa = password_challenge_results[0]
-        # if isinstance(tfa, list) and tfa[0][2] == 'TWO_STEP_VERIFICATION':
-        #     tfa_code = self._get_tfa_info('2-step verification code')
-        #
-        #     if not tfa_code:
-        #         self._downloader.report_warning(
-        #             'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
-        #             '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
-        #         return False
-        #
-        #     tfa_code = remove_start(tfa_code, 'G-')
-        #     print('tfa', tfa_code)
-        #     tfa_results = req(
-        #         self._TFA_URL,
-        #         self._TFA_REQ_TEMPLATE.format(user_hash, tfa_code),
-        #         'Submitting TFA code', 'Unable to submit TFA code')
-        #
-        #     TODO
+        tfa = try_get(res, lambda x: x[0][0], list)
+        if tfa:
+            tfa_str = try_get(tfa, lambda x: x[2], compat_str)
+            if tfa_str == 'TWO_STEP_VERIFICATION':
+                # SEND_SUCCESS - TFA code has been successfully sent to phone
+                # QUOTA_EXCEEDED - reached the limit of TFA codes
+                status = try_get(tfa, lambda x: x[5], compat_str)
+                if status == 'QUOTA_EXCEEDED':
+                    warn('Exceeded the limit of TFA codes, try later')
+                    return False
+
+                tl = try_get(challenge_results, lambda x: x[1][2], compat_str)
+                if not tl:
+                    warn('Unable to extract TL')
+                    return False
+
+                tfa_code = self._get_tfa_info('2-step verification code')
+
+                if not tfa_code:
+                    warn(
+                        'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
+                        '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
+                    return False
+
+                tfa_code = remove_start(tfa_code, 'G-')
+
+                tfa_req = [
+                    user_hash, None, 2, None,
+                    [
+                        9, None, None, None, None, None, None, None,
+                        [None, tfa_code, True, 2]
+                    ]]
+
+                tfa_results = req(
+                    self._TFA_URL.format(tl), tfa_req,
+                    'Submitting TFA code', 'Unable to submit TFA code')
+
+                if tfa_results is False:
+                    return False
+
+                tfa_res = try_get(tfa_results, lambda x: x[0][5], list)
+                if tfa_res:
+                    tfa_msg = try_get(tfa_res, lambda x: x[5], compat_str)
+                    warn(
+                        'Unable to finish TFA: %s' % 'Invalid TFA code'
+                        if tfa_msg == 'INCORRECT_ANSWER_ENTERED' else tfa_msg)
+                    return False
+
+                check_cookie_url = try_get(
+                    tfa_results, lambda x: x[0][-1][2], compat_str)
+        else:
+            check_cookie_url = try_get(res, lambda x: x[2], compat_str)
+
+        if not check_cookie_url:
+            warn('Unable to extract CheckCookie URL')
+            return False
 
         check_cookie_results = self._download_webpage(
-            password_challenge_results[2], None, 'Checking cookie')
+            check_cookie_url, None, 'Checking cookie', fatal=False)
 
-        if '>Sign out<' not in check_cookie_results:
-            self._downloader.report_warning('Unable to log in')
+        if check_cookie_results is False:
+            return False
+
+        if 'https://myaccount.google.com/' not in check_cookie_results:
+            warn('Unable to log in')
             return False
 
         return True

From 3892a9f4ab54a0da5386b6ff612dd5ef9ff70267 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 May 2017 04:44:54 +0700
Subject: [PATCH 261/611] [ChangeLog] Actualize

---
 ChangeLog | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 7cceec10f..7a344c40f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,12 +1,32 @@
 version <unreleased>
 
+Common
+* [extractor/common] Fix typo in _extract_akamai_formats
++ [postprocessor/ffmpeg] Embed chapters into media file with --add-metadata
++ [extractor/common] Introduce chapters meta field
+
 Extractors
+* [youtube] Fix authentication (#12820, #12927, #12973, #12992, #12993, #12995,
+  #13003)
 * [bilibili] Fix video downloading (#13001)
+* [rmcdecouverte] Fix extraction (#12937)
+* [theplatform] Extract chapters
+* [bandcamp] Fix thumbnail extraction (#12980)
+* [pornhub] Extend URL regular expression (#12996)
++ [youtube] Extract chapters
++ [nrk] Extract chapters
++ [vice] Add support for ooyala embeds in article pages
 + [vice] Support vice articles (#12968)
 * [vice] Fix extraction for non en_us videos (#12967)
 * [gdcvault] Fix extraction for some videos (#12733)
+* [pbs] Improve multipart video support (#12981)
+* [laola1tv] Fix extraction (#12880)
 + [cda] Support birthday verification (#12789)
 * [leeco] Fix extraction (#12974)
++ [pbs] Extract chapters
+* [amp] Imporove thumbnail and subtitles extraction
+* [foxsports] Fix extraction (#12945)
+- [coub] Remove comment count extraction (#12941)
 
 
 version 2017.05.01

From 4ac0f573ef10d4e2bd883b63936e3fe1a41c8abb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 May 2017 04:51:34 +0700
Subject: [PATCH 262/611] release 2017.05.07

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 5 +++--
 youtube_dl/version.py     | 2 +-
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index fc221594f..86d11e142 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.01*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.01**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.07*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.07**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.05.01
+[debug] youtube-dl version 2017.05.07
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 7a344c40f..5418bb95f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.05.07
 
 Common
 * [extractor/common] Fix typo in _extract_akamai_formats
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index e3c038c48..d867dfe65 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -879,9 +879,10 @@
  - **VGTV**: VGTV, BTTV, FTV, Aftenposten and Aftonbladet
  - **vh1.com**
  - **Viafree**
- - **Vice**
+ - **vice**
+ - **vice:article**
+ - **vice:show**
  - **Viceland**
- - **ViceShow**
  - **Vidbit**
  - **Viddler**
  - **Videa**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index c19ac49b0..dc707071a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.05.01'
+__version__ = '2017.05.07'

From 2eeb588efe9a7df4b2dcd90de9e461e8ff4a40fa Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 7 May 2017 08:58:34 +0100
Subject: [PATCH 263/611] [nbc] improve extraction(closes #12364)

---
 youtube_dl/extractor/nbc.py | 98 ++++++++++++-------------------------
 1 file changed, 31 insertions(+), 67 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index d2a44d05d..3b31ca3ef 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 
 class NBCIE(AdobePassIE):
-    _VALID_URL = r'https?://(?:www\.)?nbc\.com/(?:[^/]+/)+(?P<id>n?\d+)'
+    _VALID_URL = r'https?://(?:www\.)?nbc\.com/[^/]+/video/[^/]+/(?P<id>n?\d+)'
 
     _TESTS = [
         {
@@ -36,16 +36,6 @@ class NBCIE(AdobePassIE):
                 'skip_download': True,
             },
         },
-        {
-            'url': 'http://www.nbc.com/the-tonight-show/episodes/176',
-            'info_dict': {
-                'id': '176',
-                'ext': 'flv',
-                'title': 'Ricky Gervais, Steven Van Zandt, ILoveMakonnen',
-                'description': 'A brand new episode of The Tonight Show welcomes Ricky Gervais, Steven Van Zandt and ILoveMakonnen.',
-            },
-            'skip': '404 Not Found',
-        },
         {
             'url': 'http://www.nbc.com/saturday-night-live/video/star-wars-teaser/2832821',
             'info_dict': {
@@ -63,11 +53,6 @@ class NBCIE(AdobePassIE):
             },
             'skip': 'Only works from US',
         },
-        {
-            # This video has expired but with an escaped embedURL
-            'url': 'http://www.nbc.com/parenthood/episode-guide/season-5/just-like-at-home/515',
-            'only_matching': True,
-        },
         {
             # HLS streams requires the 'hdnea3' cookie
             'url': 'http://www.nbc.com/Kings/video/goliath/n1806',
@@ -89,58 +74,37 @@ class NBCIE(AdobePassIE):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        info = {
-            '_type': 'url_transparent',
-            'ie_key': 'ThePlatform',
-            'id': video_id,
+        video_data = self._download_json(
+            'https://api.nbc.com/v3/videos', video_id, query={
+                'filter[permalink]': url,
+            })['data'][0]['attributes']
+        query = {
+            'mbr': 'true',
+            'manifest': 'm3u',
+        }
+        video_id = video_data['guid']
+        title = video_data['title']
+        if video_data.get('entitlement') == 'auth':
+            resource = self._get_mvpd_resource(
+                'nbcentertainment', title, video_id,
+                video_data.get('vChipRating'))
+            query['auth'] = self._extract_mvpd_auth(
+                url, video_id, 'nbcentertainment', resource)
+        theplatform_url = smuggle_url(update_url_query(
+            'http://link.theplatform.com/s/NnzsPC/media/guid/2410887629/' + video_id,
+            query), {'force_smil_url': True})
+        return {
+            '_type': 'url_transparent',
+            'id': video_id,
+            'title': title,
+            'url': theplatform_url,
+            'description': video_data.get('description'),
+            'keywords': video_data.get('keywords'),
+            'season_number': int_or_none(video_data.get('seasonNumber')),
+            'episode_number': int_or_none(video_data.get('episodeNumber')),
+            'series': video_data.get('showName'),
+            'ie_key': 'ThePlatform',
         }
-        video_data = None
-        preload = self._search_regex(
-            r'PRELOAD\s*=\s*({.+})', webpage, 'preload data', default=None)
-        if preload:
-            preload_data = self._parse_json(preload, video_id)
-            path = compat_urllib_parse_urlparse(url).path.rstrip('/')
-            entity_id = preload_data.get('xref', {}).get(path)
-            video_data = preload_data.get('entities', {}).get(entity_id)
-        if video_data:
-            query = {
-                'mbr': 'true',
-                'manifest': 'm3u',
-            }
-            video_id = video_data['guid']
-            title = video_data['title']
-            if video_data.get('entitlement') == 'auth':
-                resource = self._get_mvpd_resource(
-                    'nbcentertainment', title, video_id,
-                    video_data.get('vChipRating'))
-                query['auth'] = self._extract_mvpd_auth(
-                    url, video_id, 'nbcentertainment', resource)
-            theplatform_url = smuggle_url(update_url_query(
-                'http://link.theplatform.com/s/NnzsPC/media/guid/2410887629/' + video_id,
-                query), {'force_smil_url': True})
-            info.update({
-                'id': video_id,
-                'title': title,
-                'url': theplatform_url,
-                'description': video_data.get('description'),
-                'keywords': video_data.get('keywords'),
-                'season_number': int_or_none(video_data.get('seasonNumber')),
-                'episode_number': int_or_none(video_data.get('episodeNumber')),
-                'series': video_data.get('showName'),
-            })
-        else:
-            theplatform_url = unescapeHTML(lowercase_escape(self._html_search_regex(
-                [
-                    r'(?:class="video-player video-player-full" data-mpx-url|class="player" src)="(.*?)"',
-                    r'<iframe[^>]+src="((?:https?:)?//player\.theplatform\.com/[^"]+)"',
-                    r'"embedURL"\s*:\s*"([^"]+)"'
-                ],
-                webpage, 'theplatform url').replace('_no_endcard', '').replace('\\/', '/')))
-            if theplatform_url.startswith('//'):
-                theplatform_url = 'http:' + theplatform_url
-            info['url'] = smuggle_url(theplatform_url, {'source_url': url})
-        return info
 
 
 class NBCSportsVPlayerIE(InfoExtractor):

From 52294cdda761ad08785e7118ae8e121ceef257ec Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 7 May 2017 09:31:14 +0100
Subject: [PATCH 264/611] [nbc] remove unused imports and extract permalink
 from modified urls

---
 youtube_dl/extractor/nbc.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 3b31ca3ef..62db70b43 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -5,10 +5,8 @@ import re
 from .common import InfoExtractor
 from .theplatform import ThePlatformIE
 from .adobepass import AdobePassIE
-from ..compat import compat_urllib_parse_urlparse
 from ..utils import (
     find_xpath_attr,
-    lowercase_escape,
     smuggle_url,
     unescapeHTML,
     update_url_query,
@@ -17,7 +15,7 @@ from ..utils import (
 
 
 class NBCIE(AdobePassIE):
-    _VALID_URL = r'https?://(?:www\.)?nbc\.com/[^/]+/video/[^/]+/(?P<id>n?\d+)'
+    _VALID_URL = r'(?P<permalink>https?://(?:www\.)?nbc\.com/[^/]+/video/[^/]+/(?P<id>n?\d+))'
 
     _TESTS = [
         {
@@ -73,10 +71,10 @@ class NBCIE(AdobePassIE):
     ]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        permalink, video_id = re.match(self._VALID_URL, url).groups()
         video_data = self._download_json(
             'https://api.nbc.com/v3/videos', video_id, query={
-                'filter[permalink]': url,
+                'filter[permalink]': permalink,
             })['data'][0]['attributes']
         query = {
             'mbr': 'true',

From 3d40084b835afe49adff7eca67dec26bc9b95c0a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 May 2017 20:03:38 +0700
Subject: [PATCH 265/611] [nuevo] Pass headers to _extract_nuevo

---
 youtube_dl/extractor/nuevo.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nuevo.py b/youtube_dl/extractor/nuevo.py
index 87fb94d1f..be1e09d37 100644
--- a/youtube_dl/extractor/nuevo.py
+++ b/youtube_dl/extractor/nuevo.py
@@ -10,9 +10,10 @@ from ..utils import (
 
 
 class NuevoBaseIE(InfoExtractor):
-    def _extract_nuevo(self, config_url, video_id):
+    def _extract_nuevo(self, config_url, video_id, headers={}):
         config = self._download_xml(
-            config_url, video_id, transform_source=lambda s: s.strip())
+            config_url, video_id, transform_source=lambda s: s.strip(),
+            headers=headers)
 
         title = xpath_text(config, './title', 'title', fatal=True).strip()
         video_id = xpath_text(config, './mediaid', default=video_id)

From b6eb74e340fd523fb0c710b3141d6f9c696069f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 May 2017 20:08:58 +0700
Subject: [PATCH 266/611] [nonktube] Add extractor (closes #8647, closes
 #13024)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/nonktube.py   | 37 ++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+)
 create mode 100644 youtube_dl/extractor/nonktube.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 1d7495910..968cca9d2 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -663,6 +663,7 @@ from .nintendo import NintendoIE
 from .njpwworld import NJPWWorldIE
 from .nobelprize import NobelPrizeIE
 from .noco import NocoIE
+from .nonktube import NonkTubeIE
 from .noovo import NoovoIE
 from .normalboots import NormalbootsIE
 from .nosvideo import NosVideoIE
diff --git a/youtube_dl/extractor/nonktube.py b/youtube_dl/extractor/nonktube.py
new file mode 100644
index 000000000..ba6007cd1
--- /dev/null
+++ b/youtube_dl/extractor/nonktube.py
@@ -0,0 +1,37 @@
+from __future__ import unicode_literals
+
+from .nuevo import NuevoBaseIE
+
+
+class NonkTubeIE(NuevoBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?nonktube\.com/(?:(?:video|embed)/|media/nuevo/embed\.php\?.*?\bid=)(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.nonktube.com/video/118636/sensual-wife-uncensored-fucked-in-hairy-pussy-and-facialized',
+        'info_dict': {
+            'id': '118636',
+            'ext': 'mp4',
+            'title': 'Sensual Wife Uncensored Fucked In Hairy Pussy And Facialized',
+            'age_limit': 18,
+            'duration': 1150.98,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }, {
+        'url': 'https://www.nonktube.com/embed/118636',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        nuevo_url = 'https://www.nonktube.com/media/nuevo/config.php?key=%s' % video_id
+
+        info = self._extract_nuevo(
+            nuevo_url, video_id, headers={
+                'Referer': 'https://www.nonktube.com/media/nuevo/player.swf?config=%s' % nuevo_url
+            })
+        info.update({
+            'age_limit': 18
+        })
+        return info

From bf82b8732353f45582cfefc6cced01721fbf4154 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 May 2017 20:13:22 +0700
Subject: [PATCH 267/611] [nonktube] Use econfig nuevo URL

---
 youtube_dl/extractor/nonktube.py | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/nonktube.py b/youtube_dl/extractor/nonktube.py
index ba6007cd1..63e58aae2 100644
--- a/youtube_dl/extractor/nonktube.py
+++ b/youtube_dl/extractor/nonktube.py
@@ -25,13 +25,9 @@ class NonkTubeIE(NuevoBaseIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        nuevo_url = 'https://www.nonktube.com/media/nuevo/config.php?key=%s' % video_id
-
         info = self._extract_nuevo(
-            nuevo_url, video_id, headers={
-                'Referer': 'https://www.nonktube.com/media/nuevo/player.swf?config=%s' % nuevo_url
-            })
-        info.update({
-            'age_limit': 18
-        })
+            'https://www.nonktube.com/media/nuevo/econfig.php?key=%s'
+            % video_id, video_id)
+
+        info['age_limit'] = 18
         return info

From 04c09f19618345203a787c300e4ecf99f1ff9b02 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 8 May 2017 15:00:57 +0100
Subject: [PATCH 268/611] [turner] extract thumbnail and is_live and strip
 description

---
 youtube_dl/extractor/turner.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/turner.py b/youtube_dl/extractor/turner.py
index 1c0be9fc6..efeb677ee 100644
--- a/youtube_dl/extractor/turner.py
+++ b/youtube_dl/extractor/turner.py
@@ -13,6 +13,7 @@ from ..utils import (
     xpath_attr,
     update_url_query,
     ExtractorError,
+    strip_or_none,
 )
 
 
@@ -163,17 +164,21 @@ class TurnerBaseIE(AdobePassIE):
             'height': int_or_none(image.get('height')),
         } for image in video_data.findall('images/image')]
 
+        is_live = xpath_text(video_data, 'isLive') == 'true'
+
         return {
             'id': video_id,
-            'title': title,
+            'title': self._live_title(title) if is_live else title,
             'formats': formats,
             'subtitles': subtitles,
             'thumbnails': thumbnails,
-            'description': xpath_text(video_data, 'description'),
+            'thumbnail': xpath_text(video_data, 'poster'),
+            'description': strip_or_none(xpath_text(video_data, 'description')),
             'duration': parse_duration(xpath_text(video_data, 'length') or xpath_text(video_data, 'trt')),
             'timestamp': self._extract_timestamp(video_data),
             'upload_date': xpath_attr(video_data, 'metas', 'version'),
             'series': xpath_text(video_data, 'showTitle'),
             'season_number': int_or_none(xpath_text(video_data, 'seasonNumber')),
             'episode_number': int_or_none(xpath_text(video_data, 'episodeNumber')),
+            'is_live': is_live,
         }

From 3b859145c2ad4efd5250fc0199174f1ec2068b34 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 8 May 2017 15:01:10 +0100
Subject: [PATCH 269/611] [adultswim] Fix Extraction(closes #8640)(closes
 #10950)(closes closes #11042)(closes #12121)

- add support for adobe pass authentication
- add support for live streams
- add support for show pages
---
 youtube_dl/extractor/adultswim.py | 285 +++++++++++-------------------
 1 file changed, 106 insertions(+), 179 deletions(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 989505c82..9c37ea33c 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -5,91 +5,52 @@ import re
 
 from .turner import TurnerBaseIE
 from ..utils import (
-    ExtractorError,
     int_or_none,
+    strip_or_none,
 )
 
 
 class AdultSwimIE(TurnerBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?adultswim\.com/videos/(?P<is_playlist>playlists/)?(?P<show_path>[^/]+)/(?P<episode_path>[^/?#]+)/?'
+    _VALID_URL = r'https?://(?:www\.)?adultswim\.com/videos/(?P<show_path>[^/?#]+)(?:/(?P<episode_path>[^/?#]+))?'
 
     _TESTS = [{
         'url': 'http://adultswim.com/videos/rick-and-morty/pilot',
-        'playlist': [
-            {
-                'md5': '247572debc75c7652f253c8daa51a14d',
-                'info_dict': {
-                    'id': 'rQxZvXQ4ROaSOqq-or2Mow-0',
-                    'ext': 'flv',
-                    'title': 'Rick and Morty - Pilot Part 1',
-                    'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
-                },
-            },
-            {
-                'md5': '77b0e037a4b20ec6b98671c4c379f48d',
-                'info_dict': {
-                    'id': 'rQxZvXQ4ROaSOqq-or2Mow-3',
-                    'ext': 'flv',
-                    'title': 'Rick and Morty - Pilot Part 4',
-                    'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
-                },
-            },
-        ],
         'info_dict': {
             'id': 'rQxZvXQ4ROaSOqq-or2Mow',
+            'ext': 'mp4',
             'title': 'Rick and Morty - Pilot',
-            'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
-        },
-        'skip': 'This video is only available for registered users',
-    }, {
-        'url': 'http://www.adultswim.com/videos/playlists/american-parenting/putting-francine-out-of-business/',
-        'playlist': [
-            {
-                'md5': '2eb5c06d0f9a1539da3718d897f13ec5',
-                'info_dict': {
-                    'id': '-t8CamQlQ2aYZ49ItZCFog-0',
-                    'ext': 'flv',
-                    'title': 'American Dad - Putting Francine Out of Business',
-                    'description': 'Stan hatches a plan to get Francine out of the real estate business.Watch more American Dad on [adult swim].'
-                },
-            }
-        ],
-        'info_dict': {
-            'id': '-t8CamQlQ2aYZ49ItZCFog',
-            'title': 'American Dad - Putting Francine Out of Business',
-            'description': 'Stan hatches a plan to get Francine out of the real estate business.Watch more American Dad on [adult swim].'
-        },
-    }, {
-        'url': 'http://www.adultswim.com/videos/tim-and-eric-awesome-show-great-job/dr-steve-brule-for-your-wine/',
-        'playlist': [
-            {
-                'md5': '3e346a2ab0087d687a05e1e7f3b3e529',
-                'info_dict': {
-                    'id': 'sY3cMUR_TbuE4YmdjzbIcQ-0',
-                    'ext': 'mp4',
-                    'title': 'Tim and Eric Awesome Show Great Job! - Dr. Steve Brule, For Your Wine',
-                    'description': 'Dr. Brule reports live from Wine Country with a special report on wines.  \r\nWatch Tim and Eric Awesome Show Great Job! episode #20, "Embarrassed" on Adult Swim.\r\n\r\n',
-                },
-            }
-        ],
-        'info_dict': {
-            'id': 'sY3cMUR_TbuE4YmdjzbIcQ',
-            'title': 'Tim and Eric Awesome Show Great Job! - Dr. Steve Brule, For Your Wine',
-            'description': 'Dr. Brule reports live from Wine Country with a special report on wines.  \r\nWatch Tim and Eric Awesome Show Great Job! episode #20, "Embarrassed" on Adult Swim.\r\n\r\n',
+            'description': 'Rick moves in with his daughter\'s family and establishes himself as a bad influence on his grandson, Morty.',
+            'timestamp': 1493267400,
+            'upload_date': '20170427',
         },
         'params': {
             # m3u8 download
             'skip_download': True,
-        }
+        },
+        'expected_warnings': ['Unable to download f4m manifest'],
+    }, {
+        'url': 'http://www.adultswim.com/videos/tim-and-eric-awesome-show-great-job/dr-steve-brule-for-your-wine/',
+        'info_dict': {
+            'id': 'sY3cMUR_TbuE4YmdjzbIcQ',
+            'ext': 'mp4',
+            'title': 'Tim and Eric Awesome Show Great Job! - Dr. Steve Brule, For Your Wine',
+            'description': 'Dr. Brule reports live from Wine Country with a special report on wines.  \nWatch Tim and Eric Awesome Show Great Job! episode #20, "Embarrassed" on Adult Swim.',
+            'upload_date': '20080124',
+            'timestamp': 1201150800,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
-        # heroMetadata.trailer
         'url': 'http://www.adultswim.com/videos/decker/inside-decker-a-new-hero/',
         'info_dict': {
             'id': 'I0LQFQkaSUaFp8PnAWHhoQ',
             'ext': 'mp4',
             'title': 'Decker - Inside Decker: A New Hero',
-            'description': 'md5:c916df071d425d62d70c86d4399d3ee0',
-            'duration': 249.008,
+            'description': 'The guys recap the conclusion of the season. They announce a new hero, take a peek into the Victorville Film Archive and welcome back the talented James Dean.',
+            'timestamp': 1469480460,
+            'upload_date': '20160725',
         },
         'params': {
             # m3u8 download
@@ -97,136 +58,102 @@ class AdultSwimIE(TurnerBaseIE):
         },
         'expected_warnings': ['Unable to download f4m manifest'],
     }, {
-        'url': 'http://www.adultswim.com/videos/toonami/friday-october-14th-2016/',
+        'url': 'http://www.adultswim.com/videos/attack-on-titan',
         'info_dict': {
-            'id': 'eYiLsKVgQ6qTC6agD67Sig',
-            'title': 'Toonami - Friday, October 14th, 2016',
-            'description': 'md5:99892c96ffc85e159a428de85c30acde',
+            'id': 'b7A69dzfRzuaXIECdxW8XQ',
+            'title': 'Attack on Titan',
+            'description': 'md5:6c8e003ea0777b47013e894767f5e114',
+        },
+        'playlist_mincount': 12,
+    }, {
+        'url': 'http://www.adultswim.com/videos/streams/williams-stream',
+        'info_dict': {
+            'id': 'd8DEBj7QRfetLsRgFnGEyg',
+            'ext': 'mp4',
+            'title': r're:^Williams Stream \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
+            'description': 'original programming',
         },
-        'playlist': [{
-            'md5': '',
-            'info_dict': {
-                'id': 'eYiLsKVgQ6qTC6agD67Sig',
-                'ext': 'mp4',
-                'title': 'Toonami - Friday, October 14th, 2016',
-                'description': 'md5:99892c96ffc85e159a428de85c30acde',
-            },
-        }],
         'params': {
             # m3u8 download
             'skip_download': True,
         },
-        'expected_warnings': ['Unable to download f4m manifest'],
     }]
 
-    @staticmethod
-    def find_video_info(collection, slug):
-        for video in collection.get('videos'):
-            if video.get('slug') == slug:
-                return video
-
-    @staticmethod
-    def find_collection_by_linkURL(collections, linkURL):
-        for collection in collections:
-            if collection.get('linkURL') == linkURL:
-                return collection
-
-    @staticmethod
-    def find_collection_containing_video(collections, slug):
-        for collection in collections:
-            for video in collection.get('videos'):
-                if video.get('slug') == slug:
-                    return collection, video
-        return None, None
-
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        show_path = mobj.group('show_path')
-        episode_path = mobj.group('episode_path')
-        is_playlist = True if mobj.group('is_playlist') else False
+        show_path, episode_path = re.match(self._VALID_URL, url).groups()
+        display_id = episode_path or show_path
+        webpage = self._download_webpage(url, display_id)
+        initial_data = self._parse_json(self._search_regex(
+            r'AS_INITIAL_DATA(?:__)?\s*=\s*({.+?});',
+            webpage, 'initial data'), display_id)
 
-        webpage = self._download_webpage(url, episode_path)
+        is_stream = show_path == 'streams'
+        if is_stream:
+            if not episode_path:
+                episode_path = 'live-stream'
 
-        # Extract the value of `bootstrappedData` from the Javascript in the page.
-        bootstrapped_data = self._parse_json(self._search_regex(
-            r'var bootstrappedData = ({.*});', webpage, 'bootstraped data'), episode_path)
+            video_data = next(stream for stream_path, stream in initial_data['streams'].items() if stream_path == episode_path)
+            video_id = video_data.get('stream')
 
-        # Downloading videos from a /videos/playlist/ URL needs to be handled differently.
-        # NOTE: We are only downloading one video (the current one) not the playlist
-        if is_playlist:
-            collections = bootstrapped_data['playlists']['collections']
-            collection = self.find_collection_by_linkURL(collections, show_path)
-            video_info = self.find_video_info(collection, episode_path)
-
-            show_title = video_info['showTitle']
-            segment_ids = [video_info['videoPlaybackID']]
+            if not video_id:
+                entries = []
+                for episode in video_data.get('archiveEpisodes', []):
+                    episode_url = episode.get('url')
+                    if not episode_url:
+                        continue
+                    entries.append(self.url_result(
+                        episode_url, 'AdultSwim', episode.get('id')))
+                return self.playlist_result(
+                    entries, video_data.get('id'), video_data.get('title'),
+                    strip_or_none(video_data.get('description')))
         else:
-            collections = bootstrapped_data['show']['collections']
-            collection, video_info = self.find_collection_containing_video(collections, episode_path)
-            # Video wasn't found in the collections, let's try `slugged_video`.
-            if video_info is None:
-                if bootstrapped_data.get('slugged_video', {}).get('slug') == episode_path:
-                    video_info = bootstrapped_data['slugged_video']
-            if not video_info:
-                video_info = bootstrapped_data.get(
-                    'heroMetadata', {}).get('trailer', {}).get('video')
-            if not video_info:
-                video_info = bootstrapped_data.get('onlineOriginals', [None])[0]
-            if not video_info:
-                raise ExtractorError('Unable to find video info')
+            show_data = initial_data['show']
 
-            show = bootstrapped_data['show']
-            show_title = show['title']
-            stream = video_info.get('stream')
-            if stream and stream.get('videoPlaybackID'):
-                segment_ids = [stream['videoPlaybackID']]
-            elif video_info.get('clips'):
-                segment_ids = [clip['videoPlaybackID'] for clip in video_info['clips']]
-            elif video_info.get('videoPlaybackID'):
-                segment_ids = [video_info['videoPlaybackID']]
-            elif video_info.get('id'):
-                segment_ids = [video_info['id']]
-            else:
-                if video_info.get('auth') is True:
-                    raise ExtractorError(
-                        'This video is only available via cable service provider subscription that'
-                        ' is not currently supported. You may want to use --cookies.', expected=True)
-                else:
-                    raise ExtractorError('Unable to find stream or clips')
+            if not episode_path:
+                entries = []
+                for video in show_data.get('videos', []):
+                    slug = video.get('slug')
+                    if not slug:
+                        continue
+                    entries.append(self.url_result(
+                        'http://adultswim.com/videos/%s/%s' % (show_path, slug),
+                        'AdultSwim', video.get('id')))
+                return self.playlist_result(
+                    entries, show_data.get('id'), show_data.get('title'),
+                    strip_or_none(show_data.get('metadata', {}).get('description')))
 
-        episode_id = video_info['id']
-        episode_title = video_info['title']
-        episode_description = video_info.get('description')
-        episode_duration = int_or_none(video_info.get('duration'))
-        view_count = int_or_none(video_info.get('views'))
+            video_data = show_data['sluggedVideo']
+            video_id = video_data['id']
 
-        entries = []
-        for part_num, segment_id in enumerate(segment_ids):
-            segement_info = self._extract_cvp_info(
-                'http://www.adultswim.com/videos/api/v0/assets?id=%s&platform=desktop' % segment_id,
-                segment_id, {
-                    'secure': {
-                        'media_src': 'http://androidhls-secure.cdn.turner.com/adultswim/big',
-                        'tokenizer_src': 'http://www.adultswim.com/astv/mvpd/processors/services/token_ipadAdobe.do',
-                    },
-                })
-            segment_title = '%s - %s' % (show_title, episode_title)
-            if len(segment_ids) > 1:
-                segment_title += ' Part %d' % (part_num + 1)
-            segement_info.update({
-                'id': segment_id,
-                'title': segment_title,
-                'description': episode_description,
+        info = self._extract_cvp_info(
+            'http://www.adultswim.com/videos/api/v0/assets?id=' + video_id,
+            video_id, {
+                'secure': {
+                    'media_src': 'http://androidhls-secure.cdn.turner.com/adultswim/big',
+                    'tokenizer_src': 'http://www.adultswim.com/astv/mvpd/processors/services/token_ipadAdobe.do',
+                },
+            }, {
+                'url': url,
+                'site_name': 'AdultSwim',
+                'auth_required': video_data.get('auth'),
             })
-            entries.append(segement_info)
 
-        return {
-            '_type': 'playlist',
-            'id': episode_id,
-            'display_id': episode_path,
-            'entries': entries,
-            'title': '%s - %s' % (show_title, episode_title),
-            'description': episode_description,
-            'duration': episode_duration,
-            'view_count': view_count,
-        }
+        info.update({
+            'id': video_id,
+            'display_id': display_id,
+            'description': info.get('description') or strip_or_none(video_data.get('description')),
+        })
+        if not is_stream:
+            info.update({
+                'duration': info.get('duration') or int_or_none(video_data.get('duration')),
+                'timestamp': info.get('timestamp') or int_or_none(video_data.get('launch_date')),
+                'season_number': info.get('season_number') or int_or_none(video_data.get('season_number')),
+                'episode': info['title'],
+                'episode_number': info.get('episode_number') or int_or_none(video_data.get('episode_number')),
+            })
+
+            info['series'] = video_data.get('collection_title') or info.get('series')
+            if info['series'] and info['series'] != info['title']:
+                info['title'] = '%s - %s' % (info['series'], info['title'])
+
+        return info

From 8fa17117df723ab0bb4b0105142ea861df663ba5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 8 May 2017 18:13:58 +0100
Subject: [PATCH 270/611] [funimation] fix authentication(closes #13021)

---
 youtube_dl/extractor/funimation.py | 40 ++++++++++++++----------------
 1 file changed, 18 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index e44a2a87f..c1559be0a 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -20,6 +20,7 @@ class FunimationIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?funimation(?:\.com|now\.uk)/shows/[^/]+/(?P<id>[^/?#&]+)'
 
     _NETRC_MACHINE = 'funimation'
+    _TOKEN = None
 
     _TESTS = [{
         'url': 'https://www.funimation.com/shows/hacksign/role-play/',
@@ -67,27 +68,19 @@ class FunimationIE(InfoExtractor):
         (username, password) = self._get_login_info()
         if username is None:
             return
-        data = urlencode_postdata({
-            'email_field': username,
-            'password_field': password,
-        })
-        user_agent = self._extract_cloudflare_session_ua(self._LOGIN_URL)
-        if not user_agent:
-            user_agent = 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0'
-        login_request = sanitized_Request(self._LOGIN_URL, data, headers={
-            'User-Agent': user_agent,
-            'Content-Type': 'application/x-www-form-urlencoded'
-        })
-        login_page = self._download_webpage(
-            login_request, None, 'Logging in as %s' % username)
-        if any(p in login_page for p in ('funimation.com/logout', '>Log Out<')):
-            return
-        error = self._html_search_regex(
-            r'(?s)<div[^>]+id=["\']errorMessages["\'][^>]*>(.+?)</div>',
-            login_page, 'error messages', default=None)
-        if error:
-            raise ExtractorError('Unable to login: %s' % error, expected=True)
-        raise ExtractorError('Unable to log in')
+        try:
+            data = self._download_json(
+                'https://prod-api-funimationnow.dadcdigital.com/api/auth/login/',
+                None, 'Logging in as %s' % username, data=urlencode_postdata({
+                    'username': username,
+                    'password': password,
+                }))
+            self._TOKEN = data['token']
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+                error = self._parse_json(e.cause.read().decode(), None)['error']
+                raise ExtractorError(error, expected=True)
+            raise
 
     def _real_initialize(self):
         self._login()
@@ -125,9 +118,12 @@ class FunimationIE(InfoExtractor):
         description = self._html_search_meta(['description', 'og:description'], webpage, fatal=True)
 
         try:
+            headers = {}
+            if self._TOKEN:
+                headers['Authorization'] = 'Token %s' % self._TOKEN
             sources = self._download_json(
                 'https://prod-api-funimationnow.dadcdigital.com/api/source/catalog/video/%s/signed/' % video_id,
-                video_id)['items']
+                video_id, headers=headers)['items']
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
                 error = self._parse_json(e.cause.read(), video_id)['errors'][0]

From 804181dda9cbb7f2cbb8305bb4097908312a83f6 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 8 May 2017 18:50:25 +0100
Subject: [PATCH 271/611] [funimation] remove codes related to old login method
 and update test

---
 youtube_dl/extractor/funimation.py | 26 ++++++--------------------
 1 file changed, 6 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index c1559be0a..8c37509ec 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -2,15 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_HTTPError,
-    compat_urllib_parse_unquote_plus,
-)
+from ..compat import compat_HTTPError
 from ..utils import (
     determine_ext,
     int_or_none,
     js_to_json,
-    sanitized_Request,
     ExtractorError,
     urlencode_postdata
 )
@@ -39,31 +35,21 @@ class FunimationIE(InfoExtractor):
     }, {
         'url': 'https://www.funimation.com/shows/attack-on-titan-junior-high/broadcast-dub-preview/',
         'info_dict': {
-            'id': '9635',
+            'id': '210051',
             'display_id': 'broadcast-dub-preview',
             'ext': 'mp4',
             'title': 'Attack on Titan: Junior High - Broadcast Dub Preview',
-            'description': 'md5:f8ec49c0aff702a7832cd81b8a44f803',
             'thumbnail': r're:https?://.*\.(?:jpg|png)',
         },
-        'skip': 'Access without user interaction is forbidden by CloudFlare',
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         'url': 'https://www.funimationnow.uk/shows/puzzle-dragons-x/drop-impact/simulcast/',
         'only_matching': True,
     }]
 
-    _LOGIN_URL = 'http://www.funimation.com/login'
-
-    def _extract_cloudflare_session_ua(self, url):
-        ci_session_cookie = self._get_cookies(url).get('ci_session')
-        if ci_session_cookie:
-            ci_session = compat_urllib_parse_unquote_plus(ci_session_cookie.value)
-            # ci_session is a string serialized by PHP function serialize()
-            # This case is simple enough to use regular expressions only
-            return self._search_regex(
-                r'"user_agent";s:\d+:"([^"]+)"', ci_session, 'user agent',
-                default=None)
-
     def _login(self):
         (username, password) = self._get_login_info()
         if username is None:

From e9137224b3ef638cce3dac024814397fd020f3b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 May 2017 01:14:02 +0700
Subject: [PATCH 272/611] [YoutubeDL] Force restrict filenames when no locale
 is set for python 2 as well (#13027)

---
 youtube_dl/YoutubeDL.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index eb465c425..4c33d494a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -370,10 +370,10 @@ class YoutubeDL(object):
                 else:
                     raise
 
-        if (sys.version_info >= (3,) and sys.platform != 'win32' and
+        if (sys.platform != 'win32' and
                 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
                 not params.get('restrictfilenames', False)):
-            # On Python 3, the Unicode filesystem API will throw errors (#1474)
+            # Unicode filesystem API will throw errors (#1474, #13027)
             self.report_warning(
                 'Assuming --restrict-filenames since file system encoding '
                 'cannot encode all characters. '

From afa0200bf05671be5a380a79206309bdd4f886d6 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 8 May 2017 20:03:26 +0100
Subject: [PATCH 273/611] [vrv] extract dash formats and subtitles

---
 youtube_dl/extractor/vrv.py | 47 +++++++++++++++++++++++++++----------
 1 file changed, 34 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/vrv.py b/youtube_dl/extractor/vrv.py
index 487047fd7..9959627c0 100644
--- a/youtube_dl/extractor/vrv.py
+++ b/youtube_dl/extractor/vrv.py
@@ -112,21 +112,41 @@ class VRVIE(VRVBaseIE):
 
         audio_locale = streams_json.get('audio_locale')
         formats = []
-        for stream_id, stream in streams_json.get('streams', {}).get('adaptive_hls', {}).items():
-            stream_url = stream.get('url')
-            if not stream_url:
-                continue
-            stream_id = stream_id or audio_locale
-            m3u8_formats = self._extract_m3u8_formats(
-                stream_url, video_id, 'mp4', m3u8_id=stream_id,
-                note='Downloading %s m3u8 information' % stream_id,
-                fatal=False)
-            if audio_locale:
-                for f in m3u8_formats:
-                    f['language'] = audio_locale
-            formats.extend(m3u8_formats)
+        for stream_type, streams in streams_json.get('streams', {}).items():
+            if stream_type in ('adaptive_hls', 'adaptive_dash'):
+                for stream in streams.values():
+                    stream_url = stream.get('url')
+                    if not stream_url:
+                        continue
+                    stream_id = stream.get('hardsub_locale') or audio_locale
+                    format_id = '%s-%s' % (stream_type.split('_')[1], stream_id)
+                    if stream_type == 'adaptive_hls':
+                        adaptive_formats = self._extract_m3u8_formats(
+                            stream_url, video_id, 'mp4', m3u8_id=format_id,
+                            note='Downloading %s m3u8 information' % stream_id,
+                            fatal=False)
+                    else:
+                        adaptive_formats = self._extract_mpd_formats(
+                            stream_url, video_id, mpd_id=format_id,
+                            note='Downloading %s MPD information' % stream_id,
+                            fatal=False)
+                    if audio_locale:
+                        for f in adaptive_formats:
+                            if f.get('acodec') != 'none':
+                                f['language'] = audio_locale
+                    formats.extend(adaptive_formats)
         self._sort_formats(formats)
 
+        subtitles = {}
+        for subtitle in streams_json.get('subtitles', {}).values():
+            subtitle_url = subtitle.get('url')
+            if not subtitle_url:
+                continue
+            subtitles.setdefault(subtitle.get('locale', 'en-US'), []).append({
+                'url': subtitle_url,
+                'ext': subtitle.get('format', 'ass'),
+            })
+
         thumbnails = []
         for thumbnail in video_data.get('images', {}).get('thumbnails', []):
             thumbnail_url = thumbnail.get('source')
@@ -142,6 +162,7 @@ class VRVIE(VRVBaseIE):
             'id': video_id,
             'title': title,
             'formats': formats,
+            'subtitles': subtitles,
             'thumbnails': thumbnails,
             'description': video_data.get('description'),
             'duration': float_or_none(video_data.get('duration_ms'), 1000),

From 5996d21aeac73509eee060bd26dbec95ede5e2eb Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 9 May 2017 00:47:37 +0100
Subject: [PATCH 274/611] [cspan] add support for brightcove live embeds(closes
 #13028)

---
 youtube_dl/extractor/brightcove.py | 20 +++++++++++++++++---
 youtube_dl/extractor/cspan.py      | 15 +++++++++++++++
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 3f017a2b1..0ed59bcbc 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -5,6 +5,7 @@ import re
 import json
 
 from .common import InfoExtractor
+from .adobepass import AdobePassIE
 from ..compat import (
     compat_etree_fromstring,
     compat_parse_qs,
@@ -448,7 +449,7 @@ class BrightcoveLegacyIE(InfoExtractor):
         return info
 
 
-class BrightcoveNewIE(InfoExtractor):
+class BrightcoveNewIE(AdobePassIE):
     IE_NAME = 'brightcove:new'
     _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>\d+|ref:[^&]+)'
     _TESTS = [{
@@ -602,6 +603,20 @@ class BrightcoveNewIE(InfoExtractor):
                 raise ExtractorError(message, expected=True)
             raise
 
+        errors = json_data.get('errors')
+        if errors and errors[0].get('error_subcode') == 'TVE_AUTH':
+            custom_fields = json_data['custom_fields']
+            tve_token = self._extract_mvpd_auth(
+                smuggled_data['source_url'], video_id,
+                custom_fields['bcadobepassrequestorid'],
+                custom_fields['bcadobepassresourceid'])
+            json_data = self._download_json(
+                api_url, video_id, headers={
+                    'Accept': 'application/json;pk=%s' % policy_key
+                }, query={
+                    'tveToken': tve_token,
+                })
+
         title = json_data['name'].strip()
 
         formats = []
@@ -667,7 +682,6 @@ class BrightcoveNewIE(InfoExtractor):
                     })
                 formats.append(f)
 
-        errors = json_data.get('errors')
         if not formats and errors:
             error = errors[0]
             raise ExtractorError(
@@ -684,7 +698,7 @@ class BrightcoveNewIE(InfoExtractor):
 
         is_live = False
         duration = float_or_none(json_data.get('duration'), 1000)
-        if duration and duration < 0:
+        if duration is not None and duration <= 0:
             is_live = True
 
         return {
diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index d4576160b..171820e27 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -10,6 +10,7 @@ from ..utils import (
     smuggle_url,
     determine_ext,
     ExtractorError,
+    extract_attributes,
 )
 from .senateisvp import SenateISVPIE
 from .ustream import UstreamIE
@@ -68,6 +69,7 @@ class CSpanIE(InfoExtractor):
             'uploader_id': '12987475',
         },
     }]
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -78,6 +80,19 @@ class CSpanIE(InfoExtractor):
         if ustream_url:
             return self.url_result(ustream_url, UstreamIE.ie_key())
 
+        if '&vod' not in url:
+            bc = self._search_regex(
+                r"(<[^>]+id='brightcove-player-embed'[^>]+>)",
+                webpage, 'brightcove embed', default=None)
+            if bc:
+                bc_attr = extract_attributes(bc)
+                bc_url = self.BRIGHTCOVE_URL_TEMPLATE % (
+                    bc_attr.get('data-bcaccountid', '3162030207001'),
+                    bc_attr.get('data-noprebcplayerid', 'SyGGpuJy3g'),
+                    bc_attr.get('data-newbcplayerid', 'default'),
+                    bc_attr['data-bcid'])
+                return self.url_result(smuggle_url(bc_url, {'source_url': url}))
+
         # We first look for clipid, because clipprog always appears before
         patterns = [r'id=\'clip(%s)\'\s*value=\'([0-9]+)\'' % t for t in ('id', 'prog')]
         results = list(filter(None, (re.search(p, webpage) for p in patterns)))

From b972fb037bc4a07694df6c010a25f12aedb9f731 Mon Sep 17 00:00:00 2001
From: Rasmus Rendal <rasmusrendal@gmail.com>
Date: Sun, 7 May 2017 13:04:23 +0200
Subject: [PATCH 275/611] [drtv] Lower preference for SignLanguage formats
 (closes #13013)

---
 youtube_dl/extractor/drtv.py | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index e4917014a..e6f72ac50 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -45,6 +45,18 @@ class DRTVIE(InfoExtractor):
             'upload_date': '20160902',
             'duration': 131.4,
         },
+    }, {
+        'url': 'https://www.dr.dk/tv/se/historien-om-danmark/-/historien-om-danmark-stenalder',
+        'md5': '7c8ca12e6c3d3e3edd59ba5a9b7ca10a',
+        'info_dict': {
+            'id': 'historien-om-danmark-stenalder',
+            'ext': 'mp4',
+            'title': 'Historien om Danmark: Stenalder (1)',
+            'description': 'Én fascinerende historie om tusindvis af år, hvor vores land bliver skabt ud af is og vand, og hvor de første danskere ankommer til vores egn. Det bliver en rejse ind i urtiden og det liv, som urtidsjægerne har levet i skovene og ved havet og helt frem til bondestenalderen. Gennem skeletfund afslører eksperter, hvordan vores forfædre har set ud i stenalderen og hvorfor stenaldermennesket byggede de imponerende jættestuer, som ligger overalt i det danske.',
+            'timestamp': 1490401996,
+            'upload_date': '20170325',
+            'duration': 3502.04,
+        },
     }]
 
     def _real_extract(self, url):
@@ -85,7 +97,11 @@ class DRTVIE(InfoExtractor):
             kind = asset.get('Kind')
             if kind == 'Image':
                 thumbnail = asset.get('Uri')
-            elif kind in ('VideoResource', 'AudioResource'):
+            preference = 0
+
+            sign_language = asset.get('Target') == 'SignLanguage'
+
+            if kind in ('VideoResource', 'AudioResource'):
                 duration = float_or_none(asset.get('DurationInMilliseconds'), 1000)
                 restricted_to_denmark = asset.get('RestrictedToDenmark')
                 spoken_subtitles = asset.get('Target') == 'SpokenSubtitles'
@@ -95,10 +111,12 @@ class DRTVIE(InfoExtractor):
                         continue
                     target = link.get('Target')
                     format_id = target or ''
-                    preference = None
                     if spoken_subtitles:
                         preference = -1
                         format_id += '-spoken-subtitles'
+                    if sign_language:
+                        preference = -1
+                        format_id += "-sign-language"
                     if target == 'HDS':
                         f4m_formats = self._extract_f4m_formats(
                             uri + '?hdcore=3.3.0&plugin=aasp-3.3.0.99.43',

From 8d65880e247d32e47dfb42616b0734639d4c070f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 May 2017 15:37:09 +0700
Subject: [PATCH 276/611] [drtv] Improve extraction and update tests (closes
 #13013, closes #13016)

---
 youtube_dl/extractor/drtv.py | 36 +++++++++++++++++-------------------
 1 file changed, 17 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index e6f72ac50..c84624f1e 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -20,7 +20,7 @@ class DRTVIE(InfoExtractor):
     IE_NAME = 'drtv'
     _TESTS = [{
         'url': 'https://www.dr.dk/tv/se/boern/ultra/klassen-ultra/klassen-darlig-taber-10',
-        'md5': '25e659cccc9a2ed956110a299fdf5983',
+        'md5': '7ae17b4e18eb5d29212f424a7511c184',
         'info_dict': {
             'id': 'klassen-darlig-taber-10',
             'ext': 'mp4',
@@ -30,32 +30,36 @@ class DRTVIE(InfoExtractor):
             'upload_date': '20160823',
             'duration': 606.84,
         },
-        'params': {
-            'skip_download': True,
-        },
     }, {
+        # embed
         'url': 'https://www.dr.dk/nyheder/indland/live-christianias-rydning-af-pusher-street-er-i-gang',
-        'md5': '2c37175c718155930f939ef59952474a',
         'info_dict': {
             'id': 'christiania-pusher-street-ryddes-drdkrjpo',
             'ext': 'mp4',
             'title': 'LIVE Christianias rydning af Pusher Street er i gang',
-            'description': '- Det er det fedeste, der er sket i 20 år, fortæller christianit til DR Nyheder.',
+            'description': 'md5:2a71898b15057e9b97334f61d04e6eb5',
             'timestamp': 1472800279,
             'upload_date': '20160902',
             'duration': 131.4,
         },
+        'params': {
+            'skip_download': True,
+        },
     }, {
+        # with SignLanguage formats
         'url': 'https://www.dr.dk/tv/se/historien-om-danmark/-/historien-om-danmark-stenalder',
-        'md5': '7c8ca12e6c3d3e3edd59ba5a9b7ca10a',
         'info_dict': {
             'id': 'historien-om-danmark-stenalder',
             'ext': 'mp4',
             'title': 'Historien om Danmark: Stenalder (1)',
-            'description': 'Én fascinerende historie om tusindvis af år, hvor vores land bliver skabt ud af is og vand, og hvor de første danskere ankommer til vores egn. Det bliver en rejse ind i urtiden og det liv, som urtidsjægerne har levet i skovene og ved havet og helt frem til bondestenalderen. Gennem skeletfund afslører eksperter, hvordan vores forfædre har set ud i stenalderen og hvorfor stenaldermennesket byggede de imponerende jættestuer, som ligger overalt i det danske.',
+            'description': 'md5:8c66dcbc1669bbc6f873879880f37f2a',
             'timestamp': 1490401996,
             'upload_date': '20170325',
             'duration': 3502.04,
+            'formats': 'mincount:20',
+        },
+        'params': {
+            'skip_download': True,
         },
     }]
 
@@ -97,26 +101,20 @@ class DRTVIE(InfoExtractor):
             kind = asset.get('Kind')
             if kind == 'Image':
                 thumbnail = asset.get('Uri')
-            preference = 0
-
-            sign_language = asset.get('Target') == 'SignLanguage'
-
-            if kind in ('VideoResource', 'AudioResource'):
+            elif kind in ('VideoResource', 'AudioResource'):
                 duration = float_or_none(asset.get('DurationInMilliseconds'), 1000)
                 restricted_to_denmark = asset.get('RestrictedToDenmark')
-                spoken_subtitles = asset.get('Target') == 'SpokenSubtitles'
+                asset_target = asset.get('Target')
                 for link in asset.get('Links', []):
                     uri = link.get('Uri')
                     if not uri:
                         continue
                     target = link.get('Target')
                     format_id = target or ''
-                    if spoken_subtitles:
+                    preference = None
+                    if asset_target in ('SpokenSubtitles', 'SignLanguage'):
                         preference = -1
-                        format_id += '-spoken-subtitles'
-                    if sign_language:
-                        preference = -1
-                        format_id += "-sign-language"
+                        format_id += '-%s' % asset_target
                     if target == 'HDS':
                         f4m_formats = self._extract_f4m_formats(
                             uri + '?hdcore=3.3.0&plugin=aasp-3.3.0.99.43',

From 5d0968f0af3ce2a7da9a5f3098c6436f07c661aa Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 9 May 2017 11:14:29 +0100
Subject: [PATCH 277/611] [packtpub] add support for authentication(closes
 #12622)

---
 youtube_dl/extractor/packtpub.py | 39 +++++++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/packtpub.py b/youtube_dl/extractor/packtpub.py
index 881f3bcc7..bb668c999 100644
--- a/youtube_dl/extractor/packtpub.py
+++ b/youtube_dl/extractor/packtpub.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_str
+from ..compat import (
+    compat_str,
+    compat_HTTPError,
+)
 from ..utils import (
     clean_html,
     ExtractorError,
@@ -11,6 +14,7 @@ from ..utils import (
     strip_or_none,
     unified_timestamp,
     urljoin,
+    urlencode_postdata,
 )
 
 
@@ -34,6 +38,32 @@ class PacktPubIE(PacktPubBaseIE):
             'upload_date': '20170331',
         },
     }
+    _NETRC_MACHINE = 'packtpub'
+    _TOKEN = None
+
+    def _real_initialize(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+        webpage = self._download_webpage(self._PACKT_BASE, None)
+        login_form = self._form_hidden_inputs(
+            'packt-user-login-form', webpage)
+        login_form.update({
+            'email': username,
+            'password': password,
+        })
+        self._download_webpage(
+            self._PACKT_BASE, None, 'Logging in as %s' % username,
+            data=urlencode_postdata(login_form))
+        try:
+            self._TOKEN = self._download_json(
+                '%s/users/tokens/sessions' % self._MAPT_REST, None,
+                'Downloading Authorization Token')['data']['token']
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code in (401, 404):
+                message = self._parse_json(e.cause.read().decode(), None)['message']
+                raise ExtractorError(message, expected=True)
+            raise
 
     def _handle_error(self, response):
         if response.get('status') != 'success':
@@ -51,14 +81,17 @@ class PacktPubIE(PacktPubBaseIE):
         course_id, chapter_id, video_id = mobj.group(
             'course_id', 'chapter_id', 'id')
 
+        headers = {}
+        if self._TOKEN:
+            headers['Authorization'] = self._TOKEN
         video = self._download_json(
             '%s/users/me/products/%s/chapters/%s/sections/%s'
             % (self._MAPT_REST, course_id, chapter_id, video_id), video_id,
-            'Downloading JSON video')['data']
+            'Downloading JSON video', headers=headers)['data']
 
         content = video.get('content')
         if not content:
-            raise ExtractorError('This video is locked', expected=True)
+            self.raise_login_required('This video is locked')
 
         video_url = content['file']
 

From 6d1ded75021ca76f14c4fe8d3aa698704bae1dd9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 May 2017 23:07:01 +0700
Subject: [PATCH 278/611] [francetv] Adapt to site redesign (closes #13034)

---
 youtube_dl/extractor/extractors.py |   4 +-
 youtube_dl/extractor/francetv.py   | 217 +++++++++++------------------
 2 files changed, 86 insertions(+), 135 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 968cca9d2..61a6f1013 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -350,9 +350,9 @@ from .foxsports import FoxSportsIE
 from .franceculture import FranceCultureIE
 from .franceinter import FranceInterIE
 from .francetv import (
-    PluzzIE,
-    FranceTvInfoIE,
     FranceTVIE,
+    FranceTVEmbedIE,
+    FranceTVInfoIE,
     GenerationQuoiIE,
     CultureboxIE,
 )
diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 48d43ae58..6e1a537a0 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -21,11 +21,13 @@ from .dailymotion import (
 
 
 class FranceTVBaseInfoExtractor(InfoExtractor):
-    def _extract_video(self, video_id, catalogue):
+    def _extract_video(self, video_id, catalogue=None):
         info = self._download_json(
-            'http://webservices.francetelevisions.fr/tools/getInfosOeuvre/v2/?idDiffusion=%s&catalogue=%s'
-            % (video_id, catalogue),
-            video_id, 'Downloading video JSON')
+            'https://sivideo.webservices.francetelevisions.fr/tools/getInfosOeuvre/v2/',
+            video_id, 'Downloading video JSON', query={
+                'idDiffusion': video_id,
+                'catalogue': catalogue or '',
+            })
 
         if info.get('status') == 'NOK':
             raise ExtractorError(
@@ -109,27 +111,94 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
         }
 
 
-class PluzzIE(FranceTVBaseInfoExtractor):
-    IE_NAME = 'pluzz.francetv.fr'
-    _VALID_URL = r'https?://(?:m\.)?pluzz\.francetv\.fr/videos/(?P<id>.+?)\.html'
+class FranceTVIE(FranceTVBaseInfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?france\.tv/(?:[^/]+/)+(?P<id>[^/]+)\.html'
 
-    # Can't use tests, videos expire in 7 days
+    _TESTS = [{
+        'url': 'https://www.france.tv/france-2/13h15-le-dimanche/140921-les-mysteres-de-jesus.html',
+        'info_dict': {
+            'id': '157550144',
+            'ext': 'mp4',
+            'title': '13h15, le dimanche... - Les mystères de Jésus',
+            'description': 'md5:75efe8d4c0a8205e5904498ffe1e1a42',
+            'timestamp': 1494156300,
+            'upload_date': '20170507',
+        },
+        'params': {
+            # m3u8 downloads
+            'skip_download': True,
+        },
+    }, {
+        # france3
+        'url': 'https://www.france.tv/france-3/des-chiffres-et-des-lettres/139063-emission-du-mardi-9-mai-2017.html',
+        'only_matching': True,
+    }, {
+        # france4
+        'url': 'https://www.france.tv/france-4/hero-corp/saison-1/134151-apres-le-calme.html',
+        'only_matching': True,
+    }, {
+        # france5
+        'url': 'https://www.france.tv/france-5/c-a-dire/saison-10/137013-c-a-dire.html',
+        'only_matching': True,
+    }, {
+        # franceo
+        'url': 'https://www.france.tv/france-o/archipels/132249-mon-ancetre-l-esclave.html',
+        'only_matching': True,
+    }, {
+        # france2 live
+        'url': 'https://www.france.tv/france-2/direct.html',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.france.tv/documentaires/histoire/136517-argentine-les-500-bebes-voles-de-la-dictature.html',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.france.tv/jeux-et-divertissements/divertissements/133965-le-web-contre-attaque.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
 
         webpage = self._download_webpage(url, display_id)
 
-        video_id = self._html_search_meta(
-            'id_video', webpage, 'video id', default=None)
+        catalogue = None
+        video_id = self._search_regex(
+            r'data-main-video=(["\'])(?P<id>(?:(?!\1).)+)\1',
+            webpage, 'video id', default=None, group='id')
+
         if not video_id:
-            video_id = self._search_regex(
-                r'data-diffusion=["\'](\d+)', webpage, 'video id')
-
-        return self._extract_video(video_id, 'Pluzz')
+            video_id, catalogue = self._html_search_regex(
+                r'(?:href=|player\.setVideo\(\s*)"http://videos?\.francetv\.fr/video/([^@]+@[^"]+)"',
+                webpage, 'video ID').split('@')
+        return self._extract_video(video_id, catalogue)
 
 
-class FranceTvInfoIE(FranceTVBaseInfoExtractor):
+class FranceTVEmbedIE(FranceTVBaseInfoExtractor):
+    _VALID_URL = r'https?://embed\.francetv\.fr/*\?.*?\bue=(?P<id>[^&]+)'
+
+    _TEST = {
+        'url': 'http://embed.francetv.fr/?ue=7fd581a2ccf59d2fc5719c5c13cf6961',
+        'info_dict': {
+            'id': 'NI_983319',
+            'ext': 'mp4',
+            'title': 'Le Pen Reims',
+            'upload_date': '20170505',
+            'timestamp': 1493981780,
+            'duration': 16,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        video = self._download_json(
+            'http://api-embed.webservices.francetelevisions.fr/key/%s' % video_id,
+            video_id)
+
+        return self._extract_video(video['video_id'], video.get('catalog'))
+
+
+class FranceTVInfoIE(FranceTVBaseInfoExtractor):
     IE_NAME = 'francetvinfo.fr'
     _VALID_URL = r'https?://(?:www|mobile|france3-regions)\.francetvinfo\.fr/(?:[^/]+/)*(?P<title>[^/?#&.]+)'
 
@@ -233,124 +302,6 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
         return self._extract_video(video_id, catalogue)
 
 
-class FranceTVIE(FranceTVBaseInfoExtractor):
-    IE_NAME = 'francetv'
-    IE_DESC = 'France 2, 3, 4, 5 and Ô'
-    _VALID_URL = r'''(?x)
-                    https?://
-                        (?:
-                            (?:www\.)?france[2345o]\.fr/
-                                (?:
-                                    emissions/[^/]+/(?:videos|diffusions)|
-                                    emission/[^/]+|
-                                    videos|
-                                    jt
-                                )
-                            /|
-                            embed\.francetv\.fr/\?ue=
-                        )
-                        (?P<id>[^/?]+)
-                    '''
-
-    _TESTS = [
-        # france2
-        {
-            'url': 'http://www.france2.fr/emissions/13h15-le-samedi-le-dimanche/videos/75540104',
-            'md5': 'c03fc87cb85429ffd55df32b9fc05523',
-            'info_dict': {
-                'id': '109169362',
-                'ext': 'flv',
-                'title': '13h15, le dimanche...',
-                'description': 'md5:9a0932bb465f22d377a449be9d1a0ff7',
-                'upload_date': '20140914',
-                'timestamp': 1410693600,
-            },
-        },
-        # france3
-        {
-            'url': 'http://www.france3.fr/emissions/pieces-a-conviction/diffusions/13-11-2013_145575',
-            'md5': '679bb8f8921f8623bd658fa2f8364da0',
-            'info_dict': {
-                'id': '000702326_CAPP_PicesconvictionExtrait313022013_120220131722_Au',
-                'ext': 'mp4',
-                'title': 'Le scandale du prix des médicaments',
-                'description': 'md5:1384089fbee2f04fc6c9de025ee2e9ce',
-                'upload_date': '20131113',
-                'timestamp': 1384380000,
-            },
-        },
-        # france4
-        {
-            'url': 'http://www.france4.fr/emissions/hero-corp/videos/rhozet_herocorp_bonus_1_20131106_1923_06112013172108_F4',
-            'md5': 'a182bf8d2c43d88d46ec48fbdd260c1c',
-            'info_dict': {
-                'id': 'rhozet_herocorp_bonus_1_20131106_1923_06112013172108_F4',
-                'ext': 'mp4',
-                'title': 'Hero Corp Making of - Extrait 1',
-                'description': 'md5:c87d54871b1790679aec1197e73d650a',
-                'upload_date': '20131106',
-                'timestamp': 1383766500,
-            },
-        },
-        # france5
-        {
-            'url': 'http://www.france5.fr/emissions/c-a-dire/videos/quels_sont_les_enjeux_de_cette_rentree_politique__31-08-2015_908948?onglet=tous&page=1',
-            'md5': 'f6c577df3806e26471b3d21631241fd0',
-            'info_dict': {
-                'id': '123327454',
-                'ext': 'flv',
-                'title': 'C à dire ?! - Quels sont les enjeux de cette rentrée politique ?',
-                'description': 'md5:4a0d5cb5dce89d353522a84462bae5a4',
-                'upload_date': '20150831',
-                'timestamp': 1441035120,
-            },
-        },
-        # franceo
-        {
-            'url': 'http://www.franceo.fr/jt/info-soir/18-07-2015',
-            'md5': '47d5816d3b24351cdce512ad7ab31da8',
-            'info_dict': {
-                'id': '125377621',
-                'ext': 'flv',
-                'title': 'Infô soir',
-                'description': 'md5:01b8c6915a3d93d8bbbd692651714309',
-                'upload_date': '20150718',
-                'timestamp': 1437241200,
-                'duration': 414,
-            },
-        },
-        {
-            # francetv embed
-            'url': 'http://embed.francetv.fr/?ue=8d7d3da1e3047c42ade5a5d7dfd3fc87',
-            'info_dict': {
-                'id': 'EV_30231',
-                'ext': 'flv',
-                'title': 'Alcaline, le concert avec Calogero',
-                'description': 'md5:61f08036dcc8f47e9cfc33aed08ffaff',
-                'upload_date': '20150226',
-                'timestamp': 1424989860,
-                'duration': 5400,
-            },
-        },
-        {
-            'url': 'http://www.france4.fr/emission/highlander/diffusion-du-17-07-2015-04h05',
-            'only_matching': True,
-        },
-        {
-            'url': 'http://www.franceo.fr/videos/125377617',
-            'only_matching': True,
-        }
-    ]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        video_id, catalogue = self._html_search_regex(
-            r'(?:href=|player\.setVideo\(\s*)"http://videos?\.francetv\.fr/video/([^@]+@[^"]+)"',
-            webpage, 'video ID').split('@')
-        return self._extract_video(video_id, catalogue)
-
-
 class GenerationQuoiIE(InfoExtractor):
     IE_NAME = 'france2.fr:generation-quoi'
     _VALID_URL = r'https?://generation-quoi\.france2\.fr/portrait/(?P<id>[^/?#]+)'

From cbd84b58175144fa898c5dfd32c58d37ffaee2c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 May 2017 23:17:22 +0700
Subject: [PATCH 279/611] [ChangeLog] Actualize

---
 ChangeLog | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 5418bb95f..5137266f6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,26 @@
+version <unreleased>
+
+Core
+* [YoutubeDL] Force --restrict-filenames when no locale is set on all python
+  versions (#13027)
+
+Extractors
+* [francetv] Adapt to site redesign (#13034)
++ [packtpub] Add support for authentication (#12622)
+* [drtv] Lower preference for SignLanguage formats (#13013, #13016)
++ [cspan] Add support for brightcove live embeds (#13028)
+* [vrv] Extract DASH formats and subtitles
+* [funimation] Fix authentication (#13021)
+* [adultswim] Fix extraction (#8640, #10950, #11042, #12121)
+    + Add support for Adobe Pass authentication
+    + Add support for live streams
+    + Add support for show pages
+* [turner] Extract thumbnail, is_live and strip description
++ [nonktube] Add support for nonktube.com (#8647, #13024)
++ [nuevo] Pass headers to _extract_nuevo
+* [nbc] Improve extraction (#12364)
+
+
 version 2017.05.07
 
 Common

From a7ed6b341cce27b6b5731969d39711b15dca8d25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 May 2017 04:20:13 +0700
Subject: [PATCH 280/611] release 2017.05.09

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 5 +++--
 youtube_dl/version.py     | 2 +-
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 86d11e142..f7742ddcf 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.07*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.07**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.09*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.09**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.05.07
+[debug] youtube-dl version 2017.05.09
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 5137266f6..4775f8695 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.05.09
 
 Core
 * [YoutubeDL] Force --restrict-filenames when no locale is set on all python
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d867dfe65..2f27686b3 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -281,7 +281,8 @@
  - **france2.fr:generation-quoi**
  - **FranceCulture**
  - **FranceInter**
- - **francetv**: France 2, 3, 4, 5 and Ô
+ - **FranceTV**
+ - **FranceTVEmbed**
  - **francetvinfo.fr**
  - **Freesound**
  - **freespeech.org**
@@ -530,6 +531,7 @@
  - **NJPWWorld**: 新日本プロレスワールド
  - **NobelPrize**
  - **Noco**
+ - **NonkTube**
  - **Noovo**
  - **Normalboots**
  - **NosVideo**
@@ -602,7 +604,6 @@
  - **pluralsight**
  - **pluralsight:course**
  - **plus.google**: Google Plus
- - **pluzz.francetv.fr**
  - **podomatic**
  - **Pokemon**
  - **PolskieRadio**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index dc707071a..33a0c2130 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.05.07'
+__version__ = '2017.05.09'

From 39ee2638194cb40bfc6652ecb2fca2455e87e327 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 10 May 2017 08:50:30 +0100
Subject: [PATCH 281/611] use platform=desktop in assets url(closes #13041)

---
 youtube_dl/extractor/adultswim.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 9c37ea33c..acc4ce38d 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -126,7 +126,7 @@ class AdultSwimIE(TurnerBaseIE):
             video_id = video_data['id']
 
         info = self._extract_cvp_info(
-            'http://www.adultswim.com/videos/api/v0/assets?id=' + video_id,
+            'http://www.adultswim.com/videos/api/v0/assets?platform=desktop&id=' + video_id,
             video_id, {
                 'secure': {
                     'media_src': 'http://androidhls-secure.cdn.turner.com/adultswim/big',

From 3166b1f0ac16769190f6ab1e7e5b0e45bb63d482 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 May 2017 22:35:10 +0700
Subject: [PATCH 282/611] [myspace] Improve _VALID_URL (closes #13040)

---
 youtube_dl/extractor/myspace.py | 100 ++++++++++++++++----------------
 1 file changed, 49 insertions(+), 51 deletions(-)

diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index f281238c9..e164d5940 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -12,64 +12,62 @@ from ..utils import (
 
 
 class MySpaceIE(InfoExtractor):
-    _VALID_URL = r'https?://myspace\.com/([^/]+)/(?P<mediatype>video/[^/]+/|music/song/.*?)(?P<id>\d+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        myspace\.com/[^/]+/
+                        (?P<mediatype>
+                            video/[^/]+/(?P<video_id>\d+)|
+                            music/song/[^/?#&]+-(?P<song_id>\d+)-\d+(?:[/?#&]|$)
+                        )
+                    '''
 
-    _TESTS = [
-        {
-            'url': 'https://myspace.com/fiveminutestothestage/video/little-big-town/109594919',
-            'md5': '9c1483c106f4a695c47d2911feed50a7',
-            'info_dict': {
-                'id': '109594919',
-                'ext': 'mp4',
-                'title': 'Little Big Town',
-                'description': 'This country quartet was all smiles while playing a sold out show at the Pacific Amphitheatre in Orange County, California.',
-                'uploader': 'Five Minutes to the Stage',
-                'uploader_id': 'fiveminutestothestage',
-                'timestamp': 1414108751,
-                'upload_date': '20141023',
-            },
+    _TESTS = [{
+        'url': 'https://myspace.com/fiveminutestothestage/video/little-big-town/109594919',
+        'md5': '9c1483c106f4a695c47d2911feed50a7',
+        'info_dict': {
+            'id': '109594919',
+            'ext': 'mp4',
+            'title': 'Little Big Town',
+            'description': 'This country quartet was all smiles while playing a sold out show at the Pacific Amphitheatre in Orange County, California.',
+            'uploader': 'Five Minutes to the Stage',
+            'uploader_id': 'fiveminutestothestage',
+            'timestamp': 1414108751,
+            'upload_date': '20141023',
         },
+    }, {
         # songs
-        {
-            'url': 'https://myspace.com/killsorrow/music/song/of-weakened-soul...-93388656-103880681',
-            'md5': '1d7ee4604a3da226dd69a123f748b262',
-            'info_dict': {
-                'id': '93388656',
-                'ext': 'm4a',
-                'title': 'Of weakened soul...',
-                'uploader': 'Killsorrow',
-                'uploader_id': 'killsorrow',
-            },
-        }, {
-            'add_ie': ['Youtube'],
-            'url': 'https://myspace.com/threedaysgrace/music/song/animal-i-have-become-28400208-28218041',
-            'info_dict': {
-                'id': 'xqds0B_meys',
-                'ext': 'webm',
-                'title': 'Three Days Grace - Animal I Have Become',
-                'description': 'md5:8bd86b3693e72a077cf863a8530c54bb',
-                'uploader': 'ThreeDaysGraceVEVO',
-                'uploader_id': 'ThreeDaysGraceVEVO',
-                'upload_date': '20091002',
-            },
-        }, {
-            'add_ie': ['Youtube'],
-            'url': 'https://myspace.com/starset2/music/song/first-light-95799905-106964426',
-            'info_dict': {
-                'id': 'ypWvQgnJrSU',
-                'ext': 'mp4',
-                'title': 'Starset - First Light',
-                'description': 'md5:2d5db6c9d11d527683bcda818d332414',
-                'uploader': 'Yumi K',
-                'uploader_id': 'SorenPromotions',
-                'upload_date': '20140725',
-            }
+        'url': 'https://myspace.com/killsorrow/music/song/of-weakened-soul...-93388656-103880681',
+        'md5': '1d7ee4604a3da226dd69a123f748b262',
+        'info_dict': {
+            'id': '93388656',
+            'ext': 'm4a',
+            'title': 'Of weakened soul...',
+            'uploader': 'Killsorrow',
+            'uploader_id': 'killsorrow',
         },
-    ]
+    }, {
+        'add_ie': ['Youtube'],
+        'url': 'https://myspace.com/threedaysgrace/music/song/animal-i-have-become-28400208-28218041',
+        'info_dict': {
+            'id': 'xqds0B_meys',
+            'ext': 'webm',
+            'title': 'Three Days Grace - Animal I Have Become',
+            'description': 'md5:8bd86b3693e72a077cf863a8530c54bb',
+            'uploader': 'ThreeDaysGraceVEVO',
+            'uploader_id': 'ThreeDaysGraceVEVO',
+            'upload_date': '20091002',
+        },
+    }, {
+        'url': 'https://myspace.com/starset2/music/song/first-light-95799905-106964426',
+        'only_matching': True,
+    }, {
+        'url': 'https://myspace.com/thelargemouthbassband/music/song/02-pure-eyes.mp3-94422330-105113388',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = mobj.group('video_id') or mobj.group('song_id')
         is_song = mobj.group('mediatype').startswith('music/song')
         webpage = self._download_webpage(url, video_id)
         player_url = self._search_regex(

From 7274f3d0e9e7d76d00fa3d438df1e1dc43143435 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 12 May 2017 18:44:10 +0800
Subject: [PATCH 283/611] [douyu] Support Douyu shows (closes #12228)

---
 ChangeLog                          |  6 +++
 youtube_dl/extractor/douyutv.py    | 82 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  5 +-
 3 files changed, 92 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 4775f8695..43431ac93 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractor
++ [douyu] Support Douyu shows (#12228)
+
+
 version 2017.05.09
 
 Core
diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index d22133d24..9757f4422 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -3,11 +3,14 @@ from __future__ import unicode_literals
 
 import time
 import hashlib
+import re
 
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     unescapeHTML,
+    unified_strdate,
+    urljoin,
 )
 
 
@@ -117,3 +120,82 @@ class DouyuTVIE(InfoExtractor):
             'uploader': uploader,
             'is_live': True,
         }
+
+
+class DouyuShowIE(InfoExtractor):
+    _VALID_URL = r'https?://v(?:mobile)?\.douyu\.com/show/(?P<id>[0-9a-zA-Z]+)'
+
+    _TESTS = [{
+        'url': 'https://v.douyu.com/show/rjNBdvnVXNzvE2yw',
+        'md5': '0c2cfd068ee2afe657801269b2d86214',
+        'info_dict': {
+            'id': 'rjNBdvnVXNzvE2yw',
+            'ext': 'mp4',
+            'title': '陈一发儿：砒霜 我有个室友系列！04-01 22点场',
+            'duration': 7150.08,
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'uploader': '陈一发儿',
+            'uploader_id': 'XrZwYelr5wbK',
+            'uploader_url': 'https://v.douyu.com/author/XrZwYelr5wbK',
+            'upload_date': '20170402',
+        },
+    }, {
+        'url': 'https://vmobile.douyu.com/show/rjNBdvnVXNzvE2yw',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        url = url.replace('vmobile.', 'v.')
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        room_info = self._parse_json(self._search_regex(
+            r'var\s+\$ROOM\s*=\s*({.+});', webpage, 'room info'), video_id)
+
+        video_info = None
+
+        for trial in range(5):
+            # Sometimes Douyu rejects our request. Let's try it more times
+            try:
+                video_info = self._download_json(
+                    'https://vmobile.douyu.com/video/getInfo', video_id,
+                    query={'vid': video_id},
+                    headers={
+                        'Referer': url,
+                        'x-requested-with': 'XMLHttpRequest',
+                    })
+                break
+            except ExtractorError:
+                self._sleep(1, video_id)
+
+        if not video_info:
+            raise ExtractorError('Can\'t fetch video info')
+
+        formats = self._extract_m3u8_formats(
+            video_info['data']['video_url'], video_id,
+            entry_protocol='m3u8_native', ext='mp4')
+
+        upload_date = unified_strdate(self._html_search_regex(
+            r'<em>上传时间：</em><span>([^<]+)</span>', webpage,
+            'upload date', fatal=False))
+
+        uploader = uploader_id = uploader_url = None
+        mobj = re.search(
+            r'(?m)<a[^>]+href="/author/([0-9a-zA-Z]+)".+?<strong[^>]+title="([^"]+)"',
+            webpage)
+        if mobj:
+            uploader_id, uploader = mobj.groups()
+            uploader_url = urljoin(url, '/author/' + uploader_id)
+
+        return {
+            'id': video_id,
+            'title': room_info['name'],
+            'formats': formats,
+            'duration': room_info.get('duration'),
+            'thumbnail': room_info.get('pic'),
+            'upload_date': upload_date,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'uploader_url': uploader_url,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 61a6f1013..510c35fcb 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -251,7 +251,10 @@ from .democracynow import DemocracynowIE
 from .dfb import DFBIE
 from .dhm import DHMIE
 from .dotsub import DotsubIE
-from .douyutv import DouyuTVIE
+from .douyutv import (
+    DouyuShowIE,
+    DouyuTVIE,
+)
 from .dplay import (
     DPlayIE,
     DPlayItIE,

From 116283ff640ec77de7359ddaffaa8dfbb50f6e17 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 12 May 2017 19:15:33 +0800
Subject: [PATCH 284/611] [liveleak] Fix extraction (#12053)

---
 ChangeLog                        |  1 +
 youtube_dl/extractor/liveleak.py | 83 +++++++++++++-------------------
 2 files changed, 34 insertions(+), 50 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 43431ac93..96e4ce8a1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractor
+* [liveleak] Fix extraction (#12053)
 + [douyu] Support Douyu shows (#12228)
 
 
diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index c7de65353..c54519636 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import json
 import re
 
 from .common import InfoExtractor
@@ -11,10 +10,10 @@ class LiveLeakIE(InfoExtractor):
     _VALID_URL = r'https?://(?:\w+\.)?liveleak\.com/view\?(?:.*?)i=(?P<id>[\w_]+)(?:.*)'
     _TESTS = [{
         'url': 'http://www.liveleak.com/view?i=757_1364311680',
-        'md5': '50f79e05ba149149c1b4ea961223d5b3',
+        'md5': '0813c2430bea7a46bf13acf3406992f4',
         'info_dict': {
             'id': '757_1364311680',
-            'ext': 'flv',
+            'ext': 'mp4',
             'description': 'extremely bad day for this guy..!',
             'uploader': 'ljfriel2',
             'title': 'Most unlucky car accident',
@@ -22,7 +21,7 @@ class LiveLeakIE(InfoExtractor):
         }
     }, {
         'url': 'http://www.liveleak.com/view?i=f93_1390833151',
-        'md5': 'b13a29626183c9d33944e6a04f41aafc',
+        'md5': 'd3f1367d14cc3c15bf24fbfbe04b9abf',
         'info_dict': {
             'id': 'f93_1390833151',
             'ext': 'mp4',
@@ -32,6 +31,7 @@ class LiveLeakIE(InfoExtractor):
             'thumbnail': r're:^https?://.*\.jpg$'
         }
     }, {
+        # Prochan embed
         'url': 'http://www.liveleak.com/view?i=4f7_1392687779',
         'md5': '42c6d97d54f1db107958760788c5f48f',
         'info_dict': {
@@ -41,11 +41,13 @@ class LiveLeakIE(InfoExtractor):
             'uploader': 'CapObveus',
             'title': 'Man is Fatally Struck by Reckless Car While Packing up a Moving Truck',
             'age_limit': 18,
-        }
+        },
+        'skip': 'Video is dead',
     }, {
         # Covers https://github.com/rg3/youtube-dl/pull/5983
+        # Multiple resolutions
         'url': 'http://www.liveleak.com/view?i=801_1409392012',
-        'md5': '0b3bec2d888c20728ca2ad3642f0ef15',
+        'md5': 'c3a449dbaca5c0d1825caecd52a57d7b',
         'info_dict': {
             'id': '801_1409392012',
             'ext': 'mp4',
@@ -93,57 +95,38 @@ class LiveLeakIE(InfoExtractor):
             webpage, 'age limit', default=None))
         video_thumbnail = self._og_search_thumbnail(webpage)
 
-        sources_raw = self._search_regex(
-            r'(?s)sources:\s*(\[.*?\]),', webpage, 'video URLs', default=None)
-        if sources_raw is None:
-            alt_source = self._search_regex(
-                r'(file: ".*?"),', webpage, 'video URL', default=None)
-            if alt_source:
-                sources_raw = '[{ %s}]' % alt_source
-            else:
-                # Maybe an embed?
-                embed_url = self._search_regex(
-                    r'<iframe[^>]+src="(https?://(?:www\.)?(?:prochan|youtube)\.com/embed[^"]+)"',
-                    webpage, 'embed URL')
-                return {
-                    '_type': 'url_transparent',
-                    'url': embed_url,
-                    'id': video_id,
-                    'title': video_title,
-                    'description': video_description,
-                    'uploader': video_uploader,
-                    'age_limit': age_limit,
-                }
+        entries = self._parse_html5_media_entries(url, webpage, video_id)
+        if not entries:
+            # Maybe an embed?
+            embed_url = self._search_regex(
+                r'<iframe[^>]+src="((?:https?:)?//(?:www\.)?(?:prochan|youtube)\.com/embed[^"]+)"',
+                webpage, 'embed URL')
+            return {
+                '_type': 'url_transparent',
+                'url': embed_url,
+                'id': video_id,
+                'title': video_title,
+                'description': video_description,
+                'uploader': video_uploader,
+                'age_limit': age_limit,
+            }
 
-        sources_json = re.sub(r'\s([a-z]+):\s', r'"\1": ', sources_raw)
-        sources = json.loads(sources_json)
+        info_dict = entries[0]
 
-        formats = [{
-            'format_id': '%s' % i,
-            'format_note': s.get('label'),
-            'url': s['file'],
-        } for i, s in enumerate(sources)]
+        for a_format in info_dict['formats']:
+            if not a_format.get('height'):
+                a_format['height'] = self._search_regex(
+                    r'([0-9]+)p\.mp4', a_format['url'], 'height label', default=None)
 
-        for i, s in enumerate(sources):
-            # Removing '.h264_*.mp4' gives the raw video, which is essentially
-            # the same video without the LiveLeak logo at the top (see
-            # https://github.com/rg3/youtube-dl/pull/4768)
-            orig_url = re.sub(r'\.h264_.+?\.mp4', '', s['file'])
-            if s['file'] != orig_url:
-                formats.append({
-                    'format_id': 'original-%s' % i,
-                    'format_note': s.get('label'),
-                    'url': orig_url,
-                    'preference': 1,
-                })
-        self._sort_formats(formats)
+        self._sort_formats(info_dict['formats'])
 
-        return {
+        info_dict.update({
             'id': video_id,
             'title': video_title,
             'description': video_description,
             'uploader': video_uploader,
-            'formats': formats,
             'age_limit': age_limit,
             'thumbnail': video_thumbnail,
-        }
+        })
+
+        return info_dict

From 6c52477f595176d16f063407bcf44439b9c26022 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 12 May 2017 16:17:32 +0100
Subject: [PATCH 285/611] [condenast] improve embed support

---
 youtube_dl/extractor/condenast.py | 81 ++++++++++++++++++++-----------
 youtube_dl/extractor/generic.py   | 16 ------
 2 files changed, 54 insertions(+), 43 deletions(-)

diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index d3463b874..0c3f0c0e4 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -16,7 +16,6 @@ from ..utils import (
     mimetype2ext,
     orderedSet,
     parse_iso8601,
-    remove_end,
 )
 
 
@@ -50,10 +49,17 @@ class CondeNastIE(InfoExtractor):
         'wmagazine': 'W Magazine',
     }
 
-    _VALID_URL = r'https?://(?:video|www|player)\.(?P<site>%s)\.com/(?P<type>watch|series|video|embed(?:js)?)/(?P<id>[^/?#]+)' % '|'.join(_SITES.keys())
+    _VALID_URL = r'''(?x)https?://(?:video|www|player(?:-backend)?)\.(?:%s)\.com/
+        (?:
+            (?:
+                embed(?:js)?|
+                (?:script|inline)/video
+            )/(?P<id>[0-9a-f]{24})(?:/(?P<player_id>[0-9a-f]{24}))?(?:.+?\btarget=(?P<target>[^&]+))?|
+            (?P<type>watch|series|video)/(?P<display_id>[^/?#]+)
+        )''' % '|'.join(_SITES.keys())
     IE_DESC = 'Condé Nast media group: %s' % ', '.join(sorted(_SITES.values()))
 
-    EMBED_URL = r'(?:https?:)?//player\.(?P<site>%s)\.com/(?P<type>embed(?:js)?)/.+?' % '|'.join(_SITES.keys())
+    EMBED_URL = r'(?:https?:)?//player(?:-backend)?\.(?:%s)\.com/(?:embed(?:js)?|(?:script|inline)/video)/.+?' % '|'.join(_SITES.keys())
 
     _TESTS = [{
         'url': 'http://video.wired.com/watch/3d-printed-speakers-lit-with-led',
@@ -89,6 +95,12 @@ class CondeNastIE(InfoExtractor):
             'upload_date': '20150916',
             'timestamp': 1442434955,
         }
+    }, {
+        'url': 'https://player.cnevids.com/inline/video/59138decb57ac36b83000005.js?target=js-cne-player',
+        'only_matching': True,
+    }, {
+        'url': 'http://player-backend.cnevids.com/script/video/59138decb57ac36b83000005.js',
+        'only_matching': True,
     }]
 
     def _extract_series(self, url, webpage):
@@ -104,7 +116,7 @@ class CondeNastIE(InfoExtractor):
         entries = [self.url_result(build_url(path), 'CondeNast') for path in paths]
         return self.playlist_result(entries, playlist_title=title)
 
-    def _extract_video(self, webpage, url_type):
+    def _extract_video_params(self, webpage):
         query = {}
         params = self._search_regex(
             r'(?s)var params = {(.+?)}[;,]', webpage, 'player params', default=None)
@@ -123,17 +135,30 @@ class CondeNastIE(InfoExtractor):
                 'playerId': params['data-player'],
                 'target': params['id'],
             })
-        video_id = query['videoId']
+        return query
+
+    def _extract_video(self, params):
+        video_id = params['videoId']
+
         video_info = None
-        info_page = self._download_json(
-            'http://player.cnevids.com/player/video.js',
-            video_id, 'Downloading video info', fatal=False, query=query)
-        if info_page:
-            video_info = info_page.get('video')
-        if not video_info:
+        if params.get('playerId'):
+            info_page = self._download_json(
+                'http://player.cnevids.com/player/video.js',
+                video_id, 'Downloading video info', fatal=False, query=params)
+            if info_page:
+                video_info = info_page.get('video')
+            if not video_info:
+                info_page = self._download_webpage(
+                    'http://player.cnevids.com/player/loader.js',
+                    video_id, 'Downloading loader info', query=params)
+        else:
             info_page = self._download_webpage(
-                'http://player.cnevids.com/player/loader.js',
-                video_id, 'Downloading loader info', query=query)
+                'https://player.cnevids.com/inline/video/%s.js' % video_id,
+                video_id, 'Downloading inline info', query={
+                    'target': params.get('target', 'embedplayer')
+                })
+
+        if not video_info:
             video_info = self._parse_json(
                 self._search_regex(
                     r'(?s)var\s+config\s*=\s*({.+?});', info_page, 'config'),
@@ -161,9 +186,7 @@ class CondeNastIE(InfoExtractor):
             })
         self._sort_formats(formats)
 
-        info = self._search_json_ld(
-            webpage, video_id, fatal=False) if url_type != 'embed' else {}
-        info.update({
+        return {
             'id': video_id,
             'formats': formats,
             'title': title,
@@ -174,22 +197,26 @@ class CondeNastIE(InfoExtractor):
             'series': video_info.get('series_title'),
             'season': video_info.get('season_title'),
             'timestamp': parse_iso8601(video_info.get('premiere_date')),
-        })
-        return info
+            'categories': video_info.get('categories'),
+        }
 
     def _real_extract(self, url):
-        site, url_type, item_id = re.match(self._VALID_URL, url).groups()
+        video_id, player_id, target, url_type, display_id = re.match(self._VALID_URL, url).groups()
 
-        # Convert JS embed to regular embed
-        if url_type == 'embedjs':
-            parsed_url = compat_urlparse.urlparse(url)
-            url = compat_urlparse.urlunparse(parsed_url._replace(
-                path=remove_end(parsed_url.path, '.js').replace('/embedjs/', '/embed/')))
-            url_type = 'embed'
+        if video_id:
+            return self._extract_video({
+                'videoId': video_id,
+                'playerId': player_id,
+                'target': target,
+            })
 
-        webpage = self._download_webpage(url, item_id)
+        webpage = self._download_webpage(url, display_id)
 
         if url_type == 'series':
             return self._extract_series(url, webpage)
         else:
-            return self._extract_video(webpage, url_type)
+            params = self._extract_video_params(webpage)
+            info = self._search_json_ld(
+                webpage, display_id, fatal=False)
+            info.update(self._extract_video(params))
+            return info
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b06f43446..635043c9e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2146,22 +2146,6 @@ class GenericIE(InfoExtractor):
         if svt_url:
             return self.url_result(svt_url, 'SVT')
 
-        # Look for embedded condenast player
-        matches = re.findall(
-            r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="(https?://player\.cnevids\.com/embed/[^"]+")',
-            webpage)
-        if matches:
-            return {
-                '_type': 'playlist',
-                'entries': [{
-                    '_type': 'url',
-                    'ie_key': 'CondeNast',
-                    'url': ma,
-                } for ma in matches],
-                'title': video_title,
-                'id': video_id,
-            }
-
         # Look for Bandcamp pages with custom domain
         mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage)
         if mobj is not None:

From 7ad4362357efe2d1b769c8cfd758af81e3131261 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 12 May 2017 16:32:38 +0100
Subject: [PATCH 286/611] [thescene] fix extraction(closes #13061)

---
 youtube_dl/extractor/thescene.py | 36 ++++++--------------------------
 1 file changed, 6 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/thescene.py b/youtube_dl/extractor/thescene.py
index b8504f0eb..cd642355c 100644
--- a/youtube_dl/extractor/thescene.py
+++ b/youtube_dl/extractor/thescene.py
@@ -3,10 +3,6 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 
 from ..compat import compat_urlparse
-from ..utils import (
-    int_or_none,
-    qualities,
-)
 
 
 class TheSceneIE(InfoExtractor):
@@ -24,6 +20,9 @@ class TheSceneIE(InfoExtractor):
             'season': 'Ready To Wear Spring 2013',
             'tags': list,
             'categories': list,
+            'upload_date': '20120913',
+            'timestamp': 1347512400,
+            'uploader': 'vogue',
         },
     }
 
@@ -37,32 +36,9 @@ class TheSceneIE(InfoExtractor):
             self._html_search_regex(
                 r'id=\'js-player-script\'[^>]+src=\'(.+?)\'', webpage, 'player url'))
 
-        player = self._download_webpage(player_url, display_id)
-        info = self._parse_json(
-            self._search_regex(
-                r'(?m)video\s*:\s*({.+?}),$', player, 'info json'),
-            display_id)
-
-        video_id = info['id']
-        title = info['title']
-
-        qualities_order = qualities(('low', 'high'))
-        formats = [{
-            'format_id': '{0}-{1}'.format(f['type'].split('/')[0], f['quality']),
-            'url': f['src'],
-            'quality': qualities_order(f['quality']),
-        } for f in info['sources']]
-        self._sort_formats(formats)
-
         return {
-            'id': video_id,
+            '_type': 'url_transparent',
             'display_id': display_id,
-            'title': title,
-            'formats': formats,
-            'thumbnail': info.get('poster_frame'),
-            'duration': int_or_none(info.get('duration')),
-            'series': info.get('series_title'),
-            'season': info.get('season_title'),
-            'tags': info.get('tags'),
-            'categories': info.get('categories'),
+            'url': player_url,
+            'ie_key': 'CondeNast',
         }

From 58bb4402834bc3ed5ec0046ee5a8d71e56532134 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 May 2017 21:51:58 +0700
Subject: [PATCH 287/611] [extractor/generic] Extract wistia embed code into
 separate method

---
 youtube_dl/extractor/generic.py | 30 +++++-------------------------
 youtube_dl/extractor/wistia.py  | 22 ++++++++++++++++++++++
 2 files changed, 27 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 635043c9e..8f9097945 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -88,6 +88,7 @@ from .rutube import RutubeIE
 from .limelight import LimelightBaseIE
 from .anvato import AnvatoIE
 from .washingtonpost import WashingtonPostIE
+from .wistia import WistiaIE
 
 
 class GenericIE(InfoExtractor):
@@ -2111,36 +2112,15 @@ class GenericIE(InfoExtractor):
                     playlists, video_id, video_title, lambda p: '//dailymotion.com/playlist/%s' % p)
 
         # Look for embedded Wistia player
-        match = re.search(
-            r'<(?:meta[^>]+?content|iframe[^>]+?src)=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
-        if match:
-            embed_url = self._proto_relative_url(
-                unescapeHTML(match.group('url')))
+        wistia_url = WistiaIE._extract_url(webpage)
+        if wistia_url:
             return {
                 '_type': 'url_transparent',
-                'url': embed_url,
-                'ie_key': 'Wistia',
+                'url': self._proto_relative_url(wistia_url),
+                'ie_key': WistiaIE.ie_key(),
                 'uploader': video_uploader,
             }
 
-        match = re.search(r'(?:id=["\']wistia_|data-wistia-?id=["\']|Wistia\.embed\(["\'])(?P<id>[^"\']+)', webpage)
-        if match:
-            return {
-                '_type': 'url_transparent',
-                'url': 'wistia:%s' % match.group('id'),
-                'ie_key': 'Wistia',
-                'uploader': video_uploader,
-            }
-
-        match = re.search(
-            r'''(?sx)
-                <script[^>]+src=(["'])(?:https?:)?//fast\.wistia\.com/assets/external/E-v1\.js\1[^>]*>.*?
-                <div[^>]+class=(["']).*?\bwistia_async_(?P<id>[a-z0-9]+)\b.*?\2
-            ''', webpage)
-        if match:
-            return self.url_result(self._proto_relative_url(
-                'wistia:%s' % match.group('id')), 'Wistia')
-
         # Look for SVT player
         svt_url = SVTIE._extract_url(webpage)
         if svt_url:
diff --git a/youtube_dl/extractor/wistia.py b/youtube_dl/extractor/wistia.py
index c634b8dec..2182d6fd4 100644
--- a/youtube_dl/extractor/wistia.py
+++ b/youtube_dl/extractor/wistia.py
@@ -1,10 +1,13 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     int_or_none,
     float_or_none,
+    unescapeHTML,
 )
 
 
@@ -34,6 +37,25 @@ class WistiaIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_url(webpage):
+        match = re.search(
+            r'<(?:meta[^>]+?content|iframe[^>]+?src)=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
+        if match:
+            return unescapeHTML(match.group('url'))
+
+        match = re.search(r'(?:id=["\']wistia_|data-wistia-?id=["\']|Wistia\.embed\(["\'])(?P<id>[^"\']+)', webpage)
+        if match:
+            return 'wistia:%s' % match.group('id')
+
+        match = re.search(
+            r'''(?sx)
+                <script[^>]+src=(["'])(?:https?:)?//fast\.wistia\.com/assets/external/E-v1\.js\1[^>]*>.*?
+                <div[^>]+class=(["']).*?\bwistia_async_(?P<id>[a-z0-9]+)\b.*?\2
+            ''', webpage)
+        if match:
+            return 'wistia:%s' % match.group('id')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From 7fc60f4ee9129c67aecacd6f9164e98dbf7601f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 May 2017 21:52:59 +0700
Subject: [PATCH 288/611] [upskill] Add extractor (closes #13043)

---
 youtube_dl/extractor/extractors.py |   4 +
 youtube_dl/extractor/upskill.py    | 176 +++++++++++++++++++++++++++++
 2 files changed, 180 insertions(+)
 create mode 100644 youtube_dl/extractor/upskill.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 510c35fcb..811519925 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1102,6 +1102,10 @@ from .uplynk import (
     UplynkIE,
     UplynkPreplayIE,
 )
+from .upskill import (
+    UpskillIE,
+    UpskillCourseIE,
+)
 from .urort import UrortIE
 from .urplay import URPlayIE
 from .usanetwork import USANetworkIE
diff --git a/youtube_dl/extractor/upskill.py b/youtube_dl/extractor/upskill.py
new file mode 100644
index 000000000..30297b4dd
--- /dev/null
+++ b/youtube_dl/extractor/upskill.py
@@ -0,0 +1,176 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from .wistia import WistiaIE
+from ..compat import compat_str
+from ..utils import (
+    clean_html,
+    ExtractorError,
+    get_element_by_class,
+    urlencode_postdata,
+    urljoin,
+)
+
+
+class UpskillBaseIE(InfoExtractor):
+    _LOGIN_URL = 'http://upskillcourses.com/sign_in'
+    _NETRC_MACHINE = 'upskill'
+
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        username, password = self._get_login_info()
+        if username is None:
+            return
+
+        login_page, urlh = self._download_webpage_handle(
+            self._LOGIN_URL, None, 'Downloading login page')
+
+        login_url = compat_str(urlh.geturl())
+
+        login_form = self._hidden_inputs(login_page)
+
+        login_form.update({
+            'user[email]': username,
+            'user[password]': password,
+        })
+
+        post_url = self._search_regex(
+            r'<form[^>]+action=(["\'])(?P<url>(?:(?!\1).)+)\1', login_page,
+            'post url', default=login_url, group='url')
+
+        if not post_url.startswith('http'):
+            post_url = urljoin(login_url, post_url)
+
+        response = self._download_webpage(
+            post_url, None, 'Logging in',
+            data=urlencode_postdata(login_form),
+            headers={
+                'Content-Type': 'application/x-www-form-urlencoded',
+                'Referer': login_url,
+            })
+
+        # Successful login
+        if any(re.search(p, response) for p in (
+                r'class=["\']user-signout',
+                r'<a[^>]+\bhref=["\']/sign_out',
+                r'>\s*Log out\s*<')):
+            return
+
+        message = get_element_by_class('alert', response)
+        if message is not None:
+            raise ExtractorError(
+                'Unable to login: %s' % clean_html(message), expected=True)
+
+        raise ExtractorError('Unable to log in')
+
+
+class UpskillIE(UpskillBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?upskillcourses\.com/courses/[^/]+/lectures/(?P<id>\d+)'
+
+    _TESTS = [{
+        'url': 'http://upskillcourses.com/courses/essential-web-developer-course/lectures/1747100',
+        'info_dict': {
+            'id': 'uzw6zw58or',
+            'ext': 'mp4',
+            'title': 'Welcome to the Course!',
+            'description': 'md5:8d66c13403783370af62ca97a7357bdd',
+            'duration': 138.763,
+            'timestamp': 1479846621,
+            'upload_date': '20161122',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://upskillcourses.com/courses/119763/lectures/1747100',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        wistia_url = WistiaIE._extract_url(webpage)
+        if not wistia_url:
+            if any(re.search(p, webpage) for p in (
+                    r'class=["\']lecture-contents-locked',
+                    r'>\s*Lecture contents locked',
+                    r'id=["\']lecture-locked')):
+                self.raise_login_required('Lecture contents locked')
+
+        title = self._og_search_title(webpage, default=None)
+
+        return {
+            '_type': 'url_transparent',
+            'url': wistia_url,
+            'ie_key': WistiaIE.ie_key(),
+            'title': title,
+        }
+
+
+class UpskillCourseIE(UpskillBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?upskillcourses\.com/courses/(?:enrolled/)?(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'http://upskillcourses.com/courses/essential-web-developer-course/',
+        'info_dict': {
+            'id': '119763',
+            'title': 'The Essential Web Developer Course (Free)',
+        },
+        'playlist_count': 192,
+    }, {
+        'url': 'http://upskillcourses.com/courses/119763/',
+        'only_matching': True,
+    }, {
+        'url': 'http://upskillcourses.com/courses/enrolled/119763',
+        'only_matching': True,
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return False if UpskillIE.suitable(url) else super(
+            UpskillCourseIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        course_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, course_id)
+
+        course_id = self._search_regex(
+            r'data-course-id=["\'](\d+)', webpage, 'course id',
+            default=course_id)
+
+        entries = []
+
+        for mobj in re.finditer(
+                r'(?s)(?P<li><li[^>]+class=(["\'])(?:(?!\2).)*?section-item[^>]+>.+?</li>)',
+                webpage):
+            li = mobj.group('li')
+            if 'fa-youtube-play' not in li:
+                continue
+            lecture_url = self._search_regex(
+                r'<a[^>]+href=(["\'])(?P<url>(?:(?!\1).)+)\1', li,
+                'lecture url', default=None, group='url')
+            if not lecture_url:
+                continue
+            lecture_id = self._search_regex(
+                r'/lectures/(\d+)', lecture_url, 'lecture id', default=None)
+            title = self._html_search_regex(
+                r'<span[^>]+class=["\']lecture-name[^>]+>([^<]+)', li,
+                'title', default=None)
+            entries.append(
+                self.url_result(
+                    urljoin('http://upskillcourses.com/', lecture_url),
+                    ie=UpskillIE.ie_key(), video_id=lecture_id,
+                    video_title=clean_html(title)))
+
+        course_title = self._html_search_regex(
+            (r'(?s)<img[^>]+class=["\']course-image[^>]+>\s*<h\d>(.+?)</h',
+             r'(?s)<h\d[^>]+class=["\']course-title[^>]+>(.+?)</h'),
+            webpage, 'course title', fatal=False)
+
+        return self.playlist_result(entries, course_id, course_title)

From 12f01118b0bd621e3e3df813a774fa0a9c1cfd73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 May 2017 21:57:00 +0700
Subject: [PATCH 289/611] [francetv] Add support for mobile.france.tv (closes
 #13068)

---
 youtube_dl/extractor/francetv.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 6e1a537a0..546d5caa0 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -112,7 +112,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
 
 
 class FranceTVIE(FranceTVBaseInfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?france\.tv/(?:[^/]+/)+(?P<id>[^/]+)\.html'
+    _VALID_URL = r'https?://(?:(?:www\.)?france\.tv|mobile\.france\.tv)/(?:[^/]+/)+(?P<id>[^/]+)\.html'
 
     _TESTS = [{
         'url': 'https://www.france.tv/france-2/13h15-le-dimanche/140921-les-mysteres-de-jesus.html',
@@ -154,6 +154,9 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
     }, {
         'url': 'https://www.france.tv/jeux-et-divertissements/divertissements/133965-le-web-contre-attaque.html',
         'only_matching': True,
+    }, {
+        'url': 'https://mobile.france.tv/france-5/c-dans-l-air/137347-emission-du-vendredi-12-mai-2017.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From fa26734e0760549b7b727017106e9d649bb695ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 00:03:15 +0700
Subject: [PATCH 290/611] [postprocessor/metadatafromtitle] Add support regex
 syntax for --metadata-from-title (closes #13065)

---
 youtube_dl/options.py                         | 11 ++++++-----
 youtube_dl/postprocessor/metadatafromtitle.py |  4 +++-
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 52309fb84..57f2e2c1e 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -814,11 +814,12 @@ def parseOpts(overrideArguments=None):
         '--metadata-from-title',
         metavar='FORMAT', dest='metafromtitle',
         help='Parse additional metadata like song title / artist from the video title. '
-             'The format syntax is the same as --output, '
-             'the parsed parameters replace existing values. '
-             'Additional templates: %(album)s, %(artist)s. '
-             'Example: --metadata-from-title "%(artist)s - %(title)s" matches a title like '
-             '"Coldplay - Paradise"')
+             'The format syntax is the same as --output. Regular expression with '
+             'named capture groups may also be used. '
+             'The parsed parameters replace existing values. '
+             'Example: --metadata-from-title "%(artist)s - %(title)s" matches a title like ' 
+             '"Coldplay - Paradise". '
+             'Example (regex): --metadata-from-title "(?P<artist>.+?) - (?P<title>.+)"')
     postproc.add_option(
         '--xattrs',
         action='store_true', dest='xattrs', default=False,
diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dl/postprocessor/metadatafromtitle.py
index a7d637a3c..c73f02447 100644
--- a/youtube_dl/postprocessor/metadatafromtitle.py
+++ b/youtube_dl/postprocessor/metadatafromtitle.py
@@ -9,7 +9,9 @@ class MetadataFromTitlePP(PostProcessor):
     def __init__(self, downloader, titleformat):
         super(MetadataFromTitlePP, self).__init__(downloader)
         self._titleformat = titleformat
-        self._titleregex = self.format_to_regex(titleformat)
+        self._titleregex = (self.format_to_regex(titleformat)
+                            if re.search(r'%\(\w+\)s', titleformat)
+                            else titleformat)
 
     def format_to_regex(self, fmt):
         r"""

From b845766597ea360ea474fac6764694ab34e8ef60 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 00:32:50 +0700
Subject: [PATCH 291/611] [imdb] Relax _VALID_URL (closes #13056)

---
 youtube_dl/extractor/imdb.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index f95c00c73..3ff672a89 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -13,7 +13,7 @@ from ..utils import (
 class ImdbIE(InfoExtractor):
     IE_NAME = 'imdb'
     IE_DESC = 'Internet Movie Database trailers'
-    _VALID_URL = r'https?://(?:www|m)\.imdb\.com/(?:video/[^/]+/|title/tt\d+.*?#lb-|videoplayer/)vi(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www|m)\.imdb\.com/(?:video|title).+?[/-]vi(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.imdb.com/video/imdb/vi2524815897',
@@ -35,6 +35,9 @@ class ImdbIE(InfoExtractor):
     }, {
         'url': 'http://www.imdb.com/videoplayer/vi1562949145',
         'only_matching': True,
+    }, {
+        'url': 'http://www.imdb.com/title/tt4218696/videoplayer/vi2608641561',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 851a01aed6c8fd2d7fa0bccb9f7fca7b95f99f16 Mon Sep 17 00:00:00 2001
From: Luca Steeb <contact@luca-steeb.com>
Date: Sat, 13 May 2017 19:57:02 +0200
Subject: [PATCH 292/611] [aljazeera] Extend _VALID_URL

---
 youtube_dl/extractor/aljazeera.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/aljazeera.py b/youtube_dl/extractor/aljazeera.py
index 388e578d5..c68be3134 100644
--- a/youtube_dl/extractor/aljazeera.py
+++ b/youtube_dl/extractor/aljazeera.py
@@ -4,9 +4,9 @@ from .common import InfoExtractor
 
 
 class AlJazeeraIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?aljazeera\.com/programmes/.*?/(?P<id>[^/]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?aljazeera\.com/(?:programmes|video)/.*?/(?P<id>[^/]+)\.html'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.aljazeera.com/programmes/the-slum/2014/08/deliverance-201482883754237240.html',
         'info_dict': {
             'id': '3792260579001',
@@ -19,7 +19,10 @@ class AlJazeeraIE(InfoExtractor):
         },
         'add_ie': ['BrightcoveNew'],
         'skip': 'Not accessible from Travis CI server',
-    }
+    }, {
+        'url': 'http://www.aljazeera.com/video/news/2017/05/sierra-leone-709-carat-diamond-auctioned-170511100111930.html',
+        'only_matching': True,
+    }]
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/665003303001/default_default/index.html?videoId=%s'
 
     def _real_extract(self, url):

From efe9316703efa5dc1561dc1c5159a77c838f5ab0 Mon Sep 17 00:00:00 2001
From: phaer <phaer@phaer.org>
Date: Sun, 8 Jan 2017 15:49:54 +0100
Subject: [PATCH 293/611] [orf:radio] Fix extraction

Since oe1.orf.at has been updated, both ORF radios supported by youtube_dl
use the same API. This commit honors this fact by merging both extractors
into one.
---
 youtube_dl/extractor/extractors.py |   2 +-
 youtube_dl/extractor/orf.py        | 113 +++++++++++++----------------
 2 files changed, 53 insertions(+), 62 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 811519925..dc4f9f24b 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -736,8 +736,8 @@ from .openload import OpenloadIE
 from .ora import OraTVIE
 from .orf import (
     ORFTVthekIE,
-    ORFOE1IE,
     ORFFM4IE,
+    ORFOE1IE,
     ORFIPTVIE,
 )
 from .packtpub import (
diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 1e2c54e68..10e4a2e1c 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -2,8 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import calendar
-import datetime
 
 from .common import InfoExtractor
 from ..compat import compat_str
@@ -144,77 +142,25 @@ class ORFTVthekIE(InfoExtractor):
         }
 
 
-class ORFOE1IE(InfoExtractor):
-    IE_NAME = 'orf:oe1'
-    IE_DESC = 'Radio Österreich 1'
-    _VALID_URL = r'https?://oe1\.orf\.at/(?:programm/|konsole\?.*?\btrack_id=)(?P<id>[0-9]+)'
-
-    # Audios on ORF radio are only available for 7 days, so we can't add tests.
-    _TESTS = [{
-        'url': 'http://oe1.orf.at/konsole?show=on_demand#?track_id=394211',
-        'only_matching': True,
-    }, {
-        'url': 'http://oe1.orf.at/konsole?show=ondemand&track_id=443608&load_day=/programm/konsole/tag/20160726',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        show_id = self._match_id(url)
-        data = self._download_json(
-            'http://oe1.orf.at/programm/%s/konsole' % show_id,
-            show_id
-        )
-
-        timestamp = datetime.datetime.strptime('%s %s' % (
-            data['item']['day_label'],
-            data['item']['time']
-        ), '%d.%m.%Y %H:%M')
-        unix_timestamp = calendar.timegm(timestamp.utctimetuple())
-
-        return {
-            'id': show_id,
-            'title': data['item']['title'],
-            'url': data['item']['url_stream'],
-            'ext': 'mp3',
-            'description': data['item'].get('info'),
-            'timestamp': unix_timestamp
-        }
-
-
-class ORFFM4IE(InfoExtractor):
-    IE_NAME = 'orf:fm4'
-    IE_DESC = 'radio FM4'
-    _VALID_URL = r'https?://fm4\.orf\.at/(?:7tage/?#|player/)(?P<date>[0-9]+)/(?P<show>\w+)'
-
-    _TEST = {
-        'url': 'http://fm4.orf.at/player/20160110/IS/',
-        'md5': '01e736e8f1cef7e13246e880a59ad298',
-        'info_dict': {
-            'id': '2016-01-10_2100_tl_54_7DaysSun13_11244',
-            'ext': 'mp3',
-            'title': 'Im Sumpf',
-            'description': 'md5:384c543f866c4e422a55f66a62d669cd',
-            'duration': 7173,
-            'timestamp': 1452456073,
-            'upload_date': '20160110',
-        },
-        'skip': 'Live streams on FM4 got deleted soon',
-    }
-
+class ORFRadioIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
+        station = mobj.group('station')
         show_date = mobj.group('date')
         show_id = mobj.group('show')
 
+        if station == 'fm4':
+            show_id = '4%s' % show_id
+
         data = self._download_json(
-            'http://audioapi.orf.at/fm4/json/2.0/broadcasts/%s/4%s' % (show_date, show_id),
+            'http://audioapi.orf.at/%s/api/json/current/broadcast/%s/%s' % (station, show_id, show_date),
             show_id
         )
 
         def extract_entry_dict(info, title, subtitle):
             return {
                 'id': info['loopStreamId'].replace('.mp3', ''),
-                'url': 'http://loopstream01.apa.at/?channel=fm4&id=%s' % info['loopStreamId'],
+                'url': 'http://loopstream01.apa.at/?channel=%s&id=%s' % (station, info['loopStreamId']),
                 'title': title,
                 'description': subtitle,
                 'duration': (info['end'] - info['start']) / 1000,
@@ -233,6 +179,51 @@ class ORFFM4IE(InfoExtractor):
         }
 
 
+class ORFFM4IE(ORFRadioIE):
+    IE_NAME = 'orf:fm4'
+    IE_DESC = 'radio FM4'
+    _VALID_URL = r'https?://(?P<station>fm4)\.orf\.at/(?:7tage/?#|player/)(?P<date>[0-9]+)/(?P<show>\w+)'
+
+    _TESTS = [
+        {
+            'url': 'http://fm4.orf.at/player/20170107/CC',
+            'md5': '2b0be47375432a7ef104453432a19212',
+            'info_dict': {
+                'id': '2017-01-07_2100_tl_54_7DaysSat18_31295',
+                'ext': 'mp3',
+                'title': 'Solid Steel Radioshow',
+                'description': 'Die Mixshow von Coldcut und Ninja Tune.',
+                'duration': 3599,
+                'timestamp': 1483819257,
+                'upload_date': '20170107',
+            },
+            'skip': 'Shows from ORF radios are only available for 7 days.'
+        }
+    ]
+
+
+class ORFOE1IE(ORFRadioIE):
+    IE_NAME = 'orf:oe1'
+    IE_DESC = 'Radio Österreich 1'
+    _VALID_URL = r'https?://(?P<station>oe1)\.orf\.at/(?:7tage/?#|player/)(?P<date>[0-9]+)/(?P<show>\w+)'
+
+    _TESTS = [
+       {
+            'url': 'http://oe1.orf.at/player/20170108/456544',
+            'md5': '34d8a6e67ea888293741c86a099b745b',
+            'info_dict': {
+                'id': '2017-01-08_0759_tl_51_7DaysSun6_256141',
+                'ext': 'mp3',
+                'title': 'Morgenjournal',
+                'duration': 609,
+                'timestamp': 1483858796,
+                'upload_date': '20170108',
+            },
+            'skip': 'Shows from ORF radios are only available for 7 days.'
+        }
+    ]
+
+
 class ORFIPTVIE(InfoExtractor):
     IE_NAME = 'orf:iptv'
     IE_DESC = 'iptv.ORF.at'

From 1339ecb2f8cfe96821ba6284cd470e846634f110 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 04:30:29 +0700
Subject: [PATCH 294/611] [orf:radio] Cleanup _VALID_URLs (closes #11643)

---
 youtube_dl/extractor/orf.py | 62 +++++++++++++++++--------------------
 1 file changed, 29 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 10e4a2e1c..cc296eabd 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -182,46 +182,42 @@ class ORFRadioIE(InfoExtractor):
 class ORFFM4IE(ORFRadioIE):
     IE_NAME = 'orf:fm4'
     IE_DESC = 'radio FM4'
-    _VALID_URL = r'https?://(?P<station>fm4)\.orf\.at/(?:7tage/?#|player/)(?P<date>[0-9]+)/(?P<show>\w+)'
+    _VALID_URL = r'https?://(?P<station>fm4)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
 
-    _TESTS = [
-        {
-            'url': 'http://fm4.orf.at/player/20170107/CC',
-            'md5': '2b0be47375432a7ef104453432a19212',
-            'info_dict': {
-                'id': '2017-01-07_2100_tl_54_7DaysSat18_31295',
-                'ext': 'mp3',
-                'title': 'Solid Steel Radioshow',
-                'description': 'Die Mixshow von Coldcut und Ninja Tune.',
-                'duration': 3599,
-                'timestamp': 1483819257,
-                'upload_date': '20170107',
-            },
-            'skip': 'Shows from ORF radios are only available for 7 days.'
-        }
-    ]
+    _TEST = {
+        'url': 'http://fm4.orf.at/player/20170107/CC',
+        'md5': '2b0be47375432a7ef104453432a19212',
+        'info_dict': {
+            'id': '2017-01-07_2100_tl_54_7DaysSat18_31295',
+            'ext': 'mp3',
+            'title': 'Solid Steel Radioshow',
+            'description': 'Die Mixshow von Coldcut und Ninja Tune.',
+            'duration': 3599,
+            'timestamp': 1483819257,
+            'upload_date': '20170107',
+        },
+        'skip': 'Shows from ORF radios are only available for 7 days.'
+    }
 
 
 class ORFOE1IE(ORFRadioIE):
     IE_NAME = 'orf:oe1'
     IE_DESC = 'Radio Österreich 1'
-    _VALID_URL = r'https?://(?P<station>oe1)\.orf\.at/(?:7tage/?#|player/)(?P<date>[0-9]+)/(?P<show>\w+)'
+    _VALID_URL = r'https?://(?P<station>oe1)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
 
-    _TESTS = [
-       {
-            'url': 'http://oe1.orf.at/player/20170108/456544',
-            'md5': '34d8a6e67ea888293741c86a099b745b',
-            'info_dict': {
-                'id': '2017-01-08_0759_tl_51_7DaysSun6_256141',
-                'ext': 'mp3',
-                'title': 'Morgenjournal',
-                'duration': 609,
-                'timestamp': 1483858796,
-                'upload_date': '20170108',
-            },
-            'skip': 'Shows from ORF radios are only available for 7 days.'
-        }
-    ]
+    _TEST = {
+        'url': 'http://oe1.orf.at/player/20170108/456544',
+        'md5': '34d8a6e67ea888293741c86a099b745b',
+        'info_dict': {
+            'id': '2017-01-08_0759_tl_51_7DaysSun6_256141',
+            'ext': 'mp3',
+            'title': 'Morgenjournal',
+            'duration': 609,
+            'timestamp': 1483858796,
+            'upload_date': '20170108',
+        },
+        'skip': 'Shows from ORF radios are only available for 7 days.'
+    }
 
 
 class ORFIPTVIE(InfoExtractor):

From 0de136341a9b6384098e14e4a0a1a5be438abdf2 Mon Sep 17 00:00:00 2001
From: Timendum <timedum@gmail.com>
Date: Tue, 2 May 2017 16:17:15 +0200
Subject: [PATCH 295/611] [mediaset] Add extractor

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/mediaset.py   | 118 +++++++++++++++++++++++++++++
 2 files changed, 119 insertions(+)
 create mode 100644 youtube_dl/extractor/mediaset.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index dc4f9f24b..ed603eb29 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -545,6 +545,7 @@ from .mangomolo import (
 )
 from .matchtv import MatchTVIE
 from .mdr import MDRIE
+from .mediaset import MediasetIE
 from .medici import MediciIE
 from .meipai import MeipaiIE
 from .melonvod import MelonVODIE
diff --git a/youtube_dl/extractor/mediaset.py b/youtube_dl/extractor/mediaset.py
new file mode 100644
index 000000000..4b1489524
--- /dev/null
+++ b/youtube_dl/extractor/mediaset.py
@@ -0,0 +1,118 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import (
+    InfoExtractor,
+    ExtractorError
+)
+from ..utils import (
+    determine_ext,
+    parse_duration,
+    unified_strdate
+)
+
+
+class MediasetIE(InfoExtractor):
+    _VALID_URL = r'https?://www\.video\.mediaset\.it/(?:(?:video|on-demand)/(?:.+)_|player/playerIFrame(?:Twitter)?\.shtml\?id=)(?P<id>[0-9]+)(?:.html|&.+)'
+    _TESTS = [{
+        # full episode
+        'url': 'http://www.video.mediaset.it/video/hello_goodbye/full/quarta-puntata_661824.html',
+        'md5': '9b75534d42c44ecef7bf1ffeacb7f85d',
+        'info_dict': {
+            'id': '661824',
+            'ext': 'mp4',
+            'title': 'Quarta puntata',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:7183696d6df570e3412a5ef74b27c5e2',
+            'uploader': 'mediaset'
+        }
+    }, {
+        # on demand
+        'url': 'http://www.video.mediaset.it/video/domenica_live/interviste/il-fenomeno-elettra-lamborghini_716283.html',
+        'md5': '81c57566bf2ee02e995f5342f079ca25',
+        'info_dict': {
+            'id': '716283',
+            'ext': 'mp4',
+            'title': 'Il fenomeno Elettra Lamborghini',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:dabf0e7cf48fc6d0a3417b989028748a',
+            'uploader': 'mediaset'
+        }
+    }, {
+        # clip
+        'url': 'http://www.video.mediaset.it/video/gogglebox/clip/un-grande-classico-della-commedia-sexy_661680.html',
+        'md5': '189ca72fe399db80dbfa595a4abf42d0',
+        'info_dict': {
+            'id': '661680',
+            'ext': 'mp4',
+            'title': 'Un grande classico della commedia sexy',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'Un film che riesce a risvegliare i sensi di Gigi.',
+            'uploader': 'mediaset'
+        }
+    }, {
+        # iframe simple
+        'url': 'http://www.video.mediaset.it/player/playerIFrame.shtml?id=665924&autoplay=true',
+        'md5': '308430901e55e1ad83dddb4be2a4454a',
+        'info_dict': {
+            'id': '665924',
+            'ext': 'mp4',
+            'title': 'Gianna Nannini incontra i fan a Milano',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'La cantante parla del nuovo libro',
+            'uploader': 'mediaset'
+        }
+    }, {
+        # iframe twitter (from http://www.wittytv.it/se-prima-mi-fidavo-zero/)
+        'url': 'https://www.video.mediaset.it/player/playerIFrameTwitter.shtml?id=665104&playrelated=false&autoplay=false&related=true&hidesocial=true',
+        'md5': '6f53a834b3b5eac1ebc2037ccf7194d0',
+        'info_dict': {
+            'id': '665104',
+            'ext': 'mp4',
+            'title': '\"Se prima mi fidavo zero...\"',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'Una piccola anteprima della prossima puntata del Trono Classico',
+            'uploader': 'mediaset'
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        formats = []
+        uploader = None
+        categories = None
+
+        mediainfo = self._download_json(
+            'http://plr.video.mediaset.it/html/metainfo.sjson?id=%s' % video_id,
+            video_id, 'Downloading video info JSON').get('video')
+
+        if 'brand-info' in mediainfo:
+            uploader = mediainfo.get('brand-info').get('publisher')
+            categories = [mediainfo.get('brand-info').get('category')]
+
+        cnd = self._download_json(
+            'http://cdnsel01.mediaset.net/GetCdn.aspx?streamid=%s&format=json' % video_id,
+            video_id, 'Downloading video CND JSON')
+
+        if not cnd.get('videoList'):
+            raise ExtractorError('Video not found')
+
+        for media_url in cnd.get('videoList'):
+            formats.append({
+                'url': media_url,
+                'ext': determine_ext(media_url)
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': mediainfo.get('title'),
+            'formats': formats,
+            'description': mediainfo.get('short-description'),
+            'uploader': uploader,
+            'thumbnail': mediainfo.get('thumbnail'),
+            'duration': parse_duration(mediainfo.get('duration')),
+            'release_date': unified_strdate(mediainfo.get('production-date')),
+            'webpage_url': mediainfo.get('url'),
+            'categories': categories
+        }

From 56f9c77f0e715a26b749d494c99a3f0b076bbbf8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 05:27:51 +0700
Subject: [PATCH 296/611] [mediaset] Improve extraction (closes #12708, closes
 #12964)

---
 youtube_dl/extractor/mediaset.py | 128 ++++++++++++++-----------------
 1 file changed, 56 insertions(+), 72 deletions(-)

diff --git a/youtube_dl/extractor/mediaset.py b/youtube_dl/extractor/mediaset.py
index 4b1489524..89e23ca9b 100644
--- a/youtube_dl/extractor/mediaset.py
+++ b/youtube_dl/extractor/mediaset.py
@@ -1,19 +1,25 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .common import (
-    InfoExtractor,
-    ExtractorError
-)
+from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     determine_ext,
     parse_duration,
-    unified_strdate
+    try_get,
+    unified_strdate,
 )
 
 
 class MediasetIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.video\.mediaset\.it/(?:(?:video|on-demand)/(?:.+)_|player/playerIFrame(?:Twitter)?\.shtml\?id=)(?P<id>[0-9]+)(?:.html|&.+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?video\.mediaset\.it/
+                        (?:
+                            (?:video|on-demand)/(?:[^/]+/)+[^/]+_|
+                            player/playerIFrame(?:Twitter)?\.shtml\?.*?\bid=
+                        )(?P<id>[0-9]+)
+                    '''
     _TESTS = [{
         # full episode
         'url': 'http://www.video.mediaset.it/video/hello_goodbye/full/quarta-puntata_661824.html',
@@ -22,97 +28,75 @@ class MediasetIE(InfoExtractor):
             'id': '661824',
             'ext': 'mp4',
             'title': 'Quarta puntata',
-            'thumbnail': 're:^https?://.*\.jpg$',
             'description': 'md5:7183696d6df570e3412a5ef74b27c5e2',
-            'uploader': 'mediaset'
-        }
-    }, {
-        # on demand
-        'url': 'http://www.video.mediaset.it/video/domenica_live/interviste/il-fenomeno-elettra-lamborghini_716283.html',
-        'md5': '81c57566bf2ee02e995f5342f079ca25',
-        'info_dict': {
-            'id': '716283',
-            'ext': 'mp4',
-            'title': 'Il fenomeno Elettra Lamborghini',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'md5:dabf0e7cf48fc6d0a3417b989028748a',
-            'uploader': 'mediaset'
-        }
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 1414,
+            'creator': 'mediaset',
+            'release_date': '20161107',
+            'series': 'Hello Goodbye',
+            'categories': ['reality'],
+        },
+        'expected_warnings': ['is not a supported codec'],
     }, {
         # clip
         'url': 'http://www.video.mediaset.it/video/gogglebox/clip/un-grande-classico-della-commedia-sexy_661680.html',
-        'md5': '189ca72fe399db80dbfa595a4abf42d0',
-        'info_dict': {
-            'id': '661680',
-            'ext': 'mp4',
-            'title': 'Un grande classico della commedia sexy',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'Un film che riesce a risvegliare i sensi di Gigi.',
-            'uploader': 'mediaset'
-        }
+        'only_matching': True,
     }, {
         # iframe simple
         'url': 'http://www.video.mediaset.it/player/playerIFrame.shtml?id=665924&autoplay=true',
-        'md5': '308430901e55e1ad83dddb4be2a4454a',
-        'info_dict': {
-            'id': '665924',
-            'ext': 'mp4',
-            'title': 'Gianna Nannini incontra i fan a Milano',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'La cantante parla del nuovo libro',
-            'uploader': 'mediaset'
-        }
+        'only_matching': True,
     }, {
         # iframe twitter (from http://www.wittytv.it/se-prima-mi-fidavo-zero/)
         'url': 'https://www.video.mediaset.it/player/playerIFrameTwitter.shtml?id=665104&playrelated=false&autoplay=false&related=true&hidesocial=true',
-        'md5': '6f53a834b3b5eac1ebc2037ccf7194d0',
-        'info_dict': {
-            'id': '665104',
-            'ext': 'mp4',
-            'title': '\"Se prima mi fidavo zero...\"',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'Una piccola anteprima della prossima puntata del Trono Classico',
-            'uploader': 'mediaset'
-        }
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
+        video_list = self._download_json(
+            'http://cdnsel01.mediaset.net/GetCdn.aspx',
+            video_id, 'Downloading video CDN JSON', query={
+                'streamid': video_id,
+                'format': 'json',
+            })['videoList']
+
         formats = []
-        uploader = None
-        categories = None
+        for format_url in video_list:
+            if '.ism' in format_url:
+                formats.extend(self._extract_ism_formats(
+                    format_url, video_id, ism_id='mss', fatal=False))
+            else:
+                formats.append({
+                    'url': format_url,
+                    'format_id': determine_ext(format_url),
+                })
+        self._sort_formats(formats)
 
         mediainfo = self._download_json(
-            'http://plr.video.mediaset.it/html/metainfo.sjson?id=%s' % video_id,
-            video_id, 'Downloading video info JSON').get('video')
+            'http://plr.video.mediaset.it/html/metainfo.sjson',
+            video_id, 'Downloading video info JSON', query={
+                'id': video_id,
+            })['video']
 
-        if 'brand-info' in mediainfo:
-            uploader = mediainfo.get('brand-info').get('publisher')
-            categories = [mediainfo.get('brand-info').get('category')]
+        title = mediainfo['title']
 
-        cnd = self._download_json(
-            'http://cdnsel01.mediaset.net/GetCdn.aspx?streamid=%s&format=json' % video_id,
-            video_id, 'Downloading video CND JSON')
-
-        if not cnd.get('videoList'):
-            raise ExtractorError('Video not found')
-
-        for media_url in cnd.get('videoList'):
-            formats.append({
-                'url': media_url,
-                'ext': determine_ext(media_url)
-            })
-        self._sort_formats(formats)
+        creator = try_get(
+            mediainfo, lambda x: x['brand-info']['publisher'], compat_str)
+        category = try_get(
+            mediainfo, lambda x: x['brand-info']['category'], compat_str)
+        categories = [category] if category else None
 
         return {
             'id': video_id,
-            'title': mediainfo.get('title'),
-            'formats': formats,
+            'title': title,
             'description': mediainfo.get('short-description'),
-            'uploader': uploader,
             'thumbnail': mediainfo.get('thumbnail'),
             'duration': parse_duration(mediainfo.get('duration')),
+            'creator': creator,
             'release_date': unified_strdate(mediainfo.get('production-date')),
             'webpage_url': mediainfo.get('url'),
-            'categories': categories
+            'series': mediainfo.get('brand-value'),
+            'categories': categories,
+            'formats': formats,
         }

From 76d5a36391480294b6c26437ccfa34cb43145a7e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 06:11:45 +0700
Subject: [PATCH 297/611] [extractor/common] Respect Width and Height
 attributes in ISM manifests

---
 youtube_dl/extractor/common.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b9ad8461a..0bbb1103b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2001,6 +2001,12 @@ class InfoExtractor(object):
             compat_etree_fromstring(ism.encode('utf-8')), urlh.geturl(), ism_id)
 
     def _parse_ism_formats(self, ism_doc, ism_url, ism_id=None):
+        """
+        Parse formats from ISM manifest.
+        References:
+         1. [MS-SSTR]: Smooth Streaming Protocol,
+            https://msdn.microsoft.com/en-us/library/ff469518.aspx
+        """
         if ism_doc.get('IsLive') == 'TRUE' or ism_doc.find('Protection') is not None:
             return []
 
@@ -2022,8 +2028,11 @@ class InfoExtractor(object):
                     self.report_warning('%s is not a supported codec' % fourcc)
                     continue
                 tbr = int(track.attrib['Bitrate']) // 1000
-                width = int_or_none(track.get('MaxWidth'))
-                height = int_or_none(track.get('MaxHeight'))
+                # [1] does not mention Width and Height attributes. However,
+                # they're often present while MaxWidth and MaxHeight are
+                # missing, so should be used as fallbacks
+                width = int_or_none(track.get('MaxWidth') or track.get('Width'))
+                height = int_or_none(track.get('MaxHeight') or track.get('Height'))
                 sampling_rate = int_or_none(track.get('SamplingRate'))
 
                 track_url_pattern = re.sub(r'{[Bb]itrate}', track.attrib['Bitrate'], url_pattern)

From 946826eec75aeb2fe0e9edc8e734b6bfd89ad7c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 06:17:34 +0700
Subject: [PATCH 298/611] [extractor/generic] Remove duplicate limelight code

---
 youtube_dl/extractor/generic.py | 23 -----------------------
 1 file changed, 23 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8f9097945..dde65fa96 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2519,29 +2519,6 @@ class GenericIE(InfoExtractor):
             return self.playlist_result(
                 limelight_urls, video_id, video_title, video_description)
 
-        mobj = re.search(r'LimelightPlayer\.doLoad(Media|Channel|ChannelList)\(["\'](?P<id>[a-z0-9]{32})', webpage)
-        if mobj:
-            lm = {
-                'Media': 'media',
-                'Channel': 'channel',
-                'ChannelList': 'channel_list',
-            }
-            return self.url_result(smuggle_url('limelight:%s:%s' % (
-                lm[mobj.group(1)], mobj.group(2)), {'source_url': url}),
-                'Limelight%s' % mobj.group(1), mobj.group(2))
-
-        mobj = re.search(
-            r'''(?sx)
-                <object[^>]+class=(["\'])LimelightEmbeddedPlayerFlash\1[^>]*>.*?
-                    <param[^>]+
-                        name=(["\'])flashVars\2[^>]+
-                        value=(["\'])(?:(?!\3).)*mediaId=(?P<id>[a-z0-9]{32})
-            ''', webpage)
-        if mobj:
-            return self.url_result(smuggle_url(
-                'limelight:media:%s' % mobj.group('id'),
-                {'source_url': url}), 'LimelightMedia', mobj.group('id'))
-
         # Look for Anvato embeds
         anvato_urls = AnvatoIE._extract_urls(self, webpage, video_id)
         if anvato_urls:

From ca04de463d17cc8bf53dafb19eb25e6dff8bd239 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 06:28:40 +0700
Subject: [PATCH 299/611] [mediaset] Add support for shortcut

---
 youtube_dl/extractor/mediaset.py | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/mediaset.py b/youtube_dl/extractor/mediaset.py
index 89e23ca9b..3ebc7f2e1 100644
--- a/youtube_dl/extractor/mediaset.py
+++ b/youtube_dl/extractor/mediaset.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
@@ -13,12 +15,15 @@ from ..utils import (
 
 class MediasetIE(InfoExtractor):
     _VALID_URL = r'''(?x)
-                    https?://
-                        (?:www\.)?video\.mediaset\.it/
-                        (?:
-                            (?:video|on-demand)/(?:[^/]+/)+[^/]+_|
-                            player/playerIFrame(?:Twitter)?\.shtml\?.*?\bid=
-                        )(?P<id>[0-9]+)
+                    (?:
+                        mediaset:|
+                        https?://
+                            (?:www\.)?video\.mediaset\.it/
+                            (?:
+                                (?:video|on-demand)/(?:[^/]+/)+[^/]+_|
+                                player/playerIFrame(?:Twitter)?\.shtml\?.*?\bid=
+                            )
+                    )(?P<id>[0-9]+)
                     '''
     _TESTS = [{
         # full episode
@@ -49,6 +54,9 @@ class MediasetIE(InfoExtractor):
         # iframe twitter (from http://www.wittytv.it/se-prima-mi-fidavo-zero/)
         'url': 'https://www.video.mediaset.it/player/playerIFrameTwitter.shtml?id=665104&playrelated=false&autoplay=false&related=true&hidesocial=true',
         'only_matching': True,
+    }, {
+        'url': 'mediaset:661824',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 5d29af3d15eaec8958a4fad00103e18112f0c172 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 06:29:16 +0700
Subject: [PATCH 300/611] [extractor/generic] Add support for mediaset embeds

---
 youtube_dl/extractor/generic.py  | 7 +++++++
 youtube_dl/extractor/mediaset.py | 8 ++++++++
 2 files changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index dde65fa96..32f38c401 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -89,6 +89,7 @@ from .limelight import LimelightBaseIE
 from .anvato import AnvatoIE
 from .washingtonpost import WashingtonPostIE
 from .wistia import WistiaIE
+from .mediaset import MediasetIE
 
 
 class GenericIE(InfoExtractor):
@@ -2648,6 +2649,12 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 wapo_urls, video_id, video_title, ie=WashingtonPostIE.ie_key())
 
+        # Look for Mediaset embeds
+        mediaset_urls = MediasetIE._extract_urls(webpage)
+        if mediaset_urls:
+            return self.playlist_from_matches(
+                mediaset_urls, video_id, video_title, ie=MediasetIE.ie_key())
+
         # Looking for http://schema.org/VideoObject
         json_ld = self._search_json_ld(
             webpage, video_id, default={}, expected_type='VideoObject')
diff --git a/youtube_dl/extractor/mediaset.py b/youtube_dl/extractor/mediaset.py
index 3ebc7f2e1..9a0e41663 100644
--- a/youtube_dl/extractor/mediaset.py
+++ b/youtube_dl/extractor/mediaset.py
@@ -59,6 +59,14 @@ class MediasetIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_urls(webpage):
+        return [
+            mobj.group('url')
+            for mobj in re.finditer(
+                r'<iframe\b[^>]+\bsrc=(["\'])(?P<url>https?://(?:www\.)?video\.mediaset\.it/player/playerIFrame(?:Twitter)?\.shtml\?.*?\bid=\d+.*?)\1',
+                webpage)]
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From d105a7edc6e06c4eaf92838217913dde18aaae05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 06:39:47 +0700
Subject: [PATCH 301/611] [mediaset] Fix upload date

---
 youtube_dl/extractor/mediaset.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mediaset.py b/youtube_dl/extractor/mediaset.py
index 9a0e41663..9760eafd5 100644
--- a/youtube_dl/extractor/mediaset.py
+++ b/youtube_dl/extractor/mediaset.py
@@ -37,7 +37,7 @@ class MediasetIE(InfoExtractor):
             'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 1414,
             'creator': 'mediaset',
-            'release_date': '20161107',
+            'upload_date': '20161107',
             'series': 'Hello Goodbye',
             'categories': ['reality'],
         },
@@ -110,7 +110,7 @@ class MediasetIE(InfoExtractor):
             'thumbnail': mediainfo.get('thumbnail'),
             'duration': parse_duration(mediainfo.get('duration')),
             'creator': creator,
-            'release_date': unified_strdate(mediainfo.get('production-date')),
+            'upload_date': unified_strdate(mediainfo.get('production-date')),
             'webpage_url': mediainfo.get('url'),
             'series': mediainfo.get('brand-value'),
             'categories': categories,

From 2b8e6a68f8cef6a5e2fd2987446bc19ddc20bbaa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 06:40:19 +0700
Subject: [PATCH 302/611] [extractor/generic] Add test for mediaset embed

---
 youtube_dl/extractor/generic.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 32f38c401..c108d4a8a 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1720,6 +1720,19 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': [WashingtonPostIE.ie_key()],
         },
+        {
+            # Mediaset embed
+            'url': 'http://www.tgcom24.mediaset.it/politica/serracchiani-voglio-vivere-in-una-societa-aperta-reazioni-sproporzionate-_3071354-201702a.shtml',
+            'info_dict': {
+                'id': '720642',
+                'ext': 'mp4',
+                'title': 'Serracchiani: "Voglio vivere in una società aperta, con tutela del patto di fiducia"',
+            },
+            'params': {
+                'skip_download': True,
+            },
+            'add_ie': [MediasetIE.ie_key()],
+        },
         # {
         #     # TODO: find another test
         #     # http://schema.org/VideoObject

From 166d12b00cfbc45f60f052c541f567c22a80bb85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 07:33:24 +0700
Subject: [PATCH 303/611] [options] PEP 8

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 57f2e2c1e..3021a6f41 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -817,7 +817,7 @@ def parseOpts(overrideArguments=None):
              'The format syntax is the same as --output. Regular expression with '
              'named capture groups may also be used. '
              'The parsed parameters replace existing values. '
-             'Example: --metadata-from-title "%(artist)s - %(title)s" matches a title like ' 
+             'Example: --metadata-from-title "%(artist)s - %(title)s" matches a title like '
              '"Coldplay - Paradise". '
              'Example (regex): --metadata-from-title "(?P<artist>.+?) - (?P<title>.+)"')
     postproc.add_option(

From 09b866e171a6cef8578a2a37f5bd766ae6bfbf50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 07:28:16 +0700
Subject: [PATCH 304/611] [ChangeLog] Actualize

---
 ChangeLog | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 96e4ce8a1..0dd213e6d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,23 @@
 version <unreleased>
 
+Core
++ [extractor/common] Respect Width and Height attributes in ISM manifests
++ [postprocessor/metadatafromtitle] Add support regular expression syntax for
+  --metadata-from-title (#13065)
+
 Extractor
++ [mediaset] Add support for video.mediaset.it (#12708, #12964)
+* [orf:radio] Fix extraction (#11643, #12926)
+* [aljazeera] Extend URL regular expression (#13053)
+* [imdb] Relax URL regular expression (#13056)
++ [francetv] Add support for mobile.france.tv (#13068)
++ [upskill] Add support for upskillcourses.com (#13043)
+* [thescene] Fix extraction (#13061)
+* [condenast] Improve embed support
 * [liveleak] Fix extraction (#12053)
 + [douyu] Support Douyu shows (#12228)
+* [myspace] Improve URL regular expression (#13040)
+* [adultswim] Use desktop platform in assets URL (#13041)
 
 
 version 2017.05.09

From 6f3c632c24c6f5d1d653273c48e9a612bb04a873 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 07:38:40 +0700
Subject: [PATCH 305/611] release 2017.05.14

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 ChangeLog                 |  2 +-
 README.md                 | 14 ++++++++------
 docs/supportedsites.md    |  4 ++++
 youtube_dl/version.py     |  2 +-
 5 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index f7742ddcf..2955e833d 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.09*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.09**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.14*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.14**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.05.09
+[debug] youtube-dl version 2017.05.14
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 0dd213e6d..91c77515f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.05.14
 
 Core
 + [extractor/common] Respect Width and Height attributes in ISM manifests
diff --git a/README.md b/README.md
index 1ecd2005a..dc0be1f40 100644
--- a/README.md
+++ b/README.md
@@ -400,12 +400,14 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     --add-metadata                   Write metadata to the video file
     --metadata-from-title FORMAT     Parse additional metadata like song title /
                                      artist from the video title. The format
-                                     syntax is the same as --output, the parsed
-                                     parameters replace existing values.
-                                     Additional templates: %(album)s,
-                                     %(artist)s. Example: --metadata-from-title
-                                     "%(artist)s - %(title)s" matches a title
-                                     like "Coldplay - Paradise"
+                                     syntax is the same as --output. Regular
+                                     expression with named capture groups may
+                                     also be used. The parsed parameters replace
+                                     existing values. Example: --metadata-from-
+                                     title "%(artist)s - %(title)s" matches a
+                                     title like "Coldplay - Paradise". Example
+                                     (regex): --metadata-from-title
+                                     "(?P<artist>.+?) - (?P<title>.+)"
     --xattrs                         Write metadata to the video file's xattrs
                                      (using dublin core and xdg standards)
     --fixup POLICY                   Automatically correct known faults of the
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 2f27686b3..aa6c118f2 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -216,6 +216,7 @@
  - **DiscoveryVR**
  - **Disney**
  - **Dotsub**
+ - **DouyuShow**
  - **DouyuTV**: 斗鱼
  - **DPlay**
  - **DPlayIt**
@@ -433,6 +434,7 @@
  - **MDR**: MDR.DE and KiKA
  - **media.ccc.de**
  - **Medialaan**
+ - **Mediaset**
  - **Medici**
  - **Meipai**: 美拍
  - **MelonVOD**
@@ -861,6 +863,8 @@
  - **uol.com.br**
  - **uplynk**
  - **uplynk:preplay**
+ - **Upskill**
+ - **UpskillCourse**
  - **Urort**: NRK P3 Urørt
  - **URPlay**
  - **USANetwork**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 33a0c2130..9b074fafc 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.05.09'
+__version__ = '2017.05.14'

From c360e641e9f3bd1c5111d03e5d9d90e5b6abdf02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 09:55:40 +0700
Subject: [PATCH 306/611] [dailymotion] Extend _VALID_URL (closes #13079)

---
 youtube_dl/extractor/dailymotion.py | 143 ++++++++++++++--------------
 1 file changed, 69 insertions(+), 74 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 441114d19..f8db76c18 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -38,7 +38,7 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
 
 
 class DailymotionIE(DailymotionBaseInfoExtractor):
-    _VALID_URL = r'(?i)(?:https?://)?(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(?:embed|swf|#)/)?video/(?P<id>[^/?_]+)'
+    _VALID_URL = r'(?i)https?://(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(?:(?:embed|swf|#)/)?video|swf)/(?P<id>[^/?_]+)'
     IE_NAME = 'dailymotion'
 
     _FORMATS = [
@@ -49,87 +49,82 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
         ('stream_h264_hd1080_url', 'hd180'),
     ]
 
-    _TESTS = [
-        {
-            'url': 'http://www.dailymotion.com/video/x5kesuj_office-christmas-party-review-jason-bateman-olivia-munn-t-j-miller_news',
-            'md5': '074b95bdee76b9e3654137aee9c79dfe',
-            'info_dict': {
-                'id': 'x5kesuj',
-                'ext': 'mp4',
-                'title': 'Office Christmas Party Review –  Jason Bateman, Olivia Munn, T.J. Miller',
-                'description': 'Office Christmas Party Review -  Jason Bateman, Olivia Munn, T.J. Miller',
-                'thumbnail': r're:^https?:.*\.(?:jpg|png)$',
-                'duration': 187,
-                'timestamp': 1493651285,
-                'upload_date': '20170501',
-                'uploader': 'Deadline',
-                'uploader_id': 'x1xm8ri',
-                'age_limit': 0,
-                'view_count': int,
-            },
+    _TESTS = [{
+        'url': 'http://www.dailymotion.com/video/x5kesuj_office-christmas-party-review-jason-bateman-olivia-munn-t-j-miller_news',
+        'md5': '074b95bdee76b9e3654137aee9c79dfe',
+        'info_dict': {
+            'id': 'x5kesuj',
+            'ext': 'mp4',
+            'title': 'Office Christmas Party Review –  Jason Bateman, Olivia Munn, T.J. Miller',
+            'description': 'Office Christmas Party Review -  Jason Bateman, Olivia Munn, T.J. Miller',
+            'thumbnail': r're:^https?:.*\.(?:jpg|png)$',
+            'duration': 187,
+            'timestamp': 1493651285,
+            'upload_date': '20170501',
+            'uploader': 'Deadline',
+            'uploader_id': 'x1xm8ri',
+            'age_limit': 0,
+            'view_count': int,
         },
-        {
-            'url': 'https://www.dailymotion.com/video/x2iuewm_steam-machine-models-pricing-listed-on-steam-store-ign-news_videogames',
-            'md5': '2137c41a8e78554bb09225b8eb322406',
-            'info_dict': {
-                'id': 'x2iuewm',
-                'ext': 'mp4',
-                'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News',
-                'description': 'Several come bundled with the Steam Controller.',
-                'thumbnail': r're:^https?:.*\.(?:jpg|png)$',
-                'duration': 74,
-                'timestamp': 1425657362,
-                'upload_date': '20150306',
-                'uploader': 'IGN',
-                'uploader_id': 'xijv66',
-                'age_limit': 0,
-                'view_count': int,
-            },
-            'skip': 'video gone',
+    }, {
+        'url': 'https://www.dailymotion.com/video/x2iuewm_steam-machine-models-pricing-listed-on-steam-store-ign-news_videogames',
+        'md5': '2137c41a8e78554bb09225b8eb322406',
+        'info_dict': {
+            'id': 'x2iuewm',
+            'ext': 'mp4',
+            'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News',
+            'description': 'Several come bundled with the Steam Controller.',
+            'thumbnail': r're:^https?:.*\.(?:jpg|png)$',
+            'duration': 74,
+            'timestamp': 1425657362,
+            'upload_date': '20150306',
+            'uploader': 'IGN',
+            'uploader_id': 'xijv66',
+            'age_limit': 0,
+            'view_count': int,
         },
+        'skip': 'video gone',
+    }, {
         # Vevo video
-        {
-            'url': 'http://www.dailymotion.com/video/x149uew_katy-perry-roar-official_musi',
-            'info_dict': {
-                'title': 'Roar (Official)',
-                'id': 'USUV71301934',
-                'ext': 'mp4',
-                'uploader': 'Katy Perry',
-                'upload_date': '20130905',
-            },
-            'params': {
-                'skip_download': True,
-            },
-            'skip': 'VEVO is only available in some countries',
+        'url': 'http://www.dailymotion.com/video/x149uew_katy-perry-roar-official_musi',
+        'info_dict': {
+            'title': 'Roar (Official)',
+            'id': 'USUV71301934',
+            'ext': 'mp4',
+            'uploader': 'Katy Perry',
+            'upload_date': '20130905',
         },
+        'params': {
+            'skip_download': True,
+        },
+        'skip': 'VEVO is only available in some countries',
+    }, {
         # age-restricted video
-        {
-            'url': 'http://www.dailymotion.com/video/xyh2zz_leanna-decker-cyber-girl-of-the-year-desires-nude-playboy-plus_redband',
-            'md5': '0d667a7b9cebecc3c89ee93099c4159d',
-            'info_dict': {
-                'id': 'xyh2zz',
-                'ext': 'mp4',
-                'title': 'Leanna Decker - Cyber Girl Of The Year Desires Nude [Playboy Plus]',
-                'uploader': 'HotWaves1012',
-                'age_limit': 18,
-            },
-            'skip': 'video gone',
+        'url': 'http://www.dailymotion.com/video/xyh2zz_leanna-decker-cyber-girl-of-the-year-desires-nude-playboy-plus_redband',
+        'md5': '0d667a7b9cebecc3c89ee93099c4159d',
+        'info_dict': {
+            'id': 'xyh2zz',
+            'ext': 'mp4',
+            'title': 'Leanna Decker - Cyber Girl Of The Year Desires Nude [Playboy Plus]',
+            'uploader': 'HotWaves1012',
+            'age_limit': 18,
         },
+        'skip': 'video gone',
+    }, {
         # geo-restricted, player v5
-        {
-            'url': 'http://www.dailymotion.com/video/xhza0o',
-            'only_matching': True,
-        },
+        'url': 'http://www.dailymotion.com/video/xhza0o',
+        'only_matching': True,
+    }, {
         # with subtitles
-        {
-            'url': 'http://www.dailymotion.com/video/x20su5f_the-power-of-nightmares-1-the-rise-of-the-politics-of-fear-bbc-2004_news',
-            'only_matching': True,
-        },
-        {
-            'url': 'http://www.dailymotion.com/swf/video/x3n92nf',
-            'only_matching': True,
-        }
-    ]
+        'url': 'http://www.dailymotion.com/video/x20su5f_the-power-of-nightmares-1-the-rise-of-the-politics-of-fear-bbc-2004_news',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.dailymotion.com/swf/video/x3n92nf',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.dailymotion.com/swf/x3ss1m_funny-magic-trick-barry-and-stuart_fun',
+        'only_matching': True,
+    }]
 
     @staticmethod
     def _extract_urls(webpage):

From 60f5c9fb191b1ffc11e69dece379a8ad2de22207 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 12:33:33 +0700
Subject: [PATCH 307/611] [utils] Recognize more audio codecs (#13081)

---
 youtube_dl/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index aa1900436..4293a77f5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2360,11 +2360,11 @@ def parse_codecs(codecs_str):
         if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v'):
             if not vcodec:
                 vcodec = full_codec
-        elif codec in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3'):
+        elif codec in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'):
             if not acodec:
                 acodec = full_codec
         else:
-            write_string('WARNING: Unknown codec %s' % full_codec, sys.stderr)
+            write_string('WARNING: Unknown codec %s\n' % full_codec, sys.stderr)
     if not vcodec and not acodec:
         if len(splited_codecs) == 2:
             return {

From 1c45b7a8a9ac50b87b9b1a540e9d9e875ac22a0b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 12:47:19 +0700
Subject: [PATCH 308/611] [dailymail] Fix sources extraction (closes #13057)

---
 youtube_dl/extractor/dailymail.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dailymail.py b/youtube_dl/extractor/dailymail.py
index 98c835bf1..538565c66 100644
--- a/youtube_dl/extractor/dailymail.py
+++ b/youtube_dl/extractor/dailymail.py
@@ -2,9 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     int_or_none,
     determine_protocol,
+    try_get,
     unescapeHTML,
 )
 
@@ -28,8 +30,14 @@ class DailyMailIE(InfoExtractor):
         video_data = self._parse_json(self._search_regex(
             r"data-opts='({.+?})'", webpage, 'video data'), video_id)
         title = unescapeHTML(video_data['title'])
-        video_sources = self._download_json(video_data.get(
-            'sources', {}).get('url') or 'http://www.dailymail.co.uk/api/player/%s/video-sources.json' % video_id, video_id)
+
+        sources_url = (try_get(
+            video_data,
+            (lambda x: x['plugins']['sources']['url'],
+             lambda x: x['sources']['url']), compat_str) or
+            'http://www.dailymail.co.uk/api/player/%s/video-sources.json' % video_id)
+
+        video_sources = self._download_json(sources_url, video_id)
 
         formats = []
         for rendition in video_sources['renditions']:

From 89fd03079b83e1dfae016e4e28746b00befe8b2d Mon Sep 17 00:00:00 2001
From: mrBliss <dewinant@gmail.com>
Date: Mon, 15 May 2017 16:46:55 +0200
Subject: [PATCH 309/611] [vier] Improve extraction

+ Add support for authentication
* Bypass authentication when no credentials provded
* Improve extraction robustness
---
 youtube_dl/extractor/vier.py | 74 +++++++++++++++++++++++++++++++++++-
 1 file changed, 72 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index 5ef7635b6..b6efaa284 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -5,12 +5,17 @@ import re
 import itertools
 
 from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    urlencode_postdata,
+)
 
 
 class VierIE(InfoExtractor):
     IE_NAME = 'vier'
     IE_DESC = 'vier.be and vijf.be'
     _VALID_URL = r'https?://(?:www\.)?(?P<site>vier|vijf)\.be/(?:[^/]+/videos/(?P<display_id>[^/]+)(?:/(?P<id>\d+))?|video/v3/embed/(?P<embed_id>\d+))'
+    _NETRC_MACHINE = 'vier'
     _TESTS = [{
         'url': 'http://www.vier.be/planb/videos/het-wordt-warm-de-moestuin/16129',
         'info_dict': {
@@ -24,19 +29,47 @@ class VierIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
+        # 'skip': 'Requires account credentials',
     }, {
         'url': 'http://www.vijf.be/temptationisland/videos/zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas/2561614',
         'info_dict': {
             'id': '2561614',
             'display_id': 'zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas',
             'ext': 'mp4',
-            'title': 'ZO grappig: Temptation Island hosts moeten kiezen tussen onmogelijke dilemma\'s',
+            'title': 'EXTRA: Temptation Island hosts moeten kiezen tussen onmogelijke dilemma\'s',
             'description': 'Het spel is simpel: Annelien Coorevits en Rick Brandsteder krijgen telkens 2 dilemma\'s voorgeschoteld en ze MOETEN een keuze maken.',
         },
         'params': {
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.vier.be/janigaat/videos/jani-gaat-naar-tokio-aflevering-4/2674839',
+        'info_dict': {
+            'id': '2674839',
+            'display_id': 'jani-gaat-naar-tokio-aflevering-4',
+            'ext': 'mp4',
+            'title': 'Jani gaat naar Tokio - Aflevering 4',
+            'description': 'Bekijk hier de volledige vierde aflevering van het 2de seizoen van Jani gaat...',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+        'skip': 'Requires account credentials',
+    }, {
+        'url': 'http://www.vier.be/janigaat/videos/jani-gaat-naar-tokio-aflevering-4/2674839',
+        'info_dict': {
+            'id': '2674839',
+            'display_id': 'jani-gaat-naar-tokio-aflevering-4',
+            'ext': 'mp4',
+            'title': 'jani-gaat-naar-tokio-aflevering-4',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+        'expected_warnings': ['Log in to extract metadata'],
     }, {
         'url': 'http://www.vier.be/planb/videos/mieren-herders-van-de-bladluizen',
         'only_matching': True,
@@ -45,17 +78,54 @@ class VierIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    def _real_initialize(self):
+        self._logged_in = False
+
+    def _login(self, site):
+        username, password = self._get_login_info()
+        if username is None or password is None:
+            return
+
+        login_page = self._download_webpage(
+            'http://www.%s.be/user/login' % site,
+            None, note='Logging in', errnote='Unable to log in',
+            data=urlencode_postdata({
+                'form_id': 'user_login',
+                'name': username,
+                'pass': password,
+            }),
+            headers={'Content-Type': 'application/x-www-form-urlencoded'})
+
+        login_error = self._html_search_regex(
+            r'(?s)<div class="messages error">\s*<div>\s*<h2.+?</h2>(.+?)<',
+            login_page, 'login error', default=None)
+        if login_error:
+            self.report_warning('Unable to log in: %s' % login_error)
+        else:
+            self._logged_in = True
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         embed_id = mobj.group('embed_id')
         display_id = mobj.group('display_id') or embed_id
+        video_id = mobj.group('id') or embed_id
         site = mobj.group('site')
 
+        if not self._logged_in:
+            self._login(site)
+
         webpage = self._download_webpage(url, display_id)
 
+        if r'id="user-login"' in webpage:
+            self.report_warning(
+                'Log in to extract metadata', video_id=display_id)
+            webpage = self._download_webpage(
+                'http://www.%s.be/video/v3/embed/%s' % (site, video_id),
+                display_id)
+
         video_id = self._search_regex(
             [r'data-nid="(\d+)"', r'"nid"\s*:\s*"(\d+)"'],
-            webpage, 'video id')
+            webpage, 'video id', default=video_id)
         application = self._search_regex(
             [r'data-application="([^"]+)"', r'"application"\s*:\s*"([^"]+)"'],
             webpage, 'application', default=site + '_vod')

From 7073015a23f96ce8ca0400286051d53cb9237a9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 May 2017 22:00:53 +0700
Subject: [PATCH 310/611] [vier] PEP 8 and cleanup

---
 youtube_dl/extractor/vier.py | 27 ++++++++++-----------------
 1 file changed, 10 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index b6efaa284..a98bd5c81 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -5,10 +5,7 @@ import re
 import itertools
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    urlencode_postdata,
-)
+from ..utils import urlencode_postdata
 
 
 class VierIE(InfoExtractor):
@@ -18,6 +15,7 @@ class VierIE(InfoExtractor):
     _NETRC_MACHINE = 'vier'
     _TESTS = [{
         'url': 'http://www.vier.be/planb/videos/het-wordt-warm-de-moestuin/16129',
+        'md5': 'e4ae2054a6b040ef1e289e20d111b46e',
         'info_dict': {
             'id': '16129',
             'display_id': 'het-wordt-warm-de-moestuin',
@@ -25,22 +23,16 @@ class VierIE(InfoExtractor):
             'title': 'Het wordt warm in De Moestuin',
             'description': 'De vele uren werk eisen hun tol. Wim droomt van assistentie...',
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-        # 'skip': 'Requires account credentials',
     }, {
         'url': 'http://www.vijf.be/temptationisland/videos/zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas/2561614',
         'info_dict': {
             'id': '2561614',
             'display_id': 'zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas',
             'ext': 'mp4',
-            'title': 'EXTRA: Temptation Island hosts moeten kiezen tussen onmogelijke dilemma\'s',
-            'description': 'Het spel is simpel: Annelien Coorevits en Rick Brandsteder krijgen telkens 2 dilemma\'s voorgeschoteld en ze MOETEN een keuze maken.',
+            'title': 'md5:84f45fe48b8c1fa296a7f6d208d080a7',
+            'description': 'md5:0356d4981e58b8cbee19355cbd51a8fe',
         },
         'params': {
-            # m3u8 download
             'skip_download': True,
         },
     }, {
@@ -50,14 +42,15 @@ class VierIE(InfoExtractor):
             'display_id': 'jani-gaat-naar-tokio-aflevering-4',
             'ext': 'mp4',
             'title': 'Jani gaat naar Tokio - Aflevering 4',
-            'description': 'Bekijk hier de volledige vierde aflevering van het 2de seizoen van Jani gaat...',
+            'description': 'md5:2d169e8186ae4247e50c99aaef97f7b2',
         },
         'params': {
-            # m3u8 download
             'skip_download': True,
         },
         'skip': 'Requires account credentials',
     }, {
+        # Requires account credentials but bypassed extraction via v3/embed page
+        # without metadata
         'url': 'http://www.vier.be/janigaat/videos/jani-gaat-naar-tokio-aflevering-4/2674839',
         'info_dict': {
             'id': '2674839',
@@ -66,12 +59,12 @@ class VierIE(InfoExtractor):
             'title': 'jani-gaat-naar-tokio-aflevering-4',
         },
         'params': {
-            # m3u8 download
             'skip_download': True,
         },
         'expected_warnings': ['Log in to extract metadata'],
     }, {
-        'url': 'http://www.vier.be/planb/videos/mieren-herders-van-de-bladluizen',
+        # Without video id in URL
+        'url': 'http://www.vier.be/planb/videos/dit-najaar-plan-b',
         'only_matching': True,
     }, {
         'url': 'http://www.vier.be/video/v3/embed/16129',
@@ -125,7 +118,7 @@ class VierIE(InfoExtractor):
 
         video_id = self._search_regex(
             [r'data-nid="(\d+)"', r'"nid"\s*:\s*"(\d+)"'],
-            webpage, 'video id', default=video_id)
+            webpage, 'video id', default=video_id or display_id)
         application = self._search_regex(
             [r'data-application="([^"]+)"', r'"application"\s*:\s*"([^"]+)"'],
             webpage, 'application', default=site + '_vod')

From 6f766798049418c15e3c1b62c046e507093993dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 May 2017 22:11:34 +0700
Subject: [PATCH 311/611] [extractor/common] Add support for schemeless URLs in
 _extract_wowza_formats (closes #13088, closes #13092)

---
 youtube_dl/extractor/common.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 0bbb1103b..74b6f1197 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2205,8 +2205,9 @@ class InfoExtractor(object):
 
     def _extract_wowza_formats(self, url, video_id, m3u8_entry_protocol='m3u8_native', skip_protocols=[]):
         url = re.sub(r'/(?:manifest|playlist|jwplayer)\.(?:m3u8|f4m|mpd|smil)', '', url)
-        url_base = self._search_regex(r'(?:https?|rtmp|rtsp)(://[^?]+)', url, 'format url')
-        http_base_url = 'http' + url_base
+        url_base = self._search_regex(
+            r'(?:(?:https?|rtmp|rtsp):)?(//[^?]+)', url, 'format url')
+        http_base_url = self._proto_relative_url(url_base, scheme='http:')
         formats = []
         if 'm3u8' not in skip_protocols:
             formats.extend(self._extract_m3u8_formats(

From f2e2f0c777dc9e541d89b742ac12e7ad37d8cb8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 May 2017 22:19:33 +0700
Subject: [PATCH 312/611] [extractor/common] Fix rtmp and rtsp formats' URLs in
 _extract_wowza_formats

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 74b6f1197..fec39da8b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2207,7 +2207,7 @@ class InfoExtractor(object):
         url = re.sub(r'/(?:manifest|playlist|jwplayer)\.(?:m3u8|f4m|mpd|smil)', '', url)
         url_base = self._search_regex(
             r'(?:(?:https?|rtmp|rtsp):)?(//[^?]+)', url, 'format url')
-        http_base_url = self._proto_relative_url(url_base, scheme='http:')
+        http_base_url = '%s:%s' % ('http', url_base)
         formats = []
         if 'm3u8' not in skip_protocols:
             formats.extend(self._extract_m3u8_formats(
@@ -2241,7 +2241,7 @@ class InfoExtractor(object):
             for protocol in ('rtmp', 'rtsp'):
                 if protocol not in skip_protocols:
                     formats.append({
-                        'url': protocol + url_base,
+                        'url': '%s:%s' % (protocol, url_base),
                         'format_id': protocol,
                         'protocol': protocol,
                     })

From 36b226d48fd0b366dcf99a646550181639301eb2 Mon Sep 17 00:00:00 2001
From: mrBliss <dewinant@gmail.com>
Date: Fri, 24 Mar 2017 10:34:35 +0100
Subject: [PATCH 313/611] [vier] Extract more info

Extract the `episode_number` and `upload_date`. Also extract the real
`description`.
---
 youtube_dl/extractor/vier.py | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index a98bd5c81..7fb718343 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -5,7 +5,11 @@ import re
 import itertools
 
 from .common import InfoExtractor
-from ..utils import urlencode_postdata
+from ..utils import (
+    urlencode_postdata,
+    int_or_none,
+    unified_strdate,
+)
 
 
 class VierIE(InfoExtractor):
@@ -22,6 +26,7 @@ class VierIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Het wordt warm in De Moestuin',
             'description': 'De vele uren werk eisen hun tol. Wim droomt van assistentie...',
+            'upload_date': '20121025',
         },
     }, {
         'url': 'http://www.vijf.be/temptationisland/videos/zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas/2561614',
@@ -31,6 +36,7 @@ class VierIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'md5:84f45fe48b8c1fa296a7f6d208d080a7',
             'description': 'md5:0356d4981e58b8cbee19355cbd51a8fe',
+            'upload_date': '20170228',
         },
         'params': {
             'skip_download': True,
@@ -42,7 +48,9 @@ class VierIE(InfoExtractor):
             'display_id': 'jani-gaat-naar-tokio-aflevering-4',
             'ext': 'mp4',
             'title': 'Jani gaat naar Tokio - Aflevering 4',
-            'description': 'md5:2d169e8186ae4247e50c99aaef97f7b2',
+            'description': 'md5:aa8d611541db6ae9e863125704511f88',
+            'upload_date': '20170501',
+            'episode_number': 4,
         },
         'params': {
             'skip_download': True,
@@ -131,14 +139,28 @@ class VierIE(InfoExtractor):
         self._sort_formats(formats)
 
         title = self._og_search_title(webpage, default=display_id)
-        description = self._og_search_description(webpage, default=None)
         thumbnail = self._og_search_thumbnail(webpage, default=None)
+        description = self._html_search_regex(
+            r'''(?x)<div\ class="[^"]*field-type-text-with-summary[^"]*">\s*
+                      (?:<div\ class="[^"]+">\s*)*
+                     <p>\s*(?:<span>)?(.+?)</''',
+            webpage, 'description', default=None)
+        episode_number = int_or_none(self._search_regex(
+            r'(?i)aflevering (\d+)', title, 'episode_number', default=None,
+            fatal=False))
+        upload_date = unified_strdate(self._html_search_regex(
+            r'''(?x)<div\ class="[^"]*field-name-post-date[^"]*">\s*
+                    (?:<div\ class="[^"]+">\s*)*
+                      (\d{2}/\d{2}/\d{4})''',
+            webpage, 'upload_date', default=None))
 
         return {
             'id': video_id,
             'display_id': display_id,
             'title': title,
             'description': description,
+            'episode_number': episode_number,
+            'upload_date': upload_date,
             'thumbnail': thumbnail,
             'formats': formats,
         }

From 650bd9471697390a9cdef2a54d35ec6dd5dc49ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 May 2017 23:38:27 +0700
Subject: [PATCH 314/611] [vier] Relax regexes and extract more metadata
 (closes #12539)

---
 youtube_dl/extractor/vier.py | 39 ++++++++++++++++++++++--------------
 1 file changed, 24 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index 7fb718343..3e67eb8c2 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -27,6 +27,8 @@ class VierIE(InfoExtractor):
             'title': 'Het wordt warm in De Moestuin',
             'description': 'De vele uren werk eisen hun tol. Wim droomt van assistentie...',
             'upload_date': '20121025',
+            'series': 'Plan B',
+            'tags': ['De Moestuin', 'Moestuin', 'meisjes', 'Tomaat', 'Wim', 'Droom'],
         },
     }, {
         'url': 'http://www.vijf.be/temptationisland/videos/zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas/2561614',
@@ -37,6 +39,8 @@ class VierIE(InfoExtractor):
             'title': 'md5:84f45fe48b8c1fa296a7f6d208d080a7',
             'description': 'md5:0356d4981e58b8cbee19355cbd51a8fe',
             'upload_date': '20170228',
+            'series': 'Temptation Island',
+            'tags': list,
         },
         'params': {
             'skip_download': True,
@@ -50,7 +54,9 @@ class VierIE(InfoExtractor):
             'title': 'Jani gaat naar Tokio - Aflevering 4',
             'description': 'md5:aa8d611541db6ae9e863125704511f88',
             'upload_date': '20170501',
+            'series': 'Jani gaat',
             'episode_number': 4,
+            'tags': ['Jani Gaat', 'Volledige Aflevering'],
         },
         'params': {
             'skip_download': True,
@@ -135,33 +141,36 @@ class VierIE(InfoExtractor):
             webpage, 'filename')
 
         playlist_url = 'http://vod.streamcloud.be/%s/_definst_/mp4:%s.mp4/playlist.m3u8' % (application, filename)
-        formats = self._extract_wowza_formats(playlist_url, display_id, skip_protocols=['dash'])
+        formats = self._extract_wowza_formats(
+            playlist_url, display_id, skip_protocols=['dash'])
         self._sort_formats(formats)
 
         title = self._og_search_title(webpage, default=display_id)
-        thumbnail = self._og_search_thumbnail(webpage, default=None)
         description = self._html_search_regex(
-            r'''(?x)<div\ class="[^"]*field-type-text-with-summary[^"]*">\s*
-                      (?:<div\ class="[^"]+">\s*)*
-                     <p>\s*(?:<span>)?(.+?)</''',
-            webpage, 'description', default=None)
-        episode_number = int_or_none(self._search_regex(
-            r'(?i)aflevering (\d+)', title, 'episode_number', default=None,
-            fatal=False))
+            r'(?s)<div\b[^>]+\bclass=(["\'])[^>]*?\bfield-type-text-with-summary\b[^>]*?\1[^>]*>.*?<p>(?P<value>.+?)</p>',
+            webpage, 'description', default=None, group='value')
+        thumbnail = self._og_search_thumbnail(webpage, default=None)
         upload_date = unified_strdate(self._html_search_regex(
-            r'''(?x)<div\ class="[^"]*field-name-post-date[^"]*">\s*
-                    (?:<div\ class="[^"]+">\s*)*
-                      (\d{2}/\d{2}/\d{4})''',
-            webpage, 'upload_date', default=None))
+            r'(?s)<div\b[^>]+\bclass=(["\'])[^>]*?\bfield-name-post-date\b[^>]*?\1[^>]*>.*?(?P<value>\d{2}/\d{2}/\d{4})',
+            webpage, 'upload date', default=None, group='value'))
+
+        series = self._search_regex(
+            r'data-program=(["\'])(?P<value>(?:(?!\1).)+)\1', webpage,
+            'series', default=None, group='value')
+        episode_number = int_or_none(self._search_regex(
+            r'(?i)aflevering (\d+)', title, 'episode number', default=None))
+        tags = re.findall(r'<a\b[^>]+\bhref=["\']/tags/[^>]+>([^<]+)<', webpage)
 
         return {
             'id': video_id,
             'display_id': display_id,
             'title': title,
             'description': description,
-            'episode_number': episode_number,
-            'upload_date': upload_date,
             'thumbnail': thumbnail,
+            'upload_date': upload_date,
+            'series': series,
+            'episode_number': episode_number,
+            'tags': tags,
             'formats': formats,
         }
 

From 7769f837018cbfa6cab615b8ae938528f895ce8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 May 2017 22:18:33 +0700
Subject: [PATCH 315/611] [jsinterp] Add support for quoted names and indexers
 (closes #13123, closes #13130)

---
 youtube_dl/jsinterp.py | 33 +++++++++++++++++----------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 24cdec28c..038910049 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -6,6 +6,7 @@ import re
 
 from .utils import (
     ExtractorError,
+    remove_quotes,
 )
 
 _OPERATORS = [
@@ -57,7 +58,6 @@ class JSInterpreter(object):
 
     def interpret_expression(self, expr, local_vars, allow_recursion):
         expr = expr.strip()
-
         if expr == '':  # Empty expression
             return None
 
@@ -121,11 +121,19 @@ class JSInterpreter(object):
             pass
 
         m = re.match(
-            r'(?P<var>%s)\.(?P<member>[^(]+)(?:\(+(?P<args>[^()]*)\))?$' % _NAME_RE,
+            r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE, expr)
+        if m:
+            val = local_vars[m.group('in')]
+            idx = self.interpret_expression(
+                m.group('idx'), local_vars, allow_recursion - 1)
+            return val[idx]
+
+        m = re.match(
+            r'(?P<var>%s)(?:\.(?P<member>[^(]+)|\[(?P<member2>[^]]+)\])\s*(?:\(+(?P<args>[^()]*)\))?$' % _NAME_RE,
             expr)
         if m:
             variable = m.group('var')
-            member = m.group('member')
+            member = remove_quotes(m.group('member') or m.group('member2'))
             arg_str = m.group('args')
 
             if variable in local_vars:
@@ -173,14 +181,6 @@ class JSInterpreter(object):
 
             return obj[member](argvals)
 
-        m = re.match(
-            r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE, expr)
-        if m:
-            val = local_vars[m.group('in')]
-            idx = self.interpret_expression(
-                m.group('idx'), local_vars, allow_recursion - 1)
-            return val[idx]
-
         for op, opfunc in _OPERATORS:
             m = re.match(r'(?P<x>.+?)%s(?P<y>.+)' % re.escape(op), expr)
             if not m:
@@ -211,21 +211,22 @@ class JSInterpreter(object):
         raise ExtractorError('Unsupported JS expression %r' % expr)
 
     def extract_object(self, objname):
+        _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')'''
         obj = {}
         obj_m = re.search(
             (r'(?<!this\.)%s\s*=\s*\{' % re.escape(objname)) +
-            r'\s*(?P<fields>([a-zA-Z$0-9]+\s*:\s*function\(.*?\)\s*\{.*?\}(?:,\s*)?)*)' +
-            r'\}\s*;',
+            r'\s*(?P<fields>(%s\s*:\s*function\(.*?\)\s*\{.*?\}(?:,\s*)?)*)' +
+            r'\}\s*;' % _FUNC_NAME_RE,
             self.code)
         fields = obj_m.group('fields')
         # Currently, it only supports function definitions
         fields_m = re.finditer(
-            r'(?P<key>[a-zA-Z$0-9]+)\s*:\s*function'
-            r'\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}',
+            r'(?P<key>%s)\s*:\s*function'
+            r'\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}' % _FUNC_NAME_RE,
             fields)
         for f in fields_m:
             argnames = f.group('args').split(',')
-            obj[f.group('key')] = self.build_function(argnames, f.group('code'))
+            obj[remove_quotes(f.group('key'))] = self.build_function(argnames, f.group('code'))
 
         return obj
 

From 3bc1eea0d841aab62a43ac2358ebdb376d507c01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 May 2017 22:29:25 +0700
Subject: [PATCH 316/611] [ChangeLog] Actualize

---
 ChangeLog | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 91c77515f..06efad4e6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,22 @@
+version <unreleased>
+
+Core
++ [jsinterp] Add support for quoted names and indexers (#13123, #13124, #13125,
+  #13126, #13128, #13129, #13130, #13131, #13132)
++ [extractor/common] Add support for schemeless URLs in _extract_wowza_formats
+  (#13088, #13092)
++ [utils] Recognize more audio codecs (#13081)
+
+Extractors
++ [vier] Extract more metadata (#12539)
+* [vier] Improve extraction (#12801)
+    + Add support for authentication
+    * Bypass authentication when no credentials provided
+    * Improve extraction robustness
+* [dailymail] Fix sources extraction (#13057)
+* [dailymotion] Extend URL regular expression (#13079)
+
+
 version 2017.05.14
 
 Core

From 9d63e57d1f6e828a7c9ff3356763c7666bd0202a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 May 2017 22:30:37 +0700
Subject: [PATCH 317/611] release 2017.05.18

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 2955e833d..71a2631b9 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.14*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.14**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.18*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.18**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.05.14
+[debug] youtube-dl version 2017.05.18
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 06efad4e6..443b3aaf6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.05.18
 
 Core
 + [jsinterp] Add support for quoted names and indexers (#13123, #13124, #13125,
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9b074fafc..7fa23d274 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.05.14'
+__version__ = '2017.05.18'

From 9221d5d7a84fd778d54806b3d72b58392481f2bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 May 2017 22:37:32 +0700
Subject: [PATCH 318/611] [ChangeLog] Fix typo

---
 ChangeLog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 443b3aaf6..476387002 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -24,7 +24,7 @@ Core
 + [postprocessor/metadatafromtitle] Add support regular expression syntax for
   --metadata-from-title (#13065)
 
-Extractor
+Extractors
 + [mediaset] Add support for video.mediaset.it (#12708, #12964)
 * [orf:radio] Fix extraction (#11643, #12926)
 * [aljazeera] Extend URL regular expression (#13053)

From 0e2d626ddd2c0c4a8e369026812c7092f5d17de2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 May 2017 22:57:38 +0700
Subject: [PATCH 319/611] [jsinterp] Fix typo and cleanup regexes (closes
 #13134)

---
 youtube_dl/jsinterp.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 038910049..7bda59610 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -214,15 +214,18 @@ class JSInterpreter(object):
         _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')'''
         obj = {}
         obj_m = re.search(
-            (r'(?<!this\.)%s\s*=\s*\{' % re.escape(objname)) +
-            r'\s*(?P<fields>(%s\s*:\s*function\(.*?\)\s*\{.*?\}(?:,\s*)?)*)' +
-            r'\}\s*;' % _FUNC_NAME_RE,
+            r'''(?x)
+                (?<!this\.)%s\s*=\s*{\s*
+                    (?P<fields>(%s\s*:\s*function\s*\(.*?\)\s*{.*?}(?:,\s*)?)*)
+                }\s*;
+            ''' % (re.escape(objname), _FUNC_NAME_RE),
             self.code)
         fields = obj_m.group('fields')
         # Currently, it only supports function definitions
         fields_m = re.finditer(
-            r'(?P<key>%s)\s*:\s*function'
-            r'\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}' % _FUNC_NAME_RE,
+            r'''(?x)
+                (?P<key>%s)\s*:\s*function\s*\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}
+            ''' % _FUNC_NAME_RE,
             fields)
         for f in fields_m:
             argnames = f.group('args').split(',')

From bcaa1dd0609fea4a5ef108c27f53f399c4d24765 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 May 2017 22:58:14 +0700
Subject: [PATCH 320/611] [ChangeLog] Actualize

---
 ChangeLog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 476387002..606e60313 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Core
+* [jsinterp] Fix typo and cleanup regular expressions (#13134)
+
+
 version 2017.05.18
 
 Core

From bfdf6fcc6677531a296d51ba97e110f0a4556fbf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 May 2017 23:00:03 +0700
Subject: [PATCH 321/611] release 2017.05.18.1

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 71a2631b9..b714fcfa8 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.18*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.18**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.18.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.18.1**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.05.18
+[debug] youtube-dl version 2017.05.18.1
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 606e60313..1637876b3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.05.18.1
 
 Core
 * [jsinterp] Fix typo and cleanup regular expressions (#13134)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7fa23d274..5e963e71c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.05.18'
+__version__ = '2017.05.18.1'

From 00cb0faca8a2875355651611bd90c4e8a856a2bf Mon Sep 17 00:00:00 2001
From: Logan B <einstein95@users.noreply.github.com>
Date: Sat, 20 May 2017 02:54:57 +1200
Subject: [PATCH 322/611] [mitele] Update app key regex

---
 youtube_dl/extractor/mitele.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 28b743cca..964dc542c 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -136,11 +136,9 @@ class MiTeleIE(InfoExtractor):
             video_id, 'Downloading gigya script')
 
         # Get a appKey/uuid for getting the session key
-        appKey_var = self._search_regex(
-            r'value\s*\(\s*["\']appGridApplicationKey["\']\s*,\s*([0-9a-f]+)',
-            gigya_sc, 'appKey variable')
         appKey = self._search_regex(
-            r'var\s+%s\s*=\s*["\']([0-9a-f]+)' % appKey_var, gigya_sc, 'appKey')
+            r'constant\s*\(\s*["\']_appGridApplicationKey["\']\s*,\s*["\']([0-9a-f]+)',
+            gigya_sc, 'appKey')
 
         session_json = self._download_json(
             'https://appgrid-api.cloud.accedo.tv/session',

From 5f6fbcea08bc6473cf24c8f2ffee38bc01c80de0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 19 May 2017 22:29:54 +0700
Subject: [PATCH 323/611] [hitbox] Add support for smashcast.tv (closes #13154)

---
 youtube_dl/extractor/hitbox.py | 52 ++++++++++++++++++++--------------
 1 file changed, 30 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/hitbox.py b/youtube_dl/extractor/hitbox.py
index e21ebb8fb..1d905dc81 100644
--- a/youtube_dl/extractor/hitbox.py
+++ b/youtube_dl/extractor/hitbox.py
@@ -16,8 +16,8 @@ from ..utils import (
 
 class HitboxIE(InfoExtractor):
     IE_NAME = 'hitbox'
-    _VALID_URL = r'https?://(?:www\.)?hitbox\.tv/video/(?P<id>[0-9]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?(?:hitbox|smashcast)\.tv/(?:[^/]+/)*videos?/(?P<id>[0-9]+)'
+    _TESTS = [{
         'url': 'http://www.hitbox.tv/video/203213',
         'info_dict': {
             'id': '203213',
@@ -38,13 +38,15 @@ class HitboxIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
-    }
+    }, {
+        'url': 'https://www.smashcast.tv/hitboxlive/videos/203213',
+        'only_matching': True,
+    }]
 
     def _extract_metadata(self, url, video_id):
         thumb_base = 'https://edge.sf.hitbox.tv'
         metadata = self._download_json(
-            '%s/%s' % (url, video_id), video_id,
-            'Downloading metadata JSON')
+            '%s/%s' % (url, video_id), video_id, 'Downloading metadata JSON')
 
         date = 'media_live_since'
         media_type = 'livestream'
@@ -63,14 +65,15 @@ class HitboxIE(InfoExtractor):
         views = int_or_none(video_meta.get('media_views'))
         timestamp = parse_iso8601(video_meta.get(date), ' ')
         categories = [video_meta.get('category_name')]
-        thumbs = [
-            {'url': thumb_base + video_meta.get('media_thumbnail'),
-             'width': 320,
-             'height': 180},
-            {'url': thumb_base + video_meta.get('media_thumbnail_large'),
-             'width': 768,
-             'height': 432},
-        ]
+        thumbs = [{
+            'url': thumb_base + video_meta.get('media_thumbnail'),
+            'width': 320,
+            'height': 180
+        }, {
+            'url': thumb_base + video_meta.get('media_thumbnail_large'),
+            'width': 768,
+            'height': 432
+        }]
 
         return {
             'id': video_id,
@@ -90,7 +93,7 @@ class HitboxIE(InfoExtractor):
         video_id = self._match_id(url)
 
         player_config = self._download_json(
-            'https://www.hitbox.tv/api/player/config/video/%s' % video_id,
+            'https://www.smashcast.tv/api/player/config/video/%s' % video_id,
             video_id, 'Downloading video JSON')
 
         formats = []
@@ -121,8 +124,7 @@ class HitboxIE(InfoExtractor):
         self._sort_formats(formats)
 
         metadata = self._extract_metadata(
-            'https://www.hitbox.tv/api/media/video',
-            video_id)
+            'https://www.smashcast.tv/api/media/video', video_id)
         metadata['formats'] = formats
 
         return metadata
@@ -130,8 +132,8 @@ class HitboxIE(InfoExtractor):
 
 class HitboxLiveIE(HitboxIE):
     IE_NAME = 'hitbox:live'
-    _VALID_URL = r'https?://(?:www\.)?hitbox\.tv/(?!video)(?P<id>.+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?(?:hitbox|smashcast)\.tv/(?P<id>[^/?#&]+)'
+    _TESTS = [{
         'url': 'http://www.hitbox.tv/dimak',
         'info_dict': {
             'id': 'dimak',
@@ -146,13 +148,20 @@ class HitboxLiveIE(HitboxIE):
             # live
             'skip_download': True,
         },
-    }
+    }, {
+        'url': 'https://www.smashcast.tv/dimak',
+        'only_matching': True,
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return False if HitboxIE.suitable(url) else super(HitboxLiveIE, cls).suitable(url)
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         player_config = self._download_json(
-            'https://www.hitbox.tv/api/player/config/live/%s' % video_id,
+            'https://www.smashcast.tv/api/player/config/live/%s' % video_id,
             video_id)
 
         formats = []
@@ -197,8 +206,7 @@ class HitboxLiveIE(HitboxIE):
         self._sort_formats(formats)
 
         metadata = self._extract_metadata(
-            'https://www.hitbox.tv/api/media/live',
-            video_id)
+            'https://www.smashcast.tv/api/media/live', video_id)
         metadata['formats'] = formats
         metadata['is_live'] = True
         metadata['title'] = self._live_title(metadata.get('title'))

From 8fffac6927b4144bfb3e473b58fa52313042ecbb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 19 May 2017 23:11:02 +0700
Subject: [PATCH 324/611] [njpwworld] Fix extraction (closes #13162)

---
 youtube_dl/extractor/njpwworld.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/njpwworld.py b/youtube_dl/extractor/njpwworld.py
index f5e3f6815..9b5ad5a9f 100644
--- a/youtube_dl/extractor/njpwworld.py
+++ b/youtube_dl/extractor/njpwworld.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
+    extract_attributes,
     get_element_by_class,
     urlencode_postdata,
 )
@@ -56,17 +57,24 @@ class NJPWWorldIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         formats = []
-        for player_url, kind in re.findall(r'<a[^>]+href="(/player[^"]+)".+?<img[^>]+src="[^"]+qf_btn_([^".]+)', webpage):
-            player_url = compat_urlparse.urljoin(url, player_url)
-
+        for mobj in re.finditer(r'<a[^>]+\bhref=(["\'])/player.+?[^>]*>', webpage):
+            player = extract_attributes(mobj.group(0))
+            player_path = player.get('href')
+            if not player_path:
+                continue
+            kind = self._search_regex(
+                r'(low|high)$', player.get('class') or '', 'kind',
+                default='low')
+            player_url = compat_urlparse.urljoin(url, player_path)
             player_page = self._download_webpage(
                 player_url, video_id, note='Downloading player page')
-
             entries = self._parse_html5_media_entries(
                 player_url, player_page, video_id, m3u8_id='hls-%s' % kind,
-                m3u8_entry_protocol='m3u8_native',
-                preference=2 if 'hq' in kind else 1)
-            formats.extend(entries[0]['formats'])
+                m3u8_entry_protocol='m3u8_native')
+            kind_formats = entries[0]['formats']
+            for f in kind_formats:
+                f['quality'] = 2 if kind == 'high' else 1
+            formats.extend(kind_formats)
 
         self._sort_formats(formats)
 

From 77d682da9dc2e62a1505cccb77408b8fa638a7f1 Mon Sep 17 00:00:00 2001
From: vobe <vobe@jackal.fr>
Date: Fri, 19 May 2017 20:18:03 +0200
Subject: [PATCH 325/611] [toypics] Fix extraction

---
 youtube_dl/extractor/toypics.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/toypics.py b/youtube_dl/extractor/toypics.py
index 938e05076..774fa565c 100644
--- a/youtube_dl/extractor/toypics.py
+++ b/youtube_dl/extractor/toypics.py
@@ -24,15 +24,16 @@ class ToypicsIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         page = self._download_webpage(url, video_id)
-        video_url = self._html_search_regex(
-            r'src:\s+"(http://static[0-9]+\.toypics\.net/flvideo/[^"]+)"', page, 'video URL')
-        title = self._html_search_regex(
-            r'<title>Toypics - ([^<]+)</title>', page, 'title')
+        formats = self._parse_html5_media_entries(url, page, video_id)[0]['formats']
+        title = self._html_search_regex([
+            r'<h1[^>]+class=["\']view-video-title[^>]+>([^<]+)</h',
+            r'<title>([^<]+) - Toypics</title>',
+        ], page, 'title')
         username = self._html_search_regex(
-            r'toypics.net/([^/"]+)" class="user-name">', page, 'username')
+            r'More videos from <strong>([^<]+)</strong>', page, 'username')
         return {
             'id': video_id,
-            'url': video_url,
+            'formats': formats,
             'title': title,
             'uploader': username,
             'age_limit': 18,
@@ -71,7 +72,7 @@ class ToypicsUserIE(InfoExtractor):
                 note='Downloading page %d/%d' % (n, page_count))
             urls.extend(
                 re.findall(
-                    r'<p class="video-entry-title">\s+<a href="(https?://videos.toypics.net/view/[^"]+)">',
+                    r'<div[^>]+class=["\']preview[^>]+>\s*<a[^>]+href="(https?://videos.toypics.net/view/[^"]+)"',
                     lpage))
 
         return {

From 2ca29f1aafda58fd7fd6db55d8b99eb89e07f2df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 20 May 2017 01:28:42 +0700
Subject: [PATCH 326/611] [toypics] Improve and modernize

---
 youtube_dl/extractor/toypics.py | 34 ++++++++++++++++++---------------
 1 file changed, 19 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/toypics.py b/youtube_dl/extractor/toypics.py
index 774fa565c..f705a06c9 100644
--- a/youtube_dl/extractor/toypics.py
+++ b/youtube_dl/extractor/toypics.py
@@ -6,43 +6,48 @@ import re
 
 
 class ToypicsIE(InfoExtractor):
-    IE_DESC = 'Toypics user profile'
-    _VALID_URL = r'https?://videos\.toypics\.net/view/(?P<id>[0-9]+)/.*'
+    IE_DESC = 'Toypics video'
+    _VALID_URL = r'https?://videos\.toypics\.net/view/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://videos.toypics.net/view/514/chancebulged,-2-1/',
         'md5': '16e806ad6d6f58079d210fe30985e08b',
         'info_dict': {
             'id': '514',
             'ext': 'mp4',
-            'title': 'Chance-Bulge\'d, 2',
+            'title': "Chance-Bulge'd, 2",
             'age_limit': 18,
             'uploader': 'kidsune',
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        page = self._download_webpage(url, video_id)
-        formats = self._parse_html5_media_entries(url, page, video_id)[0]['formats']
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        formats = self._parse_html5_media_entries(
+            url, webpage, video_id)[0]['formats']
         title = self._html_search_regex([
             r'<h1[^>]+class=["\']view-video-title[^>]+>([^<]+)</h',
             r'<title>([^<]+) - Toypics</title>',
-        ], page, 'title')
-        username = self._html_search_regex(
-            r'More videos from <strong>([^<]+)</strong>', page, 'username')
+        ], webpage, 'title')
+
+        uploader = self._html_search_regex(
+            r'More videos from <strong>([^<]+)</strong>', webpage, 'uploader',
+            fatal=False)
+
         return {
             'id': video_id,
             'formats': formats,
             'title': title,
-            'uploader': username,
+            'uploader': uploader,
             'age_limit': 18,
         }
 
 
 class ToypicsUserIE(InfoExtractor):
     IE_DESC = 'Toypics user profile'
-    _VALID_URL = r'https?://videos\.toypics\.net/(?P<username>[^/?]+)(?:$|[?#])'
+    _VALID_URL = r'https?://videos\.toypics\.net/(?!view)(?P<id>[^/?#&]+)'
     _TEST = {
         'url': 'http://videos.toypics.net/Mikey',
         'info_dict': {
@@ -52,8 +57,7 @@ class ToypicsUserIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        username = mobj.group('username')
+        username = self._match_id(url)
 
         profile_page = self._download_webpage(
             url, username, note='Retrieving profile page')
@@ -72,7 +76,7 @@ class ToypicsUserIE(InfoExtractor):
                 note='Downloading page %d/%d' % (n, page_count))
             urls.extend(
                 re.findall(
-                    r'<div[^>]+class=["\']preview[^>]+>\s*<a[^>]+href="(https?://videos.toypics.net/view/[^"]+)"',
+                    r'<div[^>]+class=["\']preview[^>]+>\s*<a[^>]+href="(https?://videos\.toypics\.net/view/[^"]+)"',
                     lpage))
 
         return {

From daa4e9ff90e9c017238dae48e9862ebe02bc3ff2 Mon Sep 17 00:00:00 2001
From: Nii-90 <Nii-90@users.noreply.github.com>
Date: Sat, 20 May 2017 09:50:46 -0400
Subject: [PATCH 327/611] [adobepass] Add support for Brighthouse MSO

---
 youtube_dl/extractor/adobepass.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index 7da96c65c..d57ad85c2 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -36,6 +36,11 @@ MSO_INFO = {
         'username_field': 'Ecom_User_ID',
         'password_field': 'Ecom_Password',
     },
+    'Brighthouse': {
+        'name': 'Bright House Networks | Spectrum',
+        'username_field': 'j_username',
+        'password_field': 'j_password',
+    },
     'Charter_Direct': {
         'name': 'Charter Spectrum',
         'username_field': 'IDToken1',

From 35a2d221a35caf8aad533b5f8ef6d7b12acd4e41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 20 May 2017 23:06:30 +0700
Subject: [PATCH 328/611] [toggle] Relax _VALID_URL (closes #13172)

---
 youtube_dl/extractor/toggle.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/toggle.py b/youtube_dl/extractor/toggle.py
index c54b876d3..348d6ecdf 100644
--- a/youtube_dl/extractor/toggle.py
+++ b/youtube_dl/extractor/toggle.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 class ToggleIE(InfoExtractor):
     IE_NAME = 'toggle'
-    _VALID_URL = r'https?://video\.toggle\.sg/(?:en|zh)/(?:series|clips|movies)/(?:[^/]+/)+(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://video\.toggle\.sg/(?:en|zh)/(?:[^/]+/){2,}(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://video.toggle.sg/en/series/lion-moms-tif/trailers/lion-moms-premier/343115',
         'info_dict': {
@@ -73,6 +73,12 @@ class ToggleIE(InfoExtractor):
     }, {
         'url': 'http://video.toggle.sg/en/movies/seven-days/321936',
         'only_matching': True,
+    }, {
+        'url': 'https://video.toggle.sg/en/tv-show/news/may-2017-cna-singapore-tonight/fri-19-may-2017/512456',
+        'only_matching': True,
+    }, {
+        'url': 'http://video.toggle.sg/en/channels/eleven-plus/401585',
+        'only_matching': True,
     }]
 
     _FORMAT_PREFERENCES = {

From fdc20f87a62ff973b97c9ea2b0e68b394960b8fa Mon Sep 17 00:00:00 2001
From: Protuhj <ihatespam234@gmail.com>
Date: Sat, 20 May 2017 14:11:37 -0400
Subject: [PATCH 329/611] [youtube] Fix DASH manifest signature decryption
 (closes #8944)

---
 youtube_dl/extractor/youtube.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 44a39282f..ae832cac5 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1435,6 +1435,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             age_gate = False
             video_info = None
+            sts = ''
             # Try looking directly into the video webpage
             ytplayer_config = self._get_ytplayer_config(video_id, video_webpage)
             if ytplayer_config:
@@ -1451,6 +1452,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         args['ypc_vid'], YoutubeIE.ie_key(), video_id=args['ypc_vid'])
                 if args.get('livestream') == '1' or args.get('live_playback') == 1:
                     is_live = True
+                sts = ytplayer_config.get('sts', '')
             if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):
                 # We also try looking in get_video_info since it may contain different dashmpd
                 # URL that points to a DASH manifest with possibly different itag set (some itags
@@ -1461,8 +1463,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 self.report_video_info_webpage_download(video_id)
                 for el_type in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']:
                     video_info_url = (
-                        '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
-                        % (proto, video_id, el_type))
+                        '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en&sts=%s'
+                        % (proto, video_id, el_type, sts))
                     video_info_webpage = self._download_webpage(
                         video_info_url,
                         video_id, note=False,

From dc4e4f90a285dfebd060da55973ad3d1bc88ab7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 May 2017 01:18:56 +0700
Subject: [PATCH 330/611] [youtube] Modernize

---
 youtube_dl/extractor/youtube.py | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ae832cac5..d66693c0c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1435,7 +1435,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             age_gate = False
             video_info = None
-            sts = ''
+            sts = None
             # Try looking directly into the video webpage
             ytplayer_config = self._get_ytplayer_config(video_id, video_webpage)
             if ytplayer_config:
@@ -1452,7 +1452,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         args['ypc_vid'], YoutubeIE.ie_key(), video_id=args['ypc_vid'])
                 if args.get('livestream') == '1' or args.get('live_playback') == 1:
                     is_live = True
-                sts = ytplayer_config.get('sts', '')
+                sts = ytplayer_config.get('sts')
             if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):
                 # We also try looking in get_video_info since it may contain different dashmpd
                 # URL that points to a DASH manifest with possibly different itag set (some itags
@@ -1461,14 +1461,25 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 # The general idea is to take a union of itags of both DASH manifests (for example
                 # video with such 'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093)
                 self.report_video_info_webpage_download(video_id)
-                for el_type in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']:
-                    video_info_url = (
-                        '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en&sts=%s'
-                        % (proto, video_id, el_type, sts))
+                for el in ('info', 'embedded', 'detailpage', 'vevo', ''):
+                    query = {
+                        'video_id': video_id,
+                        'ps': 'default',
+                        'eurl': '',
+                        'gl': 'US',
+                        'hl': 'en',
+                    }
+                    if el:
+                        query['el'] = el
+                    if sts:
+                        query['sts'] = sts
                     video_info_webpage = self._download_webpage(
-                        video_info_url,
+                        '%s://www.youtube.com/get_video_info' % proto,
                         video_id, note=False,
-                        errnote='unable to download video info webpage')
+                        errnote='unable to download video info webpage',
+                        fatal=False, query=query)
+                    if not video_info_webpage:
+                        continue
                     get_video_info = compat_parse_qs(video_info_webpage)
                     if get_video_info.get('use_cipher_signature') != ['True']:
                         add_dash_mpd(get_video_info)

From afdb387cd8f9f54ee666cf1f64416d877f4eba19 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Caletka?= <ondrej@caletka.cz>
Date: Sun, 21 May 2017 10:41:52 +0200
Subject: [PATCH 331/611] [streamcz] Add support for subtitles

---
 youtube_dl/extractor/streamcz.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/streamcz.py b/youtube_dl/extractor/streamcz.py
index 9e533103c..58e0b4c80 100644
--- a/youtube_dl/extractor/streamcz.py
+++ b/youtube_dl/extractor/streamcz.py
@@ -26,7 +26,7 @@ class StreamCZIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.stream.cz/peklonataliri/765767-ecka-pro-deti',
-        'md5': '6d3ca61a8d0633c9c542b92fcb936b0c',
+        'md5': '934bb6a6d220d99c010783c9719960d5',
         'info_dict': {
             'id': '765767',
             'ext': 'mp4',
@@ -37,7 +37,7 @@ class StreamCZIE(InfoExtractor):
         },
     }, {
         'url': 'http://www.stream.cz/blanik/10002447-tri-roky-pro-mazanka',
-        'md5': 'e54a254fb8b871968fd8403255f28589',
+        'md5': '849a88c1e1ca47d41403c2ba5e59e261',
         'info_dict': {
             'id': '10002447',
             'ext': 'mp4',
@@ -85,6 +85,14 @@ class StreamCZIE(InfoExtractor):
         else:
             title = data['name']
 
+        subtitles = {}
+        srt_url = data.get('subtitles_srt')
+        if srt_url:
+            subtitles['cs'] = [{
+                'ext': 'srt',
+                'url': srt_url,
+            }]
+
         return {
             'id': video_id,
             'title': title,
@@ -93,4 +101,5 @@ class StreamCZIE(InfoExtractor):
             'description': data.get('web_site_text'),
             'duration': int_or_none(data.get('duration')),
             'view_count': int_or_none(data.get('views')),
+            'subtitles': subtitles,
         }

From a609e61a90ad8f3b472e2e221f9f75c8408b65f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 May 2017 23:40:07 +0700
Subject: [PATCH 332/611] [downloader/external] Pass -loglevel to ffmpeg
 downloader (closes #13183)

---
 youtube_dl/downloader/external.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index e78169a0d..db018fa89 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -212,6 +212,11 @@ class FFmpegFD(ExternalFD):
 
         args = [ffpp.executable, '-y']
 
+        for log_level in ('quiet', 'verbose'):
+            if self.params.get(log_level, False):
+                args += ['-loglevel', log_level]
+                break
+
         seekable = info_dict.get('_seekable')
         if seekable is not None:
             # setting -seekable prevents ffmpeg from guessing if the server

From 0b75e42dfb36e973e7448de4b031bde24bd3e614 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 May 2017 00:00:49 +0700
Subject: [PATCH 333/611] Credit @zurfyx for atresplayer improvements (#12548)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 1bdb74285..7fd47d5fb 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -212,3 +212,4 @@ Xiao Di Guan
 Thomas Winant
 Daniel Twardowski
 Jeremie Jarosh
+Gerard Rovira

From 06c1b3ce070a36bc769b599cb36bc71dd59bed86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 May 2017 00:07:31 +0700
Subject: [PATCH 334/611] Credit @mphe for streamango (#12643)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 7fd47d5fb..487a2bdab 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -213,3 +213,4 @@ Thomas Winant
 Daniel Twardowski
 Jeremie Jarosh
 Gerard Rovira
+Marvin Ewald

From 01f3c8e29003b3b82f33849975e36962ba34372c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 May 2017 00:08:14 +0700
Subject: [PATCH 335/611] Credit @fredbourni for noovo (#12792)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 487a2bdab..b5e8c074b 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -214,3 +214,4 @@ Daniel Twardowski
 Jeremie Jarosh
 Gerard Rovira
 Marvin Ewald
+Frédéric Bournival

From 9dc5ab041f62d9e968f9c08f26fc98b92d819a4e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 May 2017 00:15:44 +0700
Subject: [PATCH 336/611] [ChangeLog] Actualize

---
 ChangeLog | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 1637876b3..373fa12a6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,19 @@
+version <unreleased>
+
+Core
++ [downloader/external] Pass -loglevel to ffmpeg downloader (#13183)
++ [adobepass] Add support for Bright House Networks (#13149)
+
+Extractors
++ [streamcz] Add support for subtitles (#13174)
+* [youtube] Fix DASH manifest signature decryption (#8944, #13156)
+* [toggle] Relax URL regular expression (#13172)
+* [toypics] Fix extraction (#13077)
+* [njpwworld] Fix extraction (#13162, #13169)
++ [hitbox] Add support for smashcast.tv (#13154)
+* [mitele] Update app key regular expression (#13158)
+
+
 version 2017.05.18.1
 
 Core

From 2570e851677f1930c32b953424fa6fe8d8769856 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 May 2017 00:17:48 +0700
Subject: [PATCH 337/611] release 2017.05.23

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 +-
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index b714fcfa8..730a07ed1 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.18.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.18.1**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.23*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.23**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.05.18.1
+[debug] youtube-dl version 2017.05.23
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 373fa12a6..75a95f0b0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.05.23
 
 Core
 + [downloader/external] Pass -loglevel to ffmpeg downloader (#13183)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index aa6c118f2..5545cff23 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -803,7 +803,7 @@
  - **ToonGoggles**
  - **Tosh**: Tosh.0
  - **tou.tv**
- - **Toypics**: Toypics user profile
+ - **Toypics**: Toypics video
  - **ToypicsUser**: Toypics user profile
  - **TrailerAddict** (Currently broken)
  - **Trilulilu**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5e963e71c..c006eba76 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.05.18.1'
+__version__ = '2017.05.23'

From de53511201589f12b252680cfb2c35ac73fa8c9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 May 2017 00:41:53 +0700
Subject: [PATCH 338/611] Credit @timendum for rai (#11790) and mediaset
 (#12964)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index b5e8c074b..9a8a72265 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -215,3 +215,4 @@ Jeremie Jarosh
 Gerard Rovira
 Marvin Ewald
 Frédéric Bournival
+Timendum

From 0551f1b07b41bf64b4b8c1150abec2903e37a81f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 May 2017 21:47:40 +0700
Subject: [PATCH 339/611] Credit @gritstub for vevo fix (#12879)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 9a8a72265..f80c34a08 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -216,3 +216,4 @@ Gerard Rovira
 Marvin Ewald
 Frédéric Bournival
 Timendum
+gritstub

From 884d09f3304992837be140e1e22ff44a87cbf6ff Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 24 May 2017 09:44:41 +0100
Subject: [PATCH 340/611] [cbsnews] fix extraction for 60 Minutes videos

---
 youtube_dl/extractor/cbsnews.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 17bb9af4f..651ae2665 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -65,7 +65,7 @@ class CBSNewsIE(CBSIE):
 
         item = video_info['item'] if 'item' in video_info else video_info
         guid = item['mpxRefId']
-        return self._extract_video_info(guid)
+        return self._extract_video_info(guid, 'cbsnews')
 
 
 class CBSNewsLiveVideoIE(InfoExtractor):

From e834f04400ba419b842743224510ca8abaf933c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 May 2017 22:58:16 +0700
Subject: [PATCH 341/611] [vimeo] Fix formats' sorting (closes #13189)

---
 youtube_dl/extractor/vimeo.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 61cc469bf..a483c8409 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -151,10 +151,16 @@ class VimeoBaseInfoExtractor(InfoExtractor):
                     else:
                         mpd_manifest_urls = [(format_id, manifest_url)]
                     for f_id, m_url in mpd_manifest_urls:
-                        formats.extend(self._extract_mpd_formats(
+                        mpd_formats = self._extract_mpd_formats(
                             m_url.replace('/master.json', '/master.mpd'), video_id, f_id,
                             'Downloading %s MPD information' % cdn_name,
-                            fatal=False))
+                            fatal=False)
+                        for f in mpd_formats:
+                            if f.get('vcodec') == 'none':
+                                f['preference'] = -50
+                            elif f.get('acodec') == 'none':
+                                f['preference'] = -40
+                        formats.extend(mpd_formats)
 
         subtitles = {}
         text_tracks = config['request'].get('text_tracks')

From 5192ee17e79ec103abf1e5a06faa0b0019ab00f2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 25 May 2017 22:04:12 +0800
Subject: [PATCH 342/611] [postprocessor/ffmpeg] Fix metadata filename handling
 on Python 2

Fixes #13182
---
 ChangeLog                          | 6 ++++++
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 75a95f0b0..9e4f53093 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Core
+* [postprocessor/ffmpeg] Fix metadata filename handling on Python 2 (#13182)
+
+
 version 2017.05.23
 
 Core
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index c91ec8588..f021ea8fd 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -444,7 +444,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
 
         chapters = info.get('chapters', [])
         if chapters:
-            metadata_filename = encodeFilename(replace_extension(filename, 'meta'))
+            metadata_filename = replace_extension(filename, 'meta')
             with io.open(metadata_filename, 'wt', encoding='utf-8') as f:
                 def ffmpeg_escape(text):
                     return re.sub(r'(=|;|#|\\|\n)', r'\\\1', text)

From b7f8749304aca8e8f338f40a1b2877e4ac732c73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 May 2017 22:28:26 +0700
Subject: [PATCH 343/611] [udemy] Fix extraction for outputs' format entries
 without URL (closes #13192)

---
 youtube_dl/extractor/udemy.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index dae1aa3c6..160be1b1b 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -52,6 +52,10 @@ class UdemyIE(InfoExtractor):
         # new URL schema
         'url': 'https://www.udemy.com/electric-bass-right-from-the-start/learn/v4/t/lecture/4580906',
         'only_matching': True,
+    }, {
+        # no url in outputs format entry
+        'url': 'https://www.udemy.com/learn-web-development-complete-step-by-step-guide-to-success/learn/v4/t/lecture/4125812',
+        'only_matching': True,
     }]
 
     def _extract_course_info(self, webpage, video_id):
@@ -219,7 +223,7 @@ class UdemyIE(InfoExtractor):
 
         def extract_output_format(src, f_id):
             return {
-                'url': src['url'],
+                'url': src.get('url'),
                 'format_id': '%sp' % (src.get('height') or f_id),
                 'width': int_or_none(src.get('width')),
                 'height': int_or_none(src.get('height')),

From 59ed87cbd9ea08c889514a05b646141004f432a1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 May 2017 14:15:46 +0800
Subject: [PATCH 344/611] [youku] Fix extraction (closes #13191)

---
 ChangeLog                     |   3 +
 youtube_dl/extractor/youku.py | 214 +++++++---------------------------
 2 files changed, 45 insertions(+), 172 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 9e4f53093..6a05657ab 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -3,6 +3,9 @@ version <unreleased>
 Core
 * [postprocessor/ffmpeg] Fix metadata filename handling on Python 2 (#13182)
 
+Extractors
+* [youku] Fix extraction (#13191)
+
 
 version 2017.05.23
 
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 73ebe5759..37be440fa 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import base64
 import itertools
 import random
 import re
@@ -9,15 +8,9 @@ import string
 import time
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_ord,
-    compat_str,
-    compat_urllib_parse_urlencode,
-)
 from ..utils import (
     ExtractorError,
     get_element_by_attribute,
-    try_get,
 )
 
 
@@ -35,9 +28,9 @@ class YoukuIE(InfoExtractor):
         # MD5 is unstable
         'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html',
         'info_dict': {
-            'id': 'XMTc1ODE5Njcy_part1',
+            'id': 'XMTc1ODE5Njcy',
             'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',
-            'ext': 'flv'
+            'ext': 'mp4',
         }
     }, {
         'url': 'http://player.youku.com/player.php/sid/XNDgyMDQ2NTQw/v.swf',
@@ -46,25 +39,24 @@ class YoukuIE(InfoExtractor):
         'url': 'http://v.youku.com/v_show/id_XODgxNjg1Mzk2_ev_1.html',
         'info_dict': {
             'id': 'XODgxNjg1Mzk2',
+            'ext': 'mp4',
             'title': '武媚娘传奇 85',
         },
-        'playlist_count': 11,
-        'skip': 'Available in China only',
     }, {
         'url': 'http://v.youku.com/v_show/id_XMTI1OTczNDM5Mg==.html',
         'info_dict': {
             'id': 'XMTI1OTczNDM5Mg',
+            'ext': 'mp4',
             'title': '花千骨 04',
         },
-        'playlist_count': 13,
     }, {
         'url': 'http://v.youku.com/v_show/id_XNjA1NzA2Njgw.html',
         'note': 'Video protected with password',
         'info_dict': {
             'id': 'XNjA1NzA2Njgw',
+            'ext': 'mp4',
             'title': '邢義田复旦讲座之想象中的胡人—从“左衽孔子”说起',
         },
-        'playlist_count': 19,
         'params': {
             'videopassword': '100600',
         },
@@ -73,130 +65,16 @@ class YoukuIE(InfoExtractor):
         'url': 'http://v.youku.com/v_show/id_XOTUxMzg4NDMy.html',
         'info_dict': {
             'id': 'XOTUxMzg4NDMy',
+            'ext': 'mp4',
             'title': '我的世界☆明月庄主☆车震猎杀☆杀人艺术Minecraft',
         },
-        'playlist_count': 6,
     }]
 
-    def construct_video_urls(self, data):
-        # get sid, token
-        def yk_t(s1, s2):
-            ls = list(range(256))
-            t = 0
-            for i in range(256):
-                t = (t + ls[i] + compat_ord(s1[i % len(s1)])) % 256
-                ls[i], ls[t] = ls[t], ls[i]
-            s = bytearray()
-            x, y = 0, 0
-            for i in range(len(s2)):
-                y = (y + 1) % 256
-                x = (x + ls[y]) % 256
-                ls[x], ls[y] = ls[y], ls[x]
-                s.append(compat_ord(s2[i]) ^ ls[(ls[x] + ls[y]) % 256])
-            return bytes(s)
-
-        sid, token = yk_t(
-            b'becaf9be', base64.b64decode(data['security']['encrypt_string'].encode('ascii'))
-        ).decode('ascii').split('_')
-
-        # get oip
-        oip = data['security']['ip']
-
-        fileid_dict = {}
-        for stream in data['stream']:
-            if stream.get('channel_type') == 'tail':
-                continue
-            format = stream.get('stream_type')
-            fileid = try_get(
-                stream, lambda x: x['segs'][0]['fileid'],
-                compat_str) or stream['stream_fileid']
-            fileid_dict[format] = fileid
-
-        def get_fileid(format, n):
-            number = hex(int(str(n), 10))[2:].upper()
-            if len(number) == 1:
-                number = '0' + number
-            streamfileids = fileid_dict[format]
-            fileid = streamfileids[0:8] + number + streamfileids[10:]
-            return fileid
-
-        # get ep
-        def generate_ep(format, n):
-            fileid = get_fileid(format, n)
-            ep_t = yk_t(
-                b'bf7e5f01',
-                ('%s_%s_%s' % (sid, fileid, token)).encode('ascii')
-            )
-            ep = base64.b64encode(ep_t).decode('ascii')
-            return ep
-
-        # generate video_urls
-        video_urls_dict = {}
-        for stream in data['stream']:
-            if stream.get('channel_type') == 'tail':
-                continue
-            format = stream.get('stream_type')
-            video_urls = []
-            for dt in stream['segs']:
-                n = str(stream['segs'].index(dt))
-                param = {
-                    'K': dt['key'],
-                    'hd': self.get_hd(format),
-                    'myp': 0,
-                    'ypp': 0,
-                    'ctype': 12,
-                    'ev': 1,
-                    'token': token,
-                    'oip': oip,
-                    'ep': generate_ep(format, n)
-                }
-                video_url = \
-                    'http://k.youku.com/player/getFlvPath/' + \
-                    'sid/' + sid + \
-                    '_00' + \
-                    '/st/' + self.parse_ext_l(format) + \
-                    '/fileid/' + get_fileid(format, n) + '?' + \
-                    compat_urllib_parse_urlencode(param)
-                video_urls.append(video_url)
-            video_urls_dict[format] = video_urls
-
-        return video_urls_dict
-
     @staticmethod
     def get_ysuid():
         return '%d%s' % (int(time.time()), ''.join([
             random.choice(string.ascii_letters) for i in range(3)]))
 
-    def get_hd(self, fm):
-        hd_id_dict = {
-            '3gp': '0',
-            '3gphd': '1',
-            'flv': '0',
-            'flvhd': '0',
-            'mp4': '1',
-            'mp4hd': '1',
-            'mp4hd2': '1',
-            'mp4hd3': '1',
-            'hd2': '2',
-            'hd3': '3',
-        }
-        return hd_id_dict[fm]
-
-    def parse_ext_l(self, fm):
-        ext_dict = {
-            '3gp': 'flv',
-            '3gphd': 'mp4',
-            'flv': 'flv',
-            'flvhd': 'flv',
-            'mp4': 'mp4',
-            'mp4hd': 'mp4',
-            'mp4hd2': 'flv',
-            'mp4hd3': 'flv',
-            'hd2': 'flv',
-            'hd3': 'flv',
-        }
-        return ext_dict[fm]
-
     def get_format_name(self, fm):
         _dict = {
             '3gp': 'h6',
@@ -210,32 +88,40 @@ class YoukuIE(InfoExtractor):
             'hd2': 'h2',
             'hd3': 'h1',
         }
-        return _dict[fm]
+        return _dict.get(fm)
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         self._set_cookie('youku.com', '__ysuid', self.get_ysuid())
+        self._set_cookie('youku.com', 'xreferrer', 'http://www.youku.com')
 
-        def retrieve_data(req_url, note):
-            headers = {
-                'Referer': req_url,
-            }
-            headers.update(self.geo_verification_headers())
-            self._set_cookie('youku.com', 'xreferrer', 'http://www.youku.com')
-
-            raw_data = self._download_json(req_url, video_id, note=note, headers=headers)
-
-            return raw_data['data']
-
-        video_password = self._downloader.params.get('videopassword')
+        _, urlh = self._download_webpage_handle(
+            'https://log.mmstat.com/eg.js', video_id, 'Retrieving cna info')
+        # The etag header is '"foobar"'; let's remove the double quotes
+        cna = urlh.headers['etag'][1:-1]
 
         # request basic data
-        basic_data_url = 'http://play.youku.com/play/get.json?vid=%s&ct=12' % video_id
-        if video_password:
-            basic_data_url += '&pwd=%s' % video_password
+        basic_data_params = {
+            'vid': video_id,
+            'ccode': '0401',
+            'client_ip': '192.168.1.1',
+            'utid': cna,
+            'client_ts': time.time() / 1000,
+        }
 
-        data = retrieve_data(basic_data_url, 'Downloading JSON metadata')
+        video_password = self._downloader.params.get('videopassword')
+        if video_password:
+            basic_data_params['password'] = video_password
+
+        headers = {
+            'Referer': url,
+        }
+        headers.update(self.geo_verification_headers())
+        data = self._download_json(
+            'https://ups.youku.com/ups/get.json', video_id,
+            'Downloading JSON metadata',
+            query=basic_data_params, headers=headers)['data']
 
         error = data.get('error')
         if error:
@@ -255,37 +141,21 @@ class YoukuIE(InfoExtractor):
         # get video title
         title = data['video']['title']
 
-        # generate video_urls_dict
-        video_urls_dict = self.construct_video_urls(data)
-
-        # construct info
-        entries = [{
-            'id': '%s_part%d' % (video_id, i + 1),
-            'title': title,
-            'formats': [],
-            # some formats are not available for all parts, we have to detect
-            # which one has all
-        } for i in range(max(len(v.get('segs')) for v in data['stream']))]
-        for stream in data['stream']:
-            if stream.get('channel_type') == 'tail':
-                continue
-            fm = stream.get('stream_type')
-            video_urls = video_urls_dict[fm]
-            for video_url, seg, entry in zip(video_urls, stream['segs'], entries):
-                entry['formats'].append({
-                    'url': video_url,
-                    'format_id': self.get_format_name(fm),
-                    'ext': self.parse_ext_l(fm),
-                    'filesize': int(seg['size']),
-                    'width': stream.get('width'),
-                    'height': stream.get('height'),
-                })
+        formats = [{
+            'url': stream['m3u8_url'],
+            'format_id': self.get_format_name(stream.get('stream_type')),
+            'ext': 'mp4',
+            'protocol': 'm3u8_native',
+            'filesize': int(stream.get('size')),
+            'width': stream.get('width'),
+            'height': stream.get('height'),
+        } for stream in data['stream'] if stream.get('channel_type') != 'tail']
+        self._sort_formats(formats)
 
         return {
-            '_type': 'multi_video',
             'id': video_id,
             'title': title,
-            'entries': entries,
+            'formats': formats,
         }
 
 

From 5552c9eb0fece567f7dda13810939fca32d7d65a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 May 2017 21:58:18 +0800
Subject: [PATCH 345/611] [utils] Recognize more patterns in strip_jsonp()

Used in Youku Show pages
---
 ChangeLog           | 1 +
 test/test_utils.py  | 8 ++++++++
 youtube_dl/utils.py | 7 ++++++-
 3 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 6a05657ab..d6e980c5a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Core
++ [utils] strip_jsonp() can recognize more patterns
 * [postprocessor/ffmpeg] Fix metadata filename handling on Python 2 (#13182)
 
 Extractors
diff --git a/test/test_utils.py b/test/test_utils.py
index f31559e71..d7e05817c 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -678,6 +678,14 @@ class TestUtil(unittest.TestCase):
         d = json.loads(stripped)
         self.assertEqual(d, {'status': 'success'})
 
+        stripped = strip_jsonp('window.cb && window.cb({"status": "success"});')
+        d = json.loads(stripped)
+        self.assertEqual(d, {'status': 'success'})
+
+        stripped = strip_jsonp('window.cb && cb({"status": "success"});')
+        d = json.loads(stripped)
+        self.assertEqual(d, {'status': 'success'})
+
     def test_uppercase_escape(self):
         self.assertEqual(uppercase_escape('aä'), 'aä')
         self.assertEqual(uppercase_escape('\\U0001d550'), '𝕐')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4293a77f5..6c84bfe0f 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2211,7 +2211,12 @@ def parse_age_limit(s):
 
 def strip_jsonp(code):
     return re.sub(
-        r'(?s)^[a-zA-Z0-9_.$]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
+        r'''(?sx)^
+            (?:window\.)?(?P<func_name>[a-zA-Z0-9_.$]+)
+            (?:\s*&&\s*(?P=func_name))?
+            \s*\(\s*(?P<callback_data>.*)\);?
+            \s*?(?://[^\n]*)*$''',
+        r'\g<callback_data>', code)
 
 
 def js_to_json(code):

From d3d4ba7f241bb3c8d6478a36917ff6c81e97a85d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 May 2017 21:59:16 +0800
Subject: [PATCH 346/611] [youku:show] Fix extraction

---
 ChangeLog                     |  1 +
 youtube_dl/extractor/youku.py | 67 +++++++++++++++++++++--------------
 2 files changed, 41 insertions(+), 27 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index d6e980c5a..2ac471f8e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -5,6 +5,7 @@ Core
 * [postprocessor/ffmpeg] Fix metadata filename handling on Python 2 (#13182)
 
 Extractors
+* [youku:show] Fix extraction
 * [youku] Fix extraction (#13191)
 
 
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 37be440fa..e081cfc19 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -10,7 +10,10 @@ import time
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    get_element_by_attribute,
+    get_element_by_class,
+    js_to_json,
+    strip_jsonp,
+    urljoin,
 )
 
 
@@ -160,49 +163,59 @@ class YoukuIE(InfoExtractor):
 
 
 class YoukuShowIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?youku\.com/show_page/id_(?P<id>[0-9a-z]+)\.html'
+    _VALID_URL = r'https?://list\.youku\.com/show/id_(?P<id>[0-9a-z]+)\.html'
     IE_NAME = 'youku:show'
 
     _TEST = {
-        'url': 'http://www.youku.com/show_page/id_zc7c670be07ff11e48b3f.html',
+        'url': 'http://list.youku.com/show/id_zc7c670be07ff11e48b3f.html',
         'info_dict': {
             'id': 'zc7c670be07ff11e48b3f',
             'title': '花千骨 未删减版',
-            'description': 'md5:578d4f2145ae3f9128d9d4d863312910',
+            'description': 'md5:a1ae6f5618571bbeb5c9821f9c81b558',
         },
         'playlist_count': 50,
     }
 
     _PAGE_SIZE = 40
 
-    def _find_videos_in_page(self, webpage):
-        videos = re.findall(
-            r'<li><a[^>]+href="(?P<url>https?://v\.youku\.com/[^"]+)"[^>]+title="(?P<title>[^"]+)"', webpage)
-        return [
-            self.url_result(video_url, YoukuIE.ie_key(), title)
-            for video_url, title in videos]
-
     def _real_extract(self, url):
         show_id = self._match_id(url)
         webpage = self._download_webpage(url, show_id)
 
-        entries = self._find_videos_in_page(webpage)
-
-        playlist_title = self._html_search_regex(
-            r'<span[^>]+class="name">([^<]+)</span>', webpage, 'playlist title', fatal=False)
-        detail_div = get_element_by_attribute('class', 'detail', webpage) or ''
-        playlist_description = self._html_search_regex(
-            r'<span[^>]+style="display:none"[^>]*>([^<]+)</span>',
-            detail_div, 'playlist description', fatal=False)
-
-        for idx in itertools.count(1):
-            episodes_page = self._download_webpage(
-                'http://www.youku.com/show_episode/id_%s.html' % show_id,
-                show_id, query={'divid': 'reload_%d' % (idx * self._PAGE_SIZE + 1)},
-                note='Downloading episodes page %d' % idx)
-            new_entries = self._find_videos_in_page(episodes_page)
+        entries = []
+        page_config = self._parse_json(self._search_regex(
+            r'var\s+PageConfig\s*=\s*({.+});', webpage, 'page config'),
+            show_id, transform_source=js_to_json)
+        for idx in itertools.count(0):
+            if idx == 0:
+                playlist_data_url = 'http://list.youku.com/show/module'
+                query = {'id': page_config['showid'], 'tab': 'point'}
+            else:
+                playlist_data_url = 'http://list.youku.com/show/point'
+                query = {
+                    'id': page_config['showid'],
+                    'stage': 'reload_%d' % (self._PAGE_SIZE * idx + 1),
+                }
+            query['callback'] = 'cb'
+            playlist_data = self._download_json(
+                playlist_data_url, show_id, query=query,
+                note='Downloading playlist data page %d' % (idx + 1),
+                transform_source=lambda s: js_to_json(strip_jsonp(s)))['html']
+            video_urls = re.findall(
+                r'<div[^>]+class="p-thumb"[^<]+<a[^>]+href="([^"]+)"',
+                playlist_data)
+            new_entries = [
+                self.url_result(urljoin(url, video_url), YoukuIE.ie_key())
+                for video_url in video_urls]
             entries.extend(new_entries)
             if len(new_entries) < self._PAGE_SIZE:
                 break
 
-        return self.playlist_result(entries, show_id, playlist_title, playlist_description)
+        desc = self._html_search_meta('description', webpage, fatal=False)
+        playlist_title = desc.split(',')[0] if desc else None
+        detail_li = get_element_by_class('p-intro', webpage)
+        playlist_description = get_element_by_class(
+            'intro-more', detail_li) if detail_li else None
+
+        return self.playlist_result(
+            entries, show_id, playlist_title, playlist_description)

From c130f0a37b5c31fa559373ec319f79830c591c22 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 May 2017 23:04:42 +0800
Subject: [PATCH 347/611] [tudou] Merge into youku extractor (fixes #12214)

Also, there are no tudou playlists anymore. All playlist URLs points to youku
playlists.
---
 ChangeLog                          |   1 +
 youtube_dl/extractor/extractors.py |   5 --
 youtube_dl/extractor/tudou.py      | 132 -----------------------------
 youtube_dl/extractor/youku.py      |  16 +++-
 4 files changed, 15 insertions(+), 139 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 2ac471f8e..936ba512c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -5,6 +5,7 @@ Core
 * [postprocessor/ffmpeg] Fix metadata filename handling on Python 2 (#13182)
 
 Extractors
+* [tudou] Merge into youku extractor (#12214)
 * [youku:show] Fix extraction
 * [youku] Fix extraction (#13191)
 
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index ed603eb29..558c910d4 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1019,11 +1019,6 @@ from .trilulilu import TriluliluIE
 from .trutv import TruTVIE
 from .tube8 import Tube8IE
 from .tubitv import TubiTvIE
-from .tudou import (
-    TudouIE,
-    TudouPlaylistIE,
-    TudouAlbumIE,
-)
 from .tumblr import TumblrIE
 from .tunein import (
     TuneInClipIE,
diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index 2aae55e7e..7421378a8 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -3,138 +3,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_str
-from ..utils import (
-    ExtractorError,
-    int_or_none,
-    InAdvancePagedList,
-    float_or_none,
-    unescapeHTML,
-)
-
-
-class TudouIE(InfoExtractor):
-    IE_NAME = 'tudou'
-    _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:(?:programs|wlplay)/view|(?:listplay|albumplay)/[\w-]{11})/(?P<id>[\w-]{11})'
-    _TESTS = [{
-        'url': 'http://www.tudou.com/listplay/zzdE77v6Mmo/2xN2duXMxmw.html',
-        'md5': '140a49ed444bd22f93330985d8475fcb',
-        'info_dict': {
-            'id': '159448201',
-            'ext': 'f4v',
-            'title': '卡马乔国足开大脚长传冲吊集锦',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'timestamp': 1372113489000,
-            'description': '卡马乔卡家军，开大脚先进战术不完全集锦！',
-            'duration': 289.04,
-            'view_count': int,
-            'filesize': int,
-        }
-    }, {
-        'url': 'http://www.tudou.com/programs/view/ajX3gyhL0pc/',
-        'info_dict': {
-            'id': '117049447',
-            'ext': 'f4v',
-            'title': 'La Sylphide-Bolshoi-Ekaterina Krysanova & Vyacheslav Lopatin 2012',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'timestamp': 1349207518000,
-            'description': 'md5:294612423894260f2dcd5c6c04fe248b',
-            'duration': 5478.33,
-            'view_count': int,
-            'filesize': int,
-        }
-    }]
-
-    _PLAYER_URL = 'http://js.tudouui.com/bin/lingtong/PortalPlayer_177.swf'
-
-    # Translated from tudou/tools/TVCHelper.as in PortalPlayer_193.swf
-    # 0001, 0002 and 4001 are not included as they indicate temporary issues
-    TVC_ERRORS = {
-        '0003': 'The video is deleted or does not exist',
-        '1001': 'This video is unavailable due to licensing issues',
-        '1002': 'This video is unavailable as it\'s under review',
-        '1003': 'This video is unavailable as it\'s under review',
-        '3001': 'Password required',
-        '5001': 'This video is available in Mainland China only due to licensing issues',
-        '7001': 'This video is unavailable',
-        '8001': 'This video is unavailable due to licensing issues',
-    }
-
-    def _url_for_id(self, video_id, quality=None):
-        info_url = 'http://v2.tudou.com/f?id=' + compat_str(video_id)
-        if quality:
-            info_url += '&hd' + quality
-        xml_data = self._download_xml(info_url, video_id, 'Opening the info XML page')
-        error = xml_data.attrib.get('error')
-        if error is not None:
-            raise ExtractorError('Tudou said: %s' % error, expected=True)
-        final_url = xml_data.text
-        return final_url
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        item_data = self._download_json(
-            'http://www.tudou.com/tvp/getItemInfo.action?ic=%s' % video_id, video_id)
-
-        youku_vcode = item_data.get('vcode')
-        if youku_vcode:
-            return self.url_result('youku:' + youku_vcode, ie='Youku')
-
-        if not item_data.get('itemSegs'):
-            tvc_code = item_data.get('tvcCode')
-            if tvc_code:
-                err_msg = self.TVC_ERRORS.get(tvc_code)
-                if err_msg:
-                    raise ExtractorError('Tudou said: %s' % err_msg, expected=True)
-                raise ExtractorError('Unexpected error %s returned from Tudou' % tvc_code)
-            raise ExtractorError('Unxpected error returned from Tudou')
-
-        title = unescapeHTML(item_data['kw'])
-        description = item_data.get('desc')
-        thumbnail_url = item_data.get('pic')
-        view_count = int_or_none(item_data.get('playTimes'))
-        timestamp = int_or_none(item_data.get('pt'))
-
-        segments = self._parse_json(item_data['itemSegs'], video_id)
-        # It looks like the keys are the arguments that have to be passed as
-        # the hd field in the request url, we pick the higher
-        # Also, filter non-number qualities (see issue #3643).
-        quality = sorted(filter(lambda k: k.isdigit(), segments.keys()),
-                         key=lambda k: int(k))[-1]
-        parts = segments[quality]
-        len_parts = len(parts)
-        if len_parts > 1:
-            self.to_screen('%s: found %s parts' % (video_id, len_parts))
-
-        def part_func(partnum):
-            part = parts[partnum]
-            part_id = part['k']
-            final_url = self._url_for_id(part_id, quality)
-            ext = (final_url.split('?')[0]).split('.')[-1]
-            return [{
-                'id': '%s' % part_id,
-                'url': final_url,
-                'ext': ext,
-                'title': title,
-                'thumbnail': thumbnail_url,
-                'description': description,
-                'view_count': view_count,
-                'timestamp': timestamp,
-                'duration': float_or_none(part.get('seconds'), 1000),
-                'filesize': int_or_none(part.get('size')),
-                'http_headers': {
-                    'Referer': self._PLAYER_URL,
-                },
-            }]
-
-        entries = InAdvancePagedList(part_func, len_parts, 1)
-
-        return {
-            '_type': 'multi_video',
-            'entries': entries,
-            'id': video_id,
-            'title': title,
-        }
 
 
 class TudouPlaylistIE(InfoExtractor):
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index e081cfc19..ab97466d2 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -22,7 +22,9 @@ class YoukuIE(InfoExtractor):
     IE_DESC = '优酷'
     _VALID_URL = r'''(?x)
         (?:
-            http://(?:v|player)\.youku\.com/(?:v_show/id_|player\.php/sid/)|
+            https?://(
+                (?:v|player)\.youku\.com/(?:v_show/id_|player\.php/sid/)|
+                video\.tudou\.com/v/)|
             youku:)
         (?P<id>[A-Za-z0-9]+)(?:\.html|/v\.swf|)
     '''
@@ -71,6 +73,16 @@ class YoukuIE(InfoExtractor):
             'ext': 'mp4',
             'title': '我的世界☆明月庄主☆车震猎杀☆杀人艺术Minecraft',
         },
+    }, {
+        'url': 'http://video.tudou.com/v/XMjIyNzAzMTQ4NA==.html?f=46177805',
+        'info_dict': {
+            'id': 'XMjIyNzAzMTQ4NA',
+            'ext': 'mp4',
+            'title': '卡马乔国足开大脚长传冲吊集锦',
+        },
+    }, {
+        'url': 'http://video.tudou.com/v/XMjE4ODI3OTg2MA==.html',
+        'only_matching': True,
     }]
 
     @staticmethod
@@ -107,7 +119,7 @@ class YoukuIE(InfoExtractor):
         # request basic data
         basic_data_params = {
             'vid': video_id,
-            'ccode': '0401',
+            'ccode': '0402' if 'tudou.com' in url else '0401',
             'client_ip': '192.168.1.1',
             'utid': cna,
             'client_ts': time.time() / 1000,

From 97067db2ae32b18ef573af99b91d36026b4b1d05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 May 2017 22:12:24 +0700
Subject: [PATCH 348/611] [bbc] Add support for authentication

---
 youtube_dl/extractor/bbc.py | 40 +++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index dd65b8d86..7cd4446b4 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -6,14 +6,18 @@ import itertools
 
 from .common import InfoExtractor
 from ..utils import (
+    clean_html,
     dict_get,
     ExtractorError,
     float_or_none,
+    get_element_by_class,
     int_or_none,
     parse_duration,
     parse_iso8601,
     try_get,
     unescapeHTML,
+    urlencode_postdata,
+    urljoin,
 )
 from ..compat import (
     compat_etree_fromstring,
@@ -38,6 +42,9 @@ class BBCCoUkIE(InfoExtractor):
                         (?P<id>%s)(?!/(?:episodes|broadcasts|clips))
                     ''' % _ID_REGEX
 
+    _LOGIN_URL = 'https://account.bbc.com/signin'
+    _NETRC_MACHINE = 'bbc'
+
     _MEDIASELECTOR_URLS = [
         # Provides HQ HLS streams with even better quality that pc mediaset but fails
         # with geolocation in some cases when it's even not geo restricted at all (e.g.
@@ -227,6 +234,39 @@ class BBCCoUkIE(InfoExtractor):
 
     _USP_RE = r'/([^/]+?)\.ism(?:\.hlsv2\.ism)?/[^/]+\.m3u8'
 
+    def _login(self):
+        username, password = self._get_login_info()
+        if username is None:
+            return
+
+        login_page = self._download_webpage(
+            self._LOGIN_URL, None, 'Downloading signin page')
+
+        login_form = self._hidden_inputs(login_page)
+
+        login_form.update({
+            'username': username,
+            'password': password,
+        })
+
+        post_url = urljoin(self._LOGIN_URL, self._search_regex(
+            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
+            'post url', default=self._LOGIN_URL, group='url'))
+
+        response, urlh = self._download_webpage_handle(
+            post_url, None, 'Logging in', data=urlencode_postdata(login_form),
+            headers={'Referer': self._LOGIN_URL})
+
+        if self._LOGIN_URL in urlh.geturl():
+            error = clean_html(get_element_by_class('form-message', response))
+            if error:
+                raise ExtractorError(
+                    'Unable to login: %s' % error, expected=True)
+            raise ExtractorError('Unable to log in')
+
+    def _real_initialize(self):
+        self._login()
+
     class MediaSelectionError(Exception):
         def __init__(self, id):
             self.id = id

From fd545fc6d1a5b489830e80c421f7e09264e0c80f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 May 2017 12:43:14 +0800
Subject: [PATCH 349/611] Revert "[youtube] Don't use the DASH manifest from
 'get_video_info' if 'use_cipher_signature' is True (#5118)"

This reverts commit 87dc451108b278eb5e5600a0159bdbd337272392.
---
 youtube_dl/extractor/youtube.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index d66693c0c..d6c098706 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1481,8 +1481,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     if not video_info_webpage:
                         continue
                     get_video_info = compat_parse_qs(video_info_webpage)
-                    if get_video_info.get('use_cipher_signature') != ['True']:
-                        add_dash_mpd(get_video_info)
+                    add_dash_mpd(get_video_info)
                     if not video_info:
                         video_info = get_video_info
                     if 'token' in get_video_info:

From 98a061894176aa791e645dda0285c04da41241f1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 May 2017 23:21:32 +0800
Subject: [PATCH 350/611] [ChangeLog] Update after the fix for #11381

---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index 936ba512c..a622f704d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -5,6 +5,7 @@ Core
 * [postprocessor/ffmpeg] Fix metadata filename handling on Python 2 (#13182)
 
 Extractors
++ [youtube] DASH MPDs with cipher signatures are recognized now (#11381)
 * [tudou] Merge into youku extractor (#12214)
 * [youku:show] Fix extraction
 * [youku] Fix extraction (#13191)

From cc304ce588ca95ddeca67d376e6656eb2bb2b988 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 May 2017 22:27:56 +0700
Subject: [PATCH 351/611] [ChangeLog] Actualize

---
 ChangeLog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index a622f704d..c38f414f4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -6,9 +6,13 @@ Core
 
 Extractors
 + [youtube] DASH MPDs with cipher signatures are recognized now (#11381)
++ [bbc] Add support for authentication
 * [tudou] Merge into youku extractor (#12214)
 * [youku:show] Fix extraction
 * [youku] Fix extraction (#13191)
+* [udemy] Fix extraction for outputs' format entries without URL (#13192)
+* [vimeo] Fix formats' sorting (#13189)
+* [cbsnews] Fix extraction for 60 Minutes videos (#12861)
 
 
 version 2017.05.23

From 28dbde9cc3e9921737b7677409fef073ea207363 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 May 2017 22:29:16 +0700
Subject: [PATCH 352/611] release 2017.05.26

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 ---
 youtube_dl/version.py     | 2 +-
 4 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 730a07ed1..5ca69ffa1 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.23*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.23**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.26*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.26**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.05.23
+[debug] youtube-dl version 2017.05.26
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index c38f414f4..ff6bccf61 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.05.26
 
 Core
 + [utils] strip_jsonp() can recognize more patterns
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 5545cff23..9791d585b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -810,9 +810,6 @@
  - **TruTV**
  - **Tube8**
  - **TubiTv**
- - **tudou**
- - **tudou:album**
- - **tudou:playlist**
  - **Tumblr**
  - **tunein:clip**
  - **tunein:program**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index c006eba76..16d91dfad 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.05.23'
+__version__ = '2017.05.26'

From 7f4c3a7439a66e522a4cd5d7ef3f0fb19bf155af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 May 2017 22:42:27 +0700
Subject: [PATCH 353/611] [cbsnews] Fix extraction (closes #13205)

---
 youtube_dl/extractor/cbsnews.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 651ae2665..5b61c3f52 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -61,11 +61,17 @@ class CBSNewsIE(CBSIE):
 
         video_info = self._parse_json(self._html_search_regex(
             r'(?:<ul class="media-list items" id="media-related-items"><li data-video-info|<div id="cbsNewsVideoPlayer" data-video-player-options)=\'({.+?})\'',
-            webpage, 'video JSON info'), video_id)
+            webpage, 'video JSON info', default='{}'), video_id, fatal=False)
 
-        item = video_info['item'] if 'item' in video_info else video_info
-        guid = item['mpxRefId']
-        return self._extract_video_info(guid, 'cbsnews')
+        if video_info:
+            item = video_info['item'] if 'item' in video_info else video_info
+        else:
+            state = self._parse_json(self._search_regex(
+                r'data-cbsvideoui-options=(["\'])(?P<json>{.+?})\1', webpage,
+                'playlist JSON info', group='json'), video_id)['state']
+            item = state['playlist'][state['pid']]
+
+        return self._extract_video_info(item['mpxRefId'], 'cbsnews')
 
 
 class CBSNewsLiveVideoIE(InfoExtractor):

From d16c0121b91f8f2b1b7131f3dcad06902d39613c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 27 May 2017 00:06:45 +0800
Subject: [PATCH 354/611] [youku] Extract more metadata (closes #10433)

---
 ChangeLog                     |  6 +++++
 youtube_dl/extractor/youku.py | 46 ++++++++++++++++++++++++++++++++++-
 2 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index ff6bccf61..65a3ca468 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
++ [youku] Extract more metadata (#10433)
+
+
 version 2017.05.26
 
 Core
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index ab97466d2..dcce15d77 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -12,6 +12,7 @@ from ..utils import (
     ExtractorError,
     get_element_by_class,
     js_to_json,
+    str_or_none,
     strip_jsonp,
     urljoin,
 )
@@ -36,6 +37,12 @@ class YoukuIE(InfoExtractor):
             'id': 'XMTc1ODE5Njcy',
             'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',
             'ext': 'mp4',
+            'duration': 74.73,
+            'thumbnail': r're:^https?://.*',
+            'uploader': '。躲猫猫、',
+            'uploader_id': '36017967',
+            'uploader_url': 'http://i.youku.com/u/UMTQ0MDcxODY4',
+            'tags': list,
         }
     }, {
         'url': 'http://player.youku.com/player.php/sid/XNDgyMDQ2NTQw/v.swf',
@@ -46,6 +53,12 @@ class YoukuIE(InfoExtractor):
             'id': 'XODgxNjg1Mzk2',
             'ext': 'mp4',
             'title': '武媚娘传奇 85',
+            'duration': 1999.61,
+            'thumbnail': r're:^https?://.*',
+            'uploader': '疯狂豆花',
+            'uploader_id': '62583473',
+            'uploader_url': 'http://i.youku.com/u/UMjUwMzMzODky',
+            'tags': list,
         },
     }, {
         'url': 'http://v.youku.com/v_show/id_XMTI1OTczNDM5Mg==.html',
@@ -53,6 +66,12 @@ class YoukuIE(InfoExtractor):
             'id': 'XMTI1OTczNDM5Mg',
             'ext': 'mp4',
             'title': '花千骨 04',
+            'duration': 2363,
+            'thumbnail': r're:^https?://.*',
+            'uploader': '放剧场-花千骨',
+            'uploader_id': '772849359',
+            'uploader_url': 'http://i.youku.com/u/UMzA5MTM5NzQzNg==',
+            'tags': list,
         },
     }, {
         'url': 'http://v.youku.com/v_show/id_XNjA1NzA2Njgw.html',
@@ -61,6 +80,12 @@ class YoukuIE(InfoExtractor):
             'id': 'XNjA1NzA2Njgw',
             'ext': 'mp4',
             'title': '邢義田复旦讲座之想象中的胡人—从“左衽孔子”说起',
+            'duration': 7264.5,
+            'thumbnail': r're:^https?://.*',
+            'uploader': 'FoxJin1006',
+            'uploader_id': '322014285',
+            'uploader_url': 'http://i.youku.com/u/UMTI4ODA1NzE0MA==',
+            'tags': list,
         },
         'params': {
             'videopassword': '100600',
@@ -72,6 +97,12 @@ class YoukuIE(InfoExtractor):
             'id': 'XOTUxMzg4NDMy',
             'ext': 'mp4',
             'title': '我的世界☆明月庄主☆车震猎杀☆杀人艺术Minecraft',
+            'duration': 702.08,
+            'thumbnail': r're:^https?://.*',
+            'uploader': '明月庄主moon',
+            'uploader_id': '38465621',
+            'uploader_url': 'http://i.youku.com/u/UMTUzODYyNDg0',
+            'tags': list,
         },
     }, {
         'url': 'http://video.tudou.com/v/XMjIyNzAzMTQ4NA==.html?f=46177805',
@@ -79,6 +110,12 @@ class YoukuIE(InfoExtractor):
             'id': 'XMjIyNzAzMTQ4NA',
             'ext': 'mp4',
             'title': '卡马乔国足开大脚长传冲吊集锦',
+            'duration': 289,
+            'thumbnail': r're:^https?://.*',
+            'uploader': '阿卜杜拉之星',
+            'uploader_id': '2382249',
+            'uploader_url': 'http://i.youku.com/u/UOTUyODk5Ng==',
+            'tags': list,
         },
     }, {
         'url': 'http://video.tudou.com/v/XMjE4ODI3OTg2MA==.html',
@@ -154,7 +191,8 @@ class YoukuIE(InfoExtractor):
                 raise ExtractorError(msg)
 
         # get video title
-        title = data['video']['title']
+        video_data = data['video']
+        title = video_data['title']
 
         formats = [{
             'url': stream['m3u8_url'],
@@ -171,6 +209,12 @@ class YoukuIE(InfoExtractor):
             'id': video_id,
             'title': title,
             'formats': formats,
+            'duration': video_data.get('seconds'),
+            'thumbnail': video_data.get('logo'),
+            'uploader': video_data.get('username'),
+            'uploader_id': str_or_none(video_data.get('userid')),
+            'uploader_url': data.get('uploader', {}).get('homepage'),
+            'tags': video_data.get('tags'),
         }
 
 

From 20e2c9de0414f18dff26020045445cf91cad6b70 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 26 May 2017 20:00:44 +0100
Subject: [PATCH 355/611] [adn] fix formats extraction

---
 youtube_dl/extractor/adn.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/adn.py b/youtube_dl/extractor/adn.py
index 66caf6a81..39f80b23f 100644
--- a/youtube_dl/extractor/adn.py
+++ b/youtube_dl/extractor/adn.py
@@ -15,6 +15,7 @@ from ..utils import (
     intlist_to_bytes,
     srt_subtitles_timecode,
     strip_or_none,
+    urljoin,
 )
 
 
@@ -31,25 +32,28 @@ class ADNIE(InfoExtractor):
             'description': 'md5:2f7b5aa76edbc1a7a92cedcda8a528d5',
         }
     }
+    _BASE_URL = 'http://animedigitalnetwork.fr'
 
     def _get_subtitles(self, sub_path, video_id):
         if not sub_path:
             return None
 
         enc_subtitles = self._download_webpage(
-            'http://animedigitalnetwork.fr/' + sub_path,
-            video_id, fatal=False)
+            urljoin(self._BASE_URL, sub_path),
+            video_id, fatal=False, headers={
+                'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:53.0) Gecko/20100101 Firefox/53.0',
+            })
         if not enc_subtitles:
             return None
 
         # http://animedigitalnetwork.fr/components/com_vodvideo/videojs/adn-vjs.min.js
         dec_subtitles = intlist_to_bytes(aes_cbc_decrypt(
             bytes_to_intlist(base64.b64decode(enc_subtitles[24:])),
-            bytes_to_intlist(b'\nd\xaf\xd2J\xd0\xfc\xe1\xfc\xdf\xb61\xe8\xe1\xf0\xcc'),
+            bytes_to_intlist(b'\x1b\xe0\x29\x61\x38\x94\x24\x00\x12\xbd\xc5\x80\xac\xce\xbe\xb0'),
             bytes_to_intlist(base64.b64decode(enc_subtitles[:24]))
         ))
         subtitles_json = self._parse_json(
-            dec_subtitles[:-compat_ord(dec_subtitles[-1])],
+            dec_subtitles[:-compat_ord(dec_subtitles[-1])].decode(),
             None, fatal=False)
         if not subtitles_json:
             return None
@@ -103,9 +107,16 @@ class ADNIE(InfoExtractor):
         metas = options.get('metas') or {}
         title = metas.get('title') or video_info['title']
         links = player_config.get('links') or {}
+        if not links:
+            links_url = player_config['linksurl']
+            links_data = self._download_json(urljoin(
+                self._BASE_URL, links_url), video_id)
+            links = links_data.get('links') or {}
 
         formats = []
         for format_id, qualities in links.items():
+            if not isinstance(qualities, dict):
+                continue
             for load_balancer_url in qualities.values():
                 load_balancer_data = self._download_json(
                     load_balancer_url, video_id, fatal=False) or {}

From b407d8533d3956a7c27ad42cbde9a877c36df72c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 27 May 2017 23:05:02 +0800
Subject: [PATCH 356/611] [utils] Drop an compatibility wrapper for Python <
 2.6

addinfourl.getcode is added since Python 2.6a1. As youtube-dl now
requires 2.6+, this is no longer necessary.

See https://github.com/python/cpython/commit/9b0d46db115c10767b240a0a64286214b50fe6ad
---
 youtube_dl/utils.py | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 6c84bfe0f..d79410e58 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -932,14 +932,6 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
         except zlib.error:
             return zlib.decompress(data)
 
-    @staticmethod
-    def addinfourl_wrapper(stream, headers, url, code):
-        if hasattr(compat_urllib_request.addinfourl, 'getcode'):
-            return compat_urllib_request.addinfourl(stream, headers, url, code)
-        ret = compat_urllib_request.addinfourl(stream, headers, url)
-        ret.code = code
-        return ret
-
     def http_request(self, req):
         # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
         # always respected by websites, some tend to give out URLs with non percent-encoded
@@ -991,13 +983,13 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
                     break
                 else:
                     raise original_ioerror
-            resp = self.addinfourl_wrapper(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
+            resp = compat_urllib_request.addinfourl(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
             resp.msg = old_resp.msg
             del resp.headers['Content-encoding']
         # deflate
         if resp.headers.get('Content-encoding', '') == 'deflate':
             gz = io.BytesIO(self.deflate(resp.read()))
-            resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
+            resp = compat_urllib_request.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
             resp.msg = old_resp.msg
             del resp.headers['Content-encoding']
         # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see

From 03327bc9a62bba1e6181e22e0ada82fb2f087fb0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 May 2017 22:36:58 +0700
Subject: [PATCH 357/611] [cbsinteractive] Relax _VALID_URL (closes #13213)

---
 youtube_dl/extractor/cbsinteractive.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cbsinteractive.py b/youtube_dl/extractor/cbsinteractive.py
index 86f7eee20..681d63e29 100644
--- a/youtube_dl/extractor/cbsinteractive.py
+++ b/youtube_dl/extractor/cbsinteractive.py
@@ -8,7 +8,7 @@ from ..utils import int_or_none
 
 
 class CBSInteractiveIE(CBSIE):
-    _VALID_URL = r'https?://(?:www\.)?(?P<site>cnet|zdnet)\.com/(?:videos|video/share)/(?P<id>[^/?]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<site>cnet|zdnet)\.com/(?:videos|video(?:/share)?)/(?P<id>[^/?]+)'
     _TESTS = [{
         'url': 'http://www.cnet.com/videos/hands-on-with-microsofts-windows-8-1-update/',
         'info_dict': {
@@ -60,6 +60,9 @@ class CBSInteractiveIE(CBSIE):
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.zdnet.com/video/huawei-matebook-x-video/',
+        'only_matching': True,
     }]
 
     MPX_ACCOUNTS = {

From 1e0d65f0bded292440d55ca37449038c8b5d9344 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Sun, 14 May 2017 10:04:42 +0200
Subject: [PATCH 358/611] [beam:vod] Add extractor

---
 youtube_dl/extractor/beampro.py    | 121 ++++++++++++++++++++++++++---
 youtube_dl/extractor/extractors.py |   5 +-
 2 files changed, 116 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/beampro.py b/youtube_dl/extractor/beampro.py
index f3a9e3278..79c5fe5cd 100644
--- a/youtube_dl/extractor/beampro.py
+++ b/youtube_dl/extractor/beampro.py
@@ -6,16 +6,30 @@ from ..utils import (
     ExtractorError,
     clean_html,
     compat_str,
+    float_or_none,
     int_or_none,
     parse_iso8601,
     try_get,
+    urljoin,
 )
 
 
-class BeamProLiveIE(InfoExtractor):
+class BeamProBaseIE(InfoExtractor):
+    _RATINGS = {'family': 0, 'teen': 13, '18+': 18}
+
+    def _extract_channel_info(self, chan):
+        user_id = chan.get('userId') or try_get(chan, lambda x: x['user']['id'])
+        return {
+            'uploader': chan.get('token') or try_get(
+                chan, lambda x: x['user']['username'], compat_str),
+            'uploader_id': compat_str(user_id) if user_id else None,
+            'age_limit': self._RATINGS.get(chan.get('audience')),
+        }
+
+
+class BeamProLiveIE(BeamProBaseIE):
     IE_NAME = 'Beam:live'
     _VALID_URL = r'https?://(?:\w+\.)?beam\.pro/(?P<id>[^/?#&]+)'
-    _RATINGS = {'family': 0, 'teen': 13, '18+': 18}
     _TEST = {
         'url': 'http://www.beam.pro/niterhayven',
         'info_dict': {
@@ -38,6 +52,10 @@ class BeamProLiveIE(InfoExtractor):
         },
     }
 
+    @classmethod
+    def suitable(cls, url):
+        return False if BeamProVodIE.suitable(url) else super(BeamProLiveIE, cls).suitable(url)
+
     def _real_extract(self, url):
         channel_name = self._match_id(url)
 
@@ -55,19 +73,104 @@ class BeamProLiveIE(InfoExtractor):
             channel_name, ext='mp4', m3u8_id='hls', fatal=False)
         self._sort_formats(formats)
 
-        user_id = chan.get('userId') or try_get(chan, lambda x: x['user']['id'])
-
-        return {
+        info = {
             'id': compat_str(chan.get('id') or channel_name),
             'title': self._live_title(chan.get('name') or channel_name),
             'description': clean_html(chan.get('description')),
             'thumbnail': try_get(chan, lambda x: x['thumbnail']['url'], compat_str),
             'timestamp': parse_iso8601(chan.get('updatedAt')),
-            'uploader': chan.get('token') or try_get(
-                chan, lambda x: x['user']['username'], compat_str),
-            'uploader_id': compat_str(user_id) if user_id else None,
-            'age_limit': self._RATINGS.get(chan.get('audience')),
             'is_live': True,
             'view_count': int_or_none(chan.get('viewersTotal')),
             'formats': formats,
         }
+        info.update(self._extract_channel_info(chan))
+
+        return info
+
+
+class BeamProVodIE(BeamProBaseIE):
+    IE_NAME = 'Beam:vod'
+    _VALID_URL = r'https?://(?:\w+\.)?beam\.pro/[^/?#&]+.*[?&]vod=(?P<id>\d+)'
+    _TEST = {
+        'url': 'https://beam.pro/willow8714?vod=2259830',
+        'md5': 'b2431e6e8347dc92ebafb565d368b76b',
+        'info_dict': {
+            'id': '2259830',
+            'ext': 'mp4',
+            'title': 'willow8714\'s Channel',
+            'duration': 6828.15,
+            'thumbnail': r're:https://.*source\.png$',
+            'timestamp': 1494046474,
+            'upload_date': '20170506',
+            'uploader': 'willow8714',
+            'uploader_id': '6085379',
+            'age_limit': 13,
+            'view_count': int,
+        },
+    }
+
+    def _extract_format(self, vod, vod_type):
+        if not vod.get('baseUrl'):
+            return []
+
+        if vod_type == 'hls':
+            filename, protocol = 'manifest.m3u8', 'm3u8'
+        elif vod_type == 'raw':
+            filename, protocol = 'source.mp4', 'https'
+        else:
+            return []
+
+        data = vod.get('data') or {}
+
+        format_id = [vod_type]
+        if 'Height' in data:
+            format_id.append('%sp' % data['Height'])
+
+        return [{
+            'url': urljoin(vod['baseUrl'], filename),
+            'format_id': '-'.join(format_id),
+            'ext': 'mp4',
+            'protocol': protocol,
+            'width': int_or_none(data.get('Width')),
+            'height': int_or_none(data.get('Height')),
+            'fps': int_or_none(data.get('Fps')),
+            'tbr': int_or_none(data.get('Bitrate'), 1000),
+        }]
+
+    def _real_extract(self, url):
+        vod_id = self._match_id(url)
+
+        vod_info = self._download_json(
+            'https://beam.pro/api/v1/recordings/%s' % vod_id, vod_id)
+
+        state = vod_info.get('state')
+        if state != 'AVAILABLE':
+            raise ExtractorError(
+                'VOD %s is not available (state: %s)' % (vod_id, state), expected=True)
+
+        formats = []
+        thumbnail_url = None
+
+        for vod in vod_info['vods']:
+            vod_type = vod.get('format')
+            if vod_type in ('hls', 'raw'):
+                formats.extend(self._extract_format(vod, vod_type))
+            elif vod_type == 'thumbnail':
+                thumbnail_url = urljoin(vod.get('baseUrl'), 'source.png')
+
+        self._sort_formats(formats)
+
+        info = {
+            'id': vod_id,
+            'title': vod_info.get('name') or vod_id,
+            'duration': float_or_none(vod_info.get('duration')),
+            'thumbnail': thumbnail_url,
+            'timestamp': parse_iso8601(vod_info.get('createdAt')),
+            'view_count': int_or_none(vod_info.get('viewsTotal')),
+            'formats': formats,
+        }
+
+        chan = vod_info.get('channel') or {}
+        info.update(self._extract_channel_info(chan))
+
+        return info
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 558c910d4..aecd08ac1 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -98,7 +98,10 @@ from .bbc import (
     BBCCoUkPlaylistIE,
     BBCIE,
 )
-from .beampro import BeamProLiveIE
+from .beampro import (
+    BeamProLiveIE,
+    BeamProVodIE,
+)
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
 from .bellmedia import BellMediaIE

From 6bceb36b9939b2fcc8417cc734fa9e376d67e4f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 May 2017 05:39:28 +0700
Subject: [PATCH 359/611] [beam] Improve and add support for mixer.com (closes
 #13032)

---
 youtube_dl/extractor/beampro.py | 52 ++++++++++++++++++++-------------
 1 file changed, 32 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/beampro.py b/youtube_dl/extractor/beampro.py
index 79c5fe5cd..2eaec1ab4 100644
--- a/youtube_dl/extractor/beampro.py
+++ b/youtube_dl/extractor/beampro.py
@@ -15,6 +15,7 @@ from ..utils import (
 
 
 class BeamProBaseIE(InfoExtractor):
+    _API_BASE = 'https://mixer.com/api/v1'
     _RATINGS = {'family': 0, 'teen': 13, '18+': 18}
 
     def _extract_channel_info(self, chan):
@@ -28,10 +29,10 @@ class BeamProBaseIE(InfoExtractor):
 
 
 class BeamProLiveIE(BeamProBaseIE):
-    IE_NAME = 'Beam:live'
-    _VALID_URL = r'https?://(?:\w+\.)?beam\.pro/(?P<id>[^/?#&]+)'
+    IE_NAME = 'Mixer:live'
+    _VALID_URL = r'https?://(?:\w+\.)?(?:beam\.pro|mixer\.com)/(?P<id>[^/?#&]+)'
     _TEST = {
-        'url': 'http://www.beam.pro/niterhayven',
+        'url': 'http://mixer.com/niterhayven',
         'info_dict': {
             'id': '261562',
             'ext': 'mp4',
@@ -52,6 +53,8 @@ class BeamProLiveIE(BeamProBaseIE):
         },
     }
 
+    _MANIFEST_URL_TEMPLATE = '%s/channels/%%s/manifest.%%s' % BeamProBaseIE._API_BASE
+
     @classmethod
     def suitable(cls, url):
         return False if BeamProVodIE.suitable(url) else super(BeamProLiveIE, cls).suitable(url)
@@ -60,7 +63,7 @@ class BeamProLiveIE(BeamProBaseIE):
         channel_name = self._match_id(url)
 
         chan = self._download_json(
-            'https://beam.pro/api/v1/channels/%s' % channel_name, channel_name)
+            '%s/channels/%s' % (self._API_BASE, channel_name), channel_name)
 
         if chan.get('online') is False:
             raise ExtractorError(
@@ -68,16 +71,22 @@ class BeamProLiveIE(BeamProBaseIE):
 
         channel_id = chan['id']
 
+        def manifest_url(kind):
+            return self._MANIFEST_URL_TEMPLATE % (channel_id, kind)
+
         formats = self._extract_m3u8_formats(
-            'https://beam.pro/api/v1/channels/%s/manifest.m3u8' % channel_id,
-            channel_name, ext='mp4', m3u8_id='hls', fatal=False)
+            manifest_url('m3u8'), channel_name, ext='mp4', m3u8_id='hls',
+            fatal=False)
+        formats.extend(self._extract_smil_formats(
+            manifest_url('smil'), channel_name, fatal=False))
         self._sort_formats(formats)
 
         info = {
             'id': compat_str(chan.get('id') or channel_name),
             'title': self._live_title(chan.get('name') or channel_name),
             'description': clean_html(chan.get('description')),
-            'thumbnail': try_get(chan, lambda x: x['thumbnail']['url'], compat_str),
+            'thumbnail': try_get(
+                chan, lambda x: x['thumbnail']['url'], compat_str),
             'timestamp': parse_iso8601(chan.get('updatedAt')),
             'is_live': True,
             'view_count': int_or_none(chan.get('viewersTotal')),
@@ -89,10 +98,10 @@ class BeamProLiveIE(BeamProBaseIE):
 
 
 class BeamProVodIE(BeamProBaseIE):
-    IE_NAME = 'Beam:vod'
-    _VALID_URL = r'https?://(?:\w+\.)?beam\.pro/[^/?#&]+.*[?&]vod=(?P<id>\d+)'
+    IE_NAME = 'Mixer:vod'
+    _VALID_URL = r'https?://(?:\w+\.)?(?:beam\.pro|mixer\.com)/[^/?#&]+\?.*?\bvod=(?P<id>\d+)'
     _TEST = {
-        'url': 'https://beam.pro/willow8714?vod=2259830',
+        'url': 'https://mixer.com/willow8714?vod=2259830',
         'md5': 'b2431e6e8347dc92ebafb565d368b76b',
         'info_dict': {
             'id': '2259830',
@@ -107,23 +116,27 @@ class BeamProVodIE(BeamProBaseIE):
             'age_limit': 13,
             'view_count': int,
         },
+        'params': {
+            'skip_download': True,
+        },
     }
 
-    def _extract_format(self, vod, vod_type):
+    @staticmethod
+    def _extract_format(vod, vod_type):
         if not vod.get('baseUrl'):
             return []
 
         if vod_type == 'hls':
-            filename, protocol = 'manifest.m3u8', 'm3u8'
+            filename, protocol = 'manifest.m3u8', 'm3u8_native'
         elif vod_type == 'raw':
             filename, protocol = 'source.mp4', 'https'
         else:
-            return []
+            assert False
 
-        data = vod.get('data') or {}
+        data = vod.get('data') if isinstance(vod.get('data'), dict) else {}
 
         format_id = [vod_type]
-        if 'Height' in data:
+        if isinstance(data.get('Height'), compat_str):
             format_id.append('%sp' % data['Height'])
 
         return [{
@@ -141,12 +154,13 @@ class BeamProVodIE(BeamProBaseIE):
         vod_id = self._match_id(url)
 
         vod_info = self._download_json(
-            'https://beam.pro/api/v1/recordings/%s' % vod_id, vod_id)
+            '%s/recordings/%s' % (self._API_BASE, vod_id), vod_id)
 
         state = vod_info.get('state')
         if state != 'AVAILABLE':
             raise ExtractorError(
-                'VOD %s is not available (state: %s)' % (vod_id, state), expected=True)
+                'VOD %s is not available (state: %s)' % (vod_id, state),
+                expected=True)
 
         formats = []
         thumbnail_url = None
@@ -169,8 +183,6 @@ class BeamProVodIE(BeamProBaseIE):
             'view_count': int_or_none(vod_info.get('viewsTotal')),
             'formats': formats,
         }
-
-        chan = vod_info.get('channel') or {}
-        info.update(self._extract_channel_info(chan))
+        info.update(self._extract_channel_info(vod_info.get('channel') or {}))
 
         return info

From 765522345fcdf726df9b520314fb8802c2fa8459 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Kundr=C3=A1t?= <jkt@flaska.net>
Date: Fri, 12 May 2017 17:16:50 +0200
Subject: [PATCH 360/611] [dvtv] Parse adaptive formats as well

The old code hit an error when it attempted to parse the string
"adaptive" for video height. Actually parsing the returned playlists is
a good idea because it adds more output formats, including some
audio-only-ones.
---
 youtube_dl/extractor/dvtv.py | 31 ++++++++++++++++++++++++-------
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/dvtv.py b/youtube_dl/extractor/dvtv.py
index 974c69dbc..b1ae37d78 100644
--- a/youtube_dl/extractor/dvtv.py
+++ b/youtube_dl/extractor/dvtv.py
@@ -71,6 +71,14 @@ class DVTVIE(InfoExtractor):
     }, {
         'url': 'http://video.aktualne.cz/v-cechach-poprve-zazni-zelenkova-zrestaurovana-mse/r~45b4b00483ec11e4883b002590604f2e/',
         'only_matching': True,
+    }, {
+        'url': 'https://video.aktualne.cz/dvtv/zeman-si-jen-leci-mindraky-sobotku-nenavidi-a-babis-se-mu-te/r~960cdb3a365a11e7a83b0025900fea04/',
+        'md5': 'f8efe9656017da948369aa099788c8ea',
+        'info_dict': {
+            'id': '3c496fec365911e7a6500025900fea04',
+            'ext': 'm3u8',
+            'title': 'Zeman si jen léčí mindráky, Sobotku nenávidí a Babiš se mu teď hodí, tvrdí Kmenta',
+        }
     }]
 
     def _parse_video_metadata(self, js, video_id):
@@ -79,13 +87,22 @@ class DVTVIE(InfoExtractor):
         formats = []
         for video in metadata['sources']:
             ext = video['type'][6:]
-            formats.append({
-                'url': video['file'],
-                'ext': ext,
-                'format_id': '%s-%s' % (ext, video['label']),
-                'height': int(video['label'].rstrip('p')),
-                'fps': 25,
-            })
+            if video['label'] != 'adaptive':
+                formats.append({
+                    'url': video['file'],
+                    'ext': ext,
+                    'format_id': '%s-%s' % (ext, video['label']),
+                    'height': int(video['label'].rstrip('p')),
+                    'fps': 25,
+                })
+            elif video['type'] == 'application/dash+xml':
+                formats.extend(self._extract_mpd_formats(video['file'],
+                                                         video_id,
+                                                         fatal=False))
+            elif video['type'] == 'application/vnd.apple.mpegurl':
+                formats.extend(self._extract_m3u8_formats(video['file'],
+                                                          video_id,
+                                                          fatal=False))
 
         self._sort_formats(formats)
 

From 4d9fc40100eee316da261c9e60eed9dab5c8ef71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 May 2017 06:19:21 +0700
Subject: [PATCH 361/611] [dvtv] Improve and fix playlists support (closes
 #13063)

---
 youtube_dl/extractor/dvtv.py | 108 ++++++++++++++++++++++-------------
 1 file changed, 67 insertions(+), 41 deletions(-)

diff --git a/youtube_dl/extractor/dvtv.py b/youtube_dl/extractor/dvtv.py
index b1ae37d78..e85c58bd5 100644
--- a/youtube_dl/extractor/dvtv.py
+++ b/youtube_dl/extractor/dvtv.py
@@ -5,9 +5,12 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    js_to_json,
-    unescapeHTML,
+    determine_ext,
     ExtractorError,
+    int_or_none,
+    js_to_json,
+    mimetype2ext,
+    unescapeHTML,
 )
 
 
@@ -24,14 +27,7 @@ class DVTVIE(InfoExtractor):
             'id': 'dc0768de855511e49e4b0025900fea04',
             'ext': 'mp4',
             'title': 'Vondra o Českém století: Při pohledu na Havla mi bylo trapně',
-        }
-    }, {
-        'url': 'http://video.aktualne.cz/dvtv/stropnicky-policie-vrbetice-preventivne-nekontrolovala/r~82ed4322849211e4a10c0025900fea04/',
-        'md5': '6388f1941b48537dbd28791f712af8bf',
-        'info_dict': {
-            'id': '72c02230849211e49f60002590604f2e',
-            'ext': 'mp4',
-            'title': 'Stropnický: Policie Vrbětice preventivně nekontrolovala',
+            'duration': 1484,
         }
     }, {
         'url': 'http://video.aktualne.cz/dvtv/dvtv-16-12-2014-utok-talibanu-boj-o-kliniku-uprchlici/r~973eb3bc854e11e498be002590604f2e/',
@@ -44,72 +40,100 @@ class DVTVIE(InfoExtractor):
             'info_dict': {
                 'id': 'b0b40906854d11e4bdad0025900fea04',
                 'ext': 'mp4',
-                'title': 'Drtinová Veselovský TV 16. 12. 2014: Témata dne'
+                'title': 'Drtinová Veselovský TV 16. 12. 2014: Témata dne',
+                'description': 'md5:0916925dea8e30fe84222582280b47a0',
+                'timestamp': 1418760010,
+                'upload_date': '20141216',
             }
         }, {
             'md5': '5f7652a08b05009c1292317b449ffea2',
             'info_dict': {
                 'id': '420ad9ec854a11e4bdad0025900fea04',
                 'ext': 'mp4',
-                'title': 'Školní masakr možná změní boj s Talibanem, říká novinářka'
+                'title': 'Školní masakr možná změní boj s Talibanem, říká novinářka',
+                'description': 'md5:ff2f9f6de73c73d7cef4f756c1c1af42',
+                'timestamp': 1418760010,
+                'upload_date': '20141216',
             }
         }, {
             'md5': '498eb9dfa97169f409126c617e2a3d64',
             'info_dict': {
                 'id': '95d35580846a11e4b6d20025900fea04',
                 'ext': 'mp4',
-                'title': 'Boj o kliniku: Veřejný zájem, nebo právo na majetek?'
+                'title': 'Boj o kliniku: Veřejný zájem, nebo právo na majetek?',
+                'description': 'md5:889fe610a70fee5511dc3326a089188e',
+                'timestamp': 1418760010,
+                'upload_date': '20141216',
             }
         }, {
             'md5': 'b8dc6b744844032dab6ba3781a7274b9',
             'info_dict': {
                 'id': '6fe14d66853511e4833a0025900fea04',
                 'ext': 'mp4',
-                'title': 'Pánek: Odmítání syrských uprchlíků je ostudou české vlády'
+                'title': 'Pánek: Odmítání syrských uprchlíků je ostudou české vlády',
+                'description': 'md5:544f86de6d20c4815bea11bf2ac3004f',
+                'timestamp': 1418760010,
+                'upload_date': '20141216',
             }
         }],
-    }, {
-        'url': 'http://video.aktualne.cz/v-cechach-poprve-zazni-zelenkova-zrestaurovana-mse/r~45b4b00483ec11e4883b002590604f2e/',
-        'only_matching': True,
     }, {
         'url': 'https://video.aktualne.cz/dvtv/zeman-si-jen-leci-mindraky-sobotku-nenavidi-a-babis-se-mu-te/r~960cdb3a365a11e7a83b0025900fea04/',
         'md5': 'f8efe9656017da948369aa099788c8ea',
         'info_dict': {
             'id': '3c496fec365911e7a6500025900fea04',
-            'ext': 'm3u8',
+            'ext': 'mp4',
             'title': 'Zeman si jen léčí mindráky, Sobotku nenávidí a Babiš se mu teď hodí, tvrdí Kmenta',
-        }
+            'duration': 1103,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://video.aktualne.cz/v-cechach-poprve-zazni-zelenkova-zrestaurovana-mse/r~45b4b00483ec11e4883b002590604f2e/',
+        'only_matching': True,
     }]
 
     def _parse_video_metadata(self, js, video_id):
-        metadata = self._parse_json(js, video_id, transform_source=js_to_json)
+        data = self._parse_json(js, video_id, transform_source=js_to_json)
+
+        title = unescapeHTML(data['title'])
 
         formats = []
-        for video in metadata['sources']:
-            ext = video['type'][6:]
-            if video['label'] != 'adaptive':
+        for video in data['sources']:
+            video_url = video.get('file')
+            if not video_url:
+                continue
+            video_type = video.get('type')
+            ext = determine_ext(video_url, mimetype2ext(video_type))
+            if video_type == 'application/vnd.apple.mpegurl' or ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False))
+            elif video_type == 'application/dash+xml' or ext == 'mpd':
+                formats.extend(self._extract_mpd_formats(
+                    video_url, video_id, mpd_id='dash', fatal=False))
+            else:
+                label = video.get('label')
+                height = self._search_regex(
+                    r'^(\d+)[pP]', label or '', 'height', default=None)
+                format_id = ['http']
+                for f in (ext, label):
+                    if f:
+                        format_id.append(f)
                 formats.append({
-                    'url': video['file'],
-                    'ext': ext,
-                    'format_id': '%s-%s' % (ext, video['label']),
-                    'height': int(video['label'].rstrip('p')),
-                    'fps': 25,
+                    'url': video_url,
+                    'format_id': '-'.join(format_id),
+                    'height': int_or_none(height),
                 })
-            elif video['type'] == 'application/dash+xml':
-                formats.extend(self._extract_mpd_formats(video['file'],
-                                                         video_id,
-                                                         fatal=False))
-            elif video['type'] == 'application/vnd.apple.mpegurl':
-                formats.extend(self._extract_m3u8_formats(video['file'],
-                                                          video_id,
-                                                          fatal=False))
-
         self._sort_formats(formats)
 
         return {
-            'id': metadata['mediaid'],
-            'title': unescapeHTML(metadata['title']),
-            'thumbnail': self._proto_relative_url(metadata['image'], 'http:'),
+            'id': data.get('mediaid') or video_id,
+            'title': title,
+            'description': data.get('description'),
+            'thumbnail': data.get('image'),
+            'duration': int_or_none(data.get('duration')),
+            'timestamp': int_or_none(data.get('pubtime')),
             'formats': formats
         }
 
@@ -120,7 +144,7 @@ class DVTVIE(InfoExtractor):
 
         # single video
         item = self._search_regex(
-            r"(?s)embedData[0-9a-f]{32}\['asset'\]\s*=\s*(\{.+?\});",
+            r'(?s)embedData[0-9a-f]{32}\[["\']asset["\']\]\s*=\s*(\{.+?\});',
             webpage, 'video', default=None, fatal=False)
 
         if item:
@@ -130,6 +154,8 @@ class DVTVIE(InfoExtractor):
         items = re.findall(
             r"(?s)BBX\.context\.assets\['[0-9a-f]{32}'\]\.push\(({.+?})\);",
             webpage)
+        if not items:
+            items = re.findall(r'(?s)var\s+asset\s*=\s*({.+?});\n', webpage)
 
         if items:
             return {

From 4050be78e5c2e5b77a747c20db0a12e11a218db5 Mon Sep 17 00:00:00 2001
From: midas02 <midas02@users.noreply.github.com>
Date: Tue, 18 Apr 2017 05:44:33 +0200
Subject: [PATCH 362/611] [medialaan] Fix videos with missing videoUrl

A rough trick to get around the two different json styles medialaan seems to be using.
Fix for these example videos:
https://vtmkzoom.be/video?aid=45724
https://vtmkzoom.be/video?aid=45425
---
 youtube_dl/extractor/medialaan.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/medialaan.py b/youtube_dl/extractor/medialaan.py
index 6e067474b..fc8f2550b 100644
--- a/youtube_dl/extractor/medialaan.py
+++ b/youtube_dl/extractor/medialaan.py
@@ -17,7 +17,7 @@ from ..utils import (
 class MedialaanIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                     https?://
-                        (?:www\.)?
+                        (?:www\.|nieuws\.)?
                         (?:
                             (?P<site_id>vtm|q2|vtmkzoom)\.be/
                             (?:
@@ -85,6 +85,22 @@ class MedialaanIE(InfoExtractor):
         # clip
         'url': 'http://vtmkzoom.be/k3-dansstudio/een-nieuw-seizoen-van-k3-dansstudio',
         'only_matching': True,
+    }, {
+        # http/s redirect
+        'url': 'https://vtmkzoom.be/video?aid=45724',
+        'info_dict': {
+            'id': '257136373657000',
+            'ext': 'mp4',
+            'title': 'K3 Dansstudio Ushuaia afl.6',
+        },
+        'params': {
+                'skip_download': True,
+        },
+        'skip': 'Requires account credentials',
+    }, {
+        # nieuws.vtm.be
+        'url': 'https://nieuws.vtm.be/stadion/stadion/genk-nog-moeilijk-programma',
+        'only_matching': True,
     }]
 
     def _real_initialize(self):
@@ -146,6 +162,8 @@ class MedialaanIE(InfoExtractor):
                 video_id, transform_source=lambda s: '[%s]' % s, fatal=False)
             if player:
                 video = player[-1]
+                if video['videoUrl'] in ('http', 'https'):
+                    return self.url_result(video['url'], MedialaanIE.ie_key())
                 info = {
                     'id': video_id,
                     'url': video['videoUrl'],

From 703751add4ec7b7b8a66d892ea893e62763faca0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 May 2017 06:25:50 +0700
Subject: [PATCH 363/611] [medialaan] PEP 8 (closes #12774)

---
 youtube_dl/extractor/medialaan.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/medialaan.py b/youtube_dl/extractor/medialaan.py
index fc8f2550b..4c32fbc2c 100644
--- a/youtube_dl/extractor/medialaan.py
+++ b/youtube_dl/extractor/medialaan.py
@@ -94,7 +94,7 @@ class MedialaanIE(InfoExtractor):
             'title': 'K3 Dansstudio Ushuaia afl.6',
         },
         'params': {
-                'skip_download': True,
+            'skip_download': True,
         },
         'skip': 'Requires account credentials',
     }, {

From 5a6d1da442579fc06e458406dba0f8b60102dc1c Mon Sep 17 00:00:00 2001
From: motophil <phil@rrr.de>
Date: Sun, 19 Mar 2017 12:54:02 +0100
Subject: [PATCH 364/611] [gaskrank] Fix extraction

---
 youtube_dl/extractor/gaskrank.py | 38 ++++++++------------------------
 1 file changed, 9 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/gaskrank.py b/youtube_dl/extractor/gaskrank.py
index 36ba7d8cf..baf513665 100644
--- a/youtube_dl/extractor/gaskrank.py
+++ b/youtube_dl/extractor/gaskrank.py
@@ -6,7 +6,6 @@ from .common import InfoExtractor
 from ..utils import (
     float_or_none,
     int_or_none,
-    js_to_json,
     unified_strdate,
 )
 
@@ -49,19 +48,12 @@ class GaskrankIE(InfoExtractor):
 
     def _real_extract(self, url):
         """extract information from gaskrank.tv"""
-        def fix_json(code):
-            """Removes trailing comma in json: {{},} --> {{}}"""
-            return re.sub(r',\s*}', r'}', js_to_json(code))
 
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
         categories = [re.match(self._VALID_URL, url).group('categories')]
-        title = self._search_regex(
-            r'movieName\s*:\s*\'([^\']*)\'',
-            webpage, 'title')
-        thumbnail = self._search_regex(
-            r'poster\s*:\s*\'([^\']*)\'',
-            webpage, 'thumbnail', default=None)
+        title = self._og_search_title(webpage, default=None) or self._html_search_meta(
+            'title', webpage, fatal=True)
 
         mobj = re.search(
             r'Video von:\s*(?P<uploader_id>[^|]*?)\s*\|\s*vom:\s*(?P<upload_date>[0-9][0-9]\.[0-9][0-9]\.[0-9][0-9][0-9][0-9])',
@@ -89,29 +81,14 @@ class GaskrankIE(InfoExtractor):
         if average_rating:
             average_rating = float_or_none(average_rating.replace(',', '.'))
 
-        playlist = self._parse_json(
-            self._search_regex(
-                r'playlist\s*:\s*\[([^\]]*)\]',
-                webpage, 'playlist', default='{}'),
-            display_id, transform_source=fix_json, fatal=False)
-
         video_id = self._search_regex(
             r'https?://movies\.gaskrank\.tv/([^-]*?)(-[^\.]*)?\.mp4',
-            playlist.get('0').get('src'), 'video id')
+            webpage, 'video id')
 
-        formats = []
-        for key in playlist:
-            formats.append({
-                'url': playlist[key]['src'],
-                'format_id': key,
-                'quality': playlist[key].get('quality')})
-        self._sort_formats(formats, field_preference=['format_id'])
-
-        return {
+        entry = self._parse_html5_media_entries(url, webpage, video_id)[0]
+        entry.update({
             'id': video_id,
             'title': title,
-            'formats': formats,
-            'thumbnail': thumbnail,
             'categories': categories,
             'display_id': display_id,
             'uploader_id': uploader_id,
@@ -120,4 +97,7 @@ class GaskrankIE(InfoExtractor):
             'tags': tags,
             'view_count': view_count,
             'average_rating': average_rating,
-        }
+        })
+        self._sort_formats(entry['formats'])
+
+        return entry

From 7e688d2f6ade3e39423161c7837e48ca2b8243a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 May 2017 06:46:52 +0700
Subject: [PATCH 365/611] [gaskrank] Improve (closes #12493)

---
 youtube_dl/extractor/gaskrank.py | 74 ++++++++++++++++----------------
 1 file changed, 36 insertions(+), 38 deletions(-)

diff --git a/youtube_dl/extractor/gaskrank.py b/youtube_dl/extractor/gaskrank.py
index baf513665..1726a6704 100644
--- a/youtube_dl/extractor/gaskrank.py
+++ b/youtube_dl/extractor/gaskrank.py
@@ -11,50 +11,48 @@ from ..utils import (
 
 
 class GaskrankIE(InfoExtractor):
-    """InfoExtractor for gaskrank.tv"""
-    _VALID_URL = r'https?://(?:www\.)?gaskrank\.tv/tv/(?P<categories>[^/]+)/(?P<id>[^/]+)\.html?'
-    _TESTS = [
-        {
-            'url': 'http://www.gaskrank.tv/tv/motorrad-fun/strike-einparken-durch-anfaenger-crash-mit-groesserem-flurschaden.htm',
-            'md5': '1ae88dbac97887d85ebd1157a95fc4f9',
-            'info_dict': {
-                'id': '201601/26955',
-                'ext': 'mp4',
-                'title': 'Strike! Einparken können nur Männer - Flurschaden hält sich in Grenzen *lol*',
-                'thumbnail': r're:^https?://.*\.jpg$',
-                'categories': ['motorrad-fun'],
-                'display_id': 'strike-einparken-durch-anfaenger-crash-mit-groesserem-flurschaden',
-                'uploader_id': 'Bikefun',
-                'upload_date': '20170110',
-                'uploader_url': None,
-            }
-        },
-        {
-            'url': 'http://www.gaskrank.tv/tv/racing/isle-of-man-tt-2011-michael-du-15920.htm',
-            'md5': 'c33ee32c711bc6c8224bfcbe62b23095',
-            'info_dict': {
-                'id': '201106/15920',
-                'ext': 'mp4',
-                'title': 'Isle of Man - Michael Dunlop vs Guy Martin - schwindelig kucken',
-                'thumbnail': r're:^https?://.*\.jpg$',
-                'categories': ['racing'],
-                'display_id': 'isle-of-man-tt-2011-michael-du-15920',
-                'uploader_id': 'IOM',
-                'upload_date': '20160506',
-                'uploader_url': 'www.iomtt.com',
-            }
+    _VALID_URL = r'https?://(?:www\.)?gaskrank\.tv/tv/(?P<categories>[^/]+)/(?P<id>[^/]+)\.htm'
+    _TESTS = [{
+        'url': 'http://www.gaskrank.tv/tv/motorrad-fun/strike-einparken-durch-anfaenger-crash-mit-groesserem-flurschaden.htm',
+        'md5': '1ae88dbac97887d85ebd1157a95fc4f9',
+        'info_dict': {
+            'id': '201601/26955',
+            'ext': 'mp4',
+            'title': 'Strike! Einparken können nur Männer - Flurschaden hält sich in Grenzen *lol*',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'categories': ['motorrad-fun'],
+            'display_id': 'strike-einparken-durch-anfaenger-crash-mit-groesserem-flurschaden',
+            'uploader_id': 'Bikefun',
+            'upload_date': '20170110',
+            'uploader_url': None,
         }
-    ]
+    }, {
+        'url': 'http://www.gaskrank.tv/tv/racing/isle-of-man-tt-2011-michael-du-15920.htm',
+        'md5': 'c33ee32c711bc6c8224bfcbe62b23095',
+        'info_dict': {
+            'id': '201106/15920',
+            'ext': 'mp4',
+            'title': 'Isle of Man - Michael Dunlop vs Guy Martin - schwindelig kucken',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'categories': ['racing'],
+            'display_id': 'isle-of-man-tt-2011-michael-du-15920',
+            'uploader_id': 'IOM',
+            'upload_date': '20170523',
+            'uploader_url': 'www.iomtt.com',
+        }
+    }]
 
     def _real_extract(self, url):
-        """extract information from gaskrank.tv"""
-
         display_id = self._match_id(url)
+
         webpage = self._download_webpage(url, display_id)
-        categories = [re.match(self._VALID_URL, url).group('categories')]
-        title = self._og_search_title(webpage, default=None) or self._html_search_meta(
+
+        title = self._og_search_title(
+            webpage, default=None) or self._html_search_meta(
             'title', webpage, fatal=True)
 
+        categories = [re.match(self._VALID_URL, url).group('categories')]
+
         mobj = re.search(
             r'Video von:\s*(?P<uploader_id>[^|]*?)\s*\|\s*vom:\s*(?P<upload_date>[0-9][0-9]\.[0-9][0-9]\.[0-9][0-9][0-9][0-9])',
             webpage)
@@ -83,7 +81,7 @@ class GaskrankIE(InfoExtractor):
 
         video_id = self._search_regex(
             r'https?://movies\.gaskrank\.tv/([^-]*?)(-[^\.]*)?\.mp4',
-            webpage, 'video id')
+            webpage, 'video id', default=display_id)
 
         entry = self._parse_html5_media_entries(url, webpage, video_id)[0]
         entry.update({

From bc22df29c49523e41abb84eb09232784b2e07d34 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Wed, 26 Apr 2017 21:21:17 +0200
Subject: [PATCH 366/611] [abcnews] Add support for embed URLs

---
 youtube_dl/extractor/abcnews.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/abcnews.py b/youtube_dl/extractor/abcnews.py
index 4f56c4c11..e18be2497 100644
--- a/youtube_dl/extractor/abcnews.py
+++ b/youtube_dl/extractor/abcnews.py
@@ -12,7 +12,11 @@ from ..compat import compat_urlparse
 
 class AbcNewsVideoIE(AMPIE):
     IE_NAME = 'abcnews:video'
-    _VALID_URL = r'https?://abcnews\.go\.com/[^/]+/video/(?P<display_id>[0-9a-z-]+)-(?P<id>\d+)'
+    _VALID_URL = r'''(?x)
+        https?://abcnews\.go\.com/
+            (?:[^/]+/video/(?P<display_id>[0-9a-z-]+)-|
+               video/embed\?id=)
+       (?P<id>\d+)'''
 
     _TESTS = [{
         'url': 'http://abcnews.go.com/ThisWeek/video/week-exclusive-irans-foreign-minister-zarif-20411932',
@@ -29,6 +33,17 @@ class AbcNewsVideoIE(AMPIE):
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        'url': 'http://abcnews.go.com/video/embed?id=46979033',
+        'md5': 'd19d160cfa8372fc197f406d4821dd9f',
+        'info_dict': {
+            'id': '46979033',
+            'ext': 'flv',
+            'title': 'Nikki Haley reacts to French election results',
+            'description': 'md5:df1b5ad1b8ad76c4d4e01d3c67d6b38e',
+            'duration': 274,
+            'thumbnail': r're:^https?://.*\.jpg$',
+        },
     }, {
         'url': 'http://abcnews.go.com/2020/video/2020-husband-stands-teacher-jail-student-affairs-26119478',
         'only_matching': True,

From 164fcbfeb75787cfeb1f0a84188ba4c11ac4a009 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 May 2017 07:05:50 +0700
Subject: [PATCH 367/611] [abcnews] Improve and remove duplicate test (closes
 #12851)

---
 youtube_dl/extractor/abcnews.py | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/abcnews.py b/youtube_dl/extractor/abcnews.py
index e18be2497..74d54560c 100644
--- a/youtube_dl/extractor/abcnews.py
+++ b/youtube_dl/extractor/abcnews.py
@@ -13,10 +13,14 @@ from ..compat import compat_urlparse
 class AbcNewsVideoIE(AMPIE):
     IE_NAME = 'abcnews:video'
     _VALID_URL = r'''(?x)
-        https?://abcnews\.go\.com/
-            (?:[^/]+/video/(?P<display_id>[0-9a-z-]+)-|
-               video/embed\?id=)
-       (?P<id>\d+)'''
+                    https?://
+                        abcnews\.go\.com/
+                        (?:
+                            [^/]+/video/(?P<display_id>[0-9a-z-]+)-|
+                            video/embed\?.*?\bid=
+                        )
+                        (?P<id>\d+)
+                    '''
 
     _TESTS = [{
         'url': 'http://abcnews.go.com/ThisWeek/video/week-exclusive-irans-foreign-minister-zarif-20411932',
@@ -35,15 +39,7 @@ class AbcNewsVideoIE(AMPIE):
         },
     }, {
         'url': 'http://abcnews.go.com/video/embed?id=46979033',
-        'md5': 'd19d160cfa8372fc197f406d4821dd9f',
-        'info_dict': {
-            'id': '46979033',
-            'ext': 'flv',
-            'title': 'Nikki Haley reacts to French election results',
-            'description': 'md5:df1b5ad1b8ad76c4d4e01d3c67d6b38e',
-            'duration': 274,
-            'thumbnail': r're:^https?://.*\.jpg$',
-        },
+        'only_matching': True,
     }, {
         'url': 'http://abcnews.go.com/2020/video/2020-husband-stands-teacher-jail-student-affairs-26119478',
         'only_matching': True,

From 855520427458f68076143c4079b038f4d15e5a13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 May 2017 07:50:15 +0700
Subject: [PATCH 368/611] [xhamster] Extract categories (closes #11728)

---
 youtube_dl/extractor/xhamster.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 7b6703714..7cf0629e7 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -4,6 +4,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    clean_html,
     dict_get,
     ExtractorError,
     int_or_none,
@@ -25,6 +26,7 @@ class XHamsterIE(InfoExtractor):
             'uploader': 'Ruseful2011',
             'duration': 893,
             'age_limit': 18,
+            'categories': ['Fake Hub', 'Amateur', 'MILFs', 'POV', 'Boss', 'Office', 'Oral', 'Reality', 'Sexy'],
         },
     }, {
         'url': 'http://xhamster.com/movies/2221348/britney_spears_sexy_booty.html?hd',
@@ -36,6 +38,7 @@ class XHamsterIE(InfoExtractor):
             'uploader': 'jojo747400',
             'duration': 200,
             'age_limit': 18,
+            'categories': ['Britney Spears', 'Celebrities', 'HD Videos', 'Sexy', 'Sexy Booty'],
         },
         'params': {
             'skip_download': True,
@@ -51,6 +54,7 @@ class XHamsterIE(InfoExtractor):
             'uploader': 'parejafree',
             'duration': 72,
             'age_limit': 18,
+            'categories': ['Amateur', 'Blowjobs'],
         },
         'params': {
             'skip_download': True,
@@ -152,6 +156,12 @@ class XHamsterIE(InfoExtractor):
 
         self._sort_formats(formats)
 
+        categories_html = self._search_regex(
+            r'(?s)<table.+?(<span>Categories:.+?)</table>', webpage,
+            'categories', default=None)
+        categories = [clean_html(category) for category in re.findall(
+            r'<a[^>]+>(.+?)</a>', categories_html)] if categories_html else None
+
         return {
             'id': video_id,
             'title': title,
@@ -165,6 +175,7 @@ class XHamsterIE(InfoExtractor):
             'dislike_count': int_or_none(dislike_count),
             'comment_count': int_or_none(comment_count),
             'age_limit': age_limit,
+            'categories': categories,
             'formats': formats,
         }
 

From 78e210dea554144c8c1890c617808b0acdab9bfc Mon Sep 17 00:00:00 2001
From: fiocfun <fiocfun@gmail.com>
Date: Sat, 27 May 2017 13:28:08 +0300
Subject: [PATCH 369/611] [xhamster] Fix author and like/dislike count
 extraction

---
 youtube_dl/extractor/xhamster.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 7cf0629e7..ae0c167c4 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -108,7 +108,7 @@ class XHamsterIE(InfoExtractor):
             webpage, 'upload date', fatal=False))
 
         uploader = self._html_search_regex(
-            r'<span[^>]+itemprop=["\']author[^>]+><a[^>]+href=["\'].+?xhamster\.com/user/[^>]+>(?P<uploader>.+?)</a>',
+            r'<span[^>]+?itemprop=["\']author[^>]+?><a[^>]+?href=["\'][^>]+?><span[^>]+?itemprop=["\']name[^>]+?>(.+?)</span',
             webpage, 'uploader', default='anonymous')
 
         thumbnail = self._search_regex(
@@ -124,7 +124,7 @@ class XHamsterIE(InfoExtractor):
             r'content=["\']User(?:View|Play)s:(\d+)',
             webpage, 'view count', fatal=False))
 
-        mobj = re.search(r"hint='(?P<likecount>\d+) Likes / (?P<dislikecount>\d+) Dislikes'", webpage)
+        mobj = re.search(r'hint=[\'"](?P<likecount>\d+) Likes / (?P<dislikecount>\d+) Dislikes[\'"]', webpage)
         (like_count, dislike_count) = (mobj.group('likecount'), mobj.group('dislikecount')) if mobj else (None, None)
 
         mobj = re.search(r'</label>Comments \((?P<commentcount>\d+)\)</div>', webpage)

From a846173d9304c958bd7bedb89fe767490c885ae1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 May 2017 07:55:56 +0700
Subject: [PATCH 370/611] [xhamster] Simplify (closes #13216)

---
 youtube_dl/extractor/xhamster.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index ae0c167c4..6987b2e3a 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -108,7 +108,7 @@ class XHamsterIE(InfoExtractor):
             webpage, 'upload date', fatal=False))
 
         uploader = self._html_search_regex(
-            r'<span[^>]+?itemprop=["\']author[^>]+?><a[^>]+?href=["\'][^>]+?><span[^>]+?itemprop=["\']name[^>]+?>(.+?)</span',
+            r'<span[^>]+itemprop=["\']author[^>]+><a[^>]+><span[^>]+>([^<]+)',
             webpage, 'uploader', default='anonymous')
 
         thumbnail = self._search_regex(
@@ -124,7 +124,7 @@ class XHamsterIE(InfoExtractor):
             r'content=["\']User(?:View|Play)s:(\d+)',
             webpage, 'view count', fatal=False))
 
-        mobj = re.search(r'hint=[\'"](?P<likecount>\d+) Likes / (?P<dislikecount>\d+) Dislikes[\'"]', webpage)
+        mobj = re.search(r'hint=[\'"](?P<likecount>\d+) Likes / (?P<dislikecount>\d+) Dislikes', webpage)
         (like_count, dislike_count) = (mobj.group('likecount'), mobj.group('dislikecount')) if mobj else (None, None)
 
         mobj = re.search(r'</label>Comments \((?P<commentcount>\d+)\)</div>', webpage)

From a49eccdfa7e1ba4d834a6f65d15ef88a37bd78dd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 28 May 2017 00:47:41 +0800
Subject: [PATCH 371/611] [youtube] Parse player_url if format URLs are
 encrypted or DASH MPDs are requested

Fixes #13211
---
 ChangeLog                       |  2 ++
 youtube_dl/extractor/youtube.py | 11 ++++++-----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 65a3ca468..c8dd70fe5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,8 @@
 version <unreleased>
 
 Extractors
+* [youtube] Fix DASH MPD extraction for videos with non-encrypted format URLs
+  (#13211)
 + [youku] Extract more metadata (#10433)
 
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index d6c098706..40ac1a019 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1715,12 +1715,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 format_id = url_data['itag'][0]
                 url = url_data['url'][0]
 
-                if 'sig' in url_data:
-                    url += '&signature=' + url_data['sig'][0]
-                elif 's' in url_data:
-                    encrypted_sig = url_data['s'][0]
+                if 's' in url_data or self._downloader.params.get('youtube_include_dash_manifest', True):
                     ASSETS_RE = r'"assets":.+?"js":\s*("[^"]+")'
-
                     jsplayer_url_json = self._search_regex(
                         ASSETS_RE,
                         embed_webpage if age_gate else video_webpage,
@@ -1741,6 +1737,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                             video_webpage, 'age gate player URL')
                         player_url = json.loads(player_url_json)
 
+                if 'sig' in url_data:
+                    url += '&signature=' + url_data['sig'][0]
+                elif 's' in url_data:
+                    encrypted_sig = url_data['s'][0]
+
                     if self._downloader.params.get('verbose'):
                         if player_url is None:
                             player_version = 'unknown'

From 99bea8d298ae68a14bd86675db1331bef09bf193 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 May 2017 00:33:24 +0700
Subject: [PATCH 372/611] [ChangeLog] Actualize

---
 ChangeLog | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index c8dd70fe5..dfe505e95 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -3,7 +3,18 @@ version <unreleased>
 Extractors
 * [youtube] Fix DASH MPD extraction for videos with non-encrypted format URLs
   (#13211)
+* [xhamster] Fix uploader and like/dislike count extraction (#13216))
++ [xhamster] Extract categories (#11728)
++ [abcnews] Add support for embed URLs (#12851)
+* [gaskrank] Fix extraction (#12493)
+* [medialaan] Fix videos with missing videoUrl (#12774)
+* [dvtv] Fix playlist support
++ [dvtv] Add support for DASH and HLS formats (#3063)
++ [beam:vod] Add support for beam.pro/mixer.com VODs (#13032))
+* [cbsinteractive] Relax URL regular expression (#13213)
+* [adn] Fix formats extraction
 + [youku] Extract more metadata (#10433)
+* [cbsnews] Fix extraction (#13205)
 
 
 version 2017.05.26

From 941ea38ef535ec2923c224a88cdedeae3eb42a68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 May 2017 00:42:18 +0700
Subject: [PATCH 373/611] release 2017.05.29

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 ++-
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 5ca69ffa1..24a0d72c5 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.26*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.26**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.29*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.29**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.05.26
+[debug] youtube-dl version 2017.05.29
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index dfe505e95..7d78c87e6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.05.29
 
 Extractors
 * [youtube] Fix DASH MPD extraction for videos with non-encrypted format URLs
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 9791d585b..8ddc800b4 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -93,7 +93,6 @@
  - **bbc.co.uk:article**: BBC articles
  - **bbc.co.uk:iplayer:playlist**
  - **bbc.co.uk:playlist**
- - **Beam:live**
  - **Beatport**
  - **Beeg**
  - **BehindKink**
@@ -453,6 +452,8 @@
  - **mixcloud:playlist**
  - **mixcloud:stream**
  - **mixcloud:user**
+ - **Mixer:live**
+ - **Mixer:vod**
  - **MLB**
  - **Mnet**
  - **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 16d91dfad..d4ee43122 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.05.26'
+__version__ = '2017.05.29'

From d10d0e3cf8c38fba27d873fd3cb60572327fe5dc Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 29 May 2017 14:58:07 +0800
Subject: [PATCH 374/611] [README.md] Add an example for how to use .netrc on
 Windows

That's a Python bug: http://bugs.python.org/issue28334
Most likely it will be fixed in Python 3.7: https://github.com/python/cpython/pull/123
---
 README.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index dc0be1f40..284ce8ca3 100644
--- a/README.md
+++ b/README.md
@@ -474,7 +474,10 @@ machine twitch login my_twitch_account_name password my_twitch_password
 ```
 To activate authentication with the `.netrc` file you should pass `--netrc` to youtube-dl or place it in the [configuration file](#configuration).
 
-On Windows you may also need to setup the `%HOME%` environment variable manually.
+On Windows you may also need to setup the `%HOME%` environment variable manually. For example:
+```
+set HOME=%USERPROFILE%
+```
 
 # OUTPUT TEMPLATE
 

From 6b7ce85cdcba2449455fb79e84ce2014b18b5779 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 May 2017 23:50:48 +0700
Subject: [PATCH 375/611] [README.md] Mention http_dash_segments protocol

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 284ce8ca3..fa901a46d 100644
--- a/README.md
+++ b/README.md
@@ -652,7 +652,7 @@ Also filtering work for comparisons `=` (equals), `!=` (not equals), `^=` (begin
  - `acodec`: Name of the audio codec in use
  - `vcodec`: Name of the video codec in use
  - `container`: Name of the container format
- - `protocol`: The protocol that will be used for the actual download, lower-case (`http`, `https`, `rtsp`, `rtmp`, `rtmpe`, `mms`, `f4m`, `ism`, `m3u8`, or `m3u8_native`)
+ - `protocol`: The protocol that will be used for the actual download, lower-case (`http`, `https`, `rtsp`, `rtmp`, `rtmpe`, `mms`, `f4m`, `ism`, `http_dash_segments`, `m3u8`, or `m3u8_native`)
  - `format_id`: A short description of the format
 
 Note that none of the aforementioned meta fields are guaranteed to be present since this solely depends on the metadata obtained by particular extractor, i.e. the metadata offered by the video hoster.

From c56ad5c97580f4883bef375427caa62c851dc7a8 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 31 May 2017 15:43:54 +0100
Subject: [PATCH 376/611] [packtpub] Fix authentication(closes #13240)

---
 youtube_dl/extractor/packtpub.py | 23 ++++++++---------------
 1 file changed, 8 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/packtpub.py b/youtube_dl/extractor/packtpub.py
index bb668c999..8ed3c6347 100644
--- a/youtube_dl/extractor/packtpub.py
+++ b/youtube_dl/extractor/packtpub.py
@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+import json
 import re
 
 from .common import InfoExtractor
@@ -14,7 +15,6 @@ from ..utils import (
     strip_or_none,
     unified_timestamp,
     urljoin,
-    urlencode_postdata,
 )
 
 
@@ -45,22 +45,15 @@ class PacktPubIE(PacktPubBaseIE):
         (username, password) = self._get_login_info()
         if username is None:
             return
-        webpage = self._download_webpage(self._PACKT_BASE, None)
-        login_form = self._form_hidden_inputs(
-            'packt-user-login-form', webpage)
-        login_form.update({
-            'email': username,
-            'password': password,
-        })
-        self._download_webpage(
-            self._PACKT_BASE, None, 'Logging in as %s' % username,
-            data=urlencode_postdata(login_form))
         try:
             self._TOKEN = self._download_json(
-                '%s/users/tokens/sessions' % self._MAPT_REST, None,
-                'Downloading Authorization Token')['data']['token']
+                self._MAPT_REST + '/users/tokens', None,
+                'Downloading Authorization Token', data=json.dumps({
+                    'email': username,
+                    'password': password,
+                }).encode())['data']['access']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code in (401, 404):
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code in (400, 401, 404):
                 message = self._parse_json(e.cause.read().decode(), None)['message']
                 raise ExtractorError(message, expected=True)
             raise
@@ -83,7 +76,7 @@ class PacktPubIE(PacktPubBaseIE):
 
         headers = {}
         if self._TOKEN:
-            headers['Authorization'] = self._TOKEN
+            headers['Authorization'] = 'Bearer ' + self._TOKEN
         video = self._download_json(
             '%s/users/me/products/%s/chapters/%s/sections/%s'
             % (self._MAPT_REST, course_id, chapter_id, video_id), video_id,

From 87b5184a0d7e32f595fca19b1ec227164ccd0acd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 31 May 2017 23:56:32 +0700
Subject: [PATCH 377/611] [drbonanza] Fix extraction (closes #13231)

---
 youtube_dl/extractor/drbonanza.py | 146 +++++++-----------------------
 1 file changed, 35 insertions(+), 111 deletions(-)

diff --git a/youtube_dl/extractor/drbonanza.py b/youtube_dl/extractor/drbonanza.py
index 79ec212c8..164e97c36 100644
--- a/youtube_dl/extractor/drbonanza.py
+++ b/youtube_dl/extractor/drbonanza.py
@@ -1,135 +1,59 @@
 from __future__ import unicode_literals
 
-import json
 import re
 
 from .common import InfoExtractor
 from ..utils import (
-    int_or_none,
-    parse_iso8601,
+    js_to_json,
+    parse_duration,
+    unescapeHTML,
 )
 
 
 class DRBonanzaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dr\.dk/bonanza/(?:[^/]+/)+(?:[^/])+?(?:assetId=(?P<id>\d+))?(?:[#&]|$)'
-
-    _TESTS = [{
-        'url': 'http://www.dr.dk/bonanza/serie/portraetter/Talkshowet.htm?assetId=65517',
+    _VALID_URL = r'https?://(?:www\.)?dr\.dk/bonanza/[^/]+/\d+/[^/]+/(?P<id>\d+)/(?P<display_id>[^/?#&]+)'
+    _TEST = {
+        'url': 'http://www.dr.dk/bonanza/serie/154/matador/40312/matador---0824-komme-fremmede-',
         'info_dict': {
-            'id': '65517',
+            'id': '40312',
+            'display_id': 'matador---0824-komme-fremmede-',
             'ext': 'mp4',
-            'title': 'Talkshowet - Leonard Cohen',
-            'description': 'md5:8f34194fb30cd8c8a30ad8b27b70c0ca',
+            'title': 'MATADOR - 08:24. "Komme fremmede".',
+            'description': 'md5:77b4c1ac4d4c1b9d610ab4395212ff84',
             'thumbnail': r're:^https?://.*\.(?:gif|jpg)$',
-            'timestamp': 1295537932,
-            'upload_date': '20110120',
-            'duration': 3664,
+            'duration': 4613,
         },
-        'params': {
-            'skip_download': True,  # requires rtmp
-        },
-    }, {
-        'url': 'http://www.dr.dk/bonanza/radio/serie/sport/fodbold.htm?assetId=59410',
-        'md5': '6dfe039417e76795fb783c52da3de11d',
-        'info_dict': {
-            'id': '59410',
-            'ext': 'mp3',
-            'title': 'EM fodbold 1992 Danmark - Tyskland finale Transmission',
-            'description': 'md5:501e5a195749480552e214fbbed16c4e',
-            'thumbnail': r're:^https?://.*\.(?:gif|jpg)$',
-            'timestamp': 1223274900,
-            'upload_date': '20081006',
-            'duration': 7369,
-        },
-    }]
+    }
 
     def _real_extract(self, url):
-        url_id = self._match_id(url)
-        webpage = self._download_webpage(url, url_id)
+        mobj = re.match(self._VALID_URL, url)
+        video_id, display_id = mobj.group('id', 'display_id')
 
-        if url_id:
-            info = json.loads(self._html_search_regex(r'({.*?%s.*})' % url_id, webpage, 'json'))
-        else:
-            # Just fetch the first video on that page
-            info = json.loads(self._html_search_regex(r'bonanzaFunctions.newPlaylist\(({.*})\)', webpage, 'json'))
+        webpage = self._download_webpage(url, display_id)
 
-        asset_id = str(info['AssetId'])
-        title = info['Title'].rstrip(' \'\"-,.:;!?')
-        duration = int_or_none(info.get('Duration'), scale=1000)
-        # First published online. "FirstPublished" contains the date for original airing.
-        timestamp = parse_iso8601(
-            re.sub(r'\.\d+$', '', info['Created']))
+        info = self._parse_html5_media_entries(
+            url, webpage, display_id, m3u8_id='hls',
+            m3u8_entry_protocol='m3u8_native')[0]
+        self._sort_formats(info['formats'])
 
-        def parse_filename_info(url):
-            match = re.search(r'/\d+_(?P<width>\d+)x(?P<height>\d+)x(?P<bitrate>\d+)K\.(?P<ext>\w+)$', url)
-            if match:
-                return {
-                    'width': int(match.group('width')),
-                    'height': int(match.group('height')),
-                    'vbr': int(match.group('bitrate')),
-                    'ext': match.group('ext')
-                }
-            match = re.search(r'/\d+_(?P<bitrate>\d+)K\.(?P<ext>\w+)$', url)
-            if match:
-                return {
-                    'vbr': int(match.group('bitrate')),
-                    'ext': match.group(2)
-                }
-            return {}
+        asset = self._parse_json(
+            self._search_regex(
+                r'(?s)currentAsset\s*=\s*({.+?})\s*</script', webpage, 'asset'),
+            display_id, transform_source=js_to_json)
 
-        video_types = ['VideoHigh', 'VideoMid', 'VideoLow']
-        preferencemap = {
-            'VideoHigh': -1,
-            'VideoMid': -2,
-            'VideoLow': -3,
-            'Audio': -4,
-        }
+        title = unescapeHTML(asset['AssetTitle']).strip()
 
-        formats = []
-        for file in info['Files']:
-            if info['Type'] == 'Video':
-                if file['Type'] in video_types:
-                    format = parse_filename_info(file['Location'])
-                    format.update({
-                        'url': file['Location'],
-                        'format_id': file['Type'].replace('Video', ''),
-                        'preference': preferencemap.get(file['Type'], -10),
-                    })
-                    if format['url'].startswith('rtmp'):
-                        rtmp_url = format['url']
-                        format['rtmp_live'] = True  # --resume does not work
-                        if '/bonanza/' in rtmp_url:
-                            format['play_path'] = rtmp_url.split('/bonanza/')[1]
-                    formats.append(format)
-                elif file['Type'] == 'Thumb':
-                    thumbnail = file['Location']
-            elif info['Type'] == 'Audio':
-                if file['Type'] == 'Audio':
-                    format = parse_filename_info(file['Location'])
-                    format.update({
-                        'url': file['Location'],
-                        'format_id': file['Type'],
-                        'vcodec': 'none',
-                    })
-                    formats.append(format)
-                elif file['Type'] == 'Thumb':
-                    thumbnail = file['Location']
+        def extract(field):
+            return self._search_regex(
+                r'<div[^>]+>\s*<p>%s:<p>\s*</div>\s*<div[^>]+>\s*<p>([^<]+)</p>' % field,
+                webpage, field, default=None)
 
-        description = '%s\n%s\n%s\n' % (
-            info['Description'], info['Actors'], info['Colophon'])
-
-        self._sort_formats(formats)
-
-        display_id = re.sub(r'[^\w\d-]', '', re.sub(r' ', '-', title.lower())) + '-' + asset_id
-        display_id = re.sub(r'-+', '-', display_id)
-
-        return {
-            'id': asset_id,
+        info.update({
+            'id': asset.get('AssetId') or video_id,
             'display_id': display_id,
             'title': title,
-            'formats': formats,
-            'description': description,
-            'thumbnail': thumbnail,
-            'timestamp': timestamp,
-            'duration': duration,
-        }
+            'description': extract('Programinfo'),
+            'duration': parse_duration(extract('Tid')),
+            'thumbnail': asset.get('AssetImageUrl'),
+        })
+        return info

From 4489d418163def3bc94fc1b9be999ae946621530 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Jun 2017 00:15:15 +0700
Subject: [PATCH 378/611] [francetv] Relax _VALID_URL

---
 youtube_dl/extractor/francetv.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 546d5caa0..2bcbb3e39 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -112,7 +112,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
 
 
 class FranceTVIE(FranceTVBaseInfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www\.)?france\.tv|mobile\.france\.tv)/(?:[^/]+/)+(?P<id>[^/]+)\.html'
+    _VALID_URL = r'https?://(?:(?:www\.)?france\.tv|mobile\.france\.tv)/(?:[^/]+/)*(?P<id>[^/]+)\.html'
 
     _TESTS = [{
         'url': 'https://www.france.tv/france-2/13h15-le-dimanche/140921-les-mysteres-de-jesus.html',
@@ -157,6 +157,9 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
     }, {
         'url': 'https://mobile.france.tv/france-5/c-dans-l-air/137347-emission-du-vendredi-12-mai-2017.html',
         'only_matching': True,
+    }, {
+        'url': 'https://www.france.tv/142749-rouge-sang.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From f7a747ce59036dca4451037bff4b75cf358e4ad5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Jun 2017 22:19:52 +0700
Subject: [PATCH 379/611] [1tv] Lower preference for http formats (closes
 #13246)

---
 youtube_dl/extractor/firsttv.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/firsttv.py b/youtube_dl/extractor/firsttv.py
index 081c71842..4803a22c8 100644
--- a/youtube_dl/extractor/firsttv.py
+++ b/youtube_dl/extractor/firsttv.py
@@ -102,6 +102,8 @@ class FirstTVIE(InfoExtractor):
                     'format_id': f.get('name'),
                     'tbr': tbr,
                     'source_preference': quality(f.get('name')),
+                    # quality metadata of http formats may be incorrect
+                    'preference': -1,
                 })
             # m3u8 URL format is reverse engineered from [1] (search for
             # master.m3u8). dashEdges (that is currently balancer-vod.1tv.ru)

From 39d4c1be4d601a3297157315652637ca52000965 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Jun 2017 23:29:45 +0700
Subject: [PATCH 380/611] [youtube] Improve chapters extraction (closes #13247)

---
 test/test_youtube_chapters.py   | 7 +++++++
 youtube_dl/extractor/youtube.py | 6 ++++++
 2 files changed, 13 insertions(+)

diff --git a/test/test_youtube_chapters.py b/test/test_youtube_chapters.py
index cb12f8384..324ca8525 100644
--- a/test/test_youtube_chapters.py
+++ b/test/test_youtube_chapters.py
@@ -254,6 +254,13 @@ class TestYoutubeChapters(unittest.TestCase):
                 'title': '3 - Из серпов луны...[Iz serpov luny]',
             }]
         ),
+        (
+            # https://www.youtube.com/watch?v=xZW70zEasOk
+            # time point more than duration
+            '''● LCS Spring finals: Saturday and Sunday from <a href="#" onclick="yt.www.watch.player.seekTo(13*60+30);return false;">13:30</a> outside the venue! <br />● PAX East: Fri, Sat & Sun - more info in tomorrows video on the main channel!''',
+            283,
+            []
+        ),
     ]
 
     def test_youtube_chapters(self):
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 40ac1a019..bf4f4e139 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1353,10 +1353,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             start_time = parse_duration(time_point)
             if start_time is None:
                 continue
+            if start_time > duration:
+                break
             end_time = (duration if next_num == len(chapter_lines)
                         else parse_duration(chapter_lines[next_num][1]))
             if end_time is None:
                 continue
+            if end_time > duration:
+                end_time = duration
+            if start_time > end_time:
+                break
             chapter_title = re.sub(
                 r'<a[^>]+>[^<]+</a>', '', chapter_line).strip(' \t-')
             chapter_title = re.sub(r'\s+', ' ', chapter_title)

From f129c3f349b41ef8f454d67af761e1c15ad30a92 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jun 2017 01:03:51 +0700
Subject: [PATCH 381/611] [safari] Fix typo (closes #13252)

---
 youtube_dl/extractor/safari.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index c3aec1edd..794e4ee6f 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -67,7 +67,7 @@ class SafariBaseIE(InfoExtractor):
                 'Login failed; make sure your credentials are correct and try again.',
                 expected=True)
 
-        SafariBaseIE.LOGGED_IN = True
+        self.LOGGED_IN = True
 
         self.to_screen('Login successful')
 

From 4bede0d8f5b6fc8d8e46ee240f808935e03eafa2 Mon Sep 17 00:00:00 2001
From: CeruleanSky <CeruleanSky@users.noreply.github.com>
Date: Sat, 3 Jun 2017 08:14:23 -0400
Subject: [PATCH 382/611] [YoutubeDL] Don't emit ANSI escape codes on Windows

---
 youtube_dl/YoutubeDL.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4c33d494a..7efa0c948 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -498,24 +498,25 @@ class YoutubeDL(object):
     def to_console_title(self, message):
         if not self.params.get('consoletitle', False):
             return
-        if compat_os_name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
-            # c_wchar_p() might not be necessary if `message` is
-            # already of type unicode()
-            ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
+        if compat_os_name == 'nt':
+            if ctypes.windll.kernel32.GetConsoleWindow():
+                # c_wchar_p() might not be necessary if `message` is
+                # already of type unicode()
+                ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
         elif 'TERM' in os.environ:
             self._write_string('\033]0;%s\007' % message, self._screen_file)
 
     def save_console_title(self):
         if not self.params.get('consoletitle', False):
             return
-        if 'TERM' in os.environ:
+        if compat_os_name != 'nt' and 'TERM' in os.environ:
             # Save the title on stack
             self._write_string('\033[22;0t', self._screen_file)
 
     def restore_console_title(self):
         if not self.params.get('consoletitle', False):
             return
-        if 'TERM' in os.environ:
+        if compat_os_name != 'nt' and 'TERM' in os.environ:
             # Restore the title from stack
             self._write_string('\033[23;0t', self._screen_file)
 

From cd750b731c41cd9e84c7b16f7c22b6a34da18452 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 3 Jun 2017 22:07:47 +0700
Subject: [PATCH 383/611] [godtv] Remove extractor (closes #13175)

---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/godtv.py      | 66 ------------------------------
 2 files changed, 67 deletions(-)
 delete mode 100644 youtube_dl/extractor/godtv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index aecd08ac1..31e790155 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -392,7 +392,6 @@ from .globo import (
 from .go import GoIE
 from .go90 import Go90IE
 from .godtube import GodTubeIE
-from .godtv import GodTVIE
 from .golem import GolemIE
 from .googledrive import GoogleDriveIE
 from .googleplus import GooglePlusIE
diff --git a/youtube_dl/extractor/godtv.py b/youtube_dl/extractor/godtv.py
deleted file mode 100644
index c5d3b4e6a..000000000
--- a/youtube_dl/extractor/godtv.py
+++ /dev/null
@@ -1,66 +0,0 @@
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from .ooyala import OoyalaIE
-from ..utils import js_to_json
-
-
-class GodTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?god\.tv(?:/[^/]+)*/(?P<id>[^/?#&]+)'
-    _TESTS = [{
-        'url': 'http://god.tv/jesus-image/video/jesus-conference-2016/randy-needham',
-        'info_dict': {
-            'id': 'lpd3g2MzE6D1g8zFAKz8AGpxWcpu6o_3',
-            'ext': 'mp4',
-            'title': 'Randy Needham',
-            'duration': 3615.08,
-        },
-        'params': {
-            'skip_download': True,
-        }
-    }, {
-        'url': 'http://god.tv/playlist/bible-study',
-        'info_dict': {
-            'id': 'bible-study',
-        },
-        'playlist_mincount': 37,
-    }, {
-        'url': 'http://god.tv/node/15097',
-        'only_matching': True,
-    }, {
-        'url': 'http://god.tv/live/africa',
-        'only_matching': True,
-    }, {
-        'url': 'http://god.tv/liveevents',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, display_id)
-
-        settings = self._parse_json(
-            self._search_regex(
-                r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);',
-                webpage, 'settings', default='{}'),
-            display_id, transform_source=js_to_json, fatal=False)
-
-        ooyala_id = None
-
-        if settings:
-            playlist = settings.get('playlist')
-            if playlist and isinstance(playlist, list):
-                entries = [
-                    OoyalaIE._build_url_result(video['content_id'])
-                    for video in playlist if video.get('content_id')]
-                if entries:
-                    return self.playlist_result(entries, display_id)
-            ooyala_id = settings.get('ooyala', {}).get('content_id')
-
-        if not ooyala_id:
-            ooyala_id = self._search_regex(
-                r'["\']content_id["\']\s*:\s*(["\'])(?P<id>[\w-]+)\1',
-                webpage, 'ooyala id', group='id')
-
-        return OoyalaIE._build_url_result(ooyala_id)

From cd138d8bd44ad9845453f7a7fa945f9740425809 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Jun 2017 15:54:19 +0700
Subject: [PATCH 384/611] [pornhub:playlist] Fix extraction (closes #13281)

---
 youtube_dl/extractor/pornhub.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 1dcc8df00..d90c6ce1a 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -252,11 +252,14 @@ class PornHubPlaylistBaseIE(InfoExtractor):
 
         playlist = self._parse_json(
             self._search_regex(
-                r'playlistObject\s*=\s*({.+?});', webpage, 'playlist'),
-            playlist_id)
+                r'(?:playlistObject|PLAYLIST_VIEW)\s*=\s*({.+?});', webpage,
+                'playlist', default='{}'),
+            playlist_id, fatal=False)
+        title = playlist.get('title') or self._search_regex(
+            r'>Videos\s+in\s+(.+?)\s+[Pp]laylist<', webpage, 'title', fatal=False)
 
         return self.playlist_result(
-            entries, playlist_id, playlist.get('title'), playlist.get('description'))
+            entries, playlist_id, title, playlist.get('description'))
 
 
 class PornHubPlaylistIE(PornHubPlaylistBaseIE):

From 9edcdac90cc871e6d6e497dc0fb099e40b99ff02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Jun 2017 20:39:55 +0700
Subject: [PATCH 385/611] [pornhub:uservideos] Add missing raise

---
 youtube_dl/extractor/pornhub.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index d90c6ce1a..e032817f2 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -299,6 +299,7 @@ class PornHubUserVideosIE(PornHubPlaylistBaseIE):
             except ExtractorError as e:
                 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
                     break
+                raise
             page_entries = self._extract_entries(webpage)
             if not page_entries:
                 break

From 62bafabc099de4ededf9e136e3c11e878749d066 Mon Sep 17 00:00:00 2001
From: Adam Voss <vossad01@gmail.com>
Date: Sun, 4 Jun 2017 08:47:05 -0500
Subject: [PATCH 386/611] [bandcamp:weekly] Add extractor

---
 youtube_dl/extractor/bandcamp.py   | 67 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  2 +-
 2 files changed, 68 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 489d0ba53..54fa8634c 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -17,6 +17,7 @@ from ..utils import (
     parse_filesize,
     unescapeHTML,
     update_url_query,
+    unified_strdate,
 )
 
 
@@ -222,6 +223,10 @@ class BandcampAlbumIE(InfoExtractor):
         'playlist_count': 2,
     }]
 
+    @classmethod
+    def suitable(cls, url):
+        return False if BandcampWeeklyIE.suitable(url) else super(BandcampAlbumIE, cls).suitable(url)
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         uploader_id = mobj.group('subdomain')
@@ -250,3 +255,65 @@ class BandcampAlbumIE(InfoExtractor):
             'title': title,
             'entries': entries,
         }
+
+
+class BandcampWeeklyIE(InfoExtractor):
+    IE_NAME = 'Bandcamp:bandcamp_weekly'
+    _VALID_URL = r'https?://(?:www\.)?bandcamp\.com/?\?(?:.*&)?show=(?P<id>\d+)(?:$|[&#])'
+    _TESTS = [{
+        'url': 'https://bandcamp.com/?show=224',
+        'md5': 'b00df799c733cf7e0c567ed187dea0fd',
+        'info_dict': {
+            'id': '224',
+            'ext': 'opus',
+            'title': 'BC Weekly April 4th 2017: Magic Moments',
+            'description': 'Stones Throw\'s Vex Ruffin, plus up and coming singer Salami Rose Joe Louis, in conversation about their fantastic DIY albums.',
+        }
+    }, {
+        'url': 'https://bandcamp.com/?blah/blah@&show=228',
+        'only_matching': True
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        blob = self._parse_json(
+            self._search_regex(
+                r'data-blob=(["\'])(?P<blob>{.+?})\1', webpage,
+                'blob', group='blob'),
+            video_id, transform_source=unescapeHTML)
+
+        show = blob['bcw_show']
+
+        # This is desired because any invalid show id redirects to `bandcamp.com`
+        # which happens to expose the latest Bandcamp Weekly episode.
+        video_id = compat_str(show['show_id'])
+
+        def to_format_dictionaries(audio_stream):
+            dictionaries = [{'format_id': kvp[0], 'url': kvp[1]} for kvp in audio_stream.items()]
+            known_extensions = ['mp3', 'opus']
+
+            for dictionary in dictionaries:
+                for ext in known_extensions:
+                    if ext in dictionary['format_id']:
+                        dictionary['ext'] = ext
+                        break
+
+            return dictionaries
+
+        formats = to_format_dictionaries(show['audio_stream'])
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': show['audio_title'] + ': ' + show['subtitle'],
+            'description': show.get('desc'),
+            'duration': float_or_none(show.get('audio_duration')),
+            'webpage_url': 'https://bandcamp.com/?show=' + video_id,
+            'is_live': False,
+            'release_date': unified_strdate(show.get('published_date')),
+            'series': 'Bandcamp Weekly',
+            'episode_id': compat_str(video_id),
+            'formats': formats
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 31e790155..e1907314d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -90,7 +90,7 @@ from .azmedien import (
 )
 from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
-from .bandcamp import BandcampIE, BandcampAlbumIE
+from .bandcamp import BandcampIE, BandcampAlbumIE, BandcampWeeklyIE
 from .bbc import (
     BBCCoUkIE,
     BBCCoUkArticleIE,

From 6d923aab35f186ca5c4c7fce6b7c43145aa192db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Jun 2017 23:21:30 +0700
Subject: [PATCH 387/611] [bandcamp:weekly] Improve and extract more metadata
 (closes #12758)

---
 youtube_dl/extractor/bandcamp.py | 78 ++++++++++++++++++++++----------
 1 file changed, 54 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 54fa8634c..9ddb9af17 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -14,6 +14,7 @@ from ..utils import (
     ExtractorError,
     float_or_none,
     int_or_none,
+    KNOWN_EXTENSIONS,
     parse_filesize,
     unescapeHTML,
     update_url_query,
@@ -22,7 +23,7 @@ from ..utils import (
 
 
 class BandcampIE(InfoExtractor):
-    _VALID_URL = r'https?://.*?\.bandcamp\.com/track/(?P<title>.*)'
+    _VALID_URL = r'https?://.*?\.bandcamp\.com/track/(?P<title>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://youtube-dl.bandcamp.com/track/youtube-dl-test-song',
         'md5': 'c557841d5e50261777a6585648adf439',
@@ -156,7 +157,7 @@ class BandcampIE(InfoExtractor):
 
 class BandcampAlbumIE(InfoExtractor):
     IE_NAME = 'Bandcamp:album'
-    _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<album_id>[^?#]+)|/?(?:$|[?#]))'
+    _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<album_id>[^/?#&]+))?'
 
     _TESTS = [{
         'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1',
@@ -225,7 +226,9 @@ class BandcampAlbumIE(InfoExtractor):
 
     @classmethod
     def suitable(cls, url):
-        return False if BandcampWeeklyIE.suitable(url) else super(BandcampAlbumIE, cls).suitable(url)
+        return (False
+                if BandcampWeeklyIE.suitable(url) or BandcampIE.suitable(url)
+                else super(BandcampAlbumIE, cls).suitable(url))
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -258,16 +261,22 @@ class BandcampAlbumIE(InfoExtractor):
 
 
 class BandcampWeeklyIE(InfoExtractor):
-    IE_NAME = 'Bandcamp:bandcamp_weekly'
-    _VALID_URL = r'https?://(?:www\.)?bandcamp\.com/?\?(?:.*&)?show=(?P<id>\d+)(?:$|[&#])'
+    IE_NAME = 'Bandcamp:weekly'
+    _VALID_URL = r'https?://(?:www\.)?bandcamp\.com/?\?(?:.*?&)?show=(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://bandcamp.com/?show=224',
         'md5': 'b00df799c733cf7e0c567ed187dea0fd',
         'info_dict': {
             'id': '224',
             'ext': 'opus',
-            'title': 'BC Weekly April 4th 2017: Magic Moments',
-            'description': 'Stones Throw\'s Vex Ruffin, plus up and coming singer Salami Rose Joe Louis, in conversation about their fantastic DIY albums.',
+            'title': 'BC Weekly April 4th 2017 - Magic Moments',
+            'description': 'md5:5d48150916e8e02d030623a48512c874',
+            'duration': 5829.77,
+            'release_date': '20170404',
+            'series': 'Bandcamp Weekly',
+            'episode': 'Magic Moments',
+            'episode_number': 208,
+            'episode_id': '224',
         }
     }, {
         'url': 'https://bandcamp.com/?blah/blah@&show=228',
@@ -288,32 +297,53 @@ class BandcampWeeklyIE(InfoExtractor):
 
         # This is desired because any invalid show id redirects to `bandcamp.com`
         # which happens to expose the latest Bandcamp Weekly episode.
-        video_id = compat_str(show['show_id'])
+        show_id = int_or_none(show.get('show_id')) or int_or_none(video_id)
 
-        def to_format_dictionaries(audio_stream):
-            dictionaries = [{'format_id': kvp[0], 'url': kvp[1]} for kvp in audio_stream.items()]
-            known_extensions = ['mp3', 'opus']
-
-            for dictionary in dictionaries:
-                for ext in known_extensions:
-                    if ext in dictionary['format_id']:
-                        dictionary['ext'] = ext
-                        break
-
-            return dictionaries
-
-        formats = to_format_dictionaries(show['audio_stream'])
+        formats = []
+        for format_id, format_url in show['audio_stream'].items():
+            if not isinstance(format_url, compat_str):
+                continue
+            for known_ext in KNOWN_EXTENSIONS:
+                if known_ext in format_id:
+                    ext = known_ext
+                    break
+            else:
+                ext = None
+            formats.append({
+                'format_id': format_id,
+                'url': format_url,
+                'ext': ext,
+                'vcodec': 'none',
+            })
         self._sort_formats(formats)
 
+        title = show.get('audio_title') or 'Bandcamp Weekly'
+        subtitle = show.get('subtitle')
+        if subtitle:
+            title += ' - %s' % subtitle
+
+        episode_number = None
+        seq = blob.get('bcw_seq')
+
+        if seq and isinstance(seq, list):
+            try:
+                episode_number = next(
+                    int_or_none(e.get('episode_number'))
+                    for e in seq
+                    if isinstance(e, dict) and int_or_none(e.get('id')) == show_id)
+            except StopIteration:
+                pass
+
         return {
             'id': video_id,
-            'title': show['audio_title'] + ': ' + show['subtitle'],
-            'description': show.get('desc'),
+            'title': title,
+            'description': show.get('desc') or show.get('short_desc'),
             'duration': float_or_none(show.get('audio_duration')),
-            'webpage_url': 'https://bandcamp.com/?show=' + video_id,
             'is_live': False,
             'release_date': unified_strdate(show.get('published_date')),
             'series': 'Bandcamp Weekly',
+            'episode': show.get('subtitle'),
+            'episode_number': episode_number,
             'episode_id': compat_str(video_id),
             'formats': formats
         }

From 43e6579558290db833171272412a2d4f007e853d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Jun 2017 23:22:19 +0700
Subject: [PATCH 388/611] Credit @adamvoss for bandcamp:weekly (#12758)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index f80c34a08..f5a08e974 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -217,3 +217,4 @@ Marvin Ewald
 Frédéric Bournival
 Timendum
 gritstub
+Adam Voss

From e1b90cc3db81c96abdf9a532aa4816b571ad5e88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Jun 2017 00:35:41 +0700
Subject: [PATCH 389/611] Credit @mikf for beam:vod (#13032)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index f5a08e974..bd61240c0 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -218,3 +218,4 @@ Frédéric Bournival
 Timendum
 gritstub
 Adam Voss
+Mike Fährmann

From f8ba3fda4dedfe1e480b6b929bc374766c722234 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Jun 2017 00:38:44 +0700
Subject: [PATCH 390/611] Credit @jktjkt for dvtv formats (#13063)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index bd61240c0..e2bdebe5e 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -219,3 +219,4 @@ Timendum
 gritstub
 Adam Voss
 Mike Fährmann
+Jan Kundrát

From 130880ba48fe545c9464ac5854cbc88915f84b52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Jun 2017 00:43:38 +0700
Subject: [PATCH 391/611] [ChangeLog] Actualize

---
 ChangeLog | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 7d78c87e6..639693413 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,20 @@
+version <unreleased>
+
+Core
+* [YoutubeDL] Don't emit ANSI escape codes on Windows (#13270)
+
+Extractors
++ [bandcamp:weekly] Add support for bandcamp weekly (#12758)
+* [pornhub:playlist] Fix extraction (#13281)
+- [godtv] Remove extractor (#13175)
+* [safari] Fix typo (#13252)
+* [youtube] Improve chapters extraction (#13247)
+* [1tv] Lower preference for HTTP formats (#13246)
+* [francetv] Relax URL regular expression
+* [drbonanza] Fix extraction (#13231)
+* [packtpub] Fix authentication (#13240)
+
+
 version 2017.05.29
 
 Extractors

From 537191826f0c778ed3e2eaef4168a3aa358fb7e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Jun 2017 00:48:07 +0700
Subject: [PATCH 392/611] release 2017.06.05

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 +-
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 24a0d72c5..ce1e46085 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.29*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.29**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.06.05*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.06.05**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.05.29
+[debug] youtube-dl version 2017.06.05
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 639693413..c605c5bd6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.06.05
 
 Core
 * [YoutubeDL] Don't emit ANSI escape codes on Windows (#13270)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 8ddc800b4..fe022d158 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -87,6 +87,7 @@
  - **bambuser:channel**
  - **Bandcamp**
  - **Bandcamp:album**
+ - **Bandcamp:weekly**
  - **bangumi.bilibili.com**: BiliBili番剧
  - **bbc**: BBC
  - **bbc.co.uk**: BBC iPlayer
@@ -310,7 +311,6 @@
  - **Go**
  - **Go90**
  - **GodTube**
- - **GodTV**
  - **Golem**
  - **GoogleDrive**
  - **Goshgay**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d4ee43122..d3aa39977 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.05.29'
+__version__ = '2017.06.05'

From e1db730d86302f1f96f20d44510e8fb060366bff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 6 Jun 2017 00:13:57 +0700
Subject: [PATCH 393/611] [tvplayer] Fix extraction (closes #13291)

---
 youtube_dl/extractor/tvplayer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tvplayer.py b/youtube_dl/extractor/tvplayer.py
index ebde6053f..8f8686a65 100644
--- a/youtube_dl/extractor/tvplayer.py
+++ b/youtube_dl/extractor/tvplayer.py
@@ -48,7 +48,7 @@ class TVPlayerIE(InfoExtractor):
             'https://tvplayer.com/watch/context', display_id,
             'Downloading JSON context', query={
                 'resource': resource_id,
-                'nonce': token,
+                'gen': token,
             })
 
         validate = context['validate']

From 0a3924e74622175dc9798e01e60d6bd3b3b36018 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 6 Jun 2017 21:21:44 +0700
Subject: [PATCH 394/611] [rtlnl] Improve _VALID_URL (closes #13295)

---
 youtube_dl/extractor/rtlnl.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 54076de28..3e22998c6 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -15,7 +15,7 @@ class RtlNlIE(InfoExtractor):
         https?://(?:www\.)?
         (?:
             rtlxl\.nl/[^\#]*\#!/[^/]+/|
-            rtl\.nl/system/videoplayer/(?:[^/]+/)+(?:video_)?embed\.html\b.+?\buuid=
+            rtl\.nl/(?:system/videoplayer/(?:[^/]+/)+(?:video_)?embed\.html\b.+?\buuid=|video/)
         )
         (?P<id>[0-9a-f-]+)'''
 
@@ -70,6 +70,9 @@ class RtlNlIE(InfoExtractor):
     }, {
         'url': 'http://rtlxl.nl/?_ga=1.204735956.572365465.1466978370#!/rtl-nieuws-132237/3c487912-023b-49ac-903e-2c5d79f8410f',
         'only_matching': True,
+    }, {
+        'url': 'https://www.rtl.nl/video/c603c9c2-601d-4b5e-8175-64f1e942dc7d/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From eb703e538035b0ebdc076be0ba982aac97d9ef08 Mon Sep 17 00:00:00 2001
From: Luca Steeb <contact@luca-steeb.com>
Date: Mon, 5 Jun 2017 20:17:43 +0200
Subject: [PATCH 395/611] [streamango] Make title optional

---
 youtube_dl/extractor/streamango.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/streamango.py b/youtube_dl/extractor/streamango.py
index aa4fad162..cf03d5b09 100644
--- a/youtube_dl/extractor/streamango.py
+++ b/youtube_dl/extractor/streamango.py
@@ -21,6 +21,13 @@ class StreamangoIE(InfoExtractor):
             'ext': 'mp4',
             'title': '20170315_150006.mp4',
         }
+    }, {
+        'url': 'https://streamango.com/embed/foqebrpftarclpob/asdf_asd_2_mp4',
+        'info_dict': {
+            'id': 'foqebrpftarclpob',
+            'ext': 'mp4',
+            'title': 'foqebrpftarclpob',
+        }
     }, {
         'url': 'https://streamango.com/embed/clapasobsptpkdfe/20170315_150006_mp4',
         'only_matching': True,
@@ -31,7 +38,7 @@ class StreamangoIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        title = self._og_search_title(webpage)
+        title = self._og_search_title(webpage, default=video_id)
 
         formats = []
         for format_ in re.findall(r'({[^}]*\bsrc\s*:\s*[^}]*})', webpage):

From 1508da30c28f2047a6d9d1dcebd529fa6ea56f15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 7 Jun 2017 21:53:13 +0700
Subject: [PATCH 396/611] [streamango] Skip download for test (closes #13292)

---
 youtube_dl/extractor/streamango.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/streamango.py b/youtube_dl/extractor/streamango.py
index cf03d5b09..a9e34c027 100644
--- a/youtube_dl/extractor/streamango.py
+++ b/youtube_dl/extractor/streamango.py
@@ -22,12 +22,16 @@ class StreamangoIE(InfoExtractor):
             'title': '20170315_150006.mp4',
         }
     }, {
+        # no og:title
         'url': 'https://streamango.com/embed/foqebrpftarclpob/asdf_asd_2_mp4',
         'info_dict': {
             'id': 'foqebrpftarclpob',
             'ext': 'mp4',
             'title': 'foqebrpftarclpob',
-        }
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'https://streamango.com/embed/clapasobsptpkdfe/20170315_150006_mp4',
         'only_matching': True,

From a355b57f5893997567fe635f730a7e88d57cbca1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Jun 2017 21:52:19 +0700
Subject: [PATCH 397/611] [README.md] Clarify output template references
 (closes #13316)

---
 youtube_dl/options.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 3021a6f41..79e9fd12c 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -310,7 +310,7 @@ def parseOpts(overrideArguments=None):
         metavar='FILTER', dest='match_filter', default=None,
         help=(
             'Generic video filter. '
-            'Specify any key (see help for -o for a list of available keys) to '
+            'Specify any key (see the "OUTPUT TEMPLATE" for a list of available keys) to '
             'match if the key is present, '
             '!key to check if the key is not present, '
             'key > NUMBER (like "comment_count > 12", also works with '
@@ -618,7 +618,7 @@ def parseOpts(overrideArguments=None):
     verbosity.add_option(
         '-j', '--dump-json',
         action='store_true', dest='dumpjson', default=False,
-        help='Simulate, quiet but print JSON information. See --output for a description of available keys.')
+        help='Simulate, quiet but print JSON information. See the "OUTPUT TEMPLATE" for a description of available keys.')
     verbosity.add_option(
         '-J', '--dump-single-json',
         action='store_true', dest='dump_single_json', default=False,

From 76e6378358a618a20051e9f9fd38e43af3169683 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Jun 2017 22:02:42 +0700
Subject: [PATCH 398/611] [README.md] Improve man page formatting

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index fa901a46d..2189552a9 100644
--- a/README.md
+++ b/README.md
@@ -535,13 +535,14 @@ The basic usage is not to set any template arguments when downloading a single f
  - `playlist_id` (string): Playlist identifier
  - `playlist_title` (string): Playlist title
 
-
 Available for the video that belongs to some logical chapter or section:
+
  - `chapter` (string): Name or title of the chapter the video belongs to
  - `chapter_number` (numeric): Number of the chapter the video belongs to
  - `chapter_id` (string): Id of the chapter the video belongs to
 
 Available for the video that is an episode of some series or programme:
+
  - `series` (string): Title of the series or programme the video episode belongs to
  - `season` (string): Title of the season the video episode belongs to
  - `season_number` (numeric): Number of the season the video episode belongs to
@@ -551,6 +552,7 @@ Available for the video that is an episode of some series or programme:
  - `episode_id` (string): Id of the video episode
 
 Available for the media that is a track or a part of a music album:
+
  - `track` (string): Title of the track
  - `track_number` (numeric): Number of the track within an album or a disc
  - `track_id` (string): Id of the track

From c9969434183c562eb9935aa20f147f234aa61e53 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Jun 2017 22:53:14 +0700
Subject: [PATCH 399/611] [YoutubeDL] Sanitize more fields (#13313)

---
 youtube_dl/YoutubeDL.py | 52 +++++++++++++++++++++++++++++------------
 1 file changed, 37 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 7efa0c948..c05103bb6 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -58,6 +58,7 @@ from .utils import (
     format_bytes,
     formatSeconds,
     GeoRestrictedError,
+    int_or_none,
     ISO3166Utils,
     locked_file,
     make_HTTPS_handler,
@@ -302,6 +303,17 @@ class YoutubeDL(object):
                         postprocessor.
     """
 
+    _NUMERIC_FIELDS = set((
+        'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
+        'timestamp', 'upload_year', 'upload_month', 'upload_day',
+        'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
+        'average_rating', 'comment_count', 'age_limit',
+        'start_time', 'end_time',
+        'chapter_number', 'season_number', 'episode_number',
+        'track_number', 'disc_number', 'release_year',
+        'playlist_index',
+    ))
+
     params = None
     _ies = []
     _pps = []
@@ -639,22 +651,11 @@ class YoutubeDL(object):
                     r'%%(\1)0%dd' % field_size_compat_map[mobj.group('field')],
                     outtmpl)
 
-            NUMERIC_FIELDS = set((
-                'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
-                'timestamp', 'upload_year', 'upload_month', 'upload_day',
-                'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
-                'average_rating', 'comment_count', 'age_limit',
-                'start_time', 'end_time',
-                'chapter_number', 'season_number', 'episode_number',
-                'track_number', 'disc_number', 'release_year',
-                'playlist_index',
-            ))
-
             # Missing numeric fields used together with integer presentation types
             # in format specification will break the argument substitution since
             # string 'NA' is returned for missing fields. We will patch output
             # template for missing fields to meet string presentation type.
-            for numeric_field in NUMERIC_FIELDS:
+            for numeric_field in self._NUMERIC_FIELDS:
                 if numeric_field not in template_dict:
                     # As of [1] format syntax is:
                     #  %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type
@@ -1345,9 +1346,28 @@ class YoutubeDL(object):
         if 'title' not in info_dict:
             raise ExtractorError('Missing "title" field in extractor result')
 
-        if not isinstance(info_dict['id'], compat_str):
-            self.report_warning('"id" field is not a string - forcing string conversion')
-            info_dict['id'] = compat_str(info_dict['id'])
+        def report_force_conversion(field, field_not, conversion):
+            self.report_warning(
+                '"%s" field is not %s - forcing %s conversion, there is an error in extractor'
+                % (field, field_not, conversion))
+
+        def sanitize_string_field(info, string_field):
+            field = info.get(string_field)
+            if field is None or isinstance(field, compat_str):
+                return
+            report_force_conversion(string_field, 'a string', 'string')
+            info[string_field] = compat_str(field)
+
+        def sanitize_numeric_fields(info):
+            for numeric_field in self._NUMERIC_FIELDS:
+                field = info.get(numeric_field)
+                if field is None or isinstance(field, compat_numeric_types):
+                    continue
+                report_force_conversion(numeric_field, 'numeric', 'int')
+                info[numeric_field] = int_or_none(field)
+
+        sanitize_string_field(info_dict, 'id')
+        sanitize_numeric_fields(info_dict)
 
         if 'playlist' not in info_dict:
             # It isn't part of a playlist
@@ -1435,6 +1455,8 @@ class YoutubeDL(object):
             if 'url' not in format:
                 raise ExtractorError('Missing "url" key in result (index %d)' % i)
 
+            sanitize_string_field(format, 'format_id')
+            sanitize_numeric_fields(format)
             format['url'] = sanitize_url(format['url'])
 
             if format.get('format_id') is None:

From 931adf8cc18e1cc2199d0d4ba5d852f8119ebdb7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Jun 2017 22:54:30 +0700
Subject: [PATCH 400/611] [liveleak] Ensure height is int (closes #13313)

---
 youtube_dl/extractor/liveleak.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index c54519636..b2247a84d 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -115,8 +115,9 @@ class LiveLeakIE(InfoExtractor):
 
         for a_format in info_dict['formats']:
             if not a_format.get('height'):
-                a_format['height'] = self._search_regex(
-                    r'([0-9]+)p\.mp4', a_format['url'], 'height label', default=None)
+                a_format['height'] = int_or_none(self._search_regex(
+                    r'([0-9]+)p\.mp4', a_format['url'], 'height label',
+                    default=None))
 
         self._sort_formats(info_dict['formats'])
 

From 4244a13a1d7420d8e8a2a51bc4e7d2fa17e9f844 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Jun 2017 23:20:48 +0700
Subject: [PATCH 401/611] [safari] Improve authentication detection (closes
 #13319)

---
 youtube_dl/extractor/safari.py | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 794e4ee6f..909a6ba97 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -16,7 +16,6 @@ from ..utils import (
 
 class SafariBaseIE(InfoExtractor):
     _LOGIN_URL = 'https://www.safaribooksonline.com/accounts/login/'
-    _SUCCESSFUL_LOGIN_REGEX = r'<a href="/accounts/logout/"[^>]*>Sign Out</a>'
     _NETRC_MACHINE = 'safari'
 
     _API_BASE = 'https://www.safaribooksonline.com/api/v1'
@@ -28,10 +27,6 @@ class SafariBaseIE(InfoExtractor):
         self._login()
 
     def _login(self):
-        # We only need to log in once for courses or individual videos
-        if self.LOGGED_IN:
-            return
-
         (username, password) = self._get_login_info()
         if username is None:
             return
@@ -39,11 +34,17 @@ class SafariBaseIE(InfoExtractor):
         headers = std_headers.copy()
         if 'Referer' not in headers:
             headers['Referer'] = self._LOGIN_URL
-        login_page_request = sanitized_Request(self._LOGIN_URL, headers=headers)
 
         login_page = self._download_webpage(
-            login_page_request, None,
-            'Downloading login form')
+            self._LOGIN_URL, None, 'Downloading login form', headers=headers)
+
+        def is_logged(webpage):
+            return any(re.search(p, webpage) for p in (
+                r'href=["\']/accounts/logout/', r'>Sign Out<'))
+
+        if is_logged(login_page):
+            self.LOGGED_IN = True
+            return
 
         csrf = self._html_search_regex(
             r"name='csrfmiddlewaretoken'\s+value='([^']+)'",
@@ -62,15 +63,13 @@ class SafariBaseIE(InfoExtractor):
         login_page = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 
-        if re.search(self._SUCCESSFUL_LOGIN_REGEX, login_page) is None:
+        if not is_logged(login_page):
             raise ExtractorError(
                 'Login failed; make sure your credentials are correct and try again.',
                 expected=True)
 
         self.LOGGED_IN = True
 
-        self.to_screen('Login successful')
-
 
 class SafariIE(SafariBaseIE):
     IE_NAME = 'safari'

From 1693bebe4d3e605001b1df8b6e1f238c2e3ff897 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Jun 2017 00:16:42 +0700
Subject: [PATCH 402/611] [sohu] Fix numeric fields

---
 youtube_dl/extractor/sohu.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 7da12cef8..a62ed84f1 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -8,7 +8,11 @@ from ..compat import (
     compat_str,
     compat_urllib_parse_urlencode,
 )
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    try_get,
+)
 
 
 class SohuIE(InfoExtractor):
@@ -169,10 +173,11 @@ class SohuIE(InfoExtractor):
                 formats.append({
                     'url': video_url,
                     'format_id': format_id,
-                    'filesize': data['clipsBytes'][i],
-                    'width': data['width'],
-                    'height': data['height'],
-                    'fps': data['fps'],
+                    'filesize': int_or_none(
+                        try_get(data, lambda x: x['clipsBytes'][i])),
+                    'width': int_or_none(data.get('width')),
+                    'height': int_or_none(data.get('height')),
+                    'fps': int_or_none(data.get('fps')),
                 })
             self._sort_formats(formats)
 

From 15aeeb11882c9597614ce6e249968bfdf7830e8a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Jun 2017 00:20:07 +0700
Subject: [PATCH 403/611] [flickr] Ensure format id is string

---
 youtube_dl/extractor/flickr.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index a8e1bf42a..9f166efd4 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -1,7 +1,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
+from ..compat import (
+    compat_str,
+    compat_urllib_parse_urlencode,
+)
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -81,7 +84,7 @@ class FlickrIE(InfoExtractor):
 
             formats = []
             for stream in streams['stream']:
-                stream_type = str(stream.get('type'))
+                stream_type = compat_str(stream.get('type'))
                 formats.append({
                     'format_id': stream_type,
                     'url': stream['_content'],

From cc69a3de1b0980d34e21719fd4c264575d620bb4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Jun 2017 00:22:14 +0700
Subject: [PATCH 404/611] [foxgay] Ensure height is int

---
 youtube_dl/extractor/foxgay.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/foxgay.py b/youtube_dl/extractor/foxgay.py
index e887ae488..512a10645 100644
--- a/youtube_dl/extractor/foxgay.py
+++ b/youtube_dl/extractor/foxgay.py
@@ -5,6 +5,7 @@ import itertools
 from .common import InfoExtractor
 from ..utils import (
     get_element_by_id,
+    int_or_none,
     remove_end,
 )
 
@@ -46,7 +47,7 @@ class FoxgayIE(InfoExtractor):
 
         formats = [{
             'url': source,
-            'height': resolution,
+            'height': int_or_none(resolution),
         } for source, resolution in zip(
             video_data['sources'], video_data.get('resolutions', itertools.repeat(None)))]
 

From 87f5646937f52643a32f80210adf2647e3bd5d25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Jun 2017 00:24:23 +0700
Subject: [PATCH 405/611] [gfycat] Ensure filesize is int

---
 youtube_dl/extractor/gfycat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gfycat.py b/youtube_dl/extractor/gfycat.py
index 884700c52..45ccc11c1 100644
--- a/youtube_dl/extractor/gfycat.py
+++ b/youtube_dl/extractor/gfycat.py
@@ -82,7 +82,7 @@ class GfycatIE(InfoExtractor):
             video_url = gfy.get('%sUrl' % format_id)
             if not video_url:
                 continue
-            filesize = gfy.get('%sSize' % format_id)
+            filesize = int_or_none(gfy.get('%sSize' % format_id))
             formats.append({
                 'url': video_url,
                 'format_id': format_id,

From e4d2e76d8ed6d125f671987b855c6d3eef1b38f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Jun 2017 00:27:11 +0700
Subject: [PATCH 406/611] [golem] Ensure format id is string

---
 youtube_dl/extractor/golem.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/golem.py b/youtube_dl/extractor/golem.py
index 2bfb99040..47a068e74 100644
--- a/youtube_dl/extractor/golem.py
+++ b/youtube_dl/extractor/golem.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_str,
     compat_urlparse,
 )
 from ..utils import (
@@ -46,7 +47,7 @@ class GolemIE(InfoExtractor):
                 continue
 
             formats.append({
-                'format_id': e.tag,
+                'format_id': compat_str(e.tag),
                 'url': compat_urlparse.urljoin(self._PREFIX, url),
                 'height': self._int(e.get('height'), 'height'),
                 'width': self._int(e.get('width'), 'width'),

From 34646967ba8a09c4868f08f1903425b6d721c33d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Jun 2017 00:29:20 +0700
Subject: [PATCH 407/611] [jove] Ensure comment count is int

---
 youtube_dl/extractor/jove.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/jove.py b/youtube_dl/extractor/jove.py
index f9a034b78..27e0e37f6 100644
--- a/youtube_dl/extractor/jove.py
+++ b/youtube_dl/extractor/jove.py
@@ -65,9 +65,9 @@ class JoveIE(InfoExtractor):
             webpage, 'description', fatal=False)
         publish_date = unified_strdate(self._html_search_meta(
             'citation_publication_date', webpage, 'publish date', fatal=False))
-        comment_count = self._html_search_regex(
+        comment_count = int(self._html_search_regex(
             r'<meta name="num_comments" content="(\d+) Comments?"',
-            webpage, 'comment count', fatal=False)
+            webpage, 'comment count', fatal=False))
 
         return {
             'id': video_id,

From f5521ea209d1b20ec0bc6eeeb16e99101d05c854 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Jun 2017 00:30:23 +0700
Subject: [PATCH 408/611] [sexu] Ensure height is int

---
 youtube_dl/extractor/sexu.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sexu.py b/youtube_dl/extractor/sexu.py
index 5e22ea730..3df51520b 100644
--- a/youtube_dl/extractor/sexu.py
+++ b/youtube_dl/extractor/sexu.py
@@ -32,8 +32,9 @@ class SexuIE(InfoExtractor):
         formats = [{
             'url': source['file'].replace('\\', ''),
             'format_id': source.get('label'),
-            'height': self._search_regex(
-                r'^(\d+)[pP]', source.get('label', ''), 'height', default=None),
+            'height': int(self._search_regex(
+                r'^(\d+)[pP]', source.get('label', ''), 'height',
+                default=None)),
         } for source in sources if source.get('file')]
         self._sort_formats(formats)
 

From 751583042299138ef4deef5862e6fe2ada078417 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Jun 2017 00:31:47 +0700
Subject: [PATCH 409/611] [turbo] Ensure format id is string

---
 youtube_dl/extractor/turbo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/turbo.py b/youtube_dl/extractor/turbo.py
index 25aa9c58e..be3eaa5c2 100644
--- a/youtube_dl/extractor/turbo.py
+++ b/youtube_dl/extractor/turbo.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -49,7 +50,7 @@ class TurboIE(InfoExtractor):
         for child in item:
             m = re.search(r'url_video_(?P<quality>.+)', child.tag)
             if m:
-                quality = m.group('quality')
+                quality = compat_str(m.group('quality'))
                 formats.append({
                     'format_id': quality,
                     'url': child.text,

From 1afd0b0da70b6ed709c610aff98786d71511a629 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Jun 2017 00:40:03 +0700
Subject: [PATCH 410/611] [extractor/common] Return unicode string from
 _match_id

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index fec39da8b..f027447c8 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -376,7 +376,7 @@ class InfoExtractor(object):
             cls._VALID_URL_RE = re.compile(cls._VALID_URL)
         m = cls._VALID_URL_RE.match(url)
         assert m
-        return m.group('id')
+        return compat_str(m.group('id'))
 
     @classmethod
     def working(cls):

From 4e3637034cf2b3630e6f9c6d34aa0177a8c83950 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Jun 2017 23:55:05 +0700
Subject: [PATCH 411/611] [extractor/generic] Ensure format id is unicode
 string

---
 youtube_dl/extractor/generic.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c108d4a8a..8ef1a2980 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -10,6 +10,7 @@ from .common import InfoExtractor
 from .youtube import YoutubeIE
 from ..compat import (
     compat_etree_fromstring,
+    compat_str,
     compat_urllib_parse_unquote,
     compat_urlparse,
     compat_xml_parse_error,
@@ -1907,14 +1908,14 @@ class GenericIE(InfoExtractor):
         content_type = head_response.headers.get('Content-Type', '').lower()
         m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>[^;\s]+)', content_type)
         if m:
-            format_id = m.group('format_id')
+            format_id = compat_str(m.group('format_id'))
             if format_id.endswith('mpegurl'):
                 formats = self._extract_m3u8_formats(url, video_id, 'mp4')
             elif format_id == 'f4m':
                 formats = self._extract_f4m_formats(url, video_id)
             else:
                 formats = [{
-                    'format_id': m.group('format_id'),
+                    'format_id': format_id,
                     'url': url,
                     'vcodec': 'none' if m.group('type') == 'audio' else None
                 }]

From 7409af9eb322b0e76a4ae2d713020c45d178fabe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jun 2017 08:56:53 +0700
Subject: [PATCH 412/611] [msn] Fix formats extraction

---
 youtube_dl/extractor/msn.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/msn.py b/youtube_dl/extractor/msn.py
index 1473bcf48..650731fdc 100644
--- a/youtube_dl/extractor/msn.py
+++ b/youtube_dl/extractor/msn.py
@@ -68,10 +68,6 @@ class MSNIE(InfoExtractor):
             format_url = file_.get('url')
             if not format_url:
                 continue
-            ext = determine_ext(format_url)
-            if ext == 'ism':
-                formats.extend(self._extract_ism_formats(
-                    format_url + '/Manifest', display_id, 'mss', fatal=False))
             if 'm3u8' in format_url:
                 # m3u8_native should not be used here until
                 # https://github.com/rg3/youtube-dl/issues/9913 is fixed
@@ -79,6 +75,9 @@ class MSNIE(InfoExtractor):
                     format_url, display_id, 'mp4',
                     m3u8_id='hls', fatal=False)
                 formats.extend(m3u8_formats)
+            elif determine_ext(format_url) == 'ism':
+                formats.extend(self._extract_ism_formats(
+                    format_url + '/Manifest', display_id, 'mss', fatal=False))
             else:
                 formats.append({
                     'url': format_url,

From 6e999fbc120dfdbd8ec5ca056105420fd7cbffe8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jun 2017 19:44:44 +0700
Subject: [PATCH 413/611] [newgrounds] Improve formats and uploader extraction
 (closes #13346)

---
 youtube_dl/extractor/newgrounds.py | 48 ++++++++++++++++++++++++++----
 1 file changed, 43 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py
index 9bea610c8..ae4fe95ef 100644
--- a/youtube_dl/extractor/newgrounds.py
+++ b/youtube_dl/extractor/newgrounds.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import int_or_none
 
 
 class NewgroundsIE(InfoExtractor):
@@ -23,24 +24,61 @@ class NewgroundsIE(InfoExtractor):
             'title': 'Dancin',
             'uploader': 'Squirrelman82',
         },
+    }, {
+        # source format unavailable, additional mp4 formats
+        'url': 'http://www.newgrounds.com/portal/view/689400',
+        'info_dict': {
+            'id': '689400',
+            'ext': 'mp4',
+            'title': 'ZTV News Episode 8',
+            'uploader': 'BennettTheSage',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
         media_id = self._match_id(url)
+
         webpage = self._download_webpage(url, media_id)
 
         title = self._html_search_regex(
             r'<title>([^>]+)</title>', webpage, 'title')
 
-        uploader = self._html_search_regex(
-            r'Author\s*<a[^>]+>([^<]+)', webpage, 'uploader', fatal=False)
+        video_url = self._parse_json(self._search_regex(
+            r'"url"\s*:\s*("[^"]+"),', webpage, ''), media_id)
 
-        music_url = self._parse_json(self._search_regex(
-            r'"url":("[^"]+"),', webpage, ''), media_id)
+        formats = [{
+            'url': video_url,
+            'format_id': 'source',
+            'quality': 1,
+        }]
+
+        max_resolution = int_or_none(self._search_regex(
+            r'max_resolution["\']\s*:\s*(\d+)', webpage, 'max resolution',
+            default=None))
+        if max_resolution:
+            url_base = video_url.rpartition('.')[0]
+            for resolution in (360, 720, 1080):
+                if resolution > max_resolution:
+                    break
+                formats.append({
+                    'url': '%s.%dp.mp4' % (url_base, resolution),
+                    'format_id': '%dp' % resolution,
+                    'height': resolution,
+                })
+
+        self._check_formats(formats, media_id)
+        self._sort_formats(formats)
+
+        uploader = self._html_search_regex(
+            r'(?:Author|Writer)\s*<a[^>]+>([^<]+)', webpage, 'uploader',
+            fatal=False)
 
         return {
             'id': media_id,
             'title': title,
-            'url': music_url,
             'uploader': uploader,
+            'formats': formats,
         }

From 70e7967202efa0acb6202ceedf87049ba72b94e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jun 2017 20:50:33 +0700
Subject: [PATCH 414/611] [newgrounds:playlist] Add extractor (closes #10611)

---
 youtube_dl/extractor/extractors.py |  5 ++-
 youtube_dl/extractor/newgrounds.py | 56 +++++++++++++++++++++++++++++-
 2 files changed, 59 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e1907314d..7e45232dd 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -636,7 +636,10 @@ from .neteasemusic import (
     NetEaseMusicProgramIE,
     NetEaseMusicDjRadioIE,
 )
-from .newgrounds import NewgroundsIE
+from .newgrounds import (
+    NewgroundsIE,
+    NewgroundsPlaylistIE,
+)
 from .newstube import NewstubeIE
 from .nextmedia import (
     NextMediaIE,
diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py
index ae4fe95ef..dc183caec 100644
--- a/youtube_dl/extractor/newgrounds.py
+++ b/youtube_dl/extractor/newgrounds.py
@@ -1,7 +1,12 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    extract_attributes,
+    int_or_none,
+)
 
 
 class NewgroundsIE(InfoExtractor):
@@ -82,3 +87,52 @@ class NewgroundsIE(InfoExtractor):
             'uploader': uploader,
             'formats': formats,
         }
+
+
+class NewgroundsPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?newgrounds\.com/(?:collection|[^/]+/search/[^/]+)/(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://www.newgrounds.com/collection/cats',
+        'info_dict': {
+            'id': 'cats',
+            'title': 'Cats',
+        },
+        'playlist_mincount': 46,
+    }, {
+        'url': 'http://www.newgrounds.com/portal/search/author/ZONE-SAMA',
+        'info_dict': {
+            'id': 'ZONE-SAMA',
+            'title': 'Portal Search: ZONE-SAMA',
+        },
+        'playlist_mincount': 47,
+    }, {
+        'url': 'http://www.newgrounds.com/audio/search/title/cats',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        title = self._search_regex(
+            r'<title>([^>]+)</title>', webpage, 'title', default=None)
+
+        # cut left menu
+        webpage = self._search_regex(
+            r'(?s)<div[^>]+\bclass=["\']column wide(.+)',
+            webpage, 'wide column', default=webpage)
+
+        entries = []
+        for a, path, media_id in re.findall(
+                r'(<a[^>]+\bhref=["\']/?((?:portal/view|audio/listen)/(\d+))[^>]+>)',
+                webpage):
+            a_class = extract_attributes(a).get('class')
+            if a_class not in ('item-portalsubmission', 'item-audiosubmission'):
+                continue
+            entries.append(
+                self.url_result(
+                    'https://www.newgrounds.com/%s' % path,
+                    ie=NewgroundsIE.ie_key(), video_id=media_id))
+
+        return self.playlist_result(entries, playlist_id, title)

From 2ae2ffda5eae9c64d40d2fec839ba5deb07717f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jun 2017 21:27:22 +0700
Subject: [PATCH 415/611] [utils] Improve unified_timestamp

---
 test/test_utils.py  | 1 +
 youtube_dl/utils.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index d7e05817c..41b094d89 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -340,6 +340,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unified_timestamp('May 16, 2016 11:15 PM'), 1463440500)
         self.assertEqual(unified_timestamp('Feb 7, 2016 at 6:35 pm'), 1454870100)
         self.assertEqual(unified_timestamp('2017-03-30T17:52:41Q'), 1490896361)
+        self.assertEqual(unified_timestamp('Sep 11, 2013 | 5:49 AM'), 1378878540)
 
     def test_determine_ext(self):
         self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d79410e58..16bf49408 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1179,7 +1179,7 @@ def unified_timestamp(date_str, day_first=True):
     if date_str is None:
         return None
 
-    date_str = date_str.replace(',', ' ')
+    date_str = re.sub(r'[,|]', '', date_str)
 
     pm_delta = 12 if re.search(r'(?i)PM', date_str) else 0
     timezone, date_str = extract_timezone(date_str)

From 28a4d6cce89d159f9fe701e6cf716c7e3ffa4415 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jun 2017 21:27:32 +0700
Subject: [PATCH 416/611] [newgrounds] Extract more metadata (closes #13232)

---
 youtube_dl/extractor/newgrounds.py | 38 ++++++++++++++++++++++++++----
 1 file changed, 33 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py
index dc183caec..0e26f8399 100644
--- a/youtube_dl/extractor/newgrounds.py
+++ b/youtube_dl/extractor/newgrounds.py
@@ -6,6 +6,9 @@ from .common import InfoExtractor
 from ..utils import (
     extract_attributes,
     int_or_none,
+    parse_duration,
+    parse_filesize,
+    unified_timestamp,
 )
 
 
@@ -19,7 +22,10 @@ class NewgroundsIE(InfoExtractor):
             'ext': 'mp3',
             'title': 'B7 - BusMode',
             'uploader': 'Burn7',
-        }
+            'timestamp': 1378878540,
+            'upload_date': '20130911',
+            'duration': 143,
+        },
     }, {
         'url': 'https://www.newgrounds.com/portal/view/673111',
         'md5': '3394735822aab2478c31b1004fe5e5bc',
@@ -28,6 +34,8 @@ class NewgroundsIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Dancin',
             'uploader': 'Squirrelman82',
+            'timestamp': 1460256780,
+            'upload_date': '20160410',
         },
     }, {
         # source format unavailable, additional mp4 formats
@@ -37,6 +45,8 @@ class NewgroundsIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'ZTV News Episode 8',
             'uploader': 'BennettTheSage',
+            'timestamp': 1487965140,
+            'upload_date': '20170224',
         },
         'params': {
             'skip_download': True,
@@ -51,11 +61,11 @@ class NewgroundsIE(InfoExtractor):
         title = self._html_search_regex(
             r'<title>([^>]+)</title>', webpage, 'title')
 
-        video_url = self._parse_json(self._search_regex(
+        media_url = self._parse_json(self._search_regex(
             r'"url"\s*:\s*("[^"]+"),', webpage, ''), media_id)
 
         formats = [{
-            'url': video_url,
+            'url': media_url,
             'format_id': 'source',
             'quality': 1,
         }]
@@ -64,7 +74,7 @@ class NewgroundsIE(InfoExtractor):
             r'max_resolution["\']\s*:\s*(\d+)', webpage, 'max resolution',
             default=None))
         if max_resolution:
-            url_base = video_url.rpartition('.')[0]
+            url_base = media_url.rpartition('.')[0]
             for resolution in (360, 720, 1080):
                 if resolution > max_resolution:
                     break
@@ -77,14 +87,32 @@ class NewgroundsIE(InfoExtractor):
         self._check_formats(formats, media_id)
         self._sort_formats(formats)
 
-        uploader = self._html_search_regex(
+        uploader = self._search_regex(
             r'(?:Author|Writer)\s*<a[^>]+>([^<]+)', webpage, 'uploader',
             fatal=False)
 
+        timestamp = unified_timestamp(self._search_regex(
+            r'<dt>Uploaded</dt>\s*<dd>([^<]+)', webpage, 'timestamp',
+            default=None))
+        duration = parse_duration(self._search_regex(
+            r'<dd>Song\s*</dd><dd>.+?</dd><dd>([^<]+)', webpage, 'duration',
+            default=None))
+
+        filesize_approx = parse_filesize(self._html_search_regex(
+            r'<dd>Song\s*</dd><dd>(.+?)</dd>', webpage, 'filesize',
+            default=None))
+        if len(formats) == 1:
+            formats[0]['filesize_approx'] = filesize_approx
+
+        if '<dd>Song' in webpage:
+            formats[0]['vcodec'] = 'none'
+
         return {
             'id': media_id,
             'title': title,
             'uploader': uploader,
+            'timestamp': timestamp,
+            'duration': duration,
             'formats': formats,
         }
 

From 624bd0104c3c0f367ecb129c438da57398668801 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jun 2017 21:36:19 +0700
Subject: [PATCH 417/611] [rutv] Add support for testplayer.vgtrk.com (closes
 #13347)

---
 youtube_dl/extractor/rutv.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index a5e672c0a..d2713c19a 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -13,11 +13,15 @@ from ..utils import (
 class RUTVIE(InfoExtractor):
     IE_DESC = 'RUTV.RU'
     _VALID_URL = r'''(?x)
-        https?://player\.(?:rutv\.ru|vgtrk\.com)/
-            (?P<path>flash\d+v/container\.swf\?id=
-            |iframe/(?P<type>swf|video|live)/id/
-            |index/iframe/cast_id/)
-            (?P<id>\d+)'''
+                    https?://
+                        (?:test)?player\.(?:rutv\.ru|vgtrk\.com)/
+                        (?P<path>
+                            flash\d+v/container\.swf\?id=|
+                            iframe/(?P<type>swf|video|live)/id/|
+                            index/iframe/cast_id/
+                        )
+                        (?P<id>\d+)
+                    '''
 
     _TESTS = [
         {
@@ -99,17 +103,21 @@ class RUTVIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            'url': 'https://testplayer.vgtrk.com/iframe/live/id/19201/showZoomBtn/false/isPlay/true/',
+            'only_matching': True,
+        },
     ]
 
     @classmethod
     def _extract_url(cls, webpage):
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.(?:rutv\.ru|vgtrk\.com)/(?:iframe/(?:swf|video|live)/id|index/iframe/cast_id)/.+?)\1', webpage)
+            r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:test)?player\.(?:rutv\.ru|vgtrk\.com)/(?:iframe/(?:swf|video|live)/id|index/iframe/cast_id)/.+?)\1', webpage)
         if mobj:
             return mobj.group('url')
 
         mobj = re.search(
-            r'<meta[^>]+?property=(["\'])og:video\1[^>]+?content=(["\'])(?P<url>https?://player\.(?:rutv\.ru|vgtrk\.com)/flash\d+v/container\.swf\?id=.+?\2)',
+            r'<meta[^>]+?property=(["\'])og:video\1[^>]+?content=(["\'])(?P<url>https?://(?:test)?player\.(?:rutv\.ru|vgtrk\.com)/flash\d+v/container\.swf\?id=.+?\2)',
             webpage)
         if mobj:
             return mobj.group('url')

From 16bc958287b25566dbe22d22274d4c992278d2f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Jun 2017 00:14:04 +0700
Subject: [PATCH 418/611] [xfileshare] Modernize and pass referrer

---
 youtube_dl/extractor/xfileshare.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 13f8be6cb..bfec17e3c 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -130,12 +130,12 @@ class XFileShareIE(InfoExtractor):
             if countdown:
                 self._sleep(countdown, video_id)
 
-            post = urlencode_postdata(fields)
-
-            req = sanitized_Request(url, post)
-            req.add_header('Content-type', 'application/x-www-form-urlencoded')
-
-            webpage = self._download_webpage(req, video_id, 'Downloading video page')
+            webpage = self._download_webpage(
+                url, video_id, 'Downloading video page',
+                data=urlencode_postdata(fields), headers={
+                    'Referer': url,
+                    'Content-type': 'application/x-www-form-urlencoded',
+                })
 
         title = (self._search_regex(
             (r'style="z-index: [0-9]+;">([^<]+)</span>',

From 534863e057b155cd71035f05600444c2be62ca95 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Jun 2017 00:16:47 +0700
Subject: [PATCH 419/611] [xfileshare] Add support for rapidvideo (closes
 #13348)

---
 youtube_dl/extractor/xfileshare.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index bfec17e3c..32e16bc73 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -30,6 +30,7 @@ class XFileShareIE(InfoExtractor):
         (r'vidabc\.com', 'Vid ABC'),
         (r'vidbom\.com', 'VidBom'),
         (r'vidlo\.us', 'vidlo'),
+        (r'rapidvideo\.(?:cool|org)', 'RapidVideo.TV'),
     )
 
     IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1])
@@ -109,6 +110,9 @@ class XFileShareIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.rapidvideo.cool/b667kprndr8w',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 72b409559c984bb116015ada55acd38120b24cc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Jun 2017 01:50:32 +0700
Subject: [PATCH 420/611] [compat] Introduce compat_HTMLParseError

---
 youtube_dl/compat.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 39527117f..bbc499eda 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2322,6 +2322,19 @@ try:
 except ImportError:  # Python 2
     from HTMLParser import HTMLParser as compat_HTMLParser
 
+try:  # Python 2
+    from HTMLParser import HTMLParseError as compat_HTMLParseError
+except ImportError:  # Python <3.4
+    try:
+        from html.parser import HTMLParseError as compat_HTMLParseError
+    except ImportError:  # Python >3.4
+
+        # HTMLParseError has been deprecated in Python 3.3 and removed in
+        # Python 3.5. Introducing dummy exception for Python >3.5 for compatible
+        # and uniform cross-version exceptiong handling
+        class compat_HTMLParseError(Exception):
+            pass
+
 try:
     from subprocess import DEVNULL
     compat_subprocess_get_DEVNULL = lambda: DEVNULL

From b4a3d461e4a00dfc60047b667aa3136c8b03eda8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Jun 2017 01:52:24 +0700
Subject: [PATCH 421/611] [utils] Handle HTMLParseError in extract_attributes
 (closes #13349)

---
 test/test_utils.py  | 2 ++
 youtube_dl/utils.py | 9 +++++++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 41b094d89..2b93b3604 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -916,6 +916,8 @@ class TestUtil(unittest.TestCase):
             supports_outside_bmp = False
         if supports_outside_bmp:
             self.assertEqual(extract_attributes('<e x="Smile &#128512;!">'), {'x': 'Smile \U0001f600!'})
+        # Malformed HTML should not break attributes extraction on older Python
+        self.assertEqual(extract_attributes('<mal"formed/>'), {})
 
     def test_clean_html(self):
         self.assertEqual(clean_html('a:\nb'), 'a: b')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 16bf49408..1973bd483 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -36,6 +36,7 @@ import xml.etree.ElementTree
 import zlib
 
 from .compat import (
+    compat_HTMLParseError,
     compat_HTMLParser,
     compat_basestring,
     compat_chr,
@@ -409,8 +410,12 @@ def extract_attributes(html_element):
     but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
     """
     parser = HTMLAttributeParser()
-    parser.feed(html_element)
-    parser.close()
+    try:
+        parser.feed(html_element)
+        parser.close()
+    # Older Python may throw HTMLParseError in case of malformed HTML
+    except compat_HTMLParseError:
+        pass
     return parser.attrs
 
 

From bf87c36c93aee81a1bf26c8213262724955d52a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Jun 2017 02:01:12 +0700
Subject: [PATCH 422/611] [xfileshare] PEP 8

---
 youtube_dl/extractor/xfileshare.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 32e16bc73..a32789ef4 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -10,7 +10,6 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     NO_DEFAULT,
-    sanitized_Request,
     urlencode_postdata,
 )
 

From 9932ac5c58080e52b3111f5f8dec15c65c4079ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Jun 2017 02:01:15 +0700
Subject: [PATCH 423/611] [ChangeLog] Actualize

---
 ChangeLog | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index c605c5bd6..19f160c81 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,36 @@
+version <unreleased>
+
+Core
+* [utils] Handle compat_HTMLParseError in extract_attributes (#13349)
++ [compat] Introduce compat_HTMLParseError
+* [utils] Improve unified_timestamp
+* [extractor/generic] Ensure format id is unicode string
+* [extractor/common] Return unicode string from _match_id
++ [YoutubeDL] Sanitize more fields (#13313)
+
+Extractors
++ [xfileshare] Add support for rapidvideo.tv (#13348)
+* [xfileshare] Modernize and pass Referer
++ [rutv] Add support for testplayer.vgtrk.com (#13347)
++ [newgrounds] Extract more metadata (#13232)
++ [newgrounds:playlist] Add support for playlists (#10611)
+* [newgrounds] Improve formats and uploader extraction (#13346)
+* [msn] Fix formats extraction
+* [turbo] Ensure format id is string
+* [sexu] Ensure height is int
+* [jove] Ensure comment count is int
+* [golem] Ensure format id is string
+* [gfycat] Ensure filesize is int
+* [foxgay] Ensure height is int
+* [flickr] Ensure format id is string
+* [sohu] Fix numeric fields
+* [safari] Improve authentication detection (#13319)
+* [liveleak] Ensure height is int (#13313)
+* [streamango] Make title optional (#13292)
+* [rtlnl] Improve URL regular expression (#13295)
+* [tvplayer] Fix extraction (#13291)
+
+
 version 2017.06.05
 
 Core

From cb1e6d898532e86b0752117337b7519d22c63274 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Jun 2017 02:23:17 +0700
Subject: [PATCH 424/611] release 2017.06.12

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 ChangeLog                 |  2 +-
 README.md                 | 28 ++++++++++++++--------------
 docs/supportedsites.md    |  3 ++-
 youtube_dl/version.py     |  2 +-
 5 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index ce1e46085..b8806e7b3 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.06.05*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.06.05**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.06.12*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.06.12**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.06.05
+[debug] youtube-dl version 2017.06.12
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 19f160c81..e3b7f7d60 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.06.12
 
 Core
 * [utils] Handle compat_HTMLParseError in extract_attributes (#13349)
diff --git a/README.md b/README.md
index 2189552a9..fe2bebc2a 100644
--- a/README.md
+++ b/README.md
@@ -145,18 +145,18 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     --max-views COUNT                Do not download any videos with more than
                                      COUNT views
     --match-filter FILTER            Generic video filter. Specify any key (see
-                                     help for -o for a list of available keys)
-                                     to match if the key is present, !key to
-                                     check if the key is not present, key >
-                                     NUMBER (like "comment_count > 12", also
-                                     works with >=, <, <=, !=, =) to compare
-                                     against a number, key = 'LITERAL' (like
-                                     "uploader = 'Mike Smith'", also works with
-                                     !=) to match against a string literal and &
-                                     to require multiple matches. Values which
-                                     are not known are excluded unless you put a
-                                     question mark (?) after the operator. For
-                                     example, to only match videos that have
+                                     the "OUTPUT TEMPLATE" for a list of
+                                     available keys) to match if the key is
+                                     present, !key to check if the key is not
+                                     present, key > NUMBER (like "comment_count
+                                     > 12", also works with >=, <, <=, !=, =) to
+                                     compare against a number, key = 'LITERAL'
+                                     (like "uploader = 'Mike Smith'", also works
+                                     with !=) to match against a string literal
+                                     and & to require multiple matches. Values
+                                     which are not known are excluded unless you
+                                     put a question mark (?) after the operator.
+                                     For example, to only match videos that have
                                      been liked more than 100 times and disliked
                                      less than 50 times (or the dislike
                                      functionality is not available at the given
@@ -277,8 +277,8 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     --get-filename                   Simulate, quiet but print output filename
     --get-format                     Simulate, quiet but print output format
     -j, --dump-json                  Simulate, quiet but print JSON information.
-                                     See --output for a description of available
-                                     keys.
+                                     See the "OUTPUT TEMPLATE" for a description
+                                     of available keys.
     -J, --dump-single-json           Simulate, quiet but print JSON information
                                      for each command-line argument. If the URL
                                      refers to a playlist, dump the whole
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index fe022d158..7189f31d9 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -512,6 +512,7 @@
  - **netease:song**: 网易云音乐
  - **Netzkino**
  - **Newgrounds**
+ - **NewgroundsPlaylist**
  - **Newstube**
  - **NextMedia**: 蘋果日報
  - **NextMediaActionNews**: 蘋果日報 - 動新聞
@@ -974,7 +975,7 @@
  - **WSJArticle**
  - **XBef**
  - **XboxClips**
- - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To, XVIDSTAGE, Vid ABC, VidBom, vidlo
+ - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To, XVIDSTAGE, Vid ABC, VidBom, vidlo, RapidVideo.TV
  - **XHamster**
  - **XHamsterEmbed**
  - **xiami:album**: 虾米音乐 - 专辑
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d3aa39977..a7386c3a8 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.06.05'
+__version__ = '2017.06.12'

From b081f53b08e791d9e688bfa1a17caf3bb3630693 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Jun 2017 02:35:57 +0700
Subject: [PATCH 425/611] [compat] Add compat_HTMLParseError to __all__

---
 youtube_dl/compat.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index bbc499eda..7ef327451 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2895,6 +2895,7 @@ else:
 
 
 __all__ = [
+    'compat_HTMLParseError',
     'compat_HTMLParser',
     'compat_HTTPError',
     'compat_basestring',

From 97fa1f8dc4df553624b76a3da2b00edc47744a16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 13 Jun 2017 23:15:06 +0700
Subject: [PATCH 426/611] [corus] Add support for history.ca (closes #13359)

---
 youtube_dl/extractor/corus.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/corus.py b/youtube_dl/extractor/corus.py
index 7b2f5008b..9cdd34636 100644
--- a/youtube_dl/extractor/corus.py
+++ b/youtube_dl/extractor/corus.py
@@ -8,7 +8,16 @@ from ..utils import int_or_none
 
 
 class CorusIE(ThePlatformFeedIE):
-    _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:globaltv|etcanada)\.com|(?:hgtv|foodnetwork|slice)\.ca)/(?:video/|(?:[^/]+/)+(?:videos/[a-z0-9-]+-|video\.html\?.*?\bv=))(?P<id>\d+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?
+                        (?P<domain>
+                            (?:globaltv|etcanada)\.com|
+                            (?:hgtv|foodnetwork|slice|history)\.ca
+                        )
+                        /(?:video/|(?:[^/]+/)+(?:videos/[a-z0-9-]+-|video\.html\?.*?\bv=))
+                        (?P<id>\d+)
+                    '''
     _TESTS = [{
         'url': 'http://www.hgtv.ca/shows/bryan-inc/videos/movie-night-popcorn-with-bryan-870923331648/',
         'md5': '05dcbca777bf1e58c2acbb57168ad3a6',
@@ -27,6 +36,9 @@ class CorusIE(ThePlatformFeedIE):
     }, {
         'url': 'http://etcanada.com/video/873675331955/meet-the-survivor-game-changers-castaways-part-2/',
         'only_matching': True,
+    }, {
+        'url': 'http://www.history.ca/the-world-without-canada/video/full-episodes/natural-resources/video.html?v=955054659646#video',
+        'only_matching': True,
     }]
 
     _TP_FEEDS = {
@@ -50,6 +62,10 @@ class CorusIE(ThePlatformFeedIE):
             'feed_id': '5tUJLgV2YNJ5',
             'account_id': 2414427935,
         },
+        'history': {
+            'feed_id': 'tQFx_TyyEq4J',
+            'account_id': 2369613659,
+        },
     }
 
     def _real_extract(self, url):

From b5dc33daa96fd59ed5e0dc38cb5bb5157433a72b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 13 Jun 2017 23:27:27 +0700
Subject: [PATCH 427/611] [corus] Add support for showcase.ca

---
 youtube_dl/extractor/corus.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/corus.py b/youtube_dl/extractor/corus.py
index 9cdd34636..807a29eea 100644
--- a/youtube_dl/extractor/corus.py
+++ b/youtube_dl/extractor/corus.py
@@ -13,7 +13,7 @@ class CorusIE(ThePlatformFeedIE):
                         (?:www\.)?
                         (?P<domain>
                             (?:globaltv|etcanada)\.com|
-                            (?:hgtv|foodnetwork|slice|history)\.ca
+                            (?:hgtv|foodnetwork|slice|history|showcase)\.ca
                         )
                         /(?:video/|(?:[^/]+/)+(?:videos/[a-z0-9-]+-|video\.html\?.*?\bv=))
                         (?P<id>\d+)
@@ -39,6 +39,9 @@ class CorusIE(ThePlatformFeedIE):
     }, {
         'url': 'http://www.history.ca/the-world-without-canada/video/full-episodes/natural-resources/video.html?v=955054659646#video',
         'only_matching': True,
+    }, {
+        'url': 'http://www.showcase.ca/eyewitness/video/eyewitness++106/video.html?v=955070531919&p=1&s=da#video',
+        'only_matching': True,
     }]
 
     _TP_FEEDS = {
@@ -66,6 +69,10 @@ class CorusIE(ThePlatformFeedIE):
             'feed_id': 'tQFx_TyyEq4J',
             'account_id': 2369613659,
         },
+        'showcase': {
+            'feed_id': '9H6qyshBZU3E',
+            'account_id': 2414426607,
+        },
     }
 
     def _real_extract(self, url):

From 7dd5415cd0e824b00e6abf9a18d55701d52babec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Jun 2017 21:33:40 +0700
Subject: [PATCH 428/611] [npo] Improve _VALID_URL (closes #13376)

---
 youtube_dl/extractor/npo.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 79296f0ef..5f8b6def1 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -35,7 +35,7 @@ class NPOIE(NPOBaseIE):
                         https?://
                             (?:www\.)?
                             (?:
-                                npo\.nl/(?!live|radio)(?:[^/]+/){2}|
+                                npo\.nl/(?!(?:live|radio)/)(?:[^/]+/){2}|
                                 ntr\.nl/(?:[^/]+/){2,}|
                                 omroepwnl\.nl/video/fragment/[^/]+__|
                                 zapp\.nl/[^/]+/[^/]+/
@@ -150,6 +150,9 @@ class NPOIE(NPOBaseIE):
         # live stream
         'url': 'npo:LI_NL1_4188102',
         'only_matching': True,
+    }, {
+        'url': 'http://www.npo.nl/radio-gaga/13-06-2017/BNN_101383373',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 0a268c6e11e9fb55f41b474497997bff61d97cd2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Jun 2017 22:02:15 +0700
Subject: [PATCH 429/611] [extractor/common] Improve jwplayer formats
 extraction (closes #13379)

---
 youtube_dl/extractor/common.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f027447c8..941385ae2 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2328,6 +2328,8 @@ class InfoExtractor(object):
         urls = []
         formats = []
         for source in jwplayer_sources_data:
+            if not isinstance(source, dict):
+                continue
             source_url = self._proto_relative_url(source.get('file'))
             if not source_url:
                 continue

From 5215f45327a1ca84c73b98df05c1977520cfb0a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marcin=20Cie=C5=9Blak?= <saper@saper.info>
Date: Wed, 14 Jun 2017 23:13:04 +0200
Subject: [PATCH 430/611] [disney] Add support for disneychannel.de

---
 youtube_dl/extractor/disney.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/disney.py b/youtube_dl/extractor/disney.py
index 939d1338c..968c4c7fd 100644
--- a/youtube_dl/extractor/disney.py
+++ b/youtube_dl/extractor/disney.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 class DisneyIE(InfoExtractor):
     _VALID_URL = r'''(?x)
-        https?://(?P<domain>(?:[^/]+\.)?(?:disney\.[a-z]{2,3}(?:\.[a-z]{2})?|disney(?:(?:me|latino)\.com|turkiye\.com\.tr)|(?:starwars|marvelkids)\.com))/(?:(?:embed/|(?:[^/]+/)+[\w-]+-)(?P<id>[a-z0-9]{24})|(?:[^/]+/)?(?P<display_id>[^/?#]+))'''
+        https?://(?P<domain>(?:[^/]+\.)?(?:disney\.[a-z]{2,3}(?:\.[a-z]{2})?|disney(?:(?:me|latino)\.com|turkiye\.com\.tr|channel\.de)|(?:starwars|marvelkids)\.com))/(?:(?:embed/|(?:[^/]+/)+[\w-]+-)(?P<id>[a-z0-9]{24})|(?:[^/]+/)?(?P<display_id>[^/?#]+))'''
     _TESTS = [{
         # Disney.EmbedVideo
         'url': 'http://video.disney.com/watch/moana-trailer-545ed1857afee5a0ec239977',
@@ -68,6 +68,9 @@ class DisneyIE(InfoExtractor):
     }, {
         'url': 'http://disneyjunior.en.disneyme.com/dj/watch-my-friends-tigger-and-pooh-promo',
         'only_matching': True,
+    }, {
+        'url': 'http://disneychannel.de/sehen/soy-luna-folge-118-5518518987ba27f3cc729268',
+        'only_matching': True,
     }, {
         'url': 'http://disneyjunior.disney.com/galactech-the-galactech-grab-galactech-an-admiral-rescue',
         'only_matching': True,

From 0bf4b71b753534ac4a93d9893dcd7333b5f229b3 Mon Sep 17 00:00:00 2001
From: gfabiano <gfabiano40@gmail.com>
Date: Wed, 14 Jun 2017 23:16:50 +0200
Subject: [PATCH 431/611] [4tube] Fix extraction (closes #13381)

---
 youtube_dl/extractor/fourtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index 9776c8422..e3fd08bcf 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -85,11 +85,11 @@ class FourTubeIE(InfoExtractor):
             media_id = params[0]
             sources = ['%s' % p for p in params[2]]
 
-        token_url = 'http://tkn.4tube.com/{0}/desktop/{1}'.format(
+        token_url = 'https://tkn.kodicdn.com/{0}/desktop/{1}'.format(
             media_id, '+'.join(sources))
         headers = {
             b'Content-Type': b'application/x-www-form-urlencoded',
-            b'Origin': b'http://www.4tube.com',
+            b'Origin': b'https://www.4tube.com',
         }
         token_req = sanitized_Request(token_url, b'{}', headers)
         tokens = self._download_json(token_req, video_id)

From ca270371718fda18b3331dd645a200a365b096c9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 15 Jun 2017 11:19:03 +0800
Subject: [PATCH 432/611] [bilibili] Fix extraction of videos with double
 quotes in titles

Closes #13387
---
 ChangeLog                        |  6 ++++++
 youtube_dl/extractor/bilibili.py | 18 +++++++++++++++++-
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index e3b7f7d60..9b45f08e3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [bilibili] Fix extraction of videos with double quotes in titles (#13387)
+
+
 version 2017.06.12
 
 Core
diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 1e3f25515..1e57310d6 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -54,6 +54,22 @@ class BiliBiliIE(InfoExtractor):
             'description': '如果你是神明，并且能够让妄想成为现实。那你会进行怎么样的妄想？是淫靡的世界？独裁社会？毁灭性的制裁？还是……2015年，涩谷。从6年前发生的大灾害“涩谷地震”之后复兴了的这个街区里新设立的私立高中...',
         },
         'skip': 'Geo-restricted to China',
+    }, {
+        # Title with double quotes
+        'url': 'http://www.bilibili.com/video/av8903802/',
+        'info_dict': {
+            'id': '8903802',
+            'ext': 'mp4',
+            'title': '阿滴英文｜英文歌分享#6 "Closer',
+            'description': '滴妹今天唱Closer給你聽! 有史以来，被推最多次也是最久的歌曲，其实歌词跟我原本想像差蛮多的，不过还是好听！ 微博@阿滴英文',
+            'uploader': '阿滴英文',
+            'uploader_id': '65880958',
+            'timestamp': 1488382620,
+            'upload_date': '20170301',
+        },
+        'params': {
+            'skip_download': True,  # Test metadata only
+        },
     }]
 
     _APP_KEY = '84956560bc028eb7'
@@ -135,7 +151,7 @@ class BiliBiliIE(InfoExtractor):
                 'formats': formats,
             })
 
-        title = self._html_search_regex('<h1[^>]+title="([^"]+)">', webpage, 'title')
+        title = self._html_search_regex('<h1[^>]*>([^<]+)</h1>', webpage, 'title')
         description = self._html_search_meta('description', webpage)
         timestamp = unified_timestamp(self._html_search_regex(
             r'<time[^>]+datetime="([^"]+)"', webpage, 'upload time', default=None))

From 6a9cb29509a3b1fa2c1ffce82acb2fa7a2699c29 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 15 Jun 2017 13:04:36 +0800
Subject: [PATCH 433/611] [extractor/common] Fix json dumping with --geo-bypass

The line "[debug] Using fake IP %s (%s) as X-Forwarded-For." was printed
to stdout even with -j/-J, which breaks the resultant JSON.
---
 ChangeLog                      | 3 +++
 youtube_dl/extractor/common.py | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 9b45f08e3..5cd0b3393 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,8 @@
 version <unreleased>
 
+Core
+* [extractor/common] Fix json dumping with --geo-bypass
+
 Extractors
 * [bilibili] Fix extraction of videos with double quotes in titles (#13387)
 
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 941385ae2..6e415ea41 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -420,7 +420,7 @@ class InfoExtractor(object):
             if country_code:
                 self._x_forwarded_for_ip = GeoUtils.random_ipv4(country_code)
                 if self._downloader.params.get('verbose', False):
-                    self._downloader.to_stdout(
+                    self._downloader.to_screen(
                         '[debug] Using fake IP %s (%s) as X-Forwarded-For.'
                         % (self._x_forwarded_for_ip, country_code.upper()))
 

From 0ea6efbb7a1434b83c871577845383a472324cb5 Mon Sep 17 00:00:00 2001
From: gfabiano <gfabiano40@gmail.com>
Date: Wed, 14 Jun 2017 23:38:21 +0200
Subject: [PATCH 434/611] [xfileshare] Add support for fastvideo.me

---
 youtube_dl/extractor/xfileshare.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index a32789ef4..74aac85a3 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -30,6 +30,7 @@ class XFileShareIE(InfoExtractor):
         (r'vidbom\.com', 'VidBom'),
         (r'vidlo\.us', 'vidlo'),
         (r'rapidvideo\.(?:cool|org)', 'RapidVideo.TV'),
+        (r'fastvideo\.me', 'FastVideo.me'),
     )
 
     IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1])
@@ -112,6 +113,9 @@ class XFileShareIE(InfoExtractor):
     }, {
         'url': 'http://www.rapidvideo.cool/b667kprndr8w',
         'only_matching': True,
+    }, {
+        'url': 'http://www.fastvideo.me/k8604r8nk8sn/FAST_FURIOUS_8_-_Trailer_italiano_ufficiale.mp4.html',
+        'only_matching': True
     }]
 
     def _real_extract(self, url):

From 96a2daa1ee5de9ddff5d0180fccb8376725de60c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Jun 2017 23:40:39 +0700
Subject: [PATCH 435/611] [extractor/common] Improve jwplayer subtitles
 extraction

---
 youtube_dl/extractor/common.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6e415ea41..9751ab021 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2299,6 +2299,8 @@ class InfoExtractor(object):
             tracks = video_data.get('tracks')
             if tracks and isinstance(tracks, list):
                 for track in tracks:
+                    if not isinstance(track, dict):
+                        continue
                     if track.get('kind') != 'captions':
                         continue
                     track_url = urljoin(base_url, track.get('file'))

From b230fefc3c436b4d3ff278a11552b8a964b7056e Mon Sep 17 00:00:00 2001
From: Jakub Adam Wieczorek <ja.wieczorek@student.uw.edu.pl>
Date: Thu, 15 Jun 2017 23:57:56 +0200
Subject: [PATCH 436/611] [polskieradio] Fix extraction

---
 youtube_dl/extractor/polskieradio.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/polskieradio.py b/youtube_dl/extractor/polskieradio.py
index 2ac1fcb0b..978d6f813 100644
--- a/youtube_dl/extractor/polskieradio.py
+++ b/youtube_dl/extractor/polskieradio.py
@@ -65,7 +65,7 @@ class PolskieRadioIE(InfoExtractor):
         webpage = self._download_webpage(url, playlist_id)
 
         content = self._search_regex(
-            r'(?s)<div[^>]+class="audio atarticle"[^>]*>(.+?)<script>',
+            r'(?s)<div[^>]+class="\s*this-article\s*"[^>]*>(.+?)<div[^>]+class="tags"[^>]*>',
             webpage, 'content')
 
         timestamp = unified_timestamp(self._html_search_regex(

From 4f90d2aeacccab6dc461e92236480701c4a0900b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 17 Jun 2017 17:09:24 +0800
Subject: [PATCH 437/611] [Makefile] Excluding __pycache__ correctly (#13400)

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 023556391..84ccce2b3 100644
--- a/Makefile
+++ b/Makefile
@@ -101,7 +101,7 @@ youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-
 		--exclude '*.pyc' \
 		--exclude '*.pyo' \
 		--exclude '*~' \
-		--exclude '__pycache' \
+		--exclude '__pycache__' \
 		--exclude '.git' \
 		--exclude 'testdata' \
 		--exclude 'docs/_build' \

From 473e87064b7b60ea68147401a2e6487b715f25c8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 17 Jun 2017 17:37:25 +0800
Subject: [PATCH 438/611] [devscripts/prepare_manpage] Fix deprecated escape
 sequence on py36

---
 devscripts/prepare_manpage.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/devscripts/prepare_manpage.py b/devscripts/prepare_manpage.py
index f9fe63f1f..76bf873e1 100644
--- a/devscripts/prepare_manpage.py
+++ b/devscripts/prepare_manpage.py
@@ -8,7 +8,7 @@ import re
 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 README_FILE = os.path.join(ROOT_DIR, 'README.md')
 
-PREFIX = '''%YOUTUBE-DL(1)
+PREFIX = r'''%YOUTUBE-DL(1)
 
 # NAME
 

From be80986ed9175af0a0fb216edfdfaeeb9769d1dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Jun 2017 19:01:27 +0700
Subject: [PATCH 439/611] [postprocessor/metadatafromtitle] Fix missing
 optional meta fields (closes #13408)

---
 youtube_dl/postprocessor/metadatafromtitle.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dl/postprocessor/metadatafromtitle.py
index c73f02447..f5c14d974 100644
--- a/youtube_dl/postprocessor/metadatafromtitle.py
+++ b/youtube_dl/postprocessor/metadatafromtitle.py
@@ -35,11 +35,14 @@ class MetadataFromTitlePP(PostProcessor):
         title = info['title']
         match = re.match(self._titleregex, title)
         if match is None:
-            self._downloader.to_screen('[fromtitle] Could not interpret title of video as "%s"' % self._titleformat)
+            self._downloader.to_screen(
+                '[fromtitle] Could not interpret title of video as "%s"'
+                % self._titleformat)
             return [], info
         for attribute, value in match.groupdict().items():
-            value = match.group(attribute)
             info[attribute] = value
-            self._downloader.to_screen('[fromtitle] parsed ' + attribute + ': ' + value)
+            self._downloader.to_screen(
+                '[fromtitle] parsed %s: %s'
+                % (attribute, value if value is not None else 'NA'))
 
         return [], info

From b08e235f09b360262b97ef361a2321839a0534cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Jun 2017 23:14:24 +0700
Subject: [PATCH 440/611] [compat] Fix compat_shlex_quote on Windows (closes
 #5889, closes #10254)

---
 youtube_dl/compat.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 7ef327451..9e4e13bcf 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2617,14 +2617,22 @@ except ImportError:  # Python 2
                 parsed_result[name] = [value]
         return parsed_result
 
-try:
-    from shlex import quote as compat_shlex_quote
-except ImportError:  # Python < 3.3
+
+compat_os_name = os._name if os.name == 'java' else os.name
+
+
+if compat_os_name == 'nt':
     def compat_shlex_quote(s):
-        if re.match(r'^[-_\w./]+$', s):
-            return s
-        else:
-            return "'" + s.replace("'", "'\"'\"'") + "'"
+        return s if re.match(r'^[-_\w./]+$', s) else '"%s"' % s.replace('"', '\\"')
+else:
+    try:
+        from shlex import quote as compat_shlex_quote
+    except ImportError:  # Python < 3.3
+        def compat_shlex_quote(s):
+            if re.match(r'^[-_\w./]+$', s):
+                return s
+            else:
+                return "'" + s.replace("'", "'\"'\"'") + "'"
 
 
 try:
@@ -2649,9 +2657,6 @@ def compat_ord(c):
         return ord(c)
 
 
-compat_os_name = os._name if os.name == 'java' else os.name
-
-
 if sys.version_info >= (3, 0):
     compat_getenv = os.getenv
     compat_expanduser = os.path.expanduser

From 8b6ac49ecc873f787f647f1e623af989f85de644 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Jun 2017 23:15:57 +0700
Subject: [PATCH 441/611] [postprocessor/execafterdownload] Encode command line
 (closes #13407)

---
 youtube_dl/postprocessor/execafterdownload.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py
index 90630c2d7..64dabe790 100644
--- a/youtube_dl/postprocessor/execafterdownload.py
+++ b/youtube_dl/postprocessor/execafterdownload.py
@@ -4,7 +4,10 @@ import subprocess
 
 from .common import PostProcessor
 from ..compat import compat_shlex_quote
-from ..utils import PostProcessingError
+from ..utils import (
+    encodeArgument,
+    PostProcessingError,
+)
 
 
 class ExecAfterDownloadPP(PostProcessor):
@@ -20,7 +23,7 @@ class ExecAfterDownloadPP(PostProcessor):
         cmd = cmd.replace('{}', compat_shlex_quote(information['filepath']))
 
         self._downloader.to_screen('[exec] Executing command: %s' % cmd)
-        retCode = subprocess.call(cmd, shell=True)
+        retCode = subprocess.call(encodeArgument(cmd), shell=True)
         if retCode != 0:
             raise PostProcessingError(
                 'Command returned error code %d' % retCode)

From aefce8e6dca8a0ae784468a5e1a4ba01f1471380 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Jun 2017 23:48:58 +0700
Subject: [PATCH 442/611] [utils] Use compat_shlex_quote in shell_quote

---
 youtube_dl/utils.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 1973bd483..39860e9d1 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -22,7 +22,6 @@ import locale
 import math
 import operator
 import os
-import pipes
 import platform
 import random
 import re
@@ -1535,7 +1534,7 @@ def shell_quote(args):
         if isinstance(a, bytes):
             # We may get a filename encoded with 'encodeFilename'
             a = a.decode(encoding)
-        quoted_args.append(pipes.quote(a))
+        quoted_args.append(compat_shlex_quote(a))
     return ' '.join(quoted_args)
 
 

From 1433734c35dbfe16193ddcb6e59f55b58e76ba74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Jun 2017 23:50:21 +0700
Subject: [PATCH 443/611] [downloader/common] Use utils.shell_quote for debug
 command line

---
 youtube_dl/downloader/common.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 5d6621147..77242dacc 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -8,10 +8,11 @@ import random
 
 from ..compat import compat_os_name
 from ..utils import (
+    decodeArgument,
     encodeFilename,
     error_to_compat_str,
-    decodeArgument,
     format_bytes,
+    shell_quote,
     timeconvert,
 )
 
@@ -381,10 +382,5 @@ class FileDownloader(object):
         if exe is None:
             exe = os.path.basename(str_args[0])
 
-        try:
-            import pipes
-            shell_quote = lambda args: ' '.join(map(pipes.quote, str_args))
-        except ImportError:
-            shell_quote = repr
         self.to_screen('[debug] %s command line: %s' % (
             exe, shell_quote(str_args)))

From a6f29820c6b4911a3c0a38ede6cff0bb3efdab9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Jun 2017 00:15:43 +0700
Subject: [PATCH 444/611] [ChangeLog] Actualize

---
 ChangeLog | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 5cd0b3393..b738d2063 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,10 +1,25 @@
 version <unreleased>
 
 Core
+* [downloader/common] Use utils.shell_quote for debug command line
+* [utils] Use compat_shlex_quote in shell_quote
+* [postprocessor/execafterdownload] Encode command line (#13407)
+* [compat] Fix compat_shlex_quote on Windows (#5889, #10254)
+* [postprocessor/metadatafromtitle] Fix missing optional meta fields processing
+   in --metadata-from-title (#13408)
 * [extractor/common] Fix json dumping with --geo-bypass
++ [extractor/common] Improve jwplayer subtitles extraction
++ [extractor/common] Improve jwplayer formats extraction (#13379)
 
 Extractors
+* [polskieradio] Fix extraction (#13392)
++ [xfileshare] Add support for fastvideo.me (#13385)
 * [bilibili] Fix extraction of videos with double quotes in titles (#13387)
+* [4tube] Fix extraction (#13381, #13382)
++ [disney] Add support for disneychannel.de (#13383)
+* [npo] Improve URL regular expression (#13376)
++ [corus] Add support for showcase.ca
++ [corus] Add support for history.ca (#13359)
 
 
 version 2017.06.12

From eaa006d1bd6a5fd1d3c482a04963068061fd2682 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Jun 2017 00:16:49 +0700
Subject: [PATCH 445/611] release 2017.06.18

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 +-
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index b8806e7b3..002c1274a 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.06.12*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.06.12**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.06.18*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.06.18**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.06.12
+[debug] youtube-dl version 2017.06.18
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index b738d2063..29d21e4e4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.06.18
 
 Core
 * [downloader/common] Use utils.shell_quote for debug command line
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 7189f31d9..0f21be0a2 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -975,7 +975,7 @@
  - **WSJArticle**
  - **XBef**
  - **XboxClips**
- - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To, XVIDSTAGE, Vid ABC, VidBom, vidlo, RapidVideo.TV
+ - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To, XVIDSTAGE, Vid ABC, VidBom, vidlo, RapidVideo.TV, FastVideo.me
  - **XHamster**
  - **XHamsterEmbed**
  - **xiami:album**: 虾米音乐 - 专辑
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a7386c3a8..8782a6a1e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.06.12'
+__version__ = '2017.06.18'

From 560050669b1e98bc76c797256c6e21719541b2ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Jun 2017 20:18:51 +0700
Subject: [PATCH 446/611] [asiancrush] Add extractor (closes #13420)

---
 youtube_dl/extractor/asiancrush.py | 93 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  4 ++
 2 files changed, 97 insertions(+)
 create mode 100644 youtube_dl/extractor/asiancrush.py

diff --git a/youtube_dl/extractor/asiancrush.py b/youtube_dl/extractor/asiancrush.py
new file mode 100644
index 000000000..594c88c9c
--- /dev/null
+++ b/youtube_dl/extractor/asiancrush.py
@@ -0,0 +1,93 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from .kaltura import KalturaIE
+from ..utils import (
+    extract_attributes,
+    remove_end,
+    urlencode_postdata,
+)
+
+
+class AsianCrushIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?asiancrush\.com/video/(?:[^/]+/)?0+(?P<id>\d+)v\b'
+    _TESTS = [{
+        'url': 'https://www.asiancrush.com/video/012869v/women-who-flirt/',
+        'md5': 'c3b740e48d0ba002a42c0b72857beae6',
+        'info_dict': {
+            'id': '1_y4tmjm5r',
+            'ext': 'mp4',
+            'title': 'Women Who Flirt',
+            'description': 'md5:3db14e9186197857e7063522cb89a805',
+            'timestamp': 1496936429,
+            'upload_date': '20170608',
+            'uploader_id': 'craig@crifkin.com',
+        },
+    }, {
+        'url': 'https://www.asiancrush.com/video/she-was-pretty/011886v-pretty-episode-3/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        data = self._download_json(
+            'https://www.asiancrush.com/wp-admin/admin-ajax.php', video_id,
+            data=urlencode_postdata({
+                'postid': video_id,
+                'action': 'get_channel_kaltura_vars',
+            }))
+
+        entry_id = data['entry_id']
+
+        return self.url_result(
+            'kaltura:%s:%s' % (data['partner_id'], entry_id),
+            ie=KalturaIE.ie_key(), video_id=entry_id,
+            video_title=data.get('vid_label'))
+
+
+class AsianCrushPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?asiancrush\.com/series/0+(?P<id>\d+)s\b'
+    _TEST = {
+        'url': 'https://www.asiancrush.com/series/012481s/scholar-walks-night/',
+        'info_dict': {
+            'id': '12481',
+            'title': 'Scholar Who Walks the Night',
+            'description': 'md5:7addd7c5132a09fd4741152d96cce886',
+        },
+        'playlist_count': 20,
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = []
+
+        for mobj in re.finditer(
+                r'<a[^>]+href=(["\'])(?P<url>%s.*?)\1[^>]*>' % AsianCrushIE._VALID_URL,
+                webpage):
+            attrs = extract_attributes(mobj.group(0))
+            if attrs.get('class') == 'clearfix':
+                entries.append(self.url_result(
+                    mobj.group('url'), ie=AsianCrushIE.ie_key()))
+
+        title = remove_end(
+            self._html_search_regex(
+                r'(?s)<h1\b[^>]\bid=["\']movieTitle[^>]+>(.+?)</h1>', webpage,
+                'title', default=None) or self._og_search_title(
+                webpage, default=None) or self._html_search_meta(
+                'twitter:title', webpage, 'title',
+                default=None) or self._search_regex(
+                r'<title>([^<]+)</title>', webpage, 'title', fatal=False),
+            ' | AsianCrush')
+
+        description = self._og_search_description(
+            webpage, default=None) or self._html_search_meta(
+            'twitter:description', webpage, 'description', fatal=False)
+
+        return self.playlist_result(entries, playlist_id, title, description)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 7e45232dd..edbb4bdde 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -71,6 +71,10 @@ from .arte import (
     TheOperaPlatformIE,
     ArteTVPlaylistIE,
 )
+from .asiancrush import (
+    AsianCrushIE,
+    AsianCrushPlaylistIE,
+)
 from .atresplayer import AtresPlayerIE
 from .atttechchannel import ATTTechChannelIE
 from .atvat import ATVAtIE

From cc2ffe5afe692bccc9214110bd253b87f4361856 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 19 Jun 2017 16:20:36 +0800
Subject: [PATCH 447/611] [pandora.tv] Fix upload_date extraction (closes
 #12846)

---
 ChangeLog                         |  6 ++++++
 youtube_dl/extractor/pandoratv.py | 25 ++++++++++++++++++++++---
 2 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 29d21e4e4..7f077fd24 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [pandora.tv] Fix upload_date extraction (#12846)
+
+
 version 2017.06.18
 
 Core
diff --git a/youtube_dl/extractor/pandoratv.py b/youtube_dl/extractor/pandoratv.py
index 89c95fffb..fc7bd3411 100644
--- a/youtube_dl/extractor/pandoratv.py
+++ b/youtube_dl/extractor/pandoratv.py
@@ -19,7 +19,7 @@ class PandoraTVIE(InfoExtractor):
     IE_NAME = 'pandora.tv'
     IE_DESC = '판도라TV'
     _VALID_URL = r'https?://(?:.+?\.)?channel\.pandora\.tv/channel/video\.ptv\?'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://jp.channel.pandora.tv/channel/video.ptv?c1=&prgid=53294230&ch_userid=mikakim&ref=main&lot=cate_01_2',
         'info_dict': {
             'id': '53294230',
@@ -34,7 +34,26 @@ class PandoraTVIE(InfoExtractor):
             'view_count': int,
             'like_count': int,
         }
-    }
+    }, {
+        'url': 'http://channel.pandora.tv/channel/video.ptv?ch_userid=gogoucc&prgid=54721744',
+        'info_dict': {
+            'id': '54721744',
+            'ext': 'flv',
+            'title': '[HD] JAPAN COUNTDOWN 170423',
+            'description': '[HD] JAPAN COUNTDOWN 170423',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 1704.9,
+            'upload_date': '20170423',
+            'uploader': 'GOGO_UCC',
+            'uploader_id': 'gogoucc',
+            'view_count': int,
+            'like_count': int,
+        },
+        'params': {
+            # Test metadata only
+            'skip_download': True,
+        },
+    }]
 
     def _real_extract(self, url):
         qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
@@ -86,7 +105,7 @@ class PandoraTVIE(InfoExtractor):
             'description': info.get('body'),
             'thumbnail': info.get('thumbnail') or info.get('poster'),
             'duration': float_or_none(info.get('runtime'), 1000) or parse_duration(info.get('time')),
-            'upload_date': info['fid'][:8] if isinstance(info.get('fid'), compat_str) else None,
+            'upload_date': info['fid'].split('/')[-1][:8] if isinstance(info.get('fid'), compat_str) else None,
             'uploader': info.get('nickname'),
             'uploader_id': info.get('upload_userid'),
             'view_count': str_to_int(info.get('hit')),

From 2d2132ac6e75932c9c7e6c143f20156482043729 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 19 Jun 2017 22:53:39 +0700
Subject: [PATCH 448/611] [adobepass] Fix extraction on older python 2.6

---
 youtube_dl/extractor/adobepass.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index d57ad85c2..3dfc632e7 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -6,7 +6,10 @@ import time
 import xml.etree.ElementTree as etree
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_kwargs,
+    compat_urlparse,
+)
 from ..utils import (
     unescapeHTML,
     urlencode_postdata,
@@ -1317,7 +1320,8 @@ class AdobePassIE(InfoExtractor):
         headers = kwargs.get('headers', {})
         headers.update(self.geo_verification_headers())
         kwargs['headers'] = headers
-        return super(AdobePassIE, self)._download_webpage_handle(*args, **kwargs)
+        return super(AdobePassIE, self)._download_webpage_handle(
+            *args, **compat_kwargs(kwargs))
 
     @staticmethod
     def _get_mvpd_resource(provider_id, title, guid, rating):

From fee00b3884922f6cb44926349df788d0fc589811 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 19 Jun 2017 22:57:37 +0700
Subject: [PATCH 449/611] [viu] Fix extraction on older python 2.6

---
 youtube_dl/extractor/viu.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viu.py b/youtube_dl/extractor/viu.py
index db6a65d2e..5cf93591c 100644
--- a/youtube_dl/extractor/viu.py
+++ b/youtube_dl/extractor/viu.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_str
+from ..compat import (
+    compat_kwargs,
+    compat_str,
+)
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -36,7 +39,8 @@ class ViuBaseIE(InfoExtractor):
         headers.update(kwargs.get('headers', {}))
         kwargs['headers'] = headers
         response = self._download_json(
-            'https://www.viu.com/api/' + path, *args, **kwargs)['response']
+            'https://www.viu.com/api/' + path, *args,
+            **compat_kwargs(kwargs))['response']
         if response.get('status') != 'success':
             raise ExtractorError('%s said: %s' % (
                 self.IE_NAME, response['message']), expected=True)

From a1de83e5f01cc220fa45caee80b9159cc555609d Mon Sep 17 00:00:00 2001
From: Orn <orngudjonsson@gmail.com>
Date: Thu, 15 Jun 2017 22:29:27 +0000
Subject: [PATCH 450/611] [ruv] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/ruv.py        | 31 ++++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+)
 create mode 100644 youtube_dl/extractor/ruv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index edbb4bdde..e97691daa 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -875,6 +875,7 @@ from .rutube import (
 )
 from .rutv import RUTVIE
 from .ruutu import RuutuIE
+from .ruv import RuvIE
 from .sandia import SandiaIE
 from .safari import (
     SafariIE,
diff --git a/youtube_dl/extractor/ruv.py b/youtube_dl/extractor/ruv.py
new file mode 100644
index 000000000..518fb5027
--- /dev/null
+++ b/youtube_dl/extractor/ruv.py
@@ -0,0 +1,31 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class RuvIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ruv\.is/sarpurinn/ruv/\w+/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://ruv.is/sarpurinn/ruv/frettir/20170614',
+        'md5': 'a07ea1ebaba64082d90323b1c96f264b',
+        'info_dict': {
+            'id': '20170614',
+            'ext': 'mp4',
+            'title': 'Fréttir',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._og_search_title(webpage)
+        video_url = self._html_search_regex(r'video\.src\s*=\s*["\'](.+?)["\']', webpage, 'video URL')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': video_url,
+            'ext': 'mp4'
+        }

From 85cbcede5b9ad2b059e799bb9047264c2a38b7fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 19 Jun 2017 23:45:09 +0700
Subject: [PATCH 451/611] [ruv] Improve, extract all formats and metadata
 (closes #13396)

---
 youtube_dl/extractor/ruv.py | 96 ++++++++++++++++++++++++++++++++-----
 1 file changed, 83 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/ruv.py b/youtube_dl/extractor/ruv.py
index 518fb5027..8f3cc4095 100644
--- a/youtube_dl/extractor/ruv.py
+++ b/youtube_dl/extractor/ruv.py
@@ -2,30 +2,100 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    unified_timestamp,
+)
 
 
 class RuvIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ruv\.is/sarpurinn/ruv/\w+/(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'http://ruv.is/sarpurinn/ruv/frettir/20170614',
-        'md5': 'a07ea1ebaba64082d90323b1c96f264b',
+    _VALID_URL = r'https?://(?:www\.)?ruv\.is/(?:sarpurinn/[^/]+|node)/(?P<id>[^/]+(?:/\d+)?)'
+    _TESTS = [{
+        # m3u8
+        'url': 'http://ruv.is/sarpurinn/ruv-aukaras/fh-valur/20170516',
+        'md5': '66347652f4e13e71936817102acc1724',
         'info_dict': {
-            'id': '20170614',
+            'id': '1144499',
+            'display_id': 'fh-valur/20170516',
             'ext': 'mp4',
-            'title': 'Fréttir',
-        }
-    }
+            'title': 'FH - Valur',
+            'description': 'Bein útsending frá 3. leik FH og Vals í úrslitum Olísdeildar karla í handbolta.',
+            'timestamp': 1494963600,
+            'upload_date': '20170516',
+        },
+    }, {
+        # mp3
+        'url': 'http://ruv.is/sarpurinn/ras-2/morgunutvarpid/20170619',
+        'md5': '395ea250c8a13e5fdb39d4670ef85378',
+        'info_dict': {
+            'id': '1153630',
+            'display_id': 'morgunutvarpid/20170619',
+            'ext': 'mp3',
+            'title': 'Morgunútvarpið',
+            'description': 'md5:a4cf1202c0a1645ca096b06525915418',
+            'timestamp': 1497855000,
+            'upload_date': '20170619',
+        },
+    }, {
+        'url': 'http://ruv.is/sarpurinn/ruv/frettir/20170614',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.ruv.is/node/1151854',
+        'only_matching': True,
+    }, {
+        'url': 'http://ruv.is/sarpurinn/klippa/secret-soltice-hefst-a-morgun',
+        'only_matching': True,
+    }, {
+        'url': 'http://ruv.is/sarpurinn/ras-1/morgunvaktin/20170619',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
 
         title = self._og_search_title(webpage)
-        video_url = self._html_search_regex(r'video\.src\s*=\s*["\'](.+?)["\']', webpage, 'video URL')
+
+        FIELD_RE = r'video\.%s\s*=\s*(["\'])(?P<url>(?:(?!\1).)+)\1'
+
+        media_url = self._html_search_regex(
+            FIELD_RE % 'src', webpage, 'video URL', group='url')
+
+        video_id = self._search_regex(
+            r'<link\b[^>]+\bhref=["\']https?://www\.ruv\.is/node/(\d+)',
+            webpage, 'video id', default=display_id)
+
+        ext = determine_ext(media_url)
+
+        if ext == 'm3u8':
+            formats = self._extract_m3u8_formats(
+                media_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                m3u8_id='hls')
+        elif ext == 'mp3':
+            formats = [{
+                'format_id': 'mp3',
+                'url': media_url,
+                'vcodec': 'none',
+            }]
+        else:
+            formats = [{
+                'url': media_url,
+            }]
+
+        description = self._og_search_description(webpage, default=None)
+        thumbnail = self._og_search_thumbnail(
+            webpage, default=None) or self._search_regex(
+            FIELD_RE % 'poster', webpage, 'thumbnail', fatal=False)
+        timestamp = unified_timestamp(self._html_search_meta(
+            'article:published_time', webpage, 'timestamp', fatal=False))
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': title,
-            'url': video_url,
-            'ext': 'mp4'
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'formats': formats,
         }

From 1641ca402d03b36d28bab94eb898997cadf69993 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 20 Jun 2017 01:27:59 +0700
Subject: [PATCH 452/611] [vimeo] Add fallback mp4 extension for original
 format

---
 youtube_dl/extractor/vimeo.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index a483c8409..c3f71b45e 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -615,7 +615,10 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 if download_url and not source_file.get('is_cold') and not source_file.get('is_defrosting'):
                     source_name = source_file.get('public_name', 'Original')
                     if self._is_valid_url(download_url, video_id, '%s video' % source_name):
-                        ext = source_file.get('extension', determine_ext(download_url)).lower()
+                        ext = (try_get(
+                            source_file, lambda x: x['extension'],
+                            compat_str) or determine_ext(
+                            download_url, None) or 'mp4').lower()
                         formats.append({
                             'url': download_url,
                             'ext': ext,

From 6ce79d7ac067ef5c286661923b8df6dc7dc7b0da Mon Sep 17 00:00:00 2001
From: Giuseppe Fabiano <gfabiano40@gmail.com>
Date: Mon, 19 Jun 2017 23:07:00 +0200
Subject: [PATCH 453/611] [abcotvs] Fix test md5

---
 youtube_dl/extractor/abcotvs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/abcotvs.py b/youtube_dl/extractor/abcotvs.py
index 76e98132b..03b92a39c 100644
--- a/youtube_dl/extractor/abcotvs.py
+++ b/youtube_dl/extractor/abcotvs.py
@@ -22,7 +22,7 @@ class ABCOTVSIE(InfoExtractor):
                 'display_id': 'east-bay-museum-celebrates-vintage-synthesizers',
                 'ext': 'mp4',
                 'title': 'East Bay museum celebrates vintage synthesizers',
-                'description': 'md5:a4f10fb2f2a02565c1749d4adbab4b10',
+                'description': 'md5:24ed2bd527096ec2a5c67b9d5a9005f3',
                 'thumbnail': r're:^https?://.*\.jpg$',
                 'timestamp': 1421123075,
                 'upload_date': '20150113',

From 048b55804da21c2b2978cd2f710b2c7b438e24ba Mon Sep 17 00:00:00 2001
From: Giuseppe Fabiano <gfabiano40@gmail.com>
Date: Mon, 19 Jun 2017 23:30:45 +0200
Subject: [PATCH 454/611] [watchindianporn] Fix extraction (closes #13411)

---
 youtube_dl/extractor/watchindianporn.py | 50 +++++++------------------
 1 file changed, 14 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/watchindianporn.py b/youtube_dl/extractor/watchindianporn.py
index ed099beea..fadc539ee 100644
--- a/youtube_dl/extractor/watchindianporn.py
+++ b/youtube_dl/extractor/watchindianporn.py
@@ -4,11 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    unified_strdate,
-    parse_duration,
-    int_or_none,
-)
+from ..utils import parse_duration
 
 
 class WatchIndianPornIE(InfoExtractor):
@@ -23,11 +19,8 @@ class WatchIndianPornIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Hot milf from kerala shows off her gorgeous large breasts on camera',
             'thumbnail': r're:^https?://.*\.jpg$',
-            'uploader': 'LoveJay',
-            'upload_date': '20160428',
             'duration': 226,
             'view_count': int,
-            'comment_count': int,
             'categories': list,
             'age_limit': 18,
         }
@@ -40,51 +33,36 @@ class WatchIndianPornIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        video_url = self._html_search_regex(
-            r"url: escape\('([^']+)'\)", webpage, 'url')
+        info_dict = self._parse_html5_media_entries(url, webpage, video_id)[0]
 
-        title = self._html_search_regex(
-            r'<h2 class="he2"><span>(.*?)</span>',
-            webpage, 'title')
-        thumbnail = self._html_search_regex(
-            r'<span id="container"><img\s+src="([^"]+)"',
-            webpage, 'thumbnail', fatal=False)
-
-        uploader = self._html_search_regex(
-            r'class="aupa">\s*(.*?)</a>',
-            webpage, 'uploader')
-        upload_date = unified_strdate(self._html_search_regex(
-            r'Added: <strong>(.+?)</strong>', webpage, 'upload date', fatal=False))
+        title = self._html_search_regex((
+            r'<title>(.+?)\s*-\s*Indian\s+Porn</title>',
+            r'<h4>(.+?)</h4>'
+        ), webpage, 'title')
 
         duration = parse_duration(self._search_regex(
-            r'<td>Time:\s*</td>\s*<td align="right"><span>\s*(.+?)\s*</span>',
+            r'Time:\s*<strong>\s*(.+?)\s*</strong>',
             webpage, 'duration', fatal=False))
 
-        view_count = int_or_none(self._search_regex(
-            r'<td>Views:\s*</td>\s*<td align="right"><span>\s*(\d+)\s*</span>',
+        view_count = int(self._search_regex(
+            r'(?s)Time:\s*<strong>.*?</strong>.*?<strong>\s*(\d+)\s*</strong>',
             webpage, 'view count', fatal=False))
-        comment_count = int_or_none(self._search_regex(
-            r'<td>Comments:\s*</td>\s*<td align="right"><span>\s*(\d+)\s*</span>',
-            webpage, 'comment count', fatal=False))
 
         categories = re.findall(
-            r'<a href="[^"]+/search/video/desi"><span>([^<]+)</span></a>',
+            r'<a[^>]+class=[\'"]categories[\'"][^>]*>\s*([^<]+)\s*</a>',
             webpage)
 
-        return {
+        info_dict.update({
             'id': video_id,
             'display_id': display_id,
-            'url': video_url,
             'http_headers': {
                 'Referer': url,
             },
             'title': title,
-            'thumbnail': thumbnail,
-            'uploader': uploader,
-            'upload_date': upload_date,
             'duration': duration,
             'view_count': view_count,
-            'comment_count': comment_count,
             'categories': categories,
             'age_limit': 18,
-        }
+        })
+
+        return info_dict

From 9be9ec5980c1c53642eb112ccced3246ac8a391e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 20 Jun 2017 22:58:33 +0700
Subject: [PATCH 455/611] [googledrive] Fix formats' sorting (closes #13443)

---
 youtube_dl/extractor/googledrive.py | 37 +++++++++++++++++++----------
 1 file changed, 25 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index fec36cbbb..9705cfadd 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -69,19 +69,32 @@ class GoogleDriveIE(InfoExtractor):
             r'"fmt_stream_map"\s*,\s*"([^"]+)', webpage, 'fmt stream map').split(',')
         fmt_list = self._search_regex(r'"fmt_list"\s*,\s*"([^"]+)', webpage, 'fmt_list').split(',')
 
+        resolutions = {}
+        for fmt in fmt_list:
+            mobj = re.search(
+                r'^(?P<format_id>\d+)/(?P<width>\d+)[xX](?P<height>\d+)', fmt)
+            if mobj:
+                resolutions[mobj.group('format_id')] = (
+                    int(mobj.group('width')), int(mobj.group('height')))
+
         formats = []
-        for fmt, fmt_stream in zip(fmt_list, fmt_stream_map):
-            fmt_id, fmt_url = fmt_stream.split('|')
-            resolution = fmt.split('/')[1]
-            width, height = resolution.split('x')
-            formats.append({
-                'url': lowercase_escape(fmt_url),
-                'format_id': fmt_id,
-                'resolution': resolution,
-                'width': int_or_none(width),
-                'height': int_or_none(height),
-                'ext': self._FORMATS_EXT[fmt_id],
-            })
+        for fmt_stream in fmt_stream_map:
+            fmt_stream_split = fmt_stream.split('|')
+            if len(fmt_stream_split) < 2:
+                continue
+            format_id, format_url = fmt_stream_split[:2]
+            f = {
+                'url': lowercase_escape(format_url),
+                'format_id': format_id,
+                'ext': self._FORMATS_EXT[format_id],
+            }
+            resolution = resolutions.get(format_id)
+            if resolution:
+                f.update({
+                    'width': resolution[0],
+                    'height': resolution[0],
+                })
+            formats.append(f)
         self._sort_formats(formats)
 
         return {

From 97b6e3011370a851b942bca144afb7cb08a57f5d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 22 Jun 2017 00:20:45 +0700
Subject: [PATCH 456/611] [youporn] Fix title extraction (closes #13456)

---
 youtube_dl/extractor/youporn.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 34ab878a4..7bc2cefc7 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -68,9 +68,12 @@ class YouPornIE(InfoExtractor):
         webpage = self._download_webpage(request, display_id)
 
         title = self._search_regex(
-            [r'(?:video_titles|videoTitle)\s*[:=]\s*(["\'])(?P<title>.+?)\1',
-             r'<h1[^>]+class=["\']heading\d?["\'][^>]*>([^<])<'],
-            webpage, 'title', group='title')
+            [r'(?:video_titles|videoTitle|title)\s*[:=]\s*(["\'])(?P<title>(?:(?!\1).)+)\1',
+             r'<h1[^>]+class=["\']heading\d?["\'][^>]*>(?P<title>[^<]+)<'],
+            webpage, 'title', group='title',
+            default=None) or self._og_search_title(
+            webpage, default=None) or self._html_search_meta(
+            'title', webpage, fatal=True)
 
         links = []
 

From d4893e764bbda206b6194884e67b0acaf231d0d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 22 Jun 2017 00:40:15 +0700
Subject: [PATCH 457/611] [youporn] Improve formats extraction

---
 youtube_dl/extractor/youporn.py | 32 ++++++++++++++++++++++++--------
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 7bc2cefc7..64e3c3c94 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     int_or_none,
     sanitized_Request,
@@ -68,7 +69,7 @@ class YouPornIE(InfoExtractor):
         webpage = self._download_webpage(request, display_id)
 
         title = self._search_regex(
-            [r'(?:video_titles|videoTitle|title)\s*[:=]\s*(["\'])(?P<title>(?:(?!\1).)+)\1',
+            [r'(?:video_titles|videoTitle)\s*[:=]\s*(["\'])(?P<title>(?:(?!\1).)+)\1',
              r'<h1[^>]+class=["\']heading\d?["\'][^>]*>(?P<title>[^<]+)<'],
             webpage, 'title', group='title',
             default=None) or self._og_search_title(
@@ -77,22 +78,37 @@ class YouPornIE(InfoExtractor):
 
         links = []
 
+        # Main source
+        definitions = self._parse_json(
+            self._search_regex(
+                r'mediaDefinition\s*=\s*(\[.+?\]);', webpage,
+                'media definitions', default='[]'),
+            video_id, fatal=False)
+        if definitions:
+            for definition in definitions:
+                if not isinstance(definition, dict):
+                    continue
+                video_url = definition.get('videoUrl')
+                if isinstance(video_url, compat_str) and video_url:
+                    links.append(video_url)
+
+        # Fallback #1, this also contains extra low quality 180p format
+        for _, link in re.findall(r'<a[^>]+href=(["\'])(http.+?)\1[^>]+title=["\']Download [Vv]ideo', webpage):
+            links.append(link)
+
+        # Fallback #2 (unavailable as at 22.06.2017)
         sources = self._search_regex(
             r'(?s)sources\s*:\s*({.+?})', webpage, 'sources', default=None)
         if sources:
             for _, link in re.findall(r'[^:]+\s*:\s*(["\'])(http.+?)\1', sources):
                 links.append(link)
 
-        # Fallback #1
+        # Fallback #3 (unavailable as at 22.06.2017)
         for _, link in re.findall(
-                r'(?:videoUrl|videoSrc|videoIpadUrl|html5PlayerSrc)\s*[:=]\s*(["\'])(http.+?)\1', webpage):
+                r'(?:videoSrc|videoIpadUrl|html5PlayerSrc)\s*[:=]\s*(["\'])(http.+?)\1', webpage):
             links.append(link)
 
-        # Fallback #2, this also contains extra low quality 180p format
-        for _, link in re.findall(r'<a[^>]+href=(["\'])(http.+?)\1[^>]+title=["\']Download [Vv]ideo', webpage):
-            links.append(link)
-
-        # Fallback #3, encrypted links
+        # Fallback #4, encrypted links (unavailable as at 22.06.2017)
         for _, encrypted_link in re.findall(
                 r'encryptedQuality\d{3,4}URL\s*=\s*(["\'])([\da-zA-Z+/=]+)\1', webpage):
             links.append(aes_decrypt_text(encrypted_link, title, 32).decode('utf-8'))

From 18166bb8e8db6bbeb1f279e236b9808e7d197dd8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 22 Jun 2017 00:47:02 +0700
Subject: [PATCH 458/611] [youporn] Fix upload date extraction

---
 youtube_dl/extractor/youporn.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 64e3c3c94..547adefeb 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -27,7 +27,7 @@ class YouPornIE(InfoExtractor):
             'description': 'Love & Sex Answers: http://bit.ly/DanAndJenn -- Is It Unhealthy To Masturbate Daily?',
             'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': 'Ask Dan And Jennifer',
-            'upload_date': '20101221',
+            'upload_date': '20101217',
             'average_rating': int,
             'view_count': int,
             'comment_count': int,
@@ -46,7 +46,7 @@ class YouPornIE(InfoExtractor):
             'description': 'http://sweetlivegirls.com Big Tits Awesome Brunette On amazing webcam show.mp4',
             'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': 'Unknown',
-            'upload_date': '20111125',
+            'upload_date': '20110418',
             'average_rating': int,
             'view_count': int,
             'comment_count': int,
@@ -143,7 +143,8 @@ class YouPornIE(InfoExtractor):
             r'(?s)<div[^>]+class=["\']submitByLink["\'][^>]*>(.+?)</div>',
             webpage, 'uploader', fatal=False)
         upload_date = unified_strdate(self._html_search_regex(
-            r'(?s)<div[^>]+class=["\']videoInfo(?:Date|Time)["\'][^>]*>(.+?)</div>',
+            [r'Date\s+[Aa]dded:\s*<span>([^<]+)',
+             r'(?s)<div[^>]+class=["\']videoInfo(?:Date|Time)["\'][^>]*>(.+?)</div>'],
             webpage, 'upload date', fatal=False))
 
         age_limit = self._rta_search(webpage)

From 0f4a5a73e70172c0accbd2e936d08988d065b3b1 Mon Sep 17 00:00:00 2001
From: Parmjit Virk <pvirk@mts.net>
Date: Thu, 22 Jun 2017 12:08:36 -0500
Subject: [PATCH 459/611] [drtuber] Fix formats extraction (fixes 12058)

---
 youtube_dl/extractor/drtuber.py | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/drtuber.py b/youtube_dl/extractor/drtuber.py
index 1eca82b3b..c5d56a9ad 100644
--- a/youtube_dl/extractor/drtuber.py
+++ b/youtube_dl/extractor/drtuber.py
@@ -44,8 +44,23 @@ class DrTuberIE(InfoExtractor):
         webpage = self._download_webpage(
             'http://www.drtuber.com/video/%s' % video_id, display_id)
 
-        video_url = self._html_search_regex(
-            r'<source src="([^"]+)"', webpage, 'video URL')
+        video_data = self._download_json(
+            'http://www.drtuber.com/player_config_json/', video_id, query={
+                'vid': video_id,
+                'embed': 0,
+                'aid': 0,
+                'domain_id': 0,
+            })
+
+        formats = []
+        for format_id, video_url in video_data['files'].items():
+            if video_url:
+                formats.append({
+                    'format_id': format_id,
+                    'quality': 2 if format_id == 'hq' else 1,
+                    'url': video_url
+                })
+        self._sort_formats(formats)
 
         title = self._html_search_regex(
             (r'class="title_watch"[^>]*><(?:p|h\d+)[^>]*>([^<]+)<',
@@ -75,7 +90,7 @@ class DrTuberIE(InfoExtractor):
         return {
             'id': video_id,
             'display_id': display_id,
-            'url': video_url,
+            'formats': formats,
             'title': title,
             'thumbnail': thumbnail,
             'like_count': like_count,

From fa3ea7223ac4d547c848e2df44504158ee0099f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jun 2017 00:42:42 +0700
Subject: [PATCH 460/611] [hgtv.com:show] Relax video config regex and update
 test (closes #13279, closes #13461)

---
 youtube_dl/extractor/hgtv.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/hgtv.py b/youtube_dl/extractor/hgtv.py
index e854300c7..4d4b06824 100644
--- a/youtube_dl/extractor/hgtv.py
+++ b/youtube_dl/extractor/hgtv.py
@@ -7,14 +7,19 @@ from .common import InfoExtractor
 class HGTVComShowIE(InfoExtractor):
     IE_NAME = 'hgtv.com:show'
     _VALID_URL = r'https?://(?:www\.)?hgtv\.com/shows/[^/]+/(?P<id>[^/?#&]+)'
-    _TEST = {
-        'url': 'http://www.hgtv.com/shows/flip-or-flop/flip-or-flop-full-episodes-videos',
+    _TESTS = [{
+        # data-module="video"
+        'url': 'http://www.hgtv.com/shows/flip-or-flop/flip-or-flop-full-episodes-season-4-videos',
         'info_dict': {
-            'id': 'flip-or-flop-full-episodes-videos',
+            'id': 'flip-or-flop-full-episodes-season-4-videos',
             'title': 'Flip or Flop Full Episodes',
         },
         'playlist_mincount': 15,
-    }
+    }, {
+        # data-deferred-module="video"
+        'url': 'http://www.hgtv.com/shows/good-bones/episodes/an-old-victorian-house-gets-a-new-facelift',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
@@ -23,7 +28,7 @@ class HGTVComShowIE(InfoExtractor):
 
         config = self._parse_json(
             self._search_regex(
-                r'(?s)data-module=["\']video["\'][^>]*>.*?<script[^>]+type=["\']text/x-config["\'][^>]*>(.+?)</script',
+                r'(?s)data-(?:deferred)?-module=["\']video["\'][^>]*>.*?<script[^>]+type=["\']text/x-config["\'][^>]*>(.+?)</script',
                 webpage, 'video config'),
             display_id)['channels'][0]
 

From ddbb4c5c3ea7a64e09834d9628e7e34e03675289 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jun 2017 02:00:19 +0700
Subject: [PATCH 461/611] [youtube] Adapt to new automatic captions rendition
 (closes #13467)

---
 youtube_dl/extractor/youtube.py | 62 ++++++++++++++++++++++-----------
 1 file changed, 41 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index bf4f4e139..77cd271ef 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1269,37 +1269,57 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     sub_lang_list[sub_lang] = sub_formats
                 return sub_lang_list
 
+            def make_captions(sub_url, sub_langs):
+                parsed_sub_url = compat_urllib_parse_urlparse(sub_url)
+                caption_qs = compat_parse_qs(parsed_sub_url.query)
+                captions = {}
+                for sub_lang in sub_langs:
+                    sub_formats = []
+                    for ext in self._SUBTITLE_FORMATS:
+                        caption_qs.update({
+                            'tlang': [sub_lang],
+                            'fmt': [ext],
+                        })
+                        sub_url = compat_urlparse.urlunparse(parsed_sub_url._replace(
+                            query=compat_urllib_parse_urlencode(caption_qs, True)))
+                        sub_formats.append({
+                            'url': sub_url,
+                            'ext': ext,
+                        })
+                    captions[sub_lang] = sub_formats
+                return captions
+
+            # New captions format as of 22.06.2017
+            player_response = args.get('player_response')
+            if player_response and isinstance(player_response, compat_str):
+                player_response = self._parse_json(
+                    player_response, video_id, fatal=False)
+                if player_response:
+                    renderer = player_response['captions']['playerCaptionsTracklistRenderer']
+                    base_url = renderer['captionTracks'][0]['baseUrl']
+                    sub_lang_list = []
+                    for lang in renderer['translationLanguages']:
+                        lang_code = lang.get('languageCode')
+                        if lang_code:
+                            sub_lang_list.append(lang_code)
+                    return make_captions(base_url, sub_lang_list)
+
             # Some videos don't provide ttsurl but rather caption_tracks and
             # caption_translation_languages (e.g. 20LmZk1hakA)
+            # Does not used anymore as of 22.06.2017
             caption_tracks = args['caption_tracks']
             caption_translation_languages = args['caption_translation_languages']
             caption_url = compat_parse_qs(caption_tracks.split(',')[0])['u'][0]
-            parsed_caption_url = compat_urllib_parse_urlparse(caption_url)
-            caption_qs = compat_parse_qs(parsed_caption_url.query)
-
-            sub_lang_list = {}
+            sub_lang_list = []
             for lang in caption_translation_languages.split(','):
                 lang_qs = compat_parse_qs(compat_urllib_parse_unquote_plus(lang))
                 sub_lang = lang_qs.get('lc', [None])[0]
-                if not sub_lang:
-                    continue
-                sub_formats = []
-                for ext in self._SUBTITLE_FORMATS:
-                    caption_qs.update({
-                        'tlang': [sub_lang],
-                        'fmt': [ext],
-                    })
-                    sub_url = compat_urlparse.urlunparse(parsed_caption_url._replace(
-                        query=compat_urllib_parse_urlencode(caption_qs, True)))
-                    sub_formats.append({
-                        'url': sub_url,
-                        'ext': ext,
-                    })
-                sub_lang_list[sub_lang] = sub_formats
-            return sub_lang_list
+                if sub_lang:
+                    sub_lang_list.append(sub_lang)
+            return make_captions(caption_url, sub_lang_list)
         # An extractor error can be raise by the download process if there are
         # no automatic captions but there are subtitles
-        except (KeyError, ExtractorError):
+        except (KeyError, IndexError, ExtractorError):
             self._downloader.report_warning(err_msg)
             return {}
 

From 38dad4737fd18ff380d38ce85ad200d1d0931b8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jun 2017 02:10:54 +0700
Subject: [PATCH 462/611] [ChangeLog] Actualize

---
 ChangeLog | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 7f077fd24..f12e74f1d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,22 @@
 version <unreleased>
 
+Core
+* [adobepass] Fix extraction on older python 2.6
+
 Extractors
+* [youtube] Adapt to new automatic captions rendition (#13467)
+* [hgtv.com:show] Relax video config regular expression (#13279, #13461)
+* [drtuber] Fix formats extraction (#12058)
+* [youporn] Fix upload date extraction
+* [youporn] Improve formats extraction
+* [youporn] Fix title extraction (#13456)
+* [googledrive] Fix formats sorting (#13443)
+* [watchindianporn] Fix extraction (#13411, #13415)
++ [vimeo] Add fallback mp4 extension for original format
++ [ruv] Add support for ruv.is (#13396)
+* [viu] Fix extraction on older python 2.6
 * [pandora.tv] Fix upload_date extraction (#12846)
++ [asiancrush] Add support for asiancrush.com (#13420)
 
 
 version 2017.06.18

From 170719414da3b516af464c4b0fb8a6a94c398e14 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jun 2017 02:13:21 +0700
Subject: [PATCH 463/611] release 2017.06.23

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 +++
 youtube_dl/version.py     | 2 +-
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 002c1274a..4e7ceafd8 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.06.18*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.06.18**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.06.23*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.06.23**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.06.18
+[debug] youtube-dl version 2017.06.23
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index f12e74f1d..746250db9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.06.23
 
 Core
 * [adobepass] Fix extraction on older python 2.6
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 0f21be0a2..e827ec0cf 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -67,6 +67,8 @@
  - **arte.tv:info**
  - **arte.tv:magazine**
  - **arte.tv:playlist**
+ - **AsianCrush**
+ - **AsianCrushPlaylist**
  - **AtresPlayer**
  - **ATTTechChannel**
  - **ATVAt**
@@ -686,6 +688,7 @@
  - **rutube:person**: Rutube person videos
  - **RUTV**: RUTV.RU
  - **Ruutu**
+ - **Ruv**
  - **safari**: safaribooksonline.com online video
  - **safari:api**
  - **safari:course**: safaribooksonline.com online courses
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8782a6a1e..dfb69ab8d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.06.18'
+__version__ = '2017.06.23'

From ac7409eec5be3e120928e66af0d5fd7f38a06c5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jun 2017 02:54:12 +0700
Subject: [PATCH 464/611] [hgtv.com:show] Fix typo

---
 youtube_dl/extractor/hgtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/hgtv.py b/youtube_dl/extractor/hgtv.py
index 4d4b06824..a4f332565 100644
--- a/youtube_dl/extractor/hgtv.py
+++ b/youtube_dl/extractor/hgtv.py
@@ -28,7 +28,7 @@ class HGTVComShowIE(InfoExtractor):
 
         config = self._parse_json(
             self._search_regex(
-                r'(?s)data-(?:deferred)?-module=["\']video["\'][^>]*>.*?<script[^>]+type=["\']text/x-config["\'][^>]*>(.+?)</script',
+                r'(?s)data-(?:deferred-)?module=["\']video["\'][^>]*>.*?<script[^>]+type=["\']text/x-config["\'][^>]*>(.+?)</script',
                 webpage, 'video config'),
             display_id)['channels'][0]
 

From 4cb18ab1b96f1302f482f73989e263933e71d2e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jun 2017 20:50:48 +0700
Subject: [PATCH 465/611] [ooyala] Skip empty format URLs (closes #13471,
 closes #13476)

---
 youtube_dl/extractor/ooyala.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 84be2b1e3..ef8adccc1 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -41,7 +41,7 @@ class OoyalaBaseIE(InfoExtractor):
             for stream in cur_auth_data['streams']:
                 s_url = base64.b64decode(
                     stream['url']['data'].encode('ascii')).decode('utf-8')
-                if s_url in urls:
+                if not s_url or s_url in urls:
                     continue
                 urls.append(s_url)
                 ext = determine_ext(s_url, None)

From 4f4dd8d797538476a5a97f90e7074df2105d7573 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jun 2017 20:56:21 +0700
Subject: [PATCH 466/611] [ooyala] Make more robust

---
 youtube_dl/extractor/ooyala.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index ef8adccc1..a25f326b5 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -3,12 +3,14 @@ import re
 import base64
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
-    int_or_none,
-    float_or_none,
-    ExtractorError,
-    unsmuggle_url,
     determine_ext,
+    ExtractorError,
+    float_or_none,
+    int_or_none,
+    try_get,
+    unsmuggle_url,
 )
 from ..compat import compat_urllib_parse_urlencode
 
@@ -39,13 +41,15 @@ class OoyalaBaseIE(InfoExtractor):
         formats = []
         if cur_auth_data['authorized']:
             for stream in cur_auth_data['streams']:
-                s_url = base64.b64decode(
-                    stream['url']['data'].encode('ascii')).decode('utf-8')
+                url_data = try_get(stream, lambda x: x['url']['data'], compat_str)
+                if not url_data:
+                    continue
+                s_url = base64.b64decode(url_data.encode('ascii')).decode('utf-8')
                 if not s_url or s_url in urls:
                     continue
                 urls.append(s_url)
                 ext = determine_ext(s_url, None)
-                delivery_type = stream['delivery_type']
+                delivery_type = stream.get('delivery_type')
                 if delivery_type == 'hls' or ext == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
                         re.sub(r'/ip(?:ad|hone)/', '/all/', s_url), embed_code, 'mp4', 'm3u8_native',
@@ -65,7 +69,7 @@ class OoyalaBaseIE(InfoExtractor):
                 else:
                     formats.append({
                         'url': s_url,
-                        'ext': ext or stream.get('delivery_type'),
+                        'ext': ext or delivery_type,
                         'vcodec': stream.get('video_codec'),
                         'format_id': delivery_type,
                         'width': int_or_none(stream.get('width')),

From b5f523ed62f6c84fe0c58274f1751e66c58282d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jun 2017 20:56:48 +0700
Subject: [PATCH 467/611] [ooyala] Add test for missing stream['url']['data']

---
 youtube_dl/extractor/ooyala.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index a25f326b5..52580baed 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -140,6 +140,11 @@ class OoyalaIE(OoyalaBaseIE):
                 'title': 'Divide Tool Path.mp4',
                 'duration': 204.405,
             }
+        },
+        {
+            # empty stream['url']['data']
+            'url': 'http://player.ooyala.com/player.js?embedCode=w2bnZtYjE6axZ_dw1Cd0hQtXd_ige2Is',
+            'only_matching': True,
         }
     ]
 

From 73af5cc817ff19d21cb432c5a4e9e37dd35a353d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jun 2017 21:18:33 +0700
Subject: [PATCH 468/611] [YoutubeDL] Skip malformed formats for better
 extraction robustness

---
 youtube_dl/YoutubeDL.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index c05103bb6..b3a6d4d3b 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1448,17 +1448,25 @@ class YoutubeDL(object):
         if not formats:
             raise ExtractorError('No video formats found!')
 
+        def is_wellformed(f):
+            url = f.get('url')
+            valid_url = url and isinstance(url, compat_str)
+            if not valid_url:
+                self.report_warning(
+                    '"url" field is missing or empty - skipping format, '
+                    'there is an error in extractor')
+            return valid_url
+
+        # Filter out malformed formats for better extraction robustness
+        formats = list(filter(is_wellformed, formats))
+
         formats_dict = {}
 
         # We check that all the formats have the format and format_id fields
         for i, format in enumerate(formats):
-            if 'url' not in format:
-                raise ExtractorError('Missing "url" key in result (index %d)' % i)
-
             sanitize_string_field(format, 'format_id')
             sanitize_numeric_fields(format)
             format['url'] = sanitize_url(format['url'])
-
             if format.get('format_id') is None:
                 format['format_id'] = compat_str(i)
             else:

From bd65f181532ab4e535b408d3ccf99723534eb326 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Jun 2017 18:33:31 +0700
Subject: [PATCH 469/611] [onetpl] Add support for videos embedded via
 pulsembed (closes #13482)

---
 youtube_dl/extractor/onet.py | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/onet.py b/youtube_dl/extractor/onet.py
index 94f57990b..58da1bc27 100644
--- a/youtube_dl/extractor/onet.py
+++ b/youtube_dl/extractor/onet.py
@@ -11,6 +11,7 @@ from ..utils import (
     get_element_by_class,
     int_or_none,
     js_to_json,
+    NO_DEFAULT,
     parse_iso8601,
     remove_start,
     strip_or_none,
@@ -198,6 +199,19 @@ class OnetPlIE(InfoExtractor):
             'upload_date': '20170214',
             'timestamp': 1487078046,
         },
+    }, {
+        # embedded via pulsembed
+        'url': 'http://film.onet.pl/pensjonat-nad-rozlewiskiem-relacja-z-planu-serialu/y428n0',
+        'info_dict': {
+            'id': '501235.965429946',
+            'ext': 'mp4',
+            'title': '"Pensjonat nad rozlewiskiem": relacja z planu serialu',
+            'upload_date': '20170622',
+            'timestamp': 1498159955,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'http://film.onet.pl/zwiastuny/ghost-in-the-shell-drugi-zwiastun-pl/5q6yl3',
         'only_matching': True,
@@ -212,13 +226,25 @@ class OnetPlIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    def _search_mvp_id(self, webpage, default=NO_DEFAULT):
+        return self._search_regex(
+            r'data-(?:params-)?mvp=["\'](\d+\.\d+)', webpage, 'mvp id',
+            default=default)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
-        mvp_id = self._search_regex(
-            r'data-params-mvp=["\'](\d+\.\d+)', webpage, 'mvp id')
+        mvp_id = self._search_mvp_id(webpage, default=None)
+
+        if not mvp_id:
+            pulsembed_url = self._search_regex(
+                r'data-src=(["\'])(?P<url>(?:https?:)?//pulsembed\.eu/.+?)\1',
+                webpage, 'pulsembed url', group='url')
+            webpage = self._download_webpage(
+                pulsembed_url, video_id, 'Downloading pulsembed webpage')
+            mvp_id = self._search_mvp_id(webpage)
 
         return self.url_result(
             'onetmvp:%s' % mvp_id, OnetMVPIE.ie_key(), video_id=mvp_id)

From 27449ad894f7d49e189686a15399ea66cfb63667 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Jun 2017 01:09:12 +0700
Subject: [PATCH 470/611] [redbulltv] Add support for lives and segments
 (closes #13486))

---
 youtube_dl/extractor/redbulltv.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/redbulltv.py b/youtube_dl/extractor/redbulltv.py
index afab62426..c5918afee 100644
--- a/youtube_dl/extractor/redbulltv.py
+++ b/youtube_dl/extractor/redbulltv.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class RedBullTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?redbull\.tv/(?:video|film)/(?P<id>AP-\w+)'
+    _VALID_URL = r'https?://(?:www\.)?redbull\.tv/(?:video|film|live)/(?:AP-\w+/segment/)?(?P<id>AP-\w+)'
     _TESTS = [{
         # film
         'url': 'https://www.redbull.tv/video/AP-1Q756YYX51W11/abc-of-wrc',
@@ -42,6 +42,22 @@ class RedBullTVIE(InfoExtractor):
             'season_number': 2,
             'episode_number': 4,
         },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        # segment
+        'url': 'https://www.redbull.tv/live/AP-1R5DX49XS1W11/segment/AP-1QSAQJ6V52111/semi-finals',
+        'info_dict': {
+            'id': 'AP-1QSAQJ6V52111',
+            'ext': 'mp4',
+            'title': 'Semi Finals - Vans Park Series Pro Tour',
+            'description': 'md5:306a2783cdafa9e65e39aa62f514fd97',
+            'duration': 11791.991,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'https://www.redbull.tv/film/AP-1MSKKF5T92111/in-motion',
         'only_matching': True,

From 23aec3d623146d06535a4f5388693c1e9a2bbfde Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Jun 2017 01:10:31 +0700
Subject: [PATCH 471/611] [redbulltv] Restore hls format prefix

---
 youtube_dl/extractor/redbulltv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/redbulltv.py b/youtube_dl/extractor/redbulltv.py
index c5918afee..5d6cc3610 100644
--- a/youtube_dl/extractor/redbulltv.py
+++ b/youtube_dl/extractor/redbulltv.py
@@ -98,7 +98,8 @@ class RedBullTVIE(InfoExtractor):
         title = info['title'].strip()
 
         formats = self._extract_m3u8_formats(
-            video['url'], video_id, 'mp4', 'm3u8_native')
+            video['url'], video_id, 'mp4', entry_protocol='m3u8_native',
+            m3u8_id='hls')
         self._sort_formats(formats)
 
         subtitles = {}

From 449c66577640a0c3f0b383204a1e7284429a61c3 Mon Sep 17 00:00:00 2001
From: james <jrabblac@gmail.com>
Date: Sat, 17 Jun 2017 17:15:41 +0200
Subject: [PATCH 472/611] [raiplay:live] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/rai.py        | 19 ++++++++++++++++++-
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e97691daa..a263c88b3 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -824,6 +824,7 @@ from .radiobremen import RadioBremenIE
 from .radiofrance import RadioFranceIE
 from .rai import (
     RaiPlayIE,
+    RaiPlayLiveIE,
     RaiIE,
 )
 from .rbmaradio import RBMARadioIE
diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 81eb9db85..ed15a5f10 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -208,10 +208,27 @@ class RaiPlayIE(RaiBaseIE):
         }
 
         info.update(relinker_info)
-
         return info
 
 
+class RaiPlayLiveIE(RaiBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?raiplay\.it/dirette/(?P<id>\w*)'
+    _TEST = {
+        'url': 'http://www.raiplay.it/dirette/rai3',
+        'only_matching': True,
+    }
+
+    def _real_extract(self, url):
+        channel = self._match_id(url)
+
+        webpage = self._download_webpage(url, channel)
+        re_id = r'<div([^>]*)data-uniquename=(["\'])[\w-]*(?P<id>%s)(\2)([^>]*?)>' % RaiBaseIE._UUID_RE
+        video_id = self._html_search_regex(re_id, webpage, 'livestream-id', group='id')
+
+        return self.url_result('http://www.raiplay.it/dirette/ContentItem-%s.html' % video_id,
+                               RaiPlayIE.ie_key(), video_id)
+
+
 class RaiIE(RaiBaseIE):
     _VALID_URL = r'https?://[^/]+\.(?:rai\.(?:it|tv)|rainews\.it)/dl/.+?-(?P<id>%s)(?:-.+?)?\.html' % RaiBaseIE._UUID_RE
     _TESTS = [{

From 9c48b5a193de754403f4a1ced78f6cf6b3893676 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Jun 2017 01:48:02 +0700
Subject: [PATCH 473/611] [raiplay:live] Improve and add test (closes #13414)

---
 youtube_dl/extractor/rai.py | 44 +++++++++++++++++++++++++++----------
 1 file changed, 32 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index ed15a5f10..e11bf8f9a 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -191,11 +191,12 @@ class RaiPlayIE(RaiBaseIE):
 
         info = {
             'id': video_id,
-            'title': title,
+            'title': self._live_title(title) if relinker_info.get(
+                'is_live') else title,
             'alt_title': media.get('subtitle'),
             'description': media.get('description'),
-            'uploader': media.get('channel'),
-            'creator': media.get('editor'),
+            'uploader': strip_or_none(media.get('channel')),
+            'creator': strip_or_none(media.get('editor')),
             'duration': parse_duration(video.get('duration')),
             'timestamp': timestamp,
             'thumbnails': thumbnails,
@@ -212,21 +213,40 @@ class RaiPlayIE(RaiBaseIE):
 
 
 class RaiPlayLiveIE(RaiBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?raiplay\.it/dirette/(?P<id>\w*)'
+    _VALID_URL = r'https?://(?:www\.)?raiplay\.it/dirette/(?P<id>[^/?#&]+)'
     _TEST = {
-        'url': 'http://www.raiplay.it/dirette/rai3',
-        'only_matching': True,
+        'url': 'http://www.raiplay.it/dirette/rainews24',
+        'info_dict': {
+            'id': 'd784ad40-e0ae-4a69-aa76-37519d238a9c',
+            'display_id': 'rainews24',
+            'ext': 'mp4',
+            'title': 're:^Diretta di Rai News 24 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 'md5:6eca31500550f9376819f174e5644754',
+            'uploader': 'Rai News 24',
+            'creator': 'Rai News 24',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):
-        channel = self._match_id(url)
+        display_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, channel)
-        re_id = r'<div([^>]*)data-uniquename=(["\'])[\w-]*(?P<id>%s)(\2)([^>]*?)>' % RaiBaseIE._UUID_RE
-        video_id = self._html_search_regex(re_id, webpage, 'livestream-id', group='id')
+        webpage = self._download_webpage(url, display_id)
 
-        return self.url_result('http://www.raiplay.it/dirette/ContentItem-%s.html' % video_id,
-                               RaiPlayIE.ie_key(), video_id)
+        video_id = self._search_regex(
+            r'data-uniquename=["\']ContentItem-(%s)' % RaiBaseIE._UUID_RE,
+            webpage, 'content id')
+
+        return {
+            '_type': 'url_transparent',
+            'ie_key': RaiPlayIE.ie_key(),
+            'url': 'http://www.raiplay.it/dirette/ContentItem-%s.html' % video_id,
+            'id': video_id,
+            'display_id': display_id,
+        }
 
 
 class RaiIE(RaiBaseIE):

From 5744cf6c03e6f89914d44be7c5d77fca3b121bef Mon Sep 17 00:00:00 2001
From: Argn0 <Argn0@users.noreply.github.com>
Date: Sat, 24 Jun 2017 20:59:15 +0200
Subject: [PATCH 474/611] [ign] Add another video id pattern (closes #13328)

---
 youtube_dl/extractor/ign.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index c45c68c1d..c1367cf51 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -89,6 +89,11 @@ class IGNIE(InfoExtractor):
             'url': 'http://me.ign.com/ar/angry-birds-2/106533/video/lrd-ldyy-lwl-lfylm-angry-birds',
             'only_matching': True,
         },
+        {
+            # videoId pattern
+            'url': 'http://www.ign.com/articles/2017/06/08/new-ducktales-short-donalds-birthday-doesnt-go-as-planned',
+            'only_matching': True,
+        },
     ]
 
     def _find_video_id(self, webpage):
@@ -98,6 +103,8 @@ class IGNIE(InfoExtractor):
             r'data-video-id="(.+?)"',
             r'<object id="vid_(.+?)"',
             r'<meta name="og:image" content=".*/(.+?)-(.+?)/.+.jpg"',
+            r'videoId&quot;\s*:\s*&quot;(.+?)&quot;',
+            r'videoId["\']\s*:\s*["\']([^"\']+?)["\']',
         ]
         return self._search_regex(res_id, webpage, 'video id', default=None)
 

From fd9ee4de8c3aa640d51c5ff42f5d8241970e894d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Jun 2017 02:14:10 +0700
Subject: [PATCH 475/611] [wsj] Add support for barrons.com (closes #13470)

---
 youtube_dl/extractor/wsj.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/wsj.py b/youtube_dl/extractor/wsj.py
index 45cfca7c5..9b5487710 100644
--- a/youtube_dl/extractor/wsj.py
+++ b/youtube_dl/extractor/wsj.py
@@ -13,7 +13,7 @@ class WSJIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                         (?:
                             https?://video-api\.wsj\.com/api-video/player/iframe\.html\?.*?\bguid=|
-                            https?://(?:www\.)?wsj\.com/video/[^/]+/|
+                            https?://(?:www\.)?(?:wsj|barrons)\.com/video/[^/]+/|
                             wsj:
                         )
                         (?P<id>[a-fA-F0-9-]{36})
@@ -35,6 +35,9 @@ class WSJIE(InfoExtractor):
     }, {
         'url': 'http://www.wsj.com/video/can-alphabet-build-a-smarter-city/359DDAA8-9AC1-489C-82E6-0429C1E430E0.html',
         'only_matching': True,
+    }, {
+        'url': 'http://www.barrons.com/video/capitalism-deserves-more-respect-from-millennials/F301217E-6F46-43AE-B8D2-B7180D642EE9.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 0c7a631b613fe8ec443e6b17ea8f7a17fb8abb5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Jun 2017 04:54:56 +0700
Subject: [PATCH 476/611] [adobepass] Add support for ATTOTT MSO (DIRECTV NOW)
 (closes #13472)

---
 youtube_dl/extractor/adobepass.py | 43 ++++++++++++++++++++++++++-----
 1 file changed, 37 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index 3dfc632e7..b83b51efb 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -15,6 +15,7 @@ from ..utils import (
     urlencode_postdata,
     unified_timestamp,
     ExtractorError,
+    NO_DEFAULT,
 )
 
 
@@ -24,6 +25,11 @@ MSO_INFO = {
         'username_field': 'username',
         'password_field': 'password',
     },
+    'ATTOTT': {
+        'name': 'DIRECTV NOW',
+        'username_field': 'email',
+        'password_field': 'loginpassword',
+    },
     'Rogers': {
         'name': 'Rogers',
         'username_field': 'UserName',
@@ -1316,6 +1322,8 @@ class AdobePassIE(InfoExtractor):
     _USER_AGENT = 'Mozilla/5.0 (X11; Linux i686; rv:47.0) Gecko/20100101 Firefox/47.0'
     _MVPD_CACHE = 'ap-mvpd'
 
+    _DOWNLOADING_LOGIN_PAGE = 'Downloading Provider Login Page'
+
     def _download_webpage_handle(self, *args, **kwargs):
         headers = kwargs.get('headers', {})
         headers.update(self.geo_verification_headers())
@@ -1365,6 +1373,21 @@ class AdobePassIE(InfoExtractor):
                 'Use --ap-mso to specify Adobe Pass Multiple-system operator Identifier '
                 'and --ap-username and --ap-password or --netrc to provide account credentials.', expected=True)
 
+        def extract_redirect_url(html, url=None, fatal=False):
+            # TODO: eliminate code duplication with generic extractor and move
+            # redirection code into _download_webpage_handle
+            REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)'
+            redirect_url = self._search_regex(
+                r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
+                r'(?:[a-z-]+="[^"]+"\s+)*?content="%s' % REDIRECT_REGEX,
+                html, 'meta refresh redirect',
+                default=NO_DEFAULT if fatal else None, fatal=fatal)
+            if not redirect_url:
+                return None
+            if url:
+                redirect_url = compat_urlparse.urljoin(url, unescapeHTML(redirect_url))
+            return redirect_url
+
         mvpd_headers = {
             'ap_42': 'anonymous',
             'ap_11': 'Linux i686',
@@ -1414,16 +1437,15 @@ class AdobePassIE(InfoExtractor):
                         if '<form name="signin"' in provider_redirect_page:
                             provider_login_page_res = provider_redirect_page_res
                         elif 'http-equiv="refresh"' in provider_redirect_page:
-                            oauth_redirect_url = self._html_search_regex(
-                                r'content="0;\s*url=([^\'"]+)',
-                                provider_redirect_page, 'meta refresh redirect')
+                            oauth_redirect_url = extract_redirect_url(
+                                provider_redirect_page, fatal=True)
                             provider_login_page_res = self._download_webpage_handle(
                                 oauth_redirect_url, video_id,
-                                'Downloading Provider Login Page')
+                                self._DOWNLOADING_LOGIN_PAGE)
                         else:
                             provider_login_page_res = post_form(
                                 provider_redirect_page_res,
-                                'Downloading Provider Login Page')
+                                self._DOWNLOADING_LOGIN_PAGE)
 
                         mvpd_confirm_page_res = post_form(
                             provider_login_page_res, 'Logging in', {
@@ -1470,8 +1492,17 @@ class AdobePassIE(InfoExtractor):
                             'Content-Type': 'application/x-www-form-urlencoded'
                         })
                 else:
+                    # Some providers (e.g. DIRECTV NOW) have another meta refresh
+                    # based redirect that should be followed.
+                    provider_redirect_page, urlh = provider_redirect_page_res
+                    provider_refresh_redirect_url = extract_redirect_url(
+                        provider_redirect_page, url=urlh.geturl())
+                    if provider_refresh_redirect_url:
+                        provider_redirect_page_res = self._download_webpage_handle(
+                            provider_refresh_redirect_url, video_id,
+                            'Downloading Provider Redirect Page (meta refresh)')
                     provider_login_page_res = post_form(
-                        provider_redirect_page_res, 'Downloading Provider Login Page')
+                        provider_redirect_page_res, self._DOWNLOADING_LOGIN_PAGE)
                     mvpd_confirm_page_res = post_form(provider_login_page_res, 'Logging in', {
                         mso_info.get('username_field', 'username'): username,
                         mso_info.get('password_field', 'password'): password,

From a5aea53fc88a87889a8ce4189e28c28386bb6bb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Jun 2017 05:13:12 +0700
Subject: [PATCH 477/611] [ChangeLog] Actualize

---
 ChangeLog | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 746250db9..d3343c760 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,20 @@
+version <unreleased>
+
+Core
++ [adobepass] Add support for DIRECTV NOW (mso ATTOTT) (#13472)
+* [YoutubeDL] Skip malformed formats for better extraction robustness
+
+Extractors
++ [wsj] Add support for barrons.com (#13470)
++ [ign] Add another video id pattern (#13328)
++ [raiplay:live] Add support for live streams (#13414)
++ [redbulltv] Add support for live videos and segments (#13486)
++ [onetpl] Add support for videos embedded via pulsembed (#13482)
+* [ooyala] Make more robust
+* [ooyala] Skip empty format URLs (#13471, #13476)
+* [hgtv.com:show] Fix typo
+
+
 version 2017.06.23
 
 Core

From a7ce8f16c4ee4b8f567d943ada8ade8a50f99860 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Jun 2017 05:16:06 +0700
Subject: [PATCH 478/611] release 2017.06.25

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 1 +
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 4e7ceafd8..82bbbdaa6 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.06.23*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.06.23**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.06.25*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.06.25**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.06.23
+[debug] youtube-dl version 2017.06.25
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index d3343c760..a0072ffe3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.06.25
 
 Core
 + [adobepass] Add support for DIRECTV NOW (mso ATTOTT) (#13472)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index e827ec0cf..010ff762c 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -644,6 +644,7 @@
  - **RadioJavan**
  - **Rai**
  - **RaiPlay**
+ - **RaiPlayLive**
  - **RBMARadio**
  - **RDS**: RDS.ca
  - **RedBullTV**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index dfb69ab8d..b6d378896 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.06.23'
+__version__ = '2017.06.25'

From 4a87de72dfbc7a0ad9496b225a865907715a7bad Mon Sep 17 00:00:00 2001
From: Luca Steeb <contact@luca-steeb.com>
Date: Sun, 25 Jun 2017 21:30:05 +0200
Subject: [PATCH 479/611] [niconico] fix sp subdomain links

---
 youtube_dl/extractor/niconico.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 8baac23e4..f268a72d5 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -83,9 +83,12 @@ class NiconicoIE(InfoExtractor):
             'uploader_id': '312',
         },
         'skip': 'The viewing period of the video you were searching for has expired.',
+    }, {
+        'url': 'http://sp.nicovideo.jp/watch/sm28964488?ss_pos=1&cp_in=wt_tg',
+        'only_matching': True,
     }]
 
-    _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.|secure\.|sp\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)'
     _NETRC_MACHINE = 'niconico'
 
     def _real_initialize(self):

From 136503e302995c4264df7cfccb62fdae94a2c680 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 26 Jun 2017 19:56:07 +0800
Subject: [PATCH 480/611] [ChangeLog] Update after #13494

---
 ChangeLog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index a0072ffe3..7b3c6c6cf 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
++ [niconico] Support sp subdomain (#13494)
+
+
 version 2017.06.25
 
 Core

From 35544690e4f4fb1b8184e57d9c6c54fe893e6a3b Mon Sep 17 00:00:00 2001
From: "Xuan Hu (Sean)" <i+github@huxuan.org>
Date: Mon, 26 Jun 2017 23:00:31 +0800
Subject: [PATCH 481/611] [pandatv] Add support for https URLs

---
 youtube_dl/extractor/pandatv.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/pandatv.py b/youtube_dl/extractor/pandatv.py
index 133cc9b88..a5ed49449 100644
--- a/youtube_dl/extractor/pandatv.py
+++ b/youtube_dl/extractor/pandatv.py
@@ -10,13 +10,13 @@ from ..utils import (
 
 class PandaTVIE(InfoExtractor):
     IE_DESC = '熊猫TV'
-    _VALID_URL = r'http://(?:www\.)?panda\.tv/(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'http://www.panda.tv/10091',
+    _VALID_URL = r'https?://(?:www\.)?panda\.tv/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://www.panda.tv/66666',
         'info_dict': {
-            'id': '10091',
+            'id': '66666',
             'title': 're:.+',
-            'uploader': '囚徒',
+            'uploader': '刘杀鸡',
             'ext': 'flv',
             'is_live': True,
         },
@@ -24,7 +24,10 @@ class PandaTVIE(InfoExtractor):
             'skip_download': True,
         },
         'skip': 'Live stream is offline',
-    }
+    }, {
+        'url': 'https://www.panda.tv/66666',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 33ffb645a6881744f8befcb813ef1f5b6958d385 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Jun 2017 22:11:09 +0700
Subject: [PATCH 482/611] [pandatv] Switch to https for API and download URLs

---
 youtube_dl/extractor/pandatv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pandatv.py b/youtube_dl/extractor/pandatv.py
index a5ed49449..c86d70771 100644
--- a/youtube_dl/extractor/pandatv.py
+++ b/youtube_dl/extractor/pandatv.py
@@ -33,7 +33,7 @@ class PandaTVIE(InfoExtractor):
         video_id = self._match_id(url)
 
         config = self._download_json(
-            'http://www.panda.tv/api_room?roomid=%s' % video_id, video_id)
+            'https://www.panda.tv/api_room?roomid=%s' % video_id, video_id)
 
         error_code = config.get('errno', 0)
         if error_code is not 0:
@@ -77,7 +77,7 @@ class PandaTVIE(InfoExtractor):
                 continue
             for pref, (ext, pl) in enumerate((('m3u8', '-hls'), ('flv', ''))):
                 formats.append({
-                    'url': 'http://pl%s%s.live.panda.tv/live_panda/%s%s%s.%s'
+                    'url': 'https://pl%s%s.live.panda.tv/live_panda/%s%s%s.%s'
                     % (pl, plflag1, room_key, live_panda, suffix[quality], ext),
                     'format_id': '%s-%s' % (k, ext),
                     'quality': quality,

From f1c051009bcf52525df22eb3a59797076551a579 Mon Sep 17 00:00:00 2001
From: Viktor Szakats <vszakats@users.noreply.github.com>
Date: Tue, 27 Jun 2017 09:20:18 -0500
Subject: [PATCH 483/611] [soundcloud] Switch to https for API requests

---
 youtube_dl/extractor/soundcloud.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 0ee4a8ff8..994c86145 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -136,7 +136,7 @@ class SoundcloudIE(InfoExtractor):
 
     @classmethod
     def _resolv_url(cls, url):
-        return 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=' + cls._CLIENT_ID
+        return 'https://api.soundcloud.com/resolve.json?url=' + url + '&client_id=' + cls._CLIENT_ID
 
     def _extract_info_dict(self, info, full_title=None, quiet=False, secret_token=None):
         track_id = compat_str(info['id'])
@@ -174,7 +174,7 @@ class SoundcloudIE(InfoExtractor):
 
         # We have to retrieve the url
         format_dict = self._download_json(
-            'http://api.soundcloud.com/i1/tracks/%s/streams' % track_id,
+            'https://api.soundcloud.com/i1/tracks/%s/streams' % track_id,
             track_id, 'Downloading track url', query={
                 'client_id': self._CLIENT_ID,
                 'secret_token': secret_token,
@@ -236,7 +236,7 @@ class SoundcloudIE(InfoExtractor):
         track_id = mobj.group('track_id')
 
         if track_id is not None:
-            info_json_url = 'http://api.soundcloud.com/tracks/' + track_id + '.json?client_id=' + self._CLIENT_ID
+            info_json_url = 'https://api.soundcloud.com/tracks/' + track_id + '.json?client_id=' + self._CLIENT_ID
             full_title = track_id
             token = mobj.group('secret_token')
             if token:
@@ -261,7 +261,7 @@ class SoundcloudIE(InfoExtractor):
 
             self.report_resolve(full_title)
 
-            url = 'http://soundcloud.com/%s' % resolve_title
+            url = 'https://soundcloud.com/%s' % resolve_title
             info_json_url = self._resolv_url(url)
         info = self._download_json(info_json_url, full_title, 'Downloading info JSON')
 
@@ -304,7 +304,7 @@ class SoundcloudSetIE(SoundcloudPlaylistBaseIE):
         # extract simple title (uploader + slug of song title)
         slug_title = mobj.group('slug_title')
         full_title = '%s/sets/%s' % (uploader, slug_title)
-        url = 'http://soundcloud.com/%s/sets/%s' % (uploader, slug_title)
+        url = 'https://soundcloud.com/%s/sets/%s' % (uploader, slug_title)
 
         token = mobj.group('token')
         if token:
@@ -410,7 +410,7 @@ class SoundcloudUserIE(SoundcloudPlaylistBaseIE):
         mobj = re.match(self._VALID_URL, url)
         uploader = mobj.group('user')
 
-        url = 'http://soundcloud.com/%s/' % uploader
+        url = 'https://soundcloud.com/%s/' % uploader
         resolv_url = self._resolv_url(url)
         user = self._download_json(
             resolv_url, uploader, 'Downloading user info')
@@ -473,7 +473,7 @@ class SoundcloudPlaylistIE(SoundcloudPlaylistBaseIE):
     _VALID_URL = r'https?://api\.soundcloud\.com/playlists/(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))?$'
     IE_NAME = 'soundcloud:playlist'
     _TESTS = [{
-        'url': 'http://api.soundcloud.com/playlists/4110309',
+        'url': 'https://api.soundcloud.com/playlists/4110309',
         'info_dict': {
             'id': '4110309',
             'title': 'TILT Brass - Bowery Poetry Club, August \'03 [Non-Site SCR 02]',

From bf2dc9cc6e1ca38955f646b67f0f6ec40aec139d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 27 Jun 2017 21:26:46 +0700
Subject: [PATCH 484/611] [soundcloud] Fix tests

---
 youtube_dl/extractor/soundcloud.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 994c86145..3f1a46bb2 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -290,7 +290,7 @@ class SoundcloudSetIE(SoundcloudPlaylistBaseIE):
             'id': '2284613',
             'title': 'The Royal Concept EP',
         },
-        'playlist_mincount': 6,
+        'playlist_mincount': 5,
     }, {
         'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep/token',
         'only_matching': True,
@@ -380,7 +380,7 @@ class SoundcloudUserIE(SoundcloudPlaylistBaseIE):
         'url': 'https://soundcloud.com/grynpyret/spotlight',
         'info_dict': {
             'id': '7098329',
-            'title': 'GRYNPYRET (Spotlight)',
+            'title': 'Grynpyret (Spotlight)',
         },
         'playlist_mincount': 1,
     }]

From 0646e34c7d511a02d8d93e840bceaa3521c2204e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 27 Jun 2017 22:25:34 +0700
Subject: [PATCH 485/611] [facebook] Add support for plugin video embeds and
 multiple embeds (closes #13493)

---
 youtube_dl/extractor/buzzfeed.py |  7 ++++---
 youtube_dl/extractor/facebook.py | 20 ++++++++++----------
 youtube_dl/extractor/generic.py  |  6 +++---
 3 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/buzzfeed.py b/youtube_dl/extractor/buzzfeed.py
index 75fa92d7c..ec411091e 100644
--- a/youtube_dl/extractor/buzzfeed.py
+++ b/youtube_dl/extractor/buzzfeed.py
@@ -84,9 +84,10 @@ class BuzzFeedIE(InfoExtractor):
                 continue
             entries.append(self.url_result(video['url']))
 
-        facebook_url = FacebookIE._extract_url(webpage)
-        if facebook_url:
-            entries.append(self.url_result(facebook_url))
+        facebook_urls = FacebookIE._extract_urls(webpage)
+        entries.extend([
+            self.url_result(facebook_url)
+            for facebook_url in facebook_urls])
 
         return {
             '_type': 'playlist',
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index b69c1ede0..4b3f6cc86 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -203,19 +203,19 @@ class FacebookIE(InfoExtractor):
     }]
 
     @staticmethod
-    def _extract_url(webpage):
-        mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>https://www\.facebook\.com/video/embed.+?)\1', webpage)
-        if mobj is not None:
-            return mobj.group('url')
-
+    def _extract_urls(webpage):
+        urls = []
+        for mobj in re.finditer(
+                r'<iframe[^>]+?src=(["\'])(?P<url>https?://www\.facebook\.com/(?:video/embed|plugins/video\.php).+?)\1',
+                webpage):
+            urls.append(mobj.group('url'))
         # Facebook API embed
         # see https://developers.facebook.com/docs/plugins/embedded-video-player
-        mobj = re.search(r'''(?x)<div[^>]+
+        for mobj in re.finditer(r'''(?x)<div[^>]+
                 class=(?P<q1>[\'"])[^\'"]*\bfb-(?:video|post)\b[^\'"]*(?P=q1)[^>]+
-                data-href=(?P<q2>[\'"])(?P<url>(?:https?:)?//(?:www\.)?facebook.com/.+?)(?P=q2)''', webpage)
-        if mobj is not None:
-            return mobj.group('url')
+                data-href=(?P<q2>[\'"])(?P<url>(?:https?:)?//(?:www\.)?facebook.com/.+?)(?P=q2)''', webpage):
+            urls.append(mobj.group('url'))
+        return urls
 
     def _login(self):
         (useremail, password) = self._get_login_info()
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8ef1a2980..760a7f9c2 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2222,9 +2222,9 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'))
 
         # Look for embedded Facebook player
-        facebook_url = FacebookIE._extract_url(webpage)
-        if facebook_url is not None:
-            return self.url_result(facebook_url, 'Facebook')
+        facebook_urls = FacebookIE._extract_urls(webpage)
+        if facebook_urls:
+            return self.playlist_from_matches(facebook_urls, video_id, video_title)
 
         # Look for embedded VK player
         mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://vk\.com/video_ext\.php.+?)\1', webpage)

From fd1c5fba6bee4de36cc3276d86c84c0abc054a2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 27 Jun 2017 22:36:54 +0700
Subject: [PATCH 486/611] [facebook] Add test for plugin video embed (#13493)

---
 youtube_dl/extractor/generic.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 760a7f9c2..2792ea3cf 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1522,6 +1522,21 @@ class GenericIE(InfoExtractor):
                 'title': 'Facebook video #599637780109885',
             },
         },
+        # Facebook <iframe> embed, plugin video
+        {
+            'url': 'http://5pillarsuk.com/2017/06/07/tariq-ramadan-disagrees-with-pr-exercise-by-imams-refusing-funeral-prayers-for-london-attackers/',
+            'info_dict': {
+                'id': '1754168231264132',
+                'ext': 'mp4',
+                'title': 'About the Imams and Religious leaders refusing to perform funeral prayers for...',
+                'uploader': 'Tariq Ramadan (official)',
+                'timestamp': 1496758379,
+                'upload_date': '20170606',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         # Facebook API embed
         {
             'url': 'http://www.lothype.com/blue-stars-2016-preview-standstill-full-show/',

From 88d6b7c2bde239ad3593de54ab894ff43020142d Mon Sep 17 00:00:00 2001
From: Parmjit Virk <pvirk@mts.net>
Date: Wed, 28 Jun 2017 10:21:35 -0500
Subject: [PATCH 487/611] [cbsnews] Relax video info regex (fixes #13284)

---
 youtube_dl/extractor/cbsnews.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 5b61c3f52..ef5911bd6 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -52,6 +52,21 @@ class CBSNewsIE(CBSIE):
                 'skip_download': True,
             },
         },
+        {
+            'url': 'http://www.cbsnews.com/news/maria-ridulph-murder-will-the-nations-oldest-cold-case-to-go-to-trial-ever-get-solved/',
+            'info_dict': {
+                'id': 'QpM5BJjBVEAUFi7ydR9LusS69DPLqPJ1',
+                'ext': 'mp4',
+                'title': 'Cold as Ice',
+                'description': 'Can a childhood memory of a friend\'s murder solve a 1957 cold case? "48 Hours" correspondent Erin Moriarty has the latest.',
+                'upload_date': '20170604',
+                'timestamp': 1496538000,
+                'uploader': 'CBSI-NEW',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
     ]
 
     def _real_extract(self, url):
@@ -60,7 +75,7 @@ class CBSNewsIE(CBSIE):
         webpage = self._download_webpage(url, video_id)
 
         video_info = self._parse_json(self._html_search_regex(
-            r'(?:<ul class="media-list items" id="media-related-items"><li data-video-info|<div id="cbsNewsVideoPlayer" data-video-player-options)=\'({.+?})\'',
+            r'(?:<ul class="media-list items" id="media-related-items"[^>]*><li data-video-info|<div id="cbsNewsVideoPlayer" data-video-player-options)=\'({.+?})\'',
             webpage, 'video JSON info', default='{}'), video_id, fatal=False)
 
         if video_info:

From a919ca0ad60c4ef6165fceadeb9f6ad70b53e7a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Jun 2017 22:30:12 +0700
Subject: [PATCH 488/611] [cbsnews] Actualize test

---
 youtube_dl/extractor/cbsnews.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index ef5911bd6..51df15fac 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -15,19 +15,23 @@ class CBSNewsIE(CBSIE):
 
     _TESTS = [
         {
-            'url': 'http://www.cbsnews.com/news/tesla-and-spacex-elon-musks-industrial-empire/',
+            # 60 minutes
+            'url': 'http://www.cbsnews.com/news/artificial-intelligence-positioned-to-be-a-game-changer/',
             'info_dict': {
-                'id': 'tesla-and-spacex-elon-musks-industrial-empire',
-                'ext': 'flv',
-                'title': 'Tesla and SpaceX: Elon Musk\'s industrial empire',
-                'thumbnail': 'http://beta.img.cbsnews.com/i/2014/03/30/60147937-2f53-4565-ad64-1bdd6eb64679/60-0330-pelley-640x360.jpg',
-                'duration': 791,
+                'id': '_B6Ga3VJrI4iQNKsir_cdFo9Re_YJHE_',
+                'ext': 'mp4',
+                'title': 'Artificial Intelligence',
+                'description': 'md5:8818145f9974431e0fb58a1b8d69613c',
+                'thumbnail': r're:^https?://.*\.jpg$',
+                'duration': 1606,
+                'uploader': 'CBSI-NEW',
+                'timestamp': 1498431900,
+                'upload_date': '20170625',
             },
             'params': {
-                # rtmp download
+                # m3u8 download
                 'skip_download': True,
             },
-            'skip': 'Subscribers only',
         },
         {
             'url': 'http://www.cbsnews.com/videos/fort-hood-shooting-army-downplays-mental-illness-as-cause-of-attack/',
@@ -53,6 +57,7 @@ class CBSNewsIE(CBSIE):
             },
         },
         {
+            # 48 hours
             'url': 'http://www.cbsnews.com/news/maria-ridulph-murder-will-the-nations-oldest-cold-case-to-go-to-trial-ever-get-solved/',
             'info_dict': {
                 'id': 'QpM5BJjBVEAUFi7ydR9LusS69DPLqPJ1',

From b2ed954fc6f19a00c7c8c12f45a6a44bd1722867 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Jun 2017 22:27:53 +0700
Subject: [PATCH 489/611] [bbccouk] Capture and output error message (closes
 #13518)

---
 youtube_dl/extractor/bbc.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 7cd4446b4..6f5f9aa3a 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -523,6 +523,12 @@ class BBCCoUkIE(InfoExtractor):
 
         webpage = self._download_webpage(url, group_id, 'Downloading video page')
 
+        error = self._search_regex(
+            r'<div\b[^>]+\bclass=["\']smp__message delta["\'][^>]*>([^<]+)<',
+            webpage, 'error', default=None)
+        if error:
+            raise ExtractorError(error, expected=True)
+
         programme_id = None
         duration = None
 

From 72d256c434169b9dbfbc1c2fbce2a14f0195a0bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Jun 2017 22:29:28 +0700
Subject: [PATCH 490/611] [bbccouk] Extend _VALID_URL

---
 youtube_dl/extractor/bbc.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 6f5f9aa3a..79ded6ba1 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -36,7 +36,7 @@ class BBCCoUkIE(InfoExtractor):
                         (?:
                             programmes/(?!articles/)|
                             iplayer(?:/[^/]+)?/(?:episode/|playlist/)|
-                            music/clips[/#]|
+                            music/(?:clips|audiovideo/popular)[/#]|
                             radio/player/
                         )
                         (?P<id>%s)(?!/(?:episodes|broadcasts|clips))
@@ -229,8 +229,10 @@ class BBCCoUkIE(InfoExtractor):
         }, {
             'url': 'http://www.bbc.co.uk/radio/player/p03cchwf',
             'only_matching': True,
-        }
-    ]
+        }, {
+            'url': 'https://www.bbc.co.uk/music/audiovideo/popular#p055bc55',
+            'only_matching': True,
+        }]
 
     _USP_RE = r'/([^/]+?)\.ism(?:\.hlsv2\.ism)?/[^/]+\.m3u8'
 

From b311b0ead22f13f7cb10a3c2802f58e0692addcc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jun 2017 21:41:05 +0700
Subject: [PATCH 491/611] [generic] Extract more generic metadata (closes
 #13527)

---
 youtube_dl/extractor/generic.py | 35 ++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 2792ea3cf..f9bff433c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2048,6 +2048,13 @@ class GenericIE(InfoExtractor):
         video_description = self._og_search_description(webpage, default=None)
         video_thumbnail = self._og_search_thumbnail(webpage, default=None)
 
+        info_dict.update({
+            'title': video_title,
+            'description': video_description,
+            'thumbnail': video_thumbnail,
+            'age_limit': age_limit,
+        })
+
         # Look for Brightcove Legacy Studio embeds
         bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
         if bc_urls:
@@ -2684,18 +2691,26 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 mediaset_urls, video_id, video_title, ie=MediasetIE.ie_key())
 
+        def merge_dicts(dict1, dict2):
+            merged = {}
+            for k, v in dict1.items():
+                if v is not None:
+                    merged[k] = v
+            for k, v in dict2.items():
+                if v is None:
+                    continue
+                if (k not in merged or
+                        (isinstance(v, compat_str) and v and
+                            isinstance(merged[k], compat_str) and
+                            not merged[k])):
+                    merged[k] = v
+            return merged
+
         # Looking for http://schema.org/VideoObject
         json_ld = self._search_json_ld(
             webpage, video_id, default={}, expected_type='VideoObject')
         if json_ld.get('url'):
-            info_dict.update({
-                'title': video_title or info_dict['title'],
-                'description': video_description,
-                'thumbnail': video_thumbnail,
-                'age_limit': age_limit
-            })
-            info_dict.update(json_ld)
-            return info_dict
+            return merge_dicts(json_ld, info_dict)
 
         # Look for HTML5 media
         entries = self._parse_html5_media_entries(url, webpage, video_id, m3u8_id='hls')
@@ -2713,9 +2728,7 @@ class GenericIE(InfoExtractor):
         if jwplayer_data:
             info = self._parse_jwplayer_data(
                 jwplayer_data, video_id, require_title=False, base_url=url)
-            if not info.get('title'):
-                info['title'] = video_title
-            return info
+            return merge_dicts(info, info_dict)
 
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):

From d4f8ce6e91aff0ba0a6a4464d5525ba66720ae86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jun 2017 21:55:45 +0700
Subject: [PATCH 492/611] [dplayit] Relax video id regex (closes #13524)

---
 youtube_dl/extractor/dplay.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index 87c5dd63e..1a41760f8 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -184,7 +184,7 @@ class DPlayItIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         info_url = self._search_regex(
-            r'url\s*:\s*["\']((?:https?:)?//[^/]+/playback/videoPlaybackInfo/\d+)',
+            r'url\s*[:=]\s*["\']((?:https?:)?//[^/]+/playback/videoPlaybackInfo/\d+)',
             webpage, 'video id')
 
         title = remove_end(self._og_search_title(webpage), ' | Dplay')

From c69701c6ab94ca2fce4fedf6390515acb55e9086 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jun 2017 22:19:06 +0700
Subject: [PATCH 493/611] [extractor/common] Improve _json_ld

---
 youtube_dl/extractor/common.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9751ab021..afeb4c5da 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1002,17 +1002,17 @@ class InfoExtractor(object):
                 item_type = e.get('@type')
                 if expected_type is not None and expected_type != item_type:
                     return info
-                if item_type == 'TVEpisode':
+                if item_type in ('TVEpisode', 'Episode'):
                     info.update({
                         'episode': unescapeHTML(e.get('name')),
                         'episode_number': int_or_none(e.get('episodeNumber')),
                         'description': unescapeHTML(e.get('description')),
                     })
                     part_of_season = e.get('partOfSeason')
-                    if isinstance(part_of_season, dict) and part_of_season.get('@type') == 'TVSeason':
+                    if isinstance(part_of_season, dict) and part_of_season.get('@type') in ('TVSeason', 'Season', 'CreativeWorkSeason'):
                         info['season_number'] = int_or_none(part_of_season.get('seasonNumber'))
                     part_of_series = e.get('partOfSeries') or e.get('partOfTVSeries')
-                    if isinstance(part_of_series, dict) and part_of_series.get('@type') == 'TVSeries':
+                    if isinstance(part_of_series, dict) and part_of_series.get('@type') in ('TVSeries', 'Series', 'CreativeWorkSeries'):
                         info['series'] = unescapeHTML(part_of_series.get('name'))
                 elif item_type == 'Article':
                     info.update({
@@ -1022,10 +1022,10 @@ class InfoExtractor(object):
                     })
                 elif item_type == 'VideoObject':
                     extract_video_object(e)
-                elif item_type == 'WebPage':
-                    video = e.get('video')
-                    if isinstance(video, dict) and video.get('@type') == 'VideoObject':
-                        extract_video_object(video)
+                    continue
+                video = e.get('video')
+                if isinstance(video, dict) and video.get('@type') == 'VideoObject':
+                    extract_video_object(video)
                 break
         return dict((k, v) for k, v in info.items() if v is not None)
 

From 54faac223505758e48c1f7ce1c7ed8c110f9bac4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jun 2017 22:20:30 +0700
Subject: [PATCH 494/611] [tastytrade] Add extractor (closes #13521)

---
 youtube_dl/extractor/tastytrade.py | 43 ++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100644 youtube_dl/extractor/tastytrade.py

diff --git a/youtube_dl/extractor/tastytrade.py b/youtube_dl/extractor/tastytrade.py
new file mode 100644
index 000000000..7fe96bd5f
--- /dev/null
+++ b/youtube_dl/extractor/tastytrade.py
@@ -0,0 +1,43 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from .ooyala import OoyalaIE
+
+
+class TastyTradeIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?tastytrade\.com/tt/shows/[^/]+/episodes/(?P<id>[^/?#&]+)'
+
+    _TESTS = [{
+        'url': 'https://www.tastytrade.com/tt/shows/market-measures/episodes/correlation-in-short-volatility-06-28-2017',
+        'info_dict': {
+            'id': 'F3bnlzbToeI6pLEfRyrlfooIILUjz4nM',
+            'ext': 'mp4',
+            'title': 'A History of Teaming',
+            'description': 'md5:2a9033db8da81f2edffa4c99888140b3',
+            'duration': 422.255,
+        },
+        'params': {
+            'skip_download': True,
+        },
+        'add_ie': ['Ooyala'],
+    }, {
+        'url': 'https://www.tastytrade.com/tt/shows/daily-dose/episodes/daily-dose-06-30-2017',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        ooyala_code = self._search_regex(
+            r'data-media-id=(["\'])(?P<code>(?:(?!\1).)+)\1',
+            webpage, 'ooyala code', group='code')
+
+        info = self._search_json_ld(webpage, display_id, fatal=False)
+        info.update({
+            '_type': 'url_transparent',
+            'ie_key': OoyalaIE.ie_key(),
+            'url': 'ooyala:%s' % ooyala_code,
+            'display_id': display_id,
+        })
+        return info

From 49174788035038b8f12f971ec04ef897df60f435 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Jul 2017 18:39:01 +0700
Subject: [PATCH 495/611] [ted] Fix extraction (closes #13535))

---
 youtube_dl/extractor/ted.py | 49 +++++++++++++++++++++++++------------
 1 file changed, 34 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 3f3c681ae..f27d0e313 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -6,7 +6,10 @@ import re
 from .common import InfoExtractor
 
 from ..compat import compat_str
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    try_get,
+)
 
 
 class TEDIE(InfoExtractor):
@@ -113,8 +116,9 @@ class TEDIE(InfoExtractor):
     }
 
     def _extract_info(self, webpage):
-        info_json = self._search_regex(r'q\("\w+.init",({.+})\)</script>',
-                                       webpage, 'info json')
+        info_json = self._search_regex(
+            r'(?s)q\(\s*"\w+.init"\s*,\s*({.+})\)\s*</script>',
+            webpage, 'info json')
         return json.loads(info_json)
 
     def _real_extract(self, url):
@@ -136,11 +140,16 @@ class TEDIE(InfoExtractor):
         webpage = self._download_webpage(url, name,
                                          'Downloading playlist webpage')
         info = self._extract_info(webpage)
-        playlist_info = info['playlist']
+
+        playlist_info = try_get(
+            info, lambda x: x['__INITIAL_DATA__']['playlist'],
+            dict) or info['playlist']
 
         playlist_entries = [
             self.url_result('http://www.ted.com/talks/' + talk['slug'], self.ie_key())
-            for talk in info['talks']
+            for talk in try_get(
+                info, lambda x: x['__INITIAL_DATA__']['talks'],
+                dict) or info['talks']
         ]
         return self.playlist_result(
             playlist_entries,
@@ -149,9 +158,14 @@ class TEDIE(InfoExtractor):
 
     def _talk_info(self, url, video_name):
         webpage = self._download_webpage(url, video_name)
-        self.report_extraction(video_name)
 
-        talk_info = self._extract_info(webpage)['talks'][0]
+        info = self._extract_info(webpage)
+
+        talk_info = try_get(
+            info, lambda x: x['__INITIAL_DATA__']['talks'][0],
+            dict) or info['talks'][0]
+
+        title = talk_info['title'].strip()
 
         external = talk_info.get('external')
         if external:
@@ -165,19 +179,27 @@ class TEDIE(InfoExtractor):
                 'url': ext_url or external['uri'],
             }
 
+        native_downloads = try_get(
+            talk_info, lambda x: x['downloads']['nativeDownloads'],
+            dict) or talk_info['nativeDownloads']
+
         formats = [{
             'url': format_url,
             'format_id': format_id,
             'format': format_id,
-        } for (format_id, format_url) in talk_info['nativeDownloads'].items() if format_url is not None]
+        } for (format_id, format_url) in native_downloads.items() if format_url is not None]
         if formats:
             for f in formats:
                 finfo = self._NATIVE_FORMATS.get(f['format_id'])
                 if finfo:
                     f.update(finfo)
 
+        player_talk = talk_info['player_talks'][0]
+
+        resources_ = player_talk.get('resources') or talk_info.get('resources')
+
         http_url = None
-        for format_id, resources in talk_info['resources'].items():
+        for format_id, resources in resources_.items():
             if format_id == 'h264':
                 for resource in resources:
                     h264_url = resource.get('file')
@@ -237,14 +259,11 @@ class TEDIE(InfoExtractor):
 
         video_id = compat_str(talk_info['id'])
 
-        thumbnail = talk_info['thumb']
-        if not thumbnail.startswith('http'):
-            thumbnail = 'http://' + thumbnail
         return {
             'id': video_id,
-            'title': talk_info['title'].strip(),
-            'uploader': talk_info['speaker'],
-            'thumbnail': thumbnail,
+            'title': title,
+            'uploader': player_talk.get('speaker') or talk_info.get('speaker'),
+            'thumbnail': player_talk.get('thumb') or talk_info.get('thumb'),
             'description': self._og_search_description(webpage),
             'subtitles': self._get_subtitles(video_id, talk_info),
             'formats': formats,

From 9b35297be1b7e5fa7372bac664ab91a7fa364760 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Jul 2017 18:39:29 +0700
Subject: [PATCH 496/611] [extractors] Add import for tastytrade

---
 youtube_dl/extractor/extractors.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index a263c88b3..bbdb4a2fe 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -973,6 +973,7 @@ from .tagesschau import (
     TagesschauIE,
 )
 from .tass import TassIE
+from .tastytrade import TastyTradeIE
 from .tbs import TBSIE
 from .tdslifeway import TDSLifewayIE
 from .teachertube import (

From 02d61a65e27c9f186ea24bce8e9f0c65f4ec508a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Jul 2017 08:00:22 +0700
Subject: [PATCH 497/611] [xfileshare] Extend format regex (closes #13536)

---
 youtube_dl/extractor/xfileshare.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 74aac85a3..ad747978d 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -157,7 +157,7 @@ class XFileShareIE(InfoExtractor):
         def extract_formats(default=NO_DEFAULT):
             urls = []
             for regex in (
-                    r'file\s*:\s*(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1',
+                    r'(?:file|src)\s*:\s*(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1',
                     r'file_link\s*=\s*(["\'])(?P<url>http(?:(?!\1).)+)\1',
                     r'addVariable\((\\?["\'])file\1\s*,\s*(\\?["\'])(?P<url>http(?:(?!\2).)+)\2\)',
                     r'<embed[^>]+src=(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1'):

From a3a6d01a9613566347ac39bac9cef3c4b8cd692f Mon Sep 17 00:00:00 2001
From: Parmjit Virk <pvirk@mts.net>
Date: Sat, 1 Jul 2017 21:09:57 -0500
Subject: [PATCH 498/611] [thisoldhouse] Fix video id extraction (closes
 #13540)

---
 youtube_dl/extractor/thisoldhouse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisoldhouse.py b/youtube_dl/extractor/thisoldhouse.py
index 197258df1..6a3efb9df 100644
--- a/youtube_dl/extractor/thisoldhouse.py
+++ b/youtube_dl/extractor/thisoldhouse.py
@@ -31,5 +31,5 @@ class ThisOldHouseIE(InfoExtractor):
         drupal_settings = self._parse_json(self._search_regex(
             r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);',
             webpage, 'drupal settings'), display_id)
-        video_id = drupal_settings['jwplatform']['video_id']
+        video_id = list(drupal_settings['comScore'])[0]
         return self.url_result('jwplatform:' + video_id, 'JWPlatform', video_id)

From 99a7e76240f3e79d770bff0029dc3321e14b2a97 Mon Sep 17 00:00:00 2001
From: Parmjit Virk <pvirk@mts.net>
Date: Sat, 1 Jul 2017 21:11:58 -0500
Subject: [PATCH 499/611] [thisoldhouse] Update test

---
 youtube_dl/extractor/thisoldhouse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisoldhouse.py b/youtube_dl/extractor/thisoldhouse.py
index 6a3efb9df..0c321354f 100644
--- a/youtube_dl/extractor/thisoldhouse.py
+++ b/youtube_dl/extractor/thisoldhouse.py
@@ -8,7 +8,7 @@ class ThisOldHouseIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?thisoldhouse\.com/(?:watch|how-to|tv-episode)/(?P<id>[^/?#]+)'
     _TESTS = [{
         'url': 'https://www.thisoldhouse.com/how-to/how-to-build-storage-bench',
-        'md5': '946f05bbaa12a33f9ae35580d2dfcfe3',
+        'md5': '568acf9ca25a639f0c4ff905826b662f',
         'info_dict': {
             'id': '2REGtUDQ',
             'ext': 'mp4',

From 50ae3f646e6c7eec187078dd43f1cb3777ccfb4c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Jul 2017 20:04:08 +0700
Subject: [PATCH 500/611] [thisoldhouse] Add more fallbacks for video id
 (closes #13541)

---
 youtube_dl/extractor/thisoldhouse.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/thisoldhouse.py b/youtube_dl/extractor/thisoldhouse.py
index 0c321354f..6ab147ad7 100644
--- a/youtube_dl/extractor/thisoldhouse.py
+++ b/youtube_dl/extractor/thisoldhouse.py
@@ -2,6 +2,8 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import try_get
 
 
 class ThisOldHouseIE(InfoExtractor):
@@ -28,8 +30,15 @@ class ThisOldHouseIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-        drupal_settings = self._parse_json(self._search_regex(
-            r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);',
-            webpage, 'drupal settings'), display_id)
-        video_id = list(drupal_settings['comScore'])[0]
+        video_id = self._search_regex(
+            (r'data-mid=(["\'])(?P<id>(?:(?!\1).)+)\1',
+             r'id=(["\'])inline-video-player-(?P<id>(?:(?!\1).)+)\1'),
+            webpage, 'video id', default=None, group='id')
+        if not video_id:
+            drupal_settings = self._parse_json(self._search_regex(
+                r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);',
+                webpage, 'drupal settings'), display_id)
+            video_id = try_get(
+                drupal_settings, lambda x: x['jwplatform']['video_id'],
+                compat_str) or list(drupal_settings['comScore'])[0]
         return self.url_result('jwplatform:' + video_id, 'JWPlatform', video_id)

From 4d9ba27bba916d4e305d41fbef0685170eb1991f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Jul 2017 20:12:40 +0700
Subject: [PATCH 501/611] [ChangeLog] Actualize

---
 ChangeLog | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 7b3c6c6cf..c17acf45e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,22 @@
 version <unreleased>
 
+Core
+* [extractor/common] Improve _json_ld
+
 Extractors
++ [thisoldhouse] Add more fallbacks for video id
+* [thisoldhouse] Fix video id extraction (#13540, #13541)
+* [xfileshare] Extend format regular expression (#13536)
+* [ted] Fix extraction (#13535)
++ [tastytrade] Add support for tastytrade.com (#13521)
+* [dplayit] Relax video id regular expression (#13524)
++ [generic] Extract more generic metadata (#13527)
++ [bbccouk] Capture and output error message (#13501, #13518)
+* [cbsnews] Relax video info regular expression (#13284, #13503)
++ [facebook] Add support for plugin video embeds and multiple embeds (#13493)
+* [soundcloud] Switch to https for API requests (#13502)
+* [pandatv] Switch to https for API and download URLs
++ [pandatv] Add support for https URLs (#13491)
 + [niconico] Support sp subdomain (#13494)
 
 

From b6c9fe416243373bcb59eb8aa5ef0baca8f3c97c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Jul 2017 20:17:10 +0700
Subject: [PATCH 502/611] release 2017.07.02

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 1 +
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 82bbbdaa6..974603507 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.06.25*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.06.25**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.07.02*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.07.02**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.06.25
+[debug] youtube-dl version 2017.07.02
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index c17acf45e..4c2011326 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.07.02
 
 Core
 * [extractor/common] Improve _json_ld
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 010ff762c..db2e2bac9 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -768,6 +768,7 @@
  - **Tagesschau**
  - **tagesschau:player**
  - **Tass**
+ - **TastyTrade**
  - **TBS**
  - **TDSLifeway**
  - **teachertube**: teachertube.com videos
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b6d378896..0db974f97 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.06.25'
+__version__ = '2017.07.02'

From 609ff8ca19f1c4c168a81121074b91cc0f0d4c47 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 5 Jul 2017 23:23:35 +0800
Subject: [PATCH 503/611] [utils] Support attributes with no values in
 get_elements_by_attribute()

---
 test/test_utils.py  | 6 ++++++
 youtube_dl/utils.py | 4 ++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 2b93b3604..30738e7cf 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1228,6 +1228,12 @@ part 3</font></u>
         self.assertEqual(get_element_by_attribute('class', 'foo', html), None)
         self.assertEqual(get_element_by_attribute('class', 'no-such-foo', html), None)
 
+        html = '''
+            <div itemprop="author" itemscope>foo</div>
+        '''
+
+        self.assertEqual(get_element_by_attribute('itemprop', 'author', html), 'foo')
+
     def test_get_elements_by_class(self):
         html = '''
             <span class="foo bar">nice</span><span class="foo bar">also nice</span>
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 39860e9d1..fdf5e29e7 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -365,9 +365,9 @@ def get_elements_by_attribute(attribute, value, html, escape_value=True):
     retlist = []
     for m in re.finditer(r'''(?xs)
         <([a-zA-Z0-9:._-]+)
-         (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'))*?
+         (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
          \s+%s=['"]?%s['"]?
-         (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'))*?
+         (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
         \s*>
         (?P<content>.*?)
         </\1>

From babbc04d459a6e8b6a2e65c522107cce1ca2cbbb Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 13 Jun 2017 16:21:26 +0800
Subject: [PATCH 504/611] [xuite] Move to the new HTML5 API and reduce # of
 requests

---
 youtube_dl/extractor/xuite.py | 88 ++++++++++++++---------------------
 1 file changed, 35 insertions(+), 53 deletions(-)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index e0818201a..0276c0dbb 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -1,14 +1,13 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import base64
-
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     ExtractorError,
+    float_or_none,
+    get_element_by_attribute,
     parse_iso8601,
-    parse_duration,
+    remove_end,
 )
 
 
@@ -24,6 +23,7 @@ class XuiteIE(InfoExtractor):
             'id': '3860914',
             'ext': 'mp3',
             'title': '孤單南半球-歐德陽',
+            'description': '孤單南半球-歐德陽',
             'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 247.246,
             'timestamp': 1314932940,
@@ -44,7 +44,7 @@ class XuiteIE(InfoExtractor):
             'duration': 596.458,
             'timestamp': 1454242500,
             'upload_date': '20160131',
-            'uploader': 'yan12125',
+            'uploader': '屁姥',
             'uploader_id': '12158353',
             'categories': ['個人短片'],
             'description': 'http://download.blender.org/peach/bigbuckbunny_movies/BigBuckBunny_320x180.mp4',
@@ -72,10 +72,10 @@ class XuiteIE(InfoExtractor):
         # from http://forgetfulbc.blogspot.com/2016/06/date.html
         'url': 'http://vlog.xuite.net/embed/cE1xbENoLTI3NDQ3MzM2LmZsdg==?ar=0&as=0',
         'info_dict': {
-            'id': 'cE1xbENoLTI3NDQ3MzM2LmZsdg==',
+            'id': '27447336',
             'ext': 'mp4',
             'title': '男女平權只是口號？專家解釋約會時男生是否該幫女生付錢 (中字)',
-            'description': 'md5:f0abdcb69df300f522a5442ef3146f2a',
+            'description': 'md5:1223810fa123b179083a3aed53574706',
             'timestamp': 1466160960,
             'upload_date': '20160617',
             'uploader': 'B.C. & Lowy',
@@ -86,29 +86,9 @@ class XuiteIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    @staticmethod
-    def base64_decode_utf8(data):
-        return base64.b64decode(data.encode('utf-8')).decode('utf-8')
-
-    @staticmethod
-    def base64_encode_utf8(data):
-        return base64.b64encode(data.encode('utf-8')).decode('utf-8')
-
-    def _extract_flv_config(self, encoded_media_id):
-        flv_config = self._download_xml(
-            'http://vlog.xuite.net/flash/player?media=%s' % encoded_media_id,
-            'flv config')
-        prop_dict = {}
-        for prop in flv_config.findall('./property'):
-            prop_id = self.base64_decode_utf8(prop.attrib['id'])
-            # CDATA may be empty in flv config
-            if not prop.text:
-                continue
-            encoded_content = self.base64_decode_utf8(prop.text)
-            prop_dict[prop_id] = compat_urllib_parse_unquote(encoded_content)
-        return prop_dict
-
     def _real_extract(self, url):
+        # /play/ URLs provide embedded video URL and more metadata
+        url = url.replace('/embed/', '/play/')
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
@@ -121,51 +101,53 @@ class XuiteIE(InfoExtractor):
                 '%s returned error: %s' % (self.IE_NAME, error_msg),
                 expected=True)
 
-        encoded_media_id = self._search_regex(
-            r'attributes\.name\s*=\s*"([^"]+)"', webpage,
-            'encoded media id', default=None)
-        if encoded_media_id is None:
-            video_id = self._html_search_regex(
-                r'data-mediaid="(\d+)"', webpage, 'media id')
-            encoded_media_id = self.base64_encode_utf8(video_id)
-        flv_config = self._extract_flv_config(encoded_media_id)
+        media_info = self._parse_json(self._search_regex(
+            r'var\s+mediaInfo\s*=\s*({.*});', webpage, 'media info'), video_id)
 
-        FORMATS = {
-            'audio': 'mp3',
-            'video': 'mp4',
-        }
+        video_id = media_info['MEDIA_ID']
 
         formats = []
-        for format_tag in ('src', 'hq_src'):
-            video_url = flv_config.get(format_tag)
+        for key in ('html5Url', 'html5HQUrl'):
+            video_url = media_info.get(key)
             if not video_url:
                 continue
             format_id = self._search_regex(
-                r'\bq=(.+?)\b', video_url, 'format id', default=format_tag)
+                r'\bq=(.+?)\b', video_url, 'format id', default=None)
             formats.append({
                 'url': video_url,
-                'ext': FORMATS.get(flv_config['type'], 'mp4'),
+                'ext': 'mp4' if format_id.isnumeric() else format_id,
                 'format_id': format_id,
                 'height': int(format_id) if format_id.isnumeric() else None,
             })
         self._sort_formats(formats)
 
-        timestamp = flv_config.get('publish_datetime')
+        timestamp = media_info.get('PUBLISH_DATETIME')
         if timestamp:
             timestamp = parse_iso8601(timestamp + ' +0800', ' ')
 
-        category = flv_config.get('category')
+        category = media_info.get('catName')
         categories = [category] if category else []
 
+        uploader = media_info.get('NICKNAME')
+        uploader_url = None
+
+        author_div = get_element_by_attribute('itemprop', 'author', webpage)
+        if author_div:
+            uploader = uploader or self._html_search_meta('name', author_div)
+            uploader_url = self._html_search_regex(
+                r'<link[^>]+itemprop="url"[^>]+href="([^"]+)"', author_div,
+                'uploader URL', fatal=False)
+
         return {
             'id': video_id,
-            'title': flv_config['title'],
-            'description': flv_config.get('description'),
-            'thumbnail': flv_config.get('thumb'),
+            'title': media_info['TITLE'],
+            'description': remove_end(media_info.get('metaDesc'), ' (Xuite 影音)'),
+            'thumbnail': media_info.get('ogImageUrl'),
             'timestamp': timestamp,
-            'uploader': flv_config.get('author_name'),
-            'uploader_id': flv_config.get('author_id'),
-            'duration': parse_duration(flv_config.get('duration')),
+            'uploader': uploader,
+            'uploader_id': media_info.get('MEMBER_ID'),
+            'uploader_url': uploader_url,
+            'duration': float_or_none(media_info.get('MEDIA_DURATION'), 1000000),
             'categories': categories,
             'formats': formats,
         }

From 0a2e1b2e30045de7834aca880d35253c5e8a3812 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Jul 2017 22:13:47 +0700
Subject: [PATCH 505/611] [vier] Adapt extraction to redesign (#13575)

---
 youtube_dl/extractor/vier.py | 45 +++++++++++++++++++++++++++++-------
 1 file changed, 37 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index 3e67eb8c2..dbd5ba9ba 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -15,7 +15,21 @@ from ..utils import (
 class VierIE(InfoExtractor):
     IE_NAME = 'vier'
     IE_DESC = 'vier.be and vijf.be'
-    _VALID_URL = r'https?://(?:www\.)?(?P<site>vier|vijf)\.be/(?:[^/]+/videos/(?P<display_id>[^/]+)(?:/(?P<id>\d+))?|video/v3/embed/(?P<embed_id>\d+))'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?(?P<site>vier|vijf)\.be/
+                        (?:
+                            (?:
+                                [^/]+/videos|
+                                video(?:/[^/]+)*
+                            )/
+                            (?P<display_id>[^/]+)(?:/(?P<id>\d+))?|
+                            (?:
+                                video/v3/embed|
+                                embed/video/public
+                            )/(?P<embed_id>\d+)
+                        )
+                    '''
     _NETRC_MACHINE = 'vier'
     _TESTS = [{
         'url': 'http://www.vier.be/planb/videos/het-wordt-warm-de-moestuin/16129',
@@ -83,6 +97,15 @@ class VierIE(InfoExtractor):
     }, {
         'url': 'http://www.vier.be/video/v3/embed/16129',
         'only_matching': True,
+    }, {
+        'url': 'https://www.vijf.be/embed/video/public/4093',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.vier.be/video/blockbusters/in-juli-en-augustus-summer-classics',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.vier.be/video/achter-de-rug/2017/achter-de-rug-seizoen-1-aflevering-6',
+        'only_matching': True,
     }]
 
     def _real_initialize(self):
@@ -133,14 +156,20 @@ class VierIE(InfoExtractor):
         video_id = self._search_regex(
             [r'data-nid="(\d+)"', r'"nid"\s*:\s*"(\d+)"'],
             webpage, 'video id', default=video_id or display_id)
-        application = self._search_regex(
-            [r'data-application="([^"]+)"', r'"application"\s*:\s*"([^"]+)"'],
-            webpage, 'application', default=site + '_vod')
-        filename = self._search_regex(
-            [r'data-filename="([^"]+)"', r'"filename"\s*:\s*"([^"]+)"'],
-            webpage, 'filename')
 
-        playlist_url = 'http://vod.streamcloud.be/%s/_definst_/mp4:%s.mp4/playlist.m3u8' % (application, filename)
+        playlist_url = self._search_regex(
+            r'data-file=(["\'])(?P<url>(?:https?:)?//[^/]+/.+?\.m3u8.*?)\1',
+            webpage, 'm3u8 url', default=None, group='url')
+
+        if not playlist_url:
+            application = self._search_regex(
+                [r'data-application="([^"]+)"', r'"application"\s*:\s*"([^"]+)"'],
+                webpage, 'application', default=site + '_vod')
+            filename = self._search_regex(
+                [r'data-filename="([^"]+)"', r'"filename"\s*:\s*"([^"]+)"'],
+                webpage, 'filename')
+            playlist_url = 'http://vod.streamcloud.be/%s/_definst_/mp4:%s.mp4/playlist.m3u8' % (application, filename)
+
         formats = self._extract_wowza_formats(
             playlist_url, display_id, skip_protocols=['dash'])
         self._sort_formats(formats)

From 655470825231eaa03b4b82cbc1314d551e72a01e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Jul 2017 23:20:50 +0700
Subject: [PATCH 506/611] [kaltura] Fix typo in subtitles extraction (closes
 #13569)

---
 youtube_dl/extractor/kaltura.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 41c1f3d96..138d4844d 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -324,7 +324,7 @@ class KalturaIE(InfoExtractor):
         if captions:
             for caption in captions.get('objects', []):
                 # Continue if caption is not ready
-                if f.get('status') != 2:
+                if caption.get('status') != 2:
                     continue
                 if not caption.get('id'):
                     continue

From dee2ff1d818bebd74990b7cebbc698f22163a43a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Jul 2017 00:25:37 +0700
Subject: [PATCH 507/611] [test_utils] Fix tests under Windows

---
 test/test_utils.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 30738e7cf..7803e5bc7 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -98,6 +98,7 @@ from youtube_dl.compat import (
     compat_chr,
     compat_etree_fromstring,
     compat_getenv,
+    compat_os_name,
     compat_setenv,
     compat_urlparse,
     compat_parse_qs,
@@ -448,7 +449,9 @@ class TestUtil(unittest.TestCase):
 
     def test_shell_quote(self):
         args = ['ffmpeg', '-i', encodeFilename('ñ€ß\'.mp4')]
-        self.assertEqual(shell_quote(args), """ffmpeg -i 'ñ€ß'"'"'.mp4'""")
+        self.assertEqual(
+            shell_quote(args),
+            """ffmpeg -i 'ñ€ß'"'"'.mp4'""" if compat_os_name != 'nt' else '''ffmpeg -i "ñ€ß'.mp4"''')
 
     def test_str_to_int(self):
         self.assertEqual(str_to_int('123,456'), 123456)
@@ -932,7 +935,7 @@ class TestUtil(unittest.TestCase):
     def test_args_to_str(self):
         self.assertEqual(
             args_to_str(['foo', 'ba/r', '-baz', '2 be', '']),
-            'foo ba/r -baz \'2 be\' \'\''
+            'foo ba/r -baz \'2 be\' \'\'' if compat_os_name != 'nt' else 'foo ba/r -baz "2 be" ""'
         )
 
     def test_parse_filesize(self):

From 60d4401c5e14d94574e2418d229dcc0c067d3559 Mon Sep 17 00:00:00 2001
From: Parmjit Virk <pvirk@mts.net>
Date: Thu, 6 Jul 2017 10:55:59 -0500
Subject: [PATCH 508/611] [espn] Extend _VALID_URL (fixes #13244)

---
 youtube_dl/extractor/espn.py | 48 ++++++++++++++++++++++++++++++------
 1 file changed, 40 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/espn.py b/youtube_dl/extractor/espn.py
index 8795e0ddf..7a7436068 100644
--- a/youtube_dl/extractor/espn.py
+++ b/youtube_dl/extractor/espn.py
@@ -10,7 +10,25 @@ from ..utils import (
 
 
 class ESPNIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:espn\.go|(?:www\.)?espn)\.com/video/clip(?:\?.*?\bid=|/_/id/)(?P<id>\d+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            (?:(?:\w+\.)+)?espn\.go|
+                            (?:www\.)?espn
+                        )\.com/
+                        (?:
+                            (?:
+                                video/clip|
+                                watch/player
+                            )
+                            (?:
+                                \?.*?\bid=|
+                                /_/id/
+                            )
+                        )
+                        (?P<id>\d+)
+                    '''
+
     _TESTS = [{
         'url': 'http://espn.go.com/video/clip?id=10365079',
         'info_dict': {
@@ -25,20 +43,34 @@ class ESPNIE(InfoExtractor):
             'skip_download': True,
         },
     }, {
-        # intl video, from http://www.espnfc.us/video/mls-highlights/150/video/2743663/must-see-moments-best-of-the-mls-season
-        'url': 'http://espn.go.com/video/clip?id=2743663',
+        'url': 'https://broadband.espn.go.com/video/clip?id=18910086',
         'info_dict': {
-            'id': '2743663',
+            'id': '18910086',
             'ext': 'mp4',
-            'title': 'Must-See Moments: Best of the MLS season',
-            'description': 'md5:4c2d7232beaea572632bec41004f0aeb',
-            'timestamp': 1449446454,
-            'upload_date': '20151207',
+            'title': 'Kyrie spins around defender for two',
+            'description': 'md5:2b0f5bae9616d26fba8808350f0d2b9b',
+            'timestamp': 1489539155,
+            'upload_date': '20170315',
         },
         'params': {
             'skip_download': True,
         },
         'expected_warnings': ['Unable to download f4m manifest'],
+    }, {
+        'url': 'http://nonredline.sports.espn.go.com/video/clip?id=19744672',
+        'only_matching': True,
+    }, {
+        'url': 'https://cdn.espn.go.com/video/clip/_/id/19771774',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.espn.com/watch/player?id=19141491',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.espn.com/watch/player?bucketId=257&id=19505875',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.espn.com/watch/player/_/id/19141491',
+        'only_matching': True,
     }, {
         'url': 'http://www.espn.com/video/clip?id=10365079',
         'only_matching': True,

From ddeff4be3fb491a82d775fcd4140fc49a443864c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Jul 2017 23:03:51 +0700
Subject: [PATCH 509/611] Credit @gfabiano for #13382, #13385, #13415

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index e2bdebe5e..2f828adb5 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -220,3 +220,4 @@ gritstub
 Adam Voss
 Mike Fährmann
 Jan Kundrát
+Giuseppe Fabiano

From ab328411d53eee079a66f373bc83cf09d207dae4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Jul 2017 23:05:27 +0700
Subject: [PATCH 510/611] Credit @orng for ruv (#13396)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 2f828adb5..ec58744cf 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -221,3 +221,4 @@ Adam Voss
 Mike Fährmann
 Jan Kundrát
 Giuseppe Fabiano
+Örn Guðjónsson

From 8a04ade86bf3a7f0a321232850a2d03a52c47819 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Jul 2017 23:06:53 +0700
Subject: [PATCH 511/611] Credit @parmjitv for #13322, #13503, #13541, #13549

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index ec58744cf..053159cc3 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -222,3 +222,4 @@ Mike Fährmann
 Jan Kundrát
 Giuseppe Fabiano
 Örn Guðjónsson
+Parmjit Virk

From 00e5c363159f7771dc3a2e33b1a38af520782372 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Jul 2017 22:22:29 +0700
Subject: [PATCH 512/611] [xhamster] Add support for new URL schema (closes
 #13593)

---
 youtube_dl/extractor/xhamster.py | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 6987b2e3a..de9ec9b9e 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -14,7 +14,15 @@ from ..utils import (
 
 
 class XHamsterIE(InfoExtractor):
-    _VALID_URL = r'(?P<proto>https?)://(?:.+?\.)?xhamster\.com/movies/(?P<id>[0-9]+)/(?P<seo>.*?)\.html(?:\?.*)?'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:.+?\.)?xhamster\.com/
+                        (?:
+                            movies/(?P<id>\d+)/(?P<display_id>[^/]*)\.html|
+                            videos/(?P<display_id_2>[^/]*)-(?P<id_2>\d+)
+                        )
+                    '''
+
     _TESTS = [{
         'url': 'http://xhamster.com/movies/1509445/femaleagent_shy_beauty_takes_the_bait.html',
         'md5': '8281348b8d3c53d39fffb377d24eac4e',
@@ -66,6 +74,10 @@ class XHamsterIE(InfoExtractor):
         # This video is visible for marcoalfa123456's friends only
         'url': 'https://it.xhamster.com/movies/7263980/la_mia_vicina.html',
         'only_matching': True,
+    }, {
+        # new URL schema
+        'url': 'https://pt.xhamster.com/videos/euro-pedal-pumping-7937821',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -81,11 +93,10 @@ class XHamsterIE(InfoExtractor):
 
         mobj = re.match(self._VALID_URL, url)
 
-        video_id = mobj.group('id')
-        seo = mobj.group('seo')
-        proto = mobj.group('proto')
-        mrss_url = '%s://xhamster.com/movies/%s/%s.html' % (proto, video_id, seo)
-        webpage = self._download_webpage(mrss_url, video_id)
+        video_id = mobj.group('id') or mobj.group('id_2')
+        display_id = mobj.group('display_id') or mobj.group('display_id_2')
+
+        webpage = self._download_webpage(url, video_id)
 
         error = self._html_search_regex(
             r'<div[^>]+id=["\']videoClosed["\'][^>]*>(.+?)</div>',

From d852c6bc5995444096a6c07b66ea075269412cf7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Jul 2017 22:49:11 +0700
Subject: [PATCH 513/611] [xhamster] Extract all formats and fix duration
 extraction (#13593)

---
 youtube_dl/extractor/xhamster.py | 75 +++++++++++++++++---------------
 1 file changed, 39 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index de9ec9b9e..c42b59e51 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     clean_html,
     dict_get,
@@ -28,6 +29,7 @@ class XHamsterIE(InfoExtractor):
         'md5': '8281348b8d3c53d39fffb377d24eac4e',
         'info_dict': {
             'id': '1509445',
+            'display_id': 'femaleagent_shy_beauty_takes_the_bait',
             'ext': 'mp4',
             'title': 'FemaleAgent Shy beauty takes the bait',
             'upload_date': '20121014',
@@ -40,6 +42,7 @@ class XHamsterIE(InfoExtractor):
         'url': 'http://xhamster.com/movies/2221348/britney_spears_sexy_booty.html?hd',
         'info_dict': {
             'id': '2221348',
+            'display_id': 'britney_spears_sexy_booty',
             'ext': 'mp4',
             'title': 'Britney Spears  Sexy Booty',
             'upload_date': '20130914',
@@ -81,18 +84,7 @@ class XHamsterIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        def extract_video_url(webpage, name):
-            return self._search_regex(
-                [r'''file\s*:\s*(?P<q>["'])(?P<mp4>.+?)(?P=q)''',
-                 r'''<a\s+href=(?P<q>["'])(?P<mp4>.+?)(?P=q)\s+class=["']mp4Thumb''',
-                 r'''<video[^>]+file=(?P<q>["'])(?P<mp4>.+?)(?P=q)[^>]*>'''],
-                webpage, name, group='mp4')
-
-        def is_hd(webpage):
-            return '<div class=\'icon iconHD\'' in webpage
-
         mobj = re.match(self._VALID_URL, url)
-
         video_id = mobj.group('id') or mobj.group('id_2')
         display_id = mobj.group('display_id') or mobj.group('display_id_2')
 
@@ -110,6 +102,39 @@ class XHamsterIE(InfoExtractor):
              r'<title[^>]*>(.+?)(?:,\s*[^,]*?\s*Porn\s*[^,]*?:\s*xHamster[^<]*| - xHamster\.com)</title>'],
             webpage, 'title')
 
+        formats = []
+        format_urls = set()
+
+        sources = self._parse_json(
+            self._search_regex(
+                r'sources\s*:\s*({.+?})\s*,?\s*\n', webpage, 'sources',
+                default='{}'),
+            video_id, fatal=False)
+        for format_id, format_url in sources.items():
+            if not isinstance(format_url, compat_str):
+                continue
+            if format_url in format_urls:
+                continue
+            format_urls.add(format_url)
+            formats.append({
+                'format_id': format_id,
+                'url': format_url,
+                'height': int_or_none(self._search_regex(
+                    r'^(\d+)[pP]', format_id, 'height', default=None))
+            })
+
+        video_url = self._search_regex(
+            [r'''file\s*:\s*(?P<q>["'])(?P<mp4>.+?)(?P=q)''',
+             r'''<a\s+href=(?P<q>["'])(?P<mp4>.+?)(?P=q)\s+class=["']mp4Thumb''',
+             r'''<video[^>]+file=(?P<q>["'])(?P<mp4>.+?)(?P=q)[^>]*>'''],
+            webpage, 'video url', group='mp4', default=None)
+        if video_url and video_url not in format_urls:
+            formats.append({
+                'url': video_url,
+            })
+
+        self._sort_formats(formats)
+
         # Only a few videos have an description
         mobj = re.search(r'<span>Description: </span>([^<]+)', webpage)
         description = mobj.group(1) if mobj else None
@@ -128,7 +153,8 @@ class XHamsterIE(InfoExtractor):
             webpage, 'thumbnail', fatal=False, group='thumbnail')
 
         duration = parse_duration(self._search_regex(
-            r'Runtime:\s*</span>\s*([\d:]+)', webpage,
+            [r'<[^<]+\bitemprop=["\']duration["\'][^<]+\bcontent=["\'](.+?)["\']',
+             r'Runtime:\s*</span>\s*([\d:]+)'], webpage,
             'duration', fatal=False))
 
         view_count = int_or_none(self._search_regex(
@@ -143,30 +169,6 @@ class XHamsterIE(InfoExtractor):
 
         age_limit = self._rta_search(webpage)
 
-        hd = is_hd(webpage)
-
-        format_id = 'hd' if hd else 'sd'
-
-        video_url = extract_video_url(webpage, format_id)
-        formats = [{
-            'url': video_url,
-            'format_id': 'hd' if hd else 'sd',
-            'preference': 1,
-        }]
-
-        if not hd:
-            mrss_url = self._search_regex(r'<link rel="canonical" href="([^"]+)', webpage, 'mrss_url')
-            webpage = self._download_webpage(mrss_url + '?hd', video_id, note='Downloading HD webpage')
-            if is_hd(webpage):
-                video_url = extract_video_url(webpage, 'hd')
-                formats.append({
-                    'url': video_url,
-                    'format_id': 'hd',
-                    'preference': 2,
-                })
-
-        self._sort_formats(formats)
-
         categories_html = self._search_regex(
             r'(?s)<table.+?(<span>Categories:.+?)</table>', webpage,
             'categories', default=None)
@@ -175,6 +177,7 @@ class XHamsterIE(InfoExtractor):
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': title,
             'description': description,
             'upload_date': upload_date,

From eadd313321f4fbacac2ec10f2f382197e401d7f7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 8 Jul 2017 15:48:05 +0800
Subject: [PATCH 514/611] [yam] Remove extractor

mymedia.yam.com is dead. An wikipedia user also pointed out that Yam's
blog service is no longer available. [1]

[1] https://zh.wikipedia.org/zh-tw/%E5%A4%A9%E7%A9%BA%E9%83%A8%E8%90%BD
---
 ChangeLog                          |   6 ++
 youtube_dl/extractor/extractors.py |   1 -
 youtube_dl/extractor/yam.py        | 123 -----------------------------
 3 files changed, 6 insertions(+), 124 deletions(-)
 delete mode 100644 youtube_dl/extractor/yam.py

diff --git a/ChangeLog b/ChangeLog
index 4c2011326..5d07c12cb 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+- [yam] Remove extractor
+
+
 version 2017.07.02
 
 Core
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index bbdb4a2fe..b83c3aba5 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1280,7 +1280,6 @@ from .yahoo import (
     YahooIE,
     YahooSearchIE,
 )
-from .yam import YamIE
 from .yandexmusic import (
     YandexMusicTrackIE,
     YandexMusicAlbumIE,
diff --git a/youtube_dl/extractor/yam.py b/youtube_dl/extractor/yam.py
deleted file mode 100644
index ef5535547..000000000
--- a/youtube_dl/extractor/yam.py
+++ /dev/null
@@ -1,123 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..compat import compat_urlparse
-from ..utils import (
-    float_or_none,
-    month_by_abbreviation,
-    ExtractorError,
-    get_element_by_attribute,
-)
-
-
-class YamIE(InfoExtractor):
-    IE_DESC = '蕃薯藤yam天空部落'
-    _VALID_URL = r'https?://mymedia\.yam\.com/m/(?P<id>\d+)'
-
-    _TESTS = [{
-        # An audio hosted on Yam
-        'url': 'http://mymedia.yam.com/m/2283921',
-        'md5': 'c011b8e262a52d5473d9c2e3c9963b9c',
-        'info_dict': {
-            'id': '2283921',
-            'ext': 'mp3',
-            'title': '發現 - 趙薇 京華煙雲主題曲',
-            'description': '發現 - 趙薇 京華煙雲主題曲',
-            'uploader_id': 'princekt',
-            'upload_date': '20080807',
-            'duration': 313.0,
-        }
-    }, {
-        # An external video hosted on YouTube
-        'url': 'http://mymedia.yam.com/m/3599430',
-        'md5': '03127cf10d8f35d120a9e8e52e3b17c6',
-        'info_dict': {
-            'id': 'CNpEoQlrIgA',
-            'ext': 'mp4',
-            'upload_date': '20150306',
-            'uploader': '新莊社大瑜伽社',
-            'description': 'md5:11e2e405311633ace874f2e6226c8b17',
-            'uploader_id': '2323agoy',
-            'title': '20090412陽明山二子坪-1',
-        },
-        'skip': 'Video does not exist',
-    }, {
-        'url': 'http://mymedia.yam.com/m/3598173',
-        'info_dict': {
-            'id': '3598173',
-            'ext': 'mp4',
-        },
-        'skip': 'cause Yam system error',
-    }, {
-        'url': 'http://mymedia.yam.com/m/3599437',
-        'info_dict': {
-            'id': '3599437',
-            'ext': 'mp4',
-        },
-        'skip': 'invalid YouTube URL',
-    }, {
-        'url': 'http://mymedia.yam.com/m/2373534',
-        'md5': '7ff74b91b7a817269d83796f8c5890b1',
-        'info_dict': {
-            'id': '2373534',
-            'ext': 'mp3',
-            'title': '林俊傑&蔡卓妍-小酒窩',
-            'description': 'md5:904003395a0fcce6cfb25028ff468420',
-            'upload_date': '20080928',
-            'uploader_id': 'onliner2',
-        }
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        page = self._download_webpage(url, video_id)
-
-        # Check for errors
-        system_msg = self._html_search_regex(
-            r'系統訊息(?:<br>|\n|\r)*([^<>]+)<br>', page, 'system message',
-            default=None)
-        if system_msg:
-            raise ExtractorError(system_msg, expected=True)
-
-        # Is it hosted externally on YouTube?
-        youtube_url = self._html_search_regex(
-            r'<embed src="(http://www.youtube.com/[^"]+)"',
-            page, 'YouTube url', default=None)
-        if youtube_url:
-            return self.url_result(youtube_url, 'Youtube')
-
-        title = self._html_search_regex(
-            r'<h1[^>]+class="heading"[^>]*>\s*(.+)\s*</h1>', page, 'title')
-
-        api_page = self._download_webpage(
-            'http://mymedia.yam.com/api/a/?pID=' + video_id, video_id,
-            note='Downloading API page')
-        api_result_obj = compat_urlparse.parse_qs(api_page)
-
-        info_table = get_element_by_attribute('class', 'info', page)
-        uploader_id = self._html_search_regex(
-            r'<!-- 發表作者 -->：[\n ]+<a href="/([a-z0-9]+)"',
-            info_table, 'uploader id', fatal=False)
-        mobj = re.search(r'<!-- 發表於 -->(?P<mon>[A-Z][a-z]{2})\s+' +
-                         r'(?P<day>\d{1,2}), (?P<year>\d{4})', page)
-        if mobj:
-            upload_date = '%s%02d%02d' % (
-                mobj.group('year'),
-                month_by_abbreviation(mobj.group('mon')),
-                int(mobj.group('day')))
-        else:
-            upload_date = None
-        duration = float_or_none(api_result_obj['totaltime'][0], scale=1000)
-
-        return {
-            'id': video_id,
-            'url': api_result_obj['mp3file'][0],
-            'title': title,
-            'description': self._html_search_meta('description', page),
-            'duration': duration,
-            'uploader_id': uploader_id,
-            'upload_date': upload_date,
-        }

From a49804816c0246d81b9d34d9f89f99fae06da887 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Jul 2017 18:12:15 +0700
Subject: [PATCH 515/611] [dailymotion] Add support for new layout (close
 #13580)

---
 youtube_dl/extractor/dailymotion.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index f8db76c18..74e991331 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -147,7 +147,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
         view_count_str = self._search_regex(
             (r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserPlays:([\s\d,.]+)"',
              r'video_views_count[^>]+>\s+([\s\d\,.]+)'),
-            webpage, 'view count', fatal=False)
+            webpage, 'view count', default=None)
         if view_count_str:
             view_count_str = re.sub(r'\s', '', view_count_str)
         view_count = str_to_int(view_count_str)
@@ -159,7 +159,9 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             [r'buildPlayer\(({.+?})\);\n',  # See https://github.com/rg3/youtube-dl/issues/7826
              r'playerV5\s*=\s*dmp\.create\([^,]+?,\s*({.+?})\);',
              r'buildPlayer\(({.+?})\);',
-             r'var\s+config\s*=\s*({.+?});'],
+             r'var\s+config\s*=\s*({.+?});',
+             # New layout regex (see https://github.com/rg3/youtube-dl/issues/13580)
+             r'__PLAYER_CONFIG__\s*=\s*({.+?});'],
             webpage, 'player v5', default=None)
         if player_v5:
             player = self._parse_json(player_v5, video_id)

From 8b347a389eaa6d545ada901c2e236a5eb2272960 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 00:26:13 +0700
Subject: [PATCH 516/611] [googledrive] Fix height extraction (closes #13603)

---
 youtube_dl/extractor/googledrive.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index 9705cfadd..c40da85c5 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -92,7 +92,7 @@ class GoogleDriveIE(InfoExtractor):
             if resolution:
                 f.update({
                     'width': resolution[0],
-                    'height': resolution[0],
+                    'height': resolution[1],
                 })
             formats.append(f)
         self._sort_formats(formats)

From 7a5773090789bec38a3f58dfb09039155919a540 Mon Sep 17 00:00:00 2001
From: rrooij <rrooij@users.noreply.github.com>
Date: Sun, 9 Jul 2017 09:21:40 +0200
Subject: [PATCH 517/611] [npo:live] Fix live stream id extraction (closes
 #13568)

---
 youtube_dl/extractor/npo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 5f8b6def1..516b1e941 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -341,7 +341,7 @@ class NPOLiveIE(NPOBaseIE):
         webpage = self._download_webpage(url, display_id)
 
         live_id = self._search_regex(
-            r'data-prid="([^"]+)"', webpage, 'live id')
+            [r'media-id="([^"]+)"', r'data-prid="([^"]+)"'], webpage, 'live id')
 
         return {
             '_type': 'url_transparent',

From 15237fcd51dca192103f08a910660616e3b241b8 Mon Sep 17 00:00:00 2001
From: mlindner <mlindner@users.noreply.github.com>
Date: Sun, 9 Jul 2017 00:54:52 -0700
Subject: [PATCH 518/611] [veoh] Extend _VALID_URL

---
 youtube_dl/extractor/veoh.py | 73 ++++++++++++++++++------------------
 1 file changed, 36 insertions(+), 37 deletions(-)

diff --git a/youtube_dl/extractor/veoh.py b/youtube_dl/extractor/veoh.py
index 0f5d68738..b20dddc5c 100644
--- a/youtube_dl/extractor/veoh.py
+++ b/youtube_dl/extractor/veoh.py
@@ -12,47 +12,46 @@ from ..utils import (
 
 
 class VeohIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?veoh\.com/(?:watch|iphone/#_Watch)/(?P<id>(?:v|yapi-)[\da-zA-Z]+)'
+    _VALID_URL = r'https?://(?:www\.)?veoh\.com/(?:watch|iphone/#_Watch)/(?P<id>(?:v|e|yapi-)[\da-zA-Z]+)'
 
-    _TESTS = [
-        {
-            'url': 'http://www.veoh.com/watch/v56314296nk7Zdmz3',
-            'md5': '620e68e6a3cff80086df3348426c9ca3',
-            'info_dict': {
-                'id': '56314296',
-                'ext': 'mp4',
-                'title': 'Straight Backs Are Stronger',
-                'uploader': 'LUMOback',
-                'description': 'At LUMOback, we believe straight backs are stronger.  The LUMOback Posture & Movement Sensor:  It gently vibrates when you slouch, inspiring improved posture and mobility.  Use the app to track your data and improve your posture over time. ',
-            },
+    _TESTS = [{
+        'url': 'http://www.veoh.com/watch/v56314296nk7Zdmz3',
+        'md5': '620e68e6a3cff80086df3348426c9ca3',
+        'info_dict': {
+            'id': '56314296',
+            'ext': 'mp4',
+            'title': 'Straight Backs Are Stronger',
+            'uploader': 'LUMOback',
+            'description': 'At LUMOback, we believe straight backs are stronger.  The LUMOback Posture & Movement Sensor:  It gently vibrates when you slouch, inspiring improved posture and mobility.  Use the app to track your data and improve your posture over time. ',
         },
-        {
-            'url': 'http://www.veoh.com/watch/v27701988pbTc4wzN?h1=Chile+workers+cover+up+to+avoid+skin+damage',
-            'md5': '4a6ff84b87d536a6a71e6aa6c0ad07fa',
-            'info_dict': {
-                'id': '27701988',
-                'ext': 'mp4',
-                'title': 'Chile workers cover up to avoid skin damage',
-                'description': 'md5:2bd151625a60a32822873efc246ba20d',
-                'uploader': 'afp-news',
-                'duration': 123,
-            },
-            'skip': 'This video has been deleted.',
+    }, {
+        'url': 'http://www.veoh.com/watch/v27701988pbTc4wzN?h1=Chile+workers+cover+up+to+avoid+skin+damage',
+        'md5': '4a6ff84b87d536a6a71e6aa6c0ad07fa',
+        'info_dict': {
+            'id': '27701988',
+            'ext': 'mp4',
+            'title': 'Chile workers cover up to avoid skin damage',
+            'description': 'md5:2bd151625a60a32822873efc246ba20d',
+            'uploader': 'afp-news',
+            'duration': 123,
         },
-        {
-            'url': 'http://www.veoh.com/watch/v69525809F6Nc4frX',
-            'md5': '4fde7b9e33577bab2f2f8f260e30e979',
-            'note': 'Embedded ooyala video',
-            'info_dict': {
-                'id': '69525809',
-                'ext': 'mp4',
-                'title': 'Doctors Alter Plan For Preteen\'s Weight Loss Surgery',
-                'description': 'md5:f5a11c51f8fb51d2315bca0937526891',
-                'uploader': 'newsy-videos',
-            },
-            'skip': 'This video has been deleted.',
+        'skip': 'This video has been deleted.',
+    }, {
+        'url': 'http://www.veoh.com/watch/v69525809F6Nc4frX',
+        'md5': '4fde7b9e33577bab2f2f8f260e30e979',
+        'note': 'Embedded ooyala video',
+        'info_dict': {
+            'id': '69525809',
+            'ext': 'mp4',
+            'title': 'Doctors Alter Plan For Preteen\'s Weight Loss Surgery',
+            'description': 'md5:f5a11c51f8fb51d2315bca0937526891',
+            'uploader': 'newsy-videos',
         },
-    ]
+        'skip': 'This video has been deleted.',
+    }, {
+        'url': 'http://www.veoh.com/watch/e152215AJxZktGS',
+        'only_matching': True,
+    }]
 
     def _extract_formats(self, source):
         formats = []

From 5af2fd7fa02734c2a23f917fb60f1c14da149d3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 15:55:04 +0700
Subject: [PATCH 519/611] [eagleplatform] Add support for another embed pattern
 (#13557)

---
 youtube_dl/extractor/eagleplatform.py | 36 ++++++++++++++++++++++-----
 1 file changed, 30 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 76d39adac..5e1de04a1 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -60,16 +60,40 @@ class EaglePlatformIE(InfoExtractor):
             webpage)
         if mobj is not None:
             return mobj.group('url')
-        # Basic usage embedding (see http://dultonmedia.github.io/eplayer/)
+        PLAYER_JS_RE = r'''
+                        <script[^>]+
+                            src=(?P<qjs>["\'])(?:https?:)?//(?P<host>(?:(?!(?P=qjs)).)+\.media\.eagleplatform\.com)/player/player\.js(?P=qjs)
+                        .+?
+                    '''
+        # "Basic usage" embedding (see http://dultonmedia.github.io/eplayer/)
         mobj = re.search(
             r'''(?xs)
-                    <script[^>]+
-                        src=(?P<q1>["\'])(?:https?:)?//(?P<host>.+?\.media\.eagleplatform\.com)/player/player\.js(?P=q1)
-                    .+?
+                    %s
                     <div[^>]+
-                        class=(?P<q2>["\'])eagleplayer(?P=q2)[^>]+
+                        class=(?P<qclass>["\'])eagleplayer(?P=qclass)[^>]+
                         data-id=["\'](?P<id>\d+)
-            ''', webpage)
+            ''' % PLAYER_JS_RE, webpage)
+        if mobj is not None:
+            return 'eagleplatform:%(host)s:%(id)s' % mobj.groupdict()
+        # Generalization of "Javascript code usage", "Combined usage" and
+        # "Usage without attaching to DOM" embeddings (see
+        # http://dultonmedia.github.io/eplayer/)
+        mobj = re.search(
+            r'''(?xs)
+                    %s
+                    <script>
+                    .+?
+                    new\s+EaglePlayer\(
+                        (?:[^,]+\s*,\s*)?
+                        {
+                            .+?
+                            \bid\s*:\s*["\']?(?P<id>\d+)
+                            .+?
+                        }
+                    \s*\)
+                    .+?
+                    </script>
+            ''' % PLAYER_JS_RE, webpage)
         if mobj is not None:
             return 'eagleplatform:%(host)s:%(id)s' % mobj.groupdict()
 

From 665e9452461abaff7127653265c78bd585acea6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 15:57:33 +0700
Subject: [PATCH 520/611] [eagleplatform] Add support for referrer protected
 videos (closes #13557)

---
 youtube_dl/extractor/eagleplatform.py | 25 ++++++++++++++++++++++---
 youtube_dl/extractor/generic.py       | 10 +++++-----
 2 files changed, 27 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 5e1de04a1..34891a362 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -11,6 +11,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     int_or_none,
+    unsmuggle_url,
 )
 
 
@@ -50,6 +51,10 @@ class EaglePlatformIE(InfoExtractor):
             'view_count': int,
         },
         'skip': 'Georestricted',
+    }, {
+        # referrer protected video (https://tvrain.ru/lite/teleshow/kak_vse_nachinalos/namin-418921/)
+        'url': 'tvrainru.media.eagleplatform.com:582306',
+        'only_matching': True,
     }]
 
     @staticmethod
@@ -103,9 +108,10 @@ class EaglePlatformIE(InfoExtractor):
         if status != 200:
             raise ExtractorError(' '.join(response['errors']), expected=True)
 
-    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', *args, **kwargs):
+    def _download_json(self, url_or_request, video_id, *args, **kwargs):
         try:
-            response = super(EaglePlatformIE, self)._download_json(url_or_request, video_id, note)
+            response = super(EaglePlatformIE, self)._download_json(
+                url_or_request, video_id, *args, **kwargs)
         except ExtractorError as ee:
             if isinstance(ee.cause, compat_HTTPError):
                 response = self._parse_json(ee.cause.read().decode('utf-8'), video_id)
@@ -117,11 +123,24 @@ class EaglePlatformIE(InfoExtractor):
         return self._download_json(url_or_request, video_id, note)['data'][0]
 
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+
         mobj = re.match(self._VALID_URL, url)
         host, video_id = mobj.group('custom_host') or mobj.group('host'), mobj.group('id')
 
+        headers = {}
+        query = {
+            'id': video_id,
+        }
+
+        referrer = smuggled_data.get('referrer')
+        if referrer:
+            headers['Referer'] = referrer
+            query['referrer'] = referrer
+
         player_data = self._download_json(
-            'http://%s/api/player_data?id=%s' % (host, video_id), video_id)
+            'http://%s/api/player_data' % host, video_id,
+            headers=headers, query=query)
 
         media = player_data['data']['playlist']['viewports'][0]['medialist'][0]
 
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index f9bff433c..7232f39db 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1185,7 +1185,7 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Kaltura'],
         },
-        # Eagle.Platform embed (generic URL)
+        # EaglePlatform embed (generic URL)
         {
             'url': 'http://lenta.ru/news/2015/03/06/navalny/',
             # Not checking MD5 as sometimes the direct HTTP link results in 404 and HLS is used
@@ -1200,7 +1200,7 @@ class GenericIE(InfoExtractor):
                 'age_limit': 0,
             },
         },
-        # ClipYou (Eagle.Platform) embed (custom URL)
+        # ClipYou (EaglePlatform) embed (custom URL)
         {
             'url': 'http://muz-tv.ru/play/7129/',
             # Not checking MD5 as sometimes the direct HTTP link results in 404 and HLS is used
@@ -2443,12 +2443,12 @@ class GenericIE(InfoExtractor):
         if kaltura_url:
             return self.url_result(smuggle_url(kaltura_url, {'source_url': url}), KalturaIE.ie_key())
 
-        # Look for Eagle.Platform embeds
+        # Look for EaglePlatform embeds
         eagleplatform_url = EaglePlatformIE._extract_url(webpage)
         if eagleplatform_url:
-            return self.url_result(eagleplatform_url, EaglePlatformIE.ie_key())
+            return self.url_result(smuggle_url(eagleplatform_url, {'referrer': url}), EaglePlatformIE.ie_key())
 
-        # Look for ClipYou (uses Eagle.Platform) embeds
+        # Look for ClipYou (uses EaglePlatform) embeds
         mobj = re.search(
             r'<iframe[^>]+src="https?://(?P<host>media\.clipyou\.ru)/index/player\?.*\brecord_id=(?P<id>\d+).*"', webpage)
         if mobj is not None:

From 250b042c7e71a6e8bbff534aa41c2b92dae1acf7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 16:02:38 +0700
Subject: [PATCH 521/611] [generic] Add tests for #13557

---
 youtube_dl/extractor/generic.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7232f39db..95c38698d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1199,6 +1199,24 @@ class GenericIE(InfoExtractor):
                 'view_count': int,
                 'age_limit': 0,
             },
+            'params': {
+                'skip_download': True,
+            },
+        },
+        # referrer protected EaglePlatform embed
+        {
+            'url': 'https://tvrain.ru/lite/teleshow/kak_vse_nachinalos/namin-418921/',
+            'info_dict': {
+                'id': '582306',
+                'ext': 'mp4',
+                'title': 'Стас Намин: «Мы нарушили девственность Кремля»',
+                'thumbnail': r're:^https?://.*\.jpg$',
+                'duration': 3382,
+                'view_count': int,
+            },
+            'params': {
+                'skip_download': True,
+            },
         },
         # ClipYou (EaglePlatform) embed (custom URL)
         {
@@ -1212,6 +1230,9 @@ class GenericIE(InfoExtractor):
                 'duration': 216,
                 'view_count': int,
             },
+            'params': {
+                'skip_download': True,
+            },
         },
         # Pladform embed
         {

From 4328ddf82b812420ffc120b4150251f751bff08c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 16:29:52 +0700
Subject: [PATCH 522/611] [extractor/common] Add support for AMP tags in
 _parse_html5_media_entries

---
 youtube_dl/extractor/common.py  |  7 +++++--
 youtube_dl/extractor/generic.py | 10 ++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index afeb4c5da..daa10885f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2132,15 +2132,18 @@ class InfoExtractor(object):
             return is_plain_url, formats
 
         entries = []
+        # amp-video and amp-audio are very similar to their HTML5 counterparts
+        # so we wll include them right here (see
+        # https://www.ampproject.org/docs/reference/components/amp-video)
         media_tags = [(media_tag, media_type, '')
                       for media_tag, media_type
-                      in re.findall(r'(?s)(<(video|audio)[^>]*/>)', webpage)]
+                      in re.findall(r'(?s)(<(?:amp-)?(video|audio)[^>]*/>)', webpage)]
         media_tags.extend(re.findall(
             # We only allow video|audio followed by a whitespace or '>'.
             # Allowing more characters may end up in significant slow down (see
             # https://github.com/rg3/youtube-dl/issues/11979, example URL:
             # http://www.porntrex.com/maps/videositemap.xml).
-            r'(?s)(<(?P<tag>video|audio)(?:\s+[^>]*)?>)(.*?)</(?P=tag)>', webpage))
+            r'(?s)(<(?P<tag>(?:amp-)?(?:video|audio))(?:\s+[^>]*)?>)(.*?)</(?P=tag)>', webpage))
         for media_tag, media_type, media_content in media_tags:
             media_info = {
                 'formats': [],
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 95c38698d..919f4f987 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1770,6 +1770,16 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': [MediasetIE.ie_key()],
         },
+        {
+            # AMP embed (see https://www.ampproject.org/docs/reference/components/amp-video)
+            'url': 'https://tvrain.ru/amp/418921/',
+            'md5': 'cc00413936695987e8de148b67d14f1d',
+            'info_dict': {
+                'id': '418921',
+                'ext': 'mp4',
+                'title': 'Стас Намин: «Мы нарушили девственность Кремля»',
+            },
+        },
         # {
         #     # TODO: find another test
         #     # http://schema.org/VideoObject

From d2b9f362fabad8f9490825456d8ed679d7159271 Mon Sep 17 00:00:00 2001
From: Christopher Smith <ctsmi@ucalgary.ca>
Date: Thu, 29 Jun 2017 13:10:45 -0600
Subject: [PATCH 523/611] [cjsw] Add extractor

---
 youtube_dl/extractor/cjsw.py       | 41 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 42 insertions(+)
 create mode 100644 youtube_dl/extractor/cjsw.py

diff --git a/youtube_dl/extractor/cjsw.py b/youtube_dl/extractor/cjsw.py
new file mode 100644
index 000000000..087cac9bc
--- /dev/null
+++ b/youtube_dl/extractor/cjsw.py
@@ -0,0 +1,41 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class CJSWIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?cjsw\.com/program/\S+/(?P<id>[0-9]+)'
+    IE_NAME = 'cjsw'
+    _TEST = {
+        'url': 'http://cjsw.com/program/freshly-squeezed/episode/20170620',
+        'md5': 'cee14d40f1e9433632c56e3d14977120',
+        'info_dict': {
+            'id': '20170620',
+            'ext': 'mp3',
+            'title': 'Freshly Squeezed',
+            'description': 'Sled Island artists featured // Live session with Phi Pho, followed by a live session with Sinzere & The Late Nights! // Stay Fresh Y\'all!!',
+        }
+    }
+
+    def _real_extract(self, url):
+        episode_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, episode_id)
+
+        title = self._search_regex(
+            r'<button[^>]+data-showname=(["\'])(?P<title>(?!\1).+?)\1[^>]*>', webpage, 'title', group='title')
+        description = self._html_search_regex(
+            r'<p>(?P<description>.+?)</p>', webpage, 'description', fatal=False)
+        formats = [{
+            'url': self._search_regex(
+                r'<button[^>]+data-audio-src=(["\'])(?P<audio_url>(?!\1).+?)\1[^>]*>', webpage, 'audio_url', group='audio_url'),
+            'ext': 'mp3',
+            'vcodec': 'none',
+        }]
+        return {
+            'id': episode_id,
+            'title': title,
+            'description': description,
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index b83c3aba5..4524fa687 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -185,6 +185,7 @@ from .chirbit import (
     ChirbitProfileIE,
 )
 from .cinchcast import CinchcastIE
+from .cjsw import CJSWIE
 from .clipfish import ClipfishIE
 from .cliphunter import CliphunterIE
 from .cliprs import ClipRsIE

From c319d1c4833f89df818fe39f4c99cdc5c9a8bf01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 17:00:45 +0700
Subject: [PATCH 524/611] [csjw] Fix issues and improve extraction (closes
 #13525)

---
 youtube_dl/extractor/cjsw.py | 57 ++++++++++++++++++++++++++----------
 1 file changed, 41 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/cjsw.py b/youtube_dl/extractor/cjsw.py
index 087cac9bc..aab6ea535 100644
--- a/youtube_dl/extractor/cjsw.py
+++ b/youtube_dl/extractor/cjsw.py
@@ -1,41 +1,66 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    unescapeHTML,
+)
 
 
 class CJSWIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?cjsw\.com/program/\S+/(?P<id>[0-9]+)'
-    IE_NAME = 'cjsw'
+    _VALID_URL = r'https?://(?:www\.)?cjsw\.com/program/(?P<program>[^/]+)/episode/(?P<id>\d+)'
     _TEST = {
         'url': 'http://cjsw.com/program/freshly-squeezed/episode/20170620',
         'md5': 'cee14d40f1e9433632c56e3d14977120',
         'info_dict': {
-            'id': '20170620',
+            'id': '91d9f016-a2e7-46c5-8dcb-7cbcd7437c41',
             'ext': 'mp3',
-            'title': 'Freshly Squeezed',
-            'description': 'Sled Island artists featured // Live session with Phi Pho, followed by a live session with Sinzere & The Late Nights! // Stay Fresh Y\'all!!',
-        }
+            'title': 'Freshly Squeezed – Episode June 20, 2017',
+            'description': 'md5:c967d63366c3898a80d0c7b0ff337202',
+            'series': 'Freshly Squeezed',
+            'episode_id': '20170620',
+        },
     }
 
     def _real_extract(self, url):
-        episode_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        program, episode_id = mobj.group('program', 'id')
+        audio_id = '%s/%s' % (program, episode_id)
 
         webpage = self._download_webpage(url, episode_id)
 
-        title = self._search_regex(
-            r'<button[^>]+data-showname=(["\'])(?P<title>(?!\1).+?)\1[^>]*>', webpage, 'title', group='title')
-        description = self._html_search_regex(
-            r'<p>(?P<description>.+?)</p>', webpage, 'description', fatal=False)
+        title = unescapeHTML(self._search_regex(
+            (r'<h1[^>]+class=["\']episode-header__title["\'][^>]*>(?P<title>[^<]+)',
+             r'data-audio-title=(["\'])(?P<title>(?:(?!\1).)+)\1'),
+            webpage, 'title', group='title'))
+
+        audio_url = self._search_regex(
+            r'<button[^>]+data-audio-src=(["\'])(?P<url>(?:(?!\1).)+)\1',
+            webpage, 'audio url', group='url')
+
+        audio_id = self._search_regex(
+            r'/([\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})\.mp3',
+            audio_url, 'audio id', default=audio_id)
+
         formats = [{
-            'url': self._search_regex(
-                r'<button[^>]+data-audio-src=(["\'])(?P<audio_url>(?!\1).+?)\1[^>]*>', webpage, 'audio_url', group='audio_url'),
-            'ext': 'mp3',
+            'url': audio_url,
+            'ext': determine_ext(audio_url, 'mp3'),
             'vcodec': 'none',
         }]
+
+        description = self._html_search_regex(
+            r'<p>(?P<description>.+?)</p>', webpage, 'description', fatal=False)
+        series = self._search_regex(
+            r'data-showname=(["\'])(?P<name>(?:(?!\1).)+)\1', webpage,
+            'series', default=program, group='name')
+
         return {
-            'id': episode_id,
+            'id': audio_id,
             'title': title,
             'description': description,
             'formats': formats,
+            'series': series,
+            'episode_id': episode_id,
         }

From 0d2f0b0357325823782884327a158aeccf4f9b49 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 17:05:11 +0700
Subject: [PATCH 525/611] [csjw] Make description optional

---
 youtube_dl/extractor/cjsw.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/cjsw.py b/youtube_dl/extractor/cjsw.py
index aab6ea535..dd271586f 100644
--- a/youtube_dl/extractor/cjsw.py
+++ b/youtube_dl/extractor/cjsw.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 class CJSWIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?cjsw\.com/program/(?P<program>[^/]+)/episode/(?P<id>\d+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://cjsw.com/program/freshly-squeezed/episode/20170620',
         'md5': 'cee14d40f1e9433632c56e3d14977120',
         'info_dict': {
@@ -22,7 +22,11 @@ class CJSWIE(InfoExtractor):
             'series': 'Freshly Squeezed',
             'episode_id': '20170620',
         },
-    }
+    }, {
+        # no description
+        'url': 'http://cjsw.com/program/road-pops/episode/20170707/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -51,7 +55,8 @@ class CJSWIE(InfoExtractor):
         }]
 
         description = self._html_search_regex(
-            r'<p>(?P<description>.+?)</p>', webpage, 'description', fatal=False)
+            r'<p>(?P<description>.+?)</p>', webpage, 'description',
+            default=None)
         series = self._search_regex(
             r'data-showname=(["\'])(?P<name>(?:(?!\1).)+)\1', webpage,
             'series', default=program, group='name')

From a02682fd13ce5ba88d2508c90559eaa7f43b65d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 17:09:44 +0700
Subject: [PATCH 526/611] Keep in sync with ffmpeg's current malformed AAC
 bitstream wording (closes #13587)

---
 youtube_dl/YoutubeDL.py            | 4 ++--
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b3a6d4d3b..60ee4b7d8 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1890,7 +1890,7 @@ class YoutubeDL(object):
                         info_dict.get('protocol') == 'm3u8' and
                         self.params.get('hls_prefer_native')):
                     if fixup_policy == 'warn':
-                        self.report_warning('%s: malformated aac bitstream.' % (
+                        self.report_warning('%s: malformed AAC bitstream detected.' % (
                             info_dict['id']))
                     elif fixup_policy == 'detect_or_warn':
                         fixup_pp = FFmpegFixupM3u8PP(self)
@@ -1899,7 +1899,7 @@ class YoutubeDL(object):
                             info_dict['__postprocessors'].append(fixup_pp)
                         else:
                             self.report_warning(
-                                '%s: malformated aac bitstream. %s'
+                                '%s: malformed AAC bitstream detected. %s'
                                 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
                     else:
                         assert fixup_policy in ('ignore', 'never')
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index f021ea8fd..51256a3fb 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -542,7 +542,7 @@ class FFmpegFixupM3u8PP(FFmpegPostProcessor):
             temp_filename = prepend_extension(filename, 'temp')
 
             options = ['-c', 'copy', '-f', 'mp4', '-bsf:a', 'aac_adtstoasc']
-            self._downloader.to_screen('[ffmpeg] Fixing malformated aac bitstream in "%s"' % filename)
+            self._downloader.to_screen('[ffmpeg] Fixing malformed AAC bitstream in "%s"' % filename)
             self.run_ffmpeg(filename, temp_filename, options)
 
             os.remove(encodeFilename(filename))

From ed84454d358f3cbfdc43dab31328b165f9c72c68 Mon Sep 17 00:00:00 2001
From: Santiago Calcagno <santicalcagno@gmail.com>
Date: Tue, 13 Jun 2017 12:32:04 -0300
Subject: [PATCH 527/611] [egghead:course] Fix extraction

---
 youtube_dl/extractor/egghead.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/egghead.py b/youtube_dl/extractor/egghead.py
index db921465e..01fcdb6cf 100644
--- a/youtube_dl/extractor/egghead.py
+++ b/youtube_dl/extractor/egghead.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -22,18 +20,18 @@ class EggheadCourseIE(InfoExtractor):
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
-        webpage = self._download_webpage(url, playlist_id)
+        api_url = 'https://egghead.io/api/v1/series/' + playlist_id
+        course = self._download_json(api_url, playlist_id)
+        title = course.get('title')
+        description = course.get('description')
 
-        title = self._html_search_regex(r'<h1 class="title">([^<]+)</h1>', webpage, 'title')
-        ul = self._search_regex(r'(?s)<ul class="series-lessons-list">(.*?)</ul>', webpage, 'session list')
-
-        found = re.findall(r'(?s)<a class="[^"]*"\s*href="([^"]+)">\s*<li class="item', ul)
-        entries = [self.url_result(m) for m in found]
+        lessons = course.get('lessons')
+        entries = [{'_type': 'url', 'ie_key': 'Wistia', 'url': 'wistia:' + l.get('wistia_id')} for l in lessons]
 
         return {
             '_type': 'playlist',
             'id': playlist_id,
             'title': title,
-            'description': self._og_search_description(webpage),
+            'description': description,
             'entries': entries,
         }

From 485cb375766df8f2ef79b7fe2915ead4ef61a01e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 17:28:42 +0700
Subject: [PATCH 528/611] [egghead:course] Improve (closes #13370)

---
 youtube_dl/extractor/egghead.py | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/egghead.py b/youtube_dl/extractor/egghead.py
index 01fcdb6cf..c86f52319 100644
--- a/youtube_dl/extractor/egghead.py
+++ b/youtube_dl/extractor/egghead.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 class EggheadCourseIE(InfoExtractor):
     IE_DESC = 'egghead.io course'
     IE_NAME = 'egghead:course'
-    _VALID_URL = r'https://egghead\.io/courses/(?P<id>[a-zA-Z_0-9-]+)'
+    _VALID_URL = r'https://egghead\.io/courses/(?P<id>[^/?#&]+)'
     _TEST = {
         'url': 'https://egghead.io/courses/professor-frisby-introduces-composable-functional-javascript',
         'playlist_count': 29,
@@ -20,18 +20,16 @@ class EggheadCourseIE(InfoExtractor):
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
-        api_url = 'https://egghead.io/api/v1/series/' + playlist_id
-        course = self._download_json(api_url, playlist_id)
-        title = course.get('title')
-        description = course.get('description')
 
-        lessons = course.get('lessons')
-        entries = [{'_type': 'url', 'ie_key': 'Wistia', 'url': 'wistia:' + l.get('wistia_id')} for l in lessons]
+        course = self._download_json(
+            'https://egghead.io/api/v1/series/%s' % playlist_id, playlist_id)
 
-        return {
-            '_type': 'playlist',
-            'id': playlist_id,
-            'title': title,
-            'description': description,
-            'entries': entries,
-        }
+        entries = [
+            self.url_result(
+                'wistia:%s' % lesson['wistia_id'], ie='Wistia',
+                video_id=lesson['wistia_id'], video_title=lesson.get('title'))
+            for lesson in course['lessons'] if lesson.get('wistia_id')]
+
+        return self.playlist_result(
+            entries, playlist_id, course.get('title'),
+            course.get('description'))

From 58179eb7d96ebef26a0083e80a2022fab4ca1558 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 17:55:40 +0700
Subject: [PATCH 529/611] [abc.net.au:iview] Extract more formats (closes
 #13492, closes #13489)

---
 youtube_dl/extractor/abc.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index 0247cabf9..60f753b95 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -3,11 +3,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
     js_to_json,
     int_or_none,
     parse_iso8601,
+    try_get,
 )
 
 
@@ -124,7 +126,20 @@ class ABCIViewIE(InfoExtractor):
         title = video_params.get('title') or video_params['seriesTitle']
         stream = next(s for s in video_params['playlist'] if s.get('type') == 'program')
 
-        formats = self._extract_akamai_formats(stream['hds-unmetered'], video_id)
+        format_urls = [
+            try_get(stream, lambda x: x['hds-unmetered'], compat_str)]
+
+        # May have higher quality video
+        sd_url = try_get(
+            stream, lambda x: x['streams']['hds']['sd'], compat_str)
+        if sd_url:
+            format_urls.append(sd_url.replace('metered', 'um'))
+
+        formats = []
+        for format_url in format_urls:
+            if format_url:
+                formats.extend(
+                    self._extract_akamai_formats(format_url, video_id))
         self._sort_formats(formats)
 
         subtitles = {}

From 256a746d21634eccad07a1e6dcafedcdf8b6181b Mon Sep 17 00:00:00 2001
From: luboss <lubos.katrinec@gmail.com>
Date: Fri, 2 Jun 2017 22:44:39 +0200
Subject: [PATCH 530/611] [joj] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/joj.py        | 56 ++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100755 youtube_dl/extractor/joj.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 4524fa687..9ee080895 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -470,6 +470,7 @@ from .jamendo import (
 )
 from .jeuxvideo import JeuxVideoIE
 from .jove import JoveIE
+from .joj import JojIE
 from .jwplatform import JWPlatformIE
 from .jpopsukitv import JpopsukiIE
 from .kaltura import KalturaIE
diff --git a/youtube_dl/extractor/joj.py b/youtube_dl/extractor/joj.py
new file mode 100755
index 000000000..2ebfec902
--- /dev/null
+++ b/youtube_dl/extractor/joj.py
@@ -0,0 +1,56 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+import re
+
+
+class JojIE(InfoExtractor):
+    _VALID_URL = r'https?://[a-z0-9]+\.joj\.sk/([^/]+/)*(?P<title_query>(?P<release_date>[0-9]{4}(-[0-9]{2}){2}).*)' # noqa
+    _TESTS = [{
+        'url': 'https://www.joj.sk/nove-byvanie/archiv/2017-05-28-nove-byvanie', # noqa
+        'info_dict': {
+            'id': 'a388ec4c-6019-4a4a-9312-b1bee194e932',
+            'ext': 'mp4',
+            'title': 'Nové Bývanie',
+            'release_date': '20170528'
+        }
+    }, {
+        'url': 'http://nasi.joj.sk/epizody/2016-09-06-stari-rodicia',
+        'info_dict': {
+            'id': 'f18b2c5f-9ea8-4941-a164-a814c53306ad',
+            'ext': 'mp4',
+            'title': 'Starí Rodičia',
+            'release_date': '20160906'
+        }
+    }]
+
+    media_src_url = 'http://n16.joj.sk/storage/'
+    xml_source_url = 'https://media.joj.sk/services/Video.php?clip='
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        release_date = mobj.group('release_date').replace('-', '')
+        webpage = self._download_webpage(url, 'id')
+        video_id = self._html_search_regex(
+            r'https?://([a-z0-9]+\.)joj\.sk/embed/(?P<video_id>[a-f0-9\-]+)',
+            webpage, 'id', group='video_id')
+        xml_playlist_url = self.xml_source_url + video_id
+        xml_playlist_et = self._download_xml(xml_playlist_url, 'XML playlist')
+        formats = []
+        for file_el in xml_playlist_et.findall('files/file'):
+            try:
+                height = int(file_el.attrib['id'].replace('p', ''))
+            except ValueError:
+                height = 0
+            formats.append({'height': height,
+                            'url': self.media_src_url + file_el.attrib['path'].replace(  # noqa
+                                'dat/', '', 1)})
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage).title(),
+            'formats': formats,
+            'release_date': release_date
+        }

From 73cf76a93fe48240bf82b1685b1403f05b793ebf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 19:05:18 +0700
Subject: [PATCH 531/611] [joj] Rewrite and add support for generic embeds
 (closes #13268)

---
 youtube_dl/extractor/generic.py |  17 +++++
 youtube_dl/extractor/joj.py     | 108 ++++++++++++++++++++++----------
 2 files changed, 93 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 919f4f987..f2c577f98 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -91,6 +91,7 @@ from .anvato import AnvatoIE
 from .washingtonpost import WashingtonPostIE
 from .wistia import WistiaIE
 from .mediaset import MediasetIE
+from .joj import JojIE
 
 
 class GenericIE(InfoExtractor):
@@ -1770,6 +1771,16 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': [MediasetIE.ie_key()],
         },
+        {
+            # JOJ.sk embeds
+            'url': 'https://www.noviny.sk/slovensko/238543-slovenskom-sa-prehnala-vlna-silnych-burok',
+            'info_dict': {
+                'id': '238543-slovenskom-sa-prehnala-vlna-silnych-burok',
+                'title': 'Slovenskom sa prehnala vlna silných búrok',
+            },
+            'playlist_mincount': 5,
+            'add_ie': [JojIE.ie_key()],
+        },
         {
             # AMP embed (see https://www.ampproject.org/docs/reference/components/amp-video)
             'url': 'https://tvrain.ru/amp/418921/',
@@ -2722,6 +2733,12 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 mediaset_urls, video_id, video_title, ie=MediasetIE.ie_key())
 
+        # Look for JOJ.sk embeds
+        joj_urls = JojIE._extract_urls(webpage)
+        if joj_urls:
+            return self.playlist_from_matches(
+                joj_urls, video_id, video_title, ie=JojIE.ie_key())
+
         def merge_dicts(dict1, dict2):
             merged = {}
             for k, v in dict1.items():
diff --git a/youtube_dl/extractor/joj.py b/youtube_dl/extractor/joj.py
index 2ebfec902..a764023e9 100755
--- a/youtube_dl/extractor/joj.py
+++ b/youtube_dl/extractor/joj.py
@@ -1,56 +1,100 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
 import re
 
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    js_to_json,
+    try_get,
+)
+
 
 class JojIE(InfoExtractor):
-    _VALID_URL = r'https?://[a-z0-9]+\.joj\.sk/([^/]+/)*(?P<title_query>(?P<release_date>[0-9]{4}(-[0-9]{2}){2}).*)' # noqa
+    _VALID_URL = r'''(?x)
+                    (?:
+                        joj:|
+                        https?://media\.joj\.sk/embed/
+                    )
+                    (?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})
+                '''
     _TESTS = [{
-        'url': 'https://www.joj.sk/nove-byvanie/archiv/2017-05-28-nove-byvanie', # noqa
+        'url': 'https://media.joj.sk/embed/a388ec4c-6019-4a4a-9312-b1bee194e932',
         'info_dict': {
             'id': 'a388ec4c-6019-4a4a-9312-b1bee194e932',
             'ext': 'mp4',
-            'title': 'Nové Bývanie',
-            'release_date': '20170528'
+            'title': 'NOVÉ BÝVANIE',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 3118,
         }
     }, {
-        'url': 'http://nasi.joj.sk/epizody/2016-09-06-stari-rodicia',
-        'info_dict': {
-            'id': 'f18b2c5f-9ea8-4941-a164-a814c53306ad',
-            'ext': 'mp4',
-            'title': 'Starí Rodičia',
-            'release_date': '20160906'
-        }
+        'url': 'joj:a388ec4c-6019-4a4a-9312-b1bee194e932',
+        'only_matching': True,
     }]
 
-    media_src_url = 'http://n16.joj.sk/storage/'
-    xml_source_url = 'https://media.joj.sk/services/Video.php?clip='
+    @staticmethod
+    def _extract_urls(webpage):
+        return re.findall(
+            r'<iframe\b[^>]+\bsrc=["\'](?P<url>(?:https?:)?//media\.joj\.sk/embed/[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
+            webpage)
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        release_date = mobj.group('release_date').replace('-', '')
-        webpage = self._download_webpage(url, 'id')
-        video_id = self._html_search_regex(
-            r'https?://([a-z0-9]+\.)joj\.sk/embed/(?P<video_id>[a-f0-9\-]+)',
-            webpage, 'id', group='video_id')
-        xml_playlist_url = self.xml_source_url + video_id
-        xml_playlist_et = self._download_xml(xml_playlist_url, 'XML playlist')
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            'https://media.joj.sk/embed/%s' % video_id, video_id)
+
+        title = self._search_regex(
+            (r'videoTitle\s*:\s*(["\'])(?P<title>(?:(?!\1).)+)\1',
+             r'<title>(?P<title>[^<]+)'), webpage, 'title',
+            default=None, group='title') or self._og_search_title(webpage)
+
+        bitrates = self._parse_json(
+            self._search_regex(
+                r'(?s)bitrates\s*=\s*({.+?});', webpage, 'bitrates',
+                default='{}'),
+            video_id, transform_source=js_to_json, fatal=False)
+
         formats = []
-        for file_el in xml_playlist_et.findall('files/file'):
-            try:
-                height = int(file_el.attrib['id'].replace('p', ''))
-            except ValueError:
-                height = 0
-            formats.append({'height': height,
-                            'url': self.media_src_url + file_el.attrib['path'].replace(  # noqa
-                                'dat/', '', 1)})
+        for format_url in try_get(bitrates, lambda x: x['mp4'], list) or []:
+            if isinstance(format_url, compat_str):
+                height = self._search_regex(
+                    r'(\d+)[pP]\.', format_url, 'height', default=None)
+                formats.append({
+                    'url': format_url,
+                    'format_id': '%sp' % height if height else None,
+                    'height': int(height),
+                })
+        if not formats:
+            playlist = self._download_xml(
+                'https://media.joj.sk/services/Video.php?clip=%s' % video_id,
+                video_id)
+            for file_el in playlist.findall('./files/file'):
+                path = file_el.get('path')
+                if not path:
+                    continue
+                format_id = file_el.get('id') or file_el.get('label')
+                formats.append({
+                    'url': 'http://n16.joj.sk/storage/%s' % path.replace(
+                        'dat/', '', 1),
+                    'format_id': format_id,
+                    'height': int_or_none(self._search_regex(
+                        r'(\d+)[pP]', format_id or path, 'height',
+                        default=None)),
+                })
         self._sort_formats(formats)
 
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        duration = int_or_none(self._search_regex(
+            r'videoDuration\s*:\s*(\d+)', webpage, 'duration', fatal=False))
+
         return {
             'id': video_id,
-            'title': self._og_search_title(webpage).title(),
+            'title': title,
+            'thumbnail': thumbnail,
+            'duration': duration,
             'formats': formats,
-            'release_date': release_date
         }

From 6e925598d68f5d5216aa3e9abed5c7706a68c891 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 19:15:48 +0700
Subject: [PATCH 532/611] [csjw] Add coding cookie

---
 youtube_dl/extractor/cjsw.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/cjsw.py b/youtube_dl/extractor/cjsw.py
index dd271586f..505bdbe16 100644
--- a/youtube_dl/extractor/cjsw.py
+++ b/youtube_dl/extractor/cjsw.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

From 71a1db89198100a0e9bc5099aeed622264690203 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 20:06:24 +0700
Subject: [PATCH 533/611] [dailymail] Add support for embeds

---
 youtube_dl/extractor/dailymail.py | 17 ++++++++++++++---
 youtube_dl/extractor/generic.py   | 21 +++++++++++++++++++++
 2 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dailymail.py b/youtube_dl/extractor/dailymail.py
index 538565c66..af3978035 100644
--- a/youtube_dl/extractor/dailymail.py
+++ b/youtube_dl/extractor/dailymail.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
@@ -12,8 +14,8 @@ from ..utils import (
 
 
 class DailyMailIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dailymail\.co\.uk/video/[^/]+/video-(?P<id>[0-9]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?dailymail\.co\.uk/(?:video/[^/]+/video-|embed/video/)(?P<id>[0-9]+)'
+    _TESTS = [{
         'url': 'http://www.dailymail.co.uk/video/tvshowbiz/video-1295863/The-Mountain-appears-sparkling-water-ad-Heavy-Bubbles.html',
         'md5': 'f6129624562251f628296c3a9ffde124',
         'info_dict': {
@@ -22,7 +24,16 @@ class DailyMailIE(InfoExtractor):
             'title': 'The Mountain appears in sparkling water ad for \'Heavy Bubbles\'',
             'description': 'md5:a93d74b6da172dd5dc4d973e0b766a84',
         }
-    }
+    }, {
+        'url': 'http://www.dailymail.co.uk/embed/video/1295863.html',
+        'only_matching': True,
+    }]
+
+    @staticmethod
+    def _extract_urls(webpage):
+        return re.findall(
+            r'<iframe\b[^>]+\bsrc=["\'](?P<url>(?:https?:)?//(?:www\.)?dailymail\.co\.uk/embed/video/\d+\.html)',
+            webpage)
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index f2c577f98..5e8890d41 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -57,6 +57,7 @@ from .dailymotion import (
     DailymotionIE,
     DailymotionCloudIE,
 )
+from .dailymail import DailyMailIE
 from .onionstudios import OnionStudiosIE
 from .viewlift import ViewLiftEmbedIE
 from .mtv import MTVServicesEmbeddedIE
@@ -760,6 +761,20 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Dailymotion'],
         },
+        # DailyMail embed
+        {
+            'url': 'http://www.bumm.sk/krimi/2017/07/05/biztonsagi-kamera-buktatta-le-az-agg-ferfit-utlegelo-apolot',
+            'info_dict': {
+                'id': '1495629',
+                'ext': 'mp4',
+                'title': 'Care worker punches elderly dementia patient in head 11 times',
+                'description': 'md5:3a743dee84e57e48ec68bf67113199a5',
+            },
+            'add_ie': ['DailyMail'],
+            'params': {
+                'skip_download': True,
+            },
+        },
         # YouTube embed
         {
             'url': 'http://www.badzine.de/ansicht/datum/2014/06/09/so-funktioniert-die-neue-englische-badminton-liga.html',
@@ -2190,6 +2205,12 @@ class GenericIE(InfoExtractor):
                 return self.playlist_from_matches(
                     playlists, video_id, video_title, lambda p: '//dailymotion.com/playlist/%s' % p)
 
+        # Look for DailyMail embeds
+        dailymail_urls = DailyMailIE._extract_urls(webpage)
+        if dailymail_urls:
+            return self.playlist_from_matches(
+                dailymail_urls, video_id, video_title, ie=DailyMailIE.ie_key())
+
         # Look for embedded Wistia player
         wistia_url = WistiaIE._extract_url(webpage)
         if wistia_url:

From 207acd8465b51d9d00d2bdda22f10858eb7f1bb5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 20:15:15 +0700
Subject: [PATCH 534/611] [ChangeLog] Actualize

---
 ChangeLog | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 5d07c12cb..edfde8b6f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,27 @@
 version <unreleased>
 
+Core
++ [extractor/common] Add support for AMP tags in _parse_html5_media_entries
++ [utils] Support attributes with no values in get_elements_by_attribute
+
 Extractors
++ [dailymail] Add support for embeds
++ [joj] Add support for joj.sk (#13268)
+* [abc.net.au:iview] Extract more formats (#13492, #13489)
+* [egghead:course] Fix extraction (#6635, #13370)
++ [cjsw] Add support for cjsw.com (#13525)
++ [eagleplatform] Add support for referrer protected videos (#13557)
++ [eagleplatform] Add support for another embed pattern (#13557)
+* [veoh] Extend URL regular expression (#13601)
+* [npo:live] Fix live stream id extraction (#13568, #13605)
+* [googledrive] Fix height extraction (#13603)
++ [dailymotion] Add support for new layout (#13580)
 - [yam] Remove extractor
+* [xhamster] Extract all formats and fix duration extraction (#13593)
++ [xhamster] Add support for new URL schema (#13593)
+* [espn] Extend URL regular expression (#13244, #13549)
+* [kaltura] Fix typo in subtitles extraction (#13569)
+* [vier] Adapt extraction to redesign (#13575)
 
 
 version 2017.07.02

From 65c416dda896f8a0023f01547e6b707dd57ed30a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 20:16:38 +0700
Subject: [PATCH 535/611] release 2017.07.09

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 ++-
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 974603507..c4314855d 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.07.02*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.07.02**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.07.09*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.07.09**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.07.02
+[debug] youtube-dl version 2017.07.09
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index edfde8b6f..c379cae71 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.07.09
 
 Core
 + [extractor/common] Add support for AMP tags in _parse_html5_media_entries
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index db2e2bac9..b6a147faf 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -154,6 +154,7 @@
  - **chirbit**
  - **chirbit:profile**
  - **Cinchcast**
+ - **CJSW**
  - **Clipfish**
  - **cliphunter**
  - **ClipRs**
@@ -369,6 +370,7 @@
  - **Jamendo**
  - **JamendoAlbum**
  - **JeuxVideo**
+ - **Joj**
  - **Jove**
  - **jpopsuki.tv**
  - **JWPlatform**
@@ -996,7 +998,6 @@
  - **XVideos**
  - **XXXYMovies**
  - **Yahoo**: Yahoo screen and movies
- - **Yam**: 蕃薯藤yam天空部落
  - **yandexmusic:album**: Яндекс.Музыка - Альбом
  - **yandexmusic:playlist**: Яндекс.Музыка - Плейлист
  - **yandexmusic:track**: Яндекс.Музыка - Трек
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0db974f97..14358a74c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.07.02'
+__version__ = '2017.07.09'

From 7bf539edcc3dc44481d5196fd01637698653ffc7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Jul 2017 00:14:41 +0700
Subject: [PATCH 536/611] [eagleplatform] Fix test

---
 youtube_dl/extractor/eagleplatform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 34891a362..42789278e 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -53,7 +53,7 @@ class EaglePlatformIE(InfoExtractor):
         'skip': 'Georestricted',
     }, {
         # referrer protected video (https://tvrain.ru/lite/teleshow/kak_vse_nachinalos/namin-418921/)
-        'url': 'tvrainru.media.eagleplatform.com:582306',
+        'url': 'eagleplatform:tvrainru.media.eagleplatform.com:582306',
         'only_matching': True,
     }]
 

From b71c18b4343d54ce8373e9a11df882aca1ae82a0 Mon Sep 17 00:00:00 2001
From: coreynicholson <coreynicholson@users.noreply.github.com>
Date: Sun, 9 Jul 2017 22:24:04 +0100
Subject: [PATCH 537/611] [vlive:playlist] Add extractor

---
 youtube_dl/extractor/extractors.py |  3 +-
 youtube_dl/extractor/vlive.py      | 56 ++++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 9ee080895..eb1541729 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1206,7 +1206,8 @@ from .vk import (
 )
 from .vlive import (
     VLiveIE,
-    VLiveChannelIE
+    VLiveChannelIE,
+    VLivePlaylistIE
 )
 from .vodlocker import VodlockerIE
 from .vodpl import VODPlIE
diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index e58940607..f3825db5c 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -49,6 +49,10 @@ class VLiveIE(InfoExtractor):
         },
     }]
 
+    @classmethod
+    def suitable(cls, url):
+        return False if VLivePlaylistIE.suitable(url) else super(VLiveIE, cls).suitable(url)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -261,3 +265,55 @@ class VLiveChannelIE(InfoExtractor):
 
         return self.playlist_result(
             entries, channel_code, channel_name)
+
+
+class VLivePlaylistIE(InfoExtractor):
+    IE_NAME = 'vlive:playlist'
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?vlive\.tv/video/(?P<video_id>[0-9]+)/playlist/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.vlive.tv/video/22867/playlist/22912',
+        'info_dict': {
+            'id': '22912',
+            'title': 'Valentine Day Message from TWICE'
+        },
+        'playlist_mincount': 9
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        video_id_match = re.match(self._VALID_URL, url)
+        assert video_id_match
+        video_id = compat_str(video_id_match.group('video_id'))
+
+        VIDEO_URL_TEMPLATE = 'http://www.vlive.tv/video/%s'
+        if self._downloader.params.get('noplaylist'):
+            self.to_screen(
+                'Downloading just video %s because of --no-playlist' % video_id)
+            return self.url_result(
+                VIDEO_URL_TEMPLATE % video_id,
+                ie=VLiveIE.ie_key(), video_id=video_id)
+
+        self.to_screen(
+            'Downloading playlist %s - add --no-playlist to just download video' % playlist_id)
+
+        webpage = self._download_webpage(
+            'http://www.vlive.tv/video/%s/playlist/%s' % (video_id, playlist_id), video_id)
+
+        playlist_name = self._html_search_regex(
+            r'<div[^>]+class="[^"]*multicam_playlist[^>]*>\s*<h3[^>]+>([^<]+)',
+            webpage, 'playlist name', fatal=False)
+
+        item_ids = self._search_regex(
+            r'\bvar\s+playlistVideoSeqs\s*=\s*(\[[^]]+\])',
+            webpage, 'playlist item ids')
+
+        entries = []
+        for item_id in self._parse_json(item_ids, playlist_id):
+            item_id = compat_str(item_id)
+            entries.append(
+                self.url_result(
+                    VIDEO_URL_TEMPLATE % item_id,
+                    ie=VLiveIE.ie_key(), video_id=item_id))
+
+        return self.playlist_result(
+            entries, playlist_id, playlist_name)

From e3cd1fcdd177613acae4198cafbff51fbbb912c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Jul 2017 04:32:24 +0700
Subject: [PATCH 538/611] [vlive:playlist] Relax and simplify

---
 youtube_dl/extractor/vlive.py | 41 +++++++++++++++++------------------
 1 file changed, 20 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index f3825db5c..77c120a57 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -280,10 +280,8 @@ class VLivePlaylistIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        playlist_id = self._match_id(url)
-        video_id_match = re.match(self._VALID_URL, url)
-        assert video_id_match
-        video_id = compat_str(video_id_match.group('video_id'))
+        mobj = re.match(self._VALID_URL, url)
+        video_id, playlist_id = mobj.group('video_id', 'id')
 
         VIDEO_URL_TEMPLATE = 'http://www.vlive.tv/video/%s'
         if self._downloader.params.get('noplaylist'):
@@ -294,26 +292,27 @@ class VLivePlaylistIE(InfoExtractor):
                 ie=VLiveIE.ie_key(), video_id=video_id)
 
         self.to_screen(
-            'Downloading playlist %s - add --no-playlist to just download video' % playlist_id)
+            'Downloading playlist %s - add --no-playlist to just download video'
+            % playlist_id)
 
         webpage = self._download_webpage(
-            'http://www.vlive.tv/video/%s/playlist/%s' % (video_id, playlist_id), video_id)
+            'http://www.vlive.tv/video/%s/playlist/%s'
+            % (video_id, playlist_id), playlist_id)
+
+        item_ids = self._parse_json(
+            self._search_regex(
+                r'playlistVideoSeqs\s*=\s*(\[[^]]+\])', webpage,
+                'playlist video seqs'),
+            playlist_id)
+
+        entries = [
+            self.url_result(
+                VIDEO_URL_TEMPLATE % item_id, ie=VLiveIE.ie_key(),
+                video_id=compat_str(item_id))
+            for item_id in item_ids]
 
         playlist_name = self._html_search_regex(
             r'<div[^>]+class="[^"]*multicam_playlist[^>]*>\s*<h3[^>]+>([^<]+)',
-            webpage, 'playlist name', fatal=False)
+            webpage, 'playlist title', fatal=False)
 
-        item_ids = self._search_regex(
-            r'\bvar\s+playlistVideoSeqs\s*=\s*(\[[^]]+\])',
-            webpage, 'playlist item ids')
-
-        entries = []
-        for item_id in self._parse_json(item_ids, playlist_id):
-            item_id = compat_str(item_id)
-            entries.append(
-                self.url_result(
-                    VIDEO_URL_TEMPLATE % item_id,
-                    ie=VLiveIE.ie_key(), video_id=item_id))
-
-        return self.playlist_result(
-            entries, playlist_id, playlist_name)
+        return self.playlist_result(entries, playlist_id, playlist_name)

From c3c94ca4a40504147fce387ffb7eb9cb43233550 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 10 Jul 2017 21:34:27 +0800
Subject: [PATCH 539/611] [giantbomb] Extract m3u8 formats (closes #13626)

---
 ChangeLog                         |  6 ++++++
 youtube_dl/extractor/giantbomb.py | 14 ++++++++++----
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index c379cae71..a37d621a9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [giantbomb] Extract m3u8 formats (#13626)
+
+
 version 2017.07.09
 
 Core
diff --git a/youtube_dl/extractor/giantbomb.py b/youtube_dl/extractor/giantbomb.py
index 29b684d35..6a1b1e96e 100644
--- a/youtube_dl/extractor/giantbomb.py
+++ b/youtube_dl/extractor/giantbomb.py
@@ -5,9 +5,10 @@ import json
 
 from .common import InfoExtractor
 from ..utils import (
-    unescapeHTML,
-    qualities,
+    determine_ext,
     int_or_none,
+    qualities,
+    unescapeHTML,
 )
 
 
@@ -15,7 +16,7 @@ class GiantBombIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?giantbomb\.com/videos/(?P<display_id>[^/]+)/(?P<id>\d+-\d+)'
     _TEST = {
         'url': 'http://www.giantbomb.com/videos/quick-look-destiny-the-dark-below/2300-9782/',
-        'md5': '57badeface303ecf6b98b812de1b9018',
+        'md5': 'c8ea694254a59246a42831155dec57ac',
         'info_dict': {
             'id': '2300-9782',
             'display_id': 'quick-look-destiny-the-dark-below',
@@ -51,11 +52,16 @@ class GiantBombIE(InfoExtractor):
         for format_id, video_url in video['videoStreams'].items():
             if format_id == 'f4m_stream':
                 continue
-            if video_url.endswith('.f4m'):
+            ext = determine_ext(video_url)
+            if ext == 'f4m':
                 f4m_formats = self._extract_f4m_formats(video_url + '?hdcore=3.3.1', display_id)
                 if f4m_formats:
                     f4m_formats[0]['quality'] = quality(format_id)
                     formats.extend(f4m_formats)
+            elif ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, display_id, ext='mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False))
             else:
                 formats.append({
                     'url': video_url,

From bb13949197458fc6bd888bbe9255c391927a997b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 7 Jun 2017 14:47:25 +0800
Subject: [PATCH 540/611] [niconico] Check login errors (#12486)

---
 youtube_dl/extractor/niconico.py | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index f268a72d5..695e32e59 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -1,23 +1,22 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
 import json
 import datetime
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_parse_qs,
     compat_urlparse,
 )
 from ..utils import (
+    determine_ext,
     ExtractorError,
     int_or_none,
     parse_duration,
     parse_iso8601,
-    sanitized_Request,
-    xpath_text,
-    determine_ext,
     urlencode_postdata,
+    xpath_text,
 )
 
 
@@ -101,19 +100,24 @@ class NiconicoIE(InfoExtractor):
             return True
 
         # Log in
+        login_ok = True
         login_form_strs = {
-            'mail': username,
+            'mail_tel': username,
             'password': password,
         }
-        login_data = urlencode_postdata(login_form_strs)
-        request = sanitized_Request(
-            'https://secure.nicovideo.jp/secure/login', login_data)
-        login_results = self._download_webpage(
-            request, None, note='Logging in', errnote='Unable to log in')
-        if re.search(r'(?i)<h1 class="mb8p4">Log in error</h1>', login_results) is not None:
+        urlh = self._request_webpage(
+            'https://account.nicovideo.jp/api/v1/login', None,
+            note='Logging in', errnote='Unable to log in',
+            data=urlencode_postdata(login_form_strs))
+        if urlh is False:
+            login_ok = False
+        else:
+            parts = compat_urlparse.urlparse(urlh.geturl())
+            if compat_parse_qs(parts.query).get('message', [None])[0] == 'cant_login':
+                login_ok = False
+        if not login_ok:
             self._downloader.report_warning('unable to log in: bad username or password')
-            return False
-        return True
+        return login_ok
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 708f6f511e1d03363c9dbac7ed42cbdebfbc7718 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 11 Jul 2017 15:04:45 +0800
Subject: [PATCH 541/611] [niconico] Fix authentication error handling (closes
 #12486)

---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index a37d621a9..db1776dce 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [niconico] Fix authentication error handling (#12486)
 * [giantbomb] Extract m3u8 formats (#13626)
 
 

From 2edfd745df4b7a764d7456e7536e47ef140de24d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 4 Jun 2017 00:41:55 +0800
Subject: [PATCH 542/611] [twitter] Extract mp4 urls via mobile API (closes
 #12726)

---
 youtube_dl/extractor/twitter.py | 123 +++++++++++++++++++++++---------
 1 file changed, 89 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 37e3bc412..2041b4199 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -7,11 +7,13 @@ from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
     determine_ext,
-    float_or_none,
-    xpath_text,
-    remove_end,
-    int_or_none,
+    dict_get,
     ExtractorError,
+    float_or_none,
+    int_or_none,
+    remove_end,
+    try_get,
+    xpath_text,
 )
 
 from .periscope import PeriscopeIE
@@ -22,6 +24,15 @@ class TwitterBaseIE(InfoExtractor):
         vmap_data = self._download_xml(vmap_url, video_id)
         return xpath_text(vmap_data, './/MediaFile').strip()
 
+    @staticmethod
+    def _search_dimensions_in_video_url(a_format, video_url):
+        m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
+        if m:
+            a_format.update({
+                'width': int(m.group('width')),
+                'height': int(m.group('height')),
+            })
+
 
 class TwitterCardIE(TwitterBaseIE):
     IE_NAME = 'twitter:card'
@@ -90,6 +101,59 @@ class TwitterCardIE(TwitterBaseIE):
         },
     ]
 
+    def _parse_media_info(self, media_info, video_id):
+        formats = []
+        for media_variant in media_info.get('variants', []):
+            media_url = media_variant['url']
+            if media_url.endswith('.m3u8'):
+                formats.extend(self._extract_m3u8_formats(media_url, video_id, ext='mp4', m3u8_id='hls'))
+            elif media_url.endswith('.mpd'):
+                formats.extend(self._extract_mpd_formats(media_url, video_id, mpd_id='dash'))
+            else:
+                vbr = int_or_none(dict_get(media_variant, ('bitRate', 'bitrate')), scale=1000)
+                a_format = {
+                    'url': media_url,
+                    'format_id': 'http-%d' % vbr if vbr else 'http',
+                    'vbr': vbr,
+                }
+                # Reported bitRate may be zero
+                if not a_format['vbr']:
+                    del a_format['vbr']
+
+                self._search_dimensions_in_video_url(a_format, media_url)
+
+                formats.append(a_format)
+        return formats
+
+    def _extract_mobile_formats(self, username, video_id):
+        webpage = self._download_webpage(
+            'https://mobile.twitter.com/%s/status/%s' % (username, video_id),
+            video_id, 'Downloading mobile webpage',
+            headers={
+                # A recent mobile UA is necessary for `gt` cookie
+                'User-Agent': 'Mozilla/5.0 (Android 6.0.1; Mobile; rv:54.0) Gecko/54.0 Firefox/54.0',
+            })
+        main_script_url = self._html_search_regex(
+            r'<script[^>]+src="([^"]+main\.[^"]+)"', webpage, 'main script URL')
+        main_script = self._download_webpage(
+            main_script_url, video_id, 'Downloading main script')
+        bearer_token = self._search_regex(
+            r'BEARER_TOKEN\s*:\s*"([^"]+)"',
+            main_script, 'bearer token')
+        guest_token = self._search_regex(
+            r'document\.cookie\s*=\s*decodeURIComponent\("gt=(\d+)',
+            webpage, 'guest token')
+        api_data = self._download_json(
+            'https://api.twitter.com/2/timeline/conversation/%s.json' % video_id,
+            video_id, 'Downloading mobile API data',
+            headers={
+                'Authorization': 'Bearer ' + bearer_token,
+                'x-guest-token': guest_token,
+            })
+        media_info = try_get(api_data, lambda o: o['globalObjects']['tweets'][video_id]
+                                                  ['extended_entities']['media'][0]['video_info']) or {}
+        return self._parse_media_info(media_info, video_id)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -117,14 +181,6 @@ class TwitterCardIE(TwitterBaseIE):
         if periscope_url:
             return self.url_result(periscope_url, PeriscopeIE.ie_key())
 
-        def _search_dimensions_in_video_url(a_format, video_url):
-            m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
-            if m:
-                a_format.update({
-                    'width': int(m.group('width')),
-                    'height': int(m.group('height')),
-                })
-
         video_url = config.get('video_url') or config.get('playlist', [{}])[0].get('source')
 
         if video_url:
@@ -135,7 +191,7 @@ class TwitterCardIE(TwitterBaseIE):
                     'url': video_url,
                 }
 
-                _search_dimensions_in_video_url(f, video_url)
+                self._search_dimensions_in_video_url(f, video_url)
 
                 formats.append(f)
 
@@ -152,29 +208,14 @@ class TwitterCardIE(TwitterBaseIE):
                 media_info = entity['mediaInfo']
 
         if media_info:
-            for media_variant in media_info['variants']:
-                media_url = media_variant['url']
-                if media_url.endswith('.m3u8'):
-                    formats.extend(self._extract_m3u8_formats(media_url, video_id, ext='mp4', m3u8_id='hls'))
-                elif media_url.endswith('.mpd'):
-                    formats.extend(self._extract_mpd_formats(media_url, video_id, mpd_id='dash'))
-                else:
-                    vbr = int_or_none(media_variant.get('bitRate'), scale=1000)
-                    a_format = {
-                        'url': media_url,
-                        'format_id': 'http-%d' % vbr if vbr else 'http',
-                        'vbr': vbr,
-                    }
-                    # Reported bitRate may be zero
-                    if not a_format['vbr']:
-                        del a_format['vbr']
-
-                    _search_dimensions_in_video_url(a_format, media_url)
-
-                    formats.append(a_format)
-
+            formats.extend(self._parse_media_info(media_info, video_id))
             duration = float_or_none(media_info.get('duration', {}).get('nanos'), scale=1e9)
 
+        username = config.get('user', {}).get('screen_name')
+        if username:
+            formats.extend(self._extract_mobile_formats(username, video_id))
+
+        self._remove_duplicate_formats(formats)
         self._sort_formats(formats)
 
         title = self._search_regex(r'<title>([^<]+)</title>', webpage, 'title')
@@ -301,6 +342,20 @@ class TwitterIE(InfoExtractor):
             'timestamp': 1474613214,
         },
         'add_ie': ['Periscope'],
+    }, {
+        # has mp4 formats via mobile API
+        'url': 'https://twitter.com/news_al3alm/status/852138619213144067',
+        'info_dict': {
+            'id': '852138619213144067',
+            'ext': 'mp4',
+            'title': 'عالم الأخبار - كلمة تاريخية بجلسة الجناسي التاريخية.. النائب خالد مؤنس العتيبي للمعارضين : اتقوا الله .. الظلم ظلمات يوم القيامة',
+            'description': 'عالم الأخبار on Twitter: "كلمة تاريخية بجلسة الجناسي التاريخية.. النائب خالد مؤنس العتيبي للمعارضين : اتقوا الله .. الظلم ظلمات يوم القيامة   https://t.co/xg6OhpyKfN"',
+            'uploader': 'عالم الأخبار',
+            'uploader_id': 'news_al3alm',
+        },
+        'params': {
+            'format': 'best[format_id^=http-]',
+        },
     }]
 
     def _real_extract(self, url):

From 7f176ac4775aed5e5a72d2f0c4b579d1b886d419 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 11 Jul 2017 15:35:19 +0800
Subject: [PATCH 543/611] [periscope] Support pscp.tv URLs in embedded frames

And fix a relevant twitter test
---
 ChangeLog                         | 1 +
 youtube_dl/extractor/periscope.py | 2 +-
 youtube_dl/extractor/twitter.py   | 5 +++--
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index db1776dce..ffb4b6994 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [periscope] Support pscp.tv URLs in embedded frames
 * [niconico] Fix authentication error handling (#12486)
 * [giantbomb] Extract m3u8 formats (#13626)
 
diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 1add6b840..bfa12edc9 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -49,7 +49,7 @@ class PeriscopeIE(PeriscopeBaseIE):
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<iframe[^>]+src=([\'"])(?P<url>(?:https?:)?//(?:www\.)?periscope\.tv/(?:(?!\1).)+)\1', webpage)
+            r'<iframe[^>]+src=([\'"])(?P<url>(?:https?:)?//(?:www\.)?(?:periscope|pscp)\.tv/(?:(?!\1).)+)\1', webpage)
         if mobj:
             return mobj.group('url')
 
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 2041b4199..e4bc7e012 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -335,10 +335,11 @@ class TwitterIE(InfoExtractor):
         'info_dict': {
             'id': '1zqKVVlkqLaKB',
             'ext': 'mp4',
-            'title': 'Sgt Kerry Schmidt - Ontario Provincial Police - Road rage, mischief, assault, rollover and fire in one occurrence',
+            'title': 'Sgt Kerry Schmidt - LIVE on #Periscope: Road rage, mischief, assault, rollover and fire in one occurrence',
+            'description': 'Sgt Kerry Schmidt on Twitter: "LIVE on #Periscope: Road rage, mischief, assault, rollover and fire in one occurrence  https://t.co/EKrVgIXF3s"',
             'upload_date': '20160923',
             'uploader_id': 'OPP_HSD',
-            'uploader': 'Sgt Kerry Schmidt - Ontario Provincial Police',
+            'uploader': 'Sgt Kerry Schmidt',
             'timestamp': 1474613214,
         },
         'add_ie': ['Periscope'],

From 9be31e771cd9481ea690c01eed398645deadc1de Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 11 Jul 2017 15:48:34 +0800
Subject: [PATCH 544/611] [twitter] Support HLS streams in vmap URLs

---
 ChangeLog                       |  1 +
 youtube_dl/extractor/twitter.py | 23 ++++++++++++++---------
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index ffb4b6994..8e6451144 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [twitter] Support HLS streams in vmap URLs
 + [periscope] Support pscp.tv URLs in embedded frames
 * [niconico] Fix authentication error handling (#12486)
 * [giantbomb] Extract m3u8 formats (#13626)
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index e4bc7e012..89eabe77e 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -20,9 +20,16 @@ from .periscope import PeriscopeIE
 
 
 class TwitterBaseIE(InfoExtractor):
-    def _get_vmap_video_url(self, vmap_url, video_id):
+    def _extract_formats_from_vmap_url(self, vmap_url, video_id):
         vmap_data = self._download_xml(vmap_url, video_id)
-        return xpath_text(vmap_data, './/MediaFile').strip()
+        video_url = xpath_text(vmap_data, './/MediaFile').strip()
+        if determine_ext(video_url) == 'm3u8':
+            return self._extract_m3u8_formats(
+                video_url, video_id, ext='mp4', m3u8_id='hls',
+                entry_protocol='m3u8_native')
+        return [{
+            'url': video_url,
+        }]
 
     @staticmethod
     def _search_dimensions_in_video_url(a_format, video_url):
@@ -197,9 +204,8 @@ class TwitterCardIE(TwitterBaseIE):
 
         vmap_url = config.get('vmapUrl') or config.get('vmap_url')
         if vmap_url:
-            formats.append({
-                'url': self._get_vmap_video_url(vmap_url, video_id),
-            })
+            formats.extend(
+                self._extract_formats_from_vmap_url(vmap_url, video_id))
 
         media_info = None
 
@@ -449,7 +455,7 @@ class TwitterAmplifyIE(TwitterBaseIE):
 
         vmap_url = self._html_search_meta(
             'twitter:amplify:vmap', webpage, 'vmap url')
-        video_url = self._get_vmap_video_url(vmap_url, video_id)
+        formats = self._extract_formats_from_vmap_url(vmap_url, video_id)
 
         thumbnails = []
         thumbnail = self._html_search_meta(
@@ -471,11 +477,10 @@ class TwitterAmplifyIE(TwitterBaseIE):
             })
 
         video_w, video_h = _find_dimension('player')
-        formats = [{
-            'url': video_url,
+        formats[0].update({
             'width': video_w,
             'height': video_h,
-        }]
+        })
 
         return {
             'id': video_id,

From e8f20ffa032a791548a66bb7b694c424673537e6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 11 Jul 2017 16:05:15 +0800
Subject: [PATCH 545/611] [vine] Make sure the title won't be empty

And fix a relevant TwitterCard test case
---
 ChangeLog                       | 1 +
 youtube_dl/extractor/twitter.py | 1 +
 youtube_dl/extractor/vine.py    | 6 ++++--
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 8e6451144..a5de3c232 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [vine] Make sure the title won't be empty
 + [twitter] Support HLS streams in vmap URLs
 + [periscope] Support pscp.tv URLs in embedded frames
 * [niconico] Fix authentication error handling (#12486)
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 89eabe77e..2ff55412b 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -91,6 +91,7 @@ class TwitterCardIE(TwitterBaseIE):
                 'uploader_id': '1189339351084113920',
                 'uploader': 'ArsenalTerje',
                 'title': 'Vine by ArsenalTerje',
+                'timestamp': 1447451307,
             },
             'add_ie': ['Vine'],
         }, {
diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 4957a07f7..46950d3a1 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -92,10 +92,12 @@ class VineIE(InfoExtractor):
 
         username = data.get('username')
 
+        alt_title = 'Vine by %s' % username if username else None
+
         return {
             'id': video_id,
-            'title': data.get('description'),
-            'alt_title': 'Vine by %s' % username if username else None,
+            'title': data.get('description') or alt_title or 'Vine video',
+            'alt_title': alt_title,
             'thumbnail': data.get('thumbnailUrl'),
             'timestamp': unified_timestamp(data.get('created')),
             'uploader': username,

From 3615bfe1b4b97cb35ddd63c455160a50c2a10961 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 11 Jul 2017 16:46:37 +0800
Subject: [PATCH 546/611] [twitter] Fix remaining tests

---
 youtube_dl/extractor/twitter.py | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 2ff55412b..6eaf360a6 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -54,7 +54,8 @@ class TwitterCardIE(TwitterBaseIE):
                 'title': 'Twitter Card',
                 'thumbnail': r're:^https?://.*\.jpg$',
                 'duration': 30.033,
-            }
+            },
+            'skip': 'Video gone',
         },
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/623160978427936768',
@@ -66,6 +67,7 @@ class TwitterCardIE(TwitterBaseIE):
                 'thumbnail': r're:^https?://.*\.jpg',
                 'duration': 80.155,
             },
+            'skip': 'Video gone',
         },
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/654001591733886977',
@@ -83,7 +85,7 @@ class TwitterCardIE(TwitterBaseIE):
         },
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/665289828897005568',
-            'md5': 'ab2745d0b0ce53319a534fccaa986439',
+            'md5': '6dabeaca9e68cbb71c99c322a4b42a11',
             'info_dict': {
                 'id': 'iBb2x00UVlv',
                 'ext': 'mp4',
@@ -96,12 +98,12 @@ class TwitterCardIE(TwitterBaseIE):
             'add_ie': ['Vine'],
         }, {
             'url': 'https://twitter.com/i/videos/tweet/705235433198714880',
-            'md5': '3846d0a07109b5ab622425449b59049d',
+            'md5': '884812a2adc8aaf6fe52b15ccbfa3b88',
             'info_dict': {
                 'id': '705235433198714880',
                 'ext': 'mp4',
                 'title': 'Twitter web player',
-                'thumbnail': r're:^https?://.*\.jpg',
+                'thumbnail': r're:^https?://.*',
             },
         }, {
             'url': 'https://twitter.com/i/videos/752274308186120192',
@@ -303,10 +305,10 @@ class TwitterIE(InfoExtractor):
         'info_dict': {
             'id': '700207533655363584',
             'ext': 'mp4',
-            'title': 'JG - BEAT PROD: @suhmeduh #Damndaniel',
-            'description': 'JG on Twitter: "BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
+            'title': 'Donte - BEAT PROD: @suhmeduh #Damndaniel',
+            'description': 'Donte on Twitter: "BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
             'thumbnail': r're:^https?://.*\.jpg',
-            'uploader': 'JG',
+            'uploader': 'Donte',
             'uploader_id': 'jaydingeer',
         },
         'params': {
@@ -318,9 +320,11 @@ class TwitterIE(InfoExtractor):
         'info_dict': {
             'id': 'MIOxnrUteUd',
             'ext': 'mp4',
-            'title': 'Dr.Pepperの飲み方 #japanese #バカ #ドクペ #電動ガン',
-            'uploader': 'TAKUMA',
-            'uploader_id': '1004126642786242560',
+            'title': 'FilmDrunk - Vine of the day',
+            'description': 'FilmDrunk on Twitter: "Vine of the day https://t.co/xmTvRdqxWf"',
+            'uploader': 'FilmDrunk',
+            'uploader_id': 'Filmdrunk',
+            'timestamp': 1402826626,
             'upload_date': '20140615',
         },
         'add_ie': ['Vine'],

From f2bb33a9868e499f4582fee24a7b67d559d33575 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Jul 2017 21:36:45 +0700
Subject: [PATCH 547/611] [ted] Fix subtitles extraction (closes #13628, closes
 #13629)

---
 youtube_dl/extractor/ted.py | 30 ++++++++++++++++--------------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index f27d0e313..06a27fd04 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -271,20 +271,22 @@ class TEDIE(InfoExtractor):
         }
 
     def _get_subtitles(self, video_id, talk_info):
-        languages = [lang['languageCode'] for lang in talk_info.get('languages', [])]
-        if languages:
-            sub_lang_list = {}
-            for l in languages:
-                sub_lang_list[l] = [
-                    {
-                        'url': 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/%s' % (video_id, l, ext),
-                        'ext': ext,
-                    }
-                    for ext in ['ted', 'srt']
-                ]
-            return sub_lang_list
-        else:
-            return {}
+        sub_lang_list = {}
+        for language in try_get(
+                talk_info,
+                (lambda x: x['downloads']['languages'],
+                 lambda x: x['languages']), list):
+            lang_code = language.get('languageCode') or language.get('ianaCode')
+            if not lang_code:
+                continue
+            sub_lang_list[lang_code] = [
+                {
+                    'url': 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/%s' % (video_id, lang_code, ext),
+                    'ext': ext,
+                }
+                for ext in ['ted', 'srt']
+            ]
+        return sub_lang_list
 
     def _watch_info(self, url, name):
         webpage = self._download_webpage(url, name)

From 9a0942ad55bba714d6eaeb9ee4f66a138ec85e17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Jul 2017 22:59:36 +0700
Subject: [PATCH 548/611] [drtv] Make HLS and HDS extraction non fatal

---
 youtube_dl/extractor/drtv.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index c84624f1e..69effba58 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -118,7 +118,7 @@ class DRTVIE(InfoExtractor):
                     if target == 'HDS':
                         f4m_formats = self._extract_f4m_formats(
                             uri + '?hdcore=3.3.0&plugin=aasp-3.3.0.99.43',
-                            video_id, preference, f4m_id=format_id)
+                            video_id, preference, f4m_id=format_id, fatal=False)
                         if kind == 'AudioResource':
                             for f in f4m_formats:
                                 f['vcodec'] = 'none'
@@ -126,7 +126,8 @@ class DRTVIE(InfoExtractor):
                     elif target == 'HLS':
                         formats.extend(self._extract_m3u8_formats(
                             uri, video_id, 'mp4', entry_protocol='m3u8_native',
-                            preference=preference, m3u8_id=format_id))
+                            preference=preference, m3u8_id=format_id,
+                            fatal=False))
                     else:
                         bitrate = link.get('Bitrate')
                         if bitrate:

From 15da37c7dc8cf14ba5ce880aa1805fceaa71fc44 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 14 Jul 2017 00:40:54 +0700
Subject: [PATCH 549/611] [YoutubeDL] Don't expand env variables in meta fields
 (closes #13637)

---
 test/test_YoutubeDL.py  |  6 ++++++
 youtube_dl/YoutubeDL.py | 15 ++++++++++++++-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 75945e38f..70989e232 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -527,6 +527,8 @@ class TestYoutubeDL(unittest.TestCase):
             'ext': 'mp4',
             'width': None,
             'height': 1080,
+            'title1': '$PATH',
+            'title2': '%PATH%',
         }
 
         def fname(templ):
@@ -545,10 +547,14 @@ class TestYoutubeDL(unittest.TestCase):
         self.assertEqual(fname('%(height)0 6d.%(ext)s'), ' 01080.mp4')
         self.assertEqual(fname('%(height)0   6d.%(ext)s'), ' 01080.mp4')
         self.assertEqual(fname('%(height)   0   6d.%(ext)s'), ' 01080.mp4')
+        self.assertEqual(fname('%%'), '%')
+        self.assertEqual(fname('%%%%'), '%%')
         self.assertEqual(fname('%%(height)06d.%(ext)s'), '%(height)06d.mp4')
         self.assertEqual(fname('%(width)06d.%(ext)s'), 'NA.mp4')
         self.assertEqual(fname('%(width)06d.%%(ext)s'), 'NA.%(ext)s')
         self.assertEqual(fname('%%(width)06d.%(ext)s'), '%(width)06d.mp4')
+        self.assertEqual(fname('Hello %(title1)s'), 'Hello $PATH')
+        self.assertEqual(fname('Hello %(title2)s'), 'Hello %PATH%')
 
     def test_format_note(self):
         ydl = YoutubeDL()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 60ee4b7d8..8730d32ef 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -20,6 +20,7 @@ import re
 import shutil
 import subprocess
 import socket
+import string
 import sys
 import time
 import tokenize
@@ -674,7 +675,19 @@ class YoutubeDL(object):
                         FORMAT_RE.format(numeric_field),
                         r'%({0})s'.format(numeric_field), outtmpl)
 
-            filename = expand_path(outtmpl % template_dict)
+            # expand_path translates '%%' into '%' and '$$' into '$'
+            # correspondingly that is not what we want since we need to keep
+            # '%%' intact for template dict substitution step. Working around
+            # with boundary-alike separator hack.
+            sep = ''.join([random.choice(string.ascii_letters) for _ in range(32)])
+            outtmpl = outtmpl.replace('%%', '%{0}%'.format(sep)).replace('$$', '${0}$'.format(sep))
+
+            # outtmpl should be expand_path'ed before template dict substitution
+            # because meta fields may contain env variables we don't want to
+            # be expanded. For example, for outtmpl "%(title)s.%(ext)s" and
+            # title "Hello $PATH", we don't want `$PATH` to be expanded.
+            filename = expand_path(outtmpl).replace(sep, '') % template_dict
+
             # Temporary fix for #4787
             # 'Treat' all problem characters by passing filename through preferredencoding
             # to workaround encoding issues with subprocess on python2 @ Windows

From f354d8480700c5e6f288bfce497a363b4c6f0859 Mon Sep 17 00:00:00 2001
From: rrooij <rrooij@users.noreply.github.com>
Date: Fri, 14 Jul 2017 17:10:17 +0200
Subject: [PATCH 550/611] [5tv] Add another video URL pattern (closes #13354)

---
 youtube_dl/extractor/fivetv.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/fivetv.py b/youtube_dl/extractor/fivetv.py
index 15736c9fe..9f9863746 100644
--- a/youtube_dl/extractor/fivetv.py
+++ b/youtube_dl/extractor/fivetv.py
@@ -43,7 +43,7 @@ class FiveTVIE(InfoExtractor):
         'info_dict': {
             'id': 'glavnoe',
             'ext': 'mp4',
-            'title': 'Итоги недели с 8 по 14 июня 2015 года',
+            'title': r're:^Итоги недели с \d+ по \d+ \w+ \d{4} года$',
             'thumbnail': r're:^https?://.*\.jpg$',
         },
     }, {
@@ -70,7 +70,8 @@ class FiveTVIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         video_url = self._search_regex(
-            r'<a[^>]+?href="([^"]+)"[^>]+?class="videoplayer"',
+            [r'<div[^>]+?class="flowplayer[^>]+?data-href="([^"]+)"',
+             r'<a[^>]+?href="([^"]+)"[^>]+?class="videoplayer"'],
             webpage, 'video url')
 
         title = self._og_search_title(webpage, default=None) or self._search_regex(

From 00dbdfc1f741b919a0add36394065ce1aeccfda8 Mon Sep 17 00:00:00 2001
From: satunnainen <satunnainen@users.noreply.github.com>
Date: Fri, 14 Jul 2017 18:11:07 +0300
Subject: [PATCH 551/611] [slideshare] Fix extraction

---
 youtube_dl/extractor/slideshare.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/slideshare.py b/youtube_dl/extractor/slideshare.py
index 74a1dc672..e89ebebe7 100644
--- a/youtube_dl/extractor/slideshare.py
+++ b/youtube_dl/extractor/slideshare.py
@@ -31,7 +31,7 @@ class SlideshareIE(InfoExtractor):
         page_title = mobj.group('title')
         webpage = self._download_webpage(url, page_title)
         slideshare_obj = self._search_regex(
-            r'\$\.extend\(slideshare_object,\s*(\{.*?\})\);',
+            r'\$\.extend\(.*?slideshare_object,\s*(\{.*?\})\);',
             webpage, 'slideshare object')
         info = json.loads(slideshare_obj)
         if info['slideshow']['type'] != 'video':

From 7d02dcfaa2589453ee3cc6c88ee27f04c252f8a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 14 Jul 2017 22:37:04 +0700
Subject: [PATCH 552/611] [youtube] Don't capture YouTube Red ad for creator
 meta field (closes #13621)

---
 youtube_dl/extractor/youtube.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 77cd271ef..4597ccb3a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -673,6 +673,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             },
         },
         # video_info is None (https://github.com/rg3/youtube-dl/issues/4421)
+        # YouTube Red ad is not captured for creator
         {
             'url': '__2ABJjxzNo',
             'info_dict': {
@@ -1649,7 +1650,21 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             video_webpage, 'license', default=None)
 
         m_music = re.search(
-            r'<h4[^>]+class="title"[^>]*>\s*Music\s*</h4>\s*<ul[^>]*>\s*<li>(?P<title>.+?) by (?P<creator>.+?)(?:\(.+?\))?</li',
+            r'''(?x)
+                <h4[^>]+class="title"[^>]*>\s*Music\s*</h4>\s*
+                <ul[^>]*>\s*
+                <li>(?P<title>.+?)
+                by (?P<creator>.+?)
+                (?:
+                    \(.+?\)|
+                    <a[^>]*
+                        (?:
+                            \bhref=["\']/red[^>]*>|             # drop possible
+                            >\s*Listen ad-free with YouTube Red # YouTube Red ad 
+                        )
+                    .*?
+                )?</li
+            ''',
             video_webpage)
         if m_music:
             video_alt_title = remove_quotes(unescapeHTML(m_music.group('title')))

From 2583c0b54e56f6dbce85a079d91a05e9b13c2dce Mon Sep 17 00:00:00 2001
From: Robin Neatherway <robin.neatherway@gmail.com>
Date: Fri, 14 Jul 2017 17:08:32 +0100
Subject: [PATCH 553/611] Fix bugs caused by typos

---
 youtube_dl/downloader/ism.py           | 3 +--
 youtube_dl/extractor/audioboom.py      | 2 +-
 youtube_dl/extractor/generic.py        | 2 +-
 youtube_dl/extractor/karrierevideos.py | 2 +-
 4 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/ism.py b/youtube_dl/downloader/ism.py
index 5f6f9faef..9b001ecff 100644
--- a/youtube_dl/downloader/ism.py
+++ b/youtube_dl/downloader/ism.py
@@ -98,7 +98,7 @@ def write_piff_header(stream, params):
 
     if is_audio:
         smhd_payload = s88.pack(0)  # balance
-        smhd_payload = u16.pack(0)  # reserved
+        smhd_payload += u16.pack(0)  # reserved
         media_header_box = full_box(b'smhd', 0, 0, smhd_payload)  # Sound Media Header
     else:
         vmhd_payload = u16.pack(0)  # graphics mode
@@ -126,7 +126,6 @@ def write_piff_header(stream, params):
         if fourcc == 'AACL':
             sample_entry_box = box(b'mp4a', sample_entry_payload)
     else:
-        sample_entry_payload = sample_entry_payload
         sample_entry_payload += u16.pack(0)  # pre defined
         sample_entry_payload += u16.pack(0)  # reserved
         sample_entry_payload += u32.pack(0) * 3  # pre defined
diff --git a/youtube_dl/extractor/audioboom.py b/youtube_dl/extractor/audioboom.py
index e48bb8972..393f381c6 100644
--- a/youtube_dl/extractor/audioboom.py
+++ b/youtube_dl/extractor/audioboom.py
@@ -43,7 +43,7 @@ class AudioBoomIE(InfoExtractor):
 
         def from_clip(field):
             if clip:
-                clip.get(field)
+                return clip.get(field)
 
         audio_url = from_clip('clipURLPriorToLoading') or self._og_search_property(
             'audio', webpage, 'audio url')
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 5e8890d41..8c2ff39d5 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2740,7 +2740,7 @@ class GenericIE(InfoExtractor):
         rutube_urls = RutubeIE._extract_urls(webpage)
         if rutube_urls:
             return self.playlist_from_matches(
-                rutube_urls, ie=RutubeIE.ie_key())
+                rutube_urls, video_id, video_title, ie=RutubeIE.ie_key())
 
         # Look for WashingtonPost embeds
         wapo_urls = WashingtonPostIE._extract_urls(webpage)
diff --git a/youtube_dl/extractor/karrierevideos.py b/youtube_dl/extractor/karrierevideos.py
index 4e9eb67bf..f236a2f78 100644
--- a/youtube_dl/extractor/karrierevideos.py
+++ b/youtube_dl/extractor/karrierevideos.py
@@ -48,7 +48,7 @@ class KarriereVideosIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         title = (self._html_search_meta('title', webpage, default=None) or
-                 self._search_regex(r'<h1 class="title">([^<]+)</h1>'))
+                 self._search_regex(r'<h1 class="title">([^<]+)</h1>', webpage, 'video title'))
 
         video_id = self._search_regex(
             r'/config/video/(.+?)\.xml', webpage, 'video id')

From 4e826cd9aec383768a7b25aa3161efd4672f9310 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Jul 2017 06:48:12 +0700
Subject: [PATCH 554/611] [nexx] Add extractor (closes #10807, closes #13465)

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/generic.py    |  22 +++
 youtube_dl/extractor/nexx.py       | 221 +++++++++++++++++++++++++++++
 3 files changed, 244 insertions(+)
 create mode 100644 youtube_dl/extractor/nexx.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index eb1541729..9d34447a9 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -653,6 +653,7 @@ from .nextmedia import (
     AppleDailyIE,
     NextTVIE,
 )
+from .nexx import NexxIE
 from .nfb import NFBIE
 from .nfl import NFLIE
 from .nhk import NhkVodIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8c2ff39d5..123a21296 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -36,6 +36,7 @@ from .brightcove import (
     BrightcoveLegacyIE,
     BrightcoveNewIE,
 )
+from .nexx import NexxIE
 from .nbc import NBCSportsVPlayerIE
 from .ooyala import OoyalaIE
 from .rutv import RUTVIE
@@ -1549,6 +1550,22 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['BrightcoveLegacy'],
         },
+        # Nexx embed
+        {
+            'url': 'https://www.funk.net/serien/5940e15073f6120001657956/items/593efbb173f6120001657503',
+            'info_dict': {
+                'id': '247746',
+                'ext': 'mp4',
+                'title': "Yesterday's Jam (OV)",
+                'description': 'md5:09bc0984723fed34e2581624a84e05f0',
+                'timestamp': 1492594816,
+                'upload_date': '20170419',
+            },
+            'params': {
+                'format': 'bestvideo',
+                'skip_download': True,
+            },
+        },
         # Facebook <iframe> embed
         {
             'url': 'https://www.hostblogger.de/blog/archives/6181-Auto-jagt-Betonmischer.html',
@@ -2133,6 +2150,11 @@ class GenericIE(InfoExtractor):
         if bc_urls:
             return self.playlist_from_matches(bc_urls, video_id, video_title, ie='BrightcoveNew')
 
+        # Look for Nexx embeds
+        nexx_urls = NexxIE._extract_urls(webpage)
+        if nexx_urls:
+            return self.playlist_from_matches(nexx_urls, video_id, video_title, ie=NexxIE.ie_key())
+
         # Look for ThePlatform embeds
         tp_urls = ThePlatformIE._extract_urls(webpage)
         if tp_urls:
diff --git a/youtube_dl/extractor/nexx.py b/youtube_dl/extractor/nexx.py
new file mode 100644
index 000000000..60b42cb7d
--- /dev/null
+++ b/youtube_dl/extractor/nexx.py
@@ -0,0 +1,221 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import hashlib
+import random
+import re
+import time
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    parse_duration,
+    try_get,
+    urlencode_postdata,
+)
+
+
+class NexxIE(InfoExtractor):
+    _VALID_URL = r'https?://api\.nexx(?:\.cloud|cdn\.com)/v3/(?P<domain_id>\d+)/videos/byid/(?P<id>\d+)'
+    _TESTS = [{
+        # movie
+        'url': 'https://api.nexx.cloud/v3/748/videos/byid/128907',
+        'md5': '16746bfc28c42049492385c989b26c4a',
+        'info_dict': {
+            'id': '128907',
+            'ext': 'mp4',
+            'title': 'Stiftung Warentest',
+            'alt_title': 'Wie ein Test abläuft',
+            'description': 'md5:d1ddb1ef63de721132abd38639cc2fd2',
+            'release_year': 2013,
+            'creator': 'SPIEGEL TV',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 2509,
+            'timestamp': 1384264416,
+            'upload_date': '20131112',
+        },
+        'params': {
+            'format': 'bestvideo',
+        },
+    }, {
+        # episode
+        'url': 'https://api.nexx.cloud/v3/741/videos/byid/247858',
+        'info_dict': {
+            'id': '247858',
+            'ext': 'mp4',
+            'title': 'Return of the Golden Child (OV)',
+            'description': 'md5:5d969537509a92b733de21bae249dc63',
+            'release_year': 2017,
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 1397,
+            'timestamp': 1495033267,
+            'upload_date': '20170517',
+            'episode_number': 2,
+            'season_number': 2,
+        },
+        'params': {
+            'format': 'bestvideo',
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://api.nexxcdn.com/v3/748/videos/byid/128907',
+        'only_matching': True,
+    }]
+
+    @staticmethod
+    def _extract_urls(webpage):
+        # Reference:
+        # 1. https://nx-s.akamaized.net/files/201510/44.pdf
+
+        entries = []
+
+        # JavaScript Integration
+        for domain_id, video_id in re.findall(
+                r'''(?isx)
+                    <script\b[^>]+\bsrc=["\']https?://require\.nexx(?:\.cloud|cdn\.com)/(\d+).+?
+                    onPLAYReady.+?
+                    _play\.init\s*\(.+?\s*,\s*(\d+)\s*,\s*.+?\)
+                ''', webpage):
+            entries.append('https://api.nexx.cloud/v3/%s/videos/byid/%s' % (domain_id, video_id))
+
+        # TODO: support more embed formats
+
+        return entries
+
+    def _handle_error(self, response):
+        status = int_or_none(try_get(
+            response, lambda x: x['metadata']['status']) or 200)
+        if 200 <= status < 300:
+            return
+        raise ExtractorError(
+            '%s said: %s' % (self.IE_NAME, response['metadata']['errorhint']),
+            expected=True)
+
+    def _call_api(self, domain_id, path, video_id, data=None, headers={}):
+        headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8'
+        result = self._download_json(
+            'https://api.nexx.cloud/v3/%s/%s' % (domain_id, path), video_id,
+            'Downloading %s JSON' % path, data=urlencode_postdata(data),
+            headers=headers)
+        self._handle_error(result)
+        return result['result']
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        domain_id, video_id = mobj.group('domain_id', 'id')
+
+        # Reverse engineered from JS code (see getDeviceID function)
+        device_id = '%d:%d:%d%d' % (
+            random.randint(1, 4), int(time.time()),
+            random.randint(1e4, 99999), random.randint(1, 9))
+
+        result = self._call_api(domain_id, 'session/init', video_id, data={
+            'nxp_devh': device_id,
+            'nxp_userh': '',
+            'precid': '0',
+            'playlicense': '0',
+            'screenx': '1920',
+            'screeny': '1080',
+            'playerversion': '6.0.00',
+            'gateway': 'html5',
+            'adGateway': '',
+            'explicitlanguage': 'en-US',
+            'addTextTemplates': '1',
+            'addDomainData': '1',
+            'addAdModel': '1',
+        }, headers={
+            'X-Request-Enable-Auth-Fallback': '1',
+        })
+
+        cid = result['general']['cid']
+
+        # As described in [1] X-Request-Token generation algorithm is
+        # as follows:
+        #   md5( operation + domain_id + domain_secret )
+        # where domain_secret is a static value that will be given by nexx.tv
+        # as per [1]. Here is how this "secret" is generated (reversed
+        # from _play.api.init function, search for clienttoken). So it's
+        # actually not static and not that much of a secret.
+        # 1. https://nexxtvstorage.blob.core.windows.net/files/201610/27.pdf
+        secret = result['device']['clienttoken'][int(device_id[0]):]
+        secret = secret[0:len(secret) - int(device_id[-1])]
+
+        op = 'byid'
+
+        # Reversed from JS code for _play.api.call function (search for
+        # X-Request-Token)
+        request_token = hashlib.md5(
+            ''.join((op, domain_id, secret)).encode('utf-8')).hexdigest()
+
+        video = self._call_api(
+            domain_id, 'videos/%s/%s' % (op, video_id), video_id, data={
+                'additionalfields': 'language,channel,actors,studio,licenseby,slug,subtitle,teaser,description',
+                'addInteractionOptions': '1',
+                'addStatusDetails': '1',
+                'addStreamDetails': '1',
+                'addCaptions': '1',
+                'addScenes': '1',
+                'addHotSpots': '1',
+                'addBumpers': '1',
+                'captionFormat': 'data',
+            }, headers={
+                'X-Request-CID': cid,
+                'X-Request-Token': request_token,
+            })
+
+        general = video['general']
+        title = general['title']
+
+        stream_data = video['streamdata']
+        language = general.get('language_raw') or ''
+
+        # TODO: reverse more cdns and formats
+
+        cdn = stream_data['cdnType']
+        assert cdn == 'azure'
+
+        azure_locator = stream_data['azureLocator']
+
+        AZURE_URL = 'http://nx-p%02d.akamaized.net/'
+
+        for secure in ('s', ''):
+            cdn_shield = stream_data.get('cdnShieldHTTP%s' % secure.upper())
+            if cdn_shield:
+                azure_base = 'http%s://%s' % (secure, cdn_shield)
+                break
+        else:
+            azure_base = AZURE_URL % int(stream_data['azureAccount'].replace('nexxplayplus', ''))
+
+        is_ml = ',' in language
+        azure_m3u8_url = '%s%s/%s_src%s.ism/Manifest(format=m3u8-aapl)' % (
+            azure_base, azure_locator, video_id, ('_manifest' if is_ml else ''))
+
+        protection_token = try_get(
+            video, lambda x: x['protectiondata']['token'], compat_str)
+        if protection_token:
+            azure_m3u8_url += '?hdnts=%s' % protection_token
+
+        formats = self._extract_m3u8_formats(
+            azure_m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native',
+            m3u8_id='%s-hls' % cdn)
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'alt_title': general.get('subtitle'),
+            'description': general.get('description'),
+            'release_year': int_or_none(general.get('year')),
+            'creator': general.get('studio') or general.get('studio_adref'),
+            'thumbnail': try_get(
+                video, lambda x: x['imagedata']['thumb'], compat_str),
+            'duration': parse_duration(general.get('runtime')),
+            'timestamp': int_or_none(general.get('uploaded')),
+            'episode_number': int_or_none(try_get(
+                video, lambda x: x['episodedata']['episode'])),
+            'season_number': int_or_none(try_get(
+                video, lambda x: x['episodedata']['season'])),
+            'formats': formats,
+        }

From c7604d79e9993e76845141a61fdf3c308af917e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Jul 2017 06:52:23 +0700
Subject: [PATCH 555/611] [spiegeltv] Delegate extraction to nexx (closes
 #13159)

---
 youtube_dl/extractor/spiegeltv.py | 113 +++---------------------------
 1 file changed, 8 insertions(+), 105 deletions(-)

diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index e1cfb8698..6ccf4c342 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -1,114 +1,17 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlparse
-from ..utils import (
-    determine_ext,
-    float_or_none,
-)
+from .nexx import NexxIE
 
 
 class SpiegeltvIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?spiegel\.tv/(?:#/)?filme/(?P<id>[\-a-z0-9]+)'
-    _TESTS = [{
-        'url': 'http://www.spiegel.tv/filme/flug-mh370/',
-        'info_dict': {
-            'id': 'flug-mh370',
-            'ext': 'm4v',
-            'title': 'Flug MH370',
-            'description': 'Das Rätsel um die Boeing 777 der Malaysia-Airlines',
-            'thumbnail': r're:http://.*\.jpg$',
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        }
-    }, {
-        'url': 'http://www.spiegel.tv/#/filme/alleskino-die-wahrheit-ueber-maenner/',
+    _VALID_URL = r'https?://(?:www\.)?spiegel\.tv/videos/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.spiegel.tv/videos/161681-flug-mh370/',
         'only_matching': True,
-    }]
+    }
 
     def _real_extract(self, url):
-        if '/#/' in url:
-            url = url.replace('/#/', '/')
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        title = self._html_search_regex(r'<h1.*?>(.*?)</h1>', webpage, 'title')
-
-        apihost = 'http://spiegeltv-ivms2-restapi.s3.amazonaws.com'
-        version_json = self._download_json(
-            '%s/version.json' % apihost, video_id,
-            note='Downloading version information')
-        version_name = version_json['version_name']
-
-        slug_json = self._download_json(
-            '%s/%s/restapi/slugs/%s.json' % (apihost, version_name, video_id),
-            video_id,
-            note='Downloading object information')
-        oid = slug_json['object_id']
-
-        media_json = self._download_json(
-            '%s/%s/restapi/media/%s.json' % (apihost, version_name, oid),
-            video_id, note='Downloading media information')
-        uuid = media_json['uuid']
-        is_wide = media_json['is_wide']
-
-        server_json = self._download_json(
-            'http://spiegeltv-prod-static.s3.amazonaws.com/projectConfigs/projectConfig.json',
-            video_id, note='Downloading server information')
-
-        format = '16x9' if is_wide else '4x3'
-
-        formats = []
-        for streamingserver in server_json['streamingserver']:
-            endpoint = streamingserver.get('endpoint')
-            if not endpoint:
-                continue
-            play_path = 'mp4:%s_spiegeltv_0500_%s.m4v' % (uuid, format)
-            if endpoint.startswith('rtmp'):
-                formats.append({
-                    'url': endpoint,
-                    'format_id': 'rtmp',
-                    'app': compat_urllib_parse_urlparse(endpoint).path[1:],
-                    'play_path': play_path,
-                    'player_path': 'http://prod-static.spiegel.tv/frontend-076.swf',
-                    'ext': 'flv',
-                    'rtmp_live': True,
-                })
-            elif determine_ext(endpoint) == 'm3u8':
-                formats.append({
-                    'url': endpoint.replace('[video]', play_path),
-                    'ext': 'm4v',
-                    'format_id': 'hls',  # Prefer hls since it allows to workaround georestriction
-                    'protocol': 'm3u8',
-                    'preference': 1,
-                    'http_headers': {
-                        'Accept-Encoding': 'deflate',  # gzip causes trouble on the server side
-                    },
-                })
-            else:
-                formats.append({
-                    'url': endpoint,
-                })
-        self._check_formats(formats, video_id)
-
-        thumbnails = []
-        for image in media_json['images']:
-            thumbnails.append({
-                'url': image['url'],
-                'width': image['width'],
-                'height': image['height'],
-            })
-
-        description = media_json['subtitle']
-        duration = float_or_none(media_json.get('duration_in_ms'), scale=1000)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'duration': duration,
-            'thumbnails': thumbnails,
-            'formats': formats,
-        }
+        return self.url_result(
+            'https://api.nexx.cloud/v3/748/videos/byid/%s'
+            % self._match_id(url), ie=NexxIE.ie_key())

From ea3f20494f64c18123c61f722a7864e3dbdde566 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Jul 2017 07:02:05 +0700
Subject: [PATCH 556/611] [youtube] PEP 8

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4597ccb3a..2e71795e7 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1660,7 +1660,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     <a[^>]*
                         (?:
                             \bhref=["\']/red[^>]*>|             # drop possible
-                            >\s*Listen ad-free with YouTube Red # YouTube Red ad 
+                            >\s*Listen ad-free with YouTube Red # YouTube Red ad
                         )
                     .*?
                 )?</li

From 961ea474b6c6965d49a58d0400d0368fa0300b19 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Jul 2017 07:02:14 +0700
Subject: [PATCH 557/611] [YoutubeDL] PEP 8

---
 youtube_dl/YoutubeDL.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8730d32ef..89c07be29 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -20,13 +20,14 @@ import re
 import shutil
 import subprocess
 import socket
-import string
 import sys
 import time
 import tokenize
 import traceback
 import random
 
+from string import ascii_letters
+
 from .compat import (
     compat_basestring,
     compat_cookiejar,
@@ -679,7 +680,7 @@ class YoutubeDL(object):
             # correspondingly that is not what we want since we need to keep
             # '%%' intact for template dict substitution step. Working around
             # with boundary-alike separator hack.
-            sep = ''.join([random.choice(string.ascii_letters) for _ in range(32)])
+            sep = ''.join([random.choice(ascii_letters) for _ in range(32)])
             outtmpl = outtmpl.replace('%%', '%{0}%'.format(sep)).replace('$$', '${0}$'.format(sep))
 
             # outtmpl should be expand_path'ed before template dict substitution

From ef78563e9c148d8a24382c282d74a960e244569e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Jul 2017 07:33:26 +0700
Subject: [PATCH 558/611] [ChangeLog] Actualize

---
 ChangeLog | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index a5de3c232..5f3ea6b5f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,11 +1,25 @@
 version <unreleased>
 
+Core
+* [YoutubeDL] Don't expand environment variables in meta fields (#13637)
+
 Extractors
+* [spiegeltv] Delegate extraction to nexx extractor (#13159)
++ [nexx] Add support for nexx.cloud (#10807, #13465)
+* [generic] Fix rutube embeds extraction (#13641)
+* [karrierevideos] Fix title extraction (#13641)
+* [youtube] Don't capture YouTube Red ad for creator meta field (#13621)
+* [slideshare] Fix extraction (#13617)
++ [5tv] Add another video URL pattern (#13354, #13606)
+* [drtv] Make HLS and HDS extraction non fatal
+* [ted] Fix subtitles extraction (#13628, #13629)
 * [vine] Make sure the title won't be empty
 + [twitter] Support HLS streams in vmap URLs
 + [periscope] Support pscp.tv URLs in embedded frames
+* [twitter] Extract mp4 urls via mobile API (#12726)
 * [niconico] Fix authentication error handling (#12486)
 * [giantbomb] Extract m3u8 formats (#13626)
++ [vlive:playlist] Add support for playlists (#13613)
 
 
 version 2017.07.09

From cea931a9e5cf682eafe5b7cdacbbe22630b7a9e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Jul 2017 07:36:05 +0700
Subject: [PATCH 559/611] release 2017.07.15

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index c4314855d..0f20d0485 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.07.09*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.07.09**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.07.15*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.07.15**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.07.09
+[debug] youtube-dl version 2017.07.15
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 5f3ea6b5f..7d71fc5e1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.07.15
 
 Core
 * [YoutubeDL] Don't expand environment variables in meta fields (#13637)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index b6a147faf..d7304ba06 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -521,6 +521,7 @@
  - **NextMedia**: 蘋果日報
  - **NextMediaActionNews**: 蘋果日報 - 動新聞
  - **NextTV**: 壹電視
+ - **Nexx**
  - **nfb**: National Film Board of Canada
  - **nfl.com**
  - **NhkVod**
@@ -942,6 +943,7 @@
  - **vk:wallpost**
  - **vlive**
  - **vlive:channel**
+ - **vlive:playlist**
  - **Vodlocker**
  - **VODPl**
  - **VODPlatform**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 14358a74c..82e166fef 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.07.09'
+__version__ = '2017.07.15'

From 94b817edebb63c3d8485e1ae27cc394dd9e21f9d Mon Sep 17 00:00:00 2001
From: troywith77 <ruitang307@gmail.com>
Date: Tue, 9 May 2017 13:10:18 +0800
Subject: [PATCH 560/611] [pearvideo] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/pear.py       | 34 ++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)
 create mode 100644 youtube_dl/extractor/pear.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 9d34447a9..75c1a3d0e 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -762,6 +762,7 @@ from .pandoratv import PandoraTVIE
 from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
+from .pear import PearIE
 from .people import PeopleIE
 from .periscope import (
     PeriscopeIE,
diff --git a/youtube_dl/extractor/pear.py b/youtube_dl/extractor/pear.py
new file mode 100644
index 000000000..77fd46852
--- /dev/null
+++ b/youtube_dl/extractor/pear.py
@@ -0,0 +1,34 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class PearIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?pearvideo\.com/video_(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.pearvideo.com/video_1076290',
+        'info_dict': {
+            'id': '1076290',
+            'ext': 'mp4',
+            'title': '小浣熊在主人家玻璃上滚石头：没砸',
+            'description': '小浣熊找到一个小石头，仿佛发现了一个宝贝。它不停地用石头按在玻璃上，滚来滚去，吸引主人注意。',
+            'url': 'http://video.pearvideo.com/mp4/short/20170508/cont-1076290-10438018-hd.mp4'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r'<h1[^>]+class="video-tt">(.+)</h1>', webpage, 'title', fatal=False)
+        description = self._html_search_regex(r'<div[^>]+class="summary"[^>]*>([^<]+)<', webpage, 'description', fatal=False)
+        url = self._html_search_regex(r'hdUrl="(.*?)"', webpage, 'url', fatal=False)
+
+        return {
+            'id': video_id,
+            'ext': 'mp4',
+            'title': title,
+            'description': description,
+            'url': url
+        }

From decf86044d17a8ec04e43a4805a0092622d976ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Jul 2017 03:06:04 +0700
Subject: [PATCH 561/611] [pearvideo] Improve (closes #13031)

---
 youtube_dl/extractor/extractors.py |  2 +-
 youtube_dl/extractor/pear.py       | 34 ----------------
 youtube_dl/extractor/pearvideo.py  | 63 ++++++++++++++++++++++++++++++
 3 files changed, 64 insertions(+), 35 deletions(-)
 delete mode 100644 youtube_dl/extractor/pear.py
 create mode 100644 youtube_dl/extractor/pearvideo.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 75c1a3d0e..28f0d3f0d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -762,7 +762,7 @@ from .pandoratv import PandoraTVIE
 from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
-from .pear import PearIE
+from .pearvideo import PearVideoIE
 from .people import PeopleIE
 from .periscope import (
     PeriscopeIE,
diff --git a/youtube_dl/extractor/pear.py b/youtube_dl/extractor/pear.py
deleted file mode 100644
index 77fd46852..000000000
--- a/youtube_dl/extractor/pear.py
+++ /dev/null
@@ -1,34 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-
-
-class PearIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pearvideo\.com/video_(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'http://www.pearvideo.com/video_1076290',
-        'info_dict': {
-            'id': '1076290',
-            'ext': 'mp4',
-            'title': '小浣熊在主人家玻璃上滚石头：没砸',
-            'description': '小浣熊找到一个小石头，仿佛发现了一个宝贝。它不停地用石头按在玻璃上，滚来滚去，吸引主人注意。',
-            'url': 'http://video.pearvideo.com/mp4/short/20170508/cont-1076290-10438018-hd.mp4'
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        title = self._html_search_regex(r'<h1[^>]+class="video-tt">(.+)</h1>', webpage, 'title', fatal=False)
-        description = self._html_search_regex(r'<div[^>]+class="summary"[^>]*>([^<]+)<', webpage, 'description', fatal=False)
-        url = self._html_search_regex(r'hdUrl="(.*?)"', webpage, 'url', fatal=False)
-
-        return {
-            'id': video_id,
-            'ext': 'mp4',
-            'title': title,
-            'description': description,
-            'url': url
-        }
diff --git a/youtube_dl/extractor/pearvideo.py b/youtube_dl/extractor/pearvideo.py
new file mode 100644
index 000000000..1d777221c
--- /dev/null
+++ b/youtube_dl/extractor/pearvideo.py
@@ -0,0 +1,63 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    qualities,
+    unified_timestamp,
+)
+
+
+class PearVideoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?pearvideo\.com/video_(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.pearvideo.com/video_1076290',
+        'info_dict': {
+            'id': '1076290',
+            'ext': 'mp4',
+            'title': '小浣熊在主人家玻璃上滚石头：没砸',
+            'description': 'md5:01d576b747de71be0ee85eb7cac25f9d',
+            'timestamp': 1494275280,
+            'upload_date': '20170508',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        quality = qualities(
+            ('ldflv', 'ld', 'sdflv', 'sd', 'hdflv', 'hd', 'src'))
+
+        formats = [{
+            'url': mobj.group('url'),
+            'format_id': mobj.group('id'),
+            'quality': quality(mobj.group('id')),
+        } for mobj in re.finditer(
+            r'(?P<id>[a-zA-Z]+)Url\s*=\s*(["\'])(?P<url>(?:https?:)?//.+?)\2',
+            webpage)]
+        self._sort_formats(formats)
+
+        title = self._search_regex(
+            (r'<h1[^>]+\bclass=(["\'])video-tt\1[^>]*>(?P<value>[^<]+)',
+             r'<[^>]+\bdata-title=(["\'])(?P<value>(?:(?!\1).)+)\1'),
+            webpage, 'title', group='value')
+        description = self._search_regex(
+            (r'<div[^>]+\bclass=(["\'])summary\1[^>]*>(?P<value>[^<]+)',
+             r'<[^>]+\bdata-summary=(["\'])(?P<value>(?:(?!\1).)+)\1'),
+            webpage, 'description', default=None,
+            group='value') or self._html_search_meta('Description', webpage)
+        timestamp = unified_timestamp(self._search_regex(
+            r'<div[^>]+\bclass=["\']date["\'][^>]*>([^<]+)',
+            webpage, 'timestamp', fatal=False))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'timestamp': timestamp,
+            'formats': formats,
+        }

From 089b97cfee8553886d33cd52b7ede178cebd7034 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Jul 2017 04:30:48 +0700
Subject: [PATCH 562/611] [nexx] Improve JS embed extraction

---
 youtube_dl/extractor/nexx.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/nexx.py b/youtube_dl/extractor/nexx.py
index 60b42cb7d..12450d4c5 100644
--- a/youtube_dl/extractor/nexx.py
+++ b/youtube_dl/extractor/nexx.py
@@ -72,13 +72,17 @@ class NexxIE(InfoExtractor):
         entries = []
 
         # JavaScript Integration
-        for domain_id, video_id in re.findall(
-                r'''(?isx)
-                    <script\b[^>]+\bsrc=["\']https?://require\.nexx(?:\.cloud|cdn\.com)/(\d+).+?
-                    onPLAYReady.+?
-                    _play\.init\s*\(.+?\s*,\s*(\d+)\s*,\s*.+?\)
-                ''', webpage):
-            entries.append('https://api.nexx.cloud/v3/%s/videos/byid/%s' % (domain_id, video_id))
+        mobj = re.search(
+            r'<script\b[^>]+\bsrc=["\']https?://require\.nexx(?:\.cloud|cdn\.com)/(?P<id>\d+)',
+            webpage)
+        if mobj:
+            domain_id = mobj.group('id')
+            for video_id in re.findall(
+                    r'(?is)onPLAYReady.+?_play\.init\s*\(.+?\s*,\s*["\']?(\d+)',
+                    webpage):
+                entries.append(
+                    'https://api.nexx.cloud/v3/%s/videos/byid/%s'
+                    % (domain_id, video_id))
 
         # TODO: support more embed formats
 

From 3f59b0154a8b6dc85425edfbb3dfdc64f41a6ecb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Jul 2017 04:32:37 +0700
Subject: [PATCH 563/611] [nexx:embed] Add extractor for iframe embeds

---
 youtube_dl/extractor/extractors.py |  5 +++-
 youtube_dl/extractor/generic.py    | 31 +++++++++++++++++++-
 youtube_dl/extractor/nexx.py       | 46 ++++++++++++++++++++++++++++++
 3 files changed, 80 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 28f0d3f0d..e8a066b83 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -653,7 +653,10 @@ from .nextmedia import (
     AppleDailyIE,
     NextTVIE,
 )
-from .nexx import NexxIE
+from .nexx import (
+    NexxIE,
+    NexxEmbedIE,
+)
 from .nfb import NFBIE
 from .nfl import NFLIE
 from .nhk import NhkVodIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 123a21296..0ab2ef2d6 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -36,7 +36,10 @@ from .brightcove import (
     BrightcoveLegacyIE,
     BrightcoveNewIE,
 )
-from .nexx import NexxIE
+from .nexx import (
+    NexxIE,
+    NexxEmbedIE,
+)
 from .nbc import NBCSportsVPlayerIE
 from .ooyala import OoyalaIE
 from .rutv import RUTVIE
@@ -1566,6 +1569,27 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        # Nexx iFrame embed
+        {
+            'url': 'http://www.spiegel.de/sptv/spiegeltv/spiegel-tv-ueber-schnellste-katapult-achterbahn-der-welt-taron-a-1137884.html',
+            'info_dict': {
+                'id': '161464',
+                'ext': 'mp4',
+                'title': 'Nervenkitzel Achterbahn',
+                'alt_title': 'Karussellbauer in Deutschland',
+                'description': 'md5:ffe7b1cc59a01f585e0569949aef73cc',
+                'release_year': 2005,
+                'creator': 'SPIEGEL TV',
+                'thumbnail': r're:^https?://.*\.jpg$',
+                'duration': 2761,
+                'timestamp': 1394021479,
+                'upload_date': '20140305',
+            },
+            'params': {
+                'format': 'bestvideo',
+                'skip_download': True,
+            },
+        },
         # Facebook <iframe> embed
         {
             'url': 'https://www.hostblogger.de/blog/archives/6181-Auto-jagt-Betonmischer.html',
@@ -2155,6 +2179,11 @@ class GenericIE(InfoExtractor):
         if nexx_urls:
             return self.playlist_from_matches(nexx_urls, video_id, video_title, ie=NexxIE.ie_key())
 
+        # Look for Nexx iFrame embeds
+        nexx_embed_urls = NexxEmbedIE._extract_urls(webpage)
+        if nexx_embed_urls:
+            return self.playlist_from_matches(nexx_embed_urls, video_id, video_title, ie=NexxEmbedIE.ie_key())
+
         # Look for ThePlatform embeds
         tp_urls = ThePlatformIE._extract_urls(webpage)
         if tp_urls:
diff --git a/youtube_dl/extractor/nexx.py b/youtube_dl/extractor/nexx.py
index 12450d4c5..e2960273e 100644
--- a/youtube_dl/extractor/nexx.py
+++ b/youtube_dl/extractor/nexx.py
@@ -88,6 +88,10 @@ class NexxIE(InfoExtractor):
 
         return entries
 
+    @staticmethod
+    def _extract_url(webpage):
+        return NexxIE._extract_urls(webpage)[0]
+
     def _handle_error(self, response):
         status = int_or_none(try_get(
             response, lambda x: x['metadata']['status']) or 200)
@@ -223,3 +227,45 @@ class NexxIE(InfoExtractor):
                 video, lambda x: x['episodedata']['season'])),
             'formats': formats,
         }
+
+
+class NexxEmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://embed\.nexx(?:\.cloud|cdn\.com)/\d+/(?P<id>[^/?#&]+)'
+    _TEST = {
+        'url': 'http://embed.nexx.cloud/748/KC1614647Z27Y7T?autoplay=1',
+        'md5': '16746bfc28c42049492385c989b26c4a',
+        'info_dict': {
+            'id': '161464',
+            'ext': 'mp4',
+            'title': 'Nervenkitzel Achterbahn',
+            'alt_title': 'Karussellbauer in Deutschland',
+            'description': 'md5:ffe7b1cc59a01f585e0569949aef73cc',
+            'release_year': 2005,
+            'creator': 'SPIEGEL TV',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 2761,
+            'timestamp': 1394021479,
+            'upload_date': '20140305',
+        },
+        'params': {
+            'format': 'bestvideo',
+            'skip_download': True,
+        },
+    }
+
+    @staticmethod
+    def _extract_urls(webpage):
+        # Reference:
+        # 1. https://nx-s.akamaized.net/files/201510/44.pdf
+
+        # iFrame Embed Integration
+        return [mobj.group('url') for mobj in re.finditer(
+                r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//embed\.nexx(?:\.cloud|cdn\.com)/\d+/(?:(?!\1).)+)\1',
+            webpage)]
+
+    def _real_extract(self, url):
+        embed_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, embed_id)
+
+        return self.url_result(NexxIE._extract_url(webpage), ie=NexxIE.ie_key())

From 749ca5eced0b9a8ed1ef79f4ee80908e0ac242c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Jul 2017 04:33:14 +0700
Subject: [PATCH 564/611] [extractor/common] Fix playlist_from_matches

---
 youtube_dl/extractor/common.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index daa10885f..748b4d59f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -730,12 +730,12 @@ class InfoExtractor(object):
             video_info['title'] = video_title
         return video_info
 
-    def playlist_from_matches(self, matches, video_id, video_title, getter=None, ie=None):
-        urlrs = orderedSet(
+    def playlist_from_matches(self, matches, playlist_id=None, playlist_title=None, getter=None, ie=None):
+        urls = orderedSet(
             self.url_result(self._proto_relative_url(getter(m) if getter else m), ie)
             for m in matches)
         return self.playlist_result(
-            urlrs, playlist_id=video_id, playlist_title=video_title)
+            urls, playlist_id=playlist_id, playlist_title=playlist_title)
 
     @staticmethod
     def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None):

From 00d06e3cfcb7bfffa3b585694525f05a6ce36af9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Jul 2017 04:38:20 +0700
Subject: [PATCH 565/611] [spiegel:article] Add support for nexx iframe embeds
 (closes #13029)

---
 youtube_dl/extractor/spiegel.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index ec1b60388..8598377b0 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from .nexx import NexxEmbedIE
 from .spiegeltv import SpiegeltvIE
 from ..compat import compat_urlparse
 from ..utils import (
@@ -143,6 +144,9 @@ class SpiegelArticleIE(InfoExtractor):
         entries = [
             self.url_result(compat_urlparse.urljoin(
                 self.http_scheme() + '//spiegel.de/', embed_path))
-            for embed_path in embeds
-        ]
-        return self.playlist_result(entries)
+            for embed_path in embeds]
+        if embeds:
+            return self.playlist_result(entries)
+
+        return self.playlist_from_matches(
+            NexxEmbedIE._extract_urls(webpage), ie=NexxEmbedIE.ie_key())

From 13eb526f111dc9b98421f3e05287980b88766409 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Jul 2017 05:23:19 +0700
Subject: [PATCH 566/611] [nexx:embed] PEP 8

---
 youtube_dl/extractor/nexx.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nexx.py b/youtube_dl/extractor/nexx.py
index e2960273e..d0235fdfe 100644
--- a/youtube_dl/extractor/nexx.py
+++ b/youtube_dl/extractor/nexx.py
@@ -260,7 +260,7 @@ class NexxEmbedIE(InfoExtractor):
 
         # iFrame Embed Integration
         return [mobj.group('url') for mobj in re.finditer(
-                r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//embed\.nexx(?:\.cloud|cdn\.com)/\d+/(?:(?!\1).)+)\1',
+            r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//embed\.nexx(?:\.cloud|cdn\.com)/\d+/(?:(?!\1).)+)\1',
             webpage)]
 
     def _real_extract(self, url):

From 7abed4e06c8935026873a9c01821d5a5b2d80d4f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Jul 2017 12:40:45 +0700
Subject: [PATCH 567/611] [crunchyroll] Relax series and season regex (closes
 #13659)

---
 youtube_dl/extractor/crunchyroll.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 2ffa4a7f8..8bdaf0c2c 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -510,7 +510,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 
         # webpage provide more accurate data than series_title from XML
         series = self._html_search_regex(
-            r'id=["\']showmedia_about_episode_num[^>]+>\s*<a[^>]+>([^<]+)',
+            r'(?s)<h\d[^>]+\bid=["\']showmedia_about_episode_num[^>]+>(.+?)</h\d',
             webpage, 'series', fatal=False)
         season = xpath_text(metadata, 'series_title')
 
@@ -518,7 +518,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         episode_number = int_or_none(xpath_text(metadata, 'episode_number'))
 
         season_number = int_or_none(self._search_regex(
-            r'(?s)<h4[^>]+id=["\']showmedia_about_episode_num[^>]+>.+?</h4>\s*<h4>\s*Season (\d+)',
+            r'(?s)<h\d[^>]+id=["\']showmedia_about_episode_num[^>]+>.+?</h\d>\s*<h4>\s*Season (\d+)',
             webpage, 'season number', default=None))
 
         return {

From 83d00044c1c9aef487f1c30bb50246e9ad039636 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Jul 2017 20:48:09 +0700
Subject: [PATCH 568/611] [adn] Improve error reporting (#13663)

---
 youtube_dl/extractor/adn.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/adn.py b/youtube_dl/extractor/adn.py
index 39f80b23f..cffdab6ca 100644
--- a/youtube_dl/extractor/adn.py
+++ b/youtube_dl/extractor/adn.py
@@ -107,11 +107,13 @@ class ADNIE(InfoExtractor):
         metas = options.get('metas') or {}
         title = metas.get('title') or video_info['title']
         links = player_config.get('links') or {}
+        error = None
         if not links:
             links_url = player_config['linksurl']
             links_data = self._download_json(urljoin(
                 self._BASE_URL, links_url), video_id)
             links = links_data.get('links') or {}
+            error = links_data.get('error')
 
         formats = []
         for format_id, qualities in links.items():
@@ -130,7 +132,8 @@ class ADNIE(InfoExtractor):
                     for f in m3u8_formats:
                         f['language'] = 'fr'
                 formats.extend(m3u8_formats)
-        error = options.get('error')
+        if not error:
+            error = options.get('error')
         if not formats and error:
             raise ExtractorError('%s said: %s' % (self.IE_NAME, error), expected=True)
         self._sort_formats(formats)

From bb176df3bbef62dcf958dcc542f778686e44db63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Jul 2017 22:19:40 +0700
Subject: [PATCH 569/611] [spiegel:article] Move test

---
 youtube_dl/extractor/generic.py | 21 ---------------------
 youtube_dl/extractor/spiegel.py | 20 ++++++++++++++++++++
 2 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 0ab2ef2d6..36c81eda9 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1569,27 +1569,6 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
-        # Nexx iFrame embed
-        {
-            'url': 'http://www.spiegel.de/sptv/spiegeltv/spiegel-tv-ueber-schnellste-katapult-achterbahn-der-welt-taron-a-1137884.html',
-            'info_dict': {
-                'id': '161464',
-                'ext': 'mp4',
-                'title': 'Nervenkitzel Achterbahn',
-                'alt_title': 'Karussellbauer in Deutschland',
-                'description': 'md5:ffe7b1cc59a01f585e0569949aef73cc',
-                'release_year': 2005,
-                'creator': 'SPIEGEL TV',
-                'thumbnail': r're:^https?://.*\.jpg$',
-                'duration': 2761,
-                'timestamp': 1394021479,
-                'upload_date': '20140305',
-            },
-            'params': {
-                'format': 'bestvideo',
-                'skip_download': True,
-            },
-        },
         # Facebook <iframe> embed
         {
             'url': 'https://www.hostblogger.de/blog/archives/6181-Auto-jagt-Betonmischer.html',
diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index 8598377b0..84298fee4 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -122,6 +122,26 @@ class SpiegelArticleIE(InfoExtractor):
 
         },
         'playlist_count': 6,
+    }, {
+        # Nexx iFrame embed
+        'url': 'http://www.spiegel.de/sptv/spiegeltv/spiegel-tv-ueber-schnellste-katapult-achterbahn-der-welt-taron-a-1137884.html',
+        'info_dict': {
+            'id': '161464',
+            'ext': 'mp4',
+            'title': 'Nervenkitzel Achterbahn',
+            'alt_title': 'Karussellbauer in Deutschland',
+            'description': 'md5:ffe7b1cc59a01f585e0569949aef73cc',
+            'release_year': 2005,
+            'creator': 'SPIEGEL TV',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 2761,
+            'timestamp': 1394021479,
+            'upload_date': '20140305',
+        },
+        'params': {
+            'format': 'bestvideo',
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):

From d20b1c6725fce956b44413cced449b7d09b12de2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 20 Jul 2017 18:14:14 +0800
Subject: [PATCH 570/611] [dispeak] Recognize sevt subdomain (closes #13276)

---
 ChangeLog                       | 6 ++++++
 youtube_dl/extractor/dispeak.py | 6 +++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 7d71fc5e1..8e442d159 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
++ [dispeak] Recognize sevt subdomain (#13276)
+
+
 version 2017.07.15
 
 Core
diff --git a/youtube_dl/extractor/dispeak.py b/youtube_dl/extractor/dispeak.py
index a78cb8a2a..c05f601e2 100644
--- a/youtube_dl/extractor/dispeak.py
+++ b/youtube_dl/extractor/dispeak.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class DigitallySpeakingIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:evt\.dispeak|events\.digitallyspeaking)\.com/(?:[^/]+/)+xml/(?P<id>[^.]+)\.xml'
+    _VALID_URL = r'https?://(?:s?evt\.dispeak|events\.digitallyspeaking)\.com/(?:[^/]+/)+xml/(?P<id>[^.]+)\.xml'
 
     _TESTS = [{
         # From http://gdcvault.com/play/1023460/Tenacious-Design-and-The-Interface
@@ -28,6 +28,10 @@ class DigitallySpeakingIE(InfoExtractor):
         # From http://www.gdcvault.com/play/1014631/Classic-Game-Postmortem-PAC
         'url': 'http://events.digitallyspeaking.com/gdc/sf11/xml/12396_1299111843500GMPX.xml',
         'only_matching': True,
+    }, {
+        # From http://www.gdcvault.com/play/1013700/Advanced-Material
+        'url': 'http://sevt.dispeak.com/ubm/gdc/eur10/xml/11256_1282118587281VNIT.xml',
+        'only_matching': True,
     }]
 
     def _parse_mp4(self, metadata):

From 85f5a74b6cf44b0c8b612c264c36eaabd958f501 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 20 Jul 2017 21:19:09 +0800
Subject: [PATCH 571/611] [tbs] Mark as broken and skip invalid tests

---
 youtube_dl/extractor/tbs.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tbs.py b/youtube_dl/extractor/tbs.py
index bf93eb868..e9474533f 100644
--- a/youtube_dl/extractor/tbs.py
+++ b/youtube_dl/extractor/tbs.py
@@ -8,6 +8,9 @@ from ..utils import extract_attributes
 
 
 class TBSIE(TurnerBaseIE):
+    # https://github.com/rg3/youtube-dl/issues/13658
+    _WORKING = False
+
     _VALID_URL = r'https?://(?:www\.)?(?P<site>tbs|tntdrama)\.com/videos/(?:[^/]+/)+(?P<id>[^/?#]+)\.html'
     _TESTS = [{
         'url': 'http://www.tbs.com/videos/people-of-earth/season-1/extras/2007318/theatrical-trailer.html',
@@ -17,7 +20,8 @@ class TBSIE(TurnerBaseIE):
             'ext': 'mp4',
             'title': 'Theatrical Trailer',
             'description': 'Catch the latest comedy from TBS, People of Earth, premiering Halloween night--Monday, October 31, at 9/8c.',
-        }
+        },
+        'skip': 'TBS videos are deleted after a while',
     }, {
         'url': 'http://www.tntdrama.com/videos/good-behavior/season-1/extras/1538823/you-better-run.html',
         'md5': 'ce53c6ead5e9f3280b4ad2031a6fab56',
@@ -26,7 +30,8 @@ class TBSIE(TurnerBaseIE):
             'ext': 'mp4',
             'title': 'You Better Run',
             'description': 'Letty Raines must figure out what she\'s running toward while running away from her past. Good Behavior premieres November 15 at 9/8c.',
-        }
+        },
+        'skip': 'TBS videos are deleted after a while',
     }]
 
     def _real_extract(self, url):

From fa63cf6c2301972b7d0ae76fb7a11c7d1a2786a9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 20 Jul 2017 22:57:51 +0800
Subject: [PATCH 572/611] [youku:show] Fix playlist extraction (closes #13248)

---
 ChangeLog                     |  1 +
 youtube_dl/extractor/youku.py | 66 ++++++++++++++++++++---------------
 2 files changed, 39 insertions(+), 28 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 8e442d159..a83523cb9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [youku:show] Fix playlist extraction (#13248)
 + [dispeak] Recognize sevt subdomain (#13276)
 
 
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index dcce15d77..4ae9adb51 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import itertools
 import random
 import re
 import string
@@ -222,50 +221,61 @@ class YoukuShowIE(InfoExtractor):
     _VALID_URL = r'https?://list\.youku\.com/show/id_(?P<id>[0-9a-z]+)\.html'
     IE_NAME = 'youku:show'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://list.youku.com/show/id_zc7c670be07ff11e48b3f.html',
         'info_dict': {
             'id': 'zc7c670be07ff11e48b3f',
-            'title': '花千骨 未删减版',
+            'title': '花千骨 DVD版',
             'description': 'md5:a1ae6f5618571bbeb5c9821f9c81b558',
         },
         'playlist_count': 50,
-    }
+    }, {
+        # Episode number not starting from 1
+        'url': 'http://list.youku.com/show/id_zefbfbd70efbfbd780bef.html',
+        'info_dict': {
+            'id': 'zefbfbd70efbfbd780bef',
+            'title': '超级飞侠3',
+            'description': 'md5:275715156abebe5ccc2a1992e9d56b98',
+        },
+        'playlist_count': 24,
+    }]
 
-    _PAGE_SIZE = 40
+    def _extract_entries(self, playlist_data_url, show_id, idx, query, url):
+        query['callback'] = 'cb'
+        playlist_data = self._download_json(
+            playlist_data_url, show_id, query=query,
+            note='Downloading playlist data page %d' % (idx + 1),
+            transform_source=lambda s: js_to_json(strip_jsonp(s)))['html']
+        drama_list = (get_element_by_class('p-drama-grid', playlist_data) or
+                      get_element_by_class('p-drama-half-row', playlist_data))
+        if drama_list is None:
+            raise ExtractorError('No episodes found')
+        video_urls = re.findall(r'<a[^>]+href="([^"]+)"', drama_list)
+        return playlist_data, [
+            self.url_result(urljoin(url, video_url), YoukuIE.ie_key())
+            for video_url in video_urls]
 
     def _real_extract(self, url):
         show_id = self._match_id(url)
         webpage = self._download_webpage(url, show_id)
 
-        entries = []
         page_config = self._parse_json(self._search_regex(
             r'var\s+PageConfig\s*=\s*({.+});', webpage, 'page config'),
             show_id, transform_source=js_to_json)
-        for idx in itertools.count(0):
-            if idx == 0:
-                playlist_data_url = 'http://list.youku.com/show/module'
-                query = {'id': page_config['showid'], 'tab': 'point'}
-            else:
-                playlist_data_url = 'http://list.youku.com/show/point'
-                query = {
+        first_page, entries = self._extract_entries(
+            'http://list.youku.com/show/module', show_id, 0, {
+                'id': page_config['showid'],
+                'tab': 'showInfo',
+            }, url)
+        # The first reload_id has the same items as first_page
+        reload_ids = re.findall('<li[^>]+data-id="([^"]+)">', first_page)[1:]
+        for idx, reload_id in enumerate(reload_ids):
+            _, new_entries = self._extract_entries(
+                'http://list.youku.com/show/episode', show_id, idx + 1, {
                     'id': page_config['showid'],
-                    'stage': 'reload_%d' % (self._PAGE_SIZE * idx + 1),
-                }
-            query['callback'] = 'cb'
-            playlist_data = self._download_json(
-                playlist_data_url, show_id, query=query,
-                note='Downloading playlist data page %d' % (idx + 1),
-                transform_source=lambda s: js_to_json(strip_jsonp(s)))['html']
-            video_urls = re.findall(
-                r'<div[^>]+class="p-thumb"[^<]+<a[^>]+href="([^"]+)"',
-                playlist_data)
-            new_entries = [
-                self.url_result(urljoin(url, video_url), YoukuIE.ie_key())
-                for video_url in video_urls]
+                    'stage': reload_id,
+                }, url)
             entries.extend(new_entries)
-            if len(new_entries) < self._PAGE_SIZE:
-                break
 
         desc = self._html_search_meta('description', webpage, fatal=False)
         playlist_title = desc.split(',')[0] if desc else None

From 3fcf346ac16e6fe1963a3eab861d6bd9c32ce6db Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 20 Jul 2017 23:20:46 +0800
Subject: [PATCH 573/611] [youku:show] Refine playlist extraction

Handle playlists that the initial page is not the first page
---
 youtube_dl/extractor/youku.py | 34 +++++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 4ae9adb51..0c4bc2eda 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -13,7 +13,6 @@ from ..utils import (
     js_to_json,
     str_or_none,
     strip_jsonp,
-    urljoin,
 )
 
 
@@ -238,13 +237,16 @@ class YoukuShowIE(InfoExtractor):
             'description': 'md5:275715156abebe5ccc2a1992e9d56b98',
         },
         'playlist_count': 24,
+    }, {
+        # Ongoing playlist. The initial page is the last one
+        'url': 'http://list.youku.com/show/id_za7c275ecd7b411e1a19e.html',
+        'only_matchine': True,
     }]
 
-    def _extract_entries(self, playlist_data_url, show_id, idx, query, url):
+    def _extract_entries(self, playlist_data_url, show_id, note, query):
         query['callback'] = 'cb'
         playlist_data = self._download_json(
-            playlist_data_url, show_id, query=query,
-            note='Downloading playlist data page %d' % (idx + 1),
+            playlist_data_url, show_id, query=query, note=note,
             transform_source=lambda s: js_to_json(strip_jsonp(s)))['html']
         drama_list = (get_element_by_class('p-drama-grid', playlist_data) or
                       get_element_by_class('p-drama-half-row', playlist_data))
@@ -252,29 +254,39 @@ class YoukuShowIE(InfoExtractor):
             raise ExtractorError('No episodes found')
         video_urls = re.findall(r'<a[^>]+href="([^"]+)"', drama_list)
         return playlist_data, [
-            self.url_result(urljoin(url, video_url), YoukuIE.ie_key())
+            self.url_result(self._proto_relative_url(video_url, 'http:'), YoukuIE.ie_key())
             for video_url in video_urls]
 
     def _real_extract(self, url):
         show_id = self._match_id(url)
         webpage = self._download_webpage(url, show_id)
 
+        entries = []
         page_config = self._parse_json(self._search_regex(
             r'var\s+PageConfig\s*=\s*({.+});', webpage, 'page config'),
             show_id, transform_source=js_to_json)
-        first_page, entries = self._extract_entries(
-            'http://list.youku.com/show/module', show_id, 0, {
+        first_page, initial_entries = self._extract_entries(
+            'http://list.youku.com/show/module', show_id,
+            note='Downloading initial playlist data page',
+            query={
                 'id': page_config['showid'],
                 'tab': 'showInfo',
-            }, url)
+            })
+        first_page_reload_id = self._html_search_regex(
+            r'<div[^>]+id="(reload_\d+)', first_page, 'first page reload id')
         # The first reload_id has the same items as first_page
-        reload_ids = re.findall('<li[^>]+data-id="([^"]+)">', first_page)[1:]
+        reload_ids = re.findall('<li[^>]+data-id="([^"]+)">', first_page)
         for idx, reload_id in enumerate(reload_ids):
+            if reload_id == first_page_reload_id:
+                entries.extend(initial_entries)
+                continue
             _, new_entries = self._extract_entries(
-                'http://list.youku.com/show/episode', show_id, idx + 1, {
+                'http://list.youku.com/show/episode', show_id,
+                note='Downloading playlist data page %d' % (idx + 1),
+                query={
                     'id': page_config['showid'],
                     'stage': reload_id,
-                }, url)
+                })
             entries.extend(new_entries)
 
         desc = self._html_search_meta('description', webpage, fatal=False)

From c653326a1425f4c271f387fde7a706bf4b52a7a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 20 Jul 2017 22:49:52 +0700
Subject: [PATCH 574/611] [funnyordie] Extract more metadata (closes #13677)

---
 youtube_dl/extractor/funnyordie.py | 64 ++++++++++++++++++++++++++----
 1 file changed, 56 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index 49409369c..f85e7de14 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -1,10 +1,14 @@
 from __future__ import unicode_literals
 
-import json
 import re
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    float_or_none,
+    int_or_none,
+    unified_timestamp,
+)
 
 
 class FunnyOrDieIE(InfoExtractor):
@@ -18,6 +22,10 @@ class FunnyOrDieIE(InfoExtractor):
             'title': 'Heart-Shaped Box: Literal Video Version',
             'description': 'md5:ea09a01bc9a1c46d9ab696c01747c338',
             'thumbnail': r're:^http:.*\.jpg$',
+            'uploader': 'DASjr',
+            'timestamp': 1317904928,
+            'upload_date': '20111006',
+            'duration': 318.3,
         },
     }, {
         'url': 'http://www.funnyordie.com/embed/e402820827',
@@ -27,6 +35,8 @@ class FunnyOrDieIE(InfoExtractor):
             'title': 'Please Use This Song (Jon Lajoie)',
             'description': 'Please use this to sell something.  www.jonlajoie.com',
             'thumbnail': r're:^http:.*\.jpg$',
+            'timestamp': 1398988800,
+            'upload_date': '20140502',
         },
         'params': {
             'skip_download': True,
@@ -100,15 +110,53 @@ class FunnyOrDieIE(InfoExtractor):
                 'url': 'http://www.funnyordie.com%s' % src,
             }]
 
-        post_json = self._search_regex(
-            r'fb_post\s*=\s*(\{.*?\});', webpage, 'post details')
-        post = json.loads(post_json)
+        timestamp = unified_timestamp(self._html_search_meta(
+            'uploadDate', webpage, 'timestamp', default=None))
+
+        uploader = self._html_search_regex(
+            r'<h\d[^>]+\bclass=["\']channel-preview-name[^>]+>(.+?)</h',
+            webpage, 'uploader', default=None)
+
+        title, description, thumbnail, duration = [None] * 4
+
+        medium = self._parse_json(
+            self._search_regex(
+                r'jsonMedium\s*=\s*({.+?});', webpage, 'JSON medium',
+                default='{}'),
+            video_id, fatal=False)
+        if medium:
+            title = medium.get('title')
+            duration = float_or_none(medium.get('duration'))
+            if not timestamp:
+                timestamp = unified_timestamp(medium.get('publishDate'))
+
+        post = self._parse_json(
+            self._search_regex(
+                r'fb_post\s*=\s*(\{.*?\});', webpage, 'post details',
+                default='{}'),
+            video_id, fatal=False)
+        if post:
+            if not title:
+                title = post.get('name')
+            description = post.get('description')
+            thumbnail = post.get('picture')
+
+        if not title:
+            title = self._og_search_title(webpage)
+        if not description:
+            description = self._og_search_description(webpage)
+        if not duration:
+            duration = int_or_none(self._html_search_meta(
+                ('video:duration', 'duration'), webpage, 'duration', default=False))
 
         return {
             'id': video_id,
-            'title': post['name'],
-            'description': post.get('description'),
-            'thumbnail': post.get('picture'),
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'timestamp': timestamp,
+            'duration': duration,
             'formats': formats,
             'subtitles': subtitles,
         }

From dc6520aa3d1fe7afc52613e392f15dde90af4844 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 20 Jul 2017 23:22:36 +0700
Subject: [PATCH 575/611] [egghead:lesson] Add extractor (#6635)

---
 youtube_dl/extractor/egghead.py    | 49 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  5 ++-
 2 files changed, 53 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/egghead.py b/youtube_dl/extractor/egghead.py
index c86f52319..e4a3046af 100644
--- a/youtube_dl/extractor/egghead.py
+++ b/youtube_dl/extractor/egghead.py
@@ -2,6 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    try_get,
+    unified_timestamp,
+)
 
 
 class EggheadCourseIE(InfoExtractor):
@@ -33,3 +38,47 @@ class EggheadCourseIE(InfoExtractor):
         return self.playlist_result(
             entries, playlist_id, course.get('title'),
             course.get('description'))
+
+
+class EggheadLessonIE(InfoExtractor):
+    IE_DESC = 'egghead.io lesson'
+    IE_NAME = 'egghead:lesson'
+    _VALID_URL = r'https://egghead\.io/lessons/(?P<id>[^/?#&]+)'
+    _TEST = {
+        'url': 'https://egghead.io/lessons/javascript-linear-data-flow-with-container-style-types-box',
+        'info_dict': {
+            'id': 'fv5yotjxcg',
+            'ext': 'mp4',
+            'title': 'Create linear data flow with container style types (Box)',
+            'description': 'md5:9aa2cdb6f9878ed4c39ec09e85a8150e',
+            'thumbnail': r're:^https?:.*\.jpg$',
+            'timestamp': 1481296768,
+            'upload_date': '20161209',
+            'duration': 304,
+            'view_count': 0,
+            'tags': ['javascript', 'free'],
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        lesson_id = self._match_id(url)
+
+        lesson = self._download_json(
+            'https://egghead.io/api/v1/lessons/%s' % lesson_id, lesson_id)
+
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'Wistia',
+            'url': 'wistia:%s' % lesson['wistia_id'],
+            'id': lesson['wistia_id'],
+            'title': lesson.get('title'),
+            'description': lesson.get('summary'),
+            'thumbnail': lesson.get('thumb_nail'),
+            'timestamp': unified_timestamp(lesson.get('published_at')),
+            'duration': int_or_none(lesson.get('duration')),
+            'view_count': int_or_none(lesson.get('plays_count')),
+            'tags': try_get(lesson, lambda x: x['tag_list'], list),
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e8a066b83..db7616caa 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -298,7 +298,10 @@ from .dw import (
 from .eagleplatform import EaglePlatformIE
 from .ebaumsworld import EbaumsWorldIE
 from .echomsk import EchoMskIE
-from .egghead import EggheadCourseIE
+from .egghead import (
+    EggheadCourseIE,
+    EggheadLessonIE,
+)
 from .ehow import EHowIE
 from .eighttracks import EightTracksIE
 from .einthusan import EinthusanIE

From 0396806f671e5828c2abdeb8048acf8b654507b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 21 Jul 2017 00:13:32 +0700
Subject: [PATCH 576/611] [YoutubeDL] Do not override id, extractor and
 extractor_key in url_transparent All these meta fields must be borrowed from
 final extractor that actually performs extraction. This commit fixes
 extractor id in download archives for url_transparent downloads. Previously,
 'transparent' extractor was erroneously used for extractor archive id, e.g.
 'eggheadlesson 4n8ugwwj5t' instead of 'wistia 4n8ugwwj5t'.

---
 test/test_YoutubeDL.py  | 7 ++++++-
 youtube_dl/YoutubeDL.py | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 70989e232..e0decb81c 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -41,6 +41,7 @@ def _make_result(formats, **kwargs):
         'id': 'testid',
         'title': 'testttitle',
         'extractor': 'testex',
+        'extractor_key': 'TestEx',
     }
     res.update(**kwargs)
     return res
@@ -761,7 +762,8 @@ class TestYoutubeDL(unittest.TestCase):
                     '_type': 'url_transparent',
                     'url': 'foo2:',
                     'ie_key': 'Foo2',
-                    'title': 'foo1 title'
+                    'title': 'foo1 title',
+                    'id': 'foo1_id',
                 }
 
         class Foo2IE(InfoExtractor):
@@ -787,6 +789,9 @@ class TestYoutubeDL(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['url'], TEST_URL)
         self.assertEqual(downloaded['title'], 'foo1 title')
+        self.assertEqual(downloaded['id'], 'testid')
+        self.assertEqual(downloaded['extractor'], 'testex')
+        self.assertEqual(downloaded['extractor_key'], 'TestEx')
 
 
 if __name__ == '__main__':
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 89c07be29..f94836d06 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -860,7 +860,7 @@ class YoutubeDL(object):
 
             force_properties = dict(
                 (k, v) for k, v in ie_result.items() if v is not None)
-            for f in ('_type', 'url', 'ie_key'):
+            for f in ('_type', 'url', 'id', 'extractor', 'extractor_key', 'ie_key'):
                 if f in force_properties:
                     del force_properties[f]
             new_result = info.copy()

From 7d9a1db1110b13e8e6b65613ebb3daf7f0ff3c4c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 22 Jul 2017 11:40:46 +0700
Subject: [PATCH 577/611] [dramafever] Remove video id from title (closes
 #13699)

---
 youtube_dl/extractor/dramafever.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index e7abc8889..03fa3aabc 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -12,6 +12,7 @@ from ..utils import (
     ExtractorError,
     clean_html,
     int_or_none,
+    remove_end,
     sanitized_Request,
     urlencode_postdata
 )
@@ -73,7 +74,7 @@ class DramaFeverIE(DramaFeverBaseIE):
         'info_dict': {
             'id': '4512.1',
             'ext': 'mp4',
-            'title': 'Cooking with Shin 4512.1',
+            'title': 'Cooking with Shin',
             'description': 'md5:a8eec7942e1664a6896fcd5e1287bfd0',
             'episode': 'Episode 1',
             'episode_number': 1,
@@ -91,7 +92,7 @@ class DramaFeverIE(DramaFeverBaseIE):
         'info_dict': {
             'id': '4826.4',
             'ext': 'mp4',
-            'title': 'Mnet Asian Music Awards 2015 4826.4',
+            'title': 'Mnet Asian Music Awards 2015',
             'description': 'md5:3ff2ee8fedaef86e076791c909cf2e91',
             'episode': 'Mnet Asian Music Awards 2015 - Part 3',
             'episode_number': 4,
@@ -122,6 +123,10 @@ class DramaFeverIE(DramaFeverBaseIE):
                     countries=self._GEO_COUNTRIES)
             raise
 
+        # title is postfixed with video id for some reason, removing
+        if info.get('title'):
+            info['title'] = remove_end(info['title'], video_id).strip()
+
         series_id, episode_number = video_id.split('.')
         episode_info = self._download_json(
             # We only need a single episode info, so restricting page size to one episode

From f76c02c87b479310b0e090216895879257b1062a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 22 Jul 2017 11:41:40 +0700
Subject: [PATCH 578/611] [dramafever] Fix tests

---
 youtube_dl/extractor/dramafever.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 03fa3aabc..9a498d72a 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -73,7 +73,7 @@ class DramaFeverIE(DramaFeverBaseIE):
         'url': 'http://www.dramafever.com/drama/4512/1/Cooking_with_Shin/',
         'info_dict': {
             'id': '4512.1',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Cooking with Shin',
             'description': 'md5:a8eec7942e1664a6896fcd5e1287bfd0',
             'episode': 'Episode 1',
@@ -81,7 +81,7 @@ class DramaFeverIE(DramaFeverBaseIE):
             'thumbnail': r're:^https?://.*\.jpg',
             'timestamp': 1404336058,
             'upload_date': '20140702',
-            'duration': 343,
+            'duration': 344,
         },
         'params': {
             # m3u8 download
@@ -91,7 +91,7 @@ class DramaFeverIE(DramaFeverBaseIE):
         'url': 'http://www.dramafever.com/drama/4826/4/Mnet_Asian_Music_Awards_2015/?ap=1',
         'info_dict': {
             'id': '4826.4',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Mnet Asian Music Awards 2015',
             'description': 'md5:3ff2ee8fedaef86e076791c909cf2e91',
             'episode': 'Mnet Asian Music Awards 2015 - Part 3',
@@ -99,7 +99,7 @@ class DramaFeverIE(DramaFeverBaseIE):
             'thumbnail': r're:^https?://.*\.jpg',
             'timestamp': 1450213200,
             'upload_date': '20151215',
-            'duration': 5602,
+            'duration': 5359,
         },
         'params': {
             # m3u8 download

From 359aa2fdd145d11a29a04f620fed95acbf142f66 Mon Sep 17 00:00:00 2001
From: dubber0 <rexa.mose@gmail.com>
Date: Sat, 22 Jul 2017 14:15:55 +0200
Subject: [PATCH 579/611] [npo] Add support for npo3.nl URLs

---
 youtube_dl/extractor/npo.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 516b1e941..fa4ef20c5 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -28,7 +28,7 @@ class NPOBaseIE(InfoExtractor):
 
 class NPOIE(NPOBaseIE):
     IE_NAME = 'npo'
-    IE_DESC = 'npo.nl and ntr.nl'
+    IE_DESC = 'npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl'
     _VALID_URL = r'''(?x)
                     (?:
                         npo:|
@@ -38,7 +38,7 @@ class NPOIE(NPOBaseIE):
                                 npo\.nl/(?!(?:live|radio)/)(?:[^/]+/){2}|
                                 ntr\.nl/(?:[^/]+/){2,}|
                                 omroepwnl\.nl/video/fragment/[^/]+__|
-                                zapp\.nl/[^/]+/[^/]+/
+                                (?:zapp|npo3)\.nl/(?:[^/]+/){2}
                             )
                         )
                         (?P<id>[^/?#]+)
@@ -146,6 +146,9 @@ class NPOIE(NPOBaseIE):
     }, {
         'url': 'http://www.zapp.nl/beste-vrienden-quiz/extra-video-s/WO_NTR_1067990',
         'only_matching': True,
+    }, {
+        'url': 'https://www.npo3.nl/3onderzoekt/16-09-2015/VPWON_1239870',
+        'only_matching': True,
     }, {
         # live stream
         'url': 'npo:LI_NL1_4188102',

From 327c8364f11f23dd919e8009c6adb021c34054fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 22 Jul 2017 21:35:14 +0700
Subject: [PATCH 580/611] [sportbox:embed] Fix extraction

---
 youtube_dl/extractor/sportbox.py | 61 ++++++++++++++++++--------------
 1 file changed, 34 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index e7bd5bf91..54497c880 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -4,7 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import js_to_json
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    js_to_json,
+)
 
 
 class SportBoxEmbedIE(InfoExtractor):
@@ -14,8 +18,10 @@ class SportBoxEmbedIE(InfoExtractor):
         'info_dict': {
             'id': '211355',
             'ext': 'mp4',
-            'title': 'В Новороссийске прошел детский турнир «Поле славы боевой»',
+            'title': '211355',
             'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 292,
+            'view_count': int,
         },
         'params': {
             # m3u8 download
@@ -24,6 +30,9 @@ class SportBoxEmbedIE(InfoExtractor):
     }, {
         'url': 'http://news.sportbox.ru/vdl/player?nid=370908&only_player=1&autostart=false&playeri=2&height=340&width=580',
         'only_matching': True,
+    }, {
+        'url': 'https://news.sportbox.ru/vdl/player/media/193095',
+        'only_matching': True,
     }]
 
     @staticmethod
@@ -37,36 +46,34 @@ class SportBoxEmbedIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
+        wjplayer_data = self._parse_json(
+            self._search_regex(
+                r'(?s)wjplayer\(({.+?})\);', webpage, 'wjplayer settings'),
+            video_id, transform_source=js_to_json)
+
         formats = []
-
-        def cleanup_js(code):
-            # desktop_advert_config contains complex Javascripts and we don't need it
-            return js_to_json(re.sub(r'desktop_advert_config.*', '', code))
-
-        jwplayer_data = self._parse_json(self._search_regex(
-            r'(?s)player\.setup\(({.+?})\);', webpage, 'jwplayer settings'), video_id,
-            transform_source=cleanup_js)
-
-        hls_url = jwplayer_data.get('hls_url')
-        if hls_url:
-            formats.extend(self._extract_m3u8_formats(
-                hls_url, video_id, ext='mp4', m3u8_id='hls'))
-
-        rtsp_url = jwplayer_data.get('rtsp_url')
-        if rtsp_url:
-            formats.append({
-                'url': rtsp_url,
-                'format_id': 'rtsp',
-            })
-
+        for source in wjplayer_data['sources']:
+            src = source.get('src')
+            if not src:
+                continue
+            if determine_ext(src) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    src, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False))
+            else:
+                formats.append({
+                    'url': src,
+                })
         self._sort_formats(formats)
 
-        title = jwplayer_data['node_title']
-        thumbnail = jwplayer_data.get('image_url')
+        view_count = int_or_none(self._search_regex(
+            r'Просмотров\s*:\s*(\d+)', webpage, 'view count', default=None))
 
         return {
             'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
+            'title': video_id,
+            'thumbnail': wjplayer_data.get('poster'),
+            'duration': int_or_none(wjplayer_data.get('duration')),
+            'view_count': view_count,
             'formats': formats,
         }

From 0017d9ad6de831384e74db14a821e4c94020c9ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Jul 2017 00:12:01 +0700
Subject: [PATCH 581/611] [YoutubeDL] Improve default format specification
 (closes #13704)

---
 test/test_YoutubeDL.py  | 11 +++++++++++
 youtube_dl/YoutubeDL.py | 31 +++++++++++++++++++++++--------
 2 files changed, 34 insertions(+), 8 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index e0decb81c..4af14f9db 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -449,6 +449,17 @@ class TestFormatSelection(unittest.TestCase):
             pass
         self.assertEqual(ydl.downloaded_info_dicts, [])
 
+    def test_default_format_spec(self):
+        ydl = YDL({'simulate': True})
+        self.assertEqual(ydl._default_format_spec({}), 'bestvideo+bestaudio/best')
+
+        ydl = YDL({'outtmpl': '-'})
+        self.assertEqual(ydl._default_format_spec({}), 'best')
+
+        ydl = YDL({})
+        self.assertEqual(ydl._default_format_spec({}, download=False), 'bestvideo+bestaudio/best')
+        self.assertEqual(ydl._default_format_spec({'is_live': True}), 'best')
+
 
 class TestYoutubeDL(unittest.TestCase):
     def test_subtitles(self):
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index f94836d06..367ae3533 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1064,6 +1064,25 @@ class YoutubeDL(object):
             return op(actual_value, comparison_value)
         return _filter
 
+    def _default_format_spec(self, info_dict, download=True):
+        req_format_list = []
+
+        def can_have_partial_formats():
+            if self.params.get('simulate', False):
+                return True
+            if not download:
+                return True
+            if self.params.get('outtmpl', DEFAULT_OUTTMPL) == '-':
+                return False
+            if info_dict.get('is_live'):
+                return False
+            merger = FFmpegMergerPP(self)
+            return merger.available and merger.can_merge()
+        if can_have_partial_formats():
+            req_format_list.append('bestvideo+bestaudio')
+        req_format_list.append('best')
+        return '/'.join(req_format_list)
+
     def build_format_selector(self, format_spec):
         def syntax_error(note, start):
             message = (
@@ -1534,14 +1553,10 @@ class YoutubeDL(object):
 
         req_format = self.params.get('format')
         if req_format is None:
-            req_format_list = []
-            if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
-                    not info_dict.get('is_live')):
-                merger = FFmpegMergerPP(self)
-                if merger.available and merger.can_merge():
-                    req_format_list.append('bestvideo+bestaudio')
-            req_format_list.append('best')
-            req_format = '/'.join(req_format_list)
+            req_format = self._default_format_spec(info_dict, download=download)
+            if self.params.get('verbose'):
+                self.to_stdout('[debug] Default format spec: %s' % req_format)
+
         format_selector = self.build_format_selector(req_format)
 
         # While in format selection we may need to have an access to the original

From e0f1fb0a27612c2398df59dd85194edfdf8cbc2a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Jul 2017 00:25:23 +0700
Subject: [PATCH 582/611] [mtv] Skip missing video parts (closes #13690)

---
 youtube_dl/extractor/mtv.py | 28 ++++++++++++++++++++++------
 youtube_dl/extractor/vh1.py | 12 ++++++++----
 2 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 8acea1461..fc098cd13 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -83,7 +83,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
                 hls_url = rendition.find('./src').text
                 formats.extend(self._extract_m3u8_formats(
                     hls_url, video_id, ext='mp4', entry_protocol='m3u8_native',
-                    m3u8_id='hls'))
+                    m3u8_id='hls', fatal=False))
             else:
                 # fms
                 try:
@@ -106,7 +106,8 @@ class MTVServicesInfoExtractor(InfoExtractor):
                     }])
                 except (KeyError, TypeError):
                     raise ExtractorError('Invalid rendition field.')
-        self._sort_formats(formats)
+        if formats:
+            self._sort_formats(formats)
         return formats
 
     def _extract_subtitles(self, mdoc, mtvn_id):
@@ -133,8 +134,11 @@ class MTVServicesInfoExtractor(InfoExtractor):
             mediagen_url += 'acceptMethods='
             mediagen_url += 'hls' if use_hls else 'fms'
 
-        mediagen_doc = self._download_xml(mediagen_url, video_id,
-                                          'Downloading video urls')
+        mediagen_doc = self._download_xml(
+            mediagen_url, video_id, 'Downloading video urls', fatal=False)
+
+        if mediagen_doc is False:
+            return None
 
         item = mediagen_doc.find('./video/item')
         if item is not None and item.get('type') == 'text':
@@ -174,6 +178,13 @@ class MTVServicesInfoExtractor(InfoExtractor):
 
         formats = self._extract_video_formats(mediagen_doc, mtvn_id, video_id)
 
+        # Some parts of complete video may be missing (e.g. missing Act 3 in
+        # http://www.southpark.de/alle-episoden/s14e01-sexual-healing)
+        if not formats:
+            return None
+
+        self._sort_formats(formats)
+
         return {
             'title': title,
             'formats': formats,
@@ -205,9 +216,14 @@ class MTVServicesInfoExtractor(InfoExtractor):
         title = xpath_text(idoc, './channel/title')
         description = xpath_text(idoc, './channel/description')
 
+        entries = []
+        for item in idoc.findall('.//item'):
+            info = self._get_video_info(item, use_hls)
+            if info:
+                entries.append(info)
+
         return self.playlist_result(
-            [self._get_video_info(item, use_hls) for item in idoc.findall('.//item')],
-            playlist_title=title, playlist_description=description)
+            entries, playlist_title=title, playlist_description=description)
 
     def _extract_triforce_mgid(self, webpage, data_zone=None, video_id=None):
         triforce_feed = self._parse_json(self._search_regex(
diff --git a/youtube_dl/extractor/vh1.py b/youtube_dl/extractor/vh1.py
index 6be3774b7..570fa45ea 100644
--- a/youtube_dl/extractor/vh1.py
+++ b/youtube_dl/extractor/vh1.py
@@ -121,7 +121,11 @@ class VH1IE(MTVIE):
         idoc = self._download_xml(
             doc_url, video_id,
             'Downloading info', transform_source=fix_xml_ampersands)
-        return self.playlist_result(
-            [self._get_video_info(item) for item in idoc.findall('.//item')],
-            playlist_id=video_id,
-        )
+
+        entries = []
+        for item in idoc.findall('.//item'):
+            info = self._get_video_info(item)
+            if info:
+                entries.append(info)
+
+        return self.playlist_result(entries, playlist_id=video_id)

From 935d6c20c00536cf39cf2844295266e64492bb10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Jul 2017 00:44:50 +0700
Subject: [PATCH 583/611] [vidio] Make duration non fatal and fix typo

---
 youtube_dl/extractor/vidio.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vidio.py b/youtube_dl/extractor/vidio.py
index 701bb1d01..01da32f1c 100644
--- a/youtube_dl/extractor/vidio.py
+++ b/youtube_dl/extractor/vidio.py
@@ -56,7 +56,8 @@ class VidioIE(InfoExtractor):
         self._sort_formats(formats)
 
         duration = int_or_none(duration or self._search_regex(
-            r'data-video-duration=(["\'])(?P<duartion>\d+)\1', webpage, 'duration'))
+            r'data-video-duration=(["\'])(?P<duration>\d+)\1', webpage,
+            'duration', fatal=False, group='duration'))
         thumbnail = thumbnail or self._og_search_thumbnail(webpage)
 
         like_count = int_or_none(self._search_regex(

From 71dde5eecf07ae3a8871e5d4a05a944097e17fb9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Jul 2017 00:59:07 +0700
Subject: [PATCH 584/611] [itv] Fix production id extraction (closes #13671)

---
 youtube_dl/extractor/itv.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index f3156804d..26c48e4b8 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -59,12 +59,18 @@ class ITVIE(InfoExtractor):
         def _add_sub_element(element, name):
             return etree.SubElement(element, _add_ns(name))
 
+        production_id = (
+            params.get('data-video-autoplay-id') or
+            '%s#001' % (
+                params.get('data-video-episode-id') or
+                video_id.replace('a', '/')))
+
         req_env = etree.Element(_add_ns('soapenv:Envelope'))
         _add_sub_element(req_env, 'soapenv:Header')
         body = _add_sub_element(req_env, 'soapenv:Body')
         get_playlist = _add_sub_element(body, ('tem:GetPlaylist'))
         request = _add_sub_element(get_playlist, 'tem:request')
-        _add_sub_element(request, 'itv:ProductionId').text = params['data-video-id']
+        _add_sub_element(request, 'itv:ProductionId').text = production_id
         _add_sub_element(request, 'itv:RequestGuid').text = compat_str(uuid.uuid4()).upper()
         vodcrid = _add_sub_element(request, 'itv:Vodcrid')
         _add_sub_element(vodcrid, 'com:Id')

From 425f41319aec6940195818e980005cef4946eb75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Jul 2017 01:06:08 +0700
Subject: [PATCH 585/611] [ChangeLog] Actualize

---
 ChangeLog | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index a83523cb9..cd8a34a86 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,28 @@
 version <unreleased>
 
+Core
+* [YoutubeDL] Improve default format specification (#13704)
+* [YoutubeDL] Do not override id, extractor and extractor_key for
+  url_transparent entities
+* [extractor/common] Fix playlist_from_matches
+
 Extractors
+* [itv] Fix production id extraction (#13671, #13703)
+* [vidio] Make duration non fatal and fix typo
+* [mtv] Skip missing video parts (#13690)
+* [sportbox:embed] Fix extraction
++ [npo] Add support for npo3.nl URLs (#13695)
+* [dramafever] Remove video id from title (#13699)
++ [egghead:lesson] Add support for lessons (#6635)
+* [funnyordie] Extract more metadata (#13677)
 * [youku:show] Fix playlist extraction (#13248)
 + [dispeak] Recognize sevt subdomain (#13276)
+* [adn] Improve error reporting (#13663)
+* [crunchyroll] Relax series and season regex (#13659)
++ [spiegel:article] Add support for nexx iframe embeds (#13029)
++ [nexx:embed] Add support for iframe embeds
+* [nexx] Improve JS embed extraction
++ [pearvideo] Add support for pearvideo.com (#13031)
 
 
 version 2017.07.15

From 0db492c02a60dbfb44514833445bf267f5319ae2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Jul 2017 01:09:09 +0700
Subject: [PATCH 586/611] release 2017.07.23

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 ChangeLog                 |  2 +-
 docs/supportedsites.md    | 13 ++++++++-----
 youtube_dl/version.py     |  2 +-
 4 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 0f20d0485..37d09d796 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.07.15*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.07.15**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.07.23*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.07.23**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.07.15
+[debug] youtube-dl version 2017.07.23
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index cd8a34a86..302d32aab 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.07.23
 
 Core
 * [YoutubeDL] Improve default format specification (#13704)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d7304ba06..eb09c470c 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -42,7 +42,7 @@
  - **Allocine**
  - **AlphaPorno**
  - **AMCNetworks**
- - **anderetijden**: npo.nl and ntr.nl
+ - **anderetijden**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **AnimeOnDemand**
  - **anitube.se**
  - **Anvato**
@@ -238,6 +238,7 @@
  - **EbaumsWorld**
  - **EchoMsk**
  - **egghead:course**: egghead.io course
+ - **egghead:lesson**: egghead.io lesson
  - **eHow**
  - **Einthusan**
  - **eitb.tv**
@@ -522,6 +523,7 @@
  - **NextMediaActionNews**: 蘋果日報 - 動新聞
  - **NextTV**: 壹電視
  - **Nexx**
+ - **NexxEmbed**
  - **nfb**: National Film Board of Canada
  - **nfl.com**
  - **NhkVod**
@@ -552,7 +554,7 @@
  - **NowTVList**
  - **nowvideo**: NowVideo
  - **Noz**
- - **npo**: npo.nl and ntr.nl
+ - **npo**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **npo.nl:live**
  - **npo.nl:radio**
  - **npo.nl:radio:fragment**
@@ -596,6 +598,7 @@
  - **Patreon**
  - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), CONNECTICUT PUBLIC TELEVISION (WEDH), KNPB Channel 5 (KNPB), SOPTV (KSYS), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), PBS 39 (WLVT), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WGCU Public Media (WGCU), WPBT2 (WPBT), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), ETV (WRLK), UNC-TV (WUNC), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), OPB (KOPB), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WTTW11 (WTTW), Iowa Public Television/IPTV (KDIN), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), WIPB-TV (WIPB), WTIU (WTIU), CET  (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), KSMQ (KSMQ), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), East Tennessee PBS (WSJK), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVPB (WVPB), WKYU-PBS (WKYU), KERA 13 (KERA), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), witf (WITF), WQED Multimedia (WQED), WMHT Educational Telecommunications (WMHT), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WGTE TV (WGTE), Lakeland Public Television (KAWE), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), MontanaPBS (KUSM), KRWG/Channel 22 (KRWG), KACV (KACV), KCOS/Channel 13 (KCOS), WCNY/Channel 24 (WCNY), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), WPSU (WPSU), WVIA Public Media Studios (WVIA), WTVI (WTVI), Western Reserve PBS (WNEO), WVIZ/PBS ideastream (WVIZ), KCTS 9 (KCTS), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KLRN (KLRN), KLRU (KLRU), WTJX Channel 12 (WTJX), WCVE PBS (WCVE), KBTC Public Television (KBTC)
  - **pcmag**
+ - **PearVideo**
  - **People**
  - **periscope**: Periscope
  - **periscope:user**: Periscope user videos
@@ -772,7 +775,7 @@
  - **tagesschau:player**
  - **Tass**
  - **TastyTrade**
- - **TBS**
+ - **TBS** (Currently broken)
  - **TDSLifeway**
  - **teachertube**: teachertube.com videos
  - **teachertube:user:collection**: teachertube.com user and collection videos
@@ -950,7 +953,7 @@
  - **VoiceRepublic**
  - **VoxMedia**
  - **Vporn**
- - **vpro**: npo.nl and ntr.nl
+ - **vpro**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **Vrak**
  - **VRT**: deredactie.be, sporza.be, cobra.be and cobra.canvas.be
  - **vrv**
@@ -976,7 +979,7 @@
  - **wholecloud**: WholeCloud
  - **Wimp**
  - **Wistia**
- - **wnl**: npo.nl and ntr.nl
+ - **wnl**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **WorldStarHipHop**
  - **wrzuta.pl**
  - **wrzuta.pl:playlist**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 82e166fef..a8dbb93e3 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.07.15'
+__version__ = '2017.07.23'

From 905d18a7aa42263c66f311ac0cdf46b2caa2f4d6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 23 Jul 2017 16:21:35 +0800
Subject: [PATCH 587/611] [options] Correctly hide login info from debug
 outputs (#13696)

Iterate over opts instead of PRIVATE_OPTS for both performance and
correctness
---
 ChangeLog             |  6 ++++++
 test/test_options.py  | 26 ++++++++++++++++++++++++++
 youtube_dl/options.py | 38 ++++++++++++++++++--------------------
 3 files changed, 50 insertions(+), 20 deletions(-)
 create mode 100644 test/test_options.py

diff --git a/ChangeLog b/ChangeLog
index 302d32aab..8e63b5c11 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Core
+* [options] Correctly hide login info from debug outputs (#13696)
+
+
 version 2017.07.23
 
 Core
diff --git a/test/test_options.py b/test/test_options.py
new file mode 100644
index 000000000..785281fe3
--- /dev/null
+++ b/test/test_options.py
@@ -0,0 +1,26 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.options import _hide_login_info
+
+
+class TestOptions(unittest.TestCase):
+    def test_hide_login_inf(self):
+        self.assertEqual(_hide_login_info(['-u', 'foo', '-p', 'bar']),
+                         ['-u', 'PRIVATE', '-p', 'PRIVATE'])
+        self.assertEqual(_hide_login_info(['-u']), ['-u'])
+        self.assertEqual(_hide_login_info(['-u', 'foo', '-u', 'bar']),
+                         ['-u', 'PRIVATE', '-u', 'PRIVATE'])
+        self.assertEqual(_hide_login_info(['--username=foo']),
+                         ['--username=PRIVATE'])
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 79e9fd12c..38439c971 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -20,6 +20,24 @@ from .utils import (
 from .version import __version__
 
 
+def _hide_login_info(opts):
+    PRIVATE_OPTS = set(['-p', '--password', '-u', '--username', '--video-password', '--ap-password', '--ap-username'])
+    eqre = re.compile('^(?P<key>' + ('|'.join(re.escape(po) for po in PRIVATE_OPTS)) + ')=.+$')
+
+    def _scrub_eq(o):
+        m = eqre.match(o)
+        if m:
+            return m.group('key') + '=PRIVATE'
+        else:
+            return o
+
+    opts = list(map(_scrub_eq, opts))
+    for idx, opt in enumerate(opts):
+        if opt in PRIVATE_OPTS and idx + 1 < len(opts):
+            opts[idx + 1] = 'PRIVATE'
+    return opts
+
+
 def parseOpts(overrideArguments=None):
     def _readOptions(filename_bytes, default=[]):
         try:
@@ -93,26 +111,6 @@ def parseOpts(overrideArguments=None):
     def _comma_separated_values_options_callback(option, opt_str, value, parser):
         setattr(parser.values, option.dest, value.split(','))
 
-    def _hide_login_info(opts):
-        PRIVATE_OPTS = ['-p', '--password', '-u', '--username', '--video-password', '--ap-password', '--ap-username']
-        eqre = re.compile('^(?P<key>' + ('|'.join(re.escape(po) for po in PRIVATE_OPTS)) + ')=.+$')
-
-        def _scrub_eq(o):
-            m = eqre.match(o)
-            if m:
-                return m.group('key') + '=PRIVATE'
-            else:
-                return o
-
-        opts = list(map(_scrub_eq, opts))
-        for private_opt in PRIVATE_OPTS:
-            try:
-                i = opts.index(private_opt)
-                opts[i + 1] = 'PRIVATE'
-            except ValueError:
-                pass
-        return opts
-
     # No need to wrap help messages if we're on a wide console
     columns = compat_get_terminal_size().columns
     max_width = columns if columns else 80

From 73095e013fb1bc4a1e676d7be77a103f0013a227 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 23 Jul 2017 16:24:18 +0800
Subject: [PATCH 588/611] [options] Typo

---
 test/test_options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_options.py b/test/test_options.py
index 785281fe3..3a25a6ba3 100644
--- a/test/test_options.py
+++ b/test/test_options.py
@@ -12,7 +12,7 @@ from youtube_dl.options import _hide_login_info
 
 
 class TestOptions(unittest.TestCase):
-    def test_hide_login_inf(self):
+    def test_hide_login_info(self):
         self.assertEqual(_hide_login_info(['-u', 'foo', '-p', 'bar']),
                          ['-u', 'PRIVATE', '-p', 'PRIVATE'])
         self.assertEqual(_hide_login_info(['-u']), ['-u'])

From e3ce912c3d767fcb1a1225d05ac64da1acab94aa Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 23 Jul 2017 16:25:17 +0800
Subject: [PATCH 589/611] [niconico] improve error reporting (#13696)

---
 youtube_dl/extractor/niconico.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 695e32e59..79b9952c3 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -147,6 +147,9 @@ class NiconicoIE(InfoExtractor):
             elif 'closed' in flv_info:
                 raise ExtractorError('Niconico videos now require logging in',
                                      expected=True)
+            elif 'error' in flv_info:
+                raise ExtractorError('%s reports error: %s' % (
+                    self.IE_NAME, flv_info['error'][0]), expected=True)
             else:
                 raise ExtractorError('Unable to find video URL')
 

From 3150976669ef2ffc9f4eee9e99a6e70730bc22fb Mon Sep 17 00:00:00 2001
From: nyuszika7h <nyuszika7h@gmail.com>
Date: Sun, 23 Jul 2017 15:33:18 +0200
Subject: [PATCH 590/611] [ISSUE_TEMPLATE_tmpl.md] Minor improvements

---
 .github/ISSUE_TEMPLATE_tmpl.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE_tmpl.md b/.github/ISSUE_TEMPLATE_tmpl.md
index df79503d3..26f61d3b4 100644
--- a/.github/ISSUE_TEMPLATE_tmpl.md
+++ b/.github/ISSUE_TEMPLATE_tmpl.md
@@ -1,16 +1,16 @@
 ## Please follow the guide below
 
 - You will be asked some questions and requested to provide some information, please read them **carefully** and answer honestly
-- Put an `x` into all the boxes [ ] relevant to your *issue* (like that [x])
-- Use *Preview* tab to see how your issue will actually look like
+- Put an `x` into all the boxes [ ] relevant to your *issue* (like this: `[x]`)
+- Use the *Preview* tab to see what your issue will actually look like
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *%(version)s*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *%(version)s*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
 - [ ] I've **verified** and **I assure** that I'm running youtube-dl **%(version)s**
 
 ### Before submitting an *issue* make sure you have:
-- [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
+- [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
 - [ ] [Searched](https://github.com/rg3/youtube-dl/search?type=Issues) the bugtracker for similar issues including closed ones
 
 ### What is the purpose of your *issue*?
@@ -28,9 +28,9 @@
 
 ### If the purpose of this *issue* is a *bug report*, *site support request* or you are not completely sure provide the full verbose output as follows:
 
-Add `-v` flag to **your command line** you run youtube-dl with, copy the **whole** output and insert it here. It should look similar to one below (replace it with **your** log inserted between triple ```):
+Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl -v <your command line>`), copy the **whole** output and insert it here. It should look similar to one below (replace it with **your** log inserted between triple ```):
+
 ```
-$ youtube-dl -v <your command line>
 [debug] System config: []
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']

From f0e31e32c940d8529353f40bd2426163c3199216 Mon Sep 17 00:00:00 2001
From: nyuszika7h <nyuszika7h@gmail.com>
Date: Sun, 23 Jul 2017 15:40:04 +0200
Subject: [PATCH 591/611] [nick] Automate geo-restriction bypass (#13711)

---
 youtube_dl/extractor/nick.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index 08a75929e..1fa19cde4 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -12,6 +12,7 @@ class NickIE(MTVServicesInfoExtractor):
     IE_NAME = 'nick.com'
     _VALID_URL = r'https?://(?:(?:www|beta)\.)?nick(?:jr)?\.com/(?:[^/]+/)?(?:videos/clip|[^/]+/videos)/(?P<id>[^/?#.]+)'
     _FEED_URL = 'http://udat.mtvnservices.com/service1/dispatch.htm'
+    _GEO_COUNTRIES = ['US']
     _TESTS = [{
         'url': 'http://www.nick.com/videos/clip/alvinnn-and-the-chipmunks-112-full-episode.html',
         'playlist': [

From 70bfab0e9ac8ddb7da67d71633c8c4b0704054cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Jul 2017 21:00:19 +0700
Subject: [PATCH 592/611] [mtv] Improve thumbnal extraction

---
 youtube_dl/extractor/mtv.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index fc098cd13..25af5ddfd 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -50,8 +50,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         thumb_node = itemdoc.find(search_path)
         if thumb_node is None:
             return None
-        else:
-            return thumb_node.attrib['url']
+        return thumb_node.get('url') or thumb_node.text or None
 
     def _extract_mobile_video_formats(self, mtvn_id):
         webpage_url = self._MOBILE_TEMPLATE % mtvn_id

From c99d6890cb46626870474e5c1092d9772096c4b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Jul 2017 21:00:56 +0700
Subject: [PATCH 593/611] [nickru] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/nick.py       | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index db7616caa..2513f2587 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -673,6 +673,7 @@ from .nick import (
     NickIE,
     NickDeIE,
     NickNightIE,
+    NickRuIE,
 )
 from .niconico import NiconicoIE, NiconicoPlaylistIE
 from .ninecninemedia import (
diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index 1fa19cde4..b688637bc 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -125,3 +125,21 @@ class NickNightIE(NickDeIE):
         return self._search_regex(
             r'mrss\s*:\s*(["\'])(?P<url>http.+?)\1', webpage,
             'mrss url', group='url')
+
+
+class NickRuIE(MTVServicesInfoExtractor):
+    IE_NAME = 'nickelodeonru'
+    _VALID_URL = r'https?://(?:www\.)nickelodeon\.ru/(?:playlist|shows|videos)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'http://www.nickelodeon.ru/shows/henrydanger/videos/episodes/3-sezon-15-seriya-licenziya-na-polyot/pmomfb#playlist/7airc6',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.nickelodeon.ru/videos/smotri-na-nickelodeon-v-iyule/g9hvh7',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        mgid = self._extract_mgid(webpage)
+        return self.url_result('http://media.mtvnservices.com/embed/%s' % mgid)

From f9c48d895b5600c82e9b55f703e68b060f25de07 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Jul 2017 23:12:43 +0700
Subject: [PATCH 594/611] [cloudy] Fix extraction (closes #13737)

---
 youtube_dl/extractor/cloudy.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cloudy.py b/youtube_dl/extractor/cloudy.py
index 9bc8dbea4..85ca20ecc 100644
--- a/youtube_dl/extractor/cloudy.py
+++ b/youtube_dl/extractor/cloudy.py
@@ -30,7 +30,11 @@ class CloudyIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(
-            'http://www.cloudy.ec/embed.php?id=%s' % video_id, video_id)
+            'https://www.cloudy.ec/embed.php', video_id, query={
+                'id': video_id,
+                'playerPage': 1,
+                'autoplay': 1,
+            })
 
         info = self._parse_html5_media_entries(url, webpage, video_id)[0]
 

From 9682666bdadec955fb8600fa3721f59b2a4b8099 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 27 Jul 2017 02:04:51 +0700
Subject: [PATCH 595/611] [amcnetworks] Make rating optional (closes #12453)

---
 youtube_dl/extractor/amcnetworks.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/amcnetworks.py b/youtube_dl/extractor/amcnetworks.py
index 3a0ec6776..dd3b18d72 100644
--- a/youtube_dl/extractor/amcnetworks.py
+++ b/youtube_dl/extractor/amcnetworks.py
@@ -3,9 +3,10 @@ from __future__ import unicode_literals
 
 from .theplatform import ThePlatformIE
 from ..utils import (
-    update_url_query,
-    parse_age_limit,
     int_or_none,
+    parse_age_limit,
+    try_get,
+    update_url_query,
 )
 
 
@@ -68,7 +69,8 @@ class AMCNetworksIE(ThePlatformIE):
         info = self._parse_theplatform_metadata(theplatform_metadata)
         video_id = theplatform_metadata['pid']
         title = theplatform_metadata['title']
-        rating = theplatform_metadata['ratings'][0]['rating']
+        rating = try_get(
+            theplatform_metadata, lambda x: x['ratings'][0]['rating'])
         auth_required = self._search_regex(
             r'window\.authRequired\s*=\s*(true|false);',
             webpage, 'auth required')

From 24e966e8dab954136dabbc497064ac63b252495b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 28 Jul 2017 12:13:19 +0200
Subject: [PATCH 596/611] [megaphone] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/generic.py    |  8 +++++
 youtube_dl/extractor/megaphone.py  | 55 ++++++++++++++++++++++++++++++
 3 files changed, 64 insertions(+)
 create mode 100644 youtube_dl/extractor/megaphone.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 2513f2587..668248648 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -558,6 +558,7 @@ from .matchtv import MatchTVIE
 from .mdr import MDRIE
 from .mediaset import MediasetIE
 from .medici import MediciIE
+from .megaphone import MegaphoneIE
 from .meipai import MeipaiIE
 from .melonvod import MelonVODIE
 from .meta import METAIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 36c81eda9..9678c32c4 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -97,6 +97,7 @@ from .washingtonpost import WashingtonPostIE
 from .wistia import WistiaIE
 from .mediaset import MediasetIE
 from .joj import JojIE
+from .megaphone import MegaphoneIE
 
 
 class GenericIE(InfoExtractor):
@@ -2790,6 +2791,13 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 joj_urls, video_id, video_title, ie=JojIE.ie_key())
 
+        # Look for megaphone.fm embeds
+        mpfn_urls = MegaphoneIE._extract_urls(webpage)
+        if mpfn_urls:
+            return self.playlist_from_matches(
+                mpfn_urls, video_id, video_title, ie=MegaphoneIE.ie_key())
+
+
         def merge_dicts(dict1, dict2):
             merged = {}
             for k, v in dict1.items():
diff --git a/youtube_dl/extractor/megaphone.py b/youtube_dl/extractor/megaphone.py
new file mode 100644
index 000000000..60e3caf0d
--- /dev/null
+++ b/youtube_dl/extractor/megaphone.py
@@ -0,0 +1,55 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import js_to_json
+
+
+class MegaphoneIE(InfoExtractor):
+    IE_NAME = 'megaphone.fm'
+    IE_DESC = 'megaphone.fm embedded players'
+    _VALID_URL = r'https://player\.megaphone\.fm/(?P<id>[A-Z0-9]+)'
+    _TEST = {
+        'url': 'https://player.megaphone.fm/GLT9749789991?"',
+        'md5': '4816a0de523eb3e972dc0dda2c191f96',
+        'info_dict': {
+            'id': 'GLT9749789991',
+            'ext': 'mp3',
+            'title': '#97 What Kind Of Idiot Gets Phished?',
+            'thumbnail': 're:^https://.*\.png.*$',
+            'duration': 1776.26375,
+            'author': 'Reply All',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._og_search_property('audio:title', webpage)
+        author = self._og_search_property('audio:artist', webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        episode_json = self._search_regex(r'(?s)var\s+episode\s*=\s*(\{.+?\});', webpage, 'episode JSON')
+        episode_data = self._parse_json(episode_json, video_id, js_to_json)
+        video_url = self._proto_relative_url(episode_data['mediaUrl'], 'https:')
+
+        formats = [{
+            'url': video_url,
+        }]
+
+        return {
+            'id': video_id,
+            'thumbnail': thumbnail,
+            'title': title,
+            'author': author,
+            'duration': episode_data['duration'],
+            'formats': formats,
+        }
+
+    @classmethod
+    def _extract_urls(cls, webpage):
+        return [m[0] for m in re.findall(
+            r'<iframe[^>]*?\ssrc=["\'](%s)' % cls._VALID_URL, webpage)]

From c5a49ff08413411174837f1034ef439b79ff774b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Jul 2017 15:02:41 +0700
Subject: [PATCH 597/611] [downloader/hls] Use redirect URL as manifest base
 (#13755)

---
 youtube_dl/downloader/hls.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 0e29c8a2a..46308cf07 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -59,9 +59,9 @@ class HlsFD(FragmentFD):
         man_url = info_dict['url']
         self.to_screen('[%s] Downloading m3u8 manifest' % self.FD_NAME)
 
-        manifest = self.ydl.urlopen(self._prepare_url(info_dict, man_url)).read()
-
-        s = manifest.decode('utf-8', 'ignore')
+        urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url))
+        man_url = urlh.geturl()
+        s = urlh.read().decode('utf-8', 'ignore')
 
         if not self.can_download(s, info_dict):
             if info_dict.get('extra_param_to_segment_url'):

From cbbe66635f3c23316f04a6f56ad57e025bc47263 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Jul 2017 15:10:19 +0700
Subject: [PATCH 598/611] [yandexdisk] Add extractor (closes #13755)

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/yandexdisk.py | 115 +++++++++++++++++++++++++++++
 2 files changed, 116 insertions(+)
 create mode 100644 youtube_dl/extractor/yandexdisk.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 668248648..852942e0d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1298,6 +1298,7 @@ from .yandexmusic import (
     YandexMusicAlbumIE,
     YandexMusicPlaylistIE,
 )
+from .yandexdisk import YandexDiskIE
 from .yesjapan import YesJapanIE
 from .yinyuetai import YinYueTaiIE
 from .ynet import YnetIE
diff --git a/youtube_dl/extractor/yandexdisk.py b/youtube_dl/extractor/yandexdisk.py
new file mode 100644
index 000000000..11729f0f7
--- /dev/null
+++ b/youtube_dl/extractor/yandexdisk.py
@@ -0,0 +1,115 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    determine_ext,
+    float_or_none,
+    int_or_none,
+    try_get,
+    urlencode_postdata,
+)
+
+
+class YandexDiskIE(InfoExtractor):
+    _VALID_URL = r'https?://yadi\.sk/i/(?P<id>[^/?#&]+)'
+
+    _TEST = {
+        'url': 'https://yadi.sk/i/VdOeDou8eZs6Y',
+        'md5': '33955d7ae052f15853dc41f35f17581c',
+        'info_dict': {
+            'id': 'VdOeDou8eZs6Y',
+            'ext': 'mp4',
+            'title': '4.mp4',
+            'duration': 168.6,
+            'uploader': 'y.botova',
+            'uploader_id': '300043621',
+            'view_count': int,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        status = self._download_webpage(
+            'https://disk.yandex.com/auth/status', video_id, query={
+                'urlOrigin': url,
+                'source': 'public',
+                'md5': 'false',
+            })
+
+        sk = self._search_regex(
+            r'(["\'])sk(?:External)?\1\s*:\s*(["\'])(?P<value>(?:(?!\2).)+)\2',
+            status, 'sk', group='value')
+
+        webpage = self._download_webpage(url, video_id)
+
+        models = self._parse_json(
+            self._search_regex(
+                r'<script[^>]+id=["\']models-client[^>]+>\s*(\[.+?\])\s*</script',
+                webpage, 'video JSON'),
+            video_id)
+
+        data = next(
+            model['data'] for model in models
+            if model.get('model') == 'resource')
+
+        video_hash = data['id']
+        title = data['name']
+
+        models = self._download_json(
+            'https://disk.yandex.com/models/', video_id,
+            data=urlencode_postdata({
+                '_model.0': 'videoInfo',
+                'id.0': video_hash,
+                '_model.1': 'do-get-resource-url',
+                'id.1': video_hash,
+                'version': '13.6',
+                'sk': sk,
+            }), query={'_m': 'videoInfo'})['models']
+
+        videos = try_get(models, lambda x: x[0]['data']['videos'], list) or []
+        source_url = try_get(
+            models, lambda x: x[1]['data']['file'], compat_str)
+
+        formats = []
+        if source_url:
+            formats.append({
+                'url': source_url,
+                'format_id': 'source',
+                'ext': determine_ext(title, 'mp4'),
+                'quality': 1,
+            })
+        for video in videos:
+            format_url = video.get('url')
+            if not format_url:
+                continue
+            if determine_ext(format_url) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False))
+            else:
+                formats.append({
+                    'url': format_url,
+                })
+        self._sort_formats(formats)
+
+        duration = float_or_none(try_get(
+            models, lambda x: x[0]['data']['duration']), 1000)
+        uploader = try_get(
+            data, lambda x: x['user']['display_name'], compat_str)
+        uploader_id = try_get(
+            data, lambda x: x['user']['uid'], compat_str)
+        view_count = int_or_none(try_get(
+            data, lambda x: x['meta']['views_counter']))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'duration': duration,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'view_count': view_count,
+            'formats': formats,
+        }

From 95908ce45382a72be26e918f8db1809b0ce81190 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Jul 2017 15:13:12 +0700
Subject: [PATCH 599/611] [extractor/generic] PEP 8

---
 youtube_dl/extractor/generic.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 9678c32c4..4b83e861b 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2797,7 +2797,6 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 mpfn_urls, video_id, video_title, ie=MegaphoneIE.ie_key())
 
-
         def merge_dicts(dict1, dict2):
             merged = {}
             for k, v in dict1.items():

From 2a7a82321135bc59364c91caddde4211f378785b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Jul 2017 15:25:32 +0700
Subject: [PATCH 600/611] [svtplay] Update API URL (closes #13767)

---
 youtube_dl/extractor/svt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index 1b5afb73e..38a505f9c 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -181,7 +181,7 @@ class SVTPlayIE(SVTBaseIE):
 
         if video_id:
             data = self._download_json(
-                'http://www.svt.se/videoplayer-api/video/%s' % video_id, video_id)
+                'https://api.svt.se/videoplayer-api/video/%s' % video_id, video_id)
             info_dict = self._extract_video(data, video_id)
             if not info_dict.get('title'):
                 info_dict['title'] = re.sub(

From c04017519da74a375d6c1c95733d921e96d8ee82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Jul 2017 15:30:53 +0700
Subject: [PATCH 601/611] [svtplay] Use geo verification proxy for API request

---
 youtube_dl/extractor/svt.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index 38a505f9c..48bc4529e 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -181,7 +181,8 @@ class SVTPlayIE(SVTBaseIE):
 
         if video_id:
             data = self._download_json(
-                'https://api.svt.se/videoplayer-api/video/%s' % video_id, video_id)
+                'https://api.svt.se/videoplayer-api/video/%s' % video_id,
+                video_id, headers=self.geo_verification_headers())
             info_dict = self._extract_video(data, video_id)
             if not info_dict.get('title'):
                 info_dict['title'] = re.sub(

From 836ef2648613f4ca565b319af4769c02e35f60f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Jul 2017 18:41:42 +0700
Subject: [PATCH 602/611] [soundcloud:trackstation] Add extractor (closes
 #13733)

---
 youtube_dl/extractor/extractors.py |   3 +-
 youtube_dl/extractor/soundcloud.py | 141 ++++++++++++++++++-----------
 2 files changed, 89 insertions(+), 55 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 852942e0d..d2c5e8030 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -935,8 +935,9 @@ from .soundcloud import (
     SoundcloudIE,
     SoundcloudSetIE,
     SoundcloudUserIE,
+    SoundcloudTrackStationIE,
     SoundcloudPlaylistIE,
-    SoundcloudSearchIE
+    SoundcloudSearchIE,
 )
 from .soundgasm import (
     SoundgasmIE,
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 3f1a46bb2..2f1b2978c 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -31,6 +31,7 @@ class SoundcloudIE(InfoExtractor):
 
     _VALID_URL = r'''(?x)^(?:https?://)?
                     (?:(?:(?:www\.|m\.)?soundcloud\.com/
+                            (?!stations/track)
                             (?P<uploader>[\w\d-]+)/
                             (?!(?:tracks|sets(?:/.+?)?|reposts|likes|spotlight)/?(?:$|[?#]))
                             (?P<title>[\w\d-]+)/?
@@ -330,7 +331,63 @@ class SoundcloudSetIE(SoundcloudPlaylistBaseIE):
         }
 
 
-class SoundcloudUserIE(SoundcloudPlaylistBaseIE):
+class SoundcloudPagedPlaylistBaseIE(SoundcloudPlaylistBaseIE):
+    _API_BASE = 'https://api.soundcloud.com'
+    _API_V2_BASE = 'https://api-v2.soundcloud.com'
+
+    def _extract_playlist(self, base_url, playlist_id, playlist_title):
+        COMMON_QUERY = {
+            'limit': 50,
+            'client_id': self._CLIENT_ID,
+            'linked_partitioning': '1',
+        }
+
+        query = COMMON_QUERY.copy()
+        query['offset'] = 0
+
+        next_href = base_url + '?' + compat_urllib_parse_urlencode(query)
+
+        entries = []
+        for i in itertools.count():
+            response = self._download_json(
+                next_href, playlist_id, 'Downloading track page %s' % (i + 1))
+
+            collection = response['collection']
+            if not collection:
+                break
+
+            def resolve_permalink_url(candidates):
+                for cand in candidates:
+                    if isinstance(cand, dict):
+                        permalink_url = cand.get('permalink_url')
+                        entry_id = self._extract_id(cand)
+                        if permalink_url and permalink_url.startswith('http'):
+                            return permalink_url, entry_id
+
+            for e in collection:
+                permalink_url, entry_id = resolve_permalink_url((e, e.get('track'), e.get('playlist')))
+                if permalink_url:
+                    entries.append(self.url_result(permalink_url, video_id=entry_id))
+
+            next_href = response.get('next_href')
+            if not next_href:
+                break
+
+            parsed_next_href = compat_urlparse.urlparse(response['next_href'])
+            qs = compat_urlparse.parse_qs(parsed_next_href.query)
+            qs.update(COMMON_QUERY)
+            next_href = compat_urlparse.urlunparse(
+                parsed_next_href._replace(query=compat_urllib_parse_urlencode(qs, True)))
+
+        return {
+            '_type': 'playlist',
+            'id': playlist_id,
+            'title': playlist_title,
+            'entries': entries,
+        }
+
+
+class SoundcloudUserIE(SoundcloudPagedPlaylistBaseIE):
     _VALID_URL = r'''(?x)
                         https?://
                             (?:(?:www|m)\.)?soundcloud\.com/
@@ -385,16 +442,13 @@ class SoundcloudUserIE(SoundcloudPlaylistBaseIE):
         'playlist_mincount': 1,
     }]
 
-    _API_BASE = 'https://api.soundcloud.com'
-    _API_V2_BASE = 'https://api-v2.soundcloud.com'
-
     _BASE_URL_MAP = {
-        'all': '%s/profile/soundcloud:users:%%s' % _API_V2_BASE,
-        'tracks': '%s/users/%%s/tracks' % _API_BASE,
-        'sets': '%s/users/%%s/playlists' % _API_V2_BASE,
-        'reposts': '%s/profile/soundcloud:users:%%s/reposts' % _API_V2_BASE,
-        'likes': '%s/users/%%s/likes' % _API_V2_BASE,
-        'spotlight': '%s/users/%%s/spotlight' % _API_V2_BASE,
+        'all': '%s/profile/soundcloud:users:%%s' % SoundcloudPagedPlaylistBaseIE._API_V2_BASE,
+        'tracks': '%s/users/%%s/tracks' % SoundcloudPagedPlaylistBaseIE._API_BASE,
+        'sets': '%s/users/%%s/playlists' % SoundcloudPagedPlaylistBaseIE._API_V2_BASE,
+        'reposts': '%s/profile/soundcloud:users:%%s/reposts' % SoundcloudPagedPlaylistBaseIE._API_V2_BASE,
+        'likes': '%s/users/%%s/likes' % SoundcloudPagedPlaylistBaseIE._API_V2_BASE,
+        'spotlight': '%s/users/%%s/spotlight' % SoundcloudPagedPlaylistBaseIE._API_V2_BASE,
     }
 
     _TITLE_MAP = {
@@ -416,57 +470,36 @@ class SoundcloudUserIE(SoundcloudPlaylistBaseIE):
             resolv_url, uploader, 'Downloading user info')
 
         resource = mobj.group('rsrc') or 'all'
-        base_url = self._BASE_URL_MAP[resource] % user['id']
 
-        COMMON_QUERY = {
-            'limit': 50,
-            'client_id': self._CLIENT_ID,
-            'linked_partitioning': '1',
-        }
+        return self._extract_playlist(
+            self._BASE_URL_MAP[resource] % user['id'], compat_str(user['id']),
+            '%s (%s)' % (user['username'], self._TITLE_MAP[resource]))
 
-        query = COMMON_QUERY.copy()
-        query['offset'] = 0
 
-        next_href = base_url + '?' + compat_urllib_parse_urlencode(query)
+class SoundcloudTrackStationIE(SoundcloudPagedPlaylistBaseIE):
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/stations/track/[^/]+/(?P<id>[^/?#&]+)'
+    IE_NAME = 'soundcloud:trackstation'
+    _TESTS = [{
+        'url': 'https://soundcloud.com/stations/track/officialsundial/your-text',
+        'info_dict': {
+            'id': '286017854',
+            'title': 'Track station: your-text',
+        },
+        'playlist_mincount': 47,
+    }]
 
-        entries = []
-        for i in itertools.count():
-            response = self._download_json(
-                next_href, uploader, 'Downloading track page %s' % (i + 1))
+    def _real_extract(self, url):
+        track_name = self._match_id(url)
 
-            collection = response['collection']
-            if not collection:
-                break
+        webpage = self._download_webpage(url, track_name)
 
-            def resolve_permalink_url(candidates):
-                for cand in candidates:
-                    if isinstance(cand, dict):
-                        permalink_url = cand.get('permalink_url')
-                        entry_id = self._extract_id(cand)
-                        if permalink_url and permalink_url.startswith('http'):
-                            return permalink_url, entry_id
+        track_id = self._search_regex(
+            r'soundcloud:track-stations:(\d+)', webpage, 'track id')
 
-            for e in collection:
-                permalink_url, entry_id = resolve_permalink_url((e, e.get('track'), e.get('playlist')))
-                if permalink_url:
-                    entries.append(self.url_result(permalink_url, video_id=entry_id))
-
-            next_href = response.get('next_href')
-            if not next_href:
-                break
-
-            parsed_next_href = compat_urlparse.urlparse(response['next_href'])
-            qs = compat_urlparse.parse_qs(parsed_next_href.query)
-            qs.update(COMMON_QUERY)
-            next_href = compat_urlparse.urlunparse(
-                parsed_next_href._replace(query=compat_urllib_parse_urlencode(qs, True)))
-
-        return {
-            '_type': 'playlist',
-            'id': compat_str(user['id']),
-            'title': '%s (%s)' % (user['username'], self._TITLE_MAP[resource]),
-            'entries': entries,
-        }
+        return self._extract_playlist(
+            '%s/stations/soundcloud:track-stations:%s/tracks'
+            % (self._API_V2_BASE, track_id),
+            track_id, 'Track station: %s' % track_name)
 
 
 class SoundcloudPlaylistIE(SoundcloudPlaylistBaseIE):

From e445850e69990502b171765343fc38317e932aca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Jul 2017 18:45:57 +0700
Subject: [PATCH 603/611] [soundcloud] Update client id

---
 youtube_dl/extractor/soundcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 2f1b2978c..2e52e092b 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -122,7 +122,7 @@ class SoundcloudIE(InfoExtractor):
         },
     ]
 
-    _CLIENT_ID = '2t9loNQH90kzJcsFCODdigxfp325aq4z'
+    _CLIENT_ID = 'JlZIsxg2hY5WnBgtn3jfS0UYCl0K8DOg'
     _IPHONE_CLIENT_ID = '376f225bf427445fc4bfb6b99b72e0bf'
 
     @staticmethod

From ca127ab2c174cdee4428eb4e192393c6ca942ac8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Jul 2017 23:07:28 +0700
Subject: [PATCH 604/611] [ard] Add support for lives (closes #13771)

---
 youtube_dl/extractor/ard.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 2d5599456..3f248b147 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -93,6 +93,7 @@ class ARDMediathekIE(InfoExtractor):
 
         duration = int_or_none(media_info.get('_duration'))
         thumbnail = media_info.get('_previewImage')
+        is_live = media_info.get('_isLive') is True
 
         subtitles = {}
         subtitle_url = media_info.get('_subtitleUrl')
@@ -106,6 +107,7 @@ class ARDMediathekIE(InfoExtractor):
             'id': video_id,
             'duration': duration,
             'thumbnail': thumbnail,
+            'is_live': is_live,
             'formats': formats,
             'subtitles': subtitles,
         }
@@ -166,9 +168,11 @@ class ARDMediathekIE(InfoExtractor):
         # determine video id from url
         m = re.match(self._VALID_URL, url)
 
+        document_id = None
+
         numid = re.search(r'documentId=([0-9]+)', url)
         if numid:
-            video_id = numid.group(1)
+            document_id = video_id = numid.group(1)
         else:
             video_id = m.group('video_id')
 
@@ -228,12 +232,16 @@ class ARDMediathekIE(InfoExtractor):
                 'formats': formats,
             }
         else:  # request JSON file
+            if not document_id:
+                video_id = self._search_regex(
+                    r'/play/(?:config|media)/(\d+)', webpage, 'media id')
             info = self._extract_media_info(
-                'http://www.ardmediathek.de/play/media/%s' % video_id, webpage, video_id)
+                'http://www.ardmediathek.de/play/media/%s' % video_id,
+                webpage, video_id)
 
         info.update({
             'id': video_id,
-            'title': title,
+            'title': self._live_title(title) if info.get('is_live') else title,
             'description': description,
             'thumbnail': thumbnail,
         })

From 198d4cb40ce9d819e8e4079058642ee96dae213b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Grzegorz=20Ruci=C5=84ski?= <grucin@gmail.com>
Date: Sat, 29 Jul 2017 20:30:04 +0200
Subject: [PATCH 605/611] [generic] Add support for another ooyala embed
 pattern (closes #13727)

---
 youtube_dl/extractor/generic.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 4b83e861b..34e814988 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -575,6 +575,19 @@ class GenericIE(InfoExtractor):
             },
             'skip': 'movie expired',
         },
+        # ooyala video embedded with http://player.ooyala.com/static/v4/production/latest/core.min.js
+        {
+            'url': 'http://wnep.com/2017/07/22/steampunk-fest-comes-to-honesdale/',
+            'info_dict': {
+                'id': 'lwYWYxYzE6V5uJMjNGyKtwwiw9ZJD7t2',
+                'ext': 'mp4',
+                'title': 'Steampunk Fest Comes to Honesdale',
+                'duration': 43.276,
+            },
+            'params': {
+                'skip_download': True,
+            }
+        },
         # embed.ly video
         {
             'url': 'http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/',
@@ -2293,6 +2306,7 @@ class GenericIE(InfoExtractor):
         # Look for Ooyala videos
         mobj = (re.search(r'player\.ooyala\.com/[^"?]+[?#][^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
                 re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
+                re.search(r'OO\.Player\.create\.apply\(\s*OO\.Player\s*,\s*op\(\s*\[\s*[\'"][^\'"]*[\'"]\s*,\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
                 re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage) or
                 re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))
         if mobj is not None:

From a0a477b885dc1dd688058924357c4935f3c935cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Jul 2017 15:48:22 +0700
Subject: [PATCH 606/611] [youjizz] Fix extraction (closes #13744)

---
 youtube_dl/extractor/youjizz.py | 80 ++++++++++++++++++++++++++++-----
 1 file changed, 68 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/youjizz.py b/youtube_dl/extractor/youjizz.py
index b50f34e9b..f33fabe19 100644
--- a/youtube_dl/extractor/youjizz.py
+++ b/youtube_dl/extractor/youjizz.py
@@ -1,39 +1,95 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    parse_duration,
+)
 
 
 class YouJizzIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:\w+\.)?youjizz\.com/videos/(?:[^/#?]+)?-(?P<id>[0-9]+)\.html(?:$|[?#])'
+    _VALID_URL = r'https?://(?:\w+\.)?youjizz\.com/videos/(?:[^/#?]*-(?P<id>\d+)\.html|embed/(?P<embed_id>\d+))'
     _TESTS = [{
         'url': 'http://www.youjizz.com/videos/zeichentrick-1-2189178.html',
-        'md5': '78fc1901148284c69af12640e01c6310',
+        'md5': 'b1e1dfaa8bb9537d8b84eeda9cf4acf4',
         'info_dict': {
             'id': '2189178',
             'ext': 'mp4',
             'title': 'Zeichentrick 1',
             'age_limit': 18,
+            'duration': 2874,
         }
     }, {
         'url': 'http://www.youjizz.com/videos/-2189178.html',
         'only_matching': True,
+    }, {
+        'url': 'https://www.youjizz.com/videos/embed/31991001',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        # YouJizz's HTML5 player has invalid HTML
-        webpage = webpage.replace('"controls', '" controls')
-        age_limit = self._rta_search(webpage)
-        video_title = self._html_search_regex(
-            r'<title>\s*(.*)\s*</title>', webpage, 'title')
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id') or mobj.group('embed_id')
 
-        info_dict = self._parse_html5_media_entries(url, webpage, video_id)[0]
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(
+            r'<title>(.+?)</title>', webpage, 'title')
+
+        formats = []
+
+        encodings = self._parse_json(
+            self._search_regex(
+                r'encodings\s*=\s*(\[.+?\]);\n', webpage, 'encodings',
+                default='[]'),
+            video_id, fatal=False)
+        for encoding in encodings:
+            if not isinstance(encoding, dict):
+                continue
+            format_url = encoding.get('filename')
+            if not isinstance(format_url, compat_str):
+                continue
+            if determine_ext(format_url) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False))
+            else:
+                format_id = encoding.get('name') or encoding.get('quality')
+                height = int_or_none(self._search_regex(
+                    r'^(\d+)[pP]', format_id, 'height', default=None))
+                formats.append({
+                    'url': format_url,
+                    'format_id': format_id,
+                    'height': height,
+                })
+
+        if formats:
+            info_dict = {
+                'formats': formats,
+            }
+        else:
+            # YouJizz's HTML5 player has invalid HTML
+            webpage = webpage.replace('"controls', '" controls')
+            info_dict = self._parse_html5_media_entries(
+                url, webpage, video_id)[0]
+
+        duration = parse_duration(self._search_regex(
+            r'<strong>Runtime:</strong>([^<]+)', webpage, 'duration',
+            default=None))
+        uploader = self._search_regex(
+            r'<strong>Uploaded By:.*?<a[^>]*>([^<]+)', webpage, 'uploader',
+            default=None)
 
         info_dict.update({
             'id': video_id,
-            'title': video_title,
-            'age_limit': age_limit,
+            'title': title,
+            'age_limit': self._rta_search(webpage),
+            'duration': duration,
+            'uploader': uploader,
         })
 
         return info_dict

From 0ed4758023ddfb4d9630ba9114ef70ef7e6ac09d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Jul 2017 19:08:44 +0700
Subject: [PATCH 607/611] [clipfish] Remove extractor

---
 youtube_dl/extractor/clipfish.py   | 67 ------------------------------
 youtube_dl/extractor/extractors.py |  1 -
 2 files changed, 68 deletions(-)
 delete mode 100644 youtube_dl/extractor/clipfish.py

diff --git a/youtube_dl/extractor/clipfish.py b/youtube_dl/extractor/clipfish.py
deleted file mode 100644
index 0920f6219..000000000
--- a/youtube_dl/extractor/clipfish.py
+++ /dev/null
@@ -1,67 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    unified_strdate,
-)
-
-
-class ClipfishIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?clipfish\.de/(?:[^/]+/)+video/(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'http://www.clipfish.de/special/ugly-americans/video/4343170/s01-e01-ugly-americans-date-in-der-hoelle/',
-        'md5': 'b9a5dc46294154c1193e2d10e0c95693',
-        'info_dict': {
-            'id': '4343170',
-            'ext': 'mp4',
-            'title': 'S01 E01 - Ugly Americans - Date in der Hölle',
-            'description': 'Mark Lilly arbeitet im Sozialdienst der Stadt New York und soll Immigranten bei ihrer Einbürgerung in die USA zur Seite stehen.',
-            'upload_date': '20161005',
-            'duration': 1291,
-            'view_count': int,
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        video_info = self._download_json(
-            'http://www.clipfish.de/devapi/id/%s?format=json&apikey=hbbtv' % video_id,
-            video_id)['items'][0]
-
-        formats = []
-
-        m3u8_url = video_info.get('media_videourl_hls')
-        if m3u8_url:
-            formats.append({
-                'url': m3u8_url.replace('de.hls.fra.clipfish.de', 'hls.fra.clipfish.de'),
-                'ext': 'mp4',
-                'format_id': 'hls',
-            })
-
-        mp4_url = video_info.get('media_videourl')
-        if mp4_url:
-            formats.append({
-                'url': mp4_url,
-                'format_id': 'mp4',
-                'width': int_or_none(video_info.get('width')),
-                'height': int_or_none(video_info.get('height')),
-                'tbr': int_or_none(video_info.get('bitrate')),
-            })
-
-        descr = video_info.get('descr')
-        if descr:
-            descr = descr.strip()
-
-        return {
-            'id': video_id,
-            'title': video_info['title'],
-            'description': descr,
-            'formats': formats,
-            'thumbnail': video_info.get('media_content_thumbnail_large') or video_info.get('media_thumbnail'),
-            'duration': int_or_none(video_info.get('media_length')),
-            'upload_date': unified_strdate(video_info.get('pubDate')),
-            'view_count': int_or_none(video_info.get('media_views'))
-        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d2c5e8030..bdc7370cd 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -186,7 +186,6 @@ from .chirbit import (
 )
 from .cinchcast import CinchcastIE
 from .cjsw import CJSWIE
-from .clipfish import ClipfishIE
 from .cliphunter import CliphunterIE
 from .cliprs import ClipRsIE
 from .clipsyndicate import ClipsyndicateIE

From 8b9f50d7cb4cfab5d505f4233c3e176a8106d6db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Jul 2017 19:09:44 +0700
Subject: [PATCH 608/611] [watchbox] Add extractor (#13739)

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/watchbox.py   | 151 +++++++++++++++++++++++++++++
 2 files changed, 152 insertions(+)
 create mode 100644 youtube_dl/extractor/watchbox.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index bdc7370cd..3489e86f0 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1244,6 +1244,7 @@ from .washingtonpost import (
     WashingtonPostArticleIE,
 )
 from .wat import WatIE
+from .watchbox import WatchBoxIE
 from .watchindianporn import WatchIndianPornIE
 from .wdr import (
     WDRIE,
diff --git a/youtube_dl/extractor/watchbox.py b/youtube_dl/extractor/watchbox.py
new file mode 100644
index 000000000..b382338fa
--- /dev/null
+++ b/youtube_dl/extractor/watchbox.py
@@ -0,0 +1,151 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    js_to_json,
+    strip_or_none,
+    try_get,
+    unified_timestamp,
+)
+
+
+class WatchBoxIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?watchbox\.de/(?P<kind>serien|filme)/(?:[^/]+/)*[^/]+-(?P<id>\d+)'
+    _TESTS = [{
+        # film
+        'url': 'https://www.watchbox.de/filme/free-jimmy-12325.html',
+        'info_dict': {
+            'id': '341368',
+            'ext': 'mp4',
+            'title': 'Free Jimmy',
+            'description': 'md5:bcd8bafbbf9dc0ef98063d344d7cc5f6',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 4890,
+            'age_limit': 16,
+            'release_year': 2009,
+        },
+        'params': {
+            'format': 'bestvideo',
+            'skip_download': True,
+        },
+        'expected_warnings': ['Failed to download m3u8 information'],
+    }, {
+        # episode
+        'url': 'https://www.watchbox.de/serien/ugly-americans-12231/staffel-1/date-in-der-hoelle-328286.html',
+        'info_dict': {
+            'id': '328286',
+            'ext': 'mp4',
+            'title': 'S01 E01 - Date in der Hölle',
+            'description': 'md5:2f31c74a8186899f33cb5114491dae2b',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 1291,
+            'age_limit': 12,
+            'release_year': 2010,
+            'series': 'Ugly Americans',
+            'season_number': 1,
+            'episode': 'Date in der Hölle',
+            'episode_number': 1,
+        },
+        'params': {
+            'format': 'bestvideo',
+            'skip_download': True,
+        },
+        'expected_warnings': ['Failed to download m3u8 information'],
+    }, {
+        'url': 'https://www.watchbox.de/serien/ugly-americans-12231/staffel-2/der-ring-des-powers-328270',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        kind, video_id = mobj.group('kind', 'id')
+
+        webpage = self._download_webpage(url, video_id)
+
+        source = self._parse_json(
+            self._search_regex(
+                r'(?s)source\s*:\s*({.+?})\s*,\s*\n', webpage, 'source',
+                default='{}'),
+            video_id, transform_source=js_to_json, fatal=False) or {}
+
+        video_id = compat_str(source.get('videoId') or video_id)
+
+        devapi = self._download_json(
+            'http://api.watchbox.de/devapi/id/%s' % video_id, video_id, query={
+                'format': 'json',
+                'apikey': 'hbbtv',
+            }, fatal=False)
+
+        item = try_get(devapi, lambda x: x['items'][0], dict) or {}
+
+        title = item.get('title') or try_get(
+            item, lambda x: x['movie']['headline_movie'],
+            compat_str) or source['title']
+
+        formats = []
+        hls_url = item.get('media_videourl_hls') or source.get('hls')
+        if hls_url:
+            formats.extend(self._extract_m3u8_formats(
+                hls_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                m3u8_id='hls', fatal=False))
+        dash_url = item.get('media_videourl_wv') or source.get('dash')
+        if dash_url:
+            formats.extend(self._extract_mpd_formats(
+                dash_url, video_id, mpd_id='dash', fatal=False))
+        mp4_url = item.get('media_videourl')
+        if mp4_url:
+            formats.append({
+                'url': mp4_url,
+                'format_id': 'mp4',
+                'width': int_or_none(item.get('width')),
+                'height': int_or_none(item.get('height')),
+                'tbr': int_or_none(item.get('bitrate')),
+            })
+        self._sort_formats(formats)
+
+        description = strip_or_none(item.get('descr'))
+        thumbnail = item.get('media_content_thumbnail_large') or source.get('poster') or item.get('media_thumbnail')
+        duration = int_or_none(item.get('media_length') or source.get('length'))
+        timestamp = unified_timestamp(item.get('pubDate'))
+        view_count = int_or_none(item.get('media_views'))
+        age_limit = int_or_none(try_get(item, lambda x: x['movie']['fsk']))
+        release_year = int_or_none(try_get(item, lambda x: x['movie']['rel_year']))
+
+        info = {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'timestamp': timestamp,
+            'view_count': view_count,
+            'age_limit': age_limit,
+            'release_year': release_year,
+            'formats': formats,
+        }
+
+        if kind.lower() == 'serien':
+            series = try_get(
+                item, lambda x: x['special']['title'],
+                compat_str) or source.get('format')
+            season_number = int_or_none(self._search_regex(
+                r'^S(\d{1,2})\s*E\d{1,2}', title, 'season number',
+                default=None) or self._search_regex(
+                    r'/staffel-(\d+)/', url, 'season number', default=None))
+            episode = source.get('title')
+            episode_number = int_or_none(self._search_regex(
+                r'^S\d{1,2}\s*E(\d{1,2})', title, 'episode number',
+                default=None))
+            info.update({
+                'series': series,
+                'season_number': season_number,
+                'episode': episode,
+                'episode_number': episode_number,
+            })
+
+        return info

From f701827e319963ca783d012f3647aa44fc0efcd6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Jul 2017 19:43:09 +0700
Subject: [PATCH 609/611] [ChangeLog] Actualize

---
 ChangeLog | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 8e63b5c11..ca3ee8a93 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,28 @@
 version <unreleased>
 
 Core
+* [downloader/hls] Use redirect URL as manifest base (#13755)
 * [options] Correctly hide login info from debug outputs (#13696)
 
+Extractors
++ [watchbox] Add support for watchbox.de (#13739)
+- [clipfish] Remove extractor
++ [youjizz] Fix extraction (#13744)
++ [generic] Add support for another ooyala embed pattern (#13727)
++ [ard] Add support for lives (#13771)
+* [soundcloud] Update client id
++ [soundcloud:trackstation] Add support for track stations (#13733)
+* [svtplay] Use geo verification proxy for API request
+* [svtplay] Update API URL (#13767)
++ [yandexdisk] Add support for yadi.sk (#13755)
++ [megaphone] Add support for megaphone.fm
+* [amcnetworks] Make rating optional (#12453)
+* [cloudy] Fix extraction (#13737)
++ [nickru] Add support for nickelodeon.ru
+* [mtv] Improve thumbnal extraction
+* [nick] Automate geo-restriction bypass (#13711)
+* [niconico] Improve error reporting (#13696)
+
 
 version 2017.07.23
 

From 5c9ea67bc0dedd15a0ed9ad05d8fcf09946ca461 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Jul 2017 20:47:31 +0700
Subject: [PATCH 610/611] release 2017.07.30.1

---
 .github/ISSUE_TEMPLATE.md | 16 ++++++++--------
 ChangeLog                 |  2 +-
 docs/supportedsites.md    |  6 +++++-
 youtube_dl/version.py     |  2 +-
 4 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 37d09d796..0421de755 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -1,16 +1,16 @@
 ## Please follow the guide below
 
 - You will be asked some questions and requested to provide some information, please read them **carefully** and answer honestly
-- Put an `x` into all the boxes [ ] relevant to your *issue* (like that [x])
-- Use *Preview* tab to see how your issue will actually look like
+- Put an `x` into all the boxes [ ] relevant to your *issue* (like this: `[x]`)
+- Use the *Preview* tab to see what your issue will actually look like
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.07.23*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.07.23**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.07.30.1*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.07.30.1**
 
 ### Before submitting an *issue* make sure you have:
-- [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
+- [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
 - [ ] [Searched](https://github.com/rg3/youtube-dl/search?type=Issues) the bugtracker for similar issues including closed ones
 
 ### What is the purpose of your *issue*?
@@ -28,14 +28,14 @@
 
 ### If the purpose of this *issue* is a *bug report*, *site support request* or you are not completely sure provide the full verbose output as follows:
 
-Add `-v` flag to **your command line** you run youtube-dl with, copy the **whole** output and insert it here. It should look similar to one below (replace it with **your** log inserted between triple ```):
+Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl -v <your command line>`), copy the **whole** output and insert it here. It should look similar to one below (replace it with **your** log inserted between triple ```):
+
 ```
-$ youtube-dl -v <your command line>
 [debug] System config: []
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.07.23
+[debug] youtube-dl version 2017.07.30.1
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index ca3ee8a93..4f03ef064 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.07.30.1
 
 Core
 * [downloader/hls] Use redirect URL as manifest base (#13755)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index eb09c470c..77aac8249 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -155,7 +155,6 @@
  - **chirbit:profile**
  - **Cinchcast**
  - **CJSW**
- - **Clipfish**
  - **cliphunter**
  - **ClipRs**
  - **Clipsyndicate**
@@ -440,6 +439,7 @@
  - **Medialaan**
  - **Mediaset**
  - **Medici**
+ - **megaphone.fm**: megaphone.fm embedded players
  - **Meipai**: 美拍
  - **MelonVOD**
  - **META**
@@ -533,6 +533,7 @@
  - **nhl.com:videocenter:category**: NHL videocenter category
  - **nick.com**
  - **nick.de**
+ - **nickelodeonru**
  - **nicknight**
  - **niconico**: ニコニコ動画
  - **NiconicoPlaylist**
@@ -734,6 +735,7 @@
  - **soundcloud:playlist**
  - **soundcloud:search**: Soundcloud search
  - **soundcloud:set**
+ - **soundcloud:trackstation**
  - **soundcloud:user**
  - **soundgasm**
  - **soundgasm:profile**
@@ -968,6 +970,7 @@
  - **washingtonpost**
  - **washingtonpost:article**
  - **wat.tv**
+ - **WatchBox**
  - **WatchIndianPorn**: Watch Indian Porn
  - **WDR**
  - **wdr:mobile**
@@ -1003,6 +1006,7 @@
  - **XVideos**
  - **XXXYMovies**
  - **Yahoo**: Yahoo screen and movies
+ - **YandexDisk**
  - **yandexmusic:album**: Яндекс.Музыка - Альбом
  - **yandexmusic:playlist**: Яндекс.Музыка - Плейлист
  - **yandexmusic:track**: Яндекс.Музыка - Трек
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a8dbb93e3..38162157d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.07.23'
+__version__ = '2017.07.30.1'

From 9118c9f18a43a2f3e4814fcc02ac8e5180077df3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Aug 2017 05:20:14 +0700
Subject: [PATCH 611/611] [nrktv] Update API host (closes #13796)

---
 youtube_dl/extractor/nrk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 3b4f51f61..18ead9426 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -237,7 +237,7 @@ class NRKTVIE(NRKBaseIE):
                             (?:/\d{2}-\d{2}-\d{4})?
                             (?:\#del=(?P<part_id>\d+))?
                     ''' % _EPISODE_RE
-    _API_HOST = 'psapi-we.nrk.no'
+    _API_HOST = 'psapi-ne.nrk.no'
 
     _TESTS = [{
         'url': 'https://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014',