Merge remote-tracking branch 'upstream/master'

2025-02-03 21:12:53 +08:00 · 2017-11-25 02:19:39 +01:00 · 2017-11-25 02:19:39 +01:00 · 3c211cff37
commit 3c211cff37
parent 8e189bb1f9 cd9ff4ec5b
44 changed files with 635 additions and 269 deletions
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@ -6,8 +6,8 @@

 ---

-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.29*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.29**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.11.15*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.11.15**

 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.10.29
+[debug] youtube-dl version 2017.11.15
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
--- a/46
+++ b/46
@ -1,3 +1,49 @@
+version <unreleased>
+
+Extractors
+* [youku] Fix extraction; update ccode (#14815)
+ [JWPlatform] support iframes (#14828)
+
+version 2017.11.15
+
+Core
+* [common] Skip Apple FairPlay m3u8 manifests (#14741)
+* [YoutubeDL] Fix playlist range optimization for --playlist-items (#14740)
+
+Extractors
+* [vshare] Capture and output error message
+* [vshare] Fix extraction (#14473)
+* [crunchyroll] Extract old RTMP formats
+* [tva] Fix extraction (#14736)
+* [gamespot] Lower preference of HTTP formats (#14652)
+* [instagram:user] Fix extraction (#14699)
+* [ccma] Fix typo (#14730)
+- Remove sensitive data from logging in messages
+* [instagram:user] Fix extraction (#14699)
+ [gamespot] Add support for article URLs (#14652)
+* [gamespot] Skip Brightcove Once HTTP formats (#14652)
+* [cartoonnetwork] Update tokenizer_src (#14666)
+ [wsj] Recognize another URL pattern (#14704)
+* [pandatv] Update API URL and sign format URLs (#14693)
+* [crunchyroll] Use old login method (#11572)
+
+
+version 2017.11.06
+
+Core
+ [extractor/common] Add protocol for f4m formats
+* [f4m] Prefer baseURL for relative URLs (#14660)
+* [extractor/common] Respect URL query in _extract_wowza_formats (14645)
+
+Extractors
+ [hotstar:playlist] Add support for playlists (#12465)
+* [hotstar] Bypass geo restriction (#14672)
+- [22tracks] Remove extractor (#11024, #14628)
+ [skysport] Sdd support ooyala videos protected with embed_token (#14641)
+* [gamespot] Extract formats referenced with new data fields (#14652)
+* [spankbang] Detect unavailable videos (#14644)
+
+
 version 2017.10.29

 Core
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@ -3,8 +3,6 @@
 - **1up.com**
 - **20min**
 - **220.ro**
- - **22tracks:genre**
- - **22tracks:track**
 - **24video**
 - **3qsdn**: 3Q SDN
 - **3sat**
@ -342,6 +340,7 @@
 - **HornBunny**
 - **HotNewHipHop**
 - **HotStar**
+ - **hotstar:playlist**
 - **Howcast**
 - **HowStuffWorks**
 - **HRTi**
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@ -948,7 +948,8 @@ class YoutubeDL(object):
                report_download(n_entries)
            else:  # iterable
                if playlistitems:
-                    entries = make_playlistitems_entries(list(ie_entries))
+                    entries = make_playlistitems_entries(list(itertools.islice(
+                        ie_entries, 0, max(playlistitems))))
                else:
                    entries = list(itertools.islice(
                        ie_entries, playliststart, playlistend))
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@ -78,7 +78,7 @@ class AnimeOnDemandIE(InfoExtractor):
            post_url = urljoin(self._LOGIN_URL, post_url)

        response = self._download_webpage(
-            post_url, None, 'Logging in as %s' % username,
+            post_url, None, 'Logging in',
            data=urlencode_postdata(login_form), headers={
                'Referer': self._LOGIN_URL,
            })
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@ -87,7 +87,7 @@ class AtresPlayerIE(InfoExtractor):
            self._LOGIN_URL, urlencode_postdata(login_form))
        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
        response = self._download_webpage(
-            request, None, 'Logging in as %s' % username)
+            request, None, 'Logging in')

        error = self._html_search_regex(
            r'(?s)<ul[^>]+class="[^"]*\blist_error\b[^"]*">(.+?)</ul>',
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@ -59,7 +59,7 @@ class BambuserIE(InfoExtractor):
            self._LOGIN_URL, urlencode_postdata(login_form))
        request.add_header('Referer', self._LOGIN_URL)
        response = self._download_webpage(
-            request, None, 'Logging in as %s' % username)
+            request, None, 'Logging in')

        login_error = self._html_search_regex(
            r'(?s)<div class="messages error">(.+?)</div>',
--- a/youtube_dl/extractor/cartoonnetwork.py
+++ b/youtube_dl/extractor/cartoonnetwork.py
@ -31,7 +31,7 @@ class CartoonNetworkIE(TurnerBaseIE):
            'http://www.cartoonnetwork.com/video-seo-svc/episodeservices/getCvpPlaylist?networkName=CN2&' + query, video_id, {
                'secure': {
                    'media_src': 'http://androidhls-secure.cdn.turner.com/toon/big',
-                    'tokenizer_src': 'http://www.cartoonnetwork.com/cntv/mvpd/processors/services/token_ipadAdobe.do',
+                    'tokenizer_src': 'https://token.vgtf.net/token/token_mobile',
                },
            }, {
                'url': url,
--- a/youtube_dl/extractor/ccma.py
+++ b/youtube_dl/extractor/ccma.py
@ -93,7 +93,7 @@ class CCMAIE(InfoExtractor):
            'description': clean_html(informacio.get('descripcio')),
            'duration': duration,
            'timestamp': timestamp,
-            'thumnails': thumbnails,
+            'thumbnails': thumbnails,
            'subtitles': subtitles,
            'formats': formats,
        }
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@ -592,19 +592,11 @@ class InfoExtractor(object):
        if not encoding:
            encoding = self._guess_encoding_from_content(content_type, webpage_bytes)
        if self._downloader.params.get('dump_intermediate_pages', False):
-            try:
-                url = url_or_request.get_full_url()
-            except AttributeError:
-                url = url_or_request
-            self.to_screen('Dumping request to ' + url)
+            self.to_screen('Dumping request to ' + urlh.geturl())
            dump = base64.b64encode(webpage_bytes).decode('ascii')
            self._downloader.to_screen(dump)
        if self._downloader.params.get('write_pages', False):
-            try:
-                url = url_or_request.get_full_url()
-            except AttributeError:
-                url = url_or_request
-            basen = '%s_%s' % (video_id, url)
+            basen = '%s_%s' % (video_id, urlh.geturl())
            if len(basen) > 240:
                h = '___' + hashlib.md5(basen.encode('utf-8')).hexdigest()
                basen = basen[:240 - len(h)] + h
@ -1356,6 +1348,9 @@ class InfoExtractor(object):
        if '#EXT-X-FAXS-CM:' in m3u8_doc:  # Adobe Flash Access
            return []

+        if re.search(r'#EXT-X-SESSION-KEY:.*?URI="skd://', m3u8_doc):  # Apple FairPlay
+            return []
+
        formats = []

        format_url = lambda u: (
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@ -38,11 +38,32 @@ class CrunchyrollBaseIE(InfoExtractor):
    _LOGIN_FORM = 'login_form'
    _NETRC_MACHINE = 'crunchyroll'

+    def _call_rpc_api(self, method, video_id, note=None, data=None):
+        data = data or {}
+        data['req'] = 'RpcApi' + method
+        data = compat_urllib_parse_urlencode(data).encode('utf-8')
+        return self._download_xml(
+            'http://www.crunchyroll.com/xml/',
+            video_id, note, fatal=False, data=data, headers={
+                'Content-Type': 'application/x-www-form-urlencoded',
+            })
+
    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return

+        self._download_webpage(
+            'https://www.crunchyroll.com/?a=formhandler',
+            None, 'Logging in', 'Wrong login info',
+            data=urlencode_postdata({
+                'formname': 'RpcApiUser_Login',
+                'next_url': 'https://www.crunchyroll.com/acct/membership',
+                'name': username,
+                'password': password,
+            }))
+
+        '''
        login_page = self._download_webpage(
            self._LOGIN_URL, None, 'Downloading login page')

@ -86,6 +107,7 @@ class CrunchyrollBaseIE(InfoExtractor):
            raise ExtractorError('Unable to login: %s' % error, expected=True)

        raise ExtractorError('Unable to log in')
+        '''

    def _real_initialize(self):
        self._login()
@ -365,15 +387,19 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
    def _get_subtitles(self, video_id, webpage):
        subtitles = {}
        for sub_id, sub_name in re.findall(r'\bssid=([0-9]+)"[^>]+?\btitle="([^"]+)', webpage):
-            sub_page = self._download_webpage(
-                'http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id=' + sub_id,
-                video_id, note='Downloading subtitles for ' + sub_name)
-            id = self._search_regex(r'id=\'([0-9]+)', sub_page, 'subtitle_id', fatal=False)
-            iv = self._search_regex(r'<iv>([^<]+)', sub_page, 'subtitle_iv', fatal=False)
-            data = self._search_regex(r'<data>([^<]+)', sub_page, 'subtitle_data', fatal=False)
-            if not id or not iv or not data:
+            sub_doc = self._call_rpc_api(
+                'Subtitle_GetXml', video_id,
+                'Downloading subtitles for ' + sub_name, data={
+                    'subtitle_script_id': sub_id,
+                })
+            if not sub_doc:
                continue
-            subtitle = self._decrypt_subtitles(data, iv, id).decode('utf-8')
+            sid = sub_doc.get('id')
+            iv = xpath_text(sub_doc, 'iv', 'subtitle iv')
+            data = xpath_text(sub_doc, 'data', 'subtitle data')
+            if not sid or not iv or not data:
+                continue
+            subtitle = self._decrypt_subtitles(data, iv, sid).decode('utf-8')
            lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False)
            if not lang_code:
                continue
@ -444,65 +470,79 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
        for fmt in available_fmts:
            stream_quality, stream_format = self._FORMAT_IDS[fmt]
            video_format = fmt + 'p'
-            streamdata_req = sanitized_Request(
-                'http://www.crunchyroll.com/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=%s&video_format=%s&video_quality=%s'
-                % (video_id, stream_format, stream_quality),
-                compat_urllib_parse_urlencode({'current_page': url}).encode('utf-8'))
-            streamdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
-            streamdata = self._download_xml(
-                streamdata_req, video_id,
-                note='Downloading media info for %s' % video_format)
-            stream_info = streamdata.find('./{default}preload/stream_info')
-            video_encode_id = xpath_text(stream_info, './video_encode_id')
-            if video_encode_id in video_encode_ids:
-                continue
-            video_encode_ids.append(video_encode_id)
+            stream_infos = []
+            streamdata = self._call_rpc_api(
+                'VideoPlayer_GetStandardConfig', video_id,
+                'Downloading media info for %s' % video_format, data={
+                    'media_id': video_id,
+                    'video_format': stream_format,
+                    'video_quality': stream_quality,
+                    'current_page': url,
+                })
+            if streamdata:
+                stream_info = streamdata.find('./{default}preload/stream_info')
+                if stream_info:
+                    stream_infos.append(stream_info)
+            stream_info = self._call_rpc_api(
+                'VideoEncode_GetStreamInfo', video_id,
+                'Downloading stream info for %s' % video_format, data={
+                    'media_id': video_id,
+                    'video_format': stream_format,
+                    'video_encode_quality': stream_quality,
+                })
+            if stream_info:
+                stream_infos.append(stream_info)
+            for stream_info in stream_infos:
+                video_encode_id = xpath_text(stream_info, './video_encode_id')
+                if video_encode_id in video_encode_ids:
+                    continue
+                video_encode_ids.append(video_encode_id)

-            video_file = xpath_text(stream_info, './file')
-            if not video_file:
-                continue
-            if video_file.startswith('http'):
-                formats.extend(self._extract_m3u8_formats(
-                    video_file, video_id, 'mp4', entry_protocol='m3u8_native',
-                    m3u8_id='hls', fatal=False))
-                continue
-
-            video_url = xpath_text(stream_info, './host')
-            if not video_url:
-                continue
-            metadata = stream_info.find('./metadata')
-            format_info = {
-                'format': video_format,
-                'format_id': video_format,
-                'height': int_or_none(xpath_text(metadata, './height')),
-                'width': int_or_none(xpath_text(metadata, './width')),
-            }
-
-            if '.fplive.net/' in video_url:
-                video_url = re.sub(r'^rtmpe?://', 'http://', video_url.strip())
-                parsed_video_url = compat_urlparse.urlparse(video_url)
-                direct_video_url = compat_urlparse.urlunparse(parsed_video_url._replace(
-                    netloc='v.lvlt.crcdn.net',
-                    path='%s/%s' % (remove_end(parsed_video_url.path, '/'), video_file.split(':')[-1])))
-                if self._is_valid_url(direct_video_url, video_id, video_format):
-                    format_info.update({
-                        'url': direct_video_url,
-                    })
-                    formats.append(format_info)
+                video_file = xpath_text(stream_info, './file')
+                if not video_file:
+                    continue
+                if video_file.startswith('http'):
+                    formats.extend(self._extract_m3u8_formats(
+                        video_file, video_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id='hls', fatal=False))
                    continue

-            format_info.update({
-                'url': video_url,
-                'play_path': video_file,
-                'ext': 'flv',
-            })
-            formats.append(format_info)
-        self._sort_formats(formats)
+                video_url = xpath_text(stream_info, './host')
+                if not video_url:
+                    continue
+                metadata = stream_info.find('./metadata')
+                format_info = {
+                    'format': video_format,
+                    'height': int_or_none(xpath_text(metadata, './height')),
+                    'width': int_or_none(xpath_text(metadata, './width')),
+                }

-        metadata = self._download_xml(
-            'http://www.crunchyroll.com/xml', video_id,
-            note='Downloading media info', query={
-                'req': 'RpcApiVideoPlayer_GetMediaMetadata',
+                if '.fplive.net/' in video_url:
+                    video_url = re.sub(r'^rtmpe?://', 'http://', video_url.strip())
+                    parsed_video_url = compat_urlparse.urlparse(video_url)
+                    direct_video_url = compat_urlparse.urlunparse(parsed_video_url._replace(
+                        netloc='v.lvlt.crcdn.net',
+                        path='%s/%s' % (remove_end(parsed_video_url.path, '/'), video_file.split(':')[-1])))
+                    if self._is_valid_url(direct_video_url, video_id, video_format):
+                        format_info.update({
+                            'format_id': 'http-' + video_format,
+                            'url': direct_video_url,
+                        })
+                        formats.append(format_info)
+                        continue
+
+                format_info.update({
+                    'format_id': 'rtmp-' + video_format,
+                    'url': video_url,
+                    'play_path': video_file,
+                    'ext': 'flv',
+                })
+                formats.append(format_info)
+        self._sort_formats(formats, ('height', 'width', 'tbr', 'fps'))
+
+        metadata = self._call_rpc_api(
+            'VideoPlayer_GetMediaMetadata', video_id,
+            note='Downloading media info', data={
                'media_id': video_id,
            })

--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@ -54,7 +54,7 @@ class DramaFeverBaseIE(AMPIE):
        request = sanitized_Request(
            self._LOGIN_URL, urlencode_postdata(login_form))
        response = self._download_webpage(
-            request, None, 'Logging in as %s' % username)
+            request, None, 'Logging in')

        if all(logout_pattern not in response
               for logout_pattern in ['href="/accounts/logout/"', '>Log out<']):
--- a/youtube_dl/extractor/drtuber.py
+++ b/youtube_dl/extractor/drtuber.py
@ -10,7 +10,7 @@ from ..utils import (


 class DrTuberIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?drtuber\.com/(?:video|embed)/(?P<id>\d+)(?:/(?P<display_id>[\w-]+))?'
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?drtuber\.com/(?:video|embed)/(?P<id>\d+)(?:/(?P<display_id>[\w-]+))?'
    _TESTS = [{
        'url': 'http://www.drtuber.com/video/1740434/hot-perky-blonde-naked-golf',
        'md5': '93e680cf2536ad0dfb7e74d94a89facd',
@ -28,6 +28,9 @@ class DrTuberIE(InfoExtractor):
    }, {
        'url': 'http://www.drtuber.com/embed/489939',
        'only_matching': True,
+    }, {
+        'url': 'http://m.drtuber.com/video/3893529/lingerie-blowjob-from-beautiful-teen',
+        'only_matching': True,
    }]

    @staticmethod
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@ -348,7 +348,6 @@ from .firstpost import FirstpostIE
 from .firsttv import FirstTVIE
 from .fivemin import FiveMinIE
 from .fivetv import FiveTVIE
-from .fktv import FKTVIE
 from .flickr import FlickrIE
 from .flipagram import FlipagramIE
 from .folketinget import FolketingetIE
@ -432,7 +431,10 @@ from .hitbox import HitboxIE, HitboxLiveIE
 from .hitrecord import HitRecordIE
 from .hornbunny import HornBunnyIE
 from .hotnewhiphop import HotNewHipHopIE
-from .hotstar import HotStarIE
+from .hotstar import (
+    HotStarIE,
+    HotStarPlaylistIE,
+)
 from .howcast import HowcastIE
 from .howstuffworks import HowStuffWorksIE
 from .hrti import (
@ -569,6 +571,7 @@ from .mangomolo import (
    MangomoloLiveIE,
 )
 from .manyvids import ManyVidsIE
+from .massengeschmacktv import MassengeschmackTVIE
 from .matchtv import MatchTVIE
 from .mdr import MDRIE
 from .mediaset import MediasetIE
@ -786,6 +789,7 @@ from .patreon import PatreonIE
 from .pbs import PBSIE
 from .pearvideo import PearVideoIE
 from .people import PeopleIE
+from .performgroup import PerformGroupIE
 from .periscope import (
    PeriscopeIE,
    PeriscopeUserIE,
--- a/youtube_dl/extractor/faz.py
+++ b/youtube_dl/extractor/faz.py
@ -1,7 +1,10 @@
 # coding: utf-8
 from __future__ import unicode_literals

+import re
+
 from .common import InfoExtractor
+from ..compat import compat_etree_fromstring
 from ..utils import (
    xpath_element,
    xpath_text,
@ -43,10 +46,15 @@ class FazIE(InfoExtractor):

        webpage = self._download_webpage(url, video_id)
        description = self._og_search_description(webpage)
-        config_xml_url = self._search_regex(
-            r'videoXMLURL\s*=\s*"([^"]+)', webpage, 'config xml url')
-        config = self._download_xml(
-            config_xml_url, video_id, 'Downloading config xml')
+        media = self._html_search_regex(
+            r"data-videojs-media='([^']+)",
+            webpage, 'media')
+        if media == 'extern':
+            perform_url = self._search_regex(
+                r"<iframe[^>]+?src='((?:http:)?//player\.performgroup\.com/eplayer/eplayer\.html#/?[0-9a-f]{26}\.[0-9a-z]{26})",
+                webpage, 'perform url')
+            return self.url_result(perform_url)
+        config = compat_etree_fromstring(media)

        encodings = xpath_element(config, 'ENCODINGS', 'encodings', True)
        formats = []
@ -55,12 +63,24 @@ class FazIE(InfoExtractor):
            if encoding is not None:
                encoding_url = xpath_text(encoding, 'FILENAME')
                if encoding_url:
-                    formats.append({
+                    tbr = xpath_text(encoding, 'AVERAGEBITRATE', 1000)
+                    if tbr:
+                        tbr = int_or_none(tbr.replace(',', '.'))
+                    f = {
                        'url': encoding_url,
                        'format_id': code.lower(),
                        'quality': pref,
-                        'tbr': int_or_none(xpath_text(encoding, 'AVERAGEBITRATE')),
-                    })
+                        'tbr': tbr,
+                        'vcodec': xpath_text(encoding, 'CODEC'),
+                    }
+                    mobj = re.search(r'(\d+)x(\d+)_(\d+)\.mp4', encoding_url)
+                    if mobj:
+                        f.update({
+                            'width': int(mobj.group(1)),
+                            'height': int(mobj.group(2)),
+                            'tbr': tbr or int(mobj.group(3)),
+                        })
+                    formats.append(f)
        self._sort_formats(formats)

        return {
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@ -1,51 +0,0 @@
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    clean_html,
-    determine_ext,
-    js_to_json,
-)
-
-
-class FKTVIE(InfoExtractor):
-    IE_NAME = 'fernsehkritik.tv'
-    _VALID_URL = r'https?://(?:www\.)?fernsehkritik\.tv/folge-(?P<id>[0-9]+)(?:/.*)?'
-
-    _TEST = {
-        'url': 'http://fernsehkritik.tv/folge-1',
-        'md5': '21f0b0c99bce7d5b524eb1b17b1c6d79',
-        'info_dict': {
-            'id': '1',
-            'ext': 'mp4',
-            'title': 'Folge 1 vom 10. April 2007',
-            'thumbnail': r're:^https?://.*\.jpg$',
-        },
-    }
-
-    def _real_extract(self, url):
-        episode = self._match_id(url)
-
-        webpage = self._download_webpage(
-            'http://fernsehkritik.tv/folge-%s/play' % episode, episode)
-        title = clean_html(self._html_search_regex(
-            '<h3>([^<]+)</h3>', webpage, 'title'))
-        thumbnail = self._search_regex(r'POSTER\s*=\s*"([^"]+)', webpage, 'thumbnail', fatal=False)
-        sources = self._parse_json(self._search_regex(r'(?s)MEDIA\s*=\s*(\[.+?\]);', webpage, 'media'), episode, js_to_json)
-
-        formats = []
-        for source in sources:
-            furl = source.get('src')
-            if furl:
-                formats.append({
-                    'url': furl,
-                    'format_id': determine_ext(furl),
-                })
-        self._sort_formats(formats)
-
-        return {
-            'id': episode,
-            'title': title,
-            'formats': formats,
-            'thumbnail': thumbnail,
-        }
--- a/youtube_dl/extractor/fox9.py
+++ b/youtube_dl/extractor/fox9.py
@ -2,7 +2,6 @@
 from __future__ import unicode_literals

 from .anvato import AnvatoIE
-from ..utils import js_to_json


 class FOX9IE(AnvatoIE):
@ -34,9 +33,9 @@ class FOX9IE(AnvatoIE):

        video_id = self._parse_json(
            self._search_regex(
-                r'AnvatoPlaylist\s*\(\s*(\[.+?\])\s*\)\s*;',
+                r"this\.videosJson\s*=\s*'(\[.+?\])';",
                webpage, 'anvato playlist'),
-            video_id, transform_source=js_to_json)[0]['video']
+            video_id)[0]['video']

        return self._get_anvato_videos(
            'anvato_epfox_app_web_prod_b3373168e12f423f41504f207000188daf88251b',
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@ -363,6 +363,6 @@ class CultureboxIE(FranceTVBaseInfoExtractor):
            raise ExtractorError('Video %s is not available' % name, expected=True)

        video_id, catalogue = self._search_regex(
-            r'"http://videos\.francetv\.fr/video/([^@]+@[^"]+)"', webpage, 'video id').split('@')
+            r'"https?://videos\.francetv\.fr/video/([^@]+@[^"]+)"', webpage, 'video id').split('@')

        return self._extract_video(video_id, catalogue)
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@ -57,7 +57,7 @@ class FunimationIE(InfoExtractor):
        try:
            data = self._download_json(
                'https://prod-api-funimationnow.dadcdigital.com/api/auth/login/',
-                None, 'Logging in as %s' % username, data=urlencode_postdata({
+                None, 'Logging in', data=urlencode_postdata({
                    'username': username,
                    'password': password,
                }))
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@ -14,7 +14,7 @@ from ..utils import (


 class GameSpotIE(OnceIE):
-    _VALID_URL = r'https?://(?:www\.)?gamespot\.com/videos/(?:[^/]+/\d+-|embed/)(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?gamespot\.com/(?:video|article)s/(?:[^/]+/\d+-|embed/)(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',
        'md5': 'b2a30deaa8654fcccd43713a6b6a4825',
@ -38,6 +38,9 @@ class GameSpotIE(OnceIE):
    }, {
        'url': 'https://www.gamespot.com/videos/embed/6439218/',
        'only_matching': True,
+    }, {
+        'url': 'https://www.gamespot.com/articles/the-last-of-us-2-receives-new-ps4-trailer/1100-6454469/',
+        'only_matching': True,
    }]

    def _real_extract(self, url):
@ -108,7 +111,8 @@ class GameSpotIE(OnceIE):
            onceux_url = self._parse_json(unescapeHTML(onceux_json), page_id).get('metadataUri')
            if onceux_url:
                formats.extend(self._extract_once_formats(re.sub(
-                    r'https?://[^/]+', 'http://once.unicornmedia.com', onceux_url)))
+                    r'https?://[^/]+', 'http://once.unicornmedia.com', onceux_url),
+                    http_formats_preference=-1))

        if not formats:
            for quality in ['sd', 'hd']:
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@ -102,6 +102,7 @@ from .joj import JojIE
 from .megaphone import MegaphoneIE
 from .vzaar import VzaarIE
 from .channel9 import Channel9IE
+from .vshare import VShareIE


 class GenericIE(InfoExtractor):
@ -1098,9 +1099,9 @@ class GenericIE(InfoExtractor):
        },
        # jwplayer rtmp
        {
-            'url': 'http://www.suffolk.edu/sjc/',
+            'url': 'http://www.suffolk.edu/sjc/live.php',
            'info_dict': {
-                'id': 'sjclive',
+                'id': 'live',
                'ext': 'flv',
                'title': 'Massachusetts Supreme Judicial Court Oral Arguments',
                'uploader': 'www.suffolk.edu',
@ -1108,7 +1109,7 @@ class GenericIE(InfoExtractor):
            'params': {
                'skip_download': True,
            },
-            'skip': 'does not contain a video anymore',
+            'skip': 'Only has video a few mornings per month, see http://www.suffolk.edu/sjc/',
        },
        # Complex jwplayer
        {
@ -1135,6 +1136,19 @@ class GenericIE(InfoExtractor):
                'skip_download': True,
            }
        },
+        {
+            # JWPlatform iframe
+            'url': 'https://www.mediaite.com/tv/dem-senator-claims-gary-cohn-faked-a-bad-connection-during-trump-call-to-get-him-off-the-phone/',
+            'md5': 'ca00a040364b5b439230e7ebfd02c4e9',
+            'info_dict': {
+                'id': 'O0c5JcKT',
+                'ext': 'mp4',
+                'upload_date': '20171122',
+                'timestamp': 1511366290,
+                'title': 'Dem Senator Claims Gary Cohn Faked a Bad Connection During Trump Call to Get Him Off the Phone',
+            },
+            'add_ie': [JWPlatformIE.ie_key()],
+        },
        {
            # Video.js embed, multiple formats
            'url': 'http://ortcam.com/solidworks-урок-6-настройка-чертежа_33f9b7351.html',
@ -1921,6 +1935,16 @@ class GenericIE(InfoExtractor):
                'title': 'Rescue Kit 14 Free Edition - Getting started',
            },
            'playlist_count': 4,
+        },
+        {
+            # vshare embed
+            'url': 'https://youtube-dl-demo.neocities.org/vshare.html',
+            'md5': '17b39f55b5497ae8b59f5fbce8e35886',
+            'info_dict': {
+                'id': '0f64ce6',
+                'title': 'vl14062007715967',
+                'ext': 'mp4',
+            }
        }
        # {
        #     # TODO: find another test
@ -2879,6 +2903,11 @@ class GenericIE(InfoExtractor):
            return self.playlist_from_matches(
                channel9_urls, video_id, video_title, ie=Channel9IE.ie_key())

+        vshare_urls = VShareIE._extract_urls(webpage)
+        if vshare_urls:
+            return self.playlist_from_matches(
+                vshare_urls, video_id, video_title, ie=VShareIE.ie_key())
+
        def merge_dicts(dict1, dict2):
            merged = {}
            for k, v in dict1.items():
--- a/youtube_dl/extractor/hotstar.py
+++ b/youtube_dl/extractor/hotstar.py
@ -1,22 +1,47 @@
 # coding: utf-8
 from __future__ import unicode_literals

+import re
+
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
-    ExtractorError,
    determine_ext,
+    ExtractorError,
    int_or_none,
 )


-class HotStarIE(InfoExtractor):
+class HotStarBaseIE(InfoExtractor):
+    _GEO_COUNTRIES = ['IN']
+
+    def _download_json(self, *args, **kwargs):
+        response = super(HotStarBaseIE, self)._download_json(*args, **kwargs)
+        if response['resultCode'] != 'OK':
+            if kwargs.get('fatal'):
+                raise ExtractorError(
+                    response['errorDescription'], expected=True)
+            return None
+        return response['resultObj']
+
+    def _download_content_info(self, content_id):
+        return self._download_json(
+            'https://account.hotstar.com/AVS/besc', content_id, query={
+                'action': 'GetAggregatedContentDetails',
+                'appVersion': '5.0.40',
+                'channel': 'PCTV',
+                'contentId': content_id,
+            })['contentInfo'][0]
+
+
+class HotStarIE(HotStarBaseIE):
    _VALID_URL = r'https?://(?:www\.)?hotstar\.com/(?:.+?[/-])?(?P<id>\d{10})'
    _TESTS = [{
        'url': 'http://www.hotstar.com/on-air-with-aib--english-1000076273',
        'info_dict': {
            'id': '1000076273',
            'ext': 'mp4',
-            'title': 'On Air With AIB - English',
+            'title': 'On Air With AIB',
            'description': 'md5:c957d8868e9bc793ccb813691cc4c434',
            'timestamp': 1447227000,
            'upload_date': '20151111',
@ -34,23 +59,11 @@ class HotStarIE(InfoExtractor):
        'only_matching': True,
    }]

-    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', fatal=True, query=None):
-        json_data = super(HotStarIE, self)._download_json(
-            url_or_request, video_id, note, fatal=fatal, query=query)
-        if json_data['resultCode'] != 'OK':
-            if fatal:
-                raise ExtractorError(json_data['errorDescription'])
-            return None
-        return json_data['resultObj']
-
    def _real_extract(self, url):
        video_id = self._match_id(url)
-        video_data = self._download_json(
-            'http://account.hotstar.com/AVS/besc', video_id, query={
-                'action': 'GetAggregatedContentDetails',
-                'channel': 'PCTV',
-                'contentId': video_id,
-            })['contentInfo'][0]
+
+        video_data = self._download_content_info(video_id)
+
        title = video_data['episodeTitle']

        if video_data.get('encrypted') == 'Y':
@ -99,3 +112,51 @@ class HotStarIE(InfoExtractor):
            'episode_number': int_or_none(video_data.get('episodeNumber')),
            'series': video_data.get('contentTitle'),
        }
+
+
+class HotStarPlaylistIE(HotStarBaseIE):
+    IE_NAME = 'hotstar:playlist'
+    _VALID_URL = r'(?P<url>https?://(?:www\.)?hotstar\.com/tv/[^/]+/(?P<content_id>\d+))/(?P<type>[^/]+)/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://www.hotstar.com/tv/pratidaan/14982/episodes/14812/9993',
+        'info_dict': {
+            'id': '14812',
+        },
+        'playlist_mincount': 75,
+    }, {
+        'url': 'http://www.hotstar.com/tv/pratidaan/14982/popular-clips/9998/9998',
+        'only_matching': True,
+    }]
+    _ITEM_TYPES = {
+        'episodes': 'EPISODE',
+        'popular-clips': 'CLIPS',
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        base_url = mobj.group('url')
+        content_id = mobj.group('content_id')
+        playlist_type = mobj.group('type')
+
+        content_info = self._download_content_info(content_id)
+        playlist_id = compat_str(content_info['categoryId'])
+
+        collection = self._download_json(
+            'https://search.hotstar.com/AVS/besc', playlist_id, query={
+                'action': 'SearchContents',
+                'appVersion': '5.0.40',
+                'channel': 'PCTV',
+                'moreFilters': 'series:%s;' % playlist_id,
+                'query': '*',
+                'searchOrder': 'last_broadcast_date desc,year desc,title asc',
+                'type': self._ITEM_TYPES.get(playlist_type, 'EPISODE'),
+            })
+
+        entries = [
+            self.url_result(
+                '%s/_/%s' % (base_url, video['contentId']),
+                ie=HotStarIE.ie_key(), video_id=video['contentId'])
+            for video in collection['response']['docs']
+            if video.get('contentId')]
+
+        return self.playlist_result(entries, playlist_id)
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@ -1,5 +1,6 @@
 from __future__ import unicode_literals

+import itertools
 import re

 from .common import InfoExtractor
@ -7,7 +8,6 @@ from ..compat import compat_str
 from ..utils import (
    get_element_by_attribute,
    int_or_none,
-    limit_length,
    lowercase_escape,
    try_get,
 )
@ -130,13 +130,21 @@ class InstagramIE(InfoExtractor):
                video_url = media.get('video_url')
                height = int_or_none(media.get('dimensions', {}).get('height'))
                width = int_or_none(media.get('dimensions', {}).get('width'))
-                description = media.get('caption')
+                description = try_get(
+                    media, lambda x: x['edge_media_to_caption']['edges'][0]['node']['text'],
+                    compat_str) or media.get('caption')
                thumbnail = media.get('display_src')
-                timestamp = int_or_none(media.get('date'))
+                timestamp = int_or_none(media.get('taken_at_timestamp') or media.get('date'))
                uploader = media.get('owner', {}).get('full_name')
                uploader_id = media.get('owner', {}).get('username')
-                like_count = int_or_none(media.get('likes', {}).get('count'))
-                comment_count = int_or_none(media.get('comments', {}).get('count'))
+
+                def get_count(key, kind):
+                    return int_or_none(try_get(
+                        media, (lambda x: x['edge_media_%s' % key]['count'],
+                                lambda x: x['%ss' % kind]['count'])))
+                like_count = get_count('preview_like', 'like')
+                comment_count = get_count('to_comment', 'comment')
+
                comments = [{
                    'author': comment.get('user', {}).get('username'),
                    'author_id': comment.get('user', {}).get('id'),
@ -212,7 +220,7 @@ class InstagramIE(InfoExtractor):


 class InstagramUserIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?instagram\.com/(?P<username>[^/]{2,})/?(?:$|[?#])'
+    _VALID_URL = r'https?://(?:www\.)?instagram\.com/(?P<id>[^/]{2,})/?(?:$|[?#])'
    IE_DESC = 'Instagram user profile'
    IE_NAME = 'instagram:user'
    _TEST = {
@ -221,82 +229,79 @@ class InstagramUserIE(InfoExtractor):
            'id': 'porsche',
            'title': 'porsche',
        },
-        'playlist_mincount': 2,
-        'playlist': [{
-            'info_dict': {
-                'id': '614605558512799803_462752227',
-                'ext': 'mp4',
-                'title': '#Porsche Intelligent Performance.',
-                'thumbnail': r're:^https?://.*\.jpg',
-                'uploader': 'Porsche',
-                'uploader_id': 'porsche',
-                'timestamp': 1387486713,
-                'upload_date': '20131219',
-            },
-        }],
+        'playlist_count': 5,
        'params': {
            'extract_flat': True,
            'skip_download': True,
+            'playlistend': 5,
        }
    }

-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        uploader_id = mobj.group('username')
+    def _entries(self, uploader_id):
+        query = {
+            '__a': 1,
+        }

-        entries = []
-        page_count = 0
-        media_url = 'http://instagram.com/%s/media' % uploader_id
-        while True:
+        def get_count(kind):
+            return int_or_none(try_get(
+                node, lambda x: x['%ss' % kind]['count']))
+
+        for page_num in itertools.count(1):
            page = self._download_json(
-                media_url, uploader_id,
-                note='Downloading page %d ' % (page_count + 1),
-            )
-            page_count += 1
+                'https://instagram.com/%s/' % uploader_id, uploader_id,
+                note='Downloading page %d' % page_num,
+                fatal=False, query=query)
+            if not page:
+                break

-            for it in page['items']:
-                if it.get('type') != 'video':
+            nodes = try_get(page, lambda x: x['user']['media']['nodes'], list)
+            if not nodes:
+                break
+
+            max_id = None
+
+            for node in nodes:
+                node_id = node.get('id')
+                if node_id:
+                    max_id = node_id
+
+                if node.get('__typename') != 'GraphVideo' and node.get('is_video') is not True:
+                    continue
+                video_id = node.get('code')
+                if not video_id:
                    continue
-                like_count = int_or_none(it.get('likes', {}).get('count'))
-                user = it.get('user', {})

-                formats = [{
-                    'format_id': k,
-                    'height': v.get('height'),
-                    'width': v.get('width'),
-                    'url': v['url'],
-                } for k, v in it['videos'].items()]
-                self._sort_formats(formats)
+                info = self.url_result(
+                    'https://instagram.com/p/%s/' % video_id,
+                    ie=InstagramIE.ie_key(), video_id=video_id)

-                thumbnails_el = it.get('images', {})
-                thumbnail = thumbnails_el.get('thumbnail', {}).get('url')
+                description = try_get(
+                    node, [lambda x: x['caption'], lambda x: x['text']['id']],
+                    compat_str)
+                thumbnail = node.get('thumbnail_src') or node.get('display_src')
+                timestamp = int_or_none(node.get('date'))

-                # In some cases caption is null, which corresponds to None
-                # in python. As a result, it.get('caption', {}) gives None
-                title = (it.get('caption') or {}).get('text', it['id'])
+                comment_count = get_count('comment')
+                like_count = get_count('like')
+                view_count = int_or_none(node.get('video_views'))

-                entries.append({
-                    'id': it['id'],
-                    'title': limit_length(title, 80),
-                    'formats': formats,
+                info.update({
+                    'description': description,
                    'thumbnail': thumbnail,
-                    'webpage_url': it.get('link'),
-                    'uploader': user.get('full_name'),
-                    'uploader_id': user.get('username'),
+                    'timestamp': timestamp,
+                    'comment_count': comment_count,
                    'like_count': like_count,
-                    'timestamp': int_or_none(it.get('created_time')),
+                    'view_count': view_count,
                })

-            if not page['items']:
-                break
-            max_id = page['items'][-1]['id'].split('_')[0]
-            media_url = (
-                'http://instagram.com/%s/media?max_id=%s' % (
-                    uploader_id, max_id))
+                yield info

-        return {
-            '_type': 'playlist',
-            'entries': entries,
-            'id': uploader_id,
-            'title': uploader_id,
-        }
+            if not max_id:
+                break
+
+            query['max_id'] = max_id
+
+    def _real_extract(self, url):
+        uploader_id = self._match_id(url)
+        return self.playlist_result(
+            self._entries(uploader_id), uploader_id, uploader_id)
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@ -24,7 +24,7 @@ class JWPlatformIE(InfoExtractor):
    @staticmethod
    def _extract_url(webpage):
        mobj = re.search(
-            r'<script[^>]+?src=["\'](?P<url>(?:https?:)?//content.jwplatform.com/players/[a-zA-Z0-9]{8})',
+            r'<(?:script|iframe)[^>]+?src=["\'](?P<url>(?:https?:)?//content.jwplatform.com/players/[a-zA-Z0-9]{8})',
            webpage)
        if mobj:
            return mobj.group('url')
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@ -114,7 +114,7 @@ class LivestreamIE(InfoExtractor):

        smil_url = video_data.get('smil_url')
        if smil_url:
-            formats.extend(self._extract_smil_formats(smil_url, video_id))
+            formats.extend(self._extract_smil_formats(smil_url, video_id, fatal=False))

        m3u8_url = video_data.get('m3u8_url')
        if m3u8_url:
--- a/youtube_dl/extractor/massengeschmacktv.py
+++ b/youtube_dl/extractor/massengeschmacktv.py
@ -0,0 +1,77 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    determine_ext,
+    int_or_none,
+    js_to_json,
+    mimetype2ext,
+    parse_filesize,
+)
+
+
+class MassengeschmackTVIE(InfoExtractor):
+    IE_NAME = 'massengeschmack.tv'
+    _VALID_URL = r'https?://(?:www\.)?massengeschmack\.tv/play/(?P<id>[^?&#]+)'
+
+    _TEST = {
+        'url': 'https://massengeschmack.tv/play/fktv202',
+        'md5': 'a9e054db9c2b5a08f0a0527cc201e8d3',
+        'info_dict': {
+            'id': 'fktv202',
+            'ext': 'mp4',
+            'title': 'Fernsehkritik-TV - Folge 202',
+        },
+    }
+
+    def _real_extract(self, url):
+        episode = self._match_id(url)
+
+        webpage = self._download_webpage(url, episode)
+        title = clean_html(self._html_search_regex(
+            '<h3>([^<]+)</h3>', webpage, 'title'))
+        thumbnail = self._search_regex(r'POSTER\s*=\s*"([^"]+)', webpage, 'thumbnail', fatal=False)
+        sources = self._parse_json(self._search_regex(r'(?s)MEDIA\s*=\s*(\[.+?\]);', webpage, 'media'), episode, js_to_json)
+
+        formats = []
+        for source in sources:
+            furl = source.get('src')
+            if not furl:
+                continue
+            furl = self._proto_relative_url(furl)
+            ext = determine_ext(furl) or mimetype2ext(source.get('type'))
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    furl, episode, 'mp4', 'm3u8_native',
+                    m3u8_id='hls', fatal=False))
+            else:
+                formats.append({
+                    'url': furl,
+                    'format_id': determine_ext(furl),
+                })
+
+        for (durl, format_id, width, height, filesize) in re.findall(r'''(?x)
+                                   <a[^>]+?href="(?P<url>(?:https:)?//[^"]+)".*?
+                                   <strong>(?P<format_id>.+?)</strong>.*?
+                                   <small>(?:(?P<width>\d+)x(?P<height>\d+))?\s+?\((?P<filesize>[\d,]+\s*[GM]iB)\)</small>
+                                ''', webpage):
+            formats.append({
+                'url': durl,
+                'format_id': format_id,
+                'width': int_or_none(width),
+                'height': int_or_none(height),
+                'filesize': parse_filesize(filesize),
+                'vcodec': 'none' if format_id.startswith('Audio') else None,
+            })
+
+        self._sort_formats(formats, ('width', 'height', 'filesize', 'tbr'))
+
+        return {
+            'id': episode,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+        }
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@ -70,7 +70,7 @@ class NocoIE(InfoExtractor):
            return

        login = self._download_json(
-            self._LOGIN_URL, None, 'Logging in as %s' % username,
+            self._LOGIN_URL, None, 'Logging in',
            data=urlencode_postdata({
                'a': 'login',
                'cookie': '1',
--- a/youtube_dl/extractor/once.py
+++ b/youtube_dl/extractor/once.py
@ -11,7 +11,7 @@ class OnceIE(InfoExtractor):
    ADAPTIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/master/playlist/%s/%s/%s/content.m3u8'
    PROGRESSIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/media/progressive/%s/%s/%s/%s/content.mp4'

-    def _extract_once_formats(self, url):
+    def _extract_once_formats(self, url, http_formats_preference=None):
        domain_id, application_id, media_item_id = re.match(
            OnceIE._VALID_URL, url).groups()
        formats = self._extract_m3u8_formats(
@ -35,6 +35,7 @@ class OnceIE(InfoExtractor):
                    'format_id': adaptive_format['format_id'].replace(
                        'hls', 'http'),
                    'protocol': 'http',
+                    'preference': http_formats_preference,
                })
                progressive_formats.append(progressive_format)
        self._check_formats(progressive_formats, media_item_id)
--- a/youtube_dl/extractor/pandatv.py
+++ b/youtube_dl/extractor/pandatv.py
@ -33,7 +33,7 @@ class PandaTVIE(InfoExtractor):
        video_id = self._match_id(url)

        config = self._download_json(
-            'https://www.panda.tv/api_room?roomid=%s' % video_id, video_id)
+            'https://www.panda.tv/api_room_v2?roomid=%s' % video_id, video_id)

        error_code = config.get('errno', 0)
        if error_code is not 0:
@ -66,6 +66,11 @@ class PandaTVIE(InfoExtractor):
            plflag1 = '4'
        live_panda = 'live_panda' if plflag0 < 1 else ''

+        plflag_auth = self._parse_json(video_info['plflag_list'], video_id)
+        sign = plflag_auth['auth']['sign']
+        ts = plflag_auth['auth']['time']
+        rid = plflag_auth['auth']['rid']
+
        quality_key = qualities(['OD', 'HD', 'SD'])
        suffix = ['_small', '_mid', '']
        formats = []
@ -77,8 +82,8 @@ class PandaTVIE(InfoExtractor):
                continue
            for pref, (ext, pl) in enumerate((('m3u8', '-hls'), ('flv', ''))):
                formats.append({
-                    'url': 'https://pl%s%s.live.panda.tv/live_panda/%s%s%s.%s'
-                    % (pl, plflag1, room_key, live_panda, suffix[quality], ext),
+                    'url': 'https://pl%s%s.live.panda.tv/live_panda/%s%s%s.%s?sign=%s&ts=%s&rid=%s'
+                    % (pl, plflag1, room_key, live_panda, suffix[quality], ext, sign, ts, rid),
                    'format_id': '%s-%s' % (k, ext),
                    'quality': quality,
                    'source_preference': pref,
--- a/youtube_dl/extractor/patreon.py
+++ b/youtube_dl/extractor/patreon.py
@ -67,7 +67,7 @@ class PatreonIE(InfoExtractor):
            'https://www.patreon.com/processLogin',
            compat_urllib_parse_urlencode(login_form).encode('utf-8')
        )
-        login_page = self._download_webpage(request, None, note='Logging in as %s' % username)
+        login_page = self._download_webpage(request, None, note='Logging in')

        if re.search(r'onLoginFailed', login_page):
            raise ExtractorError('Unable to login, incorrect username and/or password', expected=True)
--- a/youtube_dl/extractor/performgroup.py
+++ b/youtube_dl/extractor/performgroup.py
@ -0,0 +1,83 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class PerformGroupIE(InfoExtractor):
+    _VALID_URL = r'https?://player\.performgroup\.com/eplayer(?:/eplayer\.html|\.js)#/?(?P<id>[0-9a-f]{26})\.(?P<auth_token>[0-9a-z]{26})'
+    _TESTS = [{
+        # http://www.faz.net/aktuell/sport/fussball/wm-2018-playoffs-schweiz-besiegt-nordirland-1-0-15286104.html
+        'url': 'http://player.performgroup.com/eplayer/eplayer.html#d478c41c5d192f56b9aa859de8.1w4crrej5w14e1ed4s1ce4ykab',
+        'md5': '259cb03d142e2e52471e8837ecacb29f',
+        'info_dict': {
+            'id': 'xgrwobuzumes1lwjxtcdpwgxd',
+            'ext': 'mp4',
+            'title': 'Liga MX: Keine Einsicht nach Horrorfoul',
+            'description': 'md5:7cd3b459c82725b021e046ab10bf1c5b',
+            'timestamp': 1511533477,
+            'upload_date': '20171124',
+        }
+    }]
+
+    def _call_api(self, service, auth_token, content_id, referer_url):
+        return self._download_json(
+            'http://ep3.performfeeds.com/ep%s/%s/%s/' % (service, auth_token, content_id),
+            content_id, headers={
+                'Referer': referer_url,
+                'Origin': 'http://player.performgroup.com',
+            }, query={
+                '_fmt': 'json',
+            })
+
+    def _real_extract(self, url):
+        player_id, auth_token = re.search(self._VALID_URL, url).groups()
+        bootstrap = self._call_api('bootstrap', auth_token, player_id, url)
+        video = bootstrap['config']['dataSource']['sourceItems'][0]['videos'][0]
+        video_id = video['uuid']
+        vod = self._call_api('vod', auth_token, video_id, url)
+        media = vod['videos']['video'][0]['media']
+
+        formats = []
+        hls_url = media.get('hls', {}).get('url')
+        if hls_url:
+            formats.extend(self._extract_m3u8_formats(hls_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+
+        hds_url = media.get('hds', {}).get('url')
+        if hds_url:
+            formats.extend(self._extract_f4m_formats(hds_url + '?hdcore', video_id, f4m_id='hds', fatal=False))
+
+        for c in media.get('content', []):
+            c_url = c.get('url')
+            if not c_url:
+                continue
+            tbr = int_or_none(c.get('bitrate'), 1000)
+            format_id = 'http'
+            if tbr:
+                format_id += '-%d' % tbr
+            formats.append({
+                'format_id': format_id,
+                'url': c_url,
+                'tbr': tbr,
+                'width': int_or_none(c.get('width')),
+                'height': int_or_none(c.get('height')),
+                'filesize': int_or_none(c.get('fileSize')),
+                'vcodec': c.get('type'),
+                'fps': int_or_none(c.get('videoFrameRate')),
+                'vbr': int_or_none(c.get('videoRate'), 1000),
+                'abr': int_or_none(c.get('audioRate'), 1000),
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': video['title'],
+            'description': video.get('description'),
+            'thumbnail': video.get('poster'),
+            'duration': int_or_none(video.get('duration')),
+            'timestamp': int_or_none(video.get('publishedTime'), 1000),
+            'formats': formats,
+        }
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@ -116,7 +116,7 @@ class PluralsightIE(PluralsightBaseIE):
            post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)

        response = self._download_webpage(
-            post_url, None, 'Logging in as %s' % username,
+            post_url, None, 'Logging in',
            data=urlencode_postdata(login_form),
            headers={'Content-Type': 'application/x-www-form-urlencoded'})

--- a/youtube_dl/extractor/roosterteeth.py
+++ b/youtube_dl/extractor/roosterteeth.py
@ -68,7 +68,7 @@ class RoosterTeethIE(InfoExtractor):

        login_request = self._download_webpage(
            self._LOGIN_URL, None,
-            note='Logging in as %s' % username,
+            note='Logging in',
            data=urlencode_postdata(login_form),
            headers={
                'Referer': self._LOGIN_URL,
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@ -61,7 +61,7 @@ class SafariBaseIE(InfoExtractor):
        request = sanitized_Request(
            self._LOGIN_URL, urlencode_postdata(login_form), headers=headers)
        login_page = self._download_webpage(
-            request, None, 'Logging in as %s' % username)
+            request, None, 'Logging in')

        if not is_logged(login_page):
            raise ExtractorError(
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@ -7,7 +7,7 @@ from ..utils import ExtractorError


 class SpankBangIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www|[a-z]{2})\.)?spankbang\.com/(?P<id>[\da-z]+)/video'
+    _VALID_URL = r'https?://(?:(?:www|m|[a-z]{2})\.)?spankbang\.com/(?P<id>[\da-z]+)/video'
    _TESTS = [{
        'url': 'http://spankbang.com/3vvn/video/fantasy+solo',
        'md5': '1cc433e1d6aa14bc376535b8679302f7',
@ -15,7 +15,7 @@ class SpankBangIE(InfoExtractor):
            'id': '3vvn',
            'ext': 'mp4',
            'title': 'fantasy solo',
-            'description': 'Watch fantasy solo free HD porn video - 05 minutes - dillion harper masturbates on a bed free adult movies.',
+            'description': 'Watch fantasy solo free HD porn video - 05 minutes -  Babe,Masturbation,Solo,Toy  - dillion harper masturbates on a bed free adult movies sexy clips.',
            'thumbnail': r're:^https?://.*\.jpg$',
            'uploader': 'silly2587',
            'age_limit': 18,
@ -28,6 +28,10 @@ class SpankBangIE(InfoExtractor):
        # no uploader
        'url': 'http://spankbang.com/lklg/video/sex+with+anyone+wedding+edition+2',
        'only_matching': True,
+    }, {
+        # mobile page
+        'url': 'http://m.spankbang.com/1o2de/video/can+t+remember+her+name',
+        'only_matching': True,
    }]

    def _real_extract(self, url):
--- a/youtube_dl/extractor/tva.py
+++ b/youtube_dl/extractor/tva.py
@ -32,6 +32,8 @@ class TVAIE(InfoExtractor):
        video_data = self._download_json(
            'https://videos.tva.ca/proxy/item/_' + video_id, video_id, headers={
                'Accept': 'application/json',
+            }, query={
+                'appId': '5955fc5f23eec60006c951f1',
            })

        def get_attribute(key):
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@ -101,7 +101,7 @@ class TwitchBaseIE(InfoExtractor):
            fail(clean_html(login_page))

        redirect_page, handle = login_step(
-            login_page, handle, 'Logging in as %s' % username, {
+            login_page, handle, 'Logging in', {
                'username': username,
                'password': password,
            })
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@ -164,7 +164,7 @@ class UdemyIE(InfoExtractor):
        })

        response = self._download_webpage(
-            self._LOGIN_URL, None, 'Logging in as %s' % username,
+            self._LOGIN_URL, None, 'Logging in',
            data=urlencode_postdata(login_form),
            headers={
                'Referer': self._ORIGIN_URL,
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@ -99,7 +99,7 @@ class VikiBaseIE(InfoExtractor):

        login = self._call_api(
            'sessions.json', None,
-            'Logging in as %s' % username, post_data=login_form)
+            'Logging in', post_data=login_form)

        self._token = login.get('token')
        if not self._token:
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@ -67,7 +67,7 @@ class VKBaseIE(InfoExtractor):

        login_page = self._download_webpage(
            'https://login.vk.com/?act=login', None,
-            note='Logging in as %s' % username,
+            note='Logging in',
            data=urlencode_postdata(login_form))

        if re.search(r'onLoginFailed', login_page):
--- a/youtube_dl/extractor/vshare.py
+++ b/youtube_dl/extractor/vshare.py
@ -1,14 +1,21 @@
 # coding: utf-8
 from __future__ import unicode_literals

+import re
+
 from .common import InfoExtractor
+from ..compat import compat_chr
+from ..utils import (
+    decode_packed_codes,
+    ExtractorError,
+)


 class VShareIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?vshare\.io/[dv]/(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'https://vshare.io/d/0f64ce6',
-        'md5': '16d7b8fef58846db47419199ff1ab3e7',
+        'md5': '17b39f55b5497ae8b59f5fbce8e35886',
        'info_dict': {
            'id': '0f64ce6',
            'title': 'vl14062007715967',
@ -19,20 +26,49 @@ class VShareIE(InfoExtractor):
        'only_matching': True,
    }]

+    @staticmethod
+    def _extract_urls(webpage):
+        return re.findall(
+            r'<iframe[^>]+?src=["\'](?P<url>(?:https?:)?//(?:www\.)?vshare\.io/v/[^/?#&]+)',
+            webpage)
+
+    def _extract_packed(self, webpage):
+        packed = self._search_regex(
+            r'(eval\(function.+)', webpage, 'packed code')
+        unpacked = decode_packed_codes(packed)
+        digits = self._search_regex(r'\[((?:\d+,?)+)\]', unpacked, 'digits')
+        digits = [int(digit) for digit in digits.split(',')]
+        key_digit = self._search_regex(
+            r'fromCharCode\(.+?(\d+)\)}', unpacked, 'key digit')
+        chars = [compat_chr(d - int(key_digit)) for d in digits]
+        return ''.join(chars)
+
    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(
-            'https://vshare.io/d/%s' % video_id, video_id)
+            'https://vshare.io/v/%s/width-650/height-430/1' % video_id,
+            video_id)

        title = self._html_search_regex(
-            r'(?s)<div id="root-container">(.+?)<br/>', webpage, 'title')
-        video_url = self._search_regex(
-            r'<a[^>]+href=(["\'])(?P<url>(?:https?:)?//.+?)\1[^>]*>[Cc]lick\s+here',
-            webpage, 'video url', group='url')
+            r'<title>([^<]+)</title>', webpage, 'title')
+        title = title.split(' - ')[0]

-        return {
+        error = self._html_search_regex(
+            r'(?s)<div[^>]+\bclass=["\']xxx-error[^>]+>(.+?)</div', webpage,
+            'error', default=None)
+        if error:
+            raise ExtractorError(error, expected=True)
+
+        info = self._parse_html5_media_entries(
+            url, '<video>%s</video>' % self._extract_packed(webpage),
+            video_id)[0]
+
+        self._sort_formats(info['formats'])
+
+        info.update({
            'id': video_id,
            'title': title,
-            'url': video_url,
-        }
+        })
+
+        return info
--- a/youtube_dl/extractor/wsj.py
+++ b/youtube_dl/extractor/wsj.py
@ -13,7 +13,7 @@ class WSJIE(InfoExtractor):
    _VALID_URL = r'''(?x)
                        (?:
                            https?://video-api\.wsj\.com/api-video/player/iframe\.html\?.*?\bguid=|
-                            https?://(?:www\.)?(?:wsj|barrons)\.com/video/[^/]+/|
+                            https?://(?:www\.)?(?:wsj|barrons)\.com/video/(?:[^/]+/)+|
                            wsj:
                        )
                        (?P<id>[a-fA-F0-9-]{36})
@ -38,6 +38,9 @@ class WSJIE(InfoExtractor):
    }, {
        'url': 'http://www.barrons.com/video/capitalism-deserves-more-respect-from-millennials/F301217E-6F46-43AE-B8D2-B7180D642EE9.html',
        'only_matching': True,
+    }, {
+        'url': 'https://www.wsj.com/video/series/a-brief-history-of/the-modern-cell-carrier-how-we-got-here/980E2187-401D-48A1-B82B-1486CEE06CB9',
+        'only_matching': True,
    }]

    def _real_extract(self, url):
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@ -154,7 +154,7 @@ class YoukuIE(InfoExtractor):
        # request basic data
        basic_data_params = {
            'vid': video_id,
-            'ccode': '0402' if 'tudou.com' in url else '0401',
+            'ccode': '0502',
            'client_ip': '192.168.1.1',
            'utid': cna,
            'client_ts': time.time() / 1000,
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@ -1,3 +1,3 @@
 from __future__ import unicode_literals

-__version__ = '2017.10.29'
+__version__ = '2017.11.15'