youtube-dl/youtube_dl/extractor/appletrailers.py

from __future__ import unicode_literals

import re
import json

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    int_or_none,
)


class AppleTrailersIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/(?:trailers|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'
    _TESTS = [{
        "url": "http://trailers.apple.com/trailers/wb/manofsteel/",
        'info_dict': {
            'id': 'manofsteel',
        },
        "playlist": [
            {
                "md5": "d97a8e575432dbcb81b7c3acb741f8a8",
                "info_dict": {
                    "id": "manofsteel-trailer4",
                    "ext": "mov",
                    "duration": 111,
                    "title": "Trailer 4",
                    "upload_date": "20130523",
                    "uploader_id": "wb",
                },
            },
            {
                "md5": "b8017b7131b721fb4e8d6f49e1df908c",
                "info_dict": {
                    "id": "manofsteel-trailer3",
                    "ext": "mov",
                    "duration": 182,
                    "title": "Trailer 3",
                    "upload_date": "20130417",
                    "uploader_id": "wb",
                },
            },
            {
                "md5": "d0f1e1150989b9924679b441f3404d48",
                "info_dict": {
                    "id": "manofsteel-trailer",
                    "ext": "mov",
                    "duration": 148,
                    "title": "Trailer",
                    "upload_date": "20121212",
                    "uploader_id": "wb",
                },
            },
            {
                "md5": "5fe08795b943eb2e757fa95cb6def1cb",
                "info_dict": {
                    "id": "manofsteel-teaser",
                    "ext": "mov",
                    "duration": 93,
                    "title": "Teaser",
                    "upload_date": "20120721",
                    "uploader_id": "wb",
                },
            },
        ]
    }, {
        'url': 'http://trailers.apple.com/ca/metropole/autrui/',
        'only_matching': True,
    }]

    _JSON_RE = r'iTunes.playURL\((.*?)\);'

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        movie = mobj.group('movie')
        uploader_id = mobj.group('company')

        playlist_url = compat_urlparse.urljoin(url, 'includes/playlists/itunes.inc')

        def fix_html(s):
            s = re.sub(r'(?s)<script[^<]*?>.*?</script>', '', s)
            s = re.sub(r'<img ([^<]*?)>', r'<img \1/>', s)
            # The ' in the onClick attributes are not escaped, it couldn't be parsed
            # like: http://trailers.apple.com/trailers/wb/gravity/

            def _clean_json(m):
                return 'iTunes.playURL(%s);' % m.group(1).replace('\'', '&#39;')
            s = re.sub(self._JSON_RE, _clean_json, s)
            s = '<html>%s</html>' % s
            return s
        doc = self._download_xml(playlist_url, movie, transform_source=fix_html)

        playlist = []
        for li in doc.findall('./div/ul/li'):
            on_click = li.find('.//a').attrib['onClick']
            trailer_info_json = self._search_regex(self._JSON_RE,
                                                   on_click, 'trailer info')
            trailer_info = json.loads(trailer_info_json)
            title = trailer_info['title']
            video_id = movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', title).lower()
            thumbnail = li.find('.//img').attrib['src']
            upload_date = trailer_info['posted'].replace('-', '')

            runtime = trailer_info['runtime']
            m = re.search(r'(?P<minutes>[0-9]+):(?P<seconds>[0-9]{1,2})', runtime)
            duration = None
            if m:
                duration = 60 * int(m.group('minutes')) + int(m.group('seconds'))

            first_url = trailer_info['url']
            trailer_id = first_url.split('/')[-1].rpartition('_')[0].lower()
            settings_json_url = compat_urlparse.urljoin(url, 'includes/settings/%s.json' % trailer_id)
            settings = self._download_json(settings_json_url, trailer_id, 'Downloading settings json')

            formats = []
            for format in settings['metadata']['sizes']:
                # The src is a file pointing to the real video file
                format_url = re.sub(r'_(\d*p.mov)', r'_h\1', format['src'])
                formats.append({
                    'url': format_url,
                    'format': format['type'],
                    'width': int_or_none(format['width']),
                    'height': int_or_none(format['height']),
                })

            self._sort_formats(formats)

            playlist.append({
                '_type': 'video',
                'id': video_id,
                'formats': formats,
                'title': title,
                'duration': duration,
                'thumbnail': thumbnail,
                'upload_date': upload_date,
                'uploader_id': uploader_id,
                'http_headers': {
                    'User-Agent': 'QuickTime compatible (youtube-dl)',
                },
            })

        return {
            '_type': 'playlist',
            'id': movie,
            'entries': playlist,
        }
More unicode literals 2014-01-07 17:04:48 +08:00			`from __future__ import unicode_literals`

[appletrailers] Add support 2013-08-28 08:18:44 +08:00			`import re`
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-30 02:49:58 +08:00			`import json`
[appletrailers] Add support 2013-08-28 08:18:44 +08:00
			`from .common import InfoExtractor`
Fix imports and general cleanup · Import from compat what comes from compat. Yes, some names are available in utils too, but that's an implementation detail. · Use _match_id consistently whenever possible · Fix some outdated tests · Use consistent valid URL (always match the whole protocol, no ^ at start required) · Use modern test definitions 2014-12-13 19:24:42 +08:00			`from ..compat import compat_urlparse`
[appletrailers] Add support 2013-08-28 08:18:44 +08:00			`from ..utils import (`
[appletrailers] Support height-less videos 2014-08-10 19:04:45 +08:00			`int_or_none,`
[appletrailers] Add support 2013-08-28 08:18:44 +08:00			`)`


			`class AppleTrailersIE(InfoExtractor):`
[appletrailers] Extend _VALID_URL (Closes #5027) 2015-02-22 21:53:32 +08:00			`_VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/(?:trailers\|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'`
[appletrailers] Add test (#5027) 2015-02-22 21:58:39 +08:00			`_TESTS = [{`
More unicode literals 2014-01-07 17:04:48 +08:00			`"url": "http://trailers.apple.com/trailers/wb/manofsteel/",`
Extend various playlist tests 2015-02-18 07:49:10 +08:00			`'info_dict': {`
			`'id': 'manofsteel',`
			`},`
More unicode literals 2014-01-07 17:04:48 +08:00			`"playlist": [`
[appletrailers] Add support 2013-08-28 08:18:44 +08:00			`{`
More unicode literals 2014-01-07 17:04:48 +08:00			`"md5": "d97a8e575432dbcb81b7c3acb741f8a8",`
			`"info_dict": {`
[appletrailers] Modernize 2014-03-28 04:10:51 +08:00			`"id": "manofsteel-trailer4",`
			`"ext": "mov",`
More unicode literals 2014-01-07 17:04:48 +08:00			`"duration": 111,`
			`"title": "Trailer 4",`
			`"upload_date": "20130523",`
			`"uploader_id": "wb",`
[appletrailers] Add support 2013-08-28 08:18:44 +08:00			`},`
			`},`
			`{`
More unicode literals 2014-01-07 17:04:48 +08:00			`"md5": "b8017b7131b721fb4e8d6f49e1df908c",`
			`"info_dict": {`
[appletrailers] Modernize 2014-03-28 04:10:51 +08:00			`"id": "manofsteel-trailer3",`
			`"ext": "mov",`
More unicode literals 2014-01-07 17:04:48 +08:00			`"duration": 182,`
			`"title": "Trailer 3",`
			`"upload_date": "20130417",`
			`"uploader_id": "wb",`
[appletrailers] Add support 2013-08-28 08:18:44 +08:00			`},`
			`},`
			`{`
More unicode literals 2014-01-07 17:04:48 +08:00			`"md5": "d0f1e1150989b9924679b441f3404d48",`
			`"info_dict": {`
[appletrailers] Modernize 2014-03-28 04:10:51 +08:00			`"id": "manofsteel-trailer",`
			`"ext": "mov",`
More unicode literals 2014-01-07 17:04:48 +08:00			`"duration": 148,`
			`"title": "Trailer",`
			`"upload_date": "20121212",`
			`"uploader_id": "wb",`
[appletrailers] Add support 2013-08-28 08:18:44 +08:00			`},`
			`},`
			`{`
More unicode literals 2014-01-07 17:04:48 +08:00			`"md5": "5fe08795b943eb2e757fa95cb6def1cb",`
			`"info_dict": {`
[appletrailers] Modernize 2014-03-28 04:10:51 +08:00			`"id": "manofsteel-teaser",`
			`"ext": "mov",`
More unicode literals 2014-01-07 17:04:48 +08:00			`"duration": 93,`
			`"title": "Teaser",`
			`"upload_date": "20120721",`
			`"uploader_id": "wb",`
[appletrailers] Add support 2013-08-28 08:18:44 +08:00			`},`
[appletrailers] Modernize 2014-03-28 04:10:51 +08:00			`},`
[appletrailers] Add support 2013-08-28 08:18:44 +08:00			`]`
[appletrailers] Add test (#5027) 2015-02-22 21:58:39 +08:00			`}, {`
			`'url': 'http://trailers.apple.com/ca/metropole/autrui/',`
			`'only_matching': True,`
			`}]`
[appletrailers] Add support 2013-08-28 08:18:44 +08:00
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-30 02:49:58 +08:00			`_JSON_RE = r'iTunes.playURL\((.*?)\);'`

[appletrailers] Add support 2013-08-28 08:18:44 +08:00			`def _real_extract(self, url):`
			`mobj = re.match(self._VALID_URL, url)`
			`movie = mobj.group('movie')`
			`uploader_id = mobj.group('company')`

[appletrailers] Modernize 2014-03-28 04:10:51 +08:00			`playlist_url = compat_urlparse.urljoin(url, 'includes/playlists/itunes.inc')`
PEP8 applied 2014-11-24 03:41:03 +08:00
Use `_download_xml` in more extractors 2013-12-11 04:03:53 +08:00			`def fix_html(s):`
[appletrailers] Modernize 2014-03-28 04:10:51 +08:00			`s = re.sub(r'(?s)<script[^<]?>.?</script>', '', s)`
Use `_download_xml` in more extractors 2013-12-11 04:03:53 +08:00			`s = re.sub(r'<img ([^<]*?)>', r'<img \1/>', s)`
			`# The ' in the onClick attributes are not escaped, it couldn't be parsed`
			`# like: http://trailers.apple.com/trailers/wb/gravity/`
PEP8 applied 2014-11-24 03:41:03 +08:00
Use `_download_xml` in more extractors 2013-12-11 04:03:53 +08:00			`def _clean_json(m):`
[appletrailers] Modernize 2014-03-28 04:10:51 +08:00			`return 'iTunes.playURL(%s);' % m.group(1).replace('\'', ''')`
Use `_download_xml` in more extractors 2013-12-11 04:03:53 +08:00			`s = re.sub(self._JSON_RE, _clean_json, s)`
[appletrailers] Modernize 2014-11-26 19:41:24 +08:00			`s = '<html>%s</html>' % s`
Use `_download_xml` in more extractors 2013-12-11 04:03:53 +08:00			`return s`
			`doc = self._download_xml(playlist_url, movie, transform_source=fix_html)`
[appletrailers] Add support 2013-08-28 08:18:44 +08:00
			`playlist = []`
			`for li in doc.findall('./div/ul/li'):`
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-30 02:49:58 +08:00			`on_click = li.find('.//a').attrib['onClick']`
			`trailer_info_json = self._search_regex(self._JSON_RE,`
PEP8: applied even more rules 2014-11-24 04:39:15 +08:00			`on_click, 'trailer info')`
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-30 02:49:58 +08:00			`trailer_info = json.loads(trailer_info_json)`
			`title = trailer_info['title']`
[appletrailers] Add support 2013-08-28 08:18:44 +08:00			`video_id = movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', title).lower()`
			`thumbnail = li.find('.//img').attrib['src']`
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-30 02:49:58 +08:00			`upload_date = trailer_info['posted'].replace('-', '')`
[appletrailers] Add support 2013-08-28 08:18:44 +08:00
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-30 02:49:58 +08:00			`runtime = trailer_info['runtime']`
			`m = re.search(r'(?P<minutes>[0-9]+):(?P<seconds>[0-9]{1,2})', runtime)`
[appletrailers] Add support 2013-08-28 08:18:44 +08:00			`duration = None`
			`if m:`
			`duration = 60 * int(m.group('minutes')) + int(m.group('seconds'))`

[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-30 02:49:58 +08:00			`first_url = trailer_info['url']`
[appletrailers] The request for the settings must have the trailer name in lower case (fixes #1329) 2013-09-30 02:59:19 +08:00			`trailer_id = first_url.split('/')[-1].rpartition('_')[0].lower()`
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-30 02:49:58 +08:00			`settings_json_url = compat_urlparse.urljoin(url, 'includes/settings/%s.json' % trailer_id)`
[appletrailers] Modernize 2014-03-28 04:10:51 +08:00			`settings = self._download_json(settings_json_url, trailer_id, 'Downloading settings json')`
[appletrailers] Add support 2013-08-28 08:18:44 +08:00
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-30 02:49:58 +08:00			`formats = []`
			`for format in settings['metadata']['sizes']:`
			`# The src is a file pointing to the real video file`
			`format_url = re.sub(r'_(\d*p.mov)', r'_h\1', format['src'])`
			`formats.append({`
			`'url': format_url,`
			`'format': format['type'],`
[appletrailers] Support height-less videos 2014-08-10 19:04:45 +08:00			`'width': int_or_none(format['width']),`
			`'height': int_or_none(format['height']),`
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-30 02:49:58 +08:00			`})`
[appletrailers] Use centralized format selection 2013-12-25 22:24:41 +08:00
			`self._sort_formats(formats)`
[appletrailers] Add support 2013-08-28 08:18:44 +08:00
Remove the compatibility code used before the new format system was implemented 2013-12-03 21:21:06 +08:00			`playlist.append({`
[appletrailers] Add support 2013-08-28 08:18:44 +08:00			`'_type': 'video',`
			`'id': video_id,`
			`'formats': formats,`
			`'title': title,`
			`'duration': duration,`
			`'thumbnail': thumbnail,`
			`'upload_date': upload_date,`
			`'uploader_id': uploader_id,`
[extractors] Use http_headers for setting the User-Agent and the Referer 2015-01-25 01:19:58 +08:00			`'http_headers': {`
			`'User-Agent': 'QuickTime compatible (youtube-dl)',`
			`},`
Remove the compatibility code used before the new format system was implemented 2013-12-03 21:21:06 +08:00			`})`
[appletrailers] Add support 2013-08-28 08:18:44 +08:00
			`return {`
			`'_type': 'playlist',`
			`'id': movie,`
			`'entries': playlist,`
			`}`