From 17880cbaca971d631304ed6cfbe7c1365e8b4110 Mon Sep 17 00:00:00 2001 From: hodayabu Date: Tue, 21 Apr 2020 14:32:32 +0300 Subject: [PATCH] tiktok fix code --- test/ci/test_tiktok.py | 21 +++++++++--- youtube_dl/extractor/tiktok.py | 62 ++++++++++------------------------ 2 files changed, 35 insertions(+), 48 deletions(-) diff --git a/test/ci/test_tiktok.py b/test/ci/test_tiktok.py index 2fe6cc4b9..5e050e6e2 100644 --- a/test/ci/test_tiktok.py +++ b/test/ci/test_tiktok.py @@ -4,13 +4,26 @@ import youtube_dl class TikTokTestYoutubeDl(unittest.TestCase): def test_meta_data(self): - url = 'https://www.tiktok.com/@danieltbraun/video/6817099671043853574' + url = 'https://www.tiktok.com/@oriangaon/video/6807126376001441030' params = {} ydl = youtube_dl.YoutubeDL(params) info = ydl.extract_info(url, download=False) - self.assertEquals(info['share_count'], 121) - - + self.assertEquals(info['id'], '6807126376001441030') + self.assertEquals(info['url'], 'https://www.tiktok.com/@oriangaon/video/6807126376001441030') + self.assertEquals(info['title'], '#foryou #foyou Mmmmm....,,') + self.assertEquals(info['uploader'], 'Oriangaon') + self.assertEquals(info['timestamp'], 1584907616) + self.assertEquals(info['thumbnail'], + 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/d1a8fbd3e42dda3a1baa01ee9edad289') + self.assertGreaterEqual(info['view_count'], 79864) + self.assertEquals(info['uploader_id'], '6772113344733955077') + self.assertFalse(info['is_live']) + self.assertEquals(info['live_status'], 'not_live') + self.assertGreaterEqual(info['like_count'], 2213) + self.assertGreaterEqual(info['share_count'], 109) + self.assertGreaterEqual(info['comment_count'], 40) + self.assertEquals(info['duration'], 10) + self.assertEquals(info['ext'], 'mp.4') if __name__ == '__main__': diff --git a/youtube_dl/extractor/tiktok.py b/youtube_dl/extractor/tiktok.py index 140db07bc..9d742465e 100644 --- a/youtube_dl/extractor/tiktok.py +++ b/youtube_dl/extractor/tiktok.py @@ -2,7 +2,6 @@ from __future__ import unicode_literals import ast from bs4 import BeautifulSoup -import requests import json from .common import InfoExtractor from ..utils import ( @@ -11,11 +10,7 @@ from ..utils import ( int_or_none, str_or_none, try_get, - url_or_none, -) - - -# add to requirements.txt- bs4, newspaper, requests + url_or_none) class TikTokBaseIE(InfoExtractor): @@ -70,8 +65,6 @@ class TikTokBaseIE(InfoExtractor): } - - class TikTokIE(TikTokBaseIE): _VALID_URL = r'''(?x) https?:// @@ -103,55 +96,34 @@ class TikTokIE(TikTokBaseIE): }] def _real_extract(self, url): - video_id = url.split('/')[-1] - - # extract meta data using the official api - # Response json contains: provider url, title, html, author_namee, height, thumbnail_width, width, version, - # author_url, thumbnail_height, thumbnail_url, type, provider_name (tiktok) - + video_id = self._match_id(url) json_api = self._download_json('https://www.tiktok.com/oembed?url=' + url, video_id) - # extract metadata with beautifulSoup webpage = self._download_webpage(url, video_id) soup = BeautifulSoup(webpage, features="html.parser") - h2 = soup.find_all("h2", {"class": "jsx-1038045583 jsx-3192540912 jsx-2150087249 video-meta-count"}) - data = h2[0].text.split(' ') - likes_count = self.numeric_convert(data[0]) - comments_count = self.numeric_convert(data[3]) json_next_data = soup.find(id='__NEXT_DATA__') props = json_next_data.contents[0] json_data_encode = json.dumps(props.encode('utf-8')) ast_le = ast.literal_eval(json_data_encode) data_dict = json.loads(ast_le) - timestamp = self.numeric_convert(data_dict['props']['pageProps']['videoData']['itemInfos']['createTime']) - shares = data_dict['props']['pageProps']['videoData']['itemInfos']['shareCount'] - views = data_dict['props']['pageProps']['videoData']['itemInfos']['playCount'] - duration = data_dict['props']['pageProps']['videoData']['itemInfos']['video']['videoMeta']['duration'] - provider_id = data_dict['props']['pageProps']['videoData']['itemInfos']['authorId'] - - # TO-DO- check on formats + item_info = data_dict['props']['pageProps']['videoData']['itemInfos'] + timestamp = int(item_info['createTime']) + shares = item_info['shareCount'] + views = item_info['playCount'] + duration = item_info['video']['videoMeta']['duration'] + provider_id = item_info['authorId'] + comments_count = item_info['commentCount'] + likes_count = item_info['diggCount'] return self.info_dict(video_id, str(url), json_api['title'], json_api['author_name'], timestamp, json_api['thumbnail_url'], - views, provider_id, False, 'not_live', likes_count, shares, '', comments_count,duration) + views, provider_id, False, 'not_live', likes_count, shares, '', comments_count, duration) - def numeric_convert(self, unicode): - if 'K' in unicode: - unicode=unicode[:-1] - return int(float(unicode)*1000) - if 'M' in unicode: - unicode=unicode[:-1] - return int(float(unicode)*100000) - else: - return int(unicode) - - - - def info_dict (self, video_id, url, video_title, - uploader, timestamp, thumbnail, - view_count, uploader_id, is_live, live_status - , likes_count, shares_count, subtitles, comment_count, duration): + def info_dict(self, video_id, url, video_title, + uploader, timestamp, thumbnail, + view_count, uploader_id, is_live, live_status + , likes_count, shares_count, subtitles, comment_count, duration): info_dict = { 'id': video_id, 'url': url, @@ -167,11 +139,13 @@ class TikTokIE(TikTokBaseIE): 'share_count': shares_count, 'subtitles': subtitles, 'comment_count': comment_count, - 'duration': duration + 'duration': duration, + 'ext':'mp.4' } return info_dict + class TikTokUserIE(TikTokBaseIE): _VALID_URL = r'''(?x) https?://