diff --git a/Web/API_logs.txt b/Web/API_logs.txt new file mode 100644 index 0000000..25b1a65 --- /dev/null +++ b/Web/API_logs.txt @@ -0,0 +1,2 @@ +# @Author: https://github.com/Evil0ctal/ +# @Time: 2021/11/06 \ No newline at end of file diff --git a/Web/logs.txt b/Web/logs.txt new file mode 100644 index 0000000..25b1a65 --- /dev/null +++ b/Web/logs.txt @@ -0,0 +1,2 @@ +# @Author: https://github.com/Evil0ctal/ +# @Time: 2021/11/06 \ No newline at end of file diff --git a/Web/scraper.py b/Web/scraper.py new file mode 100644 index 0000000..7262fea --- /dev/null +++ b/Web/scraper.py @@ -0,0 +1,388 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# @Author: https://github.com/Evil0ctal/ +# @Time: 2021/11/06 +# @Update: 2022/04/05 +# @Function: +# 核心代码,估值1块(๑•̀ㅂ•́)و✧ +# 用于爬取Douyin/TikTok数据并以字典形式返回。 + + +import re +import json +import time +import requests +from retrying import retry + + +class Scraper: + """ + Scraper.douyin():抖音视频/图集解析,返回字典。 + Scraper.tiktok():TikTok视频解析,返回字典。 + """ + def __init__(self): + self.headers = { + 'user-agent': 'Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Mobile Safari/537.36 Edg/87.0.664.66' + } + self.tiktok_headers = { + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + "authority": "www.tiktok.com", + "Accept-Encoding": "gzip, deflate", + "Connection": "keep-alive", + "Host": "www.tiktok.com", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) coc_coc_browser/86.0.170 Chrome/80.0.3987.170 Safari/537.36", + } + + @retry(stop_max_attempt_number=6) + def douyin(self, original_url): + """ + 利用官方接口解析抖音链接信息 + :param original_url: 抖音/TikTok链接(支持长/短链接) + :return:包含信息的字典 + """ + headers = self.headers + try: + # 开始时间 + start = time.time() + # 原视频链接 + r = requests.get(url=original_url, headers=headers, allow_redirects=False) + try: + # 2021/12/11 发现抖音做了限制,会自动重定向网址,但是可以从回执头中获取 + long_url = r.headers['Location'] + except: + # 报错后判断为长链接,直接截取视频id + long_url = original_url + # 正则匹配出视频ID + key = re.findall('video/(\d+)?', long_url)[0] + # 构造抖音API链接 + api_url = f'https://www.iesdouyin.com/web/api/v2/aweme/iteminfo/?item_ids={key}' + print("正在请求抖音API链接: " + '\n' + api_url) + # 将回执以JSON格式处理 + js = json.loads(requests.get(url=api_url, headers=headers).text) + # 判断是否为图集 + if js['item_list'][0]['images'] is not None: + print("类型 = 图集") + # 类型为图集 + url_type = 'album' + # 图集标题 + album_title = str(js['item_list'][0]['desc']) + # 图集作者昵称 + album_author = str(js['item_list'][0]['author']['nickname']) + # 图集作者签名 + album_author_signature = str(js['item_list'][0]['author']['signature']) + # 图集作者UID + album_author_uid = str(js['item_list'][0]['author']['uid']) + # 图集作者抖音号 + album_author_id = str(js['item_list'][0]['author']['unique_id']) + if album_author_id == "": + # 如果作者未修改过抖音号,应使用此值以避免无法获取其抖音ID + album_author_id = str(js['item_list'][0]['author']['short_id']) + # 图集BGM链接 + if len(js['item_list'][0]['music']['play_url']['url_list']) > 0: + album_music = str(js['item_list'][0]['music']['play_url']['url_list'][0]) + else: + # 部分视频的API数据中没有BGM链接,返回None + album_music = "None" + # 图集BGM标题 + album_music_title = str(js['item_list'][0]['music']['title']) + # 图集BGM作者 + album_music_author = str(js['item_list'][0]['music']['author']) + # 图集BGM ID + album_music_id = str(js['item_list'][0]['music']['id']) + # 图集BGM MID + album_music_mid = str(js['item_list'][0]['music']['mid']) + # 图集ID + album_aweme_id = str(js['item_list'][0]['statistics']['aweme_id']) + # 评论数量 + album_comment_count = str(js['item_list'][0]['statistics']['comment_count']) + # 获赞数量 + album_digg_count = str(js['item_list'][0]['statistics']['digg_count']) + # 播放次数 + album_play_count = str(js['item_list'][0]['statistics']['play_count']) + # 分享次数 + album_share_count = str(js['item_list'][0]['statistics']['share_count']) + # 上传时间戳 + album_create_time = str(js['item_list'][0]['create_time']) + # 将话题保存在列表中 + album_hashtags = [] + for tag in js['item_list'][0]['text_extra']: + album_hashtags.append(tag['hashtag_name']) + # 将无水印图片链接保存在列表中 + images_list = [] + for data in js['item_list'][0]['images']: + images_list.append(data['url_list'][0]) + # 结束时间 + end = time.time() + # 解析时间 + analyze_time = format((end - start), '.4f') + # 将信息储存在字典中 + album_data = {'status': 'success', + 'analyze_time': (analyze_time + 's'), + 'url_type': url_type, + 'platform': 'douyin', + 'original_url': original_url, + 'api_url': api_url, + 'album_aweme_id': album_aweme_id, + 'album_title': album_title, + 'album_author': album_author, + 'album_author_signature': album_author_signature, + 'album_author_uid': album_author_uid, + 'album_author_id': album_author_id, + 'album_music': album_music, + 'album_music_title': album_music_title, + 'album_music_author': album_music_author, + 'album_music_id': album_music_id, + 'album_music_mid': album_music_mid, + 'album_comment_count': album_comment_count, + 'album_digg_count': album_digg_count, + 'album_play_count': album_play_count, + 'album_share_count': album_share_count, + 'album_create_time': album_create_time, + 'album_list': images_list, + 'album_hashtags': album_hashtags} + return album_data + else: + print("类型 = 视频") + # 类型为视频 + url_type = 'video' + # 视频标题 + video_title = str(js['item_list'][0]['desc']) + # 视频作者昵称 + video_author = str(js['item_list'][0]['author']['nickname']) + # 视频作者抖音号 + video_author_id = str(js['item_list'][0]['author']['unique_id']) + if video_author_id == "": + # 如果作者未修改过抖音号,应使用此值以避免无法获取其抖音ID + video_author_id = str(js['item_list'][0]['author']['short_id']) + # 有水印视频链接 + wm_video_url = str(js['item_list'][0]['video']['play_addr']['url_list'][0]) + # 无水印视频链接 (在回执JSON中将关键字'playwm'替换为'play'即可获得无水印地址) + nwm_video_url = str(js['item_list'][0]['video']['play_addr']['url_list'][0]).replace('playwm', 'play') + # 去水印后视频链接(2022年1月1日抖音APi获取到的URL会进行跳转,需要在Location中获取直链) + r = requests.get(url=nwm_video_url, headers=headers, allow_redirects=False) + video_url = r.headers['Location'] + # 视频背景音频 + if len(js['item_list'][0]['music']['play_url']['url_list']) > 0: + video_music = str(js['item_list'][0]['music']['play_url']['url_list'][0]) + else: + # 部分视频的API数据中没有BGM链接,返回None + video_music = "None" + # 视频作者签名 + video_author_signature = str(js['item_list'][0]['author']['signature']) + # 视频作者UID + video_author_uid = str(js['item_list'][0]['author']['uid']) + # 视频BGM标题 + video_music_title = str(js['item_list'][0]['music']['title']) + # 视频BGM作者 + video_music_author = str(js['item_list'][0]['music']['author']) + # 视频BGM ID + video_music_id = str(js['item_list'][0]['music']['id']) + # 视频BGM MID + video_music_mid = str(js['item_list'][0]['music']['mid']) + # 视频ID + video_aweme_id = str(js['item_list'][0]['statistics']['aweme_id']) + # 评论数量 + video_comment_count = str(js['item_list'][0]['statistics']['comment_count']) + # 获赞数量 + video_digg_count = str(js['item_list'][0]['statistics']['digg_count']) + # 播放次数 + video_play_count = str(js['item_list'][0]['statistics']['play_count']) + # 分享次数 + video_share_count = str(js['item_list'][0]['statistics']['share_count']) + # 上传时间戳 + video_create_time = str(js['item_list'][0]['create_time']) + # 将话题保存在列表中 + video_hashtags = [] + for tag in js['item_list'][0]['text_extra']: + video_hashtags.append(tag['hashtag_name']) + # 结束时间 + end = time.time() + # 解析时间 + analyze_time = format((end - start), '.4f') + # 返回包含数据的字典 + video_data = {'status': 'success', + 'analyze_time': (analyze_time + 's'), + 'url_type': url_type, + 'platform': 'douyin', + 'original_url': original_url, + 'api_url': api_url, + 'video_title': video_title, + 'nwm_video_url': video_url, + 'wm_video_url': wm_video_url, + 'video_aweme_id': video_aweme_id, + 'video_author': video_author, + 'video_author_signature': video_author_signature, + 'video_author_uid': video_author_uid, + 'video_author_id': video_author_id, + 'video_music': video_music, + 'video_music_title': video_music_title, + 'video_music_author': video_music_author, + 'video_music_id': video_music_id, + 'video_music_mid': video_music_mid, + 'video_comment_count': video_comment_count, + 'video_digg_count': video_digg_count, + 'video_play_count': video_play_count, + 'video_share_count': video_share_count, + 'video_create_time': video_create_time, + 'video_hashtags': video_hashtags} + return video_data + except Exception as e: + # 返回异常 + return {'status': 'failed', 'reason': e, 'function': 'Scraper.douyin()', 'value': original_url} + + @retry(stop_max_attempt_number=6) + def tiktok(self, original_url): + """ + 解析TikTok链接 + :param original_url:TikTok链接 + :return:包含信息的字典 + """ + headers = self.headers + # 开始时间 + start = time.time() + # 校验TikTok链接 + if original_url[:12] == "https://www.": + original_url = original_url + print("目标链接: ", original_url) + else: + # 从请求头中获取原始链接 + response = requests.get(url=original_url, headers=headers, allow_redirects=False) + true_link = response.headers['Location'].split("?")[0] + # TikTok请求头返回的第二种链接类型 + if '.html' in true_link: + response = requests.get(url=true_link, headers=headers, allow_redirects=False) + original_url = response.headers['Location'].split("?")[0] + print("目标链接: ", original_url) + try: + # 开始获取TikTok数据 + tiktok_headers = self.tiktok_headers + html = requests.get(url=original_url, headers=tiktok_headers) + res = re.search('") + put_error("发生了了意料之外的错误,输入值已被记录。") + put_html('

⚠详情

') + put_table([ + ['函数名', '原因', '输入值'], + [function, str(reason), value]]) + put_markdown('可能的原因:') + put_markdown('服务器可能被目标主机的防火墙限流(稍等片刻后再次尝试)') + put_markdown('输入了错误的链接(暂不支持主页链接解析)') + put_markdown('该视频已经被删除或屏蔽(你看的都是些啥(⊙_⊙)?)') + put_markdown('你可以在右上角的关于菜单中查看本站错误日志。') + put_markdown('[点击此处在GayHub上进行反馈](https://github.com/Evil0ctal/TikTokDownloader_PyWebIO/issues)') + put_html("
") + # 将错误记录在logs.txt中 + error_date = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + with open('logs.txt', 'a') as f: + f.write(error_date + ":\n" + function + ': ' + str(reason) + '\n' + "Input value: " + value + '\n') + + +def put_douyin_result(item): + # 向前端输出表格 + api = Scraper() + # 抖音数据 + douyin_date = api.douyin(item) + # API链接 + short_api_url = 'https://api.douyin.wtf/api?url=' + item + download_video = 'https://api.douyin.wtf/video?url=' + item + download_bgm = 'https://api.douyin.wtf/music?url=' + item + if douyin_date['status'] == 'success': + if douyin_date['url_type'] == 'video': + put_table([ + ['类型', '内容'], + ['格式:', douyin_date['url_type']], + ['视频直链: ', put_link('点击打开视频', douyin_date['nwm_video_url'], new_window=True)], + ['视频下载:', put_link('点击下载', download_video, new_window=True)], + ['背景音乐直链: ', put_link('点击打开音频', douyin_date['video_music'], new_window=True)], + ['背景音乐下载:', put_link('点击下载', download_bgm, new_window=True)], + ['视频标题: ', douyin_date['video_title']], + ['作者昵称: ', douyin_date['video_author']], + ['作者抖音ID: ', douyin_date['video_author_id']], + ['原视频链接: ', put_link('点击打开原视频', item, new_window=True)], + ['当前视频API链接: ', put_link('点击浏览API数据', douyin_date['api_url'], new_window=True)], + ['当前视频精简API链接: ', put_link('点击浏览API数据', short_api_url, new_window=True)] + ]) + return 'success' + else: + put_table([ + ['类型', '内容'], + ['格式:', douyin_date['url_type']], + ['背景音乐直链: ', put_link('点击打开音频', douyin_date['url_type'], new_window=True)], + ['背景音乐下载:', put_link('点击下载', download_bgm, new_window=True)], + ['视频标题: ', douyin_date['album_title']], + ['作者昵称: ', douyin_date['album_author']], + ['作者抖音ID: ', douyin_date['album_author_id']], + ['原视频链接: ', put_link('点击打开原视频', douyin_date['original_url'], new_window=True)], + ['当前视频API链接: ', put_link('点击浏览API数据', douyin_date['api_url'], new_window=True)], + ['当前视频精简API链接: ', put_link('点击浏览API数据', 'short_api_url', new_window=True)] + ]) + for i in douyin_date['album_list']: + put_table([ + ['图片直链: ', put_link('点击打开图片', i, new_window=True), put_image(i)] + ]) + return 'success' + else: + # {'status': 'failed', 'reason': e, 'function': 'API.tiktok()', 'value': original_url} + reason = douyin_date['reason'] + function = douyin_date['function'] + value = douyin_date['value'] + error_do(reason, function, value) + return 'failed' + + +def put_tiktok_result(item): + # 将TikTok结果显示在前端 + api = Scraper() + # TikTok数据 + tiktok_date = api.tiktok(item) + if tiktok_date['status'] == 'success': + # API链接 + short_api_url = 'https://api.douyin.wtf/api?url=' + item + download_video = 'https://api.douyin.wtf/video?url=' + item + download_bgm = 'https://api.douyin.wtf/music?url=' + item + put_table([ + ['类型', '内容'], + ['视频标题: ', tiktok_date['video_title']], + ['视频直链(有水印): ', put_link('点击打开视频', tiktok_date['wm_video_url'], new_window=True)], + ['视频直链(无水印): ', put_link('点击打开视频', tiktok_date['nwm_video_url'], new_window=True)], + ['视频下载(无水印):', put_link('点击下载', download_video, new_window=True)], + ['音频(名称-作者):', tiktok_date['video_music_title'] + " - " + tiktok_date['video_music_author']], + ['音频播放:', put_link('点击播放', tiktok_date['video_music_url'], new_window=True)], + ['作者昵称: ', tiktok_date['video_author_nickname']], + ['作者ID: ', tiktok_date['video_author']], + ['粉丝数量: ', tiktok_date['video_author_followerCount']], + ['关注他人数量: ', tiktok_date['video_author_followingCount']], + ['获赞总量: ', tiktok_date['video_author_heartCount']], + ['视频总量: ', tiktok_date['video_author_videoCount']], + ['原视频链接: ', put_link('点击打开原视频', item, new_window=True)], + ['当前视频API链接: ', put_link('点击浏览API数据', short_api_url, new_window=True)] + ]) + return 'success' + else: + # {'status': 'failed', 'reason': e, 'function': 'API.tiktok()', 'value': original_url} + reason = tiktok_date['reason'] + function = tiktok_date['function'] + value = tiktok_date['value'] + error_do(reason, function, value) + return 'failed' + + +def ios_pop_window(): + with popup("iOS快捷指令"): + put_text('快捷指令需要在抖音或TikTok的APP内,浏览你想要无水印保存的视频或图集。') + put_text('点击分享按钮,然后下拉找到 "抖音TikTok无水印下载" 这个选项。') + put_text('如遇到通知询问是否允许快捷指令访问xxxx (域名或服务器),需要点击允许才可以正常使用。') + put_text('该快捷指令会在你相册创建一个新的相薄方便你浏览保存到内容。') + put_html('
') + put_link('[点击获取快捷指令]', 'https://www.icloud.com/shortcuts/e8243369340548efa0d4c1888dd3c170', + new_window=True) + + +def api_document_pop_window(): + with popup("API文档"): + put_markdown("💽API文档") + put_markdown("API可将请求参数转换为需要提取的无水印视频/图片直链,配合IOS捷径可实现应用内下载。") + put_link('[中文文档]', 'https://github.com/Evil0ctal/TikTokDownloader_PyWebIO#%EF%B8%8Fapi%E4%BD%BF%E7%94%A8', + new_window=True) + put_html('
') + put_link('[English doc]', + 'https://github.com/Evil0ctal/TikTokDownloader_PyWebIO/blob/main/README.en.md#%EF%B8%8Fapi-usage', + new_window=True) + put_html('
') + put_markdown("🛰️API参考") + put_markdown('抖音/TikTok解析请求参数') + put_code('http://localhost(服务器IP):2333/api?url="复制的(抖音/TikTok)的(分享文本/链接)"\n#返回JSON') + put_markdown('抖音/TikTok视频下载请求参数') + put_code('http://localhost(服务器IP):2333/video?url="复制的抖音/TikTok链接"\n' + '# 返回mp4文件下载请求\n' + '# 大量请求时很吃服务器内存,容易崩,慎用。') + put_markdown('抖音视频/图集音频下载请求参数') + put_code('http://localhost(服务器IP):2333/music?url="复制的抖音/TikTok链接"\n' + '# 返回mp3文件下载请求\n' + '# 大量请求时很吃服务器内存,容易崩,慎用。') + + +def log_popup_window(): + with popup('错误日志'): + put_html('

⚠️关于解析失败可能的原因

') + put_markdown('服务器可能被目标主机的防火墙限流(稍等片刻后再次尝试)') + put_markdown('输入了错误的链接(暂不支持主页链接解析)') + put_markdown('该视频已经被删除或屏蔽(你看的都是些啥(⊙_⊙)?)') + put_markdown('你可以在右上角的关于菜单中查看本站错误日志。') + put_markdown('[点击此处在GayHub上进行反馈](https://github.com/Evil0ctal/TikTokDownloader_PyWebIO/issues)') + put_html('
') + put_text('点击logs.txt可下载日志:') + content = open(r'./logs.txt', 'rb').read() + put_file('logs.txt', content=content) + with open('./logs.txt', 'r') as f: + content = f.read() + put_text(str(content)) + + +def about_popup_window(): + with popup('更多信息'): + put_html('

👀访问记录

') + put_image('https://views.whatilearened.today/views/github/evil0ctal/TikTokDownload_PyWebIO.svg', + title='访问记录') + put_html('
') + put_html('

⭐Github

') + put_markdown('[TikTokDownloader_PyWebIO](https://github.com/Evil0ctal/TikTokDownloader_PyWebIO)') + put_html('
') + put_html('

🎯反馈

') + put_markdown('提交:[issues](https://github.com/Evil0ctal/TikTokDownloader_PyWebIO/issues)') + put_html('
') + put_html('

🌐视频/图集批量下载

') + put_markdown('可以使用[IDM](https://www.zhihu.com/topic/19746283/hot)之类的工具对结果页面的链接进行嗅探。') + put_markdown('如果你有更好的想法欢迎PR') + put_html('
') + put_html('

💖WeChat

') + put_markdown('微信:[Evil0ctal](https://mycyberpunk.com/)') + put_html('
') + + +@config(title=title, description=description) +def main(): + # 设置favicon + favicon_url = "https://raw.githubusercontent.com/Evil0ctal/TikTokDownloader_PyWebIO/main/favicon/android-chrome-512x512.png" + session.run_js(""" + $('#favicon32,#favicon16').remove(); + $('head').append('') + """ % favicon_url) + # 修改footer + session.run_js("""$('footer').remove()""") + # 访问记录 + view_amount = requests.get("https://views.whatilearened.today/views/github/evil0ctal/TikTokDownload_PyWebIO.svg") + put_markdown("""
😼抖音/TikTok无水印在线解析
""") + put_html('
') + put_row([put_button("快捷指令", onclick=lambda: ios_pop_window(), link_style=True, small=True), + put_button("API", onclick=lambda: api_document_pop_window(), link_style=True, small=True), + put_button("日志", onclick=lambda: log_popup_window(), link_style=True, small=True), + put_button("关于", onclick=lambda: about_popup_window(), link_style=True, small=True) + ]) + placeholder = "批量解析请直接粘贴多个口令或链接,无需使用符号分开,支持抖音和TikTok链接混合,暂时不支持作者主页链接批量解析。" + kou_ling = textarea('请将抖音或TikTok的分享口令或网址粘贴于此', type=TEXT, validate=valid_check, required=True, + placeholder=placeholder, + position=0) + if kou_ling: + if kou_ling == 'wyn': + # 好想你(小彩蛋) + with popup('给 WYN💖'): + put_text('我大约真的没有什么才华,只是因为有幸见着了你,于是这颗庸常的心中才凭空生出好些浪漫。') + put_text('真的好爱你呀!') + put_link('WYN&THB', 'https://www.wynthb.com/') + else: + url_lists = find_url(kou_ling) + total_urls = len(url_lists) + # 解析开始时间 + start = time.time() + # 放一个毫无意义的进度条 + loading() + # 成功/失败统计 + success_count = 0 + failed_count = 0 + # 遍历链接 + for url in url_lists: + if 'douyin.com' in url: + if put_douyin_result(url) == 'failed': + failed_count += 1 + continue + else: + success_count += 1 + else: + if put_tiktok_result(url) == 'failed': + failed_count += 1 + continue + else: + success_count += 1 + clear('bar') + # 解析结束时间 + end = time.time() + put_html("

") + put_text('总共收到' + str(total_urls) + '个链接') + put_text('成功: ' + str(success_count) + ' ' + '失败: ' + str(failed_count)) + put_text('解析共耗时: %.4f秒' % (end - start)) + put_link('返回主页', '/') + + +if __name__ == "__main__": + # 初始化logs.txt + date = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + with open('logs.txt', 'a') as f: + f.write("时间: " + date + " " + "程序重载完毕!" + '\n') + app.add_url_rule('/', 'webio_view', webio_view(main), methods=['GET', 'POST', 'OPTIONS']) + # 获取空闲端口 + if os.environ.get('PORT'): + port = int(os.environ.get('PORT')) + else: + # 在这里修改默认端口(记得在防火墙放行该端口) + port = 5000 + app.run(host='0.0.0.0', port=port) +