mirror of
https://github.com/Evil0ctal/Douyin_TikTok_Download_API.git
synced 2025-04-20 04:19:57 +08:00
Compare commits
117 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
8c98fb7032 | ||
|
|
56f4ce44a2 | ||
|
|
9ab8e11672 | ||
|
|
ab87be43af | ||
|
|
f197efd21d | ||
|
|
03e7be054f | ||
|
|
3c5e38234f | ||
|
|
945aa3d46d | ||
|
|
3b491db03e | ||
|
|
39f3abb5df | ||
|
|
492bf1693b | ||
|
|
f79f67c4db | ||
|
|
400edfad22 | ||
|
|
911d8bf086 | ||
|
|
a1cc2b6056 | ||
|
|
680339db15 | ||
|
|
c1cd8b8863 | ||
|
|
774b6b2890 | ||
|
|
32755dd86b | ||
|
|
16ed4eba5e | ||
|
|
71d6c64490 | ||
|
|
8ae46f7ad7 | ||
|
|
d67aed6f2b | ||
|
|
b438d69f84 | ||
|
|
be7752a812 | ||
|
|
a054dd4276 | ||
|
|
3d25688269 | ||
|
|
ad4e9c2694 | ||
|
|
fd5b7287ee | ||
|
|
54afe5bd9e | ||
|
|
72220dc0c2 | ||
|
|
3b526220dd | ||
|
|
dd8c567978 | ||
|
|
6c09b4181b | ||
|
|
f9b19d3527 | ||
|
|
d224556c69 | ||
|
|
a8be475f94 | ||
|
|
72ea6966fa | ||
|
|
00b227b3ba | ||
|
|
adcd379a8d | ||
|
|
a7f87dff00 | ||
|
|
d641f0340c | ||
|
|
b659af881b | ||
|
|
7d467ee269 | ||
|
|
4f322212a9 | ||
|
|
ca05f11953 | ||
|
|
a6933088e2 | ||
|
|
3221c41135 | ||
|
|
1587b7da77 | ||
|
|
89e2a59f22 | ||
|
|
530d856872 | ||
|
|
f61d362bf9 | ||
|
|
2cdef7c921 | ||
|
|
2980490444 | ||
|
|
6aab4924ac | ||
|
|
538dd590d0 | ||
|
|
46561948a1 | ||
|
|
a118740a2a | ||
|
|
f366393542 | ||
|
|
92b01cda4c | ||
|
|
d49291c423 | ||
|
|
c33095ad7b | ||
|
|
fce0750b58 | ||
|
|
bd650f2ad9 | ||
|
|
2efd3e3eca | ||
|
|
c88ce226c5 | ||
|
|
d75382dabf | ||
|
|
19e9c4d13b | ||
|
|
d1dca800a6 | ||
|
|
260c986e42 | ||
|
|
5c5f9befe0 | ||
|
|
9f03964aa9 | ||
|
|
7f51e46356 | ||
|
|
ed62bc3817 | ||
|
|
8fb7b1bfc1 | ||
|
|
e63778aa8f | ||
|
|
d67006ff3a | ||
|
|
2ea9651eea | ||
|
|
9aabca88f1 | ||
|
|
35f3a51f0a | ||
|
|
d0306c4482 | ||
|
|
46c06d21cc | ||
|
|
1e13e37e98 | ||
|
|
0f0f725772 | ||
|
|
0348205128 | ||
|
|
bbaf20cf8f | ||
|
|
2a0184b6f3 | ||
|
|
007c3dcbbe | ||
|
|
5b72b41d3b | ||
|
|
b09c0da781 | ||
|
|
5fffdfa7f3 | ||
|
|
6e7461fae9 | ||
|
|
a392ccd6bd | ||
|
|
f64e828d5b | ||
|
|
1341b60c4e | ||
|
|
8cfcbd6a08 | ||
|
|
e2537d8821 | ||
|
|
10f703873f | ||
|
|
9fd80d8819 | ||
|
|
cc03bb943e | ||
|
|
b19f637b31 | ||
|
|
ebd7fd28c9 | ||
|
|
a1f26e85ce | ||
|
|
286418c246 | ||
|
|
4190bd7e05 | ||
|
|
4fa7ac0fbe | ||
|
|
3473ca1af8 | ||
|
|
beda8edd64 | ||
|
|
b3d7e13207 | ||
|
|
0081af1ff2 | ||
|
|
225df62be1 | ||
|
|
eb5ed2c9ef | ||
|
|
61ec8bbf2f | ||
|
|
523c3b685d | ||
|
|
788433c0c8 | ||
|
|
527ccc178f | ||
|
|
15a7880776 |
41
Dockerfile
41
Dockerfile
@ -1,43 +1,26 @@
|
||||
# Use the official Ubuntu base image
|
||||
FROM ubuntu:jammy
|
||||
# 使用官方 Python 3.11 的轻量版镜像
|
||||
FROM python:3.11-slim
|
||||
|
||||
LABEL maintainer="Evil0ctal"
|
||||
|
||||
# Set non-interactive frontend (useful for Docker builds)
|
||||
# 设置非交互模式，避免 Docker 构建时的交互问题
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Update the package list and install Python and pip
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
python3.11 \
|
||||
python3-pip \
|
||||
python3.11-dev \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Set a working directory
|
||||
# 设置工作目录
|
||||
WORKDIR /app
|
||||
|
||||
# Copy the application source code to the container
|
||||
# 复制应用代码到容器
|
||||
COPY . /app
|
||||
|
||||
# Install virtualenv
|
||||
RUN pip3 install -i https://mirrors.aliyun.com/pypi/simple/ -U pip \
|
||||
&& pip3 config set global.index-url https://mirrors.aliyun.com/pypi/simple/ \
|
||||
&& pip3 install virtualenv
|
||||
# 使用 Aliyun 镜像源加速 pip
|
||||
RUN pip install -i https://mirrors.aliyun.com/pypi/simple/ -U pip \
|
||||
&& pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/
|
||||
|
||||
# Check virtualenv is installed
|
||||
RUN which virtualenv
|
||||
|
||||
# Create and activate virtual environment using the virtualenv command
|
||||
RUN virtualenv venv -p python3.11
|
||||
|
||||
# Set the virtual environment path
|
||||
ENV PATH="/app/venv/bin:$PATH"
|
||||
|
||||
# Install dependencies in the virtual environment
|
||||
# 安装依赖
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Make the start script executable
|
||||
# 确保启动脚本可执行
|
||||
RUN chmod +x start.sh
|
||||
|
||||
# Command to run on container start
|
||||
# 设置容器启动命令
|
||||
CMD ["./start.sh"]
|
||||
|
||||
334
README.en.md
334
README.en.md
@ -7,101 +7,121 @@
|
||||
|
||||
[English](./README.en.md)\|[Simplified Chinese](./README.md)
|
||||
|
||||
🚀"Douyin_TikTok_Download_API" is a high-performance asynchronous API that can be used out of the box[Tik Tok](https://www.douyin.com)\|[TikTok](https://www.tiktok.com)\|[Bilibili](https://www.bilibili.com)Data crawling tool supports API calling, online batch analysis and downloading.
|
||||
🚀 "Douyin_TikTok_Download_API" is a high-performance asynchronous out-of-the-box[Tik Tok](https://www.douyin.com)\|[Tiktok](https://www.tiktok.com)\|[Biliable](https://www.bilibili.com)Data crawling tool, supports API calls, online batch analysis and download.
|
||||
|
||||
[](LICENSE)[](https://github.com/Evil0ctal/Douyin_TikTok_Download_API/releases/latest)[](https://github.com/Evil0ctal/Douyin_TikTok_Download_API/stargazers)[](https://github.com/Evil0ctal/Douyin_TikTok_Download_API/network/members)[](https://github.com/Evil0ctal/Douyin_TikTok_Download_API/issues)[](https://github.com/Evil0ctal/Douyin_TikTok_Download_API/issues?q=is%3Aissue+is%3Aclosed)<br>[](https://pypi.org/project/douyin-tiktok-scraper/)[](https://pypi.org/project/douyin-tiktok-scraper/#files)[](https://pypi.org/project/douyin-tiktok-scraper/)[](https://pypi.org/project/douyin-tiktok-scraper/)<br>[](https://api.douyin.wtf/docs)[](https://api.tikhub.io/docs)<br>[](https://afdian.net/@evil0ctal)[](https://ko-fi.com/evil0ctal)[](https://www.patreon.com/evil0ctal)
|
||||
|
||||
</div>
|
||||
|
||||
## 🔊 V4.0.0 version refactoring
|
||||
## Sponsors
|
||||
|
||||
> ALL:
|
||||
These sponsors have paid to place them here,**Doinan_tics_download_api**The project will always be free and open source. If you wish to be a sponsor of this project, please check out my[GitHub Sponsor Page](https://github.com/sponsors/evil0ctal)。
|
||||
|
||||
- Removed outdated bilibili code and needs someone to rewrite it.
|
||||
- Someone in the group wants to add the analysis of Kuaishou and Xigua videos.
|
||||
- The readme is outdated and needs to be rewritten.
|
||||
- Make PyPi package
|
||||
- The config.yaml file needs to be trimmed.
|
||||
- Add parsing of user homepage.
|
||||
- iOS shortcuts need to be updated to be compatible with the latest API responses and paths.
|
||||
- Desktop downloaders or browser plug-ins can be developed if necessary.
|
||||
- Solve the problem of crawler cookie risk control.
|
||||
<div align="center">
|
||||
<a href="https://www.tikhub.io/" target="_blank">
|
||||
<img src="https://tikhub.io/wp-content/uploads/2024/11/Main-Logo.webp" width="100" alt="TikHub.io - Global Social Data & API Marketplace">
|
||||
</a>
|
||||
<div>
|
||||
<h2><b>TikHub.io</b></h2>
|
||||
<p>Your Ultimate Social Media Data & API Marketplace</p>
|
||||
<p>
|
||||
Professional data solutions for Douyin, Xiaohongshu, TikTok, Instagram, YouTube,
|
||||
Twitter, and more.<br>
|
||||
Real-time Data | Flexible APIs | Seamless Integration | Competitive Pricing with Discounts
|
||||
</p>
|
||||
<p>
|
||||
<b>Discover TikHub.io Marketplace</b><br>
|
||||
Buy and sell custom APIs, services, and social media solutions.<br>
|
||||
Join a thriving ecosystem of developers, businesses, and content creators.
|
||||
</p>
|
||||
<p><em>Trusted by leading global influencer marketing and social media intelligence platforms</em></p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
> Change
|
||||
## 👻 Introduction
|
||||
|
||||
- Run Pywebio as a sub-APP of FastAPI.
|
||||
- Rewritten the interfaces of Douyin and TikTok, thank you[@johnserf-seed](https://github.com/Johnserf-Seed)
|
||||
- The file download endpoint has been rewritten and now uses asynchronous file IO.
|
||||
- Annotations and demonstration values were added to all endpoints.
|
||||
- Organize the project file structure.
|
||||
> 🚨If you want to use a private server to run this project, please refer to:[Deployment preparations](./README.md#%EF%B8%8F%E9%83%A8%E7%BD%B2%E5%89%8D%E7%9A%84%E5%87%86%E5%A4%87%E5%B7%A5%E4%BD%9C%E8%AF%B7%E4%BB%94%E7%BB%86%E9%98%85%E8%AF%BB),[Docker deployment](./README.md#%E9%83%A8%E7%BD%B2%E6%96%B9%E5%BC%8F%E4%BA%8C-docker),[One-click deployment](./README.md#%E9%83%A8%E7%BD%B2%E6%96%B9%E5%BC%8F%E4%B8%80-linux)
|
||||
|
||||
> Remark
|
||||
|
||||
If you are interested in writing this project together, please add us on WeChat`Evil0ctal`Note: Github project reconstruction, everyone can communicate and learn from each other in the group. Advertising and illegal things are not allowed. It is purely for making friends and technical exchanges.
|
||||
|
||||
> 私有接口服务
|
||||
|
||||
Discord:[Tikhub discord](https://discord.com/invite/aMEAS8Xsvz)
|
||||
|
||||
Free Douyin/TikTok API:[Tikhub Beta Opi](https://beta.tikhub.io/)
|
||||
|
||||
## 👻Introduction
|
||||
|
||||
> 🚨If you need to use a private server to run this project, please refer to the deployment method\[[Docker deployment](./README.md#%E9%83%A8%E7%BD%B2%E6%96%B9%E5%BC%8F%E4%BA%8C-docker),[One-click deployment](./README.md#%E9%83%A8%E7%BD%B2%E6%96%B9%E5%BC%8F%E4%B8%80-linux)]
|
||||
|
||||
This project is based on[PyWebIO](https://github.com/pywebio/PyWebIO)，[FastAPI](https://fastapi.tiangolo.com/)，[HTTPX](https://www.python-httpx.org/), fast and asynchronous[Tik Tok](https://www.douyin.com/)/[TikTok](https://www.tiktok.com/)Data crawling tool, and realizes online batch parsing and downloading of videos or photo albums without watermarks, data crawling API, iOS shortcut commands without watermark downloads through the Web, etc. You can deploy or modify this project yourself to achieve more functions, or you can call it directly in your project[scraper.py](https://github.com/Evil0ctal/Douyin_TikTok_Download_API/blob/Stable/scraper.py)or install an existing[pip package](https://pypi.org/project/douyin-tiktok-scraper/)As a parsing library, it is easy to crawl data, etc.....
|
||||
This project is based on[Pydebio](https://github.com/pywebio/PyWebIO)，[Fasting](https://fastapi.tiangolo.com/)，[HTTPX](https://www.python-httpx.org/), fast asynchronous[Tik Tok](https://www.douyin.com/)/[Tiktok](https://www.tiktok.com/)Data crawling tool, and online batch analysis and downloading of watermark-free videos or picture albums through the web, data crawling API, iOS shortcuts without watermark download and other functions. You can deploy or transform this project yourself to achieve more functions, or you can call it directly in your project[scraper.py](https://github.com/Evil0ctal/Douyin_TikTok_Download_API/blob/Stable/scraper.py)Or install an existing one[pip package](https://pypi.org/project/douyin-tiktok-scraper/)As a parsing library, easy to crawl data, etc....
|
||||
|
||||
_Some simple application scenarios:_
|
||||
|
||||
_Download prohibited videos, perform data analysis, download without watermark on iOS (with[Shortcut command APP that comes with iOS](https://apps.apple.com/cn/app/%E5%BF%AB%E6%8D%B7%E6%8C%87%E4%BB%A4/id915249334)Cooperate with the API of this project to achieve in-app downloads or read clipboard downloads), etc....._
|
||||
_Download videos that are prohibited from being downloaded, perform data analysis, and download without watermark on iOS (with[iOS's shortcut command APP](https://apps.apple.com/cn/app/%E5%BF%AB%E6%8D%B7%E6%8C%87%E4%BB%A4/id915249334)In conjunction with this project API, it can realize in-app download or read clipboard download, etc...._
|
||||
|
||||
#### ⚠️Note:
|
||||
## 🔊 V4 version notes
|
||||
|
||||
- You need to solve crawler cookie risk control issues by yourself, otherwise the interface may become unusable.
|
||||
- Douyin web cookie (obtain and replace the cookie in the configuration file below):
|
||||
- <https://github.com/Evil0ctal/Douyin_TikTok_Download_API/blob/30e56e5a7f97f87d60b1045befb1f6db147f8590/crawlers/douyin/web/config.yaml#L7>
|
||||
- TikTok web-side cookies (obtain and replace the cookies in the configuration file below):
|
||||
- <https://github.com/Evil0ctal/Douyin_TikTok_Download_API/blob/30e56e5a7f97f87d60b1045befb1f6db147f8590/crawlers/tiktok/web/config.yaml#L6>
|
||||
- I turned off the online download function of the demo site. The video someone downloaded was so huge that it crashed my server. I just right-clicked on the web page parsing result and saved it...
|
||||
- The cookies of the demo site are my own and are not guaranteed to be valid for a long time. They only serve as a demonstration. If you deploy it yourself, please obtain the cookies yourself.
|
||||
- If you are interested in writing this project, please add WeChat`Evil0ctal`Note: Github project reconstruction, everyone can communicate and learn from each other in the group, and do not allow advertisements or illegal things to be made purely friends and technical communication.
|
||||
- This project uses`X-Bogus`Algorithm and`A_Bogus`The algorithm requests TikTok and TikTok's Web API.
|
||||
- Due to Douyin's risk control, please go to**Get the Douyin website cookies in the browser and replace them in config.yaml.**
|
||||
- Please read the document below before asking for an issue, and most solutions to the problem will be included in the document.
|
||||
- This project is completely free, but please follow it when using it:[Apache-2.0 license](https://github.com/Evil0ctal/Douyin_TikTok_Download_API?tab=Apache-2.0-1-ov-file#readme)
|
||||
|
||||
## 🖥Demo site: I am very vulnerable...please do not stress test (·•᷄ࡇ•᷅ )
|
||||
## 🔖TikHub.io API
|
||||
|
||||
> 😾The online download function of the demo site is closed.
|
||||
[TikHub.io](https://api.tikhub.io/)It is an API platform that provides various public data interfaces including Douyin and TikTok. If you want to support it[Doinan_tics_download_api](https://github.com/Evil0ctal/Douyin_TikTok_Download_API)We strongly recommend that you choose the project development[TikHub.io](https://api.tikhub.io/)。
|
||||
|
||||
#### Features:
|
||||
|
||||
> 📦 Out of the box
|
||||
|
||||
Simplify the usage process and quickly carry out development work using the encapsulated SDK. All API interfaces are designed according to the RESTful architecture and are described and documented using the OpenAPI specification, accompanied by example parameters to ensure that calls are easier.
|
||||
|
||||
> 💰 Cost Advantage
|
||||
|
||||
There is no preset package limit, no monthly usage threshold, all consumption is billed instantly based on the actual usage, and is billed step by step based on the user's daily request volume. At the same time, you can check in in the user's background through daily check-in, and these free amounts will not expire.
|
||||
|
||||
> ⚡️ Quick support
|
||||
|
||||
We have a huge Discord community server where administrators and other users will quickly reply to you to help you quickly resolve current issues.
|
||||
|
||||
> 🎉 Embrace open source
|
||||
|
||||
Some of the source code of TikHub will be open sourced on Github and will sponsor some open source projects.
|
||||
|
||||
#### Link:
|
||||
|
||||
- Githubub:[TIKHOB GITUB](https://github.com/TikHubIO)
|
||||
- Discord:[Tachub](https://discord.com/invite/aMEAS8Xsvz)
|
||||
- Register:[TikHub singnup](https://beta-web.tikhub.io/en-us/users/signup)
|
||||
- API Docs:[TickHub API Docs](https://api.tikhub.io/)
|
||||
|
||||
## 🖥 Demo site: I am very fragile... Please do not press test (·•᷄ࡇ•᷅ )
|
||||
|
||||
> 😾The online download function of the demo site has been turned off, and Douyin's parsing and API services cannot be guaranteed for availability on the Demo site due to cookies.
|
||||
|
||||
🍔Web APP:<https://douyin.wtf/>
|
||||
|
||||
🍟API Document:<https://douyin.wtf/docs>
|
||||
|
||||
🌭TikHub API Document:<https://api.tikhub.io/docs>
|
||||
🌭tikub APU Docuration:<https://api.tikhub.io/docs>
|
||||
|
||||
💾iOS Shortcut (shortcut command):[Shortcut release](https://github.com/Evil0ctal/Douyin_TikTok_Download_API/discussions/104?sort=top)
|
||||
💾 iOS Shortcut:[Shortcut release](https://github.com/Evil0ctal/Douyin_TikTok_Download_API/discussions/104?sort=top)
|
||||
|
||||
📦️Desktop downloader (recommended by warehouse):
|
||||
📦️Desktop downloader (recommended warehouse):
|
||||
|
||||
- [Johnserf-Seed/TikTokDownload](https://github.com/Johnserf-Seed/TikTokDownload)
|
||||
- [Johnserf-Seed/Tiktokdownload](https://github.com/Johnserf-Seed/TikTokDownload)
|
||||
- [HFrost0/bilix](https://github.com/HFrost0/bilix)
|
||||
- [Tairraos/TikDown - \[needs update\]](https://github.com/Tairraos/TikDown/)
|
||||
- [Tairraos/TikDown - \[Updated to be\]](https://github.com/Tairraos/TikDown/)
|
||||
|
||||
## ⚗️Technology stack
|
||||
## ⚗️Technology Stack
|
||||
|
||||
- [/app/web](https://github.com/Evil0ctal/Douyin_TikTok_Download_API/blob/main/app/web)-[PyWebIO](https://www.pyweb.io/)
|
||||
- [/app/api](https://github.com/Evil0ctal/Douyin_TikTok_Download_API/blob/main/app/api)-[FastAPI](https://fastapi.tiangolo.com/)
|
||||
- [/app/web](https://github.com/Evil0ctal/Douyin_TikTok_Download_API/blob/main/app/web)-[Pydebio](https://www.pyweb.io/)
|
||||
- [/app/api](https://github.com/Evil0ctal/Douyin_TikTok_Download_API/blob/main/app/api)-[Fasting](https://fastapi.tiangolo.com/)
|
||||
- [/crawlers](https://github.com/Evil0ctal/Douyin_TikTok_Download_API/blob/main/crawlers)-[HTTPX](https://www.python-httpx.org/)
|
||||
|
||||
> **_/crawlers_**
|
||||
|
||||
- Submit requests to APIs on different platforms and retrieve data. After processing, a dictionary (dict) is returned, and asynchronous support is supported.
|
||||
- Submit requests to APIs of different platforms and retrieve data, and return dictionary (dict) after processing, supports asynchronousness.
|
||||
|
||||
> **_/app/api_**
|
||||
|
||||
- Get request parameters and use`Crawlers`The related classes process the data and return it in JSON form, download the video, and cooperate with iOS shortcut commands to achieve fast calling and support asynchronous.
|
||||
- Obtain the request parameters and use`Crawlers`After processing data, the related classes return in JSON form, download videos, and implement fast calls with iOS shortcuts, and support asynchronous.
|
||||
|
||||
> **_/app/web_**
|
||||
|
||||
- use`PyWebIO`A simple web program created to process the values entered on the web page and then use them`Crawlers`The related class processing interface outputs related data on the web page.
|
||||
- use`PyWebIO`A simple web program created, process the value entered on the web page and use it`Crawlers`The related class processing interface outputs related data on the web page.
|
||||
|
||||
**_Most of the parameters of the above files can be found in the corresponding`config.yaml`Modify in_**
|
||||
**_Most of the parameters of the above files can be in the corresponding`config.yaml`Make modifications in_**
|
||||
|
||||
## 💡Project file structure
|
||||
|
||||
@ -114,6 +134,8 @@ _Download prohibited videos, perform data analysis, download without watermark o
|
||||
│ └─web
|
||||
│ └─views
|
||||
└─crawlers
|
||||
├─bilibili
|
||||
│ └─web
|
||||
├─douyin
|
||||
│ └─web
|
||||
├─hybrid
|
||||
@ -122,65 +144,83 @@ _Download prohibited videos, perform data analysis, download without watermark o
|
||||
│ └─web
|
||||
└─utils
|
||||
|
||||
## ✨Supported functions:
|
||||
## ✨Support functions:
|
||||
|
||||
- 网页端批量解析(支持抖音/TikTok混合解析)
|
||||
- Download videos or photo albums online.
|
||||
- make[pip package](https://pypi.org/project/douyin-tiktok-scraper/)Conveniently and quickly import your projects
|
||||
- [iOS shortcut commands to quickly call API](https://apps.apple.com/cn/app/%E5%BF%AB%E6%8D%B7%E6%8C%87%E4%BB%A4/id915249334)Achieve in-app download of watermark-free videos/photo albums
|
||||
- Complete API documentation ([Demo/Demonstration](https://api.douyin.wtf/docs))
|
||||
- Rich API interface:
|
||||
- Batch analysis on the web side (supports Douyin/TikTok hybrid analysis)
|
||||
- Download videos or albums online.
|
||||
- Production[pip package](https://pypi.org/project/douyin-tiktok-scraper/)方便快速导入你的项目
|
||||
- [iOS shortcuts to quickly call API](https://apps.apple.com/cn/app/%E5%BF%AB%E6%8D%B7%E6%8C%87%E4%BB%A4/id915249334)Implement watermark-free videos/pictures in-app download
|
||||
- Complete API documentation ([Demo/Demo](https://api.douyin.wtf/docs))
|
||||
- Rich API interfaces:
|
||||
- TikTok web version API
|
||||
|
||||
- Douyin web version API
|
||||
- [x] Video data analysis
|
||||
- [x] Get user homepage work data
|
||||
- [x] Obtain the data of works liked by the user's homepage
|
||||
- [x] Obtain the data of collected works on the user's homepage
|
||||
- [x] Obtain user's homepage work data
|
||||
- [x] Obtain data on the user's homepage liked works
|
||||
- [x] Obtain data on the user's homepage collection of works
|
||||
- [x] Get user homepage information
|
||||
- [x] Get user collection work data
|
||||
- [x] Get user live stream data
|
||||
- [x] Get the live streaming data of a specified user
|
||||
- [x] Get the ranking of users who give gifts in the live broadcast room
|
||||
- [x] Get single video comment data
|
||||
- [x] Get the comment reply data of the specified video
|
||||
- [x] Obtain user compiled works data
|
||||
- [x] Obtain user live streaming data
|
||||
- [x] Get live streaming data for the specified user
|
||||
- [x] Get the ranking of gift-giving users in the live broadcast room
|
||||
- [x] Get individual video comment data
|
||||
- [x] Get comments and response data for specified videos
|
||||
- [x] Generate msToken
|
||||
- [x] Generate verify_fp
|
||||
- [x] Generate verification_fp
|
||||
- [x] Generate s_v_web_id
|
||||
- [x] Generate X-Bogus parameters using interface URL
|
||||
- [x] Generate A_Bogus parameters using interface URL
|
||||
- [x] Extract a single user id
|
||||
- [x] Extract list user id
|
||||
- [x] Extract a single work id
|
||||
- [x] Extract individual works id
|
||||
- [x] Extract list work id
|
||||
- [x] Extract live broadcast room number from list
|
||||
- [x] Extract live broadcast room number from list
|
||||
|
||||
- [x] Extract list live broadcast room number
|
||||
- [x] Extract list live broadcast room number
|
||||
- TikTok web version API
|
||||
|
||||
- [x] Video data analysis
|
||||
- [x] Get user homepage work data
|
||||
- [x] Obtain the data of works liked by the user's homepage
|
||||
- [x] Obtain user's homepage work data
|
||||
- [x] Obtain data on the user's homepage liked works
|
||||
- [x] Get user homepage information
|
||||
- [x] Get fan data on user homepage
|
||||
- [x] Get user homepage follow data
|
||||
- [x] Get user homepage collection work data
|
||||
- [x] Get user homepage collection data
|
||||
- [x] Get the user's homepage fan data
|
||||
- [x] Get user's homepage follow data
|
||||
- [x] 获取用户主页合辑作品数据
|
||||
- [x] Get search data for users' homepage
|
||||
- [x] Get user homepage playlist data
|
||||
- [x] Get single video comment data
|
||||
- [x] Get the comment reply data of the specified video
|
||||
- [x] Get individual video comment data
|
||||
- [x] Get comments and response data for specified videos
|
||||
- [x] Generate msToken
|
||||
- [x] Generate ttwid
|
||||
- [x] Generate X-Bogus parameters using interface URL
|
||||
- [x] Extract a single user sec_user_id
|
||||
- [x] Extract individual user sec_user_id
|
||||
- [x] Extract list user sec_user_id
|
||||
- [x] Extract a single work id
|
||||
- [x] Extract individual works id
|
||||
- [x] Extract list work id
|
||||
- [x] Get user unique_id
|
||||
- [x] Get list unique_id
|
||||
- [x] Get the list unique_id
|
||||
- Bilibili web version API
|
||||
- [x] Get individual video details
|
||||
- [x] Get the video streaming address
|
||||
- [x] Obtain data on video works published by users
|
||||
- [x] Get all user favorites information
|
||||
- [x] Get video data in the specified favorites
|
||||
- [x] Get information about the specified user
|
||||
- [x] Get comprehensive popular video information
|
||||
- [x] Get comments for the specified video
|
||||
- [x] Get a reply to the specified comment under the video
|
||||
- [x] Get the specified user dynamics
|
||||
- [x] Get real-time video barrage
|
||||
- [x] Get information about the specified live broadcast room
|
||||
- [x] Get live video streaming
|
||||
- [x] Get the anchor who is currently broadcasting in the specified partition
|
||||
- [x] Get a list of all live partitions
|
||||
- [x] Obtain video score information through bv number
|
||||
|
||||
* * *
|
||||
|
||||
## 📦Call the parsing library (obsolete and needs to be updated):
|
||||
## 📦 Call the parsing library (deprecated and needs to be updated):
|
||||
|
||||
> 💡PyPi:<https://pypi.org/project/douyin-tiktok-scraper/>
|
||||
> 💡PIPI ：<https://pypi.org/project/douyin-tiktok-scraper/>
|
||||
|
||||
Install the parsing library:`pip install douyin-tiktok-scraper`
|
||||
|
||||
@ -201,45 +241,45 @@ asyncio.run(hybrid_parsing(url=input("Paste Douyin/TikTok/Bilibili share URL her
|
||||
|
||||
## 🗺️Supported submission formats:
|
||||
|
||||
> 💡Tip: Including but not limited to the following examples. If you encounter link parsing failure, please open a new one.[issue](https://github.com/Evil0ctal/Douyin_TikTok_Download_API/issues)
|
||||
> 💡 Tip: Includes but is not limited to the following examples. If you encounter link resolution failure, please enable a new one.[issue](https://github.com/Evil0ctal/Douyin_TikTok_Download_API/issues)
|
||||
|
||||
- Douyin sharing password (copy in APP)
|
||||
- TikTok Sharing Password (Copy within the APP)
|
||||
|
||||
```text
|
||||
7.43 pda:/ 让你在几秒钟之内记住我 https://v.douyin.com/L5pbfdP/ 复制此链接，打开Dou音搜索，直接观看视频！
|
||||
```
|
||||
|
||||
- Douyin short URL (copy within APP)
|
||||
- TikTok short URL (copy within the APP)
|
||||
|
||||
```text
|
||||
https://v.douyin.com/L4FJNR3/
|
||||
```
|
||||
|
||||
- Douyin normal URL (copy from web version)
|
||||
- Douyin Normal URL (web version copy)
|
||||
|
||||
```text
|
||||
https://www.douyin.com/video/6914948781100338440
|
||||
```
|
||||
|
||||
- Douyin discovery page URL (APP copy)
|
||||
- TikTok Discovery Page URL (APP Copy)
|
||||
|
||||
```text
|
||||
https://www.douyin.com/discover?modal_id=7069543727328398622
|
||||
```
|
||||
|
||||
- TikTok short URL (copy within APP)
|
||||
- TikTok short URL (copy within the APP)
|
||||
|
||||
```text
|
||||
https://www.tiktok.com/t/ZTR9nDNWq/
|
||||
```
|
||||
|
||||
- TikTok normal URL (copy from web version)
|
||||
- TikTok normal website address (web version copy)
|
||||
|
||||
```text
|
||||
https://www.tiktok.com/@evil0ctal/video/7156033831819037994
|
||||
```
|
||||
|
||||
- Douyin/TikTok batch URL (no need to use matching separation)
|
||||
- TikTok batch URL (no need to use matching separation)
|
||||
|
||||
```text
|
||||
https://v.douyin.com/L4NpDJ6/
|
||||
@ -250,7 +290,7 @@ https://www.tiktok.com/t/ZTR9nDNWq/
|
||||
https://www.tiktok.com/@evil0ctal/video/7156033831819037994
|
||||
```
|
||||
|
||||
## 🛰️API documentation
|
||||
## 🛰️API Documentation
|
||||
|
||||
**_API documentation:_**
|
||||
|
||||
@ -258,59 +298,71 @@ local:<http://localhost/docs>
|
||||
|
||||
Online:<https://api.douyin.wtf/docs>
|
||||
|
||||
**_API demo:_**
|
||||
**_API Demo:_**
|
||||
|
||||
- Crawl video data (TikTok or Douyin hybrid analysis)`https://api.douyin.wtf/api/hybrid/video_data?url=[视频链接/Video URL]&minimal=false`
|
||||
- Download videos/photo albums (TikTok or Douyin hybrid analysis)`https://api.douyin.wtf/api/download?url=[视频链接/Video URL]&prefix=true&with_watermark=false`
|
||||
- Crawl video data (TikTok or Douyin mixed analysis)`https://api.douyin.wtf/api/hybrid/video_data?url=[视频链接/Video URL]&minimal=false`
|
||||
- Download video/picture album (TikTok or Douyin mixed analysis)`https://api.douyin.wtf/api/download?url=[视频链接/Video URL]&prefix=true&with_watermark=false`
|
||||
|
||||
**_For more demonstrations, please see the documentation..._**
|
||||
**_For more demonstrations, please check the document content..._**
|
||||
|
||||
## ⚠️Preparation before deployment (please read carefully):
|
||||
|
||||
- You need to solve the risk control problem of crawler cookies by yourself, otherwise the interface may be unavailable. After modifying the configuration file, you need to restart the service before it takes effect. It is best to use cookies from the account you have logged in.
|
||||
- Douyin web cookies (acquire and replace cookies in the following configuration files):
|
||||
- <https://github.com/Evil0ctal/Douyin_TikTok_Download_API/blob/30e56e5a7f97f87d60b1045befb1f6db147f8590/crawlers/douyin/web/config.yaml#L7>
|
||||
- TikTok web cookies (acquire and replace cookies in the following configuration files):
|
||||
- <https://github.com/Evil0ctal/Douyin_TikTok_Download_API/blob/30e56e5a7f97f87d60b1045befb1f6db147f8590/crawlers/tiktok/web/config.yaml#L6>
|
||||
- I turned off the online download function of the demonstration site. Someone downloaded a huge video and it crashed directly on my server. You can right-click to save the video on the web parsing result page...
|
||||
- The cookies on the demo site are my own and are not guaranteed to be valid for a long time. They only serve as a demonstration. If you deploy it yourself, please get the cookies yourself.
|
||||
- HTTP 403 error will occur if you need to access the video link returned by TikTok Web API. Please use the API in this project`/api/download`The interface downloads TikTok videos. This interface has been manually closed in the demonstration site, and you need to deploy this project yourself.
|
||||
- There is one here**Video tutorial**You can refer to:**_<https://www.bilibili.com/video/BV1vE421j7NR/>_**
|
||||
|
||||
## 💻Deployment (Method 1 Linux)
|
||||
|
||||
> 💡Tips: It is best to deploy this project to a server in the United States, otherwise strange BUGs may occur.
|
||||
> 💡Tip: It is best to deploy this project to a server in the United States, otherwise strange bugs may occur.
|
||||
|
||||
Recommended for everyone to use[Digitalocean](https://www.digitalocean.com/)server, because you can have sex for free.
|
||||
Recommended to use[DigitalOcean](https://www.digitalocean.com/)server, because it can be free.
|
||||
|
||||
Use my invitation link to sign up and you can get a $200 credit, and when you spend $25 on it, I can also get a $25 reward.
|
||||
Sign up with my invitation link and you can get a credit of $200, and I can get a reward of $25 when you spend $25 on it.
|
||||
|
||||
My invitation link:
|
||||
|
||||
<https://m.do.co/c/9f72a27dec35>
|
||||
|
||||
> Use script to deploy this project with one click
|
||||
> Use scripts to deploy this project in one click
|
||||
|
||||
- This project provides a one-click deployment script that can quickly deploy this project on the server.
|
||||
- The script was tested on Ubuntu 20.04 LTS. Other systems may have problems. If there are any problems, please solve them yourself.
|
||||
- Download using wget command[install.sh](https://raw.githubusercontent.com/Evil0ctal/Douyin_TikTok_Download_API/main/bash/install.sh)to the server and run
|
||||
- This project provides one-click deployment scripts to quickly deploy the project on the server.
|
||||
- The script was tested on Ubuntu 20.04 LTS, and other systems may have problems. If there are any problems, please solve them yourself.
|
||||
- Download using wget command[install.sh](https://raw.githubusercontent.com/Evil0ctal/Douyin_TikTok_Download_API/main/bash/install.sh)Go to the server and run
|
||||
|
||||
|
||||
wget -O install.sh https://raw.githubusercontent.com/Evil0ctal/Douyin_TikTok_Download_API/main/bash/install.sh && sudo bash install.sh
|
||||
|
||||
> Start/stop service
|
||||
> Turn on/stop service
|
||||
|
||||
- Use the following commands to control running or stopping the service:
|
||||
- Use the following command to control the operation or stop of the service:
|
||||
- `sudo systemctl start Douyin_TikTok_Download_API.service`
|
||||
- `sudo systemctl stop Douyin_TikTok_Download_API.service`
|
||||
|
||||
> Turn on/off automatic operation at startup
|
||||
> Turn on/off automatically
|
||||
|
||||
- Use the following commands to set the service to run automatically at boot or cancel automatic run at boot:
|
||||
- Use the following command to set the service to automatically run on or cancel the automatic run on:
|
||||
- `sudo systemctl enable Douyin_TikTok_Download_API.service`
|
||||
- `sudo systemctl disable Douyin_TikTok_Download_API.service`
|
||||
|
||||
> Update project
|
||||
> Update the project
|
||||
|
||||
- When the project is updated, ensure that the update script is executed in the virtual environment and all dependencies are updated. Enter the project bash directory and run update.sh:
|
||||
- When the project is updated, make sure that the update script is executed in the virtual environment and update all dependencies. Enter the project bash directory and run update.sh:
|
||||
- `cd /www/wwwroot/Douyin_TikTok_Download_API/bash && sudo bash update.sh`
|
||||
|
||||
## 💽Deployment (Method 2 Docker)
|
||||
|
||||
> 💡Tip: Docker deployment is the simplest deployment method and is suitable for users who are not familiar with Linux. This method is suitable for ensuring environment consistency, isolation and quick setup.
|
||||
> Please use a server that can normally access Douyin or TikTok, otherwise strange BUG may occur.
|
||||
> 💡 Tip: Docker deployment is the easiest way to deploy, suitable for users who are not familiar with Linux. This method is suitable for ensuring environmental consistency, isolation and quick settings.
|
||||
> Please use a server that can access Douyin or TikTok normally, otherwise strange bugs may occur.
|
||||
|
||||
### Preparation
|
||||
|
||||
Before you begin, make sure Docker is installed on your system. If you haven't installed Docker yet, you can install it from[Docker official website](https://www.docker.com/products/docker-desktop/)Download and install.
|
||||
Before you begin, make sure your system has Docker installed. If Docker is not installed, you can[Docker official website](https://www.docker.com/products/docker-desktop/)Download and install.
|
||||
|
||||
### Step 1: Pull the Docker image
|
||||
|
||||
@ -320,40 +372,40 @@ First, pull the latest Douyin_TikTok_Download_API image from Docker Hub.
|
||||
docker pull evil0ctal/douyin_tiktok_download_api:latest
|
||||
```
|
||||
|
||||
Can be replaced if needed`latest`Label the specific version you need to deploy.
|
||||
If necessary, you can replace it`latest`Tags for the specific version you need to deploy.
|
||||
|
||||
### Step 2: Run the Docker container
|
||||
|
||||
After pulling the image, you can start a container from this image. Here are the commands to run the container, including basic configuration:
|
||||
After pulling the image, you can start a container from this image. The following are the commands to run the container, including the basic configuration:
|
||||
|
||||
```bash
|
||||
docker run -d --name douyin_tiktok_api -p 80:80 evil0ctal/douyin_tiktok_download_api
|
||||
```
|
||||
|
||||
Each part of this command does the following:
|
||||
Each part of this command works as follows:
|
||||
|
||||
- `-d`: Run the container in the background (detached mode).
|
||||
- `-d`: Run containers in the background (separated mode).
|
||||
- `--name douyin_tiktok_api `: Name the container`douyin_tiktok_api `。
|
||||
- `-p 80:80`: Map port 80 on the host to port 80 of the container. Adjust the port number based on your configuration or port availability.
|
||||
- `evil0ctal/douyin_tiktok_download_api`: The name of the Docker image to use.
|
||||
- `-p 80:80`: Map port 80 on the host to port 80 of the container. Adjust the port number according to your configuration or port availability.
|
||||
- `evil0ctal/douyin_tiktok_download_api`: The name of the Docker image to be used.
|
||||
|
||||
### 步骤3：验证容器是否运行
|
||||
### Step 3: Verify that the container is running
|
||||
|
||||
Check if your container is running using the following command:
|
||||
Use the following command to check if your container is running:
|
||||
|
||||
```bash
|
||||
docker ps
|
||||
```
|
||||
|
||||
This will list all active containers. Find`douyin_tiktok_api `to confirm that it is functioning properly.
|
||||
这将列出所有活动容器。查找`douyin_tiktok_api `to confirm its normal operation.
|
||||
|
||||
### Step 4: Access the App
|
||||
### Step 4: Access the application
|
||||
|
||||
Once the container is running, you should be able to pass`http://localhost`Or API client access Douyin_TikTok_Download_API. Adjust the URL if a different port is configured or accessed from a remote location.
|
||||
After the container runs, you should be able to pass`http://localhost`Or the API client access Douyin_TikTok_Download_API. If you have a different port configured or accessed from a remote location, adjust the URL.
|
||||
|
||||
### Optional: Custom Docker commands
|
||||
|
||||
For more advanced deployments, you may wish to customize Docker commands to include environment variables, volume mounts for persistent data, or other Docker parameters. Here is an example:
|
||||
For more advanced deployments, you may want to customize Docker commands, including environment variables, volume mounts for persistent data, or other Docker parameters. Here is an example:
|
||||
|
||||
```bash
|
||||
docker run -d --name douyin_tiktok_api -p 80:80 \
|
||||
@ -362,12 +414,12 @@ docker run -d --name douyin_tiktok_api -p 80:80 \
|
||||
evil0ctal/douyin_tiktok_download_api
|
||||
```
|
||||
|
||||
- `-v /path/to/your/data:/data`: Change the`/path/to/your/data`Directory mounted to the container`/data`Directory for persisting or sharing data.
|
||||
- `-e MY_ENV_VAR=my_value`: Set environment variables within the container`MY_ENV_VAR`, whose value is`my_value`。
|
||||
- `-v /path/to/your/data:/data`: Turn on the host`/path/to/your/data`The directory mounted to the container`/data`Directory, used to persist or share data.
|
||||
- `-e MY_ENV_VAR=my_value`: Set environment variables in the container`MY_ENV_VAR`, its value is`my_value`。
|
||||
|
||||
### Configuration file modification
|
||||
|
||||
Most of the project configuration can be found in the following directories:`config.yaml`File modification:
|
||||
Most of the configurations of the project can be found in the following directories`config.yaml`Modify the file:
|
||||
|
||||
- `/crawlers/douyin/web/config.yaml`
|
||||
- `/crawlers/tiktok/web/config.yaml`
|
||||
@ -375,7 +427,7 @@ Most of the project configuration can be found in the following directories:`con
|
||||
|
||||
### Step 5: Stop and remove the container
|
||||
|
||||
When you need to stop and remove containers, use the following commands:
|
||||
When you need to stop and remove the container, use the following command:
|
||||
|
||||
```bash
|
||||
# Stop
|
||||
@ -387,17 +439,17 @@ docker rm douyin_tiktok_api
|
||||
|
||||
## 📸Screenshot
|
||||
|
||||
**_API speed test (compared to official API)_**
|
||||
**_API speed test (compare the official API)_**
|
||||
|
||||
<details><summary>🔎点击展开截图</summary>
|
||||
|
||||
Douyin official API:
|
||||
TikTok official API:
|
||||
|
||||
API of this project:
|
||||
This project API:
|
||||
|
||||
TikTok official API:
|
||||
|
||||
API of this project:
|
||||
This project API:
|
||||
|
||||
</details>
|
||||
<hr>
|
||||
@ -425,4 +477,4 @@ Web main interface:
|
||||
|
||||
> Start: 2021/11/06
|
||||
|
||||
> GitHub:[@Evil0ctal](https://github.com/Evil0ctal)
|
||||
> Githubub:[@Evil0ctal](https://github.com/Evil0ctal)
|
||||
|
||||
149
README.md
149
README.md
@ -31,41 +31,35 @@
|
||||
|
||||
</div>
|
||||
|
||||
## 🔊 V4.0.0版本重构
|
||||
## 赞助商
|
||||
|
||||
> TODO:
|
||||
这些赞助商已付费放置在这里，**Douyin_TikTok_Download_API** 项目将永远免费且开源。如果您希望成为该项目的赞助商，请查看我的 [GitHub 赞助商页面](https://github.com/sponsors/evil0ctal)。
|
||||
|
||||
- 移除了过时的bilibili代码，需要有人重写。
|
||||
- 群里有人想添加快手以及西瓜视频的解析。
|
||||
- 自述文件已经过时，需要进行重写。
|
||||
- 进行PyPi包制作
|
||||
- config.yaml文件需要进行修整。
|
||||
- 添加对用户主页的解析。
|
||||
- iOS快捷指令需要更新兼容最新的API响应和路径。
|
||||
- 桌面端下载器或浏览器插件有需要可以进行开发。
|
||||
- 解决爬虫Cookie风控问题。
|
||||
<div align="center">
|
||||
<a href="https://www.tikhub.io/" target="_blank">
|
||||
<img src="https://tikhub.io/wp-content/uploads/2024/11/Main-Logo.webp" width="100" alt="TikHub.io - Global Social Data & API Marketplace">
|
||||
</a>
|
||||
<div>
|
||||
<h2><b>TikHub.io</b></h2>
|
||||
<p>Your Ultimate Social Media Data & API Marketplace</p>
|
||||
<p>
|
||||
Professional data solutions for Douyin, Xiaohongshu, TikTok, Instagram, YouTube,
|
||||
Twitter, and more.<br>
|
||||
Real-time Data | Flexible APIs | Seamless Integration | Competitive Pricing with Discounts
|
||||
</p>
|
||||
<p>
|
||||
<b>Discover TikHub.io Marketplace</b><br>
|
||||
Buy and sell custom APIs, services, and social media solutions.<br>
|
||||
Join a thriving ecosystem of developers, businesses, and content creators.
|
||||
</p>
|
||||
<p><em>Trusted by leading global influencer marketing and social media intelligence platforms</em></p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
> 更改
|
||||
|
||||
- 将Pywebio作为FastAPI的子APP一起运行。
|
||||
- 重写了抖音以及TikTok的接口，感谢 [@johnserf-seed](https://github.com/Johnserf-Seed)
|
||||
- 重写了文件下载的端点，现在使用异步文件IO。
|
||||
- 对所有端点进行了注解和演示值的添加。
|
||||
- 整理项目文件结构。
|
||||
|
||||
> 备注
|
||||
|
||||
感兴趣一起写这个项目的给请加微信`Evil0ctal`备注github项目重构，大家可以在群里互相交流学习，不允许发广告以及违法的东西，纯粹交朋友和技术交流。
|
||||
|
||||
> 私有接口服务
|
||||
|
||||
Discord: [TikHub Discord](https://discord.com/invite/aMEAS8Xsvz)
|
||||
|
||||
Free Douyin/TikTok API: [TikHub Beta API](https://beta.tikhub.io/)
|
||||
|
||||
## 👻介绍
|
||||
|
||||
> 🚨如需使用私有服务器运行本项目，请参考部署方式[[Docker部署](./README.md#%E9%83%A8%E7%BD%B2%E6%96%B9%E5%BC%8F%E4%BA%8C-docker), [一键部署](./README.md#%E9%83%A8%E7%BD%B2%E6%96%B9%E5%BC%8F%E4%B8%80-linux)]
|
||||
> 🚨如需使用私有服务器运行本项目，请参考：[部署准备工作](./README.md#%EF%B8%8F%E9%83%A8%E7%BD%B2%E5%89%8D%E7%9A%84%E5%87%86%E5%A4%87%E5%B7%A5%E4%BD%9C%E8%AF%B7%E4%BB%94%E7%BB%86%E9%98%85%E8%AF%BB), [Docker部署](./README.md#%E9%83%A8%E7%BD%B2%E6%96%B9%E5%BC%8F%E4%BA%8C-docker), [一键部署](./README.md#%E9%83%A8%E7%BD%B2%E6%96%B9%E5%BC%8F%E4%B8%80-linux)
|
||||
|
||||
本项目是基于 [PyWebIO](https://github.com/pywebio/PyWebIO)，[FastAPI](https://fastapi.tiangolo.com/)，[HTTPX](https://www.python-httpx.org/)，快速异步的[抖音](https://www.douyin.com/)/[TikTok](https://www.tiktok.com/)数据爬取工具，并通过Web端实现在线批量解析以及下载无水印视频或图集，数据爬取API，iOS快捷指令无水印下载等功能。你可以自己部署或改造本项目实现更多功能，也可以在你的项目中直接调用[scraper.py](https://github.com/Evil0ctal/Douyin_TikTok_Download_API/blob/Stable/scraper.py)或安装现有的[pip包](https://pypi.org/project/douyin-tiktok-scraper/)作为解析库轻松爬取数据等.....
|
||||
|
||||
@ -74,18 +68,46 @@ Free Douyin/TikTok API: [TikHub Beta API](https://beta.tikhub.io/)
|
||||
*下载禁止下载的视频，进行数据分析，iOS无水印下载（搭配[iOS自带的快捷指令APP](https://apps.apple.com/cn/app/%E5%BF%AB%E6%8D%B7%E6%8C%87%E4%BB%A4/id915249334)
|
||||
配合本项目API实现应用内下载或读取剪贴板下载）等.....*
|
||||
|
||||
#### ⚠️注意：
|
||||
## 🔊 V4 版本备注
|
||||
|
||||
- 感兴趣一起写这个项目的给请加微信`Evil0ctal`备注github项目重构，大家可以在群里互相交流学习，不允许发广告以及违法的东西，纯粹交朋友和技术交流。
|
||||
- 本项目使用`X-Bogus`算法以及`A_Bogus`算法请求抖音和TikTok的Web API。
|
||||
- 由于Douyin的风控，部署完本项目后请在**浏览器中获取Douyin网站的Cookie然后在config.yaml中进行替换。**
|
||||
- 请在提出issue之前先阅读下方的文档，大多数问题的解决方法都会包含在文档中。
|
||||
- 本项目是完全免费的，但使用时请遵守：[Apache-2.0 license](https://github.com/Evil0ctal/Douyin_TikTok_Download_API?tab=Apache-2.0-1-ov-file#readme)
|
||||
|
||||
## 🔖TikHub.io API
|
||||
|
||||
[TikHub.io](https://api.tikhub.io/)是一个API平台，提供包括Douyin、TikTok在内的各种公开数据接口，如果您想支持 [Douyin_TikTok_Download_API](https://github.com/Evil0ctal/Douyin_TikTok_Download_API) 项目的开发，我们强烈建议您选择[TikHub.io](https://api.tikhub.io/)。
|
||||
|
||||
#### 特点：
|
||||
|
||||
> 📦 开箱即用
|
||||
|
||||
简化使用流程，利用封装好的SDK迅速开展开发工作。所有API接口均依据RESTful架构设计，并使用OpenAPI规范进行描述和文档化，附带示例参数，确保调用更加简便。
|
||||
|
||||
> 💰 成本优势
|
||||
|
||||
不预设套餐限制，没有月度使用门槛，所有消费按实际使用量即时计费，并且根据用户每日的请求量进行阶梯式计费，同时可以通过每日签到在用户后台进行签到获取免费的额度，并且这些免费额度不会过期。
|
||||
|
||||
> ⚡️ 快速支持
|
||||
|
||||
我们有一个庞大的Discord社区服务器，管理员和其他用户会在服务器中快速的回复你，帮助你快速解决当前的问题。
|
||||
|
||||
> 🎉 拥抱开源
|
||||
|
||||
TikHub的部分源代码会开源在Github上，并且会赞助一些开源项目的作者。
|
||||
|
||||
#### 链接：
|
||||
|
||||
- Github: [TikHub Github](https://github.com/TikHubIO)
|
||||
- Discord: [TikHub Discord](https://discord.com/invite/aMEAS8Xsvz)
|
||||
- Register: [TikHub signup](https://beta-web.tikhub.io/en-us/users/signup)
|
||||
- API Docs: [TikHub API Docs](https://api.tikhub.io/)
|
||||
|
||||
- 你需要自行解决爬虫Cookie风控问题，否则可能会导致接口无法使用。
|
||||
- 抖音网页端Cookie（自行获取并替换下面配置文件中的Cookie）：
|
||||
- https://github.com/Evil0ctal/Douyin_TikTok_Download_API/blob/30e56e5a7f97f87d60b1045befb1f6db147f8590/crawlers/douyin/web/config.yaml#L7
|
||||
- TikTok网页端Cookie（自行获取并替换下面配置文件中的Cookie）：
|
||||
- https://github.com/Evil0ctal/Douyin_TikTok_Download_API/blob/30e56e5a7f97f87d60b1045befb1f6db147f8590/crawlers/tiktok/web/config.yaml#L6
|
||||
- 演示站点的在线下载功能被我关掉了，有人下的视频巨大无比直接给我服务器干崩了，自己在网页解析结果里右键保存吧...
|
||||
- 演示站点的Cookie是我自己的，不保证长期有效，只起到演示作用，自己部署的话请自行获取Cookie。
|
||||
## 🖥演示站点: 我很脆弱...请勿压测(·•᷄ࡇ•᷅ ）
|
||||
|
||||
> 😾演示站点的在线下载功能已关闭。
|
||||
> 😾演示站点的在线下载功能已关闭，并且由于Cookie原因，Douyin的解析以及API服务在Demo站点无法保证可用性。
|
||||
|
||||
🍔Web APP: [https://douyin.wtf/](https://douyin.wtf/)
|
||||
|
||||
@ -125,14 +147,16 @@ Free Douyin/TikTok API: [TikHub Beta API](https://beta.tikhub.io/)
|
||||
|
||||
```
|
||||
./Douyin_TikTok_Download_API
|
||||
├─app
|
||||
│ ├─api
|
||||
│ │ ├─endpoints
|
||||
│ │ └─models
|
||||
│ ├─download
|
||||
├─app
|
||||
│ ├─api
|
||||
│ │ ├─endpoints
|
||||
│ │ └─models
|
||||
│ ├─download
|
||||
│ └─web
|
||||
│ └─views
|
||||
└─crawlers
|
||||
├─bilibili
|
||||
│ └─web
|
||||
│ └─views
|
||||
└─crawlers
|
||||
├─douyin
|
||||
│ └─web
|
||||
├─hybrid
|
||||
@ -151,6 +175,7 @@ Free Douyin/TikTok API: [TikHub Beta API](https://beta.tikhub.io/)
|
||||
- 完善的API文档([Demo/演示](https://api.douyin.wtf/docs))
|
||||
- 丰富的API接口：
|
||||
- 抖音网页版API
|
||||
|
||||
- [x] 视频数据解析
|
||||
- [x] 获取用户主页作品数据
|
||||
- [x] 获取用户主页喜欢作品数据
|
||||
@ -166,14 +191,15 @@ Free Douyin/TikTok API: [TikHub Beta API](https://beta.tikhub.io/)
|
||||
- [x] 生成verify_fp
|
||||
- [x] 生成s_v_web_id
|
||||
- [x] 使用接口网址生成X-Bogus参数
|
||||
- [x] 使用接口网址生成A_Bogus参数
|
||||
- [x] 提取单个用户id
|
||||
- [x] 提取列表用户id
|
||||
- [x] 提取单个作品id
|
||||
- [x] 提取列表作品id
|
||||
- [x] 提取列表直播间号
|
||||
- [x] 提取列表直播间号
|
||||
|
||||
- TikTok网页版API
|
||||
|
||||
- [x] 视频数据解析
|
||||
- [x] 获取用户主页作品数据
|
||||
- [x] 获取用户主页喜欢作品数据
|
||||
@ -194,8 +220,23 @@ Free Douyin/TikTok API: [TikHub Beta API](https://beta.tikhub.io/)
|
||||
- [x] 提取列表作品id
|
||||
- [x] 获取用户unique_id
|
||||
- [x] 获取列表unique_id
|
||||
|
||||
|
||||
- 哔哩哔哩网页版API
|
||||
- [x] 获取单个视频详情信息
|
||||
- [x] 获取视频流地址
|
||||
- [x] 获取用户发布视频作品数据
|
||||
- [x] 获取用户所有收藏夹信息
|
||||
- [x] 获取指定收藏夹内视频数据
|
||||
- [x] 获取指定用户的信息
|
||||
- [x] 获取综合热门视频信息
|
||||
- [x] 获取指定视频的评论
|
||||
- [x] 获取视频下指定评论的回复
|
||||
- [x] 获取指定用户动态
|
||||
- [x] 获取视频实时弹幕
|
||||
- [x] 获取指定直播间信息
|
||||
- [x] 获取直播间视频流
|
||||
- [x] 获取指定分区正在直播的主播
|
||||
- [x] 获取所有直播分区列表
|
||||
- [x] 通过bv号获得视频分p信息
|
||||
---
|
||||
|
||||
## 📦调用解析库（已废弃需要更新）:
|
||||
@ -287,6 +328,18 @@ https://www.tiktok.com/@evil0ctal/video/7156033831819037994
|
||||
|
||||
***更多演示请查看文档内容......***
|
||||
|
||||
## ⚠️部署前的准备工作(请仔细阅读)：
|
||||
|
||||
- 你需要自行解决爬虫Cookie风控问题，否则可能会导致接口无法使用，修改完配置文件后需要重启服务才能生效，并且最好使用已经登录过的账号的Cookie。
|
||||
- 抖音网页端Cookie（自行获取并替换下面配置文件中的Cookie）：
|
||||
- https://github.com/Evil0ctal/Douyin_TikTok_Download_API/blob/30e56e5a7f97f87d60b1045befb1f6db147f8590/crawlers/douyin/web/config.yaml#L7
|
||||
- TikTok网页端Cookie（自行获取并替换下面配置文件中的Cookie）：
|
||||
- https://github.com/Evil0ctal/Douyin_TikTok_Download_API/blob/30e56e5a7f97f87d60b1045befb1f6db147f8590/crawlers/tiktok/web/config.yaml#L6
|
||||
- 演示站点的在线下载功能被我关掉了，有人下的视频巨大无比直接给我服务器干崩了，你可以在网页解析结果页面右键保存视频...
|
||||
- 演示站点的Cookie是我自己的，不保证长期有效，只起到演示作用，自己部署的话请自行获取Cookie。
|
||||
- 需要TikTok Web API返回的视频链接直接访问会发生HTTP 403错误，请使用本项目API中的`/api/download`接口对TikTok 视频进行下载，这个接口在演示站点中已经被手动关闭了，需要你自行部署本项目。
|
||||
- 这里有一个**视频教程**可以参考：***[https://www.bilibili.com/video/BV1vE421j7NR/](https://www.bilibili.com/video/BV1vE421j7NR/)***
|
||||
|
||||
## 💻部署(方式一 Linux)
|
||||
|
||||
> 💡提示：最好将本项目部署至美国地区的服务器，否则可能会出现奇怪的BUG。
|
||||
|
||||
697
app/api/endpoints/bilibili_web.py
Normal file
697
app/api/endpoints/bilibili_web.py
Normal file
@ -0,0 +1,697 @@
|
||||
from fastapi import APIRouter, Body, Query, Request, HTTPException # 导入FastAPI组件
|
||||
from app.api.models.APIResponseModel import ResponseModel, ErrorResponseModel # 导入响应模型
|
||||
|
||||
from crawlers.bilibili.web.web_crawler import BilibiliWebCrawler # 导入哔哩哔哩web爬虫
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
BilibiliWebCrawler = BilibiliWebCrawler()
|
||||
|
||||
|
||||
# 获取单个视频详情信息
|
||||
@router.get("/fetch_one_video", response_model=ResponseModel, summary="获取单个视频详情信息/Get single video data")
|
||||
async def fetch_one_video(request: Request,
|
||||
bv_id: str = Query(example="BV1M1421t7hT", description="作品id/Video id")):
|
||||
"""
|
||||
# [中文]
|
||||
### 用途:
|
||||
- 获取单个视频详情信息
|
||||
### 参数:
|
||||
- bv_id: 作品id
|
||||
### 返回:
|
||||
- 视频详情信息
|
||||
|
||||
# [English]
|
||||
### Purpose:
|
||||
- Get single video data
|
||||
### Parameters:
|
||||
- bv_id: Video id
|
||||
### Return:
|
||||
- Video data
|
||||
|
||||
# [示例/Example]
|
||||
bv_id = "BV1M1421t7hT"
|
||||
"""
|
||||
try:
|
||||
data = await BilibiliWebCrawler.fetch_one_video(bv_id)
|
||||
return ResponseModel(code=200,
|
||||
router=request.url.path,
|
||||
data=data)
|
||||
except Exception as e:
|
||||
status_code = 400
|
||||
detail = ErrorResponseModel(code=status_code,
|
||||
router=request.url.path,
|
||||
params=dict(request.query_params),
|
||||
)
|
||||
raise HTTPException(status_code=status_code, detail=detail.dict())
|
||||
|
||||
|
||||
# 获取视频流地址
|
||||
@router.get("/fetch_video_playurl", response_model=ResponseModel, summary="获取视频流地址/Get video playurl")
|
||||
async def fetch_one_video(request: Request,
|
||||
bv_id: str = Query(example="BV1y7411Q7Eq", description="作品id/Video id"),
|
||||
cid:str = Query(example="171776208", description="作品cid/Video cid")):
|
||||
"""
|
||||
# [中文]
|
||||
### 用途:
|
||||
- 获取视频流地址
|
||||
### 参数:
|
||||
- bv_id: 作品id
|
||||
- cid: 作品cid
|
||||
### 返回:
|
||||
- 视频流地址
|
||||
|
||||
# [English]
|
||||
### Purpose:
|
||||
- Get video playurl
|
||||
### Parameters:
|
||||
- bv_id: Video id
|
||||
- cid: Video cid
|
||||
### Return:
|
||||
- Video playurl
|
||||
|
||||
# [示例/Example]
|
||||
bv_id = "BV1y7411Q7Eq"
|
||||
cid = "171776208"
|
||||
"""
|
||||
try:
|
||||
data = await BilibiliWebCrawler.fetch_video_playurl(bv_id, cid)
|
||||
return ResponseModel(code=200,
|
||||
router=request.url.path,
|
||||
data=data)
|
||||
except Exception as e:
|
||||
status_code = 400
|
||||
detail = ErrorResponseModel(code=status_code,
|
||||
router=request.url.path,
|
||||
params=dict(request.query_params),
|
||||
)
|
||||
raise HTTPException(status_code=status_code, detail=detail.dict())
|
||||
|
||||
|
||||
# 获取用户发布视频作品数据
|
||||
@router.get("/fetch_user_post_videos", response_model=ResponseModel,
|
||||
summary="获取用户主页作品数据/Get user homepage video data")
|
||||
async def fetch_user_post_videos(request: Request,
|
||||
uid: str = Query(example="178360345", description="用户UID"),
|
||||
pn: int = Query(default=1, description="页码/Page number"),):
|
||||
"""
|
||||
# [中文]
|
||||
### 用途:
|
||||
- 获取用户发布的视频数据
|
||||
### 参数:
|
||||
- uid: 用户UID
|
||||
- pn: 页码
|
||||
### 返回:
|
||||
- 用户发布的视频数据
|
||||
|
||||
# [English]
|
||||
### Purpose:
|
||||
- Get user post video data
|
||||
### Parameters:
|
||||
- uid: User UID
|
||||
- pn: Page number
|
||||
### Return:
|
||||
- User posted video data
|
||||
|
||||
# [示例/Example]
|
||||
uid = "178360345"
|
||||
pn = 1
|
||||
"""
|
||||
try:
|
||||
data = await BilibiliWebCrawler.fetch_user_post_videos(uid, pn)
|
||||
return ResponseModel(code=200,
|
||||
router=request.url.path,
|
||||
data=data)
|
||||
except Exception as e:
|
||||
status_code = 400
|
||||
detail = ErrorResponseModel(code=status_code,
|
||||
router=request.url.path,
|
||||
params=dict(request.query_params),
|
||||
)
|
||||
raise HTTPException(status_code=status_code, detail=detail.dict())
|
||||
|
||||
|
||||
# 获取用户所有收藏夹信息
|
||||
@router.get("/fetch_collect_folders", response_model=ResponseModel,
|
||||
summary="获取用户所有收藏夹信息/Get user collection folders")
|
||||
async def fetch_collect_folders(request: Request,
|
||||
uid: str = Query(example="178360345", description="用户UID")):
|
||||
"""
|
||||
# [中文]
|
||||
### 用途:
|
||||
- 获取用户收藏作品数据
|
||||
### 参数:
|
||||
- uid: 用户UID
|
||||
### 返回:
|
||||
- 用户收藏夹信息
|
||||
|
||||
# [English]
|
||||
### Purpose:
|
||||
- Get user collection folders
|
||||
### Parameters:
|
||||
- uid: User UID
|
||||
### Return:
|
||||
- user collection folders
|
||||
|
||||
# [示例/Example]
|
||||
uid = "178360345"
|
||||
"""
|
||||
try:
|
||||
data = await BilibiliWebCrawler.fetch_collect_folders(uid)
|
||||
return ResponseModel(code=200,
|
||||
router=request.url.path,
|
||||
data=data)
|
||||
except Exception as e:
|
||||
status_code = 400
|
||||
detail = ErrorResponseModel(code=status_code,
|
||||
router=request.url.path,
|
||||
params=dict(request.query_params),
|
||||
)
|
||||
raise HTTPException(status_code=status_code, detail=detail.dict())
|
||||
|
||||
|
||||
# 获取指定收藏夹内视频数据
|
||||
@router.get("/fetch_user_collection_videos", response_model=ResponseModel,
|
||||
summary="获取指定收藏夹内视频数据/Gets video data from a collection folder")
|
||||
async def fetch_user_collection_videos(request: Request,
|
||||
folder_id: str = Query(example="1756059545",
|
||||
description="收藏夹id/collection folder id"),
|
||||
pn: int = Query(default=1, description="页码/Page number")
|
||||
):
|
||||
"""
|
||||
# [中文]
|
||||
### 用途:
|
||||
- 获取指定收藏夹内视频数据
|
||||
### 参数:
|
||||
- folder_id: 用户UID
|
||||
- pn: 页码
|
||||
### 返回:
|
||||
- 指定收藏夹内视频数据
|
||||
|
||||
# [English]
|
||||
### Purpose:
|
||||
- Gets video data from a collection folder
|
||||
### Parameters:
|
||||
- folder_id: collection folder id
|
||||
- pn: Page number
|
||||
### Return:
|
||||
- video data from collection folder
|
||||
|
||||
# [示例/Example]
|
||||
folder_id = "1756059545"
|
||||
pn = 1
|
||||
"""
|
||||
try:
|
||||
data = await BilibiliWebCrawler.fetch_folder_videos(folder_id, pn)
|
||||
return ResponseModel(code=200,
|
||||
router=request.url.path,
|
||||
data=data)
|
||||
except Exception as e:
|
||||
status_code = 400
|
||||
detail = ErrorResponseModel(code=status_code,
|
||||
router=request.url.path,
|
||||
params=dict(request.query_params),
|
||||
)
|
||||
raise HTTPException(status_code=status_code, detail=detail.dict())
|
||||
|
||||
|
||||
# 获取指定用户的信息
|
||||
@router.get("/fetch_user_profile", response_model=ResponseModel,
|
||||
summary="获取指定用户的信息/Get information of specified user")
|
||||
async def fetch_collect_folders(request: Request,
|
||||
uid: str = Query(example="178360345", description="用户UID")):
|
||||
"""
|
||||
# [中文]
|
||||
### 用途:
|
||||
- 获取指定用户的信息
|
||||
### 参数:
|
||||
- uid: 用户UID
|
||||
### 返回:
|
||||
- 指定用户的个人信息
|
||||
|
||||
# [English]
|
||||
### Purpose:
|
||||
- Get information of specified user
|
||||
### Parameters:
|
||||
- uid: User UID
|
||||
### Return:
|
||||
- information of specified user
|
||||
|
||||
# [示例/Example]
|
||||
uid = "178360345"
|
||||
"""
|
||||
try:
|
||||
data = await BilibiliWebCrawler.fetch_user_profile(uid)
|
||||
return ResponseModel(code=200,
|
||||
router=request.url.path,
|
||||
data=data)
|
||||
except Exception as e:
|
||||
status_code = 400
|
||||
detail = ErrorResponseModel(code=status_code,
|
||||
router=request.url.path,
|
||||
params=dict(request.query_params),
|
||||
)
|
||||
raise HTTPException(status_code=status_code, detail=detail.dict())
|
||||
|
||||
|
||||
# 获取综合热门视频信息
|
||||
@router.get("/fetch_com_popular", response_model=ResponseModel,
|
||||
summary="获取综合热门视频信息/Get comprehensive popular video information")
|
||||
async def fetch_collect_folders(request: Request,
|
||||
pn: int = Query(default=1, description="页码/Page number")):
|
||||
"""
|
||||
# [中文]
|
||||
### 用途:
|
||||
- 获取综合热门视频信息
|
||||
### 参数:
|
||||
- pn: 页码
|
||||
### 返回:
|
||||
- 综合热门视频信息
|
||||
|
||||
# [English]
|
||||
### Purpose:
|
||||
- Get comprehensive popular video information
|
||||
### Parameters:
|
||||
- pn: Page number
|
||||
### Return:
|
||||
- comprehensive popular video information
|
||||
|
||||
# [示例/Example]
|
||||
pn = 1
|
||||
"""
|
||||
try:
|
||||
data = await BilibiliWebCrawler.fetch_com_popular(pn)
|
||||
return ResponseModel(code=200,
|
||||
router=request.url.path,
|
||||
data=data)
|
||||
except Exception as e:
|
||||
status_code = 400
|
||||
detail = ErrorResponseModel(code=status_code,
|
||||
router=request.url.path,
|
||||
params=dict(request.query_params),
|
||||
)
|
||||
raise HTTPException(status_code=status_code, detail=detail.dict())
|
||||
|
||||
|
||||
# 获取指定视频的评论
|
||||
@router.get("/fetch_video_comments", response_model=ResponseModel,
|
||||
summary="获取指定视频的评论/Get comments on the specified video")
|
||||
async def fetch_collect_folders(request: Request,
|
||||
bv_id: str = Query(example="BV1M1421t7hT", description="作品id/Video id"),
|
||||
pn: int = Query(default=1, description="页码/Page number")):
|
||||
"""
|
||||
# [中文]
|
||||
### 用途:
|
||||
- 获取指定视频的评论
|
||||
### 参数:
|
||||
- bv_id: 作品id
|
||||
- pn: 页码
|
||||
### 返回:
|
||||
- 指定视频的评论数据
|
||||
|
||||
# [English]
|
||||
### Purpose:
|
||||
- Get comments on the specified video
|
||||
### Parameters:
|
||||
- bv_id: Video id
|
||||
- pn: Page number
|
||||
### Return:
|
||||
- comments of the specified video
|
||||
|
||||
# [示例/Example]
|
||||
bv_id = "BV1M1421t7hT"
|
||||
pn = 1
|
||||
"""
|
||||
try:
|
||||
data = await BilibiliWebCrawler.fetch_video_comments(bv_id, pn)
|
||||
return ResponseModel(code=200,
|
||||
router=request.url.path,
|
||||
data=data)
|
||||
except Exception as e:
|
||||
status_code = 400
|
||||
detail = ErrorResponseModel(code=status_code,
|
||||
router=request.url.path,
|
||||
params=dict(request.query_params),
|
||||
)
|
||||
raise HTTPException(status_code=status_code, detail=detail.dict())
|
||||
|
||||
|
||||
# 获取视频下指定评论的回复
|
||||
@router.get("/fetch_comment_reply", response_model=ResponseModel,
|
||||
summary="获取视频下指定评论的回复/Get reply to the specified comment")
|
||||
async def fetch_collect_folders(request: Request,
|
||||
bv_id: str = Query(example="BV1M1421t7hT", description="作品id/Video id"),
|
||||
pn: int = Query(default=1, description="页码/Page number"),
|
||||
rpid: str = Query(example="237109455120", description="回复id/Reply id")):
|
||||
"""
|
||||
# [中文]
|
||||
### 用途:
|
||||
- 获取视频下指定评论的回复
|
||||
### 参数:
|
||||
- bv_id: 作品id
|
||||
- pn: 页码
|
||||
- rpid: 回复id
|
||||
### 返回:
|
||||
- 指定评论的回复数据
|
||||
|
||||
# [English]
|
||||
### Purpose:
|
||||
- Get reply to the specified comment
|
||||
### Parameters:
|
||||
- bv_id: Video id
|
||||
- pn: Page number
|
||||
- rpid: Reply id
|
||||
### Return:
|
||||
- Reply of the specified comment
|
||||
|
||||
# [示例/Example]
|
||||
bv_id = "BV1M1421t7hT"
|
||||
pn = 1
|
||||
rpid = "237109455120"
|
||||
"""
|
||||
try:
|
||||
data = await BilibiliWebCrawler.fetch_comment_reply(bv_id, pn, rpid)
|
||||
return ResponseModel(code=200,
|
||||
router=request.url.path,
|
||||
data=data)
|
||||
except Exception as e:
|
||||
status_code = 400
|
||||
detail = ErrorResponseModel(code=status_code,
|
||||
router=request.url.path,
|
||||
params=dict(request.query_params),
|
||||
)
|
||||
raise HTTPException(status_code=status_code, detail=detail.dict())
|
||||
|
||||
|
||||
# 获取指定用户动态
|
||||
@router.get("/fetch_user_dynamic", response_model=ResponseModel,
|
||||
summary="获取指定用户动态/Get dynamic information of specified user")
|
||||
async def fetch_collect_folders(request: Request,
|
||||
uid: str = Query(example="16015678", description="用户UID"),
|
||||
offset: str = Query(default="", example="953154282154098691",
|
||||
description="开始索引/offset")):
|
||||
"""
|
||||
# [中文]
|
||||
### 用途:
|
||||
- 获取指定用户动态
|
||||
### 参数:
|
||||
- uid: 用户UID
|
||||
- offset: 开始索引
|
||||
### 返回:
|
||||
- 指定用户动态数据
|
||||
|
||||
# [English]
|
||||
### Purpose:
|
||||
- Get dynamic information of specified user
|
||||
### Parameters:
|
||||
- uid: User UID
|
||||
- offset: offset
|
||||
### Return:
|
||||
- dynamic information of specified user
|
||||
|
||||
# [示例/Example]
|
||||
uid = "178360345"
|
||||
offset = "953154282154098691"
|
||||
"""
|
||||
try:
|
||||
data = await BilibiliWebCrawler.fetch_user_dynamic(uid, offset)
|
||||
return ResponseModel(code=200,
|
||||
router=request.url.path,
|
||||
data=data)
|
||||
except Exception as e:
|
||||
status_code = 400
|
||||
detail = ErrorResponseModel(code=status_code,
|
||||
router=request.url.path,
|
||||
params=dict(request.query_params),
|
||||
)
|
||||
raise HTTPException(status_code=status_code, detail=detail.dict())
|
||||
|
||||
|
||||
# 获取视频实时弹幕
|
||||
@router.get("/fetch_video_danmaku", response_model=ResponseModel, summary="获取视频实时弹幕/Get Video Danmaku")
|
||||
async def fetch_one_video(request: Request,
|
||||
cid: str = Query(example="1639235405", description="作品cid/Video cid")):
|
||||
"""
|
||||
# [中文]
|
||||
### 用途:
|
||||
- 获取视频实时弹幕
|
||||
### 参数:
|
||||
- cid: 作品cid
|
||||
### 返回:
|
||||
- 视频实时弹幕
|
||||
|
||||
# [English]
|
||||
### Purpose:
|
||||
- Get Video Danmaku
|
||||
### Parameters:
|
||||
- cid: Video cid
|
||||
### Return:
|
||||
- Video Danmaku
|
||||
|
||||
# [示例/Example]
|
||||
cid = "1639235405"
|
||||
"""
|
||||
try:
|
||||
data = await BilibiliWebCrawler.fetch_video_danmaku(cid)
|
||||
return ResponseModel(code=200,
|
||||
router=request.url.path,
|
||||
data=data)
|
||||
except Exception as e:
|
||||
status_code = 400
|
||||
detail = ErrorResponseModel(code=status_code,
|
||||
router=request.url.path,
|
||||
params=dict(request.query_params),
|
||||
)
|
||||
raise HTTPException(status_code=status_code, detail=detail.dict())
|
||||
|
||||
|
||||
# 获取指定直播间信息
|
||||
@router.get("/fetch_live_room_detail", response_model=ResponseModel,
|
||||
summary="获取指定直播间信息/Get information of specified live room")
|
||||
async def fetch_collect_folders(request: Request,
|
||||
room_id: str = Query(example="22816111", description="直播间ID/Live room ID")):
|
||||
"""
|
||||
# [中文]
|
||||
### 用途:
|
||||
- 获取指定直播间信息
|
||||
### 参数:
|
||||
- room_id: 直播间ID
|
||||
### 返回:
|
||||
- 指定直播间信息
|
||||
|
||||
# [English]
|
||||
### Purpose:
|
||||
- Get information of specified live room
|
||||
### Parameters:
|
||||
- room_id: Live room ID
|
||||
### Return:
|
||||
- information of specified live room
|
||||
|
||||
# [示例/Example]
|
||||
room_id = "22816111"
|
||||
"""
|
||||
try:
|
||||
data = await BilibiliWebCrawler.fetch_live_room_detail(room_id)
|
||||
return ResponseModel(code=200,
|
||||
router=request.url.path,
|
||||
data=data)
|
||||
except Exception as e:
|
||||
status_code = 400
|
||||
detail = ErrorResponseModel(code=status_code,
|
||||
router=request.url.path,
|
||||
params=dict(request.query_params),
|
||||
)
|
||||
raise HTTPException(status_code=status_code, detail=detail.dict())
|
||||
|
||||
|
||||
# 获取指定直播间视频流
|
||||
@router.get("/fetch_live_videos", response_model=ResponseModel,
|
||||
summary="获取直播间视频流/Get live video data of specified room")
|
||||
async def fetch_collect_folders(request: Request,
|
||||
room_id: str = Query(example="1815229528", description="直播间ID/Live room ID")):
|
||||
"""
|
||||
# [中文]
|
||||
### 用途:
|
||||
- 获取指定直播间视频流
|
||||
### 参数:
|
||||
- room_id: 直播间ID
|
||||
### 返回:
|
||||
- 指定直播间视频流
|
||||
|
||||
# [English]
|
||||
### Purpose:
|
||||
- Get live video data of specified room
|
||||
### Parameters:
|
||||
- room_id: Live room ID
|
||||
### Return:
|
||||
- live video data of specified room
|
||||
|
||||
# [示例/Example]
|
||||
room_id = "1815229528"
|
||||
"""
|
||||
try:
|
||||
data = await BilibiliWebCrawler.fetch_live_videos(room_id)
|
||||
return ResponseModel(code=200,
|
||||
router=request.url.path,
|
||||
data=data)
|
||||
except Exception as e:
|
||||
status_code = 400
|
||||
detail = ErrorResponseModel(code=status_code,
|
||||
router=request.url.path,
|
||||
params=dict(request.query_params),
|
||||
)
|
||||
raise HTTPException(status_code=status_code, detail=detail.dict())
|
||||
|
||||
|
||||
# 获取指定分区正在直播的主播
|
||||
@router.get("/fetch_live_streamers", response_model=ResponseModel,
|
||||
summary="获取指定分区正在直播的主播/Get live streamers of specified live area")
|
||||
async def fetch_collect_folders(request: Request,
|
||||
area_id: str = Query(example="9", description="直播分区id/Live area ID"),
|
||||
pn: int = Query(default=1, description="页码/Page number")):
|
||||
"""
|
||||
# [中文]
|
||||
### 用途:
|
||||
- 获取指定分区正在直播的主播
|
||||
### 参数:
|
||||
- area_id: 直播分区id
|
||||
- pn: 页码
|
||||
### 返回:
|
||||
- 指定分区正在直播的主播
|
||||
|
||||
# [English]
|
||||
### Purpose:
|
||||
- Get live streamers of specified live area
|
||||
### Parameters:
|
||||
- area_id: Live area ID
|
||||
- pn: Page number
|
||||
### Return:
|
||||
- live streamers of specified live area
|
||||
|
||||
# [示例/Example]
|
||||
area_id = "9"
|
||||
pn = 1
|
||||
"""
|
||||
try:
|
||||
data = await BilibiliWebCrawler.fetch_live_streamers(area_id, pn)
|
||||
return ResponseModel(code=200,
|
||||
router=request.url.path,
|
||||
data=data)
|
||||
except Exception as e:
|
||||
status_code = 400
|
||||
detail = ErrorResponseModel(code=status_code,
|
||||
router=request.url.path,
|
||||
params=dict(request.query_params),
|
||||
)
|
||||
raise HTTPException(status_code=status_code, detail=detail.dict())
|
||||
|
||||
|
||||
# 获取所有直播分区列表
|
||||
@router.get("/fetch_all_live_areas", response_model=ResponseModel,
|
||||
summary="获取所有直播分区列表/Get a list of all live areas")
|
||||
async def fetch_collect_folders(request: Request,):
|
||||
"""
|
||||
# [中文]
|
||||
### 用途:
|
||||
- 获取所有直播分区列表
|
||||
### 参数:
|
||||
### 返回:
|
||||
- 所有直播分区列表
|
||||
|
||||
# [English]
|
||||
### Purpose:
|
||||
- Get a list of all live areas
|
||||
### Parameters:
|
||||
### Return:
|
||||
- list of all live areas
|
||||
|
||||
# [示例/Example]
|
||||
"""
|
||||
try:
|
||||
data = await BilibiliWebCrawler.fetch_all_live_areas()
|
||||
return ResponseModel(code=200,
|
||||
router=request.url.path,
|
||||
data=data)
|
||||
except Exception as e:
|
||||
status_code = 400
|
||||
detail = ErrorResponseModel(code=status_code,
|
||||
router=request.url.path,
|
||||
params=dict(request.query_params),
|
||||
)
|
||||
raise HTTPException(status_code=status_code, detail=detail.dict())
|
||||
|
||||
|
||||
# 通过bv号获得视频aid号
|
||||
@router.get("/bv_to_aid", response_model=ResponseModel, summary="通过bv号获得视频aid号/Generate aid by bvid")
|
||||
async def fetch_one_video(request: Request,
|
||||
bv_id: str = Query(example="BV1M1421t7hT", description="作品id/Video id")):
|
||||
"""
|
||||
# [中文]
|
||||
### 用途:
|
||||
- 通过bv号获得视频aid号
|
||||
### 参数:
|
||||
- bv_id: 作品id
|
||||
### 返回:
|
||||
- 视频aid号
|
||||
|
||||
# [English]
|
||||
### Purpose:
|
||||
- Generate aid by bvid
|
||||
### Parameters:
|
||||
- bv_id: Video id
|
||||
### Return:
|
||||
- Video aid
|
||||
|
||||
# [示例/Example]
|
||||
bv_id = "BV1M1421t7hT"
|
||||
"""
|
||||
try:
|
||||
data = await BilibiliWebCrawler.bv_to_aid(bv_id)
|
||||
return ResponseModel(code=200,
|
||||
router=request.url.path,
|
||||
data=data)
|
||||
except Exception as e:
|
||||
status_code = 400
|
||||
detail = ErrorResponseModel(code=status_code,
|
||||
router=request.url.path,
|
||||
params=dict(request.query_params),
|
||||
)
|
||||
raise HTTPException(status_code=status_code, detail=detail.dict())
|
||||
|
||||
|
||||
# 通过bv号获得视频分p信息
|
||||
@router.get("/fetch_video_parts", response_model=ResponseModel, summary="通过bv号获得视频分p信息/Get Video Parts By bvid")
|
||||
async def fetch_one_video(request: Request,
|
||||
bv_id: str = Query(example="BV1vf421i7hV", description="作品id/Video id")):
|
||||
"""
|
||||
# [中文]
|
||||
### 用途:
|
||||
- 通过bv号获得视频分p信息
|
||||
### 参数:
|
||||
- bv_id: 作品id
|
||||
### 返回:
|
||||
- 视频分p信息
|
||||
|
||||
# [English]
|
||||
### Purpose:
|
||||
- Get Video Parts By bvid
|
||||
### Parameters:
|
||||
- bv_id: Video id
|
||||
### Return:
|
||||
- Video Parts
|
||||
|
||||
# [示例/Example]
|
||||
bv_id = "BV1vf421i7hV"
|
||||
"""
|
||||
try:
|
||||
data = await BilibiliWebCrawler.fetch_video_parts(bv_id)
|
||||
return ResponseModel(code=200,
|
||||
router=request.url.path,
|
||||
data=data)
|
||||
except Exception as e:
|
||||
status_code = 400
|
||||
detail = ErrorResponseModel(code=status_code,
|
||||
router=request.url.path,
|
||||
params=dict(request.query_params),
|
||||
)
|
||||
raise HTTPException(status_code=status_code, detail=detail.dict())
|
||||
@ -13,7 +13,7 @@ DouyinWebCrawler = DouyinWebCrawler()
|
||||
# 获取单个作品数据
|
||||
@router.get("/fetch_one_video", response_model=ResponseModel, summary="获取单个作品数据/Get single video data")
|
||||
async def fetch_one_video(request: Request,
|
||||
aweme_id: str = Query(example="7345492945006595379", description="作品id/Video id")):
|
||||
aweme_id: str = Query(example="7372484719365098803", description="作品id/Video id")):
|
||||
"""
|
||||
# [中文]
|
||||
### 用途:
|
||||
@ -32,7 +32,7 @@ async def fetch_one_video(request: Request,
|
||||
- Video data
|
||||
|
||||
# [示例/Example]
|
||||
aweme_id = "7345492945006595379"
|
||||
aweme_id = "7372484719365098803"
|
||||
"""
|
||||
try:
|
||||
data = await DouyinWebCrawler.fetch_one_video(aweme_id)
|
||||
@ -470,7 +470,7 @@ async def handler_user_profile(request: Request,
|
||||
response_model=ResponseModel,
|
||||
summary="获取单个视频评论数据/Get single video comments data")
|
||||
async def fetch_video_comments(request: Request,
|
||||
aweme_id: str = Query(example="7345492945006595379", description="作品id/Video id"),
|
||||
aweme_id: str = Query(example="7372484719365098803", description="作品id/Video id"),
|
||||
cursor: int = Query(default=0, description="游标/Cursor"),
|
||||
count: int = Query(default=20, description="数量/Number")):
|
||||
"""
|
||||
@ -495,7 +495,7 @@ async def fetch_video_comments(request: Request,
|
||||
- Comments data
|
||||
|
||||
# [示例/Example]
|
||||
aweme_id = "7345492945006595379"
|
||||
aweme_id = "7372484719365098803"
|
||||
cursor = 0
|
||||
count = 20
|
||||
"""
|
||||
@ -734,6 +734,48 @@ async def generate_x_bogus(request: Request,
|
||||
raise HTTPException(status_code=status_code, detail=detail.dict())
|
||||
|
||||
|
||||
# 使用接口地址生成Abogus参数
|
||||
@router.get("/generate_a_bogus",
|
||||
response_model=ResponseModel,
|
||||
summary="使用接口网址生成A-Bogus参数/Generate A-Bogus parameter using API URL")
|
||||
async def generate_a_bogus(request: Request,
|
||||
url: str = Query(
|
||||
example="https://www.douyin.com/aweme/v1/web/aweme/detail/?device_platform=webapp&aid=6383&channel=channel_pc_web&pc_client_type=1&version_code=190500&version_name=19.5.0&cookie_enabled=true&browser_language=zh-CN&browser_platform=Win32&browser_name=Firefox&browser_online=true&engine_name=Gecko&os_name=Windows&os_version=10&platform=PC&screen_width=1920&screen_height=1080&browser_version=124.0&engine_version=122.0.0.0&cpu_core_num=12&device_memory=8&aweme_id=7372484719365098803"),
|
||||
user_agent: str = Query(
|
||||
example="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36")):
|
||||
"""
|
||||
# [中文]
|
||||
### 用途:
|
||||
- 使用接口网址生成A-Bogus参数
|
||||
### 参数:
|
||||
- url: 接口网址
|
||||
- user_agent: 用户代理，暂时不支持自定义，直接使用默认值即可。
|
||||
|
||||
# [English]
|
||||
### Purpose:
|
||||
- Generate A-Bogus parameter using API URL
|
||||
### Parameters:
|
||||
- url: API URL
|
||||
- user_agent: User agent, temporarily does not support customization, just use the default value.
|
||||
|
||||
# [示例/Example]
|
||||
url = "https://www.douyin.com/aweme/v1/web/aweme/detail/?device_platform=webapp&aid=6383&channel=channel_pc_web&pc_client_type=1&version_code=190500&version_name=19.5.0&cookie_enabled=true&browser_language=zh-CN&browser_platform=Win32&browser_name=Firefox&browser_online=true&engine_name=Gecko&os_name=Windows&os_version=10&platform=PC&screen_width=1920&screen_height=1080&browser_version=124.0&engine_version=122.0.0.0&cpu_core_num=12&device_memory=8&aweme_id=7372484719365098803"
|
||||
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36"
|
||||
"""
|
||||
try:
|
||||
a_bogus = await DouyinWebCrawler.get_a_bogus(url, user_agent)
|
||||
return ResponseModel(code=200,
|
||||
router=request.url.path,
|
||||
data=a_bogus)
|
||||
except Exception as e:
|
||||
status_code = 400
|
||||
detail = ErrorResponseModel(code=status_code,
|
||||
router=request.url.path,
|
||||
params=dict(request.query_params),
|
||||
)
|
||||
raise HTTPException(status_code=status_code, detail=detail.dict())
|
||||
|
||||
|
||||
# 提取单个用户id
|
||||
@router.get("/get_sec_user_id",
|
||||
response_model=ResponseModel,
|
||||
|
||||
@ -4,7 +4,7 @@ import zipfile
|
||||
import aiofiles
|
||||
import httpx
|
||||
import yaml
|
||||
from fastapi import APIRouter, Request # 导入FastAPI组件
|
||||
from fastapi import APIRouter, Request, Query, HTTPException # 导入FastAPI组件
|
||||
from starlette.responses import FileResponse
|
||||
|
||||
from app.api.models.APIResponseModel import ErrorResponseModel # 导入响应模型
|
||||
@ -18,25 +18,79 @@ config_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.pa
|
||||
with open(config_path, 'r', encoding='utf-8') as file:
|
||||
config = yaml.safe_load(file)
|
||||
|
||||
|
||||
async def fetch_data(url: str):
|
||||
async def fetch_data(url: str, headers: dict = None):
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
||||
}
|
||||
} if headers is None else headers.get('headers')
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(url, headers=headers)
|
||||
response.raise_for_status() # 确保响应是成功的
|
||||
return response
|
||||
|
||||
# 下载视频专用
|
||||
async def fetch_data_stream(url: str, request:Request , headers: dict = None, file_path: str = None):
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
||||
} if headers is None else headers.get('headers')
|
||||
async with httpx.AsyncClient() as client:
|
||||
# 启用流式请求
|
||||
async with client.stream("GET", url, headers=headers) as response:
|
||||
response.raise_for_status()
|
||||
|
||||
# 流式保存文件
|
||||
async with aiofiles.open(file_path, 'wb') as out_file:
|
||||
async for chunk in response.aiter_bytes():
|
||||
if await request.is_disconnected():
|
||||
print("客户端断开连接，清理未完成的文件")
|
||||
await out_file.close()
|
||||
os.remove(file_path)
|
||||
return False
|
||||
await out_file.write(chunk)
|
||||
return True
|
||||
|
||||
@router.get("/download", summary="在线下载抖音|TikTok视频/图片/Online download Douyin|TikTok video/image")
|
||||
async def download_file_hybrid(request: Request,
|
||||
url: str, prefix: bool = True, with_watermark: bool = False):
|
||||
url: str = Query(
|
||||
example="https://www.douyin.com/video/7372484719365098803",
|
||||
description="视频或图片的URL地址，也支持抖音|TikTok的分享链接，例如：https://v.douyin.com/e4J8Q7A/"),
|
||||
prefix: bool = True,
|
||||
with_watermark: bool = False):
|
||||
"""
|
||||
# [中文]
|
||||
### 用途:
|
||||
- 在线下载抖音|TikTok 无水印或有水印的视频/图片
|
||||
- 通过传入的视频URL参数，获取对应的视频或图片数据，然后下载到本地。
|
||||
- 如果你在尝试直接访问TikTok单一视频接口的JSON数据中的视频播放地址时遇到HTTP403错误，那么你可以使用此接口来下载视频。
|
||||
- 这个接口会占用一定的服务器资源，所以在Demo站点是默认关闭的，你可以在本地部署后调用此接口。
|
||||
### 参数:
|
||||
- url: 视频或图片的URL地址，也支持抖音|TikTok的分享链接，例如：https://v.douyin.com/e4J8Q7A/。
|
||||
- prefix: 下载文件的前缀，默认为True，可以在配置文件中修改。
|
||||
- with_watermark: 是否下载带水印的视频或图片，默认为False。
|
||||
### 返回:
|
||||
- 返回下载的视频或图片文件响应。
|
||||
|
||||
# [English]
|
||||
### Purpose:
|
||||
- Download Douyin|TikTok video/image with or without watermark online.
|
||||
- By passing the video URL parameter, get the corresponding video or image data, and then download it to the local.
|
||||
- If you encounter an HTTP403 error when trying to access the video playback address in the JSON data of the TikTok single video interface directly, you can use this interface to download the video.
|
||||
- This interface will occupy a certain amount of server resources, so it is disabled by default on the Demo site, you can call this interface after deploying it locally.
|
||||
### Parameters:
|
||||
- url: The URL address of the video or image, also supports Douyin|TikTok sharing links, for example: https://v.douyin.com/e4J8Q7A/.
|
||||
- prefix: The prefix of the downloaded file, the default is True, and can be modified in the configuration file.
|
||||
- with_watermark: Whether to download videos or images with watermarks, the default is False.
|
||||
### Returns:
|
||||
- Return the response of the downloaded video or image file.
|
||||
|
||||
# [示例/Example]
|
||||
url: https://www.douyin.com/video/7372484719365098803
|
||||
"""
|
||||
# 是否开启此端点/Whether to enable this endpoint
|
||||
if not config["API"]["Download_Switch"]:
|
||||
code = 400
|
||||
message = "Download endpoint is disabled."
|
||||
return ErrorResponseModel(code=code, message=message, router=request.url.path, params=dict(request.query_params))
|
||||
message = "Download endpoint is disabled in the configuration file. | 配置文件中已禁用下载端点。"
|
||||
return ErrorResponseModel(code=code, message=message, router=request.url.path,
|
||||
params=dict(request.query_params))
|
||||
|
||||
# 开始解析数据/Start parsing data
|
||||
try:
|
||||
@ -68,11 +122,19 @@ async def download_file_hybrid(request: Request,
|
||||
return FileResponse(path=file_path, media_type='video/mp4', filename=file_name)
|
||||
|
||||
# 获取视频文件
|
||||
response = await fetch_data(url)
|
||||
__headers = await HybridCrawler.TikTokWebCrawler.get_tiktok_headers() if platform == 'tiktok' else await HybridCrawler.DouyinWebCrawler.get_douyin_headers()
|
||||
# response = await fetch_data(url, headers=__headers)
|
||||
|
||||
# 保存文件
|
||||
async with aiofiles.open(file_path, 'wb') as out_file:
|
||||
await out_file.write(response.content)
|
||||
success = await fetch_data_stream(url, request, headers=__headers, file_path=file_path)
|
||||
if not success:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="An error occurred while fetching data"
|
||||
)
|
||||
|
||||
# # 保存文件
|
||||
# async with aiofiles.open(file_path, 'wb') as out_file:
|
||||
# await out_file.write(response.content)
|
||||
|
||||
# 返回文件内容
|
||||
return FileResponse(path=file_path, filename=file_name, media_type="video/mp4")
|
||||
@ -115,6 +177,6 @@ async def download_file_hybrid(request: Request,
|
||||
|
||||
# 异常处理/Exception handling
|
||||
except Exception as e:
|
||||
print(e)
|
||||
code = 400
|
||||
return ErrorResponseModel(code=code, message=str(e), router=request.url.path, params=dict(request.query_params))
|
||||
|
||||
|
||||
@ -8,7 +8,10 @@ TikTokAPPCrawler = TikTokAPPCrawler()
|
||||
|
||||
|
||||
# 获取单个作品数据
|
||||
@router.get("/fetch_one_video", response_model=ResponseModel, summary="获取单个作品数据/Get single video data")
|
||||
@router.get("/fetch_one_video",
|
||||
response_model=ResponseModel,
|
||||
summary="获取单个作品数据/Get single video data"
|
||||
)
|
||||
async def fetch_one_video(request: Request,
|
||||
aweme_id: str = Query(example="7350810998023949599", description="作品id/Video id")):
|
||||
"""
|
||||
@ -43,3 +46,4 @@ async def fetch_one_video(request: Request,
|
||||
params=dict(request.query_params),
|
||||
)
|
||||
raise HTTPException(status_code=status_code, detail=detail.dict())
|
||||
|
||||
@ -55,7 +55,7 @@ async def fetch_one_video(request: Request,
|
||||
response_model=ResponseModel,
|
||||
summary="获取用户的个人信息/Get user profile")
|
||||
async def fetch_user_profile(request: Request,
|
||||
uniqueId: str = Query(example="tiktok", description="用户uniqueId/User uniqueId"),
|
||||
uniqueId: str = Query(default="tiktok", description="用户uniqueId/User uniqueId"),
|
||||
secUid: str = Query(default="", description="用户secUid/User secUid"),):
|
||||
"""
|
||||
# [中文]
|
||||
|
||||
@ -3,6 +3,7 @@ from app.api.endpoints import (
|
||||
tiktok_web,
|
||||
tiktok_app,
|
||||
douyin_web,
|
||||
bilibili_web,
|
||||
hybrid_parsing, ios_shortcut, download,
|
||||
)
|
||||
|
||||
@ -15,6 +16,9 @@ router.include_router(tiktok_app.router, prefix="/tiktok/app", tags=["TikTok-App
|
||||
# Douyin routers
|
||||
router.include_router(douyin_web.router, prefix="/douyin/web", tags=["Douyin-Web-API"])
|
||||
|
||||
# Bilibili routers
|
||||
router.include_router(bilibili_web.router, prefix="/bilibili/web", tags=["Bilibili-Web-API"])
|
||||
|
||||
# Hybrid routers
|
||||
router.include_router(hybrid_parsing.router, prefix="/hybrid", tags=["Hybrid-API"])
|
||||
|
||||
|
||||
@ -77,6 +77,10 @@ tags_metadata = [
|
||||
"name": "TikTok-App-API",
|
||||
"description": "**(TikTok-App-API数据接口/TikTok-App-API data endpoints)**",
|
||||
},
|
||||
{
|
||||
"name": "Bilibili-Web-API",
|
||||
"description": "**(Bilibili-Web-API数据接口/Bilibili-Web-API data endpoints)**",
|
||||
},
|
||||
{
|
||||
"name": "iOS-Shortcut",
|
||||
"description": "**(iOS快捷指令数据接口/iOS-Shortcut data endpoints)**",
|
||||
@ -103,7 +107,7 @@ description = f"""
|
||||
#### 备注
|
||||
- 本项目仅供学习交流使用，不得用于违法用途，否则后果自负。
|
||||
- 如果你不想自己部署，可以直接使用我们的在线API服务：[Douyin_TikTok_Download_API](https://douyin.wtf/docs)
|
||||
- 如果你需要更稳定以及更多功能的API服务，可以使用付费API服务：[TikHub API](https://beta.tikhub.io/)
|
||||
- 如果你需要更稳定以及更多功能的API服务，可以使用付费API服务：[TikHub API](https://api.tikhub.io/)
|
||||
|
||||
### [English]
|
||||
|
||||
@ -116,7 +120,7 @@ description = f"""
|
||||
#### Note
|
||||
- This project is for learning and communication only, and shall not be used for illegal purposes, otherwise the consequences shall be borne by yourself.
|
||||
- If you do not want to deploy it yourself, you can directly use our online API service: [Douyin_TikTok_Download_API](https://douyin.wtf/docs)
|
||||
- If you need a more stable and feature-rich API service, you can use the paid API service: [TikHub API](https://beta.tikhub.io)
|
||||
- If you need a more stable and feature-rich API service, you can use the paid API service: [TikHub API](https://api.tikhub.io)
|
||||
"""
|
||||
|
||||
docs_url = config['API']['Docs_URL']
|
||||
|
||||
@ -17,14 +17,49 @@ def api_document_pop_window():
|
||||
put_markdown("----")
|
||||
put_markdown(t("> 更多接口",
|
||||
"> More APIs"))
|
||||
put_markdown(t("如果你想要使用更多且更稳定的API服务，可以使用付费API服务",
|
||||
"If you want to use more and more stable API services, you can use paid API services"))
|
||||
put_link('[TikHub API]', 'https://api.tikhub.io', new_window=True)
|
||||
put_markdown(
|
||||
t("[TikHub.io](https://beta-web.tikhub.io/en-us/users/signin)是一个API平台，提供包括Douyin、TikTok在内的各种公开数据接口，如果您想支持 [Douyin_TikTok_Download_API](https://github.com/Evil0ctal/Douyin_TikTok_Download_API) 项目的开发，我们强烈建议您选择[TikHub.io](https://beta-web.tikhub.io/en-us/users/signin)。",
|
||||
"[TikHub.io](https://beta-web.tikhub.io/en-us/users/signin) is an API platform that provides various public data interfaces including Douyin and TikTok. If you want to support the development of the [Douyin_TikTok_Download_API](https://github.com/Evil0ctal/Douyin_TikTok_Download_API) project, we strongly recommend that you choose [TikHub.io](https://beta-web.tikhub.io/en-us/users/signin)."))
|
||||
put_markdown(
|
||||
t("#### 特点：",
|
||||
"#### Features:"))
|
||||
put_markdown(
|
||||
t("> 📦 开箱即用",
|
||||
"> 📦 Ready to use"))
|
||||
put_markdown(
|
||||
t("简化使用流程，利用封装好的SDK迅速开展开发工作。所有API接口均依据RESTful架构设计，并使用OpenAPI规范进行描述和文档化，附带示例参数，确保调用更加简便。",
|
||||
"Simplify the use process and quickly carry out development work using the encapsulated SDK. All API interfaces are designed based on the RESTful architecture and described and documented using the OpenAPI specification, with example parameters attached to ensure easier calls."))
|
||||
put_markdown(
|
||||
t("> 💰 成本优势",
|
||||
"> 💰 Cost advantage"))
|
||||
put_markdown(
|
||||
t("不预设套餐限制，没有月度使用门槛，所有消费按实际使用量即时计费，并且根据用户每日的请求量进行阶梯式计费，同时可以通过每日签到在用户后台进行签到获取免费的额度，并且这些免费额度不会过期。",
|
||||
"There is no preset package limit, no monthly usage threshold, all consumption is billed in real time according to the actual usage, and billed in a step-by-step manner according to the user's daily request volume. At the same time, you can sign in daily in the user background to get free quotas, and these free quotas will not expire."))
|
||||
put_markdown(
|
||||
t("> ⚡️ 快速支持",
|
||||
"> ⚡️ Quick support"))
|
||||
put_markdown(
|
||||
t("我们有一个庞大的Discord社区服务器，管理员和其他用户会在服务器中快速的回复你，帮助你快速解决当前的问题。",
|
||||
"We have a huge Discord community server, where administrators and other users will quickly reply to you in the server and help you quickly solve the current problem."))
|
||||
put_markdown(
|
||||
t("> 🎉 拥抱开源",
|
||||
"> 🎉 Embrace open source"))
|
||||
put_markdown(
|
||||
t("TikHub的部分源代码会开源在Github上，并且会赞助一些开源项目的作者。",
|
||||
"Some of TikHub's source code will be open sourced on Github, and will sponsor some open source project authors."))
|
||||
put_markdown(
|
||||
t("#### 链接：",
|
||||
"#### Links:"))
|
||||
put_markdown(
|
||||
t("- Github: [TikHub Github](https://github.com/TikHubIO)",
|
||||
"- Github: [TikHub Github](https://github.com/TikHubIO)"))
|
||||
put_markdown(
|
||||
t("- Discord: [TikHub Discord](https://discord.com/invite/aMEAS8Xsvz)",
|
||||
"- Discord: [TikHub Discord](https://discord.com/invite/aMEAS8Xsvz)"))
|
||||
put_markdown(
|
||||
t("- Register: [TikHub signup](https://beta-web.tikhub.io/en-us/users/signup)",
|
||||
"- Register: [TikHub signup](https://beta-web.tikhub.io/en-us/users/signup)"))
|
||||
put_markdown(
|
||||
t("- API Docs: [TikHub API Docs](https://api.tikhub.io/)",
|
||||
"- API Docs: [TikHub API Docs](https://api.tikhub.io/)"))
|
||||
put_markdown("----")
|
||||
put_markdown(t("> 限时免费测试",
|
||||
"> Free test for a limited time"))
|
||||
put_markdown(t("这里也有一个测试版的API服务，你可以直接免费使用",
|
||||
"There is also a beta version of the API service, which you can use for free"))
|
||||
put_markdown(t("测试接口只会保留一段时间，不保证数据的稳定性",
|
||||
"The test interface will only be retained for a period of time, and the stability of the data is not guaranteed"))
|
||||
put_link('[TikHub Beta API]', 'https://beta.tikhub.io', new_window=True)
|
||||
|
||||
@ -19,10 +19,6 @@ with open(config_path, 'r', encoding='utf-8') as file:
|
||||
config = yaml.safe_load(file)
|
||||
|
||||
|
||||
# 网站域名/Website domain
|
||||
domain = config['Web']['Domain']
|
||||
|
||||
|
||||
# 校验输入值/Validate input value
|
||||
def valid_check(input_data: str):
|
||||
# 检索出所有链接并返回列表/Retrieve all links and return a list
|
||||
@ -65,10 +61,13 @@ def error_do(reason: str, value: str) -> None:
|
||||
"- The video has been deleted or the link is incorrect."))
|
||||
put_markdown(ViewsUtils.t("- 接口风控，请求过于频繁。",
|
||||
"- Interface risk control, request too frequent.")),
|
||||
put_markdown(ViewsUtils.t("- 没有使用有效的Cookie，如果你部署后没有替换相应的Cookie，可能会导致解析失败。",
|
||||
"- No valid Cookie is used. If you do not replace the corresponding Cookie after deployment, it may cause parsing failure."))
|
||||
put_markdown(ViewsUtils.t("> 寻求帮助:", "> Seek help:"))
|
||||
put_markdown(ViewsUtils.t(
|
||||
"- 你可以尝试再次解析，或者尝试自行部署项目，然后替换`./app/crawlers/平台文件夹/config.yaml`中的`cookie`值。",
|
||||
"- You can try to parse again, or try to deploy the project by yourself, and then replace the `cookie` value in `./app/crawlers/platform folder/config.yaml`."))
|
||||
|
||||
put_markdown(
|
||||
"- GitHub Issue: [Evil0ctal/Douyin_TikTok_Download_API](https://github.com/Evil0ctal/Douyin_TikTok_Download_API/issues)")
|
||||
put_html("<hr>")
|
||||
@ -135,43 +134,47 @@ def parse_video():
|
||||
[ViewsUtils.t('API链接', 'API URL'),
|
||||
put_link(
|
||||
ViewsUtils.t('点击查看', 'Click to view'),
|
||||
f"{domain}/api/hybrid/video_data?url={url}&minimal=false",
|
||||
f"/api/hybrid/video_data?url={url}&minimal=false",
|
||||
new_window=True)],
|
||||
[ViewsUtils.t('API链接-精简', 'API URL-Minimal'),
|
||||
put_link(ViewsUtils.t('点击查看', 'Click to view'),
|
||||
f"{domain}/api/hybrid/video_data?url={url}&minimal=true",
|
||||
f"/api/hybrid/video_data?url={url}&minimal=true",
|
||||
new_window=True)]
|
||||
|
||||
]
|
||||
# 如果是视频/If it's video
|
||||
if url_type == ViewsUtils.t('视频', 'Video'):
|
||||
# 添加视频信息
|
||||
wm_video_url_HQ = data.get('video_data').get('wm_video_url_HQ')
|
||||
nwm_video_url_HQ = data.get('video_data').get('nwm_video_url_HQ')
|
||||
if wm_video_url_HQ and nwm_video_url_HQ:
|
||||
table_list.insert(4, [ViewsUtils.t('视频链接-水印', 'Video URL-Watermark'),
|
||||
put_link(ViewsUtils.t('点击查看', 'Click to view'),
|
||||
data.get('video_data').get('wm_video_url_HQ'), new_window=True)])
|
||||
wm_video_url_HQ, new_window=True)])
|
||||
table_list.insert(5, [ViewsUtils.t('视频链接-无水印', 'Video URL-No Watermark'),
|
||||
put_link(ViewsUtils.t('点击查看', 'Click to view'),
|
||||
data.get('video_data').get('nwm_video_url_HQ'), new_window=True)])
|
||||
nwm_video_url_HQ, new_window=True)])
|
||||
table_list.insert(6, [ViewsUtils.t('视频下载-水印', 'Video Download-Watermark'),
|
||||
put_link(ViewsUtils.t('点击下载', 'Click to download'),
|
||||
f"{domain}/api/download?url={url}&prefix=true&with_watermark=true",
|
||||
f"/api/download?url={url}&prefix=true&with_watermark=true",
|
||||
new_window=True)])
|
||||
table_list.insert(7, [ViewsUtils.t('视频下载-无水印', 'Video Download-No-Watermark'),
|
||||
put_link(ViewsUtils.t('点击下载', 'Click to download'),
|
||||
f"{domain}/api/download?url={url}&prefix=true&with_watermark=false",
|
||||
f"/api/download?url={url}&prefix=true&with_watermark=false",
|
||||
new_window=True)])
|
||||
# 添加视频信息
|
||||
table_list.insert(0, [put_video(data.get('video_data').get('nwm_video_url_HQ'), poster=None, loop=True, width='50%')])
|
||||
table_list.insert(0, [
|
||||
put_video(data.get('video_data').get('nwm_video_url_HQ'), poster=None, loop=True, width='50%')])
|
||||
# 如果是图片/If it's image
|
||||
elif url_type == ViewsUtils.t('图片', 'Image'):
|
||||
# 添加图片下载链接
|
||||
table_list.insert(4, [ViewsUtils.t('图片打包下载-水印', 'Download images ZIP-Watermark'),
|
||||
put_link(ViewsUtils.t('点击下载', 'Click to download'),
|
||||
f"{domain}/api/download?url={url}&prefix=true&with_watermark=true",
|
||||
f"/api/download?url={url}&prefix=true&with_watermark=true",
|
||||
new_window=True)])
|
||||
table_list.insert(5, [ViewsUtils.t('图片打包下载-无水印', 'Download images ZIP-No-Watermark'),
|
||||
put_link(ViewsUtils.t('点击下载', 'Click to download'),
|
||||
f"{domain}/api/download?url={url}&prefix=true&with_watermark=false",
|
||||
f"/api/download?url={url}&prefix=true&with_watermark=false",
|
||||
new_window=True)])
|
||||
# 添加图片信息
|
||||
no_watermark_image_list = data.get('image_data').get('no_watermark_image_list')
|
||||
|
||||
@ -3,53 +3,59 @@
|
||||
# Set script to exit on any errors.
|
||||
set -e
|
||||
|
||||
echo 'Updating package lists...'
|
||||
echo 'Updating package lists... | 正在更新软件包列表...'
|
||||
sudo apt-get update
|
||||
|
||||
echo 'Installing Git...'
|
||||
echo 'Installing Git... | 正在安装Git...'
|
||||
sudo apt-get install -y git
|
||||
|
||||
echo 'Installing Python3...'
|
||||
echo 'Installing Python3... | 正在安装Python3...'
|
||||
sudo apt install -y python3
|
||||
|
||||
echo 'Installing PIP3...'
|
||||
echo 'Installing PIP3... | 正在安装PIP3...'
|
||||
sudo apt install -y python3-pip
|
||||
|
||||
echo 'Installing python3-venv...'
|
||||
echo 'Installing python3-venv... | 正在安装python3-venv...'
|
||||
sudo apt install -y python3-venv
|
||||
|
||||
echo 'Creating path: /www/wwwroot'
|
||||
echo 'Creating path: /www/wwwroot | 正在创建路径: /www/wwwroot'
|
||||
sudo mkdir -p /www/wwwroot
|
||||
|
||||
cd /www/wwwroot || { echo "Failed to change directory to /www/wwwroot"; exit 1; }
|
||||
cd /www/wwwroot || { echo "Failed to change directory to /www/wwwroot | 无法切换到目录 /www/wwwroot"; exit 1; }
|
||||
|
||||
echo 'Cloning Douyin_TikTok_Download_API.git from Github!'
|
||||
echo 'Cloning Douyin_TikTok_Download_API.git from Github! | 正在从Github克隆Douyin_TikTok_Download_API.git!'
|
||||
sudo git clone https://github.com/Evil0ctal/Douyin_TikTok_Download_API.git
|
||||
|
||||
cd Douyin_TikTok_Download_API/ || { echo "Failed to change directory to Douyin_TikTok_Download_API"; exit 1; }
|
||||
cd Douyin_TikTok_Download_API/ || { echo "Failed to change directory to Douyin_TikTok_Download_API | 无法切换到目录 Douyin_TikTok_Download_API"; exit 1; }
|
||||
|
||||
echo 'Creating a virtual environment'
|
||||
echo 'Creating a virtual environment | 正在创建虚拟环境'
|
||||
python3 -m venv venv
|
||||
|
||||
echo 'Activating the virtual environment'
|
||||
echo 'Activating the virtual environment | 正在激活虚拟环境'
|
||||
source venv/bin/activate
|
||||
|
||||
echo 'Setting pip to use the default PyPI index'
|
||||
echo 'Setting pip to use the default PyPI index | 设置pip使用默认PyPI索引'
|
||||
pip config set global.index-url https://pypi.org/simple/
|
||||
|
||||
echo 'Installing dependencies from requirements.txt'
|
||||
echo 'Installing pip setuptools | 安装pip setuptools'
|
||||
pip install setuptools
|
||||
|
||||
echo 'Installing dependencies from requirements.txt | 从requirements.txt安装依赖'
|
||||
pip install -r requirements.txt
|
||||
|
||||
echo 'Deactivating the virtual environment'
|
||||
echo 'Deactivating the virtual environment | 正在停用虚拟环境'
|
||||
deactivate
|
||||
|
||||
echo 'Adding Douyin_TikTok_Download_API to system service'
|
||||
echo 'Adding Douyin_TikTok_Download_API to system service | 将Douyin_TikTok_Download_API添加到系统服务'
|
||||
sudo cp daemon/* /etc/systemd/system/
|
||||
|
||||
echo 'Enabling Douyin_TikTok_Download_API service'
|
||||
echo 'Enabling Douyin_TikTok_Download_API service | 启用Douyin_TikTok_Download_API服务'
|
||||
sudo systemctl enable Douyin_TikTok_Download_API.service
|
||||
|
||||
echo 'Starting Douyin_TikTok_Download_API service'
|
||||
echo 'Starting Douyin_TikTok_Download_API service | 启动Douyin_TikTok_Download_API服务'
|
||||
sudo systemctl start Douyin_TikTok_Download_API.service
|
||||
|
||||
echo 'Douyin_TikTok_Download_API installation complete!'
|
||||
echo 'Douyin_TikTok_Download_API installation complete! | Douyin_TikTok_Download_API安装完成!'
|
||||
echo 'You can access the API at http://localhost:80 | 您可以在http://localhost:80访问API'
|
||||
echo 'You can change the port in config.yaml under the /www/wwwroot/Douyin_TikTok_Download_API directory | 您可以在/www/wwwroot/Douyin_TikTok_Download_API目录下的config.yaml中更改端口'
|
||||
echo 'If the API is not working, please change the cookie in config.yaml under the /www/wwwroot/Douyin_TikTok_Download_API/crawler/[Douyin/TikTok]/[APP/Web]/config.yaml directory | 如果API无法工作，请更改/www/wwwroot/Douyin_TikTok_Download_API/crawler/[Douyin/TikTok]/[APP/Web]/config.yaml目录下的cookie'
|
||||
|
||||
16
config.yaml
16
config.yaml
@ -30,8 +30,8 @@ API:
|
||||
Redoc_URL: /redoc # API documentation URL | API文档URL
|
||||
|
||||
# API Information
|
||||
Version: V4.0.0 # API version | API版本
|
||||
Update_Time: 2024/04/22 # API update time | API更新时间
|
||||
Version: V4.1.2 # API version | API版本
|
||||
Update_Time: 2025/03/16 # API update time | API更新时间
|
||||
Environment: Demo # API environment | API环境
|
||||
|
||||
# Download Configuration
|
||||
@ -44,9 +44,9 @@ API:
|
||||
|
||||
# iOS Shortcut
|
||||
iOS_Shortcut:
|
||||
iOS_Shortcut_Version: 6.0
|
||||
iOS_Shortcut_Update_Time: 2024/04/22
|
||||
iOS_Shortcut_Link: https://www.icloud.com/shortcuts/4465d514869e4ca585074d40328f3e0e
|
||||
iOS_Shortcut_Link_EN: https://www.icloud.com/shortcuts/58e3a2cbac784a6782f1031c6b1dd9f8
|
||||
iOS_Shortcut_Update_Note: 重新适配https://api.douyin.wtf(API-V1 3.0.0版本)
|
||||
iOS_Shortcut_Update_Note_EN: Re-adapt https://api.douyin.wtf (API-V1 3.0.0 version)
|
||||
iOS_Shortcut_Version: 7.0
|
||||
iOS_Shortcut_Update_Time: 2024/07/05
|
||||
iOS_Shortcut_Link: https://www.icloud.com/shortcuts/06f891a026df40cfa967a907feaea632
|
||||
iOS_Shortcut_Link_EN: https://www.icloud.com/shortcuts/06f891a026df40cfa967a907feaea632
|
||||
iOS_Shortcut_Update_Note: 重构了快捷指令以兼容TikHub API。
|
||||
iOS_Shortcut_Update_Note_EN: Refactored the shortcut to be compatible with the TikHub API.
|
||||
|
||||
13
crawlers/bilibili/web/config.yaml
Normal file
13
crawlers/bilibili/web/config.yaml
Normal file
@ -0,0 +1,13 @@
|
||||
TokenManager:
|
||||
bilibili:
|
||||
headers:
|
||||
'accept-language': zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6
|
||||
'origin': https://www.bilibili.com
|
||||
'referer': https://space.bilibili.com/
|
||||
'origin_2': https://space.bilibili.com
|
||||
'cookie': buvid4=748EC8F0-82E2-1672-A286-8445DDB2A80C06110-023112304-; buvid3=73EF1E2E-B7A9-78DD-F2AE-9AB2B476E27638524infoc; b_nut=1727075638; _uuid=77AA4910F-5C8F-9647-7DA3-F583C8108BD7942063infoc; buvid_fp=75b22e5d0c3dbc642b1c80956c62c7da; bili_ticket=eyJhbGciOiJIUzI1NiIsImtpZCI6InMwMyIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3MjczNDI1NTYsImlhdCI6MTcyNzA4MzI5NiwicGx0IjotMX0.G3pvk6OC4FDWBL7GNgKkkVtUMl29UtNdgok_cANoKsw; bili_ticket_expires=1727342496; header_theme_version=CLOSE; enable_web_push=DISABLE; home_feed_column=5; browser_resolution=1488-712; b_lsid=5B4EDF8A_1921EAA1BDA
|
||||
'user-agent': Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36
|
||||
|
||||
proxies:
|
||||
http:
|
||||
https:
|
||||
62
crawlers/bilibili/web/endpoints.py
Normal file
62
crawlers/bilibili/web/endpoints.py
Normal file
@ -0,0 +1,62 @@
|
||||
class BilibiliAPIEndpoints:
|
||||
|
||||
"-------------------------------------------------------域名-domain-------------------------------------------------------"
|
||||
# 哔哩哔哩接口域名
|
||||
BILIAPI_DOMAIN = "https://api.bilibili.com"
|
||||
|
||||
# 哔哩哔哩直播域名
|
||||
LIVE_DOMAIN = "https://api.live.bilibili.com"
|
||||
|
||||
"-------------------------------------------------------接口-api-------------------------------------------------------"
|
||||
# 作品信息 (Post Detail)
|
||||
POST_DETAIL = f"{BILIAPI_DOMAIN}/x/web-interface/view"
|
||||
|
||||
# 作品视频流
|
||||
VIDEO_PLAYURL = f"{BILIAPI_DOMAIN}/x/player/wbi/playurl"
|
||||
|
||||
# 用户发布视频作品数据
|
||||
USER_POST = f"{BILIAPI_DOMAIN}/x/space/wbi/arc/search"
|
||||
|
||||
# 收藏夹列表
|
||||
COLLECT_FOLDERS = f"{BILIAPI_DOMAIN}/x/v3/fav/folder/created/list-all"
|
||||
|
||||
# 收藏夹视频
|
||||
COLLECT_VIDEOS = f"{BILIAPI_DOMAIN}/x/v3/fav/resource/list"
|
||||
|
||||
# 用户个人信息
|
||||
USER_DETAIL = f"{BILIAPI_DOMAIN}/x/space/wbi/acc/info"
|
||||
|
||||
# 综合热门
|
||||
COM_POPULAR = f"{BILIAPI_DOMAIN}/x/web-interface/popular"
|
||||
|
||||
# 每周必看
|
||||
WEEKLY_POPULAR = f"{BILIAPI_DOMAIN}/x/web-interface/popular/series/one"
|
||||
|
||||
# 入站必刷
|
||||
PRECIOUS_POPULAR = f"{BILIAPI_DOMAIN}/x/web-interface/popular/precious"
|
||||
|
||||
# 视频评论
|
||||
VIDEO_COMMENTS = f"{BILIAPI_DOMAIN}/x/v2/reply"
|
||||
|
||||
# 用户动态
|
||||
USER_DYNAMIC = f"{BILIAPI_DOMAIN}/x/polymer/web-dynamic/v1/feed/space"
|
||||
|
||||
# 评论的回复
|
||||
COMMENT_REPLY = f"{BILIAPI_DOMAIN}/x/v2/reply/reply"
|
||||
|
||||
# 视频分p信息
|
||||
VIDEO_PARTS = f"{BILIAPI_DOMAIN}/x/player/pagelist"
|
||||
|
||||
# 直播间信息
|
||||
LIVEROOM_DETAIL = f"{LIVE_DOMAIN}/room/v1/Room/get_info"
|
||||
|
||||
# 直播分区列表
|
||||
LIVE_AREAS = f"{LIVE_DOMAIN}/room/v1/Area/getList"
|
||||
|
||||
# 直播间视频流
|
||||
LIVE_VIDEOS = f"{LIVE_DOMAIN}/room/v1/Room/playUrl"
|
||||
|
||||
# 正在直播的主播
|
||||
LIVE_STREAMER = f"{LIVE_DOMAIN}/xlive/web-interface/v1/second/getList"
|
||||
|
||||
|
||||
39
crawlers/bilibili/web/models.py
Normal file
39
crawlers/bilibili/web/models.py
Normal file
@ -0,0 +1,39 @@
|
||||
import time
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class BaseRequestsModel(BaseModel):
|
||||
wts: str = str(round(time.time()))
|
||||
|
||||
|
||||
class UserPostVideos(BaseRequestsModel):
|
||||
dm_img_inter: str = '{"ds":[],"wh":[3557,5674,5],"of":[154,308,154]}'
|
||||
dm_img_list: list = []
|
||||
mid: str
|
||||
pn: int
|
||||
ps: str = "20"
|
||||
|
||||
|
||||
class UserProfile(BaseRequestsModel):
|
||||
mid: str
|
||||
|
||||
|
||||
class UserDynamic(BaseRequestsModel):
|
||||
host_mid: str
|
||||
offset: str
|
||||
wts: str = str(round(time.time()))
|
||||
|
||||
|
||||
class ComPopular(BaseRequestsModel):
|
||||
pn: int
|
||||
ps: str = "20"
|
||||
web_location: str = "333.934"
|
||||
|
||||
|
||||
class PlayUrl(BaseRequestsModel):
|
||||
qn: str
|
||||
fnval: str = '4048'
|
||||
bvid: str
|
||||
cid: str
|
||||
|
||||
|
||||
104
crawlers/bilibili/web/utils.py
Normal file
104
crawlers/bilibili/web/utils.py
Normal file
@ -0,0 +1,104 @@
|
||||
from urllib.parse import urlencode
|
||||
from crawlers.bilibili.web import wrid
|
||||
from crawlers.utils.logger import logger
|
||||
from crawlers.bilibili.web.endpoints import BilibiliAPIEndpoints
|
||||
|
||||
|
||||
class EndpointGenerator:
|
||||
def __init__(self, params: dict):
|
||||
self.params = params
|
||||
|
||||
# 获取用户发布视频作品数据 生成enpoint
|
||||
async def user_post_videos_endpoint(self) -> str:
|
||||
# 添加w_rid
|
||||
endpoint = await WridManager.wrid_model_endpoint(params=self.params)
|
||||
# 拼接成最终结果并返回
|
||||
final_endpoint = BilibiliAPIEndpoints.USER_POST + '?' + endpoint
|
||||
return final_endpoint
|
||||
|
||||
# 获取视频流地址 生成enpoint
|
||||
async def video_playurl_endpoint(self) -> str:
|
||||
# 添加w_rid
|
||||
endpoint = await WridManager.wrid_model_endpoint(params=self.params)
|
||||
# 拼接成最终结果并返回
|
||||
final_endpoint = BilibiliAPIEndpoints.VIDEO_PLAYURL + '?' + endpoint
|
||||
return final_endpoint
|
||||
|
||||
# 获取指定用户的信息 生成enpoint
|
||||
async def user_profile_endpoint(self) -> str:
|
||||
# 添加w_rid
|
||||
endpoint = await WridManager.wrid_model_endpoint(params=self.params)
|
||||
# 拼接成最终结果并返回
|
||||
final_endpoint = BilibiliAPIEndpoints.USER_DETAIL + '?' + endpoint
|
||||
return final_endpoint
|
||||
|
||||
# 获取综合热门视频信息 生成enpoint
|
||||
async def com_popular_endpoint(self) -> str:
|
||||
# 添加w_rid
|
||||
endpoint = await WridManager.wrid_model_endpoint(params=self.params)
|
||||
# 拼接成最终结果并返回
|
||||
final_endpoint = BilibiliAPIEndpoints.COM_POPULAR + '?' + endpoint
|
||||
return final_endpoint
|
||||
|
||||
# 获取指定用户动态
|
||||
async def user_dynamic_endpoint(self):
|
||||
# 添加w_rid
|
||||
endpoint = await WridManager.wrid_model_endpoint(params=self.params)
|
||||
# 拼接成最终结果并返回
|
||||
final_endpoint = BilibiliAPIEndpoints.USER_DYNAMIC + '?' + endpoint
|
||||
return final_endpoint
|
||||
|
||||
|
||||
class WridManager:
|
||||
@classmethod
|
||||
async def get_encode_query(cls, params: dict) -> str:
|
||||
params['wts'] = params['wts'] + "ea1db124af3c7062474693fa704f4ff8"
|
||||
params = dict(sorted(params.items())) # 按照 key 重排参数
|
||||
# 过滤 value 中的 "!'()*" 字符
|
||||
params = {
|
||||
k: ''.join(filter(lambda chr: chr not in "!'()*", str(v)))
|
||||
for k, v
|
||||
in params.items()
|
||||
}
|
||||
query = urlencode(params) # 序列化参数
|
||||
return query
|
||||
|
||||
@classmethod
|
||||
async def wrid_model_endpoint(cls, params: dict) -> str:
|
||||
wts = params["wts"]
|
||||
encode_query = await cls.get_encode_query(params)
|
||||
# 获取w_rid参数
|
||||
w_rid = wrid.get_wrid(e=encode_query)
|
||||
params["wts"] = wts
|
||||
params["w_rid"] = w_rid
|
||||
return "&".join(f"{k}={v}" for k, v in params.items())
|
||||
|
||||
# BV号转为对应av号
|
||||
async def bv2av(bv_id: str) -> int:
|
||||
table = "fZodR9XQDSUm21yCkr6zBqiveYah8bt4xsWpHnJE7jL5VG3guMTKNPAwcF"
|
||||
s = [11, 10, 3, 8, 4, 6, 2, 9, 5, 7]
|
||||
xor = 177451812
|
||||
add_105 = 8728348608
|
||||
add_all = 8728348608 - (2 ** 31 - 1) - 1
|
||||
tr = [0] * 128
|
||||
for i in range(58):
|
||||
tr[ord(table[i])] = i
|
||||
r = 0
|
||||
for i in range(6):
|
||||
r += tr[ord(bv_id[s[i]])] * (58 ** i)
|
||||
add = add_105
|
||||
if r < add:
|
||||
add = add_all
|
||||
aid = (r - add) ^ xor
|
||||
return aid
|
||||
|
||||
# 响应分析
|
||||
class ResponseAnalyzer:
|
||||
# 用户收藏夹信息
|
||||
@classmethod
|
||||
async def collect_folders_analyze(cls, response: dict) -> dict:
|
||||
if response['data']:
|
||||
return response
|
||||
else:
|
||||
logger.warning("该用户收藏夹为空/用户设置为不可见")
|
||||
return {"code": 1, "message": "该用户收藏夹为空/用户设置为不可见"}
|
||||
435
crawlers/bilibili/web/web_crawler.py
Normal file
435
crawlers/bilibili/web/web_crawler.py
Normal file
@ -0,0 +1,435 @@
|
||||
# ==============================================================================
|
||||
# Copyright (C) 2021 Evil0ctal
|
||||
#
|
||||
# This file is part of the Douyin_TikTok_Download_API project.
|
||||
#
|
||||
# This project is licensed under the Apache License 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at:
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
# ＿＿
|
||||
# ／＞ フ
|
||||
# | _ _ l
|
||||
# ／` ミ＿xノ
|
||||
# / | Feed me Stars ⭐ ️
|
||||
# / ヽ ﾉ
|
||||
# │ | | |
|
||||
# ／￣| | | |
|
||||
# | (￣ヽ＿_ヽ_)__)
|
||||
# ＼二つ
|
||||
# ==============================================================================
|
||||
#
|
||||
# Contributor Link:
|
||||
#
|
||||
# - https://github.com/Koyomi781
|
||||
#
|
||||
# ==============================================================================
|
||||
|
||||
import asyncio # 异步I/O
|
||||
import os # 系统操作
|
||||
import time # 时间操作
|
||||
import yaml # 配置文件
|
||||
|
||||
# 基础爬虫客户端和哔哩哔哩API端点
|
||||
from crawlers.base_crawler import BaseCrawler
|
||||
from crawlers.bilibili.web.endpoints import BilibiliAPIEndpoints
|
||||
# 哔哩哔哩工具类
|
||||
from crawlers.bilibili.web.utils import EndpointGenerator, bv2av, ResponseAnalyzer
|
||||
# 数据请求模型
|
||||
from crawlers.bilibili.web.models import UserPostVideos, UserProfile, ComPopular, UserDynamic, PlayUrl
|
||||
|
||||
# 配置文件路径
|
||||
path = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
# 读取配置文件
|
||||
with open(f"{path}/config.yaml", "r", encoding="utf-8") as f:
|
||||
config = yaml.safe_load(f)
|
||||
|
||||
|
||||
class BilibiliWebCrawler:
|
||||
|
||||
# 从配置文件读取哔哩哔哩请求头
|
||||
async def get_bilibili_headers(self):
|
||||
bili_config = config['TokenManager']['bilibili']
|
||||
kwargs = {
|
||||
"headers": {
|
||||
"accept-language": bili_config["headers"]["accept-language"],
|
||||
"origin": bili_config["headers"]["origin"],
|
||||
"referer": bili_config["headers"]["referer"],
|
||||
"user-agent": bili_config["headers"]["user-agent"],
|
||||
"cookie": bili_config["headers"]["cookie"],
|
||||
},
|
||||
"proxies": {"http://": bili_config["proxies"]["http"], "https://": bili_config["proxies"]["https"]},
|
||||
}
|
||||
return kwargs
|
||||
|
||||
"-------------------------------------------------------handler接口列表-------------------------------------------------------"
|
||||
|
||||
# 获取单个视频详情信息
|
||||
async def fetch_one_video(self, bv_id: str) -> dict:
|
||||
# 获取请求头信息
|
||||
kwargs = await self.get_bilibili_headers()
|
||||
# 创建基础爬虫对象
|
||||
base_crawler = BaseCrawler(proxies=kwargs["proxies"], crawler_headers=kwargs["headers"])
|
||||
async with base_crawler as crawler:
|
||||
# 创建请求endpoint
|
||||
endpoint = f"{BilibiliAPIEndpoints.POST_DETAIL}?bvid={bv_id}"
|
||||
# 发送请求，获取请求响应结果
|
||||
response = await crawler.fetch_get_json(endpoint)
|
||||
return response
|
||||
|
||||
# 获取视频流地址
|
||||
async def fetch_video_playurl(self, bv_id: str, cid: str, qn: str = "64") -> dict:
|
||||
# 获取请求头信息
|
||||
kwargs = await self.get_bilibili_headers()
|
||||
# 创建基础爬虫对象
|
||||
base_crawler = BaseCrawler(proxies=kwargs["proxies"], crawler_headers=kwargs["headers"])
|
||||
async with base_crawler as crawler:
|
||||
# 通过模型生成基本请求参数
|
||||
params = PlayUrl(bvid=bv_id, cid=cid, qn=qn)
|
||||
# 创建请求endpoint
|
||||
generator = EndpointGenerator(params.dict())
|
||||
endpoint = await generator.video_playurl_endpoint()
|
||||
# 发送请求，获取请求响应结果
|
||||
response = await crawler.fetch_get_json(endpoint)
|
||||
return response
|
||||
|
||||
# 获取用户发布视频作品数据
|
||||
async def fetch_user_post_videos(self, uid: str, pn: int) -> dict:
|
||||
"""
|
||||
:param uid: 用户uid
|
||||
:param pn: 页码
|
||||
:return:
|
||||
"""
|
||||
# 获取请求头信息
|
||||
kwargs = await self.get_bilibili_headers()
|
||||
# 创建基础爬虫对象
|
||||
base_crawler = BaseCrawler(proxies=kwargs["proxies"], crawler_headers=kwargs["headers"])
|
||||
async with base_crawler as crawler:
|
||||
# 通过模型生成基本请求参数
|
||||
params = UserPostVideos(mid=uid, pn=pn)
|
||||
# 创建请求endpoint
|
||||
generator = EndpointGenerator(params.dict())
|
||||
endpoint = await generator.user_post_videos_endpoint()
|
||||
# 发送请求，获取请求响应结果
|
||||
response = await crawler.fetch_get_json(endpoint)
|
||||
return response
|
||||
|
||||
# 获取用户所有收藏夹信息
|
||||
async def fetch_collect_folders(self, uid: str) -> dict:
|
||||
# 获取请求头信息
|
||||
kwargs = await self.get_bilibili_headers()
|
||||
# 创建基础爬虫对象
|
||||
base_crawler = BaseCrawler(proxies=kwargs["proxies"], crawler_headers=kwargs["headers"])
|
||||
async with base_crawler as crawler:
|
||||
# 创建请求endpoint
|
||||
endpoint = f"{BilibiliAPIEndpoints.COLLECT_FOLDERS}?up_mid={uid}"
|
||||
# 发送请求，获取请求响应结果
|
||||
response = await crawler.fetch_get_json(endpoint)
|
||||
# 分析响应结果
|
||||
result_dict = await ResponseAnalyzer.collect_folders_analyze(response=response)
|
||||
return result_dict
|
||||
|
||||
# 获取指定收藏夹内视频数据
|
||||
async def fetch_folder_videos(self, folder_id: str, pn: int) -> dict:
|
||||
"""
|
||||
:param folder_id: 收藏夹id-- 可从<获取用户所有收藏夹信息>获得
|
||||
:param pn: 页码
|
||||
:return:
|
||||
"""
|
||||
# 获取请求头信息
|
||||
kwargs = await self.get_bilibili_headers()
|
||||
# 创建基础爬虫对象
|
||||
base_crawler = BaseCrawler(proxies=kwargs["proxies"], crawler_headers=kwargs["headers"])
|
||||
# 发送请求，获取请求响应结果
|
||||
async with base_crawler as crawler:
|
||||
endpoint = f"{BilibiliAPIEndpoints.COLLECT_VIDEOS}?media_id={folder_id}&pn={pn}&ps=20&keyword=&order=mtime&type=0&tid=0&platform=web"
|
||||
response = await crawler.fetch_get_json(endpoint)
|
||||
return response
|
||||
|
||||
# 获取指定用户的信息
|
||||
async def fetch_user_profile(self, uid: str) -> dict:
|
||||
# 获取请求头信息
|
||||
kwargs = await self.get_bilibili_headers()
|
||||
# 创建基础爬虫对象
|
||||
base_crawler = BaseCrawler(proxies=kwargs["proxies"], crawler_headers=kwargs["headers"])
|
||||
async with base_crawler as crawler:
|
||||
# 通过模型生成基本请求参数
|
||||
params = UserProfile(mid=uid)
|
||||
# 创建请求endpoint
|
||||
generator = EndpointGenerator(params.dict())
|
||||
endpoint = await generator.user_profile_endpoint()
|
||||
# 发送请求，获取请求响应结果
|
||||
response = await crawler.fetch_get_json(endpoint)
|
||||
return response
|
||||
|
||||
# 获取综合热门视频信息
|
||||
async def fetch_com_popular(self, pn: int) -> dict:
|
||||
# 获取请求头信息
|
||||
kwargs = await self.get_bilibili_headers()
|
||||
# 创建基础爬虫对象
|
||||
base_crawler = BaseCrawler(proxies=kwargs["proxies"], crawler_headers=kwargs["headers"])
|
||||
async with base_crawler as crawler:
|
||||
# 通过模型生成基本请求参数
|
||||
params = ComPopular(pn=pn)
|
||||
# 创建请求endpoint
|
||||
generator = EndpointGenerator(params.dict())
|
||||
endpoint = await generator.com_popular_endpoint()
|
||||
# 发送请求，获取请求响应结果
|
||||
response = await crawler.fetch_get_json(endpoint)
|
||||
return response
|
||||
|
||||
# 获取指定视频的评论
|
||||
async def fetch_video_comments(self, bv_id: str, pn: int) -> dict:
|
||||
# 评论排序 -- 1:按点赞数排序. 0:按时间顺序排序
|
||||
sort = 1
|
||||
# 获取请求头信息
|
||||
kwargs = await self.get_bilibili_headers()
|
||||
# 创建基础爬虫对象
|
||||
base_crawler = BaseCrawler(proxies=kwargs["proxies"], crawler_headers=kwargs["headers"])
|
||||
async with base_crawler as crawler:
|
||||
# 创建请求endpoint
|
||||
endpoint = f"{BilibiliAPIEndpoints.VIDEO_COMMENTS}?type=1&oid={bv_id}&sort={sort}&nohot=0&ps=20&pn={pn}"
|
||||
# 发送请求，获取请求响应结果
|
||||
response = await crawler.fetch_get_json(endpoint)
|
||||
return response
|
||||
|
||||
# 获取视频下指定评论的回复
|
||||
async def fetch_comment_reply(self, bv_id: str, pn: int, rpid: str) -> dict:
|
||||
"""
|
||||
:param bv_id: 目标视频bv号
|
||||
:param pn: 页码
|
||||
:param rpid: 目标评论id，可通过fetch_video_comments获得
|
||||
:return:
|
||||
"""
|
||||
# 获取请求头信息
|
||||
kwargs = await self.get_bilibili_headers()
|
||||
# 创建基础爬虫对象
|
||||
base_crawler = BaseCrawler(proxies=kwargs["proxies"], crawler_headers=kwargs["headers"])
|
||||
async with base_crawler as crawler:
|
||||
# 创建请求endpoint
|
||||
endpoint = f"{BilibiliAPIEndpoints.COMMENT_REPLY}?type=1&oid={bv_id}&root={rpid}&&ps=20&pn={pn}"
|
||||
# 发送请求，获取请求响应结果
|
||||
response = await crawler.fetch_get_json(endpoint)
|
||||
return response
|
||||
|
||||
# 获取指定用户动态
|
||||
async def fetch_user_dynamic(self, uid: str, offset: str) -> dict:
|
||||
# 获取请求头信息
|
||||
kwargs = await self.get_bilibili_headers()
|
||||
# 创建基础爬虫对象
|
||||
base_crawler = BaseCrawler(proxies=kwargs["proxies"], crawler_headers=kwargs["headers"])
|
||||
async with base_crawler as crawler:
|
||||
# 通过模型生成基本请求参数
|
||||
params = UserDynamic(host_mid=uid, offset=offset)
|
||||
# 创建请求endpoint
|
||||
generator = EndpointGenerator(params.dict())
|
||||
endpoint = await generator.user_dynamic_endpoint()
|
||||
print(endpoint)
|
||||
# 发送请求，获取请求响应结果
|
||||
response = await crawler.fetch_get_json(endpoint)
|
||||
return response
|
||||
|
||||
# 获取视频实时弹幕
|
||||
async def fetch_video_danmaku(self, cid: str):
|
||||
# 获取请求头信息
|
||||
kwargs = await self.get_bilibili_headers()
|
||||
# 创建基础爬虫对象
|
||||
base_crawler = BaseCrawler(proxies=kwargs["proxies"], crawler_headers=kwargs["headers"])
|
||||
async with base_crawler as crawler:
|
||||
# 创建请求endpoint
|
||||
endpoint = f"https://comment.bilibili.com/{cid}.xml"
|
||||
# 发送请求，获取请求响应结果
|
||||
response = await crawler.fetch_response(endpoint)
|
||||
return response.text
|
||||
|
||||
# 获取指定直播间信息
|
||||
async def fetch_live_room_detail(self, room_id: str) -> dict:
|
||||
# 获取请求头信息
|
||||
kwargs = await self.get_bilibili_headers()
|
||||
# 创建基础爬虫对象
|
||||
base_crawler = BaseCrawler(proxies=kwargs["proxies"], crawler_headers=kwargs["headers"])
|
||||
async with base_crawler as crawler:
|
||||
# 创建请求endpoint
|
||||
endpoint = f"{BilibiliAPIEndpoints.LIVEROOM_DETAIL}?room_id={room_id}"
|
||||
# 发送请求，获取请求响应结果
|
||||
response = await crawler.fetch_get_json(endpoint)
|
||||
return response
|
||||
|
||||
# 获取指定直播间视频流
|
||||
async def fetch_live_videos(self, room_id: str) -> dict:
|
||||
# 获取请求头信息
|
||||
kwargs = await self.get_bilibili_headers()
|
||||
# 创建基础爬虫对象
|
||||
base_crawler = BaseCrawler(proxies=kwargs["proxies"], crawler_headers=kwargs["headers"])
|
||||
async with base_crawler as crawler:
|
||||
# 创建请求endpoint
|
||||
endpoint = f"{BilibiliAPIEndpoints.LIVE_VIDEOS}?cid={room_id}&quality=4"
|
||||
# 发送请求，获取请求响应结果
|
||||
response = await crawler.fetch_get_json(endpoint)
|
||||
return response
|
||||
|
||||
# 获取指定分区正在直播的主播
|
||||
async def fetch_live_streamers(self, area_id: str, pn: int):
|
||||
# 获取请求头信息
|
||||
kwargs = await self.get_bilibili_headers()
|
||||
# 创建基础爬虫对象
|
||||
base_crawler = BaseCrawler(proxies=kwargs["proxies"], crawler_headers=kwargs["headers"])
|
||||
async with base_crawler as crawler:
|
||||
# 创建请求endpoint
|
||||
endpoint = f"{BilibiliAPIEndpoints.LIVE_STREAMER}?platform=web&parent_area_id={area_id}&page={pn}"
|
||||
# 发送请求，获取请求响应结果
|
||||
response = await crawler.fetch_get_json(endpoint)
|
||||
return response
|
||||
|
||||
"-------------------------------------------------------utils接口列表-------------------------------------------------------"
|
||||
|
||||
# 通过bv号获得视频aid号
|
||||
async def bv_to_aid(self, bv_id: str) -> int:
|
||||
aid = await bv2av(bv_id=bv_id)
|
||||
return aid
|
||||
|
||||
# 通过bv号获得视频分p信息
|
||||
async def fetch_video_parts(self, bv_id: str) -> str:
|
||||
# 获取请求头信息
|
||||
kwargs = await self.get_bilibili_headers()
|
||||
# 创建基础爬虫对象
|
||||
base_crawler = BaseCrawler(proxies=kwargs["proxies"], crawler_headers=kwargs["headers"])
|
||||
async with base_crawler as crawler:
|
||||
# 创建请求endpoint
|
||||
endpoint = f"{BilibiliAPIEndpoints.VIDEO_PARTS}?bvid={bv_id}"
|
||||
# 发送请求，获取请求响应结果
|
||||
response = await crawler.fetch_get_json(endpoint)
|
||||
return response
|
||||
|
||||
# 获取所有直播分区列表
|
||||
async def fetch_all_live_areas(self) -> dict:
|
||||
# 获取请求头信息
|
||||
kwargs = await self.get_bilibili_headers()
|
||||
# 创建基础爬虫对象
|
||||
base_crawler = BaseCrawler(proxies=kwargs["proxies"], crawler_headers=kwargs["headers"])
|
||||
async with base_crawler as crawler:
|
||||
# 创建请求endpoint
|
||||
endpoint = BilibiliAPIEndpoints.LIVE_AREAS
|
||||
# 发送请求，获取请求响应结果
|
||||
response = await crawler.fetch_get_json(endpoint)
|
||||
return response
|
||||
|
||||
"-------------------------------------------------------main-------------------------------------------------------"
|
||||
|
||||
async def main(self):
|
||||
"""-------------------------------------------------------handler接口列表-------------------------------------------------------"""
|
||||
|
||||
# 获取单个作品数据
|
||||
# bv_id = 'BV1M1421t7hT'
|
||||
# result = await self.fetch_one_video(bv_id=bv_id)
|
||||
# print(result)
|
||||
|
||||
# 获取视频流地址
|
||||
# bv_id = 'BV1y7411Q7Eq'
|
||||
# cid = '171776208'
|
||||
# result = await self.fetch_video_playurl(bv_id=bv_id, cid=cid)
|
||||
# print(result)
|
||||
|
||||
# 获取用户发布作品数据
|
||||
# uid = '94510621'
|
||||
# pn = 1
|
||||
# result = await self.fetch_user_post_videos(uid=uid, pn=pn)
|
||||
# print(result)
|
||||
|
||||
# 获取用户所有收藏夹信息
|
||||
# uid = '178360345'
|
||||
# reslut = await self.fetch_collect_folders(uid=uid)
|
||||
# print(reslut)
|
||||
|
||||
# 获取用户指定收藏夹内视频数据
|
||||
# folder_id = '1756059545' # 收藏夹id，可从<获取用户所有收藏夹信息>获得
|
||||
# pn = 1
|
||||
# result = await self.fetch_folder_videos(folder_id=folder_id, pn=pn)
|
||||
# print(result)
|
||||
|
||||
# 获取指定用户的信息
|
||||
# uid = '178360345'
|
||||
# result = await self.fetch_user_profile(uid=uid)
|
||||
# print(result)
|
||||
|
||||
# 获取综合热门信息
|
||||
# pn = 1 # 页码
|
||||
# result = await self.fetch_com_popular(pn=pn)
|
||||
# print(result)
|
||||
|
||||
# 获取指定视频的评论(不登录只能获取一页的评论)
|
||||
# bv_id = "BV1M1421t7hT"
|
||||
# pn = 1
|
||||
# result = await self.fetch_video_comments(bv_id=bv_id, pn=pn)
|
||||
# print(result)
|
||||
|
||||
# 获取视频下指定评论的回复(不登录只能获取一页的评论)
|
||||
# bv_id = "BV1M1421t7hT"
|
||||
# rpid = "237109455120"
|
||||
# pn = 1
|
||||
# result = await self.fetch_comment_reply(bv_id=bv_id, pn=pn, rpid=rpid)
|
||||
# print(result)
|
||||
|
||||
# 获取指定用户动态
|
||||
# uid = "16015678"
|
||||
# offset = "" # 翻页索引，为空即从最新动态开始
|
||||
# result = await self.fetch_user_dynamic(uid=uid, offset=offset)
|
||||
# print(result)
|
||||
|
||||
# 获取视频实时弹幕
|
||||
# cid = "1639235405"
|
||||
# result = await self.fetch_video_danmaku(cid=cid)
|
||||
# print(result)
|
||||
|
||||
# 获取指定直播间信息
|
||||
# room_id = "1815229528"
|
||||
# result = await self.fetch_live_room_detail(room_id=room_id)
|
||||
# print(result)
|
||||
|
||||
# 获取直播间视频流
|
||||
# room_id = "1815229528"
|
||||
# result = await self.fetch_live_videos(room_id=room_id)
|
||||
# print(result)
|
||||
|
||||
# 获取指定分区正在直播的主播
|
||||
pn = 1
|
||||
area_id = '9'
|
||||
result = await self.fetch_live_streamers(area_id=area_id, pn=pn)
|
||||
print(result)
|
||||
|
||||
"-------------------------------------------------------utils接口列表-------------------------------------------------------"
|
||||
# 通过bv号获得视频aid号
|
||||
# bv_id = 'BV1M1421t7hT'
|
||||
# aid = await self.get_aid(bv_id=bv_id)
|
||||
# print(aid)
|
||||
|
||||
# 通过bv号获得视频分p信息
|
||||
# bv_id = "BV1vf421i7hV"
|
||||
# result = await self.fetch_video_parts(bv_id=bv_id)
|
||||
# print(result)
|
||||
|
||||
# 获取所有直播分区列表
|
||||
# result = await self.fetch_all_live_areas()
|
||||
# print(result)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# 初始化
|
||||
BilibiliWebCrawler = BilibiliWebCrawler()
|
||||
|
||||
# 开始时间
|
||||
start = time.time()
|
||||
|
||||
asyncio.run(BilibiliWebCrawler.main())
|
||||
|
||||
# 结束时间
|
||||
end = time.time()
|
||||
print(f"耗时：{end - start}")
|
||||
186
crawlers/bilibili/web/wrid.py
Normal file
186
crawlers/bilibili/web/wrid.py
Normal file
@ -0,0 +1,186 @@
|
||||
import urllib.parse
|
||||
|
||||
def srotl(t, e):
|
||||
return (t << e) | (t >> (32 - e))
|
||||
|
||||
def tendian(t):
|
||||
if isinstance(t, int):
|
||||
return (16711935 & srotl(t, 8)) | (4278255360 & srotl(t, 24))
|
||||
for e in range(len(t)):
|
||||
t[e] = tendian(t[e])
|
||||
return t
|
||||
|
||||
# 没问题
|
||||
def tbytes_to_words(t):
|
||||
e = []
|
||||
r = 0
|
||||
for n in range(len(t)):
|
||||
if r >> 5 >= len(e):
|
||||
e.append(0)
|
||||
e[r >> 5] |= t[n] << (24 - r % 32)
|
||||
r += 8
|
||||
return e
|
||||
|
||||
def jbinstring_to_bytes(t):
|
||||
e = []
|
||||
for n in range(len(t)):
|
||||
e.append(ord(t[n]) & 255)
|
||||
return e
|
||||
|
||||
# 没问题
|
||||
def estring_to_bytes(t):
|
||||
return jbinstring_to_bytes(urllib.parse.unquote(urllib.parse.quote(t)))
|
||||
|
||||
def _ff(t, e, n, r, o, i, a):
|
||||
# 计算中间值 c
|
||||
c = t + ((e & n) | (~e & r)) + (o & 0xFFFFFFFF) + a
|
||||
# 将 c 转换为 32 位无符号整数
|
||||
c = c & 0xFFFFFFFF
|
||||
# 左移和右移操作
|
||||
c = (c << i | c >> (32 - i)) & 0xFFFFFFFF
|
||||
# 返回结果
|
||||
return (c + e) & 0xFFFFFFFF
|
||||
|
||||
def _gg(t, e, n, r, o, i, a):
|
||||
# 计算中间值 c
|
||||
c = t + ((e & r) | (n & ~r)) + (o & 0xFFFFFFFF) + a
|
||||
# 将 c 转换为 32 位无符号整数
|
||||
c = c & 0xFFFFFFFF
|
||||
# 左移和右移操作
|
||||
c = (c << i | c >> (32 - i)) & 0xFFFFFFFF
|
||||
# 返回结果
|
||||
return (c + e) & 0xFFFFFFFF
|
||||
|
||||
def _hh(t, e, n, r, o, i, a):
|
||||
# 计算中间值 c
|
||||
c = t + (e ^ n ^ r) + (o & 0xFFFFFFFF) + a
|
||||
# 将 c 转换为 32 位无符号整数
|
||||
c = c & 0xFFFFFFFF
|
||||
# 左移和右移操作
|
||||
c = (c << i | c >> (32 - i)) & 0xFFFFFFFF
|
||||
# 返回结果
|
||||
return (c + e) & 0xFFFFFFFF
|
||||
|
||||
def _ii(t, e, n, r, o, i, a):
|
||||
# 计算中间值 c
|
||||
c = t + (n ^ (e | ~r)) + (o & 0xFFFFFFFF) + a
|
||||
# 将 c 转换为 32 位无符号整数
|
||||
c = c & 0xFFFFFFFF
|
||||
# 左移和右移操作
|
||||
c = (c << i | c >> (32 - i)) & 0xFFFFFFFF
|
||||
# 返回结果
|
||||
return (c + e) & 0xFFFFFFFF
|
||||
|
||||
def o(i, a):
|
||||
if isinstance(i, str):
|
||||
i = estring_to_bytes(i)
|
||||
elif isinstance(i, (list, tuple)):
|
||||
i = list(i)
|
||||
elif not isinstance(i, (list, bytearray)):
|
||||
i = str(i)
|
||||
c = tbytes_to_words(i)
|
||||
u = 8 * len(i)
|
||||
s, l, f, p = 1732584193, -271733879, -1732584194, 271733878
|
||||
|
||||
for d in range(len(c)):
|
||||
c[d] = (16711935 & (c[d] << 8 | c[d] >> 24)) | (4278255360 & (c[d] << 24 | c[d] >> 8))
|
||||
|
||||
# 确保列表 c 的长度足够大
|
||||
while len(c) <= (14 + ((u + 64 >> 9) << 4)):
|
||||
c.append(0)
|
||||
|
||||
c[u >> 5] |= 128 << (u % 32)
|
||||
c[14 + ((u + 64 >> 9) << 4)] = u
|
||||
|
||||
h, v, y, m = _ff, _gg, _hh, _ii
|
||||
for d in range(0, len(c), 16):
|
||||
g, b, w, A = s, l, f, p
|
||||
# 确保在访问索引之前扩展列表的长度
|
||||
while len(c) <= d + 15:
|
||||
c.append(0)
|
||||
s = h(s, l, f, p, c[d + 0], 7, -680876936)
|
||||
p = h(p, s, l, f, c[d + 1], 12, -389564586)
|
||||
f = h(f, p, s, l, c[d + 2], 17, 606105819)
|
||||
l = h(l, f, p, s, c[d + 3], 22, -1044525330)
|
||||
s = h(s, l, f, p, c[d + 4], 7, -176418897)
|
||||
p = h(p, s, l, f, c[d + 5], 12, 1200080426)
|
||||
f = h(f, p, s, l, c[d + 6], 17, -1473231341)
|
||||
l = h(l, f, p, s, c[d + 7], 22, -45705983)
|
||||
s = h(s, l, f, p, c[d + 8], 7, 1770035416)
|
||||
p = h(p, s, l, f, c[d + 9], 12, -1958414417)
|
||||
f = h(f, p, s, l, c[d + 10], 17, -42063)
|
||||
l = h(l, f, p, s, c[d + 11], 22, -1990404162)
|
||||
s = h(s, l, f, p, c[d + 12], 7, 1804603682)
|
||||
p = h(p, s, l, f, c[d + 13], 12, -40341101)
|
||||
f = h(f, p, s, l, c[d + 14], 17, -1502002290)
|
||||
s = v(s, l := h(l, f, p, s, c[d + 15], 22, 1236535329), f, p, c[d + 1], 5, -165796510)
|
||||
p = v(p, s, l, f, c[d + 6], 9, -1069501632)
|
||||
f = v(f, p, s, l, c[d + 11], 14, 643717713)
|
||||
l = v(l, f, p, s, c[d + 0], 20, -373897302)
|
||||
s = v(s, l, f, p, c[d + 5], 5, -701558691)
|
||||
p = v(p, s, l, f, c[d + 10], 9, 38016083)
|
||||
f = v(f, p, s, l, c[d + 15], 14, -660478335)
|
||||
l = v(l, f, p, s, c[d + 4], 20, -405537848)
|
||||
s = v(s, l, f, p, c[d + 9], 5, 568446438)
|
||||
p = v(p, s, l, f, c[d + 14], 9, -1019803690)
|
||||
f = v(f, p, s, l, c[d + 3], 14, -187363961)
|
||||
l = v(l, f, p, s, c[d + 8], 20, 1163531501)
|
||||
s = v(s, l, f, p, c[d + 13], 5, -1444681467)
|
||||
p = v(p, s, l, f, c[d + 2], 9, -51403784)
|
||||
f = v(f, p, s, l, c[d + 7], 14, 1735328473)
|
||||
s = y(s, l := v(l, f, p, s, c[d + 12], 20, -1926607734), f, p, c[d + 5], 4, -378558)
|
||||
p = y(p, s, l, f, c[d + 8], 11, -2022574463)
|
||||
f = y(f, p, s, l, c[d + 11], 16, 1839030562)
|
||||
l = y(l, f, p, s, c[d + 14], 23, -35309556)
|
||||
s = y(s, l, f, p, c[d + 1], 4, -1530992060)
|
||||
p = y(p, s, l, f, c[d + 4], 11, 1272893353)
|
||||
f = y(f, p, s, l, c[d + 7], 16, -155497632)
|
||||
l = y(l, f, p, s, c[d + 10], 23, -1094730640)
|
||||
s = y(s, l, f, p, c[d + 13], 4, 681279174)
|
||||
p = y(p, s, l, f, c[d + 0], 11, -358537222)
|
||||
f = y(f, p, s, l, c[d + 3], 16, -722521979)
|
||||
l = y(l, f, p, s, c[d + 6], 23, 76029189)
|
||||
s = y(s, l, f, p, c[d + 9], 4, -640364487)
|
||||
p = y(p, s, l, f, c[d + 12], 11, -421815835)
|
||||
f = y(f, p, s, l, c[d + 15], 16, 530742520)
|
||||
s = m(s, l := y(l, f, p, s, c[d + 2], 23, -995338651), f, p, c[d + 0], 6, -198630844)
|
||||
p = m(p, s, l, f, c[d + 7], 10, 1126891415)
|
||||
f = m(f, p, s, l, c[d + 14], 15, -1416354905)
|
||||
l = m(l, f, p, s, c[d + 5], 21, -57434055)
|
||||
s = m(s, l, f, p, c[d + 12], 6, 1700485571)
|
||||
p = m(p, s, l, f, c[d + 3], 10, -1894986606)
|
||||
f = m(f, p, s, l, c[d + 10], 15, -1051523)
|
||||
l = m(l, f, p, s, c[d + 1], 21, -2054922799)
|
||||
s = m(s, l, f, p, c[d + 8], 6, 1873313359)
|
||||
p = m(p, s, l, f, c[d + 15], 10, -30611744)
|
||||
f = m(f, p, s, l, c[d + 6], 15, -1560198380)
|
||||
l = m(l, f, p, s, c[d + 13], 21, 1309151649)
|
||||
s = m(s, l, f, p, c[d + 4], 6, -145523070)
|
||||
p = m(p, s, l, f, c[d + 11], 10, -1120210379)
|
||||
f = m(f, p, s, l, c[d + 2], 15, 718787259)
|
||||
l = m(l, f, p, s, c[d + 9], 21, -343485551)
|
||||
|
||||
s = (s + g) >> 0 & 0xFFFFFFFF
|
||||
l = (l + b) >> 0 & 0xFFFFFFFF
|
||||
f = (f + w) >> 0 & 0xFFFFFFFF
|
||||
p = (p + A) >> 0 & 0xFFFFFFFF
|
||||
|
||||
return tendian([s, l, f, p])
|
||||
|
||||
def twords_to_bytes(t):
|
||||
e = []
|
||||
for n in range(0, 32 * len(t), 8):
|
||||
e.append((t[n >> 5] >> (24 - n % 32)) & 255)
|
||||
return e
|
||||
|
||||
def tbytes_to_hex(t):
|
||||
e = []
|
||||
for n in range(len(t)):
|
||||
e.append(hex(t[n] >> 4)[2:])
|
||||
e.append(hex(t[n] & 15)[2:])
|
||||
return ''.join(e)
|
||||
|
||||
def get_wrid(e):
|
||||
n = None
|
||||
i = twords_to_bytes(o(e, n))
|
||||
return tbytes_to_hex(i)
|
||||
635
crawlers/douyin/web/abogus.py
Normal file
635
crawlers/douyin/web/abogus.py
Normal file
@ -0,0 +1,635 @@
|
||||
"""
|
||||
Original Author:
|
||||
This file is from https://github.com/JoeanAmier/TikTokDownloader
|
||||
And is licensed under the GNU General Public License v3.0
|
||||
If you use this code, please keep this license and the original author information.
|
||||
|
||||
Modified by:
|
||||
And this file is now a part of the https://github.com/Evil0ctal/Douyin_TikTok_Download_API open-source project.
|
||||
This project is licensed under the Apache License 2.0, and the original author information is kept.
|
||||
|
||||
Purpose:
|
||||
This file is used to generate the `a_bogus` parameter for the Douyin Web API.
|
||||
|
||||
Changes Made:
|
||||
1. Changed the ua_code to compatible with the current config file User-Agent string in https://github.com/Evil0ctal/Douyin_TikTok_Download_API/blob/main/crawlers/douyin/web/config.yaml
|
||||
"""
|
||||
|
||||
from random import choice
|
||||
from random import randint
|
||||
from random import random
|
||||
from re import compile
|
||||
from time import time
|
||||
from urllib.parse import urlencode
|
||||
from urllib.parse import quote
|
||||
from gmssl import sm3, func
|
||||
|
||||
__all__ = ["ABogus", ]
|
||||
|
||||
|
||||
class ABogus:
|
||||
__filter = compile(r'%([0-9A-F]{2})')
|
||||
__arguments = [0, 1, 14]
|
||||
__ua_key = "\u0000\u0001\u000e"
|
||||
__end_string = "cus"
|
||||
__version = [1, 0, 1, 5]
|
||||
__browser = "1536|742|1536|864|0|0|0|0|1536|864|1536|864|1536|742|24|24|MacIntel"
|
||||
__reg = [
|
||||
1937774191,
|
||||
1226093241,
|
||||
388252375,
|
||||
3666478592,
|
||||
2842636476,
|
||||
372324522,
|
||||
3817729613,
|
||||
2969243214,
|
||||
]
|
||||
__str = {
|
||||
"s0": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=",
|
||||
"s1": "Dkdpgh4ZKsQB80/Mfvw36XI1R25+WUAlEi7NLboqYTOPuzmFjJnryx9HVGcaStCe=",
|
||||
"s2": "Dkdpgh4ZKsQB80/Mfvw36XI1R25-WUAlEi7NLboqYTOPuzmFjJnryx9HVGcaStCe=",
|
||||
"s3": "ckdp1h4ZKsUB80/Mfvw36XIgR25+WQAlEi7NLboqYTOPuzmFjJnryx9HVGDaStCe",
|
||||
"s4": "Dkdpgh2ZmsQB80/MfvV36XI1R45-WUAlEixNLwoqYTOPuzKFjJnry79HbGcaStCe",
|
||||
}
|
||||
|
||||
def __init__(self,
|
||||
# user_agent: str = USERAGENT,
|
||||
platform: str = None, ):
|
||||
self.chunk = []
|
||||
self.size = 0
|
||||
self.reg = self.__reg[:]
|
||||
# self.ua_code = self.generate_ua_code(user_agent)
|
||||
# Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36
|
||||
self.ua_code = [
|
||||
76,
|
||||
98,
|
||||
15,
|
||||
131,
|
||||
97,
|
||||
245,
|
||||
224,
|
||||
133,
|
||||
122,
|
||||
199,
|
||||
241,
|
||||
166,
|
||||
79,
|
||||
34,
|
||||
90,
|
||||
191,
|
||||
128,
|
||||
126,
|
||||
122,
|
||||
98,
|
||||
66,
|
||||
11,
|
||||
14,
|
||||
40,
|
||||
49,
|
||||
110,
|
||||
110,
|
||||
173,
|
||||
67,
|
||||
96,
|
||||
138,
|
||||
252]
|
||||
self.browser = self.generate_browser_info(
|
||||
platform) if platform else self.__browser
|
||||
self.browser_len = len(self.browser)
|
||||
self.browser_code = self.char_code_at(self.browser)
|
||||
|
||||
@classmethod
|
||||
def list_1(cls, random_num=None, a=170, b=85, c=45, ) -> list:
|
||||
return cls.random_list(
|
||||
random_num,
|
||||
a,
|
||||
b,
|
||||
1,
|
||||
2,
|
||||
5,
|
||||
c & a,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def list_2(cls, random_num=None, a=170, b=85, ) -> list:
|
||||
return cls.random_list(
|
||||
random_num,
|
||||
a,
|
||||
b,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def list_3(cls, random_num=None, a=170, b=85, ) -> list:
|
||||
return cls.random_list(
|
||||
random_num,
|
||||
a,
|
||||
b,
|
||||
1,
|
||||
0,
|
||||
5,
|
||||
0,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def random_list(
|
||||
a: float = None,
|
||||
b=170,
|
||||
c=85,
|
||||
d=0,
|
||||
e=0,
|
||||
f=0,
|
||||
g=0,
|
||||
) -> list:
|
||||
r = a or (random() * 10000)
|
||||
v = [
|
||||
r,
|
||||
int(r) & 255,
|
||||
int(r) >> 8,
|
||||
]
|
||||
s = v[1] & b | d
|
||||
v.append(s)
|
||||
s = v[1] & c | e
|
||||
v.append(s)
|
||||
s = v[2] & b | f
|
||||
v.append(s)
|
||||
s = v[2] & c | g
|
||||
v.append(s)
|
||||
return v[-4:]
|
||||
|
||||
@staticmethod
|
||||
def from_char_code(*args):
|
||||
return "".join(chr(code) for code in args)
|
||||
|
||||
@classmethod
|
||||
def generate_string_1(
|
||||
cls,
|
||||
random_num_1=None,
|
||||
random_num_2=None,
|
||||
random_num_3=None,
|
||||
):
|
||||
return cls.from_char_code(*cls.list_1(random_num_1)) + cls.from_char_code(
|
||||
*cls.list_2(random_num_2)) + cls.from_char_code(*cls.list_3(random_num_3))
|
||||
|
||||
def generate_string_2(
|
||||
self,
|
||||
url_params: str,
|
||||
method="GET",
|
||||
start_time=0,
|
||||
end_time=0,
|
||||
) -> str:
|
||||
a = self.generate_string_2_list(
|
||||
url_params,
|
||||
method,
|
||||
start_time,
|
||||
end_time,
|
||||
)
|
||||
e = self.end_check_num(a)
|
||||
a.extend(self.browser_code)
|
||||
a.append(e)
|
||||
return self.rc4_encrypt(self.from_char_code(*a), "y")
|
||||
|
||||
def generate_string_2_list(
|
||||
self,
|
||||
url_params: str,
|
||||
method="GET",
|
||||
start_time=0,
|
||||
end_time=0,
|
||||
) -> list:
|
||||
start_time = start_time or int(time() * 1000)
|
||||
end_time = end_time or (start_time + randint(4, 8))
|
||||
params_array = self.generate_params_code(url_params)
|
||||
method_array = self.generate_method_code(method)
|
||||
return self.list_4(
|
||||
(end_time >> 24) & 255,
|
||||
params_array[21],
|
||||
self.ua_code[23],
|
||||
(end_time >> 16) & 255,
|
||||
params_array[22],
|
||||
self.ua_code[24],
|
||||
(end_time >> 8) & 255,
|
||||
(end_time >> 0) & 255,
|
||||
(start_time >> 24) & 255,
|
||||
(start_time >> 16) & 255,
|
||||
(start_time >> 8) & 255,
|
||||
(start_time >> 0) & 255,
|
||||
method_array[21],
|
||||
method_array[22],
|
||||
int(end_time / 256 / 256 / 256 / 256) >> 0,
|
||||
int(start_time / 256 / 256 / 256 / 256) >> 0,
|
||||
self.browser_len,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def reg_to_array(a):
|
||||
o = [0] * 32
|
||||
for i in range(8):
|
||||
c = a[i]
|
||||
o[4 * i + 3] = (255 & c)
|
||||
c >>= 8
|
||||
o[4 * i + 2] = (255 & c)
|
||||
c >>= 8
|
||||
o[4 * i + 1] = (255 & c)
|
||||
c >>= 8
|
||||
o[4 * i] = (255 & c)
|
||||
|
||||
return o
|
||||
|
||||
def compress(self, a):
|
||||
f = self.generate_f(a)
|
||||
i = self.reg[:]
|
||||
for o in range(64):
|
||||
c = self.de(i[0], 12) + i[4] + self.de(self.pe(o), o)
|
||||
c = (c & 0xFFFFFFFF)
|
||||
c = self.de(c, 7)
|
||||
s = (c ^ self.de(i[0], 12)) & 0xFFFFFFFF
|
||||
|
||||
u = self.he(o, i[0], i[1], i[2])
|
||||
u = (u + i[3] + s + f[o + 68]) & 0xFFFFFFFF
|
||||
|
||||
b = self.ve(o, i[4], i[5], i[6])
|
||||
b = (b + i[7] + c + f[o]) & 0xFFFFFFFF
|
||||
|
||||
i[3] = i[2]
|
||||
i[2] = self.de(i[1], 9)
|
||||
i[1] = i[0]
|
||||
i[0] = u
|
||||
|
||||
i[7] = i[6]
|
||||
i[6] = self.de(i[5], 19)
|
||||
i[5] = i[4]
|
||||
i[4] = (b ^ self.de(b, 9) ^ self.de(b, 17)) & 0xFFFFFFFF
|
||||
|
||||
for l in range(8):
|
||||
self.reg[l] = (self.reg[l] ^ i[l]) & 0xFFFFFFFF
|
||||
|
||||
@classmethod
|
||||
def generate_f(cls, e):
|
||||
r = [0] * 132
|
||||
|
||||
for t in range(16):
|
||||
r[t] = (e[4 * t] << 24) | (e[4 * t + 1] <<
|
||||
16) | (e[4 * t + 2] << 8) | e[4 * t + 3]
|
||||
r[t] &= 0xFFFFFFFF
|
||||
|
||||
for n in range(16, 68):
|
||||
a = r[n - 16] ^ r[n - 9] ^ cls.de(r[n - 3], 15)
|
||||
a = a ^ cls.de(a, 15) ^ cls.de(a, 23)
|
||||
r[n] = (a ^ cls.de(r[n - 13], 7) ^ r[n - 6]) & 0xFFFFFFFF
|
||||
|
||||
for n in range(68, 132):
|
||||
r[n] = (r[n - 68] ^ r[n - 64]) & 0xFFFFFFFF
|
||||
|
||||
return r
|
||||
|
||||
@staticmethod
|
||||
def pad_array(arr, length=60):
|
||||
while len(arr) < length:
|
||||
arr.append(0)
|
||||
return arr
|
||||
|
||||
def fill(self, length=60):
|
||||
size = 8 * self.size
|
||||
self.chunk.append(128)
|
||||
self.chunk = self.pad_array(self.chunk, length)
|
||||
for i in range(4):
|
||||
self.chunk.append((size >> 8 * (3 - i)) & 255)
|
||||
|
||||
@staticmethod
|
||||
def list_4(
|
||||
a: int,
|
||||
b: int,
|
||||
c: int,
|
||||
d: int,
|
||||
e: int,
|
||||
f: int,
|
||||
g: int,
|
||||
h: int,
|
||||
i: int,
|
||||
j: int,
|
||||
k: int,
|
||||
m: int,
|
||||
n: int,
|
||||
o: int,
|
||||
p: int,
|
||||
q: int,
|
||||
r: int,
|
||||
) -> list:
|
||||
return [
|
||||
44,
|
||||
a,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
24,
|
||||
b,
|
||||
n,
|
||||
0,
|
||||
c,
|
||||
d,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
0,
|
||||
239,
|
||||
e,
|
||||
o,
|
||||
f,
|
||||
g,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
h,
|
||||
0,
|
||||
0,
|
||||
14,
|
||||
i,
|
||||
j,
|
||||
0,
|
||||
k,
|
||||
m,
|
||||
3,
|
||||
p,
|
||||
1,
|
||||
q,
|
||||
1,
|
||||
r,
|
||||
0,
|
||||
0,
|
||||
0]
|
||||
|
||||
@staticmethod
|
||||
def end_check_num(a: list):
|
||||
r = 0
|
||||
for i in a:
|
||||
r ^= i
|
||||
return r
|
||||
|
||||
@classmethod
|
||||
def decode_string(cls, url_string, ):
|
||||
decoded = cls.__filter.sub(cls.replace_func, url_string)
|
||||
return decoded
|
||||
|
||||
@staticmethod
|
||||
def replace_func(match):
|
||||
return chr(int(match.group(1), 16))
|
||||
|
||||
@staticmethod
|
||||
def de(e, r):
|
||||
r %= 32
|
||||
return ((e << r) & 0xFFFFFFFF) | (e >> (32 - r))
|
||||
|
||||
@staticmethod
|
||||
def pe(e):
|
||||
return 2043430169 if 0 <= e < 16 else 2055708042
|
||||
|
||||
@staticmethod
|
||||
def he(e, r, t, n):
|
||||
if 0 <= e < 16:
|
||||
return (r ^ t ^ n) & 0xFFFFFFFF
|
||||
elif 16 <= e < 64:
|
||||
return (r & t | r & n | t & n) & 0xFFFFFFFF
|
||||
raise ValueError
|
||||
|
||||
@staticmethod
|
||||
def ve(e, r, t, n):
|
||||
if 0 <= e < 16:
|
||||
return (r ^ t ^ n) & 0xFFFFFFFF
|
||||
elif 16 <= e < 64:
|
||||
return (r & t | ~r & n) & 0xFFFFFFFF
|
||||
raise ValueError
|
||||
|
||||
@staticmethod
|
||||
def convert_to_char_code(a):
|
||||
d = []
|
||||
for i in a:
|
||||
d.append(ord(i))
|
||||
return d
|
||||
|
||||
@staticmethod
|
||||
def split_array(arr, chunk_size=64):
|
||||
result = []
|
||||
for i in range(0, len(arr), chunk_size):
|
||||
result.append(arr[i:i + chunk_size])
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def char_code_at(s):
|
||||
return [ord(char) for char in s]
|
||||
|
||||
def write(self, e, ):
|
||||
self.size = len(e)
|
||||
if isinstance(e, str):
|
||||
e = self.decode_string(e)
|
||||
e = self.char_code_at(e)
|
||||
if len(e) <= 64:
|
||||
self.chunk = e
|
||||
else:
|
||||
chunks = self.split_array(e, 64)
|
||||
for i in chunks[:-1]:
|
||||
self.compress(i)
|
||||
self.chunk = chunks[-1]
|
||||
|
||||
def reset(self, ):
|
||||
self.chunk = []
|
||||
self.size = 0
|
||||
self.reg = self.__reg[:]
|
||||
|
||||
def sum(self, e, length=60):
|
||||
self.reset()
|
||||
self.write(e)
|
||||
self.fill(length)
|
||||
self.compress(self.chunk)
|
||||
return self.reg_to_array(self.reg)
|
||||
|
||||
@classmethod
|
||||
def generate_result_unit(cls, n, s):
|
||||
r = ""
|
||||
for i, j in zip(range(18, -1, -6), (16515072, 258048, 4032, 63)):
|
||||
r += cls.__str[s][(n & j) >> i]
|
||||
return r
|
||||
|
||||
@classmethod
|
||||
def generate_result_end(cls, s, e="s4"):
|
||||
r = ""
|
||||
b = ord(s[120]) << 16
|
||||
r += cls.__str[e][(b & 16515072) >> 18]
|
||||
r += cls.__str[e][(b & 258048) >> 12]
|
||||
r += "=="
|
||||
return r
|
||||
|
||||
@classmethod
|
||||
def generate_result(cls, s, e="s4"):
|
||||
# r = ""
|
||||
# for i in range(len(s)//4):
|
||||
# b = ((ord(s[i * 3]) << 16) | (ord(s[i * 3 + 1]))
|
||||
# << 8) | ord(s[i * 3 + 2])
|
||||
# r += cls.generate_result_unit(b, e)
|
||||
# return r
|
||||
|
||||
r = []
|
||||
|
||||
for i in range(0, len(s), 3):
|
||||
if i + 2 < len(s):
|
||||
n = (
|
||||
(ord(s[i]) << 16)
|
||||
| (ord(s[i + 1]) << 8)
|
||||
| ord(s[i + 2])
|
||||
)
|
||||
elif i + 1 < len(s):
|
||||
n = (ord(s[i]) << 16) | (
|
||||
ord(s[i + 1]) << 8
|
||||
)
|
||||
else:
|
||||
n = ord(s[i]) << 16
|
||||
|
||||
for j, k in zip(range(18, -1, -6),
|
||||
(0xFC0000, 0x03F000, 0x0FC0, 0x3F)):
|
||||
if j == 6 and i + 1 >= len(s):
|
||||
break
|
||||
if j == 0 and i + 2 >= len(s):
|
||||
break
|
||||
r.append(cls.__str[e][(n & k) >> j])
|
||||
|
||||
r.append("=" * ((4 - len(r) % 4) % 4))
|
||||
return "".join(r)
|
||||
|
||||
@classmethod
|
||||
def generate_args_code(cls):
|
||||
a = []
|
||||
for j in range(24, -1, -8):
|
||||
a.append(cls.__arguments[0] >> j)
|
||||
a.append(cls.__arguments[1] / 256)
|
||||
a.append(cls.__arguments[1] % 256)
|
||||
a.append(cls.__arguments[1] >> 24)
|
||||
a.append(cls.__arguments[1] >> 16)
|
||||
for j in range(24, -1, -8):
|
||||
a.append(cls.__arguments[2] >> j)
|
||||
return [int(i) & 255 for i in a]
|
||||
|
||||
def generate_method_code(self, method: str = "GET") -> list[int]:
|
||||
return self.sm3_to_array(self.sm3_to_array(method + self.__end_string))
|
||||
# return self.sum(self.sum(method + self.__end_string))
|
||||
|
||||
def generate_params_code(self, params: str) -> list[int]:
|
||||
return self.sm3_to_array(self.sm3_to_array(params + self.__end_string))
|
||||
# return self.sum(self.sum(params + self.__end_string))
|
||||
|
||||
@classmethod
|
||||
def sm3_to_array(cls, data: str | list) -> list[int]:
|
||||
"""
|
||||
代码参考: https://github.com/Johnserf-Seed/f2/blob/main/f2/utils/abogus.py
|
||||
|
||||
计算请求体的 SM3 哈希值，并将结果转换为整数数组
|
||||
Calculate the SM3 hash value of the request body and convert the result to an array of integers
|
||||
|
||||
Args:
|
||||
data (Union[str, List[int]]): 输入数据 (Input data).
|
||||
|
||||
Returns:
|
||||
List[int]: 哈希值的整数数组 (Array of integers representing the hash value).
|
||||
"""
|
||||
|
||||
if isinstance(data, str):
|
||||
b = data.encode("utf-8")
|
||||
else:
|
||||
b = bytes(data) # 将 List[int] 转换为字节数组
|
||||
|
||||
# 将字节数组转换为适合 sm3.sm3_hash 函数处理的列表格式
|
||||
h = sm3.sm3_hash(func.bytes_to_list(b))
|
||||
|
||||
# 将十六进制字符串结果转换为十进制整数列表
|
||||
return [int(h[i: i + 2], 16) for i in range(0, len(h), 2)]
|
||||
|
||||
@classmethod
|
||||
def generate_browser_info(cls, platform: str = "Win32") -> str:
|
||||
inner_width = randint(1280, 1920)
|
||||
inner_height = randint(720, 1080)
|
||||
outer_width = randint(inner_width, 1920)
|
||||
outer_height = randint(inner_height, 1080)
|
||||
screen_x = 0
|
||||
screen_y = choice((0, 30))
|
||||
value_list = [
|
||||
inner_width,
|
||||
inner_height,
|
||||
outer_width,
|
||||
outer_height,
|
||||
screen_x,
|
||||
screen_y,
|
||||
0,
|
||||
0,
|
||||
outer_width,
|
||||
outer_height,
|
||||
outer_width,
|
||||
outer_height,
|
||||
inner_width,
|
||||
inner_height,
|
||||
24,
|
||||
24,
|
||||
platform,
|
||||
]
|
||||
return "|".join(str(i) for i in value_list)
|
||||
|
||||
@staticmethod
|
||||
def rc4_encrypt(plaintext, key):
|
||||
s = list(range(256))
|
||||
j = 0
|
||||
|
||||
for i in range(256):
|
||||
j = (j + s[i] + ord(key[i % len(key)])) % 256
|
||||
s[i], s[j] = s[j], s[i]
|
||||
|
||||
i = 0
|
||||
j = 0
|
||||
cipher = []
|
||||
|
||||
for k in range(len(plaintext)):
|
||||
i = (i + 1) % 256
|
||||
j = (j + s[i]) % 256
|
||||
s[i], s[j] = s[j], s[i]
|
||||
t = (s[i] + s[j]) % 256
|
||||
cipher.append(chr(s[t] ^ ord(plaintext[k])))
|
||||
|
||||
return ''.join(cipher)
|
||||
|
||||
def get_value(self,
|
||||
url_params: dict | str,
|
||||
method="GET",
|
||||
start_time=0,
|
||||
end_time=0,
|
||||
random_num_1=None,
|
||||
random_num_2=None,
|
||||
random_num_3=None,
|
||||
) -> str:
|
||||
string_1 = self.generate_string_1(
|
||||
random_num_1,
|
||||
random_num_2,
|
||||
random_num_3,
|
||||
)
|
||||
string_2 = self.generate_string_2(urlencode(url_params) if isinstance(
|
||||
url_params, dict) else url_params, method, start_time, end_time, )
|
||||
string = string_1 + string_2
|
||||
# return self.generate_result(
|
||||
# string, "s4") + self.generate_result_end(string, "s4")
|
||||
return self.generate_result(string, "s4")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
bogus = ABogus()
|
||||
USERAGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36"
|
||||
url_str = "https://www.douyin.com/aweme/v1/web/aweme/detail/?device_platform=webapp&aid=6383&channel=channel_pc_web&pc_client_type=1&version_code=190500&version_name=19.5.0&cookie_enabled=true&browser_language=zh-CN&browser_platform=Win32&browser_name=Firefox&browser_online=true&engine_name=Gecko&os_name=Windows&os_version=10&platform=PC&screen_width=1920&screen_height=1080&browser_version=124.0&engine_version=122.0.0.0&cpu_core_num=12&device_memory=8&aweme_id=7345492945006595379"
|
||||
# 将url参数转换为字典
|
||||
url_params = dict([param.split("=")
|
||||
for param in url_str.split("?")[1].split("&")])
|
||||
print(f"URL参数: {url_params}")
|
||||
a_bogus = bogus.get_value(url_params, )
|
||||
# 使用url编码a_bogus
|
||||
a_bogus = quote(a_bogus, safe='')
|
||||
print(a_bogus)
|
||||
print(USERAGENT)
|
||||
@ -2,15 +2,21 @@ TokenManager:
|
||||
douyin:
|
||||
headers:
|
||||
Accept-Language: zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2
|
||||
# 不要这里的修改User-Agent，请保持默认，否则会导致请求失败。
|
||||
# Do not modify User-Agent here, please keep the default, otherwise it will cause request failure.
|
||||
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36
|
||||
Referer: https://www.douyin.com/
|
||||
Cookie: __ac_nonce=06629f03b000ca74fd5d0; __ac_signature=_02B4Z6wo00f018OnaAwAAIDCyYGzp89TogfDh2yAAJbJ17; s_v_web_id=verify_lvetxr14_jPt0ZofI_p5Nw_44FG_BGSL_SVodoPoGUlGG; ttwid=1%7CtCoZxPEZnSPKZQhxavM2S9G9udonQu1yTp4eeGEhmOU%7C1714024522%7Ce9786696e4a0d08ac93356835424aece59d214e5f5c1d054dcb62cc18a48b829; IsDouyinActive=true; home_can_add_dy_2_desktop=%220%22; dy_swidth=1463; dy_sheight=915; stream_recommend_feed_params=%22%7B%5C%22cookie_enabled%5C%22%3Atrue%2C%5C%22screen_width%5C%22%3A1463%2C%5C%22screen_height%5C%22%3A915%2C%5C%22browser_online%5C%22%3Atrue%2C%5C%22cpu_core_num%5C%22%3A32%2C%5C%22device_memory%5C%22%3A8%2C%5C%22downlink%5C%22%3A7.85%2C%5C%22effective_type%5C%22%3A%5C%224g%5C%22%2C%5C%22round_trip_time%5C%22%3A100%7D%22
|
||||
# 你唯一需要修改的地方就是这里的Cookie，然后保存后重启程序即可。
|
||||
# The only place you need to modify is the Cookie here, and then save and restart the program.
|
||||
Cookie: __ac_nonce=067d687ac00d70af16eab; __ac_signature=_02B4Z6wo00f018O6kmgAAIDAR1H8JrcivBPDi5bAAJdBcf; ttwid=1%7C46sVJ6G5zO0ZRKBqbFef2B13U3CqP9gLwQEH8IV2y6A%7C1742112685%7Cae649397cca7dde21884d5f8e3e3d53eb2361aa64af04cd6889fa71d7f23344b; UIFID_TEMP=986fab8dfc2c74111fac2b883dbdee67777473ded35e2c4bebbf68cc8b91739da61f6b365ad9795b0aa3a8bddce6cc3e39c5d4fd4bad667aaefd3d3ec08baac66fe3b215343f12d8aae84e0a24048f44; douyin.com; device_web_cpu_core=16; device_web_memory_size=-1; architecture=amd64; hevc_supported=true; IsDouyinActive=true; home_can_add_dy_2_desktop=%220%22; dy_swidth=1835; dy_sheight=1147; stream_recommend_feed_params=%22%7B%5C%22cookie_enabled%5C%22%3Atrue%2C%5C%22screen_width%5C%22%3A1835%2C%5C%22screen_height%5C%22%3A1147%2C%5C%22browser_online%5C%22%3Atrue%2C%5C%22cpu_core_num%5C%22%3A16%2C%5C%22device_memory%5C%22%3A0%2C%5C%22downlink%5C%22%3A%5C%22%5C%22%2C%5C%22effective_type%5C%22%3A%5C%22%5C%22%2C%5C%22round_trip_time%5C%22%3A0%7D%22; strategyABtestKey=%221742112685.842%22; volume_info=%7B%22isUserMute%22%3Afalse%2C%22isMute%22%3Afalse%2C%22volume%22%3A0.5%7D; stream_player_status_params=%22%7B%5C%22is_auto_play%5C%22%3A0%2C%5C%22is_full_screen%5C%22%3A0%2C%5C%22is_full_webscreen%5C%22%3A0%2C%5C%22is_mute%5C%22%3A0%2C%5C%22is_speed%5C%22%3A1%2C%5C%22is_visible%5C%22%3A1%7D%22; xgplayer_user_id=835787001711; fpk1=U2FsdGVkX19Ke0llbjXpGOOr1Jeel/2GnaSJz41VO3mAFs271jC0hG7gdWlk+2pYLM4GF8TVGtwClCJIXsTKUw==; fpk2=2333b8d335abc6e14aef1caed0ae26fc; s_v_web_id=verify_m8bcww86_XfwSCnmj_5i3F_4Joq_8edO_9gRH9JENh07f; csrf_session_id=6f34e666e71445c9d39d8d06a347a13f; FORCE_LOGIN=%7B%22videoConsumedRemainSeconds%22%3A180%7D; biz_trace_id=c34e5eaf; passport_csrf_token=ab84b3e39ad78e719b236035a27379c0; passport_csrf_token_default=ab84b3e39ad78e719b236035a27379c0; __security_mc_1_s_sdk_crypt_sdk=ac2d56c3-44cd-a161; __security_mc_1_s_sdk_cert_key=ccf2bd2d-4718-b8de; __security_mc_1_s_sdk_sign_data_key_web_protect=9995d368-4e45-b17f; bd_ticket_guard_client_data=eyJiZC10aWNrZXQtZ3VhcmQtdmVyc2lvbiI6MiwiYmQtdGlja2V0LWd1YXJkLWl0ZXJhdGlvbi12ZXJzaW9uIjoxLCJiZC10aWNrZXQtZ3VhcmQtcmVlLXB1YmxpYy1rZXkiOiJCUHR2ZDlUeGU4UlhPaWdIczFqaStJWityQlF4UWZMKytiL2drbXlYUmNrelNua1lQUjJTRVZHVlo4MWFCU0EvSW4xSnBmbzN3TFlvSnhIZTZTV29DTmc9IiwiYmQtdGlja2V0LWd1YXJkLXdlYi12ZXJzaW9uIjoyfQ%3D%3D; bd_ticket_guard_client_web_domain=2; xg_device_score=8.208487995540095; sdk_source_info=7e276470716a68645a606960273f276364697660272927676c715a6d6069756077273f276364697660272927666d776a68605a607d71606b766c6a6b5a7666776c7571273f275e58272927666a6b766a69605a696c6061273f27636469766027292762696a6764695a7364776c6467696076273f275e5827292771273f27303035353c3337343437313234272927676c715a75776a716a666a69273f2763646976602778; bit_env=LVdHnIescW9BCGpo5gGuqIlwNfgj757SBdMhdZXBSWjPWbxp9Nv_B2vUt_LtEvr-ioRv0E9b8N8HWiOHe20JqcUhO4YmpIM6gB83hjgqZfmAhYEbzJR7z2bRViJaPg4xeoyGhwdjwK_Bzogp6uoUs4ov-P4JYzMh78i7jaY5Pzd6h3CaVO-eUKnTiFfUlJo_jmhSfHXGdwkurXwR4lO_UnU4Loqa0YlmDiyi0fPxURFIN5t4Ny6Ua8LLSYcUrBXHlXoQ5G4bQN4XqwuWwT9YauexXbkotU1Jv8pMJUiAhlFIMjbvfTutTSnOXJLoH_JsR_doifURl0wf8CIa_OcYw-A2VglrpbaFU6HDVTKbSRKovzIMY9bUwl_4EAiLBf87g2BU0Uz1MHd_lGNdH3ImEWpLtdRvUsW_KD7q87rPsEGVTceyQ5U3ZlETqoEOwOiggNGu5lL_1O8lt8_7eydeGA%3D%3D; gulu_source_res=eyJwX2luIjoiM2Y3NGJhZDgxMzc3OThkNmVkN2U5ZjM3NDMzNGJkYjMwNzRhYjI0ZWJhMDZkMzdmYWNiNjgzNTY2ZjY0OGUyNCJ9; passport_auth_mix_state=c534f2qcgpohqv4juisp74wq28e90snz
|
||||
|
||||
proxies:
|
||||
http:
|
||||
https:
|
||||
|
||||
msToken:
|
||||
# 不要修改下面的内容。
|
||||
# Do not modify the content below.
|
||||
url: https://mssdk.bytedance.com/web/report
|
||||
magic: 538969122
|
||||
version: 1
|
||||
@ -19,5 +25,7 @@ TokenManager:
|
||||
User-Agent: 5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.47
|
||||
|
||||
ttwid:
|
||||
# 不要修改下面的内容。
|
||||
# Do not modify the content below.
|
||||
url: https://ttwid.bytedance.com/ttwid/union/register/
|
||||
data: '{"region":"cn","aid":1768,"needFid":false,"service":"www.ixigua.com","migrate_info":{"ticket":"","source":"node"},"cbUrlProtocol":"https","union":true}'
|
||||
@ -10,26 +10,35 @@ class BaseRequestModel(BaseModel):
|
||||
aid: str = "6383"
|
||||
channel: str = "channel_pc_web"
|
||||
pc_client_type: int = 1
|
||||
version_code: str = "190500"
|
||||
version_name: str = "19.5.0"
|
||||
version_code: str = "290100"
|
||||
version_name: str = "29.1.0"
|
||||
cookie_enabled: str = "true"
|
||||
screen_width: int = 1920
|
||||
screen_height: int = 1080
|
||||
browser_language: str = "zh-CN"
|
||||
browser_platform: str = "Win32"
|
||||
browser_name: str = "Firefox"
|
||||
browser_version: str = "124.0"
|
||||
browser_name: str = "Chrome"
|
||||
browser_version: str = "130.0.0.0"
|
||||
browser_online: str = "true"
|
||||
engine_name: str = "Gecko"
|
||||
engine_version: str = "122.0.0.0"
|
||||
engine_name: str = "Blink"
|
||||
engine_version: str = "130.0.0.0"
|
||||
os_name: str = "Windows"
|
||||
os_version: str = "10"
|
||||
cpu_core_num: int = 12
|
||||
device_memory: int = 8
|
||||
platform: str = "PC"
|
||||
# downlink: int = 10
|
||||
# effective_type: str = "4g"
|
||||
# round_trip_time: int = 100
|
||||
downlink: str = "10"
|
||||
effective_type: str = "4g"
|
||||
from_user_page: str = "1"
|
||||
locate_query: str = "false"
|
||||
need_time_list: str = "1"
|
||||
pc_libra_divert: str = "Windows"
|
||||
publish_video_strategy_type: str = "2"
|
||||
round_trip_time: str = "0"
|
||||
show_live_replay_strategy: str = "1"
|
||||
time_list_query: str = "0"
|
||||
whale_cut_token: str = ""
|
||||
update_version_code: str = "170400"
|
||||
msToken: str = TokenManager.gen_real_msToken()
|
||||
|
||||
|
||||
|
||||
@ -31,27 +31,25 @@
|
||||
# - https://github.com/Johnserf-Seed
|
||||
#
|
||||
# ==============================================================================
|
||||
|
||||
|
||||
import re
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
import urllib
|
||||
from pathlib import Path
|
||||
from typing import Union
|
||||
from urllib.parse import urlencode, quote
|
||||
|
||||
# import execjs
|
||||
import httpx
|
||||
import qrcode
|
||||
import random
|
||||
import asyncio
|
||||
import yaml
|
||||
|
||||
from typing import Union
|
||||
from pathlib import Path
|
||||
from crawlers.douyin.web.xbogus import XBogus as XB
|
||||
from crawlers.douyin.web.abogus import ABogus as AB
|
||||
|
||||
from crawlers.utils.logger import logger
|
||||
from crawlers.utils.utils import (
|
||||
gen_random_str,
|
||||
get_timestamp,
|
||||
extract_valid_urls,
|
||||
split_filename,
|
||||
)
|
||||
from crawlers.utils.api_exceptions import (
|
||||
APIError,
|
||||
APIConnectionError,
|
||||
@ -60,11 +58,13 @@ from crawlers.utils.api_exceptions import (
|
||||
APIUnauthorizedError,
|
||||
APINotFoundError,
|
||||
)
|
||||
|
||||
from crawlers.douyin.web.xbogus import XBogus as XB
|
||||
|
||||
from urllib.parse import quote
|
||||
import os
|
||||
from crawlers.utils.logger import logger
|
||||
from crawlers.utils.utils import (
|
||||
gen_random_str,
|
||||
get_timestamp,
|
||||
extract_valid_urls,
|
||||
split_filename,
|
||||
)
|
||||
|
||||
# 配置文件路径
|
||||
# Read the configuration file
|
||||
@ -116,38 +116,38 @@ class TokenManager:
|
||||
|
||||
msToken = str(httpx.Cookies(response.cookies).get("msToken"))
|
||||
if len(msToken) not in [120, 128]:
|
||||
raise APIResponseError("{0} 内容不符合要求".format("msToken"))
|
||||
raise APIResponseError("响应内容：{0}， Douyin msToken API 的响应内容不符合要求。".format(msToken))
|
||||
|
||||
return msToken
|
||||
|
||||
except httpx.RequestError as exc:
|
||||
# 捕获所有与 httpx 请求相关的异常情况 (Captures all httpx request-related exceptions)
|
||||
raise APIConnectionError(
|
||||
"请求端点失败，请检查当前网络环境。 链接：{0}，代理：{1}，异常类名：{2}，异常详细信息：{3}"
|
||||
.format(cls.token_conf["url"], cls.proxies, cls.__name__, exc)
|
||||
)
|
||||
# except httpx.RequestError as exc:
|
||||
# # 捕获所有与 httpx 请求相关的异常情况 (Captures all httpx request-related exceptions)
|
||||
# raise APIConnectionError(
|
||||
# "请求端点失败，请检查当前网络环境。 链接：{0}，代理：{1}，异常类名：{2}，异常详细信息：{3}"
|
||||
# .format(cls.token_conf["url"], cls.proxies, cls.__name__, exc)
|
||||
# )
|
||||
#
|
||||
# except httpx.HTTPStatusError as e:
|
||||
# # 捕获 httpx 的状态代码错误 (captures specific status code errors from httpx)
|
||||
# if e.response.status_code == 401:
|
||||
# raise APIUnauthorizedError(
|
||||
# "参数验证失败，请更新 Douyin_TikTok_Download_API 配置文件中的 {0}，以匹配 {1} 新规则"
|
||||
# .format("msToken", "douyin")
|
||||
# )
|
||||
#
|
||||
# elif e.response.status_code == 404:
|
||||
# raise APINotFoundError("{0} 无法找到API端点".format("msToken"))
|
||||
# else:
|
||||
# raise APIResponseError(
|
||||
# "链接：{0}，状态码 {1}：{2} ".format(
|
||||
# e.response.url, e.response.status_code, e.response.text
|
||||
# )
|
||||
# )
|
||||
|
||||
except httpx.HTTPStatusError as e:
|
||||
# 捕获 httpx 的状态代码错误 (captures specific status code errors from httpx)
|
||||
if e.response.status_code == 401:
|
||||
raise APIUnauthorizedError(
|
||||
"参数验证失败，请更新 F2 配置文件中的 {0}，以匹配 {1} 新规则"
|
||||
.format("msToken", "douyin")
|
||||
)
|
||||
|
||||
elif e.response.status_code == 404:
|
||||
raise APINotFoundError("{0} 无法找到API端点".format("msToken"))
|
||||
else:
|
||||
raise APIResponseError(
|
||||
"链接：{0}，状态码 {1}：{2} ".format(
|
||||
e.response.url, e.response.status_code, e.response.text
|
||||
)
|
||||
)
|
||||
|
||||
except APIError as e:
|
||||
except Exception as e:
|
||||
# 返回虚假的msToken (Return a fake msToken)
|
||||
logger.error("msToken API错误：{0}".format(e))
|
||||
logger.info("生成虚假的msToken")
|
||||
logger.error("请求Douyin msToken API时发生错误：{0}".format(e))
|
||||
logger.info("将使用本地生成的虚假msToken参数，以继续请求。")
|
||||
return cls.gen_false_msToken()
|
||||
|
||||
@classmethod
|
||||
@ -184,7 +184,7 @@ class TokenManager:
|
||||
# 捕获 httpx 的状态代码错误 (captures specific status code errors from httpx)
|
||||
if e.response.status_code == 401:
|
||||
raise APIUnauthorizedError(
|
||||
"参数验证失败，请更新 F2 配置文件中的 {0}，以匹配 {1} 新规则"
|
||||
"参数验证失败，请更新 Douyin_TikTok_Download_API 配置文件中的 {0}，以匹配 {1} 新规则"
|
||||
.format("ttwid", "douyin")
|
||||
)
|
||||
|
||||
@ -234,6 +234,8 @@ class VerifyFpManager:
|
||||
|
||||
|
||||
class BogusManager:
|
||||
|
||||
# 字符串方法生成X-Bogus参数
|
||||
@classmethod
|
||||
def xb_str_2_endpoint(cls, endpoint: str, user_agent: str) -> str:
|
||||
try:
|
||||
@ -243,6 +245,7 @@ class BogusManager:
|
||||
|
||||
return final_endpoint[0]
|
||||
|
||||
# 字典方法生成X-Bogus参数
|
||||
@classmethod
|
||||
def xb_model_2_endpoint(cls, base_endpoint: str, params: dict, user_agent: str) -> str:
|
||||
if not isinstance(params, dict):
|
||||
@ -262,6 +265,44 @@ class BogusManager:
|
||||
|
||||
return final_endpoint
|
||||
|
||||
# 字符串方法生成A-Bogus参数
|
||||
# TODO: 未完成测试，暂时不提交至主分支。
|
||||
# @classmethod
|
||||
# def ab_str_2_endpoint_js_ver(cls, endpoint: str, user_agent: str) -> str:
|
||||
# try:
|
||||
# # 获取请求参数
|
||||
# endpoint_query_params = urllib.parse.urlparse(endpoint).query
|
||||
# # 确定A-Bogus JS文件路径
|
||||
# js_path = os.path.dirname(os.path.abspath(__file__))
|
||||
# a_bogus_js_path = os.path.join(js_path, 'a_bogus.js')
|
||||
# with open(a_bogus_js_path, 'r', encoding='utf-8') as file:
|
||||
# js_code = file.read()
|
||||
# # 此处需要使用Node环境
|
||||
# # - 安装Node.js
|
||||
# # - 安装execjs库
|
||||
# # - 安装NPM依赖
|
||||
# # - npm install jsdom
|
||||
# node_runtime = execjs.get('Node')
|
||||
# context = node_runtime.compile(js_code)
|
||||
# arg = [0, 1, 0, endpoint_query_params, "", user_agent]
|
||||
# a_bougus = quote(context.call('get_a_bogus', arg), safe='')
|
||||
# return a_bougus
|
||||
# except Exception as e:
|
||||
# raise RuntimeError("生成A-Bogus失败: {0})".format(e))
|
||||
|
||||
# 字典方法生成A-Bogus参数，感谢 @JoeanAmier 提供的纯Python版本算法。
|
||||
@classmethod
|
||||
def ab_model_2_endpoint(cls, params: dict, user_agent: str) -> str:
|
||||
if not isinstance(params, dict):
|
||||
raise TypeError("参数必须是字典类型")
|
||||
|
||||
try:
|
||||
ab_value = AB().get_value(params, )
|
||||
except Exception as e:
|
||||
raise RuntimeError("生成A-Bogus失败: {0})".format(e))
|
||||
|
||||
return quote(ab_value, safe='')
|
||||
|
||||
|
||||
class SecUserIdFetcher:
|
||||
# 预编译正则表达式
|
||||
@ -365,6 +406,7 @@ class SecUserIdFetcher:
|
||||
class AwemeIdFetcher:
|
||||
# 预编译正则表达式
|
||||
_DOUYIN_VIDEO_URL_PATTERN = re.compile(r"video/([^/?]*)")
|
||||
_DOUYIN_VIDEO_URL_PATTERN_NEW = re.compile(r"[?&]vid=(\d+)")
|
||||
_DOUYIN_NOTE_URL_PATTERN = re.compile(r"note/([^/?]*)")
|
||||
_DOUYIN_DISCOVER_URL_PATTERN = re.compile(r"modal_id=([0-9]+)")
|
||||
|
||||
@ -377,62 +419,44 @@ class AwemeIdFetcher:
|
||||
url (str): 输入的url (Input url)
|
||||
|
||||
Returns:
|
||||
str: 匹配到的aweme_id (Matched aweme_id)。
|
||||
str: 匹配到的aweme_id (Matched aweme_id)
|
||||
"""
|
||||
|
||||
if not isinstance(url, str):
|
||||
raise TypeError("参数必须是字符串类型")
|
||||
|
||||
# 提取有效URL
|
||||
url = extract_valid_urls(url)
|
||||
|
||||
if url is None:
|
||||
raise (
|
||||
APINotFoundError("输入的URL不合法。类名：{0}".format(cls.__name__))
|
||||
)
|
||||
|
||||
# 重定向到完整链接
|
||||
transport = httpx.AsyncHTTPTransport(retries=5)
|
||||
async with httpx.AsyncClient(
|
||||
transport=transport, proxies=TokenManager.proxies, timeout=10
|
||||
transport=transport, proxy=None, timeout=10
|
||||
) as client:
|
||||
try:
|
||||
response = await client.get(url, follow_redirects=True)
|
||||
response.raise_for_status()
|
||||
|
||||
video_pattern = cls._DOUYIN_VIDEO_URL_PATTERN
|
||||
note_pattern = cls._DOUYIN_NOTE_URL_PATTERN
|
||||
discover_pattern = cls._DOUYIN_DISCOVER_URL_PATTERN
|
||||
response_url = str(response.url)
|
||||
|
||||
# 2024-4-22
|
||||
# 嵌套如果超过3层需要修改此处代码 (If the nesting exceeds 3 layers, you need to modify this code)
|
||||
match = video_pattern.search(str(response.url))
|
||||
if video_pattern.search(str(response.url)):
|
||||
aweme_id = match.group(1)
|
||||
else:
|
||||
match = note_pattern.search(str(response.url))
|
||||
# 按顺序尝试匹配视频ID
|
||||
for pattern in [
|
||||
cls._DOUYIN_VIDEO_URL_PATTERN,
|
||||
cls._DOUYIN_VIDEO_URL_PATTERN_NEW,
|
||||
cls._DOUYIN_NOTE_URL_PATTERN,
|
||||
cls._DOUYIN_DISCOVER_URL_PATTERN
|
||||
]:
|
||||
match = pattern.search(response_url)
|
||||
if match:
|
||||
aweme_id = match.group(1)
|
||||
else:
|
||||
match = discover_pattern.search(str(response.url))
|
||||
if match:
|
||||
aweme_id = match.group(1)
|
||||
else:
|
||||
raise APIResponseError(
|
||||
"未在响应的地址中找到aweme_id，检查链接是否为作品页"
|
||||
)
|
||||
return aweme_id
|
||||
return match.group(1)
|
||||
|
||||
raise APIResponseError("未在响应的地址中找到 aweme_id，检查链接是否为作品页")
|
||||
|
||||
except httpx.RequestError as exc:
|
||||
# 捕获所有与 httpx 请求相关的异常情况 (Captures all httpx request-related exceptions)
|
||||
raise APIConnectionError("请求端点失败，请检查当前网络环境。 链接：{0}，代理：{1}，异常类名：{2}，异常详细信息：{3}"
|
||||
.format(url, TokenManager.proxies, cls.__name__, exc)
|
||||
raise APIConnectionError(
|
||||
f"请求端点失败，请检查当前网络环境。链接：{url}，代理：{TokenManager.proxies}，异常类名：{cls.__name__}，异常详细信息：{exc}"
|
||||
)
|
||||
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise APIResponseError("链接：{0}，状态码 {1}：{2} ".format(
|
||||
e.response.url, e.response.status_code, e.response.text
|
||||
)
|
||||
raise APIResponseError(
|
||||
f"链接：{e.response.url}，状态码 {e.response.status_code}：{e.response.text}"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
||||
@ -34,14 +34,21 @@
|
||||
|
||||
|
||||
import asyncio # 异步I/O
|
||||
import time # 时间操作
|
||||
import yaml # 配置文件
|
||||
import os # 系统操作
|
||||
import time # 时间操作
|
||||
from urllib.parse import urlencode, quote # URL编码
|
||||
import yaml # 配置文件
|
||||
|
||||
# 基础爬虫客户端和抖音API端点
|
||||
from crawlers.base_crawler import BaseCrawler
|
||||
from crawlers.douyin.web.endpoints import DouyinAPIEndpoints
|
||||
|
||||
# 抖音接口数据请求模型
|
||||
from crawlers.douyin.web.models import (
|
||||
BaseRequestModel, LiveRoomRanking, PostComments,
|
||||
PostCommentsReply, PostDetail,
|
||||
UserProfile, UserCollection, UserLike, UserLive,
|
||||
UserLive2, UserMix, UserPost
|
||||
)
|
||||
# 抖音应用的工具类
|
||||
from crawlers.douyin.web.utils import (AwemeIdFetcher, # Aweme ID获取
|
||||
BogusManager, # XBogus管理
|
||||
@ -52,14 +59,6 @@ from crawlers.douyin.web.utils import (AwemeIdFetcher, # Aweme ID获取
|
||||
extract_valid_urls # URL提取
|
||||
)
|
||||
|
||||
# 抖音接口数据请求模型
|
||||
from crawlers.douyin.web.models import (
|
||||
BaseRequestModel, LiveRoomRanking, PostComments,
|
||||
PostCommentsReply, PostDanmaku, PostDetail,
|
||||
UserProfile, UserCollection, UserLike, UserLive,
|
||||
UserLive2, UserMix, UserPost
|
||||
)
|
||||
|
||||
# 配置文件路径
|
||||
path = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
@ -96,9 +95,17 @@ class DouyinWebCrawler:
|
||||
# 创建一个作品详情的BaseModel参数
|
||||
params = PostDetail(aweme_id=aweme_id)
|
||||
# 生成一个作品详情的带有加密参数的Endpoint
|
||||
endpoint = BogusManager.xb_model_2_endpoint(
|
||||
DouyinAPIEndpoints.POST_DETAIL, params.dict(), kwargs["headers"]["User-Agent"]
|
||||
)
|
||||
# 2024年6月12日22:41:44 由于XBogus加密已经失效，所以不再使用XBogus加密参数，转移至a_bogus加密参数。
|
||||
# endpoint = BogusManager.xb_model_2_endpoint(
|
||||
# DouyinAPIEndpoints.POST_DETAIL, params.dict(), kwargs["headers"]["User-Agent"]
|
||||
# )
|
||||
|
||||
# 生成一个作品详情的带有a_bogus加密参数的Endpoint
|
||||
params_dict = params.dict()
|
||||
params_dict["msToken"] = ''
|
||||
a_bogus = BogusManager.ab_model_2_endpoint(params_dict, kwargs["headers"]["User-Agent"])
|
||||
endpoint = f"{DouyinAPIEndpoints.POST_DETAIL}?{urlencode(params_dict)}&a_bogus={a_bogus}"
|
||||
|
||||
response = await crawler.fetch_get_json(endpoint)
|
||||
return response
|
||||
|
||||
@ -108,9 +115,17 @@ class DouyinWebCrawler:
|
||||
base_crawler = BaseCrawler(proxies=kwargs["proxies"], crawler_headers=kwargs["headers"])
|
||||
async with base_crawler as crawler:
|
||||
params = UserPost(sec_user_id=sec_user_id, max_cursor=max_cursor, count=count)
|
||||
endpoint = BogusManager.xb_model_2_endpoint(
|
||||
DouyinAPIEndpoints.USER_POST, params.dict(), kwargs["headers"]["User-Agent"]
|
||||
)
|
||||
# endpoint = BogusManager.xb_model_2_endpoint(
|
||||
# DouyinAPIEndpoints.USER_POST, params.dict(), kwargs["headers"]["User-Agent"]
|
||||
# )
|
||||
# response = await crawler.fetch_get_json(endpoint)
|
||||
|
||||
# 生成一个用户发布作品数据的带有a_bogus加密参数的Endpoint
|
||||
params_dict = params.dict()
|
||||
params_dict["msToken"] = ''
|
||||
a_bogus = BogusManager.ab_model_2_endpoint(params_dict, kwargs["headers"]["User-Agent"])
|
||||
endpoint = f"{DouyinAPIEndpoints.USER_POST}?{urlencode(params_dict)}&a_bogus={a_bogus}"
|
||||
|
||||
response = await crawler.fetch_get_json(endpoint)
|
||||
return response
|
||||
|
||||
@ -120,9 +135,16 @@ class DouyinWebCrawler:
|
||||
base_crawler = BaseCrawler(proxies=kwargs["proxies"], crawler_headers=kwargs["headers"])
|
||||
async with base_crawler as crawler:
|
||||
params = UserLike(sec_user_id=sec_user_id, max_cursor=max_cursor, count=count)
|
||||
endpoint = BogusManager.xb_model_2_endpoint(
|
||||
DouyinAPIEndpoints.USER_FAVORITE_A, params.dict(), kwargs["headers"]["User-Agent"]
|
||||
)
|
||||
# endpoint = BogusManager.xb_model_2_endpoint(
|
||||
# DouyinAPIEndpoints.USER_FAVORITE_A, params.dict(), kwargs["headers"]["User-Agent"]
|
||||
# )
|
||||
# response = await crawler.fetch_get_json(endpoint)
|
||||
|
||||
params_dict = params.dict()
|
||||
params_dict["msToken"] = ''
|
||||
a_bogus = BogusManager.ab_model_2_endpoint(params_dict, kwargs["headers"]["User-Agent"])
|
||||
endpoint = f"{DouyinAPIEndpoints.USER_FAVORITE_A}?{urlencode(params_dict)}&a_bogus={a_bogus}"
|
||||
|
||||
response = await crawler.fetch_get_json(endpoint)
|
||||
return response
|
||||
|
||||
@ -275,6 +297,21 @@ class DouyinWebCrawler:
|
||||
}
|
||||
return result
|
||||
|
||||
# 使用接口地址生成Ab参数
|
||||
async def get_a_bogus(self, url: str, user_agent: str):
|
||||
endpoint = url.split("?")[0]
|
||||
# 将URL参数转换为dict
|
||||
params = dict([i.split("=") for i in url.split("?")[1].split("&")])
|
||||
# 去除URL中的msToken参数
|
||||
params["msToken"] = ""
|
||||
a_bogus = BogusManager.ab_model_2_endpoint(params, user_agent)
|
||||
result = {
|
||||
"url": f"{endpoint}?{urlencode(params)}&a_bogus={a_bogus}",
|
||||
"a_bogus": a_bogus,
|
||||
"user_agent": user_agent
|
||||
}
|
||||
return result
|
||||
|
||||
# 提取单个用户id
|
||||
async def get_sec_user_id(self, url: str):
|
||||
return await SecUserIdFetcher.get_sec_user_id(url)
|
||||
@ -315,7 +352,7 @@ class DouyinWebCrawler:
|
||||
"""-------------------------------------------------------handler接口列表-------------------------------------------------------"""
|
||||
|
||||
# 获取单一视频信息
|
||||
# aweme_id = "7345492945006595379"
|
||||
# aweme_id = "7372484719365098803"
|
||||
# result = await self.fetch_one_video(aweme_id)
|
||||
# print(result)
|
||||
|
||||
@ -398,6 +435,11 @@ class DouyinWebCrawler:
|
||||
|
||||
"""-------------------------------------------------------utils接口列表-------------------------------------------------------"""
|
||||
|
||||
# 获取抖音Web的游客Cookie
|
||||
# user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36"
|
||||
# result = await self.fetch_douyin_web_guest_cookie(user_agent)
|
||||
# print(result)
|
||||
|
||||
# 生成真实msToken
|
||||
# result = await self.gen_real_msToken()
|
||||
# print(result)
|
||||
|
||||
@ -1,3 +1,36 @@
|
||||
# ==============================================================================
|
||||
# Copyright (C) 2021 Evil0ctal
|
||||
#
|
||||
# This file is part of the Douyin_TikTok_Download_API project.
|
||||
#
|
||||
# This project is licensed under the Apache License 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at:
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
# ＿＿
|
||||
# ／＞ フ
|
||||
# | _ _ l
|
||||
# ／` ミ＿xノ
|
||||
# / | Feed me Stars ⭐ ️
|
||||
# / ヽ ﾉ
|
||||
# │ | | |
|
||||
# ／￣| | | |
|
||||
# | (￣ヽ＿_ヽ_)__)
|
||||
# ＼二つ
|
||||
# ==============================================================================
|
||||
#
|
||||
# Contributor Link:
|
||||
# - https://github.com/Evil0ctal
|
||||
#
|
||||
# ==============================================================================
|
||||
|
||||
import asyncio
|
||||
|
||||
from crawlers.douyin.web.web_crawler import DouyinWebCrawler # 导入抖音Web爬虫
|
||||
@ -24,8 +57,13 @@ class HybridCrawler:
|
||||
elif "tiktok" in url:
|
||||
platform = "tiktok"
|
||||
aweme_id = await self.TikTokWebCrawler.get_aweme_id(url)
|
||||
|
||||
# 2024-09-14: Switch to TikTokAPPCrawler instead of TikTokWebCrawler
|
||||
# data = await self.TikTokWebCrawler.fetch_one_video(aweme_id)
|
||||
# data = data.get("itemInfo").get("itemStruct")
|
||||
|
||||
data = await self.TikTokAPPCrawler.fetch_one_video(aweme_id)
|
||||
# $.aweme_type
|
||||
# $.imagePost exists if aweme_type is photo
|
||||
aweme_type = data.get("aweme_type")
|
||||
else:
|
||||
raise ValueError("hybrid_parsing_single_video: Cannot judge the video source from the URL.")
|
||||
@ -51,6 +89,7 @@ class HybridCrawler:
|
||||
}
|
||||
# 判断链接类型/Judge link type
|
||||
url_type = url_type_code_dict.get(aweme_type, 'video')
|
||||
# print(f"url_type: {url_type}")
|
||||
|
||||
"""
|
||||
以下为(视频||图片)数据处理的四个方法,如果你需要自定义数据处理请在这里修改.
|
||||
@ -124,13 +163,22 @@ class HybridCrawler:
|
||||
# TikTok视频数据处理/TikTok video data processing
|
||||
if url_type == 'video':
|
||||
# 将信息储存在字典中/Store information in a dictionary
|
||||
wm_video = data['video']['download_addr']['url_list'][0]
|
||||
# wm_video = data['video']['downloadAddr']
|
||||
# wm_video = data['video']['download_addr']['url_list'][0]
|
||||
wm_video = (
|
||||
data.get('video', {})
|
||||
.get('download_addr', {})
|
||||
.get('url_list', [None])[0]
|
||||
)
|
||||
|
||||
api_data = {
|
||||
'video_data':
|
||||
{
|
||||
'wm_video_url': wm_video,
|
||||
'wm_video_url_HQ': wm_video,
|
||||
# 'nwm_video_url': data['video']['playAddr'],
|
||||
'nwm_video_url': data['video']['play_addr']['url_list'][0],
|
||||
# 'nwm_video_url_HQ': data['video']['bitrateInfo'][0]['PlayAddr']['UrlList'][0]
|
||||
'nwm_video_url_HQ': data['video']['bit_rate'][0]['play_addr']['url_list'][0]
|
||||
}
|
||||
}
|
||||
@ -157,7 +205,9 @@ class HybridCrawler:
|
||||
async def main(self):
|
||||
# 测试混合解析单一视频接口/Test hybrid parsing single video endpoint
|
||||
# url = "https://v.douyin.com/L4FJNR3/"
|
||||
url = "https://www.tiktok.com/@evil0ctal/video/7156033831819037994"
|
||||
# url = "https://www.tiktok.com/@taylorswift/video/7359655005701311786"
|
||||
url = "https://www.tiktok.com/@flukegk83/video/7360734489271700753"
|
||||
# url = "https://www.tiktok.com/@minecraft/photo/7369296852669205791"
|
||||
minimal = True
|
||||
result = await self.hybrid_parsing_single_video(url, minimal=minimal)
|
||||
print(result)
|
||||
|
||||
@ -43,11 +43,17 @@ from crawlers.base_crawler import BaseCrawler
|
||||
from crawlers.tiktok.app.endpoints import TikTokAPIEndpoints
|
||||
from crawlers.utils.utils import model_to_query_string
|
||||
|
||||
# 重试机制
|
||||
from tenacity import *
|
||||
|
||||
# TikTok接口数据请求模型
|
||||
from crawlers.tiktok.app.models import (
|
||||
BaseRequestModel, FeedVideoDetail
|
||||
)
|
||||
|
||||
# 标记已废弃的方法
|
||||
from crawlers.utils.deprecated import deprecated
|
||||
|
||||
# 配置文件路径
|
||||
path = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
@ -66,14 +72,18 @@ class TikTokAPPCrawler:
|
||||
"User-Agent": tiktok_config["headers"]["User-Agent"],
|
||||
"Referer": tiktok_config["headers"]["Referer"],
|
||||
"Cookie": tiktok_config["headers"]["Cookie"],
|
||||
"x-ladon": "Hello From Evil0ctal!",
|
||||
},
|
||||
"proxies": {"http://": None, "https://": None},
|
||||
"proxies": {"http://": tiktok_config["proxies"]["http"],
|
||||
"https://": tiktok_config["proxies"]["https"]}
|
||||
}
|
||||
return kwargs
|
||||
|
||||
"""-------------------------------------------------------handler接口列表-------------------------------------------------------"""
|
||||
|
||||
# 获取单个作品数据
|
||||
# @deprecated("TikTok APP fetch_one_video is deprecated and will be removed in a future release. Use Web API instead. | TikTok APP fetch_one_video 已弃用，将在将来的版本中删除。请改用Web API。")
|
||||
@retry(stop=stop_after_attempt(10), wait=wait_fixed(1))
|
||||
async def fetch_one_video(self, aweme_id: str):
|
||||
# 获取TikTok的实时Cookie
|
||||
kwargs = await self.get_tiktok_headers()
|
||||
|
||||
@ -3,14 +3,18 @@ TokenManager:
|
||||
headers:
|
||||
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36
|
||||
Referer: https://www.tiktok.com/
|
||||
Cookie: tt_csrf_token=YmksDB6a-h4cT2fF7JpORI2O9UBMCWjsntIc; ttwid=1%7C0FVb9fFc-sjDG2UdJwdC1AirqYozQ0xfbAS4N72vN2Y%7C1713886256%7C78a9d83445b82b73ca8d4e0cf024ea6cdf1329b7f3866c826b0a69a300ebce46; ak_bmsc=51B1D53481A3A4E4D0CEFF2BCF622DA2~000000000000000000000000000000~YAAQ7uIsF6c4j+SOAQAAANmUCxfRGVXZ4D9xnO97l1yDw0OWyomnVkNY7IUKaggUja0kQzFQ+WG4xaxBcPt0AN0n26KeHXGGKgHYpHPUMUBHGHQGDtE4RLyy7U+LPbSJCqVaSDiPuzxHht0YUIbWogvrFmBfkP4ohcmjkZxWtEI9qQ4Whaobb2CFHGdKNt0zlVNBjJQ3uYRAvUe12zSBynQB18y6QhE8goneRkCEw9VIeft2pFIwNQ8tkWWEjDt6wHNaqeND7eASg5WLzYskWbTt6bPAOhSNRLJ38HZrOB5QNg+xxN5uuCSYmjMXCl8SkvQr91pInmOng+V898FLLBQtefs95whvbpfE0mKwBk5Cz2TkkHcUJa/IoC0CLmNqoEk3AtKxpw/J; tt_chain_token=46Xkv2ukMzyJ2e7XU7y0AQ==; bm_sv=A2E67B998DE8E6A4F1C2C02485467446~YAAQ7uIsF6g4j+SOAQAABdqUCxf1J/K4dYG0k7bbw2m5rFujdlSqMoCKDubu4R602nFvbY6zWC5puJczBv3IXwJJRpQxxR03wDCMVlKTCqjQvgDs8BoCuoNQxfY2fdS+F3bKut2lxXPQ2qctqz4kHBrgspJArHn/zu/IuKCIeSzmV4KcyxW6Zvw3/xMRA0MeHgyuHsTRBS+VrFk8Ju2NbJWWC8uSHbLCM/dhFT7/ktw8RE30r24XpQmhLpVTsUSC~1; tiktok_webapp_theme=light; msToken=ySXERzKCE0QUG0cCg6TWLw3wfEB-6kh6kAfuzhzjcQvmV1jBFloSgIsT9xk-QXFVdI99U1Fqb9mhUpIOldoDkjdZwskB8rvt66MHZaHnvBRZRtOKtTYsWT8osDyQXDVZWdPkvyE598h9; passport_csrf_token=1a47d95ebf68fc3648b0018ee75afc9f; passport_csrf_token_default=1a47d95ebf68fc3648b0018ee75afc9f; perf_feed_cache={%22expireTimestamp%22:1714057200000%2C%22itemIds%22:[%227346425092966206766%22%2C%227353812964207594795%22%2C%227343343741916171563%22]}; msToken=yWwG-ITrCnjJbx5ltBa9FTXdCImOJrl-wtQJSQH3afeEumWZcbo_qcrF6F7-NjYcrG6JVxtJiOU208REZeCSgXEZrrs5_65K741fQ7PSzCGOhz6vUyycq3Xvj4Mu-S0kJ6SqyltHnpJp
|
||||
# 你唯一需要修改的地方就是这里的Cookie，然后保存后重启程序即可。
|
||||
# The only place you need to modify is the Cookie here, and then save and restart the program.
|
||||
Cookie: tt_csrf_token=bwnaRGd9-B-0ce8ntqw9jtGzAdvzTRKNpBl0; ak_bmsc=75A1956756DE42FD14ED069AAE7A8780~000000000000000000000000000000~YAAQXCw+F8jpmBGQAQAAIfGsFBj+ZEGzR/ZeiuPpMtItu0QQUQRmjBX2kADliy6QA9rZSfrxRUZc9zuRrI4/xbIrAwA/nkdguGpa+v3QSn/1sk5uP2aqLVm0eYB/SGNafa2h2QvIPbLNiSCRhgq1GalZJL4+udqDnyBRJWE74nin74bZwrVDvCX1s8M2hWqZ9/jTkdm4sfwON9MdJIEtjAPlddQ4gxoqjPoWhfnrm24dhPT4OjL1B8QP1mgurj7zJGspqD53VcjkAl65gHVxp3dwZ5WbPYpqrh9j8wo2u/Wh6uhX+0HWmkv5yVZyTyYQTl3/ilPp9G4CuIUi84gaPLjNYea9AEnphNX0ywzDa6/yegfqyE6r3wqBBDCrR1xRM98YEB4A5PV7pw==; tt_chain_token=ljZFLdRDfyfDflXMg5XGpg==; tiktok_webapp_theme_auto_dark_ab=1; tiktok_webapp_theme=dark; perf_feed_cache={%22expireTimestamp%22:1718503200000%2C%22itemIds%22:[%227348816520216186158%22%2C%227356022137678810410%22%2C%227349561209340857630%22]}; s_v_web_id=verify_lxe3l432_JnDE5WWo_URef_4WrS_88IM_fd1CqEXZs4dZ; passport_csrf_token=af197f073ed95f4dc2636f24d55566a6; passport_csrf_token_default=af197f073ed95f4dc2636f24d55566a6; ttwid=1%7CuNT4GcgvvOjH8rTETh9d9xti_QDJjlcnSK2V7djIpuc%7C1718333954%7Cf81b989a495aedff91302da4d0a3ab6055dea486fb203a4326b37d5a5346ad0c; msToken=1Mhpyi8MlaZjM6bbLDVUhCj_6C0kEO_1_Nb62ByXLg7wy_vLnBxdMFpKclhf4HYnEjCghk2Gq47ZM5jPj3L1yFxQUZvq4oPLo1b2Wfe_33RE94uIxdiR-eSueWbcYDDgOj1Pn9Wyid5Uf5fzBQ7xxFA=; bm_sv=9ADBA7BE06EC41817F117E2279F1410C~YAAQXCw+F8bsmBGQAQAAzSewFBg2fP3Zd0aky2x7S13D97O64xi8EXhoKORBnPQyCHlh0iSlh63FFjoy6peDWaF3lkWaTly3Z7I7WvWk1GCntnYzpJaSCE5EO2OL38zPWpHcgGWuekluvptHXsheedNEefN4SUHVMt4jJynWNeTKrao0RmNLkH4zGs7QO6+MPCt94QFvNfLjBRr0wVcXlN/hx9m6kcvCyzsBBqEnpugoYvZ0SMA+INsKI5PDfQz1~1; msToken=449_l3kdcLmnEHdDP0uACa5EcPVL1NbpjyVv8yah61EwxIPZRDlGwpGIkpIjH0Tk-CDtoKwFrDdP1v2AOpwmdoIz5oQzPEXCdyfGzcVXCHbwMX1fwPxMHpea5yFPUYEDlNWaCFlgLnejRdWeN5sB_lE=
|
||||
|
||||
proxies:
|
||||
http:
|
||||
https:
|
||||
|
||||
msToken:
|
||||
url: https://mssdk-sg.tiktok.com/web/common?msToken=1Ab-7YxR9lUHSem0PraI_XzdKmpHb6j50L8AaXLAd2aWTdoJCYLfX_67rVQFE4UwwHVHmyG_NfIipqrlLT3kCXps-5PYlNAqtdwEg7TrDyTAfCKyBrOLmhMUjB55oW8SPZ4_EkNxNFUdV7MquA==
|
||||
# 不要修改下面的内容。
|
||||
# Do not modify the content below.
|
||||
url: https://mssdk.tiktokw.us/web/report?msToken=1Ab-7YxR9lUHSem0PraI_XzdKmpHb6j50L8AaXLAd2aWTdoJCYLfX_67rVQFE4UwwHVHmyG_NfIipqrlLT3kCXps-5PYlNAqtdwEg7TrDyTAfCKyBrOLmhMUjB55oW8SPZ4_EkNxNFUdV7MquA==
|
||||
magic: 538969122
|
||||
version: 1
|
||||
dataType: 8
|
||||
@ -18,6 +22,8 @@ TokenManager:
|
||||
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36
|
||||
|
||||
ttwid:
|
||||
# 不要修改下面的内容。
|
||||
# Do not modify the content below.
|
||||
url: https://www.tiktok.com/ttwid/check/
|
||||
data: '{"aid":1988,"service":"www.tiktok.com","union":false,"unionHost":"","needFid":false,"fid":"","migrate_priority":0}'
|
||||
cookie: tt_csrf_token=YmksDB6a-h4cT2fF7JpORI2O9UBMCWjsntIc; ttwid=1%7C0FVb9fFc-sjDG2UdJwdC1AirqYozQ0xfbAS4N72vN2Y%7C1713886256%7C78a9d83445b82b73ca8d4e0cf024ea6cdf1329b7f3866c826b0a69a300ebce46; ak_bmsc=51B1D53481A3A4E4D0CEFF2BCF622DA2~000000000000000000000000000000~YAAQ7uIsF6c4j+SOAQAAANmUCxfRGVXZ4D9xnO97l1yDw0OWyomnVkNY7IUKaggUja0kQzFQ+WG4xaxBcPt0AN0n26KeHXGGKgHYpHPUMUBHGHQGDtE4RLyy7U+LPbSJCqVaSDiPuzxHht0YUIbWogvrFmBfkP4ohcmjkZxWtEI9qQ4Whaobb2CFHGdKNt0zlVNBjJQ3uYRAvUe12zSBynQB18y6QhE8goneRkCEw9VIeft2pFIwNQ8tkWWEjDt6wHNaqeND7eASg5WLzYskWbTt6bPAOhSNRLJ38HZrOB5QNg+xxN5uuCSYmjMXCl8SkvQr91pInmOng+V898FLLBQtefs95whvbpfE0mKwBk5Cz2TkkHcUJa/IoC0CLmNqoEk3AtKxpw/J; tt_chain_token=46Xkv2ukMzyJ2e7XU7y0AQ==; bm_sv=A2E67B998DE8E6A4F1C2C02485467446~YAAQ7uIsF6g4j+SOAQAABdqUCxf1J/K4dYG0k7bbw2m5rFujdlSqMoCKDubu4R602nFvbY6zWC5puJczBv3IXwJJRpQxxR03wDCMVlKTCqjQvgDs8BoCuoNQxfY2fdS+F3bKut2lxXPQ2qctqz4kHBrgspJArHn/zu/IuKCIeSzmV4KcyxW6Zvw3/xMRA0MeHgyuHsTRBS+VrFk8Ju2NbJWWC8uSHbLCM/dhFT7/ktw8RE30r24XpQmhLpVTsUSC~1; tiktok_webapp_theme=light; msToken=ySXERzKCE0QUG0cCg6TWLw3wfEB-6kh6kAfuzhzjcQvmV1jBFloSgIsT9xk-QXFVdI99U1Fqb9mhUpIOldoDkjdZwskB8rvt66MHZaHnvBRZRtOKtTYsWT8osDyQXDVZWdPkvyE598h9; passport_csrf_token=1a47d95ebf68fc3648b0018ee75afc9f; passport_csrf_token_default=1a47d95ebf68fc3648b0018ee75afc9f; perf_feed_cache={%22expireTimestamp%22:1714057200000%2C%22itemIds%22:[%227346425092966206766%22%2C%227353812964207594795%22%2C%227343343741916171563%22]}; msToken=yWwG-ITrCnjJbx5ltBa9FTXdCImOJrl-wtQJSQH3afeEumWZcbo_qcrF6F7-NjYcrG6JVxtJiOU208REZeCSgXEZrrs5_65K741fQ7PSzCGOhz6vUyycq3Xvj4Mu-S0kJ6SqyltHnpJp
|
||||
|
||||
@ -22,7 +22,8 @@ class BaseRequestModel(BaseModel):
|
||||
)
|
||||
channel: str = "tiktok_web"
|
||||
cookie_enabled: str = "true"
|
||||
device_id: int = 7349090360347690538
|
||||
device_id: int = 7380187414842836523
|
||||
odinId: int = 7404669909585003563
|
||||
device_platform: str = "web_pc"
|
||||
focus_state: str = "true"
|
||||
from_page: str = "user"
|
||||
@ -49,11 +50,47 @@ class UserProfile(BaseRequestModel):
|
||||
uniqueId: str
|
||||
|
||||
|
||||
class UserPost(BaseRequestModel):
|
||||
class UserPost(BaseModel):
|
||||
WebIdLastTime: str = "1714385892"
|
||||
aid: str = "1988"
|
||||
app_language: str = "zh-Hans"
|
||||
app_name: str = "tiktok_web"
|
||||
browser_language: str = "zh-CN"
|
||||
browser_name: str = "Mozilla"
|
||||
browser_online: str = "true"
|
||||
browser_platform: str = "Win32"
|
||||
browser_version: str = "5.0%20%28Windows%29"
|
||||
channel: str = "tiktok_web"
|
||||
cookie_enabled: str = "true"
|
||||
count: int = 20
|
||||
coverFormat: int = 2
|
||||
count: int = 35
|
||||
cursor: int = 0
|
||||
data_collection_enabled: str = "true"
|
||||
device_id: str = "7380187414842836523"
|
||||
device_platform: str = "web_pc"
|
||||
focus_state: str = "true"
|
||||
from_page: str = "user"
|
||||
history_len: str = "3"
|
||||
is_fullscreen: str = "false"
|
||||
is_page_visible: str = "true"
|
||||
language: str = "zh-Hans"
|
||||
locate_item_id: str = ""
|
||||
needPinnedItemIds: str = "true"
|
||||
odinId: str = "7404669909585003563"
|
||||
os: str = "windows"
|
||||
# 0：默认排序，1：热门排序，2：最旧排序
|
||||
post_item_list_request_type: int = 0
|
||||
priority_region: str = "US"
|
||||
referer: str = ""
|
||||
region: str = "US"
|
||||
screen_height: str = "827"
|
||||
screen_width: str = "1323"
|
||||
secUid: str
|
||||
tz_name: str = "America%2FLos_Angeles"
|
||||
user_is_login: str = "true"
|
||||
webcast_language: str = "zh-Hans"
|
||||
msToken: str = "SXtP7K0MMFlQmzpuWfZoxAlAaKqt-2p8oAbOHFBw-k3TA2g4jE_FXrFKf3i38lR-xNh_bV1_qfTPRnj4PXbkBfrVD2iAazeUkASIASHT0pu-Bx2_POx7O3nBBHZe2SI7CPsanerdclxHht1hcoUTlg%3D%3D"
|
||||
_signature: str = "_02B4Z6wo000017oyWOQAAIDD9xNhTSnfaDu6MFxAAIlj23"
|
||||
|
||||
|
||||
class UserLike(BaseRequestModel):
|
||||
|
||||
@ -76,9 +76,6 @@ class TokenManager:
|
||||
|
||||
msToken = str(httpx.Cookies(response.cookies).get("msToken"))
|
||||
|
||||
if len(msToken) not in [148]:
|
||||
raise APIResponseError("{0} 内容不符合要求".format("msToken"))
|
||||
|
||||
return msToken
|
||||
|
||||
# except httpx.RequestError as exc:
|
||||
@ -90,7 +87,7 @@ class TokenManager:
|
||||
# except httpx.HTTPStatusError as e:
|
||||
# # 捕获 httpx 的状态代码错误 (captures specific status code errors from httpx)
|
||||
# if response.status_code == 401:
|
||||
# raise APIUnauthorizedError("参数验证失败，请更新 F2 配置文件中的 {0}，以匹配 {1} 新规则"
|
||||
# raise APIUnauthorizedError("参数验证失败，请更新 Douyin_TikTok_Download_API 配置文件中的 {0}，以匹配 {1} 新规则"
|
||||
# .format("msToken", "tiktok")
|
||||
# )
|
||||
#
|
||||
@ -104,8 +101,10 @@ class TokenManager:
|
||||
|
||||
except Exception as e:
|
||||
# 返回虚假的msToken (Return a fake msToken)
|
||||
logger.error("msToken API错误：{0}".format(e))
|
||||
logger.info("生成虚假的msToken")
|
||||
logger.error("生成TikTok msToken API错误：{0}".format(e))
|
||||
logger.info("当前网络无法正常访问TikTok服务器，已经使用虚假msToken以继续运行。")
|
||||
logger.info("并且TikTok相关API大概率无法正常使用，请在(/tiktok/web/config.yaml)中更新代理。")
|
||||
logger.info("如果你不需要使用TikTok相关API，请忽略此消息。")
|
||||
return cls.gen_false_msToken()
|
||||
|
||||
@classmethod
|
||||
@ -149,7 +148,7 @@ class TokenManager:
|
||||
except httpx.HTTPStatusError as e:
|
||||
# 捕获 httpx 的状态代码错误 (captures specific status code errors from httpx)
|
||||
if response.status_code == 401:
|
||||
raise APIUnauthorizedError("参数验证失败，请更新 F2 配置文件中的 {0}，以匹配 {1} 新规则"
|
||||
raise APIUnauthorizedError("参数验证失败，请更新 Douyin_TikTok_Download_API 配置文件中的 {0}，以匹配 {1} 新规则"
|
||||
.format("ttwid", "tiktok")
|
||||
)
|
||||
|
||||
@ -188,7 +187,7 @@ class TokenManager:
|
||||
except httpx.HTTPStatusError as e:
|
||||
# 捕获 httpx 的状态代码错误 (captures specific status code errors from httpx)
|
||||
if response.status_code == 401:
|
||||
raise APIUnauthorizedError("参数验证失败，请更新 F2 配置文件中的 {0}，以匹配 {1} 新规则"
|
||||
raise APIUnauthorizedError("参数验证失败，请更新 Douyin_TikTok_Download_API 配置文件中的 {0}，以匹配 {1} 新规则"
|
||||
.format("odin_tt", "tiktok")
|
||||
)
|
||||
|
||||
@ -431,15 +430,17 @@ class SecUserIdFetcher:
|
||||
class AwemeIdFetcher:
|
||||
# https://www.tiktok.com/@scarlettjonesuk/video/7255716763118226715
|
||||
# https://www.tiktok.com/@scarlettjonesuk/video/7255716763118226715?is_from_webapp=1&sender_device=pc&web_id=7306060721837852167
|
||||
# https://www.tiktok.com/@zoyapea5/photo/7370061866879454469
|
||||
|
||||
# 预编译正则表达式
|
||||
_TIKTOK_AWEMEID_PARREN = re.compile(r"video/(\d*)")
|
||||
_TIKTOK_NOTFOUND_PARREN = re.compile(r"notfound")
|
||||
_TIKTOK_AWEMEID_PATTERN = re.compile(r"video/(\d+)")
|
||||
_TIKTOK_PHOTOID_PATTERN = re.compile(r"photo/(\d+)")
|
||||
_TIKTOK_NOTFOUND_PATTERN = re.compile(r"notfound")
|
||||
|
||||
@classmethod
|
||||
async def get_aweme_id(cls, url: str) -> str:
|
||||
"""
|
||||
获取TikTok作品aweme_id
|
||||
获取TikTok作品aweme_id或photo_id
|
||||
Args:
|
||||
url: 作品链接
|
||||
Return:
|
||||
@ -454,11 +455,27 @@ class AwemeIdFetcher:
|
||||
url = extract_valid_urls(url)
|
||||
|
||||
if url is None:
|
||||
raise (
|
||||
APINotFoundError("输入的URL不合法。类名：{0}".format(cls.__name__))
|
||||
)
|
||||
raise APINotFoundError("输入的URL不合法。类名：{0}".format(cls.__name__))
|
||||
|
||||
transport = httpx.AsyncHTTPTransport(retries=5)
|
||||
# 处理不是短连接的情况
|
||||
if "tiktok" and "@" in url:
|
||||
print(f"输入的URL无需重定向: {url}")
|
||||
video_match = cls._TIKTOK_AWEMEID_PATTERN.search(url)
|
||||
photo_match = cls._TIKTOK_PHOTOID_PATTERN.search(url)
|
||||
|
||||
if not video_match and not photo_match:
|
||||
raise APIResponseError("未在响应中找到 aweme_id 或 photo_id")
|
||||
|
||||
aweme_id = video_match.group(1) if video_match else photo_match.group(1)
|
||||
|
||||
if aweme_id is None:
|
||||
raise RuntimeError("获取 aweme_id 或 photo_id 失败，{0}".format(url))
|
||||
|
||||
return aweme_id
|
||||
|
||||
# 处理短连接的情况，根据重定向后的链接获取aweme_id
|
||||
print(f"输入的URL需要重定向: {url}")
|
||||
transport = httpx.AsyncHTTPTransport(retries=10)
|
||||
async with httpx.AsyncClient(
|
||||
transport=transport, proxies=TokenManager.proxies, timeout=10
|
||||
) as client:
|
||||
@ -466,32 +483,28 @@ class AwemeIdFetcher:
|
||||
response = await client.get(url, follow_redirects=True)
|
||||
|
||||
if response.status_code in {200, 444}:
|
||||
if cls._TIKTOK_NOTFOUND_PARREN.search(str(response.url)):
|
||||
if cls._TIKTOK_NOTFOUND_PATTERN.search(str(response.url)):
|
||||
raise APINotFoundError("页面不可用，可能是由于区域限制（代理）造成的。类名: {0}"
|
||||
.format(cls.__name__)
|
||||
)
|
||||
|
||||
match = cls._TIKTOK_AWEMEID_PARREN.search(str(response.url))
|
||||
if not match:
|
||||
raise APIResponseError(
|
||||
"未在响应中找到 {0}".format("aweme_id")
|
||||
)
|
||||
video_match = cls._TIKTOK_AWEMEID_PATTERN.search(str(response.url))
|
||||
photo_match = cls._TIKTOK_PHOTOID_PATTERN.search(str(response.url))
|
||||
|
||||
aweme_id = match.group(1)
|
||||
if not video_match and not photo_match:
|
||||
raise APIResponseError("未在响应中找到 aweme_id 或 photo_id")
|
||||
|
||||
aweme_id = video_match.group(1) if video_match else photo_match.group(1)
|
||||
|
||||
if aweme_id is None:
|
||||
raise RuntimeError(
|
||||
"获取 {0} 失败，{1}".format("aweme_id", response.url)
|
||||
)
|
||||
raise RuntimeError("获取 aweme_id 或 photo_id 失败，{0}".format(response.url))
|
||||
|
||||
return aweme_id
|
||||
else:
|
||||
raise ConnectionError(
|
||||
"接口状态码异常 {0}，请检查重试".format(response.status_code)
|
||||
)
|
||||
raise ConnectionError("接口状态码异常 {0}，请检查重试".format(response.status_code))
|
||||
|
||||
except httpx.RequestError as exc:
|
||||
# 捕获所有与 httpx 请求相关的异常情况 (Captures all httpx request-related exceptions)
|
||||
# 捕获所有与 httpx 请求相关的异常情况
|
||||
raise APIConnectionError("请求端点失败，请检查当前网络环境。 链接：{0}，代理：{1}，异常类名：{2}，异常详细信息：{3}"
|
||||
.format(url, TokenManager.proxies, cls.__name__, exc)
|
||||
)
|
||||
|
||||
@ -89,7 +89,8 @@ class TikTokWebCrawler:
|
||||
"Referer": tiktok_config["headers"]["Referer"],
|
||||
"Cookie": tiktok_config["headers"]["Cookie"],
|
||||
},
|
||||
"proxies": {"http://": None, "https://": None},
|
||||
"proxies": {"http://": tiktok_config["proxies"]["http"],
|
||||
"https://": tiktok_config["proxies"]["https"]}
|
||||
}
|
||||
return kwargs
|
||||
|
||||
@ -133,7 +134,7 @@ class TikTokWebCrawler:
|
||||
kwargs = await self.get_tiktok_headers()
|
||||
# proxies = {"http://": 'http://43.159.29.191:24144', "https://": 'http://43.159.29.191:24144'}
|
||||
# 创建一个基础爬虫
|
||||
base_crawler = BaseCrawler(proxies=None, crawler_headers=kwargs["headers"])
|
||||
base_crawler = BaseCrawler(proxies=kwargs["proxies"], crawler_headers=kwargs["headers"])
|
||||
async with base_crawler as crawler:
|
||||
# 创建一个用户作品的BaseModel参数
|
||||
params = UserPost(secUid=secUid, cursor=cursor, count=count, coverFormat=coverFormat)
|
||||
@ -216,7 +217,7 @@ class TikTokWebCrawler:
|
||||
kwargs = await self.get_tiktok_headers()
|
||||
# proxies = {"http://": 'http://43.159.18.174:25263', "https://": 'http://43.159.18.174:25263'}
|
||||
# 创建一个基础爬虫
|
||||
base_crawler = BaseCrawler(proxies=None, crawler_headers=kwargs["headers"])
|
||||
base_crawler = BaseCrawler(proxies=kwargs["proxies"], crawler_headers=kwargs["headers"])
|
||||
async with base_crawler as crawler:
|
||||
# 创建一个作品评论的BaseModel参数
|
||||
params = PostComment(aweme_id=aweme_id, cursor=cursor, count=count, current_region=current_region)
|
||||
@ -343,7 +344,7 @@ class TikTokWebCrawler:
|
||||
|
||||
async def main(self):
|
||||
# 获取单个作品数据
|
||||
# item_id = "7339393672959757570"
|
||||
# item_id = "7369296852669205791"
|
||||
# response = await self.fetch_one_video(item_id)
|
||||
# print(response)
|
||||
|
||||
|
||||
18
crawlers/utils/deprecated.py
Normal file
18
crawlers/utils/deprecated.py
Normal file
@ -0,0 +1,18 @@
|
||||
import warnings
|
||||
import functools
|
||||
|
||||
|
||||
def deprecated(message):
|
||||
def decorator(func):
|
||||
@functools.wraps(func)
|
||||
async def wrapper(*args, **kwargs):
|
||||
warnings.warn(
|
||||
f"{func.__name__} is deprecated: {message}",
|
||||
DeprecationWarning,
|
||||
stacklevel=2
|
||||
)
|
||||
return await func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
@ -79,7 +79,7 @@ class LogManager(metaclass=Singleton):
|
||||
if getattr(self, "_initialized", False): # 防止重复初始化
|
||||
return
|
||||
|
||||
self.logger = logging.getLogger("TikHub_Crawlers")
|
||||
self.logger = logging.getLogger("Douyin_TikTok_Download_API_Crawlers")
|
||||
self.logger.setLevel(logging.INFO)
|
||||
self.log_dir = None
|
||||
self._initialized = True
|
||||
@ -145,7 +145,7 @@ class LogManager(metaclass=Singleton):
|
||||
|
||||
|
||||
def log_setup(log_to_console=True):
|
||||
logger = logging.getLogger("TikHub_Crawlers")
|
||||
logger = logging.getLogger("Douyin_TikTok_Download_API_Crawlers")
|
||||
if logger.hasHandlers():
|
||||
# logger已经被设置，不做任何操作
|
||||
return logger
|
||||
|
||||
@ -7,9 +7,9 @@ services: # 定义服务列表
|
||||
container_name: douyin_tiktok_download_api # 容器名称
|
||||
restart: always # 容器退出后总是重启
|
||||
volumes: # 挂载卷配置
|
||||
- ./douyin_tiktok_download_api/douyin_web/config.yaml:/crawlers/douyin/web/config.yaml
|
||||
- ./douyin_tiktok_download_api/tiktok_web/config.yaml:/crawlers/tiktok/web/config.yaml
|
||||
- ./douyin_tiktok_download_api/tiktok_app/config.yaml:/crawlers/tiktok/app/config.yaml
|
||||
- ./douyin_tiktok_download_api/douyin_web/config.yaml:/app/crawlers/douyin/web/config.yaml
|
||||
- ./douyin_tiktok_download_api/tiktok_web/config.yaml:/app/crawlers/tiktok/web/config.yaml
|
||||
- ./douyin_tiktok_download_api/tiktok_app/config.yaml:/app/crawlers/tiktok/app/config.yaml
|
||||
environment: # 环境变量配置
|
||||
TZ: Asia/Shanghai # 设置时区为亚洲/上海
|
||||
PUID: 1026 # 设置容器内部的用户 ID
|
||||
|
||||
@ -14,7 +14,7 @@ importlib_resources==6.4.0
|
||||
lz4==4.3.3
|
||||
markdown-it-py==3.0.0
|
||||
mdurl==0.1.2
|
||||
numpy==1.24.4
|
||||
numpy
|
||||
pycryptodomex==3.20.0
|
||||
pydantic==2.7.0
|
||||
pydantic_core==2.18.1
|
||||
@ -34,3 +34,5 @@ ua-parser==0.18.0
|
||||
user-agents==2.2.0
|
||||
uvicorn==0.29.0
|
||||
websockets==12.0
|
||||
gmssl==3.2.2
|
||||
tenacity~=9.0.0
|
||||
11
start.sh
11
start.sh
@ -1,11 +1,4 @@
|
||||
#!/bin/sh
|
||||
|
||||
# Activating the virtual environment
|
||||
# shellcheck disable=SC2039
|
||||
source /www/wwwroot/Douyin_TikTok_Download_API/venv/bin/activate
|
||||
|
||||
# Starting the Python application
|
||||
python start.py
|
||||
|
||||
# Deactivating the virtual environment (optional, since the script is ending)
|
||||
deactivate
|
||||
# Starting the Python application directly using python3
|
||||
python3 start.py
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user