From 7d467ee2694ac5304972cfbb6fc9440679f5958d Mon Sep 17 00:00:00 2001 From: Evil0ctal Date: Wed, 25 Sep 2024 20:30:42 -0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A7:=20Update=20to=20V4.0.9=20->=20#48?= =?UTF-8?q?1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.yaml | 4 +- crawlers/bilibili/web/web_crawler.py | 57 ++++++++++++++++++++++------ 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/config.yaml b/config.yaml index ad27ea1..2b54550 100644 --- a/config.yaml +++ b/config.yaml @@ -30,8 +30,8 @@ API: Redoc_URL: /redoc # API documentation URL | API文档URL # API Information - Version: V4.0.8 # API version | API版本 - Update_Time: 2024/09/16 # API update time | API更新时间 + Version: V4.0.9 # API version | API版本 + Update_Time: 2024/09/25 # API update time | API更新时间 Environment: Demo # API environment | API环境 # Download Configuration diff --git a/crawlers/bilibili/web/web_crawler.py b/crawlers/bilibili/web/web_crawler.py index f4fb0ed..0685455 100644 --- a/crawlers/bilibili/web/web_crawler.py +++ b/crawlers/bilibili/web/web_crawler.py @@ -1,3 +1,37 @@ +# ============================================================================== +# Copyright (C) 2021 Evil0ctal +# +# This file is part of the Douyin_TikTok_Download_API project. +# +# This project is licensed under the Apache License 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +#         __ +#        />  フ +#       |  _  _ l +#       /` ミ_xノ +#      /      | Feed me Stars ⭐ ️ +#     /  ヽ   ノ +#     │  | | | +#  / ̄|   | | | +#  | ( ̄ヽ__ヽ_)__) +#  \二つ +# ============================================================================== +# +# Contributor Link: +# +# - https://github.com/Koyomi781 +# +# ============================================================================== + import asyncio # 异步I/O import os # 系统操作 import time # 时间操作 @@ -11,7 +45,6 @@ from crawlers.bilibili.web.utils import EndpointGenerator, bv2av, ResponseAnalyz # 数据请求模型 from crawlers.bilibili.web.models import UserPostVideos, UserProfile, ComPopular, UserDynamic, PlayUrl - # 配置文件路径 path = os.path.abspath(os.path.dirname(__file__)) @@ -27,17 +60,18 @@ class BilibiliWebCrawler: bili_config = config['TokenManager']['bilibili'] kwargs = { "headers": { - "accept-language": bili_config["headers"]["accept-language"], - "origin": bili_config["headers"]["origin"], - "referer": bili_config["headers"]["referer"], - "user-agent": bili_config["headers"]["user-agent"], - "cookie": bili_config["headers"]["cookie"], + "accept-language": bili_config["headers"]["accept-language"], + "origin": bili_config["headers"]["origin"], + "referer": bili_config["headers"]["referer"], + "user-agent": bili_config["headers"]["user-agent"], + "cookie": bili_config["headers"]["cookie"], }, "proxies": {"http://": bili_config["proxies"]["http"], "https://": bili_config["proxies"]["https"]}, } return kwargs "-------------------------------------------------------handler接口列表-------------------------------------------------------" + # 获取单个视频详情信息 async def fetch_one_video(self, bv_id: str) -> dict: # 获取请求头信息 @@ -116,8 +150,8 @@ class BilibiliWebCrawler: base_crawler = BaseCrawler(proxies=kwargs["proxies"], crawler_headers=kwargs["headers"]) # 发送请求,获取请求响应结果 async with base_crawler as crawler: - endpoint = f"{BilibiliAPIEndpoints.COLLECT_VIDEOS}?media_id={folder_id}&pn={pn}&ps=20&keyword=&order=mtime&type=0&tid=0&platform=web" - response = await crawler.fetch_get_json(endpoint) + endpoint = f"{BilibiliAPIEndpoints.COLLECT_VIDEOS}?media_id={folder_id}&pn={pn}&ps=20&keyword=&order=mtime&type=0&tid=0&platform=web" + response = await crawler.fetch_get_json(endpoint) return response # 获取指定用户的信息 @@ -256,6 +290,7 @@ class BilibiliWebCrawler: return response "-------------------------------------------------------utils接口列表-------------------------------------------------------" + # 通过bv号获得视频aid号 async def bv_to_aid(self, bv_id: str) -> int: aid = await bv2av(bv_id=bv_id) @@ -288,9 +323,10 @@ class BilibiliWebCrawler: return response "-------------------------------------------------------main-------------------------------------------------------" - async def main(self): - "-------------------------------------------------------handler接口列表-------------------------------------------------------" + async def main(self): + """-------------------------------------------------------handler接口列表-------------------------------------------------------""" + # 获取单个作品数据 # bv_id = 'BV1M1421t7hT' # result = await self.fetch_one_video(bv_id=bv_id) @@ -385,7 +421,6 @@ class BilibiliWebCrawler: # print(result) - if __name__ == '__main__': # 初始化 BilibiliWebCrawler = BilibiliWebCrawler()