spider/Douyin_TikTok_Download_API
1
0
Fork 0
mirror of https://github.com/Evil0ctal/Douyin_TikTok_Download_API.git synced 2025-04-12 03:37:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix tiktok crawlers to respect proxies from config.yaml

Browse Source
This commit is contained in:
oho 2024-11-17 17:35:38 +08:00
parent b659af881b
commit d641f0340c
2 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
crawlers/tiktok/app/app_crawler.py
View File

@ -74,7 +74,8 @@ class TikTokAPPCrawler:
"Cookie": tiktok_config["headers"]["Cookie"],
"x-ladon": "Hello From Evil0ctal!",
},
"proxies": {"http://": None, "https://": None},
"proxies": {"http://": tiktok_config["proxies"]["http"],
"https://": tiktok_config["proxies"]["https"]}
}
return kwargs

7
crawlers/tiktok/web/web_crawler.py
View File

@ -89,7 +89,8 @@ class TikTokWebCrawler:
"Referer": tiktok_config["headers"]["Referer"],
"Cookie": tiktok_config["headers"]["Cookie"],
},
"proxies": {"http://": None, "https://": None},
"proxies": {"http://": tiktok_config["proxies"]["http"],
"https://": tiktok_config["proxies"]["https"]}
}
return kwargs
@ -133,7 +134,7 @@ class TikTokWebCrawler:
kwargs = await self.get_tiktok_headers()
# proxies = {"http://": 'http://43.159.29.191:24144', "https://": 'http://43.159.29.191:24144'}
# 创建一个基础爬虫
base_crawler = BaseCrawler(proxies=None, crawler_headers=kwargs["headers"])
base_crawler = BaseCrawler(proxies=kwargs["proxies"], crawler_headers=kwargs["headers"])
async with base_crawler as crawler:
# 创建一个用户作品的BaseModel参数
params = UserPost(secUid=secUid, cursor=cursor, count=count, coverFormat=coverFormat)
@ -216,7 +217,7 @@ class TikTokWebCrawler:
kwargs = await self.get_tiktok_headers()
# proxies = {"http://": 'http://43.159.18.174:25263', "https://": 'http://43.159.18.174:25263'}
# 创建一个基础爬虫
base_crawler = BaseCrawler(proxies=None, crawler_headers=kwargs["headers"])
base_crawler = BaseCrawler(proxies=kwargs["proxies"], crawler_headers=kwargs["headers"])
async with base_crawler as crawler:
# 创建一个作品评论的BaseModel参数
params = PostComment(aweme_id=aweme_id, cursor=cursor, count=count, current_region=current_region)