spider/EasySpider
1
0
Fork 0
mirror of https://github.com/NaiboWang/EasySpider.git synced 2025-04-21 12:15:16 +08:00
Code Issues Packages Projects Releases Wiki Activity

Deal with data:

Browse Source
This commit is contained in:
naibo 2023-11-23 06:33:18 +08:00
parent 4025e255a0
commit c197ff1970
4 changed files with 34 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.temp_to_pub/EasySpider_windows_x64/execution_instances/135.json Normal file
View File

File diff suppressed because one or more lines are too long

1
.temp_to_pub/EasySpider_windows_x64/execution_instances/136.json Normal file
View File

File diff suppressed because one or more lines are too long

1
.temp_to_pub/EasySpider_windows_x64/tasks/236.json Normal file
View File

File diff suppressed because one or more lines are too long

41
ExecuteStage/easyspider_executestage.py
View File

@ -894,6 +894,7 @@ class BrowserThread(Thread):
thisHistoryLength = 0 thisHistoryLength = 0
self.history["index"] = thisHistoryLength self.history["index"] = thisHistoryLength
self.history["handle"] = thisHandle self.history["handle"] = thisHandle
thisHitoryURL = self.browser.current_url
if int(node["parameters"]["loopType"]) == 0: # 单个元素循环 if int(node["parameters"]["loopType"]) == 0: # 单个元素循环
# 无跳转标签页操作 # 无跳转标签页操作
count = 0 # 执行次数 count = 0 # 执行次数
@ -1033,14 +1034,19 @@ class BrowserThread(Thread):
self.browser.execute_script('window.stop()') self.browser.execute_script('window.stop()')
except: except:
pass pass
ti = 0
if self.browser.current_url.startswith("data:"): if self.browser.current_url.startswith("data:"):
try: while self.browser.current_url != thisHitoryURL: # 如果执行完一次循环之后网址发生了变化
self.browser.execute_script("history.go(1)") # 如果是data:开头的网址，就前进一步 try:
except: # 超时的情况下 self.browser.execute_script("history.go(1)") # 如果是data:开头的网址，就前进一步
pass except: # 超时的情况下
pass
ti += 1
if self.browser.current_url == thisHitoryURL or ti > thisHistoryLength: # 如果执行完一次循环之后网址发生了变化
break
time.sleep(2) time.sleep(2)
elements = self.browser.find_elements(By.XPATH, elements = self.browser.find_elements(By.XPATH,
xpath, iframe=node["parameters"]["iframe"]) xpath, iframe=node["parameters"]["iframe"])
if index > 0: if index > 0:
index -= 1 # 如果是data:开头的网址，就要重试一次 index -= 1 # 如果是data:开头的网址，就要重试一次
if int(node["parameters"]["breakMode"]) > 0: # 如果设置了退出循环的脚本条件 if int(node["parameters"]["breakMode"]) > 0: # 如果设置了退出循环的脚本条件
@ -1110,14 +1116,29 @@ class BrowserThread(Thread):
self.browser.execute_script('window.stop()') self.browser.execute_script('window.stop()')
except: except:
pass pass
# if self.browser.current_url.startswith("data:"):
# try:
# self.browser.execute_script("history.go(1)") # 如果是data:开头的网址，就前进一步
# except: # 超时的情况下
# pass
# time.sleep(2)
# elements = self.browser.find_elements(By.XPATH,
# xpath, iframe=node["parameters"]["iframe"])
# if index > 0:
# index -= 1 # 如果是data:开头的网址，就要重试一次
ti = 0
if self.browser.current_url.startswith("data:"): if self.browser.current_url.startswith("data:"):
try: while self.browser.current_url != thisHitoryURL: # 如果执行完一次循环之后网址发生了变化
self.browser.execute_script("history.go(1)") # 如果是data:开头的网址，就前进一步 try:
except: # 超时的情况下 self.browser.execute_script("history.go(1)") # 如果是data:开头的网址，就前进一步
pass except: # 超时的情况下
pass
ti += 1
if self.browser.current_url == thisHitoryURL or ti > thisHistoryLength: # 如果执行完一次循环之后网址发生了变化
break
time.sleep(2) time.sleep(2)
elements = self.browser.find_elements(By.XPATH, elements = self.browser.find_elements(By.XPATH,
xpath, iframe=node["parameters"]["iframe"]) xpath, iframe=node["parameters"]["iframe"])
if index > 0: if index > 0:
index -= 1 # 如果是data:开头的网址，就要重试一次 index -= 1 # 如果是data:开头的网址，就要重试一次
except NoSuchElementException: except NoSuchElementException: