spider/EasySpider
1
0
Fork 0
mirror of https://github.com/NaiboWang/EasySpider.git synced 2025-04-22 22:10:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

Logic for handles of text list

Browse Source
This commit is contained in:
naibo 2023-12-30 23:10:31 +08:00
parent c5a4b11dfb
commit 7d247d68ec
10 changed files with 48 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ElectronJS/src/taskGrid/FlowChart.html
View File

@ -394,6 +394,7 @@
<option :value = 5>Run Python code on current environment (the "exec" operation)</option>
<option :value = 6>Get value of a Python expression (the "eval" operation)</option>
<option :value = 7>Pause program execution (such as when the captcha box appears)</option>
<option :value = 12>Exit Program</option>
<option :value = 8>Refresh page</option>
<option :value = 9>Send Email</option>
<option :value = 10>Clear all field values</option>

3
ElectronJS/src/taskGrid/FlowChart.js
View File

@ -204,6 +204,9 @@ let app = new Vue({
case 11:
this.nowNode["title"] = LANG("生成新行", "Generate New Row");
break;
case 12:
this.nowNode["title"] = LANG("退出程序", "Exit Program");
break;
case -1: // 跳转到其他操作时，不改变标题
break;
default: // 默认情况

1
ElectronJS/src/taskGrid/FlowChart_CN.html
View File

@ -394,6 +394,7 @@
<option :value = 5>在执行环境下运行Python代码（exec操作）</option>
<option :value = 6>在执行环境下获得Python表达式值（eval操作）</option>
<option :value = 7>暂停程序执行（如检测到验证码框出现时暂停执行）</option>
<option :value = 12>退出程序</option>
<option :value = 8>刷新页面</option>
<option :value = 9>发送邮件</option>
<option :value = 10>清空所有字段值</option>

2
ElectronJS/src/taskGrid/executeTask.html
View File

@ -69,7 +69,7 @@
<div class="row" style="margin-top: 40px;">
<div class="col-md-7" id="taskInfo" style="margin:0 auto" v-if="show">
<div class="col-md-8" id="taskInfo" style="margin:0 auto" v-if="show">
<div id="tipCustom" class="alert alert-success alert-dismissible fade show"
style="display: none; z-index: 1000">
{{tip | lang}}

2
ElectronJS/src/taskGrid/newTask.html
View File

@ -23,7 +23,7 @@
<body>
<div class="row" style="margin-top: 40px" id="newTask">
<div class="col-md-6" style="margin:0 auto;" style="text-align: center;">
<div class="col-md-8" style="margin:0 auto;" style="text-align: center;">
<nav aria-label="breadcrumb">
<ol class="breadcrumb" style="padding-left:0;background-color: white">
<li class="breadcrumb-item" @click="gotoHome"><a href="#">{{"Home~首页" | lang}}</a></li>

2
ElectronJS/src/taskGrid/taskInfo.html
View File

@ -37,7 +37,7 @@
<div class="row" style="margin-top: 40px;">
<div class="col-md-7" style="margin:0 auto" id="taskInfo" v-if="show">
<div class="col-md-8" style="margin:0 auto" id="taskInfo" v-if="show">
<nav aria-label="breadcrumb">
<ol class="breadcrumb" style="padding-left:0;background-color: white">
<li class="breadcrumb-item" @click="gotoHome"><a href="#">{{"Home~首页" | lang}}</a></li>

1
ElectronJS/tasks/313.json Normal file
View File

File diff suppressed because one or more lines are too long

1
ElectronJS/tasks/316.json Normal file
View File

@ -0,0 +1 @@
{"id":316,"name":"京东全球版-专业的综合网上购物商城","url":"https://www.jd.com","links":"https://www.jd.com","create_time":"2023-12-30 22:35:04","update_time":"2023-12-30 22:35:12","version":"0.6.0","saveThreshold":10,"quitWaitTime":60,"environment":0,"maximizeWindow":0,"maxViewLength":15,"recordLog":1,"outputFormat":"csv","saveName":"current_time","dataWriteMode":1,"inputExcel":"","startFromExit":0,"pauseKey":"p","containJudge":false,"browser":"chrome","removeDuplicate":0,"desc":"https://www.jd.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.jd.com","desc":"要采集的网址列表，多行以\\n分开","type":"text","exampleValue":"https://www.jd.com"}],"outputParameters":[{"id":0,"name":"自定义操作","desc":"自定义操作返回的数据","type":"text","recordASField":0,"exampleValue":""}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"url":"https://www.jd.com","links":"https://www.jd.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":2,"index":2,"parentId":0,"type":0,"option":5,"title":"自定义操作","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"clear":0,"newLine":1,"codeMode":12,"code":"","waitTime":0,"recordASField":0,"paraType":"text","emailConfig":{"host":"","port":465,"username":"","password":"","from":"","to":"","subject":"","content":""}}}]}

2
ExecuteStage/.vscode/launch.json vendored
View File

@ -12,7 +12,7 @@
"justMyCode": false,
// "args": ["--ids", "[7]", "--read_type", "remote", "--headless", "0"]
// "args": ["--ids", "[9]", "--read_type", "remote", "--headless", "0", "--saved_file_name", "YOUTUBE"]
"args": ["--ids", "[70]", "--headless", "0", "--user_data", "0", "--keyboard", "0",
"args": ["--ids", "[72]", "--headless", "0", "--user_data", "0", "--keyboard", "0",
"--read_type", "remote"]
// "args": "--ids '[97]' --user_data 1 --server_address http://localhost:8074 --config_folder '/Users/naibo/Documents/EasySpider/ElectronJS/' --headless 0 --read_type remote --config_file_name config.json --saved_file_name"
}

43
ExecuteStage/easyspider_executestage.py
View File

@ -863,6 +863,17 @@ class BrowserThread(Thread):
line = new_line(self.outputParameters,
self.maxViewLength, self.outputParametersRecord)
self.OUTPUT.append(line)
elif codeMode == 12: # 退出程序
self.print_and_log("根据设置的自定义操作，任务已退出|Task exited according to custom operation")
self.saveData(exit=True)
self.browser.quit()
self.print_and_log("正在清理临时用户目录……|Cleaning up temporary user directory...")
try:
shutil.rmtree(self.option["tmp_user_data_folder"])
except:
pass
self.print_and_log("清理完成！|Clean up completed!")
os._exit(0)
else: # 0 1 5 6
output = self.execute_code(
codeMode, code, max_wait_time, iframe=params["iframe"])
@ -1148,12 +1159,13 @@ class BrowserThread(Thread):
if self.browser.current_url == thisHistoryURL or ti > thisHistoryLength: # 如果执行完一次循环之后网址发生了变化
break
time.sleep(2)
if element == None: # 不固定元素列表
element = self.browser.find_elements(By.XPATH, xpath, iframe=node["parameters"]["iframe"])
else: # 固定元素列表
element = self.browser.find_element(By.XPATH, xpath, iframe=node["parameters"]["iframe"])
# if index > 0:
# index -= 1 # 如果是data:开头的网址，就要重试一次
if xpath != "":
if element == None: # 不固定元素列表
element = self.browser.find_elements(By.XPATH, xpath, iframe=node["parameters"]["iframe"])
else: # 固定元素列表
element = self.browser.find_element(By.XPATH, xpath, iframe=node["parameters"]["iframe"])
# if index > 0:
# index -= 1 # 如果是data:开头的网址，就要重试一次
else:
if element == None:
element = elements
@ -1463,6 +1475,25 @@ class BrowserThread(Thread):
code = get_output_code(output)
if code <= 0:
break
try:
changed_handle = self.browser.current_window_handle != thisHandle
except: # 如果网页被意外关闭了的情况下
self.browser.switch_to.window(
self.browser.window_handles[-1])
changed_handle = self.browser.window_handles[-1] != thisHandle
if changed_handle: # 如果执行完一次循环之后标签页的位置发生了变化
try:
while True: # 一直关闭窗口直到当前标签页
self.browser.close() # 关闭使用完的标签页
self.browser.switch_to.window(
self.browser.window_handles[-1])
if self.browser.current_window_handle == thisHandle:
break
except Exception as e:
self.print_and_log("关闭标签页发生错误：", e)
self.print_and_log(
"Error occurred while closing tab: ", e)
index, _ = self.handleHistory(node, "", thisHistoryURL, thisHistoryLength, index)
elif int(node["parameters"]["loopType"]) == 4: # 固定网址列表
# tempList = node["parameters"]["textList"].split("\r\n")
urlList = list(