spider/EasySpider
1
0
Fork 0
mirror of https://github.com/NaiboWang/EasySpider.git synced 2025-04-19 10:19:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

Two new custom operations

Browse Source
This commit is contained in:
naibo 2023-12-30 16:05:29 +08:00
parent 2a241010a9
commit c5a4b11dfb
9 changed files with 44 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.temp_to_pub/Note - The various folders within this directory are not directly usable software.txt
View File

@ -1 +1 @@
Note: The various folders within this directory are not directly usable software, but temporary folders used by the author at the time of release. Please visit the official website to download readily usable software packages: https://www.easyspider.cn
Note: The various folders within this directory are not directly usable software, but temporary folders used by the author at the time of release. Please visit the official website to download readily usable software packages: https://www.easyspider.net

5
ElectronJS/main.js
View File

@ -913,10 +913,7 @@ async function beginInvoke(msg, ws) {
xpath = parent_xpath + xpath;
}
let elementInfo = {iframe: param.iframe, xpath: xpath, id: -1};
let element = await findElementAcrossAllWindows(
elementInfo,
(notifyBrowser = false)
);
let element = await findElementAcrossAllWindows(elementInfo);
if (element != null) {
await execute_js(param.beforeJS, element, param.beforeJSWaitTime);
if (param.contentType == 0) {

11
ElectronJS/src/taskGrid/FlowChart.html
View File

@ -396,6 +396,8 @@
<option :value = 7>Pause program execution (such as when the captcha box appears)</option>
<option :value = 8>Refresh page</option>
<option :value = 9>Send Email</option>
<option :value = 10>Clear all field values</option>
<option :value = 11>Generate new data row</option>
</select>
<div v-if='nowNode["parameters"]["codeMode"] < 3 || nowNode["parameters"]["codeMode"] >= 5 && nowNode["parameters"]["codeMode"] <=6'>
<label>Code (Use Field["FieldName"] to input the lastest value of a field): </label>
@ -486,7 +488,12 @@ Please note that this feature does not support assigning values to variables. In
<label>Email content:</label>
<textarea spellcheck=false onkeydown="inputDelete(event)" class="form-control" rows="2" v-model='nowNode["parameters"]["emailConfig"]["content"]' placeholder="Write the email content here"></textarea>
</div>
<div v-if='nowNode["parameters"]["codeMode"] == 10'>
<label>This action can clear all field values, such as when used before starting a web scraping task to clear all values.</label>
</div>
<div v-if='nowNode["parameters"]["codeMode"] == 11'>
<label>This action can generate a new row of data, such as when designing a web scraping task to not generate rows of data temporarily, and instead generate a new row of data once all fields have been extracted.</label>
</div>
</div>
<div class="elements" v-if="nodeType==6">
@ -708,8 +715,8 @@ If the expression returns a value greater than 0 or evaluates to True, the opera
<input spellcheck=false onkeydown="inputDelete(event)" id="serviceDescription" name="serviceDescription" class="form-control"></input>
<label>Export Data Format (Excel/CSV/TXT/Database):</label>
<select id="outputFormat" class="form-control">
<option value="xlsx">XLSX (Excel file, recommended use CSV format when single cell exceeds 500 characters)</option>
<option value="csv">CSV (Recommended for collecting long articles)</option>
<option value="xlsx">XLSX (Excel file, recommended use CSV format when single cell exceeds 500 characters)</option>
<option value="txt">TXT</option>
<option value="json">JSON</option>
<option value="mysql">MySQL Database (recommended for large amounts of data)</option>

6
ElectronJS/src/taskGrid/FlowChart.js
View File

@ -198,6 +198,12 @@ let app = new Vue({
case 9:
this.nowNode["title"] = LANG("发送邮件", "Send Email");
break;
case 10:
this.nowNode["title"] = LANG("清空字段值", "Clear Field Value");
break;
case 11:
this.nowNode["title"] = LANG("生成新行", "Generate New Row");
break;
case -1: // 跳转到其他操作时，不改变标题
break;
default: // 默认情况

11
ElectronJS/src/taskGrid/FlowChart_CN.html
View File

@ -396,6 +396,8 @@
<option :value = 7>暂停程序执行（如检测到验证码框出现时暂停执行）</option>
<option :value = 8>刷新页面</option>
<option :value = 9>发送邮件</option>
<option :value = 10>清空所有字段值</option>
<option :value = 11>生成新数据行</option>
</select>
<div v-if='nowNode["parameters"]["codeMode"] < 3 || nowNode["parameters"]["codeMode"] >= 5 && nowNode["parameters"]["codeMode"] <=6'>
<label>代码/脚本内容（用Field["字段名"]来输入某字段/自定义操作的最新提取/返回值）： </label>
@ -486,7 +488,12 @@ print(emotlib.emoji()) # 使用其中的函数。
<label>邮件内容：</label>
<textarea spellcheck=false onkeydown="inputDelete(event)" class="form-control" rows="2" v-model='nowNode["parameters"]["emailConfig"]["content"]' placeholder="这里写邮件内容"></textarea>
</div>
<div v-if='nowNode["parameters"]["codeMode"] == 10'>
<label>此操作可以清空所有字段值，如用于爬虫任务开始前清空所有字段值。</label>
</div>
<div v-if='nowNode["parameters"]["codeMode"] == 11'>
<label>此操作可以生成新数据行，如用于爬虫任务设计时暂不生成数据行，等所有字段提取结束后统一生成新数据行。</label>
</div>
</div>
<div class="elements" v-if="nodeType==6">
@ -708,8 +715,8 @@ print(emotlib.emoji()) # 使用其中的函数。
<input spellcheck=false onkeydown="inputDelete(event)" id="serviceDescription" name="serviceDescription" class="form-control"></input>
<label>导出数据格式（Excel/CSV/TXT/数据库，<a href="https://www.bilibili.com/video/BV1os4y1679S/" target="_blank">查看MySQL操作教程</a>）：</label>
<select id="outputFormat" class="form-control">
<option value = "xlsx">XLSX（即EXCEL文件，建议单个单元格长度超过500时使用CSV格式存储）</option>
<option value = "csv">CSV（采集长文章推荐使用此格式）</option>
<option value = "xlsx">XLSX（即EXCEL文件，建议单个单元格长度超过500时使用CSV格式存储）</option>
<option value = "txt">TXT</option>
<option value = "json">JSON</option>
<option value = "mysql">MySQL数据库（大量数据推荐使用）</option>

1
ElectronJS/tasks/314.json Normal file
View File

File diff suppressed because one or more lines are too long

2
ExecuteStage/.vscode/launch.json vendored
View File

@ -12,7 +12,7 @@
"justMyCode": false,
// "args": ["--ids", "[7]", "--read_type", "remote", "--headless", "0"]
// "args": ["--ids", "[9]", "--read_type", "remote", "--headless", "0", "--saved_file_name", "YOUTUBE"]
"args": ["--ids", "[13]", "--headless", "0", "--user_data", "0", "--keyboard", "0",
"args": ["--ids", "[70]", "--headless", "0", "--user_data", "0", "--keyboard", "0",
"--read_type", "remote"]
// "args": "--ids '[97]' --user_data 1 --server_address http://localhost:8074 --config_folder '/Users/naibo/Documents/EasySpider/ElectronJS/' --headless 0 --read_type remote --config_file_name config.json --saved_file_name"
}

6
ExecuteStage/easyspider_executestage.py
View File

@ -857,6 +857,12 @@ class BrowserThread(Thread):
self.print_and_log("根据设置的自定义操作，任务已刷新页面|Task refreshed page according to custom operation")
elif codeMode == 9: # 发送邮件
send_email(node["parameters"]["emailConfig"])
elif codeMode == 10: # 清空所有字段值
self.clearOutputParameters()
elif codeMode == 11: # 生成新的数据行
line = new_line(self.outputParameters,
self.maxViewLength, self.outputParametersRecord)
self.OUTPUT.append(line)
else: # 0 1 5 6
output = self.execute_code(
codeMode, code, max_wait_time, iframe=params["iframe"])

13
ExecuteStage/utils.py
View File

@ -539,10 +539,17 @@ def write_to_excel(file_name, data, types, record):
for i in range(len(line)):
if record[i]:
to_write.append(line[i])
ws.append(to_write)
try:
ws.append(to_write)
except:
print("写入Excel文件失败，请检查数据类型是否正确。")
print("Failed to write to Excel file, please check if the data type is correct.")
# 保存工作簿
wb.save(file_name)
try:
wb.save(file_name)
except:
print("保存Excel文件失败，请检查文件是否被其他程序打开。")
print("Failed to save Excel file, please check if the file is opened by other programs.")
class Time:
def __init__(self, type1=""):