From c3bab575f47a8c2354917783564afc8927934763 Mon Sep 17 00:00:00 2001 From: naibo Date: Tue, 12 Dec 2023 01:30:13 +0800 Subject: [PATCH] Bug Fix and Picture download update --- .../execution_instances/6.json | 1 + .../EasySpider_windows_x64/tasks/255.json | 2 +- ElectronJS/main.js | 20 +++-- ElectronJS/src/taskGrid/FlowChart.css | 1 + ElectronJS/src/taskGrid/FlowChart.html | 64 ++++++++-------- ElectronJS/src/taskGrid/FlowChart_CN.html | 64 ++++++++-------- ElectronJS/tasks/255.json | 1 + ElectronJS/tasks/256.json | 1 + ExecuteStage/.vscode/launch.json | 3 +- ExecuteStage/easyspider_executestage.py | 75 ++++++++++--------- ExecuteStage/utils.py | 72 +++++++++++++----- Extension/manifest_v3/src/popup.css | 11 +++ Extension/manifest_v3/src/popup.html | 11 +++ Extension/manifest_v3/src/popup.js | 3 + 14 files changed, 205 insertions(+), 124 deletions(-) create mode 100644 .temp_to_pub/EasySpider_windows_x64/execution_instances/6.json create mode 100644 ElectronJS/tasks/255.json create mode 100644 ElectronJS/tasks/256.json create mode 100644 Extension/manifest_v3/src/popup.css create mode 100644 Extension/manifest_v3/src/popup.html create mode 100644 Extension/manifest_v3/src/popup.js diff --git a/.temp_to_pub/EasySpider_windows_x64/execution_instances/6.json b/.temp_to_pub/EasySpider_windows_x64/execution_instances/6.json new file mode 100644 index 0000000..854b1a4 --- /dev/null +++ b/.temp_to_pub/EasySpider_windows_x64/execution_instances/6.json @@ -0,0 +1 @@ +{"id":6,"name":"京东全球版-专业的综合网上购物商城","url":"https://www.jd.com","links":"https://www.jd.com","create_time":"12/11/2023, 7:29:08 AM","update_time":"12/11/2023, 8:02:45 AM","version":"0.6.0","saveThreshold":10,"quitWaitTime":60,"environment":0,"maximizeWindow":0,"maxViewLength":15,"recordLog":1,"outputFormat":"xlsx","saveName":"current_time","inputExcel":"","startFromExit":0,"pauseKey":"p","containJudge":false,"desc":"https://www.jd.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.jd.com","desc":"要采集的网址列表，多行以\\n分开","type":"text","exampleValue":"https://www.jd.com"},{"id":1,"name":"inputText_1","nodeName":"输入文字","nodeId":4,"desc":"要输入的文本，如京东搜索框输入：电脑","type":"text","exampleValue":"Field[\"参数1\"]","value":"Field[\"参数1\"]"}],"outputParameters":[{"id":0,"name":"参数1","desc":"","type":"text","recordASField":1,"exampleValue":"/手机/数码"}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"url":"https://www.jd.com","links":"https://www.jd.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":2,"index":2,"parentId":0,"type":1,"option":8,"title":"循环采集数据","sequence":[3,4],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":1,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"exitElement":"//body","historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":["/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]","//div[contains(., '/手机/数码')]","//DIV[@class='LeftSide_menu_item__SBMWC LeftSide_text_space__2UhbG ']","/html/body/div[last()-5]/div/div[last()-4]/div/div[last()-2]/div/div/div/div[last()-1]/div[last()-12]"]}},{"id":3,"index":3,"parentId":2,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"clear":0,"newLine":1,"paras":[{"nodeType":0,"contentType":7,"relative":true,"name":"参数1","desc":"","extractType":0,"relativeXPath":"","allXPaths":"","exampleValues":[{"num":0,"value":"/手机/数码"}],"unique_index":"kfd57x5t8clq048p17","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}],"loopType":1}},{"id":4,"index":4,"parentId":2,"type":0,"option":4,"title":"输入文字","sequence":[],"isInLoop":true,"position":1,"parameters":{"history":3,"tabIndex":-1,"useLoop":false,"xpath":"//*[@id=\"key\"]","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"value":"Field[\"参数1\"]","index":0,"allXPaths":["/html/body/div[4]/div[1]/div[2]/div[1]/input[1]","//input[contains(., '')]","id(\"key\")","//INPUT[@class='text defcolor']","/html/body/div[last()-6]/div/div[last()-2]/div/input"]}}]} \ No newline at end of file diff --git a/.temp_to_pub/EasySpider_windows_x64/tasks/255.json b/.temp_to_pub/EasySpider_windows_x64/tasks/255.json index 2a74487..6906f3a 100644 --- a/.temp_to_pub/EasySpider_windows_x64/tasks/255.json +++ b/.temp_to_pub/EasySpider_windows_x64/tasks/255.json @@ -1 +1 @@ -{"id":255,"name":"京东全球版-专业的综合网上购物商城","url":"https://www.jd.com","links":"https://www.jd.com","create_time":"12/11/2023, 7:29:08 AM","update_time":"12/11/2023, 7:30:45 AM","version":"0.6.0","saveThreshold":10,"quitWaitTime":60,"environment":0,"maximizeWindow":0,"maxViewLength":15,"recordLog":1,"outputFormat":"xlsx","saveName":"current_time","inputExcel":"","startFromExit":0,"pauseKey":"p","containJudge":false,"desc":"https://www.jd.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.jd.com","desc":"要采集的网址列表，多行以\\n分开","type":"text","exampleValue":"https://www.jd.com"},{"id":1,"name":"inputText_1","nodeName":"输入文字","nodeId":4,"desc":"要输入的文本，如京东搜索框输入：电脑","type":"text","exampleValue":"Field[\"参数1\"]","value":"Field[\"参数1\"]"}],"outputParameters":[{"id":0,"name":"参数1","desc":"","type":"text","recordASField":1,"exampleValue":"/手机/数码"}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"url":"https://www.jd.com","links":"https://www.jd.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":2,"index":2,"parentId":0,"type":1,"option":8,"title":"循环采集数据","sequence":[3,4],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":1,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"exitElement":"//body","historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":["/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]","//div[contains(., '/手机/数码')]","//DIV[@class='LeftSide_menu_item__SBMWC LeftSide_text_space__2UhbG ']","/html/body/div[last()-5]/div/div[last()-4]/div/div[last()-2]/div/div/div/div[last()-1]/div[last()-12]"]}},{"id":3,"index":3,"parentId":2,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"clear":0,"newLine":1,"paras":[{"nodeType":0,"contentType":8,"relative":true,"name":"参数1","desc":"","extractType":0,"relativeXPath":"","allXPaths":"","exampleValues":[{"num":0,"value":"/手机/数码"}],"unique_index":"kfd57x5t8clq048p17","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}],"loopType":1}},{"id":4,"index":4,"parentId":2,"type":0,"option":4,"title":"输入文字","sequence":[],"isInLoop":true,"position":1,"parameters":{"history":3,"tabIndex":-1,"useLoop":false,"xpath":"//*[@id=\"key\"]","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"value":"Field[\"参数1\"]","index":0,"allXPaths":["/html/body/div[4]/div[1]/div[2]/div[1]/input[1]","//input[contains(., '')]","id(\"key\")","//INPUT[@class='text defcolor']","/html/body/div[last()-6]/div/div[last()-2]/div/input"]}}]} \ No newline at end of file +{"id":255,"name":"京东全球版-专业的综合网上购物商城","url":"https://www.jd.com","links":"https://www.jd.com","create_time":"12/11/2023, 7:29:08 AM","update_time":"12/11/2023, 8:02:45 AM","version":"0.6.0","saveThreshold":10,"quitWaitTime":60,"environment":0,"maximizeWindow":0,"maxViewLength":15,"recordLog":1,"outputFormat":"xlsx","saveName":"current_time","inputExcel":"","startFromExit":0,"pauseKey":"p","containJudge":false,"desc":"https://www.jd.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.jd.com","desc":"要采集的网址列表，多行以\\n分开","type":"text","exampleValue":"https://www.jd.com"},{"id":1,"name":"inputText_1","nodeName":"输入文字","nodeId":4,"desc":"要输入的文本，如京东搜索框输入：电脑","type":"text","exampleValue":"Field[\"参数1\"]","value":"Field[\"参数1\"]"}],"outputParameters":[{"id":0,"name":"参数1","desc":"","type":"text","recordASField":1,"exampleValue":"/手机/数码"}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"url":"https://www.jd.com","links":"https://www.jd.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":2,"index":2,"parentId":0,"type":1,"option":8,"title":"循环采集数据","sequence":[3,4],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":1,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"exitElement":"//body","historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":["/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]","//div[contains(., '/手机/数码')]","//DIV[@class='LeftSide_menu_item__SBMWC LeftSide_text_space__2UhbG ']","/html/body/div[last()-5]/div/div[last()-4]/div/div[last()-2]/div/div/div/div[last()-1]/div[last()-12]"]}},{"id":3,"index":3,"parentId":2,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"clear":0,"newLine":1,"paras":[{"nodeType":0,"contentType":7,"relative":true,"name":"参数1","desc":"","extractType":0,"relativeXPath":"","allXPaths":"","exampleValues":[{"num":0,"value":"/手机/数码"}],"unique_index":"kfd57x5t8clq048p17","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}],"loopType":1}},{"id":4,"index":4,"parentId":2,"type":0,"option":4,"title":"输入文字","sequence":[],"isInLoop":true,"position":1,"parameters":{"history":3,"tabIndex":-1,"useLoop":false,"xpath":"//*[@id=\"key\"]","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"value":"Field[\"参数1\"]","index":0,"allXPaths":["/html/body/div[4]/div[1]/div[2]/div[1]/input[1]","//input[contains(., '')]","id(\"key\")","//INPUT[@class='text defcolor']","/html/body/div[last()-6]/div/div[last()-2]/div/input"]}}]} \ No newline at end of file diff --git a/ElectronJS/main.js b/ElectronJS/main.js index 026b907..89f0fae 100644 --- a/ElectronJS/main.js +++ b/ElectronJS/main.js @@ -198,7 +198,7 @@ async function findElement(driver, by, value, iframe = false) { } } -async function findElementAcrossAllWindows(msg, notifyBrowser = true) { +async function findElementAcrossAllWindows(msg, notifyBrowser = true, scrollIntoView = true) { let handles = await driver.getAllWindowHandles(); // console.log("handles", handles); let content_handle = current_handle; @@ -266,6 +266,16 @@ async function findElementAcrossAllWindows(msg, notifyBrowser = true) { if (element == null && notifyBrowser) { notify_browser("无法找到元素，请检查XPath是否正确：" + xpath, "Cannot find the element, please check if the XPath is correct: " + xpath, "warning"); } + if (element != null && scrollIntoView) { + // 浏览器切换到元素位置稍微靠上的位置 + try { + // let script = `arguments[0].scrollIntoView(true);`; + let script = `arguments[0].scrollIntoView({block: "center", inline: "center"});`; + await driver.executeScript(script, element); + } catch (e) { + console.log("Cannot scrollIntoView"); + } + } return element; } @@ -308,7 +318,7 @@ async function beginInvoke(msg, ws) { keyInfo = keyInfo.replace(//gi, ''); enter = true; } - let element = await findElementAcrossAllWindows(msg); + let element = await findElementAcrossAllWindows(msg, notifyBrowser = true, scrollIntoView = false); await element.sendKeys(Key.HOME, Key.chord(Key.SHIFT, Key.END), keyInfo); if (enter) { await element.sendKeys(Key.ENTER); @@ -321,7 +331,7 @@ async function beginInvoke(msg, ws) { let type = message.type; console.log("FROM Browser: ", message); if (type.indexOf("Click") >= 0 || type.indexOf("Move") >= 0) { - let element = await findElementAcrossAllWindows(message); + let element = await findElementAcrossAllWindows(message, notifyBrowser = true, scrollIntoView = false); if (type.indexOf("Click") >= 0) { await click_element(element); } else if (type.indexOf("Move") >= 0) { @@ -336,7 +346,7 @@ async function beginInvoke(msg, ws) { console.log(e); } } else if (msg.type == 4) { //试运行功能 - try{ + try { let flowchart_url = flowchart_window.webContents.getURL(); } catch { flowchart_window = null; @@ -562,7 +572,7 @@ async function execute_js(js, element, wait_time = 3) { if (js.length != 0) { try { await driver.executeScript(js, element); - if(wait_time == 0){ + if (wait_time == 0) { wait_time = 30000; } await new Promise(resolve => setTimeout(resolve, wait_time)); diff --git a/ElectronJS/src/taskGrid/FlowChart.css b/ElectronJS/src/taskGrid/FlowChart.css index 92e4664..4b219bc 100644 --- a/ElectronJS/src/taskGrid/FlowChart.css +++ b/ElectronJS/src/taskGrid/FlowChart.css @@ -38,6 +38,7 @@ border-radius: 7px; /*border: skyblue solid;*/ border: #007bff solid; + border-color: transparent; /*background: rgb(73, 156, 189);*/ background: rgb(10, 120, 200); color: white; diff --git a/ElectronJS/src/taskGrid/FlowChart.html b/ElectronJS/src/taskGrid/FlowChart.html index 224f9e1..f519631 100644 --- a/ElectronJS/src/taskGrid/FlowChart.html +++ b/ElectronJS/src/taskGrid/FlowChart.html @@ -113,10 +113,10 @@ @@ -159,10 +159,10 @@ @@ -414,8 +414,8 @@ This option is an advanced feature that allows directly returning the expression Please note that this feature does not support assigning values to variables. In other words, you cannot write something like `self.myVar = 1`. If you want to perform assignment operations, please select the previous option, "Run Python code on current environment (the "exec" operation)"

Whether to record the output/return value of the execution as a field:

- - - - + + + +

Set value (not applicable for "Switch to the next option" mode)

@@ -568,9 +568,9 @@ If the expression returns a value greater than 0 or evaluates to True, the loop