mirror of
https://github.com/NaiboWang/EasySpider.git
synced 2025-04-21 16:25:18 +08:00
Bug Fix and Picture download update
This commit is contained in:
naibo
parent
17587218f5
commit
c3bab575f4
@ -0,0 +1 @@
|
||||
{"id":6,"name":"京东全球版-专业的综合网上购物商城","url":"https://www.jd.com","links":"https://www.jd.com","create_time":"12/11/2023, 7:29:08 AM","update_time":"12/11/2023, 8:02:45 AM","version":"0.6.0","saveThreshold":10,"quitWaitTime":60,"environment":0,"maximizeWindow":0,"maxViewLength":15,"recordLog":1,"outputFormat":"xlsx","saveName":"current_time","inputExcel":"","startFromExit":0,"pauseKey":"p","containJudge":false,"desc":"https://www.jd.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.jd.com","desc":"要采集的网址列表，多行以\\n分开","type":"text","exampleValue":"https://www.jd.com"},{"id":1,"name":"inputText_1","nodeName":"输入文字","nodeId":4,"desc":"要输入的文本，如京东搜索框输入：电脑","type":"text","exampleValue":"Field[\"参数1\"]","value":"Field[\"参数1\"]"}],"outputParameters":[{"id":0,"name":"参数1","desc":"","type":"text","recordASField":1,"exampleValue":"/手机/数码"}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"url":"https://www.jd.com","links":"https://www.jd.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":2,"index":2,"parentId":0,"type":1,"option":8,"title":"循环采集数据","sequence":[3,4],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":1,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"exitElement":"//body","historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":["/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]","//div[contains(., '/手机/数码')]","//DIV[@class='LeftSide_menu_item__SBMWC LeftSide_text_space__2UhbG ']","/html/body/div[last()-5]/div/div[last()-4]/div/div[last()-2]/div/div/div/div[last()-1]/div[last()-12]"]}},{"id":3,"index":3,"parentId":2,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"clear":0,"newLine":1,"paras":[{"nodeType":0,"contentType":7,"relative":true,"name":"参数1","desc":"","extractType":0,"relativeXPath":"","allXPaths":"","exampleValues":[{"num":0,"value":"/手机/数码"}],"unique_index":"kfd57x5t8clq048p17","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}],"loopType":1}},{"id":4,"index":4,"parentId":2,"type":0,"option":4,"title":"输入文字","sequence":[],"isInLoop":true,"position":1,"parameters":{"history":3,"tabIndex":-1,"useLoop":false,"xpath":"//*[@id=\"key\"]","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"value":"Field[\"参数1\"]","index":0,"allXPaths":["/html/body/div[4]/div[1]/div[2]/div[1]/input[1]","//input[contains(., '')]","id(\"key\")","//INPUT[@class='text defcolor']","/html/body/div[last()-6]/div/div[last()-2]/div/input"]}}]}
|
||||
@ -1 +1 @@
|
||||
{"id":255,"name":"京东全球版-专业的综合网上购物商城","url":"https://www.jd.com","links":"https://www.jd.com","create_time":"12/11/2023, 7:29:08 AM","update_time":"12/11/2023, 7:30:45 AM","version":"0.6.0","saveThreshold":10,"quitWaitTime":60,"environment":0,"maximizeWindow":0,"maxViewLength":15,"recordLog":1,"outputFormat":"xlsx","saveName":"current_time","inputExcel":"","startFromExit":0,"pauseKey":"p","containJudge":false,"desc":"https://www.jd.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.jd.com","desc":"要采集的网址列表，多行以\\n分开","type":"text","exampleValue":"https://www.jd.com"},{"id":1,"name":"inputText_1","nodeName":"输入文字","nodeId":4,"desc":"要输入的文本，如京东搜索框输入：电脑","type":"text","exampleValue":"Field[\"参数1\"]","value":"Field[\"参数1\"]"}],"outputParameters":[{"id":0,"name":"参数1","desc":"","type":"text","recordASField":1,"exampleValue":"/手机/数码"}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"url":"https://www.jd.com","links":"https://www.jd.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":2,"index":2,"parentId":0,"type":1,"option":8,"title":"循环采集数据","sequence":[3,4],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":1,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"exitElement":"//body","historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":["/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]","//div[contains(., '/手机/数码')]","//DIV[@class='LeftSide_menu_item__SBMWC LeftSide_text_space__2UhbG ']","/html/body/div[last()-5]/div/div[last()-4]/div/div[last()-2]/div/div/div/div[last()-1]/div[last()-12]"]}},{"id":3,"index":3,"parentId":2,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"clear":0,"newLine":1,"paras":[{"nodeType":0,"contentType":8,"relative":true,"name":"参数1","desc":"","extractType":0,"relativeXPath":"","allXPaths":"","exampleValues":[{"num":0,"value":"/手机/数码"}],"unique_index":"kfd57x5t8clq048p17","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}],"loopType":1}},{"id":4,"index":4,"parentId":2,"type":0,"option":4,"title":"输入文字","sequence":[],"isInLoop":true,"position":1,"parameters":{"history":3,"tabIndex":-1,"useLoop":false,"xpath":"//*[@id=\"key\"]","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"value":"Field[\"参数1\"]","index":0,"allXPaths":["/html/body/div[4]/div[1]/div[2]/div[1]/input[1]","//input[contains(., '')]","id(\"key\")","//INPUT[@class='text defcolor']","/html/body/div[last()-6]/div/div[last()-2]/div/input"]}}]}
|
||||
{"id":255,"name":"京东全球版-专业的综合网上购物商城","url":"https://www.jd.com","links":"https://www.jd.com","create_time":"12/11/2023, 7:29:08 AM","update_time":"12/11/2023, 8:02:45 AM","version":"0.6.0","saveThreshold":10,"quitWaitTime":60,"environment":0,"maximizeWindow":0,"maxViewLength":15,"recordLog":1,"outputFormat":"xlsx","saveName":"current_time","inputExcel":"","startFromExit":0,"pauseKey":"p","containJudge":false,"desc":"https://www.jd.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.jd.com","desc":"要采集的网址列表，多行以\\n分开","type":"text","exampleValue":"https://www.jd.com"},{"id":1,"name":"inputText_1","nodeName":"输入文字","nodeId":4,"desc":"要输入的文本，如京东搜索框输入：电脑","type":"text","exampleValue":"Field[\"参数1\"]","value":"Field[\"参数1\"]"}],"outputParameters":[{"id":0,"name":"参数1","desc":"","type":"text","recordASField":1,"exampleValue":"/手机/数码"}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"url":"https://www.jd.com","links":"https://www.jd.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":2,"index":2,"parentId":0,"type":1,"option":8,"title":"循环采集数据","sequence":[3,4],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":1,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"exitElement":"//body","historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":["/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]","//div[contains(., '/手机/数码')]","//DIV[@class='LeftSide_menu_item__SBMWC LeftSide_text_space__2UhbG ']","/html/body/div[last()-5]/div/div[last()-4]/div/div[last()-2]/div/div/div/div[last()-1]/div[last()-12]"]}},{"id":3,"index":3,"parentId":2,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"clear":0,"newLine":1,"paras":[{"nodeType":0,"contentType":7,"relative":true,"name":"参数1","desc":"","extractType":0,"relativeXPath":"","allXPaths":"","exampleValues":[{"num":0,"value":"/手机/数码"}],"unique_index":"kfd57x5t8clq048p17","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}],"loopType":1}},{"id":4,"index":4,"parentId":2,"type":0,"option":4,"title":"输入文字","sequence":[],"isInLoop":true,"position":1,"parameters":{"history":3,"tabIndex":-1,"useLoop":false,"xpath":"//*[@id=\"key\"]","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"value":"Field[\"参数1\"]","index":0,"allXPaths":["/html/body/div[4]/div[1]/div[2]/div[1]/input[1]","//input[contains(., '')]","id(\"key\")","//INPUT[@class='text defcolor']","/html/body/div[last()-6]/div/div[last()-2]/div/input"]}}]}
|
||||
@ -198,7 +198,7 @@ async function findElement(driver, by, value, iframe = false) {
|
||||
}
|
||||
}
|
||||
|
||||
async function findElementAcrossAllWindows(msg, notifyBrowser = true) {
|
||||
async function findElementAcrossAllWindows(msg, notifyBrowser = true, scrollIntoView = true) {
|
||||
let handles = await driver.getAllWindowHandles();
|
||||
// console.log("handles", handles);
|
||||
let content_handle = current_handle;
|
||||
@ -266,6 +266,16 @@ async function findElementAcrossAllWindows(msg, notifyBrowser = true) {
|
||||
if (element == null && notifyBrowser) {
|
||||
notify_browser("无法找到元素，请检查XPath是否正确：" + xpath, "Cannot find the element, please check if the XPath is correct: " + xpath, "warning");
|
||||
}
|
||||
if (element != null && scrollIntoView) {
|
||||
// 浏览器切换到元素位置稍微靠上的位置
|
||||
try {
|
||||
// let script = `arguments[0].scrollIntoView(true);`;
|
||||
let script = `arguments[0].scrollIntoView({block: "center", inline: "center"});`;
|
||||
await driver.executeScript(script, element);
|
||||
} catch (e) {
|
||||
console.log("Cannot scrollIntoView");
|
||||
}
|
||||
}
|
||||
return element;
|
||||
}
|
||||
|
||||
@ -308,7 +318,7 @@ async function beginInvoke(msg, ws) {
|
||||
keyInfo = keyInfo.replace(/<enter>/gi, '');
|
||||
enter = true;
|
||||
}
|
||||
let element = await findElementAcrossAllWindows(msg);
|
||||
let element = await findElementAcrossAllWindows(msg, notifyBrowser = true, scrollIntoView = false);
|
||||
await element.sendKeys(Key.HOME, Key.chord(Key.SHIFT, Key.END), keyInfo);
|
||||
if (enter) {
|
||||
await element.sendKeys(Key.ENTER);
|
||||
@ -321,7 +331,7 @@ async function beginInvoke(msg, ws) {
|
||||
let type = message.type;
|
||||
console.log("FROM Browser: ", message);
|
||||
if (type.indexOf("Click") >= 0 || type.indexOf("Move") >= 0) {
|
||||
let element = await findElementAcrossAllWindows(message);
|
||||
let element = await findElementAcrossAllWindows(message, notifyBrowser = true, scrollIntoView = false);
|
||||
if (type.indexOf("Click") >= 0) {
|
||||
await click_element(element);
|
||||
} else if (type.indexOf("Move") >= 0) {
|
||||
@ -336,7 +346,7 @@ async function beginInvoke(msg, ws) {
|
||||
console.log(e);
|
||||
}
|
||||
} else if (msg.type == 4) { //试运行功能
|
||||
try{
|
||||
try {
|
||||
let flowchart_url = flowchart_window.webContents.getURL();
|
||||
} catch {
|
||||
flowchart_window = null;
|
||||
@ -562,7 +572,7 @@ async function execute_js(js, element, wait_time = 3) {
|
||||
if (js.length != 0) {
|
||||
try {
|
||||
await driver.executeScript(js, element);
|
||||
if(wait_time == 0){
|
||||
if (wait_time == 0) {
|
||||
wait_time = 30000;
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, wait_time));
|
||||
|
||||
@ -38,6 +38,7 @@
|
||||
border-radius: 7px;
|
||||
/*border: skyblue solid;*/
|
||||
border: #007bff solid;
|
||||
border-color: transparent;
|
||||
/*background: rgb(73, 156, 189);*/
|
||||
background: rgb(10, 120, 200);
|
||||
color: white;
|
||||
|
||||
@ -113,10 +113,10 @@
|
||||
<input onkeydown="inputDelete(event)" class="form-control" v-model.number="nowNode['parameters']['maxWaitTime']" type="number" required></input>
|
||||
<label>After executed, whether scroll down:</label>
|
||||
<select v-model='nowNode["parameters"]["scrollType"]' class="form-control">
|
||||
<option value = 0>No scrolling</option>
|
||||
<option value = 1>Scroll one screen</option>
|
||||
<option value = 2>Scroll to the end</option>
|
||||
<option value = 3>Keep scrolling until the page data does not change</option>
|
||||
<option :vaule = 0>No scrolling</option>
|
||||
<option :vaule = 1>Scroll one screen</option>
|
||||
<option :vaule = 2>Scroll to the end</option>
|
||||
<option :vaule = 3>Keep scrolling until the page data does not change</option>
|
||||
</select>
|
||||
<label>Scroll Times (the wait time after scrolling <b>ineffective</b> when the scrolling type is set to <b>no scrolling</b>):</label>
|
||||
<input onkeydown="inputDelete(event)" class="form-control" v-model.number="nowNode['parameters']['scrollCount']" type="number" required></input>
|
||||
@ -159,10 +159,10 @@
|
||||
</select>
|
||||
<label>Whether to scroll down after clicking:</label>
|
||||
<select v-model='nowNode["parameters"]["scrollType"]' class="form-control">
|
||||
<option value = 0>No Scrolling</option>
|
||||
<option value = 1>Scroll one screen</option>
|
||||
<option value = 2>Scroll to the end</option>
|
||||
<option value = 3>Keep scrolling until the page data does not change</option>
|
||||
<option :vaule = 0>No Scrolling</option>
|
||||
<option :vaule = 1>Scroll one screen</option>
|
||||
<option :vaule = 2>Scroll to the end</option>
|
||||
<option :vaule = 3>Keep scrolling until the page data does not change</option>
|
||||
</select>
|
||||
<label>Scroll Times:</label>
|
||||
<input onkeydown="inputDelete(event)" class="form-control" v-model.number="nowNode['parameters']['scrollCount']" type="number" required></input>
|
||||
@ -414,8 +414,8 @@ This option is an advanced feature that allows directly returning the expression
|
||||
Please note that this feature does not support assigning values to variables. In other words, you cannot write something like `self.myVar = 1`. If you want to perform assignment operations, please select the previous option, "Run Python code on current environment (the "exec" operation)"</pre>
|
||||
<p style="margin-top: 15px">Whether to record the output/return value of the execution as a field: </p>
|
||||
<p><select v-model='nowNode["parameters"]["recordASField"]' class="form-control">
|
||||
<option value = 0>No</option>
|
||||
<option value = 1>Yes</option>
|
||||
<option :vaule = 0>No</option>
|
||||
<option :vaule = 1>Yes</option>
|
||||
</select></p>
|
||||
<p><label>Convert parameter type to:</label>
|
||||
<select v-model='nowNode["parameters"]["paraType"]' class="form-control">
|
||||
@ -490,10 +490,10 @@ Please note that this feature does not support assigning values to variables. In
|
||||
<div v-if="!useLoop">
|
||||
<p>Option switch Mode</p>
|
||||
<select class="form-control" v-model='nowNode["parameters"]["optionMode"]'>
|
||||
<option value=0>Switch to the next option</option>
|
||||
<option value=1>Switch options by index (0 is the first option)</option>
|
||||
<option value=2>Switch options by option value</option>
|
||||
<option value=3>Switch options by option text</option>
|
||||
<option :vaule = 0>Switch to the next option</option>
|
||||
<option :vaule = 1>Switch options by index (0 is the first option)</option>
|
||||
<option :vaule = 2>Switch options by option value</option>
|
||||
<option :vaule = 3>Switch options by option text</option>
|
||||
</select>
|
||||
<p>Set value (not applicable for "Switch to the next option" mode)</p>
|
||||
<input class="form-control" id="selectValue" v-model='nowNode["parameters"]["optionValue"]' autoFocus="autofocus" type="text"></input>
|
||||
@ -568,9 +568,9 @@ If the expression returns a value greater than 0 or evaluates to True, the loop
|
||||
<div>
|
||||
<p><label>(Advanced Operation) Define loop exit condition using code/script; or you can add a <b>Custom Action</b>, then select the "Exit Loop" option:</label></p>
|
||||
<select v-model='nowNode["parameters"]["breakMode"]' class="form-control" style="font-weight: bold">
|
||||
<option value=0>Do not set script (even if a script is written below, it will not be executed)</option>
|
||||
<option value=1>JavaScript script (start with 'return ')</option>
|
||||
<option value=2>Operating system-level command</option>
|
||||
<option :vaule = 0>Do not set script (even if a script is written below, it will not be executed)</option>
|
||||
<option :vaule = 1>JavaScript script (start with 'return ')</option>
|
||||
<option :vaule = 2>Operating system-level command</option>
|
||||
</select>
|
||||
<div>
|
||||
<textarea style="margin-top: 10px" onkeydown="inputDelete(event)" class="form-control" rows="2"
|
||||
@ -584,10 +584,10 @@ If the expression returns a value greater than 0 or evaluates to True, the loop
|
||||
<input onkeydown="inputDelete(event)" required type="number" class="form-control" v-model.number='list.nl[index.nowNodeIndex]["parameters"]["historyWait"]'></input>
|
||||
<label>After executed, whether scroll down:</label>
|
||||
<select v-model='nowNode["parameters"]["scrollType"]' class="form-control">
|
||||
<option value = 0>No Scrolling</option>
|
||||
<option value = 1>Scroll one screen</option>
|
||||
<option value = 2>Scroll to the end</option>
|
||||
<option value = 3>Keep scrolling until the page data does not change</option>
|
||||
<option :vaule = 0>No Scrolling</option>
|
||||
<option :vaule = 1>Scroll one screen</option>
|
||||
<option :vaule = 2>Scroll to the end</option>
|
||||
<option :vaule = 3>Keep scrolling until the page data does not change</option>
|
||||
</select>
|
||||
<label>Scroll Times:</label>
|
||||
<input onkeydown="inputDelete(event)" class="form-control" v-model.number="nowNode['parameters']['scrollCount']" type="number" required></input>
|
||||
@ -604,15 +604,15 @@ If the expression returns a value greater than 0 or evaluates to True, the loop
|
||||
<p><input onkeydown="inputDelete(event)" type="checkbox" v-model='nowNode["parameters"]["iframe"]'></input>Operation is in iframe</p>
|
||||
<label>Condition Type:</label>
|
||||
<select v-model='TClass' class="form-control">
|
||||
<option value = 0>No Condition</option>
|
||||
<option value = 1>Text inside current page</option>
|
||||
<option value = 2>Element inside current page</option>
|
||||
<option v-if="nowNode['isInLoop']" value = 3>Text inside current loop</option>
|
||||
<option v-if="nowNode['isInLoop']" value = 4>Element inside current loop</option>
|
||||
<option value = 5>Return value of JavaScript command (start with 'return ')</option>
|
||||
<option value = 6>Return value of system command</option>
|
||||
<option v-if="nowNode['isInLoop']" value = 7>Return value of JavaScript command for the current loop item</option>
|
||||
<option value = 8>Return value of Python code under current environment</option>
|
||||
<option :value = 0>No Condition</option>
|
||||
<option :value = 1>Text inside current page</option>
|
||||
<option :value = 2>Element inside current page</option>
|
||||
<option v-if="nowNode['isInLoop']" :value = 3>Text inside current loop</option>
|
||||
<option v-if="nowNode['isInLoop']" :value = 4>Element inside current loop</option>
|
||||
<option :value = 5>Return value of JavaScript command (start with 'return ')</option>
|
||||
<option :value = 6>Return value of system command</option>
|
||||
<option v-if="nowNode['isInLoop']" :value = 7>Return value of JavaScript command for the current loop item</option>
|
||||
<option :value = 8>Return value of Python code under current environment</option>
|
||||
</select>
|
||||
|
||||
<div v-if='TClass > 0 && TClass < 5'>
|
||||
@ -653,8 +653,8 @@ If the expression returns a value greater than 0 or evaluates to True, the opera
|
||||
<input onkeydown="inputDelete(event)" required type="number" class="form-control" v-model.number='list.nl[index.nowNodeIndex]["parameters"]["wait"]'></input>
|
||||
<label>Wait Type</label>
|
||||
<select v-model='list.nl[index.nowNodeIndex]["parameters"]["waitType"]' class="form-control">
|
||||
<option value = 0>Fixed wait (set to wait for 10 seconds then it will wait for 10 seconds)</option>
|
||||
<option value = 1>Random wait (set to wait for 10 seconds then it will randomly wait for 10 × 0.5 - 10 × 1.5 seconds)</option>
|
||||
<option :vaule = 0>Fixed wait (set to wait for 10 seconds then it will wait for 10 seconds)</option>
|
||||
<option :vaule = 1>Random wait (set to wait for 10 seconds then it will randomly wait for 10 × 0.5 - 10 × 1.5 seconds)</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -113,10 +113,10 @@
|
||||
<input onkeydown="inputDelete(event)" class="form-control" v-model.number="nowNode['parameters']['maxWaitTime']" type="number" required></input>
|
||||
<label>打开网页后是否向下滚动：</label>
|
||||
<select v-model='nowNode["parameters"]["scrollType"]' class="form-control">
|
||||
<option value = 0>不滚动</option>
|
||||
<option value = 1>向下滚动一屏</option>
|
||||
<option value = 2>滚动到底部</option>
|
||||
<option value = 3>一直滚动直到页面内容无变化（需设置好滚动后的等待时间用于检测页面变化）</option>
|
||||
<option :value = 0>不滚动</option>
|
||||
<option :value = 1>向下滚动一屏</option>
|
||||
<option :value = 2>滚动到底部</option>
|
||||
<option :value = 3>一直滚动直到页面内容无变化（需设置好滚动后的等待时间用于检测页面变化）</option>
|
||||
</select>
|
||||
<label>滚动次数（滚动类型设置为<b>不滚动</b>或<b>一直滚动</b>时请忽略此项）：</label>
|
||||
<input onkeydown="inputDelete(event)" class="form-control" v-model.number="nowNode['parameters']['scrollCount']" type="number" required></input>
|
||||
@ -159,10 +159,10 @@
|
||||
</select>
|
||||
<label>点击后是否向下滚动页面：</label>
|
||||
<select v-model='nowNode["parameters"]["scrollType"]' class="form-control">
|
||||
<option value = 0>不滚动</option>
|
||||
<option value = 1>向下滚动一屏</option>
|
||||
<option value = 2>滚动到底部</option>
|
||||
<option value = 3>一直滚动直到页面内容无变化（需设置好滚动后的等待时间用于检测页面变化）</option>
|
||||
<option :vaule = 0>不滚动</option>
|
||||
<option :vaule = 1>向下滚动一屏</option>
|
||||
<option :vaule = 2>滚动到底部</option>
|
||||
<option :vaule = 3>一直滚动直到页面内容无变化（需设置好滚动后的等待时间用于检测页面变化）</option>
|
||||
</select>
|
||||
<label>滚动次数（滚动类型设置为<b>不滚动</b>或<b>一直滚动</b>时请忽略此项）：</label>
|
||||
<input onkeydown="inputDelete(event)" class="form-control" v-model.number="nowNode['parameters']['scrollCount']" type="number" required></input>
|
||||
@ -490,10 +490,10 @@ print(emotlib.emoji()) # 使用其中的函数。
|
||||
<div v-if="!useLoop">
|
||||
<p>切换模式</p>
|
||||
<select class="form-control" v-model='nowNode["parameters"]["optionMode"]'>
|
||||
<option value=0>切换到下一个选项</option>
|
||||
<option value=1>按索引值切换选项（第一个选项索引值为0）</option>
|
||||
<option value=2>按选项值切换选项</option>
|
||||
<option value=3>按选项文本切换选项</option>
|
||||
<option :value=0>切换到下一个选项</option>
|
||||
<option :value=1>按索引值切换选项（第一个选项索引值为0）</option>
|
||||
<option :value=2>按选项值切换选项</option>
|
||||
<option :value=3>按选项文本切换选项</option>
|
||||
</select>
|
||||
<p>设定值（不适用于切换到下一个选项模式）</p>
|
||||
<input class="form-control" id="selectValue" v-model='nowNode["parameters"]["optionValue"]' autoFocus="autofocus" type="text"></input>
|
||||
@ -568,9 +568,9 @@ print(emotlib.emoji()) # 使用其中的函数。
|
||||
<div>
|
||||
<p><label>（自定义操作）使用代码/脚本定义循环退出条件（也可以在流程中添加<b>自定义操作</b>，然后选择<b>退出循环</b>选项）： </label></p>
|
||||
<select v-model='nowNode["parameters"]["breakMode"]' class="form-control" style="font-weight: bold">
|
||||
<option value = 0>不设置脚本（选择这个下面写了脚本也不会执行）</option>
|
||||
<option value = 1>JavaScript脚本返回值（需以return 开头）</option>
|
||||
<option value = 2>操作系统级别命令</option>
|
||||
<option :value = 0>不设置脚本（选择这个下面写了脚本也不会执行）</option>
|
||||
<option :value = 1>JavaScript脚本返回值（需以return 开头）</option>
|
||||
<option :value = 2>操作系统级别命令</option>
|
||||
</select>
|
||||
<div>
|
||||
<textarea style="margin-top: 10px" onkeydown="inputDelete(event)" class="form-control" rows="2"
|
||||
@ -584,10 +584,10 @@ print(emotlib.emoji()) # 使用其中的函数。
|
||||
<input onkeydown="inputDelete(event)" required type="number" class="form-control" v-model.number='list.nl[index.nowNodeIndex]["parameters"]["historyWait"]'></input>
|
||||
<label>执行完是否向下滚动：</label>
|
||||
<select v-model='nowNode["parameters"]["scrollType"]' class="form-control">
|
||||
<option value = 0>不滚动</option>
|
||||
<option value = 1>向下滚动一屏</option>
|
||||
<option value = 2>滚动到底部</option>
|
||||
<option value = 3>一直滚动直到页面内容无变化（需设置好滚动后的等待时间用于检测页面变化）</option>
|
||||
<option :value = 0>不滚动</option>
|
||||
<option :value = 1>向下滚动一屏</option>
|
||||
<option :value = 2>滚动到底部</option>
|
||||
<option :value = 3>一直滚动直到页面内容无变化（需设置好滚动后的等待时间用于检测页面变化）</option>
|
||||
</select>
|
||||
<label>滚动次数（滚动类型设置为<b>不滚动</b>或<b>一直滚动</b>时请忽略此项）：</label>
|
||||
<input onkeydown="inputDelete(event)" class="form-control" v-model.number="nowNode['parameters']['scrollCount']" type="number" required></input>
|
||||
@ -596,23 +596,23 @@ print(emotlib.emoji()) # 使用其中的函数。
|
||||
</div>
|
||||
|
||||
<div class="elements" v-if="nodeType==9">
|
||||
<label>判断条件是从左往右判断的，即如果最左边的条件分支的判断条件满足，则执行最左边分支内的操作，否则判断从左向右第二个分支的条件是否满足，依此类推。</label>
|
||||
<label>判断条件是从左往右判断的，即如果最左边的条件分支的判断条件满足，则执行最左边分支内的操作，否则判断从左向右第二个分支的条件是否满足，以此类推。</label>
|
||||
</div>
|
||||
|
||||
<div class="elements" v-if="nodeType==10">
|
||||
<label>判断条件是从左往右判断的，即如果最左边的条件分支的判断条件满足，则执行最左边分支内的操作，否则判断从左向右第二个分支的条件是否满足，依此类推。</label>
|
||||
<label>判断条件是从左往右判断的，即如果最左边的条件分支的判断条件满足，则执行最左边分支内的操作，否则判断从左向右第二个分支的条件是否满足，以此类推。</label>
|
||||
<p><input onkeydown="inputDelete(event)" type="checkbox" v-model='nowNode["parameters"]["iframe"]'></input>在iframe内操作</p>
|
||||
<label>条件类型：</label>
|
||||
<select v-model='TClass' class="form-control">
|
||||
<option value = 0>无条件</option>
|
||||
<option value = 1>当前页面包括文本</option>
|
||||
<option value = 2>当前页面包括元素</option>
|
||||
<option v-if="nowNode['isInLoop']" value = 3>当前循环项包括文本</option>
|
||||
<option v-if="nowNode['isInLoop']" value = 4>当前循环项包括元素</option>
|
||||
<option value = 5>JavaScript命令返回值（需以return 开头）</option>
|
||||
<option value = 6>系统命令返回值</option>
|
||||
<option v-if="nowNode['isInLoop']" value = 7>针对当前循环项的JavaScript命令返回值（需以return 开头）</option>
|
||||
<option value = 8>执行环境下的Python表达式值（eval操作）</option>
|
||||
<option :value = 0>无条件</option>
|
||||
<option :value = 1>当前页面包括文本</option>
|
||||
<option :value = 2>当前页面包括元素</option>
|
||||
<option v-if="nowNode['isInLoop']" :value = 3>当前循环项包括文本</option>
|
||||
<option v-if="nowNode['isInLoop']" :value = 4>当前循环项包括元素</option>
|
||||
<option :value = 5>JavaScript命令返回值（需以return 开头）</option>
|
||||
<option :value = 6>系统命令返回值</option>
|
||||
<option v-if="nowNode['isInLoop']" :value = 7>针对当前循环项的JavaScript命令返回值（需以return 开头）</option>
|
||||
<option :value = 8>执行环境下的Python表达式值（eval操作）</option>
|
||||
</select>
|
||||
|
||||
<div v-if='TClass > 0 && TClass < 5'>
|
||||
@ -653,8 +653,8 @@ print(emotlib.emoji()) # 使用其中的函数。
|
||||
<input onkeydown="inputDelete(event)" required type="number" class="form-control" v-model.number='list.nl[index.nowNodeIndex]["parameters"]["wait"]'></input>
|
||||
<label style="margin-top:5px">等待类型</label>
|
||||
<select v-model='list.nl[index.nowNodeIndex]["parameters"]["waitType"]' class="form-control">
|
||||
<option value = 0>固定等待（设置等10秒就等10秒）</option>
|
||||
<option value = 1>随机等待（设置等10秒会随机等10×0.5 - 10 × 1.5 秒）</option>
|
||||
<option :value = 0>固定等待（设置等10秒就等10秒）</option>
|
||||
<option :value = 1>随机等待（设置等10秒会随机等10×0.5 - 10 × 1.5 秒）</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
1
ElectronJS/tasks/255.json
Normal file
1
ElectronJS/tasks/255.json
Normal file
@ -0,0 +1 @@
|
||||
{"id":255,"name":"京东全球版-专业的综合网上购物商城","url":"https://www.jd.com","links":"https://www.jd.com","create_time":"12/12/2023, 12:05:21 AM","update_time":"12/12/2023, 12:05:21 AM","version":"0.6.0","saveThreshold":10,"quitWaitTime":60,"environment":0,"maximizeWindow":0,"maxViewLength":15,"recordLog":1,"outputFormat":"xlsx","saveName":"current_time","inputExcel":"","startFromExit":0,"pauseKey":"p","containJudge":false,"desc":"https://www.jd.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.jd.com","desc":"要采集的网址列表，多行以\\n分开","type":"text","exampleValue":"https://www.jd.com"}],"outputParameters":[{"id":0,"name":"参数1_图片地址","desc":"","type":"text","recordASField":1,"exampleValue":"//storage.360buyimg.com/i.imageUpload/31383731303839333234345f7031363633333436303135373830_mid.jpg"}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"url":"https://www.jd.com","links":"https://www.jd.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":2,"index":2,"parentId":0,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"clear":0,"newLine":1,"paras":[{"nodeType":4,"contentType":0,"relative":false,"name":"参数1_图片地址","desc":"","extractType":0,"relativeXPath":"/html/body/div[5]/div[1]/div[1]/div[1]/div[3]/div[1]/div[1]/div[1]/a[1]/img[1]","allXPaths":["/html/body/div[5]/div[1]/div[1]/div[1]/div[3]/div[1]/div[1]/div[1]/a[1]/img[1]","//img[contains(., '')]","/html/body/div[last()-5]/div/div[last()-4]/div/div/div/div[last()-6]/div[last()-1]/a/img"],"exampleValues":[{"num":0,"value":"//storage.360buyimg.com/i.imageUpload/31383731303839333234345f7031363633333436303135373830_mid.jpg"}],"unique_index":"dxuiztdeptblq13uco8","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":1}]}}]}
|
||||
1
ElectronJS/tasks/256.json
Normal file
1
ElectronJS/tasks/256.json
Normal file
File diff suppressed because one or more lines are too long
3
ExecuteStage/.vscode/launch.json
vendored
3
ExecuteStage/.vscode/launch.json
vendored
@ -12,7 +12,8 @@
|
||||
"justMyCode": false,
|
||||
// "args": ["--ids", "[7]", "--read_type", "remote", "--headless", "0"]
|
||||
// "args": ["--ids", "[9]", "--read_type", "remote", "--headless", "0", "--saved_file_name", "YOUTUBE"]
|
||||
"args": ["--ids", "[0]", "--headless", "0", "--user_data", "0", "--keyboard", "0"]
|
||||
"args": ["--ids", "[20]", "--headless", "0", "--user_data", "0", "--keyboard", "0",
|
||||
"--read_type", "remote"]
|
||||
// "args": "--ids '[97]' --user_data 1 --server_address http://localhost:8074 --config_folder '/Users/naibo/Documents/EasySpider/ElectronJS/' --headless 0 --read_type remote --config_file_name config.json --saved_file_name"
|
||||
}
|
||||
]
|
||||
|
||||
@ -855,40 +855,45 @@ class BrowserThread(Thread):
|
||||
self.recordLog(e)
|
||||
self.recordLog("Wait element not found")
|
||||
self.recordLog("执行节点|Execute node:", node["title"])
|
||||
# 根据不同选项执行不同操作
|
||||
if node["option"] == 0 or node["option"] == 10: # root操作,条件分支操作
|
||||
for i in node["sequence"]: # 从根节点开始向下读取
|
||||
self.executeNode(i, loopValue, loopPath, index)
|
||||
elif node["option"] == 1: # 打开网页操作
|
||||
# if not (nodeId == 1 and self.service["cloudflare"] == 1):
|
||||
self.openPage(node["parameters"], loopValue)
|
||||
elif node["option"] == 2: # 点击元素
|
||||
self.clickElement(node["parameters"], loopValue, loopPath, index)
|
||||
elif node["option"] == 3: # 提取数据
|
||||
# 针对提取数据操作，设置操作开始的步骤，用于不小心关闭后的恢复的增量采集
|
||||
if self.totalSteps >= self.startSteps:
|
||||
self.getData(node["parameters"], loopValue, node["isInLoop"],
|
||||
parentPath=loopPath, index=index)
|
||||
try:
|
||||
# 根据不同选项执行不同操作
|
||||
if node["option"] == 0 or node["option"] == 10: # root操作,条件分支操作
|
||||
for i in node["sequence"]: # 从根节点开始向下读取
|
||||
self.executeNode(i, loopValue, loopPath, index)
|
||||
elif node["option"] == 1: # 打开网页操作
|
||||
# if not (nodeId == 1 and self.service["cloudflare"] == 1):
|
||||
self.openPage(node["parameters"], loopValue)
|
||||
elif node["option"] == 2: # 点击元素
|
||||
self.clickElement(node["parameters"], loopValue, loopPath, index)
|
||||
elif node["option"] == 3: # 提取数据
|
||||
# 针对提取数据操作，设置操作开始的步骤，用于不小心关闭后的恢复的增量采集
|
||||
if self.totalSteps >= self.startSteps:
|
||||
self.getData(node["parameters"], loopValue, node["isInLoop"],
|
||||
parentPath=loopPath, index=index)
|
||||
self.saveData()
|
||||
else:
|
||||
# self.getDataStep += 1
|
||||
self.print_and_log("跳过第" + str(self.totalSteps) + "次提取数据。")
|
||||
self.print_and_log(
|
||||
"Skip the " + str(self.totalSteps) + "th data extraction.")
|
||||
self.totalSteps += 1 # 总步数加一
|
||||
elif node["option"] == 4: # 输入文字
|
||||
self.inputInfo(node["parameters"], loopValue)
|
||||
elif node["option"] == 5: # 自定义操作
|
||||
self.customOperation(node, loopValue, loopPath, index)
|
||||
self.saveData()
|
||||
else:
|
||||
# self.getDataStep += 1
|
||||
self.print_and_log("跳过第" + str(self.totalSteps) + "次提取数据。")
|
||||
self.print_and_log(
|
||||
"Skip the " + str(self.totalSteps) + "th data extraction.")
|
||||
self.totalSteps += 1 # 总步数加一
|
||||
elif node["option"] == 4: # 输入文字
|
||||
self.inputInfo(node["parameters"], loopValue)
|
||||
elif node["option"] == 5: # 自定义操作
|
||||
self.customOperation(node, loopValue, loopPath, index)
|
||||
self.saveData()
|
||||
elif node["option"] == 6: # 切换下拉框
|
||||
self.switchSelect(node["parameters"], loopValue)
|
||||
elif node["option"] == 7: # 鼠标移动到元素上
|
||||
self.moveToElement(node["parameters"], loopValue, loopPath, index)
|
||||
elif node["option"] == 8: # 循环
|
||||
self.loopExecute(node, loopValue, loopPath, index) # 执行循环
|
||||
elif node["option"] == 9: # 条件分支
|
||||
self.judgeExecute(node, loopValue, loopPath, index)
|
||||
elif node["option"] == 6: # 切换下拉框
|
||||
self.switchSelect(node["parameters"], loopValue)
|
||||
elif node["option"] == 7: # 鼠标移动到元素上
|
||||
self.moveToElement(node["parameters"], loopValue, loopPath, index)
|
||||
elif node["option"] == 8: # 循环
|
||||
self.loopExecute(node, loopValue, loopPath, index) # 执行循环
|
||||
elif node["option"] == 9: # 条件分支
|
||||
self.judgeExecute(node, loopValue, loopPath, index)
|
||||
except Exception as e:
|
||||
self.print_and_log("执行节点<" + node["title"] + ">时出错，将继续执行，错误为：", e)
|
||||
self.print_and_log("Error executing node <" + node["title"] + ">, will continue to execute, error is:", e)
|
||||
|
||||
|
||||
# 执行完之后进行等待
|
||||
if node["option"] != 0 and node["option"] != 2: # 点击元素操作单独定义等待时间操作
|
||||
@ -1637,7 +1642,7 @@ class BrowserThread(Thread):
|
||||
downloadPic = 0
|
||||
if downloadPic == 1:
|
||||
download_image(self, content, "Data/Task_" +
|
||||
str(self.id) + "/" + self.saveName + "/")
|
||||
str(self.id) + "/" + self.saveName + "/", element)
|
||||
else: # 普通节点
|
||||
content = element.text
|
||||
elif p["contentType"] == 1: # 只采集当期元素下的文本，不包括子元素
|
||||
@ -1662,7 +1667,7 @@ class BrowserThread(Thread):
|
||||
downloadPic = 0
|
||||
if downloadPic == 1:
|
||||
download_image(self, content, "Data/Task_" +
|
||||
str(self.id) + "/" + self.saveName + "/")
|
||||
str(self.id) + "/" + self.saveName + "/", element)
|
||||
else:
|
||||
command = 'var arr = [];\
|
||||
var content = arguments[0];\
|
||||
|
||||
@ -16,6 +16,8 @@ from lxml import etree
|
||||
import smtplib
|
||||
from email.mime.text import MIMEText
|
||||
from email.header import Header
|
||||
import urllib.request
|
||||
import base64
|
||||
|
||||
def send_email(config):
|
||||
"""
|
||||
@ -151,39 +153,73 @@ def detect_optimizable(para, ignoreWaitElement=True, waitElement=""):
|
||||
|
||||
|
||||
|
||||
def download_image(browser, url, save_directory):
|
||||
def download_image(browser, url, save_directory, element=None):
|
||||
# 定义浏览器头信息
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
||||
}
|
||||
if is_valid_url(url):
|
||||
if url.startswith("data:image"):
|
||||
base64_data = url.split(",")[1]
|
||||
image_data = base64.b64decode(base64_data)
|
||||
# 提取文件名
|
||||
file_name = str(uuid.uuid4()) + '.png'
|
||||
# 构建保存路径
|
||||
save_path = os.path.join(save_directory, file_name)
|
||||
# 保存图片到本地
|
||||
with open(save_path, 'wb') as file:
|
||||
file.write(image_data)
|
||||
browser.print_and_log("图片已成功下载到:", save_path)
|
||||
browser.print_and_log(
|
||||
"The image has been successfully downloaded to:", save_path)
|
||||
elif is_valid_url(url):
|
||||
try:
|
||||
# 发送 GET 请求获取图片数据
|
||||
response = requests.get(url, headers=headers)
|
||||
# 提取文件名
|
||||
file_name = url.split('/')[-1].split("?")[0]
|
||||
|
||||
# 生成唯一的新文件名
|
||||
new_file_name = file_name + '_' + \
|
||||
str(uuid.uuid4()) + '_' + file_name
|
||||
|
||||
# 构建保存路径
|
||||
save_path = os.path.join(save_directory, new_file_name)
|
||||
# 发送 GET 请求获取图片数据，加载浏览器的cookies
|
||||
s = requests.session()
|
||||
cookies = browser.browser.get_cookies()
|
||||
for cookie in cookies:
|
||||
s.cookies.set(cookie['name'], cookie['value'])
|
||||
response = s.get(url, headers=headers)
|
||||
# 检查响应状态码是否为成功状态
|
||||
if response.status_code == requests.codes.ok:
|
||||
# 提取文件名
|
||||
file_name = url.split('/')[-1].split("?")[0]
|
||||
|
||||
# 生成唯一的新文件名
|
||||
new_file_name = file_name + '_' + \
|
||||
str(uuid.uuid4()) + '_' + file_name
|
||||
|
||||
# 构建保存路径
|
||||
save_path = os.path.join(save_directory, new_file_name)
|
||||
|
||||
# 保存图片到本地
|
||||
with open(save_path, 'wb') as file:
|
||||
file.write(response.content)
|
||||
|
||||
browser.print_and_log("图片已成功下载到:", save_path)
|
||||
browser.print_and_log(
|
||||
"The image has been successfully downloaded to:", save_path)
|
||||
else:
|
||||
browser.print_and_log("下载图片失败，请检查此图片链接是否有效:", url)
|
||||
browser.print_and_log(
|
||||
"Failed to download image, please check if this image link is valid:", url)
|
||||
# browser.print_and_log(f"直接下载图片失败，状态码为:{response.status_code}，尝试使用Selenium下载图片...")
|
||||
# browser.print_and_log(
|
||||
# f"Failed to download image directly, status code is: {response.status_code}, try to download image using Selenium...")
|
||||
JS = "var xhr = new XMLHttpRequest(); xhr.open('GET', '" + url +"', true); xhr.responseType = 'blob'; xhr.onload = function() {var reader = new FileReader(); reader.readAsDataURL(xhr.response); reader.onloadend = function() { var base64data = reader.result;}}; xhr.send();"""
|
||||
base64data = browser.browser.execute_script(JS)
|
||||
if base64data:
|
||||
image_data = base64data.b64decode(base64data.split(",")[1])
|
||||
with open(save_path, 'wb') as file:
|
||||
file.write(image_data)
|
||||
browser.print_and_log("图片已成功下载到:", save_path)
|
||||
browser.print_and_log(
|
||||
"The image has been successfully downloaded to:", save_path)
|
||||
else:
|
||||
browser.print_and_log("下载图片失败，只能使用元素截图功能下载图片。")
|
||||
browser.print_and_log("Failed to download image, can only download image using element screenshot function.")
|
||||
# 使用元素截图功能下载图片
|
||||
try:
|
||||
element.screenshot(save_path)
|
||||
browser.print_and_log("图片截图已保存到:", save_path)
|
||||
browser.print_and_log(
|
||||
"The image screenshot has been saved to:", save_path)
|
||||
except Exception as e:
|
||||
browser.print_and_log("下载图片失败|Error downloading image: ", e)
|
||||
except Exception as e:
|
||||
browser.print_and_log("下载图片失败|Error downloading image: ", e)
|
||||
else:
|
||||
|
||||
11
Extension/manifest_v3/src/popup.css
Normal file
11
Extension/manifest_v3/src/popup.css
Normal file
@ -0,0 +1,11 @@
|
||||
body {
|
||||
/*width: 400px; !* 设置popup的宽度 *!*/
|
||||
}
|
||||
|
||||
h1 {
|
||||
/*color: blue; !* 设置标题颜色 *!*/
|
||||
}
|
||||
|
||||
button {
|
||||
background-color: green; /* 设置按钮背景颜色 */
|
||||
}
|
||||
11
Extension/manifest_v3/src/popup.html
Normal file
11
Extension/manifest_v3/src/popup.html
Normal file
@ -0,0 +1,11 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Popup 示例</title>
|
||||
<link rel="stylesheet" type="text/css" href="popup.css">
|
||||
</head>
|
||||
<body>
|
||||
<!-- <h2>EasySpider Extension</h2>-->
|
||||
EasySpider Extension, please do not disable me.
|
||||
</body>
|
||||
</html>
|
||||
3
Extension/manifest_v3/src/popup.js
Normal file
3
Extension/manifest_v3/src/popup.js
Normal file
@ -0,0 +1,3 @@
|
||||
document.getElementById('clickme').addEventListener('click', () => {
|
||||
alert('Hello, World!');
|
||||
});
|
||||
Loading…
x
Reference in New Issue
Block a user