From 62435c037ee75c535fc5ad731cf23cf4fa68eeed Mon Sep 17 00:00:00 2001 From: naibo Date: Fri, 7 Jul 2023 22:36:58 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8F=AF=E4=BB=A5=E9=80=89=E6=8B=A9=E9=83=A8?= =?UTF-8?q?=E5=88=86=E5=AD=97=E6=AE=B5=E4=B8=8D=E8=BE=93=E5=87=BA=E5=88=B0?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E6=88=96=E6=95=B0=E6=8D=AE=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ElectronJS/tasks/154.json | 1 + ExecuteStage/.vscode/launch.json | 2 +- ExecuteStage/easyspider_executestage.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 ElectronJS/tasks/154.json diff --git a/ElectronJS/tasks/154.json b/ElectronJS/tasks/154.json new file mode 100644 index 0000000..95e1cfa --- /dev/null +++ b/ElectronJS/tasks/154.json @@ -0,0 +1 @@ +{"id":154,"name":"中国地震台网——历史查询","url":"http://www.ceic.ac.cn/history","links":"http://www.ceic.ac.cn/history","create_time":"7/7/2023, 10:34:28 PM","version":"0.3.5","saveThreshold":10,"cloudflare":0,"environment":0,"maxViewLength":15,"outputFormat":"mysql","saveName":"地震台网","containJudge":true,"desc":"http://www.ceic.ac.cn/history","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"http://www.ceic.ac.cn/history","desc":"要采集的网址列表，多行以\\n分开","type":"text","exampleValue":"http://www.ceic.ac.cn/history"},{"id":1,"name":"inputText_1","nodeName":"输入文字","nodeId":2,"desc":"要输入的文本，如京东搜索框输入：电脑","type":"text","exampleValue":"1","value":"1"},{"id":2,"name":"loopTimes_循环_2","nodeId":4,"nodeName":"循环","desc":"循环循环执行的次数（0代表无限循环）","type":"int","exampleValue":0,"value":0}],"outputParameters":[{"id":0,"name":"参数1_文本","desc":"","type":"text","recordASField":1,"exampleValue":"4.0"},{"id":1,"name":"发震时刻","desc":"","type":"datetime","recordASField":1,"exampleValue":"2023-06-2900:43:34"},{"id":2,"name":"纬度","desc":"","type":"double","recordASField":0,"exampleValue":"34.35"},{"id":3,"name":"经度","desc":"","type":"double","recordASField":1,"exampleValue":"89.02"},{"id":4,"name":"深度（千米）","desc":"","type":"int","recordASField":1,"exampleValue":"8"},{"id":5,"name":"参考位置","desc":"","type":"text","recordASField":1,"exampleValue":"西藏那曲市安多县"},{"id":6,"name":"位置链接","desc":"","type":"text","recordASField":1,"exampleValue":"https://news.ceic.ac.cn/CC20230629004335.html"},{"id":7,"name":"自定义操作","desc":"自定义操作返回的数据","type":"text","recordASField":0,"exampleValue":""},{"id":8,"name":"自定义操作","desc":"自定义操作返回的数据","type":"text","recordASField":0,"exampleValue":""}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2,3,4],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0,"waitType":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"http://www.ceic.ac.cn/history","links":"http://www.ceic.ac.cn/history","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":2,"index":2,"parentId":0,"type":0,"option":4,"title":"输入文字","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"//*[@id=\"weidu1\"]","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"value":"1","allXPaths":["/html/body/div[1]/div[3]/div[1]/div[1]/div[1]/form[1]/div[2]/input[1]","//input[contains(., '')]","id(\"weidu1\")","//INPUT[@class='span1']","//INPUT[@name='weidu1']","/html/body/div[last()-3]/div[last()-1]/div/div/div[last()-1]/form/div[last()-3]/input[last()-1]"]}},{"id":3,"index":3,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":2,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"//*[@id=\"search\"]","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/div[1]/div[3]/div[1]/div[1]/div[1]/form[1]/div[5]/a[1]","//a[contains(., '查询')]","id(\"search\")","//A[@class='check']","/html/body/div[last()-3]/div[last()-1]/div/div/div[last()-1]/form/div/a"]}},{"id":4,"index":4,"parentId":0,"type":1,"option":8,"title":"循环","sequence":[6,12,5],"isInLoop":false,"position":3,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"//*[contains(@class, \"pagination\")]/ul[1]/li[last()-1]/a[1]","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":0,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":["/html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div[2]/div[1]/div[1]/ul[1]/li[10]/a[1]","//a[contains(., '»')]","/html/body/div[last()-3]/div[last()-1]/div/div/div/div/div/div/div/ul/li[last()-1]/a"]}},{"id":7,"index":5,"parentId":4,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":true,"position":2,"parameters":{"history":4,"tabIndex":-1,"useLoop":true,"xpath":"//*[contains(@class, \"pagination\")]/ul[1]/li[10]/a[1]","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div[2]/div[1]/div[1]/ul[1]/li[10]/a[1]","//a[contains(., '»')]","/html/body/div[last()-3]/div[last()-1]/div/div/div/div/div/div/div/ul/li[last()-1]/a"],"loopType":0}},{"id":5,"index":6,"parentId":4,"type":1,"option":8,"title":"循环","sequence":[8,7],"isInLoop":true,"position":0,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":2,"pathList":"//*[contains(@class, \"speed-table1\")]/tbody[1]/tr[2]\n//*[contains(@class, \"speed-table1\")]/tbody[1]/tr[3]\n//*[contains(@class, \"speed-table1\")]/tbody[1]/tr[4]\n//*[contains(@class, \"speed-table1\")]/tbody[1]/tr[5]\n//*[contains(@class, \"speed-table1\")]/tbody[1]/tr[6]\n//*[contains(@class, \"speed-table1\")]/tbody[1]/tr[7]\n//*[contains(@class, \"speed-table1\")]/tbody[1]/tr[8]\n//*[contains(@class, \"speed-table1\")]/tbody[1]/tr[9]\n//*[contains(@class, \"speed-table1\")]/tbody[1]/tr[10]\n//*[contains(@class, \"speed-table1\")]/tbody[1]/tr[11]\n//*[contains(@class, \"tr-red\")]\n//*[contains(@class, \"speed-table1\")]/tbody[1]/tr[13]\n//*[contains(@class, \"speed-table1\")]/tbody[1]/tr[14]\n//*[contains(@class, \"speed-table1\")]/tbody[1]/tr[15]\n//*[contains(@class, \"speed-table1\")]/tbody[1]/tr[16]\n//*[contains(@class, \"speed-table1\")]/tbody[1]/tr[17]\n//*[contains(@class, \"speed-table1\")]/tbody[1]/tr[18]\n//*[contains(@class, \"speed-table1\")]/tbody[1]/tr[19]\n//*[contains(@class, \"speed-table1\")]/tbody[1]/tr[20]\n//*[contains(@class, \"speed-table1\")]/tbody[1]/tr[21]","textList":"","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":""}},{"id":9,"index":7,"parentId":5,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"paras":[{"nodeType":0,"contentType":1,"relative":true,"name":"参数1_文本","desc":"","relativeXPath":"/td[1]","allXPaths":["/td[1]","//td[contains(., '4.0')]","/html/body/div[last()-3]/div[last()-1]/div/div/div/div/div[last()-1]/table/tbody/tr[last()-19]/td[last()-5]"],"exampleValues":[{"num":0,"value":"4.0"}],"unique_index":"/td[1]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":0,"contentType":1,"relative":true,"name":"发震时刻","desc":"","relativeXPath":"/td[2]","allXPaths":["/td[2]","//td[contains(., '2023-06-29')]","/html/body/div[last()-3]/div[last()-1]/div/div/div/div/div[last()-1]/table/tbody/tr[last()-19]/td[last()-4]"],"exampleValues":[{"num":0,"value":"2023-06-2900:43:34"}],"unique_index":"/td[2]","iframe":false,"default":"","paraType":"datetime","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":0,"contentType":1,"relative":true,"name":"纬度","desc":"","relativeXPath":"/td[3]","allXPaths":["/td[3]","//td[contains(., '34.35')]","/html/body/div[last()-3]/div[last()-1]/div/div/div/div/div[last()-1]/table/tbody/tr[last()-19]/td[last()-3]"],"exampleValues":[{"num":0,"value":"34.35"}],"unique_index":"/td[3]","iframe":false,"default":"","paraType":"double","recordASField":0,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":0,"contentType":1,"relative":true,"name":"经度","desc":"","relativeXPath":"/td[4]","allXPaths":["/td[4]","//td[contains(., '89.02')]","/html/body/div[last()-3]/div[last()-1]/div/div/div/div/div[last()-1]/table/tbody/tr[last()-19]/td[last()-2]"],"exampleValues":[{"num":0,"value":"89.02"}],"unique_index":"/td[4]","iframe":false,"default":"","paraType":"double","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":0,"contentType":1,"relative":true,"name":"深度（千米）","desc":"","relativeXPath":"/td[5]","allXPaths":["/td[5]","//td[contains(., '8')]","/html/body/div[last()-3]/div[last()-1]/div/div/div/div/div[last()-1]/table/tbody/tr[last()-19]/td[last()-1]"],"exampleValues":[{"num":0,"value":"8"}],"unique_index":"/td[5]","iframe":false,"default":"","paraType":"int","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":1,"contentType":0,"relative":true,"name":"参考位置","desc":"","relativeXPath":"/td[6]/a[1]","allXPaths":["/td[6]/a[1]","//a[contains(., '西藏那曲市安多县')]","id(\"cid\")","/html/body/div[last()-3]/div[last()-1]/div/div/div/div/div[last()-1]/table/tbody/tr[last()-19]/td/a"],"exampleValues":[{"num":0,"value":"西藏那曲市安多县"}],"unique_index":"/td[6]/a[1]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":2,"contentType":0,"relative":true,"name":"位置链接","desc":"","relativeXPath":"/td[6]/a[1]","allXPaths":["/td[6]/a[1]","//a[contains(., '西藏那曲市安多县')]","id(\"cid\")","/html/body/div[last()-3]/div[last()-1]/div/div/div/div/div[last()-1]/table/tbody/tr[last()-19]/td/a"],"exampleValues":[{"num":0,"value":"https://news.ceic.ac.cn/CC20230629004335.html"}],"unique_index":"/td[6]/a[1]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}],"loopType":2}},{"id":8,"index":8,"parentId":5,"type":2,"option":9,"title":"判断条件","sequence":[9],"isInLoop":true,"position":0,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0}},{"id":11,"parentId":8,"index":9,"type":3,"option":10,"title":"条件分支","sequence":[11],"isInLoop":true,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"class":"3","value":"四川","code":"","waitTime":0},"position":0},{"id":-1,"parentId":7,"index":10,"type":3,"option":10,"title":"条件分支","sequence":[],"isInLoop":true,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"class":0,"value":"","code":"","waitTime":0},"position":1},{"id":13,"index":11,"parentId":11,"type":0,"option":5,"title":"自定义操作","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"codeMode":"3","code":"","waitTime":0,"recordASField":0,"paraType":"text"}},{"id":6,"index":12,"parentId":4,"type":2,"option":9,"title":"判断条件","sequence":[13],"isInLoop":true,"position":1,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0}},{"id":10,"parentId":6,"index":13,"type":3,"option":10,"title":"条件分支","sequence":[15],"isInLoop":true,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"class":"1","value":"克什米尔","code":"","waitTime":0},"position":0},{"id":-1,"parentId":6,"index":14,"type":3,"option":10,"title":"条件分支","sequence":[],"isInLoop":true,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"class":0,"value":"","code":"","waitTime":0},"position":1},{"id":12,"index":15,"parentId":10,"type":0,"option":5,"title":"自定义操作","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"codeMode":"3","code":"","waitTime":0,"recordASField":0,"paraType":"text"}}]} \ No newline at end of file diff --git a/ExecuteStage/.vscode/launch.json b/ExecuteStage/.vscode/launch.json index 6c24531..d0a4d21 100644 --- a/ExecuteStage/.vscode/launch.json +++ b/ExecuteStage/.vscode/launch.json @@ -12,7 +12,7 @@ "justMyCode": true, // "args": ["--id", "[7]", "--read_type", "remote", "--headless", "0"] // "args": ["--id", "[9]", "--read_type", "remote", "--headless", "0", "--saved_file_name", "YOUTUBE"] - "args": ["--id", "[69]", "--headless", "0", "--user_data", "1"] + "args": ["--id", "[73]", "--headless", "0", "--user_data", "1"] } ] } \ No newline at end of file diff --git a/ExecuteStage/easyspider_executestage.py b/ExecuteStage/easyspider_executestage.py index 8aca3ed..4f84572 100644 --- a/ExecuteStage/easyspider_executestage.py +++ b/ExecuteStage/easyspider_executestage.py @@ -360,6 +360,7 @@ class BrowserThread(Thread): paras = node["parameters"] codeMode = int(paras["codeMode"]) code = paras["code"] + output = "" max_wait_time = int(paras["waitTime"]) if codeMode == 2: # 使用循环的情况下，传入的clickPath就是实际的xpath try: