diff --git a/ElectronJS/package-lock.json b/ElectronJS/package-lock.json index 6d6f770..4480478 100644 --- a/ElectronJS/package-lock.json +++ b/ElectronJS/package-lock.json @@ -26,7 +26,7 @@ "@electron-forge/maker-rpm": "^6.0.5", "@electron-forge/maker-squirrel": "^6.0.5", "@electron-forge/maker-zip": "^6.0.5", - "electron": "^25.0.1" + "electron": "^20.0.1" } }, "node_modules/@electron-forge/cli": { @@ -1676,6 +1676,17 @@ "typedarray": "^0.0.6" } }, + "node_modules/config-chain": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.13.tgz", + "integrity": "sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==", + "dev": true, + "optional": true, + "dependencies": { + "ini": "^1.3.4", + "proto-list": "~1.2.1" + } + }, "node_modules/console-control-strings": { "version": "1.1.0", "dev": true, @@ -1966,6 +1977,12 @@ "minimatch": "^3.0.4" } }, + "node_modules/duplexer3": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.5.tgz", + "integrity": "sha512-1A8za6ws41LQgv9HrE/66jyC5yuSjQ3L/KOpFtoBilsAK2iA2wuS5rTt1OCzIvtS2V7nVmedsUU+DGRcjBmOYA==", + "dev": true + }, "node_modules/eastasianwidth": { "version": "0.2.0", "dev": true, @@ -1977,21 +1994,21 @@ "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, "node_modules/electron": { - "version": "25.2.0", - "resolved": "https://registry.npmjs.org/electron/-/electron-25.2.0.tgz", - "integrity": "sha512-I/rhcW2sV2fyiveVSBr2N7v5ZiCtdGY0UiNCDZgk2fpSC+irQjbeh7JT2b4vWmJ2ogOXBjqesrN9XszTIG6DHg==", + "version": "20.3.12", + "resolved": "https://registry.npmjs.org/electron/-/electron-20.3.12.tgz", + "integrity": "sha512-CuCZKhwdSXaUNIoEuPVZ25YH18zmox3wEM8Acwcy9tlpD8Jiuq3ji3RZ98eVWJalrpqbdE9LtTD/sLC86GIkLg==", "dev": true, "hasInstallScript": true, "dependencies": { - "@electron/get": "^2.0.0", - "@types/node": "^18.11.18", + "@electron/get": "^1.14.1", + "@types/node": "^16.11.26", "extract-zip": "^2.0.1" }, "bin": { "electron": "cli.js" }, "engines": { - "node": ">= 12.20.55" + "node": ">= 10.17.0" } }, "node_modules/electron-installer-common": { @@ -2159,6 +2176,226 @@ "node": ">= 4.0.0" } }, + "node_modules/electron/node_modules/@electron/get": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@electron/get/-/get-1.14.1.tgz", + "integrity": "sha512-BrZYyL/6m0ZXz/lDxy/nlVhQz+WF+iPS6qXolEU8atw7h6v1aYkjwJZ63m+bJMBTxDE66X+r2tPS4a/8C82sZw==", + "dev": true, + "dependencies": { + "debug": "^4.1.1", + "env-paths": "^2.2.0", + "fs-extra": "^8.1.0", + "got": "^9.6.0", + "progress": "^2.0.3", + "semver": "^6.2.0", + "sumchecker": "^3.0.1" + }, + "engines": { + "node": ">=8.6" + }, + "optionalDependencies": { + "global-agent": "^3.0.0", + "global-tunnel-ng": "^2.7.1" + } + }, + "node_modules/electron/node_modules/@sindresorhus/is": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz", + "integrity": "sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/electron/node_modules/@szmarczak/http-timer": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-1.1.2.tgz", + "integrity": "sha512-XIB2XbzHTN6ieIjfIMV9hlVcfPU26s2vafYWQcZHWXHOxiaRZYEDKEwdl129Zyg50+foYV2jCgtrqSA6qNuNSA==", + "dev": true, + "dependencies": { + "defer-to-connect": "^1.0.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/electron/node_modules/@types/node": { + "version": "16.18.38", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.18.38.tgz", + "integrity": "sha512-6sfo1qTulpVbkxECP+AVrHV9OoJqhzCsfTNp5NIG+enM4HyM3HvZCO798WShIXBN0+QtDIcutJCjsVYnQP5rIQ==", + "dev": true + }, + "node_modules/electron/node_modules/cacheable-request": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz", + "integrity": "sha512-Oj3cAGPCqOZX7Rz64Uny2GYAZNliQSqfbePrgAQ1wKAihYmCUnraBtJtKcGR4xz7wF+LoJC+ssFZvv5BgF9Igg==", + "dev": true, + "dependencies": { + "clone-response": "^1.0.2", + "get-stream": "^5.1.0", + "http-cache-semantics": "^4.0.0", + "keyv": "^3.0.0", + "lowercase-keys": "^2.0.0", + "normalize-url": "^4.1.0", + "responselike": "^1.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/electron/node_modules/decompress-response": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", + "integrity": "sha512-BzRPQuY1ip+qDonAOz42gRm/pg9F768C+npV/4JOsxRC2sq+Rlk+Q4ZCAsOhnIaMrgarILY+RMUIvMmmX1qAEA==", + "dev": true, + "dependencies": { + "mimic-response": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/electron/node_modules/defer-to-connect": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.1.3.tgz", + "integrity": "sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ==", + "dev": true + }, + "node_modules/electron/node_modules/fs-extra": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + }, + "engines": { + "node": ">=6 <7 || >=8" + } + }, + "node_modules/electron/node_modules/got": { + "version": "9.6.0", + "resolved": "https://registry.npmjs.org/got/-/got-9.6.0.tgz", + "integrity": "sha512-R7eWptXuGYxwijs0eV+v3o6+XH1IqVK8dJOEecQfTmkncw9AV4dcw/Dhxi8MdlqPthxxpZyizMzyg8RTmEsG+Q==", + "dev": true, + "dependencies": { + "@sindresorhus/is": "^0.14.0", + "@szmarczak/http-timer": "^1.1.2", + "cacheable-request": "^6.0.0", + "decompress-response": "^3.3.0", + "duplexer3": "^0.1.4", + "get-stream": "^4.1.0", + "lowercase-keys": "^1.0.1", + "mimic-response": "^1.0.1", + "p-cancelable": "^1.0.0", + "to-readable-stream": "^1.0.0", + "url-parse-lax": "^3.0.0" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/electron/node_modules/got/node_modules/get-stream": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", + "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "dev": true, + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/electron/node_modules/got/node_modules/lowercase-keys": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", + "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/electron/node_modules/json-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.0.tgz", + "integrity": "sha512-CuUqjv0FUZIdXkHPI8MezCnFCdaTAacej1TZYulLoAg1h/PhwkdXFN4V/gzY4g+fMBCOV2xF+rp7t2XD2ns/NQ==", + "dev": true + }, + "node_modules/electron/node_modules/jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==", + "dev": true, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/electron/node_modules/keyv": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-3.1.0.tgz", + "integrity": "sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA==", + "dev": true, + "dependencies": { + "json-buffer": "3.0.0" + } + }, + "node_modules/electron/node_modules/normalize-url": { + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.1.tgz", + "integrity": "sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/electron/node_modules/p-cancelable": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-1.1.0.tgz", + "integrity": "sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/electron/node_modules/responselike": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/responselike/-/responselike-1.0.2.tgz", + "integrity": "sha512-/Fpe5guzJk1gPqdJLJR5u7eG/gNY4nImjbRDaVWVMRhne55TCmj2i9Q+54PBRfatRC8v/rIiv9BN0pMd9OV5EQ==", + "dev": true, + "dependencies": { + "lowercase-keys": "^1.0.0" + } + }, + "node_modules/electron/node_modules/responselike/node_modules/lowercase-keys": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", + "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/electron/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/electron/node_modules/universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", + "dev": true, + "engines": { + "node": ">= 4.0.0" + } + }, "node_modules/emoji-regex": { "version": "8.0.0", "dev": true, @@ -2917,6 +3154,22 @@ "which": "bin/which" } }, + "node_modules/global-tunnel-ng": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/global-tunnel-ng/-/global-tunnel-ng-2.7.1.tgz", + "integrity": "sha512-4s+DyciWBV0eK148wqXxcmVAbFVPqtc3sEtUE/GTQfuU80rySLcMhUmHKSHI7/LDj8q0gDYI1lIhRRB7ieRAqg==", + "dev": true, + "optional": true, + "dependencies": { + "encodeurl": "^1.0.2", + "lodash": "^4.17.10", + "npm-conf": "^1.1.3", + "tunnel": "^0.0.6" + }, + "engines": { + "node": ">=0.10" + } + }, "node_modules/globalthis": { "version": "1.0.3", "dev": true, @@ -4043,6 +4296,30 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/npm-conf": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/npm-conf/-/npm-conf-1.1.3.tgz", + "integrity": "sha512-Yic4bZHJOt9RCFbRP3GgpqhScOY4HH3V2P8yBj6CeYq118Qr+BLXqT2JvpJ00mryLESpgOxf5XlFv4ZjXxLScw==", + "dev": true, + "optional": true, + "dependencies": { + "config-chain": "^1.1.11", + "pify": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/npm-conf/node_modules/pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==", + "dev": true, + "optional": true, + "engines": { + "node": ">=4" + } + }, "node_modules/npm-run-path": { "version": "2.0.2", "dev": true, @@ -4451,6 +4728,15 @@ "node": ">=6" } }, + "node_modules/prepend-http": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz", + "integrity": "sha512-ravE6m9Atw9Z/jjttRUZ+clIXogdghyZAuWJ3qEzjT+jI/dL1ifAqhZeC5VHzQp1MSt1+jxKkFNemj/iO7tVUA==", + "dev": true, + "engines": { + "node": ">=4" + } + }, "node_modules/process-nextick-args": { "version": "2.0.1", "license": "MIT" @@ -4475,6 +4761,13 @@ "node": ">=10" } }, + "node_modules/proto-list": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", + "integrity": "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==", + "dev": true, + "optional": true + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -5339,6 +5632,15 @@ "tmp": "^0.2.0" } }, + "node_modules/to-readable-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-1.0.0.tgz", + "integrity": "sha512-Iq25XBt6zD5npPhlLVXGFN3/gyR2/qODcKNNyTMd4vbm39HUaOiAM4PMq0eMVC/Tkxz+Zjdsc55g9yyz+Yq00Q==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/to-regex-range": { "version": "5.0.1", "dev": true, @@ -5387,6 +5689,16 @@ "dev": true, "license": "0BSD" }, + "node_modules/tunnel": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", + "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==", + "dev": true, + "optional": true, + "engines": { + "node": ">=0.6.11 <=0.7.0 || >=0.7.3" + } + }, "node_modules/type-fest": { "version": "0.21.3", "dev": true, @@ -5453,6 +5765,18 @@ "node": ">= 0.8" } }, + "node_modules/url-parse-lax": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz", + "integrity": "sha512-NjFKA0DidqPa5ciFcSrXnAltTtzz84ogy+NebPvfEgAck0+TNg4UJ4IN+fB7zRZfbgUf0syOo9MDxFkDSMuFaQ==", + "dev": true, + "dependencies": { + "prepend-http": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/username": { "version": "5.1.0", "dev": true, diff --git a/ElectronJS/package.json b/ElectronJS/package.json index ca601a2..e19f66f 100644 --- a/ElectronJS/package.json +++ b/ElectronJS/package.json @@ -27,7 +27,7 @@ "@electron-forge/maker-rpm": "^6.0.5", "@electron-forge/maker-squirrel": "^6.0.5", "@electron-forge/maker-zip": "^6.0.5", - "electron": "^25.0.1" + "electron": "^20.0.1" }, "repository": "https://github.com/NaiboWang/EasySpider", "dependencies": { diff --git a/ElectronJS/src/taskGrid/FlowChart_CN.html b/ElectronJS/src/taskGrid/FlowChart_CN.html index d2a5cb9..3ba120c 100644 --- a/ElectronJS/src/taskGrid/FlowChart_CN.html +++ b/ElectronJS/src/taskGrid/FlowChart_CN.html @@ -524,8 +524,10 @@ CSV XLSX(EXCEL) - MySQL + MySQL数据库 + 导出文件名/数据库表格名称(如果值为“Time”则名称为执行任务时的时间): + 浏览器模拟类型: 电脑端 diff --git a/ElectronJS/src/taskGrid/invokeTask.html b/ElectronJS/src/taskGrid/invokeTask.html index 8e228ac..7d6fb34 100644 --- a/ElectronJS/src/taskGrid/invokeTask.html +++ b/ElectronJS/src/taskGrid/invokeTask.html @@ -118,26 +118,38 @@ {{"Argument Value 2_1~参数值2_2" | lang}} ... + + ... + ... + ... + {{"The 'Invoke Name' is shown at the main table of this page, such as 'urlList_0' is the invoke name of the open page operation.~“调用名称”在本页面的主表格中显示,例如“urlList_0”是“打开网页“操作的调用名称。" | lang}} {{"You can just put part of the arguments in the Excel file, and the values of the rest of the arguments will be set to default. Example table for this task is:~您可以只在Excel文件中放入部分参数,其余参数值将被设置为默认值。一个针对此任务的表格示例为:" | lang}} + + + + {{task.inputParameters[i-1]["name"]}} + + + - {{task.inputParameters[i-1]["name"]}} - - - - - {{task.inputParameters[i-1]['value']}} + {{getLine(i,0)}} - - {{task.inputParameters[i-1]['value']}} + {{getLine(i,1)}} + + + + + + ... @@ -276,7 +288,18 @@ url = "taskInfo.html?id="+getUrlParam("id")+"&lang=en&type="+getUrlParam("type")+"&wsport="+getUrlParam("wsport")+"&backEndAddressServiceWrapper=" + app.$data.backEndAddressServiceWrapper } window.location.href = url; - }, readFromExcel: function(){ + }, getLine: function(i, index){ + const value = this.task.inputParameters[i-1]["value"].toString(); + const parts = value.split("\n"); + if(parts.length > index){ + return parts[index]; + } else if(this.task.inputParameters[i-1]["name"].indexOf("url") >=0){ + return parts[0]; + } else { + return ""; + } + }, + readFromExcel: function(){ $('#excelModal').modal('show'); }, submitFile: function() { let form_data = new FormData(); diff --git a/ElectronJS/src/taskGrid/logic_CN.js b/ElectronJS/src/taskGrid/logic_CN.js index 1c0d645..26fbd65 100644 --- a/ElectronJS/src/taskGrid/logic_CN.js +++ b/ElectronJS/src/taskGrid/logic_CN.js @@ -429,6 +429,7 @@ function saveService(type) { "environment": environment, "maxViewLength": parseInt($("#maxViewLength").val()), "outputFormat": $("#outputFormat").val(), + "saveName": $("#saveName").val(), "containJudge": containJudge, "desc": serviceDescription, "inputParameters": inputParameters, diff --git a/ElectronJS/tasks/63.json b/ElectronJS/tasks/63.json index 8ce18af..4474089 100644 --- a/ElectronJS/tasks/63.json +++ b/ElectronJS/tasks/63.json @@ -1 +1 @@ -{"id":63,"name":"地震台网","url":"http://www.ceic.ac.cn/history","links":"http://www.ceic.ac.cn/history","create_time":"5/21/2023, 5:59:42 PM","containJudge":false,"desc":"http://www.ceic.ac.cn/history","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"http://www.ceic.ac.cn/history","desc":"要采集的网址列表,多行以\\n分开","type":"string","exampleValue":"http://www.ceic.ac.cn/history"},{"id":1,"name":"loopText_1","nodeId":2,"nodeName":"循环","desc":"要输入的文本/网址,多行以\\n分开","type":"string","exampleValue":"1\n2\n3\n4","value":"1\n2\n3\n4"},{"id":2,"name":"loopTimes_循环_2","nodeId":5,"nodeName":"循环","desc":"循环循环执行的次数(0代表无限循环)","type":"int","exampleValue":2,"value":2}],"outputParameters":[{"id":0,"name":"参数1_文本","desc":"","type":"string","exampleValue":"3.12023-05-09 14:12:2241.0778.8410新疆阿克苏地区乌什县"}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"http://www.ceic.ac.cn/history","links":"http://www.ceic.ac.cn/history","maxWaitTime":10,"scrollType":0,"scrollCount":0}},{"id":2,"index":2,"parentId":0,"type":1,"option":8,"title":"循环","sequence":[3,4,5],"isInLoop":false,"position":1,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":0,"loopType":"3","pathList":"","textList":"1\n2\n3\n4","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":"1","breakCode":"return window.innerHeight > 500","breakCodeWaitTime":0}},{"id":3,"index":3,"parentId":2,"type":0,"option":4,"title":"输入文字","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":4,"tabIndex":0,"useLoop":true,"xpath":"//*[@id=\"weidu1\"]","wait":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"value":"1","allXPaths":["/html/body/div[1]/div[3]/div[1]/div[1]/div[1]/form[1]/div[2]/input[1]","//input[contains(., '')]","id(\"weidu1\")","//INPUT[@class='span1']","//INPUT[@name='weidu1']"]}},{"id":4,"index":4,"parentId":2,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":true,"position":1,"parameters":{"history":4,"tabIndex":0,"useLoop":false,"xpath":"//*[@id=\"search\"]","wait":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/div[1]/div[3]/div[1]/div[1]/div[1]/form[1]/div[5]/a[1]","//a[contains(., '查询')]","id(\"search\")","//A[@class='check']"]}},{"id":5,"index":5,"parentId":2,"type":1,"option":8,"title":"循环","sequence":[7,6],"isInLoop":true,"position":2,"parameters":{"history":4,"tabIndex":0,"useLoop":false,"xpath":"//a[contains(., '»')]","wait":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":0,"loopType":0,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":2,"historyWait":2,"breakMode":"1","breakCode":"return window.innerWidth > 500","breakCodeWaitTime":0,"allXPaths":["/html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div[2]/div[1]/div[1]/ul[1]/li[10]/a[1]","//a[contains(., '»')]"]}},{"id":7,"index":6,"parentId":5,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":true,"position":1,"parameters":{"history":4,"tabIndex":0,"useLoop":true,"xpath":"//*[contains(@class, \"pagination\")]/ul[1]/li[10]/a[1]","wait":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div[2]/div[1]/div[1]/ul[1]/li[10]/a[1]","//a[contains(., '»')]"],"loopType":0}},{"id":6,"index":7,"parentId":5,"type":1,"option":8,"title":"循环","sequence":[8],"isInLoop":true,"position":0,"parameters":{"history":4,"tabIndex":0,"useLoop":false,"xpath":"/html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div[1]/table[1]/tbody[1]/tr","wait":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":0,"loopType":1,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":["/html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div[1]/table[1]/tbody[1]/tr[1]","//tr[contains(., '震级(M)发震时刻(')]","//TR[@class='speed-tr-h1']"]}},{"id":8,"index":8,"parentId":6,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":4,"tabIndex":0,"useLoop":false,"xpath":"","wait":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"paras":[{"nodeType":0,"contentType":0,"relative":true,"name":"参数1_文本","desc":"","extractType":0,"relativeXPath":"","allXPaths":"","exampleValues":[{"num":0,"value":"3.12023-05-09 14:12:2241.0778.8410新疆阿克苏地区乌什县"},{"num":1,"value":"震级(M)发震时刻(UTC+8)纬度(°)经度(°)深度(千米)参考位置"},{"num":2,"value":"2.32023-05-09 12:31:4637.51115.0710河北邢台市宁晋县"},{"num":3,"value":"3.02023-05-08 04:06:4629.72103.2618四川眉山市洪雅县"},{"num":4,"value":"3.32023-05-06 21:23:4529.61102.038四川甘孜州泸定县"},{"num":5,"value":"3.02023-05-06 20:39:1130.67117.0510安徽安庆市宜秀区"},{"num":6,"value":"3.72023-05-06 13:01:5729.15105.4510四川泸州市泸县"},{"num":7,"value":"3.12023-05-06 10:19:4347.73130.858黑龙江鹤岗市萝北县"},{"num":8,"value":"5.42023-05-06 01:47:1741.45142.1040日本北海道地区"},{"num":9,"value":"5.72023-05-05 20:58:0537.35137.3510日本本州西岸近海"},{"num":10,"value":"4.32023-05-05 20:33:1928.08105.128四川宜宾市兴文县"},{"num":11,"value":"6.32023-05-05 13:42:0537.40137.4010日本本州西岸近海"},{"num":12,"value":"3.12023-05-05 11:32:0425.1298.1312云南德宏州盈江县"},{"num":13,"value":"3.22023-05-05 10:28:3925.1498.118云南德宏州盈江县"},{"num":14,"value":"3.42023-05-05 08:35:2635.7979.8610新疆和田地区和田县"},{"num":15,"value":"4.52023-05-04 23:38:2123.46121.3310台湾花莲县"},{"num":16,"value":"2.12023-05-04 19:00:3138.46117.979河北沧州市黄骅市海域"},{"num":17,"value":"3.02023-05-04 15:53:4640.8083.9115新疆阿克苏地区沙雅县"},{"num":18,"value":"3.32023-05-04 14:06:2428.92102.4715四川雅安市石棉县"},{"num":19,"value":"3.32023-05-04 11:46:2433.34119.2814江苏扬州市宝应县"},{"num":20,"value":"4.92023-05-04 10:15:2828.14105.1210四川宜宾市兴文县"}],"default":"","beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}],"loopType":1}}]} \ No newline at end of file +{"id":63,"name":"地震台网","url":"http://www.ceic.ac.cn/history","links":"http://www.ceic.ac.cn/history\nhttp://www.ceic.ac.cn/history2","create_time":"7/7/2023, 2:15:59 AM","version":"0.3.5","saveThreshold":10,"cloudflare":0,"environment":0,"maxViewLength":15,"outputFormat":"xlsx","saveName":"地震信息","containJudge":false,"desc":"http://www.ceic.ac.cn/history","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"http://www.ceic.ac.cn/history\nhttp://www.ceic.ac.cn/history2","desc":"要采集的网址列表,多行以\\n分开","type":"string","exampleValue":"http://www.ceic.ac.cn/history\nhttp://www.ceic.ac.cn/history2"},{"id":1,"name":"loopText_1","nodeId":2,"nodeName":"循环","desc":"要输入的文本/网址,多行以\\n分开","type":"string","exampleValue":"1\n2\n3\n4","value":"1\n2\n3\n4"},{"id":2,"name":"loopTimes_循环_2","nodeId":5,"nodeName":"循环","desc":"循环循环执行的次数(0代表无限循环)","type":"int","exampleValue":2,"value":2}],"outputParameters":[{"id":0,"name":"参数1_文本","desc":"","type":"string","exampleValue":"3.12023-05-09 14:12:2241.0778.8410新疆阿克苏地区乌什县"}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"http://www.ceic.ac.cn/history","links":"http://www.ceic.ac.cn/history\nhttp://www.ceic.ac.cn/history2","maxWaitTime":10,"scrollType":0,"scrollCount":0,"cookies":""}},{"id":2,"index":2,"parentId":0,"type":1,"option":8,"title":"循环","sequence":[3,4,5],"isInLoop":false,"position":1,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":0,"loopType":"3","pathList":"","textList":"1\n2\n3\n4","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":"1","breakCode":"return window.innerHeight > 500","breakCodeWaitTime":0}},{"id":3,"index":3,"parentId":2,"type":0,"option":4,"title":"输入文字","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":4,"tabIndex":0,"useLoop":true,"xpath":"//*[@id=\"weidu1\"]","wait":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"value":"1","allXPaths":["/html/body/div[1]/div[3]/div[1]/div[1]/div[1]/form[1]/div[2]/input[1]","//input[contains(., '')]","id(\"weidu1\")","//INPUT[@class='span1']","//INPUT[@name='weidu1']"]}},{"id":4,"index":4,"parentId":2,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":true,"position":1,"parameters":{"history":4,"tabIndex":0,"useLoop":false,"xpath":"//*[@id=\"search\"]","wait":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/div[1]/div[3]/div[1]/div[1]/div[1]/form[1]/div[5]/a[1]","//a[contains(., '查询')]","id(\"search\")","//A[@class='check']"]}},{"id":5,"index":5,"parentId":2,"type":1,"option":8,"title":"循环","sequence":[7,6],"isInLoop":true,"position":2,"parameters":{"history":4,"tabIndex":0,"useLoop":false,"xpath":"//a[contains(., '»')]","wait":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":0,"loopType":0,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":2,"historyWait":2,"breakMode":"1","breakCode":"return window.innerWidth > 500","breakCodeWaitTime":0,"allXPaths":["/html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div[2]/div[1]/div[1]/ul[1]/li[10]/a[1]","//a[contains(., '»')]"]}},{"id":7,"index":6,"parentId":5,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":true,"position":1,"parameters":{"history":4,"tabIndex":0,"useLoop":true,"xpath":"//*[contains(@class, \"pagination\")]/ul[1]/li[10]/a[1]","wait":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div[2]/div[1]/div[1]/ul[1]/li[10]/a[1]","//a[contains(., '»')]"],"loopType":0}},{"id":6,"index":7,"parentId":5,"type":1,"option":8,"title":"循环","sequence":[8],"isInLoop":true,"position":0,"parameters":{"history":4,"tabIndex":0,"useLoop":false,"xpath":"/html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div[1]/table[1]/tbody[1]/tr","wait":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":0,"loopType":1,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":["/html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div[1]/table[1]/tbody[1]/tr[1]","//tr[contains(., '震级(M)发震时刻(')]","//TR[@class='speed-tr-h1']"]}},{"id":8,"index":8,"parentId":6,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":4,"tabIndex":0,"useLoop":false,"xpath":"","wait":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"paras":[{"nodeType":0,"contentType":0,"relative":true,"name":"参数1_文本","desc":"","extractType":0,"relativeXPath":"","allXPaths":"","exampleValues":[{"num":0,"value":"3.12023-05-09 14:12:2241.0778.8410新疆阿克苏地区乌什县"},{"num":1,"value":"震级(M)发震时刻(UTC+8)纬度(°)经度(°)深度(千米)参考位置"},{"num":2,"value":"2.32023-05-09 12:31:4637.51115.0710河北邢台市宁晋县"},{"num":3,"value":"3.02023-05-08 04:06:4629.72103.2618四川眉山市洪雅县"},{"num":4,"value":"3.32023-05-06 21:23:4529.61102.038四川甘孜州泸定县"},{"num":5,"value":"3.02023-05-06 20:39:1130.67117.0510安徽安庆市宜秀区"},{"num":6,"value":"3.72023-05-06 13:01:5729.15105.4510四川泸州市泸县"},{"num":7,"value":"3.12023-05-06 10:19:4347.73130.858黑龙江鹤岗市萝北县"},{"num":8,"value":"5.42023-05-06 01:47:1741.45142.1040日本北海道地区"},{"num":9,"value":"5.72023-05-05 20:58:0537.35137.3510日本本州西岸近海"},{"num":10,"value":"4.32023-05-05 20:33:1928.08105.128四川宜宾市兴文县"},{"num":11,"value":"6.32023-05-05 13:42:0537.40137.4010日本本州西岸近海"},{"num":12,"value":"3.12023-05-05 11:32:0425.1298.1312云南德宏州盈江县"},{"num":13,"value":"3.22023-05-05 10:28:3925.1498.118云南德宏州盈江县"},{"num":14,"value":"3.42023-05-05 08:35:2635.7979.8610新疆和田地区和田县"},{"num":15,"value":"4.52023-05-04 23:38:2123.46121.3310台湾花莲县"},{"num":16,"value":"2.12023-05-04 19:00:3138.46117.979河北沧州市黄骅市海域"},{"num":17,"value":"3.02023-05-04 15:53:4640.8083.9115新疆阿克苏地区沙雅县"},{"num":18,"value":"3.32023-05-04 14:06:2428.92102.4715四川雅安市石棉县"},{"num":19,"value":"3.32023-05-04 11:46:2433.34119.2814江苏扬州市宝应县"},{"num":20,"value":"4.92023-05-04 10:15:2828.14105.1210四川宜宾市兴文县"}],"default":"","beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}],"loopType":1}},{"id":-1,"index":9,"parentId":2,"type":0,"option":4,"title":"输入文字","sequence":[],"isInLoop":true,"position":3,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"value":""}}]} \ No newline at end of file diff --git a/ExecuteStage/.vscode/launch.json b/ExecuteStage/.vscode/launch.json index bbbe8d7..ce66f1e 100644 --- a/ExecuteStage/.vscode/launch.json +++ b/ExecuteStage/.vscode/launch.json @@ -12,7 +12,7 @@ "justMyCode": true, // "args": ["--id", "[7]", "--read_type", "remote", "--headless", "0"] // "args": ["--id", "[9]", "--read_type", "remote", "--headless", "0", "--saved_file_name", "YOUTUBE"] - "args": ["--id", "[16]", "--headless", "0", "--user_data", "1"] + "args": ["--id", "[124]", "--headless", "0", "--user_data", "1"] } ] } \ No newline at end of file diff --git a/ExecuteStage/easyspider_executestage.py b/ExecuteStage/easyspider_executestage.py index 7b51e85..f3a9207 100644 --- a/ExecuteStage/easyspider_executestage.py +++ b/ExecuteStage/easyspider_executestage.py @@ -48,16 +48,38 @@ desired_capabilities["pageLoadStrategy"] = "none" class BrowserThread(Thread): - def __init__(self, browser_t, id, service, version, event, config): + def __init__(self, browser_t, id, service, version, event, saveName, config): Thread.__init__(self) self.browser = browser_t self.config = config self.id = id self.event = event - self.saveName = saveName + try: + self.saveName = service["saveName"] # 保存文件的名字 + except: + now = datetime.now() + # 将时间格式化为精确到毫秒的字符串 + self.saveName = now.strftime("%Y_%m_%d_%H_%M_%S_%f") self.log = "" self.OUTPUT = "" self.SAVED = False + + # 名称设定 + if saveName != "": # 命令行覆盖保存名称 + self.saveName = saveName # 保存文件的名字 + elif self.saveName == "Time": + # 获取当前时间 + now = datetime.now() + # 将时间格式化为精确到毫秒的字符串 + self.saveName = now.strftime("%Y_%m_%d_%H_%M_%S_%f") + + print("Save Name for task ID", i, "is:", self.saveName) + print("任务ID", i, "的保存文件名为:", self.saveName) + if not os.path.exists("Data/Task_" + str(i)): + os.mkdir("Data/Task_" + str(i)) + if not os.path.exists("Data/Task_" + str(i) + "/" + self.saveName): + os.mkdir("Data/Task_" + str(i) + "/" + self.saveName) # 创建保存文件夹用来保存截图 + stealth_path = driver_path[:driver_path.find( "chromedriver")] + "stealth.min.js" with open(stealth_path, 'r') as f: @@ -200,7 +222,10 @@ class BrowserThread(Thread): self.log = "" def scrollDown(self, para, rt=""): - time.sleep(para["scrollWaitTime"]) # 下拉前等待 + try: + time.sleep(para["scrollWaitTime"]) # 下拉前等待 + except: + pass scrollType = int(para["scrollType"]) try: if scrollType != 0 and para["scrollCount"] > 0: # 控制屏幕向下滚动 @@ -212,7 +237,10 @@ class BrowserThread(Thread): body.send_keys(Keys.PAGE_DOWN) elif scrollType == 2: body.send_keys(Keys.END) - time.sleep(para["scrollWaitTime"]) # 下拉完等待 + try: + time.sleep(para["scrollWaitTime"]) # 下拉完等待 + except: + pass except: self.Log('Time out after set seconds when scrolling. ') self.recordLog('Time out after set seconds when scrolling') @@ -226,7 +254,10 @@ class BrowserThread(Thread): body.send_keys(Keys.PGDN) elif scrollType == 2: body.send_keys(Keys.END) - time.sleep(para["scrollWaitTime"]) # 下拉完等待 + try: + time.sleep(para["scrollWaitTime"]) # 下拉完等待 + except: + pass if rt != "": rt.end() @@ -862,7 +893,10 @@ class BrowserThread(Thread): self.Log('Time out after set seconds when loading clicked page') self.recordLog( 'Time out after set seconds when loading clicked page') - self.browser.execute_script('window.stop()') + try: + self.browser.execute_script('window.stop()') + except: + pass except Exception as e: self.Log(e) self.recordLog(str(e)) @@ -1336,19 +1370,6 @@ if __name__ == '__main__': for i in c.id: print(options) print("id: ", i) - if c.saved_file_name != "": - saveName = c.saved_file_name # 保存文件的名字 - else: - # 获取当前时间 - now = datetime.now() - # 将时间格式化为精确到毫秒的字符串 - saveName = now.strftime("%Y_%m_%d_%H_%M_%S_%f") - print("Save Name for task ID", i, "is:", saveName) - print("任务ID", i, "的保存文件名为:", saveName) - if not os.path.exists("Data/Task_" + str(i)): - os.mkdir("Data/Task_" + str(i)) - if not os.path.exists("Data/Task_" + str(i) + "/" + saveName): - os.mkdir("Data/Task_" + str(i) + "/" + saveName) # 创建保存文件夹用来保存截图 if c.read_type == "remote": print("remote") content = requests.get( @@ -1402,7 +1423,7 @@ if __name__ == '__main__': event = Event() event.set() thread = BrowserThread(browser_t, i, service, - c.version, event, config=config) + c.version, event, c.saved_file_name, config=config) print("Thread with task id: ", i, " is created") threads.append(thread) thread.start()