diff --git a/.temp_to_pub/EasySpider_windows_x64/execution_instances/29.json b/.temp_to_pub/EasySpider_windows_x64/execution_instances/29.json new file mode 100644 index 0000000..b4ea003 --- /dev/null +++ b/.temp_to_pub/EasySpider_windows_x64/execution_instances/29.json @@ -0,0 +1 @@ +{"id":29,"name":"Index of /groups/802/1/files/public/docs2018","url":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","links":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","create_time":"8/20/2023, 9:16:31 PM","update_time":"8/20/2023, 9:16:31 PM","version":"0.5.0","saveThreshold":10,"cloudflare":0,"environment":0,"maxViewLength":15,"recordLog":1,"outputFormat":"xlsx","saveName":"current_time","inputExcel":"","startFromExit":0,"containJudge":false,"desc":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","desc":"要采集的网址列表，多行以\\n分开","type":"text","exampleValue":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/"}],"outputParameters":[],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"url":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","links":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":2,"index":2,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/pre[1]/a[6]","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/pre[1]/a[6]","//a[contains(., '60802-Bai-')]","/html/body/pre/a[last()-541]"]}}]} \ No newline at end of file diff --git a/.temp_to_pub/EasySpider_windows_x64/info.log b/.temp_to_pub/EasySpider_windows_x64/info.log index 9a4263e..fa86d62 100644 --- a/.temp_to_pub/EasySpider_windows_x64/info.log +++ b/.temp_to_pub/EasySpider_windows_x64/info.log @@ -445,3 +445,176 @@ cd ../ start EasySpider/resources/app/chrome_win64/easyspider_executestage.exe --id [28] --server_address http://localhost:8074 --user_data 0 ) else (start chrome_win64/easyspider_executestage.exe --id [28] --server_address http://localhost:8074 --user_data 0 ) ) +Server has started. +server_address: http://localhost:8074 +x64 +D:\Documents\Projects\EasySpider\.temp_to_pub\EasySpider_windows_x64\EasySpider\resources\app\chrome_win64\chromedriver_win64.exe D:\Documents\Projects\EasySpider\.temp_to_pub\EasySpider_windows_x64\EasySpider\resources\app\chrome_win64\chrome.exe D:\Documents\Projects\EasySpider\.temp_to_pub\EasySpider_windows_x64\EasySpider\resources\app\chrome_win64\execute.bat +win32 +Server listening on http://localhost:8075 +Not Windows 7 + + +GET A MESSAGE: { type: 0, message: { id: 2 } } +set socket_flowchart + + +GET A MESSAGE: { type: 0, message: { id: 0 } } +set socket_window + + +GET A MESSAGE: { type: 0, message: { id: 1 } } +set socket_start + + +GET A MESSAGE: { type: 0, message: { id: 'C1391855', title: 'Start' } } +Set handle_pair for id: C1391855 to 82F895FF7DA09A31865D372B371F747F , title is: Start + + +GET A MESSAGE: { type: 0, message: { id: 1 } } +set socket_start + + +GET A MESSAGE: { type: 0, message: { id: 'C23790568', title: 'New Task' } } +Set handle_pair for id: C23790568 to 82F895FF7DA09A31865D372B371F747F , title is: New Task + + +GET A MESSAGE: { + type: 3, + from: 0, + message: { + pipe: '{"type":"openPage","url":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","links":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/"}' + } +} +FROM Browser: { + type: 'openPage', + url: 'https://grouper.ieee.org/groups/802/1/files/public/docs2018/', + links: 'https://grouper.ieee.org/groups/802/1/files/public/docs2018/' +} +Iframe: undefined + + +GET A MESSAGE: { type: 1, message: { id: -1 } } +Window { + id: 10361598, + processId: 82352, + path: 'D:\\Documents\\Projects\\EasySpider\\.temp_to_pub\\EasySpider_windows_x64\\EasySpider\\resources\\app\\chrome_win64\\chrome.exe' +} + + +GET A MESSAGE: { + type: 0, + message: { + id: 'C33457954', + title: 'Index of /groups/802/1/files/public/docs2018' + } +} +Set handle_pair for id: C33457954 to 82F895FF7DA09A31865D372B371F747F , title is: Index of /groups/802/1/files/public/docs2018 + + +GET A MESSAGE: { + type: 3, + from: 0, + message: { + pipe: `{"type":"singleClick","id":"C33457954","history":4,"tabIndex":-1,"useLoop":false,"iframe":false,"xpath":"/html/body/pre[1]/a[6]","allXPaths":["/html/body/pre[1]/a[6]","//a[contains(., '60802-Bai-')]","/html/body/pre/a[last()-541]"]}` + } +} +FROM Browser: { + type: 'singleClick', + id: 'C33457954', + history: 4, + tabIndex: -1, + useLoop: false, + iframe: false, + xpath: '/html/body/pre[1]/a[6]', + allXPaths: [ + '/html/body/pre[1]/a[6]', + "//a[contains(., '60802-Bai-')]", + '/html/body/pre/a[last()-541]' + ] +} +Iframe: false +handles [ '82F895FF7DA09A31865D372B371F747F' ] +C33457954 82F895FF7DA09A31865D372B371F747F +current_handle 82F895FF7DA09A31865D372B371F747F +switch to handle: 82F895FF7DA09A31865D372B371F747F + + +GET A MESSAGE: { type: 0, message: { id: 'C81368640', title: '' } } +Set handle_pair for id: C81368640 to 82F895FF7DA09A31865D372B371F747F , title is: + + +GET A MESSAGE: { type: 0, message: { id: 1 } } +set socket_start + + +GET A MESSAGE: { type: 0, message: { id: 1 } } +set socket_start + + +GET A MESSAGE: { + type: 5, + message: { + id: -1, + user_data_folder: '', + mysql_config_path: '-1', + execute_type: 1 + } +} +{ + id: -1, + user_data_folder: '', + mysql_config_path: '-1', + execute_type: 1 +} +0.json +1.json +10.json +11.json +12.json +13.json +14.json +15.json +16.json +17.json +18.json +19.json +2.json +20.json +21.json +22.json +23.json +24.json +25.json +26.json +27.json +28.json +3.json +4.json +5.json +6.json +7.json +8.json +9.json + + +GET A MESSAGE: { + type: 5, + message: { + id: 29, + user_data_folder: '', + mysql_config_path: './mysql_config.json', + execute_type: 1 + } +} +{ + id: 29, + user_data_folder: '', + mysql_config_path: './mysql_config.json', + execute_type: 1 +} + +D:\Documents\Projects\EasySpider\.temp_to_pub\EasySpider_windows_x64>if exist EasySpider (start EasySpider/resources/app/chrome_win64/easyspider_executestage.exe --id [29] --server_address http://localhost:8074 --user_data 0 ) else (if exist resources ( +cd ../ + start EasySpider/resources/app/chrome_win64/easyspider_executestage.exe --id [29] --server_address http://localhost:8074 --user_data 0 +) else (start chrome_win64/easyspider_executestage.exe --id [29] --server_address http://localhost:8074 --user_data 0 ) ) + diff --git a/.temp_to_pub/EasySpider_windows_x64/tasks/213.json b/.temp_to_pub/EasySpider_windows_x64/tasks/213.json new file mode 100644 index 0000000..ed2d5ff --- /dev/null +++ b/.temp_to_pub/EasySpider_windows_x64/tasks/213.json @@ -0,0 +1 @@ +{"id":213,"name":"Index of /groups/802/1/files/public/docs2018","url":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","links":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","create_time":"8/20/2023, 9:16:31 PM","update_time":"8/20/2023, 9:16:31 PM","version":"0.5.0","saveThreshold":10,"cloudflare":0,"environment":0,"maxViewLength":15,"recordLog":1,"outputFormat":"xlsx","saveName":"current_time","inputExcel":"","startFromExit":0,"containJudge":false,"desc":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","desc":"要采集的网址列表，多行以\\n分开","type":"text","exampleValue":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/"}],"outputParameters":[],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"url":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","links":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":2,"index":2,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/pre[1]/a[6]","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/pre[1]/a[6]","//a[contains(., '60802-Bai-')]","/html/body/pre/a[last()-541]"]}}]} \ No newline at end of file diff --git a/ElectronJS/EasySpider_en.crx b/ElectronJS/EasySpider_en.crx new file mode 100644 index 0000000..fe23f22 Binary files /dev/null and b/ElectronJS/EasySpider_en.crx differ diff --git a/ElectronJS/EasySpider_zh.crx b/ElectronJS/EasySpider_zh.crx new file mode 100644 index 0000000..257bfc9 Binary files /dev/null and b/ElectronJS/EasySpider_zh.crx differ diff --git a/ElectronJS/clean_and_release_win32.cmd b/ElectronJS/clean_and_release_win32.cmd index 7f584dc..5c5c260 100644 --- a/ElectronJS/clean_and_release_win32.cmd +++ b/ElectronJS/clean_and_release_win32.cmd @@ -15,6 +15,7 @@ copy ..\ExecuteStage\easyspider_executestage.py ..\.temp_to_pub\EasySpider_windo copy ..\ExecuteStage\myChrome.py ..\.temp_to_pub\EasySpider_windows_x32\Code copy ..\ExecuteStage\utils.py ..\.temp_to_pub\EasySpider_windows_x32\Code copy ..\ExecuteStage\requirements.txt ..\.temp_to_pub\EasySpider_windows_x32\Code +copy ..\ExecuteStage\Readme.md ..\.temp_to_pub\EasySpider_windows_x32\Code xcopy ..\ExecuteStage\undetected_chromedriver_ES ..\.temp_to_pub\EasySpider_windows_x32\Code\undetected_chromedriver_ES /E /I /Y xcopy ..\ExecuteStage\.vscode ..\.temp_to_pub\EasySpider_windows_x32\Code\.vscode /E /I /Y rmdir /s /q ..\.temp_to_pub\EasySpider_windows_x32\user_data diff --git a/ElectronJS/clean_and_release_win64.cmd b/ElectronJS/clean_and_release_win64.cmd index 12548c9..2fd399a 100644 --- a/ElectronJS/clean_and_release_win64.cmd +++ b/ElectronJS/clean_and_release_win64.cmd @@ -15,6 +15,7 @@ copy ..\ExecuteStage\easyspider_executestage.py ..\.temp_to_pub\EasySpider_windo copy ..\ExecuteStage\myChrome.py ..\.temp_to_pub\EasySpider_windows_x64\Code copy ..\ExecuteStage\utils.py ..\.temp_to_pub\EasySpider_windows_x64\Code copy ..\ExecuteStage\requirements.txt ..\.temp_to_pub\EasySpider_windows_x64\Code +copy ..\ExecuteStage\Readme.md ..\.temp_to_pub\EasySpider_windows_x64\Code xcopy ..\ExecuteStage\undetected_chromedriver_ES ..\.temp_to_pub\EasySpider_windows_x64\Code\undetected_chromedriver_ES /E /I /Y xcopy ..\ExecuteStage\.vscode ..\.temp_to_pub\EasySpider_windows_x64\Code\.vscode /E /I /Y rmdir /s /q ..\.temp_to_pub\EasySpider_windows_x64\user_data diff --git a/ElectronJS/package_linux64.sh b/ElectronJS/package_linux64.sh index 5af15d8..d42d9e7 100755 --- a/ElectronJS/package_linux64.sh +++ b/ElectronJS/package_linux64.sh @@ -23,6 +23,7 @@ cp ../ExecuteStage/easyspider_executestage.py ../.temp_to_pub/EasySpider_Linux_x cp ../ExecuteStage/myChrome.py ../.temp_to_pub/EasySpider_Linux_x64/Code cp ../ExecuteStage/utils.py ../.temp_to_pub/EasySpider_Linux_x64/Code cp ../ExecuteStage/requirements.txt ../.temp_to_pub/EasySpider_Linux_x64/Code +cp ../ExecuteStage/Readme.md ../.temp_to_pub/EasySpider_Linux_x64/Code cp -Rf ../ExecuteStage/undetected_chromedriver_ES ../.temp_to_pub/EasySpider_Linux_x64/Code cp -Rf ../ExecuteStage/.vscode ../.temp_to_pub/EasySpider_Linux_x64/Code chmod 777 ../.temp_to_pub/EasySpider_Linux_x64/easy-spider.sh diff --git a/ElectronJS/package_macos.sh b/ElectronJS/package_macos.sh index c8d807f..6f72278 100755 --- a/ElectronJS/package_macos.sh +++ b/ElectronJS/package_macos.sh @@ -23,5 +23,6 @@ cp ../ExecuteStage/easyspider_executestage.py ../.temp_to_pub/EasySpider_MacOS_a cp ../ExecuteStage/myChrome.py ../.temp_to_pub/EasySpider_MacOS_all_arch/Code cp ../ExecuteStage/utils.py ../.temp_to_pub/EasySpider_MacOS_all_arch/Code cp ../ExecuteStage/requirements.txt ../.temp_to_pub/EasySpider_MacOS_all_arch/Code +cp ../ExecuteStage/Readme.md ../.temp_to_pub/EasySpider_MacOS_all_arch/Code cp -Rf ../ExecuteStage/undetected_chromedriver_ES ../.temp_to_pub/EasySpider_MacOS_all_arch/Code cp -Rf ../ExecuteStage/.vscode ../.temp_to_pub/EasySpider_MacOS_all_arch/Code