spider/EasySpider
1
0
Fork 0
mirror of https://github.com/NaiboWang/EasySpider.git synced 2025-04-16 16:26:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add Readme.md to Code

Browse Source
This commit is contained in:
naibo 2023-08-22 18:38:21 +08:00
parent d4f89ddc13
commit 329e46e051
9 changed files with 179 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.temp_to_pub/EasySpider_windows_x64/execution_instances/29.json Normal file
View File

@ -0,0 +1 @@
{"id":29,"name":"Index of /groups/802/1/files/public/docs2018","url":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","links":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","create_time":"8/20/2023, 9:16:31 PM","update_time":"8/20/2023, 9:16:31 PM","version":"0.5.0","saveThreshold":10,"cloudflare":0,"environment":0,"maxViewLength":15,"recordLog":1,"outputFormat":"xlsx","saveName":"current_time","inputExcel":"","startFromExit":0,"containJudge":false,"desc":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","desc":"要采集的网址列表，多行以\\n分开","type":"text","exampleValue":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/"}],"outputParameters":[],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"url":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","links":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":2,"index":2,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/pre[1]/a[6]","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/pre[1]/a[6]","//a[contains(., '60802-Bai-')]","/html/body/pre/a[last()-541]"]}}]}

173
.temp_to_pub/EasySpider_windows_x64/info.log
View File

@ -445,3 +445,176 @@ cd ../
start EasySpider/resources/app/chrome_win64/easyspider_executestage.exe --id [28] --server_address http://localhost:8074 --user_data 0
) else (start chrome_win64/easyspider_executestage.exe --id [28] --server_address http://localhost:8074 --user_data 0 ) )
Server has started.
server_address: http://localhost:8074
x64
D:\Documents\Projects\EasySpider\.temp_to_pub\EasySpider_windows_x64\EasySpider\resources\app\chrome_win64\chromedriver_win64.exe D:\Documents\Projects\EasySpider\.temp_to_pub\EasySpider_windows_x64\EasySpider\resources\app\chrome_win64\chrome.exe D:\Documents\Projects\EasySpider\.temp_to_pub\EasySpider_windows_x64\EasySpider\resources\app\chrome_win64\execute.bat
win32
Server listening on http://localhost:8075
Not Windows 7
GET A MESSAGE: { type: 0, message: { id: 2 } }
set socket_flowchart
GET A MESSAGE: { type: 0, message: { id: 0 } }
set socket_window
GET A MESSAGE: { type: 0, message: { id: 1 } }
set socket_start
GET A MESSAGE: { type: 0, message: { id: 'C1391855', title: 'Start' } }
Set handle_pair for id: C1391855 to 82F895FF7DA09A31865D372B371F747F , title is: Start
GET A MESSAGE: { type: 0, message: { id: 1 } }
set socket_start
GET A MESSAGE: { type: 0, message: { id: 'C23790568', title: 'New Task' } }
Set handle_pair for id: C23790568 to 82F895FF7DA09A31865D372B371F747F , title is: New Task
GET A MESSAGE: {
type: 3,
from: 0,
message: {
pipe: '{"type":"openPage","url":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","links":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/"}'
}
}
FROM Browser: {
type: 'openPage',
url: 'https://grouper.ieee.org/groups/802/1/files/public/docs2018/',
links: 'https://grouper.ieee.org/groups/802/1/files/public/docs2018/'
}
Iframe: undefined
GET A MESSAGE: { type: 1, message: { id: -1 } }
Window {
id: 10361598,
processId: 82352,
path: 'D:\\Documents\\Projects\\EasySpider\\.temp_to_pub\\EasySpider_windows_x64\\EasySpider\\resources\\app\\chrome_win64\\chrome.exe'
}
GET A MESSAGE: {
type: 0,
message: {
id: 'C33457954',
title: 'Index of /groups/802/1/files/public/docs2018'
}
}
Set handle_pair for id: C33457954 to 82F895FF7DA09A31865D372B371F747F , title is: Index of /groups/802/1/files/public/docs2018
GET A MESSAGE: {
type: 3,
from: 0,
message: {
pipe: `{"type":"singleClick","id":"C33457954","history":4,"tabIndex":-1,"useLoop":false,"iframe":false,"xpath":"/html/body/pre[1]/a[6]","allXPaths":["/html/body/pre[1]/a[6]","//a[contains(., '60802-Bai-')]","/html/body/pre/a[last()-541]"]}`
}
}
FROM Browser: {
type: 'singleClick',
id: 'C33457954',
history: 4,
tabIndex: -1,
useLoop: false,
iframe: false,
xpath: '/html/body/pre[1]/a[6]',
allXPaths: [
'/html/body/pre[1]/a[6]',
"//a[contains(., '60802-Bai-')]",
'/html/body/pre/a[last()-541]'
]
}
Iframe: false
handles [ '82F895FF7DA09A31865D372B371F747F' ]
C33457954 82F895FF7DA09A31865D372B371F747F
current_handle 82F895FF7DA09A31865D372B371F747F
switch to handle: 82F895FF7DA09A31865D372B371F747F
GET A MESSAGE: { type: 0, message: { id: 'C81368640', title: '' } }
Set handle_pair for id: C81368640 to 82F895FF7DA09A31865D372B371F747F , title is:
GET A MESSAGE: { type: 0, message: { id: 1 } }
set socket_start
GET A MESSAGE: { type: 0, message: { id: 1 } }
set socket_start
GET A MESSAGE: {
type: 5,
message: {
id: -1,
user_data_folder: '',
mysql_config_path: '-1',
execute_type: 1
}
}
{
id: -1,
user_data_folder: '',
mysql_config_path: '-1',
execute_type: 1
}
0.json
1.json
10.json
11.json
12.json
13.json
14.json
15.json
16.json
17.json
18.json
19.json
2.json
20.json
21.json
22.json
23.json
24.json
25.json
26.json
27.json
28.json
3.json
4.json
5.json
6.json
7.json
8.json
9.json
GET A MESSAGE: {
type: 5,
message: {
id: 29,
user_data_folder: '',
mysql_config_path: './mysql_config.json',
execute_type: 1
}
}
{
id: 29,
user_data_folder: '',
mysql_config_path: './mysql_config.json',
execute_type: 1
}
D:\Documents\Projects\EasySpider\.temp_to_pub\EasySpider_windows_x64>if exist EasySpider (start EasySpider/resources/app/chrome_win64/easyspider_executestage.exe --id [29] --server_address http://localhost:8074 --user_data 0 ) else (if exist resources (
cd ../
start EasySpider/resources/app/chrome_win64/easyspider_executestage.exe --id [29] --server_address http://localhost:8074 --user_data 0
) else (start chrome_win64/easyspider_executestage.exe --id [29] --server_address http://localhost:8074 --user_data 0 ) )

1
.temp_to_pub/EasySpider_windows_x64/tasks/213.json Normal file
View File

@ -0,0 +1 @@
{"id":213,"name":"Index of /groups/802/1/files/public/docs2018","url":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","links":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","create_time":"8/20/2023, 9:16:31 PM","update_time":"8/20/2023, 9:16:31 PM","version":"0.5.0","saveThreshold":10,"cloudflare":0,"environment":0,"maxViewLength":15,"recordLog":1,"outputFormat":"xlsx","saveName":"current_time","inputExcel":"","startFromExit":0,"containJudge":false,"desc":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","desc":"要采集的网址列表，多行以\\n分开","type":"text","exampleValue":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/"}],"outputParameters":[],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"url":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","links":"https://grouper.ieee.org/groups/802/1/files/public/docs2018/","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":2,"index":2,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/pre[1]/a[6]","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/pre[1]/a[6]","//a[contains(., '60802-Bai-')]","/html/body/pre/a[last()-541]"]}}]}

BIN
ElectronJS/EasySpider_en.crx Normal file
View File

Binary file not shown.

BIN
ElectronJS/EasySpider_zh.crx Normal file
View File

Binary file not shown.

1
ElectronJS/clean_and_release_win32.cmd
View File

@ -15,6 +15,7 @@ copy ..\ExecuteStage\easyspider_executestage.py ..\.temp_to_pub\EasySpider_windo
copy ..\ExecuteStage\myChrome.py ..\.temp_to_pub\EasySpider_windows_x32\Code
copy ..\ExecuteStage\utils.py ..\.temp_to_pub\EasySpider_windows_x32\Code
copy ..\ExecuteStage\requirements.txt ..\.temp_to_pub\EasySpider_windows_x32\Code
copy ..\ExecuteStage\Readme.md ..\.temp_to_pub\EasySpider_windows_x32\Code
xcopy ..\ExecuteStage\undetected_chromedriver_ES ..\.temp_to_pub\EasySpider_windows_x32\Code\undetected_chromedriver_ES /E /I /Y
xcopy ..\ExecuteStage\.vscode ..\.temp_to_pub\EasySpider_windows_x32\Code\.vscode /E /I /Y
rmdir /s /q ..\.temp_to_pub\EasySpider_windows_x32\user_data

1
ElectronJS/clean_and_release_win64.cmd
View File

@ -15,6 +15,7 @@ copy ..\ExecuteStage\easyspider_executestage.py ..\.temp_to_pub\EasySpider_windo
copy ..\ExecuteStage\myChrome.py ..\.temp_to_pub\EasySpider_windows_x64\Code
copy ..\ExecuteStage\utils.py ..\.temp_to_pub\EasySpider_windows_x64\Code
copy ..\ExecuteStage\requirements.txt ..\.temp_to_pub\EasySpider_windows_x64\Code
copy ..\ExecuteStage\Readme.md ..\.temp_to_pub\EasySpider_windows_x64\Code
xcopy ..\ExecuteStage\undetected_chromedriver_ES ..\.temp_to_pub\EasySpider_windows_x64\Code\undetected_chromedriver_ES /E /I /Y
xcopy ..\ExecuteStage\.vscode ..\.temp_to_pub\EasySpider_windows_x64\Code\.vscode /E /I /Y
rmdir /s /q ..\.temp_to_pub\EasySpider_windows_x64\user_data

1
ElectronJS/package_linux64.sh
View File

@ -23,6 +23,7 @@ cp ../ExecuteStage/easyspider_executestage.py ../.temp_to_pub/EasySpider_Linux_x
cp ../ExecuteStage/myChrome.py ../.temp_to_pub/EasySpider_Linux_x64/Code
cp ../ExecuteStage/utils.py ../.temp_to_pub/EasySpider_Linux_x64/Code
cp ../ExecuteStage/requirements.txt ../.temp_to_pub/EasySpider_Linux_x64/Code
cp ../ExecuteStage/Readme.md ../.temp_to_pub/EasySpider_Linux_x64/Code
cp -Rf ../ExecuteStage/undetected_chromedriver_ES ../.temp_to_pub/EasySpider_Linux_x64/Code
cp -Rf ../ExecuteStage/.vscode ../.temp_to_pub/EasySpider_Linux_x64/Code
chmod 777 ../.temp_to_pub/EasySpider_Linux_x64/easy-spider.sh

1
ElectronJS/package_macos.sh
View File

@ -23,5 +23,6 @@ cp ../ExecuteStage/easyspider_executestage.py ../.temp_to_pub/EasySpider_MacOS_a
cp ../ExecuteStage/myChrome.py ../.temp_to_pub/EasySpider_MacOS_all_arch/Code
cp ../ExecuteStage/utils.py ../.temp_to_pub/EasySpider_MacOS_all_arch/Code
cp ../ExecuteStage/requirements.txt ../.temp_to_pub/EasySpider_MacOS_all_arch/Code
cp ../ExecuteStage/Readme.md ../.temp_to_pub/EasySpider_MacOS_all_arch/Code
cp -Rf ../ExecuteStage/undetected_chromedriver_ES ../.temp_to_pub/EasySpider_MacOS_all_arch/Code
cp -Rf ../ExecuteStage/.vscode ../.temp_to_pub/EasySpider_MacOS_all_arch/Code