spider/EasySpider
1
0
Fork 0
mirror of https://github.com/NaiboWang/EasySpider.git synced 2025-04-21 21:25:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update Readme

Browse Source
This commit is contained in:
naibo 2023-07-08 06:42:51 +08:00
parent a003df9626
commit 021ccf12d0
8 changed files with 17 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.temp_to_pub/EasySpider_windows_x64/Readme.txt
View File

@ -4,7 +4,7 @@ Welcome to promote this software to other friends.
This version is for Windows 10 x64 and above. This version is for Windows 10 x64 and above.
The Windows version supports **Windows 10 and above**. There is no direct usable version available for Windows 7, but the [32-bit version](https://github.com/NaiboWang/EasySpider/releases/download/v0.2.0/EasySpider_0.2.0_windows_386.7z) of [v0.2.0](https://github.com/NaiboWang/EasySpider/releases/download/v0.2.0) is available and can be run by compiling the software yourself. Therefore, if you want to use Windows 7 for data collection, please download the 32-bit version of v0.2.0 or download the code and compile it yourself. The Windows version supports **Windows 10 and above**. If you want to use EasySpider on windows 7, please download the Windows x86 version of EasySpider.
Video Tutorial: https://youtube.com/playlist?list=PL0kEFEkWrT7mt9MUlEBV2DTo1QsaanUTp Video Tutorial: https://youtube.com/playlist?list=PL0kEFEkWrT7mt9MUlEBV2DTo1QsaanUTp

1
.temp_to_pub/EasySpider_windows_x64/execution_instances/0.json Normal file
View File

@ -0,0 +1 @@
{"id":0,"name":"京东全球版-专业的综合网上购物商城","url":"https://www.jd.com","links":"https://www.jd.com","create_time":"7/8/2023, 6:27:01 AM","update_time":"7/8/2023, 6:27:01 AM","version":"0.3.5","saveThreshold":10,"cloudflare":0,"environment":0,"maxViewLength":15,"outputFormat":"xlsx","saveName":"current_time","containJudge":false,"desc":"https://www.jd.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.jd.com","desc":"要采集的网址列表，多行以\\n分开","type":"text","exampleValue":"https://www.jd.com"}],"outputParameters":[{"id":0,"name":"参数1_链接文本","desc":"","type":"text","recordASField":1,"exampleValue":"手机"},{"id":1,"name":"参数2_链接地址","desc":"","type":"text","recordASField":1,"exampleValue":"https://shouji.jd.com/"}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0,"waitType":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"https://www.jd.com","links":"https://www.jd.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":2,"index":2,"parentId":0,"type":1,"option":8,"title":"循环","sequence":[3],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div/a","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":1,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":["/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/a[1]","//a[contains(., '手机')]","/html/body/div[last()-5]/div/div[last()-4]/div/div[last()-2]/div/div/div/div[last()-1]/div[last()-12]/a[last()-1]"]}},{"id":3,"index":3,"parentId":2,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"paras":[{"nodeType":1,"contentType":0,"relative":true,"name":"参数1_链接文本","desc":"","extractType":0,"relativeXPath":"","allXPaths":"","exampleValues":[{"num":0,"value":"手机"}],"unique_index":"vaoyqp5ljeljt5coby","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":2,"contentType":0,"relative":true,"name":"参数2_链接地址","desc":"","relativeXPath":"","allXPaths":"","exampleValues":[{"num":0,"value":"https://shouji.jd.com/"}],"unique_index":"vaoyqp5ljeljt5coby","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}],"loopType":1}}]}

1
.temp_to_pub/EasySpider_windows_x64/tasks/107.json Normal file
View File

@ -0,0 +1 @@
{"id":107,"name":"京东全球版-专业的综合网上购物商城","url":"https://www.jd.com","links":"https://www.jd.com","create_time":"7/8/2023, 6:27:01 AM","update_time":"7/8/2023, 6:27:01 AM","version":"0.3.5","saveThreshold":10,"cloudflare":0,"environment":0,"maxViewLength":15,"outputFormat":"xlsx","saveName":"current_time","containJudge":false,"desc":"https://www.jd.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.jd.com","desc":"要采集的网址列表，多行以\\n分开","type":"text","exampleValue":"https://www.jd.com"}],"outputParameters":[{"id":0,"name":"参数1_链接文本","desc":"","type":"text","recordASField":1,"exampleValue":"手机"},{"id":1,"name":"参数2_链接地址","desc":"","type":"text","recordASField":1,"exampleValue":"https://shouji.jd.com/"}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0,"waitType":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"https://www.jd.com","links":"https://www.jd.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":2,"index":2,"parentId":0,"type":1,"option":8,"title":"循环","sequence":[3],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div/a","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":1,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":["/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/a[1]","//a[contains(., '手机')]","/html/body/div[last()-5]/div/div[last()-4]/div/div[last()-2]/div/div/div/div[last()-1]/div[last()-12]/a[last()-1]"]}},{"id":3,"index":3,"parentId":2,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"paras":[{"nodeType":1,"contentType":0,"relative":true,"name":"参数1_链接文本","desc":"","extractType":0,"relativeXPath":"","allXPaths":"","exampleValues":[{"num":0,"value":"手机"}],"unique_index":"vaoyqp5ljeljt5coby","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":2,"contentType":0,"relative":true,"name":"参数2_链接地址","desc":"","relativeXPath":"","allXPaths":"","exampleValues":[{"num":0,"value":"https://shouji.jd.com/"}],"unique_index":"vaoyqp5ljeljt5coby","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}],"loopType":1}}]}

2
.temp_to_pub/EasySpider_windows_x64/软件使用说明.txt
View File

@ -4,7 +4,7 @@
支持Windows 10 x64及以上版本。 支持Windows 10 x64及以上版本。
Windows 7此版本无直接可用版本（因为Chrome 109是最后一个支持Windows 7的Chrome版本），但v0.2.0的32位版本可用，且可以通过自行编译软件来运行，因此如想使用Windows 7采集数据，请下载v0.2.0的32位版本或自行下载代码并编译：https://github.com/NaiboWang/EasySpider/releases/tag/v0.2.0 Windows 7此请下载Windows的32位版本使用。
视频教程：https://www.bilibili.com/video/BV1th411A7ey/ 视频教程：https://www.bilibili.com/video/BV1th411A7ey/

4
.temp_to_pub/EasySpider_windows_x86/readme.txt
View File

@ -2,9 +2,7 @@ Official Site: https://www.easyspider.net
Welcome to promote this software to other friends. Welcome to promote this software to other friends.
This version is for Windows 10 x32 and above. This version is for Windows 7 and above, including both 32-bit and 64-bit version. Please note that this version of the Chrome browser will always remain at version 109 and will not update with Chrome updates (for compatibility with Windows 7). Therefore, if you want to use the latest version of the Chrome browser for data scraping, please run the x64 version of EasySpider on Windows 10 x64 or higher systems.
The Windows version supports **Windows 10 and above**. There is no direct usable version available for Windows 7, but the [32-bit version](https://github.com/NaiboWang/EasySpider/releases/download/v0.2.0/EasySpider_0.2.0_windows_386.7z) of [v0.2.0](https://github.com/NaiboWang/EasySpider/releases/download/v0.2.0) is available and can be run by compiling the software yourself. Therefore, if you want to use Windows 7 for data collection, please download the 32-bit version of v0.2.0 or download the code and compile it yourself.
Video Tutorial: https://youtube.com/playlist?list=PL0kEFEkWrT7mt9MUlEBV2DTo1QsaanUTp Video Tutorial: https://youtube.com/playlist?list=PL0kEFEkWrT7mt9MUlEBV2DTo1QsaanUTp

4
.temp_to_pub/EasySpider_windows_x86/软件使用说明.txt
View File

@ -2,9 +2,7 @@
官方网址: https://www.easyspider.cn 官方网址: https://www.easyspider.cn
支持Windows 10 x32及以上版本。 支持Windows 7及以上版本，包括32位系统和64位系统。注意此版本的Chrome浏览器永远都是109不会随着Chrome更新而更新（为了兼容Win 7系统），因此如果想用最新版Chrome浏览器采集数据，请在Windows 10 x64及以上系统上运行x64版本的EasySpider。
Windows 7此版本无直接可用版本（因为Chrome 109是最后一个支持Windows 7的Chrome版本），但v0.2.0的32位版本可用，且可以通过自行编译软件来运行，因此如想使用Windows 7采集数据，请下载v0.2.0的32位版本或自行下载代码并编译：https://github.com/NaiboWang/EasySpider/releases/tag/v0.2.0
视频教程：https://www.bilibili.com/video/BV1th411A7ey/ 视频教程：https://www.bilibili.com/video/BV1th411A7ey/

2
ExecuteStage/.vscode/launch.json vendored
View File

@ -12,7 +12,7 @@
"justMyCode": true, "justMyCode": true,
// "args": ["--id", "[7]", "--read_type", "remote", "--headless", "0"] // "args": ["--id", "[7]", "--read_type", "remote", "--headless", "0"]
// "args": ["--id", "[9]", "--read_type", "remote", "--headless", "0", "--saved_file_name", "YOUTUBE"] // "args": ["--id", "[9]", "--read_type", "remote", "--headless", "0", "--saved_file_name", "YOUTUBE"]
"args": ["--id", "[5]", "--headless", "0", "--user_data", "1"] "args": ["--id", "[0]", "--headless", "0", "--user_data", "1"]
} }
] ]
} }

25
ExecuteStage/easyspider_executestage.py
View File

@ -1345,19 +1345,19 @@ if __name__ == '__main__':
if sys.platform == "win32" and platform.architecture()[0] == "32bit": if sys.platform == "win32" and platform.architecture()[0] == "32bit":
options.binary_location = os.path.join( options.binary_location = os.path.join(
os.getcwd(), "EasySpider/resources/app/chrome_win32/chrome.exe") # 指定chrome位置 os.getcwd(), "EasySpider/resources/app/chrome_win32/chrome.exe") # 指定chrome位置
options.add_extension("EasySpider/resources/app/XPathHelper.crx")
driver_path = os.path.join( driver_path = os.path.join(
os.getcwd(), "EasySpider/resources/app/chrome_win32/chromedriver_win32.exe") os.getcwd(), "EasySpider/resources/app/chrome_win32/chromedriver_win32.exe")
option.add_extension("EasySpider/resources/app/XPathHelper.crx")
elif sys.platform == "win32" and platform.architecture()[0] == "64bit": elif sys.platform == "win32" and platform.architecture()[0] == "64bit":
options.binary_location = os.path.join( options.binary_location = os.path.join(
os.getcwd(), "EasySpider/resources/app/chrome_win64/chrome.exe") os.getcwd(), "EasySpider/resources/app/chrome_win64/chrome.exe")
driver_path = os.path.join( driver_path = os.path.join(
os.getcwd(), "EasySpider/resources/app/chrome_win64/chromedriver_win64.exe") os.getcwd(), "EasySpider/resources/app/chrome_win64/chromedriver_win64.exe")
options.add_extension("EasySpider/resources/app/XPathHelper.crx") option.add_extension("EasySpider/resources/app/XPathHelper.crx")
elif sys.platform == "linux" and platform.architecture()[0] == "64bit": elif sys.platform == "linux" and platform.architecture()[0] == "64bit":
options.binary_location = "EasySpider/resources/app/chrome_linux64/chrome" options.binary_location = "EasySpider/resources/app/chrome_linux64/chrome"
options.add_extension("EasySpider/resources/app/XPathHelper.crx")
driver_path = "EasySpider/resources/app/chrome_linux64/chromedriver_linux64" driver_path = "EasySpider/resources/app/chrome_linux64/chromedriver_linux64"
option.add_extension("EasySpider/resources/app/XPathHelper.crx")
else: else:
print("Unsupported platform") print("Unsupported platform")
sys.exit() sys.exit()
@ -1370,21 +1370,16 @@ if __name__ == '__main__':
# # option.binary_location = "C:\\Users\\q9823\\AppData\\Local\\Google\\Chrome\\Application\\chrome.exe" # # option.binary_location = "C:\\Users\\q9823\\AppData\\Local\\Google\\Chrome\\Application\\chrome.exe"
# driver_path = "./Chrome/chromedriver.exe" # driver_path = "./Chrome/chromedriver.exe"
elif os.path.exists(os.getcwd()+"/../ElectronJS"): elif os.path.exists(os.getcwd()+"/../ElectronJS"):
if os.getcwd().find("ElectronJS") >= 0: # 软件dev用 # 软件dev用
print("Finding chromedriver in EasySpider", print("Finding chromedriver in EasySpider",
os.getcwd()) os.getcwd()+"/ElectronJS")
options.binary_location = "chrome_win64/chrome.exe" option.binary_location = "../ElectronJS/chrome_win64/chrome.exe" # 指定chrome位置
driver_path = "chrome_win64/chromedriver_win64.exe" driver_path = "../ElectronJS/chrome_win64/chromedriver_win64.exe"
options.add_extension("../ElectronJS/XPathHelper.crx") option.add_extension("../ElectronJS/XPathHelper.crx")
else: # 直接在executeStage文件夹内使用python easyspider_executestage.py时的路径
print("Finding chromedriver in EasySpider",
os.getcwd()+"/ElectronJS")
option.binary_location = "../ElectronJS/chrome_win64/chrome.exe" # 指定chrome位置
driver_path = "../ElectronJS/chrome_win64/chromedriver_win64.exe"
option.add_extension("../ElectronJS/XPathHelper.crx")
else: else:
options.binary_location = "./chrome.exe" # 指定chrome位置 options.binary_location = "./chrome.exe" # 指定chrome位置
driver_path = "./chromedriver.exe" driver_path = "./chromedriver.exe"
option.add_extension("XPathHelper.crx")
option.add_experimental_option( option.add_experimental_option(
'excludeSwitches', ['enable-automation']) # 以开发者模式 'excludeSwitches', ['enable-automation']) # 以开发者模式