spider/EasySpider
1
0
Fork 0
mirror of https://github.com/NaiboWang/EasySpider.git synced 2025-04-20 10:05:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

Input Function Update and extension bug fix

Browse Source
This commit is contained in:
naibo 2023-03-24 22:21:46 +08:00
parent bbccc26ce7
commit 8a404eecf4
16 changed files with 546 additions and 401 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
ElectronJS/EasySpider_en.crx
View File

Binary file not shown.

BIN
ElectronJS/EasySpider_zh.crx
View File

Binary file not shown.

2
ElectronJS/config.json
View File

@ -1 +1 @@
{"webserver_address":"http://localhost","webserver_port":8074,"user_data_folder":"./user_data","absolute_user_data_folder":"C:\\Users\\Naibo\\Desktop\\EasySpider\\ElectronJS\\user_data"} {"webserver_address":"http://localhost","webserver_port":8074,"user_data_folder":"./user_data","absolute_user_data_folder":"D:\\Documents\\Projects\\EasySpider\\ElectronJS\\user_data"}

521
ElectronJS/main.js
View File

@ -1,12 +1,12 @@
// Modules to control application life and create native browser window // Modules to control application life and create native browser window
const { app, BrowserWindow, dialog, ipcMain, screen } = require('electron'); const {app, BrowserWindow, dialog, ipcMain, screen} = require('electron');
app.commandLine.appendSwitch("--disable-http-cache"); app.commandLine.appendSwitch("--disable-http-cache");
const { Builder, By, Key, until } = require("selenium-webdriver"); const {Builder, By, Key, until} = require("selenium-webdriver");
const chrome = require('selenium-webdriver/chrome'); const chrome = require('selenium-webdriver/chrome');
const { ServiceBuilder } = require('selenium-webdriver/chrome'); const {ServiceBuilder} = require('selenium-webdriver/chrome');
const { rootCertificates } = require('tls'); const {rootCertificates} = require('tls');
const { exit } = require('process'); const {exit} = require('process');
const {windowManager} = require("node-window-manager"); const {windowManager} = require("node-window-manager");
const path = require('path'); const path = require('path');
const fs = require('fs'); const fs = require('fs');
@ -25,27 +25,30 @@ let chromeBinaryPath = "";
let execute_path = ""; let execute_path = "";
console.log(process.arch); console.log(process.arch);
if (process.platform === 'win32' && process.arch === 'ia32') { if (process.platform === 'win32' && process.arch === 'ia32') {
driverPath = path.join(__dirname, "chrome_win32/chromedriver_win32.exe"); driverPath = path.join(__dirname, "chrome_win32/chromedriver_win32.exe");
chromeBinaryPath = path.join(__dirname, "chrome_win32/chrome.exe"); chromeBinaryPath = path.join(__dirname, "chrome_win32/chrome.exe");
execute_path = path.join(__dirname, "chrome_win32/execute.bat"); execute_path = path.join(__dirname, "chrome_win32/execute.bat");
} else if(process.platform === 'win32' && process.arch === 'x64') { } else if (process.platform === 'win32' && process.arch === 'x64') {
driverPath = path.join(__dirname, "chrome_win64/chromedriver_win64.exe"); driverPath = path.join(__dirname, "chrome_win64/chromedriver_win64.exe");
chromeBinaryPath = path.join(__dirname, "chrome_win64/chrome.exe"); chromeBinaryPath = path.join(__dirname, "chrome_win64/chrome.exe");
execute_path = path.join(__dirname, "chrome_win64/execute.bat"); execute_path = path.join(__dirname, "chrome_win64/execute.bat");
} else if (process.platform === 'darwin') { } else if (process.platform === 'darwin') {
driverPath = path.join(__dirname, "chromedriver_mac64"); driverPath = path.join(__dirname, "chromedriver_mac64");
chromeBinaryPath = path.join(__dirname, "chrome_mac64.app/Contents/MacOS/Google Chrome"); chromeBinaryPath = path.join(__dirname, "chrome_mac64.app/Contents/MacOS/Google Chrome");
execute_path = path.join(__dirname, "easyspider_executestage"); execute_path = path.join(__dirname, "easyspider_executestage");
} else if (process.platform === 'linux') { } else if (process.platform === 'linux') {
driverPath = path.join(__dirname, "chrome_linux64/chromedriver_linux64"); driverPath = path.join(__dirname, "chrome_linux64/chromedriver_linux64");
chromeBinaryPath = path.join(__dirname, "chrome_linux64/chrome"); chromeBinaryPath = path.join(__dirname, "chrome_linux64/chrome");
execute_path = path.join(__dirname, "chrome_linux64/execute.sh"); execute_path = path.join(__dirname, "chrome_linux64/execute.sh");
} }
console.log(driverPath, chromeBinaryPath, execute_path); console.log(driverPath, chromeBinaryPath, execute_path);
let language = "en"; let language = "en";
let driver = null; let driver = null;
let mainWindow = null; let mainWindow = null;
let flowchart_window = null; let flowchart_window = null;
let current_handle = null;
let old_handles = [];
let handle_pairs = {};
// var ffi = require('ffi-napi'); // var ffi = require('ffi-napi');
// var libm = ffi.Library('libm', { // var libm = ffi.Library('libm', {
@ -63,248 +66,326 @@ let flowchart_window = null;
// testt(); // testt();
function createWindow() { function createWindow() {
// Create the browser window. // Create the browser window.
mainWindow = new BrowserWindow({ mainWindow = new BrowserWindow({
width: 520, width: 520,
height: 750, height: 750,
webPreferences: { webPreferences: {
preload: path.join(__dirname, 'src/js/preload.js') preload: path.join(__dirname, 'src/js/preload.js')
}, },
icon: iconPath, icon: iconPath,
// frame: false, //取消window自带的关闭最小化等 // frame: false, //取消window自带的关闭最小化等
resizable: false //禁止改变主窗口尺寸 resizable: false //禁止改变主窗口尺寸
}) })
// and load the index.html of the app. // and load the index.html of the app.
// mainWindow.loadFile('src/index.html'); // mainWindow.loadFile('src/index.html');
mainWindow.loadURL(server_address + '/index.html?user_data_folder=' + config.user_data_folder); mainWindow.loadURL(server_address + '/index.html?user_data_folder=' + config.user_data_folder);
// 隐藏菜单栏 // 隐藏菜单栏
const { Menu } = require('electron'); const {Menu} = require('electron');
Menu.setApplicationMenu(null); Menu.setApplicationMenu(null);
mainWindow.on('close', function (e) { mainWindow.on('close', function (e) {
if (process.platform !== 'darwin'){ if (process.platform !== 'darwin') {
app.quit(); app.quit();
} }
}); });
// mainWindow.webContents.openDevTools(); // mainWindow.webContents.openDevTools();
// Open the DevTools. // Open the DevTools.
// mainWindow.webContents.openDevTools() // mainWindow.webContents.openDevTools()
} }
function beginInvoke(msg) { async function beginInvoke(msg) {
if (msg.type == 1) { if (msg.type == 1) {
if (msg.message.id != -1) { if (msg.message.id != -1) {
let url = ""; let url = "";
if (language == "zh"){ if (language == "zh") {
url = server_address + `/taskGrid/FlowChart_CN.html?id=${msg.message.id}&wsport=${websocket_port}&backEndAddressServiceWrapper=` + server_address; url = server_address + `/taskGrid/FlowChart_CN.html?id=${msg.message.id}&wsport=${websocket_port}&backEndAddressServiceWrapper=` + server_address;
} else if(language == "en"){ } else if (language == "en") {
url = server_address + `/taskGrid/FlowChart.html?id=${msg.message.id}&wsport=${websocket_port}&backEndAddressServiceWrapper=` + server_address; url = server_address + `/taskGrid/FlowChart.html?id=${msg.message.id}&wsport=${websocket_port}&backEndAddressServiceWrapper=` + server_address;
} }
console.log(url); console.log(url);
flowchart_window.loadURL(url); flowchart_window.loadURL(url);
} }
mainWindow.hide(); mainWindow.hide();
const window = windowManager.getActiveWindow(); const window = windowManager.getActiveWindow();
// Prints the currently focused window bounds. // Prints the currently focused window bounds.
console.log(window); console.log(window);
// This method has to be called on macOS before changing the window's bounds, otherwise it will throw an error. // This method has to be called on macOS before changing the window's bounds, otherwise it will throw an error.
// It will prompt an accessibility permission request dialog, if needed. // It will prompt an accessibility permission request dialog, if needed.
if(window != undefined){ if (window != undefined) {
windowManager.requestAccessibility(); windowManager.requestAccessibility();
// Sets the active window's bounds. // Sets the active window's bounds.
let size = screen.getPrimaryDisplay().workAreaSize let size = screen.getPrimaryDisplay().workAreaSize
let width = parseInt(size.width) let width = parseInt(size.width)
let height = parseInt(size.height * 0.6) let height = parseInt(size.height * 0.6)
window.setBounds({ x: 0, y: size.height * 0.4, height:height, width:width }); window.setBounds({x: 0, y: size.height * 0.4, height: height, width: width});
} }
flowchart_window.show(); flowchart_window.show();
} else if (msg.type == 2) { } else if (msg.type == 2) {
//keyboard //keyboard
// const robot = require("@jitsi/robotjs"); // const robot = require("@jitsi/robotjs");
let keyInfo = msg.message.keyboardStr; let keyInfo = msg.message.keyboardStr;
driver.findElement(By.xpath(msg.message.xpath)).sendKeys(Key.HOME, Key.chord(Key.SHIFT, Key.END), keyInfo); let handles = await driver.getAllWindowHandles();
// robot.keyTap("a", "control"); console.log("handles", handles);
// robot.keyTap("backspace"); let exit = false;
// robot.typeString(keyInfo); let content_handle = handle_pairs[msg.message.id];
// robot.keyTap("shift"); console.log(msg.message.id, content_handle);
// robot.keyTap("shift"); let order = [...handles.filter(handle => handle != current_handle && handle != content_handle), current_handle, content_handle]; //搜索顺序
} else if (msg.type == 3) { let len = order.length;
try { while (true) {
if (msg.from == 0) { // console.log("handles");
socket_flowchart.send(msg.message.pipe); //直接把消息转接 try{
} let h = order[len - 1];
else { console.log("current_handle", current_handle);
socket_window.send(msg.message.pipe); if(h != null && handles.includes(h)){
} await driver.switchTo().window(h);
} catch { current_handle = h;
dialog.showErrorBox("Error", "Please open the flowchart window first"); console.log("switch to handle: ", h);
} }
} else if (msg.type == 5){ // await driver.executeScript("window.stop();");
var child = require('child_process').execFile; // console.log("executeScript");
// 参数顺序： 1. task id 2. server address 3. saved_file_name 4. "remote" or "local" 5. user_data_folder let element = await driver.findElement(By.xpath(msg.message.xpath));
// var parameters = [msg.message.id, server_address]; console.log("Find Element at handle: ", current_handle);
let parameters = []; await element.sendKeys(Key.HOME, Key.chord(Key.SHIFT, Key.END), keyInfo);
console.log(msg.message) console.log("send key");
if(msg.message.user_data_folder == null || msg.message.user_data_folder == undefined || msg.message.user_data_folder == ""){ break;
parameters = ["--id", msg.message.id, "--server_address", server_address, "--user_data", 0]; } catch (error) {
} else { console.log("len", len);
let user_data_folder_path = __dirname.indexOf("resources")>=0 && __dirname.indexOf("app")>=0? path.join(__dirname, "../../..", msg.message.user_data_folder): path.join(__dirname, msg.message.user_data_folder); len = len - 1;
parameters = ["--id", msg.message.id, "--server_address", server_address, "--user_data", 1]; if (len == 0) {
config.user_data_folder = msg.message.user_data_folder; break;
config.absolute_user_data_folder = user_data_folder_path; }
fs.writeFileSync(path.join(task_server.getDir(), "config.json"), JSON.stringify(config)); }
} // .then(function (element) {
// child('Chrome/easyspider_executestage.exe', parameters, function(err,stdout, stderr) { // console.log("element", element, handles);
// console.log(stdout); // element.sendKeys(Key.HOME, Key.chord(Key.SHIFT, Key.END), keyInfo);
// }); // exit = true;
// }, function (error) {
// console.log("error", error);
// len = len - 1;
// if (len == 0) {
// exit = true;
// }
// }
// );
}
// let handles = driver.getAllWindowHandles();
// driver.switchTo().window(handles[handles.length - 1]);
// driver.findElement(By.xpath(msg.message.xpath)).sendKeys(Key.HOME, Key.chord(Key.SHIFT, Key.END), keyInfo);
// robot.keyTap("a", "control");
// robot.keyTap("backspace");
// robot.typeString(keyInfo);
// robot.keyTap("shift");
// robot.keyTap("shift");
} else if (msg.type == 3) {
try {
if (msg.from == 0) {
socket_flowchart.send(msg.message.pipe); //直接把消息转接
let message = JSON.parse(msg.message.pipe);
let type = message.type;
if(type.indexOf("Click")>=0){
await new Promise(resolve => setTimeout(resolve, 2000)); //等两秒
let handles = await driver.getAllWindowHandles();
if(arrayDifference(handles, old_handles).length > 0){
old_handles = handles;
current_handle = handles[handles.length - 1];
console.log("New tab opened, change current_handle to: ", current_handle);
}
}
} else {
socket_window.send(msg.message.pipe);
}
} catch {
dialog.showErrorBox("Error", "Please open the flowchart window first");
}
} else if (msg.type == 5) {
var child = require('child_process').execFile;
// 参数顺序： 1. task id 2. server address 3. saved_file_name 4. "remote" or "local" 5. user_data_folder
// var parameters = [msg.message.id, server_address];
let parameters = [];
console.log(msg.message)
if (msg.message.user_data_folder == null || msg.message.user_data_folder == undefined || msg.message.user_data_folder == "") {
parameters = ["--id", msg.message.id, "--server_address", server_address, "--user_data", 0];
} else {
let user_data_folder_path = __dirname.indexOf("resources") >= 0 && __dirname.indexOf("app") >= 0 ? path.join(__dirname, "../../..", msg.message.user_data_folder) : path.join(__dirname, msg.message.user_data_folder);
parameters = ["--id", msg.message.id, "--server_address", server_address, "--user_data", 1];
config.user_data_folder = msg.message.user_data_folder;
config.absolute_user_data_folder = user_data_folder_path;
fs.writeFileSync(path.join(task_server.getDir(), "config.json"), JSON.stringify(config));
}
// child('Chrome/easyspider_executestage.exe', parameters, function(err,stdout, stderr) {
// console.log(stdout);
// });
let spawn = require("child_process").spawn; let spawn = require("child_process").spawn;
let child_process = spawn(execute_path, parameters); let child_process = spawn(execute_path, parameters);
child_process.stdout.on('data', function (data) { child_process.stdout.on('data', function (data) {
console.log(data.toString()); console.log(data.toString());
}); });
} }
} }
const WebSocket = require('ws'); const WebSocket = require('ws');
let socket_window = null; let socket_window = null;
let socket_start = null; let socket_start = null;
let socket_flowchart = null; let socket_flowchart = null;
var wss = new WebSocket.Server({ port: websocket_port }); let wss = new WebSocket.Server({port: websocket_port});
wss.on('connection', function (ws) { wss.on('connection', function (ws) {
ws.on('message', function (message, isBinary) { ws.on('message', async function (message, isBinary) {
let msg = JSON.parse(message.toString()); let msg = JSON.parse(message.toString());
// console.log(msg, msg.type, msg.message); // console.log(msg, msg.type, msg.message);
if (msg.type == 0) { if (msg.type == 0) {
if (msg.message.id == 0) { if (msg.message.id == 0) {
socket_window = ws; socket_window = ws;
console.log("set socket_window") console.log("set socket_window")
} } else if (msg.message.id == 1) {
else if (msg.message.id == 1) { socket_start = ws;
socket_start = ws; console.log("set socket_start")
console.log("set socket_start") } else if (msg.message.id == 2) {
} socket_flowchart = ws;
else if (msg.message.id == 2) { console.log("set socket_flowchart");
socket_flowchart = ws; } else{
console.log("set socket_flowchart"); await new Promise(resolve => setTimeout(resolve, 2300));
} handle_pairs[msg.message.id] = current_handle;
} else { console.log("Set handle_pair for id: ", msg.message.id, " to ", current_handle);
beginInvoke(msg); // console.log("handle_pairs: ", handle_pairs);
} }
}); } else if (msg.type == 10) {
let leave_handle = handle_pairs[msg.message.id];
if (leave_handle!=null && leave_handle!=undefined && leave_handle!="")
{
await driver.switchTo().window(leave_handle);
console.log("Switch to handle: ", leave_handle);
current_handle = leave_handle;
}
}
else {
await beginInvoke(msg);
}
});
}); });
console.log(process.platform); console.log(process.platform);
async function runBrowser(lang="en", user_data_folder='') { async function runBrowser(lang = "en", user_data_folder = '') {
const serviceBuilder = new ServiceBuilder(driverPath); const serviceBuilder = new ServiceBuilder(driverPath);
let options = new chrome.Options(); let options = new chrome.Options();
language = lang; options.addArguments('--disable-blink-features=AutomationControlled');
if(lang=="en") { language = lang;
options.addExtensions(path.join(__dirname, "EasySpider_en.crx")); if (lang == "en") {
} else if(lang=="zh") { options.addExtensions(path.join(__dirname, "EasySpider_en.crx"));
options.addExtensions(path.join(__dirname, "EasySpider_zh.crx")); } else if (lang == "zh") {
} options.addExtensions(path.join(__dirname, "EasySpider_zh.crx"));
options.setChromeBinaryPath(chromeBinaryPath); }
if(user_data_folder != ""){ options.setChromeBinaryPath(chromeBinaryPath);
let dir = __dirname.indexOf("resources")>=0 && __dirname.indexOf("app")>=0? path.join(__dirname, "../../..", user_data_folder): path.join(__dirname, user_data_folder); if (user_data_folder != "") {
options.addArguments("--user-data-dir=" + dir); let dir = __dirname.indexOf("resources") >= 0 && __dirname.indexOf("app") >= 0 ? path.join(__dirname, "../../..", user_data_folder) : path.join(__dirname, user_data_folder);
config.user_data_folder = user_data_folder; options.addArguments("--user-data-dir=" + dir);
fs.writeFileSync(path.join(task_server.getDir(), "config.json"), JSON.stringify(config)); config.user_data_folder = user_data_folder;
} fs.writeFileSync(path.join(task_server.getDir(), "config.json"), JSON.stringify(config));
driver = new Builder() }
.forBrowser('chrome') driver = new Builder()
.setChromeOptions(options) .forBrowser('chrome')
.setChromeService(serviceBuilder) .setChromeOptions(options)
.build(); .setChromeService(serviceBuilder)
const cdpConnection = await driver.createCDPConnection("page"); .build();
let stealth_path = path.join(__dirname, "stealth.min.js"); await driver.manage().setTimeouts({implicit: 10000, pageLoad: 10000, script: 10000});
let stealth = fs.readFileSync(stealth_path, 'utf8'); await driver.executeScript("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})");
await cdpConnection.execute('Page.addScriptToEvaluateOnNewDocument', { const cdpConnection = await driver.createCDPConnection("page");
source: stealth, let stealth_path = path.join(__dirname, "stealth.min.js");
let stealth = fs.readFileSync(stealth_path, 'utf8');
await cdpConnection.execute('Page.addScriptToEvaluateOnNewDocument', {
source: stealth,
}); });
try { try {
await driver.get(server_address + "/taskGrid/taskList.html?wsport="+websocket_port+"&backEndAddressServiceWrapper=" + server_address + "&lang=" + lang); await driver.get(server_address + "/taskGrid/taskList.html?wsport=" + websocket_port + "&backEndAddressServiceWrapper=" + server_address + "&lang=" + lang);
} finally { old_handles = await driver.getAllWindowHandles();
// await driver.quit(); // 退出浏览器 current_handle = old_handles[old_handles.length - 1];
} } finally {
// await driver.quit(); // 退出浏览器
}
} }
function handleOpenBrowser(event, lang="en", user_data_folder= "") { function handleOpenBrowser(event, lang = "en", user_data_folder = "") {
const webContents = event.sender; const webContents = event.sender;
const win = BrowserWindow.fromWebContents(webContents); const win = BrowserWindow.fromWebContents(webContents);
runBrowser(lang, user_data_folder); runBrowser(lang, user_data_folder);
let size = screen.getPrimaryDisplay().workAreaSize; let size = screen.getPrimaryDisplay().workAreaSize;
let width = parseInt(size.width); let width = parseInt(size.width);
let height = parseInt(size.height * 0.6); let height = parseInt(size.height * 0.6);
flowchart_window = new BrowserWindow({ flowchart_window = new BrowserWindow({
x:0, x: 0,
y:0, y: 0,
width: width, width: width,
height: height, height: height,
icon: iconPath, icon: iconPath,
}); });
let url = ""; let url = "";
let id = -1; let id = -1;
if (lang == "en") { if (lang == "en") {
url = server_address + `/taskGrid/FlowChart.html?id=${id}&wsport=${websocket_port}&backEndAddressServiceWrapper=` + server_address; url = server_address + `/taskGrid/FlowChart.html?id=${id}&wsport=${websocket_port}&backEndAddressServiceWrapper=` + server_address;
}else if(lang == "zh") { } else if (lang == "zh") {
url = server_address + `/taskGrid/FlowChart_CN.html?id=${id}&wsport=${websocket_port}&backEndAddressServiceWrapper=` + server_address; url = server_address + `/taskGrid/FlowChart_CN.html?id=${id}&wsport=${websocket_port}&backEndAddressServiceWrapper=` + server_address;
} }
// and load the index.html of the app. // and load the index.html of the app.
flowchart_window.loadURL(url); flowchart_window.loadURL(url);
flowchart_window.hide(); flowchart_window.hide();
flowchart_window.on('close', function (event) { flowchart_window.on('close', function (event) {
mainWindow.show(); mainWindow.show();
driver.quit(); driver.quit();
}); });
} }
function handleOpenInvoke(event, lang="en"){ function handleOpenInvoke(event, lang = "en") {
const window = new BrowserWindow({icon: iconPath}); const window = new BrowserWindow({icon: iconPath});
let url = ""; let url = "";
language = lang; language = lang;
if (lang == "en") { if (lang == "en") {
url = server_address + `/taskGrid/taskList.html?type=1&wsport=${websocket_port}&backEndAddressServiceWrapper=` + server_address; url = server_address + `/taskGrid/taskList.html?type=1&wsport=${websocket_port}&backEndAddressServiceWrapper=` + server_address;
}else if(lang == "zh") { } else if (lang == "zh") {
url = server_address + `/taskGrid/taskList.html?type=1&wsport=${websocket_port}&backEndAddressServiceWrapper=` + server_address + "&lang=zh"; url = server_address + `/taskGrid/taskList.html?type=1&wsport=${websocket_port}&backEndAddressServiceWrapper=` + server_address + "&lang=zh";
} }
// and load the index.html of the app. // and load the index.html of the app.
window.loadURL(url); window.loadURL(url);
window.maximize(); window.maximize();
mainWindow.hide(); mainWindow.hide();
window.on('close', function (event) { window.on('close', function (event) {
mainWindow.show(); mainWindow.show();
}); });
} }
// This method will be called when Electron has finished // This method will be called when Electron has finished
// initialization and is ready to create browser windows. // initialization and is ready to create browser windows.
// Some APIs can only be used after this event occurs. // Some APIs can only be used after this event occurs.
app.whenReady().then(() => { app.whenReady().then(() => {
ipcMain.on('start-design', handleOpenBrowser); ipcMain.on('start-design', handleOpenBrowser);
ipcMain.on('start-invoke', handleOpenInvoke); ipcMain.on('start-invoke', handleOpenInvoke);
createWindow(); createWindow();
app.on('activate', function () { app.on('activate', function () {
// On macOS it's common to re-create a window in the app when the // On macOS it's common to re-create a window in the app when the
// dock icon is clicked and there are no other windows open. // dock icon is clicked and there are no other windows open.
if (BrowserWindow.getAllWindows().length === 0){ if (BrowserWindow.getAllWindows().length === 0) {
createWindow(); createWindow();
} }
}) })
}) })
// Quit when all windows are closed, except on macOS. There, it's common // Quit when all windows are closed, except on macOS. There, it's common
// for applications and their menu bar to stay active until the user quits // for applications and their menu bar to stay active until the user quits
// explicitly with Cmd + Q. // explicitly with Cmd + Q.
app.on('window-all-closed', function () { app.on('window-all-closed', function () {
if (process.platform !== 'darwin'){ if (process.platform !== 'darwin') {
app.quit(); app.quit();
} }
}) })
// In this file you can include the rest of your app's specific main process // In this file you can include the rest of your app's specific main process
// code. You can also put them in separate files and require them here. // code. You can also put them in separate files and require them here.
function arrayDifference(arr1, arr2) {
return arr1.filter(item => !arr2.includes(item));
}

BIN
Extension/manifest_v3/EasySpider_en.crx
View File

Binary file not shown.

BIN
Extension/manifest_v3/EasySpider_zh.crx
View File

Binary file not shown.

10
Extension/manifest_v3/src/background.ts
View File

@ -25,9 +25,17 @@ chrome.runtime.onMessage.addListener(function(request, sender, sendResponse) {
} else if (request.type == 2) { } else if (request.type == 2) {
let message = { let message = {
type: 2, //消息类型，2代表键盘输入 type: 2, //消息类型，2代表键盘输入
message: { "keyboardStr": request.value, "xpath": request.xpath } // {}全选{BS}退格 message: { "keyboardStr": request.value, "xpath": request.xpath, "id": request.id } // {}全选{BS}退格
}; };
ws.send(JSON.stringify(message)); ws.send(JSON.stringify(message));
chrome.tabs.query({active: true, currentWindow: true}, function(tabs) {
// 获取当前选项卡的 ID
const tabId = tabs[0].id;
const url = tabs[0].url;
// 停止当前页面的加载
chrome.tabs.executeScript(tabId, {code: 'window.stop();'});
});
} else if (request.type == 3) { } else if (request.type == 3) {
let tmsg = request.msg; let tmsg = request.msg;
tmsg.tabIndex = nowTabIndex; //赋值当前tab的id tmsg.tabIndex = nowTabIndex; //赋值当前tab的id

14
Extension/manifest_v3/src/content-scripts/global.js
View File

@ -17,8 +17,22 @@ export var global = {
defaultbgColor: 'rgba(221,221,255,0.8)', defaultbgColor: 'rgba(221,221,255,0.8)',
boxShadowColor: "blue 0px 0px 5px", boxShadowColor: "blue 0px 0px 5px",
lang: config.language, lang: config.language,
id: "C" + Math.floor(Math.random() * (99999999)).toString(),
ws: null,
}; };
global.ws = new WebSocket("ws://localhost:8084");
global.ws.onopen = function() {
// Web Socket 已连接上，使用 send() 方法发送数据
console.log("已连接");
let message = {
type: 0, //消息类型，0代表连接操作
message: {
id: global.id, //socket id
}
};
this.send(JSON.stringify(message));
};
export function getOS () { export function getOS () {
if (navigator.userAgent.indexOf('Window') > 0) { if (navigator.userAgent.indexOf('Window') > 0) {

81
Extension/manifest_v3/src/content-scripts/main.js
View File

@ -114,6 +114,17 @@ document.addEventListener("mousemove", function() {
}); });
window.addEventListener("beforeunload", function(event) {
event.preventDefault();
let message = {
type: 10,
message: {
id: global.id, //socket id
}
};
global.ws.send(JSON.stringify(message));
});
//点击没反应时候的替代方案 //点击没反应时候的替代方案
document.onkeydown = function(event) { document.onkeydown = function(event) {
// console.log("keydown"); // console.log("keydown");
@ -153,6 +164,41 @@ function generateToolkit() {
if (difference > 0) { if (difference > 0) {
$(".tooldrag").css("cssText", "height:" + (26 + difference) + "px!important") $(".tooldrag").css("cssText", "height:" + (26 + difference) + "px!important")
} }
//实现提示框拖拽功能
$('.tooldrag').mousedown(function(e) {
// e.pageX
var positionDiv = $(this).offset();
var distanceX = e.pageX - positionDiv.left;
var distanceY = e.pageY - positionDiv.top;
//alert(distanceX)
// alert(positionDiv.left);
$(document).mousemove(function(e) {
var x = e.clientX - distanceX;
var y = e.clientY - distanceY;
if (x < 0) {
x = 0;
} else if (x > window.innerWidth - $('.tooldrag').outerWidth(true)) {
x = window.innerWidth - $('.tooldrag').outerWidth(true);
}
if (y < 0) {
y = 0;
} else if (y > window.innerHeight - $('.tooldrag').outerHeight(true)) {
y = window.innerHeight - $('.tooldrag').outerHeight(true);
}
$('.tooltips').css({
'right': window.innerWidth - x - $('.tooltips').outerWidth(true) + 'px',
'bottom': window.innerHeight - y - $('.tooltips').outerHeight(true) + 'px',
});
});
$(document).mouseup(function() {
$(document).off('mousemove');
});
});
timer = setInterval(function() { //时刻监测相应元素是否存在(防止出现如百度一样元素消失重写body的情况)，如果不存在，添加进来 timer = setInterval(function() { //时刻监测相应元素是否存在(防止出现如百度一样元素消失重写body的情况)，如果不存在，添加进来
if (document.body != null && document.getElementById("wrapperToolkit") == null) { if (document.body != null && document.getElementById("wrapperToolkit") == null) {
this.clearInterval(); //先取消原来的计时器，再设置新的计时器 this.clearInterval(); //先取消原来的计时器，再设置新的计时器
@ -166,38 +212,3 @@ function generateToolkit() {
//Vue元素 //Vue元素
generateToolkit(); generateToolkit();
//实现提示框拖拽功能
$('.tooldrag').mousedown(function(e) {
// e.pageX
var positionDiv = $(this).offset();
var distanceX = e.pageX - positionDiv.left;
var distanceY = e.pageY - positionDiv.top;
//alert(distanceX)
// alert(positionDiv.left);
$(document).mousemove(function(e) {
var x = e.clientX - distanceX;
var y = e.clientY - distanceY;
if (x < 0) {
x = 0;
} else if (x > window.innerWidth - $('.tooldrag').outerWidth(true)) {
x = window.innerWidth - $('.tooldrag').outerWidth(true);
}
if (y < 0) {
y = 0;
} else if (y > window.innerHeight - $('.tooldrag').outerHeight(true)) {
y = window.innerHeight - $('.tooldrag').outerHeight(true);
}
$('.tooltips').css({
'right': window.innerWidth - x - $('.tooltips').outerWidth(true) + 'px',
'bottom': window.innerHeight - y - $('.tooltips').outerHeight(true) + 'px',
});
});
$(document).mouseup(function() {
$(document).off('mousemove');
});
});

5
Extension/manifest_v3/src/content-scripts/messageInteraction.js
View File

@ -23,9 +23,10 @@ export function input(value) {
"xpath": readXPath(global.nodeList[0]["node"], 0), "xpath": readXPath(global.nodeList[0]["node"], 0),
"value": value, "value": value,
}; };
let msg = { "type": 3, msg: message }; let msg = { type: 3, msg: message };
window.stop();
chrome.runtime.sendMessage(msg); chrome.runtime.sendMessage(msg);
msg = { "type": 2, value: value, xpath: message.xpath }; msg = { type: 2, value: value, xpath: message.xpath, id: global.id};
chrome.runtime.sendMessage(msg); chrome.runtime.sendMessage(msg);
} }

140
Extension/manifest_v3/src/content-scripts/toolkit.vue
View File

@ -571,145 +571,5 @@ export default {
</script> </script>
<style> <style>
.tooltips {
width: 330px;
min-height: 300px;
background-color: white;
position: fixed;
z-index: 2147483647;
right: 30px;
bottom: 30px;
font-size: 13px !important;
font-weight: normal !important;
border: solid navy 2px;
-webkit-user-select: none;
/* 文字不可被选中 */
}
.tooldrag {
background-color: navy;
width: 100%;
text-align: center;
font-size: 13px;
height: 26px !important;
padding-top: 8px !important;
color: white;
}
.realcontent {
text-align: left;
padding-top: 10px !important;
padding-bottom: 80px !important;
padding-left: 20px !important;
padding-right: 10px !important;
}
.innercontent {
text-align: left;
padding-top: 5px !important;
padding-left: 12px !important;
}
.innercontent a {
display: inline-block;
text-decoration: none;
margin-top: 2px !important;
font-size: 13px;
color: navy !important;
cursor: pointer;
text-decoration: none !important;
}
.innercontent a:hover {
color: blue !important;
}
.innercontent span {
font-size: 20px;
color: navy;
line-height: normal;
padding-left: 5px !important;
}
.tooltips button {
margin-top: 7px !important;
font-size: 13px;
border-radius: 5px;
border: solid 2px navy;
background-color: white;
color: navy;
width: 100px;
height: 30px;
cursor: pointer;
margin-left: 15px !important;
}
.tooltips input[type=text] {
display: block;
margin-top: 7px !important;
padding-left: 5px !important;
margin-bottom: 7px !important;
font-size: 15px;
border-radius: 5px;
border: solid 2px navy;
width: 204px;
height: 30px;
}
.tooltips button:hover {
color: blue;
}
/* 下面用来对冻结表格首行元素和固定表格宽度和高度设定样式 */
.toolkitcontain {
border: 1px solid #cdd !important;
/*width: 280px !important;*/
/* 上面的宽度设定很重要 */
height: 150px;
overflow: auto;
margin-top: 10px !important;
position: relative;
}
.toolkitcontain table {
table-layout: fixed;
border-spacing: 0!important;
word-break: break-all;
word-wrap: break-word;
text-overflow: ellipsis;
overflow: hidden;
white-space: nowrap;
width: 100%;
}
.toolkitcontain th,
.toolkitcontain td,
.toolkitcontain tr {
border: 1px solid rgb(78, 78, 78) !important;
height: 25px !important;
width: 100px !important;
text-align: center !important;
font-weight: normal !important;
overflow: hidden !important;
font-size: 11px !important;
padding-left: 1px !important;
padding-right: 0px !important;
padding-top: 0px !important;
padding-bottom: 0px !important;
vertical-align: middle!important;
}
.toolkitcontain .toolkittb2 {
position: sticky;
top: 0px;
margin-bottom: 0px;
background-color: azure;
z-index: 1000;
}
.toolkitcontain .toolkittb4 {
position: absolute;
}
</style> </style>

3
Extension/manifest_v3/src/manifest.json
View File

@ -27,6 +27,7 @@
"http://*/*", "http://*/*",
"https://*/*" "https://*/*"
], ],
"css": ["style/toolkit.css"],
"js": ["content-scripts/main.js"], "js": ["content-scripts/main.js"],
"run_at": "document_end", "run_at": "document_end",
"all_frames": false "all_frames": false
@ -42,6 +43,6 @@
} }
], ],
"permissions": [ "permissions": [
"identity", "storage", "tabs" "identity", "storage", "tabs","scripting"
] ]
} }

141
Extension/manifest_v3/src/style/toolkit.css Normal file
View File

@ -0,0 +1,141 @@
.tooltips {
width: 330px;
min-height: 300px;
background-color: white;
position: fixed;
z-index: 2147483647;
right: 30px;
bottom: 30px;
font-size: 13px !important;
font-weight: normal !important;
border: solid navy 2px;
-webkit-user-select: none;
/* 文字不可被选中 */
}
.tooldrag {
background-color: navy;
width: 100%;
text-align: center;
font-size: 13px;
height: 26px !important;
padding-top: 8px !important;
color: white;
}
.realcontent {
text-align: left;
padding-top: 10px !important;
padding-bottom: 80px !important;
padding-left: 20px !important;
padding-right: 10px !important;
}
.innercontent {
text-align: left;
padding-top: 5px !important;
padding-left: 12px !important;
}
.innercontent a {
display: inline-block;
text-decoration: none;
margin-top: 2px !important;
font-size: 13px;
color: navy !important;
cursor: pointer;
text-decoration: none !important;
}
.innercontent a:hover {
color: blue !important;
}
.innercontent span {
font-size: 20px;
color: navy;
line-height: normal;
padding-left: 5px !important;
}
.tooltips button {
margin-top: 7px !important;
font-size: 13px;
border-radius: 5px;
border: solid 2px navy;
background-color: white;
color: navy;
width: 100px;
height: 30px;
cursor: pointer;
margin-left: 15px !important;
}
.tooltips input[type=text] {
display: block;
margin-top: 7px !important;
padding-left: 5px !important;
margin-bottom: 7px !important;
font-size: 15px;
border-radius: 5px;
border: solid 2px navy;
width: 204px;
height: 30px;
}
.tooltips button:hover {
color: blue;
}
/* 下面用来对冻结表格首行元素和固定表格宽度和高度设定样式 */
.toolkitcontain {
border: 1px solid #cdd !important;
/*width: 280px !important;*/
/* 上面的宽度设定很重要 */
height: 150px;
overflow: auto;
margin-top: 10px !important;
position: relative;
}
.toolkitcontain table {
table-layout: fixed;
border-spacing: 0!important;
word-break: break-all;
word-wrap: break-word;
text-overflow: ellipsis;
overflow: hidden;
white-space: nowrap;
width: 100%;
}
.toolkitcontain th,
.toolkitcontain td,
.toolkitcontain tr {
border: 1px solid rgb(78, 78, 78) !important;
height: 25px !important;
width: 100px !important;
text-align: center !important;
font-weight: normal !important;
overflow: hidden !important;
font-size: 11px !important;
padding-left: 1px !important;
padding-right: 0px !important;
padding-top: 0px !important;
padding-bottom: 0px !important;
vertical-align: middle!important;
}
.toolkitcontain .toolkittb2 {
position: sticky;
top: 0px;
margin-bottom: 0px;
background-color: azure;
z-index: 1000;
}
.toolkitcontain .toolkittb4 {
position: absolute;
}

11
Releases/EasySpider_windows_amd64/Data/task_0_11752920.csv Normal file
View File

@ -0,0 +1,11 @@
参数1_文本
吸毒的人最初是怎样沾上毒品的？
LSP是什么意思？
如何看待微软研究院发表的 GPT-4 测评文章，认为 GPT-4 可以被视作AGI的早期版本？
有哪些嵌套了好几层逻辑的经典故事？
百度还有希望崛起吗？
谁能告诉我你觉得最可悲的事是什么？
小张买了河豚鱼挂车把手上，转头鱼被王大妈偷了后，煮给孙子吃，孙子被毒死，民事责任与刑事责任怎么认定?
为何 Linus 一个人就能写出这么强的系统，中国却做不出来？
鸿门宴明明是一个失败的宴席，为什么却能进入中学课本？
男人最无声的炫富是什么？
1 参数1_文本
2 吸毒的人最初是怎样沾上毒品的？
3 LSP是什么意思？
4 如何看待微软研究院发表的 GPT-4 测评文章，认为 GPT-4 可以被视作AGI的早期版本？
5 有哪些嵌套了好几层逻辑的经典故事？
6 百度还有希望崛起吗？
7 谁能告诉我你觉得最可悲的事是什么？
8 小张买了河豚鱼挂车把手上，转头鱼被王大妈偷了后，煮给孙子吃，孙子被毒死，民事责任与刑事责任怎么认定?
9 为何 Linus 一个人就能写出这么强的系统，中国却做不出来？
10 鸿门宴明明是一个失败的宴席，为什么却能进入中学课本？
11 男人最无声的炫富是什么？

16
Releases/EasySpider_windows_amd64/Data/task_0_11752920_log.txt Normal file
View File

@ -0,0 +1,16 @@
openPage
Loading page: https://www.zhihu.com
loop
getData
getData
getData
pathNotFound: //*[contains(@class, "css-0")]/div[5]/div[1]/div[1]/div[1]/h2[1]/div[1]
getData
getData
pathNotFound: //*[contains(@class, "css-0")]/div[8]/div[1]/div[1]/div[1]/h2[1]/div[1]
getData
getData
getData
getData
getData
Done!

1
Releases/EasySpider_windows_amd64/execution_instances/0.json Normal file
View File

@ -0,0 +1 @@
{"id":0,"name":"知乎_登录后采集","url":"https://www.zhihu.com","links":"https://www.zhihu.com","containJudge":false,"desc":"https://www.zhihu.com\n使用带用户配置的浏览器模式来先手工登录后保存信息，再接着执行。","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.zhihu.com","desc":"要采集的网址列表,多行以\\n分开","type":"string","exampleValue":"https://www.zhihu.com"}],"outputParameters":[{"id":0,"name":"参数1_文本","desc":"","type":"string","exampleValue":"历史上有哪些通过“正当手段”干出不正当事的人物？"}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"url":"https://www.zhihu.com","links":"https://www.zhihu.com","scrollType":0,"scrollCount":0}},{"id":2,"index":2,"parentId":0,"type":1,"option":8,"title":"循环","sequence":[3],"isInLoop":false,"position":1,"parameters":{"history":5,"tabIndex":-1,"useLoop":false,"xpath":"","wait":0,"scrollType":0,"scrollCount":0,"loopType":2,"pathList":"//*[contains(@class, \"css-0\")]/div[2]/div[1]/div[1]/div[1]/h2[1]/div[1]\n//*[contains(@class, \"css-0\")]/div[3]/div[1]/div[1]/div[1]/h2[1]/div[1]\n//*[contains(@class, \"css-0\")]/div[4]/div[1]/div[1]/div[1]/h2[1]/div[1]\n//*[contains(@class, \"css-0\")]/div[5]/div[1]/div[1]/div[1]/h2[1]/div[1]\n//*[contains(@class, \"css-0\")]/div[6]/div[1]/div[1]/div[1]/h2[1]/div[1]\n//*[contains(@class, \"css-0\")]/div[7]/div[1]/div[1]/div[1]/h2[1]/div[1]\n//*[contains(@class, \"css-0\")]/div[8]/div[1]/div[1]/div[1]/h2[1]/div[1]\n//*[contains(@class, \"css-0\")]/div[9]/div[1]/div[1]/div[1]/h2[1]/div[1]\n//*[contains(@class, \"css-0\")]/div[10]/div[1]/div[1]/div[1]/h2[1]/div[1]\n//*[contains(@class, \"css-0\")]/div[11]/div[1]/div[1]/div[1]/h2[1]/div[1]\n//*[contains(@class, \"css-0\")]/div[12]/div[1]/div[1]/div[1]/h2[1]/div[1]\n//*[contains(@class, \"css-0\")]/div[13]/div[1]/div[1]/div[1]/h2[1]/div[1]","textList":"","exitCount":0,"historyWait":2}},{"id":3,"index":3,"parentId":2,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":5,"tabIndex":-1,"useLoop":false,"xpath":"","wait":0,"paras":[{"nodeType":0,"contentType":0,"relative":true,"name":"参数1_文本","desc":"","relativeXpath":"","exampleValues":[{"num":0,"value":"历史上有哪些通过“正当手段”干出不正当事的人物？"},{"num":1,"value":"新加坡有哪些不好的地方？"},{"num":2,"value":"孙悟空可以秒杀山村老尸那样的厉鬼吗？"},{"num":3,"value":"为什么渐渐厌倦玩《原神》了?"},{"num":4,"value":"历史上有哪些著名的考古乌龙事件?"},{"num":5,"value":"苹果公司为什么能把用户调教得这么好？"},{"num":6,"value":"哪个瞬间让你发现了世界的bug？"},{"num":7,"value":"假如中国的院士，想为亲属谋体制内的工作，难度大吗？为什么？"},{"num":8,"value":"你一直珍藏的视频是哪个？"},{"num":9,"value":"如何评价《原神》角色艾莉丝？"},{"num":10,"value":"索罗斯如何做空的英镑、泰铢？为什么做空香港失败了？"},{"num":11,"value":"如何在婚前认清并杜绝王力宏这种男人？"}],"default":""}],"loopType":2}}]}