mirror of
https://github.com/NaiboWang/EasySpider.git
synced 2025-04-20 04:39:57 +08:00
JSON Format Support
This commit is contained in:
naibo
parent
7c5f9a4139
commit
528ae7a132
@ -639,6 +639,7 @@ If the expression returns a value greater than 0 or evaluates to True, the opera
|
||||
<option value = "xlsx">XLSX (EXCEL, note that a single Excel cell can save up to 32767 characters)</option>
|
||||
<option value = "csv">CSV</option>
|
||||
<option value = "txt">TXT</option>
|
||||
<option value = "json">JSON</option>
|
||||
<option value = "mysql">MySQL Database</option>
|
||||
</select>
|
||||
<label>Export File Name/Database Table Name (Can use ../ to represent relative path to change the file save location,the keyword "current_time" will be replaced with the timestamp when the task is executed):</label>
|
||||
|
||||
@ -640,6 +640,7 @@ print(emotlib.emoji()) # 使用其中的函数。
|
||||
<option value = "xlsx">XLSX（即EXCEL文件，注意Excel单个单元格最多可存储32767字符）</option>
|
||||
<option value = "csv">CSV</option>
|
||||
<option value = "txt">TXT</option>
|
||||
<option value = "json">JSON</option>
|
||||
<option value = "mysql">MySQL数据库</option>
|
||||
</select>
|
||||
<label>导出文件名/数据库表格名称（可使用../表示相对路径以改变文件保存位置，名称中的“current_time”会被替换为执行任务时的时间戳）：</label>
|
||||
|
||||
File diff suppressed because one or more lines are too long
1
ElectronJS/tasks/200.json
Normal file
1
ElectronJS/tasks/200.json
Normal file
@ -0,0 +1 @@
|
||||
{"id":200,"name":"图片下载","url":"https://www.jd.com","links":"https://www.jd.com","create_time":"7/24/2023, 1:33:01 PM","update_time":"7/24/2023, 2:08:02 PM","version":"0.5.0","saveThreshold":10,"cloudflare":0,"environment":0,"maxViewLength":15,"recordLog":1,"outputFormat":"xlsx","saveName":"current_time","inputExcel":"","startFromExit":0,"containJudge":false,"desc":"https://www.jd.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.jd.com","desc":"要采集的网址列表，多行以\\n分开","type":"text","exampleValue":"https://www.jd.com"}],"outputParameters":[{"id":0,"name":"参数1_图片地址","desc":"","type":"text","recordASField":1,"exampleValue":"//m.360buyimg.com/babel/jfs/t1/160456/7/37206/196421/649c09faFeab01f59/cc5ea5b81653b3a5.png"}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"url":"https://www.jd.com","links":"https://www.jd.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":2,"index":2,"parentId":0,"type":1,"option":8,"title":"循环","sequence":[3],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/div[5]/div[1]/div[1]/div[1]/div[2]/div[1]/div[1]/div[1]/div[2]/div[1]/div/div[1]/div[1]/a[1]/img[1]","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":1,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":["/html/body/div[5]/div[1]/div[1]/div[1]/div[2]/div[1]/div[1]/div[1]/div[2]/div[1]/div[1]/div[1]/div[1]/a[1]/img[1]","//img[contains(., '')]","/html/body/div[last()-5]/div/div[last()-4]/div/div[last()-1]/div/div[last()-1]/div/div[last()-1]/div/div[last()-8]/div/div/a/img"]}},{"id":3,"index":3,"parentId":2,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"clear":0,"paras":[{"nodeType":4,"contentType":0,"relative":true,"name":"参数1_图片地址","desc":"","extractType":0,"relativeXPath":"","allXPaths":"","exampleValues":[{"num":0,"value":"//m.360buyimg.com/babel/jfs/t1/160456/7/37206/196421/649c09faFeab01f59/cc5ea5b81653b3a5.png"}],"unique_index":"wujk9g2wu7lkgfly5m","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"arguments[0].src = arguments[0].src.replace(\"a\",\"b\")","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":1}],"loopType":1}}]}
|
||||
1
ElectronJS/tasks/201.json
Normal file
1
ElectronJS/tasks/201.json
Normal file
@ -0,0 +1 @@
|
||||
{"id":201,"name":"京东全球版-专业的综合网上购物商城","url":"https://www.jd.com","links":"https://www.jd.com","create_time":"7/24/2023, 2:03:04 PM","update_time":"7/24/2023, 2:06:26 PM","version":"0.5.0","saveThreshold":10,"cloudflare":0,"environment":0,"maxViewLength":15,"recordLog":1,"outputFormat":"json","saveName":"TEST","inputExcel":"","startFromExit":0,"containJudge":false,"desc":"https://www.jd.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.jd.com","desc":"要采集的网址列表，多行以\\n分开","type":"text","exampleValue":"https://www.jd.com"}],"outputParameters":[{"id":0,"name":"参数1_链接文本","desc":"","type":"text","recordASField":1,"exampleValue":"手机"},{"id":1,"name":"参数2_链接地址","desc":"","type":"text","recordASField":1,"exampleValue":"https://shouji.jd.com/"}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"url":"https://www.jd.com","links":"https://www.jd.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":2,"index":2,"parentId":0,"type":1,"option":8,"title":"循环","sequence":[3],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div/a","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":1,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":["/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/a[1]","//a[contains(., '手机')]","/html/body/div[last()-6]/div/div[last()-4]/div/div[last()-2]/div/div/div/div[last()-1]/div[last()-12]/a[last()-1]"]}},{"id":3,"index":3,"parentId":2,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"clear":0,"paras":[{"nodeType":1,"contentType":0,"relative":true,"name":"参数1_链接文本","desc":"","extractType":0,"relativeXPath":"","allXPaths":"","exampleValues":[{"num":0,"value":"手机"}],"unique_index":"g8qhgp3k42llkggokun","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":2,"contentType":0,"relative":true,"name":"参数2_链接地址","desc":"","relativeXPath":"","allXPaths":"","exampleValues":[{"num":0,"value":"https://shouji.jd.com/"}],"unique_index":"g8qhgp3k42llkggokun","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}],"loopType":1}}]}
|
||||
2
ExecuteStage/.vscode/launch.json
vendored
2
ExecuteStage/.vscode/launch.json
vendored
@ -12,7 +12,7 @@
|
||||
"justMyCode": false,
|
||||
// "args": ["--id", "[7]", "--read_type", "remote", "--headless", "0"]
|
||||
// "args": ["--id", "[9]", "--read_type", "remote", "--headless", "0", "--saved_file_name", "YOUTUBE"]
|
||||
"args": ["--id", "[8]", "--headless", "0", "--user_data", "1", "--keyboard", "0"]
|
||||
"args": ["--id", "[16]", "--headless", "0", "--user_data", "1", "--keyboard", "0"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -1,6 +1,6 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# import atexit
|
||||
from utils import download_image, get_output_code, isnotnull, lowercase_tags_in_xpath, myMySQL, new_line, on_press_creator, on_release_creator, replace_field_values, write_to_csv, write_to_excel
|
||||
from utils import download_image, get_output_code, isnotnull, lowercase_tags_in_xpath, myMySQL, new_line, on_press_creator, on_release_creator, replace_field_values, write_to_csv, write_to_excel, write_to_json
|
||||
from myChrome import MyChrome
|
||||
from threading import Thread, Event
|
||||
from PIL import Image
|
||||
@ -152,27 +152,24 @@ class BrowserThread(Thread):
|
||||
filter(isnotnull, service["links"].split("\n"))) # 要执行的link的列表
|
||||
self.OUTPUT = [] # 采集的数据
|
||||
self.writeMode = 1 # 写入模式，0为新建，1为追加
|
||||
if self.outputFormat == "csv" or self.outputFormat == "txt":
|
||||
if self.outputFormat == "csv" or self.outputFormat == "txt" or self.outputFormat == "xlsx":
|
||||
if not os.path.exists("Data/Task_" + str(self.id) + "/" + self.saveName + '.' + self.outputFormat):
|
||||
self.OUTPUT.append([]) # 添加表头
|
||||
self.writeMode = 0
|
||||
elif self.outputFormat == "xlsx":
|
||||
if not os.path.exists("Data/Task_" + str(self.id) + "/" + self.saveName + '.xlsx'):
|
||||
self.OUTPUT.append([]) # 添加表头
|
||||
self.writeMode = 0
|
||||
elif self.outputFormat == "json":
|
||||
self.writeMode = 3 # JSON模式无需判断是否存在文件
|
||||
elif self.outputFormat == "mysql":
|
||||
self.mysql = myMySQL(config["mysql_config_path"])
|
||||
self.mysql.create_table(self.saveName, service["outputParameters"])
|
||||
self.writeMode = 2
|
||||
if self.writeMode == 1:
|
||||
self.print_and_log("追加模式")
|
||||
self.print_and_log("Append Mode")
|
||||
elif self.writeMode == 0:
|
||||
self.print_and_log("新建模式")
|
||||
self.print_and_log("New Mode")
|
||||
if self.writeMode == 0:
|
||||
self.print_and_log("新建模式|Create Mode")
|
||||
elif self.writeMode == 1:
|
||||
self.print_and_log("追加模式|Append Mode")
|
||||
elif self.writeMode == 2:
|
||||
self.print_and_log("MySQL模式")
|
||||
self.print_and_log("MySQL Mode")
|
||||
self.print_and_log("MySQL模式|MySQL Mode")
|
||||
elif self.writeMode == 3:
|
||||
self.print_and_log("JSON模式|JSON Mode")
|
||||
self.containJudge = service["containJudge"] # 是否含有判断语句
|
||||
self.outputParameters = {}
|
||||
self.service = service
|
||||
@ -401,6 +398,10 @@ class BrowserThread(Thread):
|
||||
str(self.id) + "/" + self.saveName + '.xlsx'
|
||||
write_to_excel(
|
||||
file_name, self.OUTPUT, self.outputParametersTypes, self.outputParametersRecord)
|
||||
elif self.outputFormat == "json":
|
||||
file_name = "Data/Task_" + \
|
||||
str(self.id) + "/" + self.saveName + '.json'
|
||||
write_to_json(file_name, self.OUTPUT, self.outputParametersTypes, self.outputParametersRecord, self.outputParameters.keys())
|
||||
elif self.outputFormat == "mysql":
|
||||
self.mysql.write_to_mysql(
|
||||
self.OUTPUT, self.outputParametersRecord, self.outputParametersTypes)
|
||||
@ -1395,7 +1396,7 @@ class BrowserThread(Thread):
|
||||
except:
|
||||
downloadPic = 0
|
||||
if downloadPic == 1:
|
||||
download_image(content, "Data/Task_" +
|
||||
download_image(self, content, "Data/Task_" +
|
||||
str(self.id) + "/" + self.saveName + "/")
|
||||
else: # 普通节点
|
||||
content = element.text
|
||||
@ -1420,7 +1421,7 @@ class BrowserThread(Thread):
|
||||
except:
|
||||
downloadPic = 0
|
||||
if downloadPic == 1:
|
||||
download_image(content, "Data/Task_" +
|
||||
download_image(self, content, "Data/Task_" +
|
||||
str(self.id) + "/" + self.saveName + "/")
|
||||
else:
|
||||
command = 'var arr = [];\
|
||||
|
||||
@ -95,7 +95,7 @@ def on_release_creator(event, press_time):
|
||||
# time.sleep(1) # 每秒检查一次
|
||||
|
||||
|
||||
def download_image(url, save_directory):
|
||||
def download_image(browser, url, save_directory):
|
||||
# 定义浏览器头信息
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
||||
@ -120,15 +120,15 @@ def download_image(url, save_directory):
|
||||
with open(save_path, 'wb') as file:
|
||||
file.write(response.content)
|
||||
|
||||
print("图片已成功下载到:", save_path)
|
||||
print("The image has been successfully downloaded to:", save_path)
|
||||
browser.print_and_log("图片已成功下载到:", save_path)
|
||||
browser.print_and_log("The image has been successfully downloaded to:", save_path)
|
||||
else:
|
||||
print("下载图片失败，请检查此图片链接是否有效:", url)
|
||||
print(
|
||||
browser.print_and_log("下载图片失败，请检查此图片链接是否有效:", url)
|
||||
browser.print_and_log(
|
||||
"Failed to download image, please check if this image link is valid:", url)
|
||||
else:
|
||||
print("下载图片失败，请检查此图片链接是否有效:", url)
|
||||
print("Failed to download image, please check if this image link is valid:", url)
|
||||
browser.print_and_log("下载图片失败，请检查此图片链接是否有效:", url)
|
||||
browser.print_and_log("Failed to download image, please check if this image link is valid:", url)
|
||||
|
||||
|
||||
def get_output_code(output):
|
||||
@ -182,6 +182,41 @@ def replace_field_values(orginal_text, outputParameters):
|
||||
return replaced_text
|
||||
|
||||
|
||||
def write_to_json(file_name, data, types, record, keys):
|
||||
keys = list(keys)
|
||||
# Prepare empty list for data
|
||||
data_to_write = []
|
||||
# Tranform data and append to list
|
||||
for line in data:
|
||||
to_write = {}
|
||||
for i in range(len(line)):
|
||||
if types[i] == "int" or types[i] == "bigInt":
|
||||
try:
|
||||
line[i] = int(line[i])
|
||||
except:
|
||||
line[i] = 0
|
||||
elif types[i] == "double":
|
||||
try:
|
||||
line[i] = float(line[i])
|
||||
except:
|
||||
line[i] = 0.0
|
||||
if record[i]:
|
||||
to_write.update({keys[i]: line[i]})
|
||||
data_to_write.append(to_write)
|
||||
|
||||
try:
|
||||
# read data from JSON
|
||||
with open(file_name, 'r', encoding='utf-8') as f:
|
||||
json_data = json.load(f)
|
||||
except:
|
||||
json_data = []
|
||||
|
||||
json_data.extend(data_to_write)
|
||||
|
||||
# write data to JSON
|
||||
with open(file_name, 'w', encoding='utf-8') as f:
|
||||
json.dump(json_data, f, ensure_ascii=False)
|
||||
|
||||
def write_to_excel(file_name, data, types, record):
|
||||
first = False
|
||||
if os.path.exists(file_name):
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user