From 128320651835cdb169ac7a07ec4e4232418c48b7 Mon Sep 17 00:00:00 2001 From: naibo Date: Sat, 15 Jul 2023 07:42:22 +0800 Subject: [PATCH] Change OCR to ddddocr --- ElectronJS/tasks/181.json | 1 + ExecuteStage/.vscode/launch.json | 2 +- ExecuteStage/easyspider_executestage.py | 82 +++++++++++++++---------- ExecuteStage/generateEXE_win64.cmd | 3 +- ExecuteStage/requirements.txt | 1 + 5 files changed, 53 insertions(+), 36 deletions(-) create mode 100644 ElectronJS/tasks/181.json diff --git a/ElectronJS/tasks/181.json b/ElectronJS/tasks/181.json new file mode 100644 index 0000000..67f9018 --- /dev/null +++ b/ElectronJS/tasks/181.json @@ -0,0 +1 @@ +{"id":181,"name":"OCR","url":"https://www.jd.com","links":"https://www.jd.com","create_time":"7/15/2023, 6:00:09 AM","update_time":"7/15/2023, 6:01:09 AM","version":"0.3.6","saveThreshold":10,"cloudflare":0,"environment":0,"maxViewLength":15,"outputFormat":"xlsx","saveName":"current_time","inputExcel":"","startFromExit":0,"containJudge":false,"desc":"https://www.jd.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.jd.com","desc":"要采集的网址列表，多行以\\n分开","type":"text","exampleValue":"https://www.jd.com"}],"outputParameters":[{"id":0,"name":"参数1_文本","desc":"","type":"text","recordASField":1,"exampleValue":"/手机/数码"},{"id":1,"name":"参数3_图片地址","desc":"","recordASField":1,"exampleValue":"//m.360buyimg.com/babel/jfs/t1/160456/7/37206/196421/649c09faFeab01f59/cc5ea5b81653b3a5.png"}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2,4],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0,"waitType":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"https://www.jd.com","links":"https://www.jd.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":2,"index":2,"parentId":0,"type":1,"option":8,"title":"循环","sequence":[3],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":1,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":["/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]","//div[contains(., '/手机/数码')]","//DIV[@class='LeftSide_menu_item__SBMWC LeftSide_text_space__2UhbG ']","/html/body/div[last()-6]/div/div[last()-4]/div/div[last()-2]/div/div/div/div[last()-1]/div[last()-12]"]}},{"id":4,"index":3,"parentId":2,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"clear":0,"paras":[{"nodeType":0,"contentType":8,"relative":true,"name":"参数1_文本","desc":"","extractType":0,"relativeXPath":"","allXPaths":"","exampleValues":[{"num":0,"value":"/手机/数码"}],"unique_index":"nih1pc92uudlk34f3og","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}],"loopType":1}},{"id":3,"index":4,"parentId":0,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":false,"position":2,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"clear":0,"paras":[{"nodeType":4,"contentType":8,"relative":false,"name":"参数3_图片地址","desc":"","extractType":0,"relativeXPath":"/html/body/div[5]/div[1]/div[1]/div[1]/div[2]/div[1]/div[1]/div[1]/div[2]/div[1]/div[2]/div[1]/div[1]/a[1]/img[1]","allXPaths":["/html/body/div[5]/div[1]/div[1]/div[1]/div[2]/div[1]/div[1]/div[1]/div[2]/div[1]/div[2]/div[1]/div[1]/a[1]/img[1]","//img[contains(., '')]","/html/body/div[last()-5]/div/div[last()-4]/div/div[last()-1]/div/div[last()-1]/div/div[last()-1]/div/div[last()-7]/div/div/a/img"],"exampleValues":[{"num":0,"value":"//m.360buyimg.com/babel/jfs/t1/160456/7/37206/196421/649c09faFeab01f59/cc5ea5b81653b3a5.png"}],"unique_index":"hr2mndh1ty6lk34fgeu","iframe":false,"default":"","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}]}}]} \ No newline at end of file diff --git a/ExecuteStage/.vscode/launch.json b/ExecuteStage/.vscode/launch.json index 00f4832..32c7152 100644 --- a/ExecuteStage/.vscode/launch.json +++ b/ExecuteStage/.vscode/launch.json @@ -12,7 +12,7 @@ "justMyCode": false, // "args": ["--id", "[7]", "--read_type", "remote", "--headless", "0"] // "args": ["--id", "[9]", "--read_type", "remote", "--headless", "0", "--saved_file_name", "YOUTUBE"] - "args": ["--id", "[79]", "--headless", "0", "--user_data", "1"] + "args": ["--id", "[81]", "--headless", "0", "--user_data", "1", "--keyboard", "0"] } ] } \ No newline at end of file diff --git a/ExecuteStage/easyspider_executestage.py b/ExecuteStage/easyspider_executestage.py index 365fa04..0158361 100644 --- a/ExecuteStage/easyspider_executestage.py +++ b/ExecuteStage/easyspider_executestage.py @@ -13,8 +13,11 @@ import sys # import hashlib import time import requests +from ddddocr import DdddOcr from urllib.parse import urljoin from lxml import etree +import onnxruntime +onnxruntime.set_default_logger_severity(3) # 隐藏onnxruntime的日志 # import undetected_chromedriver as uc from pynput.keyboard import Key, Listener from selenium.webdriver.chrome.options import Options @@ -38,7 +41,7 @@ from openpyxl import load_workbook, Workbook import csv import os from commandline_config import Config -import pytesseract +# import pytesseract from PIL import Image # import uuid from threading import Thread, Event @@ -236,8 +239,8 @@ class BrowserThread(Thread): except: node["parameters"]["recordASField"] += 1 if para["contentType"] == 8: - print("默认的OCR识别功能如果觉得不好用，可以自行修改源码get_content函数->contentType == 8的位置换成自己想要的OCR模型然后自己编译运行；或者可以先设置采集内容类型为“元素截图”把图片保存下来，然后用自定义操作调用自己写的程序，程序的功能是读取这个最新生成的图片，然后用好用的模型，如PaddleOCR把图片识别出来，然后把返回值返回给程序作为参数输出。") - print("If you think the default OCR function is not good enough, you can modify the source code get_content function -> contentType == 8 position to your own OCR model and then compile and run it; or you can first set the content type of the crawler to \"Element Screenshot\" to save the picture, and then call your own program with custom operations. The function of the program is to read the latest generated picture, then use a good model, such as PaddleOCR to recognize the picture, and then return the return value as a parameter output to the program.") + print("默认的ddddocr识别功能如果觉得不好用，可以自行修改源码get_content函数->contentType == 8的位置换成自己想要的OCR模型然后自己编译运行；或者可以先设置采集内容类型为“元素截图”把图片保存下来，然后用自定义操作调用自己写的程序，程序的功能是读取这个最新生成的图片，然后用好用的模型，如PaddleOCR把图片识别出来，然后把返回值返回给程序作为参数输出。") + print("If you think the default ddddocr function is not good enough, you can modify the source code get_content function -> contentType == 8 position to your own OCR model and then compile and run it; or you can first set the content type of the crawler to \"Element Screenshot\" to save the picture, and then call your own program with custom operations. The function of the program is to read the latest generated picture, then use a good model, such as PaddleOCR to recognize the picture, and then return the return value as a parameter output to the program.") if para["beforeJS"] == "" and para["afterJS"] == "" and para["contentType"] <= 1 and para["nodeType"] <= 2: para["optimizable"] = True else: @@ -1367,34 +1370,43 @@ class BrowserThread(Thread): screenshot_stream = io.BytesIO(screenshot) # 使用Pillow库打开截图，并转换为灰度图像 image = Image.open(screenshot_stream).convert('L') + temp_name = "OCR_" + str(time.time()) + ".png" + location = "Data/Task_" + str(self.id) + "/" + self.saveName + "/" + temp_name + image.save(location) + ocr = DdddOcr() + with open(location, 'rb') as f: + image_bytes = f.read() + content = ocr.classification(image_bytes) + os.remove(location) # 使用Tesseract OCR引擎识别图像中的文本 - content = pytesseract.image_to_string(image, lang='chi_sim+eng') + # content = pytesseract.image_to_string(image, lang='chi_sim+eng') except Exception as e: - try: - print("识别中文失败，尝试只识别英文") - print("Failed to recognize Chinese, try to recognize English only") - screenshot = element.screenshot_as_png - screenshot_stream = io.BytesIO(screenshot) - # 使用Pillow库打开截图，并转换为灰度图像 - image = Image.open(screenshot_stream).convert('L') - # 使用Tesseract OCR引擎识别图像中的文本 - content = pytesseract.image_to_string(image, lang='eng') - except Exception as e: - content = "OCR Error" - print(e) - if sys.platform == "win32": - print("要使用OCR识别功能，你需要安装Tesseract-OCR并将其添加到环境变量PATH中（添加后需重启EasySpider）：https://blog.csdn.net/u010454030/article/details/80515501") - print("\nhttps://www.bilibili.com/video/BV1GP411y7u4/") - elif sys.platform == "darwin": - print( - "注意以上错误，要使用OCR识别功能，你需要安装Tesseract-OCR并将其添加到环境变量PATH中（添加后需重启EasySpider）：https://zhuanlan.zhihu.com/p/146044810") - elif sys.platform == "linux": - print( - "注意以上错误，要使用OCR识别功能，你需要安装Tesseract-OCR并将其添加到环境变量PATH中（添加后需重启EasySpider）：https://zhuanlan.zhihu.com/p/420259031") - else: - print("注意以上错误，要使用OCR识别功能，你需要安装Tesseract-OCR并将其添加到环境变量PATH中（添加后需重启EasySpider）：https://blog.csdn.net/u010454030/article/details/80515501") - print("\nhttps://www.bilibili.com/video/BV1GP411y7u4/") - print("To use OCR, You need to install Tesseract-OCR and add it to the environment variable PATH (need to restart EasySpider after you put in PATH): https://tesseract-ocr.github.io/tessdoc/Installation.html") + # try: + # print(e) + # print("识别中文失败，尝试只识别英文") + # print("Failed to recognize Chinese, try to recognize English only") + # screenshot = element.screenshot_as_png + # screenshot_stream = io.BytesIO(screenshot) + # # 使用Pillow库打开截图，并转换为灰度图像 + # image = Image.open(screenshot_stream).convert('L') + # # 使用Tesseract OCR引擎识别图像中的文本 + # # content = pytesseract.image_to_string(image, lang='eng') + # except Exception as e: + content = "OCR Error" + print(e) + # if sys.platform == "win32": + # print("要使用OCR识别功能，你需要安装Tesseract-OCR并将其添加到环境变量PATH中（添加后需重启EasySpider）：https://blog.csdn.net/u010454030/article/details/80515501") + # print("\nhttps://www.bilibili.com/video/BV1GP411y7u4/") + # elif sys.platform == "darwin": + # print( + # "注意以上错误，要使用OCR识别功能，你需要安装Tesseract-OCR并将其添加到环境变量PATH中（添加后需重启EasySpider）：https://zhuanlan.zhihu.com/p/146044810") + # elif sys.platform == "linux": + # print( + # "注意以上错误，要使用OCR识别功能，你需要安装Tesseract-OCR并将其添加到环境变量PATH中（添加后需重启EasySpider）：https://zhuanlan.zhihu.com/p/420259031") + # else: + # print("注意以上错误，要使用OCR识别功能，你需要安装Tesseract-OCR并将其添加到环境变量PATH中（添加后需重启EasySpider）：https://blog.csdn.net/u010454030/article/details/80515501") + # print("\nhttps://www.bilibili.com/video/BV1GP411y7u4/") + # print("To use OCR, You need to install Tesseract-OCR and add it to the environment variable PATH (need to restart EasySpider after you put in PATH): https://tesseract-ocr.github.io/tessdoc/Installation.html") elif p["contentType"] == 9: content = self.execute_code( 2, p["JS"], p["JSWaitTime"], element, iframe=p["iframe"]) @@ -1642,6 +1654,7 @@ if __name__ == '__main__': "read_type": "remote", "headless": False, "server_address": "http://localhost:8074", + "keyboard": True, # 是否监听键盘输入 "version": "0.3.6", } c = Config(config) @@ -1849,13 +1862,14 @@ if __name__ == '__main__': # print("过Cloudflare验证模式有时候会不稳定，如果无法通过验证则需要隔几分钟重试一次，或者可以更换新的用户信息文件夹再执行任务。") # print("Passing the Cloudflare verification mode is sometimes unstable. If the verification fails, you need to try again every few minutes, or you can change to a new user information folder and then execute the task.") # 使用监听器监听键盘输入 - try: + try: + if c.keyboard: with Listener(on_press=on_press_creator(press_time, event), on_release=on_release_creator(event, press_time)) as listener: listener.join() - except: - pass - # print("您的操作系统不支持暂停功能。") - # print("Your operating system does not support the pause function.") + except: + pass + # print("您的操作系统不支持暂停功能。") + # print("Your operating system does not support the pause function.") # print("线程长度：", len(threads) ) diff --git a/ExecuteStage/generateEXE_win64.cmd b/ExecuteStage/generateEXE_win64.cmd index e94cac9..f04376f 100644 --- a/ExecuteStage/generateEXE_win64.cmd +++ b/ExecuteStage/generateEXE_win64.cmd @@ -1,5 +1,6 @@ rmdir /s /q build rmdir /s /q dist -pyinstaller -F --icon=favicon.ico easyspider_executestage.py +@REM pyinstaller -F --icon=favicon.ico easyspider_executestage.py +pyinstaller -F --icon=favicon.ico --add-data "C:\Python311\Lib\site-packages\onnxruntime\capi\onnxruntime_providers_shared.dll;onnxruntime\capi" --add-data "C:\Python311\Lib\site-packages\ddddocr\common.onnx;ddddocr" easyspider_executestage.py del ..\ElectronJS\chrome_win64\easyspider_executestage.exe copy dist\easyspider_executestage.exe ..\ElectronJS\chrome_win64\easyspider_executestage.exe \ No newline at end of file diff --git a/ExecuteStage/requirements.txt b/ExecuteStage/requirements.txt index cf8010e..2eb20b8 100644 --- a/ExecuteStage/requirements.txt +++ b/ExecuteStage/requirements.txt @@ -7,5 +7,6 @@ pytesseract==0.3.10 openpyxl==3.1.2 pymysql==1.1.0 lxml==4.9.2 +ddddocr==1.0.6 pynput==1.7.6 undetected-chromedriver==3.4.7 \ No newline at end of file