From 71c751a8cdd7f181ee63bc2593b26d92797b5ae7 Mon Sep 17 00:00:00 2001 From: naibo Date: Sat, 1 Jul 2023 04:48:06 +0800 Subject: [PATCH] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E5=8E=8B=E7=BC=A9=E7=9A=84?= =?UTF-8?q?=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .temp_to_pub/EasySpider_Linux_x64/readme.txt | 5 +- .../EasySpider_Linux_x64/软件使用说明.txt | 4 +- .../EasySpider_MacOS_all_arch/readme.txt | 4 +- .../软件使用说明.txt | 4 +- .../EasySpider_windows_x64/Readme.txt | 4 +- .../EasySpider_windows_x64/tasks/48.json | 1 + .../EasySpider_windows_x64/软件使用说明.txt | 4 +- .../EasySpider_windows_x86/readme.txt | 4 +- .../EasySpider_windows_x86/软件使用说明.txt | 4 +- .temp_to_pub/compress.py | 80 +++++++++++++++++++ ElectronJS/change_version.py | 3 + ElectronJS/update_chrome.py | 2 +- ElectronJS/发布一条龙脚本.md | 7 ++ ElectronJS/每次发布之前要检查的事项.md | 2 - ExecuteStage/easyspider_executestage.py | 4 +- 15 files changed, 118 insertions(+), 14 deletions(-) create mode 100644 .temp_to_pub/EasySpider_windows_x64/tasks/48.json create mode 100644 .temp_to_pub/compress.py create mode 100644 ElectronJS/发布一条龙脚本.md delete mode 100644 ElectronJS/每次发布之前要检查的事项.md diff --git a/.temp_to_pub/EasySpider_Linux_x64/readme.txt b/.temp_to_pub/EasySpider_Linux_x64/readme.txt index 41b1761..9949bf4 100644 --- a/.temp_to_pub/EasySpider_Linux_x64/readme.txt +++ b/.temp_to_pub/EasySpider_Linux_x64/readme.txt @@ -2,7 +2,7 @@ To open the EasySpider, please open your terminal, and then type: ./easy-spider.sh Then EasySpider will be opened, and don't close the terminal when running EasySpider. -Official Site: https://github.com/NaiboWang/EasySpider +Official Site: https://www.easyspider.net Welcome to promote this software to other friends. @@ -13,9 +13,10 @@ Video Tutorial: https://youtube.com/playlist?list=PL0kEFEkWrT7mt9MUlEBV2DTo1Qsaa Tasks can be imported from other machines by simply placing the .json files from the "tasks" folder of those machines into the "tasks" folder of this directory. Similarly, execution instance files can be imported by copying the .json files from the "execution_instances" folder. Note that only files named with a number greater than 0 are supported in both folders. - ======Version Update Instruction====== +Please see more new features for version greater than v0.3.2 at github release page: https://github.com/NaiboWang/EasySpider/releases + -----v0.3.2----- ## Update Instruction diff --git a/.temp_to_pub/EasySpider_Linux_x64/软件使用说明.txt b/.temp_to_pub/EasySpider_Linux_x64/软件使用说明.txt index 0d077c4..152bae7 100644 --- a/.temp_to_pub/EasySpider_Linux_x64/软件使用说明.txt +++ b/.temp_to_pub/EasySpider_Linux_x64/软件使用说明.txt @@ -4,7 +4,7 @@ ./easy-spider.sh 注意软件运行过程中不要关闭terminal。 -官方网址: https://github.com/NaiboWang/EasySpider +官方网址: https://www.easyspider.cn 支持Ubuntu 20.04, Debian, Deepin x64及以上版本。 @@ -15,6 +15,8 @@ ======版本更新说明====== +v0.3.2以上版本更新说明请查看Github Release Pages页面：https://github.com/NaiboWang/EasySpider/releases + -----v0.3.2----- ## 更新说明 diff --git a/.temp_to_pub/EasySpider_MacOS_all_arch/readme.txt b/.temp_to_pub/EasySpider_MacOS_all_arch/readme.txt index 567308e..5909012 100644 --- a/.temp_to_pub/EasySpider_MacOS_all_arch/readme.txt +++ b/.temp_to_pub/EasySpider_MacOS_all_arch/readme.txt @@ -1,4 +1,4 @@ -Official Site: https://github.com/NaiboWang/EasySpider +Official Site: https://www.easyspider.net Welcome to promote this software to other friends. @@ -12,6 +12,8 @@ You can import tasks from other machines by simply opening the EasySpider softwa ======Version Update Instruction====== +Please see more new features for version greater than v0.3.2 at github release page: https://github.com/NaiboWang/EasySpider/releases + -----v0.3.2----- ## Update Instruction diff --git a/.temp_to_pub/EasySpider_MacOS_all_arch/软件使用说明.txt b/.temp_to_pub/EasySpider_MacOS_all_arch/软件使用说明.txt index 13ea66e..535d71b 100644 --- a/.temp_to_pub/EasySpider_MacOS_all_arch/软件使用说明.txt +++ b/.temp_to_pub/EasySpider_MacOS_all_arch/软件使用说明.txt @@ -1,6 +1,6 @@ 欢迎将软件宣传给更多需要的朋友！ -官方网址: https://github.com/NaiboWang/EasySpider +官方网址: https://www.easyspider.cn 支持MacOS系统，包括Intel芯片和Arm芯片，如酷睿i7和M1芯片，最低MacOS系统版本为11.x。 @@ -13,6 +13,8 @@ ======版本更新说明====== +v0.3.2以上版本更新说明请查看Github Release Pages页面：https://github.com/NaiboWang/EasySpider/releases + -----v0.3.2----- ## 更新说明 diff --git a/.temp_to_pub/EasySpider_windows_x64/Readme.txt b/.temp_to_pub/EasySpider_windows_x64/Readme.txt index 2675796..a4395d7 100644 --- a/.temp_to_pub/EasySpider_windows_x64/Readme.txt +++ b/.temp_to_pub/EasySpider_windows_x64/Readme.txt @@ -1,4 +1,4 @@ -Official Site: https://github.com/NaiboWang/EasySpider +Official Site: https://www.easyspider.net Welcome to promote this software to other friends. @@ -15,6 +15,8 @@ Tasks can be imported from other machines by simply placing the .json files from ======Version Update Instructions====== +Please see more new features for version greater than v0.3.2 at github release page: https://github.com/NaiboWang/EasySpider/releases + -----v0.3.2----- ## Update Instruction diff --git a/.temp_to_pub/EasySpider_windows_x64/tasks/48.json b/.temp_to_pub/EasySpider_windows_x64/tasks/48.json new file mode 100644 index 0000000..a634436 --- /dev/null +++ b/.temp_to_pub/EasySpider_windows_x64/tasks/48.json @@ -0,0 +1 @@ +{"id":48,"name":"Discuz! X3.5 正式版【2022-12-21】 - Discuz! 官方发布 - Discuz!官方免费开源建站系统 - Powered by Discuz!","url":"https://discuz.dismall.com/thread-3857678-1-1.html","links":"https://discuz.dismall.com/thread-3857678-1-1.html","create_time":"7/1/2023, 4:30:15 AM","version":"0.3.3","containJudge":false,"desc":"https://discuz.dismall.com/thread-3857678-1-1.html","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://discuz.dismall.com/thread-3857678-1-1.html","desc":"要采集的网址列表，多行以\\n分开","type":"string","exampleValue":"https://discuz.dismall.com/thread-3857678-1-1.html"}],"outputParameters":[{"id":0,"name":"参数1_文本","desc":"","type":"string","exampleValue":"\n尊敬的各位站长/开发者：\n       在2022年12月底，很高兴为大家带来Discuz!X3.5的正式版，正式版提供从X3.5以下版本的升级。\n\nX3.5运行环境要求：\n       安全提示：我们强烈建议您使用仍在开发团队支持期内的操作系统、Web服务器、PHP、数据库、内存缓存等软件，超出支持期的软件可能会对您的站点带来未知的安全隐患。 性能提示：当 MySQL < 5.7 或 MariaDB < 10.2 时， InnoDB 性能下降较为严重，因此在生产系统上运行的站点应升级版本至 MySQL >= 5.7 或 MariaDB >= 10.2 以避免此问题。\n软件名称最低要求推荐版本其他事项PHP>= 5.6.07.3 - 8.2依赖 XML 扩展、 JSON 扩展、 GD 扩展 >= 1.0MySQL>= 5.5.35.7 - 8.0如使用 MariaDB ，推荐版本为 >= 10.2\nX3.5核心亮点：\n1、支持PHP8\n2、支持MYSQL8\n3、支持IPV6，支持多IP库、支持CDN不同模式IP获取，且支持扩展获取模式\n4、支持UTF8MB4（支持emoji表情、更多字符的支持）\n5、支持InnoDb（云数据库已普遍推荐使用）\n6、支持全模块开关（包括论坛，从此Discuz!X不只是论坛，而是全方案的建站系统）\n7、全新的支付系统，自带微信支付、支付宝支付、QQ钱包支付，支持扩展接入更多支付体系，支持第三方调用支付\n8、内置安全手机底层，支持第三方SMS接口接入\n9、更完善、现代化的手机版\n10、全新的后台风格\n11、更完善的HTTPS支持\n12、HTML5全面取代Flash\n\n另有数百项功能改进……\n\n下载地址：\nhttps://www.dismall.com/thread-14660-1-1.html\nhttps://gitee.com/Discuz/DiscuzX/attach_files\n\n\n\n\n\n\n"}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"https://discuz.dismall.com/thread-3857678-1-1.html","links":"https://discuz.dismall.com/thread-3857678-1-1.html","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1}},{"id":2,"index":2,"parentId":0,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"","wait":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"paras":[{"nodeType":0,"contentType":0,"relative":false,"name":"参数1_文本","desc":"","extractType":0,"relativeXPath":"/html/body/div[6]/div[4]/div[2]/div[1]/table[1]/tbody[1]/tr[1]/td[1]/div[2]/div[1]/div[1]/table[1]/tbody[1]/tr[1]/td[1]","allXPaths":["/html/body/div[6]/div[4]/div[2]/div[1]/table[1]/tbody[1]/tr[1]/td[1]/div[2]/div[1]/div[1]/table[1]/tbody[1]/tr[1]/td[1]","//td[contains(., '尊敬的各位站长/开')]","id(\"postmessage_29875312\")","//TD[@class='t_f']","/html/body/div[last()-5]/div[last()-1]/div[last()-3]/div[last()-1]/table/tbody/tr[last()-4]/td/div/div/div[last()-2]/table/tbody/tr/td"],"exampleValues":[{"num":0,"value":"\n尊敬的各位站长/开发者：\n       在2022年12月底，很高兴为大家带来Discuz!X3.5的正式版，正式版提供从X3.5以下版本的升级。\n\nX3.5运行环境要求：\n       安全提示：我们强烈建议您使用仍在开发团队支持期内的操作系统、Web服务器、PHP、数据库、内存缓存等软件，超出支持期的软件可能会对您的站点带来未知的安全隐患。 性能提示：当 MySQL < 5.7 或 MariaDB < 10.2 时， InnoDB 性能下降较为严重，因此在生产系统上运行的站点应升级版本至 MySQL >= 5.7 或 MariaDB >= 10.2 以避免此问题。\n软件名称最低要求推荐版本其他事项PHP>= 5.6.07.3 - 8.2依赖 XML 扩展、 JSON 扩展、 GD 扩展 >= 1.0MySQL>= 5.5.35.7 - 8.0如使用 MariaDB ，推荐版本为 >= 10.2\nX3.5核心亮点：\n1、支持PHP8\n2、支持MYSQL8\n3、支持IPV6，支持多IP库、支持CDN不同模式IP获取，且支持扩展获取模式\n4、支持UTF8MB4（支持emoji表情、更多字符的支持）\n5、支持InnoDb（云数据库已普遍推荐使用）\n6、支持全模块开关（包括论坛，从此Discuz!X不只是论坛，而是全方案的建站系统）\n7、全新的支付系统，自带微信支付、支付宝支付、QQ钱包支付，支持扩展接入更多支付体系，支持第三方调用支付\n8、内置安全手机底层，支持第三方SMS接口接入\n9、更完善、现代化的手机版\n10、全新的后台风格\n11、更完善的HTTPS支持\n12、HTML5全面取代Flash\n\n另有数百项功能改进……\n\n下载地址：\nhttps://www.dismall.com/thread-14660-1-1.html\nhttps://gitee.com/Discuz/DiscuzX/attach_files\n\n\n\n\n\n\n"}],"unique_index":"4d7ayvvi162ljj13jcl","default":"","beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}]}}]} \ No newline at end of file diff --git a/.temp_to_pub/EasySpider_windows_x64/软件使用说明.txt b/.temp_to_pub/EasySpider_windows_x64/软件使用说明.txt index 6f8dbc8..3c21b0c 100644 --- a/.temp_to_pub/EasySpider_windows_x64/软件使用说明.txt +++ b/.temp_to_pub/EasySpider_windows_x64/软件使用说明.txt @@ -1,6 +1,6 @@ 欢迎将软件宣传给更多需要的朋友！ -官方网址: https://github.com/NaiboWang/EasySpider +官方网址: https://www.easyspider.cn 支持Windows 10 x64及以上版本。 @@ -15,6 +15,8 @@ Windows 7此版本无直接可用版本（因为Chrome 109是最后一个支持W ======版本更新说明====== +v0.3.2以上版本更新说明请查看Github Release Pages页面：https://github.com/NaiboWang/EasySpider/releases + -----v0.3.2----- ## 更新说明 diff --git a/.temp_to_pub/EasySpider_windows_x86/readme.txt b/.temp_to_pub/EasySpider_windows_x86/readme.txt index c02d027..e28a462 100644 --- a/.temp_to_pub/EasySpider_windows_x86/readme.txt +++ b/.temp_to_pub/EasySpider_windows_x86/readme.txt @@ -1,4 +1,4 @@ -Official Site: https://github.com/NaiboWang/EasySpider +Official Site: https://www.easyspider.net Welcome to promote this software to other friends. @@ -15,6 +15,8 @@ Tasks can be imported from other machines by simply placing the .json files from ======Version New Features====== +Please see more new features for version greater than v0.3.2 at github release page: https://github.com/NaiboWang/EasySpider/releases + -----v0.3.2----- ## Update Instruction diff --git a/.temp_to_pub/EasySpider_windows_x86/软件使用说明.txt b/.temp_to_pub/EasySpider_windows_x86/软件使用说明.txt index ffcaf37..bce0841 100644 --- a/.temp_to_pub/EasySpider_windows_x86/软件使用说明.txt +++ b/.temp_to_pub/EasySpider_windows_x86/软件使用说明.txt @@ -1,6 +1,6 @@ 欢迎将软件宣传给更多需要的朋友！ -官方网址: https://github.com/NaiboWang/EasySpider +官方网址: https://www.easyspider.cn 支持Windows 10 x32及以上版本。 @@ -15,6 +15,8 @@ Windows 7此版本无直接可用版本（因为Chrome 109是最后一个支持W ======版本更新说明====== +v0.3.2以上版本更新说明请查看Github Release Pages页面：https://github.com/NaiboWang/EasySpider/releases + -----v0.3.2----- ## 更新说明 diff --git a/.temp_to_pub/compress.py b/.temp_to_pub/compress.py new file mode 100644 index 0000000..e8cf303 --- /dev/null +++ b/.temp_to_pub/compress.py @@ -0,0 +1,80 @@ +import glob +import json +import os +import re +import subprocess +import sys +import requests +import platform +import shutil +import zipfile +import urllib.request +import py7zr + +def compress_folder_to_7z(folder_path, output_file): + if os.path.exists(output_file): + os.remove(output_file) + with py7zr.SevenZipFile(output_file, 'w') as archive: + archive.writeall(folder_path, output_file) + +def compress_folder_to_7z_split(folder_path, output_file): + if os.path.exists(output_file): + os.remove(output_file) + file_name = os.path.basename(output_file) + file_dir = os.path.dirname(output_file) + + # 获取文件名的前缀 + file_prefix = os.path.splitext(file_name)[0] + + # 构建分卷文件的路径模式 + split_file_pattern = os.path.join(file_dir, file_prefix + ".7z.*") + + # 获取匹配的分卷文件列表 + split_files = glob.glob(split_file_pattern) + + # 删除分卷文件 + for split_file in split_files: + os.remove(split_file) + + # 压缩文件夹 + subprocess.call(["7z", "a", "-v95m", output_file, folder_path]) + +easyspider_version = "0.3.3" + +if __name__ == "__main__": + + if sys.platform == "win32" and platform.architecture()[0] == "64bit": + file_name = f"EasySpider_{easyspider_version}_windows_x64.7z" + if os.path.exists("./EasySpider_windows_x64/user_data"): + shutil.rmtree("./EasySpider_windows_x64/user_data") + shutil.rmtree("./EasySpider_windows_x64/Data") + shutil.rmtree("./EasySpider_windows_x64/execution_instances") + os.mkdir("./EasySpider_windows_x64/Data") + os.mkdir("./EasySpider_windows_x64/execution_instances") + compress_folder_to_7z_split("./EasySpider_windows_x64", file_name) + print(f"Compress {file_name} Split successfully!") + compress_folder_to_7z("./EasySpider_windows_x64", file_name) + print(f"Compress {file_name} successfully!") + elif sys.platform == "win32" and platform.architecture()[0] == "32bit": + file_name = f"EasySpider_{easyspider_version}_windows_x86.7z" + if os.path.exists("./EasySpider_windows_x86/user_data"): + shutil.rmtree("./EasySpider_windows_x86/user_data") + shutil.rmtree("./EasySpider_windows_x86/Data") + shutil.rmtree("./EasySpider_windows_x86/execution_instances") + os.mkdir("./EasySpider_windows_x86/Data") + os.mkdir("./EasySpider_windows_x86/execution_instances") + compress_folder_to_7z("./EasySpider_windows_x64", file_name) + print(f"Compress {file_name} successfully!") + elif sys.platform == "linux" and platform.architecture()[0] == "64bit": + file_name = f"EasySpider_{easyspider_version}_Linux_x64.7z" + if os.path.exists("./EasySpider_Linux_x64/user_data"): + shutil.rmtree("./EasySpider_Linux_x64/user_data") + shutil.rmtree("./EasySpider_Linux_x64/Data") + shutil.rmtree("./EasySpider_Linux_x64/execution_instances") + os.mkdir("./EasySpider_Linux_x64/Data") + os.mkdir("./EasySpider_Linux_x64/execution_instances") + # compress_folder_to_7z("./EasySpider_Linux_x64", file_name) + print(f"Compress {file_name} successfully!") + elif sys.platform == "darwin" and platform.architecture()[0] == "64bit": + pass + diff --git a/ElectronJS/change_version.py b/ElectronJS/change_version.py index 48b6afb..7d995b7 100644 --- a/ElectronJS/change_version.py +++ b/ElectronJS/change_version.py @@ -30,6 +30,9 @@ version = "0.3.3" if __name__ == "__main__": + file_path = "../.temp_to_pub/compress.py" + update_file_version(file_path, version, key='easyspider_version = "') + file_path = "./src/taskGrid/logic.js" update_file_version(file_path, version, key='"version": "') diff --git a/ElectronJS/update_chrome.py b/ElectronJS/update_chrome.py index 6b73eef..11541d3 100644 --- a/ElectronJS/update_chrome.py +++ b/ElectronJS/update_chrome.py @@ -93,4 +93,4 @@ if __name__ == "__main__": elif sys.platform == "darwin" and platform.architecture()[0] == "64bit": pass - print("Done!") \ No newline at end of file + print("Done and don't forget to generate executestage EXEcutable program!") \ No newline at end of file diff --git a/ElectronJS/发布一条龙脚本.md b/ElectronJS/发布一条龙脚本.md new file mode 100644 index 0000000..80d8624 --- /dev/null +++ b/ElectronJS/发布一条龙脚本.md @@ -0,0 +1,7 @@ +1. python change_version.py 更改版本号。 +2. python generate_chrome.py 更新最新的chrome及chromedriver。 +3. 在ExecuteStage目录下执行generateEXE生成运行阶段可执行文件。 +4. 在当前目录下执行package_操作系统版本打包。 +5. 对windows需要额外执行clean_and_release*.cmd打包。 +6. 在.temp_to_pub目录下执行python compress.py压缩成最终发布包。 + diff --git a/ElectronJS/每次发布之前要检查的事项.md b/ElectronJS/每次发布之前要检查的事项.md deleted file mode 100644 index ffc2592..0000000 --- a/ElectronJS/每次发布之前要检查的事项.md +++ /dev/null @@ -1,2 +0,0 @@ -- python generate_chrome.py 更新最新的chrome及chromedriver -- python change_version.py 更改版本号 diff --git a/ExecuteStage/easyspider_executestage.py b/ExecuteStage/easyspider_executestage.py index 5de57c8..2bd0016 100644 --- a/ExecuteStage/easyspider_executestage.py +++ b/ExecuteStage/easyspider_executestage.py @@ -263,7 +263,7 @@ class BrowserThread(Thread): elif int(codeMode) == 1: self.recordLog("Execute System Call:" + code) self.recordLog("执行系统命令:" + code) - # 执行系统命令，超时时间为5秒 + # 执行系统命令 try: # output = subprocess.run(code, capture_output=True, text=True, timeout=max_wait_time, encoding="utf-8", shell=True) output = subprocess.run(code, capture_output=True, text=True, timeout=max_wait_time, shell=True) @@ -271,7 +271,7 @@ class BrowserThread(Thread): output = output.stdout print(output) except subprocess.TimeoutExpired: - # 命令执行时间超过5秒，抛出异常 + # 命令执行时间超过指定值，抛出异常 self.recordLog("Command timed out") self.recordLog("命令执行超时") except Exception as e: