commit 0fffebef4e4dfb64fd0613968fe544f4eac9da5c Author: rnet Date: Sun Mar 10 14:29:00 2024 +0800 fore: 可用版本v0.1.0固定与提交 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c14af9c --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +.DS_Store +node_modules/ +output/ +npm-debug.log +yarn-error.log +.idea +.vscode +package-lock.json +yarn.lock +.history +*.swp +build/ diff --git a/.npmignore b/.npmignore new file mode 100644 index 0000000..5da7fce --- /dev/null +++ b/.npmignore @@ -0,0 +1,4 @@ +/* +!bin/ +!browser/ +!utils/ diff --git a/.release-it.js b/.release-it.js new file mode 100644 index 0000000..fa81b75 --- /dev/null +++ b/.release-it.js @@ -0,0 +1,30 @@ +module.exports = { + github: { + release: true + }, + git: { + commitMessage: "release: v${version}" + }, + npm: { + publish: false + }, + hooks: { + "after:bump": "echo 更新版本成功" + }, + plugins: { + '@release-it/conventional-changelog': { + preset: 'conventionalcommits', + infile: 'CHANGELOG.md', + sameFile: true, + releaseRules: [ + { type: 'feat', release: 'minor' }, + { type: 'fix', release: 'patch' }, + { type: 'docs', release: 'patch' }, + { type: 'style', release: 'patch' }, + { type: 'refactor', release: 'patch' }, + { type: 'perf', release: 'patch' }, + { type: 'test', release: 'patch' }, + ], + }, + }, +}; diff --git a/README.md b/README.md new file mode 100644 index 0000000..8c5f78c --- /dev/null +++ b/README.md @@ -0,0 +1,120 @@ +

+
+ sdenv +

+ +sdenv是一个javascript运行时补环境框架，与github上其它补环境框架存在较大区别，sdenv是站在巨人的肩膀上实现的，依赖于jsdom的强大dom仿真能力，sdenv可以真实模拟浏览器执行环境，作者在固定随机数与添加[sdenv-extend](https://github.com/pysunday/sdenv-extend)的部分插件后可以达到**瑞数vmp代码在sdenv运行生成的cookie值与浏览器生成的cookie值一致**。 + +* sdenv专用jsdom版本：[sdenv-jsdom](https://github.com/pysunday/sdenv-jsdom) +* sdenv多端环境提取：[sdenv-extend](https://github.com/pysunday/sdenv-extend) + +## 依赖 + +作者开发时使用的是`v20.10.0`版本node，预期最低要求是18版本，由于未做其它版本可用性测试，因此建议使用sdenv的node版本大于等于`v20.10.0` + +## 安装 + +由于`document.all`需要由c代码动态生成，而固定编译环境下的编译产物只能在相同编译环境下运行，因此安装sdenv后需要动态编译生成node文件 + +1. 安装：`npm i sdenv` +2. 编译c代码：`cd node_modules/sdenv && yarn build` + +**在编译过程未实现自动化之前可直接clone项目使用** + +## 使用 + +因为项目核心功能基于jsdom，且jsdom对dom的实现非常完善，因此使用sdenv之前建议有一定html与javascript语言开发基础，然后参考example目录下的样例文件: + +1. [use-local](https://github.com/pysunday/sdenv/example/use-local/README.md) + ```javascript + const fs = require('fs'); + const path = require('path'); + const { Script } = require("vm"); + const logger = require('../../utils/logger'); + const browser = require('../../browser/'); + const { jsdomFromText } = require('../../utils/jsdom'); + + const baseUrl = "https://wcjs.sbj.cnipa.gov.cn" + + const files = { + html: path.resolve(__dirname, 'output/makecode_input_html.html'), + js: path.resolve(__dirname, 'output/makecode_input_js.js'), + ts: path.resolve(__dirname, 'output/makecode_input_ts.json'), + } + + function getFile(name) { + const filepath = files[name]; + if (!filepath) throw new Error(`getFile: ${name}错误`); + if (!fs.existsSync(filepath)) throw new Error(`文件${filepath}不存在，请使用rs-reverse工具先获取文件`); + return fs.readFileSync(filepath); + } + + function initBrowser(window, cookieJar) { + window.$_ts = JSON.parse(getFile('ts')); + window.onbeforeunload = async (url) => { + const cookies = cookieJar.getCookieStringSync(baseUrl); + logger.debug('生成cookie：', cookies); + process.exit(); + } + browser(window, 'chrome'); + } + + async function loadPages() { + const htmltext = getFile('html'); + const jstext = getFile('js'); + const [jsdomer, cookieJar] = jsdomFromText({ + url: `${baseUrl}/sgtmi`, + referrer: `${baseUrl}/sgtmi`, + contentType: "text/html", + runScripts: "outside-only", + }) + const dom = jsdomer(htmltext); + initBrowser(dom.window, cookieJar); + new Script(jstext).runInContext(dom.getInternalVMContext()); + } + + loadPages() + ``` +2. [use-remote](https://github.com/pysunday/sdenv/example/use-remote/README.md) + ```javascript + process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0" + const logger = require('../../utils/logger'); + const browser = require('../../browser/'); + const { jsdomFromUrl } = require('../../utils/jsdom'); + + const baseUrl = "https://wcjs.sbj.cnipa.gov.cn" + + async function loadPages() { + const [jsdomer, cookieJar] = jsdomFromUrl({ + userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', + }); + const dom = await jsdomer(`${baseUrl}/sgtmi`); + window = dom.window + window.onbeforeunload = async (url) => { + const cookies = cookieJar.getCookieStringSync(baseUrl); + logger.debug('生成cookie：', cookies); + process.exit(); + } + browser(window, 'chrome'); + } + + loadPages() + ``` + +## sdenv-extend使用说明 + +为了模拟浏览器执行环境，需要将node环境与浏览器环境共有代码进行提取，并提供返回环境对象用于sdenv内window与dom内容补充使用。 + +sdenv-extend初始化只执行一次，初始化成功后生成的环境对象可以使用`Object.sdenv()`(vm中使用非node)获取。 + +sdenv-extend具体功能可参考项目内README文档。 + +## 声明 + +该项目的开发基于瑞数vmp网站，不能保证在其它反爬虫产品稳定使用，出现问题请及时提issues或者提pull参与共建! + +由于初期版本只做了chrome浏览器的拟真，且项目文档不完善，作者会陆续补充，可以加入技术交流群与订阅号持续关注！ + +添加作者微信进技术交流群：howduudu_tech(备注sdenv) + +订阅号会定期发表技术文章：码功 diff --git a/bin/documentAll.cc b/bin/documentAll.cc new file mode 100644 index 0000000..1953f37 --- /dev/null +++ b/bin/documentAll.cc @@ -0,0 +1,51 @@ +#include +namespace documentAll { + +using v8::Context; +using v8::Function; +using v8::FunctionCallbackInfo; +using v8::FunctionTemplate; +using v8::Isolate; +using v8::Local; +using v8::Number; +using v8::Object; +using v8::ObjectTemplate; +using v8::String; +using v8::Value; +using v8::Null; +using v8::Array; + +void MyFunctionCallback(const FunctionCallbackInfo& args) { + Isolate* isolate = args.GetIsolate(); + args.GetReturnValue().Set(Null(isolate)); +} + +void GetDocumentAll(const FunctionCallbackInfo& args) { + Isolate* isolate = args.GetIsolate(); + Local context = isolate->GetCurrentContext(); + Local obj_template = ObjectTemplate::New(isolate); + obj_template->MarkAsUndetectable(); + obj_template->SetCallAsFunctionHandler(MyFunctionCallback); + Local obj = obj_template->NewInstance(context).ToLocalChecked(); + if (args.Length() > 0 && args[0]->IsObject()) { + Local argObj = args[0]->ToObject(context).ToLocalChecked(); + Local propertyNames = argObj->GetPropertyNames(context).ToLocalChecked(); + for (uint32_t i = 0; i < propertyNames->Length(); ++i) { + Local key = propertyNames->Get(context, i).ToLocalChecked(); + Local value = argObj->Get(context, key).ToLocalChecked(); + (void)obj->Set(context, key, value); + } + } + args.GetReturnValue().Set(obj); +} + +void Init(Local exports, Local module) { + Isolate* isolate = exports->GetIsolate(); + Local context = isolate->GetCurrentContext(); + Local method_template = FunctionTemplate::New(isolate, GetDocumentAll); + exports->Set(context, String::NewFromUtf8(isolate, "getDocumentAll").ToLocalChecked(), method_template->GetFunction(context).ToLocalChecked()).FromJust(); +} + +NODE_MODULE(NODE_GYP_MODULE_NAME, Init) +} + diff --git a/bin/documentAll.node b/bin/documentAll.node new file mode 100755 index 0000000..a55d02d Binary files /dev/null and b/bin/documentAll.node differ diff --git a/binding.gyp b/binding.gyp new file mode 100644 index 0000000..69465b9 --- /dev/null +++ b/binding.gyp @@ -0,0 +1,11 @@ +{ + "targets": [ + { + "target_name": "documentAll", + "sources": [ + "bin/documentAll.cc", + ] + } + ] +} + diff --git a/browser/chrome/RTCPeerConnection.js b/browser/chrome/RTCPeerConnection.js new file mode 100644 index 0000000..efe3d58 --- /dev/null +++ b/browser/chrome/RTCPeerConnection.js @@ -0,0 +1,19 @@ +const sdenv = require('sdenv-extend').sdenv(); +const window = sdenv.memory.sdWindow; + +function RTCPeerConnection() { + if (!(this instanceof RTCPeerConnection)) { + throw new TypeError("Uncaught TypeError: Failed to construct 'RTCPeerConnection': Please use the 'new' operator, this DOM object constructor cannot be called as a function."); + } + this.createDataChannel = function(...params) { + // window.console.log(`【RTCPeerConnection RTCPeerConnection】调用，参数：${params}`) + } + this.createOffer = function(...params) { + // window.console.log(`【RTCPeerConnection createOffer】调用，参数：${params}`) + } +} +sdenv.tools.setNativeFuncName(RTCPeerConnection, 'RTCPeerConnection'); +sdenv.tools.setNativeObjName(RTCPeerConnection.prototype, 'RTCPeerConnection'); + + +window.RTCPeerConnection = RTCPeerConnection; diff --git a/browser/chrome/chrome.js b/browser/chrome/chrome.js new file mode 100644 index 0000000..95ec3a6 --- /dev/null +++ b/browser/chrome/chrome.js @@ -0,0 +1,39 @@ +const sdenv = require('sdenv-extend').sdenv(); + +sdenv.memory.sdWindow.chrome = { + app: { + isInstalled: false, + InstallState: { + DISABLED: "disabled", + INSTALLED: "installed", + NOT_INSTALLED: "not_installed", + }, + RunningState: { + CANNOT_RUN: "cannot_run", + READY_TO_RUN: "ready_to_run", + RUNNING: "running", + }, + getDetails: function () {}, + getIsInstalled: function() {}, + installState: function() {}, + runningState: function() {}, + }, + csi: function() {}, + loadTimes: function() { + return { + "requestTime": 1700779741.985, + "startLoadTime": 1700779741.985, + "commitLoadTime": 1700779742.021, + "finishDocumentLoadTime": 0, + "finishLoadTime": 0, + "firstPaintTime": 0, + "firstPaintAfterLoadTime": 0, + "navigationType": "Reload", + "wasFetchedViaSpdy": false, + "wasNpnNegotiated": true, + "npnNegotiatedProtocol": "http/1.1", + "wasAlternateProtocolAvailable": false, + "connectionInfo": "http/1.1" + } + } +} diff --git a/browser/chrome/ctorRegistry.js b/browser/chrome/ctorRegistry.js new file mode 100644 index 0000000..3da437a --- /dev/null +++ b/browser/chrome/ctorRegistry.js @@ -0,0 +1,12 @@ +const logger = require('@utils/logger'); +const utils = require('sdenv-jsdom/lib/jsdom/living/generated/utils.js'); +const sdenv = require('sdenv-extend').sdenv(); +const window = sdenv.memory.sdWindow; + +const ctorRegistry = window[utils.ctorRegistrySymbol] +window[utils.ctorRegistrySymbol] = new window.Proxy(ctorRegistry, { + get(target, propKey, receiver) { + logger.trace('proxy ctorRegistry get', propKey); + return window.Reflect.get(target, propKey, receiver); + } +}) diff --git a/browser/chrome/document.js b/browser/chrome/document.js new file mode 100644 index 0000000..35d56e9 --- /dev/null +++ b/browser/chrome/document.js @@ -0,0 +1,5 @@ +const getDocumentAll = require('@bin/documentAll').getDocumentAll; +const sdenv = require('sdenv-extend').sdenv(); +const window = sdenv.memory.sdWindow; + +window.document.all = getDocumentAll({ length: 3 }); diff --git a/browser/chrome/index.js b/browser/chrome/index.js new file mode 100644 index 0000000..5fa491f --- /dev/null +++ b/browser/chrome/index.js @@ -0,0 +1,12 @@ +require('./window'); +require('./document'); +require('./navigation'); +require('./navigator'); +require('./chrome'); +require('./visualViewport'); +require('./styleMedia'); +// require('./webkitRequestFileSystem'); +require('./ctorRegistry'); +require('./location'); +require('./indexedDB'); +require('./RTCPeerConnection'); diff --git a/browser/chrome/indexedDB.js b/browser/chrome/indexedDB.js new file mode 100644 index 0000000..81c3630 --- /dev/null +++ b/browser/chrome/indexedDB.js @@ -0,0 +1,16 @@ +const sdenv = require('sdenv-extend').sdenv(); +const window = sdenv.memory.sdWindow; + +const IDBFactory = function IDBFactory() { + throw new TypeError("Illegal constructor"); +} + +const indexedDB = { + __proto__: IDBFactory.prototype +}; + +sdenv.tools.setNativeFuncName(IDBFactory, 'IDBFactory'); +sdenv.tools.setNativeObjName(indexedDB, 'IDBFactory'); + +window.IDBFactory = IDBFactory; +window.indexedDB = indexedDB; diff --git a/browser/chrome/location.js b/browser/chrome/location.js new file mode 100644 index 0000000..3f2bfc3 --- /dev/null +++ b/browser/chrome/location.js @@ -0,0 +1,10 @@ +const sdenv = require('sdenv-extend').sdenv(); +const window = sdenv.memory.sdWindow; + +Object.defineProperty(window.location, 'replace', { + ...Object.getOwnPropertyDescriptor(window.location, 'replace'), + writable: false, + value: function(url) { + sdenv.tools.exit({ url }); + } +}); diff --git a/browser/chrome/navigation.js b/browser/chrome/navigation.js new file mode 100644 index 0000000..f74d09f --- /dev/null +++ b/browser/chrome/navigation.js @@ -0,0 +1,20 @@ +const sdenv = require('sdenv-extend').sdenv(); +const window = sdenv.memory.sdWindow; + +[window.Navigation, window.navigation] = sdenv.tools.getNativeProto('Navigation', 'navigation', { + canGoBack: false, + canGoForward: false, + oncurrententrychange: null, + onnavigate: null, + onnavigateerror: null, + onnavigatesuccess: null, + transition: null, + currentEntry: { + id: 'c72e7c89-2c22-47b6-86b8-e83db973ad22', + index: 1, + key: 'd6cc1590-0028-48e9-b6e7-b489d28d8481', + ondispose: null, + sameDocument: true, + url: 'http://example.com', + } +}); diff --git a/browser/chrome/navigator.js b/browser/chrome/navigator.js new file mode 100644 index 0000000..ba18f84 --- /dev/null +++ b/browser/chrome/navigator.js @@ -0,0 +1,48 @@ +const sdenv = require('sdenv-extend').sdenv(); +const window = sdenv.memory.sdWindow; + +const DeprecatedStorageQuota = function DeprecatedStorageQuota() { + throw new TypeError("Illegal constructor"); +}; +DeprecatedStorageQuota.prototype = { + queryUsageAndQuota() { + }, + requestQuota() { + }, +}; +sdenv.tools.setObjName(DeprecatedStorageQuota.prototype, "DeprecatedStorageQuota"); +const NetworkInformation = function NetworkInformation() { + throw new TypeError("Illegal constructor"); +} +sdenv.tools.setObjName(NetworkInformation.prototype, "NetworkInformation"); +class NavigatorCustomize { + get webkitPersistentStorage() { + return { __proto__: DeprecatedStorageQuota.prototype }; + } + get connection() { + return { + __proto__: NetworkInformation.prototype, + downlink: 3.85, + effectiveType: "4g", + onchange: null, + rtt: 100, + saveData: false, + }; + } + get userAgent() { + return 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'; + } + get appVersion() { + return '5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36' + } + get platform() { + return 'MacIntel'; + } + get vendor() { + return "Google Inc."; + } +}; +sdenv.tools.mixin(window.navigator, NavigatorCustomize.prototype, ['userAgent', 'platform', 'appVersion', 'vendor']); +Object.keys(window.navigator.__proto__).forEach(name => { + sdenv.tools.setFuncNative(Object.getOwnPropertyDescriptor(window.navigator.__proto__, name)?.get, 'get'); +}) diff --git a/browser/chrome/styleMedia.js b/browser/chrome/styleMedia.js new file mode 100644 index 0000000..716e8eb --- /dev/null +++ b/browser/chrome/styleMedia.js @@ -0,0 +1,6 @@ +const sdenv = require('sdenv-extend').sdenv(); +const window = sdenv.memory.sdWindow; + +window.styleMedia = sdenv.tools.getNativeProto('StyleMedia', 'styleMedia', { + type: 'screen' +})[1]; diff --git a/browser/chrome/visualViewport.js b/browser/chrome/visualViewport.js new file mode 100644 index 0000000..4e5b703 --- /dev/null +++ b/browser/chrome/visualViewport.js @@ -0,0 +1,14 @@ +const sdenv = require('sdenv-extend').sdenv(); +const window = sdenv.memory.sdWindow; + +[window.VisualViewport, window.visualViewport] = sdenv.tools.getNativeProto('VisualViewport', 'visualViewport', { + height: 904, + offsetLeft: 0, + offsetTop: 0, + onresize: null, + onscroll: null, + pageLeft: 0, + pageTop: 0, + scale: 1, + width: 1066, +}); diff --git a/browser/chrome/webkitRequestFileSystem.js b/browser/chrome/webkitRequestFileSystem.js new file mode 100644 index 0000000..326a611 --- /dev/null +++ b/browser/chrome/webkitRequestFileSystem.js @@ -0,0 +1,10 @@ +const sdenv = require('sdenv-extend').sdenv(); +const window = sdenv.memory.sdWindow; + +const webkitRequestFileSystem = function webkitRequestFileSystem(type, size, successCallback, errorCallback) { + if (typeof successCallback === 'function') { + window.setTimeout(successCallback, 0); + } +}; +sdenv.tools.setNativeFuncName(webkitRequestFileSystem, 'webkitRequestFileSystem') +window.webkitRequestFileSystem = webkitRequestFileSystem; diff --git a/browser/chrome/window.js b/browser/chrome/window.js new file mode 100644 index 0000000..ad7aad1 --- /dev/null +++ b/browser/chrome/window.js @@ -0,0 +1,16 @@ +const logger = require('@utils/logger'); +const sdenv = require('sdenv-extend').sdenv(); +const window = sdenv.memory.sdWindow; + +window.fetch = function fetch() {}; +sdenv.tools.setFuncNative(window.fetch); +window.Request = function Request() {}; +sdenv.tools.setFuncNative(window.Request); +window.closed = false; +window.opener = null; +window.clientInformation = window.navigator; +window.isSecureContext = false; +window.open = function(url) { + sdenv.tools.exit({ url }); +} +// window.console = logger; diff --git a/browser/index.js b/browser/index.js new file mode 100644 index 0000000..62c2b3c --- /dev/null +++ b/browser/index.js @@ -0,0 +1,12 @@ +require('module-alias/register'); +const sdenvExtend = require('sdenv-extend'); + +module.exports = (win, type = 'chrome') => { + new sdenvExtend({ + memory: { + sdenvExtend, + } + }, win); + require(`@/browser/${type}`); + return new sdenvExtend(); +} diff --git a/example/use-local/README.md b/example/use-local/README.md new file mode 100644 index 0000000..24a3025 --- /dev/null +++ b/example/use-local/README.md @@ -0,0 +1,11 @@ +该example通过执行本地js文件生成cookie，请求网站方式可以参考`example/use-remote`. + +output目录是使用[rs-reverse](https://github.com/pysunday/rs-reverse)下载的文件目录，命令：`npx -p rs-reverse@latest --registry=https://registry.npmjs.org rs-reverse makecode -u https://wcjs.sbj.cnipa.gov.cn/sgtmi` + +会用到三个文件： + +1. 网页html：`output/makecode_input_html.html` +2. 网页中的js外链：`output/makecode_input_js.js` +3. 网页中提取的$_ts：`output/makecode_input_ts.json` + +文件存在后执行命令`node example/use-local/index.js`生成cookie。 diff --git a/example/use-local/index.js b/example/use-local/index.js new file mode 100644 index 0000000..2129daf --- /dev/null +++ b/example/use-local/index.js @@ -0,0 +1,47 @@ +const fs = require('fs'); +const path = require('path'); +const { Script } = require("vm"); +const logger = require('../../utils/logger'); +const browser = require('../../browser/'); +const { jsdomFromText } = require('../../utils/jsdom'); + +const baseUrl = "https://wcjs.sbj.cnipa.gov.cn" + +const files = { + html: path.resolve(__dirname, 'output/makecode_input_html.html'), + js: path.resolve(__dirname, 'output/makecode_input_js.js'), + ts: path.resolve(__dirname, 'output/makecode_input_ts.json'), +} + +function getFile(name) { + const filepath = files[name]; + if (!filepath) throw new Error(`getFile: ${name}错误`); + if (!fs.existsSync(filepath)) throw new Error(`文件${filepath}不存在，请使用rs-reverse工具先获取文件`); + return fs.readFileSync(filepath); +} + +function initBrowser(window, cookieJar) { + window.$_ts = JSON.parse(getFile('ts')); + window.onbeforeunload = async (url) => { + const cookies = cookieJar.getCookieStringSync(baseUrl); + logger.debug('生成cookie：', cookies); + process.exit(); + } + browser(window, 'chrome'); +} + +async function loadPages() { + const htmltext = getFile('html'); + const jstext = getFile('js'); + const [jsdomer, cookieJar] = jsdomFromText({ + url: `${baseUrl}/sgtmi`, + referrer: `${baseUrl}/sgtmi`, + contentType: "text/html", + runScripts: "outside-only", + }) + const dom = jsdomer(htmltext); + initBrowser(dom.window, cookieJar); + new Script(jstext).runInContext(dom.getInternalVMContext()); +} + +loadPages() diff --git a/example/use-proxy/README.md b/example/use-proxy/README.md new file mode 100644 index 0000000..4f21744 --- /dev/null +++ b/example/use-proxy/README.md @@ -0,0 +1 @@ +该example通过代理请求网站生成cookie，作者用于开发使用，用户无需关注 diff --git a/example/use-proxy/index.js b/example/use-proxy/index.js new file mode 100644 index 0000000..fa1204a --- /dev/null +++ b/example/use-proxy/index.js @@ -0,0 +1,21 @@ +process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0" +const logger = require('../../utils/logger'); +const browser = require('../../browser/'); +const { jsdomFromUrl } = require('../../utils/jsdom'); + +const [jsdomer, cookieJar] = jsdomFromUrl({ + proxy: "http://127.0.0.1:7759", +}) + +const baseUrl = "https://wcjs.sbj.cnipa.gov.cn" + +async function loadPages() { + const dom = await jsdomer(`${baseUrl}/first`); + browser(dom.window, 'chrome'); + dom.window.onbeforeunload = async (url) => { + const cookies = cookieJar.getCookieStringSync(baseUrl); + logger.debug('cookieJar：', cookies); + } +} +loadPages() + diff --git a/example/use-remote/README.md b/example/use-remote/README.md new file mode 100644 index 0000000..86e9223 --- /dev/null +++ b/example/use-remote/README.md @@ -0,0 +1,3 @@ +该example通过请求网站并执行网站js代码生成cookie，如执行本地代码请参考`example/use-local` + +执行命令：`node example/use-remote/index.js` diff --git a/example/use-remote/index.js b/example/use-remote/index.js new file mode 100644 index 0000000..de9f63e --- /dev/null +++ b/example/use-remote/index.js @@ -0,0 +1,22 @@ +process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0" +const logger = require('../../utils/logger'); +const browser = require('../../browser/'); +const { jsdomFromUrl } = require('../../utils/jsdom'); + +const baseUrl = "https://wcjs.sbj.cnipa.gov.cn" + +async function loadPages() { + const [jsdomer, cookieJar] = jsdomFromUrl({ + userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', + }); + const dom = await jsdomer(`${baseUrl}/sgtmi`); + window = dom.window + window.onbeforeunload = async (url) => { + const cookies = cookieJar.getCookieStringSync(baseUrl); + logger.debug('生成cookie：', cookies); + process.exit(); + } + browser(window, 'chrome'); +} + +loadPages() diff --git a/logo.ico b/logo.ico new file mode 100644 index 0000000..47ce6c1 Binary files /dev/null and b/logo.ico differ diff --git a/package.json b/package.json new file mode 100644 index 0000000..cffc38c --- /dev/null +++ b/package.json @@ -0,0 +1,51 @@ +{ + "name": "sdenv", + "version": "0.1.0", + "description": "", + "main": "main.js", + "directories": { + "test": "test" + }, + "scripts": { + "test": "jest ./test/", + "test:debug": "node --inspect-brk node_modules/.bin/jest --runInBand ./test/", + "build": "node-gyp rebuild && cp build/Release/*.node ./bin/", + "release": "release-it" + }, + "logLevel": "debug", + "author": "pysunday", + "license": "ISC", + "dependencies": { + "bindings": "^1.5.0", + "canvas": "^2.11.2", + "jest": "^29.7.0", + "lodash": "^4.17.21", + "log4js": "^6.9.1", + "module-alias": "^2.2.3", + "node-addon-api": "^7.0.0", + "sdenv-extend": "^1.1.0", + "sdenv-jsdom": "^1.1.0" + }, + "devDependencies": { + "release-it": "^17.0.1" + }, + "gypfile": true, + "jest": { + "moduleNameMapper": { + "@/(.*)": "/$1", + "@utils/(.*)": "/utils/$1", + "@handler/(.*)": "/handler/$1", + "@bin/(.*)": "/bin/$1" + } + }, + "engines": { + "node": ">=20.10.0" + }, + "_moduleAliases": { + "@": ".", + "@handler": "./handler", + "@utils": "./utils", + "@bin": "./bin", + "@jsdom": "sdenv-jsdom/lib/jsdom/" + } +} diff --git a/test/documentAll.test.js b/test/documentAll.test.js new file mode 100644 index 0000000..9df73c1 --- /dev/null +++ b/test/documentAll.test.js @@ -0,0 +1,23 @@ +const getDocumentAll = require('../bin/documentAll.node').getDocumentAll; + +describe('模拟document.all检测', () => { + const da = getDocumentAll({ length: 1 }); + console.log( + '运行：getDocumentAll({ length: 1 })，返回：', da, + '\n运行：getDocumentAll({ length: 1 }) == undefined，返回：', da == undefined, + '\n运行：getDocumentAll({ length: 1 })()，返回：', da(), + '\n运行：typeof getDocumentAll({ length: 1 })，返回：', typeof da, + ); + test('getDocumentAll({ length: 1 }).length === 1', () => { + expect(da.length).toBe(1); + }); + test('getDocumentAll({ length: 1 }) == undefined', () => { + expect(da == undefined).toBe(true); + }); + test('typeof getDocumentAll({ length: 1 })', () => { + expect(typeof da).toBe('undefined'); + }); + test('getDocumentAll({ length: 1 })() === null', () => { + expect(da()).toBe(null); + }); +}); diff --git a/test/form.test.js b/test/form.test.js new file mode 100644 index 0000000..104f299 --- /dev/null +++ b/test/form.test.js @@ -0,0 +1,11 @@ +const jsdom = require('sdenv-jsdom'); +const { JSDOM, CookieJar } = jsdom; + +describe('form特性检测', () => { + test('子元素存在name属性', () => { + const dom = new JSDOM('
'); + const action = dom.window.document.getElementsByTagName('form')[0].action; + expect(action !== 'https://target.url/path/to').toBe(true); + expect(dom.window.document.getElementById('username')).toBe(action); + }); +}); diff --git a/utils/jsdom.js b/utils/jsdom.js new file mode 100644 index 0000000..e055684 --- /dev/null +++ b/utils/jsdom.js @@ -0,0 +1,46 @@ +const jsdom = require('sdenv-jsdom'); +const logger = require('./logger'); +const { JSDOM, CookieJar } = jsdom; + +exports.jsdomFromUrl = (config, ua) => { + const resourceLoader = new jsdom.ResourceLoader({ + strictSSL: false, + ...config, + }); + const virtualConsole = new jsdom.VirtualConsole(); + virtualConsole.sendTo({ + log: logger.log.bind(logger), + warn: logger.warn.bind(logger), + error: logger.error.bind(logger), + }); + const cookieJar = new CookieJar() + const options = { + pretendToBeVisual: true, + runScripts: "dangerously", + resources: resourceLoader, + cookieJar, + virtualConsole, + } + return [(url) => { + return JSDOM.fromURL(url, options); + }, cookieJar]; +}; + +exports.jsdomFromText = (config) => { + const virtualConsole = new jsdom.VirtualConsole(); + virtualConsole.sendTo({ + log: logger.log.bind(logger), + warn: logger.warn.bind(logger), + error: logger.error.bind(logger), + }); + const cookieJar = new CookieJar() + const options = { + pretendToBeVisual: true, + cookieJar, + virtualConsole, + ...config, + } + return [(text) => { + return new JSDOM(text, options); + }, cookieJar]; +} diff --git a/utils/logger.js b/utils/logger.js new file mode 100644 index 0000000..140c6c0 --- /dev/null +++ b/utils/logger.js @@ -0,0 +1,16 @@ +const paths = require('./paths'); +const pkg = require(paths.package); +const log4js = require('log4js'); + +log4js.configure({ + appenders: { + console: { type: 'console' } + }, + categories: { + default: { appenders: ['console'], level: 'info' } + } +}); +const logger = log4js.getLogger(pkg.name); +logger.level = pkg.logLevel || 'debug'; + +module.exports = logger; diff --git a/utils/paths.js b/utils/paths.js new file mode 100644 index 0000000..ef0fd61 --- /dev/null +++ b/utils/paths.js @@ -0,0 +1,25 @@ +const path = require('path'); +const fs = require('fs'); + +const appDirectory = (() => { + // 返回项目根目录 + const plist = fs.realpathSync(process.cwd()).split('/'); + while (!fs.existsSync(path.resolve(plist.join('/'), 'package.json'))) { + plist.pop(); + if (plist.length === 0) return false; + } + return plist.join('/'); +})(); +const resolveApp = (...relativePath) => path.resolve(appDirectory, ...relativePath); + +module.exports = { + basePath: resolveApp(''), + homePath: __dirname, + modulePath: resolveApp('node_modules'), + binPath: resolveApp('node_modules/.bin/'), + package: path.resolve('package.json'), + resolve: resolveApp, + handlerPath: resolveApp('handler'), + configPath: resolveApp('config'), + configResolve: (...p) => resolveApp('config', ...p), +}; diff --git a/utils/readConfig.js b/utils/readConfig.js new file mode 100644 index 0000000..e779e19 --- /dev/null +++ b/utils/readConfig.js @@ -0,0 +1,9 @@ +const paths = require('./paths'); + +module.exports = (filename, def = {}) => { + try { + return require(paths.configResolve(`${filename}.json`)); + } catch(e) { + return def || {}; + } +}