From 0fffebef4e4dfb64fd0613968fe544f4eac9da5c Mon Sep 17 00:00:00 2001 From: rnet Date: Sun, 10 Mar 2024 14:29:00 +0800 Subject: [PATCH] =?UTF-8?q?fore:=20=E5=8F=AF=E7=94=A8=E7=89=88=E6=9C=ACv0.?= =?UTF-8?q?1.0=E5=9B=BA=E5=AE=9A=E4=B8=8E=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 12 +++ .npmignore | 4 + .release-it.js | 30 ++++++ README.md | 120 ++++++++++++++++++++++ bin/documentAll.cc | 51 +++++++++ bin/documentAll.node | Bin 0 -> 54397 bytes binding.gyp | 11 ++ browser/chrome/RTCPeerConnection.js | 19 ++++ browser/chrome/chrome.js | 39 +++++++ browser/chrome/ctorRegistry.js | 12 +++ browser/chrome/document.js | 5 + browser/chrome/index.js | 12 +++ browser/chrome/indexedDB.js | 16 +++ browser/chrome/location.js | 10 ++ browser/chrome/navigation.js | 20 ++++ browser/chrome/navigator.js | 48 +++++++++ browser/chrome/styleMedia.js | 6 ++ browser/chrome/visualViewport.js | 14 +++ browser/chrome/webkitRequestFileSystem.js | 10 ++ browser/chrome/window.js | 16 +++ browser/index.js | 12 +++ example/use-local/README.md | 11 ++ example/use-local/index.js | 47 +++++++++ example/use-proxy/README.md | 1 + example/use-proxy/index.js | 21 ++++ example/use-remote/README.md | 3 + example/use-remote/index.js | 22 ++++ logo.ico | Bin 0 -> 16958 bytes package.json | 51 +++++++++ test/documentAll.test.js | 23 +++++ test/form.test.js | 11 ++ utils/jsdom.js | 46 +++++++++ utils/logger.js | 16 +++ utils/paths.js | 25 +++++ utils/readConfig.js | 9 ++ 35 files changed, 753 insertions(+) create mode 100644 .gitignore create mode 100644 .npmignore create mode 100644 .release-it.js create mode 100644 README.md create mode 100644 bin/documentAll.cc create mode 100755 bin/documentAll.node create mode 100644 binding.gyp create mode 100644 browser/chrome/RTCPeerConnection.js create mode 100644 browser/chrome/chrome.js create mode 100644 browser/chrome/ctorRegistry.js create mode 100644 browser/chrome/document.js create mode 100644 browser/chrome/index.js create mode 100644 browser/chrome/indexedDB.js create mode 100644 browser/chrome/location.js create mode 100644 browser/chrome/navigation.js create mode 100644 browser/chrome/navigator.js create mode 100644 browser/chrome/styleMedia.js create mode 100644 browser/chrome/visualViewport.js create mode 100644 browser/chrome/webkitRequestFileSystem.js create mode 100644 browser/chrome/window.js create mode 100644 browser/index.js create mode 100644 example/use-local/README.md create mode 100644 example/use-local/index.js create mode 100644 example/use-proxy/README.md create mode 100644 example/use-proxy/index.js create mode 100644 example/use-remote/README.md create mode 100644 example/use-remote/index.js create mode 100644 logo.ico create mode 100644 package.json create mode 100644 test/documentAll.test.js create mode 100644 test/form.test.js create mode 100644 utils/jsdom.js create mode 100644 utils/logger.js create mode 100644 utils/paths.js create mode 100644 utils/readConfig.js diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c14af9c --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +.DS_Store +node_modules/ +output/ +npm-debug.log +yarn-error.log +.idea +.vscode +package-lock.json +yarn.lock +.history +*.swp +build/ diff --git a/.npmignore b/.npmignore new file mode 100644 index 0000000..5da7fce --- /dev/null +++ b/.npmignore @@ -0,0 +1,4 @@ +/* +!bin/ +!browser/ +!utils/ diff --git a/.release-it.js b/.release-it.js new file mode 100644 index 0000000..fa81b75 --- /dev/null +++ b/.release-it.js @@ -0,0 +1,30 @@ +module.exports = { + github: { + release: true + }, + git: { + commitMessage: "release: v${version}" + }, + npm: { + publish: false + }, + hooks: { + "after:bump": "echo 更新版本成功" + }, + plugins: { + '@release-it/conventional-changelog': { + preset: 'conventionalcommits', + infile: 'CHANGELOG.md', + sameFile: true, + releaseRules: [ + { type: 'feat', release: 'minor' }, + { type: 'fix', release: 'patch' }, + { type: 'docs', release: 'patch' }, + { type: 'style', release: 'patch' }, + { type: 'refactor', release: 'patch' }, + { type: 'perf', release: 'patch' }, + { type: 'test', release: 'patch' }, + ], + }, + }, +}; diff --git a/README.md b/README.md new file mode 100644 index 0000000..8c5f78c --- /dev/null +++ b/README.md @@ -0,0 +1,120 @@ +

+
+ sdenv +

+ +sdenv是一个javascript运行时补环境框架，与github上其它补环境框架存在较大区别，sdenv是站在巨人的肩膀上实现的，依赖于jsdom的强大dom仿真能力，sdenv可以真实模拟浏览器执行环境，作者在固定随机数与添加[sdenv-extend](https://github.com/pysunday/sdenv-extend)的部分插件后可以达到**瑞数vmp代码在sdenv运行生成的cookie值与浏览器生成的cookie值一致**。 + +* sdenv专用jsdom版本：[sdenv-jsdom](https://github.com/pysunday/sdenv-jsdom) +* sdenv多端环境提取：[sdenv-extend](https://github.com/pysunday/sdenv-extend) + +## 依赖 + +作者开发时使用的是`v20.10.0`版本node，预期最低要求是18版本，由于未做其它版本可用性测试，因此建议使用sdenv的node版本大于等于`v20.10.0` + +## 安装 + +由于`document.all`需要由c代码动态生成，而固定编译环境下的编译产物只能在相同编译环境下运行，因此安装sdenv后需要动态编译生成node文件 + +1. 安装：`npm i sdenv` +2. 编译c代码：`cd node_modules/sdenv && yarn build` + +**在编译过程未实现自动化之前可直接clone项目使用** + +## 使用 + +因为项目核心功能基于jsdom，且jsdom对dom的实现非常完善，因此使用sdenv之前建议有一定html与javascript语言开发基础，然后参考example目录下的样例文件: + +1. [use-local](https://github.com/pysunday/sdenv/example/use-local/README.md) + ```javascript + const fs = require('fs'); + const path = require('path'); + const { Script } = require("vm"); + const logger = require('../../utils/logger'); + const browser = require('../../browser/'); + const { jsdomFromText } = require('../../utils/jsdom'); + + const baseUrl = "https://wcjs.sbj.cnipa.gov.cn" + + const files = { + html: path.resolve(__dirname, 'output/makecode_input_html.html'), + js: path.resolve(__dirname, 'output/makecode_input_js.js'), + ts: path.resolve(__dirname, 'output/makecode_input_ts.json'), + } + + function getFile(name) { + const filepath = files[name]; + if (!filepath) throw new Error(`getFile: ${name}错误`); + if (!fs.existsSync(filepath)) throw new Error(`文件${filepath}不存在，请使用rs-reverse工具先获取文件`); + return fs.readFileSync(filepath); + } + + function initBrowser(window, cookieJar) { + window.$_ts = JSON.parse(getFile('ts')); + window.onbeforeunload = async (url) => { + const cookies = cookieJar.getCookieStringSync(baseUrl); + logger.debug('生成cookie：', cookies); + process.exit(); + } + browser(window, 'chrome'); + } + + async function loadPages() { + const htmltext = getFile('html'); + const jstext = getFile('js'); + const [jsdomer, cookieJar] = jsdomFromText({ + url: `${baseUrl}/sgtmi`, + referrer: `${baseUrl}/sgtmi`, + contentType: "text/html", + runScripts: "outside-only", + }) + const dom = jsdomer(htmltext); + initBrowser(dom.window, cookieJar); + new Script(jstext).runInContext(dom.getInternalVMContext()); + } + + loadPages() + ``` +2. [use-remote](https://github.com/pysunday/sdenv/example/use-remote/README.md) + ```javascript + process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0" + const logger = require('../../utils/logger'); + const browser = require('../../browser/'); + const { jsdomFromUrl } = require('../../utils/jsdom'); + + const baseUrl = "https://wcjs.sbj.cnipa.gov.cn" + + async function loadPages() { + const [jsdomer, cookieJar] = jsdomFromUrl({ + userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', + }); + const dom = await jsdomer(`${baseUrl}/sgtmi`); + window = dom.window + window.onbeforeunload = async (url) => { + const cookies = cookieJar.getCookieStringSync(baseUrl); + logger.debug('生成cookie：', cookies); + process.exit(); + } + browser(window, 'chrome'); + } + + loadPages() + ``` + +## sdenv-extend使用说明 + +为了模拟浏览器执行环境，需要将node环境与浏览器环境共有代码进行提取，并提供返回环境对象用于sdenv内window与dom内容补充使用。 + +sdenv-extend初始化只执行一次，初始化成功后生成的环境对象可以使用`Object.sdenv()`(vm中使用非node)获取。 + +sdenv-extend具体功能可参考项目内README文档。 + +## 声明 + +该项目的开发基于瑞数vmp网站，不能保证在其它反爬虫产品稳定使用，出现问题请及时提issues或者提pull参与共建! + +由于初期版本只做了chrome浏览器的拟真，且项目文档不完善，作者会陆续补充，可以加入技术交流群与订阅号持续关注！ + +添加作者微信进技术交流群：howduudu_tech(备注sdenv) + +订阅号会定期发表技术文章：码功 diff --git a/bin/documentAll.cc b/bin/documentAll.cc new file mode 100644 index 0000000..1953f37 --- /dev/null +++ b/bin/documentAll.cc @@ -0,0 +1,51 @@ +#include +namespace documentAll { + +using v8::Context; +using v8::Function; +using v8::FunctionCallbackInfo; +using v8::FunctionTemplate; +using v8::Isolate; +using v8::Local; +using v8::Number; +using v8::Object; +using v8::ObjectTemplate; +using v8::String; +using v8::Value; +using v8::Null; +using v8::Array; + +void MyFunctionCallback(const FunctionCallbackInfo& args) { + Isolate* isolate = args.GetIsolate(); + args.GetReturnValue().Set(Null(isolate)); +} + +void GetDocumentAll(const FunctionCallbackInfo& args) { + Isolate* isolate = args.GetIsolate(); + Local context = isolate->GetCurrentContext(); + Local obj_template = ObjectTemplate::New(isolate); + obj_template->MarkAsUndetectable(); + obj_template->SetCallAsFunctionHandler(MyFunctionCallback); + Local obj = obj_template->NewInstance(context).ToLocalChecked(); + if (args.Length() > 0 && args[0]->IsObject()) { + Local argObj = args[0]->ToObject(context).ToLocalChecked(); + Local propertyNames = argObj->GetPropertyNames(context).ToLocalChecked(); + for (uint32_t i = 0; i < propertyNames->Length(); ++i) { + Local key = propertyNames->Get(context, i).ToLocalChecked(); + Local value = argObj->Get(context, key).ToLocalChecked(); + (void)obj->Set(context, key, value); + } + } + args.GetReturnValue().Set(obj); +} + +void Init(Local exports, Local module) { + Isolate* isolate = exports->GetIsolate(); + Local context = isolate->GetCurrentContext(); + Local method_template = FunctionTemplate::New(isolate, GetDocumentAll); + exports->Set(context, String::NewFromUtf8(isolate, "getDocumentAll").ToLocalChecked(), method_template->GetFunction(context).ToLocalChecked()).FromJust(); +} + +NODE_MODULE(NODE_GYP_MODULE_NAME, Init) +} + diff --git a/bin/documentAll.node b/bin/documentAll.node new file mode 100755 index 0000000000000000000000000000000000000000..a55d02d28531ca7d5ed12688e5776203673cdfac GIT binary patch literal 54397 zcmeI54RBP|6@c&Vl8`835*LYoB_KaRWPkEULPd88i$r1w*-%q!``DkCWW#1RynP#8 zM2(JG{Ank#*fMA!7LsICYE;T5D+wT3hMJ3~ecz)VMw8z59~4Z?iy1 z+p#m}GI#Ge_nv$1x!=9- zWopx(xCQEZe>vPA%w0*UKcQ6&cZd?y82xRzQnmX+Eg0&0e=IG&KunfGkrr-m^mpe~ zYJ-!vYdTj`eM6-n#3MI`B0(V(X^tv->UuoG{jK5-P)~_ZbIeOz*H0h|!U}hzTc}=B z=T%qN#SJo)7HGzdzVjdpLQ7P)tQq~`-cpiF?awuq4Z?Yv0;M3dM{$xNS!j+&0%`to zT&l6=4|R-3OSziB%hY-58oRjc;-x{?&l4>KA?y=kiFSW9EOf{ckBj%kqDz&xsfD8J z+6K?wR}-&lpZ4rBVZE>+gn6Lr16F-uj#d7`Z#)YjCk_N=H;SCg>;wVkFm~oX2(KvJL&kAm2QIi*7jhjrVY&n|w!7Y8Y?%@W)>0w@M1Tko z0U|&IhyW2F0z`la5CI}U1c(3;AOb{y2oM1xKm>>Y5g-CYfCvx)B0vO)01+SpM1Tko z0U|&IhyW2F0z`la5CI}U1c(3;AOb{y2>gE&$YHLb+$`77!rOCxah99)EX==m`$7}c zox_F(EW&%kEzk9SE|_{xTxae*zQ)o!a`Tkl5sRtk9~Rb!eP=-5LzWqR2N>IW9O_Uu zv95Rag8om?pMdi47sw*%^j(9S&8_e0ACeLsTwVW?lP)T{Af{YRDy`+f**g;QBR`gs`SSr)b- zFp_L6f-86%K-XeG`@6WGLnFz(L6DColQUb*uAv-Y#oReP7=I7cEAbB}_kK&)A8z3) zb6`-vPN|>S*M;R!vOEjMss~x)^w$%~M)aEt>-K_~{jIm)p3$`%xp@VwS2e8La#%li z?{Ld2Kg0D*T1;=BxXuJ^W>`yD&zG38mJQ$+*UJwju4D3SmPlIIv!-LIYiEJ=gX_5s z^B`wKo(j4P`ruq; zt&=-+a=lJ==;S<|d@O&==b%o0N+);gs*Xd!AM= zmvP_AT!vw+yFuQVtFGy>t6%8Fb#zQ&%9^%_@``96-Y!ODcR0*!HoHF*u?NShY=HpN zsW4^bj_buJK+=~m7Q?&qWl&;%4DP*SLK(t``>ZCocM3oZX&bjQRFg^shyW2F0z`la z5CI}U1c(3;AOb{y2oM1xKm>>Y5g-CYfCvx)B0vO)01+SpM1Tko0U|&IhyW2F0z`la z5CI}U1c(3;AOb{y2oM1xKm>>Y5g-CYfCvx)B0vO)01+SpM1TnV4+v=gul-j3|J(22 zImpX{y!?ch$9c*4|3g2Gmxa8X%S#6@ui)h>UN-V_JukJX!`~K3Nk8E2 zyWo#*=u@wupT3P`B0vO)01+SpM1Tko0U|&IhyW2F0z`la5CI}U1c(3;AOb{y2oM1x zKm>>Y5g-CYfCvx)B0vO)01+SpM1Tko0U|&IhyW2F0z`la5CI}U1c(3;AOb{y2oM1x zKm>>Y5g-CzbprXbbJosb>(8rhaIeK!3tYiOI4H#AxL@#xB0*7NYd5lPCb27Bf^dCZ zNoS?A#McoLLJ?V%BEGP*q%m3>4fw*I_6|AW=|tO{Otwo_O40Uf;xV} zL*7`2vNirTF(5aJ?HysCEIOTa;*B+tnCy!LL{FVpC|3rn0clBf6k@nZ_INxvXklg_ zmDl>D4er=Fn2rp-eg3ehj$4$mPf3|ql&gK=usgOg9tp^yXk?Wy5)6xyr+#H;!)h46 z#G$pIo!=MOP!nm6Vr1pt@P*?d#H9<^TOz1&7H1r@6sGE_$C=f{qBwb7cc^t%Xya*c z_{<$+9@9*m&Z|XP>zC@!z_0QoUeY5wLkML6q0Z|TkW!U5)DrQ@aS2I@4$!f}*Fn88=3vL-f=m|ElWIdC9VjTfj9sg$i{SIUO*24W_a+ zSEZR$fJwa+?GPn7QRizHV<)9lZ|q+R-81&6^E#EOj&2ByvS~st+OW&|v@WnaPk&Tz zxqx0;$GmuD2_8P>;lf99o!nfRy7TH+2M|0iylSg(522fmQ8(H?buIx2)p1D@BeM2T zDztA>O;^KPqueb?zC=Z>7-^AP)u*q^q-$0lnaY}&x^Sw&mUM&0C^vxUGQJABv0o$_ z6ovL^Fdh~KNo)zl;C%(*YsrLfGp92c4aD0aY9;%vs|>huKYcdAXw=YE5pH#M~#{mq43q*$x|>vaCG9mhMs$D0rR zaXd|rW$-tM9U)nG^|{bVi+>H*`#FytT-V|`06Lg|35Ak|;Zv{A14XX~ z40?}2Pa5>q5RktAeFlA%CAD2N=#Lomg9bf!N^1WagMP0;f6}18X3!5C^a9w=`uK}g z9Wg6uHhj*3d@1C)kmo_34|xIPV#o_2Uj}&*oH@SLxF<}_^vuqdnR<_Zs?1>hUph}_7%lz0 zmh$6r{F9{S>rBs(8T+S4^VK^&^5DlmHyRozaAHjDIl;5y1Uk1VuK{>y_ykHPd_L3$ zJL8ifcX0AEAsUT$8XT=pI|dqBPJR9x|KOMQtrq>J9RF%doTKMNL3b>Kqo^P#-8-aE zCwwbQf8*9J;Rj6*v#*PZQp_$zMA`nAeLY7W-Fu|#?vJ~B>}|1N6b_o5c7Hq+4%%%t zJAA#iGu=S8K;X2yH;7@;7ZdGKf16GANi8s%&XeL=r`w|GF*_&uV%JZ!HsqIlQo>&C z3$%(cJD$83we6qN>hXXWB%{*8Ju{MtbI;%sY8nfXNhX}^mA zH=L_}@_;pMu_yC0axyay<8Mk9w<^C$fN0a=I*p%>lNr9-C*7O_OyAG-V|>_ix^G*9ZIVTXoOnkDsc2+KkFQOFy~I|H!QiJ~G|7Y4(fzk`Fw#=s^C% zzZ_XsRq;gVx!=#Pv27}Szw^qYPdX;DeT&dm|?pN9(HeDT(Y KZvS5K$NvU~|5ZW& literal 0 HcmV?d00001 diff --git a/binding.gyp b/binding.gyp new file mode 100644 index 0000000..69465b9 --- /dev/null +++ b/binding.gyp @@ -0,0 +1,11 @@ +{ + "targets": [ + { + "target_name": "documentAll", + "sources": [ + "bin/documentAll.cc", + ] + } + ] +} + diff --git a/browser/chrome/RTCPeerConnection.js b/browser/chrome/RTCPeerConnection.js new file mode 100644 index 0000000..efe3d58 --- /dev/null +++ b/browser/chrome/RTCPeerConnection.js @@ -0,0 +1,19 @@ +const sdenv = require('sdenv-extend').sdenv(); +const window = sdenv.memory.sdWindow; + +function RTCPeerConnection() { + if (!(this instanceof RTCPeerConnection)) { + throw new TypeError("Uncaught TypeError: Failed to construct 'RTCPeerConnection': Please use the 'new' operator, this DOM object constructor cannot be called as a function."); + } + this.createDataChannel = function(...params) { + // window.console.log(`【RTCPeerConnection RTCPeerConnection】调用，参数：${params}`) + } + this.createOffer = function(...params) { + // window.console.log(`【RTCPeerConnection createOffer】调用，参数：${params}`) + } +} +sdenv.tools.setNativeFuncName(RTCPeerConnection, 'RTCPeerConnection'); +sdenv.tools.setNativeObjName(RTCPeerConnection.prototype, 'RTCPeerConnection'); + + +window.RTCPeerConnection = RTCPeerConnection; diff --git a/browser/chrome/chrome.js b/browser/chrome/chrome.js new file mode 100644 index 0000000..95ec3a6 --- /dev/null +++ b/browser/chrome/chrome.js @@ -0,0 +1,39 @@ +const sdenv = require('sdenv-extend').sdenv(); + +sdenv.memory.sdWindow.chrome = { + app: { + isInstalled: false, + InstallState: { + DISABLED: "disabled", + INSTALLED: "installed", + NOT_INSTALLED: "not_installed", + }, + RunningState: { + CANNOT_RUN: "cannot_run", + READY_TO_RUN: "ready_to_run", + RUNNING: "running", + }, + getDetails: function () {}, + getIsInstalled: function() {}, + installState: function() {}, + runningState: function() {}, + }, + csi: function() {}, + loadTimes: function() { + return { + "requestTime": 1700779741.985, + "startLoadTime": 1700779741.985, + "commitLoadTime": 1700779742.021, + "finishDocumentLoadTime": 0, + "finishLoadTime": 0, + "firstPaintTime": 0, + "firstPaintAfterLoadTime": 0, + "navigationType": "Reload", + "wasFetchedViaSpdy": false, + "wasNpnNegotiated": true, + "npnNegotiatedProtocol": "http/1.1", + "wasAlternateProtocolAvailable": false, + "connectionInfo": "http/1.1" + } + } +} diff --git a/browser/chrome/ctorRegistry.js b/browser/chrome/ctorRegistry.js new file mode 100644 index 0000000..3da437a --- /dev/null +++ b/browser/chrome/ctorRegistry.js @@ -0,0 +1,12 @@ +const logger = require('@utils/logger'); +const utils = require('sdenv-jsdom/lib/jsdom/living/generated/utils.js'); +const sdenv = require('sdenv-extend').sdenv(); +const window = sdenv.memory.sdWindow; + +const ctorRegistry = window[utils.ctorRegistrySymbol] +window[utils.ctorRegistrySymbol] = new window.Proxy(ctorRegistry, { + get(target, propKey, receiver) { + logger.trace('proxy ctorRegistry get', propKey); + return window.Reflect.get(target, propKey, receiver); + } +}) diff --git a/browser/chrome/document.js b/browser/chrome/document.js new file mode 100644 index 0000000..35d56e9 --- /dev/null +++ b/browser/chrome/document.js @@ -0,0 +1,5 @@ +const getDocumentAll = require('@bin/documentAll').getDocumentAll; +const sdenv = require('sdenv-extend').sdenv(); +const window = sdenv.memory.sdWindow; + +window.document.all = getDocumentAll({ length: 3 }); diff --git a/browser/chrome/index.js b/browser/chrome/index.js new file mode 100644 index 0000000..5fa491f --- /dev/null +++ b/browser/chrome/index.js @@ -0,0 +1,12 @@ +require('./window'); +require('./document'); +require('./navigation'); +require('./navigator'); +require('./chrome'); +require('./visualViewport'); +require('./styleMedia'); +// require('./webkitRequestFileSystem'); +require('./ctorRegistry'); +require('./location'); +require('./indexedDB'); +require('./RTCPeerConnection'); diff --git a/browser/chrome/indexedDB.js b/browser/chrome/indexedDB.js new file mode 100644 index 0000000..81c3630 --- /dev/null +++ b/browser/chrome/indexedDB.js @@ -0,0 +1,16 @@ +const sdenv = require('sdenv-extend').sdenv(); +const window = sdenv.memory.sdWindow; + +const IDBFactory = function IDBFactory() { + throw new TypeError("Illegal constructor"); +} + +const indexedDB = { + __proto__: IDBFactory.prototype +}; + +sdenv.tools.setNativeFuncName(IDBFactory, 'IDBFactory'); +sdenv.tools.setNativeObjName(indexedDB, 'IDBFactory'); + +window.IDBFactory = IDBFactory; +window.indexedDB = indexedDB; diff --git a/browser/chrome/location.js b/browser/chrome/location.js new file mode 100644 index 0000000..3f2bfc3 --- /dev/null +++ b/browser/chrome/location.js @@ -0,0 +1,10 @@ +const sdenv = require('sdenv-extend').sdenv(); +const window = sdenv.memory.sdWindow; + +Object.defineProperty(window.location, 'replace', { + ...Object.getOwnPropertyDescriptor(window.location, 'replace'), + writable: false, + value: function(url) { + sdenv.tools.exit({ url }); + } +}); diff --git a/browser/chrome/navigation.js b/browser/chrome/navigation.js new file mode 100644 index 0000000..f74d09f --- /dev/null +++ b/browser/chrome/navigation.js @@ -0,0 +1,20 @@ +const sdenv = require('sdenv-extend').sdenv(); +const window = sdenv.memory.sdWindow; + +[window.Navigation, window.navigation] = sdenv.tools.getNativeProto('Navigation', 'navigation', { + canGoBack: false, + canGoForward: false, + oncurrententrychange: null, + onnavigate: null, + onnavigateerror: null, + onnavigatesuccess: null, + transition: null, + currentEntry: { + id: 'c72e7c89-2c22-47b6-86b8-e83db973ad22', + index: 1, + key: 'd6cc1590-0028-48e9-b6e7-b489d28d8481', + ondispose: null, + sameDocument: true, + url: 'http://example.com', + } +}); diff --git a/browser/chrome/navigator.js b/browser/chrome/navigator.js new file mode 100644 index 0000000..ba18f84 --- /dev/null +++ b/browser/chrome/navigator.js @@ -0,0 +1,48 @@ +const sdenv = require('sdenv-extend').sdenv(); +const window = sdenv.memory.sdWindow; + +const DeprecatedStorageQuota = function DeprecatedStorageQuota() { + throw new TypeError("Illegal constructor"); +}; +DeprecatedStorageQuota.prototype = { + queryUsageAndQuota() { + }, + requestQuota() { + }, +}; +sdenv.tools.setObjName(DeprecatedStorageQuota.prototype, "DeprecatedStorageQuota"); +const NetworkInformation = function NetworkInformation() { + throw new TypeError("Illegal constructor"); +} +sdenv.tools.setObjName(NetworkInformation.prototype, "NetworkInformation"); +class NavigatorCustomize { + get webkitPersistentStorage() { + return { __proto__: DeprecatedStorageQuota.prototype }; + } + get connection() { + return { + __proto__: NetworkInformation.prototype, + downlink: 3.85, + effectiveType: "4g", + onchange: null, + rtt: 100, + saveData: false, + }; + } + get userAgent() { + return 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'; + } + get appVersion() { + return '5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36' + } + get platform() { + return 'MacIntel'; + } + get vendor() { + return "Google Inc."; + } +}; +sdenv.tools.mixin(window.navigator, NavigatorCustomize.prototype, ['userAgent', 'platform', 'appVersion', 'vendor']); +Object.keys(window.navigator.__proto__).forEach(name => { + sdenv.tools.setFuncNative(Object.getOwnPropertyDescriptor(window.navigator.__proto__, name)?.get, 'get'); +}) diff --git a/browser/chrome/styleMedia.js b/browser/chrome/styleMedia.js new file mode 100644 index 0000000..716e8eb --- /dev/null +++ b/browser/chrome/styleMedia.js @@ -0,0 +1,6 @@ +const sdenv = require('sdenv-extend').sdenv(); +const window = sdenv.memory.sdWindow; + +window.styleMedia = sdenv.tools.getNativeProto('StyleMedia', 'styleMedia', { + type: 'screen' +})[1]; diff --git a/browser/chrome/visualViewport.js b/browser/chrome/visualViewport.js new file mode 100644 index 0000000..4e5b703 --- /dev/null +++ b/browser/chrome/visualViewport.js @@ -0,0 +1,14 @@ +const sdenv = require('sdenv-extend').sdenv(); +const window = sdenv.memory.sdWindow; + +[window.VisualViewport, window.visualViewport] = sdenv.tools.getNativeProto('VisualViewport', 'visualViewport', { + height: 904, + offsetLeft: 0, + offsetTop: 0, + onresize: null, + onscroll: null, + pageLeft: 0, + pageTop: 0, + scale: 1, + width: 1066, +}); diff --git a/browser/chrome/webkitRequestFileSystem.js b/browser/chrome/webkitRequestFileSystem.js new file mode 100644 index 0000000..326a611 --- /dev/null +++ b/browser/chrome/webkitRequestFileSystem.js @@ -0,0 +1,10 @@ +const sdenv = require('sdenv-extend').sdenv(); +const window = sdenv.memory.sdWindow; + +const webkitRequestFileSystem = function webkitRequestFileSystem(type, size, successCallback, errorCallback) { + if (typeof successCallback === 'function') { + window.setTimeout(successCallback, 0); + } +}; +sdenv.tools.setNativeFuncName(webkitRequestFileSystem, 'webkitRequestFileSystem') +window.webkitRequestFileSystem = webkitRequestFileSystem; diff --git a/browser/chrome/window.js b/browser/chrome/window.js new file mode 100644 index 0000000..ad7aad1 --- /dev/null +++ b/browser/chrome/window.js @@ -0,0 +1,16 @@ +const logger = require('@utils/logger'); +const sdenv = require('sdenv-extend').sdenv(); +const window = sdenv.memory.sdWindow; + +window.fetch = function fetch() {}; +sdenv.tools.setFuncNative(window.fetch); +window.Request = function Request() {}; +sdenv.tools.setFuncNative(window.Request); +window.closed = false; +window.opener = null; +window.clientInformation = window.navigator; +window.isSecureContext = false; +window.open = function(url) { + sdenv.tools.exit({ url }); +} +// window.console = logger; diff --git a/browser/index.js b/browser/index.js new file mode 100644 index 0000000..62c2b3c --- /dev/null +++ b/browser/index.js @@ -0,0 +1,12 @@ +require('module-alias/register'); +const sdenvExtend = require('sdenv-extend'); + +module.exports = (win, type = 'chrome') => { + new sdenvExtend({ + memory: { + sdenvExtend, + } + }, win); + require(`@/browser/${type}`); + return new sdenvExtend(); +} diff --git a/example/use-local/README.md b/example/use-local/README.md new file mode 100644 index 0000000..24a3025 --- /dev/null +++ b/example/use-local/README.md @@ -0,0 +1,11 @@ +该example通过执行本地js文件生成cookie，请求网站方式可以参考`example/use-remote`. + +output目录是使用[rs-reverse](https://github.com/pysunday/rs-reverse)下载的文件目录，命令：`npx -p rs-reverse@latest --registry=https://registry.npmjs.org rs-reverse makecode -u https://wcjs.sbj.cnipa.gov.cn/sgtmi` + +会用到三个文件： + +1. 网页html：`output/makecode_input_html.html` +2. 网页中的js外链：`output/makecode_input_js.js` +3. 网页中提取的$_ts：`output/makecode_input_ts.json` + +文件存在后执行命令`node example/use-local/index.js`生成cookie。 diff --git a/example/use-local/index.js b/example/use-local/index.js new file mode 100644 index 0000000..2129daf --- /dev/null +++ b/example/use-local/index.js @@ -0,0 +1,47 @@ +const fs = require('fs'); +const path = require('path'); +const { Script } = require("vm"); +const logger = require('../../utils/logger'); +const browser = require('../../browser/'); +const { jsdomFromText } = require('../../utils/jsdom'); + +const baseUrl = "https://wcjs.sbj.cnipa.gov.cn" + +const files = { + html: path.resolve(__dirname, 'output/makecode_input_html.html'), + js: path.resolve(__dirname, 'output/makecode_input_js.js'), + ts: path.resolve(__dirname, 'output/makecode_input_ts.json'), +} + +function getFile(name) { + const filepath = files[name]; + if (!filepath) throw new Error(`getFile: ${name}错误`); + if (!fs.existsSync(filepath)) throw new Error(`文件${filepath}不存在，请使用rs-reverse工具先获取文件`); + return fs.readFileSync(filepath); +} + +function initBrowser(window, cookieJar) { + window.$_ts = JSON.parse(getFile('ts')); + window.onbeforeunload = async (url) => { + const cookies = cookieJar.getCookieStringSync(baseUrl); + logger.debug('生成cookie：', cookies); + process.exit(); + } + browser(window, 'chrome'); +} + +async function loadPages() { + const htmltext = getFile('html'); + const jstext = getFile('js'); + const [jsdomer, cookieJar] = jsdomFromText({ + url: `${baseUrl}/sgtmi`, + referrer: `${baseUrl}/sgtmi`, + contentType: "text/html", + runScripts: "outside-only", + }) + const dom = jsdomer(htmltext); + initBrowser(dom.window, cookieJar); + new Script(jstext).runInContext(dom.getInternalVMContext()); +} + +loadPages() diff --git a/example/use-proxy/README.md b/example/use-proxy/README.md new file mode 100644 index 0000000..4f21744 --- /dev/null +++ b/example/use-proxy/README.md @@ -0,0 +1 @@ +该example通过代理请求网站生成cookie，作者用于开发使用，用户无需关注 diff --git a/example/use-proxy/index.js b/example/use-proxy/index.js new file mode 100644 index 0000000..fa1204a --- /dev/null +++ b/example/use-proxy/index.js @@ -0,0 +1,21 @@ +process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0" +const logger = require('../../utils/logger'); +const browser = require('../../browser/'); +const { jsdomFromUrl } = require('../../utils/jsdom'); + +const [jsdomer, cookieJar] = jsdomFromUrl({ + proxy: "http://127.0.0.1:7759", +}) + +const baseUrl = "https://wcjs.sbj.cnipa.gov.cn" + +async function loadPages() { + const dom = await jsdomer(`${baseUrl}/first`); + browser(dom.window, 'chrome'); + dom.window.onbeforeunload = async (url) => { + const cookies = cookieJar.getCookieStringSync(baseUrl); + logger.debug('cookieJar：', cookies); + } +} +loadPages() + diff --git a/example/use-remote/README.md b/example/use-remote/README.md new file mode 100644 index 0000000..86e9223 --- /dev/null +++ b/example/use-remote/README.md @@ -0,0 +1,3 @@ +该example通过请求网站并执行网站js代码生成cookie，如执行本地代码请参考`example/use-local` + +执行命令：`node example/use-remote/index.js` diff --git a/example/use-remote/index.js b/example/use-remote/index.js new file mode 100644 index 0000000..de9f63e --- /dev/null +++ b/example/use-remote/index.js @@ -0,0 +1,22 @@ +process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0" +const logger = require('../../utils/logger'); +const browser = require('../../browser/'); +const { jsdomFromUrl } = require('../../utils/jsdom'); + +const baseUrl = "https://wcjs.sbj.cnipa.gov.cn" + +async function loadPages() { + const [jsdomer, cookieJar] = jsdomFromUrl({ + userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', + }); + const dom = await jsdomer(`${baseUrl}/sgtmi`); + window = dom.window + window.onbeforeunload = async (url) => { + const cookies = cookieJar.getCookieStringSync(baseUrl); + logger.debug('生成cookie：', cookies); + process.exit(); + } + browser(window, 'chrome'); +} + +loadPages() diff --git a/logo.ico b/logo.ico new file mode 100644 index 0000000000000000000000000000000000000000..47ce6c1388c2c27402f216a236e8d5124da82092 GIT binary patch literal 16958 zcmd^_4Um;p8OPsc!386iGSN_25DO4Bt8p4@Z1E!;nY1sl$#u7W?F z^Pcy-&w2jm`+cuTQpewU^Ai4@p3FWgNhT*rG8@QBTEP5)Kl74g z4m%ST!ZNrI{t3^+9?0e}ZQeg%HQWFnf#X3R!QTN2w`y)foOQ%|EL;LBK)t;XwMMJv z-8u5xBHn+Ad$z}&a4}2)eOE2qsyPa}#}L~bXoC&V2Wshuo$x1E4lS?<924i~Fl}Bd zsP`dw7JOD8ufXHrv$LQ!+98h1j?7eI`UZ4E?7wZK?}5wV6let7sPtHl`Jv1t%AE>V zz)JWxM8BIzUkk@UX?dRC|J6MnW z1spe_jv+#=>a8>`+y55$AJ{itumq+-Wt#UDXz=HlpANQd9A__(z8PE_s#|BNSYjO= znbW9mEvVad>JB&wjIq>{)yYfS!SS#H;<|c2>30m&eW2=3wJ)F^*JZWa51)bwP#ulK zc}e3qoH|_}wt%s0g!xeYaS(jkw->@zu)YW3bf|uP!?l6;W4#@jGe|!S#$;Q35ULkb z^wp915Pr9T_52cMLiOqyiRR)mvHp(CTZv^A7@Pg$zPkUhMIRlRv+%bW^#2n$IO|QZ z-cr|TjPAkr=3?B0eJ@nr2JzXB%v}7e%lZ8UHr1Y%8B6|Ta1Cq*+iN0}inzSIP{-@> zbvCHUc#Z<=4Ej=Q8^m$o{y1xIneu9levjmNHQVohgmlcO`|?%fd(L?bc7prC(=Z## zt#xqupzTC_Ij&!X9k3qmhAY7~9|tjJ=YnTVV_pewfWdx;$_c*ed9Drg^Ax-btYbW! z3qOMG&<~c8-MMS?=fF_4jO<>h(>}Ltt;;%MeO`Mu^vrw?G=MR_2i%*y|8*Dx<bswa?fP9!(woM@O?-9_QS~P9I8~PSN+ey2KX?12A0ENcusX)GH%z3@XWqiA>6m$99Qc7GC0qD z-g$5%Oo73ELf*mEA9cid!dP^?IzElr`LG0b!xJzA;!}k-_0dKd+t7agO)k9ydj%W@ zh0n#S@cg0CUl@aH;9__dY=d9HBcLYN!ohv#^CABPtm~K0IoJDO8!QCRzpr#|#bCd=KE`!xBkAc-7=MV}Huw_QP7&7}{giIAd@r@4KWzj1>oVwpsKo<|_n2%*&CjCXQE%TL@PS=wDv<=)Nz6#F&sOt};(=$mU8#1*%B^{2Jkx~p+vwNtSU(y5m8Z`=BFuqckLB96!8UTu z^L$k}M~eHq_&St7wCjIyOz9X<%jaPasBt|U1F;R>Pul(7d(IEnf>Pf0omZ>+(>6F7 zO>uqmd@z4NV~D$*bnN@f_vY_g?ch7Luiw7*ToA`VYfh(YoabltKL*Z)!TN^Fsb&4f z9^#vZjr081q$j{E=mPaGgHU!jy~X!Ohkmu)l+)@t?|3*L+!Mri=wf9fyBP3oYocq!$n+MMs&)Ow8)BS--2iI#Eck<5+HWW$JL?r|Pkq((oU0J0?>5_E7q|}kZcy2|C)Qo4U!Tq?Z8fQu{^`WE8EjMI z^L_6ah-%8Wh58*+_kerLZTpEnehSs>|KsqN|32s%SS|g9-@ikZg?_PlPpT)Q{FoK^_FYV-V)>bx$e@lI@5>(hIF|LBD$q5OB@+Sl(| z=Q-csKF))5oeSxSRF7Rx87Nl*I zmM)zNWov8v$N0l{?*F2v2i{jwS19W`@I7!0tOef_OY1GptzG@o(C{Fr$^VbbJ)k)H zxDINa`@X|k{`=s(7+k(Mr`Gg`c>L}EhoF{6!1vzb$kXfajBRiMJO`c;+rTqpTCP+o zJX;&%Kg2SDn7;{rqDx5#1A7p>WM8e!f=phd3Ow*Tc);yZvIQ zgW{-5Uk4xdxw`cs--V+gv{CVsA^*_ze~ham^JZdn-a2;ezpsJoT5-ffUk87VA+DoL+`U)ydGJOlc5E-eGk6X_Y@ zQ0NbJ8pDa;cMQ+)#<&GO0pX07Dk?t}>TvGZ@AkjziFI^?`^hG0a%hLGupMSWDR1TFg}UeB%QKle zuZO!}J-i6UU@VWrXTd%wy+)N+^Wf6aZ}?8){_p^JrZR?n)ISFX`zSXj)$hE^ufw!w zx=y$frow^IT^w_)rz10oI?skSaNKPI`^5LlYU{U69c$LH8d~8@XoQM&XS4lDvNX$| zo+N41b|y)^wxzMB4%44xvli$hv$Hvy?WIspQ#RX0y03ve?)7tTO_o1x1OE+NG_+>3 ztzFsvi8yIFb!p!u?fB-t>DsZ)y~&Jh_PV60x2`*zZEtDnY0!2x_B3jH8oQdby$xN> z+P;R)rP}`b&a5_D-x_RdTd-@QZ4cMo4Z(Ib7TTWN^*D(2b&Rvvt_|_5T@!3ucd+eU!8Y_P9cYKf-ezrIQ*V>Dx2Z3*TXTQC a*K|svrSDqm0$p1A%T8^f5Bs%Cv;PNf`p?Y( literal 0 HcmV?d00001 diff --git a/package.json b/package.json new file mode 100644 index 0000000..cffc38c --- /dev/null +++ b/package.json @@ -0,0 +1,51 @@ +{ + "name": "sdenv", + "version": "0.1.0", + "description": "", + "main": "main.js", + "directories": { + "test": "test" + }, + "scripts": { + "test": "jest ./test/", + "test:debug": "node --inspect-brk node_modules/.bin/jest --runInBand ./test/", + "build": "node-gyp rebuild && cp build/Release/*.node ./bin/", + "release": "release-it" + }, + "logLevel": "debug", + "author": "pysunday", + "license": "ISC", + "dependencies": { + "bindings": "^1.5.0", + "canvas": "^2.11.2", + "jest": "^29.7.0", + "lodash": "^4.17.21", + "log4js": "^6.9.1", + "module-alias": "^2.2.3", + "node-addon-api": "^7.0.0", + "sdenv-extend": "^1.1.0", + "sdenv-jsdom": "^1.1.0" + }, + "devDependencies": { + "release-it": "^17.0.1" + }, + "gypfile": true, + "jest": { + "moduleNameMapper": { + "@/(.*)": "/$1", + "@utils/(.*)": "/utils/$1", + "@handler/(.*)": "/handler/$1", + "@bin/(.*)": "/bin/$1" + } + }, + "engines": { + "node": ">=20.10.0" + }, + "_moduleAliases": { + "@": ".", + "@handler": "./handler", + "@utils": "./utils", + "@bin": "./bin", + "@jsdom": "sdenv-jsdom/lib/jsdom/" + } +} diff --git a/test/documentAll.test.js b/test/documentAll.test.js new file mode 100644 index 0000000..9df73c1 --- /dev/null +++ b/test/documentAll.test.js @@ -0,0 +1,23 @@ +const getDocumentAll = require('../bin/documentAll.node').getDocumentAll; + +describe('模拟document.all检测', () => { + const da = getDocumentAll({ length: 1 }); + console.log( + '运行：getDocumentAll({ length: 1 })，返回：', da, + '\n运行：getDocumentAll({ length: 1 }) == undefined，返回：', da == undefined, + '\n运行：getDocumentAll({ length: 1 })()，返回：', da(), + '\n运行：typeof getDocumentAll({ length: 1 })，返回：', typeof da, + ); + test('getDocumentAll({ length: 1 }).length === 1', () => { + expect(da.length).toBe(1); + }); + test('getDocumentAll({ length: 1 }) == undefined', () => { + expect(da == undefined).toBe(true); + }); + test('typeof getDocumentAll({ length: 1 })', () => { + expect(typeof da).toBe('undefined'); + }); + test('getDocumentAll({ length: 1 })() === null', () => { + expect(da()).toBe(null); + }); +}); diff --git a/test/form.test.js b/test/form.test.js new file mode 100644 index 0000000..104f299 --- /dev/null +++ b/test/form.test.js @@ -0,0 +1,11 @@ +const jsdom = require('sdenv-jsdom'); +const { JSDOM, CookieJar } = jsdom; + +describe('form特性检测', () => { + test('子元素存在name属性', () => { + const dom = new JSDOM('
'); + const action = dom.window.document.getElementsByTagName('form')[0].action; + expect(action !== 'https://target.url/path/to').toBe(true); + expect(dom.window.document.getElementById('username')).toBe(action); + }); +}); diff --git a/utils/jsdom.js b/utils/jsdom.js new file mode 100644 index 0000000..e055684 --- /dev/null +++ b/utils/jsdom.js @@ -0,0 +1,46 @@ +const jsdom = require('sdenv-jsdom'); +const logger = require('./logger'); +const { JSDOM, CookieJar } = jsdom; + +exports.jsdomFromUrl = (config, ua) => { + const resourceLoader = new jsdom.ResourceLoader({ + strictSSL: false, + ...config, + }); + const virtualConsole = new jsdom.VirtualConsole(); + virtualConsole.sendTo({ + log: logger.log.bind(logger), + warn: logger.warn.bind(logger), + error: logger.error.bind(logger), + }); + const cookieJar = new CookieJar() + const options = { + pretendToBeVisual: true, + runScripts: "dangerously", + resources: resourceLoader, + cookieJar, + virtualConsole, + } + return [(url) => { + return JSDOM.fromURL(url, options); + }, cookieJar]; +}; + +exports.jsdomFromText = (config) => { + const virtualConsole = new jsdom.VirtualConsole(); + virtualConsole.sendTo({ + log: logger.log.bind(logger), + warn: logger.warn.bind(logger), + error: logger.error.bind(logger), + }); + const cookieJar = new CookieJar() + const options = { + pretendToBeVisual: true, + cookieJar, + virtualConsole, + ...config, + } + return [(text) => { + return new JSDOM(text, options); + }, cookieJar]; +} diff --git a/utils/logger.js b/utils/logger.js new file mode 100644 index 0000000..140c6c0 --- /dev/null +++ b/utils/logger.js @@ -0,0 +1,16 @@ +const paths = require('./paths'); +const pkg = require(paths.package); +const log4js = require('log4js'); + +log4js.configure({ + appenders: { + console: { type: 'console' } + }, + categories: { + default: { appenders: ['console'], level: 'info' } + } +}); +const logger = log4js.getLogger(pkg.name); +logger.level = pkg.logLevel || 'debug'; + +module.exports = logger; diff --git a/utils/paths.js b/utils/paths.js new file mode 100644 index 0000000..ef0fd61 --- /dev/null +++ b/utils/paths.js @@ -0,0 +1,25 @@ +const path = require('path'); +const fs = require('fs'); + +const appDirectory = (() => { + // 返回项目根目录 + const plist = fs.realpathSync(process.cwd()).split('/'); + while (!fs.existsSync(path.resolve(plist.join('/'), 'package.json'))) { + plist.pop(); + if (plist.length === 0) return false; + } + return plist.join('/'); +})(); +const resolveApp = (...relativePath) => path.resolve(appDirectory, ...relativePath); + +module.exports = { + basePath: resolveApp(''), + homePath: __dirname, + modulePath: resolveApp('node_modules'), + binPath: resolveApp('node_modules/.bin/'), + package: path.resolve('package.json'), + resolve: resolveApp, + handlerPath: resolveApp('handler'), + configPath: resolveApp('config'), + configResolve: (...p) => resolveApp('config', ...p), +}; diff --git a/utils/readConfig.js b/utils/readConfig.js new file mode 100644 index 0000000..e779e19 --- /dev/null +++ b/utils/readConfig.js @@ -0,0 +1,9 @@ +const paths = require('./paths'); + +module.exports = (filename, def = {}) => { + try { + return require(paths.configResolve(`${filename}.json`)); + } catch(e) { + return def || {}; + } +}