mirror of
https://github.com/pysunday/sdenv.git
synced 2025-04-05 01:03:51 +08:00
fore: 可用版本v0.1.0固定与提交
This commit is contained in:
commit
0fffebef4e
12
.gitignore
vendored
Normal file
12
.gitignore
vendored
Normal file
@ -0,0 +1,12 @@
|
||||
.DS_Store
|
||||
node_modules/
|
||||
output/
|
||||
npm-debug.log
|
||||
yarn-error.log
|
||||
.idea
|
||||
.vscode
|
||||
package-lock.json
|
||||
yarn.lock
|
||||
.history
|
||||
*.swp
|
||||
build/
|
4
.npmignore
Normal file
4
.npmignore
Normal file
@ -0,0 +1,4 @@
|
||||
/*
|
||||
!bin/
|
||||
!browser/
|
||||
!utils/
|
30
.release-it.js
Normal file
30
.release-it.js
Normal file
@ -0,0 +1,30 @@
|
||||
module.exports = {
|
||||
github: {
|
||||
release: true
|
||||
},
|
||||
git: {
|
||||
commitMessage: "release: v${version}"
|
||||
},
|
||||
npm: {
|
||||
publish: false
|
||||
},
|
||||
hooks: {
|
||||
"after:bump": "echo 更新版本成功"
|
||||
},
|
||||
plugins: {
|
||||
'@release-it/conventional-changelog': {
|
||||
preset: 'conventionalcommits',
|
||||
infile: 'CHANGELOG.md',
|
||||
sameFile: true,
|
||||
releaseRules: [
|
||||
{ type: 'feat', release: 'minor' },
|
||||
{ type: 'fix', release: 'patch' },
|
||||
{ type: 'docs', release: 'patch' },
|
||||
{ type: 'style', release: 'patch' },
|
||||
{ type: 'refactor', release: 'patch' },
|
||||
{ type: 'perf', release: 'patch' },
|
||||
{ type: 'test', release: 'patch' },
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
120
README.md
Normal file
120
README.md
Normal file
@ -0,0 +1,120 @@
|
||||
<h1 align="center">
|
||||
<img width="100" height="100" src="logo.svg" alt=""><br>
|
||||
sdenv
|
||||
</h1>
|
||||
|
||||
sdenv是一个javascript运行时补环境框架,与github上其它补环境框架存在较大区别,sdenv是站在巨人的肩膀上实现的,依赖于jsdom的强大dom仿真能力,sdenv可以真实模拟浏览器执行环境,作者在固定随机数与添加[sdenv-extend](https://github.com/pysunday/sdenv-extend)的部分插件后可以达到**瑞数vmp代码在sdenv运行生成的cookie值与浏览器生成的cookie值一致**。
|
||||
|
||||
* sdenv专用jsdom版本:[sdenv-jsdom](https://github.com/pysunday/sdenv-jsdom)
|
||||
* sdenv多端环境提取:[sdenv-extend](https://github.com/pysunday/sdenv-extend)
|
||||
|
||||
## 依赖
|
||||
|
||||
作者开发时使用的是`v20.10.0`版本node,预期最低要求是18版本,由于未做其它版本可用性测试,因此建议使用sdenv的node版本大于等于`v20.10.0`
|
||||
|
||||
## 安装
|
||||
|
||||
由于`document.all`需要由c代码动态生成,而固定编译环境下的编译产物只能在相同编译环境下运行,因此安装sdenv后需要动态编译生成node文件
|
||||
|
||||
1. 安装:`npm i sdenv`
|
||||
2. 编译c代码:`cd node_modules/sdenv && yarn build`
|
||||
|
||||
**在编译过程未实现自动化之前可直接clone项目使用**
|
||||
|
||||
## 使用
|
||||
|
||||
因为项目核心功能基于jsdom,且jsdom对dom的实现非常完善,因此使用sdenv之前建议有一定html与javascript语言开发基础,然后参考example目录下的样例文件:
|
||||
|
||||
1. [use-local](https://github.com/pysunday/sdenv/example/use-local/README.md)
|
||||
```javascript
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { Script } = require("vm");
|
||||
const logger = require('../../utils/logger');
|
||||
const browser = require('../../browser/');
|
||||
const { jsdomFromText } = require('../../utils/jsdom');
|
||||
|
||||
const baseUrl = "https://wcjs.sbj.cnipa.gov.cn"
|
||||
|
||||
const files = {
|
||||
html: path.resolve(__dirname, 'output/makecode_input_html.html'),
|
||||
js: path.resolve(__dirname, 'output/makecode_input_js.js'),
|
||||
ts: path.resolve(__dirname, 'output/makecode_input_ts.json'),
|
||||
}
|
||||
|
||||
function getFile(name) {
|
||||
const filepath = files[name];
|
||||
if (!filepath) throw new Error(`getFile: ${name}错误`);
|
||||
if (!fs.existsSync(filepath)) throw new Error(`文件${filepath}不存在,请使用rs-reverse工具先获取文件`);
|
||||
return fs.readFileSync(filepath);
|
||||
}
|
||||
|
||||
function initBrowser(window, cookieJar) {
|
||||
window.$_ts = JSON.parse(getFile('ts'));
|
||||
window.onbeforeunload = async (url) => {
|
||||
const cookies = cookieJar.getCookieStringSync(baseUrl);
|
||||
logger.debug('生成cookie:', cookies);
|
||||
process.exit();
|
||||
}
|
||||
browser(window, 'chrome');
|
||||
}
|
||||
|
||||
async function loadPages() {
|
||||
const htmltext = getFile('html');
|
||||
const jstext = getFile('js');
|
||||
const [jsdomer, cookieJar] = jsdomFromText({
|
||||
url: `${baseUrl}/sgtmi`,
|
||||
referrer: `${baseUrl}/sgtmi`,
|
||||
contentType: "text/html",
|
||||
runScripts: "outside-only",
|
||||
})
|
||||
const dom = jsdomer(htmltext);
|
||||
initBrowser(dom.window, cookieJar);
|
||||
new Script(jstext).runInContext(dom.getInternalVMContext());
|
||||
}
|
||||
|
||||
loadPages()
|
||||
```
|
||||
2. [use-remote](https://github.com/pysunday/sdenv/example/use-remote/README.md)
|
||||
```javascript
|
||||
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"
|
||||
const logger = require('../../utils/logger');
|
||||
const browser = require('../../browser/');
|
||||
const { jsdomFromUrl } = require('../../utils/jsdom');
|
||||
|
||||
const baseUrl = "https://wcjs.sbj.cnipa.gov.cn"
|
||||
|
||||
async function loadPages() {
|
||||
const [jsdomer, cookieJar] = jsdomFromUrl({
|
||||
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
|
||||
});
|
||||
const dom = await jsdomer(`${baseUrl}/sgtmi`);
|
||||
window = dom.window
|
||||
window.onbeforeunload = async (url) => {
|
||||
const cookies = cookieJar.getCookieStringSync(baseUrl);
|
||||
logger.debug('生成cookie:', cookies);
|
||||
process.exit();
|
||||
}
|
||||
browser(window, 'chrome');
|
||||
}
|
||||
|
||||
loadPages()
|
||||
```
|
||||
|
||||
## sdenv-extend使用说明
|
||||
|
||||
为了模拟浏览器执行环境,需要将node环境与浏览器环境共有代码进行提取,并提供返回环境对象用于sdenv内window与dom内容补充使用。
|
||||
|
||||
sdenv-extend初始化只执行一次,初始化成功后生成的环境对象可以使用`Object.sdenv()`(vm中使用非node)获取。
|
||||
|
||||
sdenv-extend具体功能可参考项目内README文档。
|
||||
|
||||
## 声明
|
||||
|
||||
该项目的开发基于瑞数vmp网站,不能保证在其它反爬虫产品稳定使用,出现问题请及时提issues或者提pull参与共建!
|
||||
|
||||
由于初期版本只做了chrome浏览器的拟真,且项目文档不完善,作者会陆续补充,可以加入技术交流群与订阅号持续关注!
|
||||
|
||||
添加作者微信进技术交流群:howduudu_tech(备注sdenv)
|
||||
|
||||
订阅号会定期发表技术文章:码功
|
51
bin/documentAll.cc
Normal file
51
bin/documentAll.cc
Normal file
@ -0,0 +1,51 @@
|
||||
#include <node.h>
|
||||
namespace documentAll {
|
||||
|
||||
using v8::Context;
|
||||
using v8::Function;
|
||||
using v8::FunctionCallbackInfo;
|
||||
using v8::FunctionTemplate;
|
||||
using v8::Isolate;
|
||||
using v8::Local;
|
||||
using v8::Number;
|
||||
using v8::Object;
|
||||
using v8::ObjectTemplate;
|
||||
using v8::String;
|
||||
using v8::Value;
|
||||
using v8::Null;
|
||||
using v8::Array;
|
||||
|
||||
void MyFunctionCallback(const FunctionCallbackInfo<Value>& args) {
|
||||
Isolate* isolate = args.GetIsolate();
|
||||
args.GetReturnValue().Set(Null(isolate));
|
||||
}
|
||||
|
||||
void GetDocumentAll(const FunctionCallbackInfo<Value>& args) {
|
||||
Isolate* isolate = args.GetIsolate();
|
||||
Local<Context> context = isolate->GetCurrentContext();
|
||||
Local<ObjectTemplate> obj_template = ObjectTemplate::New(isolate);
|
||||
obj_template->MarkAsUndetectable();
|
||||
obj_template->SetCallAsFunctionHandler(MyFunctionCallback);
|
||||
Local<Object> obj = obj_template->NewInstance(context).ToLocalChecked();
|
||||
if (args.Length() > 0 && args[0]->IsObject()) {
|
||||
Local<Object> argObj = args[0]->ToObject(context).ToLocalChecked();
|
||||
Local<Array> propertyNames = argObj->GetPropertyNames(context).ToLocalChecked();
|
||||
for (uint32_t i = 0; i < propertyNames->Length(); ++i) {
|
||||
Local<Value> key = propertyNames->Get(context, i).ToLocalChecked();
|
||||
Local<Value> value = argObj->Get(context, key).ToLocalChecked();
|
||||
(void)obj->Set(context, key, value);
|
||||
}
|
||||
}
|
||||
args.GetReturnValue().Set(obj);
|
||||
}
|
||||
|
||||
void Init(Local<Object> exports, Local<Object> module) {
|
||||
Isolate* isolate = exports->GetIsolate();
|
||||
Local<Context> context = isolate->GetCurrentContext();
|
||||
Local<FunctionTemplate> method_template = FunctionTemplate::New(isolate, GetDocumentAll);
|
||||
exports->Set(context, String::NewFromUtf8(isolate, "getDocumentAll").ToLocalChecked(), method_template->GetFunction(context).ToLocalChecked()).FromJust();
|
||||
}
|
||||
|
||||
NODE_MODULE(NODE_GYP_MODULE_NAME, Init)
|
||||
}
|
||||
|
BIN
bin/documentAll.node
Executable file
BIN
bin/documentAll.node
Executable file
Binary file not shown.
11
binding.gyp
Normal file
11
binding.gyp
Normal file
@ -0,0 +1,11 @@
|
||||
{
|
||||
"targets": [
|
||||
{
|
||||
"target_name": "documentAll",
|
||||
"sources": [
|
||||
"bin/documentAll.cc",
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
19
browser/chrome/RTCPeerConnection.js
Normal file
19
browser/chrome/RTCPeerConnection.js
Normal file
@ -0,0 +1,19 @@
|
||||
const sdenv = require('sdenv-extend').sdenv();
|
||||
const window = sdenv.memory.sdWindow;
|
||||
|
||||
function RTCPeerConnection() {
|
||||
if (!(this instanceof RTCPeerConnection)) {
|
||||
throw new TypeError("Uncaught TypeError: Failed to construct 'RTCPeerConnection': Please use the 'new' operator, this DOM object constructor cannot be called as a function.");
|
||||
}
|
||||
this.createDataChannel = function(...params) {
|
||||
// window.console.log(`【RTCPeerConnection RTCPeerConnection】调用,参数:${params}`)
|
||||
}
|
||||
this.createOffer = function(...params) {
|
||||
// window.console.log(`【RTCPeerConnection createOffer】调用,参数:${params}`)
|
||||
}
|
||||
}
|
||||
sdenv.tools.setNativeFuncName(RTCPeerConnection, 'RTCPeerConnection');
|
||||
sdenv.tools.setNativeObjName(RTCPeerConnection.prototype, 'RTCPeerConnection');
|
||||
|
||||
|
||||
window.RTCPeerConnection = RTCPeerConnection;
|
39
browser/chrome/chrome.js
Normal file
39
browser/chrome/chrome.js
Normal file
@ -0,0 +1,39 @@
|
||||
const sdenv = require('sdenv-extend').sdenv();
|
||||
|
||||
sdenv.memory.sdWindow.chrome = {
|
||||
app: {
|
||||
isInstalled: false,
|
||||
InstallState: {
|
||||
DISABLED: "disabled",
|
||||
INSTALLED: "installed",
|
||||
NOT_INSTALLED: "not_installed",
|
||||
},
|
||||
RunningState: {
|
||||
CANNOT_RUN: "cannot_run",
|
||||
READY_TO_RUN: "ready_to_run",
|
||||
RUNNING: "running",
|
||||
},
|
||||
getDetails: function () {},
|
||||
getIsInstalled: function() {},
|
||||
installState: function() {},
|
||||
runningState: function() {},
|
||||
},
|
||||
csi: function() {},
|
||||
loadTimes: function() {
|
||||
return {
|
||||
"requestTime": 1700779741.985,
|
||||
"startLoadTime": 1700779741.985,
|
||||
"commitLoadTime": 1700779742.021,
|
||||
"finishDocumentLoadTime": 0,
|
||||
"finishLoadTime": 0,
|
||||
"firstPaintTime": 0,
|
||||
"firstPaintAfterLoadTime": 0,
|
||||
"navigationType": "Reload",
|
||||
"wasFetchedViaSpdy": false,
|
||||
"wasNpnNegotiated": true,
|
||||
"npnNegotiatedProtocol": "http/1.1",
|
||||
"wasAlternateProtocolAvailable": false,
|
||||
"connectionInfo": "http/1.1"
|
||||
}
|
||||
}
|
||||
}
|
12
browser/chrome/ctorRegistry.js
Normal file
12
browser/chrome/ctorRegistry.js
Normal file
@ -0,0 +1,12 @@
|
||||
const logger = require('@utils/logger');
|
||||
const utils = require('sdenv-jsdom/lib/jsdom/living/generated/utils.js');
|
||||
const sdenv = require('sdenv-extend').sdenv();
|
||||
const window = sdenv.memory.sdWindow;
|
||||
|
||||
const ctorRegistry = window[utils.ctorRegistrySymbol]
|
||||
window[utils.ctorRegistrySymbol] = new window.Proxy(ctorRegistry, {
|
||||
get(target, propKey, receiver) {
|
||||
logger.trace('proxy ctorRegistry get', propKey);
|
||||
return window.Reflect.get(target, propKey, receiver);
|
||||
}
|
||||
})
|
5
browser/chrome/document.js
Normal file
5
browser/chrome/document.js
Normal file
@ -0,0 +1,5 @@
|
||||
const getDocumentAll = require('@bin/documentAll').getDocumentAll;
|
||||
const sdenv = require('sdenv-extend').sdenv();
|
||||
const window = sdenv.memory.sdWindow;
|
||||
|
||||
window.document.all = getDocumentAll({ length: 3 });
|
12
browser/chrome/index.js
Normal file
12
browser/chrome/index.js
Normal file
@ -0,0 +1,12 @@
|
||||
require('./window');
|
||||
require('./document');
|
||||
require('./navigation');
|
||||
require('./navigator');
|
||||
require('./chrome');
|
||||
require('./visualViewport');
|
||||
require('./styleMedia');
|
||||
// require('./webkitRequestFileSystem');
|
||||
require('./ctorRegistry');
|
||||
require('./location');
|
||||
require('./indexedDB');
|
||||
require('./RTCPeerConnection');
|
16
browser/chrome/indexedDB.js
Normal file
16
browser/chrome/indexedDB.js
Normal file
@ -0,0 +1,16 @@
|
||||
const sdenv = require('sdenv-extend').sdenv();
|
||||
const window = sdenv.memory.sdWindow;
|
||||
|
||||
const IDBFactory = function IDBFactory() {
|
||||
throw new TypeError("Illegal constructor");
|
||||
}
|
||||
|
||||
const indexedDB = {
|
||||
__proto__: IDBFactory.prototype
|
||||
};
|
||||
|
||||
sdenv.tools.setNativeFuncName(IDBFactory, 'IDBFactory');
|
||||
sdenv.tools.setNativeObjName(indexedDB, 'IDBFactory');
|
||||
|
||||
window.IDBFactory = IDBFactory;
|
||||
window.indexedDB = indexedDB;
|
10
browser/chrome/location.js
Normal file
10
browser/chrome/location.js
Normal file
@ -0,0 +1,10 @@
|
||||
const sdenv = require('sdenv-extend').sdenv();
|
||||
const window = sdenv.memory.sdWindow;
|
||||
|
||||
Object.defineProperty(window.location, 'replace', {
|
||||
...Object.getOwnPropertyDescriptor(window.location, 'replace'),
|
||||
writable: false,
|
||||
value: function(url) {
|
||||
sdenv.tools.exit({ url });
|
||||
}
|
||||
});
|
20
browser/chrome/navigation.js
Normal file
20
browser/chrome/navigation.js
Normal file
@ -0,0 +1,20 @@
|
||||
const sdenv = require('sdenv-extend').sdenv();
|
||||
const window = sdenv.memory.sdWindow;
|
||||
|
||||
[window.Navigation, window.navigation] = sdenv.tools.getNativeProto('Navigation', 'navigation', {
|
||||
canGoBack: false,
|
||||
canGoForward: false,
|
||||
oncurrententrychange: null,
|
||||
onnavigate: null,
|
||||
onnavigateerror: null,
|
||||
onnavigatesuccess: null,
|
||||
transition: null,
|
||||
currentEntry: {
|
||||
id: 'c72e7c89-2c22-47b6-86b8-e83db973ad22',
|
||||
index: 1,
|
||||
key: 'd6cc1590-0028-48e9-b6e7-b489d28d8481',
|
||||
ondispose: null,
|
||||
sameDocument: true,
|
||||
url: 'http://example.com',
|
||||
}
|
||||
});
|
48
browser/chrome/navigator.js
Normal file
48
browser/chrome/navigator.js
Normal file
@ -0,0 +1,48 @@
|
||||
const sdenv = require('sdenv-extend').sdenv();
|
||||
const window = sdenv.memory.sdWindow;
|
||||
|
||||
const DeprecatedStorageQuota = function DeprecatedStorageQuota() {
|
||||
throw new TypeError("Illegal constructor");
|
||||
};
|
||||
DeprecatedStorageQuota.prototype = {
|
||||
queryUsageAndQuota() {
|
||||
},
|
||||
requestQuota() {
|
||||
},
|
||||
};
|
||||
sdenv.tools.setObjName(DeprecatedStorageQuota.prototype, "DeprecatedStorageQuota");
|
||||
const NetworkInformation = function NetworkInformation() {
|
||||
throw new TypeError("Illegal constructor");
|
||||
}
|
||||
sdenv.tools.setObjName(NetworkInformation.prototype, "NetworkInformation");
|
||||
class NavigatorCustomize {
|
||||
get webkitPersistentStorage() {
|
||||
return { __proto__: DeprecatedStorageQuota.prototype };
|
||||
}
|
||||
get connection() {
|
||||
return {
|
||||
__proto__: NetworkInformation.prototype,
|
||||
downlink: 3.85,
|
||||
effectiveType: "4g",
|
||||
onchange: null,
|
||||
rtt: 100,
|
||||
saveData: false,
|
||||
};
|
||||
}
|
||||
get userAgent() {
|
||||
return 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36';
|
||||
}
|
||||
get appVersion() {
|
||||
return '5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'
|
||||
}
|
||||
get platform() {
|
||||
return 'MacIntel';
|
||||
}
|
||||
get vendor() {
|
||||
return "Google Inc.";
|
||||
}
|
||||
};
|
||||
sdenv.tools.mixin(window.navigator, NavigatorCustomize.prototype, ['userAgent', 'platform', 'appVersion', 'vendor']);
|
||||
Object.keys(window.navigator.__proto__).forEach(name => {
|
||||
sdenv.tools.setFuncNative(Object.getOwnPropertyDescriptor(window.navigator.__proto__, name)?.get, 'get');
|
||||
})
|
6
browser/chrome/styleMedia.js
Normal file
6
browser/chrome/styleMedia.js
Normal file
@ -0,0 +1,6 @@
|
||||
const sdenv = require('sdenv-extend').sdenv();
|
||||
const window = sdenv.memory.sdWindow;
|
||||
|
||||
window.styleMedia = sdenv.tools.getNativeProto('StyleMedia', 'styleMedia', {
|
||||
type: 'screen'
|
||||
})[1];
|
14
browser/chrome/visualViewport.js
Normal file
14
browser/chrome/visualViewport.js
Normal file
@ -0,0 +1,14 @@
|
||||
const sdenv = require('sdenv-extend').sdenv();
|
||||
const window = sdenv.memory.sdWindow;
|
||||
|
||||
[window.VisualViewport, window.visualViewport] = sdenv.tools.getNativeProto('VisualViewport', 'visualViewport', {
|
||||
height: 904,
|
||||
offsetLeft: 0,
|
||||
offsetTop: 0,
|
||||
onresize: null,
|
||||
onscroll: null,
|
||||
pageLeft: 0,
|
||||
pageTop: 0,
|
||||
scale: 1,
|
||||
width: 1066,
|
||||
});
|
10
browser/chrome/webkitRequestFileSystem.js
Normal file
10
browser/chrome/webkitRequestFileSystem.js
Normal file
@ -0,0 +1,10 @@
|
||||
const sdenv = require('sdenv-extend').sdenv();
|
||||
const window = sdenv.memory.sdWindow;
|
||||
|
||||
const webkitRequestFileSystem = function webkitRequestFileSystem(type, size, successCallback, errorCallback) {
|
||||
if (typeof successCallback === 'function') {
|
||||
window.setTimeout(successCallback, 0);
|
||||
}
|
||||
};
|
||||
sdenv.tools.setNativeFuncName(webkitRequestFileSystem, 'webkitRequestFileSystem')
|
||||
window.webkitRequestFileSystem = webkitRequestFileSystem;
|
16
browser/chrome/window.js
Normal file
16
browser/chrome/window.js
Normal file
@ -0,0 +1,16 @@
|
||||
const logger = require('@utils/logger');
|
||||
const sdenv = require('sdenv-extend').sdenv();
|
||||
const window = sdenv.memory.sdWindow;
|
||||
|
||||
window.fetch = function fetch() {};
|
||||
sdenv.tools.setFuncNative(window.fetch);
|
||||
window.Request = function Request() {};
|
||||
sdenv.tools.setFuncNative(window.Request);
|
||||
window.closed = false;
|
||||
window.opener = null;
|
||||
window.clientInformation = window.navigator;
|
||||
window.isSecureContext = false;
|
||||
window.open = function(url) {
|
||||
sdenv.tools.exit({ url });
|
||||
}
|
||||
// window.console = logger;
|
12
browser/index.js
Normal file
12
browser/index.js
Normal file
@ -0,0 +1,12 @@
|
||||
require('module-alias/register');
|
||||
const sdenvExtend = require('sdenv-extend');
|
||||
|
||||
module.exports = (win, type = 'chrome') => {
|
||||
new sdenvExtend({
|
||||
memory: {
|
||||
sdenvExtend,
|
||||
}
|
||||
}, win);
|
||||
require(`@/browser/${type}`);
|
||||
return new sdenvExtend();
|
||||
}
|
11
example/use-local/README.md
Normal file
11
example/use-local/README.md
Normal file
@ -0,0 +1,11 @@
|
||||
该example通过执行本地js文件生成cookie,请求网站方式可以参考`example/use-remote`.
|
||||
|
||||
output目录是使用[rs-reverse](https://github.com/pysunday/rs-reverse)下载的文件目录,命令:`npx -p rs-reverse@latest --registry=https://registry.npmjs.org rs-reverse makecode -u https://wcjs.sbj.cnipa.gov.cn/sgtmi`
|
||||
|
||||
会用到三个文件:
|
||||
|
||||
1. 网页html:`output/makecode_input_html.html`
|
||||
2. 网页中的js外链:`output/makecode_input_js.js`
|
||||
3. 网页中提取的$_ts:`output/makecode_input_ts.json`
|
||||
|
||||
文件存在后执行命令`node example/use-local/index.js`生成cookie。
|
47
example/use-local/index.js
Normal file
47
example/use-local/index.js
Normal file
@ -0,0 +1,47 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { Script } = require("vm");
|
||||
const logger = require('../../utils/logger');
|
||||
const browser = require('../../browser/');
|
||||
const { jsdomFromText } = require('../../utils/jsdom');
|
||||
|
||||
const baseUrl = "https://wcjs.sbj.cnipa.gov.cn"
|
||||
|
||||
const files = {
|
||||
html: path.resolve(__dirname, 'output/makecode_input_html.html'),
|
||||
js: path.resolve(__dirname, 'output/makecode_input_js.js'),
|
||||
ts: path.resolve(__dirname, 'output/makecode_input_ts.json'),
|
||||
}
|
||||
|
||||
function getFile(name) {
|
||||
const filepath = files[name];
|
||||
if (!filepath) throw new Error(`getFile: ${name}错误`);
|
||||
if (!fs.existsSync(filepath)) throw new Error(`文件${filepath}不存在,请使用rs-reverse工具先获取文件`);
|
||||
return fs.readFileSync(filepath);
|
||||
}
|
||||
|
||||
function initBrowser(window, cookieJar) {
|
||||
window.$_ts = JSON.parse(getFile('ts'));
|
||||
window.onbeforeunload = async (url) => {
|
||||
const cookies = cookieJar.getCookieStringSync(baseUrl);
|
||||
logger.debug('生成cookie:', cookies);
|
||||
process.exit();
|
||||
}
|
||||
browser(window, 'chrome');
|
||||
}
|
||||
|
||||
async function loadPages() {
|
||||
const htmltext = getFile('html');
|
||||
const jstext = getFile('js');
|
||||
const [jsdomer, cookieJar] = jsdomFromText({
|
||||
url: `${baseUrl}/sgtmi`,
|
||||
referrer: `${baseUrl}/sgtmi`,
|
||||
contentType: "text/html",
|
||||
runScripts: "outside-only",
|
||||
})
|
||||
const dom = jsdomer(htmltext);
|
||||
initBrowser(dom.window, cookieJar);
|
||||
new Script(jstext).runInContext(dom.getInternalVMContext());
|
||||
}
|
||||
|
||||
loadPages()
|
1
example/use-proxy/README.md
Normal file
1
example/use-proxy/README.md
Normal file
@ -0,0 +1 @@
|
||||
该example通过代理请求网站生成cookie,作者用于开发使用,用户无需关注
|
21
example/use-proxy/index.js
Normal file
21
example/use-proxy/index.js
Normal file
@ -0,0 +1,21 @@
|
||||
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"
|
||||
const logger = require('../../utils/logger');
|
||||
const browser = require('../../browser/');
|
||||
const { jsdomFromUrl } = require('../../utils/jsdom');
|
||||
|
||||
const [jsdomer, cookieJar] = jsdomFromUrl({
|
||||
proxy: "http://127.0.0.1:7759",
|
||||
})
|
||||
|
||||
const baseUrl = "https://wcjs.sbj.cnipa.gov.cn"
|
||||
|
||||
async function loadPages() {
|
||||
const dom = await jsdomer(`${baseUrl}/first`);
|
||||
browser(dom.window, 'chrome');
|
||||
dom.window.onbeforeunload = async (url) => {
|
||||
const cookies = cookieJar.getCookieStringSync(baseUrl);
|
||||
logger.debug('cookieJar:', cookies);
|
||||
}
|
||||
}
|
||||
loadPages()
|
||||
|
3
example/use-remote/README.md
Normal file
3
example/use-remote/README.md
Normal file
@ -0,0 +1,3 @@
|
||||
该example通过请求网站并执行网站js代码生成cookie,如执行本地代码请参考`example/use-local`
|
||||
|
||||
执行命令:`node example/use-remote/index.js`
|
22
example/use-remote/index.js
Normal file
22
example/use-remote/index.js
Normal file
@ -0,0 +1,22 @@
|
||||
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"
|
||||
const logger = require('../../utils/logger');
|
||||
const browser = require('../../browser/');
|
||||
const { jsdomFromUrl } = require('../../utils/jsdom');
|
||||
|
||||
const baseUrl = "https://wcjs.sbj.cnipa.gov.cn"
|
||||
|
||||
async function loadPages() {
|
||||
const [jsdomer, cookieJar] = jsdomFromUrl({
|
||||
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
|
||||
});
|
||||
const dom = await jsdomer(`${baseUrl}/sgtmi`);
|
||||
window = dom.window
|
||||
window.onbeforeunload = async (url) => {
|
||||
const cookies = cookieJar.getCookieStringSync(baseUrl);
|
||||
logger.debug('生成cookie:', cookies);
|
||||
process.exit();
|
||||
}
|
||||
browser(window, 'chrome');
|
||||
}
|
||||
|
||||
loadPages()
|
51
package.json
Normal file
51
package.json
Normal file
@ -0,0 +1,51 @@
|
||||
{
|
||||
"name": "sdenv",
|
||||
"version": "0.1.0",
|
||||
"description": "",
|
||||
"main": "main.js",
|
||||
"directories": {
|
||||
"test": "test"
|
||||
},
|
||||
"scripts": {
|
||||
"test": "jest ./test/",
|
||||
"test:debug": "node --inspect-brk node_modules/.bin/jest --runInBand ./test/",
|
||||
"build": "node-gyp rebuild && cp build/Release/*.node ./bin/",
|
||||
"release": "release-it"
|
||||
},
|
||||
"logLevel": "debug",
|
||||
"author": "pysunday",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"bindings": "^1.5.0",
|
||||
"canvas": "^2.11.2",
|
||||
"jest": "^29.7.0",
|
||||
"lodash": "^4.17.21",
|
||||
"log4js": "^6.9.1",
|
||||
"module-alias": "^2.2.3",
|
||||
"node-addon-api": "^7.0.0",
|
||||
"sdenv-extend": "^1.1.0",
|
||||
"sdenv-jsdom": "^1.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"release-it": "^17.0.1"
|
||||
},
|
||||
"gypfile": true,
|
||||
"jest": {
|
||||
"moduleNameMapper": {
|
||||
"@/(.*)": "<rootDir>/$1",
|
||||
"@utils/(.*)": "<rootDir>/utils/$1",
|
||||
"@handler/(.*)": "<rootDir>/handler/$1",
|
||||
"@bin/(.*)": "<rootDir>/bin/$1"
|
||||
}
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.10.0"
|
||||
},
|
||||
"_moduleAliases": {
|
||||
"@": ".",
|
||||
"@handler": "./handler",
|
||||
"@utils": "./utils",
|
||||
"@bin": "./bin",
|
||||
"@jsdom": "sdenv-jsdom/lib/jsdom/"
|
||||
}
|
||||
}
|
23
test/documentAll.test.js
Normal file
23
test/documentAll.test.js
Normal file
@ -0,0 +1,23 @@
|
||||
const getDocumentAll = require('../bin/documentAll.node').getDocumentAll;
|
||||
|
||||
describe('模拟document.all检测', () => {
|
||||
const da = getDocumentAll({ length: 1 });
|
||||
console.log(
|
||||
'运行:getDocumentAll({ length: 1 }),返回:', da,
|
||||
'\n运行:getDocumentAll({ length: 1 }) == undefined,返回:', da == undefined,
|
||||
'\n运行:getDocumentAll({ length: 1 })(),返回:', da(),
|
||||
'\n运行:typeof getDocumentAll({ length: 1 }),返回:', typeof da,
|
||||
);
|
||||
test('getDocumentAll({ length: 1 }).length === 1', () => {
|
||||
expect(da.length).toBe(1);
|
||||
});
|
||||
test('getDocumentAll({ length: 1 }) == undefined', () => {
|
||||
expect(da == undefined).toBe(true);
|
||||
});
|
||||
test('typeof getDocumentAll({ length: 1 })', () => {
|
||||
expect(typeof da).toBe('undefined');
|
||||
});
|
||||
test('getDocumentAll({ length: 1 })() === null', () => {
|
||||
expect(da()).toBe(null);
|
||||
});
|
||||
});
|
11
test/form.test.js
Normal file
11
test/form.test.js
Normal file
@ -0,0 +1,11 @@
|
||||
const jsdom = require('sdenv-jsdom');
|
||||
const { JSDOM, CookieJar } = jsdom;
|
||||
|
||||
describe('form特性检测', () => {
|
||||
test('子元素存在name属性', () => {
|
||||
const dom = new JSDOM('<form id="__Zm9ybS5pZAo__" action="https://target.url/path/to"><input id="username" name="action"><input name="textContent" id="password"><input id="innerText" type="submit" name="id"></form>');
|
||||
const action = dom.window.document.getElementsByTagName('form')[0].action;
|
||||
expect(action !== 'https://target.url/path/to').toBe(true);
|
||||
expect(dom.window.document.getElementById('username')).toBe(action);
|
||||
});
|
||||
});
|
46
utils/jsdom.js
Normal file
46
utils/jsdom.js
Normal file
@ -0,0 +1,46 @@
|
||||
const jsdom = require('sdenv-jsdom');
|
||||
const logger = require('./logger');
|
||||
const { JSDOM, CookieJar } = jsdom;
|
||||
|
||||
exports.jsdomFromUrl = (config, ua) => {
|
||||
const resourceLoader = new jsdom.ResourceLoader({
|
||||
strictSSL: false,
|
||||
...config,
|
||||
});
|
||||
const virtualConsole = new jsdom.VirtualConsole();
|
||||
virtualConsole.sendTo({
|
||||
log: logger.log.bind(logger),
|
||||
warn: logger.warn.bind(logger),
|
||||
error: logger.error.bind(logger),
|
||||
});
|
||||
const cookieJar = new CookieJar()
|
||||
const options = {
|
||||
pretendToBeVisual: true,
|
||||
runScripts: "dangerously",
|
||||
resources: resourceLoader,
|
||||
cookieJar,
|
||||
virtualConsole,
|
||||
}
|
||||
return [(url) => {
|
||||
return JSDOM.fromURL(url, options);
|
||||
}, cookieJar];
|
||||
};
|
||||
|
||||
exports.jsdomFromText = (config) => {
|
||||
const virtualConsole = new jsdom.VirtualConsole();
|
||||
virtualConsole.sendTo({
|
||||
log: logger.log.bind(logger),
|
||||
warn: logger.warn.bind(logger),
|
||||
error: logger.error.bind(logger),
|
||||
});
|
||||
const cookieJar = new CookieJar()
|
||||
const options = {
|
||||
pretendToBeVisual: true,
|
||||
cookieJar,
|
||||
virtualConsole,
|
||||
...config,
|
||||
}
|
||||
return [(text) => {
|
||||
return new JSDOM(text, options);
|
||||
}, cookieJar];
|
||||
}
|
16
utils/logger.js
Normal file
16
utils/logger.js
Normal file
@ -0,0 +1,16 @@
|
||||
const paths = require('./paths');
|
||||
const pkg = require(paths.package);
|
||||
const log4js = require('log4js');
|
||||
|
||||
log4js.configure({
|
||||
appenders: {
|
||||
console: { type: 'console' }
|
||||
},
|
||||
categories: {
|
||||
default: { appenders: ['console'], level: 'info' }
|
||||
}
|
||||
});
|
||||
const logger = log4js.getLogger(pkg.name);
|
||||
logger.level = pkg.logLevel || 'debug';
|
||||
|
||||
module.exports = logger;
|
25
utils/paths.js
Normal file
25
utils/paths.js
Normal file
@ -0,0 +1,25 @@
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
|
||||
const appDirectory = (() => {
|
||||
// 返回项目根目录
|
||||
const plist = fs.realpathSync(process.cwd()).split('/');
|
||||
while (!fs.existsSync(path.resolve(plist.join('/'), 'package.json'))) {
|
||||
plist.pop();
|
||||
if (plist.length === 0) return false;
|
||||
}
|
||||
return plist.join('/');
|
||||
})();
|
||||
const resolveApp = (...relativePath) => path.resolve(appDirectory, ...relativePath);
|
||||
|
||||
module.exports = {
|
||||
basePath: resolveApp(''),
|
||||
homePath: __dirname,
|
||||
modulePath: resolveApp('node_modules'),
|
||||
binPath: resolveApp('node_modules/.bin/'),
|
||||
package: path.resolve('package.json'),
|
||||
resolve: resolveApp,
|
||||
handlerPath: resolveApp('handler'),
|
||||
configPath: resolveApp('config'),
|
||||
configResolve: (...p) => resolveApp('config', ...p),
|
||||
};
|
9
utils/readConfig.js
Normal file
9
utils/readConfig.js
Normal file
@ -0,0 +1,9 @@
|
||||
const paths = require('./paths');
|
||||
|
||||
module.exports = (filename, def = {}) => {
|
||||
try {
|
||||
return require(paths.configResolve(`${filename}.json`));
|
||||
} catch(e) {
|
||||
return def || {};
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user