优化爬虫,增加缓存,线程池,保存浏览器用户信息

This commit is contained in:
root
2026-06-20 15:49:40 +08:00
parent e95f4a227c
commit 09bfa9ae01
26 changed files with 1584 additions and 454 deletions
@@ -1,26 +1,17 @@
/**
* Real Browser 启动:puppeteer-real-browser(抗 Cloudflare Turnstile
* 注意:不与 puppeteer-extra 混用在同一 Browser 实例
*/
const {
BASE_LAUNCH_ARGS,
resolveChromeExecutable,
} = require('./constants');
const { resolveProfileDir } = require('./profile-dir');
/** @type {boolean|null} */
let realBrowserModuleChecked = null;
/** @type {boolean} */
let realBrowserAvailable = false;
/**
* 检测 puppeteer-real-browser 是否已安装
* @returns {boolean}
*/
function isRealBrowserAvailable() {
if (realBrowserModuleChecked !== null) {
return realBrowserAvailable;
}
if (realBrowserModuleChecked !== null) return realBrowserAvailable;
try {
require.resolve('puppeteer-real-browser');
realBrowserAvailable = true;
@@ -32,11 +23,10 @@ function isRealBrowserAvailable() {
}
/**
* 启动 real profile Browserheadless:false + Xvfb + turnstile 内置点击)
* @param {string[]} [extraArgs]
* @returns {Promise<{ browser: import('puppeteer').Browser, page: import('puppeteer').Page }>}
* @param {{ sessionKey?: string, profile?: 'standard'|'real' }} [options]
*/
async function launchRealBrowser(extraArgs = []) {
async function launchRealBrowser(extraArgs = [], options = {}) {
if (!isRealBrowserAvailable()) {
throw new Error(
'puppeteer-real-browser 未安装。请在 puppeteer-api 目录执行: npm install puppeteer-real-browser@1.4.4'
@@ -45,22 +35,28 @@ async function launchRealBrowser(extraArgs = []) {
const { connect } = require('puppeteer-real-browser');
const chromePath = resolveChromeExecutable();
const profile = options.profile === 'real' ? 'real' : 'standard';
const profileDir = options.sessionKey ? resolveProfileDir(options.sessionKey, profile) : null;
const args = [...BASE_LAUNCH_ARGS, '--mute-audio', ...extraArgs];
if (profileDir) {
args.push(`--user-data-dir=${profileDir}`);
}
const result = await connect({
headless: false,
turnstile: true,
disableXvfb: false,
customConfig: { chromePath },
args: [...BASE_LAUNCH_ARGS, '--mute-audio', ...extraArgs],
args,
});
const browser = result.browser;
const page = result.page;
if (!browser || !page) {
throw new Error('puppeteer-real-browser connect 未返回 browser/page');
}
browser.__persistentProfile = !!profileDir;
browser.__userDataDir = profileDir || '';
return { browser, page };
}