damon/links
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化爬虫，增加缓存，线程池，保存浏览器用户信息

Browse Source
This commit is contained in:
root
2026-06-20 15:49:40 +08:00
parent e95f4a227c
commit 09bfa9ae01
26 changed files with 1584 additions and 454 deletions
Download Patch File Download Diff File Expand all files Collapse all files
patches/puppeteer-api/lib/browser-factory.js
+39 -55
View File
@@ -1,34 +1,22 @@
/**
* BrowserFactory：standard / real 双轨启动 + 独立并发限流
* BrowserFactory：standard / real 双轨启动 + 温池 + 独立并发限流
*/
const {
DEFAULT_UA,
MAX_CONCURRENT_BROWSERS,
MAX_CONCURRENT_BROWSERS_REAL,
QUEUE_TIMEOUT_MS,
NAVIGATION_TIMEOUT_MS,
PAGE_DEFAULT_TIMEOUT_MS,
} = require('./constants');
const { BrowserConcurrencyLimiter } = require('./concurrency');
const { launchStandardBrowser } = require('./launch-standard');
const { launchRealBrowser, isRealBrowserAvailable } = require('./launch-real-browser');
const { browserPool } = require('./browser-pool');
const standardLimiter = new BrowserConcurrencyLimiter(MAX_CONCURRENT_BROWSERS);
const realLimiter = new BrowserConcurrencyLimiter(MAX_CONCURRENT_BROWSERS_REAL);
/**
* @typedef {Object} AntiBotConfig
* @property {boolean} [enabled]
* @property {'standard'|'real'} [profile]
* @property {boolean} [turnstile]
* @property {boolean} [solverFallback]
* @property {string} [sessionKey]
* @property {number} [challengeTimeoutMs]
*/
/**
* 规范化 antiBot 配置（未传则走 standard）
* @param {AntiBotConfig|null|undefined} antiBot
* @returns {AntiBotConfig}
*/
function normalizeAntiBot(antiBot) {
if (!antiBot || !antiBot.enabled) {
return { enabled: false, profile: 'standard' };
@@ -43,62 +31,66 @@ function normalizeAntiBot(antiBot) {
};
}
/**
* 按 profile 获取限流器
* @param {'standard'|'real'} profile
*/
function getLimiter(profile) {
return profile === 'real' ? realLimiter : standardLimiter;
}
/**
* 创建 Browser 会话
* @param {{ profile?: 'standard'|'real', extraArgs?: string[] }} options
* @returns {Promise<{ browser: import('puppeteer').Browser, page: import('puppeteer').Page|null, profile: string, cleanup: () => Promise<void> }>}
* 冷启动 Browser（供温池 launcher 使用）
* @param {{ profile?: 'standard'|'real', extraArgs?: string[], sessionKey?: string }} options
*/
async function createBrowserSession(options = {}) {
const profile = options.profile === 'real' ? 'real' : 'standard';
const extraArgs = options.extraArgs || [];
const sessionKey = options.sessionKey || '';
if (profile === 'real') {
const { browser, page } = await launchRealBrowser(extraArgs);
const { browser, page } = await launchRealBrowser(extraArgs, { sessionKey, profile: 'real' });
return {
browser,
page,
profile: 'real',
cleanup: async () => {
try {
await browser.close();
} catch (_) {
/* ignore */
cleanup: async (destroy = false) => {
if (destroy || !sessionKey) {
try {
await browser.close();
} catch (_) { /* ignore */ }
}
},
};
}
const browser = await launchStandardBrowser(extraArgs);
const browser = await launchStandardBrowser(extraArgs, { sessionKey, profile: 'standard' });
return {
browser,
page: null,
profile: 'standard',
cleanup: async () => {
try {
await browser.close();
} catch (_) {
/* ignore */
cleanup: async (destroy = false) => {
if (destroy || !sessionKey) {
try {
await browser.close();
} catch (_) { /* ignore */ }
}
},
};
}
/**
* 在并发槽位内执行 Browser 任务
* @template T
* @param {string} taskName
* @param {'standard'|'real'} profile
* @param {() => Promise<T>} fn
* @returns {Promise<T>}
* 从温池或冷启动获取 Browser
* @param {{ profile?: 'standard'|'real', extraArgs?: string[], sessionKey?: string }} options
*/
async function acquireBrowserSession(options = {}) {
const profile = options.profile === 'real' ? 'real' : 'standard';
const sessionKey = options.sessionKey || '';
const extraArgs = options.extraArgs || [];
return browserPool.acquire(sessionKey, profile, async () => createBrowserSession({
profile,
extraArgs,
sessionKey,
}));
}
async function runWithProfileLimit(taskName, profile, fn) {
const limiter = getLimiter(profile);
try {
@@ -109,23 +101,15 @@ async function runWithProfileLimit(taskName, profile, fn) {
}
}
/**
* 创建 Page 并设置 UA / viewport / cookies
* @param {import('puppeteer').Browser} browser
* @param {import('puppeteer').Page|null} existingPage
* @param {{ userAgent?: string, viewport?: object, cookies?: Array<object> }} [options]
*/
async function createManagedPage(browser, existingPage, options = {}) {
const page = existingPage || await browser.newPage();
page.setDefaultNavigationTimeout(30000);
page.setDefaultTimeout(15000);
page.setDefaultNavigationTimeout(NAVIGATION_TIMEOUT_MS);
page.setDefaultTimeout(PAGE_DEFAULT_TIMEOUT_MS);
const userAgent = options.userAgent || DEFAULT_UA;
await page.setUserAgent(userAgent);
if (options.viewport) {
await page.setViewport(options.viewport);
}
if (options.viewport) await page.setViewport(options.viewport);
if (options.cookies && options.cookies.length > 0) {
await page.setCookie(...options.cookies);
}
@@ -136,14 +120,12 @@ async function createManagedPage(browser, existingPage, options = {}) {
return page;
}
/**
* @returns {{ standard: object, real: object, realBrowserReady: boolean }}
*/
function getFactoryStats() {
return {
standard: standardLimiter.getStats(),
real: realLimiter.getStats(),
realBrowserReady: isRealBrowserAvailable(),
pool: browserPool.getStats(),
};
}
@@ -151,10 +133,12 @@ module.exports = {
normalizeAntiBot,
getLimiter,
createBrowserSession,
acquireBrowserSession,
runWithProfileLimit,
createManagedPage,
getFactoryStats,
standardLimiter,
realLimiter,
browserPool,
QUEUE_TIMEOUT_MS,
};
patches/puppeteer-api/lib/browser-pool.js
+184
View File
@@ -0,0 +1,184 @@
/**
* Browser 温池：按 profile:sessionKey 复用 Browser/Page，降低冷启动与 CF 触发
*/
const { POOL_IDLE_MS, MAX_POOLED_BROWSERS, POOL_ENABLED } = require('./constants');
class BrowserPool {
constructor() {
/** @type {Map<string, { browser: any, page: any|null, profile: string, sessionKey: string, lastUsedAt: number, inUse: boolean }>} */
this.entries = new Map();
this.hits = 0;
this.misses = 0;
this.evictions = 0;
this._sweepTimer = setInterval(() => this.evictIdle(), 30000);
this._sweepTimer.unref?.();
}
/**
* @param {string} sessionKey
* @param {'standard'|'real'} profile
*/
buildKey(sessionKey, profile) {
return `${profile}:${sessionKey}`;
}
/**
* @param {any} browser
*/
async isBrowserAlive(browser) {
if (!browser) return false;
try {
if (typeof browser.isConnected === 'function' && !browser.isConnected()) {
return false;
}
await browser.version();
return true;
} catch (_) {
return false;
}
}
/**
* @param {string} key
*/
async destroyEntry(key) {
const entry = this.entries.get(key);
if (!entry) return;
this.entries.delete(key);
try {
await entry.browser.close();
} catch (_) {
/* ignore */
}
this.evictions++;
}
enforceCapacity(excludeKey) {
if (this.entries.size < MAX_POOLED_BROWSERS) return;
const candidates = [...this.entries.entries()]
.filter(([k, e]) => k !== excludeKey && !e.inUse)
.sort((a, b) => a[1].lastUsedAt - b[1].lastUsedAt);
if (candidates.length === 0) return;
const [oldestKey] = candidates[0];
this.destroyEntry(oldestKey);
}
/**
* @param {string} sessionKey
* @param {'standard'|'real'} profile
* @param {() => Promise<{ browser: any, page: any|null, cleanup: (destroy?: boolean) => Promise<void> }>} launcher
*/
async acquire(sessionKey, profile, launcher) {
if (!POOL_ENABLED || !sessionKey) {
const launched = await launcher();
return {
browser: launched.browser,
page: launched.page,
profile,
pooled: false,
reused: false,
release: async (destroy = false) => {
await launched.cleanup(destroy);
},
};
}
const key = this.buildKey(sessionKey, profile);
const existing = this.entries.get(key);
if (existing && !existing.inUse && await this.isBrowserAlive(existing.browser)) {
existing.inUse = true;
existing.lastUsedAt = Date.now();
this.hits++;
return {
browser: existing.browser,
page: existing.page,
profile,
pooled: true,
reused: true,
release: async (destroy = false) => this.release(key, destroy),
};
}
if (existing) {
await this.destroyEntry(key);
}
this.enforceCapacity(key);
this.misses++;
const launched = await launcher();
this.entries.set(key, {
browser: launched.browser,
page: launched.page,
profile,
sessionKey,
lastUsedAt: Date.now(),
inUse: true,
});
return {
browser: launched.browser,
page: launched.page,
profile,
pooled: true,
reused: false,
release: async (destroy = false) => this.release(key, destroy),
};
}
/**
* @param {string} key
* @param {boolean} destroy
*/
async release(key, destroy = false) {
const entry = this.entries.get(key);
if (!entry) return;
entry.inUse = false;
entry.lastUsedAt = Date.now();
if (destroy || !(await this.isBrowserAlive(entry.browser))) {
await this.destroyEntry(key);
}
}
evictIdle() {
const now = Date.now();
for (const [key, entry] of this.entries.entries()) {
if (!entry.inUse && now - entry.lastUsedAt > POOL_IDLE_MS) {
this.destroyEntry(key);
}
}
}
async drain() {
clearInterval(this._sweepTimer);
const keys = [...this.entries.keys()];
for (const key of keys) {
await this.destroyEntry(key);
}
}
getStats() {
let idle = 0;
let active = 0;
for (const entry of this.entries.values()) {
if (entry.inUse) active++;
else idle++;
}
return {
enabled: POOL_ENABLED,
max: MAX_POOLED_BROWSERS,
idleMs: POOL_IDLE_MS,
size: this.entries.size,
idle,
active,
hits: this.hits,
misses: this.misses,
evictions: this.evictions,
};
}
}
const browserPool = new BrowserPool();
module.exports = {
BrowserPool,
browserPool,
};
patches/puppeteer-api/lib/cf-handler.js
+42 -22
View File
@@ -1,53 +1,75 @@
/**
* Cloudflare 挑战统一处理：检测 → 内置等待 → Captcha API 兜底
* Cloudflare 挑战统一处理：检测 → 会话快速通道 → 内置等待 → Captcha API 兜底
*/
const { detectCloudflare } = require('./cf-detector');
const { waitForTurnstile, extractTurnstileSitekey, injectTurnstileToken } = require('./turnstile-handler');
const { solveTurnstile, isCaptchaConfigured } = require('./captcha-solver');
const { isSessionLikelyValid } = require('./session-store');
/**
* @typedef {Object} CfHandleResult
* @property {boolean} success
* @property {string|null} code
* @property {string|null} challengeType
* @property {string|null} stage
* @property {boolean} solverUsed
* @property {number} elapsedMs
* @property {boolean} cfDetected
*/
/**
* 处理 Cloudflare / Turnstile 挑战（在 authActions 之前调用）
* @param {import('puppeteer').Page} page
* @param {{ turnstile?: boolean, solverFallback?: boolean, challengeTimeoutMs?: number }} antiBot
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
* @returns {Promise<CfHandleResult>}
* @param {import('./session-store').StoredSession|null} [storedSession]
*/
async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled) {
async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, storedSession = null) {
const started = Date.now();
const timeoutMs = antiBot.challengeTimeoutMs || 60000;
let cfState = await detectCloudflare(page);
if (!cfState.blocked && cfState.hasCfClearance) {
return {
success: true,
code: null,
challengeType: null,
stage: 'session_reuse',
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: false,
};
}
if (!cfState.blocked) {
return {
success: true,
code: null,
challengeType: null,
stage: null,
stage: storedSession && isSessionLikelyValid(storedSession) ? 'session_reuse' : null,
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: false,
};
}
if (storedSession && isSessionLikelyValid(storedSession)) {
console.log('[CF] 会话有效但仍被拦截，尝试 reload 一次');
try {
await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 45000) });
if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {});
cfState = await detectCloudflare(page);
if (!cfState.blocked) {
return {
success: true,
code: null,
challengeType: cfState.challengeType,
stage: 'session_reload',
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: true,
};
}
} catch (reloadErr) {
console.warn('[CF] reload 失败:', reloadErr.message);
}
}
console.log(`[CF] 检测到挑战 type=${cfState.challengeType} url=${cfState.url}`);
if (antiBot.turnstile !== false) {
const builtIn = await waitForTurnstile(page, { timeoutMs, useBuiltInClick: true });
if (builtIn.success) {
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {});
cfState = await detectCloudflare(page);
if (!cfState.blocked) {
return {
@@ -74,9 +96,7 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled) {
const apiWaitMs = Math.min(timeoutMs, 30000);
await waitForTurnstile(page, { timeoutMs: apiWaitMs });
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {});
cfState = await detectCloudflare(page);
if (!cfState.blocked) {
patches/puppeteer-api/lib/constants.js
+40 -28
View File
@@ -5,13 +5,9 @@ const fs = require('fs');
const path = require('path');
const puppeteer = require('puppeteer-extra');
/** 默认 Chrome 可执行文件路径（生产环境可通过环境变量覆盖） */
const DEFAULT_CHROME_EXECUTABLE = '/www/wwwroot/puppeteer-api/.cache/puppeteer/chrome/linux-149.0.7827.22/chrome-linux64/chrome';
/** 默认 User-Agent，与 standard / real 双轨保持一致 */
const DEFAULT_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
/** Chrome 启动基础参数：多实例 headless 场景下复用 */
const BASE_LAUNCH_ARGS = [
'--no-sandbox',
'--disable-setuid-sandbox',
@@ -26,24 +22,14 @@ const BASE_LAUNCH_ARGS = [
'--disable-crash-reporter',
];
/** puppeteer-api 专用运行时目录（与 server.js 同级 .runtime） */
const RUNTIME_DIR = path.join(__dirname, '..', '.runtime');
/**
* 确保 .runtime 子目录存在且 www 用户可写
* @param {string} name
* @returns {string}
*/
function ensureRuntimeSubdir(name) {
const dir = path.join(RUNTIME_DIR, name);
fs.mkdirSync(dir, { recursive: true, mode: 0o777 });
return dir;
}
/**
* 解析 Chrome 可执行文件路径
* @returns {string}
*/
function resolveChromeExecutable() {
const candidates = [
process.env.PUPPETEER_EXECUTABLE_PATH,
@@ -63,16 +49,12 @@ function resolveChromeExecutable() {
return bundled;
}
} catch (_) {
/* puppeteer 未内置浏览器时忽略 */
/* ignore */
}
return candidates[0] || DEFAULT_CHROME_EXECUTABLE;
}
/**
* 生产 Linux 常注入无效 DBUS 地址；删除而非设为 /dev/null
* @returns {NodeJS.ProcessEnv}
*/
function buildBrowserLaunchEnv() {
const env = { ...process.env };
delete env.DBUS_SESSION_BUS_ADDRESS;
@@ -87,10 +69,6 @@ function buildBrowserLaunchEnv() {
return env;
}
/**
* 校验 Chrome 可执行文件存在
* @returns {string}
*/
function assertChromeExecutable() {
const executablePath = resolveChromeExecutable();
if (!fs.existsSync(executablePath)) {
@@ -101,23 +79,48 @@ function assertChromeExecutable() {
return executablePath;
}
/** 并发与超时配置（可通过环境变量覆盖） */
const MAX_CONCURRENT_BROWSERS = Math.max(1, parseInt(process.env.MAX_CONCURRENT_BROWSERS || '4', 10));
const MAX_CONCURRENT_BROWSERS_REAL = Math.max(1, parseInt(process.env.MAX_CONCURRENT_BROWSERS_REAL || '2', 10));
const QUEUE_TIMEOUT_MS = Math.max(5000, parseInt(process.env.QUEUE_TIMEOUT_MS || '120000', 10));
const API_INTERCEPT_TIMEOUT_MS = Math.max(10000, parseInt(process.env.API_INTERCEPT_TIMEOUT_MS || '30000', 10));
const API_INTERCEPT_TIMEOUT_MS = Math.max(10000, parseInt(process.env.API_INTERCEPT_TIMEOUT_MS || '60000', 10));
const API_INTERCEPT_TIMEOUT_REAL_MS = Math.max(
API_INTERCEPT_TIMEOUT_MS,
parseInt(process.env.API_INTERCEPT_TIMEOUT_REAL_MS || '90000', 10)
);
/** Captcha API 配置 */
const NAVIGATION_TIMEOUT_MS = Math.max(15000, parseInt(process.env.NAVIGATION_TIMEOUT_MS || '60000', 10));
/** rebrowser-puppeteer-core / puppeteer-core 仅支持 load|domcontentloaded|networkidle0|networkidle2，不支持 commit */
const ALLOWED_NAVIGATION_WAIT_UNTIL = ['load', 'domcontentloaded', 'networkidle0', 'networkidle2'];
function resolveNavigationWaitUntil() {
const raw = (process.env.NAVIGATION_WAIT_UNTIL || 'domcontentloaded').trim();
if (ALLOWED_NAVIGATION_WAIT_UNTIL.includes(raw)) {
return raw;
}
if (raw === 'commit') {
console.warn('[constants] NAVIGATION_WAIT_UNTIL=commit 不被当前 Puppeteer 支持，已降级为 domcontentloaded');
return 'domcontentloaded';
}
console.warn(`[constants] 未知 NAVIGATION_WAIT_UNTIL=${raw}，已降级为 domcontentloaded`);
return 'domcontentloaded';
}
const NAVIGATION_WAIT_UNTIL = resolveNavigationWaitUntil();
const NAVIGATION_MAX_RETRIES = Math.max(0, parseInt(process.env.NAVIGATION_MAX_RETRIES || '1', 10));
const PAGE_DEFAULT_TIMEOUT_MS = Math.max(15000, parseInt(process.env.PAGE_DEFAULT_TIMEOUT_MS || '30000', 10));
const CAPTCHA_PROVIDER = (process.env.CAPTCHA_PROVIDER || 'capsolver').toLowerCase();
const CAPTCHA_API_KEY = process.env.CAPTCHA_API_KEY || '';
const CAPTCHA_MAX_WAIT_MS = Math.max(30000, parseInt(process.env.CAPTCHA_MAX_WAIT_MS || '120000', 10));
/** 会话 Cookie 持久化 TTL（毫秒） */
const SESSION_TTL_MS = Math.max(60000, parseInt(process.env.SESSION_TTL_MS || '1800000', 10));
const SESSION_TTL_MS = Math.max(60000, parseInt(process.env.SESSION_TTL_MS || '7200000', 10));
const PERSIST_BROWSER_PROFILE = process.env.PERSIST_BROWSER_PROFILE !== '0';
const PROFILE_MAX_AGE_MS = Math.max(0, parseInt(process.env.PROFILE_MAX_AGE_MS || String(7 * 24 * 3600 * 1000), 10));
const POOL_IDLE_MS = Math.max(60000, parseInt(process.env.POOL_IDLE_MS || '300000', 10));
const MAX_POOLED_BROWSERS = Math.max(0, parseInt(process.env.MAX_POOLED_BROWSERS || '4', 10));
const POOL_ENABLED = process.env.POOL_ENABLED !== '0' && MAX_POOLED_BROWSERS > 0;
module.exports = {
DEFAULT_CHROME_EXECUTABLE,
@@ -133,8 +136,17 @@ module.exports = {
QUEUE_TIMEOUT_MS,
API_INTERCEPT_TIMEOUT_MS,
API_INTERCEPT_TIMEOUT_REAL_MS,
NAVIGATION_TIMEOUT_MS,
NAVIGATION_WAIT_UNTIL,
NAVIGATION_MAX_RETRIES,
PAGE_DEFAULT_TIMEOUT_MS,
CAPTCHA_PROVIDER,
CAPTCHA_API_KEY,
CAPTCHA_MAX_WAIT_MS,
SESSION_TTL_MS,
PERSIST_BROWSER_PROFILE,
PROFILE_MAX_AGE_MS,
POOL_IDLE_MS,
MAX_POOLED_BROWSERS,
POOL_ENABLED,
};
patches/puppeteer-api/lib/launch-real-browser.js
+13 -17
View File
@@ -1,26 +1,17 @@
/**
* Real Browser 启动：puppeteer-real-browser（抗 Cloudflare Turnstile）
* 注意：不与 puppeteer-extra 混用在同一 Browser 实例
*/
const {
BASE_LAUNCH_ARGS,
resolveChromeExecutable,
} = require('./constants');
const { resolveProfileDir } = require('./profile-dir');
/** @type {boolean|null} */
let realBrowserModuleChecked = null;
/** @type {boolean} */
let realBrowserAvailable = false;
/**
* 检测 puppeteer-real-browser 是否已安装
* @returns {boolean}
*/
function isRealBrowserAvailable() {
if (realBrowserModuleChecked !== null) {
return realBrowserAvailable;
}
if (realBrowserModuleChecked !== null) return realBrowserAvailable;
try {
require.resolve('puppeteer-real-browser');
realBrowserAvailable = true;
@@ -32,11 +23,10 @@ function isRealBrowserAvailable() {
}
/**
* 启动 real profile Browser（headless:false + Xvfb + turnstile 内置点击）
* @param {string[]} [extraArgs]
* @returns {Promise<{ browser: import('puppeteer').Browser, page: import('puppeteer').Page }>}
* @param {{ sessionKey?: string, profile?: 'standard'|'real' }} [options]
*/
async function launchRealBrowser(extraArgs = []) {
async function launchRealBrowser(extraArgs = [], options = {}) {
if (!isRealBrowserAvailable()) {
throw new Error(
'puppeteer-real-browser 未安装。请在 puppeteer-api 目录执行: npm install puppeteer-real-browser@1.4.4'
@@ -45,22 +35,28 @@ async function launchRealBrowser(extraArgs = []) {
const { connect } = require('puppeteer-real-browser');
const chromePath = resolveChromeExecutable();
const profile = options.profile === 'real' ? 'real' : 'standard';
const profileDir = options.sessionKey ? resolveProfileDir(options.sessionKey, profile) : null;
const args = [...BASE_LAUNCH_ARGS, '--mute-audio', ...extraArgs];
if (profileDir) {
args.push(`--user-data-dir=${profileDir}`);
}
const result = await connect({
headless: false,
turnstile: true,
disableXvfb: false,
customConfig: { chromePath },
args: [...BASE_LAUNCH_ARGS, '--mute-audio', ...extraArgs],
args,
});
const browser = result.browser;
const page = result.page;
if (!browser || !page) {
throw new Error('puppeteer-real-browser connect 未返回 browser/page');
}
browser.__persistentProfile = !!profileDir;
browser.__userDataDir = profileDir || '';
return { browser, page };
}
patches/puppeteer-api/lib/launch-standard.js
+23 -18
View File
@@ -11,18 +11,21 @@ const {
buildBrowserLaunchEnv,
ensureRuntimeSubdir,
} = require('./constants');
const { resolveProfileDir } = require('./profile-dir');
// Stealth 必须在 launch 之前注册
puppeteer.use(StealthPlugin());
/**
* 启动 standard profile Browser
* @param {string[]} [extraArgs]
* @param {{ sessionKey?: string, profile?: 'standard'|'real' }} [options]
* @returns {Promise<import('puppeteer').Browser>}
*/
async function launchStandardBrowser(extraArgs = []) {
async function launchStandardBrowser(extraArgs = [], options = {}) {
const executablePath = assertChromeExecutable();
const userDataDir = fs.mkdtempSync(path.join(ensureRuntimeSubdir('chrome-profiles'), 'run-'));
const profile = options.profile === 'real' ? 'real' : 'standard';
const persistentDir = options.sessionKey ? resolveProfileDir(options.sessionKey, profile) : null;
const userDataDir = persistentDir || fs.mkdtempSync(path.join(ensureRuntimeSubdir('chrome-profiles'), 'run-'));
const persistent = !!persistentDir;
try {
const browser = await puppeteer.launch({
@@ -32,24 +35,26 @@ async function launchStandardBrowser(extraArgs = []) {
env: buildBrowserLaunchEnv(),
userDataDir,
});
const originalClose = browser.close.bind(browser);
browser.close = async () => {
try {
await originalClose();
} finally {
if (!persistent) {
const originalClose = browser.close.bind(browser);
browser.close = async () => {
try {
fs.rmSync(userDataDir, { recursive: true, force: true });
} catch (_) {
/* 清理失败不影响主流程 */
await originalClose();
} finally {
try {
fs.rmSync(userDataDir, { recursive: true, force: true });
} catch (_) { /* ignore */ }
}
}
};
};
}
browser.__userDataDir = userDataDir;
browser.__persistentProfile = persistent;
return browser;
} catch (err) {
try {
fs.rmSync(userDataDir, { recursive: true, force: true });
} catch (_) {
/* ignore */
if (!persistent) {
try {
fs.rmSync(userDataDir, { recursive: true, force: true });
} catch (_) { /* ignore */ }
}
throw err;
}
patches/puppeteer-api/lib/profile-dir.js
+77
View File
@@ -0,0 +1,77 @@
/**
* 按 sessionKey 解析持久 Chrome Profile 目录
*/
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const { RUNTIME_DIR, PERSIST_BROWSER_PROFILE, PROFILE_MAX_AGE_MS } = require('./constants');
const PROFILES_DIR = path.join(RUNTIME_DIR, 'profiles');
/**
* @param {string} sessionKey
* @param {'standard'|'real'} profile
* @returns {string|null}
*/
function resolveProfileDir(sessionKey, profile = 'standard') {
if (!sessionKey || !PERSIST_BROWSER_PROFILE) {
return null;
}
const hash = crypto.createHash('md5').update(String(sessionKey)).digest('hex');
const dir = path.join(PROFILES_DIR, `${hash}-${profile}`);
fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
return dir;
}
function profileExists(sessionKey, profile = 'standard') {
const dir = resolveProfileDir(sessionKey, profile);
if (!dir) return false;
return fs.existsSync(dir);
}
function pruneStaleProfiles() {
if (!PERSIST_BROWSER_PROFILE || PROFILE_MAX_AGE_MS <= 0) {
return 0;
}
if (!fs.existsSync(PROFILES_DIR)) {
return 0;
}
const now = Date.now();
let removed = 0;
for (const name of fs.readdirSync(PROFILES_DIR)) {
const full = path.join(PROFILES_DIR, name);
try {
const stat = fs.statSync(full);
if (!stat.isDirectory()) continue;
if (now - stat.mtimeMs > PROFILE_MAX_AGE_MS) {
fs.rmSync(full, { recursive: true, force: true });
removed++;
}
} catch (_) {
/* ignore */
}
}
return removed;
}
function getProfileStats() {
if (!fs.existsSync(PROFILES_DIR)) {
return { profilesDir: PROFILES_DIR, count: 0 };
}
const count = fs.readdirSync(PROFILES_DIR).filter((name) => {
try {
return fs.statSync(path.join(PROFILES_DIR, name)).isDirectory();
} catch (_) {
return false;
}
}).length;
return { profilesDir: PROFILES_DIR, count };
}
module.exports = {
PROFILES_DIR,
resolveProfileDir,
profileExists,
pruneStaleProfiles,
getProfileStats,
};
patches/puppeteer-api/lib/session-context.js
+38
View File
@@ -0,0 +1,38 @@
/**
* 会话上下文：合并请求 cookies 与磁盘 session，供各 API 统一使用
*/
const {
loadSession,
sessionCookiesForPage,
mergeCookies,
touchSession,
} = require('./session-store');
const { SESSION_TTL_MS } = require('./constants');
/**
* @param {{ sessionKey?: string }} antiBot
* @param {string} pageUrl
* @param {object[]} [requestCookies]
*/
function resolveSessionContext(antiBot, pageUrl, requestCookies = []) {
const sessionKey = antiBot?.sessionKey || '';
const storedSession = sessionKey ? loadSession(sessionKey) : null;
const sessionCookies = storedSession ? sessionCookiesForPage(storedSession, pageUrl) : [];
const mergedCookies = mergeCookies(requestCookies, sessionCookies);
return { sessionKey, storedSession, mergedCookies };
}
/**
* 成功过盾后滑动续期 session TTL
* @param {string} sessionKey
*/
function touchSessionIfPresent(sessionKey) {
if (sessionKey) {
touchSession(sessionKey, SESSION_TTL_MS);
}
}
module.exports = {
resolveSessionContext,
touchSessionIfPresent,
};
patches/puppeteer-api/lib/session-store.js
+83 -48
View File
@@ -1,5 +1,5 @@
/**
* 会话 Cookie 持久化：按 sessionKey 保存 cf_clearance 等，降低重复过盾概率
* 会话 Cookie 持久化：按 sessionKey 保存完整 cookie jar，降低重复过盾概率
*/
const fs = require('fs');
const path = require('path');
@@ -8,42 +8,50 @@ const { RUNTIME_DIR, SESSION_TTL_MS } = require('./constants');
const SESSIONS_DIR = path.join(RUNTIME_DIR, 'sessions');
/**
* 确保 sessions 目录存在
*/
function ensureSessionsDir() {
fs.mkdirSync(SESSIONS_DIR, { recursive: true, mode: 0o777 });
}
/**
* sessionKey 转安全文件名
* @param {string} sessionKey
* @returns {string}
*/
function keyToFilename(sessionKey) {
const hash = crypto.createHash('md5').update(sessionKey).digest('hex');
return `${hash}.json`;
}
/**
* @typedef {Object} StoredSession
* @property {string} sessionKey
* @property {number} savedAt
* @property {number} expiresAt
* @property {Array<{name:string,value:string,domain?:string,path?:string}>} cookies
* @param {{name:string,domain?:string,path?:string}} cookie
* @returns {string}
*/
function cookieIdentity(cookie) {
return `${cookie.name}|${cookie.domain || ''}|${cookie.path || '/'}`;
}
function mergeCookies(requestCookies, sessionCookies) {
const map = new Map();
for (const cookie of sessionCookies || []) {
if (cookie && cookie.name) map.set(cookieIdentity(cookie), cookie);
}
for (const cookie of requestCookies || []) {
if (cookie && cookie.name) map.set(cookieIdentity(cookie), cookie);
}
return Array.from(map.values());
}
function isSessionLikelyValid(session) {
if (!session || !Array.isArray(session.cookies)) return false;
if (session.expiresAt && Date.now() > session.expiresAt) return false;
return session.cookies.some((c) => c.name === 'cf_clearance' && c.value);
}
function hasCfClearance(session) {
if (!session || !Array.isArray(session.cookies)) return false;
return session.cookies.some((c) => c.name === 'cf_clearance' && c.value);
}
/**
* 加载已保存的会话 cookies
* @param {string} sessionKey
* @returns {StoredSession|null}
*/
function loadSession(sessionKey) {
if (!sessionKey) return null;
ensureSessionsDir();
const filePath = path.join(SESSIONS_DIR, keyToFilename(sessionKey));
if (!fs.existsSync(filePath)) return null;
try {
const raw = JSON.parse(fs.readFileSync(filePath, 'utf8'));
if (!raw || !Array.isArray(raw.cookies)) return null;
@@ -57,45 +65,36 @@ function loadSession(sessionKey) {
}
}
/**
* 保存会话 cookies（优先保留 CF 相关 cookie）
* @param {string} sessionKey
* @param {Array<{name:string,value:string,domain?:string,path?:string}>} cookies
* @param {number} [ttlMs]
*/
function saveSession(sessionKey, cookies, ttlMs = SESSION_TTL_MS) {
function touchSession(sessionKey, ttlMs = SESSION_TTL_MS) {
const session = loadSession(sessionKey);
if (!session) return;
const now = Date.now();
session.savedAt = now;
session.expiresAt = now + ttlMs;
fs.writeFileSync(path.join(SESSIONS_DIR, keyToFilename(sessionKey)), JSON.stringify(session, null, 0), { mode: 0o666 });
}
function saveSession(sessionKey, cookies, ttlMs = SESSION_TTL_MS, meta = {}) {
if (!sessionKey || !Array.isArray(cookies) || cookies.length === 0) return;
ensureSessionsDir();
const cfRelated = cookies.filter((c) =>
['cf_clearance', '__cf_bm', 'cf_chl_2'].includes(c.name)
|| c.name.startsWith('__cf')
);
const toSave = cfRelated.length > 0 ? cfRelated : cookies;
const now = Date.now();
const payload = {
sessionKey,
savedAt: now,
lastSuccessAt: now,
expiresAt: now + ttlMs,
cookies: toSave.map((c) => ({
lastHost: meta.lastHost || '',
savedProfile: meta.savedProfile || '',
cookies: cookies.map((c) => ({
name: c.name,
value: c.value,
domain: c.domain,
path: c.path || '/',
})),
};
const filePath = path.join(SESSIONS_DIR, keyToFilename(sessionKey));
fs.writeFileSync(filePath, JSON.stringify(payload, null, 0), { mode: 0o666 });
fs.writeFileSync(path.join(SESSIONS_DIR, keyToFilename(sessionKey)), JSON.stringify(payload, null, 0), { mode: 0o666 });
}
/**
* 将会话 cookies 转为 puppeteer setCookie 格式
* @param {StoredSession|null} session
* @param {string} pageUrl
* @returns {Array<{name:string,value:string,domain?:string,path?:string,url?:string}>}
*/
function sessionCookiesForPage(session, pageUrl) {
if (!session || !Array.isArray(session.cookies)) return [];
let hostname = '';
@@ -104,18 +103,54 @@ function sessionCookiesForPage(session, pageUrl) {
} catch (_) {
return session.cookies;
}
return session.cookies.map((c) => {
const cookie = { ...c };
if (!cookie.domain && hostname) {
cookie.url = `https://${hostname}/`;
}
if (!cookie.domain && hostname) cookie.url = `https://${hostname}/`;
return cookie;
});
}
function getSessionDebugInfo(sessionKey) {
if (!sessionKey) return null;
const filePath = path.join(SESSIONS_DIR, keyToFilename(sessionKey));
const session = loadSession(sessionKey);
if (!session) {
return { sessionKey, exists: false, filePath };
}
let mtime = null;
try {
mtime = fs.statSync(filePath).mtime.toISOString();
} catch (_) { /* ignore */ }
return {
sessionKey,
exists: true,
filePath,
mtime,
savedAt: session.savedAt,
expiresAt: session.expiresAt,
lastSuccessAt: session.lastSuccessAt,
lastHost: session.lastHost,
savedProfile: session.savedProfile,
cookieCount: session.cookies.length,
hasCfClearance: hasCfClearance(session),
likelyValid: isSessionLikelyValid(session),
};
}
function getSessionStats() {
ensureSessionsDir();
const count = fs.readdirSync(SESSIONS_DIR).filter((f) => f.endsWith('.json')).length;
return { sessionsDir: SESSIONS_DIR, count };
}
module.exports = {
loadSession,
saveSession,
touchSession,
sessionCookiesForPage,
mergeCookies,
isSessionLikelyValid,
hasCfClearance,
getSessionDebugInfo,
getSessionStats,
};
patches/puppeteer-api/lib/ui-pagination-runner.js
+172
View File
@@ -0,0 +1,172 @@
/**
* UI 翻页抓取：供 ui-pagination 与 auth-intercept-and-paginate 共用
*/
const crypto = require('crypto');
function generateSafeHash(obj) {
if (!obj) return '';
const clone = JSON.parse(JSON.stringify(obj));
const removeDynamicKeys = (target) => {
if (typeof target !== 'object' || target === null) return;
['timestamp', 'time', 'serverTime', 'reqId', 'requestId', 'traceId'].forEach((k) => delete target[k]);
Object.values(target).forEach((val) => removeDynamicKeys(val));
};
removeDynamicKeys(clone);
return crypto.createHash('md5').update(JSON.stringify(clone)).digest('hex');
}
function classifyPuppeteerError(err) {
const msg = (err?.message || '').toLowerCase();
if (err?.name === 'TimeoutError' || msg.includes('timeout')) return 'timeout';
if (msg.includes('net::') || msg.includes('network') || msg.includes('err_connection')
|| msg.includes('err_address_unreachable') || msg.includes('err_name_not_resolved')) return 'network';
if (msg.includes('navigation') || msg.includes('frame was detached')) return 'navigation';
if (msg.includes('target closed') || msg.includes('session closed')) return 'target_closed';
return 'unknown';
}
/**
* @param {import('puppeteer').Page} page
* @param {{
* apiUrl: string,
* nextBtnSelector: string,
* clicksToPerform: number,
* waitMs?: number,
* firstPageData?: object,
* authActions?: object[],
* executeAuthActions?: (page: import('puppeteer').Page, actions: object[]) => Promise<void>,
* }} options
*/
async function runUiPagination(page, options) {
const {
apiUrl,
nextBtnSelector,
clicksToPerform,
waitMs = 2000,
firstPageData,
authActions,
executeAuthActions,
} = options;
const capturedData = [];
const debugLogs = [];
const log = (msg) => { console.log(msg); debugLogs.push(msg); };
const seenHashes = new Set();
if (firstPageData) {
seenHashes.add(generateSafeHash(firstPageData));
}
if (Array.isArray(authActions) && authActions.length > 0 && executeAuthActions) {
log('[状态同步] 正在为翻页引擎注入授权状态...');
await executeAuthActions(page, authActions);
log('[状态同步] 授权执行完毕，等待目标列表加载...');
const authSettled = await page.waitForResponse(
(resp) => resp.url().includes(apiUrl) && resp.request().method() !== 'OPTIONS' && resp.status() >= 200 && resp.status() < 400,
{ timeout: 5000 }
).catch(() => null);
if (!authSettled) {
log('[状态同步] 未捕获到目标 API 响应，降级为固定等待 3 秒...');
await new Promise((r) => setTimeout(r, 3000));
}
}
for (let i = 0; i < clicksToPerform; i++) {
const targetPageNum = i + 2;
let pageData = null;
let attempts = 0;
const maxAttempts = 3;
await page.evaluate(() => {
document.querySelectorAll('.el-loading-mask, .ant-spin-nested-loading, .el-loading-spinner').forEach((el) => el.remove());
});
await page.waitForFunction(
() => !document.querySelector('.el-loading-mask, .ant-spin-nested-loading, .el-loading-spinner'),
{ timeout: 1000, polling: 100 }
).catch(() => new Promise((r) => setTimeout(r, 500)));
while (attempts < maxAttempts && !pageData) {
attempts++;
log(`[抓取] 准备抓取第 ${targetPageNum} 页 (尝试 ${attempts}/${maxAttempts})...`);
let responseHandler = null;
let apiTimeoutId = null;
const waitForNewPageApi = new Promise((resolve) => {
const cleanupListener = () => {
if (apiTimeoutId) { clearTimeout(apiTimeoutId); apiTimeoutId = null; }
if (responseHandler) { page.off('response', responseHandler); responseHandler = null; }
};
responseHandler = async (response) => {
const status = response.status();
if (response.url().includes(apiUrl) && response.request().method() !== 'OPTIONS' && status >= 200 && status < 400) {
try {
const responseData = await response.json();
const currentHash = generateSafeHash(responseData);
if (!seenHashes.has(currentHash)) {
seenHashes.add(currentHash);
cleanupListener();
resolve(responseData);
} else {
log('[防轮询] 抛弃重复数据，继续监听...');
}
} catch (_) { /* 非 JSON 响应跳过 */ }
}
};
page.on('response', responseHandler);
apiTimeoutId = setTimeout(() => { cleanupListener(); resolve(null); }, 15000);
});
try {
await page.waitForSelector(nextBtnSelector, { timeout: 15000 });
const clickStatus = await page.evaluate((sel) => {
const btn = document.querySelector(sel);
if (!btn) return 'not_found';
if (btn.disabled || btn.classList.contains('is-disabled') || btn.classList.contains('disabled') || btn.getAttribute('aria-disabled') === 'true') {
return 'disabled';
}
const className = btn.className || '';
if (btn.disabled === true || className.includes('disabled') || btn.getAttribute('aria-disabled') === 'true') {
return 'disabled';
}
btn.scrollIntoView({ behavior: 'instant', block: 'center' });
btn.click();
btn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
return 'success';
}, nextBtnSelector);
if (clickStatus === 'disabled') {
log(`[提示] 第 ${targetPageNum} 页按钮已被禁用，说明到底了，抓取提前结束。`);
break;
}
log('[动作] 双引擎原生点击已执行！');
} catch (clickErr) {
log(`[错误] 寻找按钮异常 (${classifyPuppeteerError(clickErr)}): ${clickErr.message}`);
}
pageData = await waitForNewPageApi;
if (!pageData && attempts < maxAttempts) {
log('[重试警报] 雷达未扫描到新数据，准备重试...');
await new Promise((r) => setTimeout(r, 500 * Math.pow(2, attempts - 1)));
}
}
if (pageData) {
capturedData.push(pageData);
log(`[成功] 斩获第 ${targetPageNum} 页数据！`);
await new Promise((r) => setTimeout(r, waitMs));
} else {
log(`[致命异常] 连续 ${maxAttempts} 次尝试均失败，放弃后续翻页。`);
break;
}
}
return { data: capturedData, debugLogs };
}
module.exports = {
runUiPagination,
generateSafeHash,
};