damon/links
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

A2C工单修复

Browse Source
This commit is contained in:
root
2026-07-03 20:12:20 +08:00
parent c3223f026e
commit 0f3eb53afe
21 changed files with 1561 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files
patches/puppeteer-api/lib/browser-factory.js
+10 -8
View File
@@ -59,11 +59,12 @@ async function createBrowserSession(options = {}) {
page,
profile: 'real',
cleanup: async (destroy = false) => {
if (destroy || !sessionKey) {
try {
await browser.close();
} catch (_) { /* ignore */ }
if (!destroy) {
return;
}
try {
await browser.close();
} catch (_) { /* ignore */ }
},
};
}
@@ -74,11 +75,12 @@ async function createBrowserSession(options = {}) {
page: null,
profile: 'standard',
cleanup: async (destroy = false) => {
if (destroy || !sessionKey) {
try {
await browser.close();
} catch (_) { /* ignore */ }
if (!destroy) {
return;
}
try {
await browser.close();
} catch (_) { /* ignore */ }
},
};
}
patches/puppeteer-api/lib/browser-pool.js
+89 -41
View File
@@ -1,12 +1,16 @@
/**
* Browser 温池：按 profile:sessionKey 复用 Browser/Page，降低冷启动与 CF 触发
* Browser 温池：按 profile:sessionKey 复用 Browser，降低冷启动与 CF 触发
*
* 同一 key 的 acquire 在 inUse 时排队等待，禁止销毁正在使用的 Browser（修复 A2C 并发误杀）。
*/
const { POOL_IDLE_MS, MAX_POOLED_BROWSERS, POOL_ENABLED } = require('./constants');
const { POOL_IDLE_MS, MAX_POOLED_BROWSERS, POOL_ENABLED, QUEUE_TIMEOUT_MS } = require('./constants');
class BrowserPool {
constructor() {
/** @type {Map<string, { browser: any, page: any|null, profile: string, sessionKey: string, lastUsedAt: number, inUse: boolean }>} */
this.entries = new Map();
/** @type {Map<string, Array<{ resolve: () => void, reject: (err: Error) => void, timer: NodeJS.Timeout|null }>>} */
this._waitQueues = new Map();
this.hits = 0;
this.misses = 0;
this.evictions = 0;
@@ -14,17 +18,10 @@ class BrowserPool {
this._sweepTimer.unref?.();
}
/**
* @param {string} sessionKey
* @param {'standard'|'real'} profile
*/
buildKey(sessionKey, profile) {
return `${profile}:${sessionKey}`;
}
/**
* @param {any} browser
*/
async isBrowserAlive(browser) {
if (!browser) return false;
try {
@@ -38,12 +35,13 @@ class BrowserPool {
}
}
/**
* @param {string} key
*/
async destroyEntry(key) {
const entry = this.entries.get(key);
if (!entry) return;
if (entry.inUse) {
console.warn(`[BrowserPool] 跳过销毁 inUse 条目 key=${key}`);
return;
}
this.entries.delete(key);
try {
await entry.browser.close();
@@ -63,11 +61,43 @@ class BrowserPool {
this.destroyEntry(oldestKey);
}
/**
* @param {string} sessionKey
* @param {'standard'|'real'} profile
* @param {() => Promise<{ browser: any, page: any|null, cleanup: (destroy?: boolean) => Promise<void> }>} launcher
*/
waitForRelease(key, timeoutMs = QUEUE_TIMEOUT_MS) {
const entry = this.entries.get(key);
if (!entry || !entry.inUse) {
return Promise.resolve();
}
return new Promise((resolve, reject) => {
const waiter = {
resolve: () => {
if (waiter.timer) clearTimeout(waiter.timer);
resolve();
},
reject,
timer: timeoutMs > 0
? setTimeout(() => {
const queue = this._waitQueues.get(key) || [];
const idx = queue.indexOf(waiter);
if (idx >= 0) queue.splice(idx, 1);
reject(new Error('POOL_ACQUIRE_TIMEOUT'));
}, timeoutMs)
: null,
};
if (!this._waitQueues.has(key)) {
this._waitQueues.set(key, []);
}
this._waitQueues.get(key).push(waiter);
console.log(`[BrowserPool] key=${key} 正在使用，排队等待 (队列 ${this._waitQueues.get(key).length})`);
});
}
_wakeNextWaiter(key) {
const queue = this._waitQueues.get(key);
if (!queue || queue.length === 0) return;
const next = queue.shift();
if (next?.timer) clearTimeout(next.timer);
next?.resolve();
}
async acquire(sessionKey, profile, launcher) {
if (!POOL_ENABLED || !sessionKey) {
const launched = await launcher();
@@ -77,29 +107,38 @@ class BrowserPool {
profile,
pooled: false,
reused: false,
release: async (destroy = false) => {
await launched.cleanup(destroy);
release: async () => {
await launched.cleanup(true);
},
};
}
const key = this.buildKey(sessionKey, profile);
const existing = this.entries.get(key);
if (existing && !existing.inUse && await this.isBrowserAlive(existing.browser)) {
existing.inUse = true;
existing.lastUsedAt = Date.now();
this.hits++;
return {
browser: existing.browser,
page: existing.page,
profile,
pooled: true,
reused: true,
release: async (destroy = false) => this.release(key, destroy),
};
}
if (existing) {
while (true) {
const existing = this.entries.get(key);
if (!existing) {
break;
}
if (existing.inUse) {
await this.waitForRelease(key);
continue;
}
if (await this.isBrowserAlive(existing.browser)) {
existing.inUse = true;
existing.lastUsedAt = Date.now();
this.hits++;
return {
browser: existing.browser,
page: null,
profile,
pooled: true,
reused: true,
release: async (destroy = false) => this.release(key, destroy),
};
}
await this.destroyEntry(key);
break;
}
this.enforceCapacity(key);
@@ -107,7 +146,7 @@ class BrowserPool {
const launched = await launcher();
this.entries.set(key, {
browser: launched.browser,
page: launched.page,
page: null,
profile,
sessionKey,
lastUsedAt: Date.now(),
@@ -124,18 +163,19 @@ class BrowserPool {
};
}
/**
* @param {string} key
* @param {boolean} destroy
*/
async release(key, destroy = false) {
const entry = this.entries.get(key);
if (!entry) return;
if (!entry) {
this._wakeNextWaiter(key);
return;
}
entry.inUse = false;
entry.lastUsedAt = Date.now();
entry.page = null;
if (destroy || !(await this.isBrowserAlive(entry.browser))) {
await this.destroyEntry(key);
}
this._wakeNextWaiter(key);
}
evictIdle() {
@@ -151,17 +191,24 @@ class BrowserPool {
clearInterval(this._sweepTimer);
const keys = [...this.entries.keys()];
for (const key of keys) {
await this.destroyEntry(key);
const entry = this.entries.get(key);
if (entry && !entry.inUse) {
await this.destroyEntry(key);
}
}
}
getStats() {
let idle = 0;
let active = 0;
let waiting = 0;
for (const entry of this.entries.values()) {
if (entry.inUse) active++;
else idle++;
}
for (const queue of this._waitQueues.values()) {
waiting += queue.length;
}
return {
enabled: POOL_ENABLED,
max: MAX_POOLED_BROWSERS,
@@ -169,6 +216,7 @@ class BrowserPool {
size: this.entries.size,
idle,
active,
waiting,
hits: this.hits,
misses: this.misses,
evictions: this.evictions,
patches/puppeteer-api/lib/captcha-proxy.js
+118
View File
@@ -0,0 +1,118 @@
/**
* CAPTCHA_PROXY 解析与 Chrome 代理注入（与 CapSolver AntiCloudflareTask 使用同一 IP）
*
* 仅在实际检测到 CF 挑战、需要 Managed 过盾时由 server.js 显式启用（useCaptchaProxy=true）。
* 格式：ip:port:username:password
*/
const CAPTCHA_PROXY_RAW = (process.env.CAPTCHA_PROXY || '').trim();
/**
* @returns {{
* host: string,
* port: string,
* username: string,
* password: string,
* chromeProxyServer: string,
* capsolverString: string,
* }|null}
*/
function parseCaptchaProxy(raw = CAPTCHA_PROXY_RAW) {
const value = (raw || '').trim();
if (value === '') {
return null;
}
const parts = value.split(':');
if (parts.length < 4) {
return null;
}
const host = parts[0];
const port = parts[1];
const username = parts[2];
const password = parts.slice(3).join(':');
if (!host || !port || !username || !password) {
return null;
}
return {
host,
port,
username,
password,
chromeProxyServer: `http://${host}:${port}`,
capsolverString: value,
};
}
/** @returns {boolean} */
function hasCaptchaProxy() {
return parseCaptchaProxy() !== null;
}
/**
* 是否属于可能走 CapSolver Managed 的工单（A2C 等）
* 注意：仅表示候选，不代表本次任务一定挂代理
* @param {object|null|undefined} antiBot
*/
function isManagedCfCandidate(antiBot) {
if (!antiBot?.enabled) {
return false;
}
if (antiBot.cfChallengeMode === 'managed') {
return true;
}
if (antiBot.cfCloudflareSolver === true) {
return true;
}
return false;
}
/**
* @deprecated 请使用 isManagedCfCandidate + 显式 useCaptchaProxy 参数
*/
function shouldUseCaptchaProxyForAntiBot(antiBot) {
return false;
}
/**
* Chrome 启动参数 --proxy-server（须 useCaptchaProxy=true 时调用）
* @returns {string[]}
*/
function getCaptchaProxyChromeArgs() {
const parsed = parseCaptchaProxy();
if (!parsed) {
return [];
}
return [
`--proxy-server=${parsed.chromeProxyServer}`,
'--proxy-bypass-list=<-loopback>,localhost,127.0.0.1',
];
}
/**
* 为 Page 设置代理认证（须配合 --proxy-server）
* @param {import('puppeteer').Page} page
* @returns {Promise<boolean>}
*/
async function applyCaptchaProxyToPage(page) {
const parsed = parseCaptchaProxy();
if (!parsed || !page) {
return false;
}
await page.authenticate({
username: parsed.username,
password: parsed.password,
});
return true;
}
module.exports = {
parseCaptchaProxy,
hasCaptchaProxy,
isManagedCfCandidate,
shouldUseCaptchaProxyForAntiBot,
getCaptchaProxyChromeArgs,
applyCaptchaProxyToPage,
};
patches/puppeteer-api/lib/captcha-solver.js
+150 -2
View File
@@ -6,6 +6,10 @@ const {
CAPTCHA_API_KEY,
CAPTCHA_MAX_WAIT_MS,
} = require('./constants');
const { sanitizePageUrlForCfSolver } = require('./cf-detector');
/** AntiCloudflareTask 所需静态代理（ip:port:user:pass），未配置则不可用 */
const CAPTCHA_PROXY = (process.env.CAPTCHA_PROXY || '').trim();
/**
* @param {string} url
@@ -113,6 +117,147 @@ async function solveVia2Captcha({ pageUrl, sitekey, apiKey }) {
throw new Error('2Captcha 求解超时');
}
/**
* CapSolver Managed Challenge（返回 cf_clearance cookie，需 CAPTCHA_PROXY）
* @param {{ pageUrl: string, apiKey: string, proxy: string, html?: string, userAgent?: string }} params
* @returns {Promise<{ cookies: Record<string, string>, userAgent?: string }>}
*/
async function solveViaCapSolverCloudflare({ pageUrl, apiKey, proxy, html, userAgent }) {
const task = {
type: 'AntiCloudflareTask',
websiteURL: pageUrl,
proxy,
};
if (html) {
task.html = html.slice(0, 500000);
}
if (userAgent) {
task.userAgent = userAgent;
}
const create = await httpJsonPost('https://api.capsolver.com/createTask', {
clientKey: apiKey,
task,
});
if (create.errorId !== 0 || !create.taskId) {
throw new Error(`CapSolver AntiCloudflare createTask 失败: ${create.errorDescription || create.errorCode || 'unknown'}`);
}
const deadline = Date.now() + CAPTCHA_MAX_WAIT_MS;
while (Date.now() < deadline) {
await new Promise((r) => setTimeout(r, 3000));
const result = await httpJsonPost('https://api.capsolver.com/getTaskResult', {
clientKey: apiKey,
taskId: create.taskId,
});
if (result.status === 'ready' && result.solution) {
const cookies = result.solution.cookies || {};
if (typeof cookies === 'string') {
return { cookies: { cf_clearance: cookies }, userAgent: result.solution.userAgent };
}
if (result.solution.token && !cookies.cf_clearance) {
return {
cookies: { cf_clearance: result.solution.token },
userAgent: result.solution.userAgent,
};
}
return {
cookies,
userAgent: result.solution.userAgent,
};
}
if (result.errorId !== 0) {
throw new Error(`CapSolver AntiCloudflare getTaskResult 失败: ${result.errorDescription || result.errorCode}`);
}
}
throw new Error('CapSolver AntiCloudflare 求解超时');
}
/**
* 统一 Cloudflare Managed Challenge 求解
* @param {{ pageUrl: string, provider?: string, apiKey?: string, proxy?: string, html?: string, userAgent?: string }} params
* @returns {Promise<{ cookies: Record<string, string>, userAgent?: string, provider: string }>}
*/
async function solveCloudflare({ pageUrl, provider, apiKey, proxy, html, userAgent }) {
const resolvedProvider = (provider || CAPTCHA_PROVIDER).toLowerCase();
const resolvedKey = apiKey || CAPTCHA_API_KEY;
const resolvedProxy = (proxy || CAPTCHA_PROXY).trim();
const cleanPageUrl = sanitizePageUrlForCfSolver(pageUrl);
if (!resolvedKey) {
throw new Error('未配置 CAPTCHA_API_KEY，无法使用 AntiCloudflareTask');
}
if (!resolvedProxy) {
throw new Error('AntiCloudflareTask 需要配置 CAPTCHA_PROXY（静态代理 ip:port:user:pass）');
}
if (resolvedProvider !== 'capsolver') {
throw new Error('AntiCloudflareTask 当前仅支持 CapSolver');
}
const { cookies, userAgent: solvedUa } = await solveViaCapSolverCloudflare({
pageUrl: cleanPageUrl,
apiKey: resolvedKey,
proxy: resolvedProxy,
html,
userAgent,
});
if (!cookies || !cookies.cf_clearance) {
throw new Error('CapSolver AntiCloudflare 未返回 cf_clearance');
}
return { cookies, userAgent: solvedUa || userAgent, provider: resolvedProvider };
}
/**
* 将 CapSolver AntiCloudflare 返回的 cookie / UA 写入页面并 reload
* @param {import('puppeteer').Page} page
* @param {string} cleanUrl
* @param {{ cookies: Record<string, string>, userAgent?: string }} solution
*/
async function applyCfClearanceSolution(page, cleanUrl, solution) {
if (solution.userAgent) {
await page.setUserAgent(solution.userAgent).catch(() => {});
}
let host = '';
let path = '/';
try {
const parsed = new URL(cleanUrl);
host = parsed.hostname;
path = parsed.pathname || '/';
} catch (_) {
host = '';
}
for (const [name, value] of Object.entries(solution.cookies || {})) {
if (!value) {
continue;
}
const cookieDomain = host.startsWith('.') ? host : `.${host}`;
await page.setCookie({
name,
value: String(value),
domain: cookieDomain,
path: '/',
secure: true,
sameSite: 'None',
}).catch(() => {});
}
console.log(`[CF] cf_clearance 已注入，goto 业务页 url=${cleanUrl}`);
await page.goto(cleanUrl, {
waitUntil: 'domcontentloaded',
timeout: 60000,
}).catch(() => {});
await page.waitForNetworkIdle({ idleTime: 500, timeout: 30000 }).catch(() => {
console.warn('[CF] cf_clearance goto 后 networkidle 超时，继续后续流程');
});
}
/**
* 统一 Turnstile 求解入口
* @param {{ pageUrl: string, sitekey: string, provider?: string, apiKey?: string }} params
@@ -121,6 +266,7 @@ async function solveVia2Captcha({ pageUrl, sitekey, apiKey }) {
async function solveTurnstile({ pageUrl, sitekey, provider, apiKey }) {
const resolvedProvider = (provider || CAPTCHA_PROVIDER).toLowerCase();
const resolvedKey = apiKey || CAPTCHA_API_KEY;
const cleanPageUrl = sanitizePageUrlForCfSolver(pageUrl);
if (!sitekey) {
throw new Error('无法提取 Turnstile sitekey');
@@ -131,9 +277,9 @@ async function solveTurnstile({ pageUrl, sitekey, provider, apiKey }) {
let token;
if (resolvedProvider === '2captcha') {
token = await solveVia2Captcha({ pageUrl, sitekey, apiKey: resolvedKey });
token = await solveVia2Captcha({ pageUrl: cleanPageUrl, sitekey, apiKey: resolvedKey });
} else {
token = await solveViaCapSolver({ pageUrl, sitekey, apiKey: resolvedKey });
token = await solveViaCapSolver({ pageUrl: cleanPageUrl, sitekey, apiKey: resolvedKey });
}
return { token, provider: resolvedProvider };
@@ -149,5 +295,7 @@ function isCaptchaConfigured() {
module.exports = {
solveTurnstile,
solveCloudflare,
applyCfClearanceSolution,
isCaptchaConfigured,
};
patches/puppeteer-api/lib/cf-api-fallback.js
+173
View File
@@ -0,0 +1,173 @@
/**
* CapSolver / 2Captcha 过盾兜底：Turnstile token 注入 或 AntiCloudflareTask cf_clearance
*
* A2C 等业务域为 CF Managed Challenge（非独立 Turnstile 挂件），须优先 AntiCloudflareTask。
*/
const { sanitizePageUrlForCfSolver } = require('./cf-detector');
const {
injectTurnstileToken,
recoverAfterTokenInject,
extractTurnstileSitekey,
} = require('./turnstile-handler');
const {
solveTurnstile,
solveCloudflare,
applyCfClearanceSolution,
} = require('./captcha-solver');
const { getSitekeyCapture } = require('./sitekey-capture');
/** @returns {boolean} */
function hasCaptchaProxy() {
return (process.env.CAPTCHA_PROXY || '').trim() !== '';
}
/**
* 是否应使用 CapSolver AntiCloudflareTask（Managed Challenge / 5s 盾）
* @param {object|null|undefined} antiBot
*/
function shouldPreferManagedCloudflareSolver(antiBot) {
if (!antiBot || antiBot.cfCloudflareSolver === false) {
return false;
}
if (antiBot.cfChallengeMode === 'managed') {
return hasCaptchaProxy();
}
if (antiBot.cfCloudflareSolver === true && hasCaptchaProxy()) {
return true;
}
return false;
}
/**
* 解析可用于 AntiTurnstileTask 的 sitekey（排除 A2C 配置的 challenge sitekey）
* @param {import('puppeteer').Page} page
* @param {object} antiBot
* @param {number} waitMs
* @returns {Promise<string|null>}
*/
async function resolveTurnstileWidgetSitekey(page, antiBot, waitMs) {
if (antiBot.cfChallengeMode === 'managed') {
return null;
}
const capture = getSitekeyCapture(page);
if (capture) {
const fromCapture = await capture.resolveForSolver(page, antiBot, waitMs);
if (fromCapture && antiBot.cfChallengeMode !== 'managed') {
const configured = String(antiBot.turnstileSitekey || '').trim();
if (configured && fromCapture === configured && antiBot.cfCloudflareSolver) {
return null;
}
return fromCapture;
}
}
const dynamic = await extractTurnstileSitekey(page);
if (dynamic) {
return dynamic;
}
const configured = String(antiBot?.turnstileSitekey || '').trim();
if (configured && !antiBot.cfCloudflareSolver) {
return configured;
}
return null;
}
/**
* CapSolver AntiCloudflareTask 求解并注入 cf_clearance
* @param {import('puppeteer').Page} page
* @param {object} params
*/
async function runManagedCloudflareSolver(page, params) {
const { cleanSolverUrl, waitForUrlSettled, timeoutMs } = params;
if (!hasCaptchaProxy()) {
throw new Error('AntiCloudflareTask 需要配置 CAPTCHA_PROXY（静态代理 ip:port:user:pass）');
}
console.log('[CF] A2C/Managed Challenge：使用 CapSolver AntiCloudflareTask');
const html = await page.content().catch(() => '');
const userAgent = await page.evaluate(() => navigator.userAgent).catch(() => '');
const { cookies, userAgent: solvedUa, provider } = await solveCloudflare({
pageUrl: cleanSolverUrl,
html,
userAgent,
});
console.log(`[CF] CapSolver AntiCloudflareTask (${provider}) 返回 cf_clearance`);
await applyCfClearanceSolution(page, cleanSolverUrl, { cookies, userAgent: solvedUa || userAgent });
if (typeof waitForUrlSettled === 'function') {
await waitForUrlSettled(page).catch(() => {});
}
return { mode: 'cloudflare', provider };
}
/**
* @param {import('puppeteer').Page} page
* @param {object} options
* @param {object} options.antiBot
* @param {string} [options.navUrl]
* @param {(page: import('puppeteer').Page) => Promise<string>} [options.waitForUrlSettled]
* @param {number} [options.timeoutMs]
* @returns {Promise<{ mode: 'turnstile'|'cloudflare', provider: string }>}
*/
async function runCaptchaApiFallback(page, options) {
const {
antiBot,
navUrl = '',
waitForUrlSettled,
timeoutMs = 60000,
} = options;
const cleanSolverUrl = sanitizePageUrlForCfSolver(navUrl || page.url());
if (shouldPreferManagedCloudflareSolver(antiBot)) {
console.log(`[CF] Captcha API 求解 pageUrl=${cleanSolverUrl} mode=managed_challenge`);
return runManagedCloudflareSolver(page, { cleanSolverUrl, waitForUrlSettled, timeoutMs });
}
const sitekey = await resolveTurnstileWidgetSitekey(
page,
antiBot,
Math.min(timeoutMs, 25000)
);
console.log(`[CF] Captcha API 求解 pageUrl=${cleanSolverUrl} sitekey=${sitekey ? 'ok' : 'missing'}`);
if (sitekey) {
try {
const { token, provider } = await solveTurnstile({ pageUrl: cleanSolverUrl, sitekey });
console.log(`[CF] Captcha API Turnstile (${provider}) 返回 token，注入页面`);
await injectTurnstileToken(page, token);
await recoverAfterTokenInject(
page,
cleanSolverUrl,
waitForUrlSettled,
Math.min(timeoutMs, 45000)
);
return { mode: 'turnstile', provider };
} catch (turnstileErr) {
const msg = (turnstileErr?.message || '').toLowerCase();
const isChallengeMismatch = msg.includes('challenge, not turnstile')
|| msg.includes('not turnstile');
if (isChallengeMismatch && antiBot.cfCloudflareSolver !== false && hasCaptchaProxy()) {
console.warn(`[CF] Turnstile 任务失败(${turnstileErr.message})，降级 AntiCloudflareTask`);
return runManagedCloudflareSolver(page, { cleanSolverUrl, waitForUrlSettled, timeoutMs });
}
throw turnstileErr;
}
}
if (antiBot?.cfCloudflareSolver !== false && hasCaptchaProxy()) {
return runManagedCloudflareSolver(page, { cleanSolverUrl, waitForUrlSettled, timeoutMs });
}
throw new Error('无法过盾：请配置 CAPTCHA_PROXY（A2C Managed Challenge）或提供有效 Turnstile sitekey');
}
module.exports = {
runCaptchaApiFallback,
shouldPreferManagedCloudflareSolver,
hasCaptchaProxy,
};
patches/puppeteer-api/lib/cf-detector.js
+35 -3
View File
@@ -20,6 +20,34 @@ function urlIndicatesCfChallenge(url) {
|| url.includes('/cdn-cgi/challenge');
}
/** CF 挑战态查询参数（CapSolver websiteURL 需剔除） */
const CF_CHALLENGE_QUERY_KEYS = [
'__cf_chl_rt_tk',
'__cf_chl_tk',
'__cf_chl_f_tk',
'__cf_chl_captcha_tk__',
];
/**
* 去掉 URL 中 CF 挑战临时参数，供 CapSolver / reload 使用
* @param {string} url
* @returns {string}
*/
function sanitizePageUrlForCfSolver(url) {
if (!url || typeof url !== 'string') {
return url || '';
}
try {
const parsed = new URL(url);
for (const key of CF_CHALLENGE_QUERY_KEYS) {
parsed.searchParams.delete(key);
}
return parsed.toString();
} catch (_) {
return url.split('?')[0] || url;
}
}
/**
* 是否要求 cf_clearance 才视为过盾完成（A2C user.a2c.chat 为 false）
* @param {object|null|undefined} antiBot
@@ -61,7 +89,9 @@ async function detectCloudflare(page, antiBot = null) {
hasTurnstileWidget,
hasChallengeRunning,
titleHasJustAMoment: titleText.includes('Just a moment'),
bodyHasJustAMoment: bodyText.includes('Just a moment') || bodyText.includes('Checking your browser'),
bodyHasJustAMoment: bodyText.includes('Just a moment')
|| bodyText.includes('Checking your browser')
|| bodyText.includes('Performing security verification'),
};
}).catch(() => ({
hasTurnstileInput: false,
@@ -77,7 +107,8 @@ async function detectCloudflare(page, antiBot = null) {
|| domSignals.titleHasJustAMoment
|| domSignals.bodyHasJustAMoment
|| domSignals.hasChallengeRunning
|| title.includes('Just a moment');
|| title.includes('Just a moment')
|| title.includes('Performing security verification');
const turnstileSignals = domSignals.hasTurnstileInput || domSignals.hasTurnstileWidget;
@@ -93,7 +124,7 @@ async function detectCloudflare(page, antiBot = null) {
if (requireClearance) {
blocked = !hasCfClearance && (realChallenge || turnstileSignals);
} else {
blocked = realChallenge;
blocked = realChallenge || (turnstileSignals && urlChallenge);
}
return {
@@ -129,4 +160,5 @@ module.exports = {
isTurnstileSolved,
urlIndicatesCfChallenge,
isCfClearanceRequired,
sanitizePageUrlForCfSolver,
};
patches/puppeteer-api/lib/cf-handler.js
+9 -11
View File
@@ -5,8 +5,9 @@
* 否则清除可能过期的 cf_clearance 并重新触发 Turnstile（Whatshub 短链 → work-order-sharing 场景）。
*/
const { detectCloudflare } = require('./cf-detector');
const { waitForTurnstile, extractTurnstileSitekey, injectTurnstileToken, waitForTurnstileSurface } = require('./turnstile-handler');
const { solveTurnstile, isCaptchaConfigured } = require('./captcha-solver');
const { waitForTurnstile } = require('./turnstile-handler');
const { isCaptchaConfigured } = require('./captcha-solver');
const { runCaptchaApiFallback } = require('./cf-api-fallback');
const { isSessionLikelyValid } = require('./session-store');
/**
@@ -257,15 +258,12 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
let sitekey = await waitForTurnstileSurface(page, Math.min(timeoutMs, 25000));
if (!sitekey) {
sitekey = await extractTurnstileSitekey(page);
}
const pageUrl = page.url();
console.log(`[CF] Captcha API 求解 pageUrl=${pageUrl} sitekey=${sitekey ? 'ok' : 'missing'}`);
const { token, provider } = await solveTurnstile({ pageUrl, sitekey });
console.log(`[CF] Captcha API (${provider}) 返回 token，注入页面`);
await injectTurnstileToken(page, token);
await runCaptchaApiFallback(page, {
antiBot,
navUrl: page.url(),
waitForUrlSettled,
timeoutMs,
});
const apiWaitMs = Math.min(timeoutMs, 30000);
await waitForTurnstile(page, { timeoutMs: apiWaitMs });
patches/puppeteer-api/lib/constants.js
+1 -1
View File
@@ -85,7 +85,7 @@ const QUEUE_TIMEOUT_MS = Math.max(5000, parseInt(process.env.QUEUE_TIMEOUT_MS ||
const API_INTERCEPT_TIMEOUT_MS = Math.max(10000, parseInt(process.env.API_INTERCEPT_TIMEOUT_MS || '60000', 10));
const API_INTERCEPT_TIMEOUT_REAL_MS = Math.max(
API_INTERCEPT_TIMEOUT_MS,
parseInt(process.env.API_INTERCEPT_TIMEOUT_REAL_MS || '90000', 10)
parseInt(process.env.API_INTERCEPT_TIMEOUT_REAL_MS || '180000', 10)
);
const NAVIGATION_TIMEOUT_MS = Math.max(15000, parseInt(process.env.NAVIGATION_TIMEOUT_MS || '60000', 10));
patches/puppeteer-api/lib/sitekey-capture.js
+182
View File
@@ -0,0 +1,182 @@
/**
* Turnstile sitekey 提前捕获：在 goto 前挂载监听，避免 60s 空等后 iframe 已销毁
*/
const {
extractSitekeyFromChallengeUrl,
extractSitekeyFromHtml,
extractTurnstileSitekey,
waitForTurnstileSurface,
} = require('./turnstile-handler');
/**
* @typedef {Object} SitekeyCaptureHandle
* @property {() => string|null} get
* @property {(page: import('puppeteer').Page, antiBot?: object) => Promise<string|null>} scanNow
* @property {(page: import('puppeteer').Page, antiBot?: object, waitMs?: number) => Promise<string|null>} resolveForSolver
* @property {() => void} detach
*/
/**
* 尝试从 URL 字符串解析并缓存 sitekey
* @param {string|null|undefined} url
* @param {{ sitekey: string|null, log: (msg: string) => void }} state
* @returns {string|null}
*/
function tryStoreSitekeyFromUrl(url, state) {
const key = extractSitekeyFromChallengeUrl(url || '');
if (key && !state.sitekey) {
state.sitekey = key;
state.log(`[CF] sitekey 捕获 (${key.slice(0, 12)}...) source=${url.slice(0, 80)}`);
}
return state.sitekey;
}
/**
* 在 page.goto 之前挂载 frame/response/target 监听
* @param {import('puppeteer').Page} page
* @param {import('puppeteer').Browser} [browser]
* @returns {SitekeyCaptureHandle}
*/
function attachSitekeyCapture(page, browser) {
const state = {
sitekey: null,
attached: false,
cleanups: [],
log: (msg) => console.log(msg),
};
if (page.__sitekeyCaptureHandle) {
return page.__sitekeyCaptureHandle;
}
const onFrameNavigated = (frame) => {
try {
tryStoreSitekeyFromUrl(frame.url(), state);
} catch (_) {
/* ignore */
}
};
page.on('framenavigated', onFrameNavigated);
state.cleanups.push(() => page.off('framenavigated', onFrameNavigated));
const onResponse = (response) => {
try {
tryStoreSitekeyFromUrl(response.url(), state);
} catch (_) {
/* ignore */
}
};
page.on('response', onResponse);
state.cleanups.push(() => page.off('response', onResponse));
if (browser && typeof browser.on === 'function') {
const onTargetCreated = (target) => {
try {
tryStoreSitekeyFromUrl(target.url(), state);
} catch (_) {
/* ignore */
}
};
browser.on('targetcreated', onTargetCreated);
state.cleanups.push(() => browser.off('targetcreated', onTargetCreated));
}
state.attached = true;
/** @type {SitekeyCaptureHandle} */
const handle = {
get: () => state.sitekey,
scanNow: async (scanPage, antiBot) => {
if (state.sitekey) {
return state.sitekey;
}
const frames = scanPage.frames();
for (const frame of frames) {
tryStoreSitekeyFromUrl(frame.url(), state);
if (state.sitekey) {
return state.sitekey;
}
}
const html = await scanPage.content().catch(() => '');
const fromHtml = extractSitekeyFromHtml(html);
if (fromHtml) {
state.sitekey = fromHtml;
state.log(`[CF] sitekey 从 HTML 提取 (${fromHtml.slice(0, 12)}...)`);
return fromHtml;
}
const browserRef = scanPage.browser();
if (browserRef) {
for (const target of browserRef.targets()) {
tryStoreSitekeyFromUrl(target.url(), state);
if (state.sitekey) {
return state.sitekey;
}
}
}
const configured = String(antiBot?.turnstileSitekey || '').trim();
if (configured && antiBot?.cfChallengeMode !== 'managed' && !antiBot?.cfCloudflareSolver) {
state.sitekey = configured;
state.log(`[CF] 使用 antiBot.turnstileSitekey (${configured.slice(0, 12)}...)`);
return configured;
}
return null;
},
resolveForSolver: async (scanPage, antiBot, waitMs = 25000) => {
let sitekey = await handle.scanNow(scanPage, antiBot);
if (sitekey) {
return sitekey;
}
sitekey = await waitForTurnstileSurface(scanPage, waitMs);
if (sitekey) {
state.sitekey = sitekey;
return sitekey;
}
sitekey = await extractTurnstileSitekey(scanPage);
if (sitekey) {
state.sitekey = sitekey;
return sitekey;
}
return handle.scanNow(scanPage, antiBot);
},
detach: () => {
for (const fn of state.cleanups) {
try {
fn();
} catch (_) {
/* ignore */
}
}
state.cleanups = [];
state.attached = false;
if (page.__sitekeyCaptureHandle === handle) {
delete page.__sitekeyCaptureHandle;
}
},
};
page.__sitekeyCaptureHandle = handle;
return handle;
}
/**
* 读取 page 上已挂载的 capture（无则 null）
* @param {import('puppeteer').Page} page
* @returns {SitekeyCaptureHandle|null}
*/
function getSitekeyCapture(page) {
return page?.__sitekeyCaptureHandle || null;
}
module.exports = {
attachSitekeyCapture,
getSitekeyCapture,
tryStoreSitekeyFromUrl,
};
patches/puppeteer-api/lib/turnstile-handler.js
+147 -19
View File
@@ -1,22 +1,52 @@
/**
* Turnstile 内置求解：等待 widget 自动完成 + token 轮询 + 多 frame sitekey 提取
*
* 支持新版 challenge-platform iframe：sitekey 在 URL 路径 /rch/{id}/{sitekey}/ 中
*/
const { detectCloudflare, isTurnstileSolved, urlIndicatesCfChallenge } = require('./cf-detector');
/**
* @typedef {Object} TurnstileResult
* @property {boolean} success
* @property {string} stage built_in|timeout|already_clear
* @property {number} elapsedMs
* 从 challenge-platform / Turnstile URL 解析 sitekey（Node 侧，可读 frame.url()）
* @param {string} url
* @returns {string|null}
*/
function extractSitekeyFromChallengeUrl(url) {
if (!url || typeof url !== 'string') {
return null;
}
const pathMatch = url.match(/\/rch\/[^/]+\/(0x4[A-Za-z0-9_-]+)\//i);
if (pathMatch) {
return pathMatch[1];
}
const queryMatch = url.match(/[?&]sitekey=([^&]+)/i);
if (queryMatch) {
return decodeURIComponent(queryMatch[1]);
}
const looseMatch = url.match(/\/(0x4[A-Za-z0-9_-]{10,})\//i);
if (looseMatch) {
return looseMatch[1];
}
return null;
}
/**
* 在单个 frame 内提取 sitekey
* 在单个 frame 内提取 sitekey（主文档 DOM + iframe src 属性）
* @param {import('puppeteer').Frame} frame
* @returns {Promise<string|null>}
*/
async function extractSitekeyFromFrame(frame) {
return frame.evaluate(() => {
const extractFromSrc = (src) => {
if (!src) return null;
const pathMatch = src.match(/\/rch\/[^/]+\/(0x4[A-Za-z0-9_-]+)\//i);
if (pathMatch) return pathMatch[1];
const queryMatch = src.match(/[?&]sitekey=([^&]+)/i);
if (queryMatch) return decodeURIComponent(queryMatch[1]);
const looseMatch = src.match(/\/(0x4[A-Za-z0-9_-]{10,})\//i);
if (looseMatch) return looseMatch[1];
return null;
};
const widget = document.querySelector('.cf-turnstile[data-sitekey], [data-sitekey]');
if (widget) {
const key = widget.getAttribute('data-sitekey');
@@ -25,12 +55,13 @@ async function extractSitekeyFromFrame(frame) {
}
}
const iframes = document.querySelectorAll('iframe[src*="challenges.cloudflare.com"], iframe[src*="challenge-platform"]');
const iframes = document.querySelectorAll(
'iframe[src*="challenges.cloudflare.com"], iframe[src*="challenge-platform"], iframe#cf-chl-widget'
);
for (const iframe of iframes) {
const src = iframe.getAttribute('src') || '';
const match = src.match(/[?&]sitekey=([^&]+)/i);
if (match) {
return decodeURIComponent(match[1]);
const fromSrc = extractFromSrc(iframe.getAttribute('src') || '');
if (fromSrc) {
return fromSrc;
}
}
@@ -38,7 +69,8 @@ async function extractSitekeyFromFrame(frame) {
for (const script of scripts) {
const text = script.textContent || '';
const match = text.match(/sitekey['"\s:=]+['"]([0-9x_a-zA-Z-]{10,})['"]/i)
|| text.match(/data-sitekey=['"]([0-9x_a-zA-Z-]{10,})['"]/i);
|| text.match(/data-sitekey=['"]([0-9x_a-zA-Z-]{10,})['"]/i)
|| text.match(/\/(0x4[A-Za-z0-9_-]{10,})\//i);
if (match) {
return match[1];
}
@@ -48,6 +80,34 @@ async function extractSitekeyFromFrame(frame) {
}).catch(() => null);
}
/**
* 从整页 HTML 文本提取 sitekey（应对 OOPIF 无法 enumerate 的场景）
* @param {string} html
* @returns {string|null}
*/
function extractSitekeyFromHtml(html) {
if (!html || typeof html !== 'string') {
return null;
}
const pathMatch = html.match(/\/rch\/[^/]+\/(0x4[A-Za-z0-9_-]+)\//i);
if (pathMatch) {
return pathMatch[1];
}
const dataMatch = html.match(/data-sitekey=["']([^"']+)["']/i);
if (dataMatch) {
return dataMatch[1];
}
const queryMatch = html.match(/[?&]sitekey=([^&"']+)/i);
if (queryMatch) {
return decodeURIComponent(queryMatch[1]);
}
const looseMatch = html.match(/\/(0x4[A-Za-z0-9_-]{10,})\//i);
if (looseMatch) {
return looseMatch[1];
}
return null;
}
/**
* 从主文档及所有子 frame 提取 Turnstile sitekey
* @param {import('puppeteer').Page} page
@@ -60,6 +120,10 @@ async function extractTurnstileSitekey(page) {
}
for (const frame of page.frames()) {
const fromFrameUrl = extractSitekeyFromChallengeUrl(frame.url());
if (fromFrameUrl) {
return fromFrameUrl;
}
if (frame === page.mainFrame()) {
continue;
}
@@ -69,15 +133,30 @@ async function extractTurnstileSitekey(page) {
}
}
const html = await page.content().catch(() => '');
const fromHtml = extractSitekeyFromHtml(html);
if (fromHtml) {
return fromHtml;
}
const browser = page.browser();
if (browser) {
for (const target of browser.targets()) {
const fromTarget = extractSitekeyFromChallengeUrl(target.url());
if (fromTarget) {
return fromTarget;
}
}
}
return null;
}
/**
* Captcha API 调用前：等待 Turnstile 挑战面出现或 URL 进入 CF 挑战态
*
* @param {import('puppeteer').Page} page
* @param {number} [timeoutMs]
* @returns {Promise<string|null>} 若已提取到 sitekey 则返回，否则 null
* @returns {Promise<string|null>}
*/
async function waitForTurnstileSurface(page, timeoutMs = 20000) {
const started = Date.now();
@@ -86,7 +165,7 @@ async function waitForTurnstileSurface(page, timeoutMs = 20000) {
while (Date.now() - started < timeoutMs) {
const sitekey = await extractTurnstileSitekey(page);
if (sitekey) {
console.log(`[CF] 已检测到 Turnstile sitekey (${sitekey.slice(0, 8)}...)`);
console.log(`[CF] 已检测到 Turnstile sitekey (${sitekey.slice(0, 12)}...)`);
return sitekey;
}
@@ -104,7 +183,8 @@ async function waitForTurnstileSurface(page, timeoutMs = 20000) {
const hasSurface = await page.evaluate(() => {
return !!document.querySelector(
'.cf-turnstile, [data-sitekey], [name="cf-turnstile-response"], '
+ 'iframe[src*="challenges.cloudflare.com"], iframe[src*="challenge-platform"]'
+ 'iframe[src*="challenges.cloudflare.com"], iframe[src*="challenge-platform"], '
+ 'iframe[id^="cf-chl-widget"]'
);
}).catch(() => false);
@@ -122,8 +202,7 @@ async function waitForTurnstileSurface(page, timeoutMs = 20000) {
/**
* 等待 Turnstile 通过（内置点击 / 自动跳转）
* @param {import('puppeteer').Page} page
* @param {{ timeoutMs?: number, pollMs?: number, useBuiltInClick?: boolean }} [options]
* @returns {Promise<TurnstileResult>}
* @param {{ timeoutMs?: number, pollMs?: number }} [options]
*/
async function waitForTurnstile(page, options = {}) {
const timeoutMs = options.timeoutMs ?? 60000;
@@ -148,7 +227,7 @@ async function waitForTurnstile(page, options = {}) {
}
const state = await detectCloudflare(page);
if (!state.blocked && !urlIndicatesCfChallenge(page.url()) && state.hasCfClearance) {
if (!state.blocked && !urlIndicatesCfChallenge(page.url())) {
return { success: true, stage: 'built_in', elapsedMs: Date.now() - started };
}
@@ -159,7 +238,27 @@ async function waitForTurnstile(page, options = {}) {
}
/**
* 将 Captcha API 返回的 token 注入页面
* 等待页面离开 CF 挑战 URL / 挑战 DOM
* @param {import('puppeteer').Page} page
* @param {number} timeoutMs
* @param {number} pollMs
*/
async function waitForChallengeUrlClear(page, timeoutMs = 45000, pollMs = 500) {
const started = Date.now();
while (Date.now() - started < timeoutMs) {
if (!urlIndicatesCfChallenge(page.url())) {
const state = await detectCloudflare(page);
if (!state.blocked) {
return true;
}
}
await new Promise((r) => setTimeout(r, pollMs));
}
return false;
}
/**
* 将 Captcha API 返回的 token 注入页面并尝试触发提交
* @param {import('puppeteer').Page} page
* @param {string} token
*/
@@ -178,10 +277,39 @@ async function injectTurnstileToken(page, token) {
}, token);
}
/**
* token 注入后等待过盾；若仍停留在挑战页则 reload 干净业务 URL
* @param {import('puppeteer').Page} page
* @param {string} cleanUrl
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
* @param {number} [timeoutMs]
*/
async function recoverAfterTokenInject(page, cleanUrl, waitForUrlSettled, timeoutMs = 45000) {
if (await waitForChallengeUrlClear(page, Math.min(timeoutMs, 12000))) {
return true;
}
if (!cleanUrl) {
return false;
}
console.log(`[CF] token 注入后仍处挑战页，reload 干净 URL: ${cleanUrl}`);
await page.goto(cleanUrl, { waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 60000) }).catch(() => {});
if (typeof waitForUrlSettled === 'function') {
await waitForUrlSettled(page).catch(() => {});
}
return waitForChallengeUrlClear(page, timeoutMs);
}
module.exports = {
waitForTurnstile,
waitForTurnstileSurface,
extractTurnstileSitekey,
extractSitekeyFromFrame,
extractSitekeyFromChallengeUrl,
extractSitekeyFromHtml,
injectTurnstileToken,
waitForChallengeUrlClear,
recoverAfterTokenInject,
};
patches/puppeteer-api/lib/ui-pagination-runner.js
+11 -2
View File
@@ -20,7 +20,8 @@ function classifyPuppeteerError(err) {
if (err?.name === 'TimeoutError' || msg.includes('timeout')) return 'timeout';
if (msg.includes('net::') || msg.includes('network') || msg.includes('err_connection')
|| msg.includes('err_address_unreachable') || msg.includes('err_name_not_resolved')) return 'network';
if (msg.includes('navigation') || msg.includes('frame was detached')) return 'navigation';
if (msg.includes('navigation') || msg.includes('frame was detached')
|| msg.includes('detached frame') || msg.includes('attempted to use detached')) return 'navigation';
if (msg.includes('target closed') || msg.includes('session closed')) return 'target_closed';
return 'unknown';
}
@@ -85,7 +86,9 @@ async function runUiPagination(page, options) {
{ timeout: 1000, polling: 100 }
).catch(() => new Promise((r) => setTimeout(r, 500)));
while (attempts < maxAttempts && !pageData) {
let paginationComplete = false;
while (attempts < maxAttempts && !pageData && !paginationComplete) {
attempts++;
log(`[抓取] 准备抓取第 ${targetPageNum} 页 (尝试 ${attempts}/${maxAttempts})...`);
@@ -139,6 +142,7 @@ async function runUiPagination(page, options) {
if (clickStatus === 'disabled') {
log(`[提示] 第 ${targetPageNum} 页按钮已被禁用，说明到底了，抓取提前结束。`);
paginationComplete = true;
break;
}
log('[动作] 双引擎原生点击已执行！');
@@ -153,6 +157,11 @@ async function runUiPagination(page, options) {
}
}
if (paginationComplete) {
log('[完成] 翻页已到底，正常结束。');
break;
}
if (pageData) {
capturedData.push(pageData);
log(`[成功] 斩获第 ${targetPageNum} 页数据！`);