优化爬虫,增加缓存,线程池,保存浏览器用户信息

This commit is contained in:
root
2026-06-20 15:49:40 +08:00
parent e95f4a227c
commit 09bfa9ae01
26 changed files with 1584 additions and 454 deletions
+42 -22
View File
@@ -1,53 +1,75 @@
/**
* Cloudflare 挑战统一处理:检测 → 内置等待 → Captcha API 兜底
* Cloudflare 挑战统一处理:检测 → 会话快速通道 → 内置等待 → Captcha API 兜底
*/
const { detectCloudflare } = require('./cf-detector');
const { waitForTurnstile, extractTurnstileSitekey, injectTurnstileToken } = require('./turnstile-handler');
const { solveTurnstile, isCaptchaConfigured } = require('./captcha-solver');
const { isSessionLikelyValid } = require('./session-store');
/**
* @typedef {Object} CfHandleResult
* @property {boolean} success
* @property {string|null} code
* @property {string|null} challengeType
* @property {string|null} stage
* @property {boolean} solverUsed
* @property {number} elapsedMs
* @property {boolean} cfDetected
*/
/**
* 处理 Cloudflare / Turnstile 挑战(在 authActions 之前调用)
* @param {import('puppeteer').Page} page
* @param {{ turnstile?: boolean, solverFallback?: boolean, challengeTimeoutMs?: number }} antiBot
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
* @returns {Promise<CfHandleResult>}
* @param {import('./session-store').StoredSession|null} [storedSession]
*/
async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled) {
async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, storedSession = null) {
const started = Date.now();
const timeoutMs = antiBot.challengeTimeoutMs || 60000;
let cfState = await detectCloudflare(page);
if (!cfState.blocked && cfState.hasCfClearance) {
return {
success: true,
code: null,
challengeType: null,
stage: 'session_reuse',
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: false,
};
}
if (!cfState.blocked) {
return {
success: true,
code: null,
challengeType: null,
stage: null,
stage: storedSession && isSessionLikelyValid(storedSession) ? 'session_reuse' : null,
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: false,
};
}
if (storedSession && isSessionLikelyValid(storedSession)) {
console.log('[CF] 会话有效但仍被拦截,尝试 reload 一次');
try {
await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 45000) });
if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {});
cfState = await detectCloudflare(page);
if (!cfState.blocked) {
return {
success: true,
code: null,
challengeType: cfState.challengeType,
stage: 'session_reload',
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: true,
};
}
} catch (reloadErr) {
console.warn('[CF] reload 失败:', reloadErr.message);
}
}
console.log(`[CF] 检测到挑战 type=${cfState.challengeType} url=${cfState.url}`);
if (antiBot.turnstile !== false) {
const builtIn = await waitForTurnstile(page, { timeoutMs, useBuiltInClick: true });
if (builtIn.success) {
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {});
cfState = await detectCloudflare(page);
if (!cfState.blocked) {
return {
@@ -74,9 +96,7 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled) {
const apiWaitMs = Math.min(timeoutMs, 30000);
await waitForTurnstile(page, { timeoutMs: apiWaitMs });
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {});
cfState = await detectCloudflare(page);
if (!cfState.blocked) {