require('dotenv').config({ path: require('path').join(__dirname, '.env') }); process.on('uncaughtException', (err) => { console.error('[FATAL] uncaughtException:', err); }); process.on('unhandledRejection', (reason) => { console.error('[FATAL] unhandledRejection:', reason); }); const express = require('express'); const fs = require('fs'); const { rewriteHttpsToHttpForLiteralIp, attachHttpIpRewriteInterceptor, } = require('./http-ip-rewrite'); const { resolveHttpRedirects, extractShareTokenFromUrl, } = require('./url-resolve'); const { DEFAULT_UA, RUNTIME_DIR, resolveChromeExecutable, MAX_CONCURRENT_BROWSERS, MAX_CONCURRENT_BROWSERS_REAL, API_INTERCEPT_TIMEOUT_MS, API_INTERCEPT_TIMEOUT_REAL_MS, NAVIGATION_TIMEOUT_MS, NAVIGATION_WAIT_UNTIL, NAVIGATION_MAX_RETRIES, PAGE_DEFAULT_TIMEOUT_MS, SESSION_TTL_MS, PERSIST_BROWSER_PROFILE, POOL_IDLE_MS, MAX_POOLED_BROWSERS, } = require('./lib/constants'); const { normalizeAntiBot, createBrowserSession, acquireBrowserSession, runWithProfileLimit, createManagedPage, getFactoryStats, standardLimiter, browserPool, } = require('./lib/browser-factory'); const { launchStandardBrowser } = require('./lib/launch-standard'); const { isRealBrowserAvailable } = require('./lib/launch-real-browser'); const { handleCloudflareChallenge, isCaptchaConfigured } = require('./lib/cf-handler'); const { detectCloudflare, urlIndicatesCfChallenge } = require('./lib/cf-detector'); const { runCaptchaApiFallback, shouldPreferManagedCloudflareSolver } = require('./lib/cf-api-fallback'); const { getCaptchaProxyChromeArgs, applyCaptchaProxyToPage, parseCaptchaProxy, hasCaptchaProxy, } = require('./lib/captcha-proxy'); const { attachSitekeyCapture } = require('./lib/sitekey-capture'); const { saveSession, getSessionDebugInfo, getSessionStats, isSessionLikelyValid, } = require('./lib/session-store'); const { resolveSessionContext, touchSessionIfPresent } = require('./lib/session-context'); const { getProfileStats, profileExists } = require('./lib/profile-dir'); const { isHuojianLongLinkUrl, isHuojianShortEntryUrl, applyHuojianBusinessHostHint, } = require('./lib/huojian-url'); const { runUiPagination } = require('./lib/ui-pagination-runner'); const app = express(); app.use(express.json({ limit: '50mb' })); let shuttingDown = false; /** HTTP 层:槽位排队 + 队列超时/关闭中 的统一错误响应 */ async function runBrowserTask(taskName, handler, profile = 'standard') { if (shuttingDown) { const err = new Error('SERVICE_SHUTTING_DOWN'); err.code = 'SERVICE_SHUTTING_DOWN'; throw err; } return runWithProfileLimit(taskName, profile, handler); } function sendTaskError(res, error, extra = {}) { if (error.code === 'QUEUE_TIMEOUT' || error.message === 'POOL_ACQUIRE_TIMEOUT') { return res.status(503).json({ success: false, error: '服务繁忙,排队超时,请稍后重试', queue: standardLimiter.getStats(), ...extra, }); } if (error.code === 'SERVICE_SHUTTING_DOWN') { return res.status(503).json({ success: false, error: '服务正在关闭', ...extra }); } const errType = classifyPuppeteerError(error); console.error(`[${errType}]`, error.message); const payload = { success: false, error: error.message, ...extra }; if (error.pageDiagnostics) { payload.page = error.pageDiagnostics; } return res.status(500).json(payload); } async function navigateToPage(page, url) { await withExponentialBackoff( () => page.goto(url, { waitUntil: NAVIGATION_WAIT_UNTIL, timeout: NAVIGATION_TIMEOUT_MS }), { maxRetries: NAVIGATION_MAX_RETRIES, baseDelayMs: 1000 } ); } async function safeCloseBrowser(browser) { if (!browser) return; try { const pages = await browser.pages(); await Promise.all(pages.map((p) => p.close().catch(() => {}))); await browser.close(); } catch (_) { try { const proc = browser.process(); if (proc && !proc.killed) proc.kill('SIGKILL'); } catch (_) { /* ignore */ } } } function classifyPuppeteerError(err) { const msg = (err?.message || '').toLowerCase(); if (err?.name === 'TimeoutError' || msg.includes('timeout')) return 'timeout'; if (msg.includes('net::') || msg.includes('network') || msg.includes('err_connection') || msg.includes('err_address_unreachable') || msg.includes('err_name_not_resolved')) return 'network'; if (msg.includes('navigation') || msg.includes('frame was detached') || msg.includes('detached frame') || msg.includes('attempted to use detached')) return 'navigation'; if (msg.includes('target closed') || msg.includes('session closed')) return 'target_closed'; return 'unknown'; } function isRetryableError(err) { const type = classifyPuppeteerError(err); return type === 'timeout' || type === 'network' || type === 'navigation'; } async function withExponentialBackoff(fn, { maxRetries = 3, baseDelayMs = 500, maxDelayMs = 8000, shouldRetry = isRetryableError, } = {}) { let lastError; for (let attempt = 0; attempt <= maxRetries; attempt++) { try { return await fn(attempt); } catch (err) { lastError = err; if (attempt >= maxRetries || !shouldRetry(err)) throw err; const delay = Math.min(baseDelayMs * Math.pow(2, attempt), maxDelayMs); await new Promise((r) => setTimeout(r, delay)); } } throw lastError; } async function waitForUrlSettled(page, { maxAttempts = 15, stableMs = 500, redirectSettleMs = 1500 } = {}) { let currentUrl = page.url(); let urlSettled = false; let settleAttempts = 0; while (!urlSettled && settleAttempts < maxAttempts) { const urlBeforeWait = currentUrl; await page.waitForFunction( (expectedUrl) => window.location.href === expectedUrl, { timeout: stableMs, polling: 100 }, urlBeforeWait ).catch(() => {}); const newUrl = page.url(); if (newUrl === currentUrl) { urlSettled = true; } else { console.log(`[路由重定向探测] 页面从 ${currentUrl} 跳转到了 ${newUrl}`); currentUrl = newUrl; await page.waitForNetworkIdle({ idleTime: 300, timeout: redirectSettleMs }).catch(() => {}); } settleAttempts++; } return currentUrl; } /** 合并 antiBot 扩展字段(postCfReady* 由 PHP 按工单类型传入,未配置则跳过) */ function resolveAntiBot(rawAntiBot) { const base = normalizeAntiBot(rawAntiBot); if (!rawAntiBot || !base.enabled) { return base; } return { ...base, postCfReadyUrlContains: rawAntiBot.postCfReadyUrlContains || '', postCfReadySelector: rawAntiBot.postCfReadySelector || '', postCfReadyApiUrl: rawAntiBot.postCfReadyApiUrl || '', postCfReadyTimeoutMs: rawAntiBot.postCfReadyTimeoutMs || 0, postCfSettleMs: rawAntiBot.postCfSettleMs || 0, postCfSpaWaitMs: rawAntiBot.postCfSpaWaitMs || 0, cfClearanceRequired: rawAntiBot.cfClearanceRequired !== false, businessHostHint: rawAntiBot.businessHostHint || '', landingUrlHint: rawAntiBot.landingUrlHint || '', turnstileSitekey: rawAntiBot.turnstileSitekey || '', cfCloudflareSolver: rawAntiBot.cfCloudflareSolver !== false, cfChallengeMode: rawAntiBot.cfChallengeMode || '', }; } /** * Whatshub 短链场景:PHP 未传 postCfReady* 时 Node 侧自动补全(避免 session_reuse 后 15s auth 超时) * @param {object} antiBot * @param {string} pageUrl */ function applyWhatshubPostCfDefaults(antiBot, pageUrl) { if (!antiBot || !antiBot.enabled) { return antiBot; } if ((antiBot.postCfReadyUrlContains || '').trim()) { return antiBot; } try { const parsed = new URL(pageUrl || ''); if (!parsed.hostname.includes('whatshub.cc') || !parsed.pathname.includes('/m/')) { return antiBot; } console.log('[CF] Whatshub 短链自动启用 postCfReady 默认配置'); return { ...antiBot, postCfReadyUrlContains: 'work-order-sharing', postCfReadySelector: '.el-input__inner', postCfReadyTimeoutMs: antiBot.postCfReadyTimeoutMs || 30000, postCfSettleMs: antiBot.postCfSettleMs || 500, postCfSpaWaitMs: antiBot.postCfSpaWaitMs || 8000, }; } catch (_) { return antiBot; } } /** * A2C 短链 → user.a2c.chat:PHP 未传 postCfReady* 时 Node 侧自动补全 * @param {object} antiBot * @param {string} pageUrl */ function applyA2cPostCfDefaults(antiBot, pageUrl) { if (!antiBot || !antiBot.enabled) { return antiBot; } if ((antiBot.postCfReadyUrlContains || '').trim()) { return antiBot; } let shouldApply = false; const sessionKey = String(antiBot.sessionKey || '').toLowerCase(); if (sessionKey.startsWith('a2c:')) { shouldApply = true; } try { const parsed = new URL(pageUrl || ''); if (parsed.hostname.includes('a2c.chat')) { shouldApply = true; } } catch (_) { /* ignore */ } if (!shouldApply) { return antiBot; } console.log('[CF] A2C 自动启用 postCfReady 默认配置'); return { ...antiBot, postCfReadyUrlContains: '/visitors/counter/share', postCfReadySelector: '.el-table', postCfReadyApiUrl: antiBot.postCfReadyApiUrl || '/api/talk/counter/share/record/list', postCfReadyTimeoutMs: antiBot.postCfReadyTimeoutMs || 60000, postCfSettleMs: antiBot.postCfSettleMs || 500, postCfSpaWaitMs: antiBot.postCfSpaWaitMs || 8000, cfClearanceRequired: antiBot.cfClearanceRequired === undefined ? false : antiBot.cfClearanceRequired, solverFallback: antiBot.solverFallback === undefined ? true : antiBot.solverFallback, cfCloudflareSolver: antiBot.cfCloudflareSolver !== false, cfChallengeMode: antiBot.cfChallengeMode || 'managed', }; } /** * 火箭短链 → 动态长链域 /gds?link=:PHP 未传 postCfReady* 时 Node 侧自动补全 * @param {object} antiBot * @param {string} pageUrl */ function applyHuojianPostCfDefaults(antiBot, pageUrl) { if (!antiBot || !antiBot.enabled) { return antiBot; } if ((antiBot.postCfReadyUrlContains || '').trim()) { return antiBot; } let shouldApply = false; const sessionKey = String(antiBot.sessionKey || '').toLowerCase(); if (sessionKey.startsWith('huojian:')) { shouldApply = true; } if (isHuojianLongLinkUrl(pageUrl) || isHuojianShortEntryUrl(pageUrl)) { shouldApply = true; } if (!shouldApply) { return antiBot; } console.log('[CF] 火箭 自动启用 postCfReady 默认配置'); return { ...antiBot, postCfReadyUrlContains: '/gds', postCfReadySelector: '.el-message-box__input', postCfReadyTimeoutMs: antiBot.postCfReadyTimeoutMs || 45000, postCfSettleMs: antiBot.postCfSettleMs || 500, postCfSpaWaitMs: antiBot.postCfSpaWaitMs || 8000, }; } /** 按 pageUrl 合并 antiBot(含 Whatshub / A2C / 火箭 默认 postCf) */ function resolveAntiBotForPage(rawAntiBot, pageUrl) { return applyHuojianPostCfDefaults( applyA2cPostCfDefaults( applyWhatshubPostCfDefaults(resolveAntiBot(rawAntiBot), pageUrl), pageUrl ), pageUrl ); } function isBusinessUrlReady(page, urlNeedle) { return !urlNeedle || page.url().includes(urlNeedle); } /** A2C 已直达 user.a2c.chat 分享页(非 yyk.ink 短链跳转场景) */ function isA2cDirectSharePage(page, urlNeedle = '') { const needle = (urlNeedle || '').trim(); if (!needle.includes('/visitors/counter/share')) { return false; } try { const parsed = new URL(page.url() || ''); return parsed.hostname.includes('a2c.chat') && parsed.pathname.includes('/visitors/counter/share'); } catch (_) { return false; } } /** 过盾后等待业务 list API 返回(A2C Vue 挂载信号) */ async function waitForPostCfListApi(page, antiBot, timeoutMs) { const apiUrl = (antiBot.postCfReadyApiUrl || '').trim(); if (!apiUrl) { return false; } console.log(`[CF] 等待业务 API 响应 api=${apiUrl} timeout=${timeoutMs}ms`); const matched = await page.waitForResponse( (resp) => { const req = resp.request(); if (req.method() === 'OPTIONS') { return false; } return resp.url().includes(apiUrl) && resp.status() >= 200 && resp.status() < 400; }, { timeout: timeoutMs } ).catch(() => null); if (matched) { console.log(`[CF] 业务 API 已响应 status=${matched.status()} url=${matched.url()}`); return true; } console.warn('[CF] 业务 API 等待超时,降级为 DOM selector 检测'); return false; } /** 等待业务 DOM:先 attached 再 visible,避免 Vue 渲染中途误判 */ async function waitForPostCfSelector(page, selector, timeoutMs) { await page.waitForSelector(selector, { timeout: timeoutMs, visible: false }); await page.waitForSelector(selector, { timeout: Math.min(15000, timeoutMs), visible: true }); } /** Whatshub 分享短链 /m/xxx/1 */ function isWhatshubShortLinkPath(url) { try { const parsed = new URL(url || ''); return parsed.hostname.includes('whatshub.cc') && parsed.pathname.includes('/m/'); } catch (_) { return false; } } /** * Whatshub 短链 + Real Browser:不覆盖 UA、不预注入磁盘 cf_clearance * @param {string} pageUrl * @param {object} antiBot * @param {object[]} mergedCookies * @param {object} extraOpts */ function buildManagedPageOptions(pageUrl, antiBot, mergedCookies, extraOpts = {}) { const opts = { ...extraOpts }; const isWhatshubShort = isWhatshubShortLinkPath(pageUrl); const isRealProfile = antiBot?.profile === 'real'; if (isWhatshubShort && isRealProfile && antiBot?.enabled) { opts.skipUserAgent = true; const filtered = (mergedCookies || []).filter((c) => c.name !== 'cf_clearance'); if (filtered.length < (mergedCookies || []).length) { console.log('[Whatshub] 短链首访:跳过磁盘 cf_clearance 预注入,避免 SPA 路由未触发'); } opts.cookies = filtered; return opts; } opts.cookies = mergedCookies; if (!opts.skipUserAgent && !opts.userAgent) { opts.userAgent = DEFAULT_UA; } return opts; } /** * A2C:短链入口时优先导航至 PHP 预解析的长链,减少 yyk.ink 跳转竞态 * @param {string} pageUrl * @param {object} antiBot */ function resolveNavigationUrl(pageUrl, antiBot) { const hint = String(antiBot?.landingUrlHint || '').trim(); if (hint === '' || !hint.includes('/visitors/counter/share')) { return pageUrl; } try { const pageHost = new URL(pageUrl).hostname; const hintHost = new URL(hint).hostname; if (hintHost.includes('a2c.chat') && pageHost !== hintHost) { console.log(`[A2C] 短链入口,直接导航至预解析长链: ${hint}`); return hint; } } catch (_) { /* ignore */ } return pageUrl; } /** * Chrome 启动参数:仅当 useCaptchaProxy=true 时注入 CAPTCHA_PROXY(与 CapSolver 同源 IP) * @param {object|null|undefined} antiBot * @param {string[]} [baseArgs] * @param {boolean} [useCaptchaProxy] */ function buildBrowserExtraArgs(antiBot, baseArgs = [], useCaptchaProxy = false) { const args = [...baseArgs]; if (!useCaptchaProxy || !hasCaptchaProxy()) { return args; } const proxyArgs = getCaptchaProxyChromeArgs(); if (proxyArgs.length > 0) { const parsed = parseCaptchaProxy(); console.log( `[Proxy] Chrome 使用 CAPTCHA_PROXY ${parsed.host}:${parsed.port}(与 CapSolver 同源 IP)` ); args.push(...proxyArgs); } return args; } /** * Managed A2C 等:仅在检测到真实 CF 挑战时切换为 CAPTCHA_PROXY Browser(无盾则直连) * 会原地更新 browserCtx.browserSession / browserCtx.page * @param {object} browserCtx */ async function ensureCaptchaProxyBrowserIfNeeded(browserCtx) { const { antiBot, profile, navigationUrl, mergedCookies, baseExtraArgs = ['--mute-audio'], pageExtraOpts = {}, } = browserCtx; let { browserSession, page } = browserCtx; if (!antiBot?.enabled || browserSession?.captchaProxyActive) { return; } if (!shouldPreferManagedCloudflareSolver(antiBot) || !hasCaptchaProxy()) { return; } const cfState = await detectCloudflare(page, antiBot); const needsProxy = cfState.blocked || urlIndicatesCfChallenge(page.url()); if (!needsProxy) { console.log('[Proxy] 未检测到 CF 挑战,Chrome 直连(不使用 CAPTCHA_PROXY)'); return; } console.log('[Proxy] 检测到 CF 挑战,切换 Browser 为 CAPTCHA_PROXY(与 CapSolver 同源 IP)'); const snapshotCookies = await page.cookies().catch(() => []); await closeTaskPage(page); await browserSession.release(true); const extraArgs = buildBrowserExtraArgs(antiBot, baseExtraArgs, true); browserSession = await acquireBrowserSession({ profile, extraArgs, sessionKey: antiBot.sessionKey || '', }); browserSession.captchaProxyActive = true; page = await createTaskPage( browserSession.browser, browserSession.reused ? null : browserSession.page, navigationUrl, antiBot, mergedCookies.length > 0 ? mergedCookies : snapshotCookies, { ...pageExtraOpts, useCaptchaProxy: true } ); if (antiBot?.enabled) { attachSitekeyCapture(page, browserSession.browser); } await prepareWhatshubShortLinkBeforeNav(page, navigationUrl, antiBot); await navigateToPage(page, navigationUrl); await waitForUrlSettled(page).catch(() => {}); if (typeof browserCtx.afterRelaunchNav === 'function') { await browserCtx.afterRelaunchNav(page); } browserCtx.browserSession = browserSession; browserCtx.page = page; } /** 任务结束关闭本次 Tab(温池复用 Browser 时保留 Browser 实例) */ async function closeTaskPage(page) { if (!page) { return; } try { if (typeof page.isClosed === 'function' && !page.isClosed()) { await page.close(); } } catch (_) { /* ignore */ } } /** * 任务页创建(含 Whatshub skipUserAgent) * 温池复用 Browser 时每次 newPage,冷启动 Real Browser 可使用 launch 返回的首个 Tab */ async function createTaskPage(browser, warmPage, pageUrl, antiBot, mergedCookies, extraOpts = {}) { const opts = buildManagedPageOptions(pageUrl, antiBot, mergedCookies, extraOpts); const useWarmPage = !!(warmPage && extraOpts.useWarmPage !== false && !extraOpts.forceNewPage); const page = useWarmPage ? warmPage : await browser.newPage(); page.setDefaultNavigationTimeout(NAVIGATION_TIMEOUT_MS); page.setDefaultTimeout(PAGE_DEFAULT_TIMEOUT_MS); if (!opts.skipUserAgent) { await page.setUserAgent(opts.userAgent || DEFAULT_UA); } if (opts.viewport) { await page.setViewport(opts.viewport); } if (opts.cookies && opts.cookies.length > 0) { await page.setCookie(...opts.cookies); } if (extraOpts.useCaptchaProxy && hasCaptchaProxy()) { await applyCaptchaProxyToPage(page); } page.on('error', (err) => console.error('[Page Error]', err.message)); page.on('pageerror', (err) => console.error('[Page JS Error]', err.message)); if (antiBot?.enabled) { attachSitekeyCapture(page, browser); } return page; } /** CF 已过但仍在短链、未到业务页(Whatshub / A2C yyk.ink 等) */ function isOnShortLinkNotBusiness(page, urlNeedle) { const url = page.url(); if (urlNeedle && url.includes(urlNeedle)) { return false; } if (isWhatshubShortLinkPath(url)) { return true; } if (urlNeedle && urlNeedle.includes('/visitors/counter/share')) { try { return !new URL(url).hostname.includes('a2c.chat'); } catch (_) { return false; } } if (urlNeedle && urlNeedle.includes('/gds')) { return !url.includes('/gds'); } return false; } /** * CF 完成后等待 Vue SPA 从短链跳转到 work-order-sharing(对齐 debug 脚本 goto+8s) * Whatshub:二次 goto 短链(保留 clearance),禁止 reload(会重新触发 __cf_chl_rt_tk) */ async function waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettledFn, pageUrl = '') { const urlNeedle = (antiBot.postCfReadyUrlContains || '').trim(); const targetUrl = pageUrl || page.url(); const onA2cDirect = isA2cDirectSharePage(page, urlNeedle); if (!urlNeedle || (!isOnShortLinkNotBusiness(page, urlNeedle) && !onA2cDirect)) { return isBusinessUrlReady(page, urlNeedle); } if (!(await isCfTrulyComplete(page, antiBot))) { return false; } const spaWaitMs = onA2cDirect ? Math.max(8000, antiBot.postCfSpaWaitMs || 8000) : Math.max(8000, antiBot.postCfSpaWaitMs || 8000); const isWhatshub = isWhatshubShortLinkPath(targetUrl); console.log( `[CF] CF 已完成,等待 SPA ${onA2cDirect ? '直链挂载' : '跳转'} settle=${spaWaitMs}ms current=${page.url()}` ); await new Promise((resolve) => setTimeout(resolve, spaWaitMs)); if (typeof waitForUrlSettledFn === 'function') { await waitForUrlSettledFn(page).catch(() => {}); } await page.waitForNetworkIdle({ idleTime: 500, timeout: 20000 }).catch(() => {}); if (onA2cDirect) { await waitForPostCfListApi(page, antiBot, Math.max(15000, antiBot.postCfReadyTimeoutMs || 60000)).catch(() => {}); } if (isBusinessUrlReady(page, urlNeedle)) { console.log(`[CF] SPA 已跳转至业务页 url=${page.url()}`); return true; } if (isWhatshub && isWhatshubShortLinkPath(targetUrl)) { console.log(`[CF] Whatshub 二次 goto 触发 SPA 路由(保留 clearance)url=${targetUrl}`); await navigateToPage(page, targetUrl); if (typeof waitForUrlSettledFn === 'function') { await waitForUrlSettledFn(page).catch(() => {}); } if (!(await isCfTrulyComplete(page, antiBot))) { console.log('[CF] 二次 goto 后 CF 未完成,等待过盾...'); const cfWait = await waitForCfTrulyComplete(page, antiBot.challengeTimeoutMs || 60000, 500, antiBot); if (!cfWait.success) { console.log(`[CF] SPA 等待后仍未到业务页 url=${page.url()}`); return false; } } console.log(`[CF] Whatshub 二次 goto 后 SPA 等待 settle=${spaWaitMs}ms current=${page.url()}`); await new Promise((resolve) => setTimeout(resolve, spaWaitMs)); await page.waitForNetworkIdle({ idleTime: 500, timeout: 15000 }).catch(() => {}); if (isBusinessUrlReady(page, urlNeedle)) { console.log(`[CF] 二次 goto 后 SPA 已跳转 url=${page.url()}`); return true; } } console.log(`[CF] SPA 等待后仍未到业务页 url=${page.url()}`); return false; } /** 非 Whatshub 场景:软 reload;Whatshub 短链改用二次 goto */ async function softReloadForSpa(page, waitForUrlSettledFn, antiBot, pageUrl = '') { const targetUrl = pageUrl || page.url(); if (isWhatshubShortLinkPath(targetUrl)) { console.log(`[CF] Whatshub 跳过 soft reload,改用二次 goto url=${targetUrl}`); return waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettledFn, targetUrl); } console.log(`[CF] 尝试软 reload(保留 clearance)url=${page.url()}`); await page.reload({ waitUntil: 'domcontentloaded', timeout: NAVIGATION_TIMEOUT_MS }).catch(() => {}); if (typeof waitForUrlSettledFn === 'function') { await waitForUrlSettledFn(page).catch(() => {}); } return waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettledFn, targetUrl); } /** CF 是否真正完成:默认需 cf_clearance;A2C 等 cfClearanceRequired=false 时仅看真实挑战 */ async function isCfTrulyComplete(page, antiBot = null) { if (urlIndicatesCfChallenge(page.url())) { return false; } if (antiBot && antiBot.cfClearanceRequired === false) { const state = await detectCloudflare(page, antiBot); return !state.blocked; } const cookies = await page.cookies(); if (!cookies.some((c) => c.name === 'cf_clearance')) { return false; } const state = await detectCloudflare(page, antiBot); return !state.blocked; } /** 轮询直到 CF 真正完成;并行扫描 sitekey 供 CapSolver 提前使用 */ async function waitForCfTrulyComplete(page, timeoutMs, pollMs = 500, antiBot = null) { const { getSitekeyCapture } = require('./lib/sitekey-capture'); const capture = getSitekeyCapture(page); const started = Date.now(); while (Date.now() - started < timeoutMs) { if (await isCfTrulyComplete(page, antiBot)) { return { success: true, elapsedMs: Date.now() - started }; } if (capture) { await capture.scanNow(page, antiBot).catch(() => {}); } await new Promise((resolve) => setTimeout(resolve, pollMs)); } return { success: false, elapsedMs: Date.now() - started }; } /** 校验 cfResult,防止 handleCloudflareChallenge 假 success */ async function finalizeCfResult(page, cfResult, antiBot = null) { if (!cfResult || !cfResult.success) { return cfResult; } if (await isCfTrulyComplete(page, antiBot)) { return cfResult; } console.log(`[CF] 过盾返回 success 但 CF 未真正完成 url=${page.url()}`); return { success: false, code: 'CF_TURNSTILE_FAILED', challengeType: cfResult.challengeType || 'turnstile', stage: 'incomplete', solverUsed: cfResult.solverUsed || false, elapsedMs: cfResult.elapsedMs || 0, cfDetected: true, }; } /** * 完整过盾:内置 Real Browser Turnstile 轮询 → Captcha API 兜底 */ async function runCfChallengeFlow(page, antiBot, waitForUrlSettled, pageUrl = '') { const started = Date.now(); const timeoutMs = Math.min(Math.max(Number(antiBot.challengeTimeoutMs) || 60000, 5000), 120000); const navUrl = pageUrl || page.url(); console.log(`[CF] 开始过盾 url=${page.url()}`); if (waitForUrlSettled) { await waitForUrlSettled(page).catch(() => {}); } if (antiBot.cfClearanceRequired === false) { const preState = await detectCloudflare(page, antiBot); if (!preState.blocked) { console.log('[CF] 免 clearance 模式:无真实 CF 挑战,跳过 Turnstile/CapSolver'); await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettled, navUrl); return { success: true, code: null, challengeType: null, stage: 'no_clearance_required', solverUsed: false, elapsedMs: Date.now() - started, cfDetected: false, }; } console.log(`[CF] 免 clearance 模式但检测到真实 CF 挑战 type=${preState.challengeType},继续过盾`); } const preferManaged = shouldPreferManagedCloudflareSolver(antiBot); if (preferManaged && antiBot.solverFallback !== false && isCaptchaConfigured()) { console.log('[CF] Managed Challenge:跳过内置 60s 轮询,直接使用 CapSolver AntiCloudflareTask'); try { if (waitForUrlSettled) { await waitForUrlSettled(page).catch(() => {}); } await runCaptchaApiFallback(page, { antiBot, navUrl, waitForUrlSettled, timeoutMs, }); const apiWait = await waitForCfTrulyComplete(page, Math.min(timeoutMs, 60000), 500, antiBot); if (apiWait.success) { if (waitForUrlSettled) { await waitForUrlSettled(page).catch(() => {}); } await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettled, navUrl); return { success: true, code: null, challengeType: 'managed', stage: 'api_managed', solverUsed: true, elapsedMs: Date.now() - started, cfDetected: true, }; } console.warn('[CF] Managed CapSolver 注入后仍未获得有效 cf_clearance'); } catch (apiErr) { console.error('[CF] Managed Challenge CapSolver 失败:', apiErr.message); return { success: false, code: 'CF_TURNSTILE_FAILED', challengeType: 'managed', stage: 'api_managed', solverUsed: true, elapsedMs: Date.now() - started, cfDetected: true, }; } return { success: false, code: 'CF_TURNSTILE_FAILED', challengeType: 'managed', stage: 'api_managed', solverUsed: true, elapsedMs: Date.now() - started, cfDetected: true, }; } if (antiBot.turnstile !== false) { console.log(`[CF] 内置 Turnstile 轮询 timeout=${timeoutMs}ms`); const builtIn = await waitForCfTrulyComplete(page, timeoutMs, 500, antiBot); if (builtIn.success) { if (waitForUrlSettled) { await waitForUrlSettled(page).catch(() => {}); } await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettled, navUrl); return { success: true, code: null, challengeType: 'turnstile', stage: 'built_in', solverUsed: false, elapsedMs: Date.now() - started, cfDetected: true, }; } console.warn(`[CF] 内置 Turnstile 未在 ${builtIn.elapsedMs}ms 内完成,尝试 Captcha API`); } if (antiBot.solverFallback !== false && isCaptchaConfigured()) { try { if (waitForUrlSettled) { await waitForUrlSettled(page).catch(() => {}); } await runCaptchaApiFallback(page, { antiBot, navUrl, waitForUrlSettled, timeoutMs, }); const apiWait = await waitForCfTrulyComplete(page, Math.min(timeoutMs, 45000), 500, antiBot); if (apiWait.success) { if (waitForUrlSettled) { await waitForUrlSettled(page).catch(() => {}); } await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettled, navUrl); return { success: true, code: null, challengeType: 'turnstile', stage: 'api', solverUsed: true, elapsedMs: Date.now() - started, cfDetected: true, }; } console.warn('[CF] Captcha API token 注入后仍未获得 cf_clearance'); } catch (apiErr) { console.error('[CF] Captcha API 兜底失败:', apiErr.message); return { success: false, code: 'CF_TURNSTILE_FAILED', challengeType: 'turnstile', stage: 'api', solverUsed: true, elapsedMs: Date.now() - started, cfDetected: true, }; } } return { success: false, code: 'CF_TURNSTILE_FAILED', challengeType: 'turnstile', stage: antiBot.solverFallback !== false && !isCaptchaConfigured() ? 'built_in' : 'timeout', solverUsed: false, elapsedMs: Date.now() - started, cfDetected: true, }; } /** 安全 CF 处理:未真正完成则走完整过盾,禁止 cf-handler 假过关 */ async function handleCloudflareChallengeSafe(page, antiBot, waitForUrlSettled, storedSession) { if (!(await isCfTrulyComplete(page, antiBot))) { return finalizeCfResult(page, await runCfChallengeFlow(page, antiBot, waitForUrlSettled), antiBot); } const result = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled, storedSession); return finalizeCfResult(page, result, antiBot); } /** 是否应 purge 陈旧 clearance(短链 SPA 未跳转时禁止 purge) */ function shouldPurgeStaleClearance(page, cfState, urlNeedle = '') { if (urlIndicatesCfChallenge(page.url())) { return false; } if (urlNeedle && isOnShortLinkNotBusiness(page, urlNeedle) && cfState.hasCfClearance) { return false; } return cfState.hasCfClearance && !cfState.blocked; } /** 业务页等待前确保 CF 已完成 */ async function ensureCfCompleteBeforeBusinessReady(page, antiBot, waitForUrlSettledFn, storedSession) { if (await isCfTrulyComplete(page, antiBot)) { return; } const cfState = await detectCloudflare(page, antiBot); console.log(`[CF] 业务页等待前 CF 未完成 blocked=${cfState.blocked} url=${page.url()}`); const cfResult = await handleCloudflareChallengeSafe(page, antiBot, waitForUrlSettledFn, storedSession); if (!cfResult.success) { const err = new Error('Cloudflare Turnstile 验证失败(业务页等待前)'); err.code = 'CF_TURNSTILE_FAILED'; throw err; } if (typeof waitForUrlSettledFn === 'function') { await waitForUrlSettledFn(page).catch(() => {}); } } /** 清除 cf_clearance(含 Chrome Profile / CDP 层) */ async function clearCfClearanceCookies(page) { const cookies = await page.cookies(); for (const c of cookies) { if (c.name !== 'cf_clearance') { continue; } const host = (c.domain || '').replace(/^\./, ''); if (host) { await page.deleteCookie({ name: c.name, url: `https://${host}/` }).catch(() => {}); } await page.deleteCookie({ name: c.name, domain: c.domain, path: c.path || '/', }).catch(() => {}); } } async function purgeCfClearanceFully(page, pageUrl) { await clearCfClearanceCookies(page); let hostname = ''; try { hostname = new URL(pageUrl).hostname; } catch (_) { return; } try { const cdp = await page.createCDPSession(); const domains = new Set([hostname, `.${hostname}`]); const cookies = await page.cookies(); for (const c of cookies) { if (c.domain) { domains.add(c.domain); } } for (const domain of domains) { await cdp.send('Network.deleteCookies', { name: 'cf_clearance', domain }).catch(() => {}); } } catch (err) { console.warn('[CF] CDP 清除 cf_clearance 失败:', err.message); } } /** * Whatshub 短链首访:清除 Chrome Profile 内陈旧 cf_clearance,确保 Turnstile+SPA 在同一 goto 生命周期 */ async function prepareWhatshubShortLinkBeforeNav(page, pageUrl, antiBot) { if (!antiBot?.enabled || antiBot.profile !== 'real' || !isWhatshubShortLinkPath(pageUrl)) { return; } const cookies = await page.cookies(); if (!cookies.some((c) => c.name === 'cf_clearance')) { return; } console.log('[Whatshub] 短链首访:清除 Profile/页面内陈旧 cf_clearance'); await purgeCfClearanceFully(page, pageUrl); } /** * CF 处理 + 业务 URL 守卫:磁盘 session 陈旧 clearance 时 purge;__cf_chl 中间态只过盾不 purge */ async function runCloudflareWithBusinessGuard(page, antiBot, waitForUrlSettled, storedSession, pageUrl, browserCtx = null) { const urlNeedle = (antiBot.postCfReadyUrlContains || '').trim(); const hasStoredSession = !!(storedSession && isSessionLikelyValid(storedSession)); async function maybeEnsureProxy(currentPage) { if (!browserCtx) { return currentPage; } browserCtx.page = currentPage; await ensureCaptchaProxyBrowserIfNeeded(browserCtx); return browserCtx.page; } async function reNavigateWithCfPurge(label) { if (urlIndicatesCfChallenge(page.url())) { console.log(`[CF] ${label}: URL 含 __cf_chl,跳过 purge,直接过盾`); return handleCloudflareChallengeSafe(page, antiBot, waitForUrlSettled, storedSession); } console.log(`[CF] ${label}: 清除陈旧 CF 并重新 goto ${pageUrl} (current=${page.url()})`); await purgeCfClearanceFully(page, pageUrl); await navigateToPage(page, pageUrl); if (waitForUrlSettled) { await waitForUrlSettled(page).catch(() => {}); } page = await maybeEnsureProxy(page); return handleCloudflareChallengeSafe(page, antiBot, waitForUrlSettled, storedSession); } let cfResult; if (urlNeedle && !isBusinessUrlReady(page, urlNeedle) && hasStoredSession) { const pre = await detectCloudflare(page, antiBot); if (shouldPurgeStaleClearance(page, pre, urlNeedle)) { cfResult = await reNavigateWithCfPurge('预检(磁盘会话-陈旧clearance)'); } else if (await isCfTrulyComplete(page, antiBot)) { await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettled, pageUrl); } } if (!cfResult) { page = await maybeEnsureProxy(page); cfResult = await handleCloudflareChallengeSafe(page, antiBot, waitForUrlSettled, storedSession); } // 短链跨域落地(如 yyk.ink → user.a2c.chat)后,业务域可能仍有 CF if (cfResult.success && !(await isCfTrulyComplete(page, antiBot))) { const crossState = await detectCloudflare(page, antiBot); if (crossState.blocked || urlIndicatesCfChallenge(page.url())) { console.log(`[CF] 跨域落地后 CF 未完成,二次过盾 url=${page.url()}`); page = await maybeEnsureProxy(page); cfResult = await finalizeCfResult( page, await runCfChallengeFlow(page, antiBot, waitForUrlSettled, page.url()), antiBot ); } } if (cfResult.success && urlNeedle && !isBusinessUrlReady(page, urlNeedle)) { await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettled, pageUrl); } let retries = 0; while ( cfResult.success && urlNeedle && !isBusinessUrlReady(page, urlNeedle) && retries < 2 ) { retries += 1; if (!(await isCfTrulyComplete(page, antiBot))) { console.log(`[CF] 业务页未达且 CF 未完成,完整过盾 重试#${retries}`); page = await maybeEnsureProxy(page); cfResult = await finalizeCfResult( page, await runCfChallengeFlow(page, antiBot, waitForUrlSettled, pageUrl), antiBot ); } else { console.log(`[CF] CF 已完成但未到业务页,等待 SPA 跳转 重试#${retries}`); const spaOk = await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettled, pageUrl); if (!spaOk && retries >= 2) { await softReloadForSpa(page, waitForUrlSettled, antiBot, pageUrl); } if (isBusinessUrlReady(page, urlNeedle)) { break; } } if (!cfResult.success) { break; } } if (browserCtx) { browserCtx.page = page; } return finalizeCfResult(page, cfResult, antiBot); } async function collectPageDiagnostics(page) { const url = page.url(); const snapshot = await page.evaluate(() => ({ url: window.location.href, title: document.title || '', bodyText: (document.body && document.body.innerText ? document.body.innerText : '').slice(0, 500), elInputInner: document.querySelectorAll('.el-input__inner').length, elTable: document.querySelectorAll('.el-table').length, btnNext: document.querySelectorAll('.btn-next').length, elPagination: document.querySelectorAll('.el-pagination').length, })).catch(() => null); return snapshot || { url, title: '', bodyText: '', elInputInner: 0 }; } async function awaitPostCfBusinessReady(page, antiBot, waitForUrlSettledFn, storedSession = null, pageUrl = '') { if (!antiBot || !antiBot.enabled) { return; } const urlContains = (antiBot.postCfReadyUrlContains || '').trim(); const selector = (antiBot.postCfReadySelector || '').trim(); if (urlContains === '' && selector === '') { return; } const navUrl = pageUrl || page.url(); await ensureCfCompleteBeforeBusinessReady(page, antiBot, waitForUrlSettledFn, storedSession); const onA2cDirect = isA2cDirectSharePage(page, urlContains); let skipSpaWait = false; if (onA2cDirect && await isCfTrulyComplete(page, antiBot)) { const domReady = selector ? await page.$(selector).catch(() => null) : true; if (domReady) { console.log('[CF] A2C 业务 DOM 已就绪,跳过重复 SPA 等待'); skipSpaWait = true; } } if (!skipSpaWait) { await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettledFn, navUrl); } const timeoutMs = Math.max(5000, antiBot.postCfReadyTimeoutMs || 30000); console.log( `[CF] 等待业务页就绪 current=${page.url()} urlContains="${urlContains}" ` + `selector="${selector}" timeout=${timeoutMs}ms` ); try { if (urlContains !== '') { await page.waitForFunction( (needle) => window.location.href.includes(needle), { timeout: timeoutMs, polling: 200 }, urlContains ); } await waitForPostCfListApi(page, antiBot, timeoutMs); if (selector !== '') { await waitForPostCfSelector(page, selector, timeoutMs); } } catch (err) { const retryMs = Math.min(30000, timeoutMs); const needsRetry = urlContains !== '' && !page.url().includes(urlContains); const needsDomRetry = urlContains !== '' && page.url().includes(urlContains) && isA2cDirectSharePage(page, urlContains); if (needsRetry) { if (!(await isCfTrulyComplete(page, antiBot))) { console.warn(`[CF] 业务页等待失败且 CF 未完成 url=${page.url()},完整过盾后重试`); try { await ensureCfCompleteBeforeBusinessReady(page, antiBot, waitForUrlSettledFn, storedSession); if (urlContains !== '') { await page.waitForFunction( (needle) => window.location.href.includes(needle), { timeout: retryMs, polling: 200 }, urlContains ); } if (selector !== '') { await waitForPostCfSelector(page, selector, retryMs); } } catch (retryErr) { retryErr.pageDiagnostics = await collectPageDiagnostics(page); throw retryErr; } } else { console.warn(`[CF] 业务页等待失败 url=${page.url()},SPA 等待 + 二次 goto`); try { await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettledFn, navUrl); if (!page.url().includes(urlContains)) { await softReloadForSpa(page, waitForUrlSettledFn, antiBot, navUrl); } if (urlContains !== '') { await page.waitForFunction( (needle) => window.location.href.includes(needle), { timeout: retryMs, polling: 200 }, urlContains ); } await waitForPostCfListApi(page, antiBot, retryMs); if (selector !== '') { await waitForPostCfSelector(page, selector, retryMs); } } catch (retryErr) { retryErr.pageDiagnostics = await collectPageDiagnostics(page); throw retryErr; } } } else if (needsDomRetry) { console.warn(`[CF] A2C URL 已就绪但 DOM 未挂载 url=${page.url()},soft reload 后重试`); try { await page.waitForNetworkIdle({ idleTime: 500, timeout: 20000 }).catch(() => {}); await softReloadForSpa(page, waitForUrlSettledFn, antiBot, navUrl); await waitForPostCfListApi(page, antiBot, retryMs); if (selector !== '') { await waitForPostCfSelector(page, selector, retryMs); } } catch (retryErr) { retryErr.pageDiagnostics = await collectPageDiagnostics(page); throw retryErr; } } else { err.pageDiagnostics = await collectPageDiagnostics(page); throw err; } } const settleMs = antiBot.postCfSettleMs || 0; if (settleMs > 0) { await new Promise((resolve) => setTimeout(resolve, settleMs)); } console.log(`[CF] 业务页就绪 url=${page.url()}`); } async function executeAuthActionsWithDiagnostics(page, authActions) { try { await executeAuthActions(page, authActions); } catch (err) { err.pageDiagnostics = await collectPageDiagnostics(page); throw err; } } async function executeAuthActions(page, authActions) { if (!Array.isArray(authActions)) return; for (const action of authActions) { if (action.type === 'type') { await page.waitForSelector(action.selector, { timeout: 15000 }); await page.type(action.selector, action.value, { delay: 100 }); } else if (action.type === 'click') { await page.waitForSelector(action.selector, { timeout: 15000 }); await page.evaluate((sel) => { const el = document.querySelector(sel); if (el) el.click(); }, action.selector); } else if (action.type === 'press') { await new Promise((r) => setTimeout(r, 200)); await page.keyboard.press(action.key); } else if (action.type === 'wait') { await new Promise((r) => setTimeout(r, action.ms)); } else if (action.type === 'vue_fill') { await page.waitForSelector(action.selector, { timeout: 15000 }); await page.waitForFunction( (sel) => { const el = document.querySelector(sel); if (!el) return false; const rect = el.getBoundingClientRect(); return rect.width > 0 && rect.height > 0; }, { timeout: 2000, polling: 100 }, action.selector ).catch(() => new Promise((r) => setTimeout(r, 500))); await page.evaluate((sel, val) => { const inputs = document.querySelectorAll(sel); for (const input of inputs) { if (input.getBoundingClientRect().width > 0 && input.getBoundingClientRect().height > 0) { input.value = val; input.dispatchEvent(new Event('input', { bubbles: true })); input.dispatchEvent(new Event('change', { bubbles: true })); input.focus(); return; } } }, action.selector, action.value); } else if (action.type === 'vue_click') { await page.waitForSelector(action.selector, { timeout: 15000 }); await page.waitForFunction( (sel) => { const el = document.querySelector(sel); if (!el) return false; const rect = el.getBoundingClientRect(); return rect.width > 0 && rect.height > 0; }, { timeout: 2000, polling: 100 }, action.selector ).catch(() => new Promise((r) => setTimeout(r, 500))); await page.evaluate((sel, matchText) => { const btns = document.querySelectorAll(sel); for (const btn of btns) { if (btn.getBoundingClientRect().width > 0 && btn.getBoundingClientRect().height > 0) { let shouldClick = false; if (matchText) { const text = (btn.textContent || btn.innerText || '').trim(); if (text.includes(matchText.trim())) { shouldClick = true; } } else { shouldClick = true; } if (shouldClick) { btn.scrollIntoView({ behavior: 'instant', block: 'center' }); const eventOpts = { bubbles: true, cancelable: true, view: window }; btn.dispatchEvent(new MouseEvent('mousedown', eventOpts)); btn.dispatchEvent(new MouseEvent('mouseup', eventOpts)); btn.dispatchEvent(new MouseEvent('click', eventOpts)); btn.click(); return; } } } }, action.selector, action.text); } } } function createApiInterceptor(page, apiUrls, defaultTimeoutMs = API_INTERCEPT_TIMEOUT_MS) { const interceptedApis = {}; let responseHandler = null; let timeoutId = null; let resolveWait = null; let waitPromise = null; const tryResolveWait = () => { if (!resolveWait || Object.keys(interceptedApis).length < apiUrls.length) { return; } if (timeoutId) { clearTimeout(timeoutId); timeoutId = null; } if (responseHandler) { page.off('response', responseHandler); responseHandler = null; } const done = resolveWait; resolveWait = null; done(); }; const cleanup = () => { if (timeoutId) { clearTimeout(timeoutId); timeoutId = null; } if (responseHandler) { page.off('response', responseHandler); responseHandler = null; } if (resolveWait) { const done = resolveWait; resolveWait = null; done(); } }; responseHandler = async (response) => { const request = response.request(); if (request.method() === 'OPTIONS') return; const matchedPath = apiUrls.find((api) => request.url().includes(api) && !interceptedApis[api]); if (matchedPath && response.status() >= 200 && response.status() < 300) { try { interceptedApis[matchedPath] = { url: request.url(), headers: request.headers(), data: JSON.parse(await response.text()), }; tryResolveWait(); } catch (_) { /* JSON 解析失败跳过 */ } } }; page.on('response', responseHandler); const beginWaiting = (timeoutMs = defaultTimeoutMs) => { if (waitPromise) { return waitPromise; } if (Object.keys(interceptedApis).length >= apiUrls.length) { waitPromise = Promise.resolve(); return waitPromise; } waitPromise = new Promise((resolve) => { resolveWait = resolve; timeoutId = setTimeout(() => { cleanup(); resolve(); }, timeoutMs); }); return waitPromise; }; const waitForApis = async (timeoutMs = defaultTimeoutMs) => { await beginWaiting(timeoutMs); return waitPromise; }; return { interceptedApis, beginWaiting, waitForApis, cleanup }; } async function seedShareTokenCookie(page, pageUrl) { try { const parsed = new URL(pageUrl); const token = parsed.searchParams.get('token'); if (!token) return; const cookieUrl = `http://${parsed.hostname}/`; await page.setCookie({ name: 'share_token', value: token, url: cookieUrl }); } catch (_) { /* ignore */ } } function resolveShareToken(...urls) { for (const url of urls) { const token = extractShareTokenFromUrl(url); if (token) return token; } return ''; } function resolveInterceptTimeout(antiBot) { const cfg = normalizeAntiBot(antiBot); return cfg.enabled && cfg.profile === 'real' ? API_INTERCEPT_TIMEOUT_REAL_MS : API_INTERCEPT_TIMEOUT_MS; } /** 并发监控:供上游调度器判断负载 */ app.get('/api/stats', (_req, res) => { const executablePath = resolveChromeExecutable(); const factoryStats = getFactoryStats(); res.json({ success: true, shuttingDown, queue: factoryStats.standard, queueReal: factoryStats.real, pool: factoryStats.pool, sessions: getSessionStats(), profiles: getProfileStats(), realBrowserReady: factoryStats.realBrowserReady, captchaConfigured: isCaptchaConfigured(), chrome: { executablePath, exists: fs.existsSync(executablePath), }, config: { maxConcurrentBrowsers: MAX_CONCURRENT_BROWSERS, maxConcurrentBrowsersReal: MAX_CONCURRENT_BROWSERS_REAL, apiInterceptTimeoutMs: API_INTERCEPT_TIMEOUT_MS, apiInterceptTimeoutRealMs: API_INTERCEPT_TIMEOUT_REAL_MS, navigationTimeoutMs: NAVIGATION_TIMEOUT_MS, navigationWaitUntil: NAVIGATION_WAIT_UNTIL, sessionTtlMs: SESSION_TTL_MS, persistBrowserProfile: PERSIST_BROWSER_PROFILE, poolIdleMs: POOL_IDLE_MS, maxPooledBrowsers: MAX_POOLED_BROWSERS, }, }); }); /** 运维调试:查看指定 sessionKey 的磁盘会话与 profile 状态 */ app.get('/api/session/:sessionKey', (req, res) => { const sessionKey = decodeURIComponent(req.params.sessionKey || ''); if (!sessionKey) { return res.status(400).json({ success: false, error: 'sessionKey 不能为空' }); } const info = getSessionDebugInfo(sessionKey); const profileStandard = getProfileStats(); res.json({ success: true, session: info, profileExists: { standard: profileExists(sessionKey, 'standard'), real: profileExists(sessionKey, 'real'), }, profilesDir: profileStandard.profilesDir, }); }); /** 运维自检:实际尝试 launch 一次 Browser(profile=standard|real) */ app.get('/api/browser-launch-test', async (req, res) => { const profile = req.query.profile === 'real' ? 'real' : 'standard'; let browser; try { await runBrowserTask(`browser-launch-test-${profile}`, async () => { if (profile === 'real') { const session = await createBrowserSession({ profile: 'real', extraArgs: ['--mute-audio'] }); browser = session.browser; const pages = await browser.pages(); return { pageCount: pages.length, profile }; } browser = await launchStandardBrowser(['--mute-audio']); const pages = await browser.pages(); return { pageCount: pages.length, profile }; }, profile); res.json({ success: true, message: 'Browser 启动成功', profile }); } catch (error) { res.status(500).json({ success: false, error: error.message, profile, chrome: { executablePath: resolveChromeExecutable(), exists: fs.existsSync(resolveChromeExecutable()), }, realBrowserReady: isRealBrowserAvailable(), runtimeDir: RUNTIME_DIR, }); } finally { if (browser) { try { await browser.close(); } catch (_) { /* ignore */ } } } }); /** 接口一:初始化授权与首屏拦截 */ app.post('/api/auth-and-intercept', async (req, res) => { const { pageUrl, apiUrls, authActions, antiBot: rawAntiBot } = req.body; if (!pageUrl || !Array.isArray(apiUrls)) { return res.status(400).json({ success: false, error: '参数错误' }); } const antiBot = resolveAntiBotForPage(rawAntiBot, pageUrl); const profile = antiBot.profile || 'standard'; const interceptTimeout = resolveInterceptTimeout(antiBot); const cfLog = { profile, cfDetected: false, turnstileStage: null, solverUsed: false, elapsedMs: 0, browserReused: false }; try { await runBrowserTask('auth-and-intercept', async () => { let browserSession = null; let interceptorCleanup = null; let page = null; const taskStarted = Date.now(); let cfDestroyBrowser = false; try { browserSession = await acquireBrowserSession({ profile, extraArgs: ['--mute-audio'], sessionKey: antiBot.sessionKey || '', }); cfLog.browserReused = !!browserSession.reused; const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl); const navigationUrl = resolveNavigationUrl(pageUrl, antiBot); page = await createTaskPage( browserSession.browser, browserSession.reused ? null : browserSession.page, navigationUrl, antiBot, mergedCookies ); const httpResolvedUrl = await resolveHttpRedirects(pageUrl).catch(() => pageUrl); if (httpResolvedUrl !== pageUrl) { console.log(`[URL解析] HTTP 重定向: ${pageUrl} -> ${httpResolvedUrl}`); } let shareToken = resolveShareToken(pageUrl, httpResolvedUrl); await seedShareTokenCookie(page, navigationUrl); if (httpResolvedUrl !== pageUrl) { await seedShareTokenCookie(page, httpResolvedUrl); } await prepareWhatshubShortLinkBeforeNav(page, navigationUrl, antiBot); await navigateToPage(page, navigationUrl); let finalPageUrl = await waitForUrlSettled(page); applyHuojianBusinessHostHint(antiBot, finalPageUrl); if (finalPageUrl !== pageUrl) { console.log(`[URL解析] 浏览器落地: ${pageUrl} -> ${finalPageUrl}`); } const browserCtx = { browserSession, page, antiBot, profile, navigationUrl, mergedCookies, baseExtraArgs: ['--mute-audio'], pageExtraOpts: {}, afterRelaunchNav: async (p) => { await seedShareTokenCookie(p, navigationUrl); }, }; await ensureCaptchaProxyBrowserIfNeeded(browserCtx); browserSession = browserCtx.browserSession; page = browserCtx.page; let interceptor; if (antiBot.enabled) { const cfResult = await runCloudflareWithBusinessGuard( page, antiBot, waitForUrlSettled, storedSession, pageUrl, browserCtx ); page = browserCtx.page; browserSession = browserCtx.browserSession; cfLog.cfDetected = cfResult.cfDetected; cfLog.turnstileStage = cfResult.stage; cfLog.solverUsed = cfResult.solverUsed; cfLog.elapsedMs = cfResult.elapsedMs; if (!cfResult.success) { cfDestroyBrowser = true; res.status(422).json({ success: false, code: cfResult.code || 'CF_TURNSTILE_FAILED', challengeType: cfResult.challengeType || 'turnstile', stage: cfResult.stage || 'timeout', error: 'Cloudflare Turnstile 验证失败', cf: cfLog, }); return; } finalPageUrl = page.url(); applyHuojianBusinessHostHint(antiBot, finalPageUrl); touchSessionIfPresent(antiBot.sessionKey); await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled, storedSession, pageUrl); } const finalShareToken = resolveShareToken(finalPageUrl); if (finalShareToken && finalShareToken !== shareToken) { shareToken = finalShareToken; await seedShareTokenCookie(page, finalPageUrl); } interceptor = createApiInterceptor(page, apiUrls, interceptTimeout); interceptorCleanup = interceptor.cleanup; await attachHttpIpRewriteInterceptor(page, 'auth-and-intercept', { shareToken }); await executeAuthActionsWithDiagnostics(page, authActions); await interceptor.waitForApis(interceptTimeout); const rawCookies = await page.cookies(); const cookiePayload = rawCookies.map((c) => ({ name: c.name, value: c.value, domain: c.domain, path: c.path, })); if (antiBot.sessionKey) { let lastHost = ''; try { lastHost = new URL(finalPageUrl).hostname; } catch (_) { /* ignore */ } saveSession(antiBot.sessionKey, cookiePayload, SESSION_TTL_MS, { lastHost, savedProfile: profile, }); } res.json({ success: true, pageUrl, finalPageUrl, interceptedApis: interceptor.interceptedApis, cookies: cookiePayload, cf: cfLog, profile, browserReused: browserSession.reused, elapsedMs: Date.now() - taskStarted, }); } finally { if (interceptorCleanup) interceptorCleanup(); await closeTaskPage(page); if (browserSession) { await browserSession.release(cfDestroyBrowser); } } }, profile); } catch (error) { if (!res.headersSent) sendTaskError(res, error, { cf: cfLog, profile }); } }); /** 接口二:脱离 UI 极速并发拉取 */ app.post('/api/batch-fetch', async (req, res) => { const { tasks, cookies, antiBot: rawAntiBot } = req.body; if (!Array.isArray(tasks) || tasks.length === 0) { return res.status(400).json({ success: false, error: 'tasks 参数错误' }); } const antiBot = normalizeAntiBot(rawAntiBot); const profile = antiBot.profile || 'standard'; try { await runBrowserTask('batch-fetch', async () => { let browserSession = null; try { browserSession = await acquireBrowserSession({ profile, extraArgs: ['--disable-web-security'], sessionKey: antiBot.sessionKey || '', }); const firstUrl = tasks[0].fullUrl; const { mergedCookies } = resolveSessionContext(antiBot, firstUrl, cookies || []); const page = await createManagedPage(browserSession.browser, browserSession.page, { cookies: mergedCookies }); const firstOrigin = new URL(firstUrl).origin; await page.setRequestInterception(true); page.on('request', (reqObj) => { if (reqObj.isInterceptResolutionHandled()) return; const url = rewriteHttpsToHttpForLiteralIp(reqObj.url()); const resourceType = reqObj.resourceType(); if (['image', 'media', 'font'].includes(resourceType)) { reqObj.abort(); return; } if (url === firstOrigin || url === firstOrigin + '/') { reqObj.respond({ status: 200, contentType: 'text/html', body: 'Context Ready' }); } else { reqObj.continue(); } }); await navigateToPage(page, firstOrigin); const FETCH_CONCURRENCY = Math.max(1, parseInt(process.env.FETCH_CONCURRENCY || '5', 10)); const FETCH_RETRY = 2; const fetchResults = await page.evaluate(async (taskList, concurrency, maxRetry) => { const out = {}; const createLimiter = (limit) => { let active = 0; const queue = []; return (fn) => new Promise((resolve, reject) => { const run = async () => { active++; try { resolve(await fn()); } catch (e) { reject(e); } finally { active--; if (queue.length) queue.shift()(); } }; if (active < limit) run(); else queue.push(run); }); }; const limit = createLimiter(concurrency); const fetchWithRetry = async (finalUrl, options, retriesLeft) => { try { const res = await fetch(finalUrl, options); return { success: true, data: await res.json() }; } catch (err) { if (retriesLeft <= 0) return { success: false, error: err.toString() }; await new Promise((r) => setTimeout(r, 300 * Math.pow(2, maxRetry - retriesLeft))); return fetchWithRetry(finalUrl, options, retriesLeft - 1); } }; const jobs = []; for (const task of taskList) { out[task.apiPath] = []; const cleanHeaders = { ...task.headers }; ['content-length', 'host', 'origin', 'referer', 'accept-encoding'].forEach((k) => delete cleanHeaders[k]); for (const params of task.paramList) { jobs.push(limit(async () => { let finalUrl = task.fullUrl; const options = { method: task.method.toUpperCase(), headers: cleanHeaders }; if (options.method === 'GET') { const qs = new URLSearchParams(params).toString(); finalUrl = finalUrl.includes('?') ? `${finalUrl}&${qs}` : `${finalUrl}?${qs}`; } else { options.body = JSON.stringify(params); } const result = await fetchWithRetry(finalUrl, options, maxRetry); out[task.apiPath].push({ params, ...result }); await new Promise((r) => setTimeout(r, 200)); })); } } await Promise.all(jobs); return out; }, tasks, FETCH_CONCURRENCY, FETCH_RETRY); res.json({ success: true, results: fetchResults, profile, browserReused: browserSession.reused }); } finally { if (browserSession) { await browserSession.release(false); } } }, profile); } catch (error) { if (!res.headersSent) sendTaskError(res, error, { profile }); } }); /** 接口三:强制 UI 模拟点击拉取 */ app.post('/api/ui-pagination', async (req, res) => { const { pageUrl, finalPageUrl, apiUrl, cookies, nextBtnSelector, clicksToPerform, waitMs = 2000, firstPageData, authActions, antiBot: rawAntiBot, } = req.body; const navigationUrl = finalPageUrl || pageUrl; const antiBot = resolveAntiBotForPage(rawAntiBot, pageUrl || navigationUrl); const profile = antiBot.profile || 'standard'; try { await runBrowserTask('ui-pagination', async () => { let browserSession = null; const debugLogs = []; let page = null; let cfDestroyBrowser = false; try { browserSession = await acquireBrowserSession({ profile, extraArgs: ['--window-size=1920,1080'], sessionKey: antiBot.sessionKey || '', }); const { storedSession, mergedCookies } = resolveSessionContext(antiBot, navigationUrl, cookies || []); page = await createTaskPage( browserSession.browser, browserSession.reused ? null : browserSession.page, navigationUrl, antiBot, mergedCookies, { viewport: { width: 1920, height: 1080 } } ); const log = (msg) => { console.log(msg); debugLogs.push(msg); }; log(`[启动] 准备访问页面: ${navigationUrl}`); if (finalPageUrl && finalPageUrl !== pageUrl) { log(`[URL解析] 使用 auth 阶段落地地址,跳过短链重定向: ${pageUrl} -> ${finalPageUrl}`); } const shareToken = resolveShareToken(finalPageUrl, pageUrl); await seedShareTokenCookie(page, navigationUrl); await prepareWhatshubShortLinkBeforeNav(page, navigationUrl, antiBot); await navigateToPage(page, navigationUrl); const settledPageUrl = await waitForUrlSettled(page); if (settledPageUrl !== navigationUrl) { log(`[URL解析] 翻页阶段落地: ${navigationUrl} -> ${settledPageUrl}`); } const browserCtx = { browserSession, page, antiBot, profile, navigationUrl, mergedCookies, baseExtraArgs: ['--window-size=1920,1080'], pageExtraOpts: { viewport: { width: 1920, height: 1080 } }, afterRelaunchNav: async (p) => { await seedShareTokenCookie(p, navigationUrl); }, }; await ensureCaptchaProxyBrowserIfNeeded(browserCtx); browserSession = browserCtx.browserSession; page = browserCtx.page; if (antiBot.enabled) { const cfResult = await runCloudflareWithBusinessGuard( page, antiBot, waitForUrlSettled, storedSession, navigationUrl, browserCtx ); page = browserCtx.page; browserSession = browserCtx.browserSession; if (!cfResult.success) { cfDestroyBrowser = true; res.status(422).json({ success: false, code: cfResult.code || 'CF_TURNSTILE_FAILED', challengeType: cfResult.challengeType, stage: cfResult.stage, error: 'Cloudflare Turnstile 验证失败(翻页阶段)', debug: debugLogs, }); return; } touchSessionIfPresent(antiBot.sessionKey); await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled, storedSession, navigationUrl); await attachHttpIpRewriteInterceptor(page, 'ui-pagination', { shareToken }); } else { await attachHttpIpRewriteInterceptor(page, 'ui-pagination', { shareToken }); } const paginationResult = await runUiPagination(page, { apiUrl, nextBtnSelector, clicksToPerform, waitMs, firstPageData, authActions, executeAuthActions, }); res.json({ success: true, data: paginationResult.data, debug: [...debugLogs, ...paginationResult.debugLogs], profile, browserReused: browserSession.reused, }); } catch (error) { if (!res.headersSent) { sendTaskError(res, error, { debug: debugLogs, profile }); } } finally { await closeTaskPage(page); if (browserSession) { await browserSession.release(cfDestroyBrowser); } } }, profile); } catch (error) { if (!res.headersSent) sendTaskError(res, error, { profile }); } }); /** 接口四:单 Browser 内 auth + 拦截 + UI 翻页(降低 CF 二次触发) */ app.post('/api/auth-intercept-and-paginate', async (req, res) => { const { pageUrl, apiUrls, authActions, antiBot: rawAntiBot, pagination, } = req.body; if (!pageUrl || !Array.isArray(apiUrls) || !pagination) { return res.status(400).json({ success: false, error: '参数错误:需要 pageUrl、apiUrls、pagination' }); } const { apiUrl, nextBtnSelector, clicksToPerform = 0, waitMs = 2000, firstPageData, } = pagination; const antiBot = resolveAntiBotForPage(rawAntiBot, pageUrl); const profile = antiBot.profile || 'standard'; const interceptTimeout = resolveInterceptTimeout(antiBot); const cfLog = { profile, cfDetected: false, turnstileStage: null, solverUsed: false, elapsedMs: 0, browserReused: false }; try { await runBrowserTask('auth-intercept-and-paginate', async () => { let browserSession = null; let interceptorCleanup = null; let page = null; const taskStarted = Date.now(); let cfDestroyBrowser = false; try { browserSession = await acquireBrowserSession({ profile, extraArgs: ['--mute-audio', '--window-size=1920,1080'], sessionKey: antiBot.sessionKey || '', }); cfLog.browserReused = !!browserSession.reused; const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl); const navigationUrl = resolveNavigationUrl(pageUrl, antiBot); page = await createTaskPage( browserSession.browser, browserSession.reused ? null : browserSession.page, navigationUrl, antiBot, mergedCookies, { viewport: { width: 1920, height: 1080 } } ); const httpResolvedUrl = await resolveHttpRedirects(pageUrl).catch(() => pageUrl); let shareToken = resolveShareToken(pageUrl, httpResolvedUrl); await seedShareTokenCookie(page, navigationUrl); await prepareWhatshubShortLinkBeforeNav(page, navigationUrl, antiBot); await navigateToPage(page, navigationUrl); let finalPageUrl = await waitForUrlSettled(page); applyHuojianBusinessHostHint(antiBot, finalPageUrl); const browserCtx = { browserSession, page, antiBot, profile, navigationUrl, mergedCookies, baseExtraArgs: ['--mute-audio', '--window-size=1920,1080'], pageExtraOpts: { viewport: { width: 1920, height: 1080 } }, afterRelaunchNav: async (p) => { await seedShareTokenCookie(p, navigationUrl); }, }; await ensureCaptchaProxyBrowserIfNeeded(browserCtx); browserSession = browserCtx.browserSession; page = browserCtx.page; let interceptor; if (antiBot.enabled) { const cfResult = await runCloudflareWithBusinessGuard( page, antiBot, waitForUrlSettled, storedSession, navigationUrl, browserCtx ); page = browserCtx.page; browserSession = browserCtx.browserSession; cfLog.cfDetected = cfResult.cfDetected; cfLog.turnstileStage = cfResult.stage; cfLog.solverUsed = cfResult.solverUsed; cfLog.elapsedMs = cfResult.elapsedMs; if (!cfResult.success) { cfDestroyBrowser = true; res.status(422).json({ success: false, code: cfResult.code || 'CF_TURNSTILE_FAILED', challengeType: cfResult.challengeType || 'turnstile', stage: cfResult.stage || 'timeout', error: 'Cloudflare Turnstile 验证失败', cf: cfLog, }); return; } finalPageUrl = page.url(); applyHuojianBusinessHostHint(antiBot, finalPageUrl); touchSessionIfPresent(antiBot.sessionKey); await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled, storedSession, navigationUrl); } interceptor = createApiInterceptor(page, apiUrls, interceptTimeout); interceptorCleanup = interceptor.cleanup; await attachHttpIpRewriteInterceptor(page, 'auth-intercept-and-paginate', { shareToken }); await executeAuthActionsWithDiagnostics(page, authActions); await interceptor.waitForApis(interceptTimeout); const rawCookies = await page.cookies(); const cookiePayload = rawCookies.map((c) => ({ name: c.name, value: c.value, domain: c.domain, path: c.path, })); if (antiBot.sessionKey) { let lastHost = ''; try { lastHost = new URL(finalPageUrl).hostname; } catch (_) { /* ignore */ } saveSession(antiBot.sessionKey, cookiePayload, SESSION_TTL_MS, { lastHost, savedProfile: profile, }); } let extraPages = []; let paginationDebug = []; if (clicksToPerform > 0 && apiUrl && nextBtnSelector) { let effectiveFirstPageData = firstPageData; if (!effectiveFirstPageData) { const matchedKey = Object.keys(interceptor.interceptedApis).find((k) => k === apiUrl || k.includes(apiUrl)); if (matchedKey) { effectiveFirstPageData = interceptor.interceptedApis[matchedKey].data; } } const paginationResult = await runUiPagination(page, { apiUrl, nextBtnSelector, clicksToPerform, waitMs, firstPageData: effectiveFirstPageData, authActions, executeAuthActions, }); extraPages = paginationResult.data; paginationDebug = paginationResult.debugLogs; } res.json({ success: true, pageUrl, finalPageUrl, interceptedApis: interceptor.interceptedApis, cookies: cookiePayload, extraPages, debug: paginationDebug, cf: cfLog, profile, browserReused: browserSession.reused, elapsedMs: Date.now() - taskStarted, }); } finally { if (interceptorCleanup) interceptorCleanup(); await closeTaskPage(page); if (browserSession) { await browserSession.release(cfDestroyBrowser); } } }, profile); } catch (error) { if (!res.headersSent) sendTaskError(res, error, { cf: cfLog, profile }); } }); const PORT = parseInt(process.env.PORT || '3001', 10); const server = app.listen(PORT, '127.0.0.1', () => { const chromePath = resolveChromeExecutable(); const chromeReady = fs.existsSync(chromePath); console.log(`🚀 Node.js 自愈型防脱轨引擎已启动 (standard: ${MAX_CONCURRENT_BROWSERS}, real: ${MAX_CONCURRENT_BROWSERS_REAL})`); console.log(`[chrome] path=${chromePath} exists=${chromeReady}`); console.log(`[real-browser] ready=${isRealBrowserAvailable()} captcha=${isCaptchaConfigured()}`); if (!chromeReady) { console.warn('[chrome] 未找到 Chrome,请在 puppeteer-api 目录执行: npx puppeteer browsers install chrome'); } }); async function gracefulShutdown(signal) { console.log(`[shutdown] 收到 ${signal},停止接收新任务...`); shuttingDown = true; server.close(); const deadline = Date.now() + 60000; while (standardLimiter.active > 0 && Date.now() < deadline) { const { active, queued } = standardLimiter.getStats(); console.log(`[shutdown] 等待在途任务完成... active=${active}, queued=${queued}`); await new Promise((r) => setTimeout(r, 1000)); } console.log('[shutdown] 排空 Browser 温池...'); await browserPool.drain(); const remaining = standardLimiter.getStats(); if (remaining.active > 0) { console.warn(`[shutdown] 超时,仍有 ${remaining.active} 个 Browser 在运行,强制退出`); process.exit(1); } console.log('[shutdown] 所有任务已完成,进程退出'); process.exit(0); } process.on('SIGTERM', () => gracefulShutdown('SIGTERM')); process.on('SIGINT', () => gracefulShutdown('SIGINT'));