root
2551 lines
99 KiB
JavaScript
Executable File
2551 lines
99 KiB
JavaScript
Executable File
require('dotenv').config({ path: require('path').join(__dirname, '.env') });
|
||
|
||
process.on('uncaughtException', (err) => {
|
||
console.error('[FATAL] uncaughtException:', err);
|
||
});
|
||
process.on('unhandledRejection', (reason) => {
|
||
console.error('[FATAL] unhandledRejection:', reason);
|
||
});
|
||
|
||
const express = require('express');
|
||
const fs = require('fs');
|
||
const {
|
||
rewriteHttpsToHttpForLiteralIp,
|
||
attachHttpIpRewriteInterceptor,
|
||
} = require('./http-ip-rewrite');
|
||
const {
|
||
resolveHttpRedirects,
|
||
extractShareTokenFromUrl,
|
||
} = require('./url-resolve');
|
||
const {
|
||
DEFAULT_UA,
|
||
RUNTIME_DIR,
|
||
resolveChromeExecutable,
|
||
MAX_CONCURRENT_BROWSERS,
|
||
MAX_CONCURRENT_BROWSERS_REAL,
|
||
API_INTERCEPT_TIMEOUT_MS,
|
||
API_INTERCEPT_TIMEOUT_REAL_MS,
|
||
NAVIGATION_TIMEOUT_MS,
|
||
NAVIGATION_WAIT_UNTIL,
|
||
NAVIGATION_MAX_RETRIES,
|
||
PAGE_DEFAULT_TIMEOUT_MS,
|
||
SESSION_TTL_MS,
|
||
PERSIST_BROWSER_PROFILE,
|
||
POOL_IDLE_MS,
|
||
MAX_POOLED_BROWSERS,
|
||
} = require('./lib/constants');
|
||
const {
|
||
normalizeAntiBot,
|
||
createBrowserSession,
|
||
acquireBrowserSession,
|
||
runWithProfileLimit,
|
||
createManagedPage,
|
||
getFactoryStats,
|
||
standardLimiter,
|
||
browserPool,
|
||
} = require('./lib/browser-factory');
|
||
const { launchStandardBrowser } = require('./lib/launch-standard');
|
||
const { isRealBrowserAvailable } = require('./lib/launch-real-browser');
|
||
const { handleCloudflareChallenge, isCaptchaConfigured } = require('./lib/cf-handler');
|
||
const { detectCloudflare, urlIndicatesCfChallenge } = require('./lib/cf-detector');
|
||
const { runCaptchaApiFallback, shouldPreferManagedCloudflareSolver } = require('./lib/cf-api-fallback');
|
||
const {
|
||
getCaptchaProxyChromeArgs,
|
||
applyCaptchaProxyToPage,
|
||
parseCaptchaProxy,
|
||
hasCaptchaProxy,
|
||
} = require('./lib/captcha-proxy');
|
||
const { attachSitekeyCapture } = require('./lib/sitekey-capture');
|
||
const {
|
||
saveSession,
|
||
getSessionDebugInfo,
|
||
getSessionStats,
|
||
isSessionLikelyValid,
|
||
loadSession,
|
||
} = require('./lib/session-store');
|
||
const { resolveSessionContext, touchSessionIfPresent } = require('./lib/session-context');
|
||
const { getProfileStats, profileExists } = require('./lib/profile-dir');
|
||
const { getProfileStats, profileExists } = require('./lib/profile-dir');
|
||
const {
|
||
isHuojianLongLinkUrl,
|
||
isHuojianShortEntryUrl,
|
||
applyHuojianBusinessHostHint,
|
||
} = require('./lib/huojian-url');
|
||
const { runUiPagination } = require('./lib/ui-pagination-runner');
|
||
|
||
const app = express();
|
||
app.use(express.json({ limit: '50mb' }));
|
||
|
||
let shuttingDown = false;
|
||
|
||
/** HTTP 层：槽位排队 + 队列超时/关闭中 的统一错误响应 */
|
||
async function runBrowserTask(taskName, handler, profile = 'standard') {
|
||
if (shuttingDown) {
|
||
const err = new Error('SERVICE_SHUTTING_DOWN');
|
||
err.code = 'SERVICE_SHUTTING_DOWN';
|
||
throw err;
|
||
}
|
||
return runWithProfileLimit(taskName, profile, handler);
|
||
}
|
||
|
||
|
||
/** PHP antiBot.purgeSession：清除磁盘 session 文件（不修改 session-context 模块） */
|
||
function purgeStoredSessionIfRequested(antiBot) {
|
||
const sessionKey = antiBot?.sessionKey || '';
|
||
if (!sessionKey || antiBot?.purgeSession !== true) {
|
||
return false;
|
||
}
|
||
const crypto = require('crypto');
|
||
const path = require('path');
|
||
const hash = crypto.createHash('md5').update(sessionKey).digest('hex');
|
||
const filePath = path.join(RUNTIME_DIR, 'sessions', `${hash}.json`);
|
||
try {
|
||
if (fs.existsSync(filePath)) {
|
||
fs.unlinkSync(filePath);
|
||
console.log(`[Session] purgeSession 已清除 sessionKey=${sessionKey}`);
|
||
return true;
|
||
}
|
||
} catch (_) { /* ignore */ }
|
||
return false;
|
||
}
|
||
|
||
function sendTaskError(res, error, extra = {}) {
|
||
if (error.code === 'QUEUE_TIMEOUT' || error.message === 'POOL_ACQUIRE_TIMEOUT') {
|
||
return res.status(503).json({
|
||
success: false,
|
||
error: '服务繁忙，排队超时，请稍后重试',
|
||
queue: standardLimiter.getStats(),
|
||
...extra,
|
||
});
|
||
}
|
||
if (error.code === 'SERVICE_SHUTTING_DOWN') {
|
||
return res.status(503).json({ success: false, error: '服务正在关闭', ...extra });
|
||
}
|
||
const errType = classifyPuppeteerError(error);
|
||
console.error(`[${errType}]`, error.message);
|
||
const payload = { success: false, error: error.message, ...extra };
|
||
if (error.pageDiagnostics) {
|
||
payload.page = error.pageDiagnostics;
|
||
}
|
||
return res.status(500).json(payload);
|
||
}
|
||
|
||
async function navigateToPage(page, url) {
|
||
await withExponentialBackoff(
|
||
() => page.goto(url, { waitUntil: NAVIGATION_WAIT_UNTIL, timeout: NAVIGATION_TIMEOUT_MS }),
|
||
{ maxRetries: NAVIGATION_MAX_RETRIES, baseDelayMs: 1000 }
|
||
);
|
||
}
|
||
|
||
async function safeCloseBrowser(browser) {
|
||
if (!browser) return;
|
||
try {
|
||
const pages = await browser.pages();
|
||
await Promise.all(pages.map((p) => p.close().catch(() => {})));
|
||
await browser.close();
|
||
} catch (_) {
|
||
try {
|
||
const proc = browser.process();
|
||
if (proc && !proc.killed) proc.kill('SIGKILL');
|
||
} catch (_) { /* ignore */ }
|
||
}
|
||
}
|
||
|
||
function classifyPuppeteerError(err) {
|
||
const msg = (err?.message || '').toLowerCase();
|
||
if (err?.name === 'TimeoutError' || msg.includes('timeout')) return 'timeout';
|
||
if (msg.includes('net::') || msg.includes('network') || msg.includes('err_connection')
|
||
|| msg.includes('err_address_unreachable') || msg.includes('err_name_not_resolved')) return 'network';
|
||
if (msg.includes('navigation') || msg.includes('frame was detached')
|
||
|| msg.includes('detached frame') || msg.includes('attempted to use detached')) return 'navigation';
|
||
if (msg.includes('target closed') || msg.includes('session closed')) return 'target_closed';
|
||
return 'unknown';
|
||
}
|
||
|
||
function isRetryableError(err) {
|
||
const type = classifyPuppeteerError(err);
|
||
return type === 'timeout' || type === 'network' || type === 'navigation';
|
||
}
|
||
|
||
async function withExponentialBackoff(fn, {
|
||
maxRetries = 3,
|
||
baseDelayMs = 500,
|
||
maxDelayMs = 8000,
|
||
shouldRetry = isRetryableError,
|
||
} = {}) {
|
||
let lastError;
|
||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||
try {
|
||
return await fn(attempt);
|
||
} catch (err) {
|
||
lastError = err;
|
||
if (attempt >= maxRetries || !shouldRetry(err)) throw err;
|
||
const delay = Math.min(baseDelayMs * Math.pow(2, attempt), maxDelayMs);
|
||
await new Promise((r) => setTimeout(r, delay));
|
||
}
|
||
}
|
||
throw lastError;
|
||
}
|
||
|
||
async function waitForUrlSettled(page, { maxAttempts = 15, stableMs = 500, redirectSettleMs = 1500 } = {}) {
|
||
let currentUrl = page.url();
|
||
let urlSettled = false;
|
||
let settleAttempts = 0;
|
||
|
||
while (!urlSettled && settleAttempts < maxAttempts) {
|
||
const urlBeforeWait = currentUrl;
|
||
await page.waitForFunction(
|
||
(expectedUrl) => window.location.href === expectedUrl,
|
||
{ timeout: stableMs, polling: 100 },
|
||
urlBeforeWait
|
||
).catch(() => {});
|
||
|
||
const newUrl = page.url();
|
||
if (newUrl === currentUrl) {
|
||
urlSettled = true;
|
||
} else {
|
||
console.log(`[路由重定向探测] 页面从 ${currentUrl} 跳转到了 ${newUrl}`);
|
||
currentUrl = newUrl;
|
||
await page.waitForNetworkIdle({ idleTime: 300, timeout: redirectSettleMs }).catch(() => {});
|
||
}
|
||
settleAttempts++;
|
||
}
|
||
return currentUrl;
|
||
}
|
||
|
||
/** 合并 antiBot 扩展字段（postCfReady* 由 PHP 按工单类型传入，未配置则跳过） */
|
||
function resolveAntiBot(rawAntiBot) {
|
||
const base = normalizeAntiBot(rawAntiBot);
|
||
if (!rawAntiBot || !base.enabled) {
|
||
return base;
|
||
}
|
||
return {
|
||
...base,
|
||
postCfReadyUrlContains: rawAntiBot.postCfReadyUrlContains || '',
|
||
postCfReadySelector: rawAntiBot.postCfReadySelector || '',
|
||
postCfReadyApiUrl: rawAntiBot.postCfReadyApiUrl || '',
|
||
postCfReadyTimeoutMs: rawAntiBot.postCfReadyTimeoutMs || 0,
|
||
postCfSettleMs: rawAntiBot.postCfSettleMs || 0,
|
||
postCfSpaWaitMs: rawAntiBot.postCfSpaWaitMs || 0,
|
||
cfClearanceRequired: rawAntiBot.cfClearanceRequired !== false,
|
||
businessHostHint: rawAntiBot.businessHostHint || '',
|
||
landingUrlHint: rawAntiBot.landingUrlHint || '',
|
||
turnstileSitekey: rawAntiBot.turnstileSitekey || '',
|
||
cfCloudflareSolver: rawAntiBot.cfCloudflareSolver !== false,
|
||
cfChallengeMode: rawAntiBot.cfChallengeMode || '',
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Whatshub 短链场景：PHP 未传 postCfReady* 时 Node 侧自动补全（避免 session_reuse 后 15s auth 超时）
|
||
* @param {object} antiBot
|
||
* @param {string} pageUrl
|
||
*/
|
||
function applyWhatshubPostCfDefaults(antiBot, pageUrl) {
|
||
if (!antiBot || !antiBot.enabled) {
|
||
return antiBot;
|
||
}
|
||
if ((antiBot.postCfReadyUrlContains || '').trim()) {
|
||
return antiBot;
|
||
}
|
||
try {
|
||
const parsed = new URL(pageUrl || '');
|
||
const isWhatshubHost = parsed.hostname.includes('whatshub.cc');
|
||
const isShortLink = parsed.pathname.includes('/m/');
|
||
const isLongLink = pageUrl.includes('work-order-sharing');
|
||
if (!isWhatshubHost || (!isShortLink && !isLongLink)) {
|
||
return antiBot;
|
||
}
|
||
console.log('[CF] Whatshub 自动启用 postCfReady 默认配置');
|
||
return {
|
||
...antiBot,
|
||
postCfReadyUrlContains: 'work-order-sharing',
|
||
postCfReadySelector: '.vxe-table, .vxe-grid',
|
||
postCfReadySelectorOptional: true,
|
||
postCfReadyTimeoutMs: antiBot.postCfReadyTimeoutMs || 60000,
|
||
postCfSettleMs: antiBot.postCfSettleMs || 500,
|
||
postCfSpaWaitMs: antiBot.postCfSpaWaitMs || 4000,
|
||
skipShortLinkCfPurgeOnReuse: antiBot.skipShortLinkCfPurgeOnReuse !== false,
|
||
};
|
||
} catch (_) {
|
||
return antiBot;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* A2C 短链 → user.a2c.chat：PHP 未传 postCfReady* 时 Node 侧自动补全
|
||
* @param {object} antiBot
|
||
* @param {string} pageUrl
|
||
*/
|
||
function applyA2cPostCfDefaults(antiBot, pageUrl) {
|
||
if (!antiBot || !antiBot.enabled) {
|
||
return antiBot;
|
||
}
|
||
if ((antiBot.postCfReadyUrlContains || '').trim()) {
|
||
return antiBot;
|
||
}
|
||
|
||
let shouldApply = false;
|
||
const sessionKey = String(antiBot.sessionKey || '').toLowerCase();
|
||
if (sessionKey.startsWith('a2c:')) {
|
||
shouldApply = true;
|
||
}
|
||
try {
|
||
const parsed = new URL(pageUrl || '');
|
||
if (parsed.hostname.includes('a2c.chat')) {
|
||
shouldApply = true;
|
||
}
|
||
} catch (_) {
|
||
/* ignore */
|
||
}
|
||
|
||
if (!shouldApply) {
|
||
return antiBot;
|
||
}
|
||
|
||
console.log('[CF] A2C 自动启用 postCfReady 默认配置');
|
||
return {
|
||
...antiBot,
|
||
postCfReadyUrlContains: '/visitors/counter/share',
|
||
postCfReadySelector: '.el-table',
|
||
postCfReadyApiUrl: antiBot.postCfReadyApiUrl || '/api/talk/counter/share/record/list',
|
||
postCfReadyTimeoutMs: antiBot.postCfReadyTimeoutMs || 60000,
|
||
postCfSettleMs: antiBot.postCfSettleMs || 500,
|
||
postCfSpaWaitMs: antiBot.postCfSpaWaitMs || 8000,
|
||
cfClearanceRequired: antiBot.cfClearanceRequired === undefined ? false : antiBot.cfClearanceRequired,
|
||
solverFallback: antiBot.solverFallback === undefined ? true : antiBot.solverFallback,
|
||
cfCloudflareSolver: antiBot.cfCloudflareSolver !== false,
|
||
cfChallengeMode: antiBot.cfChallengeMode || 'managed',
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 火箭短链 → 动态长链域 /gds?link=：PHP 未传 postCfReady* 时 Node 侧自动补全
|
||
* @param {object} antiBot
|
||
* @param {string} pageUrl
|
||
*/
|
||
function applyHuojianPostCfDefaults(antiBot, pageUrl) {
|
||
if (!antiBot || !antiBot.enabled) {
|
||
return antiBot;
|
||
}
|
||
if ((antiBot.postCfReadyUrlContains || '').trim()) {
|
||
return antiBot;
|
||
}
|
||
|
||
let shouldApply = false;
|
||
const sessionKey = String(antiBot.sessionKey || '').toLowerCase();
|
||
if (sessionKey.startsWith('huojian:')) {
|
||
shouldApply = true;
|
||
}
|
||
if (isHuojianLongLinkUrl(pageUrl) || isHuojianShortEntryUrl(pageUrl)) {
|
||
shouldApply = true;
|
||
}
|
||
|
||
if (!shouldApply) {
|
||
return antiBot;
|
||
}
|
||
|
||
console.log('[CF] 火箭 自动启用 postCfReady 默认配置');
|
||
return {
|
||
...antiBot,
|
||
postCfReadyUrlContains: '/gds',
|
||
postCfReadySelector: '.el-message-box__input',
|
||
postCfReadyTimeoutMs: antiBot.postCfReadyTimeoutMs || 45000,
|
||
postCfSettleMs: antiBot.postCfSettleMs || 500,
|
||
postCfSpaWaitMs: antiBot.postCfSpaWaitMs || 8000,
|
||
};
|
||
}
|
||
|
||
/** 按 pageUrl 合并 antiBot（含 Whatshub / A2C / 火箭 默认 postCf） */
|
||
function resolveAntiBotForPage(rawAntiBot, pageUrl) {
|
||
return applyHuojianPostCfDefaults(
|
||
applyA2cPostCfDefaults(
|
||
applyWhatshubPostCfDefaults(resolveAntiBot(rawAntiBot), pageUrl),
|
||
pageUrl
|
||
),
|
||
pageUrl
|
||
);
|
||
}
|
||
|
||
function isBusinessUrlReady(page, urlNeedle) {
|
||
return !urlNeedle || page.url().includes(urlNeedle);
|
||
}
|
||
|
||
/** A2C 已直达 user.a2c.chat 分享页（非 yyk.ink 短链跳转场景） */
|
||
function isA2cDirectSharePage(page, urlNeedle = '') {
|
||
const needle = (urlNeedle || '').trim();
|
||
if (!needle.includes('/visitors/counter/share')) {
|
||
return false;
|
||
}
|
||
try {
|
||
const parsed = new URL(page.url() || '');
|
||
return parsed.hostname.includes('a2c.chat') && parsed.pathname.includes('/visitors/counter/share');
|
||
} catch (_) {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
/** 过盾后等待业务 list API 返回（A2C Vue 挂载信号） */
|
||
async function waitForPostCfListApi(page, antiBot, timeoutMs) {
|
||
const apiUrl = (antiBot.postCfReadyApiUrl || '').trim();
|
||
if (!apiUrl) {
|
||
return false;
|
||
}
|
||
console.log(`[CF] 等待业务 API 响应 api=${apiUrl} timeout=${timeoutMs}ms`);
|
||
const matched = await page.waitForResponse(
|
||
(resp) => {
|
||
const req = resp.request();
|
||
if (req.method() === 'OPTIONS') {
|
||
return false;
|
||
}
|
||
return resp.url().includes(apiUrl) && resp.status() >= 200 && resp.status() < 400;
|
||
},
|
||
{ timeout: timeoutMs }
|
||
).catch(() => null);
|
||
if (matched) {
|
||
console.log(`[CF] 业务 API 已响应 status=${matched.status()} url=${matched.url()}`);
|
||
return true;
|
||
}
|
||
console.warn('[CF] 业务 API 等待超时，降级为 DOM selector 检测');
|
||
return false;
|
||
}
|
||
|
||
/** 等待业务 DOM：先 attached 再 visible，避免 Vue 渲染中途误判 */
|
||
async function waitForPostCfSelector(page, selector, timeoutMs) {
|
||
await page.waitForSelector(selector, { timeout: timeoutMs, visible: false });
|
||
await page.waitForSelector(selector, { timeout: Math.min(15000, timeoutMs), visible: true });
|
||
}
|
||
|
||
/** 可选 selector：URL 已到业务页时允许降级继续（Whatshub session 复用） */
|
||
async function waitForPostCfSelectorOptional(page, selector, timeoutMs, antiBot) {
|
||
const optional = antiBot?.postCfReadySelectorOptional === true;
|
||
if (!optional) {
|
||
await waitForPostCfSelector(page, selector, timeoutMs);
|
||
return;
|
||
}
|
||
const shortTimeout = Math.min(8000, timeoutMs);
|
||
try {
|
||
await waitForPostCfSelector(page, selector, shortTimeout);
|
||
} catch (err) {
|
||
const urlContains = (antiBot.postCfReadyUrlContains || '').trim();
|
||
if (urlContains && page.url().includes(urlContains)) {
|
||
console.log(`[CF] 可选 selector "${selector}" 未就绪但 URL 已到业务页，继续`);
|
||
return;
|
||
}
|
||
throw err;
|
||
}
|
||
}
|
||
|
||
/** Whatshub 分享短链 /m/xxx/1 */
|
||
function isWhatshubShortLinkPath(url) {
|
||
try {
|
||
const parsed = new URL(url || '');
|
||
return parsed.hostname.includes('whatshub.cc') && parsed.pathname.includes('/m/');
|
||
} catch (_) {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
/** Whatshub 域名判定 */
|
||
function isWhatshubHost(url) {
|
||
try {
|
||
return new URL(url || '').hostname.includes('whatshub.cc');
|
||
} catch (_) {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
/** Whatshub 密码 UI 是否可见（vxe-form / vxe-modal + Element dialog） */
|
||
async function isWhatshubPasswordDialogVisible(page) {
|
||
return page.evaluate(() => {
|
||
const isVisible = (el) => {
|
||
if (!el) return false;
|
||
const rect = el.getBoundingClientRect();
|
||
if (rect.width <= 0 || rect.height <= 0) return false;
|
||
const style = window.getComputedStyle(el);
|
||
return style.display !== 'none' && style.visibility !== 'hidden';
|
||
};
|
||
const hasVisiblePasswordInput = (root) => {
|
||
const pwd = root.querySelector('.vxe-form .el-input__inner, .el-input__inner[placeholder*="密码"]');
|
||
return pwd && isVisible(pwd);
|
||
};
|
||
const roots = document.querySelectorAll(
|
||
'form.vxe-form, .vxe-modal--wrapper, .vxe-modal, '
|
||
+ '.el-message-box, .el-overlay-message-box, .el-dialog, .el-overlay-dialog'
|
||
);
|
||
for (const root of roots) {
|
||
if (!isVisible(root)) continue;
|
||
if (hasVisiblePasswordInput(root)) return true;
|
||
}
|
||
const standaloneForm = document.querySelector('form.vxe-form');
|
||
if (standaloneForm && isVisible(standaloneForm) && hasVisiblePasswordInput(standaloneForm)) {
|
||
return true;
|
||
}
|
||
return false;
|
||
}).catch(() => false);
|
||
}
|
||
|
||
/** statistics-only 等场景：导航后立即挂 API 拦截器，避免 session 复用时响应早于监听 */
|
||
function shouldUseEarlyApiIntercept(antiBot, apiUrls) {
|
||
if (antiBot?.earlyApiIntercept === true) {
|
||
return true;
|
||
}
|
||
if (!Array.isArray(apiUrls) || apiUrls.length === 0) {
|
||
return false;
|
||
}
|
||
return apiUrls.every((u) => String(u).includes('/open/statistics'));
|
||
}
|
||
|
||
/** statistics-only 拦截时不应在密码前等待业务 API */
|
||
function shouldSkipPostCfReadyApiWait(antiBot, apiUrls = null) {
|
||
if (antiBot?.skipPostCfReadyApi) {
|
||
return true;
|
||
}
|
||
if (Array.isArray(apiUrls) && apiUrls.length > 0) {
|
||
return apiUrls.every((u) => String(u).includes('/open/statistics'));
|
||
}
|
||
return false;
|
||
}
|
||
|
||
/** Whatshub 使用 PHP 配置的 postCfSpaWaitMs；A2C 等仍保底 8s */
|
||
function resolveSpaWaitMs(antiBot, onA2cDirect, pageUrl = '') {
|
||
const configured = antiBot?.postCfSpaWaitMs || 0;
|
||
if (onA2cDirect) {
|
||
return Math.max(8000, configured || 8000);
|
||
}
|
||
if (isWhatshubShortLinkPath(pageUrl) || isWhatshubHost(pageUrl)) {
|
||
return Math.max(2000, configured || 4000);
|
||
}
|
||
return Math.max(8000, configured || 8000);
|
||
}
|
||
|
||
/** authActions 选择器超时（Whatshub session 复用时可放宽） */
|
||
function resolveAuthSelectorTimeoutMs(antiBot) {
|
||
const ms = antiBot?.authSelectorTimeoutMs;
|
||
return typeof ms === 'number' && ms > 0 ? ms : 15000;
|
||
}
|
||
|
||
/**
|
||
* Whatshub 短链 + Real Browser：不覆盖 UA、不预注入磁盘 cf_clearance
|
||
* @param {string} pageUrl
|
||
* @param {object} antiBot
|
||
* @param {object[]} mergedCookies
|
||
* @param {object} extraOpts
|
||
*/
|
||
function buildManagedPageOptions(pageUrl, antiBot, mergedCookies, extraOpts = {}) {
|
||
const opts = { ...extraOpts };
|
||
const isWhatshubShort = isWhatshubShortLinkPath(pageUrl);
|
||
const isRealProfile = antiBot?.profile === 'real';
|
||
|
||
if (isWhatshubShort && isRealProfile && antiBot?.enabled) {
|
||
opts.skipUserAgent = true;
|
||
const filtered = (mergedCookies || []).filter((c) => c.name !== 'cf_clearance');
|
||
if (filtered.length < (mergedCookies || []).length) {
|
||
console.log('[Whatshub] 短链首访：跳过磁盘 cf_clearance 预注入，避免 SPA 路由未触发');
|
||
}
|
||
opts.cookies = filtered;
|
||
return opts;
|
||
}
|
||
|
||
opts.cookies = mergedCookies;
|
||
if (!opts.skipUserAgent && !opts.userAgent) {
|
||
opts.userAgent = DEFAULT_UA;
|
||
}
|
||
return opts;
|
||
}
|
||
|
||
/**
|
||
* A2C：短链入口时优先导航至 PHP 预解析的长链，减少 yyk.ink 跳转竞态
|
||
* @param {string} pageUrl
|
||
* @param {object} antiBot
|
||
*/
|
||
function resolveNavigationUrl(pageUrl, antiBot) {
|
||
const hint = String(antiBot?.landingUrlHint || '').trim();
|
||
if (hint === '' || !hint.includes('/visitors/counter/share')) {
|
||
return pageUrl;
|
||
}
|
||
try {
|
||
const pageHost = new URL(pageUrl).hostname;
|
||
const hintHost = new URL(hint).hostname;
|
||
if (hintHost.includes('a2c.chat') && pageHost !== hintHost) {
|
||
console.log(`[A2C] 短链入口，直接导航至预解析长链: ${hint}`);
|
||
return hint;
|
||
}
|
||
} catch (_) {
|
||
/* ignore */
|
||
}
|
||
return pageUrl;
|
||
}
|
||
|
||
/**
|
||
* Chrome 启动参数：仅当 useCaptchaProxy=true 时注入 CAPTCHA_PROXY（与 CapSolver 同源 IP）
|
||
* @param {object|null|undefined} antiBot
|
||
* @param {string[]} [baseArgs]
|
||
* @param {boolean} [useCaptchaProxy]
|
||
*/
|
||
function buildBrowserExtraArgs(antiBot, baseArgs = [], useCaptchaProxy = false) {
|
||
const args = [...baseArgs];
|
||
if (!useCaptchaProxy || !hasCaptchaProxy()) {
|
||
return args;
|
||
}
|
||
const proxyArgs = getCaptchaProxyChromeArgs();
|
||
if (proxyArgs.length > 0) {
|
||
const parsed = parseCaptchaProxy();
|
||
console.log(
|
||
`[Proxy] Chrome 使用 CAPTCHA_PROXY ${parsed.host}:${parsed.port}（与 CapSolver 同源 IP）`
|
||
);
|
||
args.push(...proxyArgs);
|
||
}
|
||
return args;
|
||
}
|
||
|
||
/**
|
||
* Managed A2C 等：仅在检测到真实 CF 挑战时切换为 CAPTCHA_PROXY Browser（无盾则直连）
|
||
* 会原地更新 browserCtx.browserSession / browserCtx.page
|
||
* @param {object} browserCtx
|
||
*/
|
||
async function ensureCaptchaProxyBrowserIfNeeded(browserCtx) {
|
||
const {
|
||
antiBot,
|
||
profile,
|
||
navigationUrl,
|
||
mergedCookies,
|
||
baseExtraArgs = ['--mute-audio'],
|
||
pageExtraOpts = {},
|
||
} = browserCtx;
|
||
let { browserSession, page } = browserCtx;
|
||
|
||
if (!antiBot?.enabled || browserSession?.captchaProxyActive) {
|
||
return;
|
||
}
|
||
if (!shouldPreferManagedCloudflareSolver(antiBot) || !hasCaptchaProxy()) {
|
||
return;
|
||
}
|
||
|
||
const cfState = await detectCloudflare(page, antiBot);
|
||
const needsProxy = cfState.blocked || urlIndicatesCfChallenge(page.url());
|
||
if (!needsProxy) {
|
||
console.log('[Proxy] 未检测到 CF 挑战，Chrome 直连（不使用 CAPTCHA_PROXY）');
|
||
return;
|
||
}
|
||
|
||
console.log('[Proxy] 检测到 CF 挑战，切换 Browser 为 CAPTCHA_PROXY（与 CapSolver 同源 IP）');
|
||
|
||
const snapshotCookies = await page.cookies().catch(() => []);
|
||
await closeTaskPage(page);
|
||
await browserSession.release(true);
|
||
|
||
const extraArgs = buildBrowserExtraArgs(antiBot, baseExtraArgs, true);
|
||
browserSession = await acquireBrowserSession({
|
||
profile,
|
||
extraArgs,
|
||
sessionKey: antiBot.sessionKey || '',
|
||
});
|
||
tagBrowserSession(browserSession, profile, antiBot.sessionKey || '', extraArgs);
|
||
browserSession.captchaProxyActive = true;
|
||
|
||
const proxySessionRef = { current: browserSession };
|
||
page = await createTaskPage(
|
||
browserSession.browser,
|
||
browserSession.reused ? null : browserSession.page,
|
||
navigationUrl,
|
||
antiBot,
|
||
mergedCookies.length > 0 ? mergedCookies : snapshotCookies,
|
||
{
|
||
...pageExtraOpts,
|
||
useCaptchaProxy: true,
|
||
browserSession,
|
||
browserSessionRef: proxySessionRef,
|
||
}
|
||
);
|
||
browserSession = proxySessionRef.current;
|
||
|
||
if (antiBot?.enabled) {
|
||
attachSitekeyCapture(page, browserSession.browser);
|
||
}
|
||
|
||
await prepareWhatshubShortLinkBeforeNav(page, navigationUrl, antiBot);
|
||
await navigateToPage(page, navigationUrl);
|
||
await waitForUrlSettled(page).catch(() => {});
|
||
|
||
if (typeof browserCtx.afterRelaunchNav === 'function') {
|
||
await browserCtx.afterRelaunchNav(page);
|
||
}
|
||
|
||
browserCtx.browserSession = browserSession;
|
||
browserCtx.page = page;
|
||
}
|
||
|
||
/**
|
||
* 为温池复用 / newPage 重试附加元数据（profile、sessionKey、启动参数）
|
||
* @param {object} browserSession
|
||
* @param {'standard'|'real'} profile
|
||
* @param {string} sessionKey
|
||
* @param {string[]} launchExtraArgs
|
||
*/
|
||
function tagBrowserSession(browserSession, profile, sessionKey, launchExtraArgs = []) {
|
||
browserSession.profile = profile;
|
||
browserSession.sessionKey = sessionKey || '';
|
||
browserSession.launchExtraArgs = launchExtraArgs;
|
||
return browserSession;
|
||
}
|
||
|
||
/** @param {Error|unknown} err */
|
||
function isNewTabOpenError(err) {
|
||
const msg = String(err?.message || err || '').toLowerCase();
|
||
return msg.includes('target.createtarget') || msg.includes('failed to open a new tab');
|
||
}
|
||
|
||
/**
|
||
* 温池 Browser 若已无任何 Tab，销毁并冷启动（避免后续 newPage 失败）
|
||
* @param {object|null} browserSession
|
||
*/
|
||
async function relaunchBrowserSessionIfNoTabs(browserSession) {
|
||
if (!browserSession?.pooled || !browserSession.browser) {
|
||
return browserSession;
|
||
}
|
||
let pages = [];
|
||
try {
|
||
pages = await browserSession.browser.pages();
|
||
} catch (_) {
|
||
pages = [];
|
||
}
|
||
if (pages.length > 0) {
|
||
return browserSession;
|
||
}
|
||
|
||
console.warn('[createTaskPage] 温池 Browser pages=0，销毁并冷启动');
|
||
await browserSession.release(true);
|
||
const relaunched = await acquireBrowserSession({
|
||
profile: browserSession.profile || 'real',
|
||
extraArgs: browserSession.launchExtraArgs || ['--mute-audio'],
|
||
sessionKey: browserSession.sessionKey || '',
|
||
});
|
||
tagBrowserSession(
|
||
relaunched,
|
||
browserSession.profile || relaunched.profile || 'real',
|
||
browserSession.sessionKey || '',
|
||
browserSession.launchExtraArgs || ['--mute-audio']
|
||
);
|
||
return relaunched;
|
||
}
|
||
|
||
/** 任务结束关闭本次 Tab（温池复用 Browser 时保留至少一个 Tab） */
|
||
async function closeTaskPage(page) {
|
||
if (!page) {
|
||
return;
|
||
}
|
||
try {
|
||
if (typeof page.isClosed === 'function' && page.isClosed()) {
|
||
return;
|
||
}
|
||
const browser = typeof page.browser === 'function' ? page.browser() : null;
|
||
if (browser) {
|
||
const pages = await browser.pages().catch(() => []);
|
||
if (pages.length <= 1) {
|
||
await page.goto('about:blank', {
|
||
waitUntil: 'domcontentloaded',
|
||
timeout: Math.min(NAVIGATION_TIMEOUT_MS, 15000),
|
||
}).catch(() => {});
|
||
return;
|
||
}
|
||
}
|
||
await page.close();
|
||
} catch (_) {
|
||
/* ignore */
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 配置任务页（UA / Cookie / 事件监听）
|
||
* @param {import('puppeteer').Page} page
|
||
* @param {import('puppeteer').Browser} browser
|
||
* @param {object} opts
|
||
* @param {object} extraOpts
|
||
* @param {object|null|undefined} antiBot
|
||
*/
|
||
async function configureTaskPage(page, browser, opts, extraOpts, antiBot) {
|
||
page.setDefaultNavigationTimeout(NAVIGATION_TIMEOUT_MS);
|
||
page.setDefaultTimeout(PAGE_DEFAULT_TIMEOUT_MS);
|
||
|
||
if (!opts.skipUserAgent) {
|
||
await page.setUserAgent(opts.userAgent || DEFAULT_UA);
|
||
}
|
||
if (opts.viewport) {
|
||
await page.setViewport(opts.viewport);
|
||
}
|
||
if (opts.cookies && opts.cookies.length > 0) {
|
||
await page.setCookie(...opts.cookies);
|
||
}
|
||
|
||
if (extraOpts.useCaptchaProxy && hasCaptchaProxy()) {
|
||
await applyCaptchaProxyToPage(page);
|
||
}
|
||
|
||
page.on('error', (err) => console.error('[Page Error]', err.message));
|
||
page.on('pageerror', (err) => console.error('[Page JS Error]', err.message));
|
||
|
||
if (antiBot?.enabled) {
|
||
attachSitekeyCapture(page, browser);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 任务页创建（含 Whatshub skipUserAgent）
|
||
* 温池复用 Browser 时每次 newPage；失败则销毁温池条目并冷启动重试
|
||
*
|
||
* extraOpts.browserSession：传入时可触发 pages=0 重建与 newPage 失败重试
|
||
* extraOpts.browserSessionRef：可选 { current }，重试后回写最新 session
|
||
*/
|
||
async function createTaskPage(browser, warmPage, pageUrl, antiBot, mergedCookies, extraOpts = {}) {
|
||
const opts = buildManagedPageOptions(pageUrl, antiBot, mergedCookies, extraOpts);
|
||
let browserSession = extraOpts.browserSession || null;
|
||
let currentBrowser = browser;
|
||
let currentWarmPage = warmPage;
|
||
|
||
if (browserSession) {
|
||
browserSession = await relaunchBrowserSessionIfNoTabs(browserSession);
|
||
currentBrowser = browserSession.browser;
|
||
currentWarmPage = browserSession.reused ? null : browserSession.page;
|
||
if (extraOpts.browserSessionRef) {
|
||
extraOpts.browserSessionRef.current = browserSession;
|
||
}
|
||
}
|
||
|
||
let page = null;
|
||
for (let attempt = 0; attempt < 2; attempt++) {
|
||
const useWarmPage = !!(currentWarmPage && extraOpts.useWarmPage !== false && !extraOpts.forceNewPage);
|
||
try {
|
||
page = useWarmPage ? currentWarmPage : await currentBrowser.newPage();
|
||
break;
|
||
} catch (err) {
|
||
if (attempt === 0 && browserSession && isNewTabOpenError(err)) {
|
||
console.warn('[createTaskPage] newPage 失败，销毁温池并冷启动重试:', err.message);
|
||
await browserSession.release(true);
|
||
browserSession = await acquireBrowserSession({
|
||
profile: browserSession.profile || (antiBot?.profile === 'real' ? 'real' : 'standard'),
|
||
extraArgs: browserSession.launchExtraArgs || ['--mute-audio'],
|
||
sessionKey: browserSession.sessionKey || antiBot?.sessionKey || '',
|
||
});
|
||
tagBrowserSession(
|
||
browserSession,
|
||
browserSession.profile || (antiBot?.profile === 'real' ? 'real' : 'standard'),
|
||
browserSession.sessionKey || antiBot?.sessionKey || '',
|
||
browserSession.launchExtraArgs || ['--mute-audio']
|
||
);
|
||
if (extraOpts.browserSessionRef) {
|
||
extraOpts.browserSessionRef.current = browserSession;
|
||
}
|
||
currentBrowser = browserSession.browser;
|
||
currentWarmPage = browserSession.reused ? null : browserSession.page;
|
||
continue;
|
||
}
|
||
throw err;
|
||
}
|
||
}
|
||
|
||
if (!page) {
|
||
throw new Error('createTaskPage: 未能打开任务 Tab');
|
||
}
|
||
|
||
await configureTaskPage(page, currentBrowser, opts, extraOpts, antiBot);
|
||
return page;
|
||
}
|
||
|
||
/** CF 已过但仍在短链、未到业务页（Whatshub / A2C yyk.ink 等） */
|
||
function isOnShortLinkNotBusiness(page, urlNeedle) {
|
||
const url = page.url();
|
||
if (urlNeedle && url.includes(urlNeedle)) {
|
||
return false;
|
||
}
|
||
if (isWhatshubShortLinkPath(url)) {
|
||
return true;
|
||
}
|
||
if (urlNeedle && urlNeedle.includes('/visitors/counter/share')) {
|
||
try {
|
||
return !new URL(url).hostname.includes('a2c.chat');
|
||
} catch (_) {
|
||
return false;
|
||
}
|
||
}
|
||
if (urlNeedle && urlNeedle.includes('/gds')) {
|
||
return !url.includes('/gds');
|
||
}
|
||
return false;
|
||
}
|
||
|
||
/**
|
||
* CF 完成后等待 Vue SPA 从短链跳转到 work-order-sharing（对齐 debug 脚本 goto+8s）
|
||
* Whatshub：二次 goto 短链（保留 clearance），禁止 reload（会重新触发 __cf_chl_rt_tk）
|
||
*/
|
||
async function waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettledFn, pageUrl = '') {
|
||
const urlNeedle = (antiBot.postCfReadyUrlContains || '').trim();
|
||
const targetUrl = pageUrl || page.url();
|
||
const onA2cDirect = isA2cDirectSharePage(page, urlNeedle);
|
||
if (!urlNeedle || (!isOnShortLinkNotBusiness(page, urlNeedle) && !onA2cDirect)) {
|
||
return isBusinessUrlReady(page, urlNeedle);
|
||
}
|
||
if (!(await isCfTrulyComplete(page, antiBot))) {
|
||
return false;
|
||
}
|
||
|
||
const spaWaitMs = resolveSpaWaitMs(antiBot, onA2cDirect, targetUrl);
|
||
const isWhatshub = isWhatshubShortLinkPath(targetUrl) || isWhatshubHost(targetUrl);
|
||
|
||
console.log(
|
||
`[CF] CF 已完成，等待 SPA ${onA2cDirect ? '直链挂载' : '跳转'} settle=${spaWaitMs}ms current=${page.url()}`
|
||
);
|
||
await new Promise((resolve) => setTimeout(resolve, spaWaitMs));
|
||
|
||
if (typeof waitForUrlSettledFn === 'function') {
|
||
await waitForUrlSettledFn(page).catch(() => {});
|
||
}
|
||
await page.waitForNetworkIdle({ idleTime: 500, timeout: 20000 }).catch(() => {});
|
||
|
||
if (onA2cDirect) {
|
||
await waitForPostCfListApi(page, antiBot, Math.max(15000, antiBot.postCfReadyTimeoutMs || 60000)).catch(() => {});
|
||
}
|
||
|
||
if (isBusinessUrlReady(page, urlNeedle)) {
|
||
console.log(`[CF] SPA 已跳转至业务页 url=${page.url()}`);
|
||
return true;
|
||
}
|
||
|
||
if (isWhatshub && isWhatshubShortLinkPath(targetUrl)) {
|
||
console.log(`[CF] Whatshub 二次 goto 触发 SPA 路由（保留 clearance）url=${targetUrl}`);
|
||
await navigateToPage(page, targetUrl);
|
||
if (typeof waitForUrlSettledFn === 'function') {
|
||
await waitForUrlSettledFn(page).catch(() => {});
|
||
}
|
||
if (!(await isCfTrulyComplete(page, antiBot))) {
|
||
console.log('[CF] 二次 goto 后 CF 未完成，等待过盾...');
|
||
const cfWait = await waitForCfTrulyComplete(page, antiBot.challengeTimeoutMs || 60000, 500, antiBot);
|
||
if (!cfWait.success) {
|
||
console.log(`[CF] SPA 等待后仍未到业务页 url=${page.url()}`);
|
||
return false;
|
||
}
|
||
}
|
||
console.log(`[CF] Whatshub 二次 goto 后 SPA 等待 settle=${spaWaitMs}ms current=${page.url()}`);
|
||
await new Promise((resolve) => setTimeout(resolve, spaWaitMs));
|
||
await page.waitForNetworkIdle({ idleTime: 500, timeout: 15000 }).catch(() => {});
|
||
if (isBusinessUrlReady(page, urlNeedle)) {
|
||
console.log(`[CF] 二次 goto 后 SPA 已跳转 url=${page.url()}`);
|
||
return true;
|
||
}
|
||
}
|
||
|
||
console.log(`[CF] SPA 等待后仍未到业务页 url=${page.url()}`);
|
||
return false;
|
||
}
|
||
|
||
/** 非 Whatshub 场景：软 reload；Whatshub 短链改用二次 goto */
|
||
async function softReloadForSpa(page, waitForUrlSettledFn, antiBot, pageUrl = '') {
|
||
const targetUrl = pageUrl || page.url();
|
||
if (isWhatshubShortLinkPath(targetUrl)) {
|
||
console.log(`[CF] Whatshub 跳过 soft reload，改用二次 goto url=${targetUrl}`);
|
||
return waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettledFn, targetUrl);
|
||
}
|
||
console.log(`[CF] 尝试软 reload（保留 clearance）url=${page.url()}`);
|
||
await page.reload({ waitUntil: 'domcontentloaded', timeout: NAVIGATION_TIMEOUT_MS }).catch(() => {});
|
||
if (typeof waitForUrlSettledFn === 'function') {
|
||
await waitForUrlSettledFn(page).catch(() => {});
|
||
}
|
||
return waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettledFn, targetUrl);
|
||
}
|
||
|
||
/** CF 是否真正完成：默认需 cf_clearance；A2C 等 cfClearanceRequired=false 时仅看真实挑战 */
|
||
async function isCfTrulyComplete(page, antiBot = null) {
|
||
if (urlIndicatesCfChallenge(page.url())) {
|
||
return false;
|
||
}
|
||
if (antiBot && antiBot.cfClearanceRequired === false) {
|
||
const state = await detectCloudflare(page, antiBot);
|
||
return !state.blocked;
|
||
}
|
||
const cookies = await page.cookies();
|
||
if (!cookies.some((c) => c.name === 'cf_clearance')) {
|
||
return false;
|
||
}
|
||
const state = await detectCloudflare(page, antiBot);
|
||
return !state.blocked;
|
||
}
|
||
|
||
/** 轮询直到 CF 真正完成；并行扫描 sitekey 供 CapSolver 提前使用 */
|
||
async function waitForCfTrulyComplete(page, timeoutMs, pollMs = 500, antiBot = null) {
|
||
const { getSitekeyCapture } = require('./lib/sitekey-capture');
|
||
const capture = getSitekeyCapture(page);
|
||
const started = Date.now();
|
||
while (Date.now() - started < timeoutMs) {
|
||
if (await isCfTrulyComplete(page, antiBot)) {
|
||
return { success: true, elapsedMs: Date.now() - started };
|
||
}
|
||
if (capture) {
|
||
await capture.scanNow(page, antiBot).catch(() => {});
|
||
}
|
||
await new Promise((resolve) => setTimeout(resolve, pollMs));
|
||
}
|
||
return { success: false, elapsedMs: Date.now() - started };
|
||
}
|
||
|
||
/** 校验 cfResult，防止 handleCloudflareChallenge 假 success */
|
||
async function finalizeCfResult(page, cfResult, antiBot = null) {
|
||
if (!cfResult || !cfResult.success) {
|
||
return cfResult;
|
||
}
|
||
if (await isCfTrulyComplete(page, antiBot)) {
|
||
return cfResult;
|
||
}
|
||
console.log(`[CF] 过盾返回 success 但 CF 未真正完成 url=${page.url()}`);
|
||
return {
|
||
success: false,
|
||
code: 'CF_TURNSTILE_FAILED',
|
||
challengeType: cfResult.challengeType || 'turnstile',
|
||
stage: 'incomplete',
|
||
solverUsed: cfResult.solverUsed || false,
|
||
elapsedMs: cfResult.elapsedMs || 0,
|
||
cfDetected: true,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 完整过盾：内置 Real Browser Turnstile 轮询 → Captcha API 兜底
|
||
*/
|
||
async function runCfChallengeFlow(page, antiBot, waitForUrlSettled, pageUrl = '') {
|
||
const started = Date.now();
|
||
const timeoutMs = Math.min(Math.max(Number(antiBot.challengeTimeoutMs) || 60000, 5000), 120000);
|
||
const navUrl = pageUrl || page.url();
|
||
console.log(`[CF] 开始过盾 url=${page.url()}`);
|
||
|
||
if (waitForUrlSettled) {
|
||
await waitForUrlSettled(page).catch(() => {});
|
||
}
|
||
|
||
if (antiBot.cfClearanceRequired === false) {
|
||
const preState = await detectCloudflare(page, antiBot);
|
||
if (!preState.blocked) {
|
||
console.log('[CF] 免 clearance 模式：无真实 CF 挑战，跳过 Turnstile/CapSolver');
|
||
await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettled, navUrl);
|
||
return {
|
||
success: true,
|
||
code: null,
|
||
challengeType: null,
|
||
stage: 'no_clearance_required',
|
||
solverUsed: false,
|
||
elapsedMs: Date.now() - started,
|
||
cfDetected: false,
|
||
};
|
||
}
|
||
console.log(`[CF] 免 clearance 模式但检测到真实 CF 挑战 type=${preState.challengeType}，继续过盾`);
|
||
}
|
||
|
||
const preferManaged = shouldPreferManagedCloudflareSolver(antiBot);
|
||
|
||
if (preferManaged && antiBot.solverFallback !== false && isCaptchaConfigured()) {
|
||
console.log('[CF] Managed Challenge：跳过内置 60s 轮询，直接使用 CapSolver AntiCloudflareTask');
|
||
try {
|
||
if (waitForUrlSettled) {
|
||
await waitForUrlSettled(page).catch(() => {});
|
||
}
|
||
await runCaptchaApiFallback(page, {
|
||
antiBot,
|
||
navUrl,
|
||
waitForUrlSettled,
|
||
timeoutMs,
|
||
});
|
||
|
||
const apiWait = await waitForCfTrulyComplete(page, Math.min(timeoutMs, 60000), 500, antiBot);
|
||
if (apiWait.success) {
|
||
if (waitForUrlSettled) {
|
||
await waitForUrlSettled(page).catch(() => {});
|
||
}
|
||
await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettled, navUrl);
|
||
return {
|
||
success: true,
|
||
code: null,
|
||
challengeType: 'managed',
|
||
stage: 'api_managed',
|
||
solverUsed: true,
|
||
elapsedMs: Date.now() - started,
|
||
cfDetected: true,
|
||
};
|
||
}
|
||
console.warn('[CF] Managed CapSolver 注入后仍未获得有效 cf_clearance');
|
||
} catch (apiErr) {
|
||
console.error('[CF] Managed Challenge CapSolver 失败:', apiErr.message);
|
||
return {
|
||
success: false,
|
||
code: 'CF_TURNSTILE_FAILED',
|
||
challengeType: 'managed',
|
||
stage: 'api_managed',
|
||
solverUsed: true,
|
||
elapsedMs: Date.now() - started,
|
||
cfDetected: true,
|
||
};
|
||
}
|
||
|
||
return {
|
||
success: false,
|
||
code: 'CF_TURNSTILE_FAILED',
|
||
challengeType: 'managed',
|
||
stage: 'api_managed',
|
||
solverUsed: true,
|
||
elapsedMs: Date.now() - started,
|
||
cfDetected: true,
|
||
};
|
||
}
|
||
|
||
if (antiBot.turnstile !== false) {
|
||
console.log(`[CF] 内置 Turnstile 轮询 timeout=${timeoutMs}ms`);
|
||
const builtIn = await waitForCfTrulyComplete(page, timeoutMs, 500, antiBot);
|
||
if (builtIn.success) {
|
||
if (waitForUrlSettled) {
|
||
await waitForUrlSettled(page).catch(() => {});
|
||
}
|
||
await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettled, navUrl);
|
||
return {
|
||
success: true,
|
||
code: null,
|
||
challengeType: 'turnstile',
|
||
stage: 'built_in',
|
||
solverUsed: false,
|
||
elapsedMs: Date.now() - started,
|
||
cfDetected: true,
|
||
};
|
||
}
|
||
console.warn(`[CF] 内置 Turnstile 未在 ${builtIn.elapsedMs}ms 内完成，尝试 Captcha API`);
|
||
}
|
||
|
||
if (antiBot.solverFallback !== false && isCaptchaConfigured()) {
|
||
try {
|
||
if (waitForUrlSettled) {
|
||
await waitForUrlSettled(page).catch(() => {});
|
||
}
|
||
await runCaptchaApiFallback(page, {
|
||
antiBot,
|
||
navUrl,
|
||
waitForUrlSettled,
|
||
timeoutMs,
|
||
});
|
||
|
||
const apiWait = await waitForCfTrulyComplete(page, Math.min(timeoutMs, 45000), 500, antiBot);
|
||
if (apiWait.success) {
|
||
if (waitForUrlSettled) {
|
||
await waitForUrlSettled(page).catch(() => {});
|
||
}
|
||
await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettled, navUrl);
|
||
return {
|
||
success: true,
|
||
code: null,
|
||
challengeType: 'turnstile',
|
||
stage: 'api',
|
||
solverUsed: true,
|
||
elapsedMs: Date.now() - started,
|
||
cfDetected: true,
|
||
};
|
||
}
|
||
console.warn('[CF] Captcha API token 注入后仍未获得 cf_clearance');
|
||
} catch (apiErr) {
|
||
console.error('[CF] Captcha API 兜底失败:', apiErr.message);
|
||
return {
|
||
success: false,
|
||
code: 'CF_TURNSTILE_FAILED',
|
||
challengeType: 'turnstile',
|
||
stage: 'api',
|
||
solverUsed: true,
|
||
elapsedMs: Date.now() - started,
|
||
cfDetected: true,
|
||
};
|
||
}
|
||
}
|
||
|
||
return {
|
||
success: false,
|
||
code: 'CF_TURNSTILE_FAILED',
|
||
challengeType: 'turnstile',
|
||
stage: antiBot.solverFallback !== false && !isCaptchaConfigured() ? 'built_in' : 'timeout',
|
||
solverUsed: false,
|
||
elapsedMs: Date.now() - started,
|
||
cfDetected: true,
|
||
};
|
||
}
|
||
|
||
/** 安全 CF 处理：未真正完成则走完整过盾，禁止 cf-handler 假过关 */
|
||
async function handleCloudflareChallengeSafe(page, antiBot, waitForUrlSettled, storedSession) {
|
||
if (!(await isCfTrulyComplete(page, antiBot))) {
|
||
return finalizeCfResult(page, await runCfChallengeFlow(page, antiBot, waitForUrlSettled), antiBot);
|
||
}
|
||
const result = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled, storedSession);
|
||
return finalizeCfResult(page, result, antiBot);
|
||
}
|
||
|
||
/** 是否应 purge 陈旧 clearance（短链 SPA 未跳转时禁止 purge） */
|
||
function shouldPurgeStaleClearance(page, cfState, urlNeedle = '') {
|
||
if (urlIndicatesCfChallenge(page.url())) {
|
||
return false;
|
||
}
|
||
if (urlNeedle && isOnShortLinkNotBusiness(page, urlNeedle) && cfState.hasCfClearance) {
|
||
return false;
|
||
}
|
||
return cfState.hasCfClearance && !cfState.blocked;
|
||
}
|
||
|
||
/** 业务页等待前确保 CF 已完成 */
|
||
async function ensureCfCompleteBeforeBusinessReady(page, antiBot, waitForUrlSettledFn, storedSession) {
|
||
if (await isCfTrulyComplete(page, antiBot)) {
|
||
return;
|
||
}
|
||
const cfState = await detectCloudflare(page, antiBot);
|
||
console.log(`[CF] 业务页等待前 CF 未完成 blocked=${cfState.blocked} url=${page.url()}`);
|
||
const cfResult = await handleCloudflareChallengeSafe(page, antiBot, waitForUrlSettledFn, storedSession);
|
||
if (!cfResult.success) {
|
||
const err = new Error('Cloudflare Turnstile 验证失败（业务页等待前）');
|
||
err.code = 'CF_TURNSTILE_FAILED';
|
||
throw err;
|
||
}
|
||
if (typeof waitForUrlSettledFn === 'function') {
|
||
await waitForUrlSettledFn(page).catch(() => {});
|
||
}
|
||
}
|
||
|
||
/** 清除 cf_clearance（含 Chrome Profile / CDP 层） */
|
||
async function clearCfClearanceCookies(page) {
|
||
const cookies = await page.cookies();
|
||
for (const c of cookies) {
|
||
if (c.name !== 'cf_clearance') {
|
||
continue;
|
||
}
|
||
const host = (c.domain || '').replace(/^\./, '');
|
||
if (host) {
|
||
await page.deleteCookie({ name: c.name, url: `https://${host}/` }).catch(() => {});
|
||
}
|
||
await page.deleteCookie({
|
||
name: c.name,
|
||
domain: c.domain,
|
||
path: c.path || '/',
|
||
}).catch(() => {});
|
||
}
|
||
}
|
||
|
||
async function purgeCfClearanceFully(page, pageUrl) {
|
||
await clearCfClearanceCookies(page);
|
||
let hostname = '';
|
||
try {
|
||
hostname = new URL(pageUrl).hostname;
|
||
} catch (_) {
|
||
return;
|
||
}
|
||
try {
|
||
const cdp = await page.createCDPSession();
|
||
const domains = new Set([hostname, `.${hostname}`]);
|
||
const cookies = await page.cookies();
|
||
for (const c of cookies) {
|
||
if (c.domain) {
|
||
domains.add(c.domain);
|
||
}
|
||
}
|
||
for (const domain of domains) {
|
||
await cdp.send('Network.deleteCookies', { name: 'cf_clearance', domain }).catch(() => {});
|
||
}
|
||
} catch (err) {
|
||
console.warn('[CF] CDP 清除 cf_clearance 失败:', err.message);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Whatshub 短链首访：清除 Chrome Profile 内陈旧 cf_clearance，确保 Turnstile+SPA 在同一 goto 生命周期
|
||
* session 复用时跳过 purge，保留 clearance 加速二次同步
|
||
*/
|
||
async function prepareWhatshubShortLinkBeforeNav(page, pageUrl, antiBot, storedSession = null) {
|
||
if (!antiBot?.enabled || antiBot.profile !== 'real' || !isWhatshubShortLinkPath(pageUrl)) {
|
||
return;
|
||
}
|
||
if (antiBot.skipShortLinkCfPurgeOnReuse) {
|
||
let session = storedSession;
|
||
if (!session && antiBot.sessionKey) {
|
||
session = loadSession(antiBot.sessionKey);
|
||
}
|
||
if (session && isSessionLikelyValid(session)) {
|
||
console.log('[Whatshub] 有效 session 复用，跳过短链 cf_clearance 清除');
|
||
return;
|
||
}
|
||
}
|
||
const cookies = await page.cookies();
|
||
if (!cookies.some((c) => c.name === 'cf_clearance')) {
|
||
return;
|
||
}
|
||
console.log('[Whatshub] 短链首访：清除 Profile/页面内陈旧 cf_clearance');
|
||
await purgeCfClearanceFully(page, pageUrl);
|
||
}
|
||
|
||
/**
|
||
* CF 处理 + 业务 URL 守卫：磁盘 session 陈旧 clearance 时 purge；__cf_chl 中间态只过盾不 purge
|
||
*/
|
||
async function runCloudflareWithBusinessGuard(page, antiBot, waitForUrlSettled, storedSession, pageUrl, browserCtx = null) {
|
||
const urlNeedle = (antiBot.postCfReadyUrlContains || '').trim();
|
||
const hasStoredSession = !!(storedSession && isSessionLikelyValid(storedSession));
|
||
|
||
async function maybeEnsureProxy(currentPage) {
|
||
if (!browserCtx) {
|
||
return currentPage;
|
||
}
|
||
browserCtx.page = currentPage;
|
||
await ensureCaptchaProxyBrowserIfNeeded(browserCtx);
|
||
return browserCtx.page;
|
||
}
|
||
|
||
async function reNavigateWithCfPurge(label) {
|
||
if (urlIndicatesCfChallenge(page.url())) {
|
||
console.log(`[CF] ${label}: URL 含 __cf_chl，跳过 purge，直接过盾`);
|
||
return handleCloudflareChallengeSafe(page, antiBot, waitForUrlSettled, storedSession);
|
||
}
|
||
console.log(`[CF] ${label}: 清除陈旧 CF 并重新 goto ${pageUrl} (current=${page.url()})`);
|
||
await purgeCfClearanceFully(page, pageUrl);
|
||
await navigateToPage(page, pageUrl);
|
||
if (waitForUrlSettled) {
|
||
await waitForUrlSettled(page).catch(() => {});
|
||
}
|
||
page = await maybeEnsureProxy(page);
|
||
return handleCloudflareChallengeSafe(page, antiBot, waitForUrlSettled, storedSession);
|
||
}
|
||
|
||
let cfResult;
|
||
if (urlNeedle && !isBusinessUrlReady(page, urlNeedle) && hasStoredSession) {
|
||
const pre = await detectCloudflare(page, antiBot);
|
||
if (shouldPurgeStaleClearance(page, pre, urlNeedle)) {
|
||
cfResult = await reNavigateWithCfPurge('预检(磁盘会话-陈旧clearance)');
|
||
} else if (await isCfTrulyComplete(page, antiBot)) {
|
||
await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettled, pageUrl);
|
||
}
|
||
}
|
||
|
||
if (!cfResult) {
|
||
page = await maybeEnsureProxy(page);
|
||
cfResult = await handleCloudflareChallengeSafe(page, antiBot, waitForUrlSettled, storedSession);
|
||
}
|
||
|
||
// 短链跨域落地（如 yyk.ink → user.a2c.chat、Whatshub /m/ → work-order-sharing）后，业务域可能仍有 CF
|
||
if (cfResult.success && !(await isCfTrulyComplete(page, antiBot))) {
|
||
const crossState = await detectCloudflare(page, antiBot);
|
||
if (crossState.blocked || urlIndicatesCfChallenge(page.url())) {
|
||
const isWhatshubLongCf =
|
||
urlNeedle === 'work-order-sharing' && urlIndicatesCfChallenge(page.url());
|
||
if (isWhatshubLongCf) {
|
||
console.log(`[CF] Whatshub 长链二次过盾 url=${page.url()}`);
|
||
} else {
|
||
console.log(`[CF] 跨域落地后 CF 未完成，二次过盾 url=${page.url()}`);
|
||
}
|
||
page = await maybeEnsureProxy(page);
|
||
cfResult = await finalizeCfResult(
|
||
page,
|
||
await runCfChallengeFlow(page, antiBot, waitForUrlSettled, page.url()),
|
||
antiBot
|
||
);
|
||
}
|
||
}
|
||
|
||
if (cfResult.success && urlNeedle && !isBusinessUrlReady(page, urlNeedle)) {
|
||
await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettled, pageUrl);
|
||
}
|
||
|
||
let retries = 0;
|
||
while (
|
||
cfResult.success
|
||
&& urlNeedle
|
||
&& !isBusinessUrlReady(page, urlNeedle)
|
||
&& retries < 2
|
||
) {
|
||
retries += 1;
|
||
if (!(await isCfTrulyComplete(page, antiBot))) {
|
||
console.log(`[CF] 业务页未达且 CF 未完成，完整过盾 重试#${retries}`);
|
||
page = await maybeEnsureProxy(page);
|
||
cfResult = await finalizeCfResult(
|
||
page,
|
||
await runCfChallengeFlow(page, antiBot, waitForUrlSettled, pageUrl),
|
||
antiBot
|
||
);
|
||
} else {
|
||
console.log(`[CF] CF 已完成但未到业务页，等待 SPA 跳转 重试#${retries}`);
|
||
const spaOk = await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettled, pageUrl);
|
||
if (!spaOk && retries >= 2) {
|
||
await softReloadForSpa(page, waitForUrlSettled, antiBot, pageUrl);
|
||
}
|
||
if (isBusinessUrlReady(page, urlNeedle)) {
|
||
break;
|
||
}
|
||
}
|
||
if (!cfResult.success) {
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (browserCtx) {
|
||
browserCtx.page = page;
|
||
}
|
||
|
||
return finalizeCfResult(page, cfResult, antiBot);
|
||
}
|
||
|
||
async function collectPageDiagnostics(page) {
|
||
const url = page.url();
|
||
const snapshot = await page.evaluate(() => ({
|
||
url: window.location.href,
|
||
title: document.title || '',
|
||
bodyText: (document.body && document.body.innerText ? document.body.innerText : '').slice(0, 500),
|
||
elInputInner: document.querySelectorAll('.el-input__inner').length,
|
||
vxeForm: document.querySelectorAll('form.vxe-form').length,
|
||
vxePasswordInput: document.querySelectorAll('.vxe-form .el-input__inner').length,
|
||
vxeConfirmBtn: document.querySelectorAll('.vxe-form button.theme--primary').length,
|
||
elTable: document.querySelectorAll('.el-table').length,
|
||
vxeTable: document.querySelectorAll('.vxe-table, .vxe-grid').length,
|
||
btnNext: document.querySelectorAll('.btn-next').length,
|
||
elPagination: document.querySelectorAll('.el-pagination').length,
|
||
})).catch(() => null);
|
||
return snapshot || { url, title: '', bodyText: '', elInputInner: 0 };
|
||
}
|
||
|
||
async function awaitPostCfBusinessReady(page, antiBot, waitForUrlSettledFn, storedSession = null, pageUrl = '', apiUrls = null) {
|
||
if (!antiBot || !antiBot.enabled) {
|
||
return;
|
||
}
|
||
const urlContains = (antiBot.postCfReadyUrlContains || '').trim();
|
||
const selector = (antiBot.postCfReadySelector || '').trim();
|
||
if (urlContains === '' && selector === '') {
|
||
return;
|
||
}
|
||
|
||
const navUrl = pageUrl || page.url();
|
||
const skipApiWait = shouldSkipPostCfReadyApiWait(antiBot, apiUrls);
|
||
|
||
await ensureCfCompleteBeforeBusinessReady(page, antiBot, waitForUrlSettledFn, storedSession);
|
||
|
||
const onA2cDirect = isA2cDirectSharePage(page, urlContains);
|
||
const whatshubHost = isWhatshubHost(navUrl) || isWhatshubHost(page.url());
|
||
let skipSpaWait = false;
|
||
if (onA2cDirect && await isCfTrulyComplete(page, antiBot)) {
|
||
const domReady = selector
|
||
? await page.$(selector).catch(() => null)
|
||
: true;
|
||
if (domReady) {
|
||
console.log('[CF] A2C 业务 DOM 已就绪，跳过重复 SPA 等待');
|
||
skipSpaWait = true;
|
||
}
|
||
}
|
||
// Whatshub session 复用：已在 work-order-sharing 且无密码弹窗时跳过 SPA 固定等待
|
||
if (!skipSpaWait && whatshubHost && urlContains && page.url().includes(urlContains)) {
|
||
if (await isCfTrulyComplete(page, antiBot)) {
|
||
const pwdVisible = await isWhatshubPasswordDialogVisible(page);
|
||
const tableReady = await page.$('.vxe-table, .vxe-grid').catch(() => null);
|
||
if (!pwdVisible && (tableReady || (storedSession && isSessionLikelyValid(storedSession)))) {
|
||
console.log('[CF] Whatshub 业务页已就绪（session 复用），跳过 SPA 等待');
|
||
skipSpaWait = true;
|
||
}
|
||
}
|
||
}
|
||
if (!skipSpaWait) {
|
||
await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettledFn, navUrl);
|
||
}
|
||
|
||
const timeoutMs = Math.max(5000, antiBot.postCfReadyTimeoutMs || 30000);
|
||
console.log(
|
||
`[CF] 等待业务页就绪 current=${page.url()} urlContains="${urlContains}" `
|
||
+ `selector="${selector}" timeout=${timeoutMs}ms skipApiWait=${skipApiWait}`
|
||
);
|
||
|
||
try {
|
||
if (urlContains !== '') {
|
||
await page.waitForFunction(
|
||
(needle) => window.location.href.includes(needle),
|
||
{ timeout: timeoutMs, polling: 200 },
|
||
urlContains
|
||
);
|
||
}
|
||
if (!skipApiWait) {
|
||
await waitForPostCfListApi(page, antiBot, timeoutMs);
|
||
} else {
|
||
console.log('[CF] 跳过 pre-auth 业务 API 等待（statistics 在密码验证后触发）');
|
||
}
|
||
if (selector !== '') {
|
||
await waitForPostCfSelectorOptional(page, selector, timeoutMs, antiBot);
|
||
}
|
||
} catch (err) {
|
||
const retryMs = Math.min(30000, timeoutMs);
|
||
const needsRetry = urlContains !== '' && !page.url().includes(urlContains);
|
||
const needsDomRetry = urlContains !== '' && page.url().includes(urlContains) && isA2cDirectSharePage(page, urlContains);
|
||
if (needsRetry) {
|
||
if (!(await isCfTrulyComplete(page, antiBot))) {
|
||
console.warn(`[CF] 业务页等待失败且 CF 未完成 url=${page.url()}，完整过盾后重试`);
|
||
try {
|
||
await ensureCfCompleteBeforeBusinessReady(page, antiBot, waitForUrlSettledFn, storedSession);
|
||
if (urlContains !== '') {
|
||
await page.waitForFunction(
|
||
(needle) => window.location.href.includes(needle),
|
||
{ timeout: retryMs, polling: 200 },
|
||
urlContains
|
||
);
|
||
}
|
||
if (selector !== '') {
|
||
await waitForPostCfSelectorOptional(page, selector, retryMs, antiBot);
|
||
}
|
||
} catch (retryErr) {
|
||
retryErr.pageDiagnostics = await collectPageDiagnostics(page);
|
||
throw retryErr;
|
||
}
|
||
} else {
|
||
console.warn(`[CF] 业务页等待失败 url=${page.url()}，SPA 等待 + 二次 goto`);
|
||
try {
|
||
await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettledFn, navUrl);
|
||
if (!page.url().includes(urlContains)) {
|
||
await softReloadForSpa(page, waitForUrlSettledFn, antiBot, navUrl);
|
||
}
|
||
if (urlContains !== '') {
|
||
await page.waitForFunction(
|
||
(needle) => window.location.href.includes(needle),
|
||
{ timeout: retryMs, polling: 200 },
|
||
urlContains
|
||
);
|
||
}
|
||
if (!skipApiWait) {
|
||
await waitForPostCfListApi(page, antiBot, retryMs);
|
||
}
|
||
if (selector !== '') {
|
||
await waitForPostCfSelectorOptional(page, selector, retryMs, antiBot);
|
||
}
|
||
} catch (retryErr) {
|
||
retryErr.pageDiagnostics = await collectPageDiagnostics(page);
|
||
throw retryErr;
|
||
}
|
||
}
|
||
} else if (needsDomRetry) {
|
||
console.warn(`[CF] A2C URL 已就绪但 DOM 未挂载 url=${page.url()}，soft reload 后重试`);
|
||
try {
|
||
await page.waitForNetworkIdle({ idleTime: 500, timeout: 20000 }).catch(() => {});
|
||
await softReloadForSpa(page, waitForUrlSettledFn, antiBot, navUrl);
|
||
if (!skipApiWait) {
|
||
await waitForPostCfListApi(page, antiBot, retryMs);
|
||
}
|
||
if (selector !== '') {
|
||
await waitForPostCfSelectorOptional(page, selector, retryMs, antiBot);
|
||
}
|
||
} catch (retryErr) {
|
||
retryErr.pageDiagnostics = await collectPageDiagnostics(page);
|
||
throw retryErr;
|
||
}
|
||
} else {
|
||
err.pageDiagnostics = await collectPageDiagnostics(page);
|
||
throw err;
|
||
}
|
||
}
|
||
|
||
const settleMs = antiBot.postCfSettleMs || 0;
|
||
if (settleMs > 0) {
|
||
await new Promise((resolve) => setTimeout(resolve, settleMs));
|
||
}
|
||
console.log(`[CF] 业务页就绪 url=${page.url()}`);
|
||
}
|
||
|
||
async function executeAuthActionsWithDiagnostics(page, authActions, antiBot = null) {
|
||
try {
|
||
await executeAuthActions(page, authActions, antiBot);
|
||
} catch (err) {
|
||
err.pageDiagnostics = await collectPageDiagnostics(page);
|
||
throw err;
|
||
}
|
||
}
|
||
|
||
async function shouldSkipWhatshubAuthActions(page, antiBot) {
|
||
if (!antiBot?.authSkipIfNoPasswordDialog) {
|
||
return false;
|
||
}
|
||
const urlContains = (antiBot.postCfReadyUrlContains || 'work-order-sharing').trim();
|
||
try {
|
||
const onBusiness = !urlContains || page.url().includes(urlContains);
|
||
if (!onBusiness) {
|
||
return false;
|
||
}
|
||
const pwdVisible = await isWhatshubPasswordDialogVisible(page);
|
||
if (!pwdVisible) {
|
||
return true;
|
||
}
|
||
} catch (_) {
|
||
/* ignore */
|
||
}
|
||
return false;
|
||
}
|
||
|
||
async function waitForWhatshubPasswordFormIfNeeded(page, antiBot) {
|
||
if (!antiBot || !(isWhatshubHost(page.url()) || page.url().includes('work-order-sharing'))) {
|
||
return;
|
||
}
|
||
if (await shouldSkipWhatshubAuthActions(page, antiBot)) {
|
||
return;
|
||
}
|
||
const timeout = resolveAuthSelectorTimeoutMs(antiBot);
|
||
console.log('[Whatshub] 等待 vxe 密码表单出现...');
|
||
await page.waitForSelector('.vxe-form .el-input__inner', { visible: true, timeout }).catch(() => null);
|
||
}
|
||
|
||
async function executeAuthActions(page, authActions, antiBot = null) {
|
||
if (!Array.isArray(authActions)) return;
|
||
|
||
if (await shouldSkipWhatshubAuthActions(page, antiBot)) {
|
||
console.log('[Whatshub] Session 已登录且无密码弹窗，跳过 authActions');
|
||
return;
|
||
}
|
||
|
||
await waitForWhatshubPasswordFormIfNeeded(page, antiBot);
|
||
|
||
const selectorTimeout = resolveAuthSelectorTimeoutMs(antiBot);
|
||
|
||
for (const action of authActions) {
|
||
if (action.type === 'type') {
|
||
await page.waitForSelector(action.selector, { timeout: selectorTimeout });
|
||
await page.type(action.selector, action.value, { delay: 100 });
|
||
} else if (action.type === 'click') {
|
||
await page.waitForSelector(action.selector, { timeout: selectorTimeout });
|
||
await page.evaluate((sel) => { const el = document.querySelector(sel); if (el) el.click(); }, action.selector);
|
||
} else if (action.type === 'press') {
|
||
await new Promise((r) => setTimeout(r, 200));
|
||
await page.keyboard.press(action.key);
|
||
} else if (action.type === 'wait') {
|
||
await new Promise((r) => setTimeout(r, action.ms));
|
||
} else if (action.type === 'vue_fill') {
|
||
await page.waitForSelector(action.selector, { timeout: selectorTimeout });
|
||
await page.waitForFunction(
|
||
(sel) => {
|
||
const el = document.querySelector(sel);
|
||
if (!el) return false;
|
||
const rect = el.getBoundingClientRect();
|
||
return rect.width > 0 && rect.height > 0;
|
||
},
|
||
{ timeout: 2000, polling: 100 },
|
||
action.selector
|
||
).catch(() => new Promise((r) => setTimeout(r, 500)));
|
||
|
||
await page.evaluate((sel, val) => {
|
||
const inputs = document.querySelectorAll(sel);
|
||
for (const input of inputs) {
|
||
if (input.getBoundingClientRect().width > 0 && input.getBoundingClientRect().height > 0) {
|
||
input.value = val;
|
||
input.dispatchEvent(new Event('input', { bubbles: true }));
|
||
input.dispatchEvent(new Event('change', { bubbles: true }));
|
||
input.focus();
|
||
return;
|
||
}
|
||
}
|
||
}, action.selector, action.value);
|
||
} else if (action.type === 'vue_click') {
|
||
await page.waitForSelector(action.selector, { timeout: selectorTimeout });
|
||
await page.waitForFunction(
|
||
(sel) => {
|
||
const el = document.querySelector(sel);
|
||
if (!el) return false;
|
||
const rect = el.getBoundingClientRect();
|
||
return rect.width > 0 && rect.height > 0;
|
||
},
|
||
{ timeout: 2000, polling: 100 },
|
||
action.selector
|
||
).catch(() => new Promise((r) => setTimeout(r, 500)));
|
||
|
||
await page.evaluate((sel, matchText) => {
|
||
const btns = document.querySelectorAll(sel);
|
||
for (const btn of btns) {
|
||
if (btn.getBoundingClientRect().width > 0 && btn.getBoundingClientRect().height > 0) {
|
||
let shouldClick = false;
|
||
if (matchText) {
|
||
const text = (btn.textContent || btn.innerText || '').trim();
|
||
if (text.includes(matchText.trim())) { shouldClick = true; }
|
||
} else {
|
||
shouldClick = true;
|
||
}
|
||
if (shouldClick) {
|
||
btn.scrollIntoView({ behavior: 'instant', block: 'center' });
|
||
const eventOpts = { bubbles: true, cancelable: true, view: window };
|
||
btn.dispatchEvent(new MouseEvent('mousedown', eventOpts));
|
||
btn.dispatchEvent(new MouseEvent('mouseup', eventOpts));
|
||
btn.dispatchEvent(new MouseEvent('click', eventOpts));
|
||
btn.click();
|
||
return;
|
||
}
|
||
}
|
||
}
|
||
}, action.selector, action.text);
|
||
}
|
||
}
|
||
}
|
||
|
||
function createApiInterceptor(page, apiUrls, defaultTimeoutMs = API_INTERCEPT_TIMEOUT_MS) {
|
||
const interceptedApis = {};
|
||
let responseHandler = null;
|
||
let timeoutId = null;
|
||
let resolveWait = null;
|
||
let waitPromise = null;
|
||
|
||
const tryResolveWait = () => {
|
||
if (!resolveWait || Object.keys(interceptedApis).length < apiUrls.length) {
|
||
return;
|
||
}
|
||
if (timeoutId) {
|
||
clearTimeout(timeoutId);
|
||
timeoutId = null;
|
||
}
|
||
if (responseHandler) {
|
||
page.off('response', responseHandler);
|
||
responseHandler = null;
|
||
}
|
||
const done = resolveWait;
|
||
resolveWait = null;
|
||
done();
|
||
};
|
||
|
||
const cleanup = () => {
|
||
if (timeoutId) {
|
||
clearTimeout(timeoutId);
|
||
timeoutId = null;
|
||
}
|
||
if (responseHandler) {
|
||
page.off('response', responseHandler);
|
||
responseHandler = null;
|
||
}
|
||
if (resolveWait) {
|
||
const done = resolveWait;
|
||
resolveWait = null;
|
||
done();
|
||
}
|
||
};
|
||
|
||
responseHandler = async (response) => {
|
||
const request = response.request();
|
||
if (request.method() === 'OPTIONS') return;
|
||
const matchedPath = apiUrls.find((api) => request.url().includes(api) && !interceptedApis[api]);
|
||
if (matchedPath && response.status() >= 200 && response.status() < 300) {
|
||
try {
|
||
interceptedApis[matchedPath] = {
|
||
url: request.url(),
|
||
headers: request.headers(),
|
||
data: JSON.parse(await response.text()),
|
||
};
|
||
tryResolveWait();
|
||
} catch (_) { /* JSON 解析失败跳过 */ }
|
||
}
|
||
};
|
||
page.on('response', responseHandler);
|
||
|
||
const beginWaiting = (timeoutMs = defaultTimeoutMs) => {
|
||
if (waitPromise) {
|
||
return waitPromise;
|
||
}
|
||
if (Object.keys(interceptedApis).length >= apiUrls.length) {
|
||
waitPromise = Promise.resolve();
|
||
return waitPromise;
|
||
}
|
||
waitPromise = new Promise((resolve) => {
|
||
resolveWait = resolve;
|
||
timeoutId = setTimeout(() => {
|
||
cleanup();
|
||
resolve();
|
||
}, timeoutMs);
|
||
});
|
||
return waitPromise;
|
||
};
|
||
|
||
const waitForApis = async (timeoutMs = defaultTimeoutMs) => {
|
||
await beginWaiting(timeoutMs);
|
||
return waitPromise;
|
||
};
|
||
|
||
return { interceptedApis, beginWaiting, waitForApis, cleanup };
|
||
}
|
||
|
||
async function seedShareTokenCookie(page, pageUrl) {
|
||
try {
|
||
const parsed = new URL(pageUrl);
|
||
const token = parsed.searchParams.get('token');
|
||
if (!token) return;
|
||
const cookieUrl = `http://${parsed.hostname}/`;
|
||
await page.setCookie({ name: 'share_token', value: token, url: cookieUrl });
|
||
} catch (_) { /* ignore */ }
|
||
}
|
||
|
||
function resolveShareToken(...urls) {
|
||
for (const url of urls) {
|
||
const token = extractShareTokenFromUrl(url);
|
||
if (token) return token;
|
||
}
|
||
return '';
|
||
}
|
||
|
||
function resolveInterceptTimeout(antiBot) {
|
||
const cfg = normalizeAntiBot(antiBot);
|
||
return cfg.enabled && cfg.profile === 'real'
|
||
? API_INTERCEPT_TIMEOUT_REAL_MS
|
||
: API_INTERCEPT_TIMEOUT_MS;
|
||
}
|
||
|
||
/** 并发监控：供上游调度器判断负载 */
|
||
app.get('/api/stats', (_req, res) => {
|
||
const executablePath = resolveChromeExecutable();
|
||
const factoryStats = getFactoryStats();
|
||
res.json({
|
||
success: true,
|
||
shuttingDown,
|
||
queue: factoryStats.standard,
|
||
queueReal: factoryStats.real,
|
||
pool: factoryStats.pool,
|
||
sessions: getSessionStats(),
|
||
profiles: getProfileStats(),
|
||
realBrowserReady: factoryStats.realBrowserReady,
|
||
captchaConfigured: isCaptchaConfigured(),
|
||
chrome: {
|
||
executablePath,
|
||
exists: fs.existsSync(executablePath),
|
||
},
|
||
config: {
|
||
maxConcurrentBrowsers: MAX_CONCURRENT_BROWSERS,
|
||
maxConcurrentBrowsersReal: MAX_CONCURRENT_BROWSERS_REAL,
|
||
apiInterceptTimeoutMs: API_INTERCEPT_TIMEOUT_MS,
|
||
apiInterceptTimeoutRealMs: API_INTERCEPT_TIMEOUT_REAL_MS,
|
||
navigationTimeoutMs: NAVIGATION_TIMEOUT_MS,
|
||
navigationWaitUntil: NAVIGATION_WAIT_UNTIL,
|
||
sessionTtlMs: SESSION_TTL_MS,
|
||
persistBrowserProfile: PERSIST_BROWSER_PROFILE,
|
||
poolIdleMs: POOL_IDLE_MS,
|
||
maxPooledBrowsers: MAX_POOLED_BROWSERS,
|
||
},
|
||
});
|
||
});
|
||
|
||
/** 运维调试：查看指定 sessionKey 的磁盘会话与 profile 状态 */
|
||
app.get('/api/session/:sessionKey', (req, res) => {
|
||
const sessionKey = decodeURIComponent(req.params.sessionKey || '');
|
||
if (!sessionKey) {
|
||
return res.status(400).json({ success: false, error: 'sessionKey 不能为空' });
|
||
}
|
||
const info = getSessionDebugInfo(sessionKey);
|
||
const profileStandard = getProfileStats();
|
||
res.json({
|
||
success: true,
|
||
session: info,
|
||
profileExists: {
|
||
standard: profileExists(sessionKey, 'standard'),
|
||
real: profileExists(sessionKey, 'real'),
|
||
},
|
||
profilesDir: profileStandard.profilesDir,
|
||
});
|
||
});
|
||
|
||
/** 运维自检：实际尝试 launch 一次 Browser（profile=standard|real） */
|
||
app.get('/api/browser-launch-test', async (req, res) => {
|
||
const profile = req.query.profile === 'real' ? 'real' : 'standard';
|
||
let browser;
|
||
try {
|
||
await runBrowserTask(`browser-launch-test-${profile}`, async () => {
|
||
if (profile === 'real') {
|
||
const session = await createBrowserSession({ profile: 'real', extraArgs: ['--mute-audio'] });
|
||
browser = session.browser;
|
||
const pages = await browser.pages();
|
||
return { pageCount: pages.length, profile };
|
||
}
|
||
browser = await launchStandardBrowser(['--mute-audio']);
|
||
const pages = await browser.pages();
|
||
return { pageCount: pages.length, profile };
|
||
}, profile);
|
||
res.json({ success: true, message: 'Browser 启动成功', profile });
|
||
} catch (error) {
|
||
res.status(500).json({
|
||
success: false,
|
||
error: error.message,
|
||
profile,
|
||
chrome: {
|
||
executablePath: resolveChromeExecutable(),
|
||
exists: fs.existsSync(resolveChromeExecutable()),
|
||
},
|
||
realBrowserReady: isRealBrowserAvailable(),
|
||
runtimeDir: RUNTIME_DIR,
|
||
});
|
||
} finally {
|
||
if (browser) {
|
||
try {
|
||
await browser.close();
|
||
} catch (_) { /* ignore */ }
|
||
}
|
||
}
|
||
});
|
||
|
||
/** 接口一：初始化授权与首屏拦截 */
|
||
app.post('/api/auth-and-intercept', async (req, res) => {
|
||
const { pageUrl, apiUrls, authActions, antiBot: rawAntiBot } = req.body;
|
||
if (!pageUrl || !Array.isArray(apiUrls)) {
|
||
return res.status(400).json({ success: false, error: '参数错误' });
|
||
}
|
||
|
||
const antiBot = resolveAntiBotForPage(rawAntiBot, pageUrl);
|
||
const profile = antiBot.profile || 'standard';
|
||
const interceptTimeout = resolveInterceptTimeout(antiBot);
|
||
const cfLog = { profile, cfDetected: false, turnstileStage: null, solverUsed: false, elapsedMs: 0, browserReused: false };
|
||
|
||
try {
|
||
await runBrowserTask('auth-and-intercept', async () => {
|
||
let browserSession = null;
|
||
let interceptorCleanup = null;
|
||
let page = null;
|
||
const taskStarted = Date.now();
|
||
let cfDestroyBrowser = false;
|
||
|
||
try {
|
||
browserSession = await acquireBrowserSession({
|
||
profile,
|
||
extraArgs: ['--mute-audio'],
|
||
sessionKey: antiBot.sessionKey || '',
|
||
});
|
||
tagBrowserSession(browserSession, profile, antiBot.sessionKey || '', ['--mute-audio']);
|
||
cfLog.browserReused = !!browserSession.reused;
|
||
|
||
purgeStoredSessionIfRequested(antiBot);
|
||
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl);
|
||
const navigationUrl = resolveNavigationUrl(pageUrl, antiBot);
|
||
|
||
const authSessionRef = { current: browserSession };
|
||
page = await createTaskPage(
|
||
browserSession.browser,
|
||
browserSession.reused ? null : browserSession.page,
|
||
navigationUrl,
|
||
antiBot,
|
||
mergedCookies,
|
||
{ browserSession, browserSessionRef: authSessionRef }
|
||
);
|
||
browserSession = authSessionRef.current;
|
||
|
||
const httpResolvedUrl = await resolveHttpRedirects(pageUrl).catch(() => pageUrl);
|
||
if (httpResolvedUrl !== pageUrl) {
|
||
console.log(`[URL解析] HTTP 重定向: ${pageUrl} -> ${httpResolvedUrl}`);
|
||
}
|
||
|
||
let shareToken = resolveShareToken(pageUrl, httpResolvedUrl);
|
||
await seedShareTokenCookie(page, navigationUrl);
|
||
if (httpResolvedUrl !== pageUrl) {
|
||
await seedShareTokenCookie(page, httpResolvedUrl);
|
||
}
|
||
|
||
await prepareWhatshubShortLinkBeforeNav(page, navigationUrl, antiBot, storedSession);
|
||
|
||
let interceptor = null;
|
||
const useEarlyIntercept = shouldUseEarlyApiIntercept(antiBot, apiUrls);
|
||
if (useEarlyIntercept) {
|
||
interceptor = createApiInterceptor(page, apiUrls, interceptTimeout);
|
||
interceptorCleanup = interceptor.cleanup;
|
||
console.log('[Whatshub] statistics 早挂 API 拦截器（session 复用防漏包）');
|
||
}
|
||
|
||
await navigateToPage(page, navigationUrl);
|
||
|
||
let finalPageUrl = await waitForUrlSettled(page);
|
||
applyHuojianBusinessHostHint(antiBot, finalPageUrl);
|
||
if (finalPageUrl !== pageUrl) {
|
||
console.log(`[URL解析] 浏览器落地: ${pageUrl} -> ${finalPageUrl}`);
|
||
}
|
||
|
||
const browserCtx = {
|
||
browserSession,
|
||
page,
|
||
antiBot,
|
||
profile,
|
||
navigationUrl,
|
||
mergedCookies,
|
||
baseExtraArgs: ['--mute-audio'],
|
||
pageExtraOpts: {},
|
||
afterRelaunchNav: async (p) => {
|
||
await seedShareTokenCookie(p, navigationUrl);
|
||
},
|
||
};
|
||
await ensureCaptchaProxyBrowserIfNeeded(browserCtx);
|
||
browserSession = browserCtx.browserSession;
|
||
page = browserCtx.page;
|
||
|
||
if (antiBot.enabled) {
|
||
const cfResult = await runCloudflareWithBusinessGuard(
|
||
page, antiBot, waitForUrlSettled, storedSession, pageUrl, browserCtx
|
||
);
|
||
page = browserCtx.page;
|
||
browserSession = browserCtx.browserSession;
|
||
cfLog.cfDetected = cfResult.cfDetected;
|
||
cfLog.turnstileStage = cfResult.stage;
|
||
cfLog.solverUsed = cfResult.solverUsed;
|
||
cfLog.elapsedMs = cfResult.elapsedMs;
|
||
|
||
if (!cfResult.success) {
|
||
cfDestroyBrowser = true;
|
||
res.status(422).json({
|
||
success: false,
|
||
code: cfResult.code || 'CF_TURNSTILE_FAILED',
|
||
challengeType: cfResult.challengeType || 'turnstile',
|
||
stage: cfResult.stage || 'timeout',
|
||
error: 'Cloudflare Turnstile 验证失败',
|
||
cf: cfLog,
|
||
});
|
||
return;
|
||
}
|
||
|
||
finalPageUrl = page.url();
|
||
applyHuojianBusinessHostHint(antiBot, finalPageUrl);
|
||
touchSessionIfPresent(antiBot.sessionKey);
|
||
|
||
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled, storedSession, pageUrl, apiUrls);
|
||
}
|
||
|
||
const finalShareToken = resolveShareToken(finalPageUrl);
|
||
if (finalShareToken && finalShareToken !== shareToken) {
|
||
shareToken = finalShareToken;
|
||
await seedShareTokenCookie(page, finalPageUrl);
|
||
}
|
||
|
||
if (!interceptor) {
|
||
interceptor = createApiInterceptor(page, apiUrls, interceptTimeout);
|
||
interceptorCleanup = interceptor.cleanup;
|
||
}
|
||
await attachHttpIpRewriteInterceptor(page, 'auth-and-intercept', { shareToken });
|
||
|
||
await executeAuthActionsWithDiagnostics(page, authActions, antiBot);
|
||
await interceptor.waitForApis(interceptTimeout);
|
||
|
||
const rawCookies = await page.cookies();
|
||
const cookiePayload = rawCookies.map((c) => ({
|
||
name: c.name,
|
||
value: c.value,
|
||
domain: c.domain,
|
||
path: c.path,
|
||
}));
|
||
|
||
if (antiBot.sessionKey) {
|
||
let lastHost = '';
|
||
try {
|
||
lastHost = new URL(finalPageUrl).hostname;
|
||
} catch (_) { /* ignore */ }
|
||
saveSession(antiBot.sessionKey, cookiePayload, SESSION_TTL_MS, {
|
||
lastHost,
|
||
savedProfile: profile,
|
||
});
|
||
}
|
||
|
||
res.json({
|
||
success: true,
|
||
pageUrl,
|
||
finalPageUrl,
|
||
interceptedApis: interceptor.interceptedApis,
|
||
cookies: cookiePayload,
|
||
cf: cfLog,
|
||
profile,
|
||
browserReused: browserSession.reused,
|
||
elapsedMs: Date.now() - taskStarted,
|
||
});
|
||
} finally {
|
||
if (interceptorCleanup) interceptorCleanup();
|
||
await closeTaskPage(page);
|
||
if (browserSession) {
|
||
await browserSession.release(cfDestroyBrowser);
|
||
}
|
||
}
|
||
}, profile);
|
||
} catch (error) {
|
||
if (!res.headersSent) sendTaskError(res, error, { cf: cfLog, profile });
|
||
}
|
||
});
|
||
|
||
/** 接口二：脱离 UI 极速并发拉取 */
|
||
app.post('/api/batch-fetch', async (req, res) => {
|
||
const { tasks, cookies, antiBot: rawAntiBot } = req.body;
|
||
if (!Array.isArray(tasks) || tasks.length === 0) {
|
||
return res.status(400).json({ success: false, error: 'tasks 参数错误' });
|
||
}
|
||
|
||
const antiBot = normalizeAntiBot(rawAntiBot);
|
||
const profile = antiBot.profile || 'standard';
|
||
|
||
try {
|
||
await runBrowserTask('batch-fetch', async () => {
|
||
let browserSession = null;
|
||
try {
|
||
browserSession = await acquireBrowserSession({
|
||
profile,
|
||
extraArgs: ['--disable-web-security'],
|
||
sessionKey: antiBot.sessionKey || '',
|
||
});
|
||
|
||
const firstUrl = tasks[0].fullUrl;
|
||
const { mergedCookies } = resolveSessionContext(antiBot, firstUrl, cookies || []);
|
||
|
||
const page = await createManagedPage(browserSession.browser, browserSession.page, { cookies: mergedCookies });
|
||
const firstOrigin = new URL(firstUrl).origin;
|
||
|
||
await page.setRequestInterception(true);
|
||
page.on('request', (reqObj) => {
|
||
if (reqObj.isInterceptResolutionHandled()) return;
|
||
const url = rewriteHttpsToHttpForLiteralIp(reqObj.url());
|
||
const resourceType = reqObj.resourceType();
|
||
if (['image', 'media', 'font'].includes(resourceType)) {
|
||
reqObj.abort();
|
||
return;
|
||
}
|
||
if (url === firstOrigin || url === firstOrigin + '/') {
|
||
reqObj.respond({ status: 200, contentType: 'text/html', body: '<html><body>Context Ready</body></html>' });
|
||
} else {
|
||
reqObj.continue();
|
||
}
|
||
});
|
||
|
||
await navigateToPage(page, firstOrigin);
|
||
|
||
const FETCH_CONCURRENCY = Math.max(1, parseInt(process.env.FETCH_CONCURRENCY || '5', 10));
|
||
const FETCH_RETRY = 2;
|
||
|
||
const fetchResults = await page.evaluate(async (taskList, concurrency, maxRetry) => {
|
||
const out = {};
|
||
const createLimiter = (limit) => {
|
||
let active = 0;
|
||
const queue = [];
|
||
return (fn) => new Promise((resolve, reject) => {
|
||
const run = async () => {
|
||
active++;
|
||
try { resolve(await fn()); }
|
||
catch (e) { reject(e); }
|
||
finally {
|
||
active--;
|
||
if (queue.length) queue.shift()();
|
||
}
|
||
};
|
||
if (active < limit) run();
|
||
else queue.push(run);
|
||
});
|
||
};
|
||
const limit = createLimiter(concurrency);
|
||
|
||
const fetchWithRetry = async (finalUrl, options, retriesLeft) => {
|
||
try {
|
||
const res = await fetch(finalUrl, options);
|
||
return { success: true, data: await res.json() };
|
||
} catch (err) {
|
||
if (retriesLeft <= 0) return { success: false, error: err.toString() };
|
||
await new Promise((r) => setTimeout(r, 300 * Math.pow(2, maxRetry - retriesLeft)));
|
||
return fetchWithRetry(finalUrl, options, retriesLeft - 1);
|
||
}
|
||
};
|
||
|
||
const jobs = [];
|
||
for (const task of taskList) {
|
||
out[task.apiPath] = [];
|
||
const cleanHeaders = { ...task.headers };
|
||
['content-length', 'host', 'origin', 'referer', 'accept-encoding'].forEach((k) => delete cleanHeaders[k]);
|
||
|
||
for (const params of task.paramList) {
|
||
jobs.push(limit(async () => {
|
||
let finalUrl = task.fullUrl;
|
||
const options = { method: task.method.toUpperCase(), headers: cleanHeaders };
|
||
if (options.method === 'GET') {
|
||
const qs = new URLSearchParams(params).toString();
|
||
finalUrl = finalUrl.includes('?') ? `${finalUrl}&${qs}` : `${finalUrl}?${qs}`;
|
||
} else {
|
||
options.body = JSON.stringify(params);
|
||
}
|
||
const result = await fetchWithRetry(finalUrl, options, maxRetry);
|
||
out[task.apiPath].push({ params, ...result });
|
||
await new Promise((r) => setTimeout(r, 200));
|
||
}));
|
||
}
|
||
}
|
||
await Promise.all(jobs);
|
||
return out;
|
||
}, tasks, FETCH_CONCURRENCY, FETCH_RETRY);
|
||
|
||
res.json({ success: true, results: fetchResults, profile, browserReused: browserSession.reused });
|
||
} finally {
|
||
if (browserSession) {
|
||
await browserSession.release(false);
|
||
}
|
||
}
|
||
}, profile);
|
||
} catch (error) {
|
||
if (!res.headersSent) sendTaskError(res, error, { profile });
|
||
}
|
||
});
|
||
|
||
/** 接口三：强制 UI 模拟点击拉取 */
|
||
app.post('/api/ui-pagination', async (req, res) => {
|
||
const {
|
||
pageUrl,
|
||
finalPageUrl,
|
||
apiUrl,
|
||
cookies,
|
||
nextBtnSelector,
|
||
clicksToPerform,
|
||
waitMs = 2000,
|
||
firstPageData,
|
||
authActions,
|
||
antiBot: rawAntiBot,
|
||
} = req.body;
|
||
const navigationUrl = finalPageUrl || pageUrl;
|
||
const antiBot = resolveAntiBotForPage(rawAntiBot, pageUrl || navigationUrl);
|
||
const profile = antiBot.profile || 'standard';
|
||
|
||
try {
|
||
await runBrowserTask('ui-pagination', async () => {
|
||
let browserSession = null;
|
||
const debugLogs = [];
|
||
let page = null;
|
||
let cfDestroyBrowser = false;
|
||
|
||
try {
|
||
browserSession = await acquireBrowserSession({
|
||
profile,
|
||
extraArgs: ['--window-size=1920,1080'],
|
||
sessionKey: antiBot.sessionKey || '',
|
||
});
|
||
tagBrowserSession(browserSession, profile, antiBot.sessionKey || '', ['--window-size=1920,1080']);
|
||
|
||
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, navigationUrl, cookies || []);
|
||
|
||
const uiSessionRef = { current: browserSession };
|
||
page = await createTaskPage(
|
||
browserSession.browser,
|
||
browserSession.reused ? null : browserSession.page,
|
||
navigationUrl,
|
||
antiBot,
|
||
mergedCookies,
|
||
{
|
||
viewport: { width: 1920, height: 1080 },
|
||
browserSession,
|
||
browserSessionRef: uiSessionRef,
|
||
}
|
||
);
|
||
browserSession = uiSessionRef.current;
|
||
|
||
const log = (msg) => { console.log(msg); debugLogs.push(msg); };
|
||
log(`[启动] 准备访问页面: ${navigationUrl}`);
|
||
if (finalPageUrl && finalPageUrl !== pageUrl) {
|
||
log(`[URL解析] 使用 auth 阶段落地地址，跳过短链重定向: ${pageUrl} -> ${finalPageUrl}`);
|
||
}
|
||
|
||
const shareToken = resolveShareToken(finalPageUrl, pageUrl);
|
||
await seedShareTokenCookie(page, navigationUrl);
|
||
|
||
await prepareWhatshubShortLinkBeforeNav(page, navigationUrl, antiBot, storedSession);
|
||
await navigateToPage(page, navigationUrl);
|
||
|
||
const settledPageUrl = await waitForUrlSettled(page);
|
||
if (settledPageUrl !== navigationUrl) {
|
||
log(`[URL解析] 翻页阶段落地: ${navigationUrl} -> ${settledPageUrl}`);
|
||
}
|
||
|
||
const browserCtx = {
|
||
browserSession,
|
||
page,
|
||
antiBot,
|
||
profile,
|
||
navigationUrl,
|
||
mergedCookies,
|
||
baseExtraArgs: ['--window-size=1920,1080'],
|
||
pageExtraOpts: { viewport: { width: 1920, height: 1080 } },
|
||
afterRelaunchNav: async (p) => {
|
||
await seedShareTokenCookie(p, navigationUrl);
|
||
},
|
||
};
|
||
await ensureCaptchaProxyBrowserIfNeeded(browserCtx);
|
||
browserSession = browserCtx.browserSession;
|
||
page = browserCtx.page;
|
||
|
||
if (antiBot.enabled) {
|
||
const cfResult = await runCloudflareWithBusinessGuard(
|
||
page, antiBot, waitForUrlSettled, storedSession, navigationUrl, browserCtx
|
||
);
|
||
page = browserCtx.page;
|
||
browserSession = browserCtx.browserSession;
|
||
if (!cfResult.success) {
|
||
cfDestroyBrowser = true;
|
||
res.status(422).json({
|
||
success: false,
|
||
code: cfResult.code || 'CF_TURNSTILE_FAILED',
|
||
challengeType: cfResult.challengeType,
|
||
stage: cfResult.stage,
|
||
error: 'Cloudflare Turnstile 验证失败（翻页阶段）',
|
||
debug: debugLogs,
|
||
});
|
||
return;
|
||
}
|
||
touchSessionIfPresent(antiBot.sessionKey);
|
||
|
||
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled, storedSession, navigationUrl, apiUrl ? [apiUrl] : null);
|
||
await attachHttpIpRewriteInterceptor(page, 'ui-pagination', { shareToken });
|
||
} else {
|
||
await attachHttpIpRewriteInterceptor(page, 'ui-pagination', { shareToken });
|
||
}
|
||
|
||
const paginationResult = await runUiPagination(page, {
|
||
apiUrl,
|
||
nextBtnSelector,
|
||
clicksToPerform,
|
||
waitMs,
|
||
firstPageData,
|
||
authActions,
|
||
executeAuthActions: (p, actions) => executeAuthActions(p, actions, antiBot),
|
||
});
|
||
|
||
res.json({
|
||
success: true,
|
||
data: paginationResult.data,
|
||
debug: [...debugLogs, ...paginationResult.debugLogs],
|
||
profile,
|
||
browserReused: browserSession.reused,
|
||
});
|
||
} catch (error) {
|
||
if (!res.headersSent) {
|
||
sendTaskError(res, error, { debug: debugLogs, profile });
|
||
}
|
||
} finally {
|
||
await closeTaskPage(page);
|
||
if (browserSession) {
|
||
await browserSession.release(cfDestroyBrowser);
|
||
}
|
||
}
|
||
}, profile);
|
||
} catch (error) {
|
||
if (!res.headersSent) sendTaskError(res, error, { profile });
|
||
}
|
||
});
|
||
|
||
/** 接口四：单 Browser 内 auth + 拦截 + UI 翻页（降低 CF 二次触发） */
|
||
app.post('/api/auth-intercept-and-paginate', async (req, res) => {
|
||
const {
|
||
pageUrl,
|
||
apiUrls,
|
||
authActions,
|
||
antiBot: rawAntiBot,
|
||
pagination,
|
||
} = req.body;
|
||
|
||
if (!pageUrl || !Array.isArray(apiUrls) || !pagination) {
|
||
return res.status(400).json({ success: false, error: '参数错误：需要 pageUrl、apiUrls、pagination' });
|
||
}
|
||
|
||
const {
|
||
apiUrl,
|
||
nextBtnSelector,
|
||
clicksToPerform = 0,
|
||
waitMs = 2000,
|
||
firstPageData,
|
||
} = pagination;
|
||
|
||
const antiBot = resolveAntiBotForPage(rawAntiBot, pageUrl);
|
||
const profile = antiBot.profile || 'standard';
|
||
const interceptTimeout = resolveInterceptTimeout(antiBot);
|
||
const cfLog = { profile, cfDetected: false, turnstileStage: null, solverUsed: false, elapsedMs: 0, browserReused: false };
|
||
|
||
try {
|
||
await runBrowserTask('auth-intercept-and-paginate', async () => {
|
||
let browserSession = null;
|
||
let interceptorCleanup = null;
|
||
let page = null;
|
||
const taskStarted = Date.now();
|
||
let cfDestroyBrowser = false;
|
||
|
||
try {
|
||
browserSession = await acquireBrowserSession({
|
||
profile,
|
||
extraArgs: ['--mute-audio', '--window-size=1920,1080'],
|
||
sessionKey: antiBot.sessionKey || '',
|
||
});
|
||
tagBrowserSession(
|
||
browserSession,
|
||
profile,
|
||
antiBot.sessionKey || '',
|
||
['--mute-audio', '--window-size=1920,1080']
|
||
);
|
||
cfLog.browserReused = !!browserSession.reused;
|
||
|
||
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl);
|
||
const navigationUrl = resolveNavigationUrl(pageUrl, antiBot);
|
||
|
||
const mergedSessionRef = { current: browserSession };
|
||
page = await createTaskPage(
|
||
browserSession.browser,
|
||
browserSession.reused ? null : browserSession.page,
|
||
navigationUrl,
|
||
antiBot,
|
||
mergedCookies,
|
||
{
|
||
viewport: { width: 1920, height: 1080 },
|
||
browserSession,
|
||
browserSessionRef: mergedSessionRef,
|
||
}
|
||
);
|
||
browserSession = mergedSessionRef.current;
|
||
|
||
const httpResolvedUrl = await resolveHttpRedirects(pageUrl).catch(() => pageUrl);
|
||
|
||
let shareToken = resolveShareToken(pageUrl, httpResolvedUrl);
|
||
await seedShareTokenCookie(page, navigationUrl);
|
||
|
||
await prepareWhatshubShortLinkBeforeNav(page, navigationUrl, antiBot, storedSession);
|
||
await navigateToPage(page, navigationUrl);
|
||
let finalPageUrl = await waitForUrlSettled(page);
|
||
applyHuojianBusinessHostHint(antiBot, finalPageUrl);
|
||
|
||
const browserCtx = {
|
||
browserSession,
|
||
page,
|
||
antiBot,
|
||
profile,
|
||
navigationUrl,
|
||
mergedCookies,
|
||
baseExtraArgs: ['--mute-audio', '--window-size=1920,1080'],
|
||
pageExtraOpts: { viewport: { width: 1920, height: 1080 } },
|
||
afterRelaunchNav: async (p) => {
|
||
await seedShareTokenCookie(p, navigationUrl);
|
||
},
|
||
};
|
||
await ensureCaptchaProxyBrowserIfNeeded(browserCtx);
|
||
browserSession = browserCtx.browserSession;
|
||
page = browserCtx.page;
|
||
|
||
let interceptor;
|
||
if (antiBot.enabled) {
|
||
const cfResult = await runCloudflareWithBusinessGuard(
|
||
page, antiBot, waitForUrlSettled, storedSession, navigationUrl, browserCtx
|
||
);
|
||
page = browserCtx.page;
|
||
browserSession = browserCtx.browserSession;
|
||
cfLog.cfDetected = cfResult.cfDetected;
|
||
cfLog.turnstileStage = cfResult.stage;
|
||
cfLog.solverUsed = cfResult.solverUsed;
|
||
cfLog.elapsedMs = cfResult.elapsedMs;
|
||
|
||
if (!cfResult.success) {
|
||
cfDestroyBrowser = true;
|
||
res.status(422).json({
|
||
success: false,
|
||
code: cfResult.code || 'CF_TURNSTILE_FAILED',
|
||
challengeType: cfResult.challengeType || 'turnstile',
|
||
stage: cfResult.stage || 'timeout',
|
||
error: 'Cloudflare Turnstile 验证失败',
|
||
cf: cfLog,
|
||
});
|
||
return;
|
||
}
|
||
finalPageUrl = page.url();
|
||
applyHuojianBusinessHostHint(antiBot, finalPageUrl);
|
||
touchSessionIfPresent(antiBot.sessionKey);
|
||
|
||
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled, storedSession, navigationUrl, apiUrls);
|
||
}
|
||
|
||
interceptor = createApiInterceptor(page, apiUrls, interceptTimeout);
|
||
interceptorCleanup = interceptor.cleanup;
|
||
await attachHttpIpRewriteInterceptor(page, 'auth-intercept-and-paginate', { shareToken });
|
||
|
||
await executeAuthActionsWithDiagnostics(page, authActions, antiBot);
|
||
await interceptor.waitForApis(interceptTimeout);
|
||
|
||
const rawCookies = await page.cookies();
|
||
const cookiePayload = rawCookies.map((c) => ({
|
||
name: c.name,
|
||
value: c.value,
|
||
domain: c.domain,
|
||
path: c.path,
|
||
}));
|
||
|
||
if (antiBot.sessionKey) {
|
||
let lastHost = '';
|
||
try {
|
||
lastHost = new URL(finalPageUrl).hostname;
|
||
} catch (_) { /* ignore */ }
|
||
saveSession(antiBot.sessionKey, cookiePayload, SESSION_TTL_MS, {
|
||
lastHost,
|
||
savedProfile: profile,
|
||
});
|
||
}
|
||
|
||
let extraPages = [];
|
||
let paginationDebug = [];
|
||
if (clicksToPerform > 0 && apiUrl && nextBtnSelector) {
|
||
let effectiveFirstPageData = firstPageData;
|
||
if (!effectiveFirstPageData) {
|
||
const matchedKey = Object.keys(interceptor.interceptedApis).find((k) => k === apiUrl || k.includes(apiUrl));
|
||
if (matchedKey) {
|
||
effectiveFirstPageData = interceptor.interceptedApis[matchedKey].data;
|
||
}
|
||
}
|
||
const paginationResult = await runUiPagination(page, {
|
||
apiUrl,
|
||
nextBtnSelector,
|
||
clicksToPerform,
|
||
waitMs,
|
||
firstPageData: effectiveFirstPageData,
|
||
authActions,
|
||
executeAuthActions: (p, actions) => executeAuthActions(p, actions, antiBot),
|
||
});
|
||
extraPages = paginationResult.data;
|
||
paginationDebug = paginationResult.debugLogs;
|
||
}
|
||
|
||
res.json({
|
||
success: true,
|
||
pageUrl,
|
||
finalPageUrl,
|
||
interceptedApis: interceptor.interceptedApis,
|
||
cookies: cookiePayload,
|
||
extraPages,
|
||
debug: paginationDebug,
|
||
cf: cfLog,
|
||
profile,
|
||
browserReused: browserSession.reused,
|
||
elapsedMs: Date.now() - taskStarted,
|
||
});
|
||
} finally {
|
||
if (interceptorCleanup) interceptorCleanup();
|
||
await closeTaskPage(page);
|
||
if (browserSession) {
|
||
await browserSession.release(cfDestroyBrowser);
|
||
}
|
||
}
|
||
}, profile);
|
||
} catch (error) {
|
||
if (!res.headersSent) sendTaskError(res, error, { cf: cfLog, profile });
|
||
}
|
||
});
|
||
|
||
const PORT = parseInt(process.env.PORT || '3001', 10);
|
||
const server = app.listen(PORT, '127.0.0.1', () => {
|
||
const chromePath = resolveChromeExecutable();
|
||
const chromeReady = fs.existsSync(chromePath);
|
||
console.log(`🚀 Node.js 自愈型防脱轨引擎已启动 (standard: ${MAX_CONCURRENT_BROWSERS}, real: ${MAX_CONCURRENT_BROWSERS_REAL})`);
|
||
console.log(`[chrome] path=${chromePath} exists=${chromeReady}`);
|
||
console.log(`[real-browser] ready=${isRealBrowserAvailable()} captcha=${isCaptchaConfigured()}`);
|
||
if (!chromeReady) {
|
||
console.warn('[chrome] 未找到 Chrome，请在 puppeteer-api 目录执行: npx puppeteer browsers install chrome');
|
||
}
|
||
});
|
||
|
||
async function gracefulShutdown(signal) {
|
||
console.log(`[shutdown] 收到 ${signal}，停止接收新任务...`);
|
||
shuttingDown = true;
|
||
server.close();
|
||
|
||
const deadline = Date.now() + 60000;
|
||
while (standardLimiter.active > 0 && Date.now() < deadline) {
|
||
const { active, queued } = standardLimiter.getStats();
|
||
console.log(`[shutdown] 等待在途任务完成... active=${active}, queued=${queued}`);
|
||
await new Promise((r) => setTimeout(r, 1000));
|
||
}
|
||
|
||
console.log('[shutdown] 排空 Browser 温池...');
|
||
await browserPool.drain();
|
||
|
||
const remaining = standardLimiter.getStats();
|
||
if (remaining.active > 0) {
|
||
console.warn(`[shutdown] 超时，仍有 ${remaining.active} 个 Browser 在运行，强制退出`);
|
||
process.exit(1);
|
||
}
|
||
console.log('[shutdown] 所有任务已完成，进程退出');
|
||
process.exit(0);
|
||
}
|
||
|
||
process.on('SIGTERM', () => gracefulShutdown('SIGTERM'));
|
||
process.on('SIGINT', () => gracefulShutdown('SIGINT'));
|