A2C工单修复
This commit is contained in:
root
@@ -47,16 +47,18 @@ class AntiBotConfigBuilder
|
||||
'sessionTtlMinutes' => $ttlMinutes,
|
||||
'businessHostHint' => $businessHostHint,
|
||||
'landingUrlHint' => $landingUrlHint !== '' ? $landingUrlHint : '',
|
||||
'cfCloudflareSolver' => self::defaultCfCloudflareSolver($ticketType),
|
||||
'cfChallengeMode' => self::defaultCfChallengeMode($ticketType),
|
||||
] + self::postCfReadyExtras($ticketType);
|
||||
}
|
||||
|
||||
/**
|
||||
* 会话 scope：A2C 短链固定业务域；火箭长链用动态 host、短链用 share.{token}
|
||||
* 会话 scope：A2C 业务域+分享 id；火箭长链用动态 host、短链用 share.{token}
|
||||
*/
|
||||
public static function resolveSessionScope(string $ticketType, string $pageUrl, string $landingUrlHint = ''): string
|
||||
{
|
||||
if ($ticketType === 'a2c') {
|
||||
return self::resolveA2cSessionHost($pageUrl, $landingUrlHint);
|
||||
return self::resolveA2cSessionScope($pageUrl, $landingUrlHint);
|
||||
}
|
||||
|
||||
if ($ticketType === 'huojian') {
|
||||
@@ -135,10 +137,11 @@ class AntiBotConfigBuilder
|
||||
if ($ticketType === 'a2c') {
|
||||
return [
|
||||
'postCfReadyUrlContains' => '/visitors/counter/share',
|
||||
'postCfReadySelector' => '.btn-next',
|
||||
'postCfReadyTimeoutMs' => 30000,
|
||||
'postCfReadySelector' => '.el-table',
|
||||
'postCfReadyApiUrl' => '/api/talk/counter/share/record/list',
|
||||
'postCfReadyTimeoutMs' => 60000,
|
||||
'postCfSettleMs' => 500,
|
||||
'postCfSpaWaitMs' => 4000,
|
||||
'postCfSpaWaitMs' => 8000,
|
||||
];
|
||||
}
|
||||
|
||||
@@ -162,10 +165,6 @@ class AntiBotConfigBuilder
|
||||
|
||||
private static function defaultSolverFallback(string $ticketType): bool
|
||||
{
|
||||
if ($ticketType === 'a2c') {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -178,6 +177,26 @@ class AntiBotConfigBuilder
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* A2C 等为 CF Managed Challenge（5s 盾），非独立 Turnstile 挂件
|
||||
*/
|
||||
private static function defaultCfChallengeMode(string $ticketType): string
|
||||
{
|
||||
if ($ticketType === 'a2c') {
|
||||
return 'managed';
|
||||
}
|
||||
|
||||
return '';
|
||||
}
|
||||
|
||||
/**
|
||||
* sitekey 缺失时是否允许 CapSolver AntiCloudflareTask（需 Node 侧 CAPTCHA_PROXY）
|
||||
*/
|
||||
private static function defaultCfCloudflareSolver(string $ticketType): bool
|
||||
{
|
||||
return $ticketType === 'a2c';
|
||||
}
|
||||
|
||||
/**
|
||||
* 火箭/A2C 业务域 hint：优先长链动态 host，其次 landing 预解析
|
||||
*/
|
||||
@@ -223,6 +242,35 @@ class AntiBotConfigBuilder
|
||||
return 'user.a2c.chat';
|
||||
}
|
||||
|
||||
/**
|
||||
* A2C 会话 scope：业务域 + 分享 id（避免多工单共用温池 Browser 互相干扰）
|
||||
*/
|
||||
private static function resolveA2cSessionScope(string $pageUrl, string $landingUrlHint = ''): string
|
||||
{
|
||||
$host = self::resolveA2cSessionHost($pageUrl, $landingUrlHint);
|
||||
$shareId = self::extractA2cShareId($pageUrl);
|
||||
if ($shareId === '' && $landingUrlHint !== '') {
|
||||
$shareId = self::extractA2cShareId($landingUrlHint);
|
||||
}
|
||||
|
||||
return $shareId !== '' ? $host . ':' . $shareId : $host;
|
||||
}
|
||||
|
||||
/**
|
||||
* 从 A2C 分享 URL 提取 id 查询参数
|
||||
*/
|
||||
private static function extractA2cShareId(string $url): string
|
||||
{
|
||||
$query = (string) parse_url($url, PHP_URL_QUERY);
|
||||
if ($query === '') {
|
||||
return '';
|
||||
}
|
||||
parse_str($query, $params);
|
||||
$id = isset($params['id']) ? trim((string) $params['id']) : '';
|
||||
|
||||
return $id !== '' ? $id : '';
|
||||
}
|
||||
|
||||
private static function resolveHuojianSessionScope(string $pageUrl, string $landingUrlHint = ''): string
|
||||
{
|
||||
$scope = HuojianUrlHelper::resolveSessionScope($pageUrl);
|
||||
|
||||
@@ -7,11 +7,12 @@ namespace app\common\library\scrm\spider;
|
||||
use app\common\library\scrm\AbstractScrmSpider;
|
||||
use app\common\library\scrm\AntiBotConfigBuilder;
|
||||
use app\common\library\scrm\UnifiedScrmData;
|
||||
use app\common\service\SplitPageUrlResolver;
|
||||
|
||||
/**
|
||||
* A2C 云控蜘蛛（Real Browser 打开分享页 + 拦截加密 API + UI 翻页）
|
||||
*
|
||||
* 业务域 user.a2c.chat 无需 cf_clearance；短链 yyk.ink 仍由浏览器 follow 跳转。
|
||||
* 业务域 user.a2c.chat 在 CF 挑战页启用 CapSolver 兜底；短链 yyk.ink 预解析为长链后直达业务页。
|
||||
*/
|
||||
class A2cSpider extends AbstractScrmSpider
|
||||
{
|
||||
@@ -27,6 +28,9 @@ class A2cSpider extends AbstractScrmSpider
|
||||
|
||||
private string $password;
|
||||
|
||||
/** @var string HTTP 预解析落地 URL（供 antiBot landingUrlHint） */
|
||||
private string $landingUrlHint = '';
|
||||
|
||||
private UnifiedScrmData $unifiedData;
|
||||
|
||||
public function __construct(
|
||||
@@ -39,6 +43,7 @@ class A2cSpider extends AbstractScrmSpider
|
||||
$this->pageUrl = $pageUrl;
|
||||
$this->account = $account;
|
||||
$this->password = $password;
|
||||
$this->landingUrlHint = SplitPageUrlResolver::resolveFinalUrl($pageUrl);
|
||||
$this->unifiedData = new UnifiedScrmData();
|
||||
}
|
||||
|
||||
@@ -53,7 +58,7 @@ class A2cSpider extends AbstractScrmSpider
|
||||
'authActions' => [
|
||||
['type' => 'wait', 'ms' => 2000],
|
||||
],
|
||||
'antiBot' => AntiBotConfigBuilder::build('a2c', $this->pageUrl),
|
||||
'antiBot' => AntiBotConfigBuilder::build('a2c', $this->pageUrl, $this->landingUrlHint),
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
# puppeteer-api 环境配置示例
|
||||
# 复制为 .env 后按实际环境修改；pm2 restart server --update-env
|
||||
|
||||
# ========== CapSolver（仅 A2C 等 Managed 过盾时 Node 调用 AntiCloudflareTask）==========
|
||||
CAPTCHA_PROVIDER=capsolver
|
||||
CAPTCHA_API_KEY=
|
||||
# 静态住宅代理 ip:port:user:pass；CapSolver 过盾必填
|
||||
# Chrome 仅在「检测到 CF 挑战」时才会使用同一代理（见 server.js ensureCaptchaProxyBrowserIfNeeded）
|
||||
CAPTCHA_PROXY=
|
||||
CAPTCHA_MAX_WAIT_MS=120000
|
||||
|
||||
# ========== Standard 槽位（未走 antiBot / profile=standard 的蜘蛛）==========
|
||||
MAX_CONCURRENT_BROWSERS=4
|
||||
|
||||
# ========== Real Browser 槽位（whatshub / chatknow / huojian / a2c 共用）==========
|
||||
# 20~30 工单后台批量：建议 2~3（非 A2C 不走代理，可与 A2C 并行）
|
||||
MAX_CONCURRENT_BROWSERS_REAL=2
|
||||
|
||||
# ========== 排队（cron 一轮多工单时避免 503）==========
|
||||
QUEUE_TIMEOUT_MS=300000
|
||||
|
||||
# ========== 温池（后台批量强烈建议开启，按 sessionKey 复用 Browser）==========
|
||||
POOL_ENABLED=1
|
||||
MAX_POOLED_BROWSERS=8
|
||||
POOL_IDLE_MS=300000
|
||||
|
||||
# ========== 超时 ==========
|
||||
NAVIGATION_TIMEOUT_MS=90000
|
||||
API_INTERCEPT_TIMEOUT_MS=60000
|
||||
# Real Browser + A2C 过盾（CapSolver + SPA）建议 180s
|
||||
API_INTERCEPT_TIMEOUT_REAL_MS=180000
|
||||
|
||||
# ========== 会话复用（减少 A2C 重复过盾）==========
|
||||
SESSION_TTL_MS=7200000
|
||||
PERSIST_BROWSER_PROFILE=1
|
||||
|
||||
# ========== 服务端口 ==========
|
||||
PORT=3001
|
||||
@@ -59,11 +59,12 @@ async function createBrowserSession(options = {}) {
|
||||
page,
|
||||
profile: 'real',
|
||||
cleanup: async (destroy = false) => {
|
||||
if (destroy || !sessionKey) {
|
||||
try {
|
||||
await browser.close();
|
||||
} catch (_) { /* ignore */ }
|
||||
if (!destroy) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await browser.close();
|
||||
} catch (_) { /* ignore */ }
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -74,11 +75,12 @@ async function createBrowserSession(options = {}) {
|
||||
page: null,
|
||||
profile: 'standard',
|
||||
cleanup: async (destroy = false) => {
|
||||
if (destroy || !sessionKey) {
|
||||
try {
|
||||
await browser.close();
|
||||
} catch (_) { /* ignore */ }
|
||||
if (!destroy) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await browser.close();
|
||||
} catch (_) { /* ignore */ }
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
/**
|
||||
* Browser 温池：按 profile:sessionKey 复用 Browser/Page，降低冷启动与 CF 触发
|
||||
* Browser 温池：按 profile:sessionKey 复用 Browser，降低冷启动与 CF 触发
|
||||
*
|
||||
* 同一 key 的 acquire 在 inUse 时排队等待，禁止销毁正在使用的 Browser（修复 A2C 并发误杀）。
|
||||
*/
|
||||
const { POOL_IDLE_MS, MAX_POOLED_BROWSERS, POOL_ENABLED } = require('./constants');
|
||||
const { POOL_IDLE_MS, MAX_POOLED_BROWSERS, POOL_ENABLED, QUEUE_TIMEOUT_MS } = require('./constants');
|
||||
|
||||
class BrowserPool {
|
||||
constructor() {
|
||||
/** @type {Map<string, { browser: any, page: any|null, profile: string, sessionKey: string, lastUsedAt: number, inUse: boolean }>} */
|
||||
this.entries = new Map();
|
||||
/** @type {Map<string, Array<{ resolve: () => void, reject: (err: Error) => void, timer: NodeJS.Timeout|null }>>} */
|
||||
this._waitQueues = new Map();
|
||||
this.hits = 0;
|
||||
this.misses = 0;
|
||||
this.evictions = 0;
|
||||
@@ -14,17 +18,10 @@ class BrowserPool {
|
||||
this._sweepTimer.unref?.();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} sessionKey
|
||||
* @param {'standard'|'real'} profile
|
||||
*/
|
||||
buildKey(sessionKey, profile) {
|
||||
return `${profile}:${sessionKey}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {any} browser
|
||||
*/
|
||||
async isBrowserAlive(browser) {
|
||||
if (!browser) return false;
|
||||
try {
|
||||
@@ -38,12 +35,13 @@ class BrowserPool {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} key
|
||||
*/
|
||||
async destroyEntry(key) {
|
||||
const entry = this.entries.get(key);
|
||||
if (!entry) return;
|
||||
if (entry.inUse) {
|
||||
console.warn(`[BrowserPool] 跳过销毁 inUse 条目 key=${key}`);
|
||||
return;
|
||||
}
|
||||
this.entries.delete(key);
|
||||
try {
|
||||
await entry.browser.close();
|
||||
@@ -63,11 +61,43 @@ class BrowserPool {
|
||||
this.destroyEntry(oldestKey);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} sessionKey
|
||||
* @param {'standard'|'real'} profile
|
||||
* @param {() => Promise<{ browser: any, page: any|null, cleanup: (destroy?: boolean) => Promise<void> }>} launcher
|
||||
*/
|
||||
waitForRelease(key, timeoutMs = QUEUE_TIMEOUT_MS) {
|
||||
const entry = this.entries.get(key);
|
||||
if (!entry || !entry.inUse) {
|
||||
return Promise.resolve();
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
const waiter = {
|
||||
resolve: () => {
|
||||
if (waiter.timer) clearTimeout(waiter.timer);
|
||||
resolve();
|
||||
},
|
||||
reject,
|
||||
timer: timeoutMs > 0
|
||||
? setTimeout(() => {
|
||||
const queue = this._waitQueues.get(key) || [];
|
||||
const idx = queue.indexOf(waiter);
|
||||
if (idx >= 0) queue.splice(idx, 1);
|
||||
reject(new Error('POOL_ACQUIRE_TIMEOUT'));
|
||||
}, timeoutMs)
|
||||
: null,
|
||||
};
|
||||
if (!this._waitQueues.has(key)) {
|
||||
this._waitQueues.set(key, []);
|
||||
}
|
||||
this._waitQueues.get(key).push(waiter);
|
||||
console.log(`[BrowserPool] key=${key} 正在使用，排队等待 (队列 ${this._waitQueues.get(key).length})`);
|
||||
});
|
||||
}
|
||||
|
||||
_wakeNextWaiter(key) {
|
||||
const queue = this._waitQueues.get(key);
|
||||
if (!queue || queue.length === 0) return;
|
||||
const next = queue.shift();
|
||||
if (next?.timer) clearTimeout(next.timer);
|
||||
next?.resolve();
|
||||
}
|
||||
|
||||
async acquire(sessionKey, profile, launcher) {
|
||||
if (!POOL_ENABLED || !sessionKey) {
|
||||
const launched = await launcher();
|
||||
@@ -77,29 +107,38 @@ class BrowserPool {
|
||||
profile,
|
||||
pooled: false,
|
||||
reused: false,
|
||||
release: async (destroy = false) => {
|
||||
await launched.cleanup(destroy);
|
||||
release: async () => {
|
||||
await launched.cleanup(true);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const key = this.buildKey(sessionKey, profile);
|
||||
const existing = this.entries.get(key);
|
||||
if (existing && !existing.inUse && await this.isBrowserAlive(existing.browser)) {
|
||||
existing.inUse = true;
|
||||
existing.lastUsedAt = Date.now();
|
||||
this.hits++;
|
||||
return {
|
||||
browser: existing.browser,
|
||||
page: existing.page,
|
||||
profile,
|
||||
pooled: true,
|
||||
reused: true,
|
||||
release: async (destroy = false) => this.release(key, destroy),
|
||||
};
|
||||
}
|
||||
if (existing) {
|
||||
|
||||
while (true) {
|
||||
const existing = this.entries.get(key);
|
||||
if (!existing) {
|
||||
break;
|
||||
}
|
||||
if (existing.inUse) {
|
||||
await this.waitForRelease(key);
|
||||
continue;
|
||||
}
|
||||
if (await this.isBrowserAlive(existing.browser)) {
|
||||
existing.inUse = true;
|
||||
existing.lastUsedAt = Date.now();
|
||||
this.hits++;
|
||||
return {
|
||||
browser: existing.browser,
|
||||
page: null,
|
||||
profile,
|
||||
pooled: true,
|
||||
reused: true,
|
||||
release: async (destroy = false) => this.release(key, destroy),
|
||||
};
|
||||
}
|
||||
await this.destroyEntry(key);
|
||||
break;
|
||||
}
|
||||
|
||||
this.enforceCapacity(key);
|
||||
@@ -107,7 +146,7 @@ class BrowserPool {
|
||||
const launched = await launcher();
|
||||
this.entries.set(key, {
|
||||
browser: launched.browser,
|
||||
page: launched.page,
|
||||
page: null,
|
||||
profile,
|
||||
sessionKey,
|
||||
lastUsedAt: Date.now(),
|
||||
@@ -124,18 +163,19 @@ class BrowserPool {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} key
|
||||
* @param {boolean} destroy
|
||||
*/
|
||||
async release(key, destroy = false) {
|
||||
const entry = this.entries.get(key);
|
||||
if (!entry) return;
|
||||
if (!entry) {
|
||||
this._wakeNextWaiter(key);
|
||||
return;
|
||||
}
|
||||
entry.inUse = false;
|
||||
entry.lastUsedAt = Date.now();
|
||||
entry.page = null;
|
||||
if (destroy || !(await this.isBrowserAlive(entry.browser))) {
|
||||
await this.destroyEntry(key);
|
||||
}
|
||||
this._wakeNextWaiter(key);
|
||||
}
|
||||
|
||||
evictIdle() {
|
||||
@@ -151,17 +191,24 @@ class BrowserPool {
|
||||
clearInterval(this._sweepTimer);
|
||||
const keys = [...this.entries.keys()];
|
||||
for (const key of keys) {
|
||||
await this.destroyEntry(key);
|
||||
const entry = this.entries.get(key);
|
||||
if (entry && !entry.inUse) {
|
||||
await this.destroyEntry(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
getStats() {
|
||||
let idle = 0;
|
||||
let active = 0;
|
||||
let waiting = 0;
|
||||
for (const entry of this.entries.values()) {
|
||||
if (entry.inUse) active++;
|
||||
else idle++;
|
||||
}
|
||||
for (const queue of this._waitQueues.values()) {
|
||||
waiting += queue.length;
|
||||
}
|
||||
return {
|
||||
enabled: POOL_ENABLED,
|
||||
max: MAX_POOLED_BROWSERS,
|
||||
@@ -169,6 +216,7 @@ class BrowserPool {
|
||||
size: this.entries.size,
|
||||
idle,
|
||||
active,
|
||||
waiting,
|
||||
hits: this.hits,
|
||||
misses: this.misses,
|
||||
evictions: this.evictions,
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
/**
|
||||
* CAPTCHA_PROXY 解析与 Chrome 代理注入（与 CapSolver AntiCloudflareTask 使用同一 IP）
|
||||
*
|
||||
* 仅在实际检测到 CF 挑战、需要 Managed 过盾时由 server.js 显式启用（useCaptchaProxy=true）。
|
||||
* 格式：ip:port:username:password
|
||||
*/
|
||||
const CAPTCHA_PROXY_RAW = (process.env.CAPTCHA_PROXY || '').trim();
|
||||
|
||||
/**
|
||||
* @returns {{
|
||||
* host: string,
|
||||
* port: string,
|
||||
* username: string,
|
||||
* password: string,
|
||||
* chromeProxyServer: string,
|
||||
* capsolverString: string,
|
||||
* }|null}
|
||||
*/
|
||||
function parseCaptchaProxy(raw = CAPTCHA_PROXY_RAW) {
|
||||
const value = (raw || '').trim();
|
||||
if (value === '') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const parts = value.split(':');
|
||||
if (parts.length < 4) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const host = parts[0];
|
||||
const port = parts[1];
|
||||
const username = parts[2];
|
||||
const password = parts.slice(3).join(':');
|
||||
|
||||
if (!host || !port || !username || !password) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
host,
|
||||
port,
|
||||
username,
|
||||
password,
|
||||
chromeProxyServer: `http://${host}:${port}`,
|
||||
capsolverString: value,
|
||||
};
|
||||
}
|
||||
|
||||
/** @returns {boolean} */
|
||||
function hasCaptchaProxy() {
|
||||
return parseCaptchaProxy() !== null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 是否属于可能走 CapSolver Managed 的工单（A2C 等）
|
||||
* 注意：仅表示候选，不代表本次任务一定挂代理
|
||||
* @param {object|null|undefined} antiBot
|
||||
*/
|
||||
function isManagedCfCandidate(antiBot) {
|
||||
if (!antiBot?.enabled) {
|
||||
return false;
|
||||
}
|
||||
if (antiBot.cfChallengeMode === 'managed') {
|
||||
return true;
|
||||
}
|
||||
if (antiBot.cfCloudflareSolver === true) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated 请使用 isManagedCfCandidate + 显式 useCaptchaProxy 参数
|
||||
*/
|
||||
function shouldUseCaptchaProxyForAntiBot(antiBot) {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Chrome 启动参数 --proxy-server（须 useCaptchaProxy=true 时调用）
|
||||
* @returns {string[]}
|
||||
*/
|
||||
function getCaptchaProxyChromeArgs() {
|
||||
const parsed = parseCaptchaProxy();
|
||||
if (!parsed) {
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
`--proxy-server=${parsed.chromeProxyServer}`,
|
||||
'--proxy-bypass-list=<-loopback>,localhost,127.0.0.1',
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* 为 Page 设置代理认证（须配合 --proxy-server）
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
async function applyCaptchaProxyToPage(page) {
|
||||
const parsed = parseCaptchaProxy();
|
||||
if (!parsed || !page) {
|
||||
return false;
|
||||
}
|
||||
await page.authenticate({
|
||||
username: parsed.username,
|
||||
password: parsed.password,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
parseCaptchaProxy,
|
||||
hasCaptchaProxy,
|
||||
isManagedCfCandidate,
|
||||
shouldUseCaptchaProxyForAntiBot,
|
||||
getCaptchaProxyChromeArgs,
|
||||
applyCaptchaProxyToPage,
|
||||
};
|
||||
@@ -6,6 +6,10 @@ const {
|
||||
CAPTCHA_API_KEY,
|
||||
CAPTCHA_MAX_WAIT_MS,
|
||||
} = require('./constants');
|
||||
const { sanitizePageUrlForCfSolver } = require('./cf-detector');
|
||||
|
||||
/** AntiCloudflareTask 所需静态代理（ip:port:user:pass），未配置则不可用 */
|
||||
const CAPTCHA_PROXY = (process.env.CAPTCHA_PROXY || '').trim();
|
||||
|
||||
/**
|
||||
* @param {string} url
|
||||
@@ -113,6 +117,147 @@ async function solveVia2Captcha({ pageUrl, sitekey, apiKey }) {
|
||||
throw new Error('2Captcha 求解超时');
|
||||
}
|
||||
|
||||
/**
|
||||
* CapSolver Managed Challenge（返回 cf_clearance cookie，需 CAPTCHA_PROXY）
|
||||
* @param {{ pageUrl: string, apiKey: string, proxy: string, html?: string, userAgent?: string }} params
|
||||
* @returns {Promise<{ cookies: Record<string, string>, userAgent?: string }>}
|
||||
*/
|
||||
async function solveViaCapSolverCloudflare({ pageUrl, apiKey, proxy, html, userAgent }) {
|
||||
const task = {
|
||||
type: 'AntiCloudflareTask',
|
||||
websiteURL: pageUrl,
|
||||
proxy,
|
||||
};
|
||||
if (html) {
|
||||
task.html = html.slice(0, 500000);
|
||||
}
|
||||
if (userAgent) {
|
||||
task.userAgent = userAgent;
|
||||
}
|
||||
|
||||
const create = await httpJsonPost('https://api.capsolver.com/createTask', {
|
||||
clientKey: apiKey,
|
||||
task,
|
||||
});
|
||||
|
||||
if (create.errorId !== 0 || !create.taskId) {
|
||||
throw new Error(`CapSolver AntiCloudflare createTask 失败: ${create.errorDescription || create.errorCode || 'unknown'}`);
|
||||
}
|
||||
|
||||
const deadline = Date.now() + CAPTCHA_MAX_WAIT_MS;
|
||||
while (Date.now() < deadline) {
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
const result = await httpJsonPost('https://api.capsolver.com/getTaskResult', {
|
||||
clientKey: apiKey,
|
||||
taskId: create.taskId,
|
||||
});
|
||||
|
||||
if (result.status === 'ready' && result.solution) {
|
||||
const cookies = result.solution.cookies || {};
|
||||
if (typeof cookies === 'string') {
|
||||
return { cookies: { cf_clearance: cookies }, userAgent: result.solution.userAgent };
|
||||
}
|
||||
if (result.solution.token && !cookies.cf_clearance) {
|
||||
return {
|
||||
cookies: { cf_clearance: result.solution.token },
|
||||
userAgent: result.solution.userAgent,
|
||||
};
|
||||
}
|
||||
return {
|
||||
cookies,
|
||||
userAgent: result.solution.userAgent,
|
||||
};
|
||||
}
|
||||
if (result.errorId !== 0) {
|
||||
throw new Error(`CapSolver AntiCloudflare getTaskResult 失败: ${result.errorDescription || result.errorCode}`);
|
||||
}
|
||||
}
|
||||
throw new Error('CapSolver AntiCloudflare 求解超时');
|
||||
}
|
||||
|
||||
/**
|
||||
* 统一 Cloudflare Managed Challenge 求解
|
||||
* @param {{ pageUrl: string, provider?: string, apiKey?: string, proxy?: string, html?: string, userAgent?: string }} params
|
||||
* @returns {Promise<{ cookies: Record<string, string>, userAgent?: string, provider: string }>}
|
||||
*/
|
||||
async function solveCloudflare({ pageUrl, provider, apiKey, proxy, html, userAgent }) {
|
||||
const resolvedProvider = (provider || CAPTCHA_PROVIDER).toLowerCase();
|
||||
const resolvedKey = apiKey || CAPTCHA_API_KEY;
|
||||
const resolvedProxy = (proxy || CAPTCHA_PROXY).trim();
|
||||
const cleanPageUrl = sanitizePageUrlForCfSolver(pageUrl);
|
||||
|
||||
if (!resolvedKey) {
|
||||
throw new Error('未配置 CAPTCHA_API_KEY，无法使用 AntiCloudflareTask');
|
||||
}
|
||||
if (!resolvedProxy) {
|
||||
throw new Error('AntiCloudflareTask 需要配置 CAPTCHA_PROXY（静态代理 ip:port:user:pass）');
|
||||
}
|
||||
|
||||
if (resolvedProvider !== 'capsolver') {
|
||||
throw new Error('AntiCloudflareTask 当前仅支持 CapSolver');
|
||||
}
|
||||
|
||||
const { cookies, userAgent: solvedUa } = await solveViaCapSolverCloudflare({
|
||||
pageUrl: cleanPageUrl,
|
||||
apiKey: resolvedKey,
|
||||
proxy: resolvedProxy,
|
||||
html,
|
||||
userAgent,
|
||||
});
|
||||
|
||||
if (!cookies || !cookies.cf_clearance) {
|
||||
throw new Error('CapSolver AntiCloudflare 未返回 cf_clearance');
|
||||
}
|
||||
|
||||
return { cookies, userAgent: solvedUa || userAgent, provider: resolvedProvider };
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 CapSolver AntiCloudflare 返回的 cookie / UA 写入页面并 reload
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @param {string} cleanUrl
|
||||
* @param {{ cookies: Record<string, string>, userAgent?: string }} solution
|
||||
*/
|
||||
async function applyCfClearanceSolution(page, cleanUrl, solution) {
|
||||
if (solution.userAgent) {
|
||||
await page.setUserAgent(solution.userAgent).catch(() => {});
|
||||
}
|
||||
|
||||
let host = '';
|
||||
let path = '/';
|
||||
try {
|
||||
const parsed = new URL(cleanUrl);
|
||||
host = parsed.hostname;
|
||||
path = parsed.pathname || '/';
|
||||
} catch (_) {
|
||||
host = '';
|
||||
}
|
||||
|
||||
for (const [name, value] of Object.entries(solution.cookies || {})) {
|
||||
if (!value) {
|
||||
continue;
|
||||
}
|
||||
const cookieDomain = host.startsWith('.') ? host : `.${host}`;
|
||||
await page.setCookie({
|
||||
name,
|
||||
value: String(value),
|
||||
domain: cookieDomain,
|
||||
path: '/',
|
||||
secure: true,
|
||||
sameSite: 'None',
|
||||
}).catch(() => {});
|
||||
}
|
||||
|
||||
console.log(`[CF] cf_clearance 已注入，goto 业务页 url=${cleanUrl}`);
|
||||
await page.goto(cleanUrl, {
|
||||
waitUntil: 'domcontentloaded',
|
||||
timeout: 60000,
|
||||
}).catch(() => {});
|
||||
await page.waitForNetworkIdle({ idleTime: 500, timeout: 30000 }).catch(() => {
|
||||
console.warn('[CF] cf_clearance goto 后 networkidle 超时，继续后续流程');
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 统一 Turnstile 求解入口
|
||||
* @param {{ pageUrl: string, sitekey: string, provider?: string, apiKey?: string }} params
|
||||
@@ -121,6 +266,7 @@ async function solveVia2Captcha({ pageUrl, sitekey, apiKey }) {
|
||||
async function solveTurnstile({ pageUrl, sitekey, provider, apiKey }) {
|
||||
const resolvedProvider = (provider || CAPTCHA_PROVIDER).toLowerCase();
|
||||
const resolvedKey = apiKey || CAPTCHA_API_KEY;
|
||||
const cleanPageUrl = sanitizePageUrlForCfSolver(pageUrl);
|
||||
|
||||
if (!sitekey) {
|
||||
throw new Error('无法提取 Turnstile sitekey');
|
||||
@@ -131,9 +277,9 @@ async function solveTurnstile({ pageUrl, sitekey, provider, apiKey }) {
|
||||
|
||||
let token;
|
||||
if (resolvedProvider === '2captcha') {
|
||||
token = await solveVia2Captcha({ pageUrl, sitekey, apiKey: resolvedKey });
|
||||
token = await solveVia2Captcha({ pageUrl: cleanPageUrl, sitekey, apiKey: resolvedKey });
|
||||
} else {
|
||||
token = await solveViaCapSolver({ pageUrl, sitekey, apiKey: resolvedKey });
|
||||
token = await solveViaCapSolver({ pageUrl: cleanPageUrl, sitekey, apiKey: resolvedKey });
|
||||
}
|
||||
|
||||
return { token, provider: resolvedProvider };
|
||||
@@ -149,5 +295,7 @@ function isCaptchaConfigured() {
|
||||
|
||||
module.exports = {
|
||||
solveTurnstile,
|
||||
solveCloudflare,
|
||||
applyCfClearanceSolution,
|
||||
isCaptchaConfigured,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
/**
|
||||
* CapSolver / 2Captcha 过盾兜底：Turnstile token 注入 或 AntiCloudflareTask cf_clearance
|
||||
*
|
||||
* A2C 等业务域为 CF Managed Challenge（非独立 Turnstile 挂件），须优先 AntiCloudflareTask。
|
||||
*/
|
||||
const { sanitizePageUrlForCfSolver } = require('./cf-detector');
|
||||
const {
|
||||
injectTurnstileToken,
|
||||
recoverAfterTokenInject,
|
||||
extractTurnstileSitekey,
|
||||
} = require('./turnstile-handler');
|
||||
const {
|
||||
solveTurnstile,
|
||||
solveCloudflare,
|
||||
applyCfClearanceSolution,
|
||||
} = require('./captcha-solver');
|
||||
const { getSitekeyCapture } = require('./sitekey-capture');
|
||||
|
||||
/** @returns {boolean} */
|
||||
function hasCaptchaProxy() {
|
||||
return (process.env.CAPTCHA_PROXY || '').trim() !== '';
|
||||
}
|
||||
|
||||
/**
|
||||
* 是否应使用 CapSolver AntiCloudflareTask（Managed Challenge / 5s 盾）
|
||||
* @param {object|null|undefined} antiBot
|
||||
*/
|
||||
function shouldPreferManagedCloudflareSolver(antiBot) {
|
||||
if (!antiBot || antiBot.cfCloudflareSolver === false) {
|
||||
return false;
|
||||
}
|
||||
if (antiBot.cfChallengeMode === 'managed') {
|
||||
return hasCaptchaProxy();
|
||||
}
|
||||
if (antiBot.cfCloudflareSolver === true && hasCaptchaProxy()) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析可用于 AntiTurnstileTask 的 sitekey（排除 A2C 配置的 challenge sitekey）
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @param {object} antiBot
|
||||
* @param {number} waitMs
|
||||
* @returns {Promise<string|null>}
|
||||
*/
|
||||
async function resolveTurnstileWidgetSitekey(page, antiBot, waitMs) {
|
||||
if (antiBot.cfChallengeMode === 'managed') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const capture = getSitekeyCapture(page);
|
||||
if (capture) {
|
||||
const fromCapture = await capture.resolveForSolver(page, antiBot, waitMs);
|
||||
if (fromCapture && antiBot.cfChallengeMode !== 'managed') {
|
||||
const configured = String(antiBot.turnstileSitekey || '').trim();
|
||||
if (configured && fromCapture === configured && antiBot.cfCloudflareSolver) {
|
||||
return null;
|
||||
}
|
||||
return fromCapture;
|
||||
}
|
||||
}
|
||||
|
||||
const dynamic = await extractTurnstileSitekey(page);
|
||||
if (dynamic) {
|
||||
return dynamic;
|
||||
}
|
||||
|
||||
const configured = String(antiBot?.turnstileSitekey || '').trim();
|
||||
if (configured && !antiBot.cfCloudflareSolver) {
|
||||
return configured;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* CapSolver AntiCloudflareTask 求解并注入 cf_clearance
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @param {object} params
|
||||
*/
|
||||
async function runManagedCloudflareSolver(page, params) {
|
||||
const { cleanSolverUrl, waitForUrlSettled, timeoutMs } = params;
|
||||
|
||||
if (!hasCaptchaProxy()) {
|
||||
throw new Error('AntiCloudflareTask 需要配置 CAPTCHA_PROXY（静态代理 ip:port:user:pass）');
|
||||
}
|
||||
|
||||
console.log('[CF] A2C/Managed Challenge：使用 CapSolver AntiCloudflareTask');
|
||||
const html = await page.content().catch(() => '');
|
||||
const userAgent = await page.evaluate(() => navigator.userAgent).catch(() => '');
|
||||
const { cookies, userAgent: solvedUa, provider } = await solveCloudflare({
|
||||
pageUrl: cleanSolverUrl,
|
||||
html,
|
||||
userAgent,
|
||||
});
|
||||
console.log(`[CF] CapSolver AntiCloudflareTask (${provider}) 返回 cf_clearance`);
|
||||
await applyCfClearanceSolution(page, cleanSolverUrl, { cookies, userAgent: solvedUa || userAgent });
|
||||
if (typeof waitForUrlSettled === 'function') {
|
||||
await waitForUrlSettled(page).catch(() => {});
|
||||
}
|
||||
|
||||
return { mode: 'cloudflare', provider };
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @param {object} options
|
||||
* @param {object} options.antiBot
|
||||
* @param {string} [options.navUrl]
|
||||
* @param {(page: import('puppeteer').Page) => Promise<string>} [options.waitForUrlSettled]
|
||||
* @param {number} [options.timeoutMs]
|
||||
* @returns {Promise<{ mode: 'turnstile'|'cloudflare', provider: string }>}
|
||||
*/
|
||||
async function runCaptchaApiFallback(page, options) {
|
||||
const {
|
||||
antiBot,
|
||||
navUrl = '',
|
||||
waitForUrlSettled,
|
||||
timeoutMs = 60000,
|
||||
} = options;
|
||||
|
||||
const cleanSolverUrl = sanitizePageUrlForCfSolver(navUrl || page.url());
|
||||
|
||||
if (shouldPreferManagedCloudflareSolver(antiBot)) {
|
||||
console.log(`[CF] Captcha API 求解 pageUrl=${cleanSolverUrl} mode=managed_challenge`);
|
||||
return runManagedCloudflareSolver(page, { cleanSolverUrl, waitForUrlSettled, timeoutMs });
|
||||
}
|
||||
|
||||
const sitekey = await resolveTurnstileWidgetSitekey(
|
||||
page,
|
||||
antiBot,
|
||||
Math.min(timeoutMs, 25000)
|
||||
);
|
||||
console.log(`[CF] Captcha API 求解 pageUrl=${cleanSolverUrl} sitekey=${sitekey ? 'ok' : 'missing'}`);
|
||||
|
||||
if (sitekey) {
|
||||
try {
|
||||
const { token, provider } = await solveTurnstile({ pageUrl: cleanSolverUrl, sitekey });
|
||||
console.log(`[CF] Captcha API Turnstile (${provider}) 返回 token，注入页面`);
|
||||
await injectTurnstileToken(page, token);
|
||||
await recoverAfterTokenInject(
|
||||
page,
|
||||
cleanSolverUrl,
|
||||
waitForUrlSettled,
|
||||
Math.min(timeoutMs, 45000)
|
||||
);
|
||||
return { mode: 'turnstile', provider };
|
||||
} catch (turnstileErr) {
|
||||
const msg = (turnstileErr?.message || '').toLowerCase();
|
||||
const isChallengeMismatch = msg.includes('challenge, not turnstile')
|
||||
|| msg.includes('not turnstile');
|
||||
if (isChallengeMismatch && antiBot.cfCloudflareSolver !== false && hasCaptchaProxy()) {
|
||||
console.warn(`[CF] Turnstile 任务失败(${turnstileErr.message})，降级 AntiCloudflareTask`);
|
||||
return runManagedCloudflareSolver(page, { cleanSolverUrl, waitForUrlSettled, timeoutMs });
|
||||
}
|
||||
throw turnstileErr;
|
||||
}
|
||||
}
|
||||
|
||||
if (antiBot?.cfCloudflareSolver !== false && hasCaptchaProxy()) {
|
||||
return runManagedCloudflareSolver(page, { cleanSolverUrl, waitForUrlSettled, timeoutMs });
|
||||
}
|
||||
|
||||
throw new Error('无法过盾：请配置 CAPTCHA_PROXY（A2C Managed Challenge）或提供有效 Turnstile sitekey');
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
runCaptchaApiFallback,
|
||||
shouldPreferManagedCloudflareSolver,
|
||||
hasCaptchaProxy,
|
||||
};
|
||||
@@ -20,6 +20,34 @@ function urlIndicatesCfChallenge(url) {
|
||||
|| url.includes('/cdn-cgi/challenge');
|
||||
}
|
||||
|
||||
/** CF 挑战态查询参数（CapSolver websiteURL 需剔除） */
|
||||
const CF_CHALLENGE_QUERY_KEYS = [
|
||||
'__cf_chl_rt_tk',
|
||||
'__cf_chl_tk',
|
||||
'__cf_chl_f_tk',
|
||||
'__cf_chl_captcha_tk__',
|
||||
];
|
||||
|
||||
/**
|
||||
* 去掉 URL 中 CF 挑战临时参数，供 CapSolver / reload 使用
|
||||
* @param {string} url
|
||||
* @returns {string}
|
||||
*/
|
||||
function sanitizePageUrlForCfSolver(url) {
|
||||
if (!url || typeof url !== 'string') {
|
||||
return url || '';
|
||||
}
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
for (const key of CF_CHALLENGE_QUERY_KEYS) {
|
||||
parsed.searchParams.delete(key);
|
||||
}
|
||||
return parsed.toString();
|
||||
} catch (_) {
|
||||
return url.split('?')[0] || url;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 是否要求 cf_clearance 才视为过盾完成（A2C user.a2c.chat 为 false）
|
||||
* @param {object|null|undefined} antiBot
|
||||
@@ -61,7 +89,9 @@ async function detectCloudflare(page, antiBot = null) {
|
||||
hasTurnstileWidget,
|
||||
hasChallengeRunning,
|
||||
titleHasJustAMoment: titleText.includes('Just a moment'),
|
||||
bodyHasJustAMoment: bodyText.includes('Just a moment') || bodyText.includes('Checking your browser'),
|
||||
bodyHasJustAMoment: bodyText.includes('Just a moment')
|
||||
|| bodyText.includes('Checking your browser')
|
||||
|| bodyText.includes('Performing security verification'),
|
||||
};
|
||||
}).catch(() => ({
|
||||
hasTurnstileInput: false,
|
||||
@@ -77,7 +107,8 @@ async function detectCloudflare(page, antiBot = null) {
|
||||
|| domSignals.titleHasJustAMoment
|
||||
|| domSignals.bodyHasJustAMoment
|
||||
|| domSignals.hasChallengeRunning
|
||||
|| title.includes('Just a moment');
|
||||
|| title.includes('Just a moment')
|
||||
|| title.includes('Performing security verification');
|
||||
|
||||
const turnstileSignals = domSignals.hasTurnstileInput || domSignals.hasTurnstileWidget;
|
||||
|
||||
@@ -93,7 +124,7 @@ async function detectCloudflare(page, antiBot = null) {
|
||||
if (requireClearance) {
|
||||
blocked = !hasCfClearance && (realChallenge || turnstileSignals);
|
||||
} else {
|
||||
blocked = realChallenge;
|
||||
blocked = realChallenge || (turnstileSignals && urlChallenge);
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -129,4 +160,5 @@ module.exports = {
|
||||
isTurnstileSolved,
|
||||
urlIndicatesCfChallenge,
|
||||
isCfClearanceRequired,
|
||||
sanitizePageUrlForCfSolver,
|
||||
};
|
||||
|
||||
@@ -5,8 +5,9 @@
|
||||
* 否则清除可能过期的 cf_clearance 并重新触发 Turnstile（Whatshub 短链 → work-order-sharing 场景）。
|
||||
*/
|
||||
const { detectCloudflare } = require('./cf-detector');
|
||||
const { waitForTurnstile, extractTurnstileSitekey, injectTurnstileToken, waitForTurnstileSurface } = require('./turnstile-handler');
|
||||
const { solveTurnstile, isCaptchaConfigured } = require('./captcha-solver');
|
||||
const { waitForTurnstile } = require('./turnstile-handler');
|
||||
const { isCaptchaConfigured } = require('./captcha-solver');
|
||||
const { runCaptchaApiFallback } = require('./cf-api-fallback');
|
||||
const { isSessionLikelyValid } = require('./session-store');
|
||||
|
||||
/**
|
||||
@@ -257,15 +258,12 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
|
||||
if (waitForUrlSettled) {
|
||||
await waitForUrlSettled(page).catch(() => {});
|
||||
}
|
||||
let sitekey = await waitForTurnstileSurface(page, Math.min(timeoutMs, 25000));
|
||||
if (!sitekey) {
|
||||
sitekey = await extractTurnstileSitekey(page);
|
||||
}
|
||||
const pageUrl = page.url();
|
||||
console.log(`[CF] Captcha API 求解 pageUrl=${pageUrl} sitekey=${sitekey ? 'ok' : 'missing'}`);
|
||||
const { token, provider } = await solveTurnstile({ pageUrl, sitekey });
|
||||
console.log(`[CF] Captcha API (${provider}) 返回 token，注入页面`);
|
||||
await injectTurnstileToken(page, token);
|
||||
await runCaptchaApiFallback(page, {
|
||||
antiBot,
|
||||
navUrl: page.url(),
|
||||
waitForUrlSettled,
|
||||
timeoutMs,
|
||||
});
|
||||
|
||||
const apiWaitMs = Math.min(timeoutMs, 30000);
|
||||
await waitForTurnstile(page, { timeoutMs: apiWaitMs });
|
||||
|
||||
@@ -85,7 +85,7 @@ const QUEUE_TIMEOUT_MS = Math.max(5000, parseInt(process.env.QUEUE_TIMEOUT_MS ||
|
||||
const API_INTERCEPT_TIMEOUT_MS = Math.max(10000, parseInt(process.env.API_INTERCEPT_TIMEOUT_MS || '60000', 10));
|
||||
const API_INTERCEPT_TIMEOUT_REAL_MS = Math.max(
|
||||
API_INTERCEPT_TIMEOUT_MS,
|
||||
parseInt(process.env.API_INTERCEPT_TIMEOUT_REAL_MS || '90000', 10)
|
||||
parseInt(process.env.API_INTERCEPT_TIMEOUT_REAL_MS || '180000', 10)
|
||||
);
|
||||
|
||||
const NAVIGATION_TIMEOUT_MS = Math.max(15000, parseInt(process.env.NAVIGATION_TIMEOUT_MS || '60000', 10));
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
/**
|
||||
* Turnstile sitekey 提前捕获：在 goto 前挂载监听，避免 60s 空等后 iframe 已销毁
|
||||
*/
|
||||
const {
|
||||
extractSitekeyFromChallengeUrl,
|
||||
extractSitekeyFromHtml,
|
||||
extractTurnstileSitekey,
|
||||
waitForTurnstileSurface,
|
||||
} = require('./turnstile-handler');
|
||||
|
||||
/**
|
||||
* @typedef {Object} SitekeyCaptureHandle
|
||||
* @property {() => string|null} get
|
||||
* @property {(page: import('puppeteer').Page, antiBot?: object) => Promise<string|null>} scanNow
|
||||
* @property {(page: import('puppeteer').Page, antiBot?: object, waitMs?: number) => Promise<string|null>} resolveForSolver
|
||||
* @property {() => void} detach
|
||||
*/
|
||||
|
||||
/**
|
||||
* 尝试从 URL 字符串解析并缓存 sitekey
|
||||
* @param {string|null|undefined} url
|
||||
* @param {{ sitekey: string|null, log: (msg: string) => void }} state
|
||||
* @returns {string|null}
|
||||
*/
|
||||
function tryStoreSitekeyFromUrl(url, state) {
|
||||
const key = extractSitekeyFromChallengeUrl(url || '');
|
||||
if (key && !state.sitekey) {
|
||||
state.sitekey = key;
|
||||
state.log(`[CF] sitekey 捕获 (${key.slice(0, 12)}...) source=${url.slice(0, 80)}`);
|
||||
}
|
||||
return state.sitekey;
|
||||
}
|
||||
|
||||
/**
|
||||
* 在 page.goto 之前挂载 frame/response/target 监听
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @param {import('puppeteer').Browser} [browser]
|
||||
* @returns {SitekeyCaptureHandle}
|
||||
*/
|
||||
function attachSitekeyCapture(page, browser) {
|
||||
const state = {
|
||||
sitekey: null,
|
||||
attached: false,
|
||||
cleanups: [],
|
||||
log: (msg) => console.log(msg),
|
||||
};
|
||||
|
||||
if (page.__sitekeyCaptureHandle) {
|
||||
return page.__sitekeyCaptureHandle;
|
||||
}
|
||||
|
||||
const onFrameNavigated = (frame) => {
|
||||
try {
|
||||
tryStoreSitekeyFromUrl(frame.url(), state);
|
||||
} catch (_) {
|
||||
/* ignore */
|
||||
}
|
||||
};
|
||||
page.on('framenavigated', onFrameNavigated);
|
||||
state.cleanups.push(() => page.off('framenavigated', onFrameNavigated));
|
||||
|
||||
const onResponse = (response) => {
|
||||
try {
|
||||
tryStoreSitekeyFromUrl(response.url(), state);
|
||||
} catch (_) {
|
||||
/* ignore */
|
||||
}
|
||||
};
|
||||
page.on('response', onResponse);
|
||||
state.cleanups.push(() => page.off('response', onResponse));
|
||||
|
||||
if (browser && typeof browser.on === 'function') {
|
||||
const onTargetCreated = (target) => {
|
||||
try {
|
||||
tryStoreSitekeyFromUrl(target.url(), state);
|
||||
} catch (_) {
|
||||
/* ignore */
|
||||
}
|
||||
};
|
||||
browser.on('targetcreated', onTargetCreated);
|
||||
state.cleanups.push(() => browser.off('targetcreated', onTargetCreated));
|
||||
}
|
||||
|
||||
state.attached = true;
|
||||
|
||||
/** @type {SitekeyCaptureHandle} */
|
||||
const handle = {
|
||||
get: () => state.sitekey,
|
||||
scanNow: async (scanPage, antiBot) => {
|
||||
if (state.sitekey) {
|
||||
return state.sitekey;
|
||||
}
|
||||
|
||||
const frames = scanPage.frames();
|
||||
for (const frame of frames) {
|
||||
tryStoreSitekeyFromUrl(frame.url(), state);
|
||||
if (state.sitekey) {
|
||||
return state.sitekey;
|
||||
}
|
||||
}
|
||||
|
||||
const html = await scanPage.content().catch(() => '');
|
||||
const fromHtml = extractSitekeyFromHtml(html);
|
||||
if (fromHtml) {
|
||||
state.sitekey = fromHtml;
|
||||
state.log(`[CF] sitekey 从 HTML 提取 (${fromHtml.slice(0, 12)}...)`);
|
||||
return fromHtml;
|
||||
}
|
||||
|
||||
const browserRef = scanPage.browser();
|
||||
if (browserRef) {
|
||||
for (const target of browserRef.targets()) {
|
||||
tryStoreSitekeyFromUrl(target.url(), state);
|
||||
if (state.sitekey) {
|
||||
return state.sitekey;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const configured = String(antiBot?.turnstileSitekey || '').trim();
|
||||
if (configured && antiBot?.cfChallengeMode !== 'managed' && !antiBot?.cfCloudflareSolver) {
|
||||
state.sitekey = configured;
|
||||
state.log(`[CF] 使用 antiBot.turnstileSitekey (${configured.slice(0, 12)}...)`);
|
||||
return configured;
|
||||
}
|
||||
|
||||
return null;
|
||||
},
|
||||
resolveForSolver: async (scanPage, antiBot, waitMs = 25000) => {
|
||||
let sitekey = await handle.scanNow(scanPage, antiBot);
|
||||
if (sitekey) {
|
||||
return sitekey;
|
||||
}
|
||||
|
||||
sitekey = await waitForTurnstileSurface(scanPage, waitMs);
|
||||
if (sitekey) {
|
||||
state.sitekey = sitekey;
|
||||
return sitekey;
|
||||
}
|
||||
|
||||
sitekey = await extractTurnstileSitekey(scanPage);
|
||||
if (sitekey) {
|
||||
state.sitekey = sitekey;
|
||||
return sitekey;
|
||||
}
|
||||
|
||||
return handle.scanNow(scanPage, antiBot);
|
||||
},
|
||||
detach: () => {
|
||||
for (const fn of state.cleanups) {
|
||||
try {
|
||||
fn();
|
||||
} catch (_) {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
state.cleanups = [];
|
||||
state.attached = false;
|
||||
if (page.__sitekeyCaptureHandle === handle) {
|
||||
delete page.__sitekeyCaptureHandle;
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
page.__sitekeyCaptureHandle = handle;
|
||||
return handle;
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取 page 上已挂载的 capture（无则 null）
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @returns {SitekeyCaptureHandle|null}
|
||||
*/
|
||||
function getSitekeyCapture(page) {
|
||||
return page?.__sitekeyCaptureHandle || null;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
attachSitekeyCapture,
|
||||
getSitekeyCapture,
|
||||
tryStoreSitekeyFromUrl,
|
||||
};
|
||||
@@ -1,22 +1,52 @@
|
||||
/**
|
||||
* Turnstile 内置求解：等待 widget 自动完成 + token 轮询 + 多 frame sitekey 提取
|
||||
*
|
||||
* 支持新版 challenge-platform iframe：sitekey 在 URL 路径 /rch/{id}/{sitekey}/ 中
|
||||
*/
|
||||
const { detectCloudflare, isTurnstileSolved, urlIndicatesCfChallenge } = require('./cf-detector');
|
||||
|
||||
/**
|
||||
* @typedef {Object} TurnstileResult
|
||||
* @property {boolean} success
|
||||
* @property {string} stage built_in|timeout|already_clear
|
||||
* @property {number} elapsedMs
|
||||
* 从 challenge-platform / Turnstile URL 解析 sitekey（Node 侧，可读 frame.url()）
|
||||
* @param {string} url
|
||||
* @returns {string|null}
|
||||
*/
|
||||
function extractSitekeyFromChallengeUrl(url) {
|
||||
if (!url || typeof url !== 'string') {
|
||||
return null;
|
||||
}
|
||||
const pathMatch = url.match(/\/rch\/[^/]+\/(0x4[A-Za-z0-9_-]+)\//i);
|
||||
if (pathMatch) {
|
||||
return pathMatch[1];
|
||||
}
|
||||
const queryMatch = url.match(/[?&]sitekey=([^&]+)/i);
|
||||
if (queryMatch) {
|
||||
return decodeURIComponent(queryMatch[1]);
|
||||
}
|
||||
const looseMatch = url.match(/\/(0x4[A-Za-z0-9_-]{10,})\//i);
|
||||
if (looseMatch) {
|
||||
return looseMatch[1];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 在单个 frame 内提取 sitekey
|
||||
* 在单个 frame 内提取 sitekey（主文档 DOM + iframe src 属性）
|
||||
* @param {import('puppeteer').Frame} frame
|
||||
* @returns {Promise<string|null>}
|
||||
*/
|
||||
async function extractSitekeyFromFrame(frame) {
|
||||
return frame.evaluate(() => {
|
||||
const extractFromSrc = (src) => {
|
||||
if (!src) return null;
|
||||
const pathMatch = src.match(/\/rch\/[^/]+\/(0x4[A-Za-z0-9_-]+)\//i);
|
||||
if (pathMatch) return pathMatch[1];
|
||||
const queryMatch = src.match(/[?&]sitekey=([^&]+)/i);
|
||||
if (queryMatch) return decodeURIComponent(queryMatch[1]);
|
||||
const looseMatch = src.match(/\/(0x4[A-Za-z0-9_-]{10,})\//i);
|
||||
if (looseMatch) return looseMatch[1];
|
||||
return null;
|
||||
};
|
||||
|
||||
const widget = document.querySelector('.cf-turnstile[data-sitekey], [data-sitekey]');
|
||||
if (widget) {
|
||||
const key = widget.getAttribute('data-sitekey');
|
||||
@@ -25,12 +55,13 @@ async function extractSitekeyFromFrame(frame) {
|
||||
}
|
||||
}
|
||||
|
||||
const iframes = document.querySelectorAll('iframe[src*="challenges.cloudflare.com"], iframe[src*="challenge-platform"]');
|
||||
const iframes = document.querySelectorAll(
|
||||
'iframe[src*="challenges.cloudflare.com"], iframe[src*="challenge-platform"], iframe#cf-chl-widget'
|
||||
);
|
||||
for (const iframe of iframes) {
|
||||
const src = iframe.getAttribute('src') || '';
|
||||
const match = src.match(/[?&]sitekey=([^&]+)/i);
|
||||
if (match) {
|
||||
return decodeURIComponent(match[1]);
|
||||
const fromSrc = extractFromSrc(iframe.getAttribute('src') || '');
|
||||
if (fromSrc) {
|
||||
return fromSrc;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,7 +69,8 @@ async function extractSitekeyFromFrame(frame) {
|
||||
for (const script of scripts) {
|
||||
const text = script.textContent || '';
|
||||
const match = text.match(/sitekey['"\s:=]+['"]([0-9x_a-zA-Z-]{10,})['"]/i)
|
||||
|| text.match(/data-sitekey=['"]([0-9x_a-zA-Z-]{10,})['"]/i);
|
||||
|| text.match(/data-sitekey=['"]([0-9x_a-zA-Z-]{10,})['"]/i)
|
||||
|| text.match(/\/(0x4[A-Za-z0-9_-]{10,})\//i);
|
||||
if (match) {
|
||||
return match[1];
|
||||
}
|
||||
@@ -48,6 +80,34 @@ async function extractSitekeyFromFrame(frame) {
|
||||
}).catch(() => null);
|
||||
}
|
||||
|
||||
/**
|
||||
* 从整页 HTML 文本提取 sitekey（应对 OOPIF 无法 enumerate 的场景）
|
||||
* @param {string} html
|
||||
* @returns {string|null}
|
||||
*/
|
||||
function extractSitekeyFromHtml(html) {
|
||||
if (!html || typeof html !== 'string') {
|
||||
return null;
|
||||
}
|
||||
const pathMatch = html.match(/\/rch\/[^/]+\/(0x4[A-Za-z0-9_-]+)\//i);
|
||||
if (pathMatch) {
|
||||
return pathMatch[1];
|
||||
}
|
||||
const dataMatch = html.match(/data-sitekey=["']([^"']+)["']/i);
|
||||
if (dataMatch) {
|
||||
return dataMatch[1];
|
||||
}
|
||||
const queryMatch = html.match(/[?&]sitekey=([^&"']+)/i);
|
||||
if (queryMatch) {
|
||||
return decodeURIComponent(queryMatch[1]);
|
||||
}
|
||||
const looseMatch = html.match(/\/(0x4[A-Za-z0-9_-]{10,})\//i);
|
||||
if (looseMatch) {
|
||||
return looseMatch[1];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 从主文档及所有子 frame 提取 Turnstile sitekey
|
||||
* @param {import('puppeteer').Page} page
|
||||
@@ -60,6 +120,10 @@ async function extractTurnstileSitekey(page) {
|
||||
}
|
||||
|
||||
for (const frame of page.frames()) {
|
||||
const fromFrameUrl = extractSitekeyFromChallengeUrl(frame.url());
|
||||
if (fromFrameUrl) {
|
||||
return fromFrameUrl;
|
||||
}
|
||||
if (frame === page.mainFrame()) {
|
||||
continue;
|
||||
}
|
||||
@@ -69,15 +133,30 @@ async function extractTurnstileSitekey(page) {
|
||||
}
|
||||
}
|
||||
|
||||
const html = await page.content().catch(() => '');
|
||||
const fromHtml = extractSitekeyFromHtml(html);
|
||||
if (fromHtml) {
|
||||
return fromHtml;
|
||||
}
|
||||
|
||||
const browser = page.browser();
|
||||
if (browser) {
|
||||
for (const target of browser.targets()) {
|
||||
const fromTarget = extractSitekeyFromChallengeUrl(target.url());
|
||||
if (fromTarget) {
|
||||
return fromTarget;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Captcha API 调用前：等待 Turnstile 挑战面出现或 URL 进入 CF 挑战态
|
||||
*
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @param {number} [timeoutMs]
|
||||
* @returns {Promise<string|null>} 若已提取到 sitekey 则返回，否则 null
|
||||
* @returns {Promise<string|null>}
|
||||
*/
|
||||
async function waitForTurnstileSurface(page, timeoutMs = 20000) {
|
||||
const started = Date.now();
|
||||
@@ -86,7 +165,7 @@ async function waitForTurnstileSurface(page, timeoutMs = 20000) {
|
||||
while (Date.now() - started < timeoutMs) {
|
||||
const sitekey = await extractTurnstileSitekey(page);
|
||||
if (sitekey) {
|
||||
console.log(`[CF] 已检测到 Turnstile sitekey (${sitekey.slice(0, 8)}...)`);
|
||||
console.log(`[CF] 已检测到 Turnstile sitekey (${sitekey.slice(0, 12)}...)`);
|
||||
return sitekey;
|
||||
}
|
||||
|
||||
@@ -104,7 +183,8 @@ async function waitForTurnstileSurface(page, timeoutMs = 20000) {
|
||||
const hasSurface = await page.evaluate(() => {
|
||||
return !!document.querySelector(
|
||||
'.cf-turnstile, [data-sitekey], [name="cf-turnstile-response"], '
|
||||
+ 'iframe[src*="challenges.cloudflare.com"], iframe[src*="challenge-platform"]'
|
||||
+ 'iframe[src*="challenges.cloudflare.com"], iframe[src*="challenge-platform"], '
|
||||
+ 'iframe[id^="cf-chl-widget"]'
|
||||
);
|
||||
}).catch(() => false);
|
||||
|
||||
@@ -122,8 +202,7 @@ async function waitForTurnstileSurface(page, timeoutMs = 20000) {
|
||||
/**
|
||||
* 等待 Turnstile 通过（内置点击 / 自动跳转）
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @param {{ timeoutMs?: number, pollMs?: number, useBuiltInClick?: boolean }} [options]
|
||||
* @returns {Promise<TurnstileResult>}
|
||||
* @param {{ timeoutMs?: number, pollMs?: number }} [options]
|
||||
*/
|
||||
async function waitForTurnstile(page, options = {}) {
|
||||
const timeoutMs = options.timeoutMs ?? 60000;
|
||||
@@ -148,7 +227,7 @@ async function waitForTurnstile(page, options = {}) {
|
||||
}
|
||||
|
||||
const state = await detectCloudflare(page);
|
||||
if (!state.blocked && !urlIndicatesCfChallenge(page.url()) && state.hasCfClearance) {
|
||||
if (!state.blocked && !urlIndicatesCfChallenge(page.url())) {
|
||||
return { success: true, stage: 'built_in', elapsedMs: Date.now() - started };
|
||||
}
|
||||
|
||||
@@ -159,7 +238,27 @@ async function waitForTurnstile(page, options = {}) {
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 Captcha API 返回的 token 注入页面
|
||||
* 等待页面离开 CF 挑战 URL / 挑战 DOM
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @param {number} timeoutMs
|
||||
* @param {number} pollMs
|
||||
*/
|
||||
async function waitForChallengeUrlClear(page, timeoutMs = 45000, pollMs = 500) {
|
||||
const started = Date.now();
|
||||
while (Date.now() - started < timeoutMs) {
|
||||
if (!urlIndicatesCfChallenge(page.url())) {
|
||||
const state = await detectCloudflare(page);
|
||||
if (!state.blocked) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, pollMs));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 Captcha API 返回的 token 注入页面并尝试触发提交
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @param {string} token
|
||||
*/
|
||||
@@ -178,10 +277,39 @@ async function injectTurnstileToken(page, token) {
|
||||
}, token);
|
||||
}
|
||||
|
||||
/**
|
||||
* token 注入后等待过盾；若仍停留在挑战页则 reload 干净业务 URL
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @param {string} cleanUrl
|
||||
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
|
||||
* @param {number} [timeoutMs]
|
||||
*/
|
||||
async function recoverAfterTokenInject(page, cleanUrl, waitForUrlSettled, timeoutMs = 45000) {
|
||||
if (await waitForChallengeUrlClear(page, Math.min(timeoutMs, 12000))) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!cleanUrl) {
|
||||
return false;
|
||||
}
|
||||
|
||||
console.log(`[CF] token 注入后仍处挑战页，reload 干净 URL: ${cleanUrl}`);
|
||||
await page.goto(cleanUrl, { waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 60000) }).catch(() => {});
|
||||
if (typeof waitForUrlSettled === 'function') {
|
||||
await waitForUrlSettled(page).catch(() => {});
|
||||
}
|
||||
|
||||
return waitForChallengeUrlClear(page, timeoutMs);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
waitForTurnstile,
|
||||
waitForTurnstileSurface,
|
||||
extractTurnstileSitekey,
|
||||
extractSitekeyFromFrame,
|
||||
extractSitekeyFromChallengeUrl,
|
||||
extractSitekeyFromHtml,
|
||||
injectTurnstileToken,
|
||||
waitForChallengeUrlClear,
|
||||
recoverAfterTokenInject,
|
||||
};
|
||||
|
||||
@@ -20,7 +20,8 @@ function classifyPuppeteerError(err) {
|
||||
if (err?.name === 'TimeoutError' || msg.includes('timeout')) return 'timeout';
|
||||
if (msg.includes('net::') || msg.includes('network') || msg.includes('err_connection')
|
||||
|| msg.includes('err_address_unreachable') || msg.includes('err_name_not_resolved')) return 'network';
|
||||
if (msg.includes('navigation') || msg.includes('frame was detached')) return 'navigation';
|
||||
if (msg.includes('navigation') || msg.includes('frame was detached')
|
||||
|| msg.includes('detached frame') || msg.includes('attempted to use detached')) return 'navigation';
|
||||
if (msg.includes('target closed') || msg.includes('session closed')) return 'target_closed';
|
||||
return 'unknown';
|
||||
}
|
||||
@@ -85,7 +86,9 @@ async function runUiPagination(page, options) {
|
||||
{ timeout: 1000, polling: 100 }
|
||||
).catch(() => new Promise((r) => setTimeout(r, 500)));
|
||||
|
||||
while (attempts < maxAttempts && !pageData) {
|
||||
let paginationComplete = false;
|
||||
|
||||
while (attempts < maxAttempts && !pageData && !paginationComplete) {
|
||||
attempts++;
|
||||
log(`[抓取] 准备抓取第 ${targetPageNum} 页 (尝试 ${attempts}/${maxAttempts})...`);
|
||||
|
||||
@@ -139,6 +142,7 @@ async function runUiPagination(page, options) {
|
||||
|
||||
if (clickStatus === 'disabled') {
|
||||
log(`[提示] 第 ${targetPageNum} 页按钮已被禁用，说明到底了，抓取提前结束。`);
|
||||
paginationComplete = true;
|
||||
break;
|
||||
}
|
||||
log('[动作] 双引擎原生点击已执行！');
|
||||
@@ -153,6 +157,11 @@ async function runUiPagination(page, options) {
|
||||
}
|
||||
}
|
||||
|
||||
if (paginationComplete) {
|
||||
log('[完成] 翻页已到底，正常结束。');
|
||||
break;
|
||||
}
|
||||
|
||||
if (pageData) {
|
||||
capturedData.push(pageData);
|
||||
log(`[成功] 斩获第 ${targetPageNum} 页数据！`);
|
||||
|
||||
+431
-52
@@ -1,5 +1,12 @@
|
||||
require('dotenv').config({ path: require('path').join(__dirname, '.env') });
|
||||
|
||||
process.on('uncaughtException', (err) => {
|
||||
console.error('[FATAL] uncaughtException:', err);
|
||||
});
|
||||
process.on('unhandledRejection', (reason) => {
|
||||
console.error('[FATAL] unhandledRejection:', reason);
|
||||
});
|
||||
|
||||
const express = require('express');
|
||||
const fs = require('fs');
|
||||
const {
|
||||
@@ -41,12 +48,14 @@ const { launchStandardBrowser } = require('./lib/launch-standard');
|
||||
const { isRealBrowserAvailable } = require('./lib/launch-real-browser');
|
||||
const { handleCloudflareChallenge, isCaptchaConfigured } = require('./lib/cf-handler');
|
||||
const { detectCloudflare, urlIndicatesCfChallenge } = require('./lib/cf-detector');
|
||||
const { runCaptchaApiFallback, shouldPreferManagedCloudflareSolver } = require('./lib/cf-api-fallback');
|
||||
const {
|
||||
extractTurnstileSitekey,
|
||||
injectTurnstileToken,
|
||||
waitForTurnstileSurface,
|
||||
} = require('./lib/turnstile-handler');
|
||||
const { solveTurnstile } = require('./lib/captcha-solver');
|
||||
getCaptchaProxyChromeArgs,
|
||||
applyCaptchaProxyToPage,
|
||||
parseCaptchaProxy,
|
||||
hasCaptchaProxy,
|
||||
} = require('./lib/captcha-proxy');
|
||||
const { attachSitekeyCapture } = require('./lib/sitekey-capture');
|
||||
const {
|
||||
saveSession,
|
||||
getSessionDebugInfo,
|
||||
@@ -78,7 +87,7 @@ async function runBrowserTask(taskName, handler, profile = 'standard') {
|
||||
}
|
||||
|
||||
function sendTaskError(res, error, extra = {}) {
|
||||
if (error.code === 'QUEUE_TIMEOUT') {
|
||||
if (error.code === 'QUEUE_TIMEOUT' || error.message === 'POOL_ACQUIRE_TIMEOUT') {
|
||||
return res.status(503).json({
|
||||
success: false,
|
||||
error: '服务繁忙，排队超时，请稍后重试',
|
||||
@@ -124,7 +133,8 @@ function classifyPuppeteerError(err) {
|
||||
if (err?.name === 'TimeoutError' || msg.includes('timeout')) return 'timeout';
|
||||
if (msg.includes('net::') || msg.includes('network') || msg.includes('err_connection')
|
||||
|| msg.includes('err_address_unreachable') || msg.includes('err_name_not_resolved')) return 'network';
|
||||
if (msg.includes('navigation') || msg.includes('frame was detached')) return 'navigation';
|
||||
if (msg.includes('navigation') || msg.includes('frame was detached')
|
||||
|| msg.includes('detached frame') || msg.includes('attempted to use detached')) return 'navigation';
|
||||
if (msg.includes('target closed') || msg.includes('session closed')) return 'target_closed';
|
||||
return 'unknown';
|
||||
}
|
||||
@@ -190,12 +200,16 @@ function resolveAntiBot(rawAntiBot) {
|
||||
...base,
|
||||
postCfReadyUrlContains: rawAntiBot.postCfReadyUrlContains || '',
|
||||
postCfReadySelector: rawAntiBot.postCfReadySelector || '',
|
||||
postCfReadyApiUrl: rawAntiBot.postCfReadyApiUrl || '',
|
||||
postCfReadyTimeoutMs: rawAntiBot.postCfReadyTimeoutMs || 0,
|
||||
postCfSettleMs: rawAntiBot.postCfSettleMs || 0,
|
||||
postCfSpaWaitMs: rawAntiBot.postCfSpaWaitMs || 0,
|
||||
cfClearanceRequired: rawAntiBot.cfClearanceRequired !== false,
|
||||
businessHostHint: rawAntiBot.businessHostHint || '',
|
||||
landingUrlHint: rawAntiBot.landingUrlHint || '',
|
||||
turnstileSitekey: rawAntiBot.turnstileSitekey || '',
|
||||
cfCloudflareSolver: rawAntiBot.cfCloudflareSolver !== false,
|
||||
cfChallengeMode: rawAntiBot.cfChallengeMode || '',
|
||||
};
|
||||
}
|
||||
|
||||
@@ -265,12 +279,15 @@ function applyA2cPostCfDefaults(antiBot, pageUrl) {
|
||||
return {
|
||||
...antiBot,
|
||||
postCfReadyUrlContains: '/visitors/counter/share',
|
||||
postCfReadySelector: '.btn-next',
|
||||
postCfReadyTimeoutMs: antiBot.postCfReadyTimeoutMs || 30000,
|
||||
postCfReadySelector: '.el-table',
|
||||
postCfReadyApiUrl: antiBot.postCfReadyApiUrl || '/api/talk/counter/share/record/list',
|
||||
postCfReadyTimeoutMs: antiBot.postCfReadyTimeoutMs || 60000,
|
||||
postCfSettleMs: antiBot.postCfSettleMs || 500,
|
||||
postCfSpaWaitMs: antiBot.postCfSpaWaitMs || 4000,
|
||||
postCfSpaWaitMs: antiBot.postCfSpaWaitMs || 8000,
|
||||
cfClearanceRequired: antiBot.cfClearanceRequired === undefined ? false : antiBot.cfClearanceRequired,
|
||||
solverFallback: antiBot.solverFallback === undefined ? false : antiBot.solverFallback,
|
||||
solverFallback: antiBot.solverFallback === undefined ? true : antiBot.solverFallback,
|
||||
cfCloudflareSolver: antiBot.cfCloudflareSolver !== false,
|
||||
cfChallengeMode: antiBot.cfChallengeMode || 'managed',
|
||||
};
|
||||
}
|
||||
|
||||
@@ -326,6 +343,51 @@ function isBusinessUrlReady(page, urlNeedle) {
|
||||
return !urlNeedle || page.url().includes(urlNeedle);
|
||||
}
|
||||
|
||||
/** A2C 已直达 user.a2c.chat 分享页（非 yyk.ink 短链跳转场景） */
|
||||
function isA2cDirectSharePage(page, urlNeedle = '') {
|
||||
const needle = (urlNeedle || '').trim();
|
||||
if (!needle.includes('/visitors/counter/share')) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
const parsed = new URL(page.url() || '');
|
||||
return parsed.hostname.includes('a2c.chat') && parsed.pathname.includes('/visitors/counter/share');
|
||||
} catch (_) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** 过盾后等待业务 list API 返回（A2C Vue 挂载信号） */
|
||||
async function waitForPostCfListApi(page, antiBot, timeoutMs) {
|
||||
const apiUrl = (antiBot.postCfReadyApiUrl || '').trim();
|
||||
if (!apiUrl) {
|
||||
return false;
|
||||
}
|
||||
console.log(`[CF] 等待业务 API 响应 api=${apiUrl} timeout=${timeoutMs}ms`);
|
||||
const matched = await page.waitForResponse(
|
||||
(resp) => {
|
||||
const req = resp.request();
|
||||
if (req.method() === 'OPTIONS') {
|
||||
return false;
|
||||
}
|
||||
return resp.url().includes(apiUrl) && resp.status() >= 200 && resp.status() < 400;
|
||||
},
|
||||
{ timeout: timeoutMs }
|
||||
).catch(() => null);
|
||||
if (matched) {
|
||||
console.log(`[CF] 业务 API 已响应 status=${matched.status()} url=${matched.url()}`);
|
||||
return true;
|
||||
}
|
||||
console.warn('[CF] 业务 API 等待超时，降级为 DOM selector 检测');
|
||||
return false;
|
||||
}
|
||||
|
||||
/** 等待业务 DOM：先 attached 再 visible，避免 Vue 渲染中途误判 */
|
||||
async function waitForPostCfSelector(page, selector, timeoutMs) {
|
||||
await page.waitForSelector(selector, { timeout: timeoutMs, visible: false });
|
||||
await page.waitForSelector(selector, { timeout: Math.min(15000, timeoutMs), visible: true });
|
||||
}
|
||||
|
||||
/** Whatshub 分享短链 /m/xxx/1 */
|
||||
function isWhatshubShortLinkPath(url) {
|
||||
try {
|
||||
@@ -366,11 +428,141 @@ function buildManagedPageOptions(pageUrl, antiBot, mergedCookies, extraOpts = {}
|
||||
}
|
||||
|
||||
/**
|
||||
* 任务页创建（含 Whatshub skipUserAgent）；browser-factory 无写权限时在 server 侧实现
|
||||
* A2C：短链入口时优先导航至 PHP 预解析的长链，减少 yyk.ink 跳转竞态
|
||||
* @param {string} pageUrl
|
||||
* @param {object} antiBot
|
||||
*/
|
||||
async function createTaskPage(browser, existingPage, pageUrl, antiBot, mergedCookies, extraOpts = {}) {
|
||||
function resolveNavigationUrl(pageUrl, antiBot) {
|
||||
const hint = String(antiBot?.landingUrlHint || '').trim();
|
||||
if (hint === '' || !hint.includes('/visitors/counter/share')) {
|
||||
return pageUrl;
|
||||
}
|
||||
try {
|
||||
const pageHost = new URL(pageUrl).hostname;
|
||||
const hintHost = new URL(hint).hostname;
|
||||
if (hintHost.includes('a2c.chat') && pageHost !== hintHost) {
|
||||
console.log(`[A2C] 短链入口，直接导航至预解析长链: ${hint}`);
|
||||
return hint;
|
||||
}
|
||||
} catch (_) {
|
||||
/* ignore */
|
||||
}
|
||||
return pageUrl;
|
||||
}
|
||||
|
||||
/**
|
||||
* Chrome 启动参数：仅当 useCaptchaProxy=true 时注入 CAPTCHA_PROXY（与 CapSolver 同源 IP）
|
||||
* @param {object|null|undefined} antiBot
|
||||
* @param {string[]} [baseArgs]
|
||||
* @param {boolean} [useCaptchaProxy]
|
||||
*/
|
||||
function buildBrowserExtraArgs(antiBot, baseArgs = [], useCaptchaProxy = false) {
|
||||
const args = [...baseArgs];
|
||||
if (!useCaptchaProxy || !hasCaptchaProxy()) {
|
||||
return args;
|
||||
}
|
||||
const proxyArgs = getCaptchaProxyChromeArgs();
|
||||
if (proxyArgs.length > 0) {
|
||||
const parsed = parseCaptchaProxy();
|
||||
console.log(
|
||||
`[Proxy] Chrome 使用 CAPTCHA_PROXY ${parsed.host}:${parsed.port}（与 CapSolver 同源 IP）`
|
||||
);
|
||||
args.push(...proxyArgs);
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
/**
|
||||
* Managed A2C 等：仅在检测到真实 CF 挑战时切换为 CAPTCHA_PROXY Browser（无盾则直连）
|
||||
* 会原地更新 browserCtx.browserSession / browserCtx.page
|
||||
* @param {object} browserCtx
|
||||
*/
|
||||
async function ensureCaptchaProxyBrowserIfNeeded(browserCtx) {
|
||||
const {
|
||||
antiBot,
|
||||
profile,
|
||||
navigationUrl,
|
||||
mergedCookies,
|
||||
baseExtraArgs = ['--mute-audio'],
|
||||
pageExtraOpts = {},
|
||||
} = browserCtx;
|
||||
let { browserSession, page } = browserCtx;
|
||||
|
||||
if (!antiBot?.enabled || browserSession?.captchaProxyActive) {
|
||||
return;
|
||||
}
|
||||
if (!shouldPreferManagedCloudflareSolver(antiBot) || !hasCaptchaProxy()) {
|
||||
return;
|
||||
}
|
||||
|
||||
const cfState = await detectCloudflare(page, antiBot);
|
||||
const needsProxy = cfState.blocked || urlIndicatesCfChallenge(page.url());
|
||||
if (!needsProxy) {
|
||||
console.log('[Proxy] 未检测到 CF 挑战，Chrome 直连（不使用 CAPTCHA_PROXY）');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('[Proxy] 检测到 CF 挑战，切换 Browser 为 CAPTCHA_PROXY（与 CapSolver 同源 IP）');
|
||||
|
||||
const snapshotCookies = await page.cookies().catch(() => []);
|
||||
await closeTaskPage(page);
|
||||
await browserSession.release(true);
|
||||
|
||||
const extraArgs = buildBrowserExtraArgs(antiBot, baseExtraArgs, true);
|
||||
browserSession = await acquireBrowserSession({
|
||||
profile,
|
||||
extraArgs,
|
||||
sessionKey: antiBot.sessionKey || '',
|
||||
});
|
||||
browserSession.captchaProxyActive = true;
|
||||
|
||||
page = await createTaskPage(
|
||||
browserSession.browser,
|
||||
browserSession.reused ? null : browserSession.page,
|
||||
navigationUrl,
|
||||
antiBot,
|
||||
mergedCookies.length > 0 ? mergedCookies : snapshotCookies,
|
||||
{ ...pageExtraOpts, useCaptchaProxy: true }
|
||||
);
|
||||
|
||||
if (antiBot?.enabled) {
|
||||
attachSitekeyCapture(page, browserSession.browser);
|
||||
}
|
||||
|
||||
await prepareWhatshubShortLinkBeforeNav(page, navigationUrl, antiBot);
|
||||
await navigateToPage(page, navigationUrl);
|
||||
await waitForUrlSettled(page).catch(() => {});
|
||||
|
||||
if (typeof browserCtx.afterRelaunchNav === 'function') {
|
||||
await browserCtx.afterRelaunchNav(page);
|
||||
}
|
||||
|
||||
browserCtx.browserSession = browserSession;
|
||||
browserCtx.page = page;
|
||||
}
|
||||
|
||||
/** 任务结束关闭本次 Tab（温池复用 Browser 时保留 Browser 实例） */
|
||||
async function closeTaskPage(page) {
|
||||
if (!page) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (typeof page.isClosed === 'function' && !page.isClosed()) {
|
||||
await page.close();
|
||||
}
|
||||
} catch (_) {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 任务页创建（含 Whatshub skipUserAgent）
|
||||
* 温池复用 Browser 时每次 newPage，冷启动 Real Browser 可使用 launch 返回的首个 Tab
|
||||
*/
|
||||
async function createTaskPage(browser, warmPage, pageUrl, antiBot, mergedCookies, extraOpts = {}) {
|
||||
const opts = buildManagedPageOptions(pageUrl, antiBot, mergedCookies, extraOpts);
|
||||
const page = existingPage || await browser.newPage();
|
||||
const useWarmPage = !!(warmPage && extraOpts.useWarmPage !== false && !extraOpts.forceNewPage);
|
||||
const page = useWarmPage ? warmPage : await browser.newPage();
|
||||
page.setDefaultNavigationTimeout(NAVIGATION_TIMEOUT_MS);
|
||||
page.setDefaultTimeout(PAGE_DEFAULT_TIMEOUT_MS);
|
||||
|
||||
@@ -384,9 +576,17 @@ async function createTaskPage(browser, existingPage, pageUrl, antiBot, mergedCoo
|
||||
await page.setCookie(...opts.cookies);
|
||||
}
|
||||
|
||||
if (extraOpts.useCaptchaProxy && hasCaptchaProxy()) {
|
||||
await applyCaptchaProxyToPage(page);
|
||||
}
|
||||
|
||||
page.on('error', (err) => console.error('[Page Error]', err.message));
|
||||
page.on('pageerror', (err) => console.error('[Page JS Error]', err.message));
|
||||
|
||||
if (antiBot?.enabled) {
|
||||
attachSitekeyCapture(page, browser);
|
||||
}
|
||||
|
||||
return page;
|
||||
}
|
||||
|
||||
@@ -419,25 +619,32 @@ function isOnShortLinkNotBusiness(page, urlNeedle) {
|
||||
async function waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettledFn, pageUrl = '') {
|
||||
const urlNeedle = (antiBot.postCfReadyUrlContains || '').trim();
|
||||
const targetUrl = pageUrl || page.url();
|
||||
if (!urlNeedle || !isOnShortLinkNotBusiness(page, urlNeedle)) {
|
||||
const onA2cDirect = isA2cDirectSharePage(page, urlNeedle);
|
||||
if (!urlNeedle || (!isOnShortLinkNotBusiness(page, urlNeedle) && !onA2cDirect)) {
|
||||
return isBusinessUrlReady(page, urlNeedle);
|
||||
}
|
||||
if (!(await isCfTrulyComplete(page, antiBot))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const spaWaitMs = Math.max(8000, antiBot.postCfSpaWaitMs || 8000);
|
||||
const spaWaitMs = onA2cDirect
|
||||
? Math.max(8000, antiBot.postCfSpaWaitMs || 8000)
|
||||
: Math.max(8000, antiBot.postCfSpaWaitMs || 8000);
|
||||
const isWhatshub = isWhatshubShortLinkPath(targetUrl);
|
||||
|
||||
console.log(
|
||||
`[CF] CF 已完成，等待 SPA 跳转 settle=${spaWaitMs}ms current=${page.url()}`
|
||||
`[CF] CF 已完成，等待 SPA ${onA2cDirect ? '直链挂载' : '跳转'} settle=${spaWaitMs}ms current=${page.url()}`
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, spaWaitMs));
|
||||
|
||||
if (typeof waitForUrlSettledFn === 'function') {
|
||||
await waitForUrlSettledFn(page).catch(() => {});
|
||||
}
|
||||
await page.waitForNetworkIdle({ idleTime: 500, timeout: 15000 }).catch(() => {});
|
||||
await page.waitForNetworkIdle({ idleTime: 500, timeout: 20000 }).catch(() => {});
|
||||
|
||||
if (onA2cDirect) {
|
||||
await waitForPostCfListApi(page, antiBot, Math.max(15000, antiBot.postCfReadyTimeoutMs || 60000)).catch(() => {});
|
||||
}
|
||||
|
||||
if (isBusinessUrlReady(page, urlNeedle)) {
|
||||
console.log(`[CF] SPA 已跳转至业务页 url=${page.url()}`);
|
||||
@@ -503,13 +710,18 @@ async function isCfTrulyComplete(page, antiBot = null) {
|
||||
return !state.blocked;
|
||||
}
|
||||
|
||||
/** 轮询直到 CF 真正完成（不依赖 turnstile-handler 的 early return） */
|
||||
/** 轮询直到 CF 真正完成；并行扫描 sitekey 供 CapSolver 提前使用 */
|
||||
async function waitForCfTrulyComplete(page, timeoutMs, pollMs = 500, antiBot = null) {
|
||||
const { getSitekeyCapture } = require('./lib/sitekey-capture');
|
||||
const capture = getSitekeyCapture(page);
|
||||
const started = Date.now();
|
||||
while (Date.now() - started < timeoutMs) {
|
||||
if (await isCfTrulyComplete(page, antiBot)) {
|
||||
return { success: true, elapsedMs: Date.now() - started };
|
||||
}
|
||||
if (capture) {
|
||||
await capture.scanNow(page, antiBot).catch(() => {});
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, pollMs));
|
||||
}
|
||||
return { success: false, elapsedMs: Date.now() - started };
|
||||
@@ -566,6 +778,62 @@ async function runCfChallengeFlow(page, antiBot, waitForUrlSettled, pageUrl = ''
|
||||
console.log(`[CF] 免 clearance 模式但检测到真实 CF 挑战 type=${preState.challengeType}，继续过盾`);
|
||||
}
|
||||
|
||||
const preferManaged = shouldPreferManagedCloudflareSolver(antiBot);
|
||||
|
||||
if (preferManaged && antiBot.solverFallback !== false && isCaptchaConfigured()) {
|
||||
console.log('[CF] Managed Challenge：跳过内置 60s 轮询，直接使用 CapSolver AntiCloudflareTask');
|
||||
try {
|
||||
if (waitForUrlSettled) {
|
||||
await waitForUrlSettled(page).catch(() => {});
|
||||
}
|
||||
await runCaptchaApiFallback(page, {
|
||||
antiBot,
|
||||
navUrl,
|
||||
waitForUrlSettled,
|
||||
timeoutMs,
|
||||
});
|
||||
|
||||
const apiWait = await waitForCfTrulyComplete(page, Math.min(timeoutMs, 60000), 500, antiBot);
|
||||
if (apiWait.success) {
|
||||
if (waitForUrlSettled) {
|
||||
await waitForUrlSettled(page).catch(() => {});
|
||||
}
|
||||
await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettled, navUrl);
|
||||
return {
|
||||
success: true,
|
||||
code: null,
|
||||
challengeType: 'managed',
|
||||
stage: 'api_managed',
|
||||
solverUsed: true,
|
||||
elapsedMs: Date.now() - started,
|
||||
cfDetected: true,
|
||||
};
|
||||
}
|
||||
console.warn('[CF] Managed CapSolver 注入后仍未获得有效 cf_clearance');
|
||||
} catch (apiErr) {
|
||||
console.error('[CF] Managed Challenge CapSolver 失败:', apiErr.message);
|
||||
return {
|
||||
success: false,
|
||||
code: 'CF_TURNSTILE_FAILED',
|
||||
challengeType: 'managed',
|
||||
stage: 'api_managed',
|
||||
solverUsed: true,
|
||||
elapsedMs: Date.now() - started,
|
||||
cfDetected: true,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
code: 'CF_TURNSTILE_FAILED',
|
||||
challengeType: 'managed',
|
||||
stage: 'api_managed',
|
||||
solverUsed: true,
|
||||
elapsedMs: Date.now() - started,
|
||||
cfDetected: true,
|
||||
};
|
||||
}
|
||||
|
||||
if (antiBot.turnstile !== false) {
|
||||
console.log(`[CF] 内置 Turnstile 轮询 timeout=${timeoutMs}ms`);
|
||||
const builtIn = await waitForCfTrulyComplete(page, timeoutMs, 500, antiBot);
|
||||
@@ -592,15 +860,12 @@ async function runCfChallengeFlow(page, antiBot, waitForUrlSettled, pageUrl = ''
|
||||
if (waitForUrlSettled) {
|
||||
await waitForUrlSettled(page).catch(() => {});
|
||||
}
|
||||
let sitekey = await waitForTurnstileSurface(page, Math.min(timeoutMs, 25000));
|
||||
if (!sitekey) {
|
||||
sitekey = await extractTurnstileSitekey(page);
|
||||
}
|
||||
const pageUrlForSolver = page.url();
|
||||
console.log(`[CF] Captcha API 求解 pageUrl=${pageUrlForSolver} sitekey=${sitekey ? 'ok' : 'missing'}`);
|
||||
const { token, provider } = await solveTurnstile({ pageUrl: pageUrlForSolver, sitekey });
|
||||
console.log(`[CF] Captcha API (${provider}) 返回 token，注入页面`);
|
||||
await injectTurnstileToken(page, token);
|
||||
await runCaptchaApiFallback(page, {
|
||||
antiBot,
|
||||
navUrl,
|
||||
waitForUrlSettled,
|
||||
timeoutMs,
|
||||
});
|
||||
|
||||
const apiWait = await waitForCfTrulyComplete(page, Math.min(timeoutMs, 45000), 500, antiBot);
|
||||
if (apiWait.success) {
|
||||
@@ -744,10 +1009,19 @@ async function prepareWhatshubShortLinkBeforeNav(page, pageUrl, antiBot) {
|
||||
/**
|
||||
* CF 处理 + 业务 URL 守卫：磁盘 session 陈旧 clearance 时 purge；__cf_chl 中间态只过盾不 purge
|
||||
*/
|
||||
async function runCloudflareWithBusinessGuard(page, antiBot, waitForUrlSettled, storedSession, pageUrl) {
|
||||
async function runCloudflareWithBusinessGuard(page, antiBot, waitForUrlSettled, storedSession, pageUrl, browserCtx = null) {
|
||||
const urlNeedle = (antiBot.postCfReadyUrlContains || '').trim();
|
||||
const hasStoredSession = !!(storedSession && isSessionLikelyValid(storedSession));
|
||||
|
||||
async function maybeEnsureProxy(currentPage) {
|
||||
if (!browserCtx) {
|
||||
return currentPage;
|
||||
}
|
||||
browserCtx.page = currentPage;
|
||||
await ensureCaptchaProxyBrowserIfNeeded(browserCtx);
|
||||
return browserCtx.page;
|
||||
}
|
||||
|
||||
async function reNavigateWithCfPurge(label) {
|
||||
if (urlIndicatesCfChallenge(page.url())) {
|
||||
console.log(`[CF] ${label}: URL 含 __cf_chl，跳过 purge，直接过盾`);
|
||||
@@ -759,6 +1033,7 @@ async function runCloudflareWithBusinessGuard(page, antiBot, waitForUrlSettled,
|
||||
if (waitForUrlSettled) {
|
||||
await waitForUrlSettled(page).catch(() => {});
|
||||
}
|
||||
page = await maybeEnsureProxy(page);
|
||||
return handleCloudflareChallengeSafe(page, antiBot, waitForUrlSettled, storedSession);
|
||||
}
|
||||
|
||||
@@ -773,6 +1048,7 @@ async function runCloudflareWithBusinessGuard(page, antiBot, waitForUrlSettled,
|
||||
}
|
||||
|
||||
if (!cfResult) {
|
||||
page = await maybeEnsureProxy(page);
|
||||
cfResult = await handleCloudflareChallengeSafe(page, antiBot, waitForUrlSettled, storedSession);
|
||||
}
|
||||
|
||||
@@ -781,6 +1057,7 @@ async function runCloudflareWithBusinessGuard(page, antiBot, waitForUrlSettled,
|
||||
const crossState = await detectCloudflare(page, antiBot);
|
||||
if (crossState.blocked || urlIndicatesCfChallenge(page.url())) {
|
||||
console.log(`[CF] 跨域落地后 CF 未完成，二次过盾 url=${page.url()}`);
|
||||
page = await maybeEnsureProxy(page);
|
||||
cfResult = await finalizeCfResult(
|
||||
page,
|
||||
await runCfChallengeFlow(page, antiBot, waitForUrlSettled, page.url()),
|
||||
@@ -803,6 +1080,7 @@ async function runCloudflareWithBusinessGuard(page, antiBot, waitForUrlSettled,
|
||||
retries += 1;
|
||||
if (!(await isCfTrulyComplete(page, antiBot))) {
|
||||
console.log(`[CF] 业务页未达且 CF 未完成，完整过盾 重试#${retries}`);
|
||||
page = await maybeEnsureProxy(page);
|
||||
cfResult = await finalizeCfResult(
|
||||
page,
|
||||
await runCfChallengeFlow(page, antiBot, waitForUrlSettled, pageUrl),
|
||||
@@ -823,6 +1101,10 @@ async function runCloudflareWithBusinessGuard(page, antiBot, waitForUrlSettled,
|
||||
}
|
||||
}
|
||||
|
||||
if (browserCtx) {
|
||||
browserCtx.page = page;
|
||||
}
|
||||
|
||||
return finalizeCfResult(page, cfResult, antiBot);
|
||||
}
|
||||
|
||||
@@ -833,6 +1115,9 @@ async function collectPageDiagnostics(page) {
|
||||
title: document.title || '',
|
||||
bodyText: (document.body && document.body.innerText ? document.body.innerText : '').slice(0, 500),
|
||||
elInputInner: document.querySelectorAll('.el-input__inner').length,
|
||||
elTable: document.querySelectorAll('.el-table').length,
|
||||
btnNext: document.querySelectorAll('.btn-next').length,
|
||||
elPagination: document.querySelectorAll('.el-pagination').length,
|
||||
})).catch(() => null);
|
||||
return snapshot || { url, title: '', bodyText: '', elInputInner: 0 };
|
||||
}
|
||||
@@ -851,7 +1136,20 @@ async function awaitPostCfBusinessReady(page, antiBot, waitForUrlSettledFn, stor
|
||||
|
||||
await ensureCfCompleteBeforeBusinessReady(page, antiBot, waitForUrlSettledFn, storedSession);
|
||||
|
||||
await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettledFn, navUrl);
|
||||
const onA2cDirect = isA2cDirectSharePage(page, urlContains);
|
||||
let skipSpaWait = false;
|
||||
if (onA2cDirect && await isCfTrulyComplete(page, antiBot)) {
|
||||
const domReady = selector
|
||||
? await page.$(selector).catch(() => null)
|
||||
: true;
|
||||
if (domReady) {
|
||||
console.log('[CF] A2C 业务 DOM 已就绪，跳过重复 SPA 等待');
|
||||
skipSpaWait = true;
|
||||
}
|
||||
}
|
||||
if (!skipSpaWait) {
|
||||
await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettledFn, navUrl);
|
||||
}
|
||||
|
||||
const timeoutMs = Math.max(5000, antiBot.postCfReadyTimeoutMs || 30000);
|
||||
console.log(
|
||||
@@ -867,12 +1165,14 @@ async function awaitPostCfBusinessReady(page, antiBot, waitForUrlSettledFn, stor
|
||||
urlContains
|
||||
);
|
||||
}
|
||||
await waitForPostCfListApi(page, antiBot, timeoutMs);
|
||||
if (selector !== '') {
|
||||
await page.waitForSelector(selector, { timeout: timeoutMs, visible: true });
|
||||
await waitForPostCfSelector(page, selector, timeoutMs);
|
||||
}
|
||||
} catch (err) {
|
||||
const retryMs = Math.min(15000, timeoutMs);
|
||||
const retryMs = Math.min(30000, timeoutMs);
|
||||
const needsRetry = urlContains !== '' && !page.url().includes(urlContains);
|
||||
const needsDomRetry = urlContains !== '' && page.url().includes(urlContains) && isA2cDirectSharePage(page, urlContains);
|
||||
if (needsRetry) {
|
||||
if (!(await isCfTrulyComplete(page, antiBot))) {
|
||||
console.warn(`[CF] 业务页等待失败且 CF 未完成 url=${page.url()}，完整过盾后重试`);
|
||||
@@ -886,7 +1186,7 @@ async function awaitPostCfBusinessReady(page, antiBot, waitForUrlSettledFn, stor
|
||||
);
|
||||
}
|
||||
if (selector !== '') {
|
||||
await page.waitForSelector(selector, { timeout: retryMs, visible: true });
|
||||
await waitForPostCfSelector(page, selector, retryMs);
|
||||
}
|
||||
} catch (retryErr) {
|
||||
retryErr.pageDiagnostics = await collectPageDiagnostics(page);
|
||||
@@ -906,14 +1206,28 @@ async function awaitPostCfBusinessReady(page, antiBot, waitForUrlSettledFn, stor
|
||||
urlContains
|
||||
);
|
||||
}
|
||||
await waitForPostCfListApi(page, antiBot, retryMs);
|
||||
if (selector !== '') {
|
||||
await page.waitForSelector(selector, { timeout: retryMs, visible: true });
|
||||
await waitForPostCfSelector(page, selector, retryMs);
|
||||
}
|
||||
} catch (retryErr) {
|
||||
retryErr.pageDiagnostics = await collectPageDiagnostics(page);
|
||||
throw retryErr;
|
||||
}
|
||||
}
|
||||
} else if (needsDomRetry) {
|
||||
console.warn(`[CF] A2C URL 已就绪但 DOM 未挂载 url=${page.url()}，soft reload 后重试`);
|
||||
try {
|
||||
await page.waitForNetworkIdle({ idleTime: 500, timeout: 20000 }).catch(() => {});
|
||||
await softReloadForSpa(page, waitForUrlSettledFn, antiBot, navUrl);
|
||||
await waitForPostCfListApi(page, antiBot, retryMs);
|
||||
if (selector !== '') {
|
||||
await waitForPostCfSelector(page, selector, retryMs);
|
||||
}
|
||||
} catch (retryErr) {
|
||||
retryErr.pageDiagnostics = await collectPageDiagnostics(page);
|
||||
throw retryErr;
|
||||
}
|
||||
} else {
|
||||
err.pageDiagnostics = await collectPageDiagnostics(page);
|
||||
throw err;
|
||||
@@ -1230,6 +1544,7 @@ app.post('/api/auth-and-intercept', async (req, res) => {
|
||||
await runBrowserTask('auth-and-intercept', async () => {
|
||||
let browserSession = null;
|
||||
let interceptorCleanup = null;
|
||||
let page = null;
|
||||
const taskStarted = Date.now();
|
||||
let cfDestroyBrowser = false;
|
||||
|
||||
@@ -1242,11 +1557,12 @@ app.post('/api/auth-and-intercept', async (req, res) => {
|
||||
cfLog.browserReused = !!browserSession.reused;
|
||||
|
||||
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl);
|
||||
const navigationUrl = resolveNavigationUrl(pageUrl, antiBot);
|
||||
|
||||
const page = await createTaskPage(
|
||||
page = await createTaskPage(
|
||||
browserSession.browser,
|
||||
browserSession.page,
|
||||
pageUrl,
|
||||
browserSession.reused ? null : browserSession.page,
|
||||
navigationUrl,
|
||||
antiBot,
|
||||
mergedCookies
|
||||
);
|
||||
@@ -1257,13 +1573,13 @@ app.post('/api/auth-and-intercept', async (req, res) => {
|
||||
}
|
||||
|
||||
let shareToken = resolveShareToken(pageUrl, httpResolvedUrl);
|
||||
await seedShareTokenCookie(page, pageUrl);
|
||||
await seedShareTokenCookie(page, navigationUrl);
|
||||
if (httpResolvedUrl !== pageUrl) {
|
||||
await seedShareTokenCookie(page, httpResolvedUrl);
|
||||
}
|
||||
|
||||
await prepareWhatshubShortLinkBeforeNav(page, pageUrl, antiBot);
|
||||
await navigateToPage(page, pageUrl);
|
||||
await prepareWhatshubShortLinkBeforeNav(page, navigationUrl, antiBot);
|
||||
await navigateToPage(page, navigationUrl);
|
||||
|
||||
let finalPageUrl = await waitForUrlSettled(page);
|
||||
applyHuojianBusinessHostHint(antiBot, finalPageUrl);
|
||||
@@ -1271,11 +1587,30 @@ app.post('/api/auth-and-intercept', async (req, res) => {
|
||||
console.log(`[URL解析] 浏览器落地: ${pageUrl} -> ${finalPageUrl}`);
|
||||
}
|
||||
|
||||
const browserCtx = {
|
||||
browserSession,
|
||||
page,
|
||||
antiBot,
|
||||
profile,
|
||||
navigationUrl,
|
||||
mergedCookies,
|
||||
baseExtraArgs: ['--mute-audio'],
|
||||
pageExtraOpts: {},
|
||||
afterRelaunchNav: async (p) => {
|
||||
await seedShareTokenCookie(p, navigationUrl);
|
||||
},
|
||||
};
|
||||
await ensureCaptchaProxyBrowserIfNeeded(browserCtx);
|
||||
browserSession = browserCtx.browserSession;
|
||||
page = browserCtx.page;
|
||||
|
||||
let interceptor;
|
||||
if (antiBot.enabled) {
|
||||
const cfResult = await runCloudflareWithBusinessGuard(
|
||||
page, antiBot, waitForUrlSettled, storedSession, pageUrl
|
||||
page, antiBot, waitForUrlSettled, storedSession, pageUrl, browserCtx
|
||||
);
|
||||
page = browserCtx.page;
|
||||
browserSession = browserCtx.browserSession;
|
||||
cfLog.cfDetected = cfResult.cfDetected;
|
||||
cfLog.turnstileStage = cfResult.stage;
|
||||
cfLog.solverUsed = cfResult.solverUsed;
|
||||
@@ -1346,6 +1681,7 @@ app.post('/api/auth-and-intercept', async (req, res) => {
|
||||
});
|
||||
} finally {
|
||||
if (interceptorCleanup) interceptorCleanup();
|
||||
await closeTaskPage(page);
|
||||
if (browserSession) {
|
||||
await browserSession.release(cfDestroyBrowser);
|
||||
}
|
||||
@@ -1495,6 +1831,7 @@ app.post('/api/ui-pagination', async (req, res) => {
|
||||
await runBrowserTask('ui-pagination', async () => {
|
||||
let browserSession = null;
|
||||
const debugLogs = [];
|
||||
let page = null;
|
||||
let cfDestroyBrowser = false;
|
||||
|
||||
try {
|
||||
@@ -1506,9 +1843,9 @@ app.post('/api/ui-pagination', async (req, res) => {
|
||||
|
||||
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, navigationUrl, cookies || []);
|
||||
|
||||
const page = await createTaskPage(
|
||||
page = await createTaskPage(
|
||||
browserSession.browser,
|
||||
browserSession.page,
|
||||
browserSession.reused ? null : browserSession.page,
|
||||
navigationUrl,
|
||||
antiBot,
|
||||
mergedCookies,
|
||||
@@ -1532,10 +1869,29 @@ app.post('/api/ui-pagination', async (req, res) => {
|
||||
log(`[URL解析] 翻页阶段落地: ${navigationUrl} -> ${settledPageUrl}`);
|
||||
}
|
||||
|
||||
const browserCtx = {
|
||||
browserSession,
|
||||
page,
|
||||
antiBot,
|
||||
profile,
|
||||
navigationUrl,
|
||||
mergedCookies,
|
||||
baseExtraArgs: ['--window-size=1920,1080'],
|
||||
pageExtraOpts: { viewport: { width: 1920, height: 1080 } },
|
||||
afterRelaunchNav: async (p) => {
|
||||
await seedShareTokenCookie(p, navigationUrl);
|
||||
},
|
||||
};
|
||||
await ensureCaptchaProxyBrowserIfNeeded(browserCtx);
|
||||
browserSession = browserCtx.browserSession;
|
||||
page = browserCtx.page;
|
||||
|
||||
if (antiBot.enabled) {
|
||||
const cfResult = await runCloudflareWithBusinessGuard(
|
||||
page, antiBot, waitForUrlSettled, storedSession, navigationUrl
|
||||
page, antiBot, waitForUrlSettled, storedSession, navigationUrl, browserCtx
|
||||
);
|
||||
page = browserCtx.page;
|
||||
browserSession = browserCtx.browserSession;
|
||||
if (!cfResult.success) {
|
||||
cfDestroyBrowser = true;
|
||||
res.status(422).json({
|
||||
@@ -1578,6 +1934,7 @@ app.post('/api/ui-pagination', async (req, res) => {
|
||||
sendTaskError(res, error, { debug: debugLogs, profile });
|
||||
}
|
||||
} finally {
|
||||
await closeTaskPage(page);
|
||||
if (browserSession) {
|
||||
await browserSession.release(cfDestroyBrowser);
|
||||
}
|
||||
@@ -1619,6 +1976,7 @@ app.post('/api/auth-intercept-and-paginate', async (req, res) => {
|
||||
await runBrowserTask('auth-intercept-and-paginate', async () => {
|
||||
let browserSession = null;
|
||||
let interceptorCleanup = null;
|
||||
let page = null;
|
||||
const taskStarted = Date.now();
|
||||
let cfDestroyBrowser = false;
|
||||
|
||||
@@ -1631,11 +1989,12 @@ app.post('/api/auth-intercept-and-paginate', async (req, res) => {
|
||||
cfLog.browserReused = !!browserSession.reused;
|
||||
|
||||
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl);
|
||||
const navigationUrl = resolveNavigationUrl(pageUrl, antiBot);
|
||||
|
||||
const page = await createTaskPage(
|
||||
page = await createTaskPage(
|
||||
browserSession.browser,
|
||||
browserSession.page,
|
||||
pageUrl,
|
||||
browserSession.reused ? null : browserSession.page,
|
||||
navigationUrl,
|
||||
antiBot,
|
||||
mergedCookies,
|
||||
{ viewport: { width: 1920, height: 1080 } }
|
||||
@@ -1644,18 +2003,37 @@ app.post('/api/auth-intercept-and-paginate', async (req, res) => {
|
||||
const httpResolvedUrl = await resolveHttpRedirects(pageUrl).catch(() => pageUrl);
|
||||
|
||||
let shareToken = resolveShareToken(pageUrl, httpResolvedUrl);
|
||||
await seedShareTokenCookie(page, pageUrl);
|
||||
await seedShareTokenCookie(page, navigationUrl);
|
||||
|
||||
await prepareWhatshubShortLinkBeforeNav(page, pageUrl, antiBot);
|
||||
await navigateToPage(page, pageUrl);
|
||||
await prepareWhatshubShortLinkBeforeNav(page, navigationUrl, antiBot);
|
||||
await navigateToPage(page, navigationUrl);
|
||||
let finalPageUrl = await waitForUrlSettled(page);
|
||||
applyHuojianBusinessHostHint(antiBot, finalPageUrl);
|
||||
|
||||
const browserCtx = {
|
||||
browserSession,
|
||||
page,
|
||||
antiBot,
|
||||
profile,
|
||||
navigationUrl,
|
||||
mergedCookies,
|
||||
baseExtraArgs: ['--mute-audio', '--window-size=1920,1080'],
|
||||
pageExtraOpts: { viewport: { width: 1920, height: 1080 } },
|
||||
afterRelaunchNav: async (p) => {
|
||||
await seedShareTokenCookie(p, navigationUrl);
|
||||
},
|
||||
};
|
||||
await ensureCaptchaProxyBrowserIfNeeded(browserCtx);
|
||||
browserSession = browserCtx.browserSession;
|
||||
page = browserCtx.page;
|
||||
|
||||
let interceptor;
|
||||
if (antiBot.enabled) {
|
||||
const cfResult = await runCloudflareWithBusinessGuard(
|
||||
page, antiBot, waitForUrlSettled, storedSession, pageUrl
|
||||
page, antiBot, waitForUrlSettled, storedSession, navigationUrl, browserCtx
|
||||
);
|
||||
page = browserCtx.page;
|
||||
browserSession = browserCtx.browserSession;
|
||||
cfLog.cfDetected = cfResult.cfDetected;
|
||||
cfLog.turnstileStage = cfResult.stage;
|
||||
cfLog.solverUsed = cfResult.solverUsed;
|
||||
@@ -1677,7 +2055,7 @@ app.post('/api/auth-intercept-and-paginate', async (req, res) => {
|
||||
applyHuojianBusinessHostHint(antiBot, finalPageUrl);
|
||||
touchSessionIfPresent(antiBot.sessionKey);
|
||||
|
||||
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled, storedSession, pageUrl);
|
||||
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled, storedSession, navigationUrl);
|
||||
}
|
||||
|
||||
interceptor = createApiInterceptor(page, apiUrls, interceptTimeout);
|
||||
@@ -1744,6 +2122,7 @@ app.post('/api/auth-intercept-and-paginate', async (req, res) => {
|
||||
});
|
||||
} finally {
|
||||
if (interceptorCleanup) interceptorCleanup();
|
||||
await closeTaskPage(page);
|
||||
if (browserSession) {
|
||||
await browserSession.release(cfDestroyBrowser);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user