damon/links
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修改whatshub工单爬虫

Browse Source
This commit is contained in:
root
2026-06-29 04:54:41 +08:00
parent 6130125427
commit 9f2e904fab
14 changed files with 1254 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
patches/application/common/library/scrm/AntiBotConfigBuilder.php
+20
View File
@@ -41,6 +41,26 @@ class AntiBotConfigBuilder
'sessionKey' => $sessionKey,
'challengeTimeoutMs' => 60000,
'sessionTtlMinutes' => $ttlMinutes,
] + self::postCfReadyExtras($ticketType);
}
/**
* 按工单类型附加 CF 过盾后的业务页就绪等待（仅 Whatshub 等需要的类型）
*
* @return array<string, mixed>
*/
private static function postCfReadyExtras(string $ticketType): array
{
if ($ticketType !== 'whatshub') {
return [];
}
return [
'postCfReadyUrlContains' => 'work-order-sharing',
'postCfReadySelector' => '.el-input__inner',
'postCfReadyTimeoutMs' => 30000,
'postCfSettleMs' => 500,
'postCfSpaWaitMs' => 8000,
];
}
patches/application/common/library/scrm/spider/WhatshubSpider.php
+180 -2
View File
@@ -7,11 +7,12 @@ namespace app\common\library\scrm\spider;
use app\common\library\scrm\AbstractScrmSpider;
use app\common\library\scrm\AntiBotConfigBuilder;
use app\common\library\scrm\UnifiedScrmData;
use app\common\service\SplitTicketSyncLogger;
/**
* Whatshub 云控蜘蛛（Real Browser + Turnstile 过盾 + UI 翻页）
* Whatshub 云控蜘蛛
*
* 时序：Cloudflare Turnstile → 密码弹窗 authActions → API 拦截 → UI 翻页
* 时序：detailByShareCode API 直抓（优先）→ 失败则 Real Browser + Turnstile + UI 翻页
*/
class WhatshubSpider extends AbstractScrmSpider
{
@@ -21,9 +22,14 @@ class WhatshubSpider extends AbstractScrmSpider
/** 详情/统计 API 路径 */
private const API_DETAILS = '/api/whatshub-counter/workShare/open/statistics';
/** shareCode + password 直连接口（一次返回全量号码，无需翻页） */
private const API_DETAIL_BY_SHARE_CODE = '/api/whatshub-counter/workShare/open/detailByShareCode';
/** 默认每页条数 */
private const DEFAULT_PER_PAGE_COUNT = 20;
private const API_FETCH_TIMEOUT_SEC = 20;
private string $pageUrl;
private string $account;
@@ -45,6 +51,178 @@ class WhatshubSpider extends AbstractScrmSpider
$this->unifiedData = new UnifiedScrmData();
}
/**
* API 优先：detailByShareCode 直抓；失败则回退 Node 爬虫
*/
public function run(): UnifiedScrmData
{
$apiResult = $this->tryFetchViaShareCodeApi();
if ($apiResult instanceof UnifiedScrmData) {
SplitTicketSyncLogger::log('spider', 'whatshub api path ok', [
'count' => count($apiResult->numbers),
]);
return $apiResult;
}
SplitTicketSyncLogger::log('spider', 'whatshub api skipped or failed, fallback node', [
'pageUrl' => $this->pageUrl,
]);
return parent::run();
}
/**
* 从短链 pageUrl 解析 shareCode，例如 /m/KMYBBInp2066/1 → KMYBBInp2066
*/
private function extractShareCodeFromPageUrl(): ?string
{
$path = (string) parse_url($this->pageUrl, PHP_URL_PATH);
if ($path === '') {
return null;
}
if (preg_match('#/m/([^/]+)/#', $path, $matches)) {
return $matches[1];
}
return null;
}
/**
* 拼接 detailByShareCode 完整 GET URL
*/
private function buildShareCodeApiUrl(string $shareCode): ?string
{
$parsed = parse_url($this->pageUrl);
$scheme = isset($parsed['scheme']) ? (string) $parsed['scheme'] : 'https';
$host = isset($parsed['host']) ? (string) $parsed['host'] : '';
if ($host === '') {
return null;
}
$query = http_build_query([
'shareCode' => $shareCode,
'password' => $this->password,
]);
return $scheme . '://' . $host . self::API_DETAIL_BY_SHARE_CODE . '?' . $query;
}
/**
* 尝试通过 shareCode API 抓取；成功返回 UnifiedScrmData，否则返回 null
*/
private function tryFetchViaShareCodeApi(): ?UnifiedScrmData
{
if ($this->password === '') {
SplitTicketSyncLogger::log('spider', 'whatshub api skip: empty password', []);
return null;
}
$shareCode = $this->extractShareCodeFromPageUrl();
if ($shareCode === null || $shareCode === '') {
SplitTicketSyncLogger::log('spider', 'whatshub api skip: cannot parse shareCode', [
'pageUrl' => $this->pageUrl,
]);
return null;
}
$apiUrl = $this->buildShareCodeApiUrl($shareCode);
if ($apiUrl === null) {
SplitTicketSyncLogger::log('spider', 'whatshub api skip: invalid pageUrl host', [
'pageUrl' => $this->pageUrl,
]);
return null;
}
SplitTicketSyncLogger::log('spider', 'whatshub api request', [
'url' => preg_replace('/password=[^&]+/', 'password=***', $apiUrl),
]);
$ch = curl_init($apiUrl);
if ($ch === false) {
SplitTicketSyncLogger::log('spider', 'whatshub api curl init failed', []);
return null;
}
curl_setopt_array($ch, [
CURLOPT_RETURNTRANSFER => true,
CURLOPT_CONNECTTIMEOUT => 5,
CURLOPT_TIMEOUT => self::API_FETCH_TIMEOUT_SEC,
CURLOPT_HTTPHEADER => [
'Accept: application/json',
'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
],
]);
$response = curl_exec($ch);
$httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
$curlErr = curl_error($ch);
curl_close($ch);
if ($response === false || $curlErr !== '') {
SplitTicketSyncLogger::log('spider', 'whatshub api curl failed', [
'error' => $curlErr,
]);
return null;
}
if ($httpCode !== 200) {
SplitTicketSyncLogger::log('spider', 'whatshub api http error', [
'httpCode' => $httpCode,
]);
return null;
}
$payload = json_decode((string) $response, true);
if (!is_array($payload)) {
SplitTicketSyncLogger::log('spider', 'whatshub api invalid json', []);
return null;
}
if ((int) ($payload['code'] ?? 0) !== 200) {
SplitTicketSyncLogger::log('spider', 'whatshub api business error', [
'code' => $payload['code'] ?? null,
'msg' => $payload['msg'] ?? '',
]);
return null;
}
if (!isset($payload['data']) || !is_array($payload['data'])) {
SplitTicketSyncLogger::log('spider', 'whatshub api invalid data field', []);
return null;
}
return $this->parseShareCodeApiResponse($payload);
}
/**
* 将 detailByShareCode 响应映射为 UnifiedScrmData
*
* @param array<string, mixed> $payload
*/
private function parseShareCodeApiResponse(array $payload): UnifiedScrmData
{
$unifiedData = new UnifiedScrmData();
$todayNewSum = 0;
foreach ($payload['data'] as $item) {
if (!is_array($item) || empty($item['account'])) {
continue;
}
$number = (string) $item['account'];
$isOnline = isset($item['isOnline']) && (int) $item['isOnline'] === 1;
$dayNewFans = (int) ($item['dayNewFans'] ?? 0);
$unifiedData->addNumber($number, $isOnline, $dayNewFans);
$todayNewSum += $dayNewFans;
}
$unifiedData->todayNewCount = $todayNewSum;
$unifiedData->total = count($unifiedData->numbers);
return $unifiedData;
}
/** @return array<string, mixed> */
protected function getSpiderConfig(): array
{
patches/puppeteer-api/lib/browser-factory.js
+8 -2
View File
@@ -28,6 +28,10 @@ function normalizeAntiBot(antiBot) {
solverFallback: antiBot.solverFallback !== false,
sessionKey: antiBot.sessionKey || '',
challengeTimeoutMs: antiBot.challengeTimeoutMs || 60000,
postCfReadyUrlContains: antiBot.postCfReadyUrlContains || '',
postCfReadySelector: antiBot.postCfReadySelector || '',
postCfReadyTimeoutMs: antiBot.postCfReadyTimeoutMs || 0,
postCfSettleMs: antiBot.postCfSettleMs || 0,
};
}
@@ -106,8 +110,10 @@ async function createManagedPage(browser, existingPage, options = {}) {
page.setDefaultNavigationTimeout(NAVIGATION_TIMEOUT_MS);
page.setDefaultTimeout(PAGE_DEFAULT_TIMEOUT_MS);
const userAgent = options.userAgent || DEFAULT_UA;
await page.setUserAgent(userAgent);
if (!options.skipUserAgent) {
const userAgent = options.userAgent || DEFAULT_UA;
await page.setUserAgent(userAgent);
}
if (options.viewport) await page.setViewport(options.viewport);
if (options.cookies && options.cookies.length > 0) {
patches/puppeteer-api/lib/cf-detector.js
+18 -2
View File
@@ -2,6 +2,21 @@
* Cloudflare / Turnstile 挑战页检测
*/
/**
* URL 是否处于 CF Managed Challenge 中间态（Whatshub 短链常见 __cf_chl_rt_tk）
* @param {string} url
*/
function urlIndicatesCfChallenge(url) {
if (!url || typeof url !== 'string') {
return false;
}
return url.includes('__cf_chl_rt_tk')
|| url.includes('__cf_chl_tk')
|| url.includes('__cf_chl_f_tk')
|| url.includes('/cdn-cgi/challenge-platform')
|| url.includes('/cdn-cgi/challenge');
}
/**
* @typedef {Object} CloudflareState
* @property {boolean} blocked 是否处于 CF 挑战中
@@ -44,10 +59,10 @@ async function detectCloudflare(page) {
bodyHasJustAMoment: false,
}));
const urlChallenge = url.includes('/cdn-cgi/challenge-platform') || url.includes('/cdn-cgi/challenge');
const urlChallenge = urlIndicatesCfChallenge(url);
let challengeType = null;
if (domSignals.hasTurnstileInput || domSignals.hasTurnstileWidget) {
if (domSignals.hasTurnstileInput || domSignals.hasTurnstileWidget || url.includes('__cf_chl')) {
challengeType = 'turnstile';
} else if (urlChallenge || domSignals.titleHasJustAMoment || domSignals.bodyHasJustAMoment) {
challengeType = 'js_challenge';
@@ -94,4 +109,5 @@ async function isTurnstileSolved(page) {
module.exports = {
detectCloudflare,
isTurnstileSolved,
urlIndicatesCfChallenge,
};
patches/puppeteer-api/lib/cf-handler.js
+194 -22
View File
@@ -1,5 +1,8 @@
/**
* Cloudflare 挑战统一处理：检测 → 会话快速通道 → 内置等待 → Captcha API 兜底
*
* 当 antiBot.postCfReadyUrlContains 已配置时，session_reuse 仅在业务 URL 已到达时才走快速通道；
* 否则清除可能过期的 cf_clearance 并重新触发 Turnstile（Whatshub 短链 → work-order-sharing 场景）。
*/
const { detectCloudflare } = require('./cf-detector');
const { waitForTurnstile, extractTurnstileSitekey, injectTurnstileToken } = require('./turnstile-handler');
@@ -7,8 +10,135 @@ const { solveTurnstile, isCaptchaConfigured } = require('./captcha-solver');
const { isSessionLikelyValid } = require('./session-store');
/**
* 是否配置了 CF 后必须到达的业务 URL
* @param {object} antiBot
*/
function needsPostCfBusinessUrl(antiBot) {
return !!(antiBot?.postCfReadyUrlContains || '').trim();
}
/**
* 当前页面 URL 是否已包含业务路径片段
* @param {import('puppeteer').Page} page
* @param {{ turnstile?: boolean, solverFallback?: boolean, challengeTimeoutMs?: number }} antiBot
* @param {object} antiBot
*/
function isPostCfBusinessUrlReady(page, antiBot) {
const needle = (antiBot?.postCfReadyUrlContains || '').trim();
if (!needle) {
return true;
}
return page.url().includes(needle);
}
/**
* 清除 cf_clearance，避免「有 cookie 但 SPA 未触发跳转」的假过关
* @param {import('puppeteer').Page} page
*/
async function clearCfClearanceCookies(page) {
const cookies = await page.cookies();
for (const c of cookies) {
if (c.name !== 'cf_clearance') {
continue;
}
const host = (c.domain || '').replace(/^\./, '');
if (host) {
await page.deleteCookie({ name: c.name, url: `https://${host}/` }).catch(() => {});
}
await page.deleteCookie({
name: c.name,
domain: c.domain,
path: c.path || '/',
}).catch(() => {});
}
}
/**
* reload 后检查业务 URL 是否就绪
* @param {import('puppeteer').Page} page
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
* @param {number} timeoutMs
* @param {object} antiBot
*/
async function tryPostCfRecoverViaReload(page, waitForUrlSettled, timeoutMs, antiBot) {
if (!needsPostCfBusinessUrl(antiBot) || isPostCfBusinessUrlReady(page, antiBot)) {
return isPostCfBusinessUrlReady(page, antiBot);
}
console.log(`[CF] 业务 URL 未就绪 current=${page.url()}，尝试 reload`);
try {
await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 45000) });
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
return isPostCfBusinessUrlReady(page, antiBot);
} catch (reloadErr) {
console.warn('[CF] reload 失败:', reloadErr.message);
return false;
}
}
/**
* 清除 cf_clearance 并 reload，使 Turnstile 重新出现
* @param {import('puppeteer').Page} page
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
* @param {number} timeoutMs
*/
async function invalidateStaleCfAndReload(page, waitForUrlSettled, timeoutMs) {
console.log('[CF] 清除 cf_clearance 并 reload，重新触发 Turnstile');
await clearCfClearanceCookies(page);
await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 45000) });
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
return detectCloudflare(page);
}
/**
* session_reuse 快速返回前校验：有 postCf 要求时必须已到业务页
* @param {import('puppeteer').Page} page
* @param {object} antiBot
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
* @param {number} timeoutMs
* @param {number} started
* @param {boolean} cfDetected
*/
async function trySessionReuseFastPath(page, antiBot, waitForUrlSettled, timeoutMs, started, cfDetected) {
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
if (!needsPostCfBusinessUrl(antiBot) || isPostCfBusinessUrlReady(page, antiBot)) {
return {
success: true,
code: null,
challengeType: null,
stage: 'session_reuse',
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected,
};
}
console.log(`[CF] cf_clearance/无挑战但业务页未就绪 url=${page.url()}，降级重试`);
if (await tryPostCfRecoverViaReload(page, waitForUrlSettled, timeoutMs, antiBot)) {
return {
success: true,
code: null,
challengeType: null,
stage: 'session_reuse_reload',
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected,
};
}
return null;
}
/**
* @param {import('puppeteer').Page} page
* @param {{ turnstile?: boolean, solverFallback?: boolean, challengeTimeoutMs?: number, postCfReadyUrlContains?: string }} antiBot
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
* @param {import('./session-store').StoredSession|null} [storedSession]
*/
@@ -19,36 +149,71 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
let cfState = await detectCloudflare(page);
if (!cfState.blocked && cfState.hasCfClearance) {
return {
success: true,
code: null,
challengeType: null,
stage: 'session_reuse',
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: false,
};
const fast = await trySessionReuseFastPath(
page, antiBot, waitForUrlSettled, timeoutMs, started, false
);
if (fast) {
return fast;
}
cfState = await invalidateStaleCfAndReload(page, waitForUrlSettled, timeoutMs);
}
if (!cfState.blocked) {
return {
success: true,
code: null,
challengeType: null,
stage: storedSession && isSessionLikelyValid(storedSession) ? 'session_reuse' : null,
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: false,
};
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
if (!needsPostCfBusinessUrl(antiBot) || isPostCfBusinessUrlReady(page, antiBot)) {
return {
success: true,
code: null,
challengeType: null,
stage: storedSession && isSessionLikelyValid(storedSession) ? 'session_reuse' : null,
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: false,
};
}
if (await tryPostCfRecoverViaReload(page, waitForUrlSettled, timeoutMs, antiBot)) {
return {
success: true,
code: null,
challengeType: null,
stage: 'post_nav_reload',
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: false,
};
}
if (cfState.hasCfClearance) {
cfState = await invalidateStaleCfAndReload(page, waitForUrlSettled, timeoutMs);
} else {
return {
success: true,
code: null,
challengeType: null,
stage: 'pending_business_nav',
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: false,
};
}
}
if (storedSession && isSessionLikelyValid(storedSession)) {
console.log('[CF] 会话有效但仍被拦截，尝试 reload 一次');
try {
await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 45000) });
if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {});
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
cfState = await detectCloudflare(page);
if (!cfState.blocked) {
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
return {
success: true,
code: null,
@@ -69,7 +234,9 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
if (antiBot.turnstile !== false) {
const builtIn = await waitForTurnstile(page, { timeoutMs, useBuiltInClick: true });
if (builtIn.success) {
if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {});
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
cfState = await detectCloudflare(page);
if (!cfState.blocked) {
return {
@@ -96,7 +263,9 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
const apiWaitMs = Math.min(timeoutMs, 30000);
await waitForTurnstile(page, { timeoutMs: apiWaitMs });
if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {});
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
cfState = await detectCloudflare(page);
if (!cfState.blocked) {
@@ -138,4 +307,7 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
module.exports = {
handleCloudflareChallenge,
isCaptchaConfigured,
needsPostCfBusinessUrl,
isPostCfBusinessUrlReady,
clearCfClearanceCookies,
};
patches/puppeteer-api/lib/post-cf-ready.js
+109
View File
@@ -0,0 +1,109 @@
/**
* CF 过盾后等待业务页就绪（可选，由 antiBot.postCfReady* 驱动，未配置则跳过）
*/
/**
* 采集当前页面诊断信息（auth 失败时便于排查）
* @param {import('puppeteer').Page} page
*/
async function collectPageDiagnostics(page) {
const url = page.url();
const snapshot = await page.evaluate(() => ({
url: window.location.href,
title: document.title || '',
bodyText: (document.body && document.body.innerText ? document.body.innerText : '').slice(0, 500),
elInputInner: document.querySelectorAll('.el-input__inner').length,
})).catch(() => null);
return snapshot || { url, title: '', bodyText: '', elInputInner: 0 };
}
/**
* CF 成功后等待最终业务 URL / DOM 出现（Whatshub 等 SPA 短链跳转场景）
*
* @param {import('puppeteer').Page} page
* @param {object} antiBot normalizeAntiBot 返回值
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
* @param {number} [navigationTimeoutMs]
*/
async function awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled, navigationTimeoutMs = 60000) {
if (!antiBot || !antiBot.enabled) {
return;
}
const urlContains = (antiBot.postCfReadyUrlContains || '').trim();
const selector = (antiBot.postCfReadySelector || '').trim();
if (urlContains === '' && selector === '') {
return;
}
const timeoutMs = Math.max(5000, antiBot.postCfReadyTimeoutMs || 30000);
console.log(
`[CF] 等待业务页就绪 current=${page.url()} urlContains="${urlContains}" `
+ `selector="${selector}" timeout=${timeoutMs}ms`
);
try {
if (urlContains !== '') {
await page.waitForFunction(
(needle) => window.location.href.includes(needle),
{ timeout: timeoutMs, polling: 200 },
urlContains
);
}
if (selector !== '') {
await page.waitForSelector(selector, { timeout: timeoutMs, visible: true });
}
} catch (err) {
const retryMs = Math.min(15000, timeoutMs);
const needsRetry = urlContains !== '' && !page.url().includes(urlContains);
if (needsRetry) {
console.warn(`[CF] 业务页等待失败 url=${page.url()}，最后尝试 reload (${retryMs}ms)`);
await page.reload({ waitUntil: 'domcontentloaded', timeout: navigationTimeoutMs }).catch(() => {});
if (typeof waitForUrlSettled === 'function') {
await waitForUrlSettled(page).catch(() => {});
}
if (urlContains !== '') {
await page.waitForFunction(
(needle) => window.location.href.includes(needle),
{ timeout: retryMs, polling: 200 },
urlContains
);
}
if (selector !== '') {
await page.waitForSelector(selector, { timeout: retryMs, visible: true });
}
} else {
err.pageDiagnostics = await collectPageDiagnostics(page);
throw err;
}
}
const settleMs = antiBot.postCfSettleMs || 0;
if (settleMs > 0) {
await new Promise((resolve) => setTimeout(resolve, settleMs));
}
console.log(`[CF] 业务页就绪 url=${page.url()}`);
}
/**
* 执行 authActions，失败时附带 page 诊断
* @param {import('puppeteer').Page} page
* @param {object[]|undefined} authActions
* @param {Function} executeAuthActionsFn
*/
async function executeAuthActionsWithDiagnostics(page, authActions, executeAuthActionsFn) {
try {
await executeAuthActionsFn(page, authActions);
} catch (err) {
err.pageDiagnostics = await collectPageDiagnostics(page);
throw err;
}
}
module.exports = {
collectPageDiagnostics,
awaitPostCfBusinessReady,
executeAuthActionsWithDiagnostics,
};
patches/puppeteer-api/lib/turnstile-handler.js
+12 -6
View File
@@ -1,7 +1,7 @@
/**
* Turnstile 内置求解：等待 widget 自动完成 + token 轮询
*/
const { detectCloudflare, isTurnstileSolved } = require('./cf-detector');
const { detectCloudflare, isTurnstileSolved, urlIndicatesCfChallenge } = require('./cf-detector');
/**
* @typedef {Object} TurnstileResult
@@ -22,18 +22,24 @@ async function waitForTurnstile(page, options = {}) {
const started = Date.now();
const initial = await detectCloudflare(page);
if (!initial.blocked || initial.hasCfClearance) {
return { success: true, stage: 'already_clear', elapsedMs: Date.now() - started };
const initialUrlChallenge = urlIndicatesCfChallenge(page.url());
if ((!initial.blocked && !initialUrlChallenge) || initial.hasCfClearance) {
if (initial.hasCfClearance && !initialUrlChallenge) {
return { success: true, stage: 'already_clear', elapsedMs: Date.now() - started };
}
if (!initial.blocked && !initialUrlChallenge) {
return { success: true, stage: 'already_clear', elapsedMs: Date.now() - started };
}
}
// real browser 已开启 turnstile:true 内置点击；此处轮询等待结果
while (Date.now() - started < timeoutMs) {
if (await isTurnstileSolved(page)) {
if (await isTurnstileSolved(page) && !urlIndicatesCfChallenge(page.url())) {
return { success: true, stage: 'built_in', elapsedMs: Date.now() - started };
}
const state = await detectCloudflare(page);
if (!state.blocked) {
if (!state.blocked && !urlIndicatesCfChallenge(page.url()) && state.hasCfClearance) {
return { success: true, stage: 'built_in', elapsedMs: Date.now() - started };
}
patches/puppeteer-api/package.json
+1
View File
@@ -7,6 +7,7 @@
"start": "node server.js"
},
"dependencies": {
"dotenv": "^16.4.7",
"express": "^4.18.2",
"puppeteer": "^25.1.0",
"puppeteer-extra": "^3.3.6",
patches/puppeteer-api/server.js
+25 -3
View File
@@ -1,3 +1,5 @@
require('dotenv').config({ path: require('path').join(__dirname, '.env') });
const express = require('express');
const fs = require('fs');
const {
@@ -45,6 +47,10 @@ const {
const { resolveSessionContext, touchSessionIfPresent } = require('./lib/session-context');
const { getProfileStats, profileExists } = require('./lib/profile-dir');
const { runUiPagination } = require('./lib/ui-pagination-runner');
const {
awaitPostCfBusinessReady,
executeAuthActionsWithDiagnostics,
} = require('./lib/post-cf-ready');
const app = express();
app.use(express.json({ limit: '50mb' }));
@@ -75,7 +81,11 @@ function sendTaskError(res, error, extra = {}) {
}
const errType = classifyPuppeteerError(error);
console.error(`[${errType}]`, error.message);
return res.status(500).json({ success: false, error: error.message, ...extra });
const payload = { success: false, error: error.message, ...extra };
if (error.pageDiagnostics) {
payload.page = error.pageDiagnostics;
}
return res.status(500).json(payload);
}
async function navigateToPage(page, url) {
@@ -524,7 +534,11 @@ app.post('/api/auth-and-intercept', async (req, res) => {
await seedShareTokenCookie(page, finalPageUrl);
}
await executeAuthActions(page, authActions);
if (antiBot.enabled) {
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled);
}
await executeAuthActionsWithDiagnostics(page, authActions, executeAuthActions);
await interceptor.waitForApis(interceptTimeout);
const rawCookies = await page.cookies();
@@ -758,6 +772,10 @@ app.post('/api/ui-pagination', async (req, res) => {
touchSessionIfPresent(antiBot.sessionKey);
}
if (antiBot.enabled) {
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled);
}
const paginationResult = await runUiPagination(page, {
apiUrl,
nextBtnSelector,
@@ -874,7 +892,11 @@ app.post('/api/auth-intercept-and-paginate', async (req, res) => {
touchSessionIfPresent(antiBot.sessionKey);
}
await executeAuthActions(page, authActions);
if (antiBot.enabled) {
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled);
}
await executeAuthActionsWithDiagnostics(page, authActions, executeAuthActions);
await interceptor.waitForApis(interceptTimeout);
const rawCookies = await page.cookies();