damon/links
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修改whatshub工单爬虫

Browse Source
This commit is contained in:
root
2026-06-29 04:54:41 +08:00
parent 6130125427
commit 9f2e904fab
14 changed files with 1254 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
patches/puppeteer-api/lib/browser-factory.js
+8 -2
View File
@@ -28,6 +28,10 @@ function normalizeAntiBot(antiBot) {
solverFallback: antiBot.solverFallback !== false,
sessionKey: antiBot.sessionKey || '',
challengeTimeoutMs: antiBot.challengeTimeoutMs || 60000,
postCfReadyUrlContains: antiBot.postCfReadyUrlContains || '',
postCfReadySelector: antiBot.postCfReadySelector || '',
postCfReadyTimeoutMs: antiBot.postCfReadyTimeoutMs || 0,
postCfSettleMs: antiBot.postCfSettleMs || 0,
};
}
@@ -106,8 +110,10 @@ async function createManagedPage(browser, existingPage, options = {}) {
page.setDefaultNavigationTimeout(NAVIGATION_TIMEOUT_MS);
page.setDefaultTimeout(PAGE_DEFAULT_TIMEOUT_MS);
const userAgent = options.userAgent || DEFAULT_UA;
await page.setUserAgent(userAgent);
if (!options.skipUserAgent) {
const userAgent = options.userAgent || DEFAULT_UA;
await page.setUserAgent(userAgent);
}
if (options.viewport) await page.setViewport(options.viewport);
if (options.cookies && options.cookies.length > 0) {
patches/puppeteer-api/lib/cf-detector.js
+18 -2
View File
@@ -2,6 +2,21 @@
* Cloudflare / Turnstile 挑战页检测
*/
/**
* URL 是否处于 CF Managed Challenge 中间态（Whatshub 短链常见 __cf_chl_rt_tk）
* @param {string} url
*/
function urlIndicatesCfChallenge(url) {
if (!url || typeof url !== 'string') {
return false;
}
return url.includes('__cf_chl_rt_tk')
|| url.includes('__cf_chl_tk')
|| url.includes('__cf_chl_f_tk')
|| url.includes('/cdn-cgi/challenge-platform')
|| url.includes('/cdn-cgi/challenge');
}
/**
* @typedef {Object} CloudflareState
* @property {boolean} blocked 是否处于 CF 挑战中
@@ -44,10 +59,10 @@ async function detectCloudflare(page) {
bodyHasJustAMoment: false,
}));
const urlChallenge = url.includes('/cdn-cgi/challenge-platform') || url.includes('/cdn-cgi/challenge');
const urlChallenge = urlIndicatesCfChallenge(url);
let challengeType = null;
if (domSignals.hasTurnstileInput || domSignals.hasTurnstileWidget) {
if (domSignals.hasTurnstileInput || domSignals.hasTurnstileWidget || url.includes('__cf_chl')) {
challengeType = 'turnstile';
} else if (urlChallenge || domSignals.titleHasJustAMoment || domSignals.bodyHasJustAMoment) {
challengeType = 'js_challenge';
@@ -94,4 +109,5 @@ async function isTurnstileSolved(page) {
module.exports = {
detectCloudflare,
isTurnstileSolved,
urlIndicatesCfChallenge,
};
patches/puppeteer-api/lib/cf-handler.js
+194 -22
View File
@@ -1,5 +1,8 @@
/**
* Cloudflare 挑战统一处理：检测 → 会话快速通道 → 内置等待 → Captcha API 兜底
*
* 当 antiBot.postCfReadyUrlContains 已配置时，session_reuse 仅在业务 URL 已到达时才走快速通道；
* 否则清除可能过期的 cf_clearance 并重新触发 Turnstile（Whatshub 短链 → work-order-sharing 场景）。
*/
const { detectCloudflare } = require('./cf-detector');
const { waitForTurnstile, extractTurnstileSitekey, injectTurnstileToken } = require('./turnstile-handler');
@@ -7,8 +10,135 @@ const { solveTurnstile, isCaptchaConfigured } = require('./captcha-solver');
const { isSessionLikelyValid } = require('./session-store');
/**
* 是否配置了 CF 后必须到达的业务 URL
* @param {object} antiBot
*/
function needsPostCfBusinessUrl(antiBot) {
return !!(antiBot?.postCfReadyUrlContains || '').trim();
}
/**
* 当前页面 URL 是否已包含业务路径片段
* @param {import('puppeteer').Page} page
* @param {{ turnstile?: boolean, solverFallback?: boolean, challengeTimeoutMs?: number }} antiBot
* @param {object} antiBot
*/
function isPostCfBusinessUrlReady(page, antiBot) {
const needle = (antiBot?.postCfReadyUrlContains || '').trim();
if (!needle) {
return true;
}
return page.url().includes(needle);
}
/**
* 清除 cf_clearance，避免「有 cookie 但 SPA 未触发跳转」的假过关
* @param {import('puppeteer').Page} page
*/
async function clearCfClearanceCookies(page) {
const cookies = await page.cookies();
for (const c of cookies) {
if (c.name !== 'cf_clearance') {
continue;
}
const host = (c.domain || '').replace(/^\./, '');
if (host) {
await page.deleteCookie({ name: c.name, url: `https://${host}/` }).catch(() => {});
}
await page.deleteCookie({
name: c.name,
domain: c.domain,
path: c.path || '/',
}).catch(() => {});
}
}
/**
* reload 后检查业务 URL 是否就绪
* @param {import('puppeteer').Page} page
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
* @param {number} timeoutMs
* @param {object} antiBot
*/
async function tryPostCfRecoverViaReload(page, waitForUrlSettled, timeoutMs, antiBot) {
if (!needsPostCfBusinessUrl(antiBot) || isPostCfBusinessUrlReady(page, antiBot)) {
return isPostCfBusinessUrlReady(page, antiBot);
}
console.log(`[CF] 业务 URL 未就绪 current=${page.url()}，尝试 reload`);
try {
await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 45000) });
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
return isPostCfBusinessUrlReady(page, antiBot);
} catch (reloadErr) {
console.warn('[CF] reload 失败:', reloadErr.message);
return false;
}
}
/**
* 清除 cf_clearance 并 reload，使 Turnstile 重新出现
* @param {import('puppeteer').Page} page
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
* @param {number} timeoutMs
*/
async function invalidateStaleCfAndReload(page, waitForUrlSettled, timeoutMs) {
console.log('[CF] 清除 cf_clearance 并 reload，重新触发 Turnstile');
await clearCfClearanceCookies(page);
await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 45000) });
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
return detectCloudflare(page);
}
/**
* session_reuse 快速返回前校验：有 postCf 要求时必须已到业务页
* @param {import('puppeteer').Page} page
* @param {object} antiBot
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
* @param {number} timeoutMs
* @param {number} started
* @param {boolean} cfDetected
*/
async function trySessionReuseFastPath(page, antiBot, waitForUrlSettled, timeoutMs, started, cfDetected) {
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
if (!needsPostCfBusinessUrl(antiBot) || isPostCfBusinessUrlReady(page, antiBot)) {
return {
success: true,
code: null,
challengeType: null,
stage: 'session_reuse',
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected,
};
}
console.log(`[CF] cf_clearance/无挑战但业务页未就绪 url=${page.url()}，降级重试`);
if (await tryPostCfRecoverViaReload(page, waitForUrlSettled, timeoutMs, antiBot)) {
return {
success: true,
code: null,
challengeType: null,
stage: 'session_reuse_reload',
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected,
};
}
return null;
}
/**
* @param {import('puppeteer').Page} page
* @param {{ turnstile?: boolean, solverFallback?: boolean, challengeTimeoutMs?: number, postCfReadyUrlContains?: string }} antiBot
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
* @param {import('./session-store').StoredSession|null} [storedSession]
*/
@@ -19,36 +149,71 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
let cfState = await detectCloudflare(page);
if (!cfState.blocked && cfState.hasCfClearance) {
return {
success: true,
code: null,
challengeType: null,
stage: 'session_reuse',
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: false,
};
const fast = await trySessionReuseFastPath(
page, antiBot, waitForUrlSettled, timeoutMs, started, false
);
if (fast) {
return fast;
}
cfState = await invalidateStaleCfAndReload(page, waitForUrlSettled, timeoutMs);
}
if (!cfState.blocked) {
return {
success: true,
code: null,
challengeType: null,
stage: storedSession && isSessionLikelyValid(storedSession) ? 'session_reuse' : null,
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: false,
};
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
if (!needsPostCfBusinessUrl(antiBot) || isPostCfBusinessUrlReady(page, antiBot)) {
return {
success: true,
code: null,
challengeType: null,
stage: storedSession && isSessionLikelyValid(storedSession) ? 'session_reuse' : null,
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: false,
};
}
if (await tryPostCfRecoverViaReload(page, waitForUrlSettled, timeoutMs, antiBot)) {
return {
success: true,
code: null,
challengeType: null,
stage: 'post_nav_reload',
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: false,
};
}
if (cfState.hasCfClearance) {
cfState = await invalidateStaleCfAndReload(page, waitForUrlSettled, timeoutMs);
} else {
return {
success: true,
code: null,
challengeType: null,
stage: 'pending_business_nav',
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: false,
};
}
}
if (storedSession && isSessionLikelyValid(storedSession)) {
console.log('[CF] 会话有效但仍被拦截，尝试 reload 一次');
try {
await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 45000) });
if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {});
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
cfState = await detectCloudflare(page);
if (!cfState.blocked) {
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
return {
success: true,
code: null,
@@ -69,7 +234,9 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
if (antiBot.turnstile !== false) {
const builtIn = await waitForTurnstile(page, { timeoutMs, useBuiltInClick: true });
if (builtIn.success) {
if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {});
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
cfState = await detectCloudflare(page);
if (!cfState.blocked) {
return {
@@ -96,7 +263,9 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
const apiWaitMs = Math.min(timeoutMs, 30000);
await waitForTurnstile(page, { timeoutMs: apiWaitMs });
if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {});
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
cfState = await detectCloudflare(page);
if (!cfState.blocked) {
@@ -138,4 +307,7 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
module.exports = {
handleCloudflareChallenge,
isCaptchaConfigured,
needsPostCfBusinessUrl,
isPostCfBusinessUrlReady,
clearCfClearanceCookies,
};
patches/puppeteer-api/lib/post-cf-ready.js
+109
View File
@@ -0,0 +1,109 @@
/**
* CF 过盾后等待业务页就绪（可选，由 antiBot.postCfReady* 驱动，未配置则跳过）
*/
/**
* 采集当前页面诊断信息（auth 失败时便于排查）
* @param {import('puppeteer').Page} page
*/
async function collectPageDiagnostics(page) {
const url = page.url();
const snapshot = await page.evaluate(() => ({
url: window.location.href,
title: document.title || '',
bodyText: (document.body && document.body.innerText ? document.body.innerText : '').slice(0, 500),
elInputInner: document.querySelectorAll('.el-input__inner').length,
})).catch(() => null);
return snapshot || { url, title: '', bodyText: '', elInputInner: 0 };
}
/**
* CF 成功后等待最终业务 URL / DOM 出现（Whatshub 等 SPA 短链跳转场景）
*
* @param {import('puppeteer').Page} page
* @param {object} antiBot normalizeAntiBot 返回值
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
* @param {number} [navigationTimeoutMs]
*/
async function awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled, navigationTimeoutMs = 60000) {
if (!antiBot || !antiBot.enabled) {
return;
}
const urlContains = (antiBot.postCfReadyUrlContains || '').trim();
const selector = (antiBot.postCfReadySelector || '').trim();
if (urlContains === '' && selector === '') {
return;
}
const timeoutMs = Math.max(5000, antiBot.postCfReadyTimeoutMs || 30000);
console.log(
`[CF] 等待业务页就绪 current=${page.url()} urlContains="${urlContains}" `
+ `selector="${selector}" timeout=${timeoutMs}ms`
);
try {
if (urlContains !== '') {
await page.waitForFunction(
(needle) => window.location.href.includes(needle),
{ timeout: timeoutMs, polling: 200 },
urlContains
);
}
if (selector !== '') {
await page.waitForSelector(selector, { timeout: timeoutMs, visible: true });
}
} catch (err) {
const retryMs = Math.min(15000, timeoutMs);
const needsRetry = urlContains !== '' && !page.url().includes(urlContains);
if (needsRetry) {
console.warn(`[CF] 业务页等待失败 url=${page.url()}，最后尝试 reload (${retryMs}ms)`);
await page.reload({ waitUntil: 'domcontentloaded', timeout: navigationTimeoutMs }).catch(() => {});
if (typeof waitForUrlSettled === 'function') {
await waitForUrlSettled(page).catch(() => {});
}
if (urlContains !== '') {
await page.waitForFunction(
(needle) => window.location.href.includes(needle),
{ timeout: retryMs, polling: 200 },
urlContains
);
}
if (selector !== '') {
await page.waitForSelector(selector, { timeout: retryMs, visible: true });
}
} else {
err.pageDiagnostics = await collectPageDiagnostics(page);
throw err;
}
}
const settleMs = antiBot.postCfSettleMs || 0;
if (settleMs > 0) {
await new Promise((resolve) => setTimeout(resolve, settleMs));
}
console.log(`[CF] 业务页就绪 url=${page.url()}`);
}
/**
* 执行 authActions，失败时附带 page 诊断
* @param {import('puppeteer').Page} page
* @param {object[]|undefined} authActions
* @param {Function} executeAuthActionsFn
*/
async function executeAuthActionsWithDiagnostics(page, authActions, executeAuthActionsFn) {
try {
await executeAuthActionsFn(page, authActions);
} catch (err) {
err.pageDiagnostics = await collectPageDiagnostics(page);
throw err;
}
}
module.exports = {
collectPageDiagnostics,
awaitPostCfBusinessReady,
executeAuthActionsWithDiagnostics,
};
patches/puppeteer-api/lib/turnstile-handler.js
+12 -6
View File
@@ -1,7 +1,7 @@
/**
* Turnstile 内置求解：等待 widget 自动完成 + token 轮询
*/
const { detectCloudflare, isTurnstileSolved } = require('./cf-detector');
const { detectCloudflare, isTurnstileSolved, urlIndicatesCfChallenge } = require('./cf-detector');
/**
* @typedef {Object} TurnstileResult
@@ -22,18 +22,24 @@ async function waitForTurnstile(page, options = {}) {
const started = Date.now();
const initial = await detectCloudflare(page);
if (!initial.blocked || initial.hasCfClearance) {
return { success: true, stage: 'already_clear', elapsedMs: Date.now() - started };
const initialUrlChallenge = urlIndicatesCfChallenge(page.url());
if ((!initial.blocked && !initialUrlChallenge) || initial.hasCfClearance) {
if (initial.hasCfClearance && !initialUrlChallenge) {
return { success: true, stage: 'already_clear', elapsedMs: Date.now() - started };
}
if (!initial.blocked && !initialUrlChallenge) {
return { success: true, stage: 'already_clear', elapsedMs: Date.now() - started };
}
}
// real browser 已开启 turnstile:true 内置点击；此处轮询等待结果
while (Date.now() - started < timeoutMs) {
if (await isTurnstileSolved(page)) {
if (await isTurnstileSolved(page) && !urlIndicatesCfChallenge(page.url())) {
return { success: true, stage: 'built_in', elapsedMs: Date.now() - started };
}
const state = await detectCloudflare(page);
if (!state.blocked) {
if (!state.blocked && !urlIndicatesCfChallenge(page.url()) && state.hasCfClearance) {
return { success: true, stage: 'built_in', elapsedMs: Date.now() - started };
}
patches/puppeteer-api/package.json
+1
View File
@@ -7,6 +7,7 @@
"start": "node server.js"
},
"dependencies": {
"dotenv": "^16.4.7",
"express": "^4.18.2",
"puppeteer": "^25.1.0",
"puppeteer-extra": "^3.3.6",
patches/puppeteer-api/server.js
+25 -3
View File
@@ -1,3 +1,5 @@
require('dotenv').config({ path: require('path').join(__dirname, '.env') });
const express = require('express');
const fs = require('fs');
const {
@@ -45,6 +47,10 @@ const {
const { resolveSessionContext, touchSessionIfPresent } = require('./lib/session-context');
const { getProfileStats, profileExists } = require('./lib/profile-dir');
const { runUiPagination } = require('./lib/ui-pagination-runner');
const {
awaitPostCfBusinessReady,
executeAuthActionsWithDiagnostics,
} = require('./lib/post-cf-ready');
const app = express();
app.use(express.json({ limit: '50mb' }));
@@ -75,7 +81,11 @@ function sendTaskError(res, error, extra = {}) {
}
const errType = classifyPuppeteerError(error);
console.error(`[${errType}]`, error.message);
return res.status(500).json({ success: false, error: error.message, ...extra });
const payload = { success: false, error: error.message, ...extra };
if (error.pageDiagnostics) {
payload.page = error.pageDiagnostics;
}
return res.status(500).json(payload);
}
async function navigateToPage(page, url) {
@@ -524,7 +534,11 @@ app.post('/api/auth-and-intercept', async (req, res) => {
await seedShareTokenCookie(page, finalPageUrl);
}
await executeAuthActions(page, authActions);
if (antiBot.enabled) {
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled);
}
await executeAuthActionsWithDiagnostics(page, authActions, executeAuthActions);
await interceptor.waitForApis(interceptTimeout);
const rawCookies = await page.cookies();
@@ -758,6 +772,10 @@ app.post('/api/ui-pagination', async (req, res) => {
touchSessionIfPresent(antiBot.sessionKey);
}
if (antiBot.enabled) {
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled);
}
const paginationResult = await runUiPagination(page, {
apiUrl,
nextBtnSelector,
@@ -874,7 +892,11 @@ app.post('/api/auth-intercept-and-paginate', async (req, res) => {
touchSessionIfPresent(antiBot.sessionKey);
}
await executeAuthActions(page, authActions);
if (antiBot.enabled) {
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled);
}
await executeAuthActionsWithDiagnostics(page, authActions, executeAuthActions);
await interceptor.waitForApis(interceptTimeout);
const rawCookies = await page.cookies();