添加whatshub工单
This commit is contained in:
Executable
BIN
Binary file not shown.
BIN
Binary file not shown.
Executable
+62
@@ -0,0 +1,62 @@
|
||||
/**
|
||||
* 将发往字面量 IPv4 的 https 请求改写为 http。
|
||||
* 用于云控面板 HTTP 入口对浏览器返回 307 升级 HTTPS、但 IP 上 443 不可达的场景。
|
||||
*/
|
||||
'use strict';
|
||||
|
||||
function rewriteHttpsToHttpForLiteralIp(url) {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
if (parsed.protocol !== 'https:') {
|
||||
return url;
|
||||
}
|
||||
if (!/^\d{1,3}(\.\d{1,3}){3}$/.test(parsed.hostname)) {
|
||||
return url;
|
||||
}
|
||||
parsed.protocol = 'http:';
|
||||
return parsed.toString();
|
||||
} catch (_) {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
async function attachHttpIpRewriteInterceptor(page, contextLabel = 'page', options = {}) {
|
||||
const shareToken = typeof options.shareToken === 'string' ? options.shareToken : '';
|
||||
const blockHeavyResources = options.blockHeavyResources !== false;
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (req) => {
|
||||
if (req.isInterceptResolutionHandled()) {
|
||||
return;
|
||||
}
|
||||
if (blockHeavyResources) {
|
||||
const resourceType = req.resourceType();
|
||||
if (['image', 'media', 'font'].includes(resourceType)) {
|
||||
req.abort();
|
||||
return;
|
||||
}
|
||||
}
|
||||
const originalUrl = req.url();
|
||||
const rewrittenUrl = rewriteHttpsToHttpForLiteralIp(originalUrl);
|
||||
if (rewrittenUrl !== originalUrl) {
|
||||
const continueOptions = { url: rewrittenUrl };
|
||||
if (shareToken) {
|
||||
const headers = { ...req.headers() };
|
||||
const existingCookie = headers.cookie || headers.Cookie || '';
|
||||
if (!existingCookie.includes('share_token=')) {
|
||||
headers.cookie = existingCookie
|
||||
? `${existingCookie}; share_token=${shareToken}`
|
||||
: `share_token=${shareToken}`;
|
||||
}
|
||||
continueOptions.headers = headers;
|
||||
}
|
||||
req.continue(continueOptions);
|
||||
return;
|
||||
}
|
||||
req.continue();
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
rewriteHttpsToHttpForLiteralIp,
|
||||
attachHttpIpRewriteInterceptor,
|
||||
};
|
||||
Executable
+160
@@ -0,0 +1,160 @@
|
||||
/**
|
||||
* BrowserFactory:standard / real 双轨启动 + 独立并发限流
|
||||
*/
|
||||
const {
|
||||
DEFAULT_UA,
|
||||
MAX_CONCURRENT_BROWSERS,
|
||||
MAX_CONCURRENT_BROWSERS_REAL,
|
||||
QUEUE_TIMEOUT_MS,
|
||||
} = require('./constants');
|
||||
const { BrowserConcurrencyLimiter } = require('./concurrency');
|
||||
const { launchStandardBrowser } = require('./launch-standard');
|
||||
const { launchRealBrowser, isRealBrowserAvailable } = require('./launch-real-browser');
|
||||
|
||||
const standardLimiter = new BrowserConcurrencyLimiter(MAX_CONCURRENT_BROWSERS);
|
||||
const realLimiter = new BrowserConcurrencyLimiter(MAX_CONCURRENT_BROWSERS_REAL);
|
||||
|
||||
/**
|
||||
* @typedef {Object} AntiBotConfig
|
||||
* @property {boolean} [enabled]
|
||||
* @property {'standard'|'real'} [profile]
|
||||
* @property {boolean} [turnstile]
|
||||
* @property {boolean} [solverFallback]
|
||||
* @property {string} [sessionKey]
|
||||
* @property {number} [challengeTimeoutMs]
|
||||
*/
|
||||
|
||||
/**
|
||||
* 规范化 antiBot 配置(未传则走 standard)
|
||||
* @param {AntiBotConfig|null|undefined} antiBot
|
||||
* @returns {AntiBotConfig}
|
||||
*/
|
||||
function normalizeAntiBot(antiBot) {
|
||||
if (!antiBot || !antiBot.enabled) {
|
||||
return { enabled: false, profile: 'standard' };
|
||||
}
|
||||
return {
|
||||
enabled: true,
|
||||
profile: antiBot.profile === 'real' ? 'real' : 'standard',
|
||||
turnstile: antiBot.turnstile !== false,
|
||||
solverFallback: antiBot.solverFallback !== false,
|
||||
sessionKey: antiBot.sessionKey || '',
|
||||
challengeTimeoutMs: antiBot.challengeTimeoutMs || 60000,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 按 profile 获取限流器
|
||||
* @param {'standard'|'real'} profile
|
||||
*/
|
||||
function getLimiter(profile) {
|
||||
return profile === 'real' ? realLimiter : standardLimiter;
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建 Browser 会话
|
||||
* @param {{ profile?: 'standard'|'real', extraArgs?: string[] }} options
|
||||
* @returns {Promise<{ browser: import('puppeteer').Browser, page: import('puppeteer').Page|null, profile: string, cleanup: () => Promise<void> }>}
|
||||
*/
|
||||
async function createBrowserSession(options = {}) {
|
||||
const profile = options.profile === 'real' ? 'real' : 'standard';
|
||||
const extraArgs = options.extraArgs || [];
|
||||
|
||||
if (profile === 'real') {
|
||||
const { browser, page } = await launchRealBrowser(extraArgs);
|
||||
return {
|
||||
browser,
|
||||
page,
|
||||
profile: 'real',
|
||||
cleanup: async () => {
|
||||
try {
|
||||
await browser.close();
|
||||
} catch (_) {
|
||||
/* ignore */
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const browser = await launchStandardBrowser(extraArgs);
|
||||
return {
|
||||
browser,
|
||||
page: null,
|
||||
profile: 'standard',
|
||||
cleanup: async () => {
|
||||
try {
|
||||
await browser.close();
|
||||
} catch (_) {
|
||||
/* ignore */
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 在并发槽位内执行 Browser 任务
|
||||
* @template T
|
||||
* @param {string} taskName
|
||||
* @param {'standard'|'real'} profile
|
||||
* @param {() => Promise<T>} fn
|
||||
* @returns {Promise<T>}
|
||||
*/
|
||||
async function runWithProfileLimit(taskName, profile, fn) {
|
||||
const limiter = getLimiter(profile);
|
||||
try {
|
||||
return await limiter.run(taskName, fn);
|
||||
} catch (err) {
|
||||
if (err.message === 'QUEUE_TIMEOUT') err.code = 'QUEUE_TIMEOUT';
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建 Page 并设置 UA / viewport / cookies
|
||||
* @param {import('puppeteer').Browser} browser
|
||||
* @param {import('puppeteer').Page|null} existingPage
|
||||
* @param {{ userAgent?: string, viewport?: object, cookies?: Array<object> }} [options]
|
||||
*/
|
||||
async function createManagedPage(browser, existingPage, options = {}) {
|
||||
const page = existingPage || await browser.newPage();
|
||||
page.setDefaultNavigationTimeout(30000);
|
||||
page.setDefaultTimeout(15000);
|
||||
|
||||
const userAgent = options.userAgent || DEFAULT_UA;
|
||||
await page.setUserAgent(userAgent);
|
||||
|
||||
if (options.viewport) {
|
||||
await page.setViewport(options.viewport);
|
||||
}
|
||||
if (options.cookies && options.cookies.length > 0) {
|
||||
await page.setCookie(...options.cookies);
|
||||
}
|
||||
|
||||
page.on('error', (err) => console.error('[Page Error]', err.message));
|
||||
page.on('pageerror', (err) => console.error('[Page JS Error]', err.message));
|
||||
|
||||
return page;
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {{ standard: object, real: object, realBrowserReady: boolean }}
|
||||
*/
|
||||
function getFactoryStats() {
|
||||
return {
|
||||
standard: standardLimiter.getStats(),
|
||||
real: realLimiter.getStats(),
|
||||
realBrowserReady: isRealBrowserAvailable(),
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
normalizeAntiBot,
|
||||
getLimiter,
|
||||
createBrowserSession,
|
||||
runWithProfileLimit,
|
||||
createManagedPage,
|
||||
getFactoryStats,
|
||||
standardLimiter,
|
||||
realLimiter,
|
||||
QUEUE_TIMEOUT_MS,
|
||||
};
|
||||
Executable
+153
@@ -0,0 +1,153 @@
|
||||
/**
|
||||
* 付费 Captcha API 兜底:CapSolver / 2Captcha 统一 Turnstile 求解接口
|
||||
*/
|
||||
const {
|
||||
CAPTCHA_PROVIDER,
|
||||
CAPTCHA_API_KEY,
|
||||
CAPTCHA_MAX_WAIT_MS,
|
||||
} = require('./constants');
|
||||
|
||||
/**
|
||||
* @param {string} url
|
||||
* @param {Record<string, string>} [headers]
|
||||
* @returns {Promise<any>}
|
||||
*/
|
||||
async function httpJsonPost(url, body, headers = {}) {
|
||||
const res = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', ...headers },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
const text = await res.text();
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch (_) {
|
||||
throw new Error(`Captcha API 返回非 JSON: ${text.slice(0, 200)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} url
|
||||
* @returns {Promise<any>}
|
||||
*/
|
||||
async function httpJsonGet(url) {
|
||||
const res = await fetch(url);
|
||||
const text = await res.text();
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch (_) {
|
||||
throw new Error(`Captcha API 返回非 JSON: ${text.slice(0, 200)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* CapSolver Turnstile 求解
|
||||
* @param {{ pageUrl: string, sitekey: string, apiKey: string }} params
|
||||
* @returns {Promise<string>}
|
||||
*/
|
||||
async function solveViaCapSolver({ pageUrl, sitekey, apiKey }) {
|
||||
const create = await httpJsonPost('https://api.capsolver.com/createTask', {
|
||||
clientKey: apiKey,
|
||||
task: {
|
||||
type: 'AntiTurnstileTaskProxyLess',
|
||||
websiteURL: pageUrl,
|
||||
websiteKey: sitekey,
|
||||
},
|
||||
});
|
||||
|
||||
if (create.errorId !== 0 || !create.taskId) {
|
||||
throw new Error(`CapSolver createTask 失败: ${create.errorDescription || create.errorCode || 'unknown'}`);
|
||||
}
|
||||
|
||||
const deadline = Date.now() + CAPTCHA_MAX_WAIT_MS;
|
||||
while (Date.now() < deadline) {
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
const result = await httpJsonPost('https://api.capsolver.com/getTaskResult', {
|
||||
clientKey: apiKey,
|
||||
taskId: create.taskId,
|
||||
});
|
||||
|
||||
if (result.status === 'ready' && result.solution && result.solution.token) {
|
||||
return result.solution.token;
|
||||
}
|
||||
if (result.errorId !== 0) {
|
||||
throw new Error(`CapSolver getTaskResult 失败: ${result.errorDescription || result.errorCode}`);
|
||||
}
|
||||
}
|
||||
throw new Error('CapSolver 求解超时');
|
||||
}
|
||||
|
||||
/**
|
||||
* 2Captcha Turnstile 求解
|
||||
* @param {{ pageUrl: string, sitekey: string, apiKey: string }} params
|
||||
* @returns {Promise<string>}
|
||||
*/
|
||||
async function solveVia2Captcha({ pageUrl, sitekey, apiKey }) {
|
||||
const params = new URLSearchParams({
|
||||
key: apiKey,
|
||||
method: 'turnstile',
|
||||
sitekey,
|
||||
pageurl: pageUrl,
|
||||
json: '1',
|
||||
});
|
||||
|
||||
const create = await httpJsonGet(`https://2captcha.com/in.php?${params.toString()}`);
|
||||
if (create.status !== 1 || !create.request) {
|
||||
throw new Error(`2Captcha in.php 失败: ${create.request || create.error_text || 'unknown'}`);
|
||||
}
|
||||
|
||||
const taskId = create.request;
|
||||
const deadline = Date.now() + CAPTCHA_MAX_WAIT_MS;
|
||||
while (Date.now() < deadline) {
|
||||
await new Promise((r) => setTimeout(r, 5000));
|
||||
const result = await httpJsonGet(
|
||||
`https://2captcha.com/res.php?key=${encodeURIComponent(apiKey)}&action=get&id=${encodeURIComponent(taskId)}&json=1`
|
||||
);
|
||||
if (result.status === 1 && result.request) {
|
||||
return result.request;
|
||||
}
|
||||
if (result.request && result.request !== 'CAPCHA_NOT_READY') {
|
||||
throw new Error(`2Captcha res.php 失败: ${result.request}`);
|
||||
}
|
||||
}
|
||||
throw new Error('2Captcha 求解超时');
|
||||
}
|
||||
|
||||
/**
|
||||
* 统一 Turnstile 求解入口
|
||||
* @param {{ pageUrl: string, sitekey: string, provider?: string, apiKey?: string }} params
|
||||
* @returns {Promise<{ token: string, provider: string }>}
|
||||
*/
|
||||
async function solveTurnstile({ pageUrl, sitekey, provider, apiKey }) {
|
||||
const resolvedProvider = (provider || CAPTCHA_PROVIDER).toLowerCase();
|
||||
const resolvedKey = apiKey || CAPTCHA_API_KEY;
|
||||
|
||||
if (!sitekey) {
|
||||
throw new Error('无法提取 Turnstile sitekey');
|
||||
}
|
||||
if (!resolvedKey) {
|
||||
throw new Error('未配置 CAPTCHA_API_KEY,无法使用付费 Captcha 兜底');
|
||||
}
|
||||
|
||||
let token;
|
||||
if (resolvedProvider === '2captcha') {
|
||||
token = await solveVia2Captcha({ pageUrl, sitekey, apiKey: resolvedKey });
|
||||
} else {
|
||||
token = await solveViaCapSolver({ pageUrl, sitekey, apiKey: resolvedKey });
|
||||
}
|
||||
|
||||
return { token, provider: resolvedProvider };
|
||||
}
|
||||
|
||||
/**
|
||||
* Captcha API 是否已配置
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function isCaptchaConfigured() {
|
||||
return CAPTCHA_API_KEY.length > 0;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
solveTurnstile,
|
||||
isCaptchaConfigured,
|
||||
};
|
||||
Executable
+97
@@ -0,0 +1,97 @@
|
||||
/**
|
||||
* Cloudflare / Turnstile 挑战页检测
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} CloudflareState
|
||||
* @property {boolean} blocked 是否处于 CF 挑战中
|
||||
* @property {string|null} challengeType turnstile|js_challenge|unknown|null
|
||||
* @property {boolean} hasCfClearance 是否已有 cf_clearance cookie
|
||||
* @property {string} url 当前页面 URL
|
||||
* @property {string} title 页面 title
|
||||
*/
|
||||
|
||||
/**
|
||||
* 检测页面是否被 Cloudflare 拦截
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @returns {Promise<CloudflareState>}
|
||||
*/
|
||||
async function detectCloudflare(page) {
|
||||
const url = page.url();
|
||||
const title = await page.title().catch(() => '');
|
||||
|
||||
const cookies = await page.cookies().catch(() => []);
|
||||
const hasCfClearance = cookies.some((c) => c.name === 'cf_clearance');
|
||||
|
||||
const domSignals = await page.evaluate(() => {
|
||||
const hasTurnstileInput = !!document.querySelector('[name="cf-turnstile-response"]');
|
||||
const hasTurnstileWidget = !!document.querySelector('.cf-turnstile, [data-sitekey]');
|
||||
const hasChallengeRunning = !!document.querySelector('#challenge-running, #cf-challenge-running');
|
||||
const titleText = document.title || '';
|
||||
const bodyText = (document.body && document.body.innerText) ? document.body.innerText.slice(0, 500) : '';
|
||||
return {
|
||||
hasTurnstileInput,
|
||||
hasTurnstileWidget,
|
||||
hasChallengeRunning,
|
||||
titleHasJustAMoment: titleText.includes('Just a moment'),
|
||||
bodyHasJustAMoment: bodyText.includes('Just a moment') || bodyText.includes('Checking your browser'),
|
||||
};
|
||||
}).catch(() => ({
|
||||
hasTurnstileInput: false,
|
||||
hasTurnstileWidget: false,
|
||||
hasChallengeRunning: false,
|
||||
titleHasJustAMoment: false,
|
||||
bodyHasJustAMoment: false,
|
||||
}));
|
||||
|
||||
const urlChallenge = url.includes('/cdn-cgi/challenge-platform') || url.includes('/cdn-cgi/challenge');
|
||||
|
||||
let challengeType = null;
|
||||
if (domSignals.hasTurnstileInput || domSignals.hasTurnstileWidget) {
|
||||
challengeType = 'turnstile';
|
||||
} else if (urlChallenge || domSignals.titleHasJustAMoment || domSignals.bodyHasJustAMoment) {
|
||||
challengeType = 'js_challenge';
|
||||
}
|
||||
|
||||
const blocked = !hasCfClearance && (
|
||||
urlChallenge
|
||||
|| domSignals.titleHasJustAMoment
|
||||
|| domSignals.bodyHasJustAMoment
|
||||
|| domSignals.hasTurnstileInput
|
||||
|| domSignals.hasTurnstileWidget
|
||||
|| domSignals.hasChallengeRunning
|
||||
|| title.includes('Just a moment')
|
||||
);
|
||||
|
||||
return {
|
||||
blocked,
|
||||
challengeType,
|
||||
hasCfClearance,
|
||||
url,
|
||||
title,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 检测 Turnstile 是否已通过
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
async function isTurnstileSolved(page) {
|
||||
const cookies = await page.cookies().catch(() => []);
|
||||
if (cookies.some((c) => c.name === 'cf_clearance')) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const tokenLen = await page.evaluate(() => {
|
||||
const input = document.querySelector('[name="cf-turnstile-response"]');
|
||||
return input && input.value ? input.value.length : 0;
|
||||
}).catch(() => 0);
|
||||
|
||||
return tokenLen > 20;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
detectCloudflare,
|
||||
isTurnstileSolved,
|
||||
};
|
||||
Executable
+121
@@ -0,0 +1,121 @@
|
||||
/**
|
||||
* Cloudflare 挑战统一处理:检测 → 内置等待 → Captcha API 兜底
|
||||
*/
|
||||
const { detectCloudflare } = require('./cf-detector');
|
||||
const { waitForTurnstile, extractTurnstileSitekey, injectTurnstileToken } = require('./turnstile-handler');
|
||||
const { solveTurnstile, isCaptchaConfigured } = require('./captcha-solver');
|
||||
|
||||
/**
|
||||
* @typedef {Object} CfHandleResult
|
||||
* @property {boolean} success
|
||||
* @property {string|null} code
|
||||
* @property {string|null} challengeType
|
||||
* @property {string|null} stage
|
||||
* @property {boolean} solverUsed
|
||||
* @property {number} elapsedMs
|
||||
* @property {boolean} cfDetected
|
||||
*/
|
||||
|
||||
/**
|
||||
* 处理 Cloudflare / Turnstile 挑战(在 authActions 之前调用)
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @param {{ turnstile?: boolean, solverFallback?: boolean, challengeTimeoutMs?: number }} antiBot
|
||||
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
|
||||
* @returns {Promise<CfHandleResult>}
|
||||
*/
|
||||
async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled) {
|
||||
const started = Date.now();
|
||||
const timeoutMs = antiBot.challengeTimeoutMs || 60000;
|
||||
|
||||
let cfState = await detectCloudflare(page);
|
||||
if (!cfState.blocked) {
|
||||
return {
|
||||
success: true,
|
||||
code: null,
|
||||
challengeType: null,
|
||||
stage: null,
|
||||
solverUsed: false,
|
||||
elapsedMs: Date.now() - started,
|
||||
cfDetected: false,
|
||||
};
|
||||
}
|
||||
|
||||
console.log(`[CF] 检测到挑战 type=${cfState.challengeType} url=${cfState.url}`);
|
||||
|
||||
if (antiBot.turnstile !== false) {
|
||||
const builtIn = await waitForTurnstile(page, { timeoutMs, useBuiltInClick: true });
|
||||
if (builtIn.success) {
|
||||
if (waitForUrlSettled) {
|
||||
await waitForUrlSettled(page).catch(() => {});
|
||||
}
|
||||
cfState = await detectCloudflare(page);
|
||||
if (!cfState.blocked) {
|
||||
return {
|
||||
success: true,
|
||||
code: null,
|
||||
challengeType: cfState.challengeType,
|
||||
stage: 'built_in',
|
||||
solverUsed: false,
|
||||
elapsedMs: Date.now() - started,
|
||||
cfDetected: true,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (antiBot.solverFallback !== false && isCaptchaConfigured()) {
|
||||
try {
|
||||
const sitekey = await extractTurnstileSitekey(page);
|
||||
const pageUrl = page.url();
|
||||
const { token, provider } = await solveTurnstile({ pageUrl, sitekey });
|
||||
console.log(`[CF] Captcha API (${provider}) 返回 token,注入页面`);
|
||||
await injectTurnstileToken(page, token);
|
||||
|
||||
const apiWaitMs = Math.min(timeoutMs, 30000);
|
||||
await waitForTurnstile(page, { timeoutMs: apiWaitMs });
|
||||
|
||||
if (waitForUrlSettled) {
|
||||
await waitForUrlSettled(page).catch(() => {});
|
||||
}
|
||||
|
||||
cfState = await detectCloudflare(page);
|
||||
if (!cfState.blocked) {
|
||||
return {
|
||||
success: true,
|
||||
code: null,
|
||||
challengeType: 'turnstile',
|
||||
stage: 'api',
|
||||
solverUsed: true,
|
||||
elapsedMs: Date.now() - started,
|
||||
cfDetected: true,
|
||||
};
|
||||
}
|
||||
} catch (apiErr) {
|
||||
console.error('[CF] Captcha API 兜底失败:', apiErr.message);
|
||||
return {
|
||||
success: false,
|
||||
code: 'CF_TURNSTILE_FAILED',
|
||||
challengeType: cfState.challengeType || 'turnstile',
|
||||
stage: 'api',
|
||||
solverUsed: true,
|
||||
elapsedMs: Date.now() - started,
|
||||
cfDetected: true,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
code: 'CF_TURNSTILE_FAILED',
|
||||
challengeType: cfState.challengeType || 'unknown',
|
||||
stage: antiBot.solverFallback !== false && !isCaptchaConfigured() ? 'built_in' : 'timeout',
|
||||
solverUsed: false,
|
||||
elapsedMs: Date.now() - started,
|
||||
cfDetected: true,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
handleCloudflareChallenge,
|
||||
isCaptchaConfigured,
|
||||
};
|
||||
Executable
+82
@@ -0,0 +1,82 @@
|
||||
/**
|
||||
* Browser 并发槽位限流器:standard / real 各自独立队列
|
||||
*/
|
||||
const { QUEUE_TIMEOUT_MS } = require('./constants');
|
||||
|
||||
class BrowserConcurrencyLimiter {
|
||||
/**
|
||||
* @param {number} max 最大并发 Browser 数
|
||||
*/
|
||||
constructor(max) {
|
||||
this.max = max;
|
||||
this.active = 0;
|
||||
this.waiters = [];
|
||||
}
|
||||
|
||||
/** @returns {{ max: number, active: number, queued: number }} */
|
||||
getStats() {
|
||||
return { max: this.max, active: this.active, queued: this.waiters.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} taskName
|
||||
* @param {number} [timeoutMs]
|
||||
*/
|
||||
async acquire(taskName, timeoutMs = QUEUE_TIMEOUT_MS) {
|
||||
if (this.active < this.max) {
|
||||
this.active++;
|
||||
console.log(`[并发槽位] ${taskName} 立即获取 (${this.active}/${this.max})`);
|
||||
return;
|
||||
}
|
||||
console.log(`[并发槽位] ${taskName} 排队等待 (队列 ${this.waiters.length + 1})`);
|
||||
await new Promise((resolve, reject) => {
|
||||
const entry = {
|
||||
resolve: () => {
|
||||
this.active++;
|
||||
console.log(`[并发槽位] ${taskName} 出队执行 (${this.active}/${this.max})`);
|
||||
resolve();
|
||||
},
|
||||
reject,
|
||||
};
|
||||
if (timeoutMs > 0) {
|
||||
entry.timer = setTimeout(() => {
|
||||
const idx = this.waiters.indexOf(entry);
|
||||
if (idx >= 0) {
|
||||
this.waiters.splice(idx, 1);
|
||||
reject(new Error('QUEUE_TIMEOUT'));
|
||||
}
|
||||
}, timeoutMs);
|
||||
}
|
||||
this.waiters.push(entry);
|
||||
});
|
||||
}
|
||||
|
||||
/** @param {string} taskName */
|
||||
release(taskName) {
|
||||
this.active = Math.max(0, this.active - 1);
|
||||
const next = this.waiters.shift();
|
||||
if (next) {
|
||||
if (next.timer) clearTimeout(next.timer);
|
||||
next.resolve();
|
||||
} else {
|
||||
console.log(`[并发槽位] ${taskName} 释放 (${this.active}/${this.max})`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @template T
|
||||
* @param {string} taskName
|
||||
* @param {() => Promise<T>} fn
|
||||
* @returns {Promise<T>}
|
||||
*/
|
||||
async run(taskName, fn) {
|
||||
await this.acquire(taskName);
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
this.release(taskName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { BrowserConcurrencyLimiter };
|
||||
Executable
+140
@@ -0,0 +1,140 @@
|
||||
/**
|
||||
* puppeteer-api 共享常量与 Chrome 启动环境工具
|
||||
*/
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const puppeteer = require('puppeteer-extra');
|
||||
|
||||
/** 默认 Chrome 可执行文件路径(生产环境可通过环境变量覆盖) */
|
||||
const DEFAULT_CHROME_EXECUTABLE = '/www/wwwroot/puppeteer-api/.cache/puppeteer/chrome/linux-149.0.7827.22/chrome-linux64/chrome';
|
||||
|
||||
/** 默认 User-Agent,与 standard / real 双轨保持一致 */
|
||||
const DEFAULT_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
|
||||
|
||||
/** Chrome 启动基础参数:多实例 headless 场景下复用 */
|
||||
const BASE_LAUNCH_ARGS = [
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-gpu',
|
||||
'--disable-extensions',
|
||||
'--disable-background-networking',
|
||||
'--disable-software-rasterizer',
|
||||
'--no-first-run',
|
||||
'--no-default-browser-check',
|
||||
'--disable-breakpad',
|
||||
'--disable-crash-reporter',
|
||||
];
|
||||
|
||||
/** puppeteer-api 专用运行时目录(与 server.js 同级 .runtime) */
|
||||
const RUNTIME_DIR = path.join(__dirname, '..', '.runtime');
|
||||
|
||||
/**
|
||||
* 确保 .runtime 子目录存在且 www 用户可写
|
||||
* @param {string} name
|
||||
* @returns {string}
|
||||
*/
|
||||
function ensureRuntimeSubdir(name) {
|
||||
const dir = path.join(RUNTIME_DIR, name);
|
||||
fs.mkdirSync(dir, { recursive: true, mode: 0o777 });
|
||||
return dir;
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析 Chrome 可执行文件路径
|
||||
* @returns {string}
|
||||
*/
|
||||
function resolveChromeExecutable() {
|
||||
const candidates = [
|
||||
process.env.PUPPETEER_EXECUTABLE_PATH,
|
||||
process.env.CHROME_EXECUTABLE,
|
||||
DEFAULT_CHROME_EXECUTABLE,
|
||||
].filter(Boolean);
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (fs.existsSync(candidate)) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const bundled = puppeteer.executablePath();
|
||||
if (bundled && fs.existsSync(bundled)) {
|
||||
return bundled;
|
||||
}
|
||||
} catch (_) {
|
||||
/* puppeteer 未内置浏览器时忽略 */
|
||||
}
|
||||
|
||||
return candidates[0] || DEFAULT_CHROME_EXECUTABLE;
|
||||
}
|
||||
|
||||
/**
|
||||
* 生产 Linux 常注入无效 DBUS 地址;删除而非设为 /dev/null
|
||||
* @returns {NodeJS.ProcessEnv}
|
||||
*/
|
||||
function buildBrowserLaunchEnv() {
|
||||
const env = { ...process.env };
|
||||
delete env.DBUS_SESSION_BUS_ADDRESS;
|
||||
delete env.DBUS_SYSTEM_BUS_ADDRESS;
|
||||
|
||||
const xdgRuntime = ensureRuntimeSubdir('xdg-runtime');
|
||||
env.XDG_RUNTIME_DIR = xdgRuntime;
|
||||
env.XDG_CONFIG_HOME = ensureRuntimeSubdir('xdg-config');
|
||||
env.XDG_CACHE_HOME = ensureRuntimeSubdir('xdg-cache');
|
||||
env.TMPDIR = ensureRuntimeSubdir('tmp');
|
||||
|
||||
return env;
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验 Chrome 可执行文件存在
|
||||
* @returns {string}
|
||||
*/
|
||||
function assertChromeExecutable() {
|
||||
const executablePath = resolveChromeExecutable();
|
||||
if (!fs.existsSync(executablePath)) {
|
||||
throw new Error(
|
||||
`Chrome 可执行文件不存在: ${executablePath}。请在 puppeteer-api 目录执行: npx puppeteer browsers install chrome`
|
||||
);
|
||||
}
|
||||
return executablePath;
|
||||
}
|
||||
|
||||
/** 并发与超时配置(可通过环境变量覆盖) */
|
||||
const MAX_CONCURRENT_BROWSERS = Math.max(1, parseInt(process.env.MAX_CONCURRENT_BROWSERS || '4', 10));
|
||||
const MAX_CONCURRENT_BROWSERS_REAL = Math.max(1, parseInt(process.env.MAX_CONCURRENT_BROWSERS_REAL || '2', 10));
|
||||
const QUEUE_TIMEOUT_MS = Math.max(5000, parseInt(process.env.QUEUE_TIMEOUT_MS || '120000', 10));
|
||||
const API_INTERCEPT_TIMEOUT_MS = Math.max(10000, parseInt(process.env.API_INTERCEPT_TIMEOUT_MS || '30000', 10));
|
||||
const API_INTERCEPT_TIMEOUT_REAL_MS = Math.max(
|
||||
API_INTERCEPT_TIMEOUT_MS,
|
||||
parseInt(process.env.API_INTERCEPT_TIMEOUT_REAL_MS || '90000', 10)
|
||||
);
|
||||
|
||||
/** Captcha API 配置 */
|
||||
const CAPTCHA_PROVIDER = (process.env.CAPTCHA_PROVIDER || 'capsolver').toLowerCase();
|
||||
const CAPTCHA_API_KEY = process.env.CAPTCHA_API_KEY || '';
|
||||
const CAPTCHA_MAX_WAIT_MS = Math.max(30000, parseInt(process.env.CAPTCHA_MAX_WAIT_MS || '120000', 10));
|
||||
|
||||
/** 会话 Cookie 持久化 TTL(毫秒) */
|
||||
const SESSION_TTL_MS = Math.max(60000, parseInt(process.env.SESSION_TTL_MS || '1800000', 10));
|
||||
|
||||
module.exports = {
|
||||
DEFAULT_CHROME_EXECUTABLE,
|
||||
DEFAULT_UA,
|
||||
BASE_LAUNCH_ARGS,
|
||||
RUNTIME_DIR,
|
||||
ensureRuntimeSubdir,
|
||||
resolveChromeExecutable,
|
||||
buildBrowserLaunchEnv,
|
||||
assertChromeExecutable,
|
||||
MAX_CONCURRENT_BROWSERS,
|
||||
MAX_CONCURRENT_BROWSERS_REAL,
|
||||
QUEUE_TIMEOUT_MS,
|
||||
API_INTERCEPT_TIMEOUT_MS,
|
||||
API_INTERCEPT_TIMEOUT_REAL_MS,
|
||||
CAPTCHA_PROVIDER,
|
||||
CAPTCHA_API_KEY,
|
||||
CAPTCHA_MAX_WAIT_MS,
|
||||
SESSION_TTL_MS,
|
||||
};
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
/**
|
||||
* Real Browser 启动:puppeteer-real-browser(抗 Cloudflare Turnstile)
|
||||
* 注意:不与 puppeteer-extra 混用在同一 Browser 实例
|
||||
*/
|
||||
const {
|
||||
BASE_LAUNCH_ARGS,
|
||||
resolveChromeExecutable,
|
||||
} = require('./constants');
|
||||
|
||||
/** @type {boolean|null} */
|
||||
let realBrowserModuleChecked = null;
|
||||
|
||||
/** @type {boolean} */
|
||||
let realBrowserAvailable = false;
|
||||
|
||||
/**
|
||||
* 检测 puppeteer-real-browser 是否已安装
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function isRealBrowserAvailable() {
|
||||
if (realBrowserModuleChecked !== null) {
|
||||
return realBrowserAvailable;
|
||||
}
|
||||
try {
|
||||
require.resolve('puppeteer-real-browser');
|
||||
realBrowserAvailable = true;
|
||||
} catch (_) {
|
||||
realBrowserAvailable = false;
|
||||
}
|
||||
realBrowserModuleChecked = true;
|
||||
return realBrowserAvailable;
|
||||
}
|
||||
|
||||
/**
|
||||
* 启动 real profile Browser(headless:false + Xvfb + turnstile 内置点击)
|
||||
* @param {string[]} [extraArgs]
|
||||
* @returns {Promise<{ browser: import('puppeteer').Browser, page: import('puppeteer').Page }>}
|
||||
*/
|
||||
async function launchRealBrowser(extraArgs = []) {
|
||||
if (!isRealBrowserAvailable()) {
|
||||
throw new Error(
|
||||
'puppeteer-real-browser 未安装。请在 puppeteer-api 目录执行: npm install puppeteer-real-browser@1.4.4'
|
||||
);
|
||||
}
|
||||
|
||||
const { connect } = require('puppeteer-real-browser');
|
||||
const chromePath = resolveChromeExecutable();
|
||||
|
||||
const result = await connect({
|
||||
headless: false,
|
||||
turnstile: true,
|
||||
disableXvfb: false,
|
||||
customConfig: { chromePath },
|
||||
args: [...BASE_LAUNCH_ARGS, '--mute-audio', ...extraArgs],
|
||||
});
|
||||
|
||||
const browser = result.browser;
|
||||
const page = result.page;
|
||||
|
||||
if (!browser || !page) {
|
||||
throw new Error('puppeteer-real-browser connect 未返回 browser/page');
|
||||
}
|
||||
|
||||
return { browser, page };
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
isRealBrowserAvailable,
|
||||
launchRealBrowser,
|
||||
};
|
||||
Executable
+58
@@ -0,0 +1,58 @@
|
||||
/**
|
||||
* Standard Browser 启动:puppeteer-extra + StealthPlugin
|
||||
*/
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const puppeteer = require('puppeteer-extra');
|
||||
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
|
||||
const {
|
||||
BASE_LAUNCH_ARGS,
|
||||
assertChromeExecutable,
|
||||
buildBrowserLaunchEnv,
|
||||
ensureRuntimeSubdir,
|
||||
} = require('./constants');
|
||||
|
||||
// Stealth 必须在 launch 之前注册
|
||||
puppeteer.use(StealthPlugin());
|
||||
|
||||
/**
|
||||
* 启动 standard profile Browser
|
||||
* @param {string[]} [extraArgs]
|
||||
* @returns {Promise<import('puppeteer').Browser>}
|
||||
*/
|
||||
async function launchStandardBrowser(extraArgs = []) {
|
||||
const executablePath = assertChromeExecutable();
|
||||
const userDataDir = fs.mkdtempSync(path.join(ensureRuntimeSubdir('chrome-profiles'), 'run-'));
|
||||
|
||||
try {
|
||||
const browser = await puppeteer.launch({
|
||||
executablePath,
|
||||
headless: 'new',
|
||||
args: [...BASE_LAUNCH_ARGS, ...extraArgs],
|
||||
env: buildBrowserLaunchEnv(),
|
||||
userDataDir,
|
||||
});
|
||||
const originalClose = browser.close.bind(browser);
|
||||
browser.close = async () => {
|
||||
try {
|
||||
await originalClose();
|
||||
} finally {
|
||||
try {
|
||||
fs.rmSync(userDataDir, { recursive: true, force: true });
|
||||
} catch (_) {
|
||||
/* 清理失败不影响主流程 */
|
||||
}
|
||||
}
|
||||
};
|
||||
return browser;
|
||||
} catch (err) {
|
||||
try {
|
||||
fs.rmSync(userDataDir, { recursive: true, force: true });
|
||||
} catch (_) {
|
||||
/* ignore */
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { launchStandardBrowser };
|
||||
Executable
+121
@@ -0,0 +1,121 @@
|
||||
/**
|
||||
* 会话 Cookie 持久化:按 sessionKey 保存 cf_clearance 等,降低重复过盾概率
|
||||
*/
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
const { RUNTIME_DIR, SESSION_TTL_MS } = require('./constants');
|
||||
|
||||
const SESSIONS_DIR = path.join(RUNTIME_DIR, 'sessions');
|
||||
|
||||
/**
|
||||
* 确保 sessions 目录存在
|
||||
*/
|
||||
function ensureSessionsDir() {
|
||||
fs.mkdirSync(SESSIONS_DIR, { recursive: true, mode: 0o777 });
|
||||
}
|
||||
|
||||
/**
|
||||
* sessionKey 转安全文件名
|
||||
* @param {string} sessionKey
|
||||
* @returns {string}
|
||||
*/
|
||||
function keyToFilename(sessionKey) {
|
||||
const hash = crypto.createHash('md5').update(sessionKey).digest('hex');
|
||||
return `${hash}.json`;
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef {Object} StoredSession
|
||||
* @property {string} sessionKey
|
||||
* @property {number} savedAt
|
||||
* @property {number} expiresAt
|
||||
* @property {Array<{name:string,value:string,domain?:string,path?:string}>} cookies
|
||||
*/
|
||||
|
||||
/**
|
||||
* 加载已保存的会话 cookies
|
||||
* @param {string} sessionKey
|
||||
* @returns {StoredSession|null}
|
||||
*/
|
||||
function loadSession(sessionKey) {
|
||||
if (!sessionKey) return null;
|
||||
ensureSessionsDir();
|
||||
const filePath = path.join(SESSIONS_DIR, keyToFilename(sessionKey));
|
||||
if (!fs.existsSync(filePath)) return null;
|
||||
|
||||
try {
|
||||
const raw = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
||||
if (!raw || !Array.isArray(raw.cookies)) return null;
|
||||
if (raw.expiresAt && Date.now() > raw.expiresAt) {
|
||||
fs.unlinkSync(filePath);
|
||||
return null;
|
||||
}
|
||||
return raw;
|
||||
} catch (_) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存会话 cookies(优先保留 CF 相关 cookie)
|
||||
* @param {string} sessionKey
|
||||
* @param {Array<{name:string,value:string,domain?:string,path?:string}>} cookies
|
||||
* @param {number} [ttlMs]
|
||||
*/
|
||||
function saveSession(sessionKey, cookies, ttlMs = SESSION_TTL_MS) {
|
||||
if (!sessionKey || !Array.isArray(cookies) || cookies.length === 0) return;
|
||||
ensureSessionsDir();
|
||||
|
||||
const cfRelated = cookies.filter((c) =>
|
||||
['cf_clearance', '__cf_bm', 'cf_chl_2'].includes(c.name)
|
||||
|| c.name.startsWith('__cf')
|
||||
);
|
||||
|
||||
const toSave = cfRelated.length > 0 ? cfRelated : cookies;
|
||||
const now = Date.now();
|
||||
const payload = {
|
||||
sessionKey,
|
||||
savedAt: now,
|
||||
expiresAt: now + ttlMs,
|
||||
cookies: toSave.map((c) => ({
|
||||
name: c.name,
|
||||
value: c.value,
|
||||
domain: c.domain,
|
||||
path: c.path || '/',
|
||||
})),
|
||||
};
|
||||
|
||||
const filePath = path.join(SESSIONS_DIR, keyToFilename(sessionKey));
|
||||
fs.writeFileSync(filePath, JSON.stringify(payload, null, 0), { mode: 0o666 });
|
||||
}
|
||||
|
||||
/**
|
||||
* 将会话 cookies 转为 puppeteer setCookie 格式
|
||||
* @param {StoredSession|null} session
|
||||
* @param {string} pageUrl
|
||||
* @returns {Array<{name:string,value:string,domain?:string,path?:string,url?:string}>}
|
||||
*/
|
||||
function sessionCookiesForPage(session, pageUrl) {
|
||||
if (!session || !Array.isArray(session.cookies)) return [];
|
||||
let hostname = '';
|
||||
try {
|
||||
hostname = new URL(pageUrl).hostname;
|
||||
} catch (_) {
|
||||
return session.cookies;
|
||||
}
|
||||
|
||||
return session.cookies.map((c) => {
|
||||
const cookie = { ...c };
|
||||
if (!cookie.domain && hostname) {
|
||||
cookie.url = `https://${hostname}/`;
|
||||
}
|
||||
return cookie;
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
loadSession,
|
||||
saveSession,
|
||||
sessionCookiesForPage,
|
||||
};
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
/**
|
||||
* Turnstile 内置求解:等待 widget 自动完成 + token 轮询
|
||||
*/
|
||||
const { detectCloudflare, isTurnstileSolved } = require('./cf-detector');
|
||||
|
||||
/**
|
||||
* @typedef {Object} TurnstileResult
|
||||
* @property {boolean} success
|
||||
* @property {string} stage built_in|timeout|already_clear
|
||||
* @property {number} elapsedMs
|
||||
*/
|
||||
|
||||
/**
|
||||
* 等待 Turnstile 通过(内置点击 / 自动跳转)
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @param {{ timeoutMs?: number, pollMs?: number, useBuiltInClick?: boolean }} [options]
|
||||
* @returns {Promise<TurnstileResult>}
|
||||
*/
|
||||
async function waitForTurnstile(page, options = {}) {
|
||||
const timeoutMs = options.timeoutMs ?? 60000;
|
||||
const pollMs = options.pollMs ?? 500;
|
||||
const started = Date.now();
|
||||
|
||||
const initial = await detectCloudflare(page);
|
||||
if (!initial.blocked || initial.hasCfClearance) {
|
||||
return { success: true, stage: 'already_clear', elapsedMs: Date.now() - started };
|
||||
}
|
||||
|
||||
// real browser 已开启 turnstile:true 内置点击;此处轮询等待结果
|
||||
while (Date.now() - started < timeoutMs) {
|
||||
if (await isTurnstileSolved(page)) {
|
||||
return { success: true, stage: 'built_in', elapsedMs: Date.now() - started };
|
||||
}
|
||||
|
||||
const state = await detectCloudflare(page);
|
||||
if (!state.blocked) {
|
||||
return { success: true, stage: 'built_in', elapsedMs: Date.now() - started };
|
||||
}
|
||||
|
||||
await new Promise((r) => setTimeout(r, pollMs));
|
||||
}
|
||||
|
||||
return { success: false, stage: 'timeout', elapsedMs: Date.now() - started };
|
||||
}
|
||||
|
||||
/**
|
||||
* 从页面提取 Turnstile sitekey
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @returns {Promise<string|null>}
|
||||
*/
|
||||
async function extractTurnstileSitekey(page) {
|
||||
return page.evaluate(() => {
|
||||
const widget = document.querySelector('.cf-turnstile[data-sitekey], [data-sitekey]');
|
||||
if (widget) {
|
||||
return widget.getAttribute('data-sitekey');
|
||||
}
|
||||
const iframe = document.querySelector('iframe[src*="challenges.cloudflare.com"]');
|
||||
if (iframe && iframe.src) {
|
||||
const match = iframe.src.match(/[?&]sitekey=([^&]+)/);
|
||||
if (match) return decodeURIComponent(match[1]);
|
||||
}
|
||||
return null;
|
||||
}).catch(() => null);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 Captcha API 返回的 token 注入页面
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @param {string} token
|
||||
*/
|
||||
async function injectTurnstileToken(page, token) {
|
||||
await page.evaluate((turnstileToken) => {
|
||||
const input = document.querySelector('[name="cf-turnstile-response"]');
|
||||
if (input) {
|
||||
input.value = turnstileToken;
|
||||
input.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
input.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
}
|
||||
const form = document.querySelector('form');
|
||||
if (form) {
|
||||
form.submit();
|
||||
}
|
||||
}, token);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
waitForTurnstile,
|
||||
extractTurnstileSitekey,
|
||||
injectTurnstileToken,
|
||||
};
|
||||
+3103
File diff suppressed because it is too large
Load Diff
Executable
+20
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "puppeteer-api",
|
||||
"version": "1.0.0",
|
||||
"description": "Puppeteer Microservice",
|
||||
"main": "server.js",
|
||||
"scripts": {
|
||||
"start": "node server.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"express": "^4.18.2",
|
||||
"puppeteer": "^25.1.0",
|
||||
"puppeteer-extra": "^3.3.6",
|
||||
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
||||
"puppeteer-real-browser": "1.4.4"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"type": "commonjs"
|
||||
}
|
||||
Executable
+887
@@ -0,0 +1,887 @@
|
||||
const express = require('express');
|
||||
const fs = require('fs');
|
||||
const crypto = require('crypto');
|
||||
const {
|
||||
rewriteHttpsToHttpForLiteralIp,
|
||||
attachHttpIpRewriteInterceptor,
|
||||
} = require('./http-ip-rewrite');
|
||||
const {
|
||||
resolveHttpRedirects,
|
||||
extractShareTokenFromUrl,
|
||||
} = require('./url-resolve');
|
||||
const {
|
||||
DEFAULT_UA,
|
||||
RUNTIME_DIR,
|
||||
resolveChromeExecutable,
|
||||
MAX_CONCURRENT_BROWSERS,
|
||||
MAX_CONCURRENT_BROWSERS_REAL,
|
||||
API_INTERCEPT_TIMEOUT_MS,
|
||||
API_INTERCEPT_TIMEOUT_REAL_MS,
|
||||
} = require('./lib/constants');
|
||||
const {
|
||||
normalizeAntiBot,
|
||||
createBrowserSession,
|
||||
runWithProfileLimit,
|
||||
createManagedPage,
|
||||
getFactoryStats,
|
||||
standardLimiter,
|
||||
} = require('./lib/browser-factory');
|
||||
const { launchStandardBrowser } = require('./lib/launch-standard');
|
||||
const { isRealBrowserAvailable } = require('./lib/launch-real-browser');
|
||||
const { handleCloudflareChallenge, isCaptchaConfigured } = require('./lib/cf-handler');
|
||||
const { loadSession, saveSession, sessionCookiesForPage } = require('./lib/session-store');
|
||||
|
||||
const app = express();
|
||||
app.use(express.json({ limit: '50mb' }));
|
||||
|
||||
let shuttingDown = false;
|
||||
|
||||
/** HTTP 层:槽位排队 + 队列超时/关闭中 的统一错误响应 */
|
||||
async function runBrowserTask(taskName, handler, profile = 'standard') {
|
||||
if (shuttingDown) {
|
||||
const err = new Error('SERVICE_SHUTTING_DOWN');
|
||||
err.code = 'SERVICE_SHUTTING_DOWN';
|
||||
throw err;
|
||||
}
|
||||
return runWithProfileLimit(taskName, profile, handler);
|
||||
}
|
||||
|
||||
function sendTaskError(res, error, extra = {}) {
|
||||
if (error.code === 'QUEUE_TIMEOUT') {
|
||||
return res.status(503).json({
|
||||
success: false,
|
||||
error: '服务繁忙,排队超时,请稍后重试',
|
||||
queue: standardLimiter.getStats(),
|
||||
...extra,
|
||||
});
|
||||
}
|
||||
if (error.code === 'SERVICE_SHUTTING_DOWN') {
|
||||
return res.status(503).json({ success: false, error: '服务正在关闭', ...extra });
|
||||
}
|
||||
const errType = classifyPuppeteerError(error);
|
||||
console.error(`[${errType}]`, error.message);
|
||||
return res.status(500).json({ success: false, error: error.message, ...extra });
|
||||
}
|
||||
|
||||
function generateSafeHash(obj) {
|
||||
if (!obj) return '';
|
||||
const clone = JSON.parse(JSON.stringify(obj));
|
||||
const removeDynamicKeys = (target) => {
|
||||
if (typeof target !== 'object' || target === null) return;
|
||||
['timestamp', 'time', 'serverTime', 'reqId', 'requestId', 'traceId'].forEach((k) => delete target[k]);
|
||||
Object.values(target).forEach((val) => removeDynamicKeys(val));
|
||||
};
|
||||
removeDynamicKeys(clone);
|
||||
return crypto.createHash('md5').update(JSON.stringify(clone)).digest('hex');
|
||||
}
|
||||
|
||||
async function safeCloseBrowser(browser) {
|
||||
if (!browser) return;
|
||||
try {
|
||||
const pages = await browser.pages();
|
||||
await Promise.all(pages.map((p) => p.close().catch(() => {})));
|
||||
await browser.close();
|
||||
} catch (_) {
|
||||
try {
|
||||
const proc = browser.process();
|
||||
if (proc && !proc.killed) proc.kill('SIGKILL');
|
||||
} catch (_) { /* ignore */ }
|
||||
}
|
||||
}
|
||||
|
||||
function classifyPuppeteerError(err) {
|
||||
const msg = (err?.message || '').toLowerCase();
|
||||
if (err?.name === 'TimeoutError' || msg.includes('timeout')) return 'timeout';
|
||||
if (msg.includes('net::') || msg.includes('network') || msg.includes('err_connection')
|
||||
|| msg.includes('err_address_unreachable') || msg.includes('err_name_not_resolved')) return 'network';
|
||||
if (msg.includes('navigation') || msg.includes('frame was detached')) return 'navigation';
|
||||
if (msg.includes('target closed') || msg.includes('session closed')) return 'target_closed';
|
||||
return 'unknown';
|
||||
}
|
||||
|
||||
function isRetryableError(err) {
|
||||
const type = classifyPuppeteerError(err);
|
||||
return type === 'timeout' || type === 'network' || type === 'navigation';
|
||||
}
|
||||
|
||||
async function withExponentialBackoff(fn, {
|
||||
maxRetries = 3,
|
||||
baseDelayMs = 500,
|
||||
maxDelayMs = 8000,
|
||||
shouldRetry = isRetryableError,
|
||||
} = {}) {
|
||||
let lastError;
|
||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||
try {
|
||||
return await fn(attempt);
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
if (attempt >= maxRetries || !shouldRetry(err)) throw err;
|
||||
const delay = Math.min(baseDelayMs * Math.pow(2, attempt), maxDelayMs);
|
||||
await new Promise((r) => setTimeout(r, delay));
|
||||
}
|
||||
}
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
async function waitForUrlSettled(page, { maxAttempts = 15, stableMs = 500, redirectSettleMs = 1500 } = {}) {
|
||||
let currentUrl = page.url();
|
||||
let urlSettled = false;
|
||||
let settleAttempts = 0;
|
||||
|
||||
while (!urlSettled && settleAttempts < maxAttempts) {
|
||||
const urlBeforeWait = currentUrl;
|
||||
await page.waitForFunction(
|
||||
(expectedUrl) => window.location.href === expectedUrl,
|
||||
{ timeout: stableMs, polling: 100 },
|
||||
urlBeforeWait
|
||||
).catch(() => {});
|
||||
|
||||
const newUrl = page.url();
|
||||
if (newUrl === currentUrl) {
|
||||
urlSettled = true;
|
||||
} else {
|
||||
console.log(`[路由重定向探测] 页面从 ${currentUrl} 跳转到了 ${newUrl}`);
|
||||
currentUrl = newUrl;
|
||||
await page.waitForNetworkIdle({ idleTime: 300, timeout: redirectSettleMs }).catch(() => {});
|
||||
}
|
||||
settleAttempts++;
|
||||
}
|
||||
return currentUrl;
|
||||
}
|
||||
|
||||
async function executeAuthActions(page, authActions) {
|
||||
if (!Array.isArray(authActions)) return;
|
||||
|
||||
for (const action of authActions) {
|
||||
if (action.type === 'type') {
|
||||
await page.waitForSelector(action.selector, { timeout: 15000 });
|
||||
await page.type(action.selector, action.value, { delay: 100 });
|
||||
} else if (action.type === 'click') {
|
||||
await page.waitForSelector(action.selector, { timeout: 15000 });
|
||||
await page.evaluate((sel) => { const el = document.querySelector(sel); if (el) el.click(); }, action.selector);
|
||||
} else if (action.type === 'press') {
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
await page.keyboard.press(action.key);
|
||||
} else if (action.type === 'wait') {
|
||||
await new Promise((r) => setTimeout(r, action.ms));
|
||||
} else if (action.type === 'vue_fill') {
|
||||
await page.waitForSelector(action.selector, { timeout: 15000 });
|
||||
await page.waitForFunction(
|
||||
(sel) => {
|
||||
const el = document.querySelector(sel);
|
||||
if (!el) return false;
|
||||
const rect = el.getBoundingClientRect();
|
||||
return rect.width > 0 && rect.height > 0;
|
||||
},
|
||||
{ timeout: 2000, polling: 100 },
|
||||
action.selector
|
||||
).catch(() => new Promise((r) => setTimeout(r, 500)));
|
||||
|
||||
await page.evaluate((sel, val) => {
|
||||
const inputs = document.querySelectorAll(sel);
|
||||
for (const input of inputs) {
|
||||
if (input.getBoundingClientRect().width > 0 && input.getBoundingClientRect().height > 0) {
|
||||
input.value = val;
|
||||
input.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
input.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
input.focus();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}, action.selector, action.value);
|
||||
} else if (action.type === 'vue_click') {
|
||||
await page.waitForSelector(action.selector, { timeout: 15000 });
|
||||
await page.waitForFunction(
|
||||
(sel) => {
|
||||
const el = document.querySelector(sel);
|
||||
if (!el) return false;
|
||||
const rect = el.getBoundingClientRect();
|
||||
return rect.width > 0 && rect.height > 0;
|
||||
},
|
||||
{ timeout: 2000, polling: 100 },
|
||||
action.selector
|
||||
).catch(() => new Promise((r) => setTimeout(r, 500)));
|
||||
|
||||
await page.evaluate((sel, matchText) => {
|
||||
const btns = document.querySelectorAll(sel);
|
||||
for (const btn of btns) {
|
||||
if (btn.getBoundingClientRect().width > 0 && btn.getBoundingClientRect().height > 0) {
|
||||
let shouldClick = false;
|
||||
if (matchText) {
|
||||
const text = (btn.textContent || btn.innerText || '').trim();
|
||||
if (text.includes(matchText.trim())) { shouldClick = true; }
|
||||
} else {
|
||||
shouldClick = true;
|
||||
}
|
||||
if (shouldClick) {
|
||||
btn.scrollIntoView({ behavior: 'instant', block: 'center' });
|
||||
const eventOpts = { bubbles: true, cancelable: true, view: window };
|
||||
btn.dispatchEvent(new MouseEvent('mousedown', eventOpts));
|
||||
btn.dispatchEvent(new MouseEvent('mouseup', eventOpts));
|
||||
btn.dispatchEvent(new MouseEvent('click', eventOpts));
|
||||
btn.click();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}, action.selector, action.text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function createApiInterceptor(page, apiUrls, defaultTimeoutMs = API_INTERCEPT_TIMEOUT_MS) {
|
||||
const interceptedApis = {};
|
||||
let responseHandler = null;
|
||||
let timeoutId = null;
|
||||
let resolveWait = null;
|
||||
let waitPromise = null;
|
||||
|
||||
const tryResolveWait = () => {
|
||||
if (!resolveWait || Object.keys(interceptedApis).length < apiUrls.length) {
|
||||
return;
|
||||
}
|
||||
if (timeoutId) {
|
||||
clearTimeout(timeoutId);
|
||||
timeoutId = null;
|
||||
}
|
||||
if (responseHandler) {
|
||||
page.off('response', responseHandler);
|
||||
responseHandler = null;
|
||||
}
|
||||
const done = resolveWait;
|
||||
resolveWait = null;
|
||||
done();
|
||||
};
|
||||
|
||||
const cleanup = () => {
|
||||
if (timeoutId) {
|
||||
clearTimeout(timeoutId);
|
||||
timeoutId = null;
|
||||
}
|
||||
if (responseHandler) {
|
||||
page.off('response', responseHandler);
|
||||
responseHandler = null;
|
||||
}
|
||||
if (resolveWait) {
|
||||
const done = resolveWait;
|
||||
resolveWait = null;
|
||||
done();
|
||||
}
|
||||
};
|
||||
|
||||
responseHandler = async (response) => {
|
||||
const request = response.request();
|
||||
if (request.method() === 'OPTIONS') return;
|
||||
const matchedPath = apiUrls.find((api) => request.url().includes(api) && !interceptedApis[api]);
|
||||
if (matchedPath && response.status() >= 200 && response.status() < 300) {
|
||||
try {
|
||||
interceptedApis[matchedPath] = {
|
||||
url: request.url(),
|
||||
headers: request.headers(),
|
||||
data: JSON.parse(await response.text()),
|
||||
};
|
||||
tryResolveWait();
|
||||
} catch (_) { /* JSON 解析失败跳过 */ }
|
||||
}
|
||||
};
|
||||
page.on('response', responseHandler);
|
||||
|
||||
const beginWaiting = (timeoutMs = defaultTimeoutMs) => {
|
||||
if (waitPromise) {
|
||||
return waitPromise;
|
||||
}
|
||||
if (Object.keys(interceptedApis).length >= apiUrls.length) {
|
||||
waitPromise = Promise.resolve();
|
||||
return waitPromise;
|
||||
}
|
||||
waitPromise = new Promise((resolve) => {
|
||||
resolveWait = resolve;
|
||||
timeoutId = setTimeout(() => {
|
||||
cleanup();
|
||||
resolve();
|
||||
}, timeoutMs);
|
||||
});
|
||||
return waitPromise;
|
||||
};
|
||||
|
||||
const waitForApis = async (timeoutMs = defaultTimeoutMs) => {
|
||||
await beginWaiting(timeoutMs);
|
||||
return waitPromise;
|
||||
};
|
||||
|
||||
return { interceptedApis, beginWaiting, waitForApis, cleanup };
|
||||
}
|
||||
|
||||
async function seedShareTokenCookie(page, pageUrl) {
|
||||
try {
|
||||
const parsed = new URL(pageUrl);
|
||||
const token = parsed.searchParams.get('token');
|
||||
if (!token) return;
|
||||
const cookieUrl = `http://${parsed.hostname}/`;
|
||||
await page.setCookie({ name: 'share_token', value: token, url: cookieUrl });
|
||||
} catch (_) { /* ignore */ }
|
||||
}
|
||||
|
||||
function resolveShareToken(...urls) {
|
||||
for (const url of urls) {
|
||||
const token = extractShareTokenFromUrl(url);
|
||||
if (token) return token;
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
function resolveInterceptTimeout(antiBot) {
|
||||
const cfg = normalizeAntiBot(antiBot);
|
||||
return cfg.enabled && cfg.profile === 'real'
|
||||
? API_INTERCEPT_TIMEOUT_REAL_MS
|
||||
: API_INTERCEPT_TIMEOUT_MS;
|
||||
}
|
||||
|
||||
/** 并发监控:供上游调度器判断负载 */
|
||||
app.get('/api/stats', (_req, res) => {
|
||||
const executablePath = resolveChromeExecutable();
|
||||
const factoryStats = getFactoryStats();
|
||||
res.json({
|
||||
success: true,
|
||||
shuttingDown,
|
||||
queue: factoryStats.standard,
|
||||
queueReal: factoryStats.real,
|
||||
realBrowserReady: factoryStats.realBrowserReady,
|
||||
captchaConfigured: isCaptchaConfigured(),
|
||||
chrome: {
|
||||
executablePath,
|
||||
exists: fs.existsSync(executablePath),
|
||||
},
|
||||
config: {
|
||||
maxConcurrentBrowsers: MAX_CONCURRENT_BROWSERS,
|
||||
maxConcurrentBrowsersReal: MAX_CONCURRENT_BROWSERS_REAL,
|
||||
apiInterceptTimeoutMs: API_INTERCEPT_TIMEOUT_MS,
|
||||
apiInterceptTimeoutRealMs: API_INTERCEPT_TIMEOUT_REAL_MS,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
/** 运维自检:实际尝试 launch 一次 Browser(profile=standard|real) */
|
||||
app.get('/api/browser-launch-test', async (req, res) => {
|
||||
const profile = req.query.profile === 'real' ? 'real' : 'standard';
|
||||
let browser;
|
||||
try {
|
||||
await runBrowserTask(`browser-launch-test-${profile}`, async () => {
|
||||
if (profile === 'real') {
|
||||
const session = await createBrowserSession({ profile: 'real', extraArgs: ['--mute-audio'] });
|
||||
browser = session.browser;
|
||||
const pages = await browser.pages();
|
||||
return { pageCount: pages.length, profile };
|
||||
}
|
||||
browser = await launchStandardBrowser(['--mute-audio']);
|
||||
const pages = await browser.pages();
|
||||
return { pageCount: pages.length, profile };
|
||||
}, profile);
|
||||
res.json({ success: true, message: 'Browser 启动成功', profile });
|
||||
} catch (error) {
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: error.message,
|
||||
profile,
|
||||
chrome: {
|
||||
executablePath: resolveChromeExecutable(),
|
||||
exists: fs.existsSync(resolveChromeExecutable()),
|
||||
},
|
||||
realBrowserReady: isRealBrowserAvailable(),
|
||||
runtimeDir: RUNTIME_DIR,
|
||||
});
|
||||
} finally {
|
||||
if (browser) {
|
||||
try {
|
||||
await browser.close();
|
||||
} catch (_) { /* ignore */ }
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
/** 接口一:初始化授权与首屏拦截 */
|
||||
app.post('/api/auth-and-intercept', async (req, res) => {
|
||||
const { pageUrl, apiUrls, authActions, antiBot: rawAntiBot } = req.body;
|
||||
if (!pageUrl || !Array.isArray(apiUrls)) {
|
||||
return res.status(400).json({ success: false, error: '参数错误' });
|
||||
}
|
||||
|
||||
const antiBot = normalizeAntiBot(rawAntiBot);
|
||||
const profile = antiBot.profile || 'standard';
|
||||
const interceptTimeout = resolveInterceptTimeout(antiBot);
|
||||
const cfLog = { profile, cfDetected: false, turnstileStage: null, solverUsed: false, elapsedMs: 0 };
|
||||
|
||||
try {
|
||||
await runBrowserTask('auth-and-intercept', async () => {
|
||||
let browser;
|
||||
let sessionCleanup = null;
|
||||
let interceptorCleanup = null;
|
||||
const taskStarted = Date.now();
|
||||
|
||||
try {
|
||||
const session = await createBrowserSession({
|
||||
profile,
|
||||
extraArgs: ['--mute-audio'],
|
||||
});
|
||||
browser = session.browser;
|
||||
sessionCleanup = session.cleanup;
|
||||
|
||||
const sessionData = antiBot.sessionKey ? loadSession(antiBot.sessionKey) : null;
|
||||
const sessionCookies = sessionData
|
||||
? sessionCookiesForPage(sessionData, pageUrl)
|
||||
: [];
|
||||
|
||||
const page = await createManagedPage(browser, session.page, {
|
||||
userAgent: DEFAULT_UA,
|
||||
cookies: sessionCookies,
|
||||
});
|
||||
|
||||
const httpResolvedUrl = await resolveHttpRedirects(pageUrl).catch(() => pageUrl);
|
||||
if (httpResolvedUrl !== pageUrl) {
|
||||
console.log(`[URL解析] HTTP 重定向: ${pageUrl} -> ${httpResolvedUrl}`);
|
||||
}
|
||||
|
||||
const interceptor = createApiInterceptor(page, apiUrls, interceptTimeout);
|
||||
interceptorCleanup = interceptor.cleanup;
|
||||
|
||||
let shareToken = resolveShareToken(pageUrl, httpResolvedUrl);
|
||||
await seedShareTokenCookie(page, pageUrl);
|
||||
if (httpResolvedUrl !== pageUrl) {
|
||||
await seedShareTokenCookie(page, httpResolvedUrl);
|
||||
}
|
||||
await attachHttpIpRewriteInterceptor(page, 'auth-and-intercept', { shareToken });
|
||||
|
||||
await withExponentialBackoff(
|
||||
() => page.goto(pageUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }),
|
||||
{ maxRetries: 2, baseDelayMs: 1000 }
|
||||
);
|
||||
|
||||
let finalPageUrl = await waitForUrlSettled(page);
|
||||
if (finalPageUrl !== pageUrl) {
|
||||
console.log(`[URL解析] 浏览器落地: ${pageUrl} -> ${finalPageUrl}`);
|
||||
}
|
||||
|
||||
if (antiBot.enabled) {
|
||||
const cfResult = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled);
|
||||
cfLog.cfDetected = cfResult.cfDetected;
|
||||
cfLog.turnstileStage = cfResult.stage;
|
||||
cfLog.solverUsed = cfResult.solverUsed;
|
||||
cfLog.elapsedMs = cfResult.elapsedMs;
|
||||
|
||||
if (!cfResult.success) {
|
||||
res.status(422).json({
|
||||
success: false,
|
||||
code: cfResult.code || 'CF_TURNSTILE_FAILED',
|
||||
challengeType: cfResult.challengeType || 'turnstile',
|
||||
stage: cfResult.stage || 'timeout',
|
||||
error: 'Cloudflare Turnstile 验证失败',
|
||||
cf: cfLog,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
finalPageUrl = page.url();
|
||||
}
|
||||
|
||||
const finalShareToken = resolveShareToken(finalPageUrl);
|
||||
if (finalShareToken && finalShareToken !== shareToken) {
|
||||
shareToken = finalShareToken;
|
||||
await seedShareTokenCookie(page, finalPageUrl);
|
||||
}
|
||||
|
||||
await executeAuthActions(page, authActions);
|
||||
await interceptor.waitForApis(interceptTimeout);
|
||||
|
||||
const rawCookies = await page.cookies();
|
||||
const cookiePayload = rawCookies.map((c) => ({
|
||||
name: c.name,
|
||||
value: c.value,
|
||||
domain: c.domain,
|
||||
path: c.path,
|
||||
}));
|
||||
|
||||
if (antiBot.sessionKey) {
|
||||
saveSession(antiBot.sessionKey, cookiePayload);
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
pageUrl,
|
||||
finalPageUrl,
|
||||
interceptedApis: interceptor.interceptedApis,
|
||||
cookies: cookiePayload,
|
||||
cf: cfLog,
|
||||
profile,
|
||||
elapsedMs: Date.now() - taskStarted,
|
||||
});
|
||||
} finally {
|
||||
if (interceptorCleanup) interceptorCleanup();
|
||||
if (sessionCleanup) {
|
||||
await sessionCleanup();
|
||||
} else {
|
||||
await safeCloseBrowser(browser);
|
||||
}
|
||||
}
|
||||
}, profile);
|
||||
} catch (error) {
|
||||
if (!res.headersSent) sendTaskError(res, error, { cf: cfLog, profile });
|
||||
}
|
||||
});
|
||||
|
||||
/** 接口二:脱离 UI 极速并发拉取 */
|
||||
app.post('/api/batch-fetch', async (req, res) => {
|
||||
const { tasks, cookies, antiBot: rawAntiBot } = req.body;
|
||||
if (!Array.isArray(tasks) || tasks.length === 0) {
|
||||
return res.status(400).json({ success: false, error: 'tasks 参数错误' });
|
||||
}
|
||||
|
||||
const antiBot = normalizeAntiBot(rawAntiBot);
|
||||
const profile = antiBot.profile || 'standard';
|
||||
|
||||
try {
|
||||
await runBrowserTask('batch-fetch', async () => {
|
||||
let browser;
|
||||
let sessionCleanup = null;
|
||||
try {
|
||||
const session = await createBrowserSession({
|
||||
profile,
|
||||
extraArgs: ['--disable-web-security'],
|
||||
});
|
||||
browser = session.browser;
|
||||
sessionCleanup = session.cleanup;
|
||||
|
||||
const page = await createManagedPage(browser, session.page, { cookies });
|
||||
const firstOrigin = new URL(tasks[0].fullUrl).origin;
|
||||
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (reqObj) => {
|
||||
if (reqObj.isInterceptResolutionHandled()) return;
|
||||
const url = rewriteHttpsToHttpForLiteralIp(reqObj.url());
|
||||
const resourceType = reqObj.resourceType();
|
||||
if (['image', 'media', 'font'].includes(resourceType)) {
|
||||
reqObj.abort();
|
||||
return;
|
||||
}
|
||||
if (url === firstOrigin || url === firstOrigin + '/') {
|
||||
reqObj.respond({ status: 200, contentType: 'text/html', body: '<html><body>Context Ready</body></html>' });
|
||||
} else {
|
||||
reqObj.continue();
|
||||
}
|
||||
});
|
||||
|
||||
await withExponentialBackoff(
|
||||
() => page.goto(firstOrigin, { waitUntil: 'domcontentloaded', timeout: 30000 }),
|
||||
{ maxRetries: 2, baseDelayMs: 800 }
|
||||
);
|
||||
|
||||
const FETCH_CONCURRENCY = Math.max(1, parseInt(process.env.FETCH_CONCURRENCY || '5', 10));
|
||||
const FETCH_RETRY = 2;
|
||||
|
||||
const fetchResults = await page.evaluate(async (taskList, concurrency, maxRetry) => {
|
||||
const out = {};
|
||||
const createLimiter = (limit) => {
|
||||
let active = 0;
|
||||
const queue = [];
|
||||
return (fn) => new Promise((resolve, reject) => {
|
||||
const run = async () => {
|
||||
active++;
|
||||
try { resolve(await fn()); }
|
||||
catch (e) { reject(e); }
|
||||
finally {
|
||||
active--;
|
||||
if (queue.length) queue.shift()();
|
||||
}
|
||||
};
|
||||
if (active < limit) run();
|
||||
else queue.push(run);
|
||||
});
|
||||
};
|
||||
const limit = createLimiter(concurrency);
|
||||
|
||||
const fetchWithRetry = async (finalUrl, options, retriesLeft) => {
|
||||
try {
|
||||
const res = await fetch(finalUrl, options);
|
||||
return { success: true, data: await res.json() };
|
||||
} catch (err) {
|
||||
if (retriesLeft <= 0) return { success: false, error: err.toString() };
|
||||
await new Promise((r) => setTimeout(r, 300 * Math.pow(2, maxRetry - retriesLeft)));
|
||||
return fetchWithRetry(finalUrl, options, retriesLeft - 1);
|
||||
}
|
||||
};
|
||||
|
||||
const jobs = [];
|
||||
for (const task of taskList) {
|
||||
out[task.apiPath] = [];
|
||||
const cleanHeaders = { ...task.headers };
|
||||
['content-length', 'host', 'origin', 'referer', 'accept-encoding'].forEach((k) => delete cleanHeaders[k]);
|
||||
|
||||
for (const params of task.paramList) {
|
||||
jobs.push(limit(async () => {
|
||||
let finalUrl = task.fullUrl;
|
||||
const options = { method: task.method.toUpperCase(), headers: cleanHeaders };
|
||||
if (options.method === 'GET') {
|
||||
const qs = new URLSearchParams(params).toString();
|
||||
finalUrl = finalUrl.includes('?') ? `${finalUrl}&${qs}` : `${finalUrl}?${qs}`;
|
||||
} else {
|
||||
options.body = JSON.stringify(params);
|
||||
}
|
||||
const result = await fetchWithRetry(finalUrl, options, maxRetry);
|
||||
out[task.apiPath].push({ params, ...result });
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
}));
|
||||
}
|
||||
}
|
||||
await Promise.all(jobs);
|
||||
return out;
|
||||
}, tasks, FETCH_CONCURRENCY, FETCH_RETRY);
|
||||
|
||||
res.json({ success: true, results: fetchResults, profile });
|
||||
} finally {
|
||||
if (sessionCleanup) {
|
||||
await sessionCleanup();
|
||||
} else {
|
||||
await safeCloseBrowser(browser);
|
||||
}
|
||||
}
|
||||
}, profile);
|
||||
} catch (error) {
|
||||
if (!res.headersSent) sendTaskError(res, error, { profile });
|
||||
}
|
||||
});
|
||||
|
||||
/** 接口三:强制 UI 模拟点击拉取 */
|
||||
app.post('/api/ui-pagination', async (req, res) => {
|
||||
const {
|
||||
pageUrl,
|
||||
finalPageUrl,
|
||||
apiUrl,
|
||||
cookies,
|
||||
nextBtnSelector,
|
||||
clicksToPerform,
|
||||
waitMs = 2000,
|
||||
firstPageData,
|
||||
authActions,
|
||||
antiBot: rawAntiBot,
|
||||
} = req.body;
|
||||
const navigationUrl = finalPageUrl || pageUrl;
|
||||
const antiBot = normalizeAntiBot(rawAntiBot);
|
||||
const profile = antiBot.profile || 'standard';
|
||||
|
||||
try {
|
||||
await runBrowserTask('ui-pagination', async () => {
|
||||
let browser;
|
||||
let sessionCleanup = null;
|
||||
const capturedData = [];
|
||||
const debugLogs = [];
|
||||
const log = (msg) => { console.log(msg); debugLogs.push(msg); };
|
||||
const seenHashes = new Set();
|
||||
if (firstPageData) { seenHashes.add(generateSafeHash(firstPageData)); }
|
||||
|
||||
try {
|
||||
const session = await createBrowserSession({
|
||||
profile,
|
||||
extraArgs: ['--window-size=1920,1080'],
|
||||
});
|
||||
browser = session.browser;
|
||||
sessionCleanup = session.cleanup;
|
||||
|
||||
const page = await createManagedPage(browser, session.page, {
|
||||
viewport: { width: 1920, height: 1080 },
|
||||
cookies,
|
||||
});
|
||||
|
||||
log(`[启动] 准备访问页面: ${navigationUrl}`);
|
||||
if (finalPageUrl && finalPageUrl !== pageUrl) {
|
||||
log(`[URL解析] 使用 auth 阶段落地地址,跳过短链重定向: ${pageUrl} -> ${finalPageUrl}`);
|
||||
}
|
||||
|
||||
const shareToken = resolveShareToken(finalPageUrl, pageUrl);
|
||||
await seedShareTokenCookie(page, navigationUrl);
|
||||
await attachHttpIpRewriteInterceptor(page, 'ui-pagination', { shareToken });
|
||||
|
||||
await withExponentialBackoff(
|
||||
() => page.goto(navigationUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }),
|
||||
{ maxRetries: 2, baseDelayMs: 1000 }
|
||||
);
|
||||
|
||||
const settledPageUrl = await waitForUrlSettled(page);
|
||||
if (settledPageUrl !== navigationUrl) {
|
||||
log(`[URL解析] 翻页阶段落地: ${navigationUrl} -> ${settledPageUrl}`);
|
||||
}
|
||||
|
||||
if (antiBot.enabled) {
|
||||
const cfResult = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled);
|
||||
if (!cfResult.success) {
|
||||
res.status(422).json({
|
||||
success: false,
|
||||
code: cfResult.code || 'CF_TURNSTILE_FAILED',
|
||||
challengeType: cfResult.challengeType,
|
||||
stage: cfResult.stage,
|
||||
error: 'Cloudflare Turnstile 验证失败(翻页阶段)',
|
||||
debug: debugLogs,
|
||||
});
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (Array.isArray(authActions) && authActions.length > 0) {
|
||||
log('[状态同步] 正在为翻页引擎注入授权状态...');
|
||||
await executeAuthActions(page, authActions);
|
||||
log('[状态同步] 授权执行完毕,等待目标列表加载...');
|
||||
const authSettled = await page.waitForResponse(
|
||||
(resp) => resp.url().includes(apiUrl) && resp.request().method() !== 'OPTIONS' && resp.status() >= 200 && resp.status() < 400,
|
||||
{ timeout: 5000 }
|
||||
).catch(() => null);
|
||||
if (!authSettled) {
|
||||
log('[状态同步] 未捕获到目标 API 响应,降级为固定等待 3 秒...');
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < clicksToPerform; i++) {
|
||||
const targetPageNum = i + 2;
|
||||
let pageData = null;
|
||||
let attempts = 0;
|
||||
const maxAttempts = 3;
|
||||
|
||||
await page.evaluate(() => {
|
||||
document.querySelectorAll('.el-loading-mask, .ant-spin-nested-loading, .el-loading-spinner').forEach((el) => el.remove());
|
||||
});
|
||||
|
||||
await page.waitForFunction(
|
||||
() => !document.querySelector('.el-loading-mask, .ant-spin-nested-loading, .el-loading-spinner'),
|
||||
{ timeout: 1000, polling: 100 }
|
||||
).catch(() => new Promise((r) => setTimeout(r, 500)));
|
||||
|
||||
while (attempts < maxAttempts && !pageData) {
|
||||
attempts++;
|
||||
log(`[抓取] 准备抓取第 ${targetPageNum} 页 (尝试 ${attempts}/${maxAttempts})...`);
|
||||
|
||||
let responseHandler = null;
|
||||
let apiTimeoutId = null;
|
||||
|
||||
const waitForNewPageApi = new Promise((resolve) => {
|
||||
const cleanupListener = () => {
|
||||
if (apiTimeoutId) { clearTimeout(apiTimeoutId); apiTimeoutId = null; }
|
||||
if (responseHandler) { page.off('response', responseHandler); responseHandler = null; }
|
||||
};
|
||||
|
||||
responseHandler = async (response) => {
|
||||
const status = response.status();
|
||||
if (response.url().includes(apiUrl) && response.request().method() !== 'OPTIONS' && status >= 200 && status < 400) {
|
||||
try {
|
||||
const responseData = await response.json();
|
||||
const currentHash = generateSafeHash(responseData);
|
||||
if (!seenHashes.has(currentHash)) {
|
||||
seenHashes.add(currentHash);
|
||||
cleanupListener();
|
||||
resolve(responseData);
|
||||
} else {
|
||||
log('[防轮询] 抛弃重复数据,继续监听...');
|
||||
}
|
||||
} catch (_) { /* 非 JSON 响应跳过 */ }
|
||||
}
|
||||
};
|
||||
page.on('response', responseHandler);
|
||||
apiTimeoutId = setTimeout(() => { cleanupListener(); resolve(null); }, 15000);
|
||||
});
|
||||
|
||||
try {
|
||||
await page.waitForSelector(nextBtnSelector, { timeout: 15000 });
|
||||
|
||||
const clickStatus = await page.evaluate((sel) => {
|
||||
const btn = document.querySelector(sel);
|
||||
if (!btn) return 'not_found';
|
||||
if (btn.disabled || btn.classList.contains('is-disabled') || btn.classList.contains('disabled') || btn.getAttribute('aria-disabled') === 'true') {
|
||||
return 'disabled';
|
||||
}
|
||||
const className = btn.className || '';
|
||||
if (btn.disabled === true || className.includes('disabled') || btn.getAttribute('aria-disabled') === 'true') {
|
||||
return 'disabled';
|
||||
}
|
||||
btn.scrollIntoView({ behavior: 'instant', block: 'center' });
|
||||
btn.click();
|
||||
btn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
|
||||
return 'success';
|
||||
}, nextBtnSelector);
|
||||
|
||||
if (clickStatus === 'disabled') {
|
||||
log(`[提示] 第 ${targetPageNum} 页按钮已被禁用,说明到底了,抓取提前结束。`);
|
||||
break;
|
||||
}
|
||||
log('[动作] 双引擎原生点击已执行!');
|
||||
} catch (clickErr) {
|
||||
log(`[错误] 寻找按钮异常 (${classifyPuppeteerError(clickErr)}): ${clickErr.message}`);
|
||||
}
|
||||
|
||||
pageData = await waitForNewPageApi;
|
||||
if (!pageData && attempts < maxAttempts) {
|
||||
log('[重试警报] 雷达未扫描到新数据,准备重试...');
|
||||
await new Promise((r) => setTimeout(r, 500 * Math.pow(2, attempts - 1)));
|
||||
}
|
||||
}
|
||||
|
||||
if (pageData) {
|
||||
capturedData.push(pageData);
|
||||
log(`[成功] 斩获第 ${targetPageNum} 页数据!`);
|
||||
await new Promise((r) => setTimeout(r, waitMs));
|
||||
} else {
|
||||
log(`[致命异常] 连续 ${maxAttempts} 次尝试均失败,放弃后续翻页。`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
res.json({ success: true, data: capturedData, debug: debugLogs, profile });
|
||||
} catch (error) {
|
||||
if (!res.headersSent) {
|
||||
sendTaskError(res, error, { debug: debugLogs, profile });
|
||||
}
|
||||
} finally {
|
||||
if (sessionCleanup) {
|
||||
await sessionCleanup();
|
||||
} else {
|
||||
await safeCloseBrowser(browser);
|
||||
}
|
||||
}
|
||||
}, profile);
|
||||
} catch (error) {
|
||||
if (!res.headersSent) sendTaskError(res, error, { profile });
|
||||
}
|
||||
});
|
||||
|
||||
const PORT = parseInt(process.env.PORT || '3001', 10);
|
||||
const server = app.listen(PORT, '127.0.0.1', () => {
|
||||
const chromePath = resolveChromeExecutable();
|
||||
const chromeReady = fs.existsSync(chromePath);
|
||||
console.log(`🚀 Node.js 自愈型防脱轨引擎已启动 (standard: ${MAX_CONCURRENT_BROWSERS}, real: ${MAX_CONCURRENT_BROWSERS_REAL})`);
|
||||
console.log(`[chrome] path=${chromePath} exists=${chromeReady}`);
|
||||
console.log(`[real-browser] ready=${isRealBrowserAvailable()} captcha=${isCaptchaConfigured()}`);
|
||||
if (!chromeReady) {
|
||||
console.warn('[chrome] 未找到 Chrome,请在 puppeteer-api 目录执行: npx puppeteer browsers install chrome');
|
||||
}
|
||||
});
|
||||
|
||||
async function gracefulShutdown(signal) {
|
||||
console.log(`[shutdown] 收到 ${signal},停止接收新任务...`);
|
||||
shuttingDown = true;
|
||||
server.close();
|
||||
|
||||
const deadline = Date.now() + 60000;
|
||||
while (standardLimiter.active > 0 && Date.now() < deadline) {
|
||||
const { active, queued } = standardLimiter.getStats();
|
||||
console.log(`[shutdown] 等待在途任务完成... active=${active}, queued=${queued}`);
|
||||
await new Promise((r) => setTimeout(r, 1000));
|
||||
}
|
||||
|
||||
const remaining = standardLimiter.getStats();
|
||||
if (remaining.active > 0) {
|
||||
console.warn(`[shutdown] 超时,仍有 ${remaining.active} 个 Browser 在运行,强制退出`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log('[shutdown] 所有任务已完成,进程退出');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
process.on('SIGTERM', () => gracefulShutdown('SIGTERM'));
|
||||
process.on('SIGINT', () => gracefulShutdown('SIGINT'));
|
||||
Executable
+73
@@ -0,0 +1,73 @@
|
||||
/**
|
||||
* URL 重定向解析:在浏览器导航前尽可能解析 HTTP 级重定向链。
|
||||
*/
|
||||
'use strict';
|
||||
|
||||
const http = require('http');
|
||||
const https = require('https');
|
||||
|
||||
const REDIRECT_STATUS = new Set([301, 302, 303, 307, 308]);
|
||||
|
||||
/**
|
||||
* 发起单次 HEAD/GET,不自动跟随重定向。
|
||||
*/
|
||||
function fetchOnce(method, url, timeoutMs) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const parsed = new URL(url);
|
||||
const lib = parsed.protocol === 'https:' ? https : http;
|
||||
const req = lib.request(
|
||||
url,
|
||||
{ method, timeout: timeoutMs, headers: { 'User-Agent': 'Mozilla/5.0 (compatible; puppeteer-api/1.0)' } },
|
||||
(res) => {
|
||||
res.resume();
|
||||
resolve({ statusCode: res.statusCode || 0, headers: res.headers || {} });
|
||||
}
|
||||
);
|
||||
req.on('timeout', () => {
|
||||
req.destroy();
|
||||
reject(new Error('REDIRECT_TIMEOUT'));
|
||||
});
|
||||
req.on('error', reject);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 跟随 HTTP 重定向,返回最终 URL(无法解析时返回原始 URL)。
|
||||
*/
|
||||
async function resolveHttpRedirects(startUrl, { maxRedirects = 10, timeoutMs = 12000 } = {}) {
|
||||
let current = startUrl;
|
||||
for (let i = 0; i < maxRedirects; i++) {
|
||||
let response;
|
||||
try {
|
||||
response = await fetchOnce('HEAD', current, timeoutMs);
|
||||
} catch (_) {
|
||||
response = await fetchOnce('GET', current, timeoutMs);
|
||||
}
|
||||
if (!REDIRECT_STATUS.has(response.statusCode)) {
|
||||
return current;
|
||||
}
|
||||
const location = response.headers.location;
|
||||
if (!location) {
|
||||
return current;
|
||||
}
|
||||
current = new URL(location, current).href;
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
/**
|
||||
* 从分享页 URL 提取 token 参数(星河等云控)。
|
||||
*/
|
||||
function extractShareTokenFromUrl(pageUrl) {
|
||||
try {
|
||||
return new URL(pageUrl).searchParams.get('token') || '';
|
||||
} catch (_) {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
resolveHttpRedirects,
|
||||
extractShareTokenFromUrl,
|
||||
};
|
||||
Reference in New Issue
Block a user