root
888 lines
37 KiB
JavaScript
Executable File
888 lines
37 KiB
JavaScript
Executable File
const express = require('express');
|
||
const fs = require('fs');
|
||
const crypto = require('crypto');
|
||
const {
|
||
rewriteHttpsToHttpForLiteralIp,
|
||
attachHttpIpRewriteInterceptor,
|
||
} = require('./http-ip-rewrite');
|
||
const {
|
||
resolveHttpRedirects,
|
||
extractShareTokenFromUrl,
|
||
} = require('./url-resolve');
|
||
const {
|
||
DEFAULT_UA,
|
||
RUNTIME_DIR,
|
||
resolveChromeExecutable,
|
||
MAX_CONCURRENT_BROWSERS,
|
||
MAX_CONCURRENT_BROWSERS_REAL,
|
||
API_INTERCEPT_TIMEOUT_MS,
|
||
API_INTERCEPT_TIMEOUT_REAL_MS,
|
||
} = require('./lib/constants');
|
||
const {
|
||
normalizeAntiBot,
|
||
createBrowserSession,
|
||
runWithProfileLimit,
|
||
createManagedPage,
|
||
getFactoryStats,
|
||
standardLimiter,
|
||
} = require('./lib/browser-factory');
|
||
const { launchStandardBrowser } = require('./lib/launch-standard');
|
||
const { isRealBrowserAvailable } = require('./lib/launch-real-browser');
|
||
const { handleCloudflareChallenge, isCaptchaConfigured } = require('./lib/cf-handler');
|
||
const { loadSession, saveSession, sessionCookiesForPage } = require('./lib/session-store');
|
||
|
||
const app = express();
|
||
app.use(express.json({ limit: '50mb' }));
|
||
|
||
let shuttingDown = false;
|
||
|
||
/** HTTP 层：槽位排队 + 队列超时/关闭中 的统一错误响应 */
|
||
async function runBrowserTask(taskName, handler, profile = 'standard') {
|
||
if (shuttingDown) {
|
||
const err = new Error('SERVICE_SHUTTING_DOWN');
|
||
err.code = 'SERVICE_SHUTTING_DOWN';
|
||
throw err;
|
||
}
|
||
return runWithProfileLimit(taskName, profile, handler);
|
||
}
|
||
|
||
function sendTaskError(res, error, extra = {}) {
|
||
if (error.code === 'QUEUE_TIMEOUT') {
|
||
return res.status(503).json({
|
||
success: false,
|
||
error: '服务繁忙，排队超时，请稍后重试',
|
||
queue: standardLimiter.getStats(),
|
||
...extra,
|
||
});
|
||
}
|
||
if (error.code === 'SERVICE_SHUTTING_DOWN') {
|
||
return res.status(503).json({ success: false, error: '服务正在关闭', ...extra });
|
||
}
|
||
const errType = classifyPuppeteerError(error);
|
||
console.error(`[${errType}]`, error.message);
|
||
return res.status(500).json({ success: false, error: error.message, ...extra });
|
||
}
|
||
|
||
function generateSafeHash(obj) {
|
||
if (!obj) return '';
|
||
const clone = JSON.parse(JSON.stringify(obj));
|
||
const removeDynamicKeys = (target) => {
|
||
if (typeof target !== 'object' || target === null) return;
|
||
['timestamp', 'time', 'serverTime', 'reqId', 'requestId', 'traceId'].forEach((k) => delete target[k]);
|
||
Object.values(target).forEach((val) => removeDynamicKeys(val));
|
||
};
|
||
removeDynamicKeys(clone);
|
||
return crypto.createHash('md5').update(JSON.stringify(clone)).digest('hex');
|
||
}
|
||
|
||
async function safeCloseBrowser(browser) {
|
||
if (!browser) return;
|
||
try {
|
||
const pages = await browser.pages();
|
||
await Promise.all(pages.map((p) => p.close().catch(() => {})));
|
||
await browser.close();
|
||
} catch (_) {
|
||
try {
|
||
const proc = browser.process();
|
||
if (proc && !proc.killed) proc.kill('SIGKILL');
|
||
} catch (_) { /* ignore */ }
|
||
}
|
||
}
|
||
|
||
function classifyPuppeteerError(err) {
|
||
const msg = (err?.message || '').toLowerCase();
|
||
if (err?.name === 'TimeoutError' || msg.includes('timeout')) return 'timeout';
|
||
if (msg.includes('net::') || msg.includes('network') || msg.includes('err_connection')
|
||
|| msg.includes('err_address_unreachable') || msg.includes('err_name_not_resolved')) return 'network';
|
||
if (msg.includes('navigation') || msg.includes('frame was detached')) return 'navigation';
|
||
if (msg.includes('target closed') || msg.includes('session closed')) return 'target_closed';
|
||
return 'unknown';
|
||
}
|
||
|
||
function isRetryableError(err) {
|
||
const type = classifyPuppeteerError(err);
|
||
return type === 'timeout' || type === 'network' || type === 'navigation';
|
||
}
|
||
|
||
async function withExponentialBackoff(fn, {
|
||
maxRetries = 3,
|
||
baseDelayMs = 500,
|
||
maxDelayMs = 8000,
|
||
shouldRetry = isRetryableError,
|
||
} = {}) {
|
||
let lastError;
|
||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||
try {
|
||
return await fn(attempt);
|
||
} catch (err) {
|
||
lastError = err;
|
||
if (attempt >= maxRetries || !shouldRetry(err)) throw err;
|
||
const delay = Math.min(baseDelayMs * Math.pow(2, attempt), maxDelayMs);
|
||
await new Promise((r) => setTimeout(r, delay));
|
||
}
|
||
}
|
||
throw lastError;
|
||
}
|
||
|
||
async function waitForUrlSettled(page, { maxAttempts = 15, stableMs = 500, redirectSettleMs = 1500 } = {}) {
|
||
let currentUrl = page.url();
|
||
let urlSettled = false;
|
||
let settleAttempts = 0;
|
||
|
||
while (!urlSettled && settleAttempts < maxAttempts) {
|
||
const urlBeforeWait = currentUrl;
|
||
await page.waitForFunction(
|
||
(expectedUrl) => window.location.href === expectedUrl,
|
||
{ timeout: stableMs, polling: 100 },
|
||
urlBeforeWait
|
||
).catch(() => {});
|
||
|
||
const newUrl = page.url();
|
||
if (newUrl === currentUrl) {
|
||
urlSettled = true;
|
||
} else {
|
||
console.log(`[路由重定向探测] 页面从 ${currentUrl} 跳转到了 ${newUrl}`);
|
||
currentUrl = newUrl;
|
||
await page.waitForNetworkIdle({ idleTime: 300, timeout: redirectSettleMs }).catch(() => {});
|
||
}
|
||
settleAttempts++;
|
||
}
|
||
return currentUrl;
|
||
}
|
||
|
||
async function executeAuthActions(page, authActions) {
|
||
if (!Array.isArray(authActions)) return;
|
||
|
||
for (const action of authActions) {
|
||
if (action.type === 'type') {
|
||
await page.waitForSelector(action.selector, { timeout: 15000 });
|
||
await page.type(action.selector, action.value, { delay: 100 });
|
||
} else if (action.type === 'click') {
|
||
await page.waitForSelector(action.selector, { timeout: 15000 });
|
||
await page.evaluate((sel) => { const el = document.querySelector(sel); if (el) el.click(); }, action.selector);
|
||
} else if (action.type === 'press') {
|
||
await new Promise((r) => setTimeout(r, 200));
|
||
await page.keyboard.press(action.key);
|
||
} else if (action.type === 'wait') {
|
||
await new Promise((r) => setTimeout(r, action.ms));
|
||
} else if (action.type === 'vue_fill') {
|
||
await page.waitForSelector(action.selector, { timeout: 15000 });
|
||
await page.waitForFunction(
|
||
(sel) => {
|
||
const el = document.querySelector(sel);
|
||
if (!el) return false;
|
||
const rect = el.getBoundingClientRect();
|
||
return rect.width > 0 && rect.height > 0;
|
||
},
|
||
{ timeout: 2000, polling: 100 },
|
||
action.selector
|
||
).catch(() => new Promise((r) => setTimeout(r, 500)));
|
||
|
||
await page.evaluate((sel, val) => {
|
||
const inputs = document.querySelectorAll(sel);
|
||
for (const input of inputs) {
|
||
if (input.getBoundingClientRect().width > 0 && input.getBoundingClientRect().height > 0) {
|
||
input.value = val;
|
||
input.dispatchEvent(new Event('input', { bubbles: true }));
|
||
input.dispatchEvent(new Event('change', { bubbles: true }));
|
||
input.focus();
|
||
return;
|
||
}
|
||
}
|
||
}, action.selector, action.value);
|
||
} else if (action.type === 'vue_click') {
|
||
await page.waitForSelector(action.selector, { timeout: 15000 });
|
||
await page.waitForFunction(
|
||
(sel) => {
|
||
const el = document.querySelector(sel);
|
||
if (!el) return false;
|
||
const rect = el.getBoundingClientRect();
|
||
return rect.width > 0 && rect.height > 0;
|
||
},
|
||
{ timeout: 2000, polling: 100 },
|
||
action.selector
|
||
).catch(() => new Promise((r) => setTimeout(r, 500)));
|
||
|
||
await page.evaluate((sel, matchText) => {
|
||
const btns = document.querySelectorAll(sel);
|
||
for (const btn of btns) {
|
||
if (btn.getBoundingClientRect().width > 0 && btn.getBoundingClientRect().height > 0) {
|
||
let shouldClick = false;
|
||
if (matchText) {
|
||
const text = (btn.textContent || btn.innerText || '').trim();
|
||
if (text.includes(matchText.trim())) { shouldClick = true; }
|
||
} else {
|
||
shouldClick = true;
|
||
}
|
||
if (shouldClick) {
|
||
btn.scrollIntoView({ behavior: 'instant', block: 'center' });
|
||
const eventOpts = { bubbles: true, cancelable: true, view: window };
|
||
btn.dispatchEvent(new MouseEvent('mousedown', eventOpts));
|
||
btn.dispatchEvent(new MouseEvent('mouseup', eventOpts));
|
||
btn.dispatchEvent(new MouseEvent('click', eventOpts));
|
||
btn.click();
|
||
return;
|
||
}
|
||
}
|
||
}
|
||
}, action.selector, action.text);
|
||
}
|
||
}
|
||
}
|
||
|
||
function createApiInterceptor(page, apiUrls, defaultTimeoutMs = API_INTERCEPT_TIMEOUT_MS) {
|
||
const interceptedApis = {};
|
||
let responseHandler = null;
|
||
let timeoutId = null;
|
||
let resolveWait = null;
|
||
let waitPromise = null;
|
||
|
||
const tryResolveWait = () => {
|
||
if (!resolveWait || Object.keys(interceptedApis).length < apiUrls.length) {
|
||
return;
|
||
}
|
||
if (timeoutId) {
|
||
clearTimeout(timeoutId);
|
||
timeoutId = null;
|
||
}
|
||
if (responseHandler) {
|
||
page.off('response', responseHandler);
|
||
responseHandler = null;
|
||
}
|
||
const done = resolveWait;
|
||
resolveWait = null;
|
||
done();
|
||
};
|
||
|
||
const cleanup = () => {
|
||
if (timeoutId) {
|
||
clearTimeout(timeoutId);
|
||
timeoutId = null;
|
||
}
|
||
if (responseHandler) {
|
||
page.off('response', responseHandler);
|
||
responseHandler = null;
|
||
}
|
||
if (resolveWait) {
|
||
const done = resolveWait;
|
||
resolveWait = null;
|
||
done();
|
||
}
|
||
};
|
||
|
||
responseHandler = async (response) => {
|
||
const request = response.request();
|
||
if (request.method() === 'OPTIONS') return;
|
||
const matchedPath = apiUrls.find((api) => request.url().includes(api) && !interceptedApis[api]);
|
||
if (matchedPath && response.status() >= 200 && response.status() < 300) {
|
||
try {
|
||
interceptedApis[matchedPath] = {
|
||
url: request.url(),
|
||
headers: request.headers(),
|
||
data: JSON.parse(await response.text()),
|
||
};
|
||
tryResolveWait();
|
||
} catch (_) { /* JSON 解析失败跳过 */ }
|
||
}
|
||
};
|
||
page.on('response', responseHandler);
|
||
|
||
const beginWaiting = (timeoutMs = defaultTimeoutMs) => {
|
||
if (waitPromise) {
|
||
return waitPromise;
|
||
}
|
||
if (Object.keys(interceptedApis).length >= apiUrls.length) {
|
||
waitPromise = Promise.resolve();
|
||
return waitPromise;
|
||
}
|
||
waitPromise = new Promise((resolve) => {
|
||
resolveWait = resolve;
|
||
timeoutId = setTimeout(() => {
|
||
cleanup();
|
||
resolve();
|
||
}, timeoutMs);
|
||
});
|
||
return waitPromise;
|
||
};
|
||
|
||
const waitForApis = async (timeoutMs = defaultTimeoutMs) => {
|
||
await beginWaiting(timeoutMs);
|
||
return waitPromise;
|
||
};
|
||
|
||
return { interceptedApis, beginWaiting, waitForApis, cleanup };
|
||
}
|
||
|
||
async function seedShareTokenCookie(page, pageUrl) {
|
||
try {
|
||
const parsed = new URL(pageUrl);
|
||
const token = parsed.searchParams.get('token');
|
||
if (!token) return;
|
||
const cookieUrl = `http://${parsed.hostname}/`;
|
||
await page.setCookie({ name: 'share_token', value: token, url: cookieUrl });
|
||
} catch (_) { /* ignore */ }
|
||
}
|
||
|
||
function resolveShareToken(...urls) {
|
||
for (const url of urls) {
|
||
const token = extractShareTokenFromUrl(url);
|
||
if (token) return token;
|
||
}
|
||
return '';
|
||
}
|
||
|
||
function resolveInterceptTimeout(antiBot) {
|
||
const cfg = normalizeAntiBot(antiBot);
|
||
return cfg.enabled && cfg.profile === 'real'
|
||
? API_INTERCEPT_TIMEOUT_REAL_MS
|
||
: API_INTERCEPT_TIMEOUT_MS;
|
||
}
|
||
|
||
/** 并发监控：供上游调度器判断负载 */
|
||
app.get('/api/stats', (_req, res) => {
|
||
const executablePath = resolveChromeExecutable();
|
||
const factoryStats = getFactoryStats();
|
||
res.json({
|
||
success: true,
|
||
shuttingDown,
|
||
queue: factoryStats.standard,
|
||
queueReal: factoryStats.real,
|
||
realBrowserReady: factoryStats.realBrowserReady,
|
||
captchaConfigured: isCaptchaConfigured(),
|
||
chrome: {
|
||
executablePath,
|
||
exists: fs.existsSync(executablePath),
|
||
},
|
||
config: {
|
||
maxConcurrentBrowsers: MAX_CONCURRENT_BROWSERS,
|
||
maxConcurrentBrowsersReal: MAX_CONCURRENT_BROWSERS_REAL,
|
||
apiInterceptTimeoutMs: API_INTERCEPT_TIMEOUT_MS,
|
||
apiInterceptTimeoutRealMs: API_INTERCEPT_TIMEOUT_REAL_MS,
|
||
},
|
||
});
|
||
});
|
||
|
||
/** 运维自检：实际尝试 launch 一次 Browser（profile=standard|real） */
|
||
app.get('/api/browser-launch-test', async (req, res) => {
|
||
const profile = req.query.profile === 'real' ? 'real' : 'standard';
|
||
let browser;
|
||
try {
|
||
await runBrowserTask(`browser-launch-test-${profile}`, async () => {
|
||
if (profile === 'real') {
|
||
const session = await createBrowserSession({ profile: 'real', extraArgs: ['--mute-audio'] });
|
||
browser = session.browser;
|
||
const pages = await browser.pages();
|
||
return { pageCount: pages.length, profile };
|
||
}
|
||
browser = await launchStandardBrowser(['--mute-audio']);
|
||
const pages = await browser.pages();
|
||
return { pageCount: pages.length, profile };
|
||
}, profile);
|
||
res.json({ success: true, message: 'Browser 启动成功', profile });
|
||
} catch (error) {
|
||
res.status(500).json({
|
||
success: false,
|
||
error: error.message,
|
||
profile,
|
||
chrome: {
|
||
executablePath: resolveChromeExecutable(),
|
||
exists: fs.existsSync(resolveChromeExecutable()),
|
||
},
|
||
realBrowserReady: isRealBrowserAvailable(),
|
||
runtimeDir: RUNTIME_DIR,
|
||
});
|
||
} finally {
|
||
if (browser) {
|
||
try {
|
||
await browser.close();
|
||
} catch (_) { /* ignore */ }
|
||
}
|
||
}
|
||
});
|
||
|
||
/** 接口一：初始化授权与首屏拦截 */
|
||
app.post('/api/auth-and-intercept', async (req, res) => {
|
||
const { pageUrl, apiUrls, authActions, antiBot: rawAntiBot } = req.body;
|
||
if (!pageUrl || !Array.isArray(apiUrls)) {
|
||
return res.status(400).json({ success: false, error: '参数错误' });
|
||
}
|
||
|
||
const antiBot = normalizeAntiBot(rawAntiBot);
|
||
const profile = antiBot.profile || 'standard';
|
||
const interceptTimeout = resolveInterceptTimeout(antiBot);
|
||
const cfLog = { profile, cfDetected: false, turnstileStage: null, solverUsed: false, elapsedMs: 0 };
|
||
|
||
try {
|
||
await runBrowserTask('auth-and-intercept', async () => {
|
||
let browser;
|
||
let sessionCleanup = null;
|
||
let interceptorCleanup = null;
|
||
const taskStarted = Date.now();
|
||
|
||
try {
|
||
const session = await createBrowserSession({
|
||
profile,
|
||
extraArgs: ['--mute-audio'],
|
||
});
|
||
browser = session.browser;
|
||
sessionCleanup = session.cleanup;
|
||
|
||
const sessionData = antiBot.sessionKey ? loadSession(antiBot.sessionKey) : null;
|
||
const sessionCookies = sessionData
|
||
? sessionCookiesForPage(sessionData, pageUrl)
|
||
: [];
|
||
|
||
const page = await createManagedPage(browser, session.page, {
|
||
userAgent: DEFAULT_UA,
|
||
cookies: sessionCookies,
|
||
});
|
||
|
||
const httpResolvedUrl = await resolveHttpRedirects(pageUrl).catch(() => pageUrl);
|
||
if (httpResolvedUrl !== pageUrl) {
|
||
console.log(`[URL解析] HTTP 重定向: ${pageUrl} -> ${httpResolvedUrl}`);
|
||
}
|
||
|
||
const interceptor = createApiInterceptor(page, apiUrls, interceptTimeout);
|
||
interceptorCleanup = interceptor.cleanup;
|
||
|
||
let shareToken = resolveShareToken(pageUrl, httpResolvedUrl);
|
||
await seedShareTokenCookie(page, pageUrl);
|
||
if (httpResolvedUrl !== pageUrl) {
|
||
await seedShareTokenCookie(page, httpResolvedUrl);
|
||
}
|
||
await attachHttpIpRewriteInterceptor(page, 'auth-and-intercept', { shareToken });
|
||
|
||
await withExponentialBackoff(
|
||
() => page.goto(pageUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }),
|
||
{ maxRetries: 2, baseDelayMs: 1000 }
|
||
);
|
||
|
||
let finalPageUrl = await waitForUrlSettled(page);
|
||
if (finalPageUrl !== pageUrl) {
|
||
console.log(`[URL解析] 浏览器落地: ${pageUrl} -> ${finalPageUrl}`);
|
||
}
|
||
|
||
if (antiBot.enabled) {
|
||
const cfResult = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled);
|
||
cfLog.cfDetected = cfResult.cfDetected;
|
||
cfLog.turnstileStage = cfResult.stage;
|
||
cfLog.solverUsed = cfResult.solverUsed;
|
||
cfLog.elapsedMs = cfResult.elapsedMs;
|
||
|
||
if (!cfResult.success) {
|
||
res.status(422).json({
|
||
success: false,
|
||
code: cfResult.code || 'CF_TURNSTILE_FAILED',
|
||
challengeType: cfResult.challengeType || 'turnstile',
|
||
stage: cfResult.stage || 'timeout',
|
||
error: 'Cloudflare Turnstile 验证失败',
|
||
cf: cfLog,
|
||
});
|
||
return;
|
||
}
|
||
|
||
finalPageUrl = page.url();
|
||
}
|
||
|
||
const finalShareToken = resolveShareToken(finalPageUrl);
|
||
if (finalShareToken && finalShareToken !== shareToken) {
|
||
shareToken = finalShareToken;
|
||
await seedShareTokenCookie(page, finalPageUrl);
|
||
}
|
||
|
||
await executeAuthActions(page, authActions);
|
||
await interceptor.waitForApis(interceptTimeout);
|
||
|
||
const rawCookies = await page.cookies();
|
||
const cookiePayload = rawCookies.map((c) => ({
|
||
name: c.name,
|
||
value: c.value,
|
||
domain: c.domain,
|
||
path: c.path,
|
||
}));
|
||
|
||
if (antiBot.sessionKey) {
|
||
saveSession(antiBot.sessionKey, cookiePayload);
|
||
}
|
||
|
||
res.json({
|
||
success: true,
|
||
pageUrl,
|
||
finalPageUrl,
|
||
interceptedApis: interceptor.interceptedApis,
|
||
cookies: cookiePayload,
|
||
cf: cfLog,
|
||
profile,
|
||
elapsedMs: Date.now() - taskStarted,
|
||
});
|
||
} finally {
|
||
if (interceptorCleanup) interceptorCleanup();
|
||
if (sessionCleanup) {
|
||
await sessionCleanup();
|
||
} else {
|
||
await safeCloseBrowser(browser);
|
||
}
|
||
}
|
||
}, profile);
|
||
} catch (error) {
|
||
if (!res.headersSent) sendTaskError(res, error, { cf: cfLog, profile });
|
||
}
|
||
});
|
||
|
||
/** 接口二：脱离 UI 极速并发拉取 */
|
||
app.post('/api/batch-fetch', async (req, res) => {
|
||
const { tasks, cookies, antiBot: rawAntiBot } = req.body;
|
||
if (!Array.isArray(tasks) || tasks.length === 0) {
|
||
return res.status(400).json({ success: false, error: 'tasks 参数错误' });
|
||
}
|
||
|
||
const antiBot = normalizeAntiBot(rawAntiBot);
|
||
const profile = antiBot.profile || 'standard';
|
||
|
||
try {
|
||
await runBrowserTask('batch-fetch', async () => {
|
||
let browser;
|
||
let sessionCleanup = null;
|
||
try {
|
||
const session = await createBrowserSession({
|
||
profile,
|
||
extraArgs: ['--disable-web-security'],
|
||
});
|
||
browser = session.browser;
|
||
sessionCleanup = session.cleanup;
|
||
|
||
const page = await createManagedPage(browser, session.page, { cookies });
|
||
const firstOrigin = new URL(tasks[0].fullUrl).origin;
|
||
|
||
await page.setRequestInterception(true);
|
||
page.on('request', (reqObj) => {
|
||
if (reqObj.isInterceptResolutionHandled()) return;
|
||
const url = rewriteHttpsToHttpForLiteralIp(reqObj.url());
|
||
const resourceType = reqObj.resourceType();
|
||
if (['image', 'media', 'font'].includes(resourceType)) {
|
||
reqObj.abort();
|
||
return;
|
||
}
|
||
if (url === firstOrigin || url === firstOrigin + '/') {
|
||
reqObj.respond({ status: 200, contentType: 'text/html', body: '<html><body>Context Ready</body></html>' });
|
||
} else {
|
||
reqObj.continue();
|
||
}
|
||
});
|
||
|
||
await withExponentialBackoff(
|
||
() => page.goto(firstOrigin, { waitUntil: 'domcontentloaded', timeout: 30000 }),
|
||
{ maxRetries: 2, baseDelayMs: 800 }
|
||
);
|
||
|
||
const FETCH_CONCURRENCY = Math.max(1, parseInt(process.env.FETCH_CONCURRENCY || '5', 10));
|
||
const FETCH_RETRY = 2;
|
||
|
||
const fetchResults = await page.evaluate(async (taskList, concurrency, maxRetry) => {
|
||
const out = {};
|
||
const createLimiter = (limit) => {
|
||
let active = 0;
|
||
const queue = [];
|
||
return (fn) => new Promise((resolve, reject) => {
|
||
const run = async () => {
|
||
active++;
|
||
try { resolve(await fn()); }
|
||
catch (e) { reject(e); }
|
||
finally {
|
||
active--;
|
||
if (queue.length) queue.shift()();
|
||
}
|
||
};
|
||
if (active < limit) run();
|
||
else queue.push(run);
|
||
});
|
||
};
|
||
const limit = createLimiter(concurrency);
|
||
|
||
const fetchWithRetry = async (finalUrl, options, retriesLeft) => {
|
||
try {
|
||
const res = await fetch(finalUrl, options);
|
||
return { success: true, data: await res.json() };
|
||
} catch (err) {
|
||
if (retriesLeft <= 0) return { success: false, error: err.toString() };
|
||
await new Promise((r) => setTimeout(r, 300 * Math.pow(2, maxRetry - retriesLeft)));
|
||
return fetchWithRetry(finalUrl, options, retriesLeft - 1);
|
||
}
|
||
};
|
||
|
||
const jobs = [];
|
||
for (const task of taskList) {
|
||
out[task.apiPath] = [];
|
||
const cleanHeaders = { ...task.headers };
|
||
['content-length', 'host', 'origin', 'referer', 'accept-encoding'].forEach((k) => delete cleanHeaders[k]);
|
||
|
||
for (const params of task.paramList) {
|
||
jobs.push(limit(async () => {
|
||
let finalUrl = task.fullUrl;
|
||
const options = { method: task.method.toUpperCase(), headers: cleanHeaders };
|
||
if (options.method === 'GET') {
|
||
const qs = new URLSearchParams(params).toString();
|
||
finalUrl = finalUrl.includes('?') ? `${finalUrl}&${qs}` : `${finalUrl}?${qs}`;
|
||
} else {
|
||
options.body = JSON.stringify(params);
|
||
}
|
||
const result = await fetchWithRetry(finalUrl, options, maxRetry);
|
||
out[task.apiPath].push({ params, ...result });
|
||
await new Promise((r) => setTimeout(r, 200));
|
||
}));
|
||
}
|
||
}
|
||
await Promise.all(jobs);
|
||
return out;
|
||
}, tasks, FETCH_CONCURRENCY, FETCH_RETRY);
|
||
|
||
res.json({ success: true, results: fetchResults, profile });
|
||
} finally {
|
||
if (sessionCleanup) {
|
||
await sessionCleanup();
|
||
} else {
|
||
await safeCloseBrowser(browser);
|
||
}
|
||
}
|
||
}, profile);
|
||
} catch (error) {
|
||
if (!res.headersSent) sendTaskError(res, error, { profile });
|
||
}
|
||
});
|
||
|
||
/** 接口三：强制 UI 模拟点击拉取 */
|
||
app.post('/api/ui-pagination', async (req, res) => {
|
||
const {
|
||
pageUrl,
|
||
finalPageUrl,
|
||
apiUrl,
|
||
cookies,
|
||
nextBtnSelector,
|
||
clicksToPerform,
|
||
waitMs = 2000,
|
||
firstPageData,
|
||
authActions,
|
||
antiBot: rawAntiBot,
|
||
} = req.body;
|
||
const navigationUrl = finalPageUrl || pageUrl;
|
||
const antiBot = normalizeAntiBot(rawAntiBot);
|
||
const profile = antiBot.profile || 'standard';
|
||
|
||
try {
|
||
await runBrowserTask('ui-pagination', async () => {
|
||
let browser;
|
||
let sessionCleanup = null;
|
||
const capturedData = [];
|
||
const debugLogs = [];
|
||
const log = (msg) => { console.log(msg); debugLogs.push(msg); };
|
||
const seenHashes = new Set();
|
||
if (firstPageData) { seenHashes.add(generateSafeHash(firstPageData)); }
|
||
|
||
try {
|
||
const session = await createBrowserSession({
|
||
profile,
|
||
extraArgs: ['--window-size=1920,1080'],
|
||
});
|
||
browser = session.browser;
|
||
sessionCleanup = session.cleanup;
|
||
|
||
const page = await createManagedPage(browser, session.page, {
|
||
viewport: { width: 1920, height: 1080 },
|
||
cookies,
|
||
});
|
||
|
||
log(`[启动] 准备访问页面: ${navigationUrl}`);
|
||
if (finalPageUrl && finalPageUrl !== pageUrl) {
|
||
log(`[URL解析] 使用 auth 阶段落地地址，跳过短链重定向: ${pageUrl} -> ${finalPageUrl}`);
|
||
}
|
||
|
||
const shareToken = resolveShareToken(finalPageUrl, pageUrl);
|
||
await seedShareTokenCookie(page, navigationUrl);
|
||
await attachHttpIpRewriteInterceptor(page, 'ui-pagination', { shareToken });
|
||
|
||
await withExponentialBackoff(
|
||
() => page.goto(navigationUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }),
|
||
{ maxRetries: 2, baseDelayMs: 1000 }
|
||
);
|
||
|
||
const settledPageUrl = await waitForUrlSettled(page);
|
||
if (settledPageUrl !== navigationUrl) {
|
||
log(`[URL解析] 翻页阶段落地: ${navigationUrl} -> ${settledPageUrl}`);
|
||
}
|
||
|
||
if (antiBot.enabled) {
|
||
const cfResult = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled);
|
||
if (!cfResult.success) {
|
||
res.status(422).json({
|
||
success: false,
|
||
code: cfResult.code || 'CF_TURNSTILE_FAILED',
|
||
challengeType: cfResult.challengeType,
|
||
stage: cfResult.stage,
|
||
error: 'Cloudflare Turnstile 验证失败（翻页阶段）',
|
||
debug: debugLogs,
|
||
});
|
||
return;
|
||
}
|
||
}
|
||
|
||
if (Array.isArray(authActions) && authActions.length > 0) {
|
||
log('[状态同步] 正在为翻页引擎注入授权状态...');
|
||
await executeAuthActions(page, authActions);
|
||
log('[状态同步] 授权执行完毕，等待目标列表加载...');
|
||
const authSettled = await page.waitForResponse(
|
||
(resp) => resp.url().includes(apiUrl) && resp.request().method() !== 'OPTIONS' && resp.status() >= 200 && resp.status() < 400,
|
||
{ timeout: 5000 }
|
||
).catch(() => null);
|
||
if (!authSettled) {
|
||
log('[状态同步] 未捕获到目标 API 响应，降级为固定等待 3 秒...');
|
||
await new Promise((r) => setTimeout(r, 3000));
|
||
}
|
||
}
|
||
|
||
for (let i = 0; i < clicksToPerform; i++) {
|
||
const targetPageNum = i + 2;
|
||
let pageData = null;
|
||
let attempts = 0;
|
||
const maxAttempts = 3;
|
||
|
||
await page.evaluate(() => {
|
||
document.querySelectorAll('.el-loading-mask, .ant-spin-nested-loading, .el-loading-spinner').forEach((el) => el.remove());
|
||
});
|
||
|
||
await page.waitForFunction(
|
||
() => !document.querySelector('.el-loading-mask, .ant-spin-nested-loading, .el-loading-spinner'),
|
||
{ timeout: 1000, polling: 100 }
|
||
).catch(() => new Promise((r) => setTimeout(r, 500)));
|
||
|
||
while (attempts < maxAttempts && !pageData) {
|
||
attempts++;
|
||
log(`[抓取] 准备抓取第 ${targetPageNum} 页 (尝试 ${attempts}/${maxAttempts})...`);
|
||
|
||
let responseHandler = null;
|
||
let apiTimeoutId = null;
|
||
|
||
const waitForNewPageApi = new Promise((resolve) => {
|
||
const cleanupListener = () => {
|
||
if (apiTimeoutId) { clearTimeout(apiTimeoutId); apiTimeoutId = null; }
|
||
if (responseHandler) { page.off('response', responseHandler); responseHandler = null; }
|
||
};
|
||
|
||
responseHandler = async (response) => {
|
||
const status = response.status();
|
||
if (response.url().includes(apiUrl) && response.request().method() !== 'OPTIONS' && status >= 200 && status < 400) {
|
||
try {
|
||
const responseData = await response.json();
|
||
const currentHash = generateSafeHash(responseData);
|
||
if (!seenHashes.has(currentHash)) {
|
||
seenHashes.add(currentHash);
|
||
cleanupListener();
|
||
resolve(responseData);
|
||
} else {
|
||
log('[防轮询] 抛弃重复数据，继续监听...');
|
||
}
|
||
} catch (_) { /* 非 JSON 响应跳过 */ }
|
||
}
|
||
};
|
||
page.on('response', responseHandler);
|
||
apiTimeoutId = setTimeout(() => { cleanupListener(); resolve(null); }, 15000);
|
||
});
|
||
|
||
try {
|
||
await page.waitForSelector(nextBtnSelector, { timeout: 15000 });
|
||
|
||
const clickStatus = await page.evaluate((sel) => {
|
||
const btn = document.querySelector(sel);
|
||
if (!btn) return 'not_found';
|
||
if (btn.disabled || btn.classList.contains('is-disabled') || btn.classList.contains('disabled') || btn.getAttribute('aria-disabled') === 'true') {
|
||
return 'disabled';
|
||
}
|
||
const className = btn.className || '';
|
||
if (btn.disabled === true || className.includes('disabled') || btn.getAttribute('aria-disabled') === 'true') {
|
||
return 'disabled';
|
||
}
|
||
btn.scrollIntoView({ behavior: 'instant', block: 'center' });
|
||
btn.click();
|
||
btn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
|
||
return 'success';
|
||
}, nextBtnSelector);
|
||
|
||
if (clickStatus === 'disabled') {
|
||
log(`[提示] 第 ${targetPageNum} 页按钮已被禁用，说明到底了，抓取提前结束。`);
|
||
break;
|
||
}
|
||
log('[动作] 双引擎原生点击已执行！');
|
||
} catch (clickErr) {
|
||
log(`[错误] 寻找按钮异常 (${classifyPuppeteerError(clickErr)}): ${clickErr.message}`);
|
||
}
|
||
|
||
pageData = await waitForNewPageApi;
|
||
if (!pageData && attempts < maxAttempts) {
|
||
log('[重试警报] 雷达未扫描到新数据，准备重试...');
|
||
await new Promise((r) => setTimeout(r, 500 * Math.pow(2, attempts - 1)));
|
||
}
|
||
}
|
||
|
||
if (pageData) {
|
||
capturedData.push(pageData);
|
||
log(`[成功] 斩获第 ${targetPageNum} 页数据！`);
|
||
await new Promise((r) => setTimeout(r, waitMs));
|
||
} else {
|
||
log(`[致命异常] 连续 ${maxAttempts} 次尝试均失败，放弃后续翻页。`);
|
||
break;
|
||
}
|
||
}
|
||
|
||
res.json({ success: true, data: capturedData, debug: debugLogs, profile });
|
||
} catch (error) {
|
||
if (!res.headersSent) {
|
||
sendTaskError(res, error, { debug: debugLogs, profile });
|
||
}
|
||
} finally {
|
||
if (sessionCleanup) {
|
||
await sessionCleanup();
|
||
} else {
|
||
await safeCloseBrowser(browser);
|
||
}
|
||
}
|
||
}, profile);
|
||
} catch (error) {
|
||
if (!res.headersSent) sendTaskError(res, error, { profile });
|
||
}
|
||
});
|
||
|
||
const PORT = parseInt(process.env.PORT || '3001', 10);
|
||
const server = app.listen(PORT, '127.0.0.1', () => {
|
||
const chromePath = resolveChromeExecutable();
|
||
const chromeReady = fs.existsSync(chromePath);
|
||
console.log(`🚀 Node.js 自愈型防脱轨引擎已启动 (standard: ${MAX_CONCURRENT_BROWSERS}, real: ${MAX_CONCURRENT_BROWSERS_REAL})`);
|
||
console.log(`[chrome] path=${chromePath} exists=${chromeReady}`);
|
||
console.log(`[real-browser] ready=${isRealBrowserAvailable()} captcha=${isCaptchaConfigured()}`);
|
||
if (!chromeReady) {
|
||
console.warn('[chrome] 未找到 Chrome，请在 puppeteer-api 目录执行: npx puppeteer browsers install chrome');
|
||
}
|
||
});
|
||
|
||
async function gracefulShutdown(signal) {
|
||
console.log(`[shutdown] 收到 ${signal}，停止接收新任务...`);
|
||
shuttingDown = true;
|
||
server.close();
|
||
|
||
const deadline = Date.now() + 60000;
|
||
while (standardLimiter.active > 0 && Date.now() < deadline) {
|
||
const { active, queued } = standardLimiter.getStats();
|
||
console.log(`[shutdown] 等待在途任务完成... active=${active}, queued=${queued}`);
|
||
await new Promise((r) => setTimeout(r, 1000));
|
||
}
|
||
|
||
const remaining = standardLimiter.getStats();
|
||
if (remaining.active > 0) {
|
||
console.warn(`[shutdown] 超时，仍有 ${remaining.active} 个 Browser 在运行，强制退出`);
|
||
process.exit(1);
|
||
}
|
||
console.log('[shutdown] 所有任务已完成，进程退出');
|
||
process.exit(0);
|
||
}
|
||
|
||
process.on('SIGTERM', () => gracefulShutdown('SIGTERM'));
|
||
process.on('SIGINT', () => gracefulShutdown('SIGINT'));
|