damon/links
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化爬虫，增加缓存，线程池，保存浏览器用户信息

Browse Source
This commit is contained in:
root
2026-06-20 15:49:40 +08:00
parent e95f4a227c
commit 09bfa9ae01
26 changed files with 1584 additions and 454 deletions
Download Patch File Download Diff File Expand all files Collapse all files
patches/puppeteer-api/server.js
+274 -177
View File
@@ -1,6 +1,5 @@
const express = require('express');
const fs = require('fs');
const crypto = require('crypto');
const {
rewriteHttpsToHttpForLiteralIp,
attachHttpIpRewriteInterceptor,
@@ -17,19 +16,35 @@ const {
MAX_CONCURRENT_BROWSERS_REAL,
API_INTERCEPT_TIMEOUT_MS,
API_INTERCEPT_TIMEOUT_REAL_MS,
NAVIGATION_TIMEOUT_MS,
NAVIGATION_WAIT_UNTIL,
NAVIGATION_MAX_RETRIES,
SESSION_TTL_MS,
PERSIST_BROWSER_PROFILE,
POOL_IDLE_MS,
MAX_POOLED_BROWSERS,
} = require('./lib/constants');
const {
normalizeAntiBot,
createBrowserSession,
acquireBrowserSession,
runWithProfileLimit,
createManagedPage,
getFactoryStats,
standardLimiter,
browserPool,
} = require('./lib/browser-factory');
const { launchStandardBrowser } = require('./lib/launch-standard');
const { isRealBrowserAvailable } = require('./lib/launch-real-browser');
const { handleCloudflareChallenge, isCaptchaConfigured } = require('./lib/cf-handler');
const { loadSession, saveSession, sessionCookiesForPage } = require('./lib/session-store');
const {
saveSession,
getSessionDebugInfo,
getSessionStats,
} = require('./lib/session-store');
const { resolveSessionContext, touchSessionIfPresent } = require('./lib/session-context');
const { getProfileStats, profileExists } = require('./lib/profile-dir');
const { runUiPagination } = require('./lib/ui-pagination-runner');
const app = express();
app.use(express.json({ limit: '50mb' }));
@@ -63,16 +78,11 @@ function sendTaskError(res, error, extra = {}) {
return res.status(500).json({ success: false, error: error.message, ...extra });
}
function generateSafeHash(obj) {
if (!obj) return '';
const clone = JSON.parse(JSON.stringify(obj));
const removeDynamicKeys = (target) => {
if (typeof target !== 'object' || target === null) return;
['timestamp', 'time', 'serverTime', 'reqId', 'requestId', 'traceId'].forEach((k) => delete target[k]);
Object.values(target).forEach((val) => removeDynamicKeys(val));
};
removeDynamicKeys(clone);
return crypto.createHash('md5').update(JSON.stringify(clone)).digest('hex');
async function navigateToPage(page, url) {
await withExponentialBackoff(
() => page.goto(url, { waitUntil: NAVIGATION_WAIT_UNTIL, timeout: NAVIGATION_TIMEOUT_MS }),
{ maxRetries: NAVIGATION_MAX_RETRIES, baseDelayMs: 1000 }
);
}
async function safeCloseBrowser(browser) {
@@ -347,6 +357,9 @@ app.get('/api/stats', (_req, res) => {
shuttingDown,
queue: factoryStats.standard,
queueReal: factoryStats.real,
pool: factoryStats.pool,
sessions: getSessionStats(),
profiles: getProfileStats(),
realBrowserReady: factoryStats.realBrowserReady,
captchaConfigured: isCaptchaConfigured(),
chrome: {
@@ -358,10 +371,35 @@ app.get('/api/stats', (_req, res) => {
maxConcurrentBrowsersReal: MAX_CONCURRENT_BROWSERS_REAL,
apiInterceptTimeoutMs: API_INTERCEPT_TIMEOUT_MS,
apiInterceptTimeoutRealMs: API_INTERCEPT_TIMEOUT_REAL_MS,
navigationTimeoutMs: NAVIGATION_TIMEOUT_MS,
navigationWaitUntil: NAVIGATION_WAIT_UNTIL,
sessionTtlMs: SESSION_TTL_MS,
persistBrowserProfile: PERSIST_BROWSER_PROFILE,
poolIdleMs: POOL_IDLE_MS,
maxPooledBrowsers: MAX_POOLED_BROWSERS,
},
});
});
/** 运维调试：查看指定 sessionKey 的磁盘会话与 profile 状态 */
app.get('/api/session/:sessionKey', (req, res) => {
const sessionKey = decodeURIComponent(req.params.sessionKey || '');
if (!sessionKey) {
return res.status(400).json({ success: false, error: 'sessionKey 不能为空' });
}
const info = getSessionDebugInfo(sessionKey);
const profileStandard = getProfileStats();
res.json({
success: true,
session: info,
profileExists: {
standard: profileExists(sessionKey, 'standard'),
real: profileExists(sessionKey, 'real'),
},
profilesDir: profileStandard.profilesDir,
});
});
/** 运维自检：实际尝试 launch 一次 Browser（profile=standard|real） */
app.get('/api/browser-launch-test', async (req, res) => {
const profile = req.query.profile === 'real' ? 'real' : 'standard';
@@ -410,31 +448,28 @@ app.post('/api/auth-and-intercept', async (req, res) => {
const antiBot = normalizeAntiBot(rawAntiBot);
const profile = antiBot.profile || 'standard';
const interceptTimeout = resolveInterceptTimeout(antiBot);
const cfLog = { profile, cfDetected: false, turnstileStage: null, solverUsed: false, elapsedMs: 0 };
const cfLog = { profile, cfDetected: false, turnstileStage: null, solverUsed: false, elapsedMs: 0, browserReused: false };
try {
await runBrowserTask('auth-and-intercept', async () => {
let browser;
let sessionCleanup = null;
let browserSession = null;
let interceptorCleanup = null;
const taskStarted = Date.now();
let cfDestroyBrowser = false;
try {
const session = await createBrowserSession({
browserSession = await acquireBrowserSession({
profile,
extraArgs: ['--mute-audio'],
sessionKey: antiBot.sessionKey || '',
});
browser = session.browser;
sessionCleanup = session.cleanup;
cfLog.browserReused = !!browserSession.reused;
const sessionData = antiBot.sessionKey ? loadSession(antiBot.sessionKey) : null;
const sessionCookies = sessionData
? sessionCookiesForPage(sessionData, pageUrl)
: [];
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl);
const page = await createManagedPage(browser, session.page, {
const page = await createManagedPage(browserSession.browser, browserSession.page, {
userAgent: DEFAULT_UA,
cookies: sessionCookies,
cookies: mergedCookies,
});
const httpResolvedUrl = await resolveHttpRedirects(pageUrl).catch(() => pageUrl);
@@ -452,10 +487,7 @@ app.post('/api/auth-and-intercept', async (req, res) => {
}
await attachHttpIpRewriteInterceptor(page, 'auth-and-intercept', { shareToken });
await withExponentialBackoff(
() => page.goto(pageUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }),
{ maxRetries: 2, baseDelayMs: 1000 }
);
await navigateToPage(page, pageUrl);
let finalPageUrl = await waitForUrlSettled(page);
if (finalPageUrl !== pageUrl) {
@@ -463,13 +495,14 @@ app.post('/api/auth-and-intercept', async (req, res) => {
}
if (antiBot.enabled) {
const cfResult = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled);
const cfResult = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled, storedSession);
cfLog.cfDetected = cfResult.cfDetected;
cfLog.turnstileStage = cfResult.stage;
cfLog.solverUsed = cfResult.solverUsed;
cfLog.elapsedMs = cfResult.elapsedMs;
if (!cfResult.success) {
cfDestroyBrowser = true;
res.status(422).json({
success: false,
code: cfResult.code || 'CF_TURNSTILE_FAILED',
@@ -482,6 +515,7 @@ app.post('/api/auth-and-intercept', async (req, res) => {
}
finalPageUrl = page.url();
touchSessionIfPresent(antiBot.sessionKey);
}
const finalShareToken = resolveShareToken(finalPageUrl);
@@ -502,7 +536,14 @@ app.post('/api/auth-and-intercept', async (req, res) => {
}));
if (antiBot.sessionKey) {
saveSession(antiBot.sessionKey, cookiePayload);
let lastHost = '';
try {
lastHost = new URL(finalPageUrl).hostname;
} catch (_) { /* ignore */ }
saveSession(antiBot.sessionKey, cookiePayload, SESSION_TTL_MS, {
lastHost,
savedProfile: profile,
});
}
res.json({
@@ -513,14 +554,13 @@ app.post('/api/auth-and-intercept', async (req, res) => {
cookies: cookiePayload,
cf: cfLog,
profile,
browserReused: browserSession.reused,
elapsedMs: Date.now() - taskStarted,
});
} finally {
if (interceptorCleanup) interceptorCleanup();
if (sessionCleanup) {
await sessionCleanup();
} else {
await safeCloseBrowser(browser);
if (browserSession) {
await browserSession.release(cfDestroyBrowser);
}
}
}, profile);
@@ -541,18 +581,19 @@ app.post('/api/batch-fetch', async (req, res) => {
try {
await runBrowserTask('batch-fetch', async () => {
let browser;
let sessionCleanup = null;
let browserSession = null;
try {
const session = await createBrowserSession({
browserSession = await acquireBrowserSession({
profile,
extraArgs: ['--disable-web-security'],
sessionKey: antiBot.sessionKey || '',
});
browser = session.browser;
sessionCleanup = session.cleanup;
const page = await createManagedPage(browser, session.page, { cookies });
const firstOrigin = new URL(tasks[0].fullUrl).origin;
const firstUrl = tasks[0].fullUrl;
const { mergedCookies } = resolveSessionContext(antiBot, firstUrl, cookies || []);
const page = await createManagedPage(browserSession.browser, browserSession.page, { cookies: mergedCookies });
const firstOrigin = new URL(firstUrl).origin;
await page.setRequestInterception(true);
page.on('request', (reqObj) => {
@@ -570,10 +611,7 @@ app.post('/api/batch-fetch', async (req, res) => {
}
});
await withExponentialBackoff(
() => page.goto(firstOrigin, { waitUntil: 'domcontentloaded', timeout: 30000 }),
{ maxRetries: 2, baseDelayMs: 800 }
);
await navigateToPage(page, firstOrigin);
const FETCH_CONCURRENCY = Math.max(1, parseInt(process.env.FETCH_CONCURRENCY || '5', 10));
const FETCH_RETRY = 2;
@@ -636,12 +674,10 @@ app.post('/api/batch-fetch', async (req, res) => {
return out;
}, tasks, FETCH_CONCURRENCY, FETCH_RETRY);
res.json({ success: true, results: fetchResults, profile });
res.json({ success: true, results: fetchResults, profile, browserReused: browserSession.reused });
} finally {
if (sessionCleanup) {
await sessionCleanup();
} else {
await safeCloseBrowser(browser);
if (browserSession) {
await browserSession.release(false);
}
}
}, profile);
@@ -670,27 +706,25 @@ app.post('/api/ui-pagination', async (req, res) => {
try {
await runBrowserTask('ui-pagination', async () => {
let browser;
let sessionCleanup = null;
const capturedData = [];
let browserSession = null;
const debugLogs = [];
const log = (msg) => { console.log(msg); debugLogs.push(msg); };
const seenHashes = new Set();
if (firstPageData) { seenHashes.add(generateSafeHash(firstPageData)); }
let cfDestroyBrowser = false;
try {
const session = await createBrowserSession({
browserSession = await acquireBrowserSession({
profile,
extraArgs: ['--window-size=1920,1080'],
sessionKey: antiBot.sessionKey || '',
});
browser = session.browser;
sessionCleanup = session.cleanup;
const page = await createManagedPage(browser, session.page, {
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, navigationUrl, cookies || []);
const page = await createManagedPage(browserSession.browser, browserSession.page, {
viewport: { width: 1920, height: 1080 },
cookies,
cookies: mergedCookies,
});
const log = (msg) => { console.log(msg); debugLogs.push(msg); };
log(`[启动] 准备访问页面: ${navigationUrl}`);
if (finalPageUrl && finalPageUrl !== pageUrl) {
log(`[URL解析] 使用 auth 阶段落地地址，跳过短链重定向: ${pageUrl} -> ${finalPageUrl}`);
@@ -700,10 +734,7 @@ app.post('/api/ui-pagination', async (req, res) => {
await seedShareTokenCookie(page, navigationUrl);
await attachHttpIpRewriteInterceptor(page, 'ui-pagination', { shareToken });
await withExponentialBackoff(
() => page.goto(navigationUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }),
{ maxRetries: 2, baseDelayMs: 1000 }
);
await navigateToPage(page, navigationUrl);
const settledPageUrl = await waitForUrlSettled(page);
if (settledPageUrl !== navigationUrl) {
@@ -711,8 +742,9 @@ app.post('/api/ui-pagination', async (req, res) => {
}
if (antiBot.enabled) {
const cfResult = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled);
const cfResult = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled, storedSession);
if (!cfResult.success) {
cfDestroyBrowser = true;
res.status(422).json({
success: false,
code: cfResult.code || 'CF_TURNSTILE_FAILED',
@@ -723,125 +755,33 @@ app.post('/api/ui-pagination', async (req, res) => {
});
return;
}
touchSessionIfPresent(antiBot.sessionKey);
}
if (Array.isArray(authActions) && authActions.length > 0) {
log('[状态同步] 正在为翻页引擎注入授权状态...');
await executeAuthActions(page, authActions);
log('[状态同步] 授权执行完毕，等待目标列表加载...');
const authSettled = await page.waitForResponse(
(resp) => resp.url().includes(apiUrl) && resp.request().method() !== 'OPTIONS' && resp.status() >= 200 && resp.status() < 400,
{ timeout: 5000 }
).catch(() => null);
if (!authSettled) {
log('[状态同步] 未捕获到目标 API 响应，降级为固定等待 3 秒...');
await new Promise((r) => setTimeout(r, 3000));
}
}
const paginationResult = await runUiPagination(page, {
apiUrl,
nextBtnSelector,
clicksToPerform,
waitMs,
firstPageData,
authActions,
executeAuthActions,
});
for (let i = 0; i < clicksToPerform; i++) {
const targetPageNum = i + 2;
let pageData = null;
let attempts = 0;
const maxAttempts = 3;
await page.evaluate(() => {
document.querySelectorAll('.el-loading-mask, .ant-spin-nested-loading, .el-loading-spinner').forEach((el) => el.remove());
});
await page.waitForFunction(
() => !document.querySelector('.el-loading-mask, .ant-spin-nested-loading, .el-loading-spinner'),
{ timeout: 1000, polling: 100 }
).catch(() => new Promise((r) => setTimeout(r, 500)));
while (attempts < maxAttempts && !pageData) {
attempts++;
log(`[抓取] 准备抓取第 ${targetPageNum} 页 (尝试 ${attempts}/${maxAttempts})...`);
let responseHandler = null;
let apiTimeoutId = null;
const waitForNewPageApi = new Promise((resolve) => {
const cleanupListener = () => {
if (apiTimeoutId) { clearTimeout(apiTimeoutId); apiTimeoutId = null; }
if (responseHandler) { page.off('response', responseHandler); responseHandler = null; }
};
responseHandler = async (response) => {
const status = response.status();
if (response.url().includes(apiUrl) && response.request().method() !== 'OPTIONS' && status >= 200 && status < 400) {
try {
const responseData = await response.json();
const currentHash = generateSafeHash(responseData);
if (!seenHashes.has(currentHash)) {
seenHashes.add(currentHash);
cleanupListener();
resolve(responseData);
} else {
log('[防轮询] 抛弃重复数据，继续监听...');
}
} catch (_) { /* 非 JSON 响应跳过 */ }
}
};
page.on('response', responseHandler);
apiTimeoutId = setTimeout(() => { cleanupListener(); resolve(null); }, 15000);
});
try {
await page.waitForSelector(nextBtnSelector, { timeout: 15000 });
const clickStatus = await page.evaluate((sel) => {
const btn = document.querySelector(sel);
if (!btn) return 'not_found';
if (btn.disabled || btn.classList.contains('is-disabled') || btn.classList.contains('disabled') || btn.getAttribute('aria-disabled') === 'true') {
return 'disabled';
}
const className = btn.className || '';
if (btn.disabled === true || className.includes('disabled') || btn.getAttribute('aria-disabled') === 'true') {
return 'disabled';
}
btn.scrollIntoView({ behavior: 'instant', block: 'center' });
btn.click();
btn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
return 'success';
}, nextBtnSelector);
if (clickStatus === 'disabled') {
log(`[提示] 第 ${targetPageNum} 页按钮已被禁用，说明到底了，抓取提前结束。`);
break;
}
log('[动作] 双引擎原生点击已执行！');
} catch (clickErr) {
log(`[错误] 寻找按钮异常 (${classifyPuppeteerError(clickErr)}): ${clickErr.message}`);
}
pageData = await waitForNewPageApi;
if (!pageData && attempts < maxAttempts) {
log('[重试警报] 雷达未扫描到新数据，准备重试...');
await new Promise((r) => setTimeout(r, 500 * Math.pow(2, attempts - 1)));
}
}
if (pageData) {
capturedData.push(pageData);
log(`[成功] 斩获第 ${targetPageNum} 页数据！`);
await new Promise((r) => setTimeout(r, waitMs));
} else {
log(`[致命异常] 连续 ${maxAttempts} 次尝试均失败，放弃后续翻页。`);
break;
}
}
res.json({ success: true, data: capturedData, debug: debugLogs, profile });
res.json({
success: true,
data: paginationResult.data,
debug: [...debugLogs, ...paginationResult.debugLogs],
profile,
browserReused: browserSession.reused,
});
} catch (error) {
if (!res.headersSent) {
sendTaskError(res, error, { debug: debugLogs, profile });
}
} finally {
if (sessionCleanup) {
await sessionCleanup();
} else {
await safeCloseBrowser(browser);
if (browserSession) {
await browserSession.release(cfDestroyBrowser);
}
}
}, profile);
@@ -850,6 +790,160 @@ app.post('/api/ui-pagination', async (req, res) => {
}
});
/** 接口四：单 Browser 内 auth + 拦截 + UI 翻页（降低 CF 二次触发） */
app.post('/api/auth-intercept-and-paginate', async (req, res) => {
const {
pageUrl,
apiUrls,
authActions,
antiBot: rawAntiBot,
pagination,
} = req.body;
if (!pageUrl || !Array.isArray(apiUrls) || !pagination) {
return res.status(400).json({ success: false, error: '参数错误：需要 pageUrl、apiUrls、pagination' });
}
const {
apiUrl,
nextBtnSelector,
clicksToPerform = 0,
waitMs = 2000,
firstPageData,
} = pagination;
const antiBot = normalizeAntiBot(rawAntiBot);
const profile = antiBot.profile || 'standard';
const interceptTimeout = resolveInterceptTimeout(antiBot);
const cfLog = { profile, cfDetected: false, turnstileStage: null, solverUsed: false, elapsedMs: 0, browserReused: false };
try {
await runBrowserTask('auth-intercept-and-paginate', async () => {
let browserSession = null;
let interceptorCleanup = null;
const taskStarted = Date.now();
let cfDestroyBrowser = false;
try {
browserSession = await acquireBrowserSession({
profile,
extraArgs: ['--mute-audio', '--window-size=1920,1080'],
sessionKey: antiBot.sessionKey || '',
});
cfLog.browserReused = !!browserSession.reused;
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl);
const page = await createManagedPage(browserSession.browser, browserSession.page, {
userAgent: DEFAULT_UA,
viewport: { width: 1920, height: 1080 },
cookies: mergedCookies,
});
const httpResolvedUrl = await resolveHttpRedirects(pageUrl).catch(() => pageUrl);
const interceptor = createApiInterceptor(page, apiUrls, interceptTimeout);
interceptorCleanup = interceptor.cleanup;
let shareToken = resolveShareToken(pageUrl, httpResolvedUrl);
await seedShareTokenCookie(page, pageUrl);
await attachHttpIpRewriteInterceptor(page, 'auth-intercept-and-paginate', { shareToken });
await navigateToPage(page, pageUrl);
let finalPageUrl = await waitForUrlSettled(page);
if (antiBot.enabled) {
const cfResult = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled, storedSession);
cfLog.cfDetected = cfResult.cfDetected;
cfLog.turnstileStage = cfResult.stage;
cfLog.solverUsed = cfResult.solverUsed;
cfLog.elapsedMs = cfResult.elapsedMs;
if (!cfResult.success) {
cfDestroyBrowser = true;
res.status(422).json({
success: false,
code: cfResult.code || 'CF_TURNSTILE_FAILED',
challengeType: cfResult.challengeType || 'turnstile',
stage: cfResult.stage || 'timeout',
error: 'Cloudflare Turnstile 验证失败',
cf: cfLog,
});
return;
}
finalPageUrl = page.url();
touchSessionIfPresent(antiBot.sessionKey);
}
await executeAuthActions(page, authActions);
await interceptor.waitForApis(interceptTimeout);
const rawCookies = await page.cookies();
const cookiePayload = rawCookies.map((c) => ({
name: c.name,
value: c.value,
domain: c.domain,
path: c.path,
}));
if (antiBot.sessionKey) {
let lastHost = '';
try {
lastHost = new URL(finalPageUrl).hostname;
} catch (_) { /* ignore */ }
saveSession(antiBot.sessionKey, cookiePayload, SESSION_TTL_MS, {
lastHost,
savedProfile: profile,
});
}
let extraPages = [];
let paginationDebug = [];
if (clicksToPerform > 0 && apiUrl && nextBtnSelector) {
let effectiveFirstPageData = firstPageData;
if (!effectiveFirstPageData) {
const matchedKey = Object.keys(interceptor.interceptedApis).find((k) => k === apiUrl || k.includes(apiUrl));
if (matchedKey) {
effectiveFirstPageData = interceptor.interceptedApis[matchedKey].data;
}
}
const paginationResult = await runUiPagination(page, {
apiUrl,
nextBtnSelector,
clicksToPerform,
waitMs,
firstPageData: effectiveFirstPageData,
authActions,
executeAuthActions,
});
extraPages = paginationResult.data;
paginationDebug = paginationResult.debugLogs;
}
res.json({
success: true,
pageUrl,
finalPageUrl,
interceptedApis: interceptor.interceptedApis,
cookies: cookiePayload,
extraPages,
debug: paginationDebug,
cf: cfLog,
profile,
browserReused: browserSession.reused,
elapsedMs: Date.now() - taskStarted,
});
} finally {
if (interceptorCleanup) interceptorCleanup();
if (browserSession) {
await browserSession.release(cfDestroyBrowser);
}
}
}, profile);
} catch (error) {
if (!res.headersSent) sendTaskError(res, error, { cf: cfLog, profile });
}
});
const PORT = parseInt(process.env.PORT || '3001', 10);
const server = app.listen(PORT, '127.0.0.1', () => {
const chromePath = resolveChromeExecutable();
@@ -874,6 +968,9 @@ async function gracefulShutdown(signal) {
await new Promise((r) => setTimeout(r, 1000));
}
console.log('[shutdown] 排空 Browser 温池...');
await browserPool.drain();
const remaining = standardLimiter.getStats();
if (remaining.active > 0) {
console.warn(`[shutdown] 超时，仍有 ${remaining.active} 个 Browser 在运行，强制退出`);