damon/links
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化爬虫，增加缓存，线程池，保存浏览器用户信息

Browse Source
This commit is contained in:
root
2026-06-20 15:49:40 +08:00
parent e95f4a227c
commit 09bfa9ae01
26 changed files with 1584 additions and 454 deletions
Download Patch File Download Diff File Expand all files Collapse all files
patches/puppeteer-api/lib/browser-factory.js
+39 -55
View File
@@ -1,34 +1,22 @@
/**
* BrowserFactory：standard / real 双轨启动 + 独立并发限流
* BrowserFactory：standard / real 双轨启动 + 温池 + 独立并发限流
*/
const {
DEFAULT_UA,
MAX_CONCURRENT_BROWSERS,
MAX_CONCURRENT_BROWSERS_REAL,
QUEUE_TIMEOUT_MS,
NAVIGATION_TIMEOUT_MS,
PAGE_DEFAULT_TIMEOUT_MS,
} = require('./constants');
const { BrowserConcurrencyLimiter } = require('./concurrency');
const { launchStandardBrowser } = require('./launch-standard');
const { launchRealBrowser, isRealBrowserAvailable } = require('./launch-real-browser');
const { browserPool } = require('./browser-pool');
const standardLimiter = new BrowserConcurrencyLimiter(MAX_CONCURRENT_BROWSERS);
const realLimiter = new BrowserConcurrencyLimiter(MAX_CONCURRENT_BROWSERS_REAL);
/**
* @typedef {Object} AntiBotConfig
* @property {boolean} [enabled]
* @property {'standard'|'real'} [profile]
* @property {boolean} [turnstile]
* @property {boolean} [solverFallback]
* @property {string} [sessionKey]
* @property {number} [challengeTimeoutMs]
*/
/**
* 规范化 antiBot 配置（未传则走 standard）
* @param {AntiBotConfig|null|undefined} antiBot
* @returns {AntiBotConfig}
*/
function normalizeAntiBot(antiBot) {
if (!antiBot || !antiBot.enabled) {
return { enabled: false, profile: 'standard' };
@@ -43,62 +31,66 @@ function normalizeAntiBot(antiBot) {
};
}
/**
* 按 profile 获取限流器
* @param {'standard'|'real'} profile
*/
function getLimiter(profile) {
return profile === 'real' ? realLimiter : standardLimiter;
}
/**
* 创建 Browser 会话
* @param {{ profile?: 'standard'|'real', extraArgs?: string[] }} options
* @returns {Promise<{ browser: import('puppeteer').Browser, page: import('puppeteer').Page|null, profile: string, cleanup: () => Promise<void> }>}
* 冷启动 Browser（供温池 launcher 使用）
* @param {{ profile?: 'standard'|'real', extraArgs?: string[], sessionKey?: string }} options
*/
async function createBrowserSession(options = {}) {
const profile = options.profile === 'real' ? 'real' : 'standard';
const extraArgs = options.extraArgs || [];
const sessionKey = options.sessionKey || '';
if (profile === 'real') {
const { browser, page } = await launchRealBrowser(extraArgs);
const { browser, page } = await launchRealBrowser(extraArgs, { sessionKey, profile: 'real' });
return {
browser,
page,
profile: 'real',
cleanup: async () => {
try {
await browser.close();
} catch (_) {
/* ignore */
cleanup: async (destroy = false) => {
if (destroy || !sessionKey) {
try {
await browser.close();
} catch (_) { /* ignore */ }
}
},
};
}
const browser = await launchStandardBrowser(extraArgs);
const browser = await launchStandardBrowser(extraArgs, { sessionKey, profile: 'standard' });
return {
browser,
page: null,
profile: 'standard',
cleanup: async () => {
try {
await browser.close();
} catch (_) {
/* ignore */
cleanup: async (destroy = false) => {
if (destroy || !sessionKey) {
try {
await browser.close();
} catch (_) { /* ignore */ }
}
},
};
}
/**
* 在并发槽位内执行 Browser 任务
* @template T
* @param {string} taskName
* @param {'standard'|'real'} profile
* @param {() => Promise<T>} fn
* @returns {Promise<T>}
* 从温池或冷启动获取 Browser
* @param {{ profile?: 'standard'|'real', extraArgs?: string[], sessionKey?: string }} options
*/
async function acquireBrowserSession(options = {}) {
const profile = options.profile === 'real' ? 'real' : 'standard';
const sessionKey = options.sessionKey || '';
const extraArgs = options.extraArgs || [];
return browserPool.acquire(sessionKey, profile, async () => createBrowserSession({
profile,
extraArgs,
sessionKey,
}));
}
async function runWithProfileLimit(taskName, profile, fn) {
const limiter = getLimiter(profile);
try {
@@ -109,23 +101,15 @@ async function runWithProfileLimit(taskName, profile, fn) {
}
}
/**
* 创建 Page 并设置 UA / viewport / cookies
* @param {import('puppeteer').Browser} browser
* @param {import('puppeteer').Page|null} existingPage
* @param {{ userAgent?: string, viewport?: object, cookies?: Array<object> }} [options]
*/
async function createManagedPage(browser, existingPage, options = {}) {
const page = existingPage || await browser.newPage();
page.setDefaultNavigationTimeout(30000);
page.setDefaultTimeout(15000);
page.setDefaultNavigationTimeout(NAVIGATION_TIMEOUT_MS);
page.setDefaultTimeout(PAGE_DEFAULT_TIMEOUT_MS);
const userAgent = options.userAgent || DEFAULT_UA;
await page.setUserAgent(userAgent);
if (options.viewport) {
await page.setViewport(options.viewport);
}
if (options.viewport) await page.setViewport(options.viewport);
if (options.cookies && options.cookies.length > 0) {
await page.setCookie(...options.cookies);
}
@@ -136,14 +120,12 @@ async function createManagedPage(browser, existingPage, options = {}) {
return page;
}
/**
* @returns {{ standard: object, real: object, realBrowserReady: boolean }}
*/
function getFactoryStats() {
return {
standard: standardLimiter.getStats(),
real: realLimiter.getStats(),
realBrowserReady: isRealBrowserAvailable(),
pool: browserPool.getStats(),
};
}
@@ -151,10 +133,12 @@ module.exports = {
normalizeAntiBot,
getLimiter,
createBrowserSession,
acquireBrowserSession,
runWithProfileLimit,
createManagedPage,
getFactoryStats,
standardLimiter,
realLimiter,
browserPool,
QUEUE_TIMEOUT_MS,
};