优化爬虫,增加缓存,线程池,保存浏览器用户信息

This commit is contained in:
root
2026-06-20 15:49:40 +08:00
parent e95f4a227c
commit 09bfa9ae01
26 changed files with 1584 additions and 454 deletions
+23 -18
View File
@@ -11,18 +11,21 @@ const {
buildBrowserLaunchEnv,
ensureRuntimeSubdir,
} = require('./constants');
const { resolveProfileDir } = require('./profile-dir');
// Stealth 必须在 launch 之前注册
puppeteer.use(StealthPlugin());
/**
* 启动 standard profile Browser
* @param {string[]} [extraArgs]
* @param {{ sessionKey?: string, profile?: 'standard'|'real' }} [options]
* @returns {Promise<import('puppeteer').Browser>}
*/
async function launchStandardBrowser(extraArgs = []) {
async function launchStandardBrowser(extraArgs = [], options = {}) {
const executablePath = assertChromeExecutable();
const userDataDir = fs.mkdtempSync(path.join(ensureRuntimeSubdir('chrome-profiles'), 'run-'));
const profile = options.profile === 'real' ? 'real' : 'standard';
const persistentDir = options.sessionKey ? resolveProfileDir(options.sessionKey, profile) : null;
const userDataDir = persistentDir || fs.mkdtempSync(path.join(ensureRuntimeSubdir('chrome-profiles'), 'run-'));
const persistent = !!persistentDir;
try {
const browser = await puppeteer.launch({
@@ -32,24 +35,26 @@ async function launchStandardBrowser(extraArgs = []) {
env: buildBrowserLaunchEnv(),
userDataDir,
});
const originalClose = browser.close.bind(browser);
browser.close = async () => {
try {
await originalClose();
} finally {
if (!persistent) {
const originalClose = browser.close.bind(browser);
browser.close = async () => {
try {
fs.rmSync(userDataDir, { recursive: true, force: true });
} catch (_) {
/* 清理失败不影响主流程 */
await originalClose();
} finally {
try {
fs.rmSync(userDataDir, { recursive: true, force: true });
} catch (_) { /* ignore */ }
}
}
};
};
}
browser.__userDataDir = userDataDir;
browser.__persistentProfile = persistent;
return browser;
} catch (err) {
try {
fs.rmSync(userDataDir, { recursive: true, force: true });
} catch (_) {
/* ignore */
if (!persistent) {
try {
fs.rmSync(userDataDir, { recursive: true, force: true });
} catch (_) { /* ignore */ }
}
throw err;
}