优化爬虫,增加缓存,线程池,保存浏览器用户信息
This commit is contained in:
@@ -11,18 +11,21 @@ const {
|
||||
buildBrowserLaunchEnv,
|
||||
ensureRuntimeSubdir,
|
||||
} = require('./constants');
|
||||
const { resolveProfileDir } = require('./profile-dir');
|
||||
|
||||
// Stealth 必须在 launch 之前注册
|
||||
puppeteer.use(StealthPlugin());
|
||||
|
||||
/**
|
||||
* 启动 standard profile Browser
|
||||
* @param {string[]} [extraArgs]
|
||||
* @param {{ sessionKey?: string, profile?: 'standard'|'real' }} [options]
|
||||
* @returns {Promise<import('puppeteer').Browser>}
|
||||
*/
|
||||
async function launchStandardBrowser(extraArgs = []) {
|
||||
async function launchStandardBrowser(extraArgs = [], options = {}) {
|
||||
const executablePath = assertChromeExecutable();
|
||||
const userDataDir = fs.mkdtempSync(path.join(ensureRuntimeSubdir('chrome-profiles'), 'run-'));
|
||||
const profile = options.profile === 'real' ? 'real' : 'standard';
|
||||
const persistentDir = options.sessionKey ? resolveProfileDir(options.sessionKey, profile) : null;
|
||||
const userDataDir = persistentDir || fs.mkdtempSync(path.join(ensureRuntimeSubdir('chrome-profiles'), 'run-'));
|
||||
const persistent = !!persistentDir;
|
||||
|
||||
try {
|
||||
const browser = await puppeteer.launch({
|
||||
@@ -32,24 +35,26 @@ async function launchStandardBrowser(extraArgs = []) {
|
||||
env: buildBrowserLaunchEnv(),
|
||||
userDataDir,
|
||||
});
|
||||
const originalClose = browser.close.bind(browser);
|
||||
browser.close = async () => {
|
||||
try {
|
||||
await originalClose();
|
||||
} finally {
|
||||
if (!persistent) {
|
||||
const originalClose = browser.close.bind(browser);
|
||||
browser.close = async () => {
|
||||
try {
|
||||
fs.rmSync(userDataDir, { recursive: true, force: true });
|
||||
} catch (_) {
|
||||
/* 清理失败不影响主流程 */
|
||||
await originalClose();
|
||||
} finally {
|
||||
try {
|
||||
fs.rmSync(userDataDir, { recursive: true, force: true });
|
||||
} catch (_) { /* ignore */ }
|
||||
}
|
||||
}
|
||||
};
|
||||
};
|
||||
}
|
||||
browser.__userDataDir = userDataDir;
|
||||
browser.__persistentProfile = persistent;
|
||||
return browser;
|
||||
} catch (err) {
|
||||
try {
|
||||
fs.rmSync(userDataDir, { recursive: true, force: true });
|
||||
} catch (_) {
|
||||
/* ignore */
|
||||
if (!persistent) {
|
||||
try {
|
||||
fs.rmSync(userDataDir, { recursive: true, force: true });
|
||||
} catch (_) { /* ignore */ }
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user