优化爬虫，增加缓存，线程池，保存浏览器用户信息

2026-06-20 15:49:40 +08:00
parent e95f4a227c
commit 09bfa9ae01
26 changed files with 1584 additions and 454 deletions
@@ -0,0 +1,184 @@
+/**
+ * Browser 温池：按 profile:sessionKey 复用 Browser/Page，降低冷启动与 CF 触发
+ */
+const { POOL_IDLE_MS, MAX_POOLED_BROWSERS, POOL_ENABLED } = require('./constants');
+
+class BrowserPool {
+    constructor() {
+        /** @type {Map<string, { browser: any, page: any|null, profile: string, sessionKey: string, lastUsedAt: number, inUse: boolean }>} */
+        this.entries = new Map();
+        this.hits = 0;
+        this.misses = 0;
+        this.evictions = 0;
+        this._sweepTimer = setInterval(() => this.evictIdle(), 30000);
+        this._sweepTimer.unref?.();
+    }
+
+    /**
+     * @param {string} sessionKey
+     * @param {'standard'|'real'} profile
+     */
+    buildKey(sessionKey, profile) {
+        return `${profile}:${sessionKey}`;
+    }
+
+    /**
+     * @param {any} browser
+     */
+    async isBrowserAlive(browser) {
+        if (!browser) return false;
+        try {
+            if (typeof browser.isConnected === 'function' && !browser.isConnected()) {
+                return false;
+            }
+            await browser.version();
+            return true;
+        } catch (_) {
+            return false;
+        }
+    }
+
+    /**
+     * @param {string} key
+     */
+    async destroyEntry(key) {
+        const entry = this.entries.get(key);
+        if (!entry) return;
+        this.entries.delete(key);
+        try {
+            await entry.browser.close();
+        } catch (_) {
+            /* ignore */
+        }
+        this.evictions++;
+    }
+
+    enforceCapacity(excludeKey) {
+        if (this.entries.size < MAX_POOLED_BROWSERS) return;
+        const candidates = [...this.entries.entries()]
+            .filter(([k, e]) => k !== excludeKey && !e.inUse)
+            .sort((a, b) => a[1].lastUsedAt - b[1].lastUsedAt);
+        if (candidates.length === 0) return;
+        const [oldestKey] = candidates[0];
+        this.destroyEntry(oldestKey);
+    }
+
+    /**
+     * @param {string} sessionKey
+     * @param {'standard'|'real'} profile
+     * @param {() => Promise<{ browser: any, page: any|null, cleanup: (destroy?: boolean) => Promise<void> }>} launcher
+     */
+    async acquire(sessionKey, profile, launcher) {
+        if (!POOL_ENABLED || !sessionKey) {
+            const launched = await launcher();
+            return {
+                browser: launched.browser,
+                page: launched.page,
+                profile,
+                pooled: false,
+                reused: false,
+                release: async (destroy = false) => {
+                    await launched.cleanup(destroy);
+                },
+            };
+        }
+
+        const key = this.buildKey(sessionKey, profile);
+        const existing = this.entries.get(key);
+        if (existing && !existing.inUse && await this.isBrowserAlive(existing.browser)) {
+            existing.inUse = true;
+            existing.lastUsedAt = Date.now();
+            this.hits++;
+            return {
+                browser: existing.browser,
+                page: existing.page,
+                profile,
+                pooled: true,
+                reused: true,
+                release: async (destroy = false) => this.release(key, destroy),
+            };
+        }
+        if (existing) {
+            await this.destroyEntry(key);
+        }
+
+        this.enforceCapacity(key);
+        this.misses++;
+        const launched = await launcher();
+        this.entries.set(key, {
+            browser: launched.browser,
+            page: launched.page,
+            profile,
+            sessionKey,
+            lastUsedAt: Date.now(),
+            inUse: true,
+        });
+
+        return {
+            browser: launched.browser,
+            page: launched.page,
+            profile,
+            pooled: true,
+            reused: false,
+            release: async (destroy = false) => this.release(key, destroy),
+        };
+    }
+
+    /**
+     * @param {string} key
+     * @param {boolean} destroy
+     */
+    async release(key, destroy = false) {
+        const entry = this.entries.get(key);
+        if (!entry) return;
+        entry.inUse = false;
+        entry.lastUsedAt = Date.now();
+        if (destroy || !(await this.isBrowserAlive(entry.browser))) {
+            await this.destroyEntry(key);
+        }
+    }
+
+    evictIdle() {
+        const now = Date.now();
+        for (const [key, entry] of this.entries.entries()) {
+            if (!entry.inUse && now - entry.lastUsedAt > POOL_IDLE_MS) {
+                this.destroyEntry(key);
+            }
+        }
+    }
+
+    async drain() {
+        clearInterval(this._sweepTimer);
+        const keys = [...this.entries.keys()];
+        for (const key of keys) {
+            await this.destroyEntry(key);
+        }
+    }
+
+    getStats() {
+        let idle = 0;
+        let active = 0;
+        for (const entry of this.entries.values()) {
+            if (entry.inUse) active++;
+            else idle++;
+        }
+        return {
+            enabled: POOL_ENABLED,
+            max: MAX_POOLED_BROWSERS,
+            idleMs: POOL_IDLE_MS,
+            size: this.entries.size,
+            idle,
+            active,
+            hits: this.hits,
+            misses: this.misses,
+            evictions: this.evictions,
+        };
+    }
+}
+
+const browserPool = new BrowserPool();
+
+module.exports = {
+    BrowserPool,
+    browserPool,
+};