优化爬虫，增加缓存，线程池，保存浏览器用户信息

patches/puppeteer-api/lib/session-store.js

@@ -1,5 +1,5 @@
 /**
- * 会话 Cookie 持久化：按 sessionKey 保存 cf_clearance 等，降低重复过盾概率
+ * 会话 Cookie 持久化：按 sessionKey 保存完整 cookie jar，降低重复过盾概率
  */
 const fs = require('fs');
 const path = require('path');
@@ -8,42 +8,50 @@ const { RUNTIME_DIR, SESSION_TTL_MS } = require('./constants');
 
 const SESSIONS_DIR = path.join(RUNTIME_DIR, 'sessions');
 
-/**
- * 确保 sessions 目录存在
- */
 function ensureSessionsDir() {
     fs.mkdirSync(SESSIONS_DIR, { recursive: true, mode: 0o777 });
 }
 
-/**
- * sessionKey 转安全文件名
- * @param {string} sessionKey
- * @returns {string}
- */
 function keyToFilename(sessionKey) {
     const hash = crypto.createHash('md5').update(sessionKey).digest('hex');
     return `${hash}.json`;
 }
 
 /**
- * @typedef {Object} StoredSession
- * @property {string} sessionKey
- * @property {number} savedAt
- * @property {number} expiresAt
- * @property {Array<{name:string,value:string,domain?:string,path?:string}>} cookies
+ * @param {{name:string,domain?:string,path?:string}} cookie
+ * @returns {string}
  */
+function cookieIdentity(cookie) {
+    return `${cookie.name}|${cookie.domain || ''}|${cookie.path || '/'}`;
+}
+
+function mergeCookies(requestCookies, sessionCookies) {
+    const map = new Map();
+    for (const cookie of sessionCookies || []) {
+        if (cookie && cookie.name) map.set(cookieIdentity(cookie), cookie);
+    }
+    for (const cookie of requestCookies || []) {
+        if (cookie && cookie.name) map.set(cookieIdentity(cookie), cookie);
+    }
+    return Array.from(map.values());
+}
+
+function isSessionLikelyValid(session) {
+    if (!session || !Array.isArray(session.cookies)) return false;
+    if (session.expiresAt && Date.now() > session.expiresAt) return false;
+    return session.cookies.some((c) => c.name === 'cf_clearance' && c.value);
+}
+
+function hasCfClearance(session) {
+    if (!session || !Array.isArray(session.cookies)) return false;
+    return session.cookies.some((c) => c.name === 'cf_clearance' && c.value);
+}
 
-/**
- * 加载已保存的会话 cookies
- * @param {string} sessionKey
- * @returns {StoredSession|null}
- */
 function loadSession(sessionKey) {
     if (!sessionKey) return null;
     ensureSessionsDir();
     const filePath = path.join(SESSIONS_DIR, keyToFilename(sessionKey));
     if (!fs.existsSync(filePath)) return null;
-
     try {
         const raw = JSON.parse(fs.readFileSync(filePath, 'utf8'));
         if (!raw || !Array.isArray(raw.cookies)) return null;
@@ -57,45 +65,36 @@ function loadSession(sessionKey) {
     }
 }
 
-/**
- * 保存会话 cookies（优先保留 CF 相关 cookie）
- * @param {string} sessionKey
- * @param {Array<{name:string,value:string,domain?:string,path?:string}>} cookies
- * @param {number} [ttlMs]
- */
-function saveSession(sessionKey, cookies, ttlMs = SESSION_TTL_MS) {
+function touchSession(sessionKey, ttlMs = SESSION_TTL_MS) {
+    const session = loadSession(sessionKey);
+    if (!session) return;
+    const now = Date.now();
+    session.savedAt = now;
+    session.expiresAt = now + ttlMs;
+    fs.writeFileSync(path.join(SESSIONS_DIR, keyToFilename(sessionKey)), JSON.stringify(session, null, 0), { mode: 0o666 });
+}
+
+function saveSession(sessionKey, cookies, ttlMs = SESSION_TTL_MS, meta = {}) {
     if (!sessionKey || !Array.isArray(cookies) || cookies.length === 0) return;
     ensureSessionsDir();
-
-    const cfRelated = cookies.filter((c) =>
-        ['cf_clearance', '__cf_bm', 'cf_chl_2'].includes(c.name)
-        || c.name.startsWith('__cf')
-    );
-
-    const toSave = cfRelated.length > 0 ? cfRelated : cookies;
     const now = Date.now();
     const payload = {
         sessionKey,
         savedAt: now,
+        lastSuccessAt: now,
         expiresAt: now + ttlMs,
-        cookies: toSave.map((c) => ({
+        lastHost: meta.lastHost || '',
+        savedProfile: meta.savedProfile || '',
+        cookies: cookies.map((c) => ({
             name: c.name,
             value: c.value,
             domain: c.domain,
             path: c.path || '/',
         })),
     };
-
-    const filePath = path.join(SESSIONS_DIR, keyToFilename(sessionKey));
-    fs.writeFileSync(filePath, JSON.stringify(payload, null, 0), { mode: 0o666 });
+    fs.writeFileSync(path.join(SESSIONS_DIR, keyToFilename(sessionKey)), JSON.stringify(payload, null, 0), { mode: 0o666 });
 }
 
-/**
- * 将会话 cookies 转为 puppeteer setCookie 格式
- * @param {StoredSession|null} session
- * @param {string} pageUrl
- * @returns {Array<{name:string,value:string,domain?:string,path?:string,url?:string}>}
- */
 function sessionCookiesForPage(session, pageUrl) {
     if (!session || !Array.isArray(session.cookies)) return [];
     let hostname = '';
@@ -104,18 +103,54 @@ function sessionCookiesForPage(session, pageUrl) {
     } catch (_) {
         return session.cookies;
     }
-
     return session.cookies.map((c) => {
         const cookie = { ...c };
-        if (!cookie.domain && hostname) {
-            cookie.url = `https://${hostname}/`;
-        }
+        if (!cookie.domain && hostname) cookie.url = `https://${hostname}/`;
         return cookie;
     });
 }
 
+function getSessionDebugInfo(sessionKey) {
+    if (!sessionKey) return null;
+    const filePath = path.join(SESSIONS_DIR, keyToFilename(sessionKey));
+    const session = loadSession(sessionKey);
+    if (!session) {
+        return { sessionKey, exists: false, filePath };
+    }
+    let mtime = null;
+    try {
+        mtime = fs.statSync(filePath).mtime.toISOString();
+    } catch (_) { /* ignore */ }
+    return {
+        sessionKey,
+        exists: true,
+        filePath,
+        mtime,
+        savedAt: session.savedAt,
+        expiresAt: session.expiresAt,
+        lastSuccessAt: session.lastSuccessAt,
+        lastHost: session.lastHost,
+        savedProfile: session.savedProfile,
+        cookieCount: session.cookies.length,
+        hasCfClearance: hasCfClearance(session),
+        likelyValid: isSessionLikelyValid(session),
+    };
+}
+
+function getSessionStats() {
+    ensureSessionsDir();
+    const count = fs.readdirSync(SESSIONS_DIR).filter((f) => f.endsWith('.json')).length;
+    return { sessionsDir: SESSIONS_DIR, count };
+}
+
 module.exports = {
     loadSession,
     saveSession,
+    touchSession,
     sessionCookiesForPage,
+    mergeCookies,
+    isSessionLikelyValid,
+    hasCfClearance,
+    getSessionDebugInfo,
+    getSessionStats,
 };