/** * UI 翻页抓取:供 ui-pagination 与 auth-intercept-and-paginate 共用 */ const crypto = require('crypto'); function generateSafeHash(obj) { if (!obj) return ''; const clone = JSON.parse(JSON.stringify(obj)); const removeDynamicKeys = (target) => { if (typeof target !== 'object' || target === null) return; ['timestamp', 'time', 'serverTime', 'reqId', 'requestId', 'traceId'].forEach((k) => delete target[k]); Object.values(target).forEach((val) => removeDynamicKeys(val)); }; removeDynamicKeys(clone); return crypto.createHash('md5').update(JSON.stringify(clone)).digest('hex'); } function classifyPuppeteerError(err) { const msg = (err?.message || '').toLowerCase(); if (err?.name === 'TimeoutError' || msg.includes('timeout')) return 'timeout'; if (msg.includes('net::') || msg.includes('network') || msg.includes('err_connection') || msg.includes('err_address_unreachable') || msg.includes('err_name_not_resolved')) return 'network'; if (msg.includes('navigation') || msg.includes('frame was detached') || msg.includes('detached frame') || msg.includes('attempted to use detached')) return 'navigation'; if (msg.includes('target closed') || msg.includes('session closed')) return 'target_closed'; return 'unknown'; } /** * @param {import('puppeteer').Page} page * @param {{ * apiUrl: string, * nextBtnSelector: string, * clicksToPerform: number, * waitMs?: number, * firstPageData?: object, * authActions?: object[], * executeAuthActions?: (page: import('puppeteer').Page, actions: object[]) => Promise, * }} options */ async function runUiPagination(page, options) { const { apiUrl, nextBtnSelector, clicksToPerform, waitMs = 2000, firstPageData, authActions, executeAuthActions, } = options; const capturedData = []; const debugLogs = []; const log = (msg) => { console.log(msg); debugLogs.push(msg); }; const seenHashes = new Set(); if (firstPageData) { seenHashes.add(generateSafeHash(firstPageData)); } if (Array.isArray(authActions) && authActions.length > 0 && executeAuthActions) { log('[状态同步] 正在为翻页引擎注入授权状态...'); await executeAuthActions(page, authActions); log('[状态同步] 授权执行完毕,等待目标列表加载...'); const authSettled = await page.waitForResponse( (resp) => resp.url().includes(apiUrl) && resp.request().method() !== 'OPTIONS' && resp.status() >= 200 && resp.status() < 400, { timeout: 5000 } ).catch(() => null); if (!authSettled) { log('[状态同步] 未捕获到目标 API 响应,降级为固定等待 3 秒...'); await new Promise((r) => setTimeout(r, 3000)); } } for (let i = 0; i < clicksToPerform; i++) { const targetPageNum = i + 2; let pageData = null; let attempts = 0; const maxAttempts = 3; await page.evaluate(() => { document.querySelectorAll('.el-loading-mask, .ant-spin-nested-loading, .el-loading-spinner').forEach((el) => el.remove()); }); await page.waitForFunction( () => !document.querySelector('.el-loading-mask, .ant-spin-nested-loading, .el-loading-spinner'), { timeout: 1000, polling: 100 } ).catch(() => new Promise((r) => setTimeout(r, 500))); let paginationComplete = false; while (attempts < maxAttempts && !pageData && !paginationComplete) { attempts++; log(`[抓取] 准备抓取第 ${targetPageNum} 页 (尝试 ${attempts}/${maxAttempts})...`); let responseHandler = null; let apiTimeoutId = null; const waitForNewPageApi = new Promise((resolve) => { const cleanupListener = () => { if (apiTimeoutId) { clearTimeout(apiTimeoutId); apiTimeoutId = null; } if (responseHandler) { page.off('response', responseHandler); responseHandler = null; } }; responseHandler = async (response) => { const status = response.status(); if (response.url().includes(apiUrl) && response.request().method() !== 'OPTIONS' && status >= 200 && status < 400) { try { const responseData = await response.json(); const currentHash = generateSafeHash(responseData); if (!seenHashes.has(currentHash)) { seenHashes.add(currentHash); cleanupListener(); resolve(responseData); } else { log('[防轮询] 抛弃重复数据,继续监听...'); } } catch (_) { /* 非 JSON 响应跳过 */ } } }; page.on('response', responseHandler); apiTimeoutId = setTimeout(() => { cleanupListener(); resolve(null); }, 15000); }); try { await page.waitForSelector(nextBtnSelector, { timeout: 15000 }); const clickStatus = await page.evaluate((sel) => { const btn = document.querySelector(sel); if (!btn) return 'not_found'; if (btn.disabled || btn.classList.contains('is-disabled') || btn.classList.contains('disabled') || btn.getAttribute('aria-disabled') === 'true') { return 'disabled'; } const className = btn.className || ''; if (btn.disabled === true || className.includes('disabled') || btn.getAttribute('aria-disabled') === 'true') { return 'disabled'; } btn.scrollIntoView({ behavior: 'instant', block: 'center' }); btn.click(); btn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window })); return 'success'; }, nextBtnSelector); if (clickStatus === 'disabled') { log(`[提示] 第 ${targetPageNum} 页按钮已被禁用,说明到底了,抓取提前结束。`); paginationComplete = true; break; } log('[动作] 双引擎原生点击已执行!'); } catch (clickErr) { log(`[错误] 寻找按钮异常 (${classifyPuppeteerError(clickErr)}): ${clickErr.message}`); } pageData = await waitForNewPageApi; if (!pageData && attempts < maxAttempts) { log('[重试警报] 雷达未扫描到新数据,准备重试...'); await new Promise((r) => setTimeout(r, 500 * Math.pow(2, attempts - 1))); } } if (paginationComplete) { log('[完成] 翻页已到底,正常结束。'); break; } if (pageData) { capturedData.push(pageData); log(`[成功] 斩获第 ${targetPageNum} 页数据!`); await new Promise((r) => setTimeout(r, waitMs)); } else { log(`[致命异常] 连续 ${maxAttempts} 次尝试均失败,放弃后续翻页。`); break; } } return { data: capturedData, debugLogs }; } module.exports = { runUiPagination, generateSafeHash, };