links/patches/puppeteer-api/lib/ui-pagination-runner.js

/**
 * UI 翻页抓取：供 ui-pagination 与 auth-intercept-and-paginate 共用
 */
const crypto = require('crypto');

function generateSafeHash(obj) {
    if (!obj) return '';
    const clone = JSON.parse(JSON.stringify(obj));
    const removeDynamicKeys = (target) => {
        if (typeof target !== 'object' || target === null) return;
        ['timestamp', 'time', 'serverTime', 'reqId', 'requestId', 'traceId'].forEach((k) => delete target[k]);
        Object.values(target).forEach((val) => removeDynamicKeys(val));
    };
    removeDynamicKeys(clone);
    return crypto.createHash('md5').update(JSON.stringify(clone)).digest('hex');
}

function classifyPuppeteerError(err) {
    const msg = (err?.message || '').toLowerCase();
    if (err?.name === 'TimeoutError' || msg.includes('timeout')) return 'timeout';
    if (msg.includes('net::') || msg.includes('network') || msg.includes('err_connection')
        || msg.includes('err_address_unreachable') || msg.includes('err_name_not_resolved')) return 'network';
    if (msg.includes('navigation') || msg.includes('frame was detached')
        || msg.includes('detached frame') || msg.includes('attempted to use detached')) return 'navigation';
    if (msg.includes('target closed') || msg.includes('session closed')) return 'target_closed';
    return 'unknown';
}

/**
 * @param {import('puppeteer').Page} page
 * @param {{
 *   apiUrl: string,
 *   nextBtnSelector: string,
 *   clicksToPerform: number,
 *   waitMs?: number,
 *   firstPageData?: object,
 *   authActions?: object[],
 *   executeAuthActions?: (page: import('puppeteer').Page, actions: object[]) => Promise<void>,
 * }} options
 */
async function runUiPagination(page, options) {
    const {
        apiUrl,
        nextBtnSelector,
        clicksToPerform,
        waitMs = 2000,
        firstPageData,
        authActions,
        executeAuthActions,
    } = options;

    const capturedData = [];
    const debugLogs = [];
    const log = (msg) => { console.log(msg); debugLogs.push(msg); };
    const seenHashes = new Set();
    if (firstPageData) {
        seenHashes.add(generateSafeHash(firstPageData));
    }

    if (Array.isArray(authActions) && authActions.length > 0 && executeAuthActions) {
        log('[状态同步] 正在为翻页引擎注入授权状态...');
        await executeAuthActions(page, authActions);
        log('[状态同步] 授权执行完毕，等待目标列表加载...');
        const authSettled = await page.waitForResponse(
            (resp) => resp.url().includes(apiUrl) && resp.request().method() !== 'OPTIONS' && resp.status() >= 200 && resp.status() < 400,
            { timeout: 5000 }
        ).catch(() => null);
        if (!authSettled) {
            log('[状态同步] 未捕获到目标 API 响应，降级为固定等待 3 秒...');
            await new Promise((r) => setTimeout(r, 3000));
        }
    }

    for (let i = 0; i < clicksToPerform; i++) {
        const targetPageNum = i + 2;
        let pageData = null;
        let attempts = 0;
        const maxAttempts = 3;

        await page.evaluate(() => {
            document.querySelectorAll('.el-loading-mask, .ant-spin-nested-loading, .el-loading-spinner').forEach((el) => el.remove());
        });

        await page.waitForFunction(
            () => !document.querySelector('.el-loading-mask, .ant-spin-nested-loading, .el-loading-spinner'),
            { timeout: 1000, polling: 100 }
        ).catch(() => new Promise((r) => setTimeout(r, 500)));

        let paginationComplete = false;

        while (attempts < maxAttempts && !pageData && !paginationComplete) {
            attempts++;
            log(`[抓取] 准备抓取第 ${targetPageNum} 页 (尝试 ${attempts}/${maxAttempts})...`);

            let responseHandler = null;
            let apiTimeoutId = null;

            const waitForNewPageApi = new Promise((resolve) => {
                const cleanupListener = () => {
                    if (apiTimeoutId) { clearTimeout(apiTimeoutId); apiTimeoutId = null; }
                    if (responseHandler) { page.off('response', responseHandler); responseHandler = null; }
                };

                responseHandler = async (response) => {
                    const status = response.status();
                    if (response.url().includes(apiUrl) && response.request().method() !== 'OPTIONS' && status >= 200 && status < 400) {
                        try {
                            const responseData = await response.json();
                            const currentHash = generateSafeHash(responseData);
                            if (!seenHashes.has(currentHash)) {
                                seenHashes.add(currentHash);
                                cleanupListener();
                                resolve(responseData);
                            } else {
                                log('[防轮询] 抛弃重复数据，继续监听...');
                            }
                        } catch (_) { /* 非 JSON 响应跳过 */ }
                    }
                };
                page.on('response', responseHandler);
                apiTimeoutId = setTimeout(() => { cleanupListener(); resolve(null); }, 15000);
            });

            try {
                await page.waitForSelector(nextBtnSelector, { timeout: 15000 });

                const clickStatus = await page.evaluate((sel) => {
                    const btn = document.querySelector(sel);
                    if (!btn) return 'not_found';
                    if (btn.disabled || btn.classList.contains('is-disabled') || btn.classList.contains('disabled') || btn.getAttribute('aria-disabled') === 'true') {
                        return 'disabled';
                    }
                    const className = btn.className || '';
                    if (btn.disabled === true || className.includes('disabled') || btn.getAttribute('aria-disabled') === 'true') {
                        return 'disabled';
                    }
                    btn.scrollIntoView({ behavior: 'instant', block: 'center' });
                    btn.click();
                    btn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
                    return 'success';
                }, nextBtnSelector);

                if (clickStatus === 'disabled') {
                    log(`[提示] 第 ${targetPageNum} 页按钮已被禁用，说明到底了，抓取提前结束。`);
                    paginationComplete = true;
                    break;
                }
                log('[动作] 双引擎原生点击已执行！');
            } catch (clickErr) {
                log(`[错误] 寻找按钮异常 (${classifyPuppeteerError(clickErr)}): ${clickErr.message}`);
            }

            pageData = await waitForNewPageApi;
            if (!pageData && attempts < maxAttempts) {
                log('[重试警报] 雷达未扫描到新数据，准备重试...');
                await new Promise((r) => setTimeout(r, 500 * Math.pow(2, attempts - 1)));
            }
        }

        if (paginationComplete) {
            log('[完成] 翻页已到底，正常结束。');
            break;
        }

        if (pageData) {
            capturedData.push(pageData);
            log(`[成功] 斩获第 ${targetPageNum} 页数据！`);
            await new Promise((r) => setTimeout(r, waitMs));
        } else {
            log(`[致命异常] 连续 ${maxAttempts} 次尝试均失败，放弃后续翻页。`);
            break;
        }
    }

    return { data: capturedData, debugLogs };
}

module.exports = {
    runUiPagination,
    generateSafeHash,
};