优化爬虫，增加缓存，线程池，保存浏览器用户信息
This commit is contained in:
root
@@ -0,0 +1,172 @@
|
||||
/**
|
||||
* UI 翻页抓取：供 ui-pagination 与 auth-intercept-and-paginate 共用
|
||||
*/
|
||||
const crypto = require('crypto');
|
||||
|
||||
function generateSafeHash(obj) {
|
||||
if (!obj) return '';
|
||||
const clone = JSON.parse(JSON.stringify(obj));
|
||||
const removeDynamicKeys = (target) => {
|
||||
if (typeof target !== 'object' || target === null) return;
|
||||
['timestamp', 'time', 'serverTime', 'reqId', 'requestId', 'traceId'].forEach((k) => delete target[k]);
|
||||
Object.values(target).forEach((val) => removeDynamicKeys(val));
|
||||
};
|
||||
removeDynamicKeys(clone);
|
||||
return crypto.createHash('md5').update(JSON.stringify(clone)).digest('hex');
|
||||
}
|
||||
|
||||
function classifyPuppeteerError(err) {
|
||||
const msg = (err?.message || '').toLowerCase();
|
||||
if (err?.name === 'TimeoutError' || msg.includes('timeout')) return 'timeout';
|
||||
if (msg.includes('net::') || msg.includes('network') || msg.includes('err_connection')
|
||||
|| msg.includes('err_address_unreachable') || msg.includes('err_name_not_resolved')) return 'network';
|
||||
if (msg.includes('navigation') || msg.includes('frame was detached')) return 'navigation';
|
||||
if (msg.includes('target closed') || msg.includes('session closed')) return 'target_closed';
|
||||
return 'unknown';
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @param {{
|
||||
* apiUrl: string,
|
||||
* nextBtnSelector: string,
|
||||
* clicksToPerform: number,
|
||||
* waitMs?: number,
|
||||
* firstPageData?: object,
|
||||
* authActions?: object[],
|
||||
* executeAuthActions?: (page: import('puppeteer').Page, actions: object[]) => Promise<void>,
|
||||
* }} options
|
||||
*/
|
||||
async function runUiPagination(page, options) {
|
||||
const {
|
||||
apiUrl,
|
||||
nextBtnSelector,
|
||||
clicksToPerform,
|
||||
waitMs = 2000,
|
||||
firstPageData,
|
||||
authActions,
|
||||
executeAuthActions,
|
||||
} = options;
|
||||
|
||||
const capturedData = [];
|
||||
const debugLogs = [];
|
||||
const log = (msg) => { console.log(msg); debugLogs.push(msg); };
|
||||
const seenHashes = new Set();
|
||||
if (firstPageData) {
|
||||
seenHashes.add(generateSafeHash(firstPageData));
|
||||
}
|
||||
|
||||
if (Array.isArray(authActions) && authActions.length > 0 && executeAuthActions) {
|
||||
log('[状态同步] 正在为翻页引擎注入授权状态...');
|
||||
await executeAuthActions(page, authActions);
|
||||
log('[状态同步] 授权执行完毕，等待目标列表加载...');
|
||||
const authSettled = await page.waitForResponse(
|
||||
(resp) => resp.url().includes(apiUrl) && resp.request().method() !== 'OPTIONS' && resp.status() >= 200 && resp.status() < 400,
|
||||
{ timeout: 5000 }
|
||||
).catch(() => null);
|
||||
if (!authSettled) {
|
||||
log('[状态同步] 未捕获到目标 API 响应，降级为固定等待 3 秒...');
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < clicksToPerform; i++) {
|
||||
const targetPageNum = i + 2;
|
||||
let pageData = null;
|
||||
let attempts = 0;
|
||||
const maxAttempts = 3;
|
||||
|
||||
await page.evaluate(() => {
|
||||
document.querySelectorAll('.el-loading-mask, .ant-spin-nested-loading, .el-loading-spinner').forEach((el) => el.remove());
|
||||
});
|
||||
|
||||
await page.waitForFunction(
|
||||
() => !document.querySelector('.el-loading-mask, .ant-spin-nested-loading, .el-loading-spinner'),
|
||||
{ timeout: 1000, polling: 100 }
|
||||
).catch(() => new Promise((r) => setTimeout(r, 500)));
|
||||
|
||||
while (attempts < maxAttempts && !pageData) {
|
||||
attempts++;
|
||||
log(`[抓取] 准备抓取第 ${targetPageNum} 页 (尝试 ${attempts}/${maxAttempts})...`);
|
||||
|
||||
let responseHandler = null;
|
||||
let apiTimeoutId = null;
|
||||
|
||||
const waitForNewPageApi = new Promise((resolve) => {
|
||||
const cleanupListener = () => {
|
||||
if (apiTimeoutId) { clearTimeout(apiTimeoutId); apiTimeoutId = null; }
|
||||
if (responseHandler) { page.off('response', responseHandler); responseHandler = null; }
|
||||
};
|
||||
|
||||
responseHandler = async (response) => {
|
||||
const status = response.status();
|
||||
if (response.url().includes(apiUrl) && response.request().method() !== 'OPTIONS' && status >= 200 && status < 400) {
|
||||
try {
|
||||
const responseData = await response.json();
|
||||
const currentHash = generateSafeHash(responseData);
|
||||
if (!seenHashes.has(currentHash)) {
|
||||
seenHashes.add(currentHash);
|
||||
cleanupListener();
|
||||
resolve(responseData);
|
||||
} else {
|
||||
log('[防轮询] 抛弃重复数据，继续监听...');
|
||||
}
|
||||
} catch (_) { /* 非 JSON 响应跳过 */ }
|
||||
}
|
||||
};
|
||||
page.on('response', responseHandler);
|
||||
apiTimeoutId = setTimeout(() => { cleanupListener(); resolve(null); }, 15000);
|
||||
});
|
||||
|
||||
try {
|
||||
await page.waitForSelector(nextBtnSelector, { timeout: 15000 });
|
||||
|
||||
const clickStatus = await page.evaluate((sel) => {
|
||||
const btn = document.querySelector(sel);
|
||||
if (!btn) return 'not_found';
|
||||
if (btn.disabled || btn.classList.contains('is-disabled') || btn.classList.contains('disabled') || btn.getAttribute('aria-disabled') === 'true') {
|
||||
return 'disabled';
|
||||
}
|
||||
const className = btn.className || '';
|
||||
if (btn.disabled === true || className.includes('disabled') || btn.getAttribute('aria-disabled') === 'true') {
|
||||
return 'disabled';
|
||||
}
|
||||
btn.scrollIntoView({ behavior: 'instant', block: 'center' });
|
||||
btn.click();
|
||||
btn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
|
||||
return 'success';
|
||||
}, nextBtnSelector);
|
||||
|
||||
if (clickStatus === 'disabled') {
|
||||
log(`[提示] 第 ${targetPageNum} 页按钮已被禁用，说明到底了，抓取提前结束。`);
|
||||
break;
|
||||
}
|
||||
log('[动作] 双引擎原生点击已执行！');
|
||||
} catch (clickErr) {
|
||||
log(`[错误] 寻找按钮异常 (${classifyPuppeteerError(clickErr)}): ${clickErr.message}`);
|
||||
}
|
||||
|
||||
pageData = await waitForNewPageApi;
|
||||
if (!pageData && attempts < maxAttempts) {
|
||||
log('[重试警报] 雷达未扫描到新数据，准备重试...');
|
||||
await new Promise((r) => setTimeout(r, 500 * Math.pow(2, attempts - 1)));
|
||||
}
|
||||
}
|
||||
|
||||
if (pageData) {
|
||||
capturedData.push(pageData);
|
||||
log(`[成功] 斩获第 ${targetPageNum} 页数据！`);
|
||||
await new Promise((r) => setTimeout(r, waitMs));
|
||||
} else {
|
||||
log(`[致命异常] 连续 ${maxAttempts} 次尝试均失败，放弃后续翻页。`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return { data: capturedData, debugLogs };
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
runUiPagination,
|
||||
generateSafeHash,
|
||||
};
|
||||
Reference in New Issue
Block a user