diff --git a/_php_code/AbstractScrmSpider.class.php b/_php_code/AbstractScrmSpider.class.php index 362557a..39efeb7 100755 --- a/_php_code/AbstractScrmSpider.class.php +++ b/_php_code/AbstractScrmSpider.class.php @@ -112,14 +112,21 @@ abstract class AbstractScrmSpider // 'apiUrls' => $apiUrlsToIntercept, // 'authActions' => $config['authActions'] // ]); + $antiBot = $config['antiBot'] ?? null; + // 【阶段一】:初始化并首屏拦截 $initResult = $this->requestNode('/api/auth-and-intercept', [ 'pageUrl' => $config['pageUrl'], 'apiUrls' => $apiUrlsToIntercept, - 'authActions' => $config['authActions'] - ]); + 'authActions' => $config['authActions'], + 'antiBot' => $antiBot, + ], $this->resolveAuthInterceptTimeout($antiBot)); if (empty($initResult['success'])) { + $cfCode = (string) ($initResult['code'] ?? ''); + if ($cfCode === 'CF_TURNSTILE_FAILED') { + throw new Exception('Cloudflare Turnstile 验证失败: ' . ($initResult['stage'] ?? 'timeout')); + } throw new Exception("初始化失败: " . ($initResult['error'] ?? '未知')); } @@ -161,7 +168,8 @@ abstract class AbstractScrmSpider 'paramList' => $paramList, 'method' => $config['listMethod'] ?? 'GET' ]], - 'cookies' => $cookies + 'cookies' => $cookies, + 'antiBot' => $antiBot, ], 120); if (!empty($fetchResult['success'])) { @@ -189,7 +197,8 @@ abstract class AbstractScrmSpider 'cookies' => $cookies, 'firstPageData' => $firstPageData, // 传递原始数据源 // 🔥 核心补充:把登录剧本原封不动地传给翻页引擎! - 'authActions' => $config['authActions'] + 'authActions' => $config['authActions'], + 'antiBot' => $antiBot, ], 1200); // 增加 PHP 端的 cURL 超时时间到 400 秒,以包容多次重试产生的耗时 if (!empty($uiResult['success']) && !empty($uiResult['data'])) { @@ -220,6 +229,19 @@ abstract class AbstractScrmSpider return $unifiedData; } + /** + * Real Browser + Turnstile 首屏耗时较长,单独放宽 auth-and-intercept 超时 + * + * @param array|null $antiBot + */ + private function resolveAuthInterceptTimeout($antiBot) + { + if (!is_array($antiBot) || empty($antiBot['enabled'])) { + return 120; + } + return 180; + } + private function requestNode($endpoint, $payload, $timeout = 60) { $ch = curl_init($this->nodeHost . $endpoint); @@ -227,6 +249,7 @@ abstract class AbstractScrmSpider curl_setopt($ch, CURLOPT_POST, true); curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($payload)); curl_setopt($ch, CURLOPT_HTTPHEADER, ['Content-Type: application/json']); + curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5); curl_setopt($ch, CURLOPT_TIMEOUT, $timeout); $response = curl_exec($ch); if (curl_errno($ch)) throw new Exception(curl_error($ch)); diff --git a/_php_code/Chatknow.php b/_php_code/Chatknow.php index 6976d9a..833110a 100644 --- a/_php_code/Chatknow.php +++ b/_php_code/Chatknow.php @@ -26,6 +26,8 @@ class Chatknow extends AbstractScrmSpider protected function getSpiderConfig() { + $host = (string) parse_url($this->pageUrl, PHP_URL_HOST); + return [ 'pageUrl' => $this->pageUrl, 'apiUrls' => [self::API_LIST], @@ -43,7 +45,16 @@ class Chatknow extends AbstractScrmSpider // ['type' => 'vue_click', 'selector' => '.layui-btn', 'text' => '搜索'], ['type' => 'wait', 'ms' => 4000] - ] + ], + // ChatKnow 专用:Real Browser + Turnstile + Captcha API 兜底 + 会话复用 + 'antiBot' => [ + 'enabled' => true, + 'profile' => 'real', + 'turnstile' => true, + 'solverFallback' => true, + 'sessionKey' => 'chatknow:' . $host, + 'challengeTimeoutMs' => 60000, + ], ]; } diff --git a/_php_code/index.php b/_php_code/index.php index 65b207a..b0ed85e 100755 --- a/_php_code/index.php +++ b/_php_code/index.php @@ -11,58 +11,36 @@ require_once __DIR__ . '/SsCustomer.php'; // SS云控(Customer) require_once __DIR__ . '/Haiwang.php'; // 海王 require_once __DIR__ . '/Whatshub.php'; // Whatshub 工单平台 -try { - /* - 命令行可以测试Cloudflare防火墙 - curl -s -X POST http://127.0.0.1:3001/api/auth-and-intercept \ - -H 'Content-Type: application/json' \ - -d '{ - "pageUrl": "https://web.whatshub.cc/m/iTYsWKQH5030/1", - "apiUrls": ["/api/whatshub-counter/workShare/open/detail"], - "authActions": [ - {"type": "vue_fill", "selector": ".el-input__inner", "value": "745030"}, - {"type": "wait", "ms": 500}, - {"type": "vue_click", "selector": ".vxe-button-group .theme--primary"}, - {"type": "wait", "ms": 2200} - ], - "antiBot": { - "enabled": true, - "profile": "real", - "turnstile": true, - "solverFallback": true, - "sessionKey": "whatshub:t.flowerbells.top", - "challengeTimeoutMs": 60000 - } - }' | jq . - */ - - echo "🚀 开始执行抓取任务 (多引擎智能调度)...\n\r"; - $pageUrl = 'https://web.whatshub.cc/m/iTYsWKQH5030/1'; // PageUrl 入口授权页 - $username = ""; // 登录账号 - $password = "745030"; // 登录密码 - $spider = new Whatshub($pageUrl, $username, $password); - $finalData = $spider->run(); - - echo "✅ 任务完成!统一数据如下:\n\r"; - echo "----------------------------------------\n\r"; - echo "当日新增:{$finalData->todayNewCount} 人\n\r"; - echo "在线号码:{$finalData->totalOnline} 个\n\r"; - echo "离线号码:{$finalData->totalOffline} 个\n\r"; - echo "Total:" . $finalData->total . " 个号码\n\r"; - echo "实际总共抓取:" . count($finalData->numbers) . " 个号码\n\r"; - echo "号码列表:\n\r"; - echo dd($finalData->numbers); -} catch (Exception $e) { - echo "🚨 抓取异常:" . $e->getMessage() . "\n\r"; -} - - // try { -// echo "🚀 开始执行抓取任务 (多引擎智能调度)...\n\r"; -// $pageUrl = 'https://user.chatknow.com/child/workorder-share?shareKey=jf5t6MNrJ2mC76N'; // PageUrl 入口授权页 +// /* +// 命令行可以测试Cloudflare防火墙 +// curl -s -X POST http://127.0.0.1:3001/api/auth-and-intercept \ +// -H 'Content-Type: application/json' \ +// -d '{ +// "pageUrl": "https://web.whatshub.cc/m/iTYsWKQH5030/1", +// "apiUrls": ["/api/whatshub-counter/workShare/open/detail"], +// "authActions": [ +// {"type": "vue_fill", "selector": ".el-input__inner", "value": "745030"}, +// {"type": "wait", "ms": 500}, +// {"type": "vue_click", "selector": ".vxe-button-group .theme--primary"}, +// {"type": "wait", "ms": 2200} +// ], +// "antiBot": { +// "enabled": true, +// "profile": "real", +// "turnstile": true, +// "solverFallback": true, +// "sessionKey": "whatshub:t.flowerbells.top", +// "challengeTimeoutMs": 60000 +// } +// }' | jq . +// */ + +// echo "🚀 开始执行抓取任务 (多引擎智能调度)...\n\r"; +// $pageUrl = 'https://web.whatshub.cc/m/iTYsWKQH5030/1'; // PageUrl 入口授权页 // $username = ""; // 登录账号 -// $password = ""; // 登录密码 -// $spider = new Chatknow($pageUrl, $username, $password); +// $password = "745030"; // 登录密码 +// $spider = new Whatshub($pageUrl, $username, $password); // $finalData = $spider->run(); // echo "✅ 任务完成!统一数据如下:\n\r"; @@ -78,6 +56,28 @@ try { // echo "🚨 抓取异常:" . $e->getMessage() . "\n\r"; // } + +try { + echo "🚀 开始执行抓取任务 (多引擎智能调度)...\n\r"; + $pageUrl = 'https://user.chatknow.com/child/workorder-share?shareKey=jf5t6MNrJ2mC76N'; // PageUrl 入口授权页 + $username = ""; // 登录账号 + $password = ""; // 登录密码 + $spider = new Chatknow($pageUrl, $username, $password); + $finalData = $spider->run(); + + echo "✅ 任务完成!统一数据如下:\n\r"; + echo "----------------------------------------\n\r"; + echo "当日新增:{$finalData->todayNewCount} 人\n\r"; + echo "在线号码:{$finalData->totalOnline} 个\n\r"; + echo "离线号码:{$finalData->totalOffline} 个\n\r"; + echo "Total:" . $finalData->total . " 个号码\n\r"; + echo "实际总共抓取:" . count($finalData->numbers) . " 个号码\n\r"; + echo "号码列表:\n\r"; + echo dd($finalData->numbers); +} catch (Exception $e) { + echo "🚨 抓取异常:" . $e->getMessage() . "\n\r"; +} + // try { // echo "🚀 开始执行抓取任务 (多引擎智能调度)...\n\r"; // $pageUrl = 'https://yyk.ink/8415O53'; // PageUrl 入口授权页 diff --git a/application/common/library/scrm/AbstractScrmSpider.php b/application/common/library/scrm/AbstractScrmSpider.php index 902252a..6627f6c 100755 --- a/application/common/library/scrm/AbstractScrmSpider.php +++ b/application/common/library/scrm/AbstractScrmSpider.php @@ -68,13 +68,18 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface } $antiBot = $config['antiBot'] ?? null; + $mode = $config['paginationMode'] ?? self::MODE_FETCH; + + if ($this->shouldUseUnifiedBrowserPath($antiBot, $mode)) { + return $this->runUnifiedUiPath($config, $antiBot, $apiUrlsToIntercept, $listApi, $detailApi, $countApi); + } $initResult = $this->requestNode('/api/auth-and-intercept', [ 'pageUrl' => $config['pageUrl'], 'apiUrls' => $apiUrlsToIntercept, 'authActions' => $config['authActions'] ?? [], 'antiBot' => $antiBot, - ]); + ], $this->resolveAuthInterceptTimeout($antiBot)); if (empty($initResult['success'])) { $cfCode = (string) ($initResult['code'] ?? ''); @@ -95,8 +100,10 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface $interceptedApis = $initResult['interceptedApis']; $finalPageUrl = (string) ($initResult['finalPageUrl'] ?? ($config['pageUrl'] ?? '')); SplitTicketSyncLogger::log('spider', 'auth-and-intercept ok', [ - 'intercepted' => array_keys($interceptedApis), - 'finalPageUrl' => $finalPageUrl, + 'intercepted' => array_keys($interceptedApis), + 'finalPageUrl' => $finalPageUrl, + 'cfStage' => $initResult['cf']['turnstileStage'] ?? ($initResult['cf']['stage'] ?? null), + 'browserReused' => $initResult['browserReused'] ?? null, ]); $cookies = $initResult['cookies']; @@ -113,7 +120,6 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface $allListPagesData = [$listApiNode['data']]; $totalPages = $this->extractListTotalPages($listApiNode['data'], $countData); - $mode = $config['paginationMode'] ?? self::MODE_FETCH; SplitTicketSyncLogger::log('spider', 'pagination plan', [ 'totalPages' => $totalPages, 'mode' => $mode, @@ -182,6 +188,124 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface return $result; } + /** + * 单 Browser 路径:auth + 拦截 + UI 翻页(需 antiBot.sessionKey) + * + * @param array $config + * @param array|null $antiBot + * @param list $apiUrlsToIntercept + */ + private function runUnifiedUiPath( + array $config, + ?array $antiBot, + array $apiUrlsToIntercept, + string $listApi, + ?string $detailApi, + ?string $countApi + ): UnifiedScrmData { + $uiConfig = $this->getUiPaginationConfig(); + $sessionKey = is_array($antiBot) ? (string) ($antiBot['sessionKey'] ?? '') : ''; + + SplitTicketSyncLogger::log('spider', 'unified browser path', [ + 'sessionKey' => $sessionKey, + 'listApi' => $listApi, + ]); + + $mergedResult = $this->requestNode('/api/auth-intercept-and-paginate', [ + 'pageUrl' => $config['pageUrl'], + 'apiUrls' => $apiUrlsToIntercept, + 'authActions' => $config['authActions'] ?? [], + 'antiBot' => $antiBot, + 'pagination' => [ + 'apiUrl' => $listApi, + 'nextBtnSelector' => $uiConfig['nextBtnSelector'] ?? '', + 'waitMs' => $uiConfig['waitMs'] ?? 2000, + 'clicksToPerform' => 9999, + ], + ], 1200); + + if (empty($mergedResult['success'])) { + $cfCode = (string) ($mergedResult['code'] ?? ''); + SplitTicketSyncLogger::log('spider', 'auth-intercept-and-paginate failed', [ + 'error' => $mergedResult['error'] ?? '未知', + 'code' => $cfCode !== '' ? $cfCode : null, + 'stage' => $mergedResult['stage'] ?? null, + 'cf' => $mergedResult['cf'] ?? null, + 'sessionKey' => $sessionKey, + ]); + if ($cfCode === 'CF_TURNSTILE_FAILED') { + throw new Exception('Cloudflare Turnstile 验证失败: ' . ($mergedResult['stage'] ?? 'timeout')); + } + throw new Exception('合并同步失败: ' . (string) ($mergedResult['error'] ?? '未知')); + } + + SplitTicketSyncLogger::log('spider', 'auth-intercept-and-paginate ok', [ + 'intercepted' => array_keys($mergedResult['interceptedApis'] ?? []), + 'extraPages' => count($mergedResult['extraPages'] ?? []), + 'cfStage' => $mergedResult['cf']['turnstileStage'] ?? ($mergedResult['cf']['stage'] ?? null), + 'browserReused' => $mergedResult['browserReused'] ?? null, + 'sessionKey' => $sessionKey, + ]); + + $interceptedApis = $mergedResult['interceptedApis'] ?? []; + if (!isset($interceptedApis[$listApi])) { + throw new Exception("致命错误:未能拦截到必须的列表接口 [{$listApi}]"); + } + + $detailData = $detailApi && isset($interceptedApis[$detailApi]) + ? $interceptedApis[$detailApi]['data'] : null; + $countData = $countApi && isset($interceptedApis[$countApi]) + ? $interceptedApis[$countApi]['data'] : null; + + $listApiNode = $interceptedApis[$listApi]; + $allListPagesData = [$listApiNode['data']]; + if (!empty($mergedResult['extraPages']) && is_array($mergedResult['extraPages'])) { + foreach ($mergedResult['extraPages'] as $pageData) { + $allListPagesData[] = $pageData; + } + } + + $this->extractListTotalPages($listApiNode['data'], $countData); + + $result = $this->parseToUnifiedData($detailData, $allListPagesData); + SplitTicketSyncLogger::log('spider', 'run done', [ + 'todayNewCount' => $result->todayNewCount, + 'totalOnline' => $result->totalOnline, + 'totalOffline' => $result->totalOffline, + 'total' => $result->total, + 'numberCount' => count($result->numbers), + 'unifiedPath' => true, + ]); + return $result; + } + + /** + * @param array|null $antiBot + */ + protected function shouldUseUnifiedBrowserPath(?array $antiBot, string $mode): bool + { + if ($mode !== self::MODE_UI) { + return false; + } + if (!is_array($antiBot) || empty($antiBot['enabled'])) { + return false; + } + return !empty($antiBot['sessionKey']); + } + + /** + * Real Browser + Turnstile 首屏耗时较长,单独放宽 auth-and-intercept 超时 + * + * @param array|null $antiBot + */ + protected function resolveAuthInterceptTimeout(?array $antiBot): int + { + if (!is_array($antiBot) || empty($antiBot['enabled'])) { + return 120; + } + return 180; + } + /** * @param array $payload * @return array diff --git a/application/common/library/scrm/AntiBotConfigBuilder.php b/application/common/library/scrm/AntiBotConfigBuilder.php new file mode 100644 index 0000000..bc6ff0a --- /dev/null +++ b/application/common/library/scrm/AntiBotConfigBuilder.php @@ -0,0 +1,84 @@ +|null + */ + public static function build(string $ticketType, string $pageUrl): ?array + { + $enabledTypes = self::getEnabledTypes(); + if (!in_array($ticketType, $enabledTypes, true)) { + return null; + } + + $host = (string) parse_url($pageUrl, PHP_URL_HOST); + if ($host === '') { + return null; + } + + $sessionKey = $ticketType . ':' . $host; + $ttlMinutes = self::getSessionTtlMinutes(); + + return [ + 'enabled' => true, + 'profile' => 'real', + 'turnstile' => true, + 'solverFallback' => true, + 'sessionKey' => $sessionKey, + 'challengeTimeoutMs' => 60000, + 'sessionTtlMinutes' => $ttlMinutes, + ]; + } + + /** + * 解析 sessionKey(供 cron 分组调度使用) + */ + public static function resolveSessionKey(string $ticketType, string $pageUrl): string + { + $host = (string) parse_url($pageUrl, PHP_URL_HOST); + return $ticketType . ':' . ($host !== '' ? $host : 'unknown'); + } + + /** + * 是否对该工单类型启用 antiBot + */ + public static function isEnabledForType(string $ticketType): bool + { + return in_array($ticketType, self::getEnabledTypes(), true); + } + + /** + * @return list + */ + public static function getEnabledTypes(): array + { + $raw = (string) Config::get('site.split_scrm_antibot_types'); + if ($raw === '') { + return ['whatshub', 'chatknow']; + } + $parts = array_map('trim', explode(',', $raw)); + return array_values(array_filter($parts, static function (string $v): bool { + return $v !== ''; + })); + } + + public static function getSessionTtlMinutes(): int + { + $minutes = (int) Config::get('site.split_scrm_session_ttl_minutes'); + return $minutes > 0 ? $minutes : 120; + } +} diff --git a/application/common/library/scrm/spider/ChatknowSpider.php b/application/common/library/scrm/spider/ChatknowSpider.php index 704341a..ffd4f20 100755 --- a/application/common/library/scrm/spider/ChatknowSpider.php +++ b/application/common/library/scrm/spider/ChatknowSpider.php @@ -5,6 +5,7 @@ declare(strict_types=1); namespace app\common\library\scrm\spider; use app\common\library\scrm\AbstractScrmSpider; +use app\common\library\scrm\AntiBotConfigBuilder; use app\common\library\scrm\UnifiedScrmData; /** @@ -48,6 +49,7 @@ class ChatknowSpider extends AbstractScrmSpider 'authActions' => [ ['type' => 'wait', 'ms' => 4000], ], + 'antiBot' => AntiBotConfigBuilder::build('chatknow', $this->pageUrl), ]; } diff --git a/application/common/library/scrm/spider/WhatshubSpider.php b/application/common/library/scrm/spider/WhatshubSpider.php index 771d170..f658508 100755 --- a/application/common/library/scrm/spider/WhatshubSpider.php +++ b/application/common/library/scrm/spider/WhatshubSpider.php @@ -5,6 +5,7 @@ declare(strict_types=1); namespace app\common\library\scrm\spider; use app\common\library\scrm\AbstractScrmSpider; +use app\common\library\scrm\AntiBotConfigBuilder; use app\common\library\scrm\UnifiedScrmData; /** @@ -47,8 +48,6 @@ class WhatshubSpider extends AbstractScrmSpider /** @return array */ protected function getSpiderConfig(): array { - $host = (string) parse_url($this->pageUrl, PHP_URL_HOST); - return [ 'pageUrl' => $this->pageUrl, 'listApi' => self::API_LIST, @@ -62,15 +61,7 @@ class WhatshubSpider extends AbstractScrmSpider ['type' => 'vue_click', 'selector' => '.vxe-button-group .theme--primary'], ['type' => 'wait', 'ms' => 2200], ], - // Whatshub 专用:Real Browser + Turnstile + Captcha API 兜底 + 会话复用 - 'antiBot' => [ - 'enabled' => true, - 'profile' => 'real', - 'turnstile' => true, - 'solverFallback' => true, - 'sessionKey' => 'whatshub:' . $host, - 'challengeTimeoutMs' => 60000, - ], + 'antiBot' => AntiBotConfigBuilder::build('whatshub', $this->pageUrl), ]; } diff --git a/application/common/service/SplitTicketSyncService.php b/application/common/service/SplitTicketSyncService.php index 80c9b73..9f3de47 100755 --- a/application/common/service/SplitTicketSyncService.php +++ b/application/common/service/SplitTicketSyncService.php @@ -5,6 +5,7 @@ declare(strict_types=1); namespace app\common\service; use app\admin\model\split\Ticket; +use app\common\library\scrm\AntiBotConfigBuilder; use app\common\library\scrm\UnifiedScrmData; use think\Db; use think\Exception; @@ -107,6 +108,7 @@ class SplitTicketSyncService $list = $query->order('sync_time', 'asc')->order('id', 'asc') ->limit($maxPerRun * 5) ->select(); + $list = $this->sortTicketsBySessionKey($list); $candidateCount = count($list); SplitTicketSyncLogger::logCron('scan start', [ @@ -115,6 +117,7 @@ class SplitTicketSyncService 'autoSyncTypes' => $autoSyncTypes, 'failThreshold' => $failThreshold, 'nodeQueue' => SplitNodeHealthService::getQueueSnapshot(), + 'groupedBy' => 'sessionKey', ]); SplitTicketSyncLogger::log('cron', 'scan start', [ 'candidateCount' => $candidateCount, @@ -134,6 +137,7 @@ class SplitTicketSyncService $ticketId = (int) $ticket['id']; $ticketUrl = (string) $ticket['ticket_url']; $ticketType = (string) $ticket['ticket_type']; + $sessionKey = AntiBotConfigBuilder::resolveSessionKey($ticketType, $ticketUrl); if ($count >= $maxPerRun) { $reason = '本轮处理上限已满'; @@ -183,6 +187,7 @@ class SplitTicketSyncService 'ticketId' => $ticketId, 'ticketType' => $ticketType, 'ticketUrl' => $ticketUrl, + 'sessionKey' => $sessionKey, 'success' => !empty($result['success']), 'message' => (string) ($result['message'] ?? ''), ]; @@ -190,8 +195,8 @@ class SplitTicketSyncService } $candidateIds = array_map(static function ($row) { - return (int) $row['id']; - }, $list instanceof \think\Collection ? $list->all() : (array) $list); + return (int) (is_array($row) ? ($row['id'] ?? 0) : ($row['id'] ?? 0)); + }, $list); $openNotSynced = $this->buildOpenNotSyncedAudit( $autoSyncTypes, $failThreshold, @@ -573,4 +578,35 @@ class SplitTicketSyncService $skip = $this->shouldSkip(Ticket::get((int) $ticket['id']) ?: new Ticket($ticket)); return $skip ?? '未知原因,未进入执行队列'; } + + /** + * 按 sessionKey(ticketType:host)分组排序,使同域名工单连续执行以命中 Browser 温池 + * + * @param \think\Collection|array> $list + * @return list> + */ + private function sortTicketsBySessionKey($list): array + { + $rows = $list instanceof \think\Collection ? $list->all() : (array) $list; + usort($rows, static function ($a, $b): int { + $typeA = (string) (is_array($a) ? ($a['ticket_type'] ?? '') : ($a['ticket_type'] ?? '')); + $urlA = (string) (is_array($a) ? ($a['ticket_url'] ?? '') : ($a['ticket_url'] ?? '')); + $typeB = (string) (is_array($b) ? ($b['ticket_type'] ?? '') : ($b['ticket_type'] ?? '')); + $urlB = (string) (is_array($b) ? ($b['ticket_url'] ?? '') : ($b['ticket_url'] ?? '')); + $keyA = AntiBotConfigBuilder::resolveSessionKey($typeA, $urlA); + $keyB = AntiBotConfigBuilder::resolveSessionKey($typeB, $urlB); + if ($keyA === $keyB) { + $timeA = (int) (is_array($a) ? ($a['sync_time'] ?? 0) : ($a['sync_time'] ?? 0)); + $timeB = (int) (is_array($b) ? ($b['sync_time'] ?? 0) : ($b['sync_time'] ?? 0)); + if ($timeA === $timeB) { + $idA = (int) (is_array($a) ? ($a['id'] ?? 0) : ($a['id'] ?? 0)); + $idB = (int) (is_array($b) ? ($b['id'] ?? 0) : ($b['id'] ?? 0)); + return $idA <=> $idB; + } + return $timeA <=> $timeB; + } + return strcmp($keyA, $keyB); + }); + return $rows; + } } diff --git a/application/extra/site.php b/application/extra/site.php index f4238a5..de7eba1 100755 --- a/application/extra/site.php +++ b/application/extra/site.php @@ -44,6 +44,8 @@ return array ( ), 'split_platform_domain' => 'flowerbells.top', 'split_scrm_node_host' => 'http://127.0.0.1:3001', + 'split_scrm_antibot_types' => 'whatshub,chatknow', + 'split_scrm_session_ttl_minutes' => '120', 'split_sync_interval_a2c' => '3', 'split_sync_interval_haiwang' => '3', 'split_sync_interval_huojian' => '3', diff --git a/links.test_mZZS6.zip b/links.test_mZZS6.zip deleted file mode 100755 index 45ae0ab..0000000 Binary files a/links.test_mZZS6.zip and /dev/null differ diff --git a/patches/application/common/library/scrm/AbstractScrmSpider.php b/patches/application/common/library/scrm/AbstractScrmSpider.php index 902252a..6627f6c 100755 --- a/patches/application/common/library/scrm/AbstractScrmSpider.php +++ b/patches/application/common/library/scrm/AbstractScrmSpider.php @@ -68,13 +68,18 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface } $antiBot = $config['antiBot'] ?? null; + $mode = $config['paginationMode'] ?? self::MODE_FETCH; + + if ($this->shouldUseUnifiedBrowserPath($antiBot, $mode)) { + return $this->runUnifiedUiPath($config, $antiBot, $apiUrlsToIntercept, $listApi, $detailApi, $countApi); + } $initResult = $this->requestNode('/api/auth-and-intercept', [ 'pageUrl' => $config['pageUrl'], 'apiUrls' => $apiUrlsToIntercept, 'authActions' => $config['authActions'] ?? [], 'antiBot' => $antiBot, - ]); + ], $this->resolveAuthInterceptTimeout($antiBot)); if (empty($initResult['success'])) { $cfCode = (string) ($initResult['code'] ?? ''); @@ -95,8 +100,10 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface $interceptedApis = $initResult['interceptedApis']; $finalPageUrl = (string) ($initResult['finalPageUrl'] ?? ($config['pageUrl'] ?? '')); SplitTicketSyncLogger::log('spider', 'auth-and-intercept ok', [ - 'intercepted' => array_keys($interceptedApis), - 'finalPageUrl' => $finalPageUrl, + 'intercepted' => array_keys($interceptedApis), + 'finalPageUrl' => $finalPageUrl, + 'cfStage' => $initResult['cf']['turnstileStage'] ?? ($initResult['cf']['stage'] ?? null), + 'browserReused' => $initResult['browserReused'] ?? null, ]); $cookies = $initResult['cookies']; @@ -113,7 +120,6 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface $allListPagesData = [$listApiNode['data']]; $totalPages = $this->extractListTotalPages($listApiNode['data'], $countData); - $mode = $config['paginationMode'] ?? self::MODE_FETCH; SplitTicketSyncLogger::log('spider', 'pagination plan', [ 'totalPages' => $totalPages, 'mode' => $mode, @@ -182,6 +188,124 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface return $result; } + /** + * 单 Browser 路径:auth + 拦截 + UI 翻页(需 antiBot.sessionKey) + * + * @param array $config + * @param array|null $antiBot + * @param list $apiUrlsToIntercept + */ + private function runUnifiedUiPath( + array $config, + ?array $antiBot, + array $apiUrlsToIntercept, + string $listApi, + ?string $detailApi, + ?string $countApi + ): UnifiedScrmData { + $uiConfig = $this->getUiPaginationConfig(); + $sessionKey = is_array($antiBot) ? (string) ($antiBot['sessionKey'] ?? '') : ''; + + SplitTicketSyncLogger::log('spider', 'unified browser path', [ + 'sessionKey' => $sessionKey, + 'listApi' => $listApi, + ]); + + $mergedResult = $this->requestNode('/api/auth-intercept-and-paginate', [ + 'pageUrl' => $config['pageUrl'], + 'apiUrls' => $apiUrlsToIntercept, + 'authActions' => $config['authActions'] ?? [], + 'antiBot' => $antiBot, + 'pagination' => [ + 'apiUrl' => $listApi, + 'nextBtnSelector' => $uiConfig['nextBtnSelector'] ?? '', + 'waitMs' => $uiConfig['waitMs'] ?? 2000, + 'clicksToPerform' => 9999, + ], + ], 1200); + + if (empty($mergedResult['success'])) { + $cfCode = (string) ($mergedResult['code'] ?? ''); + SplitTicketSyncLogger::log('spider', 'auth-intercept-and-paginate failed', [ + 'error' => $mergedResult['error'] ?? '未知', + 'code' => $cfCode !== '' ? $cfCode : null, + 'stage' => $mergedResult['stage'] ?? null, + 'cf' => $mergedResult['cf'] ?? null, + 'sessionKey' => $sessionKey, + ]); + if ($cfCode === 'CF_TURNSTILE_FAILED') { + throw new Exception('Cloudflare Turnstile 验证失败: ' . ($mergedResult['stage'] ?? 'timeout')); + } + throw new Exception('合并同步失败: ' . (string) ($mergedResult['error'] ?? '未知')); + } + + SplitTicketSyncLogger::log('spider', 'auth-intercept-and-paginate ok', [ + 'intercepted' => array_keys($mergedResult['interceptedApis'] ?? []), + 'extraPages' => count($mergedResult['extraPages'] ?? []), + 'cfStage' => $mergedResult['cf']['turnstileStage'] ?? ($mergedResult['cf']['stage'] ?? null), + 'browserReused' => $mergedResult['browserReused'] ?? null, + 'sessionKey' => $sessionKey, + ]); + + $interceptedApis = $mergedResult['interceptedApis'] ?? []; + if (!isset($interceptedApis[$listApi])) { + throw new Exception("致命错误:未能拦截到必须的列表接口 [{$listApi}]"); + } + + $detailData = $detailApi && isset($interceptedApis[$detailApi]) + ? $interceptedApis[$detailApi]['data'] : null; + $countData = $countApi && isset($interceptedApis[$countApi]) + ? $interceptedApis[$countApi]['data'] : null; + + $listApiNode = $interceptedApis[$listApi]; + $allListPagesData = [$listApiNode['data']]; + if (!empty($mergedResult['extraPages']) && is_array($mergedResult['extraPages'])) { + foreach ($mergedResult['extraPages'] as $pageData) { + $allListPagesData[] = $pageData; + } + } + + $this->extractListTotalPages($listApiNode['data'], $countData); + + $result = $this->parseToUnifiedData($detailData, $allListPagesData); + SplitTicketSyncLogger::log('spider', 'run done', [ + 'todayNewCount' => $result->todayNewCount, + 'totalOnline' => $result->totalOnline, + 'totalOffline' => $result->totalOffline, + 'total' => $result->total, + 'numberCount' => count($result->numbers), + 'unifiedPath' => true, + ]); + return $result; + } + + /** + * @param array|null $antiBot + */ + protected function shouldUseUnifiedBrowserPath(?array $antiBot, string $mode): bool + { + if ($mode !== self::MODE_UI) { + return false; + } + if (!is_array($antiBot) || empty($antiBot['enabled'])) { + return false; + } + return !empty($antiBot['sessionKey']); + } + + /** + * Real Browser + Turnstile 首屏耗时较长,单独放宽 auth-and-intercept 超时 + * + * @param array|null $antiBot + */ + protected function resolveAuthInterceptTimeout(?array $antiBot): int + { + if (!is_array($antiBot) || empty($antiBot['enabled'])) { + return 120; + } + return 180; + } + /** * @param array $payload * @return array diff --git a/patches/application/common/library/scrm/AntiBotConfigBuilder.php b/patches/application/common/library/scrm/AntiBotConfigBuilder.php new file mode 100644 index 0000000..bc6ff0a --- /dev/null +++ b/patches/application/common/library/scrm/AntiBotConfigBuilder.php @@ -0,0 +1,84 @@ +|null + */ + public static function build(string $ticketType, string $pageUrl): ?array + { + $enabledTypes = self::getEnabledTypes(); + if (!in_array($ticketType, $enabledTypes, true)) { + return null; + } + + $host = (string) parse_url($pageUrl, PHP_URL_HOST); + if ($host === '') { + return null; + } + + $sessionKey = $ticketType . ':' . $host; + $ttlMinutes = self::getSessionTtlMinutes(); + + return [ + 'enabled' => true, + 'profile' => 'real', + 'turnstile' => true, + 'solverFallback' => true, + 'sessionKey' => $sessionKey, + 'challengeTimeoutMs' => 60000, + 'sessionTtlMinutes' => $ttlMinutes, + ]; + } + + /** + * 解析 sessionKey(供 cron 分组调度使用) + */ + public static function resolveSessionKey(string $ticketType, string $pageUrl): string + { + $host = (string) parse_url($pageUrl, PHP_URL_HOST); + return $ticketType . ':' . ($host !== '' ? $host : 'unknown'); + } + + /** + * 是否对该工单类型启用 antiBot + */ + public static function isEnabledForType(string $ticketType): bool + { + return in_array($ticketType, self::getEnabledTypes(), true); + } + + /** + * @return list + */ + public static function getEnabledTypes(): array + { + $raw = (string) Config::get('site.split_scrm_antibot_types'); + if ($raw === '') { + return ['whatshub', 'chatknow']; + } + $parts = array_map('trim', explode(',', $raw)); + return array_values(array_filter($parts, static function (string $v): bool { + return $v !== ''; + })); + } + + public static function getSessionTtlMinutes(): int + { + $minutes = (int) Config::get('site.split_scrm_session_ttl_minutes'); + return $minutes > 0 ? $minutes : 120; + } +} diff --git a/patches/application/common/library/scrm/spider/ChatknowSpider.php b/patches/application/common/library/scrm/spider/ChatknowSpider.php index 704341a..ffd4f20 100755 --- a/patches/application/common/library/scrm/spider/ChatknowSpider.php +++ b/patches/application/common/library/scrm/spider/ChatknowSpider.php @@ -5,6 +5,7 @@ declare(strict_types=1); namespace app\common\library\scrm\spider; use app\common\library\scrm\AbstractScrmSpider; +use app\common\library\scrm\AntiBotConfigBuilder; use app\common\library\scrm\UnifiedScrmData; /** @@ -48,6 +49,7 @@ class ChatknowSpider extends AbstractScrmSpider 'authActions' => [ ['type' => 'wait', 'ms' => 4000], ], + 'antiBot' => AntiBotConfigBuilder::build('chatknow', $this->pageUrl), ]; } diff --git a/patches/application/common/library/scrm/spider/WhatshubSpider.php b/patches/application/common/library/scrm/spider/WhatshubSpider.php index 771d170..f658508 100755 --- a/patches/application/common/library/scrm/spider/WhatshubSpider.php +++ b/patches/application/common/library/scrm/spider/WhatshubSpider.php @@ -5,6 +5,7 @@ declare(strict_types=1); namespace app\common\library\scrm\spider; use app\common\library\scrm\AbstractScrmSpider; +use app\common\library\scrm\AntiBotConfigBuilder; use app\common\library\scrm\UnifiedScrmData; /** @@ -47,8 +48,6 @@ class WhatshubSpider extends AbstractScrmSpider /** @return array */ protected function getSpiderConfig(): array { - $host = (string) parse_url($this->pageUrl, PHP_URL_HOST); - return [ 'pageUrl' => $this->pageUrl, 'listApi' => self::API_LIST, @@ -62,15 +61,7 @@ class WhatshubSpider extends AbstractScrmSpider ['type' => 'vue_click', 'selector' => '.vxe-button-group .theme--primary'], ['type' => 'wait', 'ms' => 2200], ], - // Whatshub 专用:Real Browser + Turnstile + Captcha API 兜底 + 会话复用 - 'antiBot' => [ - 'enabled' => true, - 'profile' => 'real', - 'turnstile' => true, - 'solverFallback' => true, - 'sessionKey' => 'whatshub:' . $host, - 'challengeTimeoutMs' => 60000, - ], + 'antiBot' => AntiBotConfigBuilder::build('whatshub', $this->pageUrl), ]; } diff --git a/patches/application/common/service/SplitTicketSyncService.php b/patches/application/common/service/SplitTicketSyncService.php index 80c9b73..9f3de47 100755 --- a/patches/application/common/service/SplitTicketSyncService.php +++ b/patches/application/common/service/SplitTicketSyncService.php @@ -5,6 +5,7 @@ declare(strict_types=1); namespace app\common\service; use app\admin\model\split\Ticket; +use app\common\library\scrm\AntiBotConfigBuilder; use app\common\library\scrm\UnifiedScrmData; use think\Db; use think\Exception; @@ -107,6 +108,7 @@ class SplitTicketSyncService $list = $query->order('sync_time', 'asc')->order('id', 'asc') ->limit($maxPerRun * 5) ->select(); + $list = $this->sortTicketsBySessionKey($list); $candidateCount = count($list); SplitTicketSyncLogger::logCron('scan start', [ @@ -115,6 +117,7 @@ class SplitTicketSyncService 'autoSyncTypes' => $autoSyncTypes, 'failThreshold' => $failThreshold, 'nodeQueue' => SplitNodeHealthService::getQueueSnapshot(), + 'groupedBy' => 'sessionKey', ]); SplitTicketSyncLogger::log('cron', 'scan start', [ 'candidateCount' => $candidateCount, @@ -134,6 +137,7 @@ class SplitTicketSyncService $ticketId = (int) $ticket['id']; $ticketUrl = (string) $ticket['ticket_url']; $ticketType = (string) $ticket['ticket_type']; + $sessionKey = AntiBotConfigBuilder::resolveSessionKey($ticketType, $ticketUrl); if ($count >= $maxPerRun) { $reason = '本轮处理上限已满'; @@ -183,6 +187,7 @@ class SplitTicketSyncService 'ticketId' => $ticketId, 'ticketType' => $ticketType, 'ticketUrl' => $ticketUrl, + 'sessionKey' => $sessionKey, 'success' => !empty($result['success']), 'message' => (string) ($result['message'] ?? ''), ]; @@ -190,8 +195,8 @@ class SplitTicketSyncService } $candidateIds = array_map(static function ($row) { - return (int) $row['id']; - }, $list instanceof \think\Collection ? $list->all() : (array) $list); + return (int) (is_array($row) ? ($row['id'] ?? 0) : ($row['id'] ?? 0)); + }, $list); $openNotSynced = $this->buildOpenNotSyncedAudit( $autoSyncTypes, $failThreshold, @@ -573,4 +578,35 @@ class SplitTicketSyncService $skip = $this->shouldSkip(Ticket::get((int) $ticket['id']) ?: new Ticket($ticket)); return $skip ?? '未知原因,未进入执行队列'; } + + /** + * 按 sessionKey(ticketType:host)分组排序,使同域名工单连续执行以命中 Browser 温池 + * + * @param \think\Collection|array> $list + * @return list> + */ + private function sortTicketsBySessionKey($list): array + { + $rows = $list instanceof \think\Collection ? $list->all() : (array) $list; + usort($rows, static function ($a, $b): int { + $typeA = (string) (is_array($a) ? ($a['ticket_type'] ?? '') : ($a['ticket_type'] ?? '')); + $urlA = (string) (is_array($a) ? ($a['ticket_url'] ?? '') : ($a['ticket_url'] ?? '')); + $typeB = (string) (is_array($b) ? ($b['ticket_type'] ?? '') : ($b['ticket_type'] ?? '')); + $urlB = (string) (is_array($b) ? ($b['ticket_url'] ?? '') : ($b['ticket_url'] ?? '')); + $keyA = AntiBotConfigBuilder::resolveSessionKey($typeA, $urlA); + $keyB = AntiBotConfigBuilder::resolveSessionKey($typeB, $urlB); + if ($keyA === $keyB) { + $timeA = (int) (is_array($a) ? ($a['sync_time'] ?? 0) : ($a['sync_time'] ?? 0)); + $timeB = (int) (is_array($b) ? ($b['sync_time'] ?? 0) : ($b['sync_time'] ?? 0)); + if ($timeA === $timeB) { + $idA = (int) (is_array($a) ? ($a['id'] ?? 0) : ($a['id'] ?? 0)); + $idB = (int) (is_array($b) ? ($b['id'] ?? 0) : ($b['id'] ?? 0)); + return $idA <=> $idB; + } + return $timeA <=> $timeB; + } + return strcmp($keyA, $keyB); + }); + return $rows; + } } diff --git a/patches/puppeteer-api/lib/browser-factory.js b/patches/puppeteer-api/lib/browser-factory.js index 4fa831b..b4b6476 100755 --- a/patches/puppeteer-api/lib/browser-factory.js +++ b/patches/puppeteer-api/lib/browser-factory.js @@ -1,34 +1,22 @@ /** - * BrowserFactory:standard / real 双轨启动 + 独立并发限流 + * BrowserFactory:standard / real 双轨启动 + 温池 + 独立并发限流 */ const { DEFAULT_UA, MAX_CONCURRENT_BROWSERS, MAX_CONCURRENT_BROWSERS_REAL, QUEUE_TIMEOUT_MS, + NAVIGATION_TIMEOUT_MS, + PAGE_DEFAULT_TIMEOUT_MS, } = require('./constants'); const { BrowserConcurrencyLimiter } = require('./concurrency'); const { launchStandardBrowser } = require('./launch-standard'); const { launchRealBrowser, isRealBrowserAvailable } = require('./launch-real-browser'); +const { browserPool } = require('./browser-pool'); const standardLimiter = new BrowserConcurrencyLimiter(MAX_CONCURRENT_BROWSERS); const realLimiter = new BrowserConcurrencyLimiter(MAX_CONCURRENT_BROWSERS_REAL); -/** - * @typedef {Object} AntiBotConfig - * @property {boolean} [enabled] - * @property {'standard'|'real'} [profile] - * @property {boolean} [turnstile] - * @property {boolean} [solverFallback] - * @property {string} [sessionKey] - * @property {number} [challengeTimeoutMs] - */ - -/** - * 规范化 antiBot 配置(未传则走 standard) - * @param {AntiBotConfig|null|undefined} antiBot - * @returns {AntiBotConfig} - */ function normalizeAntiBot(antiBot) { if (!antiBot || !antiBot.enabled) { return { enabled: false, profile: 'standard' }; @@ -43,62 +31,66 @@ function normalizeAntiBot(antiBot) { }; } -/** - * 按 profile 获取限流器 - * @param {'standard'|'real'} profile - */ function getLimiter(profile) { return profile === 'real' ? realLimiter : standardLimiter; } /** - * 创建 Browser 会话 - * @param {{ profile?: 'standard'|'real', extraArgs?: string[] }} options - * @returns {Promise<{ browser: import('puppeteer').Browser, page: import('puppeteer').Page|null, profile: string, cleanup: () => Promise }>} + * 冷启动 Browser(供温池 launcher 使用) + * @param {{ profile?: 'standard'|'real', extraArgs?: string[], sessionKey?: string }} options */ async function createBrowserSession(options = {}) { const profile = options.profile === 'real' ? 'real' : 'standard'; const extraArgs = options.extraArgs || []; + const sessionKey = options.sessionKey || ''; if (profile === 'real') { - const { browser, page } = await launchRealBrowser(extraArgs); + const { browser, page } = await launchRealBrowser(extraArgs, { sessionKey, profile: 'real' }); return { browser, page, profile: 'real', - cleanup: async () => { - try { - await browser.close(); - } catch (_) { - /* ignore */ + cleanup: async (destroy = false) => { + if (destroy || !sessionKey) { + try { + await browser.close(); + } catch (_) { /* ignore */ } } }, }; } - const browser = await launchStandardBrowser(extraArgs); + const browser = await launchStandardBrowser(extraArgs, { sessionKey, profile: 'standard' }); return { browser, page: null, profile: 'standard', - cleanup: async () => { - try { - await browser.close(); - } catch (_) { - /* ignore */ + cleanup: async (destroy = false) => { + if (destroy || !sessionKey) { + try { + await browser.close(); + } catch (_) { /* ignore */ } } }, }; } /** - * 在并发槽位内执行 Browser 任务 - * @template T - * @param {string} taskName - * @param {'standard'|'real'} profile - * @param {() => Promise} fn - * @returns {Promise} + * 从温池或冷启动获取 Browser + * @param {{ profile?: 'standard'|'real', extraArgs?: string[], sessionKey?: string }} options */ +async function acquireBrowserSession(options = {}) { + const profile = options.profile === 'real' ? 'real' : 'standard'; + const sessionKey = options.sessionKey || ''; + const extraArgs = options.extraArgs || []; + + return browserPool.acquire(sessionKey, profile, async () => createBrowserSession({ + profile, + extraArgs, + sessionKey, + })); +} + async function runWithProfileLimit(taskName, profile, fn) { const limiter = getLimiter(profile); try { @@ -109,23 +101,15 @@ async function runWithProfileLimit(taskName, profile, fn) { } } -/** - * 创建 Page 并设置 UA / viewport / cookies - * @param {import('puppeteer').Browser} browser - * @param {import('puppeteer').Page|null} existingPage - * @param {{ userAgent?: string, viewport?: object, cookies?: Array }} [options] - */ async function createManagedPage(browser, existingPage, options = {}) { const page = existingPage || await browser.newPage(); - page.setDefaultNavigationTimeout(30000); - page.setDefaultTimeout(15000); + page.setDefaultNavigationTimeout(NAVIGATION_TIMEOUT_MS); + page.setDefaultTimeout(PAGE_DEFAULT_TIMEOUT_MS); const userAgent = options.userAgent || DEFAULT_UA; await page.setUserAgent(userAgent); - if (options.viewport) { - await page.setViewport(options.viewport); - } + if (options.viewport) await page.setViewport(options.viewport); if (options.cookies && options.cookies.length > 0) { await page.setCookie(...options.cookies); } @@ -136,14 +120,12 @@ async function createManagedPage(browser, existingPage, options = {}) { return page; } -/** - * @returns {{ standard: object, real: object, realBrowserReady: boolean }} - */ function getFactoryStats() { return { standard: standardLimiter.getStats(), real: realLimiter.getStats(), realBrowserReady: isRealBrowserAvailable(), + pool: browserPool.getStats(), }; } @@ -151,10 +133,12 @@ module.exports = { normalizeAntiBot, getLimiter, createBrowserSession, + acquireBrowserSession, runWithProfileLimit, createManagedPage, getFactoryStats, standardLimiter, realLimiter, + browserPool, QUEUE_TIMEOUT_MS, }; diff --git a/patches/puppeteer-api/lib/browser-pool.js b/patches/puppeteer-api/lib/browser-pool.js new file mode 100644 index 0000000..f7a66e7 --- /dev/null +++ b/patches/puppeteer-api/lib/browser-pool.js @@ -0,0 +1,184 @@ +/** + * Browser 温池:按 profile:sessionKey 复用 Browser/Page,降低冷启动与 CF 触发 + */ +const { POOL_IDLE_MS, MAX_POOLED_BROWSERS, POOL_ENABLED } = require('./constants'); + +class BrowserPool { + constructor() { + /** @type {Map} */ + this.entries = new Map(); + this.hits = 0; + this.misses = 0; + this.evictions = 0; + this._sweepTimer = setInterval(() => this.evictIdle(), 30000); + this._sweepTimer.unref?.(); + } + + /** + * @param {string} sessionKey + * @param {'standard'|'real'} profile + */ + buildKey(sessionKey, profile) { + return `${profile}:${sessionKey}`; + } + + /** + * @param {any} browser + */ + async isBrowserAlive(browser) { + if (!browser) return false; + try { + if (typeof browser.isConnected === 'function' && !browser.isConnected()) { + return false; + } + await browser.version(); + return true; + } catch (_) { + return false; + } + } + + /** + * @param {string} key + */ + async destroyEntry(key) { + const entry = this.entries.get(key); + if (!entry) return; + this.entries.delete(key); + try { + await entry.browser.close(); + } catch (_) { + /* ignore */ + } + this.evictions++; + } + + enforceCapacity(excludeKey) { + if (this.entries.size < MAX_POOLED_BROWSERS) return; + const candidates = [...this.entries.entries()] + .filter(([k, e]) => k !== excludeKey && !e.inUse) + .sort((a, b) => a[1].lastUsedAt - b[1].lastUsedAt); + if (candidates.length === 0) return; + const [oldestKey] = candidates[0]; + this.destroyEntry(oldestKey); + } + + /** + * @param {string} sessionKey + * @param {'standard'|'real'} profile + * @param {() => Promise<{ browser: any, page: any|null, cleanup: (destroy?: boolean) => Promise }>} launcher + */ + async acquire(sessionKey, profile, launcher) { + if (!POOL_ENABLED || !sessionKey) { + const launched = await launcher(); + return { + browser: launched.browser, + page: launched.page, + profile, + pooled: false, + reused: false, + release: async (destroy = false) => { + await launched.cleanup(destroy); + }, + }; + } + + const key = this.buildKey(sessionKey, profile); + const existing = this.entries.get(key); + if (existing && !existing.inUse && await this.isBrowserAlive(existing.browser)) { + existing.inUse = true; + existing.lastUsedAt = Date.now(); + this.hits++; + return { + browser: existing.browser, + page: existing.page, + profile, + pooled: true, + reused: true, + release: async (destroy = false) => this.release(key, destroy), + }; + } + if (existing) { + await this.destroyEntry(key); + } + + this.enforceCapacity(key); + this.misses++; + const launched = await launcher(); + this.entries.set(key, { + browser: launched.browser, + page: launched.page, + profile, + sessionKey, + lastUsedAt: Date.now(), + inUse: true, + }); + + return { + browser: launched.browser, + page: launched.page, + profile, + pooled: true, + reused: false, + release: async (destroy = false) => this.release(key, destroy), + }; + } + + /** + * @param {string} key + * @param {boolean} destroy + */ + async release(key, destroy = false) { + const entry = this.entries.get(key); + if (!entry) return; + entry.inUse = false; + entry.lastUsedAt = Date.now(); + if (destroy || !(await this.isBrowserAlive(entry.browser))) { + await this.destroyEntry(key); + } + } + + evictIdle() { + const now = Date.now(); + for (const [key, entry] of this.entries.entries()) { + if (!entry.inUse && now - entry.lastUsedAt > POOL_IDLE_MS) { + this.destroyEntry(key); + } + } + } + + async drain() { + clearInterval(this._sweepTimer); + const keys = [...this.entries.keys()]; + for (const key of keys) { + await this.destroyEntry(key); + } + } + + getStats() { + let idle = 0; + let active = 0; + for (const entry of this.entries.values()) { + if (entry.inUse) active++; + else idle++; + } + return { + enabled: POOL_ENABLED, + max: MAX_POOLED_BROWSERS, + idleMs: POOL_IDLE_MS, + size: this.entries.size, + idle, + active, + hits: this.hits, + misses: this.misses, + evictions: this.evictions, + }; + } +} + +const browserPool = new BrowserPool(); + +module.exports = { + BrowserPool, + browserPool, +}; diff --git a/patches/puppeteer-api/lib/cf-handler.js b/patches/puppeteer-api/lib/cf-handler.js index 73d87eb..b595b2e 100755 --- a/patches/puppeteer-api/lib/cf-handler.js +++ b/patches/puppeteer-api/lib/cf-handler.js @@ -1,53 +1,75 @@ /** - * Cloudflare 挑战统一处理:检测 → 内置等待 → Captcha API 兜底 + * Cloudflare 挑战统一处理:检测 → 会话快速通道 → 内置等待 → Captcha API 兜底 */ const { detectCloudflare } = require('./cf-detector'); const { waitForTurnstile, extractTurnstileSitekey, injectTurnstileToken } = require('./turnstile-handler'); const { solveTurnstile, isCaptchaConfigured } = require('./captcha-solver'); +const { isSessionLikelyValid } = require('./session-store'); /** - * @typedef {Object} CfHandleResult - * @property {boolean} success - * @property {string|null} code - * @property {string|null} challengeType - * @property {string|null} stage - * @property {boolean} solverUsed - * @property {number} elapsedMs - * @property {boolean} cfDetected - */ - -/** - * 处理 Cloudflare / Turnstile 挑战(在 authActions 之前调用) * @param {import('puppeteer').Page} page * @param {{ turnstile?: boolean, solverFallback?: boolean, challengeTimeoutMs?: number }} antiBot * @param {(page: import('puppeteer').Page) => Promise} [waitForUrlSettled] - * @returns {Promise} + * @param {import('./session-store').StoredSession|null} [storedSession] */ -async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled) { +async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, storedSession = null) { const started = Date.now(); const timeoutMs = antiBot.challengeTimeoutMs || 60000; let cfState = await detectCloudflare(page); + + if (!cfState.blocked && cfState.hasCfClearance) { + return { + success: true, + code: null, + challengeType: null, + stage: 'session_reuse', + solverUsed: false, + elapsedMs: Date.now() - started, + cfDetected: false, + }; + } + if (!cfState.blocked) { return { success: true, code: null, challengeType: null, - stage: null, + stage: storedSession && isSessionLikelyValid(storedSession) ? 'session_reuse' : null, solverUsed: false, elapsedMs: Date.now() - started, cfDetected: false, }; } + if (storedSession && isSessionLikelyValid(storedSession)) { + console.log('[CF] 会话有效但仍被拦截,尝试 reload 一次'); + try { + await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 45000) }); + if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {}); + cfState = await detectCloudflare(page); + if (!cfState.blocked) { + return { + success: true, + code: null, + challengeType: cfState.challengeType, + stage: 'session_reload', + solverUsed: false, + elapsedMs: Date.now() - started, + cfDetected: true, + }; + } + } catch (reloadErr) { + console.warn('[CF] reload 失败:', reloadErr.message); + } + } + console.log(`[CF] 检测到挑战 type=${cfState.challengeType} url=${cfState.url}`); if (antiBot.turnstile !== false) { const builtIn = await waitForTurnstile(page, { timeoutMs, useBuiltInClick: true }); if (builtIn.success) { - if (waitForUrlSettled) { - await waitForUrlSettled(page).catch(() => {}); - } + if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {}); cfState = await detectCloudflare(page); if (!cfState.blocked) { return { @@ -74,9 +96,7 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled) { const apiWaitMs = Math.min(timeoutMs, 30000); await waitForTurnstile(page, { timeoutMs: apiWaitMs }); - if (waitForUrlSettled) { - await waitForUrlSettled(page).catch(() => {}); - } + if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {}); cfState = await detectCloudflare(page); if (!cfState.blocked) { diff --git a/patches/puppeteer-api/lib/constants.js b/patches/puppeteer-api/lib/constants.js index a259a28..4298d62 100755 --- a/patches/puppeteer-api/lib/constants.js +++ b/patches/puppeteer-api/lib/constants.js @@ -5,13 +5,9 @@ const fs = require('fs'); const path = require('path'); const puppeteer = require('puppeteer-extra'); -/** 默认 Chrome 可执行文件路径(生产环境可通过环境变量覆盖) */ const DEFAULT_CHROME_EXECUTABLE = '/www/wwwroot/puppeteer-api/.cache/puppeteer/chrome/linux-149.0.7827.22/chrome-linux64/chrome'; - -/** 默认 User-Agent,与 standard / real 双轨保持一致 */ const DEFAULT_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'; -/** Chrome 启动基础参数:多实例 headless 场景下复用 */ const BASE_LAUNCH_ARGS = [ '--no-sandbox', '--disable-setuid-sandbox', @@ -26,24 +22,14 @@ const BASE_LAUNCH_ARGS = [ '--disable-crash-reporter', ]; -/** puppeteer-api 专用运行时目录(与 server.js 同级 .runtime) */ const RUNTIME_DIR = path.join(__dirname, '..', '.runtime'); -/** - * 确保 .runtime 子目录存在且 www 用户可写 - * @param {string} name - * @returns {string} - */ function ensureRuntimeSubdir(name) { const dir = path.join(RUNTIME_DIR, name); fs.mkdirSync(dir, { recursive: true, mode: 0o777 }); return dir; } -/** - * 解析 Chrome 可执行文件路径 - * @returns {string} - */ function resolveChromeExecutable() { const candidates = [ process.env.PUPPETEER_EXECUTABLE_PATH, @@ -63,16 +49,12 @@ function resolveChromeExecutable() { return bundled; } } catch (_) { - /* puppeteer 未内置浏览器时忽略 */ + /* ignore */ } return candidates[0] || DEFAULT_CHROME_EXECUTABLE; } -/** - * 生产 Linux 常注入无效 DBUS 地址;删除而非设为 /dev/null - * @returns {NodeJS.ProcessEnv} - */ function buildBrowserLaunchEnv() { const env = { ...process.env }; delete env.DBUS_SESSION_BUS_ADDRESS; @@ -87,10 +69,6 @@ function buildBrowserLaunchEnv() { return env; } -/** - * 校验 Chrome 可执行文件存在 - * @returns {string} - */ function assertChromeExecutable() { const executablePath = resolveChromeExecutable(); if (!fs.existsSync(executablePath)) { @@ -101,23 +79,48 @@ function assertChromeExecutable() { return executablePath; } -/** 并发与超时配置(可通过环境变量覆盖) */ const MAX_CONCURRENT_BROWSERS = Math.max(1, parseInt(process.env.MAX_CONCURRENT_BROWSERS || '4', 10)); const MAX_CONCURRENT_BROWSERS_REAL = Math.max(1, parseInt(process.env.MAX_CONCURRENT_BROWSERS_REAL || '2', 10)); const QUEUE_TIMEOUT_MS = Math.max(5000, parseInt(process.env.QUEUE_TIMEOUT_MS || '120000', 10)); -const API_INTERCEPT_TIMEOUT_MS = Math.max(10000, parseInt(process.env.API_INTERCEPT_TIMEOUT_MS || '30000', 10)); +const API_INTERCEPT_TIMEOUT_MS = Math.max(10000, parseInt(process.env.API_INTERCEPT_TIMEOUT_MS || '60000', 10)); const API_INTERCEPT_TIMEOUT_REAL_MS = Math.max( API_INTERCEPT_TIMEOUT_MS, parseInt(process.env.API_INTERCEPT_TIMEOUT_REAL_MS || '90000', 10) ); -/** Captcha API 配置 */ +const NAVIGATION_TIMEOUT_MS = Math.max(15000, parseInt(process.env.NAVIGATION_TIMEOUT_MS || '60000', 10)); + +/** rebrowser-puppeteer-core / puppeteer-core 仅支持 load|domcontentloaded|networkidle0|networkidle2,不支持 commit */ +const ALLOWED_NAVIGATION_WAIT_UNTIL = ['load', 'domcontentloaded', 'networkidle0', 'networkidle2']; + +function resolveNavigationWaitUntil() { + const raw = (process.env.NAVIGATION_WAIT_UNTIL || 'domcontentloaded').trim(); + if (ALLOWED_NAVIGATION_WAIT_UNTIL.includes(raw)) { + return raw; + } + if (raw === 'commit') { + console.warn('[constants] NAVIGATION_WAIT_UNTIL=commit 不被当前 Puppeteer 支持,已降级为 domcontentloaded'); + return 'domcontentloaded'; + } + console.warn(`[constants] 未知 NAVIGATION_WAIT_UNTIL=${raw},已降级为 domcontentloaded`); + return 'domcontentloaded'; +} + +const NAVIGATION_WAIT_UNTIL = resolveNavigationWaitUntil(); +const NAVIGATION_MAX_RETRIES = Math.max(0, parseInt(process.env.NAVIGATION_MAX_RETRIES || '1', 10)); +const PAGE_DEFAULT_TIMEOUT_MS = Math.max(15000, parseInt(process.env.PAGE_DEFAULT_TIMEOUT_MS || '30000', 10)); + const CAPTCHA_PROVIDER = (process.env.CAPTCHA_PROVIDER || 'capsolver').toLowerCase(); const CAPTCHA_API_KEY = process.env.CAPTCHA_API_KEY || ''; const CAPTCHA_MAX_WAIT_MS = Math.max(30000, parseInt(process.env.CAPTCHA_MAX_WAIT_MS || '120000', 10)); -/** 会话 Cookie 持久化 TTL(毫秒) */ -const SESSION_TTL_MS = Math.max(60000, parseInt(process.env.SESSION_TTL_MS || '1800000', 10)); +const SESSION_TTL_MS = Math.max(60000, parseInt(process.env.SESSION_TTL_MS || '7200000', 10)); +const PERSIST_BROWSER_PROFILE = process.env.PERSIST_BROWSER_PROFILE !== '0'; +const PROFILE_MAX_AGE_MS = Math.max(0, parseInt(process.env.PROFILE_MAX_AGE_MS || String(7 * 24 * 3600 * 1000), 10)); + +const POOL_IDLE_MS = Math.max(60000, parseInt(process.env.POOL_IDLE_MS || '300000', 10)); +const MAX_POOLED_BROWSERS = Math.max(0, parseInt(process.env.MAX_POOLED_BROWSERS || '4', 10)); +const POOL_ENABLED = process.env.POOL_ENABLED !== '0' && MAX_POOLED_BROWSERS > 0; module.exports = { DEFAULT_CHROME_EXECUTABLE, @@ -133,8 +136,17 @@ module.exports = { QUEUE_TIMEOUT_MS, API_INTERCEPT_TIMEOUT_MS, API_INTERCEPT_TIMEOUT_REAL_MS, + NAVIGATION_TIMEOUT_MS, + NAVIGATION_WAIT_UNTIL, + NAVIGATION_MAX_RETRIES, + PAGE_DEFAULT_TIMEOUT_MS, CAPTCHA_PROVIDER, CAPTCHA_API_KEY, CAPTCHA_MAX_WAIT_MS, SESSION_TTL_MS, + PERSIST_BROWSER_PROFILE, + PROFILE_MAX_AGE_MS, + POOL_IDLE_MS, + MAX_POOLED_BROWSERS, + POOL_ENABLED, }; diff --git a/patches/puppeteer-api/lib/launch-real-browser.js b/patches/puppeteer-api/lib/launch-real-browser.js index ee8c4d3..44a3edd 100755 --- a/patches/puppeteer-api/lib/launch-real-browser.js +++ b/patches/puppeteer-api/lib/launch-real-browser.js @@ -1,26 +1,17 @@ /** * Real Browser 启动:puppeteer-real-browser(抗 Cloudflare Turnstile) - * 注意:不与 puppeteer-extra 混用在同一 Browser 实例 */ const { BASE_LAUNCH_ARGS, resolveChromeExecutable, } = require('./constants'); +const { resolveProfileDir } = require('./profile-dir'); -/** @type {boolean|null} */ let realBrowserModuleChecked = null; - -/** @type {boolean} */ let realBrowserAvailable = false; -/** - * 检测 puppeteer-real-browser 是否已安装 - * @returns {boolean} - */ function isRealBrowserAvailable() { - if (realBrowserModuleChecked !== null) { - return realBrowserAvailable; - } + if (realBrowserModuleChecked !== null) return realBrowserAvailable; try { require.resolve('puppeteer-real-browser'); realBrowserAvailable = true; @@ -32,11 +23,10 @@ function isRealBrowserAvailable() { } /** - * 启动 real profile Browser(headless:false + Xvfb + turnstile 内置点击) * @param {string[]} [extraArgs] - * @returns {Promise<{ browser: import('puppeteer').Browser, page: import('puppeteer').Page }>} + * @param {{ sessionKey?: string, profile?: 'standard'|'real' }} [options] */ -async function launchRealBrowser(extraArgs = []) { +async function launchRealBrowser(extraArgs = [], options = {}) { if (!isRealBrowserAvailable()) { throw new Error( 'puppeteer-real-browser 未安装。请在 puppeteer-api 目录执行: npm install puppeteer-real-browser@1.4.4' @@ -45,22 +35,28 @@ async function launchRealBrowser(extraArgs = []) { const { connect } = require('puppeteer-real-browser'); const chromePath = resolveChromeExecutable(); + const profile = options.profile === 'real' ? 'real' : 'standard'; + const profileDir = options.sessionKey ? resolveProfileDir(options.sessionKey, profile) : null; + const args = [...BASE_LAUNCH_ARGS, '--mute-audio', ...extraArgs]; + if (profileDir) { + args.push(`--user-data-dir=${profileDir}`); + } const result = await connect({ headless: false, turnstile: true, disableXvfb: false, customConfig: { chromePath }, - args: [...BASE_LAUNCH_ARGS, '--mute-audio', ...extraArgs], + args, }); const browser = result.browser; const page = result.page; - if (!browser || !page) { throw new Error('puppeteer-real-browser connect 未返回 browser/page'); } - + browser.__persistentProfile = !!profileDir; + browser.__userDataDir = profileDir || ''; return { browser, page }; } diff --git a/patches/puppeteer-api/lib/launch-standard.js b/patches/puppeteer-api/lib/launch-standard.js index 3ff0108..688f407 100755 --- a/patches/puppeteer-api/lib/launch-standard.js +++ b/patches/puppeteer-api/lib/launch-standard.js @@ -11,18 +11,21 @@ const { buildBrowserLaunchEnv, ensureRuntimeSubdir, } = require('./constants'); +const { resolveProfileDir } = require('./profile-dir'); -// Stealth 必须在 launch 之前注册 puppeteer.use(StealthPlugin()); /** - * 启动 standard profile Browser * @param {string[]} [extraArgs] + * @param {{ sessionKey?: string, profile?: 'standard'|'real' }} [options] * @returns {Promise} */ -async function launchStandardBrowser(extraArgs = []) { +async function launchStandardBrowser(extraArgs = [], options = {}) { const executablePath = assertChromeExecutable(); - const userDataDir = fs.mkdtempSync(path.join(ensureRuntimeSubdir('chrome-profiles'), 'run-')); + const profile = options.profile === 'real' ? 'real' : 'standard'; + const persistentDir = options.sessionKey ? resolveProfileDir(options.sessionKey, profile) : null; + const userDataDir = persistentDir || fs.mkdtempSync(path.join(ensureRuntimeSubdir('chrome-profiles'), 'run-')); + const persistent = !!persistentDir; try { const browser = await puppeteer.launch({ @@ -32,24 +35,26 @@ async function launchStandardBrowser(extraArgs = []) { env: buildBrowserLaunchEnv(), userDataDir, }); - const originalClose = browser.close.bind(browser); - browser.close = async () => { - try { - await originalClose(); - } finally { + if (!persistent) { + const originalClose = browser.close.bind(browser); + browser.close = async () => { try { - fs.rmSync(userDataDir, { recursive: true, force: true }); - } catch (_) { - /* 清理失败不影响主流程 */ + await originalClose(); + } finally { + try { + fs.rmSync(userDataDir, { recursive: true, force: true }); + } catch (_) { /* ignore */ } } - } - }; + }; + } + browser.__userDataDir = userDataDir; + browser.__persistentProfile = persistent; return browser; } catch (err) { - try { - fs.rmSync(userDataDir, { recursive: true, force: true }); - } catch (_) { - /* ignore */ + if (!persistent) { + try { + fs.rmSync(userDataDir, { recursive: true, force: true }); + } catch (_) { /* ignore */ } } throw err; } diff --git a/patches/puppeteer-api/lib/profile-dir.js b/patches/puppeteer-api/lib/profile-dir.js new file mode 100644 index 0000000..9ff306f --- /dev/null +++ b/patches/puppeteer-api/lib/profile-dir.js @@ -0,0 +1,77 @@ +/** + * 按 sessionKey 解析持久 Chrome Profile 目录 + */ +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); +const { RUNTIME_DIR, PERSIST_BROWSER_PROFILE, PROFILE_MAX_AGE_MS } = require('./constants'); + +const PROFILES_DIR = path.join(RUNTIME_DIR, 'profiles'); + +/** + * @param {string} sessionKey + * @param {'standard'|'real'} profile + * @returns {string|null} + */ +function resolveProfileDir(sessionKey, profile = 'standard') { + if (!sessionKey || !PERSIST_BROWSER_PROFILE) { + return null; + } + const hash = crypto.createHash('md5').update(String(sessionKey)).digest('hex'); + const dir = path.join(PROFILES_DIR, `${hash}-${profile}`); + fs.mkdirSync(dir, { recursive: true, mode: 0o700 }); + return dir; +} + +function profileExists(sessionKey, profile = 'standard') { + const dir = resolveProfileDir(sessionKey, profile); + if (!dir) return false; + return fs.existsSync(dir); +} + +function pruneStaleProfiles() { + if (!PERSIST_BROWSER_PROFILE || PROFILE_MAX_AGE_MS <= 0) { + return 0; + } + if (!fs.existsSync(PROFILES_DIR)) { + return 0; + } + const now = Date.now(); + let removed = 0; + for (const name of fs.readdirSync(PROFILES_DIR)) { + const full = path.join(PROFILES_DIR, name); + try { + const stat = fs.statSync(full); + if (!stat.isDirectory()) continue; + if (now - stat.mtimeMs > PROFILE_MAX_AGE_MS) { + fs.rmSync(full, { recursive: true, force: true }); + removed++; + } + } catch (_) { + /* ignore */ + } + } + return removed; +} + +function getProfileStats() { + if (!fs.existsSync(PROFILES_DIR)) { + return { profilesDir: PROFILES_DIR, count: 0 }; + } + const count = fs.readdirSync(PROFILES_DIR).filter((name) => { + try { + return fs.statSync(path.join(PROFILES_DIR, name)).isDirectory(); + } catch (_) { + return false; + } + }).length; + return { profilesDir: PROFILES_DIR, count }; +} + +module.exports = { + PROFILES_DIR, + resolveProfileDir, + profileExists, + pruneStaleProfiles, + getProfileStats, +}; diff --git a/patches/puppeteer-api/lib/session-context.js b/patches/puppeteer-api/lib/session-context.js new file mode 100644 index 0000000..a92b8f5 --- /dev/null +++ b/patches/puppeteer-api/lib/session-context.js @@ -0,0 +1,38 @@ +/** + * 会话上下文:合并请求 cookies 与磁盘 session,供各 API 统一使用 + */ +const { + loadSession, + sessionCookiesForPage, + mergeCookies, + touchSession, +} = require('./session-store'); +const { SESSION_TTL_MS } = require('./constants'); + +/** + * @param {{ sessionKey?: string }} antiBot + * @param {string} pageUrl + * @param {object[]} [requestCookies] + */ +function resolveSessionContext(antiBot, pageUrl, requestCookies = []) { + const sessionKey = antiBot?.sessionKey || ''; + const storedSession = sessionKey ? loadSession(sessionKey) : null; + const sessionCookies = storedSession ? sessionCookiesForPage(storedSession, pageUrl) : []; + const mergedCookies = mergeCookies(requestCookies, sessionCookies); + return { sessionKey, storedSession, mergedCookies }; +} + +/** + * 成功过盾后滑动续期 session TTL + * @param {string} sessionKey + */ +function touchSessionIfPresent(sessionKey) { + if (sessionKey) { + touchSession(sessionKey, SESSION_TTL_MS); + } +} + +module.exports = { + resolveSessionContext, + touchSessionIfPresent, +}; diff --git a/patches/puppeteer-api/lib/session-store.js b/patches/puppeteer-api/lib/session-store.js index 8c303c5..d2fc985 100755 --- a/patches/puppeteer-api/lib/session-store.js +++ b/patches/puppeteer-api/lib/session-store.js @@ -1,5 +1,5 @@ /** - * 会话 Cookie 持久化:按 sessionKey 保存 cf_clearance 等,降低重复过盾概率 + * 会话 Cookie 持久化:按 sessionKey 保存完整 cookie jar,降低重复过盾概率 */ const fs = require('fs'); const path = require('path'); @@ -8,42 +8,50 @@ const { RUNTIME_DIR, SESSION_TTL_MS } = require('./constants'); const SESSIONS_DIR = path.join(RUNTIME_DIR, 'sessions'); -/** - * 确保 sessions 目录存在 - */ function ensureSessionsDir() { fs.mkdirSync(SESSIONS_DIR, { recursive: true, mode: 0o777 }); } -/** - * sessionKey 转安全文件名 - * @param {string} sessionKey - * @returns {string} - */ function keyToFilename(sessionKey) { const hash = crypto.createHash('md5').update(sessionKey).digest('hex'); return `${hash}.json`; } /** - * @typedef {Object} StoredSession - * @property {string} sessionKey - * @property {number} savedAt - * @property {number} expiresAt - * @property {Array<{name:string,value:string,domain?:string,path?:string}>} cookies + * @param {{name:string,domain?:string,path?:string}} cookie + * @returns {string} */ +function cookieIdentity(cookie) { + return `${cookie.name}|${cookie.domain || ''}|${cookie.path || '/'}`; +} + +function mergeCookies(requestCookies, sessionCookies) { + const map = new Map(); + for (const cookie of sessionCookies || []) { + if (cookie && cookie.name) map.set(cookieIdentity(cookie), cookie); + } + for (const cookie of requestCookies || []) { + if (cookie && cookie.name) map.set(cookieIdentity(cookie), cookie); + } + return Array.from(map.values()); +} + +function isSessionLikelyValid(session) { + if (!session || !Array.isArray(session.cookies)) return false; + if (session.expiresAt && Date.now() > session.expiresAt) return false; + return session.cookies.some((c) => c.name === 'cf_clearance' && c.value); +} + +function hasCfClearance(session) { + if (!session || !Array.isArray(session.cookies)) return false; + return session.cookies.some((c) => c.name === 'cf_clearance' && c.value); +} -/** - * 加载已保存的会话 cookies - * @param {string} sessionKey - * @returns {StoredSession|null} - */ function loadSession(sessionKey) { if (!sessionKey) return null; ensureSessionsDir(); const filePath = path.join(SESSIONS_DIR, keyToFilename(sessionKey)); if (!fs.existsSync(filePath)) return null; - try { const raw = JSON.parse(fs.readFileSync(filePath, 'utf8')); if (!raw || !Array.isArray(raw.cookies)) return null; @@ -57,45 +65,36 @@ function loadSession(sessionKey) { } } -/** - * 保存会话 cookies(优先保留 CF 相关 cookie) - * @param {string} sessionKey - * @param {Array<{name:string,value:string,domain?:string,path?:string}>} cookies - * @param {number} [ttlMs] - */ -function saveSession(sessionKey, cookies, ttlMs = SESSION_TTL_MS) { +function touchSession(sessionKey, ttlMs = SESSION_TTL_MS) { + const session = loadSession(sessionKey); + if (!session) return; + const now = Date.now(); + session.savedAt = now; + session.expiresAt = now + ttlMs; + fs.writeFileSync(path.join(SESSIONS_DIR, keyToFilename(sessionKey)), JSON.stringify(session, null, 0), { mode: 0o666 }); +} + +function saveSession(sessionKey, cookies, ttlMs = SESSION_TTL_MS, meta = {}) { if (!sessionKey || !Array.isArray(cookies) || cookies.length === 0) return; ensureSessionsDir(); - - const cfRelated = cookies.filter((c) => - ['cf_clearance', '__cf_bm', 'cf_chl_2'].includes(c.name) - || c.name.startsWith('__cf') - ); - - const toSave = cfRelated.length > 0 ? cfRelated : cookies; const now = Date.now(); const payload = { sessionKey, savedAt: now, + lastSuccessAt: now, expiresAt: now + ttlMs, - cookies: toSave.map((c) => ({ + lastHost: meta.lastHost || '', + savedProfile: meta.savedProfile || '', + cookies: cookies.map((c) => ({ name: c.name, value: c.value, domain: c.domain, path: c.path || '/', })), }; - - const filePath = path.join(SESSIONS_DIR, keyToFilename(sessionKey)); - fs.writeFileSync(filePath, JSON.stringify(payload, null, 0), { mode: 0o666 }); + fs.writeFileSync(path.join(SESSIONS_DIR, keyToFilename(sessionKey)), JSON.stringify(payload, null, 0), { mode: 0o666 }); } -/** - * 将会话 cookies 转为 puppeteer setCookie 格式 - * @param {StoredSession|null} session - * @param {string} pageUrl - * @returns {Array<{name:string,value:string,domain?:string,path?:string,url?:string}>} - */ function sessionCookiesForPage(session, pageUrl) { if (!session || !Array.isArray(session.cookies)) return []; let hostname = ''; @@ -104,18 +103,54 @@ function sessionCookiesForPage(session, pageUrl) { } catch (_) { return session.cookies; } - return session.cookies.map((c) => { const cookie = { ...c }; - if (!cookie.domain && hostname) { - cookie.url = `https://${hostname}/`; - } + if (!cookie.domain && hostname) cookie.url = `https://${hostname}/`; return cookie; }); } +function getSessionDebugInfo(sessionKey) { + if (!sessionKey) return null; + const filePath = path.join(SESSIONS_DIR, keyToFilename(sessionKey)); + const session = loadSession(sessionKey); + if (!session) { + return { sessionKey, exists: false, filePath }; + } + let mtime = null; + try { + mtime = fs.statSync(filePath).mtime.toISOString(); + } catch (_) { /* ignore */ } + return { + sessionKey, + exists: true, + filePath, + mtime, + savedAt: session.savedAt, + expiresAt: session.expiresAt, + lastSuccessAt: session.lastSuccessAt, + lastHost: session.lastHost, + savedProfile: session.savedProfile, + cookieCount: session.cookies.length, + hasCfClearance: hasCfClearance(session), + likelyValid: isSessionLikelyValid(session), + }; +} + +function getSessionStats() { + ensureSessionsDir(); + const count = fs.readdirSync(SESSIONS_DIR).filter((f) => f.endsWith('.json')).length; + return { sessionsDir: SESSIONS_DIR, count }; +} + module.exports = { loadSession, saveSession, + touchSession, sessionCookiesForPage, + mergeCookies, + isSessionLikelyValid, + hasCfClearance, + getSessionDebugInfo, + getSessionStats, }; diff --git a/patches/puppeteer-api/lib/ui-pagination-runner.js b/patches/puppeteer-api/lib/ui-pagination-runner.js new file mode 100644 index 0000000..1424902 --- /dev/null +++ b/patches/puppeteer-api/lib/ui-pagination-runner.js @@ -0,0 +1,172 @@ +/** + * UI 翻页抓取:供 ui-pagination 与 auth-intercept-and-paginate 共用 + */ +const crypto = require('crypto'); + +function generateSafeHash(obj) { + if (!obj) return ''; + const clone = JSON.parse(JSON.stringify(obj)); + const removeDynamicKeys = (target) => { + if (typeof target !== 'object' || target === null) return; + ['timestamp', 'time', 'serverTime', 'reqId', 'requestId', 'traceId'].forEach((k) => delete target[k]); + Object.values(target).forEach((val) => removeDynamicKeys(val)); + }; + removeDynamicKeys(clone); + return crypto.createHash('md5').update(JSON.stringify(clone)).digest('hex'); +} + +function classifyPuppeteerError(err) { + const msg = (err?.message || '').toLowerCase(); + if (err?.name === 'TimeoutError' || msg.includes('timeout')) return 'timeout'; + if (msg.includes('net::') || msg.includes('network') || msg.includes('err_connection') + || msg.includes('err_address_unreachable') || msg.includes('err_name_not_resolved')) return 'network'; + if (msg.includes('navigation') || msg.includes('frame was detached')) return 'navigation'; + if (msg.includes('target closed') || msg.includes('session closed')) return 'target_closed'; + return 'unknown'; +} + +/** + * @param {import('puppeteer').Page} page + * @param {{ + * apiUrl: string, + * nextBtnSelector: string, + * clicksToPerform: number, + * waitMs?: number, + * firstPageData?: object, + * authActions?: object[], + * executeAuthActions?: (page: import('puppeteer').Page, actions: object[]) => Promise, + * }} options + */ +async function runUiPagination(page, options) { + const { + apiUrl, + nextBtnSelector, + clicksToPerform, + waitMs = 2000, + firstPageData, + authActions, + executeAuthActions, + } = options; + + const capturedData = []; + const debugLogs = []; + const log = (msg) => { console.log(msg); debugLogs.push(msg); }; + const seenHashes = new Set(); + if (firstPageData) { + seenHashes.add(generateSafeHash(firstPageData)); + } + + if (Array.isArray(authActions) && authActions.length > 0 && executeAuthActions) { + log('[状态同步] 正在为翻页引擎注入授权状态...'); + await executeAuthActions(page, authActions); + log('[状态同步] 授权执行完毕,等待目标列表加载...'); + const authSettled = await page.waitForResponse( + (resp) => resp.url().includes(apiUrl) && resp.request().method() !== 'OPTIONS' && resp.status() >= 200 && resp.status() < 400, + { timeout: 5000 } + ).catch(() => null); + if (!authSettled) { + log('[状态同步] 未捕获到目标 API 响应,降级为固定等待 3 秒...'); + await new Promise((r) => setTimeout(r, 3000)); + } + } + + for (let i = 0; i < clicksToPerform; i++) { + const targetPageNum = i + 2; + let pageData = null; + let attempts = 0; + const maxAttempts = 3; + + await page.evaluate(() => { + document.querySelectorAll('.el-loading-mask, .ant-spin-nested-loading, .el-loading-spinner').forEach((el) => el.remove()); + }); + + await page.waitForFunction( + () => !document.querySelector('.el-loading-mask, .ant-spin-nested-loading, .el-loading-spinner'), + { timeout: 1000, polling: 100 } + ).catch(() => new Promise((r) => setTimeout(r, 500))); + + while (attempts < maxAttempts && !pageData) { + attempts++; + log(`[抓取] 准备抓取第 ${targetPageNum} 页 (尝试 ${attempts}/${maxAttempts})...`); + + let responseHandler = null; + let apiTimeoutId = null; + + const waitForNewPageApi = new Promise((resolve) => { + const cleanupListener = () => { + if (apiTimeoutId) { clearTimeout(apiTimeoutId); apiTimeoutId = null; } + if (responseHandler) { page.off('response', responseHandler); responseHandler = null; } + }; + + responseHandler = async (response) => { + const status = response.status(); + if (response.url().includes(apiUrl) && response.request().method() !== 'OPTIONS' && status >= 200 && status < 400) { + try { + const responseData = await response.json(); + const currentHash = generateSafeHash(responseData); + if (!seenHashes.has(currentHash)) { + seenHashes.add(currentHash); + cleanupListener(); + resolve(responseData); + } else { + log('[防轮询] 抛弃重复数据,继续监听...'); + } + } catch (_) { /* 非 JSON 响应跳过 */ } + } + }; + page.on('response', responseHandler); + apiTimeoutId = setTimeout(() => { cleanupListener(); resolve(null); }, 15000); + }); + + try { + await page.waitForSelector(nextBtnSelector, { timeout: 15000 }); + + const clickStatus = await page.evaluate((sel) => { + const btn = document.querySelector(sel); + if (!btn) return 'not_found'; + if (btn.disabled || btn.classList.contains('is-disabled') || btn.classList.contains('disabled') || btn.getAttribute('aria-disabled') === 'true') { + return 'disabled'; + } + const className = btn.className || ''; + if (btn.disabled === true || className.includes('disabled') || btn.getAttribute('aria-disabled') === 'true') { + return 'disabled'; + } + btn.scrollIntoView({ behavior: 'instant', block: 'center' }); + btn.click(); + btn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window })); + return 'success'; + }, nextBtnSelector); + + if (clickStatus === 'disabled') { + log(`[提示] 第 ${targetPageNum} 页按钮已被禁用,说明到底了,抓取提前结束。`); + break; + } + log('[动作] 双引擎原生点击已执行!'); + } catch (clickErr) { + log(`[错误] 寻找按钮异常 (${classifyPuppeteerError(clickErr)}): ${clickErr.message}`); + } + + pageData = await waitForNewPageApi; + if (!pageData && attempts < maxAttempts) { + log('[重试警报] 雷达未扫描到新数据,准备重试...'); + await new Promise((r) => setTimeout(r, 500 * Math.pow(2, attempts - 1))); + } + } + + if (pageData) { + capturedData.push(pageData); + log(`[成功] 斩获第 ${targetPageNum} 页数据!`); + await new Promise((r) => setTimeout(r, waitMs)); + } else { + log(`[致命异常] 连续 ${maxAttempts} 次尝试均失败,放弃后续翻页。`); + break; + } + } + + return { data: capturedData, debugLogs }; +} + +module.exports = { + runUiPagination, + generateSafeHash, +}; diff --git a/patches/puppeteer-api/server.js b/patches/puppeteer-api/server.js index fa2df45..5b50a0b 100755 --- a/patches/puppeteer-api/server.js +++ b/patches/puppeteer-api/server.js @@ -1,6 +1,5 @@ const express = require('express'); const fs = require('fs'); -const crypto = require('crypto'); const { rewriteHttpsToHttpForLiteralIp, attachHttpIpRewriteInterceptor, @@ -17,19 +16,35 @@ const { MAX_CONCURRENT_BROWSERS_REAL, API_INTERCEPT_TIMEOUT_MS, API_INTERCEPT_TIMEOUT_REAL_MS, + NAVIGATION_TIMEOUT_MS, + NAVIGATION_WAIT_UNTIL, + NAVIGATION_MAX_RETRIES, + SESSION_TTL_MS, + PERSIST_BROWSER_PROFILE, + POOL_IDLE_MS, + MAX_POOLED_BROWSERS, } = require('./lib/constants'); const { normalizeAntiBot, createBrowserSession, + acquireBrowserSession, runWithProfileLimit, createManagedPage, getFactoryStats, standardLimiter, + browserPool, } = require('./lib/browser-factory'); const { launchStandardBrowser } = require('./lib/launch-standard'); const { isRealBrowserAvailable } = require('./lib/launch-real-browser'); const { handleCloudflareChallenge, isCaptchaConfigured } = require('./lib/cf-handler'); -const { loadSession, saveSession, sessionCookiesForPage } = require('./lib/session-store'); +const { + saveSession, + getSessionDebugInfo, + getSessionStats, +} = require('./lib/session-store'); +const { resolveSessionContext, touchSessionIfPresent } = require('./lib/session-context'); +const { getProfileStats, profileExists } = require('./lib/profile-dir'); +const { runUiPagination } = require('./lib/ui-pagination-runner'); const app = express(); app.use(express.json({ limit: '50mb' })); @@ -63,16 +78,11 @@ function sendTaskError(res, error, extra = {}) { return res.status(500).json({ success: false, error: error.message, ...extra }); } -function generateSafeHash(obj) { - if (!obj) return ''; - const clone = JSON.parse(JSON.stringify(obj)); - const removeDynamicKeys = (target) => { - if (typeof target !== 'object' || target === null) return; - ['timestamp', 'time', 'serverTime', 'reqId', 'requestId', 'traceId'].forEach((k) => delete target[k]); - Object.values(target).forEach((val) => removeDynamicKeys(val)); - }; - removeDynamicKeys(clone); - return crypto.createHash('md5').update(JSON.stringify(clone)).digest('hex'); +async function navigateToPage(page, url) { + await withExponentialBackoff( + () => page.goto(url, { waitUntil: NAVIGATION_WAIT_UNTIL, timeout: NAVIGATION_TIMEOUT_MS }), + { maxRetries: NAVIGATION_MAX_RETRIES, baseDelayMs: 1000 } + ); } async function safeCloseBrowser(browser) { @@ -347,6 +357,9 @@ app.get('/api/stats', (_req, res) => { shuttingDown, queue: factoryStats.standard, queueReal: factoryStats.real, + pool: factoryStats.pool, + sessions: getSessionStats(), + profiles: getProfileStats(), realBrowserReady: factoryStats.realBrowserReady, captchaConfigured: isCaptchaConfigured(), chrome: { @@ -358,10 +371,35 @@ app.get('/api/stats', (_req, res) => { maxConcurrentBrowsersReal: MAX_CONCURRENT_BROWSERS_REAL, apiInterceptTimeoutMs: API_INTERCEPT_TIMEOUT_MS, apiInterceptTimeoutRealMs: API_INTERCEPT_TIMEOUT_REAL_MS, + navigationTimeoutMs: NAVIGATION_TIMEOUT_MS, + navigationWaitUntil: NAVIGATION_WAIT_UNTIL, + sessionTtlMs: SESSION_TTL_MS, + persistBrowserProfile: PERSIST_BROWSER_PROFILE, + poolIdleMs: POOL_IDLE_MS, + maxPooledBrowsers: MAX_POOLED_BROWSERS, }, }); }); +/** 运维调试:查看指定 sessionKey 的磁盘会话与 profile 状态 */ +app.get('/api/session/:sessionKey', (req, res) => { + const sessionKey = decodeURIComponent(req.params.sessionKey || ''); + if (!sessionKey) { + return res.status(400).json({ success: false, error: 'sessionKey 不能为空' }); + } + const info = getSessionDebugInfo(sessionKey); + const profileStandard = getProfileStats(); + res.json({ + success: true, + session: info, + profileExists: { + standard: profileExists(sessionKey, 'standard'), + real: profileExists(sessionKey, 'real'), + }, + profilesDir: profileStandard.profilesDir, + }); +}); + /** 运维自检:实际尝试 launch 一次 Browser(profile=standard|real) */ app.get('/api/browser-launch-test', async (req, res) => { const profile = req.query.profile === 'real' ? 'real' : 'standard'; @@ -410,31 +448,28 @@ app.post('/api/auth-and-intercept', async (req, res) => { const antiBot = normalizeAntiBot(rawAntiBot); const profile = antiBot.profile || 'standard'; const interceptTimeout = resolveInterceptTimeout(antiBot); - const cfLog = { profile, cfDetected: false, turnstileStage: null, solverUsed: false, elapsedMs: 0 }; + const cfLog = { profile, cfDetected: false, turnstileStage: null, solverUsed: false, elapsedMs: 0, browserReused: false }; try { await runBrowserTask('auth-and-intercept', async () => { - let browser; - let sessionCleanup = null; + let browserSession = null; let interceptorCleanup = null; const taskStarted = Date.now(); + let cfDestroyBrowser = false; try { - const session = await createBrowserSession({ + browserSession = await acquireBrowserSession({ profile, extraArgs: ['--mute-audio'], + sessionKey: antiBot.sessionKey || '', }); - browser = session.browser; - sessionCleanup = session.cleanup; + cfLog.browserReused = !!browserSession.reused; - const sessionData = antiBot.sessionKey ? loadSession(antiBot.sessionKey) : null; - const sessionCookies = sessionData - ? sessionCookiesForPage(sessionData, pageUrl) - : []; + const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl); - const page = await createManagedPage(browser, session.page, { + const page = await createManagedPage(browserSession.browser, browserSession.page, { userAgent: DEFAULT_UA, - cookies: sessionCookies, + cookies: mergedCookies, }); const httpResolvedUrl = await resolveHttpRedirects(pageUrl).catch(() => pageUrl); @@ -452,10 +487,7 @@ app.post('/api/auth-and-intercept', async (req, res) => { } await attachHttpIpRewriteInterceptor(page, 'auth-and-intercept', { shareToken }); - await withExponentialBackoff( - () => page.goto(pageUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }), - { maxRetries: 2, baseDelayMs: 1000 } - ); + await navigateToPage(page, pageUrl); let finalPageUrl = await waitForUrlSettled(page); if (finalPageUrl !== pageUrl) { @@ -463,13 +495,14 @@ app.post('/api/auth-and-intercept', async (req, res) => { } if (antiBot.enabled) { - const cfResult = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled); + const cfResult = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled, storedSession); cfLog.cfDetected = cfResult.cfDetected; cfLog.turnstileStage = cfResult.stage; cfLog.solverUsed = cfResult.solverUsed; cfLog.elapsedMs = cfResult.elapsedMs; if (!cfResult.success) { + cfDestroyBrowser = true; res.status(422).json({ success: false, code: cfResult.code || 'CF_TURNSTILE_FAILED', @@ -482,6 +515,7 @@ app.post('/api/auth-and-intercept', async (req, res) => { } finalPageUrl = page.url(); + touchSessionIfPresent(antiBot.sessionKey); } const finalShareToken = resolveShareToken(finalPageUrl); @@ -502,7 +536,14 @@ app.post('/api/auth-and-intercept', async (req, res) => { })); if (antiBot.sessionKey) { - saveSession(antiBot.sessionKey, cookiePayload); + let lastHost = ''; + try { + lastHost = new URL(finalPageUrl).hostname; + } catch (_) { /* ignore */ } + saveSession(antiBot.sessionKey, cookiePayload, SESSION_TTL_MS, { + lastHost, + savedProfile: profile, + }); } res.json({ @@ -513,14 +554,13 @@ app.post('/api/auth-and-intercept', async (req, res) => { cookies: cookiePayload, cf: cfLog, profile, + browserReused: browserSession.reused, elapsedMs: Date.now() - taskStarted, }); } finally { if (interceptorCleanup) interceptorCleanup(); - if (sessionCleanup) { - await sessionCleanup(); - } else { - await safeCloseBrowser(browser); + if (browserSession) { + await browserSession.release(cfDestroyBrowser); } } }, profile); @@ -541,18 +581,19 @@ app.post('/api/batch-fetch', async (req, res) => { try { await runBrowserTask('batch-fetch', async () => { - let browser; - let sessionCleanup = null; + let browserSession = null; try { - const session = await createBrowserSession({ + browserSession = await acquireBrowserSession({ profile, extraArgs: ['--disable-web-security'], + sessionKey: antiBot.sessionKey || '', }); - browser = session.browser; - sessionCleanup = session.cleanup; - const page = await createManagedPage(browser, session.page, { cookies }); - const firstOrigin = new URL(tasks[0].fullUrl).origin; + const firstUrl = tasks[0].fullUrl; + const { mergedCookies } = resolveSessionContext(antiBot, firstUrl, cookies || []); + + const page = await createManagedPage(browserSession.browser, browserSession.page, { cookies: mergedCookies }); + const firstOrigin = new URL(firstUrl).origin; await page.setRequestInterception(true); page.on('request', (reqObj) => { @@ -570,10 +611,7 @@ app.post('/api/batch-fetch', async (req, res) => { } }); - await withExponentialBackoff( - () => page.goto(firstOrigin, { waitUntil: 'domcontentloaded', timeout: 30000 }), - { maxRetries: 2, baseDelayMs: 800 } - ); + await navigateToPage(page, firstOrigin); const FETCH_CONCURRENCY = Math.max(1, parseInt(process.env.FETCH_CONCURRENCY || '5', 10)); const FETCH_RETRY = 2; @@ -636,12 +674,10 @@ app.post('/api/batch-fetch', async (req, res) => { return out; }, tasks, FETCH_CONCURRENCY, FETCH_RETRY); - res.json({ success: true, results: fetchResults, profile }); + res.json({ success: true, results: fetchResults, profile, browserReused: browserSession.reused }); } finally { - if (sessionCleanup) { - await sessionCleanup(); - } else { - await safeCloseBrowser(browser); + if (browserSession) { + await browserSession.release(false); } } }, profile); @@ -670,27 +706,25 @@ app.post('/api/ui-pagination', async (req, res) => { try { await runBrowserTask('ui-pagination', async () => { - let browser; - let sessionCleanup = null; - const capturedData = []; + let browserSession = null; const debugLogs = []; - const log = (msg) => { console.log(msg); debugLogs.push(msg); }; - const seenHashes = new Set(); - if (firstPageData) { seenHashes.add(generateSafeHash(firstPageData)); } + let cfDestroyBrowser = false; try { - const session = await createBrowserSession({ + browserSession = await acquireBrowserSession({ profile, extraArgs: ['--window-size=1920,1080'], + sessionKey: antiBot.sessionKey || '', }); - browser = session.browser; - sessionCleanup = session.cleanup; - const page = await createManagedPage(browser, session.page, { + const { storedSession, mergedCookies } = resolveSessionContext(antiBot, navigationUrl, cookies || []); + + const page = await createManagedPage(browserSession.browser, browserSession.page, { viewport: { width: 1920, height: 1080 }, - cookies, + cookies: mergedCookies, }); + const log = (msg) => { console.log(msg); debugLogs.push(msg); }; log(`[启动] 准备访问页面: ${navigationUrl}`); if (finalPageUrl && finalPageUrl !== pageUrl) { log(`[URL解析] 使用 auth 阶段落地地址,跳过短链重定向: ${pageUrl} -> ${finalPageUrl}`); @@ -700,10 +734,7 @@ app.post('/api/ui-pagination', async (req, res) => { await seedShareTokenCookie(page, navigationUrl); await attachHttpIpRewriteInterceptor(page, 'ui-pagination', { shareToken }); - await withExponentialBackoff( - () => page.goto(navigationUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }), - { maxRetries: 2, baseDelayMs: 1000 } - ); + await navigateToPage(page, navigationUrl); const settledPageUrl = await waitForUrlSettled(page); if (settledPageUrl !== navigationUrl) { @@ -711,8 +742,9 @@ app.post('/api/ui-pagination', async (req, res) => { } if (antiBot.enabled) { - const cfResult = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled); + const cfResult = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled, storedSession); if (!cfResult.success) { + cfDestroyBrowser = true; res.status(422).json({ success: false, code: cfResult.code || 'CF_TURNSTILE_FAILED', @@ -723,125 +755,33 @@ app.post('/api/ui-pagination', async (req, res) => { }); return; } + touchSessionIfPresent(antiBot.sessionKey); } - if (Array.isArray(authActions) && authActions.length > 0) { - log('[状态同步] 正在为翻页引擎注入授权状态...'); - await executeAuthActions(page, authActions); - log('[状态同步] 授权执行完毕,等待目标列表加载...'); - const authSettled = await page.waitForResponse( - (resp) => resp.url().includes(apiUrl) && resp.request().method() !== 'OPTIONS' && resp.status() >= 200 && resp.status() < 400, - { timeout: 5000 } - ).catch(() => null); - if (!authSettled) { - log('[状态同步] 未捕获到目标 API 响应,降级为固定等待 3 秒...'); - await new Promise((r) => setTimeout(r, 3000)); - } - } + const paginationResult = await runUiPagination(page, { + apiUrl, + nextBtnSelector, + clicksToPerform, + waitMs, + firstPageData, + authActions, + executeAuthActions, + }); - for (let i = 0; i < clicksToPerform; i++) { - const targetPageNum = i + 2; - let pageData = null; - let attempts = 0; - const maxAttempts = 3; - - await page.evaluate(() => { - document.querySelectorAll('.el-loading-mask, .ant-spin-nested-loading, .el-loading-spinner').forEach((el) => el.remove()); - }); - - await page.waitForFunction( - () => !document.querySelector('.el-loading-mask, .ant-spin-nested-loading, .el-loading-spinner'), - { timeout: 1000, polling: 100 } - ).catch(() => new Promise((r) => setTimeout(r, 500))); - - while (attempts < maxAttempts && !pageData) { - attempts++; - log(`[抓取] 准备抓取第 ${targetPageNum} 页 (尝试 ${attempts}/${maxAttempts})...`); - - let responseHandler = null; - let apiTimeoutId = null; - - const waitForNewPageApi = new Promise((resolve) => { - const cleanupListener = () => { - if (apiTimeoutId) { clearTimeout(apiTimeoutId); apiTimeoutId = null; } - if (responseHandler) { page.off('response', responseHandler); responseHandler = null; } - }; - - responseHandler = async (response) => { - const status = response.status(); - if (response.url().includes(apiUrl) && response.request().method() !== 'OPTIONS' && status >= 200 && status < 400) { - try { - const responseData = await response.json(); - const currentHash = generateSafeHash(responseData); - if (!seenHashes.has(currentHash)) { - seenHashes.add(currentHash); - cleanupListener(); - resolve(responseData); - } else { - log('[防轮询] 抛弃重复数据,继续监听...'); - } - } catch (_) { /* 非 JSON 响应跳过 */ } - } - }; - page.on('response', responseHandler); - apiTimeoutId = setTimeout(() => { cleanupListener(); resolve(null); }, 15000); - }); - - try { - await page.waitForSelector(nextBtnSelector, { timeout: 15000 }); - - const clickStatus = await page.evaluate((sel) => { - const btn = document.querySelector(sel); - if (!btn) return 'not_found'; - if (btn.disabled || btn.classList.contains('is-disabled') || btn.classList.contains('disabled') || btn.getAttribute('aria-disabled') === 'true') { - return 'disabled'; - } - const className = btn.className || ''; - if (btn.disabled === true || className.includes('disabled') || btn.getAttribute('aria-disabled') === 'true') { - return 'disabled'; - } - btn.scrollIntoView({ behavior: 'instant', block: 'center' }); - btn.click(); - btn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window })); - return 'success'; - }, nextBtnSelector); - - if (clickStatus === 'disabled') { - log(`[提示] 第 ${targetPageNum} 页按钮已被禁用,说明到底了,抓取提前结束。`); - break; - } - log('[动作] 双引擎原生点击已执行!'); - } catch (clickErr) { - log(`[错误] 寻找按钮异常 (${classifyPuppeteerError(clickErr)}): ${clickErr.message}`); - } - - pageData = await waitForNewPageApi; - if (!pageData && attempts < maxAttempts) { - log('[重试警报] 雷达未扫描到新数据,准备重试...'); - await new Promise((r) => setTimeout(r, 500 * Math.pow(2, attempts - 1))); - } - } - - if (pageData) { - capturedData.push(pageData); - log(`[成功] 斩获第 ${targetPageNum} 页数据!`); - await new Promise((r) => setTimeout(r, waitMs)); - } else { - log(`[致命异常] 连续 ${maxAttempts} 次尝试均失败,放弃后续翻页。`); - break; - } - } - - res.json({ success: true, data: capturedData, debug: debugLogs, profile }); + res.json({ + success: true, + data: paginationResult.data, + debug: [...debugLogs, ...paginationResult.debugLogs], + profile, + browserReused: browserSession.reused, + }); } catch (error) { if (!res.headersSent) { sendTaskError(res, error, { debug: debugLogs, profile }); } } finally { - if (sessionCleanup) { - await sessionCleanup(); - } else { - await safeCloseBrowser(browser); + if (browserSession) { + await browserSession.release(cfDestroyBrowser); } } }, profile); @@ -850,6 +790,160 @@ app.post('/api/ui-pagination', async (req, res) => { } }); +/** 接口四:单 Browser 内 auth + 拦截 + UI 翻页(降低 CF 二次触发) */ +app.post('/api/auth-intercept-and-paginate', async (req, res) => { + const { + pageUrl, + apiUrls, + authActions, + antiBot: rawAntiBot, + pagination, + } = req.body; + + if (!pageUrl || !Array.isArray(apiUrls) || !pagination) { + return res.status(400).json({ success: false, error: '参数错误:需要 pageUrl、apiUrls、pagination' }); + } + + const { + apiUrl, + nextBtnSelector, + clicksToPerform = 0, + waitMs = 2000, + firstPageData, + } = pagination; + + const antiBot = normalizeAntiBot(rawAntiBot); + const profile = antiBot.profile || 'standard'; + const interceptTimeout = resolveInterceptTimeout(antiBot); + const cfLog = { profile, cfDetected: false, turnstileStage: null, solverUsed: false, elapsedMs: 0, browserReused: false }; + + try { + await runBrowserTask('auth-intercept-and-paginate', async () => { + let browserSession = null; + let interceptorCleanup = null; + const taskStarted = Date.now(); + let cfDestroyBrowser = false; + + try { + browserSession = await acquireBrowserSession({ + profile, + extraArgs: ['--mute-audio', '--window-size=1920,1080'], + sessionKey: antiBot.sessionKey || '', + }); + cfLog.browserReused = !!browserSession.reused; + + const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl); + + const page = await createManagedPage(browserSession.browser, browserSession.page, { + userAgent: DEFAULT_UA, + viewport: { width: 1920, height: 1080 }, + cookies: mergedCookies, + }); + + const httpResolvedUrl = await resolveHttpRedirects(pageUrl).catch(() => pageUrl); + const interceptor = createApiInterceptor(page, apiUrls, interceptTimeout); + interceptorCleanup = interceptor.cleanup; + + let shareToken = resolveShareToken(pageUrl, httpResolvedUrl); + await seedShareTokenCookie(page, pageUrl); + await attachHttpIpRewriteInterceptor(page, 'auth-intercept-and-paginate', { shareToken }); + + await navigateToPage(page, pageUrl); + let finalPageUrl = await waitForUrlSettled(page); + + if (antiBot.enabled) { + const cfResult = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled, storedSession); + cfLog.cfDetected = cfResult.cfDetected; + cfLog.turnstileStage = cfResult.stage; + cfLog.solverUsed = cfResult.solverUsed; + cfLog.elapsedMs = cfResult.elapsedMs; + + if (!cfResult.success) { + cfDestroyBrowser = true; + res.status(422).json({ + success: false, + code: cfResult.code || 'CF_TURNSTILE_FAILED', + challengeType: cfResult.challengeType || 'turnstile', + stage: cfResult.stage || 'timeout', + error: 'Cloudflare Turnstile 验证失败', + cf: cfLog, + }); + return; + } + finalPageUrl = page.url(); + touchSessionIfPresent(antiBot.sessionKey); + } + + await executeAuthActions(page, authActions); + await interceptor.waitForApis(interceptTimeout); + + const rawCookies = await page.cookies(); + const cookiePayload = rawCookies.map((c) => ({ + name: c.name, + value: c.value, + domain: c.domain, + path: c.path, + })); + + if (antiBot.sessionKey) { + let lastHost = ''; + try { + lastHost = new URL(finalPageUrl).hostname; + } catch (_) { /* ignore */ } + saveSession(antiBot.sessionKey, cookiePayload, SESSION_TTL_MS, { + lastHost, + savedProfile: profile, + }); + } + + let extraPages = []; + let paginationDebug = []; + if (clicksToPerform > 0 && apiUrl && nextBtnSelector) { + let effectiveFirstPageData = firstPageData; + if (!effectiveFirstPageData) { + const matchedKey = Object.keys(interceptor.interceptedApis).find((k) => k === apiUrl || k.includes(apiUrl)); + if (matchedKey) { + effectiveFirstPageData = interceptor.interceptedApis[matchedKey].data; + } + } + const paginationResult = await runUiPagination(page, { + apiUrl, + nextBtnSelector, + clicksToPerform, + waitMs, + firstPageData: effectiveFirstPageData, + authActions, + executeAuthActions, + }); + extraPages = paginationResult.data; + paginationDebug = paginationResult.debugLogs; + } + + res.json({ + success: true, + pageUrl, + finalPageUrl, + interceptedApis: interceptor.interceptedApis, + cookies: cookiePayload, + extraPages, + debug: paginationDebug, + cf: cfLog, + profile, + browserReused: browserSession.reused, + elapsedMs: Date.now() - taskStarted, + }); + } finally { + if (interceptorCleanup) interceptorCleanup(); + if (browserSession) { + await browserSession.release(cfDestroyBrowser); + } + } + }, profile); + } catch (error) { + if (!res.headersSent) sendTaskError(res, error, { cf: cfLog, profile }); + } +}); + const PORT = parseInt(process.env.PORT || '3001', 10); const server = app.listen(PORT, '127.0.0.1', () => { const chromePath = resolveChromeExecutable(); @@ -874,6 +968,9 @@ async function gracefulShutdown(signal) { await new Promise((r) => setTimeout(r, 1000)); } + console.log('[shutdown] 排空 Browser 温池...'); + await browserPool.drain(); + const remaining = standardLimiter.getStats(); if (remaining.active > 0) { console.warn(`[shutdown] 超时,仍有 ${remaining.active} 个 Browser 在运行,强制退出`);