diff --git a/_php_code/AbstractScrmSpider.class.php b/_php_code/AbstractScrmSpider.class.php index 39efeb7..880761d 100755 --- a/_php_code/AbstractScrmSpider.class.php +++ b/_php_code/AbstractScrmSpider.class.php @@ -75,11 +75,23 @@ abstract class AbstractScrmSpider const MODE_FETCH = 'fetch'; // 极速并发拉取 (推荐默认) const MODE_UI = 'ui_click'; // 强制 UI 点击下一页 + /** CLI 测试时输出 Node 请求/失败详情 */ + protected $verbose = false; + protected $nodeHost; public function __construct($nodeHost = 'http://127.0.0.1:3001') { - $this->nodeHost = $nodeHost; + $this->nodeHost = rtrim($nodeHost, '/'); + } + + /** + * 开启调试输出(测试脚本可调用) + */ + public function setVerbose($verbose = true) + { + $this->verbose = (bool) $verbose; + return $this; } /** ================= 子类必须实现的契约 ================= **/ @@ -97,7 +109,7 @@ abstract class AbstractScrmSpider public function run() { $config = $this->getSpiderConfig(); - + $listApi = $config['listApi']; $detailApi = $config['detailApi'] ?? null; // detail 接口是可选的 $countApi = $config['countApi'] ?? null; // 有些在线客服的总数通过单独调用接口获取 @@ -106,13 +118,14 @@ abstract class AbstractScrmSpider $apiUrlsToIntercept = [$listApi]; if ($detailApi) { $apiUrlsToIntercept[] = $detailApi; } if ($countApi) { $apiUrlsToIntercept[] = $countApi; } -// var_dump($apiUrlsToIntercept); -// dump([ -// 'pageUrl' => $config['pageUrl'], -// 'apiUrls' => $apiUrlsToIntercept, -// 'authActions' => $config['authActions'] -// ]); + $antiBot = $config['antiBot'] ?? null; + $mode = $config['paginationMode'] ?? self::MODE_FETCH; + + // antiBot + UI 翻页:单 Browser 合并路径(与生产 SplitTicket 同步一致) + if ($this->shouldUseUnifiedBrowserPath($antiBot, $mode)) { + return $this->runUnifiedUiPath($config, $antiBot, $apiUrlsToIntercept, $listApi, $detailApi, $countApi); + } // 【阶段一】:初始化并首屏拦截 $initResult = $this->requestNode('/api/auth-and-intercept', [ @@ -123,17 +136,20 @@ abstract class AbstractScrmSpider ], $this->resolveAuthInterceptTimeout($antiBot)); if (empty($initResult['success'])) { - $cfCode = (string) ($initResult['code'] ?? ''); - if ($cfCode === 'CF_TURNSTILE_FAILED') { - throw new Exception('Cloudflare Turnstile 验证失败: ' . ($initResult['stage'] ?? 'timeout')); - } - throw new Exception("初始化失败: " . ($initResult['error'] ?? '未知')); + throw new Exception($this->formatNodeFailure('初始化失败', $initResult)); } $interceptedApis = $initResult['interceptedApis']; $cookies = $initResult['cookies']; $finalPageUrl = $initResult['finalPageUrl'] ?? $config['pageUrl']; + $this->logDebug('auth-and-intercept ok', [ + 'finalPageUrl' => $finalPageUrl, + 'intercepted' => array_keys($interceptedApis), + 'cf' => $initResult['cf'] ?? null, + 'browserReused' => $initResult['browserReused'] ?? null, + ]); + // 必须拦截到 List 接口,否则无法继续 if (!isset($interceptedApis[$listApi])) { throw new Exception("致命错误:未能拦截到必须的列表接口 [{$listApi}]"); @@ -229,6 +245,140 @@ abstract class AbstractScrmSpider return $unifiedData; } + /** + * 单 Browser 路径:auth + 拦截 + UI 翻页(需 antiBot.sessionKey) + */ + protected function runUnifiedUiPath($config, $antiBot, $apiUrlsToIntercept, $listApi, $detailApi, $countApi) + { + $uiConfig = $this->getUiPaginationConfig(); + $sessionKey = is_array($antiBot) ? (string) ($antiBot['sessionKey'] ?? '') : ''; + + $this->logDebug('unified browser path', [ + 'sessionKey' => $sessionKey, + 'listApi' => $listApi, + ]); + + $mergedResult = $this->requestNode('/api/auth-intercept-and-paginate', [ + 'pageUrl' => $config['pageUrl'], + 'apiUrls' => $apiUrlsToIntercept, + 'authActions' => $config['authActions'] ?? [], + 'antiBot' => $antiBot, + 'pagination' => [ + 'apiUrl' => $listApi, + 'nextBtnSelector' => $uiConfig['nextBtnSelector'] ?? '', + 'waitMs' => $uiConfig['waitMs'] ?? 2000, + 'clicksToPerform' => 9999, + ], + ], 1200); + + if (empty($mergedResult['success'])) { + throw new Exception($this->formatNodeFailure('合并同步失败', $mergedResult)); + } + + $this->logDebug('auth-intercept-and-paginate ok', [ + 'intercepted' => array_keys($mergedResult['interceptedApis'] ?? []), + 'extraPages' => count($mergedResult['extraPages'] ?? []), + 'cf' => $mergedResult['cf'] ?? null, + 'browserReused' => $mergedResult['browserReused'] ?? null, + 'finalPageUrl' => $mergedResult['finalPageUrl'] ?? null, + ]); + + $interceptedApis = $mergedResult['interceptedApis'] ?? []; + if (!isset($interceptedApis[$listApi])) { + throw new Exception("致命错误:未能拦截到必须的列表接口 [{$listApi}]"); + } + + $detailData = $detailApi && isset($interceptedApis[$detailApi]) + ? $interceptedApis[$detailApi]['data'] : null; + $countData = $countApi && isset($interceptedApis[$countApi]) + ? $interceptedApis[$countApi]['data'] : null; + + $listApiNode = $interceptedApis[$listApi]; + $allListPagesData = [$listApiNode['data']]; + if (!empty($mergedResult['extraPages']) && is_array($mergedResult['extraPages'])) { + foreach ($mergedResult['extraPages'] as $pageData) { + $allListPagesData[] = $pageData; + } + } + + $this->extractListTotalPages($listApiNode['data'], $countData); + + return $this->parseToUnifiedData($detailData, $allListPagesData); + } + + /** + * antiBot 启用且配置了 sessionKey 时,UI 翻页走合并 Browser 路径 + */ + protected function shouldUseUnifiedBrowserPath($antiBot, $mode) + { + if ($mode !== self::MODE_UI) { + return false; + } + if (!is_array($antiBot) || empty($antiBot['enabled'])) { + return false; + } + return !empty($antiBot['sessionKey']); + } + + /** + * 格式化 Node 失败信息,附带 CF / 落地页等诊断字段 + */ + protected function formatNodeFailure($prefix, $result) + { + $cfCode = (string) ($result['code'] ?? ''); + if ($cfCode === 'CF_TURNSTILE_FAILED') { + return 'Cloudflare Turnstile 验证失败: ' . ($result['stage'] ?? 'timeout'); + } + + $parts = [$prefix . ': ' . ($result['error'] ?? '未知')]; + + if (!empty($result['finalPageUrl'])) { + $parts[] = 'finalPageUrl=' . $result['finalPageUrl']; + } + if (!empty($result['page']) && is_array($result['page'])) { + $page = $result['page']; + if (!empty($page['url'])) { + $parts[] = 'pageUrl=' . $page['url']; + } + if (isset($page['elInputInner'])) { + $parts[] = 'elInputInner=' . $page['elInputInner']; + } + } + if (!empty($result['cf']) && is_array($result['cf'])) { + $cf = $result['cf']; + $cfBits = []; + if (isset($cf['cfDetected'])) { + $cfBits[] = 'cfDetected=' . ($cf['cfDetected'] ? 'true' : 'false'); + } + if (!empty($cf['turnstileStage'])) { + $cfBits[] = 'turnstileStage=' . $cf['turnstileStage']; + } + if (isset($cf['solverUsed'])) { + $cfBits[] = 'solverUsed=' . ($cf['solverUsed'] ? 'true' : 'false'); + } + if (isset($cf['elapsedMs'])) { + $cfBits[] = 'cfElapsedMs=' . $cf['elapsedMs']; + } + if (!empty($cfBits)) { + $parts[] = implode(', ', $cfBits); + } + } + + return implode(' | ', $parts); + } + + protected function logDebug($message, $context = []) + { + if (!$this->verbose) { + return; + } + $line = '[Spider] ' . $message; + if (!empty($context)) { + $line .= ' ' . json_encode($context, JSON_UNESCAPED_UNICODE); + } + echo $line . "\n"; + } + /** * Real Browser + Turnstile 首屏耗时较长,单独放宽 auth-and-intercept 超时 * @@ -244,17 +394,76 @@ abstract class AbstractScrmSpider private function requestNode($endpoint, $payload, $timeout = 60) { - $ch = curl_init($this->nodeHost . $endpoint); + $maxAttempts = 3; + $lastDecoded = []; + for ($attempt = 1; $attempt <= $maxAttempts; $attempt++) { + $lastDecoded = $this->doRequestNode($endpoint, $payload, $timeout, $attempt); + if (!empty($lastDecoded['success'])) { + return $lastDecoded; + } + $error = (string) ($lastDecoded['error'] ?? ''); + if (!$this->shouldRetryNodeResponse($lastDecoded, $error) || $attempt >= $maxAttempts) { + return $lastDecoded; + } + $this->logDebug('node retry', [ + 'endpoint' => $endpoint, + 'attempt' => $attempt + 1, + 'error' => $error, + ]); + sleep($attempt); + } + return $lastDecoded; + } + + private function doRequestNode($endpoint, $payload, $timeout, $attempt) + { + $url = $this->nodeHost . $endpoint; + $this->logDebug('node request', [ + 'url' => $url, + 'timeout' => $timeout, + 'attempt' => $attempt, + ]); + + $ch = curl_init($url); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_POST, true); - curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($payload)); + curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($payload, JSON_UNESCAPED_UNICODE)); curl_setopt($ch, CURLOPT_HTTPHEADER, ['Content-Type: application/json']); curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5); curl_setopt($ch, CURLOPT_TIMEOUT, $timeout); $response = curl_exec($ch); - if (curl_errno($ch)) throw new Exception(curl_error($ch)); + $httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE); + if (curl_errno($ch)) { + $err = curl_error($ch); + curl_close($ch); + throw new Exception($err); + } curl_close($ch); - return json_decode($response, true); + + $decoded = json_decode((string) $response, true); + if (!is_array($decoded)) { + return ['success' => false, 'error' => 'Node 返回非 JSON', 'httpCode' => $httpCode]; + } + if ($httpCode === 503) { + $decoded['success'] = false; + $decoded['error'] = (string) ($decoded['error'] ?? '服务繁忙'); + $decoded['httpCode'] = 503; + } + return $decoded; + } + + private function shouldRetryNodeResponse($decoded, $error) + { + if ((int) ($decoded['httpCode'] ?? 0) === 503) { + return true; + } + $lower = strtolower($error); + foreach (['排队超时', 'queue_timeout', 'timeout', 'net::', 'navigation', 'connection'] as $needle) { + if ($needle !== '' && strpos($lower, $needle) !== false) { + return true; + } + } + return false; } /** diff --git a/_php_code/Whatshub.php b/_php_code/Whatshub.php index d51076a..47ec98d 100644 --- a/_php_code/Whatshub.php +++ b/_php_code/Whatshub.php @@ -7,13 +7,17 @@ class Whatshub extends AbstractScrmSpider { const API_LIST = '/api/whatshub-counter/workShare/open/detail'; // 列表API地址 const API_DETAILS = '/api/whatshub-counter/workShare/open/statistics'; // 详情API地址 + /** shareCode + password 直连接口(一次返回全量号码,无需翻页) */ + const API_DETAIL_BY_SHARE_CODE = '/api/whatshub-counter/workShare/open/detailByShareCode'; const API_COUNT = ''; // 总数API地址 const DEFAULT_PER_PAGE_COUNT = 20; // List默认每页显示的数量 + const API_FETCH_TIMEOUT_SEC = 20; + private $pageUrl; private $account; private $password; private $unifiedData; - + // 实例化时动态传入账号和密码 public function __construct($pageUrl, $account, $password, $nodeHost = 'http://127.0.0.1:3001') { @@ -21,46 +25,210 @@ class Whatshub extends AbstractScrmSpider $this->account = $account; $this->password = $password; $this->pageUrl = $pageUrl; - + $this->unifiedData = new UnifiedScrmData(); } + /** + * API 优先:detailByShareCode 直抓;失败则回退 Node 爬虫 + */ + public function run() + { + $apiResult = $this->tryFetchViaShareCodeApi(); + if ($apiResult instanceof UnifiedScrmData) { + $this->logDebug('whatshub api path ok', ['count' => count($apiResult->numbers)]); + return $apiResult; + } + + $this->logDebug('whatshub api skipped or failed, fallback node', []); + return parent::run(); + } + + /** + * 从短链 pageUrl 解析 shareCode,例如 /m/KMYBBInp2066/1 → KMYBBInp2066 + * + * @return string|null + */ + private function extractShareCodeFromPageUrl() + { + $path = (string) parse_url($this->pageUrl, PHP_URL_PATH); + if ($path === '') { + return null; + } + if (preg_match('#/m/([^/]+)/#', $path, $matches)) { + return $matches[1]; + } + return null; + } + + /** + * 拼接 detailByShareCode 完整 GET URL + * + * @param string $shareCode + * @return string|null + */ + private function buildShareCodeApiUrl($shareCode) + { + $parsed = parse_url($this->pageUrl); + $scheme = isset($parsed['scheme']) ? $parsed['scheme'] : 'https'; + $host = isset($parsed['host']) ? $parsed['host'] : ''; + if ($host === '') { + return null; + } + + $query = http_build_query([ + 'shareCode' => $shareCode, + 'password' => $this->password, + ]); + + return $scheme . '://' . $host . self::API_DETAIL_BY_SHARE_CODE . '?' . $query; + } + + /** + * 尝试通过 shareCode API 抓取;成功返回 UnifiedScrmData,否则返回 null + * + * @return UnifiedScrmData|null + */ + private function tryFetchViaShareCodeApi() + { + if ($this->password === '' || $this->password === null) { + $this->logDebug('whatshub api skip: empty password', []); + return null; + } + + $shareCode = $this->extractShareCodeFromPageUrl(); + if ($shareCode === null || $shareCode === '') { + $this->logDebug('whatshub api skip: cannot parse shareCode', ['pageUrl' => $this->pageUrl]); + return null; + } + + $apiUrl = $this->buildShareCodeApiUrl($shareCode); + if ($apiUrl === null) { + $this->logDebug('whatshub api skip: invalid pageUrl host', ['pageUrl' => $this->pageUrl]); + return null; + } + + $this->logDebug('whatshub api request', ['url' => preg_replace('/password=[^&]+/', 'password=***', $apiUrl)]); + + $ch = curl_init($apiUrl); + curl_setopt_array($ch, [ + CURLOPT_RETURNTRANSFER => true, + CURLOPT_CONNECTTIMEOUT => 5, + CURLOPT_TIMEOUT => self::API_FETCH_TIMEOUT_SEC, + CURLOPT_HTTPHEADER => [ + 'Accept: application/json', + 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + ], + ]); + + $response = curl_exec($ch); + $httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE); + $curlErr = curl_error($ch); + curl_close($ch); + + if ($response === false || $curlErr !== '') { + $this->logDebug('whatshub api curl failed', ['error' => $curlErr]); + return null; + } + + if ($httpCode !== 200) { + $this->logDebug('whatshub api http error', ['httpCode' => $httpCode]); + return null; + } + + $payload = json_decode($response, true); + if (!is_array($payload)) { + $this->logDebug('whatshub api invalid json', []); + return null; + } + + if ((int) ($payload['code'] ?? 0) !== 200) { + $this->logDebug('whatshub api business error', [ + 'code' => $payload['code'] ?? null, + 'msg' => $payload['msg'] ?? '', + ]); + return null; + } + + if (!isset($payload['data']) || !is_array($payload['data'])) { + $this->logDebug('whatshub api invalid data field', []); + return null; + } + + return $this->parseShareCodeApiResponse($payload); + } + + /** + * 将 detailByShareCode 响应映射为 UnifiedScrmData + * + * @param array $payload + * @return UnifiedScrmData + */ + private function parseShareCodeApiResponse(array $payload) + { + $unifiedData = new UnifiedScrmData(); + $todayNewSum = 0; + + foreach ($payload['data'] as $item) { + if (!is_array($item) || empty($item['account'])) { + continue; + } + + $number = (string) $item['account']; + $isOnline = isset($item['isOnline']) && (int) $item['isOnline'] === 1; + $dayNewFans = (int) ($item['dayNewFans'] ?? 0); + + $unifiedData->addNumber($number, $isOnline, $dayNewFans); + $todayNewSum += $dayNewFans; + } + + $unifiedData->todayNewCount = $todayNewSum; + $unifiedData->total = count($unifiedData->numbers); + + return $unifiedData; + } + protected function getSpiderConfig() { $host = (string) parse_url($this->pageUrl, PHP_URL_HOST); return [ 'pageUrl' => $this->pageUrl, 'apiUrls' => [self::API_LIST, self::API_DETAILS], - + // 明确指派角色 - 'listApi' => self::API_LIST, // 必须 - + 'listApi' => self::API_LIST, + 'detailApi' => self::API_DETAILS, + 'listMethod' => 'POST', 'paginationMode' => self::MODE_UI, - + 'authActions' => [ - // 1. 填入密码:寻找 class 包含 el-message-box__input 下面的任意 input - // Node.js 会自动扫描所有匹配项,并只把密码强行注入到那个“肉眼可见”的框里 + // Node antiBot.postCfReady* 会在 CF 成功后等待 work-order-sharing + .el-input__inner ['type' => 'vue_fill', 'selector' => '.el-input__inner', 'value' => $this->password], - + // 2. 停顿 500ms,让 Vue 绑定的 v-model 彻底反应过来 ['type' => 'wait', 'ms' => 500], - + // 3. 点击确认:寻找 MessageBox 底部的蓝色 primary 确认按钮 // 同样利用 vue_click 的可见性过滤,无视隐藏的旧弹窗按钮 ['type' => 'vue_click', 'selector' => '.vxe-button-group .theme--primary'], - + // 4. 等待弹窗淡出,接口开始请求 - ['type' => 'wait', 'ms' => 2200] + ['type' => 'wait', 'ms' => 2200], ], // Whatshub 专用:Real Browser + Turnstile + Captcha API 兜底 + 会话复用 - 'antiBot' => [ + // sessionKey 必须与 pageUrl 的 host 一致,例如 web.whatshub.cc + 'antiBot' => [ 'enabled' => true, 'profile' => 'real', 'turnstile' => true, 'solverFallback' => true, - 'sessionKey' => 'whatshub:' . $host, - 'challengeTimeoutMs' => 60000, + 'challengeTimeoutMs' => 40000, + 'postCfReadyUrlContains' => 'work-order-sharing', + 'postCfReadySelector' => '.el-input__inner', + 'postCfReadyTimeoutMs' => 30000, + 'postCfSettleMs' => 500, + 'postCfSpaWaitMs' => 3000, ], ]; } @@ -69,13 +237,13 @@ class Whatshub extends AbstractScrmSpider protected function extractListTotalPages($listFirstPageData, $countData = null) { $default_per_page_count = self::DEFAULT_PER_PAGE_COUNT; - $this->unifiedData->total = $listFirstPageData['data']['total']; - - if($this->unifiedData->total <= $default_per_page_count) { + $this->unifiedData->total = $listFirstPageData['total']; + + if ($this->unifiedData->total <= $default_per_page_count) { return 1; } - - return ceil($this->unifiedData->total/$default_per_page_count); + + return ceil($this->unifiedData->total / $default_per_page_count); } // 没有分页返回空数组 @@ -88,8 +256,8 @@ class Whatshub extends AbstractScrmSpider protected function getUiPaginationConfig() { return [ - 'nextBtnSelector' => '.vxe-pager--next-btn', - 'waitMs' => 1500 + 'nextBtnSelector' => '.vxe-pager--next-btn', + 'waitMs' => 1500, ]; } @@ -97,28 +265,28 @@ class Whatshub extends AbstractScrmSpider protected function parseToUnifiedData($detailData, $allListPagesData) { $unifiedData = $this->unifiedData; - + // 1. 如果捕获到了详情数据,提取今日新增 if ($detailData) { - $unifiedData->todayNewCount = (int)($detailData['data']['dayNewFans'] ?? 0); + $unifiedData->todayNewCount = (int) ($detailData['data']['dayNewFans'] ?? 0); } // 2. 循环合并了所有页数的 List 数组 foreach ($allListPagesData as $pageRaw) { - $records = $pageRaw['data']['rows'] ?? []; + $records = $pageRaw['rows'] ?? []; foreach ($records as $item) { - if(!empty($item['account'])) { + if (!empty($item['account'])) { $number = $item['account'] ?? null; $isOnline = (isset($item['isOnline']) && $item['isOnline'] == 1); - + $unifiedData->addNumber($number, $isOnline, $item['dayNewFans']); } } } - - // 🚀 3. 终极统计:所有翻页数据均已入库,此时再统计真实的 Total 总数 + + // 3. 终极统计:所有翻页数据均已入库,此时再统计真实的 Total 总数 $unifiedData->total = count($unifiedData->numbers); return $unifiedData; } -} \ No newline at end of file +} diff --git a/_php_code/index.php b/_php_code/index.php index 8ffe84a..0ee6414 100755 --- a/_php_code/index.php +++ b/_php_code/index.php @@ -11,50 +11,52 @@ require_once __DIR__ . '/SsCustomer.php'; // SS云控(Customer) require_once __DIR__ . '/Haiwang.php'; // 海王 require_once __DIR__ . '/Whatshub.php'; // Whatshub 工单平台 -// try { -// /* -// 命令行可以测试Cloudflare防火墙 -// curl -s -X POST http://127.0.0.1:3001/api/auth-and-intercept \ -// -H 'Content-Type: application/json' \ -// -d '{ -// "pageUrl": "https://web.whatshub.cc/m/iTYsWKQH5030/1", -// "apiUrls": ["/api/whatshub-counter/workShare/open/detail"], -// "authActions": [ -// {"type": "vue_fill", "selector": ".el-input__inner", "value": "745030"}, -// {"type": "wait", "ms": 500}, -// {"type": "vue_click", "selector": ".vxe-button-group .theme--primary"}, -// {"type": "wait", "ms": 2200} -// ], -// "antiBot": { -// "enabled": true, -// "profile": "real", -// "turnstile": true, -// "solverFallback": true, -// "sessionKey": "whatshub:t.flowerbells.top", -// "challengeTimeoutMs": 60000 -// } -// }' | jq . -// */ +try { + /* -// echo "🚀 开始执行抓取任务 (多引擎智能调度)...\n\r"; -// $pageUrl = 'https://web.whatshub.cc/m/iTYsWKQH5030/1'; // PageUrl 入口授权页 -// $username = ""; // 登录账号 -// $password = "745030"; // 登录密码 -// $spider = new Whatshub($pageUrl, $username, $password); -// $finalData = $spider->run(); + 命令行可以测试Cloudflare防火墙 + curl -s -X POST http://127.0.0.1:3001/api/auth-and-intercept \ + -H 'Content-Type: application/json' \ + -d '{ + "pageUrl": "https://web.whatshub.cc/m/KMYBBInp2066/1", + "apiUrls": ["/api/whatshub-counter/workShare/open/detail"], + "authActions": [ + {"type": "wait", "ms": 1000}, + {"type": "vue_fill", "selector": ".el-input__wrapper > .el-input__inner", "value": "602066"}, + {"type": "wait", "ms": 500}, + {"type": "vue_click", "selector": ".vxe-button-group .theme--primary"}, + {"type": "wait", "ms": 2200} + ], + "antiBot": { + "enabled": true, + "profile": "real", + "turnstile": true, + "solverFallback": true, + "sessionKey": "whatshub:web.whatshub.cc", + "challengeTimeoutMs": 60000 + } + }' | jq . + */ + + echo "🚀 开始执行抓取任务 (多引擎智能调度)...\n\r"; + $pageUrl = 'https://web.whatshub.cc/m/KMYBBInp2066/1'; // PageUrl 入口授权页 + $username = ""; // 登录账号 + $password = "602066"; // 登录密码 + $spider = new Whatshub($pageUrl, $username, $password); + $finalData = $spider->run(); -// echo "✅ 任务完成!统一数据如下:\n\r"; -// echo "----------------------------------------\n\r"; -// echo "当日新增:{$finalData->todayNewCount} 人\n\r"; -// echo "在线号码:{$finalData->totalOnline} 个\n\r"; -// echo "离线号码:{$finalData->totalOffline} 个\n\r"; -// echo "Total:" . $finalData->total . " 个号码\n\r"; -// echo "实际总共抓取:" . count($finalData->numbers) . " 个号码\n\r"; -// echo "号码列表:\n\r"; -// echo dd($finalData->numbers); -// } catch (Exception $e) { -// echo "🚨 抓取异常:" . $e->getMessage() . "\n\r"; -// } + echo "✅ 任务完成!统一数据如下:\n\r"; + echo "----------------------------------------\n\r"; + echo "当日新增:{$finalData->todayNewCount} 人\n\r"; + echo "在线号码:{$finalData->totalOnline} 个\n\r"; + echo "离线号码:{$finalData->totalOffline} 个\n\r"; + echo "Total:" . $finalData->total . " 个号码\n\r"; + echo "实际总共抓取:" . count($finalData->numbers) . " 个号码\n\r"; + echo "号码列表:\n\r"; + echo dd($finalData->numbers); +} catch (Exception $e) { + echo "🚨 抓取异常:" . $e->getMessage() . "\n\r"; +} // try { @@ -78,27 +80,27 @@ require_once __DIR__ . '/Whatshub.php'; // Whatshub 工单平台 // echo "🚨 抓取异常:" . $e->getMessage() . "\n\r"; // } -try { - echo "🚀 开始执行抓取任务 (多引擎智能调度)...\n\r"; - $pageUrl = 'https://yyk.ink/52oXZRG'; // PageUrl 入口授权页 - $username = ""; // 登录账号 - $password = ""; // 登录密码 - $spider = new A2c($pageUrl, $username, $password); - $finalData = $spider->run(); +// try { +// echo "🚀 开始执行抓取任务 (多引擎智能调度)...\n\r"; +// $pageUrl = 'https://yyk.ink/52oXZRG'; // PageUrl 入口授权页 +// $username = ""; // 登录账号 +// $password = ""; // 登录密码 +// $spider = new A2c($pageUrl, $username, $password); +// $finalData = $spider->run(); - echo "✅ 任务完成!统一数据如下:\n\r"; - echo "----------------------------------------\n\r"; - echo "当日新增:{$finalData->todayNewCount} 人\n\r"; - echo "在线号码:{$finalData->totalOnline} 个\n\r"; - echo "离线号码:{$finalData->totalOffline} 个\n\r"; - echo "Total:" . $finalData->total . " 个号码\n\r"; - echo "实际总共抓取:" . count($finalData->numbers) . " 个号码\n\r"; - echo "号码列表:\n\r"; - echo dd($finalData->numbers); +// echo "✅ 任务完成!统一数据如下:\n\r"; +// echo "----------------------------------------\n\r"; +// echo "当日新增:{$finalData->todayNewCount} 人\n\r"; +// echo "在线号码:{$finalData->totalOnline} 个\n\r"; +// echo "离线号码:{$finalData->totalOffline} 个\n\r"; +// echo "Total:" . $finalData->total . " 个号码\n\r"; +// echo "实际总共抓取:" . count($finalData->numbers) . " 个号码\n\r"; +// echo "号码列表:\n\r"; +// echo dd($finalData->numbers); -} catch (Exception $e) { - echo "🚨 抓取异常:" . $e->getMessage() . "\n\r"; -} +// } catch (Exception $e) { +// echo "🚨 抓取异常:" . $e->getMessage() . "\n\r"; +// } // try { // echo "🚀 开始执行<星河云控>抓取任务 (多引擎智能调度)...\n\r"; diff --git a/application/common/library/scrm/AntiBotConfigBuilder.php b/application/common/library/scrm/AntiBotConfigBuilder.php index bc6ff0a..34854d4 100644 --- a/application/common/library/scrm/AntiBotConfigBuilder.php +++ b/application/common/library/scrm/AntiBotConfigBuilder.php @@ -41,6 +41,26 @@ class AntiBotConfigBuilder 'sessionKey' => $sessionKey, 'challengeTimeoutMs' => 60000, 'sessionTtlMinutes' => $ttlMinutes, + ] + self::postCfReadyExtras($ticketType); + } + + /** + * 按工单类型附加 CF 过盾后的业务页就绪等待(仅 Whatshub 等需要的类型) + * + * @return array + */ + private static function postCfReadyExtras(string $ticketType): array + { + if ($ticketType !== 'whatshub') { + return []; + } + + return [ + 'postCfReadyUrlContains' => 'work-order-sharing', + 'postCfReadySelector' => '.el-input__inner', + 'postCfReadyTimeoutMs' => 30000, + 'postCfSettleMs' => 500, + 'postCfSpaWaitMs' => 4000, ]; } diff --git a/application/common/library/scrm/spider/WhatshubSpider.php b/application/common/library/scrm/spider/WhatshubSpider.php index f658508..d6dfae5 100755 --- a/application/common/library/scrm/spider/WhatshubSpider.php +++ b/application/common/library/scrm/spider/WhatshubSpider.php @@ -7,11 +7,12 @@ namespace app\common\library\scrm\spider; use app\common\library\scrm\AbstractScrmSpider; use app\common\library\scrm\AntiBotConfigBuilder; use app\common\library\scrm\UnifiedScrmData; +use app\common\service\SplitTicketSyncLogger; /** - * Whatshub 云控蜘蛛(Real Browser + Turnstile 过盾 + UI 翻页) + * Whatshub 云控蜘蛛 * - * 时序:Cloudflare Turnstile → 密码弹窗 authActions → API 拦截 → UI 翻页 + * 时序:detailByShareCode API 直抓(优先)→ 失败则 Real Browser + Turnstile + UI 翻页 */ class WhatshubSpider extends AbstractScrmSpider { @@ -21,9 +22,14 @@ class WhatshubSpider extends AbstractScrmSpider /** 详情/统计 API 路径 */ private const API_DETAILS = '/api/whatshub-counter/workShare/open/statistics'; + /** shareCode + password 直连接口(一次返回全量号码,无需翻页) */ + private const API_DETAIL_BY_SHARE_CODE = '/api/whatshub-counter/workShare/open/detailByShareCode'; + /** 默认每页条数 */ private const DEFAULT_PER_PAGE_COUNT = 20; + private const API_FETCH_TIMEOUT_SEC = 20; + private string $pageUrl; private string $account; @@ -45,6 +51,178 @@ class WhatshubSpider extends AbstractScrmSpider $this->unifiedData = new UnifiedScrmData(); } + /** + * API 优先:detailByShareCode 直抓;失败则回退 Node 爬虫 + */ + public function run(): UnifiedScrmData + { + $apiResult = $this->tryFetchViaShareCodeApi(); + if ($apiResult instanceof UnifiedScrmData) { + SplitTicketSyncLogger::log('spider', 'whatshub api path ok', [ + 'count' => count($apiResult->numbers), + ]); + return $apiResult; + } + + SplitTicketSyncLogger::log('spider', 'whatshub api skipped or failed, fallback node', [ + 'pageUrl' => $this->pageUrl, + ]); + + return parent::run(); + } + + /** + * 从短链 pageUrl 解析 shareCode,例如 /m/KMYBBInp2066/1 → KMYBBInp2066 + */ + private function extractShareCodeFromPageUrl(): ?string + { + $path = (string) parse_url($this->pageUrl, PHP_URL_PATH); + if ($path === '') { + return null; + } + if (preg_match('#/m/([^/]+)/#', $path, $matches)) { + return $matches[1]; + } + + return null; + } + + /** + * 拼接 detailByShareCode 完整 GET URL + */ + private function buildShareCodeApiUrl(string $shareCode): ?string + { + $parsed = parse_url($this->pageUrl); + $scheme = isset($parsed['scheme']) ? (string) $parsed['scheme'] : 'https'; + $host = isset($parsed['host']) ? (string) $parsed['host'] : ''; + if ($host === '') { + return null; + } + + $query = http_build_query([ + 'shareCode' => $shareCode, + 'password' => $this->password, + ]); + + return $scheme . '://' . $host . self::API_DETAIL_BY_SHARE_CODE . '?' . $query; + } + + /** + * 尝试通过 shareCode API 抓取;成功返回 UnifiedScrmData,否则返回 null + */ + private function tryFetchViaShareCodeApi(): ?UnifiedScrmData + { + if ($this->password === '') { + SplitTicketSyncLogger::log('spider', 'whatshub api skip: empty password', []); + return null; + } + + $shareCode = $this->extractShareCodeFromPageUrl(); + if ($shareCode === null || $shareCode === '') { + SplitTicketSyncLogger::log('spider', 'whatshub api skip: cannot parse shareCode', [ + 'pageUrl' => $this->pageUrl, + ]); + return null; + } + + $apiUrl = $this->buildShareCodeApiUrl($shareCode); + if ($apiUrl === null) { + SplitTicketSyncLogger::log('spider', 'whatshub api skip: invalid pageUrl host', [ + 'pageUrl' => $this->pageUrl, + ]); + return null; + } + + SplitTicketSyncLogger::log('spider', 'whatshub api request', [ + 'url' => preg_replace('/password=[^&]+/', 'password=***', $apiUrl), + ]); + + $ch = curl_init($apiUrl); + if ($ch === false) { + SplitTicketSyncLogger::log('spider', 'whatshub api curl init failed', []); + return null; + } + + curl_setopt_array($ch, [ + CURLOPT_RETURNTRANSFER => true, + CURLOPT_CONNECTTIMEOUT => 5, + CURLOPT_TIMEOUT => self::API_FETCH_TIMEOUT_SEC, + CURLOPT_HTTPHEADER => [ + 'Accept: application/json', + 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + ], + ]); + + $response = curl_exec($ch); + $httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE); + $curlErr = curl_error($ch); + curl_close($ch); + + if ($response === false || $curlErr !== '') { + SplitTicketSyncLogger::log('spider', 'whatshub api curl failed', [ + 'error' => $curlErr, + ]); + return null; + } + + if ($httpCode !== 200) { + SplitTicketSyncLogger::log('spider', 'whatshub api http error', [ + 'httpCode' => $httpCode, + ]); + return null; + } + + $payload = json_decode((string) $response, true); + if (!is_array($payload)) { + SplitTicketSyncLogger::log('spider', 'whatshub api invalid json', []); + return null; + } + + if ((int) ($payload['code'] ?? 0) !== 200) { + SplitTicketSyncLogger::log('spider', 'whatshub api business error', [ + 'code' => $payload['code'] ?? null, + 'msg' => $payload['msg'] ?? '', + ]); + return null; + } + + if (!isset($payload['data']) || !is_array($payload['data'])) { + SplitTicketSyncLogger::log('spider', 'whatshub api invalid data field', []); + return null; + } + + return $this->parseShareCodeApiResponse($payload); + } + + /** + * 将 detailByShareCode 响应映射为 UnifiedScrmData + * + * @param array $payload + */ + private function parseShareCodeApiResponse(array $payload): UnifiedScrmData + { + $unifiedData = new UnifiedScrmData(); + $todayNewSum = 0; + + foreach ($payload['data'] as $item) { + if (!is_array($item) || empty($item['account'])) { + continue; + } + + $number = (string) $item['account']; + $isOnline = isset($item['isOnline']) && (int) $item['isOnline'] === 1; + $dayNewFans = (int) ($item['dayNewFans'] ?? 0); + + $unifiedData->addNumber($number, $isOnline, $dayNewFans); + $todayNewSum += $dayNewFans; + } + + $unifiedData->todayNewCount = $todayNewSum; + $unifiedData->total = count($unifiedData->numbers); + + return $unifiedData; + } + /** @return array */ protected function getSpiderConfig(): array { diff --git a/patches/application/common/library/scrm/AntiBotConfigBuilder.php b/patches/application/common/library/scrm/AntiBotConfigBuilder.php index bc6ff0a..7e02ff6 100644 --- a/patches/application/common/library/scrm/AntiBotConfigBuilder.php +++ b/patches/application/common/library/scrm/AntiBotConfigBuilder.php @@ -41,6 +41,26 @@ class AntiBotConfigBuilder 'sessionKey' => $sessionKey, 'challengeTimeoutMs' => 60000, 'sessionTtlMinutes' => $ttlMinutes, + ] + self::postCfReadyExtras($ticketType); + } + + /** + * 按工单类型附加 CF 过盾后的业务页就绪等待(仅 Whatshub 等需要的类型) + * + * @return array + */ + private static function postCfReadyExtras(string $ticketType): array + { + if ($ticketType !== 'whatshub') { + return []; + } + + return [ + 'postCfReadyUrlContains' => 'work-order-sharing', + 'postCfReadySelector' => '.el-input__inner', + 'postCfReadyTimeoutMs' => 30000, + 'postCfSettleMs' => 500, + 'postCfSpaWaitMs' => 8000, ]; } diff --git a/patches/application/common/library/scrm/spider/WhatshubSpider.php b/patches/application/common/library/scrm/spider/WhatshubSpider.php index f658508..d6dfae5 100755 --- a/patches/application/common/library/scrm/spider/WhatshubSpider.php +++ b/patches/application/common/library/scrm/spider/WhatshubSpider.php @@ -7,11 +7,12 @@ namespace app\common\library\scrm\spider; use app\common\library\scrm\AbstractScrmSpider; use app\common\library\scrm\AntiBotConfigBuilder; use app\common\library\scrm\UnifiedScrmData; +use app\common\service\SplitTicketSyncLogger; /** - * Whatshub 云控蜘蛛(Real Browser + Turnstile 过盾 + UI 翻页) + * Whatshub 云控蜘蛛 * - * 时序:Cloudflare Turnstile → 密码弹窗 authActions → API 拦截 → UI 翻页 + * 时序:detailByShareCode API 直抓(优先)→ 失败则 Real Browser + Turnstile + UI 翻页 */ class WhatshubSpider extends AbstractScrmSpider { @@ -21,9 +22,14 @@ class WhatshubSpider extends AbstractScrmSpider /** 详情/统计 API 路径 */ private const API_DETAILS = '/api/whatshub-counter/workShare/open/statistics'; + /** shareCode + password 直连接口(一次返回全量号码,无需翻页) */ + private const API_DETAIL_BY_SHARE_CODE = '/api/whatshub-counter/workShare/open/detailByShareCode'; + /** 默认每页条数 */ private const DEFAULT_PER_PAGE_COUNT = 20; + private const API_FETCH_TIMEOUT_SEC = 20; + private string $pageUrl; private string $account; @@ -45,6 +51,178 @@ class WhatshubSpider extends AbstractScrmSpider $this->unifiedData = new UnifiedScrmData(); } + /** + * API 优先:detailByShareCode 直抓;失败则回退 Node 爬虫 + */ + public function run(): UnifiedScrmData + { + $apiResult = $this->tryFetchViaShareCodeApi(); + if ($apiResult instanceof UnifiedScrmData) { + SplitTicketSyncLogger::log('spider', 'whatshub api path ok', [ + 'count' => count($apiResult->numbers), + ]); + return $apiResult; + } + + SplitTicketSyncLogger::log('spider', 'whatshub api skipped or failed, fallback node', [ + 'pageUrl' => $this->pageUrl, + ]); + + return parent::run(); + } + + /** + * 从短链 pageUrl 解析 shareCode,例如 /m/KMYBBInp2066/1 → KMYBBInp2066 + */ + private function extractShareCodeFromPageUrl(): ?string + { + $path = (string) parse_url($this->pageUrl, PHP_URL_PATH); + if ($path === '') { + return null; + } + if (preg_match('#/m/([^/]+)/#', $path, $matches)) { + return $matches[1]; + } + + return null; + } + + /** + * 拼接 detailByShareCode 完整 GET URL + */ + private function buildShareCodeApiUrl(string $shareCode): ?string + { + $parsed = parse_url($this->pageUrl); + $scheme = isset($parsed['scheme']) ? (string) $parsed['scheme'] : 'https'; + $host = isset($parsed['host']) ? (string) $parsed['host'] : ''; + if ($host === '') { + return null; + } + + $query = http_build_query([ + 'shareCode' => $shareCode, + 'password' => $this->password, + ]); + + return $scheme . '://' . $host . self::API_DETAIL_BY_SHARE_CODE . '?' . $query; + } + + /** + * 尝试通过 shareCode API 抓取;成功返回 UnifiedScrmData,否则返回 null + */ + private function tryFetchViaShareCodeApi(): ?UnifiedScrmData + { + if ($this->password === '') { + SplitTicketSyncLogger::log('spider', 'whatshub api skip: empty password', []); + return null; + } + + $shareCode = $this->extractShareCodeFromPageUrl(); + if ($shareCode === null || $shareCode === '') { + SplitTicketSyncLogger::log('spider', 'whatshub api skip: cannot parse shareCode', [ + 'pageUrl' => $this->pageUrl, + ]); + return null; + } + + $apiUrl = $this->buildShareCodeApiUrl($shareCode); + if ($apiUrl === null) { + SplitTicketSyncLogger::log('spider', 'whatshub api skip: invalid pageUrl host', [ + 'pageUrl' => $this->pageUrl, + ]); + return null; + } + + SplitTicketSyncLogger::log('spider', 'whatshub api request', [ + 'url' => preg_replace('/password=[^&]+/', 'password=***', $apiUrl), + ]); + + $ch = curl_init($apiUrl); + if ($ch === false) { + SplitTicketSyncLogger::log('spider', 'whatshub api curl init failed', []); + return null; + } + + curl_setopt_array($ch, [ + CURLOPT_RETURNTRANSFER => true, + CURLOPT_CONNECTTIMEOUT => 5, + CURLOPT_TIMEOUT => self::API_FETCH_TIMEOUT_SEC, + CURLOPT_HTTPHEADER => [ + 'Accept: application/json', + 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + ], + ]); + + $response = curl_exec($ch); + $httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE); + $curlErr = curl_error($ch); + curl_close($ch); + + if ($response === false || $curlErr !== '') { + SplitTicketSyncLogger::log('spider', 'whatshub api curl failed', [ + 'error' => $curlErr, + ]); + return null; + } + + if ($httpCode !== 200) { + SplitTicketSyncLogger::log('spider', 'whatshub api http error', [ + 'httpCode' => $httpCode, + ]); + return null; + } + + $payload = json_decode((string) $response, true); + if (!is_array($payload)) { + SplitTicketSyncLogger::log('spider', 'whatshub api invalid json', []); + return null; + } + + if ((int) ($payload['code'] ?? 0) !== 200) { + SplitTicketSyncLogger::log('spider', 'whatshub api business error', [ + 'code' => $payload['code'] ?? null, + 'msg' => $payload['msg'] ?? '', + ]); + return null; + } + + if (!isset($payload['data']) || !is_array($payload['data'])) { + SplitTicketSyncLogger::log('spider', 'whatshub api invalid data field', []); + return null; + } + + return $this->parseShareCodeApiResponse($payload); + } + + /** + * 将 detailByShareCode 响应映射为 UnifiedScrmData + * + * @param array $payload + */ + private function parseShareCodeApiResponse(array $payload): UnifiedScrmData + { + $unifiedData = new UnifiedScrmData(); + $todayNewSum = 0; + + foreach ($payload['data'] as $item) { + if (!is_array($item) || empty($item['account'])) { + continue; + } + + $number = (string) $item['account']; + $isOnline = isset($item['isOnline']) && (int) $item['isOnline'] === 1; + $dayNewFans = (int) ($item['dayNewFans'] ?? 0); + + $unifiedData->addNumber($number, $isOnline, $dayNewFans); + $todayNewSum += $dayNewFans; + } + + $unifiedData->todayNewCount = $todayNewSum; + $unifiedData->total = count($unifiedData->numbers); + + return $unifiedData; + } + /** @return array */ protected function getSpiderConfig(): array { diff --git a/patches/puppeteer-api/lib/browser-factory.js b/patches/puppeteer-api/lib/browser-factory.js index b4b6476..2c5ec02 100755 --- a/patches/puppeteer-api/lib/browser-factory.js +++ b/patches/puppeteer-api/lib/browser-factory.js @@ -28,6 +28,10 @@ function normalizeAntiBot(antiBot) { solverFallback: antiBot.solverFallback !== false, sessionKey: antiBot.sessionKey || '', challengeTimeoutMs: antiBot.challengeTimeoutMs || 60000, + postCfReadyUrlContains: antiBot.postCfReadyUrlContains || '', + postCfReadySelector: antiBot.postCfReadySelector || '', + postCfReadyTimeoutMs: antiBot.postCfReadyTimeoutMs || 0, + postCfSettleMs: antiBot.postCfSettleMs || 0, }; } @@ -106,8 +110,10 @@ async function createManagedPage(browser, existingPage, options = {}) { page.setDefaultNavigationTimeout(NAVIGATION_TIMEOUT_MS); page.setDefaultTimeout(PAGE_DEFAULT_TIMEOUT_MS); - const userAgent = options.userAgent || DEFAULT_UA; - await page.setUserAgent(userAgent); + if (!options.skipUserAgent) { + const userAgent = options.userAgent || DEFAULT_UA; + await page.setUserAgent(userAgent); + } if (options.viewport) await page.setViewport(options.viewport); if (options.cookies && options.cookies.length > 0) { diff --git a/patches/puppeteer-api/lib/cf-detector.js b/patches/puppeteer-api/lib/cf-detector.js index 31c1794..195b6c7 100755 --- a/patches/puppeteer-api/lib/cf-detector.js +++ b/patches/puppeteer-api/lib/cf-detector.js @@ -2,6 +2,21 @@ * Cloudflare / Turnstile 挑战页检测 */ +/** + * URL 是否处于 CF Managed Challenge 中间态(Whatshub 短链常见 __cf_chl_rt_tk) + * @param {string} url + */ +function urlIndicatesCfChallenge(url) { + if (!url || typeof url !== 'string') { + return false; + } + return url.includes('__cf_chl_rt_tk') + || url.includes('__cf_chl_tk') + || url.includes('__cf_chl_f_tk') + || url.includes('/cdn-cgi/challenge-platform') + || url.includes('/cdn-cgi/challenge'); +} + /** * @typedef {Object} CloudflareState * @property {boolean} blocked 是否处于 CF 挑战中 @@ -44,10 +59,10 @@ async function detectCloudflare(page) { bodyHasJustAMoment: false, })); - const urlChallenge = url.includes('/cdn-cgi/challenge-platform') || url.includes('/cdn-cgi/challenge'); + const urlChallenge = urlIndicatesCfChallenge(url); let challengeType = null; - if (domSignals.hasTurnstileInput || domSignals.hasTurnstileWidget) { + if (domSignals.hasTurnstileInput || domSignals.hasTurnstileWidget || url.includes('__cf_chl')) { challengeType = 'turnstile'; } else if (urlChallenge || domSignals.titleHasJustAMoment || domSignals.bodyHasJustAMoment) { challengeType = 'js_challenge'; @@ -94,4 +109,5 @@ async function isTurnstileSolved(page) { module.exports = { detectCloudflare, isTurnstileSolved, + urlIndicatesCfChallenge, }; diff --git a/patches/puppeteer-api/lib/cf-handler.js b/patches/puppeteer-api/lib/cf-handler.js index b595b2e..e84ec14 100755 --- a/patches/puppeteer-api/lib/cf-handler.js +++ b/patches/puppeteer-api/lib/cf-handler.js @@ -1,5 +1,8 @@ /** * Cloudflare 挑战统一处理:检测 → 会话快速通道 → 内置等待 → Captcha API 兜底 + * + * 当 antiBot.postCfReadyUrlContains 已配置时,session_reuse 仅在业务 URL 已到达时才走快速通道; + * 否则清除可能过期的 cf_clearance 并重新触发 Turnstile(Whatshub 短链 → work-order-sharing 场景)。 */ const { detectCloudflare } = require('./cf-detector'); const { waitForTurnstile, extractTurnstileSitekey, injectTurnstileToken } = require('./turnstile-handler'); @@ -7,8 +10,135 @@ const { solveTurnstile, isCaptchaConfigured } = require('./captcha-solver'); const { isSessionLikelyValid } = require('./session-store'); /** + * 是否配置了 CF 后必须到达的业务 URL + * @param {object} antiBot + */ +function needsPostCfBusinessUrl(antiBot) { + return !!(antiBot?.postCfReadyUrlContains || '').trim(); +} + +/** + * 当前页面 URL 是否已包含业务路径片段 * @param {import('puppeteer').Page} page - * @param {{ turnstile?: boolean, solverFallback?: boolean, challengeTimeoutMs?: number }} antiBot + * @param {object} antiBot + */ +function isPostCfBusinessUrlReady(page, antiBot) { + const needle = (antiBot?.postCfReadyUrlContains || '').trim(); + if (!needle) { + return true; + } + return page.url().includes(needle); +} + +/** + * 清除 cf_clearance,避免「有 cookie 但 SPA 未触发跳转」的假过关 + * @param {import('puppeteer').Page} page + */ +async function clearCfClearanceCookies(page) { + const cookies = await page.cookies(); + for (const c of cookies) { + if (c.name !== 'cf_clearance') { + continue; + } + const host = (c.domain || '').replace(/^\./, ''); + if (host) { + await page.deleteCookie({ name: c.name, url: `https://${host}/` }).catch(() => {}); + } + await page.deleteCookie({ + name: c.name, + domain: c.domain, + path: c.path || '/', + }).catch(() => {}); + } +} + +/** + * reload 后检查业务 URL 是否就绪 + * @param {import('puppeteer').Page} page + * @param {(page: import('puppeteer').Page) => Promise} [waitForUrlSettled] + * @param {number} timeoutMs + * @param {object} antiBot + */ +async function tryPostCfRecoverViaReload(page, waitForUrlSettled, timeoutMs, antiBot) { + if (!needsPostCfBusinessUrl(antiBot) || isPostCfBusinessUrlReady(page, antiBot)) { + return isPostCfBusinessUrlReady(page, antiBot); + } + + console.log(`[CF] 业务 URL 未就绪 current=${page.url()},尝试 reload`); + try { + await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 45000) }); + if (waitForUrlSettled) { + await waitForUrlSettled(page).catch(() => {}); + } + return isPostCfBusinessUrlReady(page, antiBot); + } catch (reloadErr) { + console.warn('[CF] reload 失败:', reloadErr.message); + return false; + } +} + +/** + * 清除 cf_clearance 并 reload,使 Turnstile 重新出现 + * @param {import('puppeteer').Page} page + * @param {(page: import('puppeteer').Page) => Promise} [waitForUrlSettled] + * @param {number} timeoutMs + */ +async function invalidateStaleCfAndReload(page, waitForUrlSettled, timeoutMs) { + console.log('[CF] 清除 cf_clearance 并 reload,重新触发 Turnstile'); + await clearCfClearanceCookies(page); + await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 45000) }); + if (waitForUrlSettled) { + await waitForUrlSettled(page).catch(() => {}); + } + return detectCloudflare(page); +} + +/** + * session_reuse 快速返回前校验:有 postCf 要求时必须已到业务页 + * @param {import('puppeteer').Page} page + * @param {object} antiBot + * @param {(page: import('puppeteer').Page) => Promise} [waitForUrlSettled] + * @param {number} timeoutMs + * @param {number} started + * @param {boolean} cfDetected + */ +async function trySessionReuseFastPath(page, antiBot, waitForUrlSettled, timeoutMs, started, cfDetected) { + if (waitForUrlSettled) { + await waitForUrlSettled(page).catch(() => {}); + } + + if (!needsPostCfBusinessUrl(antiBot) || isPostCfBusinessUrlReady(page, antiBot)) { + return { + success: true, + code: null, + challengeType: null, + stage: 'session_reuse', + solverUsed: false, + elapsedMs: Date.now() - started, + cfDetected, + }; + } + + console.log(`[CF] cf_clearance/无挑战但业务页未就绪 url=${page.url()},降级重试`); + + if (await tryPostCfRecoverViaReload(page, waitForUrlSettled, timeoutMs, antiBot)) { + return { + success: true, + code: null, + challengeType: null, + stage: 'session_reuse_reload', + solverUsed: false, + elapsedMs: Date.now() - started, + cfDetected, + }; + } + + return null; +} + +/** + * @param {import('puppeteer').Page} page + * @param {{ turnstile?: boolean, solverFallback?: boolean, challengeTimeoutMs?: number, postCfReadyUrlContains?: string }} antiBot * @param {(page: import('puppeteer').Page) => Promise} [waitForUrlSettled] * @param {import('./session-store').StoredSession|null} [storedSession] */ @@ -19,36 +149,71 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store let cfState = await detectCloudflare(page); if (!cfState.blocked && cfState.hasCfClearance) { - return { - success: true, - code: null, - challengeType: null, - stage: 'session_reuse', - solverUsed: false, - elapsedMs: Date.now() - started, - cfDetected: false, - }; + const fast = await trySessionReuseFastPath( + page, antiBot, waitForUrlSettled, timeoutMs, started, false + ); + if (fast) { + return fast; + } + cfState = await invalidateStaleCfAndReload(page, waitForUrlSettled, timeoutMs); } if (!cfState.blocked) { - return { - success: true, - code: null, - challengeType: null, - stage: storedSession && isSessionLikelyValid(storedSession) ? 'session_reuse' : null, - solverUsed: false, - elapsedMs: Date.now() - started, - cfDetected: false, - }; + if (waitForUrlSettled) { + await waitForUrlSettled(page).catch(() => {}); + } + + if (!needsPostCfBusinessUrl(antiBot) || isPostCfBusinessUrlReady(page, antiBot)) { + return { + success: true, + code: null, + challengeType: null, + stage: storedSession && isSessionLikelyValid(storedSession) ? 'session_reuse' : null, + solverUsed: false, + elapsedMs: Date.now() - started, + cfDetected: false, + }; + } + + if (await tryPostCfRecoverViaReload(page, waitForUrlSettled, timeoutMs, antiBot)) { + return { + success: true, + code: null, + challengeType: null, + stage: 'post_nav_reload', + solverUsed: false, + elapsedMs: Date.now() - started, + cfDetected: false, + }; + } + + if (cfState.hasCfClearance) { + cfState = await invalidateStaleCfAndReload(page, waitForUrlSettled, timeoutMs); + } else { + return { + success: true, + code: null, + challengeType: null, + stage: 'pending_business_nav', + solverUsed: false, + elapsedMs: Date.now() - started, + cfDetected: false, + }; + } } if (storedSession && isSessionLikelyValid(storedSession)) { console.log('[CF] 会话有效但仍被拦截,尝试 reload 一次'); try { await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 45000) }); - if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {}); + if (waitForUrlSettled) { + await waitForUrlSettled(page).catch(() => {}); + } cfState = await detectCloudflare(page); if (!cfState.blocked) { + if (waitForUrlSettled) { + await waitForUrlSettled(page).catch(() => {}); + } return { success: true, code: null, @@ -69,7 +234,9 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store if (antiBot.turnstile !== false) { const builtIn = await waitForTurnstile(page, { timeoutMs, useBuiltInClick: true }); if (builtIn.success) { - if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {}); + if (waitForUrlSettled) { + await waitForUrlSettled(page).catch(() => {}); + } cfState = await detectCloudflare(page); if (!cfState.blocked) { return { @@ -96,7 +263,9 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store const apiWaitMs = Math.min(timeoutMs, 30000); await waitForTurnstile(page, { timeoutMs: apiWaitMs }); - if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {}); + if (waitForUrlSettled) { + await waitForUrlSettled(page).catch(() => {}); + } cfState = await detectCloudflare(page); if (!cfState.blocked) { @@ -138,4 +307,7 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store module.exports = { handleCloudflareChallenge, isCaptchaConfigured, + needsPostCfBusinessUrl, + isPostCfBusinessUrlReady, + clearCfClearanceCookies, }; diff --git a/patches/puppeteer-api/lib/post-cf-ready.js b/patches/puppeteer-api/lib/post-cf-ready.js new file mode 100644 index 0000000..66e8263 --- /dev/null +++ b/patches/puppeteer-api/lib/post-cf-ready.js @@ -0,0 +1,109 @@ +/** + * CF 过盾后等待业务页就绪(可选,由 antiBot.postCfReady* 驱动,未配置则跳过) + */ + +/** + * 采集当前页面诊断信息(auth 失败时便于排查) + * @param {import('puppeteer').Page} page + */ +async function collectPageDiagnostics(page) { + const url = page.url(); + const snapshot = await page.evaluate(() => ({ + url: window.location.href, + title: document.title || '', + bodyText: (document.body && document.body.innerText ? document.body.innerText : '').slice(0, 500), + elInputInner: document.querySelectorAll('.el-input__inner').length, + })).catch(() => null); + + return snapshot || { url, title: '', bodyText: '', elInputInner: 0 }; +} + +/** + * CF 成功后等待最终业务 URL / DOM 出现(Whatshub 等 SPA 短链跳转场景) + * + * @param {import('puppeteer').Page} page + * @param {object} antiBot normalizeAntiBot 返回值 + * @param {(page: import('puppeteer').Page) => Promise} [waitForUrlSettled] + * @param {number} [navigationTimeoutMs] + */ +async function awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled, navigationTimeoutMs = 60000) { + if (!antiBot || !antiBot.enabled) { + return; + } + + const urlContains = (antiBot.postCfReadyUrlContains || '').trim(); + const selector = (antiBot.postCfReadySelector || '').trim(); + if (urlContains === '' && selector === '') { + return; + } + + const timeoutMs = Math.max(5000, antiBot.postCfReadyTimeoutMs || 30000); + console.log( + `[CF] 等待业务页就绪 current=${page.url()} urlContains="${urlContains}" ` + + `selector="${selector}" timeout=${timeoutMs}ms` + ); + + try { + if (urlContains !== '') { + await page.waitForFunction( + (needle) => window.location.href.includes(needle), + { timeout: timeoutMs, polling: 200 }, + urlContains + ); + } + if (selector !== '') { + await page.waitForSelector(selector, { timeout: timeoutMs, visible: true }); + } + } catch (err) { + const retryMs = Math.min(15000, timeoutMs); + const needsRetry = urlContains !== '' && !page.url().includes(urlContains); + if (needsRetry) { + console.warn(`[CF] 业务页等待失败 url=${page.url()},最后尝试 reload (${retryMs}ms)`); + await page.reload({ waitUntil: 'domcontentloaded', timeout: navigationTimeoutMs }).catch(() => {}); + if (typeof waitForUrlSettled === 'function') { + await waitForUrlSettled(page).catch(() => {}); + } + if (urlContains !== '') { + await page.waitForFunction( + (needle) => window.location.href.includes(needle), + { timeout: retryMs, polling: 200 }, + urlContains + ); + } + if (selector !== '') { + await page.waitForSelector(selector, { timeout: retryMs, visible: true }); + } + } else { + err.pageDiagnostics = await collectPageDiagnostics(page); + throw err; + } + } + + const settleMs = antiBot.postCfSettleMs || 0; + if (settleMs > 0) { + await new Promise((resolve) => setTimeout(resolve, settleMs)); + } + + console.log(`[CF] 业务页就绪 url=${page.url()}`); +} + +/** + * 执行 authActions,失败时附带 page 诊断 + * @param {import('puppeteer').Page} page + * @param {object[]|undefined} authActions + * @param {Function} executeAuthActionsFn + */ +async function executeAuthActionsWithDiagnostics(page, authActions, executeAuthActionsFn) { + try { + await executeAuthActionsFn(page, authActions); + } catch (err) { + err.pageDiagnostics = await collectPageDiagnostics(page); + throw err; + } +} + +module.exports = { + collectPageDiagnostics, + awaitPostCfBusinessReady, + executeAuthActionsWithDiagnostics, +}; diff --git a/patches/puppeteer-api/lib/turnstile-handler.js b/patches/puppeteer-api/lib/turnstile-handler.js index 206e801..6545d12 100755 --- a/patches/puppeteer-api/lib/turnstile-handler.js +++ b/patches/puppeteer-api/lib/turnstile-handler.js @@ -1,7 +1,7 @@ /** * Turnstile 内置求解:等待 widget 自动完成 + token 轮询 */ -const { detectCloudflare, isTurnstileSolved } = require('./cf-detector'); +const { detectCloudflare, isTurnstileSolved, urlIndicatesCfChallenge } = require('./cf-detector'); /** * @typedef {Object} TurnstileResult @@ -22,18 +22,24 @@ async function waitForTurnstile(page, options = {}) { const started = Date.now(); const initial = await detectCloudflare(page); - if (!initial.blocked || initial.hasCfClearance) { - return { success: true, stage: 'already_clear', elapsedMs: Date.now() - started }; + const initialUrlChallenge = urlIndicatesCfChallenge(page.url()); + + if ((!initial.blocked && !initialUrlChallenge) || initial.hasCfClearance) { + if (initial.hasCfClearance && !initialUrlChallenge) { + return { success: true, stage: 'already_clear', elapsedMs: Date.now() - started }; + } + if (!initial.blocked && !initialUrlChallenge) { + return { success: true, stage: 'already_clear', elapsedMs: Date.now() - started }; + } } - // real browser 已开启 turnstile:true 内置点击;此处轮询等待结果 while (Date.now() - started < timeoutMs) { - if (await isTurnstileSolved(page)) { + if (await isTurnstileSolved(page) && !urlIndicatesCfChallenge(page.url())) { return { success: true, stage: 'built_in', elapsedMs: Date.now() - started }; } const state = await detectCloudflare(page); - if (!state.blocked) { + if (!state.blocked && !urlIndicatesCfChallenge(page.url()) && state.hasCfClearance) { return { success: true, stage: 'built_in', elapsedMs: Date.now() - started }; } diff --git a/patches/puppeteer-api/package.json b/patches/puppeteer-api/package.json index 57e6396..87857d8 100755 --- a/patches/puppeteer-api/package.json +++ b/patches/puppeteer-api/package.json @@ -7,6 +7,7 @@ "start": "node server.js" }, "dependencies": { + "dotenv": "^16.4.7", "express": "^4.18.2", "puppeteer": "^25.1.0", "puppeteer-extra": "^3.3.6", diff --git a/patches/puppeteer-api/server.js b/patches/puppeteer-api/server.js index 5b50a0b..a477128 100755 --- a/patches/puppeteer-api/server.js +++ b/patches/puppeteer-api/server.js @@ -1,3 +1,5 @@ +require('dotenv').config({ path: require('path').join(__dirname, '.env') }); + const express = require('express'); const fs = require('fs'); const { @@ -45,6 +47,10 @@ const { const { resolveSessionContext, touchSessionIfPresent } = require('./lib/session-context'); const { getProfileStats, profileExists } = require('./lib/profile-dir'); const { runUiPagination } = require('./lib/ui-pagination-runner'); +const { + awaitPostCfBusinessReady, + executeAuthActionsWithDiagnostics, +} = require('./lib/post-cf-ready'); const app = express(); app.use(express.json({ limit: '50mb' })); @@ -75,7 +81,11 @@ function sendTaskError(res, error, extra = {}) { } const errType = classifyPuppeteerError(error); console.error(`[${errType}]`, error.message); - return res.status(500).json({ success: false, error: error.message, ...extra }); + const payload = { success: false, error: error.message, ...extra }; + if (error.pageDiagnostics) { + payload.page = error.pageDiagnostics; + } + return res.status(500).json(payload); } async function navigateToPage(page, url) { @@ -524,7 +534,11 @@ app.post('/api/auth-and-intercept', async (req, res) => { await seedShareTokenCookie(page, finalPageUrl); } - await executeAuthActions(page, authActions); + if (antiBot.enabled) { + await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled); + } + + await executeAuthActionsWithDiagnostics(page, authActions, executeAuthActions); await interceptor.waitForApis(interceptTimeout); const rawCookies = await page.cookies(); @@ -758,6 +772,10 @@ app.post('/api/ui-pagination', async (req, res) => { touchSessionIfPresent(antiBot.sessionKey); } + if (antiBot.enabled) { + await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled); + } + const paginationResult = await runUiPagination(page, { apiUrl, nextBtnSelector, @@ -874,7 +892,11 @@ app.post('/api/auth-intercept-and-paginate', async (req, res) => { touchSessionIfPresent(antiBot.sessionKey); } - await executeAuthActions(page, authActions); + if (antiBot.enabled) { + await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled); + } + + await executeAuthActionsWithDiagnostics(page, authActions, executeAuthActions); await interceptor.waitForApis(interceptTimeout); const rawCookies = await page.cookies();