修改whatshub工单爬虫

_php_code/AbstractScrmSpider.class.php

@@ -75,11 +75,23 @@ abstract class AbstractScrmSpider
     const MODE_FETCH = 'fetch';      // 极速并发拉取 (推荐默认)
     const MODE_UI    = 'ui_click';   // 强制 UI 点击下一页
 
+    /** CLI 测试时输出 Node 请求/失败详情 */
+    protected $verbose = false;
+
     protected $nodeHost;
 
     public function __construct($nodeHost = 'http://127.0.0.1:3001')
     {
-        $this->nodeHost = $nodeHost;
+        $this->nodeHost = rtrim($nodeHost, '/');
+    }
+
+    /**
+     * 开启调试输出（测试脚本可调用）
+     */
+    public function setVerbose($verbose = true)
+    {
+        $this->verbose = (bool) $verbose;
+        return $this;
     }
 
     /** ================= 子类必须实现的契约 ================= **/
@@ -97,7 +109,7 @@ abstract class AbstractScrmSpider
     public function run()
     {
         $config = $this->getSpiderConfig();
-        
+
         $listApi = $config['listApi'];
         $detailApi = $config['detailApi'] ?? null; // detail 接口是可选的
         $countApi  = $config['countApi'] ?? null;   // 有些在线客服的总数通过单独调用接口获取
@@ -106,13 +118,14 @@ abstract class AbstractScrmSpider
         $apiUrlsToIntercept = [$listApi];
         if ($detailApi) { $apiUrlsToIntercept[] = $detailApi; }
         if ($countApi)  { $apiUrlsToIntercept[] = $countApi; }
-// var_dump($apiUrlsToIntercept);
-// dump([
-//             'pageUrl'     => $config['pageUrl'],
-//             'apiUrls'     => $apiUrlsToIntercept,
-//             'authActions' => $config['authActions']
-//         ]);
+
         $antiBot = $config['antiBot'] ?? null;
+        $mode = $config['paginationMode'] ?? self::MODE_FETCH;
+
+        // antiBot + UI 翻页：单 Browser 合并路径（与生产 SplitTicket 同步一致）
+        if ($this->shouldUseUnifiedBrowserPath($antiBot, $mode)) {
+            return $this->runUnifiedUiPath($config, $antiBot, $apiUrlsToIntercept, $listApi, $detailApi, $countApi);
+        }
 
         // 【阶段一】：初始化并首屏拦截
         $initResult = $this->requestNode('/api/auth-and-intercept', [
@@ -123,17 +136,20 @@ abstract class AbstractScrmSpider
         ], $this->resolveAuthInterceptTimeout($antiBot));
 
         if (empty($initResult['success'])) {
-            $cfCode = (string) ($initResult['code'] ?? '');
-            if ($cfCode === 'CF_TURNSTILE_FAILED') {
-                throw new Exception('Cloudflare Turnstile 验证失败: ' . ($initResult['stage'] ?? 'timeout'));
-            }
-            throw new Exception("初始化失败: " . ($initResult['error'] ?? '未知'));
+            throw new Exception($this->formatNodeFailure('初始化失败', $initResult));
         }
 
         $interceptedApis = $initResult['interceptedApis'];
         $cookies         = $initResult['cookies'];
         $finalPageUrl    = $initResult['finalPageUrl'] ?? $config['pageUrl'];
 
+        $this->logDebug('auth-and-intercept ok', [
+            'finalPageUrl'  => $finalPageUrl,
+            'intercepted'   => array_keys($interceptedApis),
+            'cf'            => $initResult['cf'] ?? null,
+            'browserReused' => $initResult['browserReused'] ?? null,
+        ]);
+
         // 必须拦截到 List 接口，否则无法继续
         if (!isset($interceptedApis[$listApi])) {
             throw new Exception("致命错误：未能拦截到必须的列表接口 [{$listApi}]");
@@ -229,6 +245,140 @@ abstract class AbstractScrmSpider
         return $unifiedData;
     }
 
+    /**
+     * 单 Browser 路径：auth + 拦截 + UI 翻页（需 antiBot.sessionKey）
+     */
+    protected function runUnifiedUiPath($config, $antiBot, $apiUrlsToIntercept, $listApi, $detailApi, $countApi)
+    {
+        $uiConfig = $this->getUiPaginationConfig();
+        $sessionKey = is_array($antiBot) ? (string) ($antiBot['sessionKey'] ?? '') : '';
+
+        $this->logDebug('unified browser path', [
+            'sessionKey' => $sessionKey,
+            'listApi'    => $listApi,
+        ]);
+
+        $mergedResult = $this->requestNode('/api/auth-intercept-and-paginate', [
+            'pageUrl'     => $config['pageUrl'],
+            'apiUrls'     => $apiUrlsToIntercept,
+            'authActions' => $config['authActions'] ?? [],
+            'antiBot'     => $antiBot,
+            'pagination'  => [
+                'apiUrl'          => $listApi,
+                'nextBtnSelector' => $uiConfig['nextBtnSelector'] ?? '',
+                'waitMs'          => $uiConfig['waitMs'] ?? 2000,
+                'clicksToPerform' => 9999,
+            ],
+        ], 1200);
+
+        if (empty($mergedResult['success'])) {
+            throw new Exception($this->formatNodeFailure('合并同步失败', $mergedResult));
+        }
+
+        $this->logDebug('auth-intercept-and-paginate ok', [
+            'intercepted'   => array_keys($mergedResult['interceptedApis'] ?? []),
+            'extraPages'    => count($mergedResult['extraPages'] ?? []),
+            'cf'            => $mergedResult['cf'] ?? null,
+            'browserReused' => $mergedResult['browserReused'] ?? null,
+            'finalPageUrl'  => $mergedResult['finalPageUrl'] ?? null,
+        ]);
+
+        $interceptedApis = $mergedResult['interceptedApis'] ?? [];
+        if (!isset($interceptedApis[$listApi])) {
+            throw new Exception("致命错误：未能拦截到必须的列表接口 [{$listApi}]");
+        }
+
+        $detailData = $detailApi && isset($interceptedApis[$detailApi])
+            ? $interceptedApis[$detailApi]['data'] : null;
+        $countData = $countApi && isset($interceptedApis[$countApi])
+            ? $interceptedApis[$countApi]['data'] : null;
+
+        $listApiNode = $interceptedApis[$listApi];
+        $allListPagesData = [$listApiNode['data']];
+        if (!empty($mergedResult['extraPages']) && is_array($mergedResult['extraPages'])) {
+            foreach ($mergedResult['extraPages'] as $pageData) {
+                $allListPagesData[] = $pageData;
+            }
+        }
+
+        $this->extractListTotalPages($listApiNode['data'], $countData);
+
+        return $this->parseToUnifiedData($detailData, $allListPagesData);
+    }
+
+    /**
+     * antiBot 启用且配置了 sessionKey 时，UI 翻页走合并 Browser 路径
+     */
+    protected function shouldUseUnifiedBrowserPath($antiBot, $mode)
+    {
+        if ($mode !== self::MODE_UI) {
+            return false;
+        }
+        if (!is_array($antiBot) || empty($antiBot['enabled'])) {
+            return false;
+        }
+        return !empty($antiBot['sessionKey']);
+    }
+
+    /**
+     * 格式化 Node 失败信息，附带 CF / 落地页等诊断字段
+     */
+    protected function formatNodeFailure($prefix, $result)
+    {
+        $cfCode = (string) ($result['code'] ?? '');
+        if ($cfCode === 'CF_TURNSTILE_FAILED') {
+            return 'Cloudflare Turnstile 验证失败: ' . ($result['stage'] ?? 'timeout');
+        }
+
+        $parts = [$prefix . ': ' . ($result['error'] ?? '未知')];
+
+        if (!empty($result['finalPageUrl'])) {
+            $parts[] = 'finalPageUrl=' . $result['finalPageUrl'];
+        }
+        if (!empty($result['page']) && is_array($result['page'])) {
+            $page = $result['page'];
+            if (!empty($page['url'])) {
+                $parts[] = 'pageUrl=' . $page['url'];
+            }
+            if (isset($page['elInputInner'])) {
+                $parts[] = 'elInputInner=' . $page['elInputInner'];
+            }
+        }
+        if (!empty($result['cf']) && is_array($result['cf'])) {
+            $cf = $result['cf'];
+            $cfBits = [];
+            if (isset($cf['cfDetected'])) {
+                $cfBits[] = 'cfDetected=' . ($cf['cfDetected'] ? 'true' : 'false');
+            }
+            if (!empty($cf['turnstileStage'])) {
+                $cfBits[] = 'turnstileStage=' . $cf['turnstileStage'];
+            }
+            if (isset($cf['solverUsed'])) {
+                $cfBits[] = 'solverUsed=' . ($cf['solverUsed'] ? 'true' : 'false');
+            }
+            if (isset($cf['elapsedMs'])) {
+                $cfBits[] = 'cfElapsedMs=' . $cf['elapsedMs'];
+            }
+            if (!empty($cfBits)) {
+                $parts[] = implode(', ', $cfBits);
+            }
+        }
+
+        return implode(' | ', $parts);
+    }
+
+    protected function logDebug($message, $context = [])
+    {
+        if (!$this->verbose) {
+            return;
+        }
+        $line = '[Spider] ' . $message;
+        if (!empty($context)) {
+            $line .= ' ' . json_encode($context, JSON_UNESCAPED_UNICODE);
+        }
+        echo $line . "\n";
+    }
+
     /**
      * Real Browser + Turnstile 首屏耗时较长，单独放宽 auth-and-intercept 超时
      *
@@ -244,17 +394,76 @@ abstract class AbstractScrmSpider
 
     private function requestNode($endpoint, $payload, $timeout = 60)
     {
-        $ch = curl_init($this->nodeHost . $endpoint);
+        $maxAttempts = 3;
+        $lastDecoded = [];
+        for ($attempt = 1; $attempt <= $maxAttempts; $attempt++) {
+            $lastDecoded = $this->doRequestNode($endpoint, $payload, $timeout, $attempt);
+            if (!empty($lastDecoded['success'])) {
+                return $lastDecoded;
+            }
+            $error = (string) ($lastDecoded['error'] ?? '');
+            if (!$this->shouldRetryNodeResponse($lastDecoded, $error) || $attempt >= $maxAttempts) {
+                return $lastDecoded;
+            }
+            $this->logDebug('node retry', [
+                'endpoint' => $endpoint,
+                'attempt'  => $attempt + 1,
+                'error'    => $error,
+            ]);
+            sleep($attempt);
+        }
+        return $lastDecoded;
+    }
+
+    private function doRequestNode($endpoint, $payload, $timeout, $attempt)
+    {
+        $url = $this->nodeHost . $endpoint;
+        $this->logDebug('node request', [
+            'url'     => $url,
+            'timeout' => $timeout,
+            'attempt' => $attempt,
+        ]);
+
+        $ch = curl_init($url);
         curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
         curl_setopt($ch, CURLOPT_POST, true);
-        curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($payload));
+        curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($payload, JSON_UNESCAPED_UNICODE));
         curl_setopt($ch, CURLOPT_HTTPHEADER, ['Content-Type: application/json']);
         curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5);
         curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
         $response = curl_exec($ch);
-        if (curl_errno($ch)) throw new Exception(curl_error($ch));
+        $httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
+        if (curl_errno($ch)) {
+            $err = curl_error($ch);
+            curl_close($ch);
+            throw new Exception($err);
+        }
         curl_close($ch);
-        return json_decode($response, true);
+
+        $decoded = json_decode((string) $response, true);
+        if (!is_array($decoded)) {
+            return ['success' => false, 'error' => 'Node 返回非 JSON', 'httpCode' => $httpCode];
+        }
+        if ($httpCode === 503) {
+            $decoded['success'] = false;
+            $decoded['error'] = (string) ($decoded['error'] ?? '服务繁忙');
+            $decoded['httpCode'] = 503;
+        }
+        return $decoded;
+    }
+
+    private function shouldRetryNodeResponse($decoded, $error)
+    {
+        if ((int) ($decoded['httpCode'] ?? 0) === 503) {
+            return true;
+        }
+        $lower = strtolower($error);
+        foreach (['排队超时', 'queue_timeout', 'timeout', 'net::', 'navigation', 'connection'] as $needle) {
+            if ($needle !== '' && strpos($lower, $needle) !== false) {
+                return true;
+            }
+        }
+        return false;
     }
     
     /**