修改whatshub工单爬虫

2026-06-29 04:54:41 +08:00
parent 6130125427
commit 9f2e904fab
14 changed files with 1254 additions and 147 deletions
@@ -41,6 +41,26 @@ class AntiBotConfigBuilder
            'sessionKey'         => $sessionKey,
            'challengeTimeoutMs' => 60000,
            'sessionTtlMinutes'  => $ttlMinutes,
+        ] + self::postCfReadyExtras($ticketType);
+    }
+
+    /**
+     * 按工单类型附加 CF 过盾后的业务页就绪等待（仅 Whatshub 等需要的类型）
+     *
+     * @return array<string, mixed>
+     */
+    private static function postCfReadyExtras(string $ticketType): array
+    {
+        if ($ticketType !== 'whatshub') {
+            return [];
+        }
+
+        return [
+            'postCfReadyUrlContains' => 'work-order-sharing',
+            'postCfReadySelector'    => '.el-input__inner',
+            'postCfReadyTimeoutMs'   => 30000,
+            'postCfSettleMs'         => 500,
+            'postCfSpaWaitMs'        => 4000,
        ];
    }

@@ -7,11 +7,12 @@ namespace app\common\library\scrm\spider;
 use app\common\library\scrm\AbstractScrmSpider;
 use app\common\library\scrm\AntiBotConfigBuilder;
 use app\common\library\scrm\UnifiedScrmData;
+use app\common\service\SplitTicketSyncLogger;

 /**
- * Whatshub 云控蜘蛛（Real Browser + Turnstile 过盾 + UI 翻页）
+ * Whatshub 云控蜘蛛
 *
- * 时序：Cloudflare Turnstile → 密码弹窗 authActions → API 拦截 → UI 翻页
+ * 时序：detailByShareCode API 直抓（优先）→ 失败则 Real Browser + Turnstile + UI 翻页
 */
 class WhatshubSpider extends AbstractScrmSpider
 {
@@ -21,9 +22,14 @@ class WhatshubSpider extends AbstractScrmSpider
    /** 详情/统计 API 路径 */
    private const API_DETAILS = '/api/whatshub-counter/workShare/open/statistics';

+    /** shareCode + password 直连接口（一次返回全量号码，无需翻页） */
+    private const API_DETAIL_BY_SHARE_CODE = '/api/whatshub-counter/workShare/open/detailByShareCode';
+
    /** 默认每页条数 */
    private const DEFAULT_PER_PAGE_COUNT = 20;

+    private const API_FETCH_TIMEOUT_SEC = 20;
+
    private string $pageUrl;

    private string $account;
@@ -45,6 +51,178 @@ class WhatshubSpider extends AbstractScrmSpider
        $this->unifiedData = new UnifiedScrmData();
    }

+    /**
+     * API 优先：detailByShareCode 直抓；失败则回退 Node 爬虫
+     */
+    public function run(): UnifiedScrmData
+    {
+        $apiResult = $this->tryFetchViaShareCodeApi();
+        if ($apiResult instanceof UnifiedScrmData) {
+            SplitTicketSyncLogger::log('spider', 'whatshub api path ok', [
+                'count' => count($apiResult->numbers),
+            ]);
+            return $apiResult;
+        }
+
+        SplitTicketSyncLogger::log('spider', 'whatshub api skipped or failed, fallback node', [
+            'pageUrl' => $this->pageUrl,
+        ]);
+
+        return parent::run();
+    }
+
+    /**
+     * 从短链 pageUrl 解析 shareCode，例如 /m/KMYBBInp2066/1 → KMYBBInp2066
+     */
+    private function extractShareCodeFromPageUrl(): ?string
+    {
+        $path = (string) parse_url($this->pageUrl, PHP_URL_PATH);
+        if ($path === '') {
+            return null;
+        }
+        if (preg_match('#/m/([^/]+)/#', $path, $matches)) {
+            return $matches[1];
+        }
+
+        return null;
+    }
+
+    /**
+     * 拼接 detailByShareCode 完整 GET URL
+     */
+    private function buildShareCodeApiUrl(string $shareCode): ?string
+    {
+        $parsed = parse_url($this->pageUrl);
+        $scheme = isset($parsed['scheme']) ? (string) $parsed['scheme'] : 'https';
+        $host = isset($parsed['host']) ? (string) $parsed['host'] : '';
+        if ($host === '') {
+            return null;
+        }
+
+        $query = http_build_query([
+            'shareCode' => $shareCode,
+            'password'  => $this->password,
+        ]);
+
+        return $scheme . '://' . $host . self::API_DETAIL_BY_SHARE_CODE . '?' . $query;
+    }
+
+    /**
+     * 尝试通过 shareCode API 抓取；成功返回 UnifiedScrmData，否则返回 null
+     */
+    private function tryFetchViaShareCodeApi(): ?UnifiedScrmData
+    {
+        if ($this->password === '') {
+            SplitTicketSyncLogger::log('spider', 'whatshub api skip: empty password', []);
+            return null;
+        }
+
+        $shareCode = $this->extractShareCodeFromPageUrl();
+        if ($shareCode === null || $shareCode === '') {
+            SplitTicketSyncLogger::log('spider', 'whatshub api skip: cannot parse shareCode', [
+                'pageUrl' => $this->pageUrl,
+            ]);
+            return null;
+        }
+
+        $apiUrl = $this->buildShareCodeApiUrl($shareCode);
+        if ($apiUrl === null) {
+            SplitTicketSyncLogger::log('spider', 'whatshub api skip: invalid pageUrl host', [
+                'pageUrl' => $this->pageUrl,
+            ]);
+            return null;
+        }
+
+        SplitTicketSyncLogger::log('spider', 'whatshub api request', [
+            'url' => preg_replace('/password=[^&]+/', 'password=***', $apiUrl),
+        ]);
+
+        $ch = curl_init($apiUrl);
+        if ($ch === false) {
+            SplitTicketSyncLogger::log('spider', 'whatshub api curl init failed', []);
+            return null;
+        }
+
+        curl_setopt_array($ch, [
+            CURLOPT_RETURNTRANSFER => true,
+            CURLOPT_CONNECTTIMEOUT => 5,
+            CURLOPT_TIMEOUT        => self::API_FETCH_TIMEOUT_SEC,
+            CURLOPT_HTTPHEADER     => [
+                'Accept: application/json',
+                'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+            ],
+        ]);
+
+        $response = curl_exec($ch);
+        $httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
+        $curlErr = curl_error($ch);
+        curl_close($ch);
+
+        if ($response === false || $curlErr !== '') {
+            SplitTicketSyncLogger::log('spider', 'whatshub api curl failed', [
+                'error' => $curlErr,
+            ]);
+            return null;
+        }
+
+        if ($httpCode !== 200) {
+            SplitTicketSyncLogger::log('spider', 'whatshub api http error', [
+                'httpCode' => $httpCode,
+            ]);
+            return null;
+        }
+
+        $payload = json_decode((string) $response, true);
+        if (!is_array($payload)) {
+            SplitTicketSyncLogger::log('spider', 'whatshub api invalid json', []);
+            return null;
+        }
+
+        if ((int) ($payload['code'] ?? 0) !== 200) {
+            SplitTicketSyncLogger::log('spider', 'whatshub api business error', [
+                'code' => $payload['code'] ?? null,
+                'msg'  => $payload['msg'] ?? '',
+            ]);
+            return null;
+        }
+
+        if (!isset($payload['data']) || !is_array($payload['data'])) {
+            SplitTicketSyncLogger::log('spider', 'whatshub api invalid data field', []);
+            return null;
+        }
+
+        return $this->parseShareCodeApiResponse($payload);
+    }
+
+    /**
+     * 将 detailByShareCode 响应映射为 UnifiedScrmData
+     *
+     * @param array<string, mixed> $payload
+     */
+    private function parseShareCodeApiResponse(array $payload): UnifiedScrmData
+    {
+        $unifiedData = new UnifiedScrmData();
+        $todayNewSum = 0;
+
+        foreach ($payload['data'] as $item) {
+            if (!is_array($item) || empty($item['account'])) {
+                continue;
+            }
+
+            $number = (string) $item['account'];
+            $isOnline = isset($item['isOnline']) && (int) $item['isOnline'] === 1;
+            $dayNewFans = (int) ($item['dayNewFans'] ?? 0);
+
+            $unifiedData->addNumber($number, $isOnline, $dayNewFans);
+            $todayNewSum += $dayNewFans;
+        }
+
+        $unifiedData->todayNewCount = $todayNewSum;
+        $unifiedData->total = count($unifiedData->numbers);
+
+        return $unifiedData;
+    }
+
    /** @return array<string, mixed> */
    protected function getSpiderConfig(): array
    {