优化爬虫，增加缓存，线程池，保存浏览器用户信息

2026-06-20 15:49:40 +08:00
parent e95f4a227c
commit 09bfa9ae01
26 changed files with 1584 additions and 454 deletions
@@ -68,13 +68,18 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface
        }

        $antiBot = $config['antiBot'] ?? null;
+        $mode = $config['paginationMode'] ?? self::MODE_FETCH;
+
+        if ($this->shouldUseUnifiedBrowserPath($antiBot, $mode)) {
+            return $this->runUnifiedUiPath($config, $antiBot, $apiUrlsToIntercept, $listApi, $detailApi, $countApi);
+        }

        $initResult = $this->requestNode('/api/auth-and-intercept', [
            'pageUrl'     => $config['pageUrl'],
            'apiUrls'     => $apiUrlsToIntercept,
            'authActions' => $config['authActions'] ?? [],
            'antiBot'     => $antiBot,
-        ]);
+        ], $this->resolveAuthInterceptTimeout($antiBot));

        if (empty($initResult['success'])) {
            $cfCode = (string) ($initResult['code'] ?? '');
@@ -95,8 +100,10 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface
        $interceptedApis = $initResult['interceptedApis'];
        $finalPageUrl = (string) ($initResult['finalPageUrl'] ?? ($config['pageUrl'] ?? ''));
        SplitTicketSyncLogger::log('spider', 'auth-and-intercept ok', [
-            'intercepted'  => array_keys($interceptedApis),
-            'finalPageUrl' => $finalPageUrl,
+            'intercepted'   => array_keys($interceptedApis),
+            'finalPageUrl'  => $finalPageUrl,
+            'cfStage'       => $initResult['cf']['turnstileStage'] ?? ($initResult['cf']['stage'] ?? null),
+            'browserReused' => $initResult['browserReused'] ?? null,
        ]);
        $cookies = $initResult['cookies'];

@@ -113,7 +120,6 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface
        $allListPagesData = [$listApiNode['data']];

        $totalPages = $this->extractListTotalPages($listApiNode['data'], $countData);
-        $mode = $config['paginationMode'] ?? self::MODE_FETCH;
        SplitTicketSyncLogger::log('spider', 'pagination plan', [
            'totalPages' => $totalPages,
            'mode'       => $mode,
@@ -182,6 +188,124 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface
        return $result;
    }

+    /**
+     * 单 Browser 路径：auth + 拦截 + UI 翻页（需 antiBot.sessionKey）
+     *
+     * @param array<string, mixed> $config
+     * @param array<string, mixed>|null $antiBot
+     * @param list<string> $apiUrlsToIntercept
+     */
+    private function runUnifiedUiPath(
+        array $config,
+        ?array $antiBot,
+        array $apiUrlsToIntercept,
+        string $listApi,
+        ?string $detailApi,
+        ?string $countApi
+    ): UnifiedScrmData {
+        $uiConfig = $this->getUiPaginationConfig();
+        $sessionKey = is_array($antiBot) ? (string) ($antiBot['sessionKey'] ?? '') : '';
+
+        SplitTicketSyncLogger::log('spider', 'unified browser path', [
+            'sessionKey' => $sessionKey,
+            'listApi'    => $listApi,
+        ]);
+
+        $mergedResult = $this->requestNode('/api/auth-intercept-and-paginate', [
+            'pageUrl'     => $config['pageUrl'],
+            'apiUrls'     => $apiUrlsToIntercept,
+            'authActions' => $config['authActions'] ?? [],
+            'antiBot'     => $antiBot,
+            'pagination'  => [
+                'apiUrl'          => $listApi,
+                'nextBtnSelector' => $uiConfig['nextBtnSelector'] ?? '',
+                'waitMs'          => $uiConfig['waitMs'] ?? 2000,
+                'clicksToPerform' => 9999,
+            ],
+        ], 1200);
+
+        if (empty($mergedResult['success'])) {
+            $cfCode = (string) ($mergedResult['code'] ?? '');
+            SplitTicketSyncLogger::log('spider', 'auth-intercept-and-paginate failed', [
+                'error'         => $mergedResult['error'] ?? '未知',
+                'code'          => $cfCode !== '' ? $cfCode : null,
+                'stage'         => $mergedResult['stage'] ?? null,
+                'cf'            => $mergedResult['cf'] ?? null,
+                'sessionKey'    => $sessionKey,
+            ]);
+            if ($cfCode === 'CF_TURNSTILE_FAILED') {
+                throw new Exception('Cloudflare Turnstile 验证失败: ' . ($mergedResult['stage'] ?? 'timeout'));
+            }
+            throw new Exception('合并同步失败: ' . (string) ($mergedResult['error'] ?? '未知'));
+        }
+
+        SplitTicketSyncLogger::log('spider', 'auth-intercept-and-paginate ok', [
+            'intercepted'   => array_keys($mergedResult['interceptedApis'] ?? []),
+            'extraPages'    => count($mergedResult['extraPages'] ?? []),
+            'cfStage'       => $mergedResult['cf']['turnstileStage'] ?? ($mergedResult['cf']['stage'] ?? null),
+            'browserReused' => $mergedResult['browserReused'] ?? null,
+            'sessionKey'    => $sessionKey,
+        ]);
+
+        $interceptedApis = $mergedResult['interceptedApis'] ?? [];
+        if (!isset($interceptedApis[$listApi])) {
+            throw new Exception("致命错误：未能拦截到必须的列表接口 [{$listApi}]");
+        }
+
+        $detailData = $detailApi && isset($interceptedApis[$detailApi])
+            ? $interceptedApis[$detailApi]['data'] : null;
+        $countData = $countApi && isset($interceptedApis[$countApi])
+            ? $interceptedApis[$countApi]['data'] : null;
+
+        $listApiNode = $interceptedApis[$listApi];
+        $allListPagesData = [$listApiNode['data']];
+        if (!empty($mergedResult['extraPages']) && is_array($mergedResult['extraPages'])) {
+            foreach ($mergedResult['extraPages'] as $pageData) {
+                $allListPagesData[] = $pageData;
+            }
+        }
+
+        $this->extractListTotalPages($listApiNode['data'], $countData);
+
+        $result = $this->parseToUnifiedData($detailData, $allListPagesData);
+        SplitTicketSyncLogger::log('spider', 'run done', [
+            'todayNewCount' => $result->todayNewCount,
+            'totalOnline'   => $result->totalOnline,
+            'totalOffline'  => $result->totalOffline,
+            'total'         => $result->total,
+            'numberCount'   => count($result->numbers),
+            'unifiedPath'   => true,
+        ]);
+        return $result;
+    }
+
+    /**
+     * @param array<string, mixed>|null $antiBot
+     */
+    protected function shouldUseUnifiedBrowserPath(?array $antiBot, string $mode): bool
+    {
+        if ($mode !== self::MODE_UI) {
+            return false;
+        }
+        if (!is_array($antiBot) || empty($antiBot['enabled'])) {
+            return false;
+        }
+        return !empty($antiBot['sessionKey']);
+    }
+
+    /**
+     * Real Browser + Turnstile 首屏耗时较长，单独放宽 auth-and-intercept 超时
+     *
+     * @param array<string, mixed>|null $antiBot
+     */
+    protected function resolveAuthInterceptTimeout(?array $antiBot): int
+    {
+        if (!is_array($antiBot) || empty($antiBot['enabled'])) {
+            return 120;
+        }
+        return 180;
+    }
+
    /**
     * @param array<string, mixed> $payload
     * @return array<string, mixed>
@@ -0,0 +1,84 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\common\library\scrm;
+
+use think\Config;
+
+/**
+ * 云控蜘蛛 antiBot 配置构建器
+ *
+ * 约定：sessionKey = "{ticketType}:{host}"；profile=real 当 ticket_type 在 site 配置列表中
+ */
+class AntiBotConfigBuilder
+{
+    /**
+     * 为指定工单类型与落地页 URL 构建 Node antiBot 配置；未启用类型返回 null
+     *
+     * @return array<string, mixed>|null
+     */
+    public static function build(string $ticketType, string $pageUrl): ?array
+    {
+        $enabledTypes = self::getEnabledTypes();
+        if (!in_array($ticketType, $enabledTypes, true)) {
+            return null;
+        }
+
+        $host = (string) parse_url($pageUrl, PHP_URL_HOST);
+        if ($host === '') {
+            return null;
+        }
+
+        $sessionKey = $ticketType . ':' . $host;
+        $ttlMinutes = self::getSessionTtlMinutes();
+
+        return [
+            'enabled'            => true,
+            'profile'            => 'real',
+            'turnstile'          => true,
+            'solverFallback'     => true,
+            'sessionKey'         => $sessionKey,
+            'challengeTimeoutMs' => 60000,
+            'sessionTtlMinutes'  => $ttlMinutes,
+        ];
+    }
+
+    /**
+     * 解析 sessionKey（供 cron 分组调度使用）
+     */
+    public static function resolveSessionKey(string $ticketType, string $pageUrl): string
+    {
+        $host = (string) parse_url($pageUrl, PHP_URL_HOST);
+        return $ticketType . ':' . ($host !== '' ? $host : 'unknown');
+    }
+
+    /**
+     * 是否对该工单类型启用 antiBot
+     */
+    public static function isEnabledForType(string $ticketType): bool
+    {
+        return in_array($ticketType, self::getEnabledTypes(), true);
+    }
+
+    /**
+     * @return list<string>
+     */
+    public static function getEnabledTypes(): array
+    {
+        $raw = (string) Config::get('site.split_scrm_antibot_types');
+        if ($raw === '') {
+            return ['whatshub', 'chatknow'];
+        }
+        $parts = array_map('trim', explode(',', $raw));
+        return array_values(array_filter($parts, static function (string $v): bool {
+            return $v !== '';
+        }));
+    }
+
+    public static function getSessionTtlMinutes(): int
+    {
+        $minutes = (int) Config::get('site.split_scrm_session_ttl_minutes');
+        return $minutes > 0 ? $minutes : 120;
+    }
+}
@@ -5,6 +5,7 @@ declare(strict_types=1);
 namespace app\common\library\scrm\spider;

 use app\common\library\scrm\AbstractScrmSpider;
+use app\common\library\scrm\AntiBotConfigBuilder;
 use app\common\library\scrm\UnifiedScrmData;

 /**
@@ -48,6 +49,7 @@ class ChatknowSpider extends AbstractScrmSpider
            'authActions'    => [
                ['type' => 'wait', 'ms' => 4000],
            ],
+            'antiBot'        => AntiBotConfigBuilder::build('chatknow', $this->pageUrl),
        ];
    }

@@ -5,6 +5,7 @@ declare(strict_types=1);
 namespace app\common\library\scrm\spider;

 use app\common\library\scrm\AbstractScrmSpider;
+use app\common\library\scrm\AntiBotConfigBuilder;
 use app\common\library\scrm\UnifiedScrmData;

 /**
@@ -47,8 +48,6 @@ class WhatshubSpider extends AbstractScrmSpider
    /** @return array<string, mixed> */
    protected function getSpiderConfig(): array
    {
-        $host = (string) parse_url($this->pageUrl, PHP_URL_HOST);
-
        return [
            'pageUrl'        => $this->pageUrl,
            'listApi'        => self::API_LIST,
@@ -62,15 +61,7 @@ class WhatshubSpider extends AbstractScrmSpider
                ['type' => 'vue_click', 'selector' => '.vxe-button-group .theme--primary'],
                ['type' => 'wait', 'ms' => 2200],
            ],
-            // Whatshub 专用：Real Browser + Turnstile + Captcha API 兜底 + 会话复用
-            'antiBot'        => [
-                'enabled'            => true,
-                'profile'            => 'real',
-                'turnstile'          => true,
-                'solverFallback'     => true,
-                'sessionKey'         => 'whatshub:' . $host,
-                'challengeTimeoutMs' => 60000,
-            ],
+            'antiBot'        => AntiBotConfigBuilder::build('whatshub', $this->pageUrl),
        ];
    }

@@ -5,6 +5,7 @@ declare(strict_types=1);
 namespace app\common\service;

 use app\admin\model\split\Ticket;
+use app\common\library\scrm\AntiBotConfigBuilder;
 use app\common\library\scrm\UnifiedScrmData;
 use think\Db;
 use think\Exception;
@@ -107,6 +108,7 @@ class SplitTicketSyncService
        $list = $query->order('sync_time', 'asc')->order('id', 'asc')
            ->limit($maxPerRun * 5)
            ->select();
+        $list = $this->sortTicketsBySessionKey($list);

        $candidateCount = count($list);
        SplitTicketSyncLogger::logCron('scan start', [
@@ -115,6 +117,7 @@ class SplitTicketSyncService
            'autoSyncTypes'  => $autoSyncTypes,
            'failThreshold'  => $failThreshold,
            'nodeQueue'      => SplitNodeHealthService::getQueueSnapshot(),
+            'groupedBy'      => 'sessionKey',
        ]);
        SplitTicketSyncLogger::log('cron', 'scan start', [
            'candidateCount' => $candidateCount,
@@ -134,6 +137,7 @@ class SplitTicketSyncService
            $ticketId = (int) $ticket['id'];
            $ticketUrl = (string) $ticket['ticket_url'];
            $ticketType = (string) $ticket['ticket_type'];
+            $sessionKey = AntiBotConfigBuilder::resolveSessionKey($ticketType, $ticketUrl);

            if ($count >= $maxPerRun) {
                $reason = '本轮处理上限已满';
@@ -183,6 +187,7 @@ class SplitTicketSyncService
                'ticketId'   => $ticketId,
                'ticketType' => $ticketType,
                'ticketUrl'  => $ticketUrl,
+                'sessionKey' => $sessionKey,
                'success'    => !empty($result['success']),
                'message'    => (string) ($result['message'] ?? ''),
            ];
@@ -190,8 +195,8 @@ class SplitTicketSyncService
        }

        $candidateIds = array_map(static function ($row) {
-            return (int) $row['id'];
-        }, $list instanceof \think\Collection ? $list->all() : (array) $list);
+            return (int) (is_array($row) ? ($row['id'] ?? 0) : ($row['id'] ?? 0));
+        }, $list);
        $openNotSynced = $this->buildOpenNotSyncedAudit(
            $autoSyncTypes,
            $failThreshold,
@@ -573,4 +578,35 @@ class SplitTicketSyncService
        $skip = $this->shouldSkip(Ticket::get((int) $ticket['id']) ?: new Ticket($ticket));
        return $skip ?? '未知原因，未进入执行队列';
    }
+
+    /**
+     * 按 sessionKey（ticketType:host）分组排序，使同域名工单连续执行以命中 Browser 温池
+     *
+     * @param \think\Collection|array<int, Ticket|array<string, mixed>> $list
+     * @return list<Ticket|array<string, mixed>>
+     */
+    private function sortTicketsBySessionKey($list): array
+    {
+        $rows = $list instanceof \think\Collection ? $list->all() : (array) $list;
+        usort($rows, static function ($a, $b): int {
+            $typeA = (string) (is_array($a) ? ($a['ticket_type'] ?? '') : ($a['ticket_type'] ?? ''));
+            $urlA = (string) (is_array($a) ? ($a['ticket_url'] ?? '') : ($a['ticket_url'] ?? ''));
+            $typeB = (string) (is_array($b) ? ($b['ticket_type'] ?? '') : ($b['ticket_type'] ?? ''));
+            $urlB = (string) (is_array($b) ? ($b['ticket_url'] ?? '') : ($b['ticket_url'] ?? ''));
+            $keyA = AntiBotConfigBuilder::resolveSessionKey($typeA, $urlA);
+            $keyB = AntiBotConfigBuilder::resolveSessionKey($typeB, $urlB);
+            if ($keyA === $keyB) {
+                $timeA = (int) (is_array($a) ? ($a['sync_time'] ?? 0) : ($a['sync_time'] ?? 0));
+                $timeB = (int) (is_array($b) ? ($b['sync_time'] ?? 0) : ($b['sync_time'] ?? 0));
+                if ($timeA === $timeB) {
+                    $idA = (int) (is_array($a) ? ($a['id'] ?? 0) : ($a['id'] ?? 0));
+                    $idB = (int) (is_array($b) ? ($b['id'] ?? 0) : ($b['id'] ?? 0));
+                    return $idA <=> $idB;
+                }
+                return $timeA <=> $timeB;
+            }
+            return strcmp($keyA, $keyB);
+        });
+        return $rows;
+    }
 }