damon/links
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化爬虫，增加缓存，线程池，保存浏览器用户信息

Browse Source
This commit is contained in:
root
2026-06-20 15:49:40 +08:00
parent e95f4a227c
commit 09bfa9ae01
26 changed files with 1584 additions and 454 deletions
Download Patch File Download Diff File Expand all files Collapse all files
application/common/library/scrm/AbstractScrmSpider.php
+128 -4
View File
@@ -68,13 +68,18 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface
}
$antiBot = $config['antiBot'] ?? null;
$mode = $config['paginationMode'] ?? self::MODE_FETCH;
if ($this->shouldUseUnifiedBrowserPath($antiBot, $mode)) {
return $this->runUnifiedUiPath($config, $antiBot, $apiUrlsToIntercept, $listApi, $detailApi, $countApi);
}
$initResult = $this->requestNode('/api/auth-and-intercept', [
'pageUrl' => $config['pageUrl'],
'apiUrls' => $apiUrlsToIntercept,
'authActions' => $config['authActions'] ?? [],
'antiBot' => $antiBot,
]);
], $this->resolveAuthInterceptTimeout($antiBot));
if (empty($initResult['success'])) {
$cfCode = (string) ($initResult['code'] ?? '');
@@ -95,8 +100,10 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface
$interceptedApis = $initResult['interceptedApis'];
$finalPageUrl = (string) ($initResult['finalPageUrl'] ?? ($config['pageUrl'] ?? ''));
SplitTicketSyncLogger::log('spider', 'auth-and-intercept ok', [
'intercepted' => array_keys($interceptedApis),
'finalPageUrl' => $finalPageUrl,
'intercepted' => array_keys($interceptedApis),
'finalPageUrl' => $finalPageUrl,
'cfStage' => $initResult['cf']['turnstileStage'] ?? ($initResult['cf']['stage'] ?? null),
'browserReused' => $initResult['browserReused'] ?? null,
]);
$cookies = $initResult['cookies'];
@@ -113,7 +120,6 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface
$allListPagesData = [$listApiNode['data']];
$totalPages = $this->extractListTotalPages($listApiNode['data'], $countData);
$mode = $config['paginationMode'] ?? self::MODE_FETCH;
SplitTicketSyncLogger::log('spider', 'pagination plan', [
'totalPages' => $totalPages,
'mode' => $mode,
@@ -182,6 +188,124 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface
return $result;
}
/**
* 单 Browser 路径：auth + 拦截 + UI 翻页（需 antiBot.sessionKey）
*
* @param array<string, mixed> $config
* @param array<string, mixed>|null $antiBot
* @param list<string> $apiUrlsToIntercept
*/
private function runUnifiedUiPath(
array $config,
?array $antiBot,
array $apiUrlsToIntercept,
string $listApi,
?string $detailApi,
?string $countApi
): UnifiedScrmData {
$uiConfig = $this->getUiPaginationConfig();
$sessionKey = is_array($antiBot) ? (string) ($antiBot['sessionKey'] ?? '') : '';
SplitTicketSyncLogger::log('spider', 'unified browser path', [
'sessionKey' => $sessionKey,
'listApi' => $listApi,
]);
$mergedResult = $this->requestNode('/api/auth-intercept-and-paginate', [
'pageUrl' => $config['pageUrl'],
'apiUrls' => $apiUrlsToIntercept,
'authActions' => $config['authActions'] ?? [],
'antiBot' => $antiBot,
'pagination' => [
'apiUrl' => $listApi,
'nextBtnSelector' => $uiConfig['nextBtnSelector'] ?? '',
'waitMs' => $uiConfig['waitMs'] ?? 2000,
'clicksToPerform' => 9999,
],
], 1200);
if (empty($mergedResult['success'])) {
$cfCode = (string) ($mergedResult['code'] ?? '');
SplitTicketSyncLogger::log('spider', 'auth-intercept-and-paginate failed', [
'error' => $mergedResult['error'] ?? '未知',
'code' => $cfCode !== '' ? $cfCode : null,
'stage' => $mergedResult['stage'] ?? null,
'cf' => $mergedResult['cf'] ?? null,
'sessionKey' => $sessionKey,
]);
if ($cfCode === 'CF_TURNSTILE_FAILED') {
throw new Exception('Cloudflare Turnstile 验证失败: ' . ($mergedResult['stage'] ?? 'timeout'));
}
throw new Exception('合并同步失败: ' . (string) ($mergedResult['error'] ?? '未知'));
}
SplitTicketSyncLogger::log('spider', 'auth-intercept-and-paginate ok', [
'intercepted' => array_keys($mergedResult['interceptedApis'] ?? []),
'extraPages' => count($mergedResult['extraPages'] ?? []),
'cfStage' => $mergedResult['cf']['turnstileStage'] ?? ($mergedResult['cf']['stage'] ?? null),
'browserReused' => $mergedResult['browserReused'] ?? null,
'sessionKey' => $sessionKey,
]);
$interceptedApis = $mergedResult['interceptedApis'] ?? [];
if (!isset($interceptedApis[$listApi])) {
throw new Exception("致命错误：未能拦截到必须的列表接口 [{$listApi}]");
}
$detailData = $detailApi && isset($interceptedApis[$detailApi])
? $interceptedApis[$detailApi]['data'] : null;
$countData = $countApi && isset($interceptedApis[$countApi])
? $interceptedApis[$countApi]['data'] : null;
$listApiNode = $interceptedApis[$listApi];
$allListPagesData = [$listApiNode['data']];
if (!empty($mergedResult['extraPages']) && is_array($mergedResult['extraPages'])) {
foreach ($mergedResult['extraPages'] as $pageData) {
$allListPagesData[] = $pageData;
}
}
$this->extractListTotalPages($listApiNode['data'], $countData);
$result = $this->parseToUnifiedData($detailData, $allListPagesData);
SplitTicketSyncLogger::log('spider', 'run done', [
'todayNewCount' => $result->todayNewCount,
'totalOnline' => $result->totalOnline,
'totalOffline' => $result->totalOffline,
'total' => $result->total,
'numberCount' => count($result->numbers),
'unifiedPath' => true,
]);
return $result;
}
/**
* @param array<string, mixed>|null $antiBot
*/
protected function shouldUseUnifiedBrowserPath(?array $antiBot, string $mode): bool
{
if ($mode !== self::MODE_UI) {
return false;
}
if (!is_array($antiBot) || empty($antiBot['enabled'])) {
return false;
}
return !empty($antiBot['sessionKey']);
}
/**
* Real Browser + Turnstile 首屏耗时较长，单独放宽 auth-and-intercept 超时
*
* @param array<string, mixed>|null $antiBot
*/
protected function resolveAuthInterceptTimeout(?array $antiBot): int
{
if (!is_array($antiBot) || empty($antiBot['enabled'])) {
return 120;
}
return 180;
}
/**
* @param array<string, mixed> $payload
* @return array<string, mixed>
application/common/library/scrm/AntiBotConfigBuilder.php
+84
View File
@@ -0,0 +1,84 @@
<?php
declare(strict_types=1);
namespace app\common\library\scrm;
use think\Config;
/**
* 云控蜘蛛 antiBot 配置构建器
*
* 约定：sessionKey = "{ticketType}:{host}"；profile=real 当 ticket_type 在 site 配置列表中
*/
class AntiBotConfigBuilder
{
/**
* 为指定工单类型与落地页 URL 构建 Node antiBot 配置；未启用类型返回 null
*
* @return array<string, mixed>|null
*/
public static function build(string $ticketType, string $pageUrl): ?array
{
$enabledTypes = self::getEnabledTypes();
if (!in_array($ticketType, $enabledTypes, true)) {
return null;
}
$host = (string) parse_url($pageUrl, PHP_URL_HOST);
if ($host === '') {
return null;
}
$sessionKey = $ticketType . ':' . $host;
$ttlMinutes = self::getSessionTtlMinutes();
return [
'enabled' => true,
'profile' => 'real',
'turnstile' => true,
'solverFallback' => true,
'sessionKey' => $sessionKey,
'challengeTimeoutMs' => 60000,
'sessionTtlMinutes' => $ttlMinutes,
];
}
/**
* 解析 sessionKey（供 cron 分组调度使用）
*/
public static function resolveSessionKey(string $ticketType, string $pageUrl): string
{
$host = (string) parse_url($pageUrl, PHP_URL_HOST);
return $ticketType . ':' . ($host !== '' ? $host : 'unknown');
}
/**
* 是否对该工单类型启用 antiBot
*/
public static function isEnabledForType(string $ticketType): bool
{
return in_array($ticketType, self::getEnabledTypes(), true);
}
/**
* @return list<string>
*/
public static function getEnabledTypes(): array
{
$raw = (string) Config::get('site.split_scrm_antibot_types');
if ($raw === '') {
return ['whatshub', 'chatknow'];
}
$parts = array_map('trim', explode(',', $raw));
return array_values(array_filter($parts, static function (string $v): bool {
return $v !== '';
}));
}
public static function getSessionTtlMinutes(): int
{
$minutes = (int) Config::get('site.split_scrm_session_ttl_minutes');
return $minutes > 0 ? $minutes : 120;
}
}
application/common/library/scrm/spider/ChatknowSpider.php
+2
View File
@@ -5,6 +5,7 @@ declare(strict_types=1);
namespace app\common\library\scrm\spider;
use app\common\library\scrm\AbstractScrmSpider;
use app\common\library\scrm\AntiBotConfigBuilder;
use app\common\library\scrm\UnifiedScrmData;
/**
@@ -48,6 +49,7 @@ class ChatknowSpider extends AbstractScrmSpider
'authActions' => [
['type' => 'wait', 'ms' => 4000],
],
'antiBot' => AntiBotConfigBuilder::build('chatknow', $this->pageUrl),
];
}
application/common/library/scrm/spider/WhatshubSpider.php
+2 -11
View File
@@ -5,6 +5,7 @@ declare(strict_types=1);
namespace app\common\library\scrm\spider;
use app\common\library\scrm\AbstractScrmSpider;
use app\common\library\scrm\AntiBotConfigBuilder;
use app\common\library\scrm\UnifiedScrmData;
/**
@@ -47,8 +48,6 @@ class WhatshubSpider extends AbstractScrmSpider
/** @return array<string, mixed> */
protected function getSpiderConfig(): array
{
$host = (string) parse_url($this->pageUrl, PHP_URL_HOST);
return [
'pageUrl' => $this->pageUrl,
'listApi' => self::API_LIST,
@@ -62,15 +61,7 @@ class WhatshubSpider extends AbstractScrmSpider
['type' => 'vue_click', 'selector' => '.vxe-button-group .theme--primary'],
['type' => 'wait', 'ms' => 2200],
],
// Whatshub 专用：Real Browser + Turnstile + Captcha API 兜底 + 会话复用
'antiBot' => [
'enabled' => true,
'profile' => 'real',
'turnstile' => true,
'solverFallback' => true,
'sessionKey' => 'whatshub:' . $host,
'challengeTimeoutMs' => 60000,
],
'antiBot' => AntiBotConfigBuilder::build('whatshub', $this->pageUrl),
];
}