damon/links
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化爬虫，增加缓存，线程池，保存浏览器用户信息

Browse Source
This commit is contained in:
root
2026-06-20 15:49:40 +08:00
parent e95f4a227c
commit 09bfa9ae01
26 changed files with 1584 additions and 454 deletions
Download Patch File Download Diff File Expand all files Collapse all files
patches/application/common/library/scrm/AbstractScrmSpider.php
+128 -4
View File
@@ -68,13 +68,18 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface
}
$antiBot = $config['antiBot'] ?? null;
$mode = $config['paginationMode'] ?? self::MODE_FETCH;
if ($this->shouldUseUnifiedBrowserPath($antiBot, $mode)) {
return $this->runUnifiedUiPath($config, $antiBot, $apiUrlsToIntercept, $listApi, $detailApi, $countApi);
}
$initResult = $this->requestNode('/api/auth-and-intercept', [
'pageUrl' => $config['pageUrl'],
'apiUrls' => $apiUrlsToIntercept,
'authActions' => $config['authActions'] ?? [],
'antiBot' => $antiBot,
]);
], $this->resolveAuthInterceptTimeout($antiBot));
if (empty($initResult['success'])) {
$cfCode = (string) ($initResult['code'] ?? '');
@@ -95,8 +100,10 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface
$interceptedApis = $initResult['interceptedApis'];
$finalPageUrl = (string) ($initResult['finalPageUrl'] ?? ($config['pageUrl'] ?? ''));
SplitTicketSyncLogger::log('spider', 'auth-and-intercept ok', [
'intercepted' => array_keys($interceptedApis),
'finalPageUrl' => $finalPageUrl,
'intercepted' => array_keys($interceptedApis),
'finalPageUrl' => $finalPageUrl,
'cfStage' => $initResult['cf']['turnstileStage'] ?? ($initResult['cf']['stage'] ?? null),
'browserReused' => $initResult['browserReused'] ?? null,
]);
$cookies = $initResult['cookies'];
@@ -113,7 +120,6 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface
$allListPagesData = [$listApiNode['data']];
$totalPages = $this->extractListTotalPages($listApiNode['data'], $countData);
$mode = $config['paginationMode'] ?? self::MODE_FETCH;
SplitTicketSyncLogger::log('spider', 'pagination plan', [
'totalPages' => $totalPages,
'mode' => $mode,
@@ -182,6 +188,124 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface
return $result;
}
/**
* 单 Browser 路径：auth + 拦截 + UI 翻页（需 antiBot.sessionKey）
*
* @param array<string, mixed> $config
* @param array<string, mixed>|null $antiBot
* @param list<string> $apiUrlsToIntercept
*/
private function runUnifiedUiPath(
array $config,
?array $antiBot,
array $apiUrlsToIntercept,
string $listApi,
?string $detailApi,
?string $countApi
): UnifiedScrmData {
$uiConfig = $this->getUiPaginationConfig();
$sessionKey = is_array($antiBot) ? (string) ($antiBot['sessionKey'] ?? '') : '';
SplitTicketSyncLogger::log('spider', 'unified browser path', [
'sessionKey' => $sessionKey,
'listApi' => $listApi,
]);
$mergedResult = $this->requestNode('/api/auth-intercept-and-paginate', [
'pageUrl' => $config['pageUrl'],
'apiUrls' => $apiUrlsToIntercept,
'authActions' => $config['authActions'] ?? [],
'antiBot' => $antiBot,
'pagination' => [
'apiUrl' => $listApi,
'nextBtnSelector' => $uiConfig['nextBtnSelector'] ?? '',
'waitMs' => $uiConfig['waitMs'] ?? 2000,
'clicksToPerform' => 9999,
],
], 1200);
if (empty($mergedResult['success'])) {
$cfCode = (string) ($mergedResult['code'] ?? '');
SplitTicketSyncLogger::log('spider', 'auth-intercept-and-paginate failed', [
'error' => $mergedResult['error'] ?? '未知',
'code' => $cfCode !== '' ? $cfCode : null,
'stage' => $mergedResult['stage'] ?? null,
'cf' => $mergedResult['cf'] ?? null,
'sessionKey' => $sessionKey,
]);
if ($cfCode === 'CF_TURNSTILE_FAILED') {
throw new Exception('Cloudflare Turnstile 验证失败: ' . ($mergedResult['stage'] ?? 'timeout'));
}
throw new Exception('合并同步失败: ' . (string) ($mergedResult['error'] ?? '未知'));
}
SplitTicketSyncLogger::log('spider', 'auth-intercept-and-paginate ok', [
'intercepted' => array_keys($mergedResult['interceptedApis'] ?? []),
'extraPages' => count($mergedResult['extraPages'] ?? []),
'cfStage' => $mergedResult['cf']['turnstileStage'] ?? ($mergedResult['cf']['stage'] ?? null),
'browserReused' => $mergedResult['browserReused'] ?? null,
'sessionKey' => $sessionKey,
]);
$interceptedApis = $mergedResult['interceptedApis'] ?? [];
if (!isset($interceptedApis[$listApi])) {
throw new Exception("致命错误：未能拦截到必须的列表接口 [{$listApi}]");
}
$detailData = $detailApi && isset($interceptedApis[$detailApi])
? $interceptedApis[$detailApi]['data'] : null;
$countData = $countApi && isset($interceptedApis[$countApi])
? $interceptedApis[$countApi]['data'] : null;
$listApiNode = $interceptedApis[$listApi];
$allListPagesData = [$listApiNode['data']];
if (!empty($mergedResult['extraPages']) && is_array($mergedResult['extraPages'])) {
foreach ($mergedResult['extraPages'] as $pageData) {
$allListPagesData[] = $pageData;
}
}
$this->extractListTotalPages($listApiNode['data'], $countData);
$result = $this->parseToUnifiedData($detailData, $allListPagesData);
SplitTicketSyncLogger::log('spider', 'run done', [
'todayNewCount' => $result->todayNewCount,
'totalOnline' => $result->totalOnline,
'totalOffline' => $result->totalOffline,
'total' => $result->total,
'numberCount' => count($result->numbers),
'unifiedPath' => true,
]);
return $result;
}
/**
* @param array<string, mixed>|null $antiBot
*/
protected function shouldUseUnifiedBrowserPath(?array $antiBot, string $mode): bool
{
if ($mode !== self::MODE_UI) {
return false;
}
if (!is_array($antiBot) || empty($antiBot['enabled'])) {
return false;
}
return !empty($antiBot['sessionKey']);
}
/**
* Real Browser + Turnstile 首屏耗时较长，单独放宽 auth-and-intercept 超时
*
* @param array<string, mixed>|null $antiBot
*/
protected function resolveAuthInterceptTimeout(?array $antiBot): int
{
if (!is_array($antiBot) || empty($antiBot['enabled'])) {
return 120;
}
return 180;
}
/**
* @param array<string, mixed> $payload
* @return array<string, mixed>