root
460 lines
18 KiB
PHP
Executable File
460 lines
18 KiB
PHP
Executable File
<?php
|
||
|
||
declare(strict_types=1);
|
||
|
||
namespace app\common\library\scrm;
|
||
|
||
use app\common\service\SplitTicketSyncLogger;
|
||
use Exception;
|
||
|
||
/**
|
||
* 云控蜘蛛抽象基类（Node Headless 拦截 + 翻页 + 清洗）
|
||
*/
|
||
abstract class AbstractScrmSpider implements ScrmSpiderInterface
|
||
{
|
||
public const MODE_FETCH = 'fetch';
|
||
|
||
public const MODE_UI = 'ui_click';
|
||
|
||
protected string $nodeHost;
|
||
|
||
public function __construct(string $nodeHost = 'http://127.0.0.1:3001')
|
||
{
|
||
$this->nodeHost = rtrim($nodeHost, '/');
|
||
}
|
||
|
||
/** @return array<string, mixed> */
|
||
abstract protected function getSpiderConfig(): array;
|
||
|
||
/**
|
||
* @param array<string, mixed>|null $countData
|
||
*/
|
||
abstract protected function extractListTotalPages($listFirstPageData, $countData = null);
|
||
|
||
/**
|
||
* @return array<string, mixed>
|
||
*/
|
||
abstract protected function buildListPageParams(int $page): array;
|
||
|
||
/** @return array<string, mixed> */
|
||
abstract protected function getUiPaginationConfig(): array;
|
||
|
||
/**
|
||
* @param mixed $detailData
|
||
* @param array<int, mixed> $allListPagesData
|
||
*/
|
||
abstract protected function parseToUnifiedData($detailData, array $allListPagesData): UnifiedScrmData;
|
||
|
||
public function run(): UnifiedScrmData
|
||
{
|
||
$config = $this->getSpiderConfig();
|
||
SplitTicketSyncLogger::log('spider', 'run start', [
|
||
'nodeHost' => $this->nodeHost,
|
||
'pageUrl' => $config['pageUrl'] ?? '',
|
||
'listApi' => $config['listApi'] ?? '',
|
||
'paginationMode' => $config['paginationMode'] ?? self::MODE_FETCH,
|
||
]);
|
||
|
||
$listApi = (string) ($config['listApi'] ?? '');
|
||
$detailApi = $config['detailApi'] ?? null;
|
||
$countApi = $config['countApi'] ?? null;
|
||
|
||
$apiUrlsToIntercept = [$listApi];
|
||
if ($detailApi) {
|
||
$apiUrlsToIntercept[] = $detailApi;
|
||
}
|
||
if ($countApi) {
|
||
$apiUrlsToIntercept[] = $countApi;
|
||
}
|
||
|
||
$antiBot = $config['antiBot'] ?? null;
|
||
$mode = $config['paginationMode'] ?? self::MODE_FETCH;
|
||
|
||
if ($this->shouldUseUnifiedBrowserPath($antiBot, $mode)) {
|
||
return $this->runUnifiedUiPath($config, $antiBot, $apiUrlsToIntercept, $listApi, $detailApi, $countApi);
|
||
}
|
||
|
||
$initResult = $this->requestNode('/api/auth-and-intercept', [
|
||
'pageUrl' => $config['pageUrl'],
|
||
'apiUrls' => $apiUrlsToIntercept,
|
||
'authActions' => $config['authActions'] ?? [],
|
||
'antiBot' => $antiBot,
|
||
], $this->resolveAuthInterceptTimeout($antiBot));
|
||
|
||
if (empty($initResult['success'])) {
|
||
$cfCode = (string) ($initResult['code'] ?? '');
|
||
SplitTicketSyncLogger::log('spider', 'auth-and-intercept failed', [
|
||
'error' => $initResult['error'] ?? '未知',
|
||
'code' => $cfCode !== '' ? $cfCode : null,
|
||
'challengeType' => $initResult['challengeType'] ?? null,
|
||
'stage' => $initResult['stage'] ?? null,
|
||
'cf' => $initResult['cf'] ?? null,
|
||
]);
|
||
$errMsg = (string) ($initResult['error'] ?? '未知');
|
||
if ($cfCode === 'CF_TURNSTILE_FAILED') {
|
||
throw new Exception('Cloudflare Turnstile 验证失败: ' . ($initResult['stage'] ?? 'timeout'));
|
||
}
|
||
throw new Exception('初始化失败: ' . $errMsg);
|
||
}
|
||
|
||
$interceptedApis = $initResult['interceptedApis'];
|
||
$finalPageUrl = (string) ($initResult['finalPageUrl'] ?? ($config['pageUrl'] ?? ''));
|
||
SplitTicketSyncLogger::log('spider', 'auth-and-intercept ok', [
|
||
'intercepted' => array_keys($interceptedApis),
|
||
'finalPageUrl' => $finalPageUrl,
|
||
'cfStage' => $initResult['cf']['turnstileStage'] ?? ($initResult['cf']['stage'] ?? null),
|
||
'browserReused' => $initResult['browserReused'] ?? null,
|
||
]);
|
||
$cookies = $initResult['cookies'];
|
||
|
||
if (!isset($interceptedApis[$listApi])) {
|
||
throw new Exception("致命错误：未能拦截到必须的列表接口 [{$listApi}]");
|
||
}
|
||
|
||
$detailData = $detailApi && isset($interceptedApis[$detailApi])
|
||
? $interceptedApis[$detailApi]['data'] : null;
|
||
$countData = $countApi && isset($interceptedApis[$countApi])
|
||
? $interceptedApis[$countApi]['data'] : null;
|
||
|
||
$listApiNode = $interceptedApis[$listApi];
|
||
$allListPagesData = [$listApiNode['data']];
|
||
|
||
$totalPages = $this->extractListTotalPages($listApiNode['data'], $countData);
|
||
SplitTicketSyncLogger::log('spider', 'pagination plan', [
|
||
'totalPages' => $totalPages,
|
||
'mode' => $mode,
|
||
]);
|
||
|
||
if ($totalPages > 1 || $totalPages === null) {
|
||
if ($mode === self::MODE_FETCH && $totalPages !== null) {
|
||
$paramList = [];
|
||
for ($page = 2; $page <= $totalPages; $page++) {
|
||
$paramList[] = $this->buildListPageParams($page);
|
||
}
|
||
|
||
$fetchResult = $this->requestNode('/api/batch-fetch', [
|
||
'tasks' => [[
|
||
'apiPath' => $listApi,
|
||
'fullUrl' => $listApiNode['url'],
|
||
'headers' => $listApiNode['headers'] ?? '',
|
||
'paramList' => $paramList,
|
||
'method' => $config['listMethod'] ?? 'GET',
|
||
]],
|
||
'cookies' => $cookies,
|
||
'antiBot' => $antiBot,
|
||
], 120);
|
||
|
||
if (!empty($fetchResult['success'])) {
|
||
foreach ($fetchResult['results'][$listApi] as $pResult) {
|
||
if (!empty($pResult['success'])) {
|
||
$allListPagesData[] = $pResult['data'];
|
||
}
|
||
}
|
||
}
|
||
} elseif ($mode === self::MODE_UI) {
|
||
$uiConfig = $this->getUiPaginationConfig();
|
||
$firstPageData = $listApiNode['data'];
|
||
$clicksToPerform = ($totalPages === null) ? 9999 : ($totalPages - 1);
|
||
|
||
$uiResult = $this->requestNode('/api/ui-pagination', [
|
||
'apiUrl' => $listApi,
|
||
'pageUrl' => $config['pageUrl'],
|
||
'finalPageUrl' => $finalPageUrl,
|
||
'nextBtnSelector' => $uiConfig['nextBtnSelector'] ?? '',
|
||
'waitMs' => $uiConfig['waitMs'] ?? 2000,
|
||
'clicksToPerform' => $clicksToPerform,
|
||
'cookies' => $cookies,
|
||
'firstPageData' => $firstPageData,
|
||
'authActions' => $config['authActions'] ?? [],
|
||
'antiBot' => $antiBot,
|
||
], 1200);
|
||
|
||
if (!empty($uiResult['success']) && !empty($uiResult['data'])) {
|
||
foreach ($uiResult['data'] as $pageData) {
|
||
$allListPagesData[] = $pageData;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
$result = $this->parseToUnifiedData($detailData, $allListPagesData);
|
||
SplitTicketSyncLogger::log('spider', 'run done', [
|
||
'todayNewCount' => $result->todayNewCount,
|
||
'totalOnline' => $result->totalOnline,
|
||
'totalOffline' => $result->totalOffline,
|
||
'total' => $result->total,
|
||
'numberCount' => count($result->numbers),
|
||
]);
|
||
return $result;
|
||
}
|
||
|
||
/**
|
||
* 单 Browser 路径：auth + 拦截 + UI 翻页（需 antiBot.sessionKey）
|
||
*
|
||
* @param array<string, mixed> $config
|
||
* @param array<string, mixed>|null $antiBot
|
||
* @param list<string> $apiUrlsToIntercept
|
||
*/
|
||
private function runUnifiedUiPath(
|
||
array $config,
|
||
?array $antiBot,
|
||
array $apiUrlsToIntercept,
|
||
string $listApi,
|
||
?string $detailApi,
|
||
?string $countApi
|
||
): UnifiedScrmData {
|
||
$uiConfig = $this->getUiPaginationConfig();
|
||
$sessionKey = is_array($antiBot) ? (string) ($antiBot['sessionKey'] ?? '') : '';
|
||
|
||
SplitTicketSyncLogger::log('spider', 'unified browser path', [
|
||
'sessionKey' => $sessionKey,
|
||
'listApi' => $listApi,
|
||
]);
|
||
|
||
$mergedResult = $this->requestNode('/api/auth-intercept-and-paginate', [
|
||
'pageUrl' => $config['pageUrl'],
|
||
'apiUrls' => $apiUrlsToIntercept,
|
||
'authActions' => $config['authActions'] ?? [],
|
||
'antiBot' => $antiBot,
|
||
'pagination' => [
|
||
'apiUrl' => $listApi,
|
||
'nextBtnSelector' => $uiConfig['nextBtnSelector'] ?? '',
|
||
'waitMs' => $uiConfig['waitMs'] ?? 2000,
|
||
'clicksToPerform' => 9999,
|
||
],
|
||
], 1200);
|
||
|
||
if (empty($mergedResult['success'])) {
|
||
$cfCode = (string) ($mergedResult['code'] ?? '');
|
||
SplitTicketSyncLogger::log('spider', 'auth-intercept-and-paginate failed', [
|
||
'error' => $mergedResult['error'] ?? '未知',
|
||
'code' => $cfCode !== '' ? $cfCode : null,
|
||
'stage' => $mergedResult['stage'] ?? null,
|
||
'cf' => $mergedResult['cf'] ?? null,
|
||
'sessionKey' => $sessionKey,
|
||
]);
|
||
if ($cfCode === 'CF_TURNSTILE_FAILED') {
|
||
throw new Exception('Cloudflare Turnstile 验证失败: ' . ($mergedResult['stage'] ?? 'timeout'));
|
||
}
|
||
throw new Exception('合并同步失败: ' . (string) ($mergedResult['error'] ?? '未知'));
|
||
}
|
||
|
||
SplitTicketSyncLogger::log('spider', 'auth-intercept-and-paginate ok', [
|
||
'intercepted' => array_keys($mergedResult['interceptedApis'] ?? []),
|
||
'extraPages' => count($mergedResult['extraPages'] ?? []),
|
||
'cfStage' => $mergedResult['cf']['turnstileStage'] ?? ($mergedResult['cf']['stage'] ?? null),
|
||
'browserReused' => $mergedResult['browserReused'] ?? null,
|
||
'sessionKey' => $sessionKey,
|
||
]);
|
||
|
||
$interceptedApis = $mergedResult['interceptedApis'] ?? [];
|
||
if (!isset($interceptedApis[$listApi])) {
|
||
throw new Exception("致命错误：未能拦截到必须的列表接口 [{$listApi}]");
|
||
}
|
||
|
||
$detailData = $detailApi && isset($interceptedApis[$detailApi])
|
||
? $interceptedApis[$detailApi]['data'] : null;
|
||
$countData = $countApi && isset($interceptedApis[$countApi])
|
||
? $interceptedApis[$countApi]['data'] : null;
|
||
|
||
$listApiNode = $interceptedApis[$listApi];
|
||
$allListPagesData = [$listApiNode['data']];
|
||
if (!empty($mergedResult['extraPages']) && is_array($mergedResult['extraPages'])) {
|
||
foreach ($mergedResult['extraPages'] as $pageData) {
|
||
$allListPagesData[] = $pageData;
|
||
}
|
||
}
|
||
|
||
$this->extractListTotalPages($listApiNode['data'], $countData);
|
||
|
||
$result = $this->parseToUnifiedData($detailData, $allListPagesData);
|
||
SplitTicketSyncLogger::log('spider', 'run done', [
|
||
'todayNewCount' => $result->todayNewCount,
|
||
'totalOnline' => $result->totalOnline,
|
||
'totalOffline' => $result->totalOffline,
|
||
'total' => $result->total,
|
||
'numberCount' => count($result->numbers),
|
||
'unifiedPath' => true,
|
||
]);
|
||
return $result;
|
||
}
|
||
|
||
/**
|
||
* @param array<string, mixed>|null $antiBot
|
||
*/
|
||
protected function shouldUseUnifiedBrowserPath(?array $antiBot, string $mode): bool
|
||
{
|
||
if ($mode !== self::MODE_UI) {
|
||
return false;
|
||
}
|
||
if (!is_array($antiBot) || empty($antiBot['enabled'])) {
|
||
return false;
|
||
}
|
||
return !empty($antiBot['sessionKey']);
|
||
}
|
||
|
||
/**
|
||
* Real Browser + Turnstile 首屏耗时较长，单独放宽 auth-and-intercept 超时
|
||
*
|
||
* @param array<string, mixed>|null $antiBot
|
||
*/
|
||
protected function resolveAuthInterceptTimeout(?array $antiBot): int
|
||
{
|
||
if (!is_array($antiBot) || empty($antiBot['enabled'])) {
|
||
return 120;
|
||
}
|
||
return 180;
|
||
}
|
||
|
||
/**
|
||
* @param array<string, mixed> $payload
|
||
* @return array<string, mixed>
|
||
*/
|
||
protected function requestNode(string $endpoint, array $payload, int $timeout = 60): array
|
||
{
|
||
$maxAttempts = 3;
|
||
$lastDecoded = [];
|
||
for ($attempt = 1; $attempt <= $maxAttempts; $attempt++) {
|
||
$lastDecoded = $this->doRequestNode($endpoint, $payload, $timeout, $attempt);
|
||
if (!empty($lastDecoded['success'])) {
|
||
return $lastDecoded;
|
||
}
|
||
$error = (string) ($lastDecoded['error'] ?? '');
|
||
if (!$this->shouldRetryNodeResponse($lastDecoded, $error) || $attempt >= $maxAttempts) {
|
||
return $lastDecoded;
|
||
}
|
||
$delaySeconds = $attempt;
|
||
SplitTicketSyncLogger::log('node_retry', 'retry ' . $endpoint, [
|
||
'attempt' => $attempt + 1,
|
||
'error' => $error,
|
||
'delay' => $delaySeconds,
|
||
]);
|
||
sleep($delaySeconds);
|
||
}
|
||
return $lastDecoded;
|
||
}
|
||
|
||
/**
|
||
* @param array<string, mixed> $payload
|
||
* @return array<string, mixed>
|
||
*/
|
||
private function doRequestNode(string $endpoint, array $payload, int $timeout, int $attempt): array
|
||
{
|
||
$url = $this->nodeHost . $endpoint;
|
||
$started = microtime(true);
|
||
$logPayload = SplitTicketSyncLogger::isEnabled() ? $payload : self::summarizeNodePayload($payload);
|
||
SplitTicketSyncLogger::log('node_request', 'POST ' . $endpoint, [
|
||
'url' => $url,
|
||
'timeout' => $timeout,
|
||
'attempt' => $attempt,
|
||
'payload' => $logPayload,
|
||
]);
|
||
|
||
$ch = curl_init($url);
|
||
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
||
curl_setopt($ch, CURLOPT_POST, true);
|
||
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($payload, JSON_UNESCAPED_UNICODE));
|
||
curl_setopt($ch, CURLOPT_HTTPHEADER, ['Content-Type: application/json']);
|
||
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5);
|
||
curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
|
||
$response = curl_exec($ch);
|
||
$httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||
$elapsedMs = (int) round((microtime(true) - $started) * 1000);
|
||
if (curl_errno($ch)) {
|
||
$err = curl_error($ch);
|
||
curl_close($ch);
|
||
SplitTicketSyncLogger::log('node_response', 'curl error on ' . $endpoint, [
|
||
'httpCode' => $httpCode,
|
||
'elapsedMs' => $elapsedMs,
|
||
'attempt' => $attempt,
|
||
'error' => $err,
|
||
]);
|
||
throw new Exception($err);
|
||
}
|
||
curl_close($ch);
|
||
$decoded = json_decode((string) $response, true);
|
||
$summary = is_array($decoded) ? self::summarizeNodeResponse($decoded) : ['raw' => mb_substr((string) $response, 0, 300, 'UTF-8')];
|
||
SplitTicketSyncLogger::log('node_response', 'POST ' . $endpoint, array_merge([
|
||
'httpCode' => $httpCode,
|
||
'elapsedMs' => $elapsedMs,
|
||
'attempt' => $attempt,
|
||
'responseSize' => strlen((string) $response),
|
||
], $summary));
|
||
if (!is_array($decoded)) {
|
||
return ['success' => false, 'error' => 'Node 返回非 JSON', 'httpCode' => $httpCode];
|
||
}
|
||
if ($httpCode === 503) {
|
||
$decoded['success'] = false;
|
||
$decoded['error'] = (string) ($decoded['error'] ?? '服务繁忙');
|
||
$decoded['httpCode'] = 503;
|
||
}
|
||
return $decoded;
|
||
}
|
||
|
||
/**
|
||
* @param array<string, mixed> $decoded
|
||
*/
|
||
private function shouldRetryNodeResponse(array $decoded, string $error): bool
|
||
{
|
||
if ((int) ($decoded['httpCode'] ?? 0) === 503) {
|
||
return true;
|
||
}
|
||
$lower = mb_strtolower($error, 'UTF-8');
|
||
foreach (['排队超时', 'queue_timeout', 'timeout', 'net::', 'navigation', 'connection'] as $needle) {
|
||
if ($needle !== '' && mb_strpos($lower, mb_strtolower($needle, 'UTF-8')) !== false) {
|
||
return true;
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
|
||
/**
|
||
* @param array<string, mixed> $payload
|
||
* @return array<string, mixed>
|
||
*/
|
||
private static function summarizeNodePayload(array $payload): array
|
||
{
|
||
$summary = $payload;
|
||
if (isset($summary['tasks']) && is_array($summary['tasks'])) {
|
||
$summary['tasks'] = array_map(static function ($task) {
|
||
if (!is_array($task)) {
|
||
return $task;
|
||
}
|
||
$copy = $task;
|
||
if (isset($copy['paramList']) && is_array($copy['paramList'])) {
|
||
$copy['paramList_count'] = count($copy['paramList']);
|
||
unset($copy['paramList']);
|
||
}
|
||
return $copy;
|
||
}, $summary['tasks']);
|
||
}
|
||
if (isset($summary['cookies']) && is_array($summary['cookies'])) {
|
||
$summary['cookies_count'] = count($summary['cookies']);
|
||
unset($summary['cookies']);
|
||
}
|
||
return $summary;
|
||
}
|
||
|
||
/**
|
||
* @param array<string, mixed> $decoded
|
||
* @return array<string, mixed>
|
||
*/
|
||
private static function summarizeNodeResponse(array $decoded): array
|
||
{
|
||
$summary = [
|
||
'success' => $decoded['success'] ?? null,
|
||
'error' => $decoded['error'] ?? null,
|
||
];
|
||
if (isset($decoded['interceptedApis']) && is_array($decoded['interceptedApis'])) {
|
||
$summary['interceptedApis'] = array_keys($decoded['interceptedApis']);
|
||
}
|
||
if (isset($decoded['results']) && is_array($decoded['results'])) {
|
||
$summary['resultApis'] = array_keys($decoded['results']);
|
||
}
|
||
if (isset($decoded['data']) && is_array($decoded['data'])) {
|
||
$summary['dataPages'] = count($decoded['data']);
|
||
}
|
||
return $summary;
|
||
}
|
||
}
|