";
foreach ($vars as $var) {
echo "";
// 针对布尔值和 null 的特殊处理，因为 print_r 打印这些不直观
if (is_bool($var)) {
echo "bool(" . ($var ? 'true' : 'false') . ")";
} elseif (is_null($var)) {
echo "null";
} else {
// 使用 print_r 获取格式化字符串，并用 htmlspecialchars 防止 HTML 标签被浏览器解析
$output = htmlspecialchars(print_r($var, true));
// 简单的高亮：给数组的键名加上颜色
$output = preg_replace('/\[(.*?)\]/', '[$1]', $output);
echo $output;
}
echo "";
// 如果有多个参数，用分割线隔开
if (count($vars) > 1 && $var !== end($vars)) {
echo "
";
}
}
echo "";
}
}
/**
* 美化打印变量并终止程序 (Dump and Die)
*/
if (!function_exists('dd')) {
function dd(...$vars) {
dump(...$vars);
die(1); // 终止程序执行
}
}
// 文件名: AbstractScrmSpider.php
require_once __DIR__ . '/UnifiedScrmData.class.php';
abstract class AbstractScrmSpider
{
const MODE_FETCH = 'fetch'; // 极速并发拉取 (推荐默认)
const MODE_UI = 'ui_click'; // 强制 UI 点击下一页
/** CLI 测试时输出 Node 请求/失败详情 */
protected $verbose = false;
protected $nodeHost;
public function __construct($nodeHost = 'http://127.0.0.1:3001')
{
$this->nodeHost = rtrim($nodeHost, '/');
}
/**
* 开启调试输出（测试脚本可调用）
*/
public function setVerbose($verbose = true)
{
$this->verbose = (bool) $verbose;
return $this;
}
/** ================= 子类必须实现的契约 ================= **/
abstract protected function getSpiderConfig();
// 以下方法的关注点，全部仅限于 List 接口！
abstract protected function extractListTotalPages($listFirstPageData, $countData = null);
abstract protected function buildListPageParams($page);
abstract protected function getUiPaginationConfig();
// 清洗方法直接接收两个明确的变量，告别繁杂的数组解析
abstract protected function parseToUnifiedData($detailData, $allListPagesData);
/** ================= 核心工作流 ================= **/
public function run()
{
$config = $this->getSpiderConfig();
$listApi = $config['listApi'];
$detailApi = $config['detailApi'] ?? null; // detail 接口是可选的
$countApi = $config['countApi'] ?? null; // 有些在线客服的总数通过单独调用接口获取
// 组装需要 Node 拦截的目标数组
$apiUrlsToIntercept = [$listApi];
if ($detailApi) { $apiUrlsToIntercept[] = $detailApi; }
if ($countApi) { $apiUrlsToIntercept[] = $countApi; }
$antiBot = $config['antiBot'] ?? null;
$mode = $config['paginationMode'] ?? self::MODE_FETCH;
// antiBot + UI 翻页：单 Browser 合并路径（与生产 SplitTicket 同步一致）
if ($this->shouldUseUnifiedBrowserPath($antiBot, $mode)) {
return $this->runUnifiedUiPath($config, $antiBot, $apiUrlsToIntercept, $listApi, $detailApi, $countApi);
}
// 【阶段一】：初始化并首屏拦截
$initResult = $this->requestNode('/api/auth-and-intercept', [
'pageUrl' => $config['pageUrl'],
'apiUrls' => $apiUrlsToIntercept,
'authActions' => $config['authActions'],
'antiBot' => $antiBot,
], $this->resolveAuthInterceptTimeout($antiBot));
if (empty($initResult['success'])) {
throw new Exception($this->formatNodeFailure('初始化失败', $initResult));
}
$interceptedApis = $initResult['interceptedApis'];
$cookies = $initResult['cookies'];
$finalPageUrl = $initResult['finalPageUrl'] ?? $config['pageUrl'];
$this->logDebug('auth-and-intercept ok', [
'finalPageUrl' => $finalPageUrl,
'intercepted' => array_keys($interceptedApis),
'cf' => $initResult['cf'] ?? null,
'browserReused' => $initResult['browserReused'] ?? null,
]);
// 必须拦截到 List 接口，否则无法继续
if (!isset($interceptedApis[$listApi])) {
throw new Exception("致命错误：未能拦截到必须的列表接口 [{$listApi}]");
}
// 分离 Detail 和 List 的首屏数据
$detailData = $detailApi && isset($interceptedApis[$detailApi]) ? $interceptedApis[$detailApi]['data'] : null;
$countData = $countApi && isset($interceptedApis[$countApi]) ? $interceptedApis[$countApi]['data'] : null;
// dd($detailData);
// dd($countData);
$listApiNode = $interceptedApis[$listApi];
$allListPagesData = [$listApiNode['data']]; // 初始化列表容器，装入第一页数据
// dd($allListPagesData);exit;
$totalPages = $this->extractListTotalPages($listApiNode['data'], $countData);
$mode = $config['paginationMode'] ?? self::MODE_FETCH;
// 如果总页数 > 1，触发【阶段二】
if ($totalPages > 1 || $totalPages === null) {
// 策略 1：极速 Fetch
if ($mode === self::MODE_FETCH) {
$paramList = [];
for ($page = 2; $page <= $totalPages; $page++) {
$paramList[] = $this->buildListPageParams($page);
}
$fetchResult = $this->requestNode('/api/batch-fetch', [
'tasks' => [[
'apiPath' => $listApi,
'fullUrl' => $listApiNode['url'],
'headers' => $listApiNode['headers'] ?? "",
'paramList' => $paramList,
'method' => $config['listMethod'] ?? 'GET'
]],
'cookies' => $cookies,
'antiBot' => $antiBot,
], 120);
if (!empty($fetchResult['success'])) {
foreach ($fetchResult['results'][$listApi] as $pResult) {
if ($pResult['success']) $allListPagesData[] = $pResult['data'];
}
}
}
// 策略 2：强制 UI 点击
elseif ($mode === self::MODE_UI) {
$uiConfig = $this->getUiPaginationConfig();
// 🔥 核心修改：直接将第一页的数组传给 Node，让 Node 引擎自己去剔除时间戳并生成统一哈希
$firstPageData = $listApiNode['data'];
// 🔥 核心重构 2：如果是 null 盲点模式，下发 9999 次点击任务；否则按实际计算次数
$clicksToPerform = ($totalPages === null) ? 9999 : ($totalPages - 1);
$uiResult = $this->requestNode('/api/ui-pagination', [
'apiUrl' => $listApi,
'pageUrl' => $config['pageUrl'],
'finalPageUrl' => $finalPageUrl,
'nextBtnSelector' => $uiConfig['nextBtnSelector'],
'waitMs' => $uiConfig['waitMs'] ?? 2000,
'clicksToPerform' => $clicksToPerform,
'cookies' => $cookies,
'firstPageData' => $firstPageData, // 传递原始数据源
// 🔥 核心补充：把登录剧本原封不动地传给翻页引擎！
'authActions' => $config['authActions'],
'antiBot' => $antiBot,
], 1200); // 增加 PHP 端的 cURL 超时时间到 400 秒，以包容多次重试产生的耗时
if (!empty($uiResult['success']) && !empty($uiResult['data'])) {
foreach ($uiResult['data'] as $pageData) {
$allListPagesData[] = $pageData;
}
}
}
}
// 注：如果 $totalPages <= 1，直接自然跳过上述区块，进入第三阶段。
// 【阶段三】：移交数据进行最终清洗
// return $this->parseToUnifiedData($detailData, $allListPagesData);
// 【阶段三】：解析并打包数据
$unifiedData = $this->parseToUnifiedData($detailData, $allListPagesData);
// 🔥 核心修改：引入业务对账雷达
// if (!$this->validateDataIntegrity($unifiedData)) {
// 方案 A：直接返回 false（符合你的特殊要求）
// return false;
// 方案 B（工业级推荐）：抛出自定义异常。
// 因为返回 false 会让你在外部不知道到底是“网络超时”失败的，还是“数据对账没对上”失败的。
// throw new Exception("抓取校验异常：数据未成功获取，或客服数量业务对账失败！");
// }
return $unifiedData;
}
/**
* 单 Browser 路径：auth + 拦截 + UI 翻页（需 antiBot.sessionKey）
*/
protected function runUnifiedUiPath($config, $antiBot, $apiUrlsToIntercept, $listApi, $detailApi, $countApi)
{
$uiConfig = $this->getUiPaginationConfig();
$sessionKey = is_array($antiBot) ? (string) ($antiBot['sessionKey'] ?? '') : '';
$this->logDebug('unified browser path', [
'sessionKey' => $sessionKey,
'listApi' => $listApi,
]);
$mergedResult = $this->requestNode('/api/auth-intercept-and-paginate', [
'pageUrl' => $config['pageUrl'],
'apiUrls' => $apiUrlsToIntercept,
'authActions' => $config['authActions'] ?? [],
'antiBot' => $antiBot,
'pagination' => [
'apiUrl' => $listApi,
'nextBtnSelector' => $uiConfig['nextBtnSelector'] ?? '',
'waitMs' => $uiConfig['waitMs'] ?? 2000,
'clicksToPerform' => 9999,
],
], 1200);
if (empty($mergedResult['success'])) {
throw new Exception($this->formatNodeFailure('合并同步失败', $mergedResult));
}
$this->logDebug('auth-intercept-and-paginate ok', [
'intercepted' => array_keys($mergedResult['interceptedApis'] ?? []),
'extraPages' => count($mergedResult['extraPages'] ?? []),
'cf' => $mergedResult['cf'] ?? null,
'browserReused' => $mergedResult['browserReused'] ?? null,
'finalPageUrl' => $mergedResult['finalPageUrl'] ?? null,
]);
$interceptedApis = $mergedResult['interceptedApis'] ?? [];
if (!isset($interceptedApis[$listApi])) {
throw new Exception("致命错误：未能拦截到必须的列表接口 [{$listApi}]");
}
$detailData = $detailApi && isset($interceptedApis[$detailApi])
? $interceptedApis[$detailApi]['data'] : null;
$countData = $countApi && isset($interceptedApis[$countApi])
? $interceptedApis[$countApi]['data'] : null;
$listApiNode = $interceptedApis[$listApi];
$allListPagesData = [$listApiNode['data']];
if (!empty($mergedResult['extraPages']) && is_array($mergedResult['extraPages'])) {
foreach ($mergedResult['extraPages'] as $pageData) {
$allListPagesData[] = $pageData;
}
}
$this->extractListTotalPages($listApiNode['data'], $countData);
return $this->parseToUnifiedData($detailData, $allListPagesData);
}
/**
* antiBot 启用且配置了 sessionKey 时，UI 翻页走合并 Browser 路径
*/
protected function shouldUseUnifiedBrowserPath($antiBot, $mode)
{
if ($mode !== self::MODE_UI) {
return false;
}
if (!is_array($antiBot) || empty($antiBot['enabled'])) {
return false;
}
return !empty($antiBot['sessionKey']);
}
/**
* 格式化 Node 失败信息，附带 CF / 落地页等诊断字段
*/
protected function formatNodeFailure($prefix, $result)
{
$cfCode = (string) ($result['code'] ?? '');
if ($cfCode === 'CF_TURNSTILE_FAILED') {
return 'Cloudflare Turnstile 验证失败: ' . ($result['stage'] ?? 'timeout');
}
$parts = [$prefix . ': ' . ($result['error'] ?? '未知')];
if (!empty($result['finalPageUrl'])) {
$parts[] = 'finalPageUrl=' . $result['finalPageUrl'];
}
if (!empty($result['page']) && is_array($result['page'])) {
$page = $result['page'];
if (!empty($page['url'])) {
$parts[] = 'pageUrl=' . $page['url'];
}
if (isset($page['elInputInner'])) {
$parts[] = 'elInputInner=' . $page['elInputInner'];
}
}
if (!empty($result['cf']) && is_array($result['cf'])) {
$cf = $result['cf'];
$cfBits = [];
if (isset($cf['cfDetected'])) {
$cfBits[] = 'cfDetected=' . ($cf['cfDetected'] ? 'true' : 'false');
}
if (!empty($cf['turnstileStage'])) {
$cfBits[] = 'turnstileStage=' . $cf['turnstileStage'];
}
if (isset($cf['solverUsed'])) {
$cfBits[] = 'solverUsed=' . ($cf['solverUsed'] ? 'true' : 'false');
}
if (isset($cf['elapsedMs'])) {
$cfBits[] = 'cfElapsedMs=' . $cf['elapsedMs'];
}
if (!empty($cfBits)) {
$parts[] = implode(', ', $cfBits);
}
}
return implode(' | ', $parts);
}
protected function logDebug($message, $context = [])
{
if (!$this->verbose) {
return;
}
$line = '[Spider] ' . $message;
if (!empty($context)) {
$line .= ' ' . json_encode($context, JSON_UNESCAPED_UNICODE);
}
echo $line . "\n";
}
/**
* Real Browser + Turnstile 首屏耗时较长，单独放宽 auth-and-intercept 超时
*
* @param array|null $antiBot
*/
private function resolveAuthInterceptTimeout($antiBot)
{
if (!is_array($antiBot) || empty($antiBot['enabled'])) {
return 120;
}
return 180;
}
private function requestNode($endpoint, $payload, $timeout = 60)
{
$maxAttempts = 3;
$lastDecoded = [];
for ($attempt = 1; $attempt <= $maxAttempts; $attempt++) {
$lastDecoded = $this->doRequestNode($endpoint, $payload, $timeout, $attempt);
if (!empty($lastDecoded['success'])) {
return $lastDecoded;
}
$error = (string) ($lastDecoded['error'] ?? '');
if (!$this->shouldRetryNodeResponse($lastDecoded, $error) || $attempt >= $maxAttempts) {
return $lastDecoded;
}
$this->logDebug('node retry', [
'endpoint' => $endpoint,
'attempt' => $attempt + 1,
'error' => $error,
]);
sleep($attempt);
}
return $lastDecoded;
}
private function doRequestNode($endpoint, $payload, $timeout, $attempt)
{
$url = $this->nodeHost . $endpoint;
$this->logDebug('node request', [
'url' => $url,
'timeout' => $timeout,
'attempt' => $attempt,
]);
$ch = curl_init($url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($payload, JSON_UNESCAPED_UNICODE));
curl_setopt($ch, CURLOPT_HTTPHEADER, ['Content-Type: application/json']);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5);
curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
$response = curl_exec($ch);
$httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
if (curl_errno($ch)) {
$err = curl_error($ch);
curl_close($ch);
throw new Exception($err);
}
curl_close($ch);
$decoded = json_decode((string) $response, true);
if (!is_array($decoded)) {
return ['success' => false, 'error' => 'Node 返回非 JSON', 'httpCode' => $httpCode];
}
if ($httpCode === 503) {
$decoded['success'] = false;
$decoded['error'] = (string) ($decoded['error'] ?? '服务繁忙');
$decoded['httpCode'] = 503;
}
return $decoded;
}
private function shouldRetryNodeResponse($decoded, $error)
{
if ((int) ($decoded['httpCode'] ?? 0) === 503) {
return true;
}
$lower = strtolower($error);
foreach (['排队超时', 'queue_timeout', 'timeout', 'net::', 'navigation', 'connection'] as $needle) {
if ($needle !== '' && strpos($lower, $needle) !== false) {
return true;
}
}
return false;
}
/**
* 🔥 新增：数据完整性与业务对账校验
* @param array $unifiedData 已经解析好的统一格式数据
* @return bool 校验成功返回 true，失败返回 false
*/
protected function validateDataIntegrity($unifiedData)
{
// 1. 底线校验：如果最核心的列表数据或详情数据根本就是空的，直接判定失败
if (empty($unifiedData) || !is_array($unifiedData)) {
return false;
}
// 2. 这里可以放一些全系统通用的基础校验（如果有的话）
return true;
}
}