root
485 lines
19 KiB
PHP
Executable File
485 lines
19 KiB
PHP
Executable File
<?php
|
||
|
||
/**
|
||
* 美化打印变量（不终止程序）
|
||
* 支持传入多个变量，如：dump($var1, $var2);
|
||
*/
|
||
if (!function_exists('dump')) {
|
||
function dump(...$vars) {
|
||
// 定义外层容器的 CSS 样式，使其在页面中醒目且整洁
|
||
$containerStyle = "
|
||
background-color: #282c34;
|
||
color: #abb2bf;
|
||
padding: 15px;
|
||
margin: 10px 0;
|
||
border-left: 5px solid #61afef;
|
||
border-radius: 4px;
|
||
font-family: Consolas, Monaco, 'Courier New', monospace;
|
||
font-size: 14px;
|
||
line-height: 1.6;
|
||
overflow-x: auto;
|
||
text-align: left;
|
||
box-shadow: 0 4px 6px rgba(0,0,0,0.1);
|
||
z-index: 9999;
|
||
position: relative;
|
||
";
|
||
|
||
echo "<div style=\"{$containerStyle}\">";
|
||
|
||
foreach ($vars as $var) {
|
||
echo "<pre style=\"margin: 0; padding: 5px 0;\">";
|
||
|
||
// 针对布尔值和 null 的特殊处理，因为 print_r 打印这些不直观
|
||
if (is_bool($var)) {
|
||
echo "<span style=\"color: #d19a66;\">bool</span>(" . ($var ? 'true' : 'false') . ")";
|
||
} elseif (is_null($var)) {
|
||
echo "<span style=\"color: #d19a66;\">null</span>";
|
||
} else {
|
||
// 使用 print_r 获取格式化字符串，并用 htmlspecialchars 防止 HTML 标签被浏览器解析
|
||
$output = htmlspecialchars(print_r($var, true));
|
||
|
||
// 简单的高亮：给数组的键名加上颜色
|
||
$output = preg_replace('/\[(.*?)\]/', '[<span style="color: #98c379;">$1</span>]', $output);
|
||
|
||
echo $output;
|
||
}
|
||
|
||
echo "</pre>";
|
||
|
||
// 如果有多个参数，用分割线隔开
|
||
if (count($vars) > 1 && $var !== end($vars)) {
|
||
echo "<hr style=\"border: 0; border-bottom: 1px dashed #5c6370; margin: 10px 0;\">";
|
||
}
|
||
}
|
||
|
||
echo "</div>";
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 美化打印变量并终止程序 (Dump and Die)
|
||
*/
|
||
if (!function_exists('dd')) {
|
||
function dd(...$vars) {
|
||
dump(...$vars);
|
||
die(1); // 终止程序执行
|
||
}
|
||
}
|
||
|
||
// 文件名: AbstractScrmSpider.php
|
||
|
||
require_once __DIR__ . '/UnifiedScrmData.class.php';
|
||
|
||
abstract class AbstractScrmSpider
|
||
{
|
||
const MODE_FETCH = 'fetch'; // 极速并发拉取 (推荐默认)
|
||
const MODE_UI = 'ui_click'; // 强制 UI 点击下一页
|
||
|
||
/** CLI 测试时输出 Node 请求/失败详情 */
|
||
protected $verbose = false;
|
||
|
||
protected $nodeHost;
|
||
|
||
public function __construct($nodeHost = 'http://127.0.0.1:3001')
|
||
{
|
||
$this->nodeHost = rtrim($nodeHost, '/');
|
||
}
|
||
|
||
/**
|
||
* 开启调试输出（测试脚本可调用）
|
||
*/
|
||
public function setVerbose($verbose = true)
|
||
{
|
||
$this->verbose = (bool) $verbose;
|
||
return $this;
|
||
}
|
||
|
||
/** ================= 子类必须实现的契约 ================= **/
|
||
abstract protected function getSpiderConfig();
|
||
|
||
// 以下方法的关注点，全部仅限于 List 接口！
|
||
abstract protected function extractListTotalPages($listFirstPageData, $countData = null);
|
||
abstract protected function buildListPageParams($page);
|
||
abstract protected function getUiPaginationConfig();
|
||
|
||
// 清洗方法直接接收两个明确的变量，告别繁杂的数组解析
|
||
abstract protected function parseToUnifiedData($detailData, $allListPagesData);
|
||
|
||
/** ================= 核心工作流 ================= **/
|
||
public function run()
|
||
{
|
||
$config = $this->getSpiderConfig();
|
||
|
||
$listApi = $config['listApi'];
|
||
$detailApi = $config['detailApi'] ?? null; // detail 接口是可选的
|
||
$countApi = $config['countApi'] ?? null; // 有些在线客服的总数通过单独调用接口获取
|
||
|
||
// 组装需要 Node 拦截的目标数组
|
||
$apiUrlsToIntercept = [$listApi];
|
||
if ($detailApi) { $apiUrlsToIntercept[] = $detailApi; }
|
||
if ($countApi) { $apiUrlsToIntercept[] = $countApi; }
|
||
|
||
$antiBot = $config['antiBot'] ?? null;
|
||
$mode = $config['paginationMode'] ?? self::MODE_FETCH;
|
||
|
||
// antiBot + UI 翻页：单 Browser 合并路径（与生产 SplitTicket 同步一致）
|
||
if ($this->shouldUseUnifiedBrowserPath($antiBot, $mode)) {
|
||
return $this->runUnifiedUiPath($config, $antiBot, $apiUrlsToIntercept, $listApi, $detailApi, $countApi);
|
||
}
|
||
|
||
// 【阶段一】：初始化并首屏拦截
|
||
$initResult = $this->requestNode('/api/auth-and-intercept', [
|
||
'pageUrl' => $config['pageUrl'],
|
||
'apiUrls' => $apiUrlsToIntercept,
|
||
'authActions' => $config['authActions'],
|
||
'antiBot' => $antiBot,
|
||
], $this->resolveAuthInterceptTimeout($antiBot));
|
||
|
||
if (empty($initResult['success'])) {
|
||
throw new Exception($this->formatNodeFailure('初始化失败', $initResult));
|
||
}
|
||
|
||
$interceptedApis = $initResult['interceptedApis'];
|
||
$cookies = $initResult['cookies'];
|
||
$finalPageUrl = $initResult['finalPageUrl'] ?? $config['pageUrl'];
|
||
|
||
$this->logDebug('auth-and-intercept ok', [
|
||
'finalPageUrl' => $finalPageUrl,
|
||
'intercepted' => array_keys($interceptedApis),
|
||
'cf' => $initResult['cf'] ?? null,
|
||
'browserReused' => $initResult['browserReused'] ?? null,
|
||
]);
|
||
|
||
// 必须拦截到 List 接口，否则无法继续
|
||
if (!isset($interceptedApis[$listApi])) {
|
||
throw new Exception("致命错误：未能拦截到必须的列表接口 [{$listApi}]");
|
||
}
|
||
|
||
// 分离 Detail 和 List 的首屏数据
|
||
$detailData = $detailApi && isset($interceptedApis[$detailApi]) ? $interceptedApis[$detailApi]['data'] : null;
|
||
$countData = $countApi && isset($interceptedApis[$countApi]) ? $interceptedApis[$countApi]['data'] : null;
|
||
// dd($detailData);
|
||
// dd($countData);
|
||
$listApiNode = $interceptedApis[$listApi];
|
||
$allListPagesData = [$listApiNode['data']]; // 初始化列表容器，装入第一页数据
|
||
// dd($allListPagesData);exit;
|
||
$totalPages = $this->extractListTotalPages($listApiNode['data'], $countData);
|
||
|
||
$mode = $config['paginationMode'] ?? self::MODE_FETCH;
|
||
|
||
// 如果总页数 > 1，触发【阶段二】
|
||
if ($totalPages > 1 || $totalPages === null) {
|
||
// 策略 1：极速 Fetch
|
||
if ($mode === self::MODE_FETCH) {
|
||
$paramList = [];
|
||
for ($page = 2; $page <= $totalPages; $page++) {
|
||
$paramList[] = $this->buildListPageParams($page);
|
||
}
|
||
|
||
$fetchResult = $this->requestNode('/api/batch-fetch', [
|
||
'tasks' => [[
|
||
'apiPath' => $listApi,
|
||
'fullUrl' => $listApiNode['url'],
|
||
'headers' => $listApiNode['headers'] ?? "",
|
||
'paramList' => $paramList,
|
||
'method' => $config['listMethod'] ?? 'GET'
|
||
]],
|
||
'cookies' => $cookies,
|
||
'antiBot' => $antiBot,
|
||
], 120);
|
||
|
||
if (!empty($fetchResult['success'])) {
|
||
foreach ($fetchResult['results'][$listApi] as $pResult) {
|
||
if ($pResult['success']) $allListPagesData[] = $pResult['data'];
|
||
}
|
||
}
|
||
}
|
||
// 策略 2：强制 UI 点击
|
||
elseif ($mode === self::MODE_UI) {
|
||
$uiConfig = $this->getUiPaginationConfig();
|
||
|
||
// 🔥 核心修改：直接将第一页的数组传给 Node，让 Node 引擎自己去剔除时间戳并生成统一哈希
|
||
$firstPageData = $listApiNode['data'];
|
||
// 🔥 核心重构 2：如果是 null 盲点模式，下发 9999 次点击任务；否则按实际计算次数
|
||
$clicksToPerform = ($totalPages === null) ? 9999 : ($totalPages - 1);
|
||
|
||
$uiResult = $this->requestNode('/api/ui-pagination', [
|
||
'apiUrl' => $listApi,
|
||
'pageUrl' => $config['pageUrl'],
|
||
'finalPageUrl' => $finalPageUrl,
|
||
'nextBtnSelector' => $uiConfig['nextBtnSelector'],
|
||
'waitMs' => $uiConfig['waitMs'] ?? 2000,
|
||
'clicksToPerform' => $clicksToPerform,
|
||
'cookies' => $cookies,
|
||
'firstPageData' => $firstPageData, // 传递原始数据源
|
||
// 🔥 核心补充：把登录剧本原封不动地传给翻页引擎！
|
||
'authActions' => $config['authActions'],
|
||
'antiBot' => $antiBot,
|
||
], 1200); // 增加 PHP 端的 cURL 超时时间到 400 秒，以包容多次重试产生的耗时
|
||
|
||
if (!empty($uiResult['success']) && !empty($uiResult['data'])) {
|
||
foreach ($uiResult['data'] as $pageData) {
|
||
$allListPagesData[] = $pageData;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
// 注：如果 $totalPages <= 1，直接自然跳过上述区块，进入第三阶段。
|
||
|
||
// 【阶段三】：移交数据进行最终清洗
|
||
// return $this->parseToUnifiedData($detailData, $allListPagesData);
|
||
|
||
// 【阶段三】：解析并打包数据
|
||
$unifiedData = $this->parseToUnifiedData($detailData, $allListPagesData);
|
||
|
||
// 🔥 核心修改：引入业务对账雷达
|
||
// if (!$this->validateDataIntegrity($unifiedData)) {
|
||
// 方案 A：直接返回 false（符合你的特殊要求）
|
||
// return false;
|
||
|
||
// 方案 B（工业级推荐）：抛出自定义异常。
|
||
// 因为返回 false 会让你在外部不知道到底是“网络超时”失败的，还是“数据对账没对上”失败的。
|
||
// throw new Exception("抓取校验异常：数据未成功获取，或客服数量业务对账失败！");
|
||
// }
|
||
|
||
return $unifiedData;
|
||
}
|
||
|
||
/**
|
||
* 单 Browser 路径：auth + 拦截 + UI 翻页（需 antiBot.sessionKey）
|
||
*/
|
||
protected function runUnifiedUiPath($config, $antiBot, $apiUrlsToIntercept, $listApi, $detailApi, $countApi)
|
||
{
|
||
$uiConfig = $this->getUiPaginationConfig();
|
||
$sessionKey = is_array($antiBot) ? (string) ($antiBot['sessionKey'] ?? '') : '';
|
||
|
||
$this->logDebug('unified browser path', [
|
||
'sessionKey' => $sessionKey,
|
||
'listApi' => $listApi,
|
||
]);
|
||
|
||
$mergedResult = $this->requestNode('/api/auth-intercept-and-paginate', [
|
||
'pageUrl' => $config['pageUrl'],
|
||
'apiUrls' => $apiUrlsToIntercept,
|
||
'authActions' => $config['authActions'] ?? [],
|
||
'antiBot' => $antiBot,
|
||
'pagination' => [
|
||
'apiUrl' => $listApi,
|
||
'nextBtnSelector' => $uiConfig['nextBtnSelector'] ?? '',
|
||
'waitMs' => $uiConfig['waitMs'] ?? 2000,
|
||
'clicksToPerform' => 9999,
|
||
],
|
||
], 1200);
|
||
|
||
if (empty($mergedResult['success'])) {
|
||
throw new Exception($this->formatNodeFailure('合并同步失败', $mergedResult));
|
||
}
|
||
|
||
$this->logDebug('auth-intercept-and-paginate ok', [
|
||
'intercepted' => array_keys($mergedResult['interceptedApis'] ?? []),
|
||
'extraPages' => count($mergedResult['extraPages'] ?? []),
|
||
'cf' => $mergedResult['cf'] ?? null,
|
||
'browserReused' => $mergedResult['browserReused'] ?? null,
|
||
'finalPageUrl' => $mergedResult['finalPageUrl'] ?? null,
|
||
]);
|
||
|
||
$interceptedApis = $mergedResult['interceptedApis'] ?? [];
|
||
if (!isset($interceptedApis[$listApi])) {
|
||
throw new Exception("致命错误：未能拦截到必须的列表接口 [{$listApi}]");
|
||
}
|
||
|
||
$detailData = $detailApi && isset($interceptedApis[$detailApi])
|
||
? $interceptedApis[$detailApi]['data'] : null;
|
||
$countData = $countApi && isset($interceptedApis[$countApi])
|
||
? $interceptedApis[$countApi]['data'] : null;
|
||
|
||
$listApiNode = $interceptedApis[$listApi];
|
||
$allListPagesData = [$listApiNode['data']];
|
||
if (!empty($mergedResult['extraPages']) && is_array($mergedResult['extraPages'])) {
|
||
foreach ($mergedResult['extraPages'] as $pageData) {
|
||
$allListPagesData[] = $pageData;
|
||
}
|
||
}
|
||
|
||
$this->extractListTotalPages($listApiNode['data'], $countData);
|
||
|
||
return $this->parseToUnifiedData($detailData, $allListPagesData);
|
||
}
|
||
|
||
/**
|
||
* antiBot 启用且配置了 sessionKey 时，UI 翻页走合并 Browser 路径
|
||
*/
|
||
protected function shouldUseUnifiedBrowserPath($antiBot, $mode)
|
||
{
|
||
if ($mode !== self::MODE_UI) {
|
||
return false;
|
||
}
|
||
if (!is_array($antiBot) || empty($antiBot['enabled'])) {
|
||
return false;
|
||
}
|
||
return !empty($antiBot['sessionKey']);
|
||
}
|
||
|
||
/**
|
||
* 格式化 Node 失败信息，附带 CF / 落地页等诊断字段
|
||
*/
|
||
protected function formatNodeFailure($prefix, $result)
|
||
{
|
||
$cfCode = (string) ($result['code'] ?? '');
|
||
if ($cfCode === 'CF_TURNSTILE_FAILED') {
|
||
return 'Cloudflare Turnstile 验证失败: ' . ($result['stage'] ?? 'timeout');
|
||
}
|
||
|
||
$parts = [$prefix . ': ' . ($result['error'] ?? '未知')];
|
||
|
||
if (!empty($result['finalPageUrl'])) {
|
||
$parts[] = 'finalPageUrl=' . $result['finalPageUrl'];
|
||
}
|
||
if (!empty($result['page']) && is_array($result['page'])) {
|
||
$page = $result['page'];
|
||
if (!empty($page['url'])) {
|
||
$parts[] = 'pageUrl=' . $page['url'];
|
||
}
|
||
if (isset($page['elInputInner'])) {
|
||
$parts[] = 'elInputInner=' . $page['elInputInner'];
|
||
}
|
||
}
|
||
if (!empty($result['cf']) && is_array($result['cf'])) {
|
||
$cf = $result['cf'];
|
||
$cfBits = [];
|
||
if (isset($cf['cfDetected'])) {
|
||
$cfBits[] = 'cfDetected=' . ($cf['cfDetected'] ? 'true' : 'false');
|
||
}
|
||
if (!empty($cf['turnstileStage'])) {
|
||
$cfBits[] = 'turnstileStage=' . $cf['turnstileStage'];
|
||
}
|
||
if (isset($cf['solverUsed'])) {
|
||
$cfBits[] = 'solverUsed=' . ($cf['solverUsed'] ? 'true' : 'false');
|
||
}
|
||
if (isset($cf['elapsedMs'])) {
|
||
$cfBits[] = 'cfElapsedMs=' . $cf['elapsedMs'];
|
||
}
|
||
if (!empty($cfBits)) {
|
||
$parts[] = implode(', ', $cfBits);
|
||
}
|
||
}
|
||
|
||
return implode(' | ', $parts);
|
||
}
|
||
|
||
protected function logDebug($message, $context = [])
|
||
{
|
||
if (!$this->verbose) {
|
||
return;
|
||
}
|
||
$line = '[Spider] ' . $message;
|
||
if (!empty($context)) {
|
||
$line .= ' ' . json_encode($context, JSON_UNESCAPED_UNICODE);
|
||
}
|
||
echo $line . "\n";
|
||
}
|
||
|
||
/**
|
||
* Real Browser + Turnstile 首屏耗时较长，单独放宽 auth-and-intercept 超时
|
||
*
|
||
* @param array<string, mixed>|null $antiBot
|
||
*/
|
||
private function resolveAuthInterceptTimeout($antiBot)
|
||
{
|
||
if (!is_array($antiBot) || empty($antiBot['enabled'])) {
|
||
return 120;
|
||
}
|
||
return 180;
|
||
}
|
||
|
||
private function requestNode($endpoint, $payload, $timeout = 60)
|
||
{
|
||
$maxAttempts = 3;
|
||
$lastDecoded = [];
|
||
for ($attempt = 1; $attempt <= $maxAttempts; $attempt++) {
|
||
$lastDecoded = $this->doRequestNode($endpoint, $payload, $timeout, $attempt);
|
||
if (!empty($lastDecoded['success'])) {
|
||
return $lastDecoded;
|
||
}
|
||
$error = (string) ($lastDecoded['error'] ?? '');
|
||
if (!$this->shouldRetryNodeResponse($lastDecoded, $error) || $attempt >= $maxAttempts) {
|
||
return $lastDecoded;
|
||
}
|
||
$this->logDebug('node retry', [
|
||
'endpoint' => $endpoint,
|
||
'attempt' => $attempt + 1,
|
||
'error' => $error,
|
||
]);
|
||
sleep($attempt);
|
||
}
|
||
return $lastDecoded;
|
||
}
|
||
|
||
private function doRequestNode($endpoint, $payload, $timeout, $attempt)
|
||
{
|
||
$url = $this->nodeHost . $endpoint;
|
||
$this->logDebug('node request', [
|
||
'url' => $url,
|
||
'timeout' => $timeout,
|
||
'attempt' => $attempt,
|
||
]);
|
||
|
||
$ch = curl_init($url);
|
||
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
||
curl_setopt($ch, CURLOPT_POST, true);
|
||
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($payload, JSON_UNESCAPED_UNICODE));
|
||
curl_setopt($ch, CURLOPT_HTTPHEADER, ['Content-Type: application/json']);
|
||
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5);
|
||
curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
|
||
$response = curl_exec($ch);
|
||
$httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||
if (curl_errno($ch)) {
|
||
$err = curl_error($ch);
|
||
curl_close($ch);
|
||
throw new Exception($err);
|
||
}
|
||
curl_close($ch);
|
||
|
||
$decoded = json_decode((string) $response, true);
|
||
if (!is_array($decoded)) {
|
||
return ['success' => false, 'error' => 'Node 返回非 JSON', 'httpCode' => $httpCode];
|
||
}
|
||
if ($httpCode === 503) {
|
||
$decoded['success'] = false;
|
||
$decoded['error'] = (string) ($decoded['error'] ?? '服务繁忙');
|
||
$decoded['httpCode'] = 503;
|
||
}
|
||
return $decoded;
|
||
}
|
||
|
||
private function shouldRetryNodeResponse($decoded, $error)
|
||
{
|
||
if ((int) ($decoded['httpCode'] ?? 0) === 503) {
|
||
return true;
|
||
}
|
||
$lower = strtolower($error);
|
||
foreach (['排队超时', 'queue_timeout', 'timeout', 'net::', 'navigation', 'connection'] as $needle) {
|
||
if ($needle !== '' && strpos($lower, $needle) !== false) {
|
||
return true;
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
|
||
/**
|
||
* 🔥 新增：数据完整性与业务对账校验
|
||
* @param array $unifiedData 已经解析好的统一格式数据
|
||
* @return bool 校验成功返回 true，失败返回 false
|
||
*/
|
||
protected function validateDataIntegrity($unifiedData)
|
||
{
|
||
// 1. 底线校验：如果最核心的列表数据或详情数据根本就是空的，直接判定失败
|
||
if (empty($unifiedData) || !is_array($unifiedData)) {
|
||
return false;
|
||
}
|
||
|
||
// 2. 这里可以放一些全系统通用的基础校验（如果有的话）
|
||
|
||
return true;
|
||
}
|
||
} |