完整版V1 加入爬虫功能
This commit is contained in:
root
@@ -0,0 +1,246 @@
|
||||
<?php
|
||||
|
||||
/**
|
||||
* 美化打印变量（不终止程序）
|
||||
* 支持传入多个变量，如：dump($var1, $var2);
|
||||
*/
|
||||
if (!function_exists('dump')) {
|
||||
function dump(...$vars) {
|
||||
// 定义外层容器的 CSS 样式，使其在页面中醒目且整洁
|
||||
$containerStyle = "
|
||||
background-color: #282c34;
|
||||
color: #abb2bf;
|
||||
padding: 15px;
|
||||
margin: 10px 0;
|
||||
border-left: 5px solid #61afef;
|
||||
border-radius: 4px;
|
||||
font-family: Consolas, Monaco, 'Courier New', monospace;
|
||||
font-size: 14px;
|
||||
line-height: 1.6;
|
||||
overflow-x: auto;
|
||||
text-align: left;
|
||||
box-shadow: 0 4px 6px rgba(0,0,0,0.1);
|
||||
z-index: 9999;
|
||||
position: relative;
|
||||
";
|
||||
|
||||
echo "<div style=\"{$containerStyle}\">";
|
||||
|
||||
foreach ($vars as $var) {
|
||||
echo "<pre style=\"margin: 0; padding: 5px 0;\">";
|
||||
|
||||
// 针对布尔值和 null 的特殊处理，因为 print_r 打印这些不直观
|
||||
if (is_bool($var)) {
|
||||
echo "<span style=\"color: #d19a66;\">bool</span>(" . ($var ? 'true' : 'false') . ")";
|
||||
} elseif (is_null($var)) {
|
||||
echo "<span style=\"color: #d19a66;\">null</span>";
|
||||
} else {
|
||||
// 使用 print_r 获取格式化字符串，并用 htmlspecialchars 防止 HTML 标签被浏览器解析
|
||||
$output = htmlspecialchars(print_r($var, true));
|
||||
|
||||
// 简单的高亮：给数组的键名加上颜色
|
||||
$output = preg_replace('/\[(.*?)\]/', '[<span style="color: #98c379;">$1</span>]', $output);
|
||||
|
||||
echo $output;
|
||||
}
|
||||
|
||||
echo "</pre>";
|
||||
|
||||
// 如果有多个参数，用分割线隔开
|
||||
if (count($vars) > 1 && $var !== end($vars)) {
|
||||
echo "<hr style=\"border: 0; border-bottom: 1px dashed #5c6370; margin: 10px 0;\">";
|
||||
}
|
||||
}
|
||||
|
||||
echo "</div>";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 美化打印变量并终止程序 (Dump and Die)
|
||||
*/
|
||||
if (!function_exists('dd')) {
|
||||
function dd(...$vars) {
|
||||
dump(...$vars);
|
||||
die(1); // 终止程序执行
|
||||
}
|
||||
}
|
||||
|
||||
// 文件名: AbstractScrmSpider.php
|
||||
|
||||
require_once __DIR__ . '/UnifiedScrmData.class.php';
|
||||
|
||||
abstract class AbstractScrmSpider
|
||||
{
|
||||
const MODE_FETCH = 'fetch'; // 极速并发拉取 (推荐默认)
|
||||
const MODE_UI = 'ui_click'; // 强制 UI 点击下一页
|
||||
|
||||
protected $nodeHost;
|
||||
|
||||
public function __construct($nodeHost = 'http://127.0.0.1:3001')
|
||||
{
|
||||
$this->nodeHost = $nodeHost;
|
||||
}
|
||||
|
||||
/** ================= 子类必须实现的契约 ================= **/
|
||||
abstract protected function getSpiderConfig();
|
||||
|
||||
// 以下方法的关注点，全部仅限于 List 接口！
|
||||
abstract protected function extractListTotalPages($listFirstPageData, $countData = null);
|
||||
abstract protected function buildListPageParams($page);
|
||||
abstract protected function getUiPaginationConfig();
|
||||
|
||||
// 清洗方法直接接收两个明确的变量，告别繁杂的数组解析
|
||||
abstract protected function parseToUnifiedData($detailData, $allListPagesData);
|
||||
|
||||
/** ================= 核心工作流 ================= **/
|
||||
public function run()
|
||||
{
|
||||
$config = $this->getSpiderConfig();
|
||||
|
||||
$listApi = $config['listApi'];
|
||||
$detailApi = $config['detailApi'] ?? null; // detail 接口是可选的
|
||||
$countApi = $config['countApi'] ?? null; // 有些在线客服的总数通过单独调用接口获取
|
||||
|
||||
// 组装需要 Node 拦截的目标数组
|
||||
$apiUrlsToIntercept = [$listApi];
|
||||
if ($detailApi) { $apiUrlsToIntercept[] = $detailApi; }
|
||||
if ($countApi) { $apiUrlsToIntercept[] = $countApi; }
|
||||
// var_dump($apiUrlsToIntercept);
|
||||
// 【阶段一】：初始化并首屏拦截
|
||||
$initResult = $this->requestNode('/api/auth-and-intercept', [
|
||||
'pageUrl' => $config['pageUrl'],
|
||||
'apiUrls' => $apiUrlsToIntercept,
|
||||
'authActions' => $config['authActions']
|
||||
]);
|
||||
|
||||
if (empty($initResult['success'])) {
|
||||
throw new Exception("初始化失败: " . ($initResult['error'] ?? '未知'));
|
||||
}
|
||||
|
||||
$interceptedApis = $initResult['interceptedApis'];
|
||||
$cookies = $initResult['cookies'];
|
||||
// dd($interceptedApis);
|
||||
// 必须拦截到 List 接口，否则无法继续
|
||||
if (!isset($interceptedApis[$listApi])) {
|
||||
throw new Exception("致命错误：未能拦截到必须的列表接口 [{$listApi}]");
|
||||
}
|
||||
|
||||
// 分离 Detail 和 List 的首屏数据
|
||||
$detailData = $detailApi && isset($interceptedApis[$detailApi]) ? $interceptedApis[$detailApi]['data'] : null;
|
||||
$countData = $countApi && isset($interceptedApis[$countApi]) ? $interceptedApis[$countApi]['data'] : null;
|
||||
// dd($detailData);
|
||||
// dd($countData);
|
||||
$listApiNode = $interceptedApis[$listApi];
|
||||
$allListPagesData = [$listApiNode['data']]; // 初始化列表容器，装入第一页数据
|
||||
|
||||
$totalPages = $this->extractListTotalPages($listApiNode['data'], $countData);
|
||||
|
||||
$mode = $config['paginationMode'] ?? self::MODE_FETCH;
|
||||
|
||||
// 如果总页数 > 1，触发【阶段二】
|
||||
if ($totalPages > 1 || $totalPages === null) {
|
||||
// 策略 1：极速 Fetch
|
||||
if ($mode === self::MODE_FETCH) {
|
||||
$paramList = [];
|
||||
for ($page = 2; $page <= $totalPages; $page++) {
|
||||
$paramList[] = $this->buildListPageParams($page);
|
||||
}
|
||||
|
||||
$fetchResult = $this->requestNode('/api/batch-fetch', [
|
||||
'tasks' => [[
|
||||
'apiPath' => $listApi,
|
||||
'fullUrl' => $listApiNode['url'],
|
||||
'headers' => $listApiNode['headers'] ?? "",
|
||||
'paramList' => $paramList,
|
||||
'method' => $config['listMethod'] ?? 'GET'
|
||||
]],
|
||||
'cookies' => $cookies
|
||||
], 120);
|
||||
|
||||
if (!empty($fetchResult['success'])) {
|
||||
foreach ($fetchResult['results'][$listApi] as $pResult) {
|
||||
if ($pResult['success']) $allListPagesData[] = $pResult['data'];
|
||||
}
|
||||
}
|
||||
}
|
||||
// 策略 2：强制 UI 点击
|
||||
elseif ($mode === self::MODE_UI) {
|
||||
$uiConfig = $this->getUiPaginationConfig();
|
||||
|
||||
// 🔥 核心修改：直接将第一页的数组传给 Node，让 Node 引擎自己去剔除时间戳并生成统一哈希
|
||||
$firstPageData = $listApiNode['data'];
|
||||
// 🔥 核心重构 2：如果是 null 盲点模式，下发 9999 次点击任务；否则按实际计算次数
|
||||
$clicksToPerform = ($totalPages === null) ? 9999 : ($totalPages - 1);
|
||||
|
||||
$uiResult = $this->requestNode('/api/ui-pagination', [
|
||||
'apiUrl' => $listApi,
|
||||
'pageUrl' => $config['pageUrl'],
|
||||
'nextBtnSelector' => $uiConfig['nextBtnSelector'],
|
||||
'waitMs' => $uiConfig['waitMs'] ?? 2000,
|
||||
'clicksToPerform' => $clicksToPerform,
|
||||
'cookies' => $cookies,
|
||||
'firstPageData' => $firstPageData, // 传递原始数据源
|
||||
// 🔥 核心补充：把登录剧本原封不动地传给翻页引擎！
|
||||
'authActions' => $config['authActions']
|
||||
], 1200); // 增加 PHP 端的 cURL 超时时间到 400 秒，以包容多次重试产生的耗时
|
||||
|
||||
if (!empty($uiResult['success']) && !empty($uiResult['data'])) {
|
||||
foreach ($uiResult['data'] as $pageData) {
|
||||
$allListPagesData[] = $pageData;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// 注：如果 $totalPages <= 1，直接自然跳过上述区块，进入第三阶段。
|
||||
|
||||
// 【阶段三】：移交数据进行最终清洗
|
||||
// return $this->parseToUnifiedData($detailData, $allListPagesData);
|
||||
|
||||
// 【阶段三】：解析并打包数据
|
||||
$unifiedData = $this->parseToUnifiedData($detailData, $allListPagesData);
|
||||
|
||||
// 🔥 核心修改：引入业务对账雷达
|
||||
// if (!$this->validateDataIntegrity($unifiedData)) {
|
||||
// 方案 A：直接返回 false（符合你的特殊要求）
|
||||
// return false;
|
||||
|
||||
// 方案 B（工业级推荐）：抛出自定义异常。
|
||||
// 因为返回 false 会让你在外部不知道到底是“网络超时”失败的，还是“数据对账没对上”失败的。
|
||||
// throw new Exception("抓取校验异常：数据未成功获取，或客服数量业务对账失败！");
|
||||
// }
|
||||
|
||||
return $unifiedData;
|
||||
}
|
||||
|
||||
private function requestNode($endpoint, $payload, $timeout = 60)
|
||||
{
|
||||
$ch = curl_init($this->nodeHost . $endpoint);
|
||||
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
||||
curl_setopt($ch, CURLOPT_POST, true);
|
||||
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($payload));
|
||||
curl_setopt($ch, CURLOPT_HTTPHEADER, ['Content-Type: application/json']);
|
||||
curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
|
||||
$response = curl_exec($ch);
|
||||
if (curl_errno($ch)) throw new Exception(curl_error($ch));
|
||||
curl_close($ch);
|
||||
return json_decode($response, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* 🔥 新增：数据完整性与业务对账校验
|
||||
* @param array $unifiedData 已经解析好的统一格式数据
|
||||
* @return bool 校验成功返回 true，失败返回 false
|
||||
*/
|
||||
protected function validateDataIntegrity($unifiedData)
|
||||
{
|
||||
// 1. 底线校验：如果最核心的列表数据或详情数据根本就是空的，直接判定失败
|
||||
if (empty($unifiedData) || !is_array($unifiedData)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 2. 这里可以放一些全系统通用的基础校验（如果有的话）
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user