damon/links
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化爬虫，增加缓存，线程池，保存浏览器用户信息

Browse Source
This commit is contained in:
root
2026-06-20 15:49:40 +08:00
parent e95f4a227c
commit 09bfa9ae01
26 changed files with 1584 additions and 454 deletions
Download Patch File Download Diff File Expand all files Collapse all files
_php_code/AbstractScrmSpider.class.php
+27 -4
View File
@@ -112,14 +112,21 @@ abstract class AbstractScrmSpider
// 'apiUrls' => $apiUrlsToIntercept,
// 'authActions' => $config['authActions']
// ]);
$antiBot = $config['antiBot'] ?? null;
// 【阶段一】：初始化并首屏拦截
$initResult = $this->requestNode('/api/auth-and-intercept', [
'pageUrl' => $config['pageUrl'],
'apiUrls' => $apiUrlsToIntercept,
'authActions' => $config['authActions']
]);
'authActions' => $config['authActions'],
'antiBot' => $antiBot,
], $this->resolveAuthInterceptTimeout($antiBot));
if (empty($initResult['success'])) {
$cfCode = (string) ($initResult['code'] ?? '');
if ($cfCode === 'CF_TURNSTILE_FAILED') {
throw new Exception('Cloudflare Turnstile 验证失败: ' . ($initResult['stage'] ?? 'timeout'));
}
throw new Exception("初始化失败: " . ($initResult['error'] ?? '未知'));
}
@@ -161,7 +168,8 @@ abstract class AbstractScrmSpider
'paramList' => $paramList,
'method' => $config['listMethod'] ?? 'GET'
]],
'cookies' => $cookies
'cookies' => $cookies,
'antiBot' => $antiBot,
], 120);
if (!empty($fetchResult['success'])) {
@@ -189,7 +197,8 @@ abstract class AbstractScrmSpider
'cookies' => $cookies,
'firstPageData' => $firstPageData, // 传递原始数据源
// 🔥 核心补充：把登录剧本原封不动地传给翻页引擎！
'authActions' => $config['authActions']
'authActions' => $config['authActions'],
'antiBot' => $antiBot,
], 1200); // 增加 PHP 端的 cURL 超时时间到 400 秒，以包容多次重试产生的耗时
if (!empty($uiResult['success']) && !empty($uiResult['data'])) {
@@ -220,6 +229,19 @@ abstract class AbstractScrmSpider
return $unifiedData;
}
/**
* Real Browser + Turnstile 首屏耗时较长，单独放宽 auth-and-intercept 超时
*
* @param array<string, mixed>|null $antiBot
*/
private function resolveAuthInterceptTimeout($antiBot)
{
if (!is_array($antiBot) || empty($antiBot['enabled'])) {
return 120;
}
return 180;
}
private function requestNode($endpoint, $payload, $timeout = 60)
{
$ch = curl_init($this->nodeHost . $endpoint);
@@ -227,6 +249,7 @@ abstract class AbstractScrmSpider
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($payload));
curl_setopt($ch, CURLOPT_HTTPHEADER, ['Content-Type: application/json']);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5);
curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
$response = curl_exec($ch);
if (curl_errno($ch)) throw new Exception(curl_error($ch));
_php_code/Chatknow.php
+12 -1
View File
@@ -26,6 +26,8 @@ class Chatknow extends AbstractScrmSpider
protected function getSpiderConfig()
{
$host = (string) parse_url($this->pageUrl, PHP_URL_HOST);
return [
'pageUrl' => $this->pageUrl,
'apiUrls' => [self::API_LIST],
@@ -43,7 +45,16 @@ class Chatknow extends AbstractScrmSpider
// ['type' => 'vue_click', 'selector' => '.layui-btn', 'text' => '搜索'],
['type' => 'wait', 'ms' => 4000]
]
],
// ChatKnow 专用：Real Browser + Turnstile + Captcha API 兜底 + 会话复用
'antiBot' => [
'enabled' => true,
'profile' => 'real',
'turnstile' => true,
'solverFallback' => true,
'sessionKey' => 'chatknow:' . $host,
'challengeTimeoutMs' => 60000,
],
];
}
_php_code/index.php
+50 -50
View File
@@ -11,58 +11,36 @@ require_once __DIR__ . '/SsCustomer.php'; // SS云控(Customer)
require_once __DIR__ . '/Haiwang.php'; // 海王
require_once __DIR__ . '/Whatshub.php'; // Whatshub 工单平台
try {
/*
命令行可以测试Cloudflare防火墙
curl -s -X POST http://127.0.0.1:3001/api/auth-and-intercept \
-H 'Content-Type: application/json' \
-d '{
"pageUrl": "https://web.whatshub.cc/m/iTYsWKQH5030/1",
"apiUrls": ["/api/whatshub-counter/workShare/open/detail"],
"authActions": [
{"type": "vue_fill", "selector": ".el-input__inner", "value": "745030"},
{"type": "wait", "ms": 500},
{"type": "vue_click", "selector": ".vxe-button-group .theme--primary"},
{"type": "wait", "ms": 2200}
],
"antiBot": {
"enabled": true,
"profile": "real",
"turnstile": true,
"solverFallback": true,
"sessionKey": "whatshub:t.flowerbells.top",
"challengeTimeoutMs": 60000
}
}' | jq .
*/
echo "🚀 开始执行<Whatshub 工单平台>抓取任务 (多引擎智能调度)...\n\r";
$pageUrl = 'https://web.whatshub.cc/m/iTYsWKQH5030/1'; // PageUrl 入口授权页
$username = ""; // 登录账号
$password = "745030"; // 登录密码
$spider = new Whatshub($pageUrl, $username, $password);
$finalData = $spider->run();
echo "✅ 任务完成！统一数据如下：\n\r";
echo "----------------------------------------\n\r";
echo "当日新增：{$finalData->todayNewCount} 人\n\r";
echo "在线号码：{$finalData->totalOnline} 个\n\r";
echo "离线号码：{$finalData->totalOffline} 个\n\r";
echo "Total：" . $finalData->total . " 个号码\n\r";
echo "实际总共抓取：" . count($finalData->numbers) . " 个号码\n\r";
echo "号码列表：\n\r";
echo dd($finalData->numbers);
} catch (Exception $e) {
echo "🚨 抓取异常：" . $e->getMessage() . "\n\r";
}
// try {
// echo "🚀 开始执行<ChatKnow SCRM>抓取任务 (多引擎智能调度)...\n\r";
// $pageUrl = 'https://user.chatknow.com/child/workorder-share?shareKey=jf5t6MNrJ2mC76N'; // PageUrl 入口授权页
// /*
// 命令行可以测试Cloudflare防火墙
// curl -s -X POST http://127.0.0.1:3001/api/auth-and-intercept \
// -H 'Content-Type: application/json' \
// -d '{
// "pageUrl": "https://web.whatshub.cc/m/iTYsWKQH5030/1",
// "apiUrls": ["/api/whatshub-counter/workShare/open/detail"],
// "authActions": [
// {"type": "vue_fill", "selector": ".el-input__inner", "value": "745030"},
// {"type": "wait", "ms": 500},
// {"type": "vue_click", "selector": ".vxe-button-group .theme--primary"},
// {"type": "wait", "ms": 2200}
// ],
// "antiBot": {
// "enabled": true,
// "profile": "real",
// "turnstile": true,
// "solverFallback": true,
// "sessionKey": "whatshub:t.flowerbells.top",
// "challengeTimeoutMs": 60000
// }
// }' | jq .
// */
// echo "🚀 开始执行<Whatshub 工单平台>抓取任务 (多引擎智能调度)...\n\r";
// $pageUrl = 'https://web.whatshub.cc/m/iTYsWKQH5030/1'; // PageUrl 入口授权页
// $username = ""; // 登录账号
// $password = ""; // 登录密码
// $spider = new Chatknow($pageUrl, $username, $password);
// $password = "745030"; // 登录密码
// $spider = new Whatshub($pageUrl, $username, $password);
// $finalData = $spider->run();
// echo "✅ 任务完成！统一数据如下：\n\r";
@@ -78,6 +56,28 @@ try {
// echo "🚨 抓取异常：" . $e->getMessage() . "\n\r";
// }
try {
echo "🚀 开始执行<ChatKnow SCRM>抓取任务 (多引擎智能调度)...\n\r";
$pageUrl = 'https://user.chatknow.com/child/workorder-share?shareKey=jf5t6MNrJ2mC76N'; // PageUrl 入口授权页
$username = ""; // 登录账号
$password = ""; // 登录密码
$spider = new Chatknow($pageUrl, $username, $password);
$finalData = $spider->run();
echo "✅ 任务完成！统一数据如下：\n\r";
echo "----------------------------------------\n\r";
echo "当日新增：{$finalData->todayNewCount} 人\n\r";
echo "在线号码：{$finalData->totalOnline} 个\n\r";
echo "离线号码：{$finalData->totalOffline} 个\n\r";
echo "Total：" . $finalData->total . " 个号码\n\r";
echo "实际总共抓取：" . count($finalData->numbers) . " 个号码\n\r";
echo "号码列表：\n\r";
echo dd($finalData->numbers);
} catch (Exception $e) {
echo "🚨 抓取异常：" . $e->getMessage() . "\n\r";
}
// try {
// echo "🚀 开始执行<A2c云控>抓取任务 (多引擎智能调度)...\n\r";
// $pageUrl = 'https://yyk.ink/8415O53'; // PageUrl 入口授权页