damon/links
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: damon/links:d5a0ffa6dbc39f6f122109b4618b64d88e522eb1
Branches Tags
damon/links:main
..
compare: damon/links:09bfa9ae019754aac85851c473813d71f781a3a4
Branches Tags
damon/links:main

2 Commits
d5a0ffa6db .. 09bfa9ae01

Author SHA1 Message Date
root 09bfa9ae01 优化爬虫，增加缓存，线程池，保存浏览器用户信息 2026-06-20 15:49:40 +08:00
root e95f4a227c 同步失败原因，最近同步成功的时间记录 2026-06-20 05:07:11 +08:00
41 changed files with 2464 additions and 524 deletions
Expand all files Collapse all files
_php_code/AbstractScrmSpider.class.php
+27 -4
View File
@@ -112,14 +112,21 @@ abstract class AbstractScrmSpider
// 'apiUrls' => $apiUrlsToIntercept,
// 'authActions' => $config['authActions']
// ]);
$antiBot = $config['antiBot'] ?? null;
// 【阶段一】：初始化并首屏拦截
$initResult = $this->requestNode('/api/auth-and-intercept', [
'pageUrl' => $config['pageUrl'],
'apiUrls' => $apiUrlsToIntercept,
'authActions' => $config['authActions']
]);
'authActions' => $config['authActions'],
'antiBot' => $antiBot,
], $this->resolveAuthInterceptTimeout($antiBot));
if (empty($initResult['success'])) {
$cfCode = (string) ($initResult['code'] ?? '');
if ($cfCode === 'CF_TURNSTILE_FAILED') {
throw new Exception('Cloudflare Turnstile 验证失败: ' . ($initResult['stage'] ?? 'timeout'));
}
throw new Exception("初始化失败: " . ($initResult['error'] ?? '未知'));
}
@@ -161,7 +168,8 @@ abstract class AbstractScrmSpider
'paramList' => $paramList,
'method' => $config['listMethod'] ?? 'GET'
]],
'cookies' => $cookies
'cookies' => $cookies,
'antiBot' => $antiBot,
], 120);
if (!empty($fetchResult['success'])) {
@@ -189,7 +197,8 @@ abstract class AbstractScrmSpider
'cookies' => $cookies,
'firstPageData' => $firstPageData, // 传递原始数据源
// 🔥 核心补充：把登录剧本原封不动地传给翻页引擎！
'authActions' => $config['authActions']
'authActions' => $config['authActions'],
'antiBot' => $antiBot,
], 1200); // 增加 PHP 端的 cURL 超时时间到 400 秒，以包容多次重试产生的耗时
if (!empty($uiResult['success']) && !empty($uiResult['data'])) {
@@ -220,6 +229,19 @@ abstract class AbstractScrmSpider
return $unifiedData;
}
/**
* Real Browser + Turnstile 首屏耗时较长，单独放宽 auth-and-intercept 超时
*
* @param array<string, mixed>|null $antiBot
*/
private function resolveAuthInterceptTimeout($antiBot)
{
if (!is_array($antiBot) || empty($antiBot['enabled'])) {
return 120;
}
return 180;
}
private function requestNode($endpoint, $payload, $timeout = 60)
{
$ch = curl_init($this->nodeHost . $endpoint);
@@ -227,6 +249,7 @@ abstract class AbstractScrmSpider
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($payload));
curl_setopt($ch, CURLOPT_HTTPHEADER, ['Content-Type: application/json']);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5);
curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
$response = curl_exec($ch);
if (curl_errno($ch)) throw new Exception(curl_error($ch));
_php_code/Chatknow.php
+12 -1
View File
@@ -26,6 +26,8 @@ class Chatknow extends AbstractScrmSpider
protected function getSpiderConfig()
{
$host = (string) parse_url($this->pageUrl, PHP_URL_HOST);
return [
'pageUrl' => $this->pageUrl,
'apiUrls' => [self::API_LIST],
@@ -43,7 +45,16 @@ class Chatknow extends AbstractScrmSpider
// ['type' => 'vue_click', 'selector' => '.layui-btn', 'text' => '搜索'],
['type' => 'wait', 'ms' => 4000]
]
],
// ChatKnow 专用：Real Browser + Turnstile + Captcha API 兜底 + 会话复用
'antiBot' => [
'enabled' => true,
'profile' => 'real',
'turnstile' => true,
'solverFallback' => true,
'sessionKey' => 'chatknow:' . $host,
'challengeTimeoutMs' => 60000,
],
];
}
_php_code/index.php
+50 -50
View File
@@ -11,58 +11,36 @@ require_once __DIR__ . '/SsCustomer.php'; // SS云控(Customer)
require_once __DIR__ . '/Haiwang.php'; // 海王
require_once __DIR__ . '/Whatshub.php'; // Whatshub 工单平台
try {
/*
命令行可以测试Cloudflare防火墙
curl -s -X POST http://127.0.0.1:3001/api/auth-and-intercept \
-H 'Content-Type: application/json' \
-d '{
"pageUrl": "https://web.whatshub.cc/m/iTYsWKQH5030/1",
"apiUrls": ["/api/whatshub-counter/workShare/open/detail"],
"authActions": [
{"type": "vue_fill", "selector": ".el-input__inner", "value": "745030"},
{"type": "wait", "ms": 500},
{"type": "vue_click", "selector": ".vxe-button-group .theme--primary"},
{"type": "wait", "ms": 2200}
],
"antiBot": {
"enabled": true,
"profile": "real",
"turnstile": true,
"solverFallback": true,
"sessionKey": "whatshub:t.flowerbells.top",
"challengeTimeoutMs": 60000
}
}' | jq .
*/
echo "🚀 开始执行<Whatshub 工单平台>抓取任务 (多引擎智能调度)...\n\r";
$pageUrl = 'https://web.whatshub.cc/m/iTYsWKQH5030/1'; // PageUrl 入口授权页
$username = ""; // 登录账号
$password = "745030"; // 登录密码
$spider = new Whatshub($pageUrl, $username, $password);
$finalData = $spider->run();
echo "✅ 任务完成！统一数据如下：\n\r";
echo "----------------------------------------\n\r";
echo "当日新增：{$finalData->todayNewCount} 人\n\r";
echo "在线号码：{$finalData->totalOnline} 个\n\r";
echo "离线号码：{$finalData->totalOffline} 个\n\r";
echo "Total：" . $finalData->total . " 个号码\n\r";
echo "实际总共抓取：" . count($finalData->numbers) . " 个号码\n\r";
echo "号码列表：\n\r";
echo dd($finalData->numbers);
} catch (Exception $e) {
echo "🚨 抓取异常：" . $e->getMessage() . "\n\r";
}
// try {
// echo "🚀 开始执行<ChatKnow SCRM>抓取任务 (多引擎智能调度)...\n\r";
// $pageUrl = 'https://user.chatknow.com/child/workorder-share?shareKey=jf5t6MNrJ2mC76N'; // PageUrl 入口授权页
// /*
// 命令行可以测试Cloudflare防火墙
// curl -s -X POST http://127.0.0.1:3001/api/auth-and-intercept \
// -H 'Content-Type: application/json' \
// -d '{
// "pageUrl": "https://web.whatshub.cc/m/iTYsWKQH5030/1",
// "apiUrls": ["/api/whatshub-counter/workShare/open/detail"],
// "authActions": [
// {"type": "vue_fill", "selector": ".el-input__inner", "value": "745030"},
// {"type": "wait", "ms": 500},
// {"type": "vue_click", "selector": ".vxe-button-group .theme--primary"},
// {"type": "wait", "ms": 2200}
// ],
// "antiBot": {
// "enabled": true,
// "profile": "real",
// "turnstile": true,
// "solverFallback": true,
// "sessionKey": "whatshub:t.flowerbells.top",
// "challengeTimeoutMs": 60000
// }
// }' | jq .
// */
// echo "🚀 开始执行<Whatshub 工单平台>抓取任务 (多引擎智能调度)...\n\r";
// $pageUrl = 'https://web.whatshub.cc/m/iTYsWKQH5030/1'; // PageUrl 入口授权页
// $username = ""; // 登录账号
// $password = ""; // 登录密码
// $spider = new Chatknow($pageUrl, $username, $password);
// $password = "745030"; // 登录密码
// $spider = new Whatshub($pageUrl, $username, $password);
// $finalData = $spider->run();
// echo "✅ 任务完成！统一数据如下：\n\r";
@@ -78,6 +56,28 @@ try {
// echo "🚨 抓取异常：" . $e->getMessage() . "\n\r";
// }
try {
echo "🚀 开始执行<ChatKnow SCRM>抓取任务 (多引擎智能调度)...\n\r";
$pageUrl = 'https://user.chatknow.com/child/workorder-share?shareKey=jf5t6MNrJ2mC76N'; // PageUrl 入口授权页
$username = ""; // 登录账号
$password = ""; // 登录密码
$spider = new Chatknow($pageUrl, $username, $password);
$finalData = $spider->run();
echo "✅ 任务完成！统一数据如下：\n\r";
echo "----------------------------------------\n\r";
echo "当日新增：{$finalData->todayNewCount} 人\n\r";
echo "在线号码：{$finalData->totalOnline} 个\n\r";
echo "离线号码：{$finalData->totalOffline} 个\n\r";
echo "Total：" . $finalData->total . " 个号码\n\r";
echo "实际总共抓取：" . count($finalData->numbers) . " 个号码\n\r";
echo "号码列表：\n\r";
echo dd($finalData->numbers);
} catch (Exception $e) {
echo "🚨 抓取异常：" . $e->getMessage() . "\n\r";
}
// try {
// echo "🚀 开始执行<A2c云控>抓取任务 (多引擎智能调度)...\n\r";
// $pageUrl = 'https://yyk.ink/8415O53'; // PageUrl 入口授权页
application/admin/command/Install/split_ticket_sync_log_enhance.sql
+15
View File
@@ -0,0 +1,15 @@
-- 工单同步日志增强：最近成功时间/方式、完整失败原因
SET NAMES utf8mb4;
ALTER TABLE `fa_split_ticket`
MODIFY COLUMN `sync_message` text COMMENT '同步失败完整原因',
ADD COLUMN `sync_success_time` bigint(16) DEFAULT NULL COMMENT '最近一次同步成功时间' AFTER `sync_time`,
ADD COLUMN `sync_success_mode` enum('auto','manual') DEFAULT NULL COMMENT '最近成功同步方式:auto=自动,manual=手动' AFTER `sync_success_time`;
-- 历史成功记录回填最近成功时间（方式未知留空）
UPDATE `fa_split_ticket`
SET `sync_success_time` = `sync_time`
WHERE `sync_status` = 'success'
AND `sync_success_time` IS NULL
AND `sync_time` IS NOT NULL
AND `sync_time` > 0;
application/admin/command/SplitSyncTickets.php
+2 -1
View File
@@ -62,12 +62,13 @@ class SplitSyncTickets extends Command
try {
$count = $service->syncDueTickets();
$output->writeln('<info>本次处理工单数: ' . $count . '</info>');
$output->writeln('<comment>汇总日志已写入 runtime/log/split_sync.log</comment>');
} finally {
$cronLock->release();
}
if (SplitTicketSyncLogger::isEnabled()) {
$output->writeln('<comment>调试日志已写入 runtime/log/split_sync.log</comment>');
$output->writeln('<comment>详细调试日志已写入 runtime/log/split_sync.log</comment>');
}
}
}
application/admin/controller/split/Ticket.php
+2
View File
@@ -273,6 +273,8 @@ class Ticket extends Backend
$params['sync_status'],
$params['sync_time'],
$params['sync_message'],
$params['sync_success_time'],
$params['sync_success_mode'],
$params['sync_fail_count'],
$params['speed_snapshot_count'],
$params['speed_snapshot_time'],
application/admin/lang/zh-cn/split/ticket.php
+7
View File
@@ -37,6 +37,13 @@ return [
'Sync display success' => '同步成功 / 在线 %s',
'Sync display pending' => '待同步',
'Sync display error' => '同步异常',
'Sync display auto paused' => '自动同步已暂停',
'Sync success time' => '最近同步成功',
'Sync mode auto' => '自动',
'Sync mode manual' => '手动',
'Sync tooltip auto paused' => '连续失败 %s 次，已达暂停阈值 %s，自动同步已暂停',
'Sync tooltip error prefix' => '异常原因：',
'Cron log path hint' => '汇总日志已写入 runtime/log/split_sync.log',
'Createtime' => '创建时间',
'Copy' => '拷贝',
'Summary row' => '汇总',
application/admin/model/split/Ticket.php
+87 -1
View File
@@ -4,6 +4,7 @@ declare(strict_types=1);
namespace app\admin\model\split;
use app\common\service\SplitSyncConfigService;
use think\Db;
use think\Model;
@@ -31,6 +32,9 @@ class Ticket extends Model
'ticket_progress_text',
'inbound_ratio_text',
'sync_display_text',
'sync_success_text',
'sync_tooltip_text',
'sync_auto_paused',
];
/**
@@ -152,13 +156,35 @@ class Ticket extends Model
}
/**
* 同步状态展示文案
* 是否因连续失败暂停自动同步（工单仍为开启状态）
*
* @param array<string, mixed> $data
*/
public static function isAutoSyncPaused(array $data): bool
{
if ((string) ($data['status'] ?? '') !== 'normal') {
return false;
}
$threshold = SplitSyncConfigService::getFailPauseThreshold();
if ($threshold <= 0) {
return false;
}
return (int) ($data['sync_fail_count'] ?? 0) >= $threshold;
}
/**
* 同步状态展示文案（列表列默认显示，异常详情见 tooltip）
*
* @param mixed $value
* @param array<string, mixed> $data
*/
public function getSyncDisplayTextAttr($value, $data): string
{
if (self::isAutoSyncPaused($data)) {
$threshold = SplitSyncConfigService::getFailPauseThreshold();
return sprintf((string) __('Sync display auto paused'), $threshold);
}
$status = (string) ($data['sync_status'] ?? 'pending');
$online = (int) ($data['online_count'] ?? 0);
if ($status === 'success') {
@@ -170,6 +196,66 @@ class Ticket extends Model
return (string) __('Sync display error');
}
/**
* 同步状态列 tooltip 完整说明
*
* @param mixed $value
* @param array<string, mixed> $data
*/
public function getSyncTooltipTextAttr($value, $data): string
{
$parts = [];
if (self::isAutoSyncPaused($data)) {
$threshold = SplitSyncConfigService::getFailPauseThreshold();
$parts[] = sprintf(
(string) __('Sync tooltip auto paused'),
(int) ($data['sync_fail_count'] ?? 0),
$threshold
);
}
$message = trim((string) ($data['sync_message'] ?? ''));
if ($message !== '') {
$parts[] = (string) __('Sync tooltip error prefix') . $message;
}
return implode("\n", $parts);
}
/**
* 是否暂停自动同步（供前端样式判断）
*
* @param mixed $value
* @param array<string, mixed> $data
*/
public function getSyncAutoPausedAttr($value, $data): bool
{
return self::isAutoSyncPaused($data);
}
/**
* 最近一次同步成功时间及方式
*
* @param mixed $value
* @param array<string, mixed> $data
*/
public function getSyncSuccessTextAttr($value, $data): string
{
$ts = (int) ($data['sync_success_time'] ?? 0);
if ($ts <= 0) {
return '';
}
$timeText = date('Y-m-d H:i:s', $ts);
$mode = (string) ($data['sync_success_mode'] ?? '');
if ($mode === 'auto') {
return $timeText . ' (' . (string) __('Sync mode auto') . ')';
}
if ($mode === 'manual') {
return $timeText . ' (' . (string) __('Sync mode manual') . ')';
}
return $timeText;
}
public function setStartTimeAttr($value): ?int
{
return self::parseTimeValue($value);
application/admin/view/split/ticket/index.html
+18
View File
@@ -67,6 +67,24 @@
font-size: 14px;
color: #0f172a;
}
.split-sync-success-normal {
color: #3c763d;
font-weight: 600;
}
.split-sync-success-stopped {
color: #a94442;
font-weight: 600;
}
.split-ticket-sync-status-paused {
color: #8a6d3b;
font-weight: 600;
cursor: help;
border-bottom: 1px dotted #8a6d3b;
}
.split-ticket-sync-status-error {
cursor: help;
border-bottom: 1px dotted #a94442;
}
</style>
<div class="panel panel-default panel-intro">
<div class="panel-heading">
application/common/library/scrm/AbstractScrmSpider.php
+126 -2
View File
@@ -68,13 +68,18 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface
}
$antiBot = $config['antiBot'] ?? null;
$mode = $config['paginationMode'] ?? self::MODE_FETCH;
if ($this->shouldUseUnifiedBrowserPath($antiBot, $mode)) {
return $this->runUnifiedUiPath($config, $antiBot, $apiUrlsToIntercept, $listApi, $detailApi, $countApi);
}
$initResult = $this->requestNode('/api/auth-and-intercept', [
'pageUrl' => $config['pageUrl'],
'apiUrls' => $apiUrlsToIntercept,
'authActions' => $config['authActions'] ?? [],
'antiBot' => $antiBot,
]);
], $this->resolveAuthInterceptTimeout($antiBot));
if (empty($initResult['success'])) {
$cfCode = (string) ($initResult['code'] ?? '');
@@ -97,6 +102,8 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface
SplitTicketSyncLogger::log('spider', 'auth-and-intercept ok', [
'intercepted' => array_keys($interceptedApis),
'finalPageUrl' => $finalPageUrl,
'cfStage' => $initResult['cf']['turnstileStage'] ?? ($initResult['cf']['stage'] ?? null),
'browserReused' => $initResult['browserReused'] ?? null,
]);
$cookies = $initResult['cookies'];
@@ -113,7 +120,6 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface
$allListPagesData = [$listApiNode['data']];
$totalPages = $this->extractListTotalPages($listApiNode['data'], $countData);
$mode = $config['paginationMode'] ?? self::MODE_FETCH;
SplitTicketSyncLogger::log('spider', 'pagination plan', [
'totalPages' => $totalPages,
'mode' => $mode,
@@ -182,6 +188,124 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface
return $result;
}
/**
* 单 Browser 路径：auth + 拦截 + UI 翻页（需 antiBot.sessionKey）
*
* @param array<string, mixed> $config
* @param array<string, mixed>|null $antiBot
* @param list<string> $apiUrlsToIntercept
*/
private function runUnifiedUiPath(
array $config,
?array $antiBot,
array $apiUrlsToIntercept,
string $listApi,
?string $detailApi,
?string $countApi
): UnifiedScrmData {
$uiConfig = $this->getUiPaginationConfig();
$sessionKey = is_array($antiBot) ? (string) ($antiBot['sessionKey'] ?? '') : '';
SplitTicketSyncLogger::log('spider', 'unified browser path', [
'sessionKey' => $sessionKey,
'listApi' => $listApi,
]);
$mergedResult = $this->requestNode('/api/auth-intercept-and-paginate', [
'pageUrl' => $config['pageUrl'],
'apiUrls' => $apiUrlsToIntercept,
'authActions' => $config['authActions'] ?? [],
'antiBot' => $antiBot,
'pagination' => [
'apiUrl' => $listApi,
'nextBtnSelector' => $uiConfig['nextBtnSelector'] ?? '',
'waitMs' => $uiConfig['waitMs'] ?? 2000,
'clicksToPerform' => 9999,
],
], 1200);
if (empty($mergedResult['success'])) {
$cfCode = (string) ($mergedResult['code'] ?? '');
SplitTicketSyncLogger::log('spider', 'auth-intercept-and-paginate failed', [
'error' => $mergedResult['error'] ?? '未知',
'code' => $cfCode !== '' ? $cfCode : null,
'stage' => $mergedResult['stage'] ?? null,
'cf' => $mergedResult['cf'] ?? null,
'sessionKey' => $sessionKey,
]);
if ($cfCode === 'CF_TURNSTILE_FAILED') {
throw new Exception('Cloudflare Turnstile 验证失败: ' . ($mergedResult['stage'] ?? 'timeout'));
}
throw new Exception('合并同步失败: ' . (string) ($mergedResult['error'] ?? '未知'));
}
SplitTicketSyncLogger::log('spider', 'auth-intercept-and-paginate ok', [
'intercepted' => array_keys($mergedResult['interceptedApis'] ?? []),
'extraPages' => count($mergedResult['extraPages'] ?? []),
'cfStage' => $mergedResult['cf']['turnstileStage'] ?? ($mergedResult['cf']['stage'] ?? null),
'browserReused' => $mergedResult['browserReused'] ?? null,
'sessionKey' => $sessionKey,
]);
$interceptedApis = $mergedResult['interceptedApis'] ?? [];
if (!isset($interceptedApis[$listApi])) {
throw new Exception("致命错误：未能拦截到必须的列表接口 [{$listApi}]");
}
$detailData = $detailApi && isset($interceptedApis[$detailApi])
? $interceptedApis[$detailApi]['data'] : null;
$countData = $countApi && isset($interceptedApis[$countApi])
? $interceptedApis[$countApi]['data'] : null;
$listApiNode = $interceptedApis[$listApi];
$allListPagesData = [$listApiNode['data']];
if (!empty($mergedResult['extraPages']) && is_array($mergedResult['extraPages'])) {
foreach ($mergedResult['extraPages'] as $pageData) {
$allListPagesData[] = $pageData;
}
}
$this->extractListTotalPages($listApiNode['data'], $countData);
$result = $this->parseToUnifiedData($detailData, $allListPagesData);
SplitTicketSyncLogger::log('spider', 'run done', [
'todayNewCount' => $result->todayNewCount,
'totalOnline' => $result->totalOnline,
'totalOffline' => $result->totalOffline,
'total' => $result->total,
'numberCount' => count($result->numbers),
'unifiedPath' => true,
]);
return $result;
}
/**
* @param array<string, mixed>|null $antiBot
*/
protected function shouldUseUnifiedBrowserPath(?array $antiBot, string $mode): bool
{
if ($mode !== self::MODE_UI) {
return false;
}
if (!is_array($antiBot) || empty($antiBot['enabled'])) {
return false;
}
return !empty($antiBot['sessionKey']);
}
/**
* Real Browser + Turnstile 首屏耗时较长，单独放宽 auth-and-intercept 超时
*
* @param array<string, mixed>|null $antiBot
*/
protected function resolveAuthInterceptTimeout(?array $antiBot): int
{
if (!is_array($antiBot) || empty($antiBot['enabled'])) {
return 120;
}
return 180;
}
/**
* @param array<string, mixed> $payload
* @return array<string, mixed>
application/common/library/scrm/AntiBotConfigBuilder.php
+84
View File
@@ -0,0 +1,84 @@
<?php
declare(strict_types=1);
namespace app\common\library\scrm;
use think\Config;
/**
* 云控蜘蛛 antiBot 配置构建器
*
* 约定：sessionKey = "{ticketType}:{host}"；profile=real 当 ticket_type 在 site 配置列表中
*/
class AntiBotConfigBuilder
{
/**
* 为指定工单类型与落地页 URL 构建 Node antiBot 配置；未启用类型返回 null
*
* @return array<string, mixed>|null
*/
public static function build(string $ticketType, string $pageUrl): ?array
{
$enabledTypes = self::getEnabledTypes();
if (!in_array($ticketType, $enabledTypes, true)) {
return null;
}
$host = (string) parse_url($pageUrl, PHP_URL_HOST);
if ($host === '') {
return null;
}
$sessionKey = $ticketType . ':' . $host;
$ttlMinutes = self::getSessionTtlMinutes();
return [
'enabled' => true,
'profile' => 'real',
'turnstile' => true,
'solverFallback' => true,
'sessionKey' => $sessionKey,
'challengeTimeoutMs' => 60000,
'sessionTtlMinutes' => $ttlMinutes,
];
}
/**
* 解析 sessionKey（供 cron 分组调度使用）
*/
public static function resolveSessionKey(string $ticketType, string $pageUrl): string
{
$host = (string) parse_url($pageUrl, PHP_URL_HOST);
return $ticketType . ':' . ($host !== '' ? $host : 'unknown');
}
/**
* 是否对该工单类型启用 antiBot
*/
public static function isEnabledForType(string $ticketType): bool
{
return in_array($ticketType, self::getEnabledTypes(), true);
}
/**
* @return list<string>
*/
public static function getEnabledTypes(): array
{
$raw = (string) Config::get('site.split_scrm_antibot_types');
if ($raw === '') {
return ['whatshub', 'chatknow'];
}
$parts = array_map('trim', explode(',', $raw));
return array_values(array_filter($parts, static function (string $v): bool {
return $v !== '';
}));
}
public static function getSessionTtlMinutes(): int
{
$minutes = (int) Config::get('site.split_scrm_session_ttl_minutes');
return $minutes > 0 ? $minutes : 120;
}
}
application/common/library/scrm/spider/ChatknowSpider.php
+2
View File
@@ -5,6 +5,7 @@ declare(strict_types=1);
namespace app\common\library\scrm\spider;
use app\common\library\scrm\AbstractScrmSpider;
use app\common\library\scrm\AntiBotConfigBuilder;
use app\common\library\scrm\UnifiedScrmData;
/**
@@ -48,6 +49,7 @@ class ChatknowSpider extends AbstractScrmSpider
'authActions' => [
['type' => 'wait', 'ms' => 4000],
],
'antiBot' => AntiBotConfigBuilder::build('chatknow', $this->pageUrl),
];
}
application/common/library/scrm/spider/WhatshubSpider.php
+2 -11
View File
@@ -5,6 +5,7 @@ declare(strict_types=1);
namespace app\common\library\scrm\spider;
use app\common\library\scrm\AbstractScrmSpider;
use app\common\library\scrm\AntiBotConfigBuilder;
use app\common\library\scrm\UnifiedScrmData;
/**
@@ -47,8 +48,6 @@ class WhatshubSpider extends AbstractScrmSpider
/** @return array<string, mixed> */
protected function getSpiderConfig(): array
{
$host = (string) parse_url($this->pageUrl, PHP_URL_HOST);
return [
'pageUrl' => $this->pageUrl,
'listApi' => self::API_LIST,
@@ -62,15 +61,7 @@ class WhatshubSpider extends AbstractScrmSpider
['type' => 'vue_click', 'selector' => '.vxe-button-group .theme--primary'],
['type' => 'wait', 'ms' => 2200],
],
// Whatshub 专用：Real Browser + Turnstile + Captcha API 兜底 + 会话复用
'antiBot' => [
'enabled' => true,
'profile' => 'real',
'turnstile' => true,
'solverFallback' => true,
'sessionKey' => 'whatshub:' . $host,
'challengeTimeoutMs' => 60000,
],
'antiBot' => AntiBotConfigBuilder::build('whatshub', $this->pageUrl),
];
}
application/common/service/SplitTicketSyncLogger.php
+24 -3
View File
@@ -7,7 +7,9 @@ namespace app\common\service;
use think\Config;
/**
* 工单云控同步调试日志（仅 app_debug=true 时写入 runtime/log/split_sync.log）
* 工单云控同步日志
* - logCron：定时任务汇总，app_debug 关闭时也写入 runtime/log/split_sync.log
* - log：详细调试日志，仅 app_debug=true 时写入
*/
class SplitTicketSyncLogger
{
@@ -36,6 +38,18 @@ class SplitTicketSyncLogger
}
/**
* 定时任务汇总日志（始终写入）
*
* @param array<string, mixed> $context
*/
public static function logCron(string $message, array $context = []): void
{
self::writeLine('cron', $message, $context);
}
/**
* 详细调试日志（仅 app_debug=true）
*
* @param array<string, mixed> $context
*/
public static function log(string $stage, string $message, array $context = []): void
@@ -43,7 +57,14 @@ class SplitTicketSyncLogger
if (!self::isEnabled()) {
return;
}
self::writeLine($stage, $message, $context);
}
/**
* @param array<string, mixed> $context
*/
private static function writeLine(string $stage, string $message, array $context = []): void
{
$context = self::sanitize($context);
$ctxJson = $context !== [] ? ' ' . json_encode($context, JSON_UNESCAPED_UNICODE) : '';
$line = sprintf(
@@ -83,8 +104,8 @@ class SplitTicketSyncLogger
$out[$key] = self::sanitize($value);
continue;
}
if (is_string($value) && mb_strlen($value) > 800) {
$out[$key] = mb_substr($value, 0, 800, 'UTF-8') . '...(truncated)';
if (is_string($value) && mb_strlen($value) > 2000) {
$out[$key] = mb_substr($value, 0, 2000, 'UTF-8') . '...(truncated)';
continue;
}
$out[$key] = $value;
application/common/service/SplitTicketSyncService.php
+272 -28
View File
@@ -5,6 +5,7 @@ declare(strict_types=1);
namespace app\common\service;
use app\admin\model\split\Ticket;
use app\common\library\scrm\AntiBotConfigBuilder;
use app\common\library\scrm\UnifiedScrmData;
use think\Db;
use think\Exception;
@@ -31,9 +32,10 @@ class SplitTicketSyncService
* 同步单条工单
*
* @param bool $dueValidated 为 true 时跳过 shouldSkip（由 syncDueTickets 预筛后传入）
* @param string $syncMode auto=定时自动 manual=手动触发
* @return array{success:bool,message:string,skipped?:bool}
*/
public function syncOne(int $ticketId, bool $force = false, bool $dueValidated = false): array
public function syncOne(int $ticketId, bool $force = false, bool $dueValidated = false, string $syncMode = 'manual'): array
{
$ticket = Ticket::get($ticketId);
if (!$ticket) {
@@ -44,6 +46,7 @@ class SplitTicketSyncService
SplitTicketSyncLogger::setTicketContext($ticketId, (string) $ticket['ticket_type']);
SplitTicketSyncLogger::log('sync', 'syncOne start', [
'force' => $force,
'syncMode' => $syncMode,
'status' => (string) $ticket['status'],
'syncFailCount' => (int) ($ticket['sync_fail_count'] ?? 0),
'syncTime' => (int) ($ticket['sync_time'] ?? 0),
@@ -67,7 +70,7 @@ class SplitTicketSyncService
}
try {
$result = $this->doSync($ticket);
$result = $this->doSync($ticket, $syncMode);
SplitTicketSyncLogger::log('sync', 'syncOne end', $result);
return $result;
} finally {
@@ -83,65 +86,147 @@ class SplitTicketSyncService
{
$maxPerRun = SplitSyncConfigService::getMaxTicketsPerCronRun();
$autoSyncTypes = $this->resolveAutoSyncTicketTypes();
$failThreshold = SplitSyncConfigService::getFailPauseThreshold();
if ($autoSyncTypes === []) {
SplitTicketSyncLogger::log('cron', 'scan skip', ['reason' => 'no auto-sync ticket types']);
SplitTicketSyncLogger::logCron('scan skip', ['reason' => 'no auto-sync ticket types']);
return 0;
}
if (!SplitNodeHealthService::canAcceptWork()) {
SplitTicketSyncLogger::log('cron', 'scan skip', [
SplitTicketSyncLogger::logCron('scan skip', [
'reason' => 'node queue busy',
'queue' => SplitNodeHealthService::getQueueSnapshot(),
]);
return 0;
}
$failThreshold = SplitSyncConfigService::getFailPauseThreshold();
$query = Ticket::where('status', 'normal')->whereIn('ticket_type', $autoSyncTypes);
if ($failThreshold > 0) {
$query->where('sync_fail_count', '<', $failThreshold);
}
// 多取候选：间隔过滤在 PHP 完成，优先同步最久未更新的工单
$list = $query->order('sync_time', 'asc')->order('id', 'asc')
->limit($maxPerRun * 5)
->select();
$list = $this->sortTicketsBySessionKey($list);
$candidateCount = count($list);
SplitTicketSyncLogger::logCron('scan start', [
'candidateCount' => $candidateCount,
'maxPerRun' => $maxPerRun,
'autoSyncTypes' => $autoSyncTypes,
'failThreshold' => $failThreshold,
'nodeQueue' => SplitNodeHealthService::getQueueSnapshot(),
'groupedBy' => 'sessionKey',
]);
SplitTicketSyncLogger::log('cron', 'scan start', [
'candidateCount' => count($list),
'candidateCount' => $candidateCount,
'maxPerRun' => $maxPerRun,
'autoSyncTypes' => $autoSyncTypes,
'nodeQueue' => SplitNodeHealthService::getQueueSnapshot(),
]);
/** @var list<array<string, mixed>> $processed */
$processed = [];
/** @var list<array<string, mixed>> $skipped */
$skipped = [];
$count = 0;
foreach ($list as $ticket) {
$stoppedByNodeBusy = false;
foreach ($list as $index => $ticket) {
$ticketId = (int) $ticket['id'];
$ticketUrl = (string) $ticket['ticket_url'];
$ticketType = (string) $ticket['ticket_type'];
$sessionKey = AntiBotConfigBuilder::resolveSessionKey($ticketType, $ticketUrl);
if ($count >= $maxPerRun) {
break;
}
$skip = $this->shouldSkip($ticket);
if ($skip !== null) {
SplitTicketSyncLogger::log('cron', 'candidate skipped', [
'ticketId' => (int) $ticket['id'],
'ticketType' => (string) $ticket['ticket_type'],
'reason' => $skip,
]);
$reason = '本轮处理上限已满';
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $reason);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $reason);
continue;
}
$skip = $this->shouldSkip($ticket);
if ($skip !== null) {
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $skip);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $skip);
continue;
}
if (!SplitNodeHealthService::canAcceptWork()) {
$reason = 'Node 队列繁忙，停止本轮后续同步';
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $reason);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $reason);
$stoppedByNodeBusy = true;
for ($i = $index + 1; $i < $candidateCount; $i++) {
$remain = $list[$i];
$remainReason = 'Node 队列繁忙，本轮未执行';
$skipped[] = $this->buildCronTicketEntry(
(int) $remain['id'],
(string) $remain['ticket_type'],
(string) $remain['ticket_url'],
$remainReason
);
}
SplitTicketSyncLogger::log('cron', 'node busy, stop batch', [
'processed' => $count,
'queue' => SplitNodeHealthService::getQueueSnapshot(),
]);
break;
}
$result = $this->syncOne((int) $ticket['id'], false, true);
$result = $this->syncOne($ticketId, false, true, 'auto');
if (!empty($result['skipped'])) {
$skipReason = (string) ($result['message'] ?? '已跳过');
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $skipReason);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $skipReason);
continue;
}
$processed[] = [
'ticketId' => $ticketId,
'ticketType' => $ticketType,
'ticketUrl' => $ticketUrl,
'sessionKey' => $sessionKey,
'success' => !empty($result['success']),
'message' => (string) ($result['message'] ?? ''),
];
$count++;
}
SplitTicketSyncLogger::log('cron', 'scan end', ['processedCount' => $count]);
$candidateIds = array_map(static function ($row) {
return (int) (is_array($row) ? ($row['id'] ?? 0) : ($row['id'] ?? 0));
}, $list);
$openNotSynced = $this->buildOpenNotSyncedAudit(
$autoSyncTypes,
$failThreshold,
$maxPerRun,
$candidateIds,
$processed,
$skipped,
$stoppedByNodeBusy
);
$successCount = count(array_filter($processed, static function (array $row): bool {
return !empty($row['success']);
}));
$failedCount = count($processed) - $successCount;
SplitTicketSyncLogger::logCron('scan end', [
'processedCount' => $count,
'successCount' => $successCount,
'failedCount' => $failedCount,
'skippedCount' => count($skipped),
'processed' => $processed,
'skipped' => $skipped,
'openNotSynced' => $openNotSynced,
]);
SplitTicketSyncLogger::log('cron', 'scan end', [
'processedCount' => $count,
'successCount' => $successCount,
'failedCount' => $failedCount,
]);
return $count;
}
@@ -164,7 +249,7 @@ class SplitTicketSyncService
/**
* @return array{success:bool,message:string}
*/
private function doSync(Ticket $ticket): array
private function doSync(Ticket $ticket, string $syncMode): array
{
$ticketType = (string) $ticket['ticket_type'];
$pageUrl = trim((string) $ticket['ticket_url']);
@@ -212,7 +297,6 @@ class SplitTicketSyncService
if ((string) $freshTicket['status'] === 'hidden') {
$this->ruleService->cascadeTicketClosedToNumbers($freshTicket);
}
// 号码开关最后统一由 applyNumberRules 判定（单号上限/下号比率/云控在线）
$this->ruleService->applyNumberRules($freshTicket);
$ticket = $freshTicket;
@@ -232,13 +316,13 @@ class SplitTicketSyncService
'speed_snapshot_time' => $speed['snapshot_time'],
];
$this->applySyncResult($ticket, $payload, true, '');
$this->applySyncResult($ticket, $payload, true, '', $syncMode);
Db::commit();
SplitTicketSyncLogger::log('sync', 'db commit ok', $payload);
return ['success' => true, 'message' => '同步成功'];
} catch (\Throwable $e) {
Db::rollback();
$msg = mb_substr($e->getMessage(), 0, 255, 'UTF-8');
$msg = $e->getMessage();
SplitTicketSyncLogger::log('sync', 'exception', [
'type' => get_class($e),
'message' => $msg,
@@ -282,8 +366,9 @@ class SplitTicketSyncService
/**
* @param array<string, mixed> $payload
*/
public function applySyncResult(Ticket $ticket, array $payload, bool $success, string $message = ''): void
public function applySyncResult(Ticket $ticket, array $payload, bool $success, string $message = '', string $syncMode = 'manual'): void
{
$now = time();
$data = [
'complete_count' => max(0, (int) ($payload['complete_count'] ?? 0)),
'inbound_count' => max(0, (int) ($payload['inbound_count'] ?? 0)),
@@ -293,12 +378,16 @@ class SplitTicketSyncService
'number_banned_count' => max(0, (int) ($payload['number_banned_count'] ?? 0)),
'online_count' => max(0, (int) ($payload['online_count'] ?? 0)),
'sync_status' => $success ? 'success' : 'error',
'sync_time' => time(),
'sync_message' => $success ? '' : mb_substr($message, 0, 255, 'UTF-8'),
'sync_time' => $now,
'sync_message' => $success ? '' : $message,
'sync_fail_count' => $success ? 0 : ((int) ($ticket['sync_fail_count'] ?? 0) + 1),
'speed_snapshot_count' => (int) ($payload['speed_snapshot_count'] ?? $ticket['speed_snapshot_count'] ?? 0),
'speed_snapshot_time' => (int) ($payload['speed_snapshot_time'] ?? $ticket['speed_snapshot_time'] ?? 0),
];
if ($success) {
$data['sync_success_time'] = $now;
$data['sync_success_mode'] = in_array($syncMode, ['auto', 'manual'], true) ? $syncMode : 'manual';
}
if (!$ticket->allowField(array_keys($data))->save($data)) {
throw new Exception('工单同步结果保存失败');
}
@@ -313,10 +402,9 @@ class SplitTicketSyncService
$update = [
'sync_status' => 'error',
'sync_time' => time(),
'sync_message' => mb_substr($message, 0, 255, 'UTF-8'),
'sync_message' => $message,
'sync_fail_count' => $failCount,
];
// 新建工单首次同步失败：立即关闭；已同步过的工单仍按连续失败阈值关闭
if ($neverSyncedSuccessfully || ($failThreshold > 0 && $failCount >= $failThreshold)) {
$update['status'] = 'hidden';
}
@@ -365,4 +453,160 @@ class SplitTicketSyncService
'snapshot_time' => $snapshotTime,
];
}
/**
* @return array{ticketId:int,ticketType:string,ticketUrl:string,reason:string}
*/
private function buildCronTicketEntry(int $ticketId, string $ticketType, string $ticketUrl, string $reason): array
{
return [
'ticketId' => $ticketId,
'ticketType' => $ticketType,
'ticketUrl' => $ticketUrl,
'reason' => $reason,
];
}
private function logCronCandidateSkipped(int $ticketId, string $ticketType, string $ticketUrl, string $reason): void
{
$ctx = [
'ticketId' => $ticketId,
'ticketType' => $ticketType,
'ticketUrl' => $ticketUrl,
'reason' => $reason,
];
SplitTicketSyncLogger::logCron('candidate skipped', $ctx);
SplitTicketSyncLogger::log('cron', 'candidate skipped', $ctx);
}
/**
* 开启但未在本轮同步的工单及原因（供 cron 汇总）
*
* @param list<int> $candidateIds
* @param list<array<string, mixed>> $processed
* @param list<array<string, mixed>> $skipped
* @return list<array<string, mixed>>
*/
private function buildOpenNotSyncedAudit(
array $autoSyncTypes,
int $failThreshold,
int $maxPerRun,
array $candidateIds,
array $processed,
array $skipped,
bool $stoppedByNodeBusy
): array {
$handledIds = [];
foreach ($processed as $row) {
$handledIds[(int) ($row['ticketId'] ?? 0)] = true;
}
foreach ($skipped as $row) {
$handledIds[(int) ($row['ticketId'] ?? 0)] = true;
}
$candidateIdMap = array_fill_keys($candidateIds, true);
$supportedTypes = SplitScrmSpiderFactory::listSupportedTypes();
if ($supportedTypes === []) {
return [];
}
$openList = Ticket::where('status', 'normal')
->whereIn('ticket_type', $supportedTypes)
->field('id,ticket_type,ticket_url,sync_fail_count,sync_time,sync_status')
->select();
$result = [];
foreach ($openList as $ticket) {
$ticketId = (int) $ticket['id'];
if (isset($handledIds[$ticketId])) {
continue;
}
$ticketType = (string) $ticket['ticket_type'];
$ticketUrl = (string) $ticket['ticket_url'];
$reason = $this->resolveOpenNotSyncedReason(
$ticket,
$autoSyncTypes,
$failThreshold,
$maxPerRun,
$candidateIdMap,
$stoppedByNodeBusy
);
if ($reason === null) {
continue;
}
$result[] = [
'ticketId' => $ticketId,
'ticketType' => $ticketType,
'ticketUrl' => $ticketUrl,
'reason' => $reason,
];
}
return $result;
}
/**
* @param array<string, mixed> $ticket
* @param array<string, bool> $candidateIdMap
*/
private function resolveOpenNotSyncedReason(
array $ticket,
array $autoSyncTypes,
int $failThreshold,
int $maxPerRun,
array $candidateIdMap,
bool $stoppedByNodeBusy
): ?string {
$ticketType = (string) ($ticket['ticket_type'] ?? '');
$failCount = (int) ($ticket['sync_fail_count'] ?? 0);
if ($failThreshold > 0 && $failCount >= $failThreshold) {
return sprintf('连续同步失败超过%d次已暂停自动同步', $failThreshold);
}
if (!in_array($ticketType, $autoSyncTypes, true)) {
return '该类型未配置自动同步周期';
}
if (!isset($candidateIdMap[(int) ($ticket['id'] ?? 0)])) {
return sprintf('未进入本轮候选队列(仅扫描最久未同步的前%d条)', $maxPerRun * 5);
}
if ($stoppedByNodeBusy) {
return 'Node 队列繁忙，本轮未执行';
}
$skip = $this->shouldSkip(Ticket::get((int) $ticket['id']) ?: new Ticket($ticket));
return $skip ?? '未知原因，未进入执行队列';
}
/**
* 按 sessionKey（ticketType:host）分组排序，使同域名工单连续执行以命中 Browser 温池
*
* @param \think\Collection|array<int, Ticket|array<string, mixed>> $list
* @return list<Ticket|array<string, mixed>>
*/
private function sortTicketsBySessionKey($list): array
{
$rows = $list instanceof \think\Collection ? $list->all() : (array) $list;
usort($rows, static function ($a, $b): int {
$typeA = (string) (is_array($a) ? ($a['ticket_type'] ?? '') : ($a['ticket_type'] ?? ''));
$urlA = (string) (is_array($a) ? ($a['ticket_url'] ?? '') : ($a['ticket_url'] ?? ''));
$typeB = (string) (is_array($b) ? ($b['ticket_type'] ?? '') : ($b['ticket_type'] ?? ''));
$urlB = (string) (is_array($b) ? ($b['ticket_url'] ?? '') : ($b['ticket_url'] ?? ''));
$keyA = AntiBotConfigBuilder::resolveSessionKey($typeA, $urlA);
$keyB = AntiBotConfigBuilder::resolveSessionKey($typeB, $urlB);
if ($keyA === $keyB) {
$timeA = (int) (is_array($a) ? ($a['sync_time'] ?? 0) : ($a['sync_time'] ?? 0));
$timeB = (int) (is_array($b) ? ($b['sync_time'] ?? 0) : ($b['sync_time'] ?? 0));
if ($timeA === $timeB) {
$idA = (int) (is_array($a) ? ($a['id'] ?? 0) : ($a['id'] ?? 0));
$idB = (int) (is_array($b) ? ($b['id'] ?? 0) : ($b['id'] ?? 0));
return $idA <=> $idB;
}
return $timeA <=> $timeB;
}
return strcmp($keyA, $keyB);
});
return $rows;
}
}
application/extra/site.php
+2
View File
@@ -44,6 +44,8 @@ return array (
),
'split_platform_domain' => 'flowerbells.top',
'split_scrm_node_host' => 'http://127.0.0.1:3001',
'split_scrm_antibot_types' => 'whatshub,chatknow',
'split_scrm_session_ttl_minutes' => '120',
'split_sync_interval_a2c' => '3',
'split_sync_interval_haiwang' => '3',
'split_sync_interval_huojian' => '3',
patches/application/admin/command/Install/split_ticket_sync_log_enhance.sql
+14
View File
@@ -0,0 +1,14 @@
-- 工单同步日志增强：最近成功时间/方式、完整失败原因
SET NAMES utf8mb4;
ALTER TABLE `fa_split_ticket`
MODIFY COLUMN `sync_message` text COMMENT '同步失败完整原因',
ADD COLUMN `sync_success_time` bigint(16) DEFAULT NULL COMMENT '最近一次同步成功时间' AFTER `sync_time`,
ADD COLUMN `sync_success_mode` enum('auto','manual') DEFAULT NULL COMMENT '最近成功同步方式:auto=自动,manual=手动' AFTER `sync_success_time`;
UPDATE `fa_split_ticket`
SET `sync_success_time` = `sync_time`
WHERE `sync_status` = 'success'
AND `sync_success_time` IS NULL
AND `sync_time` IS NOT NULL
AND `sync_time` > 0;
patches/application/admin/command/SplitSyncTickets.php
+2 -1
View File
@@ -62,12 +62,13 @@ class SplitSyncTickets extends Command
try {
$count = $service->syncDueTickets();
$output->writeln('<info>本次处理工单数: ' . $count . '</info>');
$output->writeln('<comment>汇总日志已写入 runtime/log/split_sync.log</comment>');
} finally {
$cronLock->release();
}
if (SplitTicketSyncLogger::isEnabled()) {
$output->writeln('<comment>调试日志已写入 runtime/log/split_sync.log</comment>');
$output->writeln('<comment>详细调试日志已写入 runtime/log/split_sync.log</comment>');
}
}
}
patches/application/admin/controller/split/Ticket.php
+2
View File
@@ -273,6 +273,8 @@ class Ticket extends Backend
$params['sync_status'],
$params['sync_time'],
$params['sync_message'],
$params['sync_success_time'],
$params['sync_success_mode'],
$params['sync_fail_count'],
$params['speed_snapshot_count'],
$params['speed_snapshot_time'],
patches/application/admin/lang/zh-cn/split/ticket.php
+7
View File
@@ -37,6 +37,13 @@ return [
'Sync display success' => '同步成功 / 在线 %s',
'Sync display pending' => '待同步',
'Sync display error' => '同步异常',
'Sync display auto paused' => '自动同步已暂停',
'Sync success time' => '最近同步成功',
'Sync mode auto' => '自动',
'Sync mode manual' => '手动',
'Sync tooltip auto paused' => '连续失败 %s 次，已达暂停阈值 %s，自动同步已暂停',
'Sync tooltip error prefix' => '异常原因：',
'Cron log path hint' => '汇总日志已写入 runtime/log/split_sync.log',
'Createtime' => '创建时间',
'Copy' => '拷贝',
'Summary row' => '汇总',
patches/application/admin/model/split/Ticket.php
+87 -1
View File
@@ -4,6 +4,7 @@ declare(strict_types=1);
namespace app\admin\model\split;
use app\common\service\SplitSyncConfigService;
use think\Db;
use think\Model;
@@ -31,6 +32,9 @@ class Ticket extends Model
'ticket_progress_text',
'inbound_ratio_text',
'sync_display_text',
'sync_success_text',
'sync_tooltip_text',
'sync_auto_paused',
];
/**
@@ -152,13 +156,35 @@ class Ticket extends Model
}
/**
* 同步状态展示文案
* 是否因连续失败暂停自动同步（工单仍为开启状态）
*
* @param array<string, mixed> $data
*/
public static function isAutoSyncPaused(array $data): bool
{
if ((string) ($data['status'] ?? '') !== 'normal') {
return false;
}
$threshold = SplitSyncConfigService::getFailPauseThreshold();
if ($threshold <= 0) {
return false;
}
return (int) ($data['sync_fail_count'] ?? 0) >= $threshold;
}
/**
* 同步状态展示文案（列表列默认显示，异常详情见 tooltip）
*
* @param mixed $value
* @param array<string, mixed> $data
*/
public function getSyncDisplayTextAttr($value, $data): string
{
if (self::isAutoSyncPaused($data)) {
$threshold = SplitSyncConfigService::getFailPauseThreshold();
return sprintf((string) __('Sync display auto paused'), $threshold);
}
$status = (string) ($data['sync_status'] ?? 'pending');
$online = (int) ($data['online_count'] ?? 0);
if ($status === 'success') {
@@ -170,6 +196,66 @@ class Ticket extends Model
return (string) __('Sync display error');
}
/**
* 同步状态列 tooltip 完整说明
*
* @param mixed $value
* @param array<string, mixed> $data
*/
public function getSyncTooltipTextAttr($value, $data): string
{
$parts = [];
if (self::isAutoSyncPaused($data)) {
$threshold = SplitSyncConfigService::getFailPauseThreshold();
$parts[] = sprintf(
(string) __('Sync tooltip auto paused'),
(int) ($data['sync_fail_count'] ?? 0),
$threshold
);
}
$message = trim((string) ($data['sync_message'] ?? ''));
if ($message !== '') {
$parts[] = (string) __('Sync tooltip error prefix') . $message;
}
return implode("\n", $parts);
}
/**
* 是否暂停自动同步（供前端样式判断）
*
* @param mixed $value
* @param array<string, mixed> $data
*/
public function getSyncAutoPausedAttr($value, $data): bool
{
return self::isAutoSyncPaused($data);
}
/**
* 最近一次同步成功时间及方式
*
* @param mixed $value
* @param array<string, mixed> $data
*/
public function getSyncSuccessTextAttr($value, $data): string
{
$ts = (int) ($data['sync_success_time'] ?? 0);
if ($ts <= 0) {
return '';
}
$timeText = date('Y-m-d H:i:s', $ts);
$mode = (string) ($data['sync_success_mode'] ?? '');
if ($mode === 'auto') {
return $timeText . ' (' . (string) __('Sync mode auto') . ')';
}
if ($mode === 'manual') {
return $timeText . ' (' . (string) __('Sync mode manual') . ')';
}
return $timeText;
}
public function setStartTimeAttr($value): ?int
{
return self::parseTimeValue($value);
patches/application/admin/view/split/ticket/index.html
+18
View File
@@ -67,6 +67,24 @@
font-size: 14px;
color: #0f172a;
}
.split-sync-success-normal {
color: #3c763d;
font-weight: 600;
}
.split-sync-success-stopped {
color: #a94442;
font-weight: 600;
}
.split-ticket-sync-status-paused {
color: #8a6d3b;
font-weight: 600;
cursor: help;
border-bottom: 1px dotted #8a6d3b;
}
.split-ticket-sync-status-error {
cursor: help;
border-bottom: 1px dotted #a94442;
}
</style>
<div class="panel panel-default panel-intro">
<div class="panel-heading">
patches/application/common/library/scrm/AbstractScrmSpider.php
+126 -2
View File
@@ -68,13 +68,18 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface
}
$antiBot = $config['antiBot'] ?? null;
$mode = $config['paginationMode'] ?? self::MODE_FETCH;
if ($this->shouldUseUnifiedBrowserPath($antiBot, $mode)) {
return $this->runUnifiedUiPath($config, $antiBot, $apiUrlsToIntercept, $listApi, $detailApi, $countApi);
}
$initResult = $this->requestNode('/api/auth-and-intercept', [
'pageUrl' => $config['pageUrl'],
'apiUrls' => $apiUrlsToIntercept,
'authActions' => $config['authActions'] ?? [],
'antiBot' => $antiBot,
]);
], $this->resolveAuthInterceptTimeout($antiBot));
if (empty($initResult['success'])) {
$cfCode = (string) ($initResult['code'] ?? '');
@@ -97,6 +102,8 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface
SplitTicketSyncLogger::log('spider', 'auth-and-intercept ok', [
'intercepted' => array_keys($interceptedApis),
'finalPageUrl' => $finalPageUrl,
'cfStage' => $initResult['cf']['turnstileStage'] ?? ($initResult['cf']['stage'] ?? null),
'browserReused' => $initResult['browserReused'] ?? null,
]);
$cookies = $initResult['cookies'];
@@ -113,7 +120,6 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface
$allListPagesData = [$listApiNode['data']];
$totalPages = $this->extractListTotalPages($listApiNode['data'], $countData);
$mode = $config['paginationMode'] ?? self::MODE_FETCH;
SplitTicketSyncLogger::log('spider', 'pagination plan', [
'totalPages' => $totalPages,
'mode' => $mode,
@@ -182,6 +188,124 @@ abstract class AbstractScrmSpider implements ScrmSpiderInterface
return $result;
}
/**
* 单 Browser 路径：auth + 拦截 + UI 翻页（需 antiBot.sessionKey）
*
* @param array<string, mixed> $config
* @param array<string, mixed>|null $antiBot
* @param list<string> $apiUrlsToIntercept
*/
private function runUnifiedUiPath(
array $config,
?array $antiBot,
array $apiUrlsToIntercept,
string $listApi,
?string $detailApi,
?string $countApi
): UnifiedScrmData {
$uiConfig = $this->getUiPaginationConfig();
$sessionKey = is_array($antiBot) ? (string) ($antiBot['sessionKey'] ?? '') : '';
SplitTicketSyncLogger::log('spider', 'unified browser path', [
'sessionKey' => $sessionKey,
'listApi' => $listApi,
]);
$mergedResult = $this->requestNode('/api/auth-intercept-and-paginate', [
'pageUrl' => $config['pageUrl'],
'apiUrls' => $apiUrlsToIntercept,
'authActions' => $config['authActions'] ?? [],
'antiBot' => $antiBot,
'pagination' => [
'apiUrl' => $listApi,
'nextBtnSelector' => $uiConfig['nextBtnSelector'] ?? '',
'waitMs' => $uiConfig['waitMs'] ?? 2000,
'clicksToPerform' => 9999,
],
], 1200);
if (empty($mergedResult['success'])) {
$cfCode = (string) ($mergedResult['code'] ?? '');
SplitTicketSyncLogger::log('spider', 'auth-intercept-and-paginate failed', [
'error' => $mergedResult['error'] ?? '未知',
'code' => $cfCode !== '' ? $cfCode : null,
'stage' => $mergedResult['stage'] ?? null,
'cf' => $mergedResult['cf'] ?? null,
'sessionKey' => $sessionKey,
]);
if ($cfCode === 'CF_TURNSTILE_FAILED') {
throw new Exception('Cloudflare Turnstile 验证失败: ' . ($mergedResult['stage'] ?? 'timeout'));
}
throw new Exception('合并同步失败: ' . (string) ($mergedResult['error'] ?? '未知'));
}
SplitTicketSyncLogger::log('spider', 'auth-intercept-and-paginate ok', [
'intercepted' => array_keys($mergedResult['interceptedApis'] ?? []),
'extraPages' => count($mergedResult['extraPages'] ?? []),
'cfStage' => $mergedResult['cf']['turnstileStage'] ?? ($mergedResult['cf']['stage'] ?? null),
'browserReused' => $mergedResult['browserReused'] ?? null,
'sessionKey' => $sessionKey,
]);
$interceptedApis = $mergedResult['interceptedApis'] ?? [];
if (!isset($interceptedApis[$listApi])) {
throw new Exception("致命错误：未能拦截到必须的列表接口 [{$listApi}]");
}
$detailData = $detailApi && isset($interceptedApis[$detailApi])
? $interceptedApis[$detailApi]['data'] : null;
$countData = $countApi && isset($interceptedApis[$countApi])
? $interceptedApis[$countApi]['data'] : null;
$listApiNode = $interceptedApis[$listApi];
$allListPagesData = [$listApiNode['data']];
if (!empty($mergedResult['extraPages']) && is_array($mergedResult['extraPages'])) {
foreach ($mergedResult['extraPages'] as $pageData) {
$allListPagesData[] = $pageData;
}
}
$this->extractListTotalPages($listApiNode['data'], $countData);
$result = $this->parseToUnifiedData($detailData, $allListPagesData);
SplitTicketSyncLogger::log('spider', 'run done', [
'todayNewCount' => $result->todayNewCount,
'totalOnline' => $result->totalOnline,
'totalOffline' => $result->totalOffline,
'total' => $result->total,
'numberCount' => count($result->numbers),
'unifiedPath' => true,
]);
return $result;
}
/**
* @param array<string, mixed>|null $antiBot
*/
protected function shouldUseUnifiedBrowserPath(?array $antiBot, string $mode): bool
{
if ($mode !== self::MODE_UI) {
return false;
}
if (!is_array($antiBot) || empty($antiBot['enabled'])) {
return false;
}
return !empty($antiBot['sessionKey']);
}
/**
* Real Browser + Turnstile 首屏耗时较长，单独放宽 auth-and-intercept 超时
*
* @param array<string, mixed>|null $antiBot
*/
protected function resolveAuthInterceptTimeout(?array $antiBot): int
{
if (!is_array($antiBot) || empty($antiBot['enabled'])) {
return 120;
}
return 180;
}
/**
* @param array<string, mixed> $payload
* @return array<string, mixed>
patches/application/common/library/scrm/AntiBotConfigBuilder.php
+84
View File
@@ -0,0 +1,84 @@
<?php
declare(strict_types=1);
namespace app\common\library\scrm;
use think\Config;
/**
* 云控蜘蛛 antiBot 配置构建器
*
* 约定：sessionKey = "{ticketType}:{host}"；profile=real 当 ticket_type 在 site 配置列表中
*/
class AntiBotConfigBuilder
{
/**
* 为指定工单类型与落地页 URL 构建 Node antiBot 配置；未启用类型返回 null
*
* @return array<string, mixed>|null
*/
public static function build(string $ticketType, string $pageUrl): ?array
{
$enabledTypes = self::getEnabledTypes();
if (!in_array($ticketType, $enabledTypes, true)) {
return null;
}
$host = (string) parse_url($pageUrl, PHP_URL_HOST);
if ($host === '') {
return null;
}
$sessionKey = $ticketType . ':' . $host;
$ttlMinutes = self::getSessionTtlMinutes();
return [
'enabled' => true,
'profile' => 'real',
'turnstile' => true,
'solverFallback' => true,
'sessionKey' => $sessionKey,
'challengeTimeoutMs' => 60000,
'sessionTtlMinutes' => $ttlMinutes,
];
}
/**
* 解析 sessionKey（供 cron 分组调度使用）
*/
public static function resolveSessionKey(string $ticketType, string $pageUrl): string
{
$host = (string) parse_url($pageUrl, PHP_URL_HOST);
return $ticketType . ':' . ($host !== '' ? $host : 'unknown');
}
/**
* 是否对该工单类型启用 antiBot
*/
public static function isEnabledForType(string $ticketType): bool
{
return in_array($ticketType, self::getEnabledTypes(), true);
}
/**
* @return list<string>
*/
public static function getEnabledTypes(): array
{
$raw = (string) Config::get('site.split_scrm_antibot_types');
if ($raw === '') {
return ['whatshub', 'chatknow'];
}
$parts = array_map('trim', explode(',', $raw));
return array_values(array_filter($parts, static function (string $v): bool {
return $v !== '';
}));
}
public static function getSessionTtlMinutes(): int
{
$minutes = (int) Config::get('site.split_scrm_session_ttl_minutes');
return $minutes > 0 ? $minutes : 120;
}
}
patches/application/common/library/scrm/spider/ChatknowSpider.php
+2
View File
@@ -5,6 +5,7 @@ declare(strict_types=1);
namespace app\common\library\scrm\spider;
use app\common\library\scrm\AbstractScrmSpider;
use app\common\library\scrm\AntiBotConfigBuilder;
use app\common\library\scrm\UnifiedScrmData;
/**
@@ -48,6 +49,7 @@ class ChatknowSpider extends AbstractScrmSpider
'authActions' => [
['type' => 'wait', 'ms' => 4000],
],
'antiBot' => AntiBotConfigBuilder::build('chatknow', $this->pageUrl),
];
}
patches/application/common/library/scrm/spider/WhatshubSpider.php
+2 -11
View File
@@ -5,6 +5,7 @@ declare(strict_types=1);
namespace app\common\library\scrm\spider;
use app\common\library\scrm\AbstractScrmSpider;
use app\common\library\scrm\AntiBotConfigBuilder;
use app\common\library\scrm\UnifiedScrmData;
/**
@@ -47,8 +48,6 @@ class WhatshubSpider extends AbstractScrmSpider
/** @return array<string, mixed> */
protected function getSpiderConfig(): array
{
$host = (string) parse_url($this->pageUrl, PHP_URL_HOST);
return [
'pageUrl' => $this->pageUrl,
'listApi' => self::API_LIST,
@@ -62,15 +61,7 @@ class WhatshubSpider extends AbstractScrmSpider
['type' => 'vue_click', 'selector' => '.vxe-button-group .theme--primary'],
['type' => 'wait', 'ms' => 2200],
],
// Whatshub 专用：Real Browser + Turnstile + Captcha API 兜底 + 会话复用
'antiBot' => [
'enabled' => true,
'profile' => 'real',
'turnstile' => true,
'solverFallback' => true,
'sessionKey' => 'whatshub:' . $host,
'challengeTimeoutMs' => 60000,
],
'antiBot' => AntiBotConfigBuilder::build('whatshub', $this->pageUrl),
];
}
patches/application/common/service/SplitTicketSyncLogger.php
+24 -3
View File
@@ -7,7 +7,9 @@ namespace app\common\service;
use think\Config;
/**
* 工单云控同步调试日志（仅 app_debug=true 时写入 runtime/log/split_sync.log）
* 工单云控同步日志
* - logCron：定时任务汇总，app_debug 关闭时也写入 runtime/log/split_sync.log
* - log：详细调试日志，仅 app_debug=true 时写入
*/
class SplitTicketSyncLogger
{
@@ -36,6 +38,18 @@ class SplitTicketSyncLogger
}
/**
* 定时任务汇总日志（始终写入）
*
* @param array<string, mixed> $context
*/
public static function logCron(string $message, array $context = []): void
{
self::writeLine('cron', $message, $context);
}
/**
* 详细调试日志（仅 app_debug=true）
*
* @param array<string, mixed> $context
*/
public static function log(string $stage, string $message, array $context = []): void
@@ -43,7 +57,14 @@ class SplitTicketSyncLogger
if (!self::isEnabled()) {
return;
}
self::writeLine($stage, $message, $context);
}
/**
* @param array<string, mixed> $context
*/
private static function writeLine(string $stage, string $message, array $context = []): void
{
$context = self::sanitize($context);
$ctxJson = $context !== [] ? ' ' . json_encode($context, JSON_UNESCAPED_UNICODE) : '';
$line = sprintf(
@@ -83,8 +104,8 @@ class SplitTicketSyncLogger
$out[$key] = self::sanitize($value);
continue;
}
if (is_string($value) && mb_strlen($value) > 800) {
$out[$key] = mb_substr($value, 0, 800, 'UTF-8') . '...(truncated)';
if (is_string($value) && mb_strlen($value) > 2000) {
$out[$key] = mb_substr($value, 0, 2000, 'UTF-8') . '...(truncated)';
continue;
}
$out[$key] = $value;
patches/application/common/service/SplitTicketSyncService.php
+272 -28
View File
@@ -5,6 +5,7 @@ declare(strict_types=1);
namespace app\common\service;
use app\admin\model\split\Ticket;
use app\common\library\scrm\AntiBotConfigBuilder;
use app\common\library\scrm\UnifiedScrmData;
use think\Db;
use think\Exception;
@@ -31,9 +32,10 @@ class SplitTicketSyncService
* 同步单条工单
*
* @param bool $dueValidated 为 true 时跳过 shouldSkip（由 syncDueTickets 预筛后传入）
* @param string $syncMode auto=定时自动 manual=手动触发
* @return array{success:bool,message:string,skipped?:bool}
*/
public function syncOne(int $ticketId, bool $force = false, bool $dueValidated = false): array
public function syncOne(int $ticketId, bool $force = false, bool $dueValidated = false, string $syncMode = 'manual'): array
{
$ticket = Ticket::get($ticketId);
if (!$ticket) {
@@ -44,6 +46,7 @@ class SplitTicketSyncService
SplitTicketSyncLogger::setTicketContext($ticketId, (string) $ticket['ticket_type']);
SplitTicketSyncLogger::log('sync', 'syncOne start', [
'force' => $force,
'syncMode' => $syncMode,
'status' => (string) $ticket['status'],
'syncFailCount' => (int) ($ticket['sync_fail_count'] ?? 0),
'syncTime' => (int) ($ticket['sync_time'] ?? 0),
@@ -67,7 +70,7 @@ class SplitTicketSyncService
}
try {
$result = $this->doSync($ticket);
$result = $this->doSync($ticket, $syncMode);
SplitTicketSyncLogger::log('sync', 'syncOne end', $result);
return $result;
} finally {
@@ -83,65 +86,147 @@ class SplitTicketSyncService
{
$maxPerRun = SplitSyncConfigService::getMaxTicketsPerCronRun();
$autoSyncTypes = $this->resolveAutoSyncTicketTypes();
$failThreshold = SplitSyncConfigService::getFailPauseThreshold();
if ($autoSyncTypes === []) {
SplitTicketSyncLogger::log('cron', 'scan skip', ['reason' => 'no auto-sync ticket types']);
SplitTicketSyncLogger::logCron('scan skip', ['reason' => 'no auto-sync ticket types']);
return 0;
}
if (!SplitNodeHealthService::canAcceptWork()) {
SplitTicketSyncLogger::log('cron', 'scan skip', [
SplitTicketSyncLogger::logCron('scan skip', [
'reason' => 'node queue busy',
'queue' => SplitNodeHealthService::getQueueSnapshot(),
]);
return 0;
}
$failThreshold = SplitSyncConfigService::getFailPauseThreshold();
$query = Ticket::where('status', 'normal')->whereIn('ticket_type', $autoSyncTypes);
if ($failThreshold > 0) {
$query->where('sync_fail_count', '<', $failThreshold);
}
// 多取候选：间隔过滤在 PHP 完成，优先同步最久未更新的工单
$list = $query->order('sync_time', 'asc')->order('id', 'asc')
->limit($maxPerRun * 5)
->select();
$list = $this->sortTicketsBySessionKey($list);
$candidateCount = count($list);
SplitTicketSyncLogger::logCron('scan start', [
'candidateCount' => $candidateCount,
'maxPerRun' => $maxPerRun,
'autoSyncTypes' => $autoSyncTypes,
'failThreshold' => $failThreshold,
'nodeQueue' => SplitNodeHealthService::getQueueSnapshot(),
'groupedBy' => 'sessionKey',
]);
SplitTicketSyncLogger::log('cron', 'scan start', [
'candidateCount' => count($list),
'candidateCount' => $candidateCount,
'maxPerRun' => $maxPerRun,
'autoSyncTypes' => $autoSyncTypes,
'nodeQueue' => SplitNodeHealthService::getQueueSnapshot(),
]);
/** @var list<array<string, mixed>> $processed */
$processed = [];
/** @var list<array<string, mixed>> $skipped */
$skipped = [];
$count = 0;
foreach ($list as $ticket) {
$stoppedByNodeBusy = false;
foreach ($list as $index => $ticket) {
$ticketId = (int) $ticket['id'];
$ticketUrl = (string) $ticket['ticket_url'];
$ticketType = (string) $ticket['ticket_type'];
$sessionKey = AntiBotConfigBuilder::resolveSessionKey($ticketType, $ticketUrl);
if ($count >= $maxPerRun) {
break;
}
$skip = $this->shouldSkip($ticket);
if ($skip !== null) {
SplitTicketSyncLogger::log('cron', 'candidate skipped', [
'ticketId' => (int) $ticket['id'],
'ticketType' => (string) $ticket['ticket_type'],
'reason' => $skip,
]);
$reason = '本轮处理上限已满';
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $reason);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $reason);
continue;
}
$skip = $this->shouldSkip($ticket);
if ($skip !== null) {
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $skip);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $skip);
continue;
}
if (!SplitNodeHealthService::canAcceptWork()) {
$reason = 'Node 队列繁忙，停止本轮后续同步';
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $reason);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $reason);
$stoppedByNodeBusy = true;
for ($i = $index + 1; $i < $candidateCount; $i++) {
$remain = $list[$i];
$remainReason = 'Node 队列繁忙，本轮未执行';
$skipped[] = $this->buildCronTicketEntry(
(int) $remain['id'],
(string) $remain['ticket_type'],
(string) $remain['ticket_url'],
$remainReason
);
}
SplitTicketSyncLogger::log('cron', 'node busy, stop batch', [
'processed' => $count,
'queue' => SplitNodeHealthService::getQueueSnapshot(),
]);
break;
}
$result = $this->syncOne((int) $ticket['id'], false, true);
$result = $this->syncOne($ticketId, false, true, 'auto');
if (!empty($result['skipped'])) {
$skipReason = (string) ($result['message'] ?? '已跳过');
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $skipReason);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $skipReason);
continue;
}
$processed[] = [
'ticketId' => $ticketId,
'ticketType' => $ticketType,
'ticketUrl' => $ticketUrl,
'sessionKey' => $sessionKey,
'success' => !empty($result['success']),
'message' => (string) ($result['message'] ?? ''),
];
$count++;
}
SplitTicketSyncLogger::log('cron', 'scan end', ['processedCount' => $count]);
$candidateIds = array_map(static function ($row) {
return (int) (is_array($row) ? ($row['id'] ?? 0) : ($row['id'] ?? 0));
}, $list);
$openNotSynced = $this->buildOpenNotSyncedAudit(
$autoSyncTypes,
$failThreshold,
$maxPerRun,
$candidateIds,
$processed,
$skipped,
$stoppedByNodeBusy
);
$successCount = count(array_filter($processed, static function (array $row): bool {
return !empty($row['success']);
}));
$failedCount = count($processed) - $successCount;
SplitTicketSyncLogger::logCron('scan end', [
'processedCount' => $count,
'successCount' => $successCount,
'failedCount' => $failedCount,
'skippedCount' => count($skipped),
'processed' => $processed,
'skipped' => $skipped,
'openNotSynced' => $openNotSynced,
]);
SplitTicketSyncLogger::log('cron', 'scan end', [
'processedCount' => $count,
'successCount' => $successCount,
'failedCount' => $failedCount,
]);
return $count;
}
@@ -164,7 +249,7 @@ class SplitTicketSyncService
/**
* @return array{success:bool,message:string}
*/
private function doSync(Ticket $ticket): array
private function doSync(Ticket $ticket, string $syncMode): array
{
$ticketType = (string) $ticket['ticket_type'];
$pageUrl = trim((string) $ticket['ticket_url']);
@@ -212,7 +297,6 @@ class SplitTicketSyncService
if ((string) $freshTicket['status'] === 'hidden') {
$this->ruleService->cascadeTicketClosedToNumbers($freshTicket);
}
// 号码开关最后统一由 applyNumberRules 判定（单号上限/下号比率/云控在线）
$this->ruleService->applyNumberRules($freshTicket);
$ticket = $freshTicket;
@@ -232,13 +316,13 @@ class SplitTicketSyncService
'speed_snapshot_time' => $speed['snapshot_time'],
];
$this->applySyncResult($ticket, $payload, true, '');
$this->applySyncResult($ticket, $payload, true, '', $syncMode);
Db::commit();
SplitTicketSyncLogger::log('sync', 'db commit ok', $payload);
return ['success' => true, 'message' => '同步成功'];
} catch (\Throwable $e) {
Db::rollback();
$msg = mb_substr($e->getMessage(), 0, 255, 'UTF-8');
$msg = $e->getMessage();
SplitTicketSyncLogger::log('sync', 'exception', [
'type' => get_class($e),
'message' => $msg,
@@ -282,8 +366,9 @@ class SplitTicketSyncService
/**
* @param array<string, mixed> $payload
*/
public function applySyncResult(Ticket $ticket, array $payload, bool $success, string $message = ''): void
public function applySyncResult(Ticket $ticket, array $payload, bool $success, string $message = '', string $syncMode = 'manual'): void
{
$now = time();
$data = [
'complete_count' => max(0, (int) ($payload['complete_count'] ?? 0)),
'inbound_count' => max(0, (int) ($payload['inbound_count'] ?? 0)),
@@ -293,12 +378,16 @@ class SplitTicketSyncService
'number_banned_count' => max(0, (int) ($payload['number_banned_count'] ?? 0)),
'online_count' => max(0, (int) ($payload['online_count'] ?? 0)),
'sync_status' => $success ? 'success' : 'error',
'sync_time' => time(),
'sync_message' => $success ? '' : mb_substr($message, 0, 255, 'UTF-8'),
'sync_time' => $now,
'sync_message' => $success ? '' : $message,
'sync_fail_count' => $success ? 0 : ((int) ($ticket['sync_fail_count'] ?? 0) + 1),
'speed_snapshot_count' => (int) ($payload['speed_snapshot_count'] ?? $ticket['speed_snapshot_count'] ?? 0),
'speed_snapshot_time' => (int) ($payload['speed_snapshot_time'] ?? $ticket['speed_snapshot_time'] ?? 0),
];
if ($success) {
$data['sync_success_time'] = $now;
$data['sync_success_mode'] = in_array($syncMode, ['auto', 'manual'], true) ? $syncMode : 'manual';
}
if (!$ticket->allowField(array_keys($data))->save($data)) {
throw new Exception('工单同步结果保存失败');
}
@@ -313,10 +402,9 @@ class SplitTicketSyncService
$update = [
'sync_status' => 'error',
'sync_time' => time(),
'sync_message' => mb_substr($message, 0, 255, 'UTF-8'),
'sync_message' => $message,
'sync_fail_count' => $failCount,
];
// 新建工单首次同步失败：立即关闭；已同步过的工单仍按连续失败阈值关闭
if ($neverSyncedSuccessfully || ($failThreshold > 0 && $failCount >= $failThreshold)) {
$update['status'] = 'hidden';
}
@@ -365,4 +453,160 @@ class SplitTicketSyncService
'snapshot_time' => $snapshotTime,
];
}
/**
* @return array{ticketId:int,ticketType:string,ticketUrl:string,reason:string}
*/
private function buildCronTicketEntry(int $ticketId, string $ticketType, string $ticketUrl, string $reason): array
{
return [
'ticketId' => $ticketId,
'ticketType' => $ticketType,
'ticketUrl' => $ticketUrl,
'reason' => $reason,
];
}
private function logCronCandidateSkipped(int $ticketId, string $ticketType, string $ticketUrl, string $reason): void
{
$ctx = [
'ticketId' => $ticketId,
'ticketType' => $ticketType,
'ticketUrl' => $ticketUrl,
'reason' => $reason,
];
SplitTicketSyncLogger::logCron('candidate skipped', $ctx);
SplitTicketSyncLogger::log('cron', 'candidate skipped', $ctx);
}
/**
* 开启但未在本轮同步的工单及原因（供 cron 汇总）
*
* @param list<int> $candidateIds
* @param list<array<string, mixed>> $processed
* @param list<array<string, mixed>> $skipped
* @return list<array<string, mixed>>
*/
private function buildOpenNotSyncedAudit(
array $autoSyncTypes,
int $failThreshold,
int $maxPerRun,
array $candidateIds,
array $processed,
array $skipped,
bool $stoppedByNodeBusy
): array {
$handledIds = [];
foreach ($processed as $row) {
$handledIds[(int) ($row['ticketId'] ?? 0)] = true;
}
foreach ($skipped as $row) {
$handledIds[(int) ($row['ticketId'] ?? 0)] = true;
}
$candidateIdMap = array_fill_keys($candidateIds, true);
$supportedTypes = SplitScrmSpiderFactory::listSupportedTypes();
if ($supportedTypes === []) {
return [];
}
$openList = Ticket::where('status', 'normal')
->whereIn('ticket_type', $supportedTypes)
->field('id,ticket_type,ticket_url,sync_fail_count,sync_time,sync_status')
->select();
$result = [];
foreach ($openList as $ticket) {
$ticketId = (int) $ticket['id'];
if (isset($handledIds[$ticketId])) {
continue;
}
$ticketType = (string) $ticket['ticket_type'];
$ticketUrl = (string) $ticket['ticket_url'];
$reason = $this->resolveOpenNotSyncedReason(
$ticket,
$autoSyncTypes,
$failThreshold,
$maxPerRun,
$candidateIdMap,
$stoppedByNodeBusy
);
if ($reason === null) {
continue;
}
$result[] = [
'ticketId' => $ticketId,
'ticketType' => $ticketType,
'ticketUrl' => $ticketUrl,
'reason' => $reason,
];
}
return $result;
}
/**
* @param array<string, mixed> $ticket
* @param array<string, bool> $candidateIdMap
*/
private function resolveOpenNotSyncedReason(
array $ticket,
array $autoSyncTypes,
int $failThreshold,
int $maxPerRun,
array $candidateIdMap,
bool $stoppedByNodeBusy
): ?string {
$ticketType = (string) ($ticket['ticket_type'] ?? '');
$failCount = (int) ($ticket['sync_fail_count'] ?? 0);
if ($failThreshold > 0 && $failCount >= $failThreshold) {
return sprintf('连续同步失败超过%d次已暂停自动同步', $failThreshold);
}
if (!in_array($ticketType, $autoSyncTypes, true)) {
return '该类型未配置自动同步周期';
}
if (!isset($candidateIdMap[(int) ($ticket['id'] ?? 0)])) {
return sprintf('未进入本轮候选队列(仅扫描最久未同步的前%d条)', $maxPerRun * 5);
}
if ($stoppedByNodeBusy) {
return 'Node 队列繁忙，本轮未执行';
}
$skip = $this->shouldSkip(Ticket::get((int) $ticket['id']) ?: new Ticket($ticket));
return $skip ?? '未知原因，未进入执行队列';
}
/**
* 按 sessionKey（ticketType:host）分组排序，使同域名工单连续执行以命中 Browser 温池
*
* @param \think\Collection|array<int, Ticket|array<string, mixed>> $list
* @return list<Ticket|array<string, mixed>>
*/
private function sortTicketsBySessionKey($list): array
{
$rows = $list instanceof \think\Collection ? $list->all() : (array) $list;
usort($rows, static function ($a, $b): int {
$typeA = (string) (is_array($a) ? ($a['ticket_type'] ?? '') : ($a['ticket_type'] ?? ''));
$urlA = (string) (is_array($a) ? ($a['ticket_url'] ?? '') : ($a['ticket_url'] ?? ''));
$typeB = (string) (is_array($b) ? ($b['ticket_type'] ?? '') : ($b['ticket_type'] ?? ''));
$urlB = (string) (is_array($b) ? ($b['ticket_url'] ?? '') : ($b['ticket_url'] ?? ''));
$keyA = AntiBotConfigBuilder::resolveSessionKey($typeA, $urlA);
$keyB = AntiBotConfigBuilder::resolveSessionKey($typeB, $urlB);
if ($keyA === $keyB) {
$timeA = (int) (is_array($a) ? ($a['sync_time'] ?? 0) : ($a['sync_time'] ?? 0));
$timeB = (int) (is_array($b) ? ($b['sync_time'] ?? 0) : ($b['sync_time'] ?? 0));
if ($timeA === $timeB) {
$idA = (int) (is_array($a) ? ($a['id'] ?? 0) : ($a['id'] ?? 0));
$idB = (int) (is_array($b) ? ($b['id'] ?? 0) : ($b['id'] ?? 0));
return $idA <=> $idB;
}
return $timeA <=> $timeB;
}
return strcmp($keyA, $keyB);
});
return $rows;
}
}
patches/public/assets/js/backend/split/ticket.js
+51 -3
View File
@@ -24,6 +24,7 @@ define(['jquery', 'bootstrap', 'backend', 'table', 'form'], function ($, undefin
sortOrder: 'desc',
fixedColumns: true,
fixedRightNumber: 1,
searchFormVisible: true,
columns: [
[
{checkbox: true},
@@ -98,6 +99,12 @@ define(['jquery', 'bootstrap', 'backend', 'table', 'form'], function ($, undefin
operate: false,
formatter: Controller.api.formatter.syncDisplay
},
{
field: 'sync_success_text',
title: __('Sync success time'),
operate: false,
formatter: Controller.api.formatter.syncSuccessDisplay
},
$.extend(
{field: 'status', title: __('Status'), searchList: Config.statusList, formatter: Table.api.formatter.toggle, yes: 'normal', no: 'hidden'},
searchSelectMeta(false)
@@ -136,6 +143,7 @@ define(['jquery', 'bootstrap', 'backend', 'table', 'form'], function ($, undefin
table.on('load-success.bs.table', function (e, res) {
Controller.api.renderSummaryRow(res);
Controller.api.initSyncTooltips(table);
var pendingIds = window.__splitTicketPendingPostAddSyncIds;
if (!pendingIds || !pendingIds.length) {
return;
@@ -309,11 +317,26 @@ define(['jquery', 'bootstrap', 'backend', 'table', 'form'], function ($, undefin
*/
finishTicketsSync: function (table) {
Controller.api.syncingTicketIds = [];
table.bootstrapTable('refresh');
table.bootstrapTable('refresh', {
silent: true
});
Controller.api.initSyncTooltips(table);
var ids = Table.api.selectedids(table);
$('.btn-sync').toggleClass('btn-disabled disabled', ids.length === 0);
},
/**
* 初始化同步状态列 tooltip（完整异常原因）
*/
initSyncTooltips: function (table) {
var $container = table.closest('.bootstrap-table');
$container.find('[data-sync-tooltip="1"]').tooltip({
container: 'body',
placement: 'auto top',
html: false
});
},
formatter: {
/**
* 操作列：编辑 → 拷贝 → 删除
@@ -407,13 +430,38 @@ define(['jquery', 'bootstrap', 'backend', 'table', 'form'], function ($, undefin
return Controller.api.formatter.syncingDisplayHtml();
}
var text = value || '';
var tooltip = row.sync_tooltip_text ? String(row.sync_tooltip_text).trim() : '';
if (tooltip !== '') {
tooltip = tooltip.replace(/\r?\n/g, '；');
}
var hasTooltip = tooltip !== '';
var color = 'danger';
if (row.sync_status === 'success') {
var extraClass = '';
if (row.sync_auto_paused) {
color = 'warning';
extraClass = ' split-ticket-sync-status-paused';
} else if (row.sync_status === 'success') {
color = 'success';
} else if (row.sync_status === 'pending') {
color = 'muted';
} else if (row.sync_status === 'error') {
extraClass = ' split-ticket-sync-status-error';
}
return '<span class="text-' + color + '">' + Fast.api.escape(text) + '</span>';
var attrs = '';
if (hasTooltip) {
attrs = ' data-sync-tooltip="1" data-toggle="tooltip" title="' + Fast.api.escape(tooltip) + '"';
}
return '<span class="text-' + color + extraClass + '"' + attrs + '>' + Fast.api.escape(text) + '</span>';
},
syncSuccessDisplay: function (value, row) {
var text = value == null ? '' : String(value).trim();
if (text === '') {
return '<span class="text-muted">-</span>';
}
var colorClass = row.status === 'normal'
? 'split-sync-success-normal'
: 'split-sync-success-stopped';
return '<span class="' + colorClass + '">' + Fast.api.escape(text) + '</span>';
},
syncingDisplayHtml: function () {
var label = (typeof Config.syncInProgressMsg !== 'undefined' && Config.syncInProgressMsg)
patches/puppeteer-api/lib/browser-factory.js
+35 -51
View File
@@ -1,34 +1,22 @@
/**
* BrowserFactory：standard / real 双轨启动 + 独立并发限流
* BrowserFactory：standard / real 双轨启动 + 温池 + 独立并发限流
*/
const {
DEFAULT_UA,
MAX_CONCURRENT_BROWSERS,
MAX_CONCURRENT_BROWSERS_REAL,
QUEUE_TIMEOUT_MS,
NAVIGATION_TIMEOUT_MS,
PAGE_DEFAULT_TIMEOUT_MS,
} = require('./constants');
const { BrowserConcurrencyLimiter } = require('./concurrency');
const { launchStandardBrowser } = require('./launch-standard');
const { launchRealBrowser, isRealBrowserAvailable } = require('./launch-real-browser');
const { browserPool } = require('./browser-pool');
const standardLimiter = new BrowserConcurrencyLimiter(MAX_CONCURRENT_BROWSERS);
const realLimiter = new BrowserConcurrencyLimiter(MAX_CONCURRENT_BROWSERS_REAL);
/**
* @typedef {Object} AntiBotConfig
* @property {boolean} [enabled]
* @property {'standard'|'real'} [profile]
* @property {boolean} [turnstile]
* @property {boolean} [solverFallback]
* @property {string} [sessionKey]
* @property {number} [challengeTimeoutMs]
*/
/**
* 规范化 antiBot 配置（未传则走 standard）
* @param {AntiBotConfig|null|undefined} antiBot
* @returns {AntiBotConfig}
*/
function normalizeAntiBot(antiBot) {
if (!antiBot || !antiBot.enabled) {
return { enabled: false, profile: 'standard' };
@@ -43,62 +31,66 @@ function normalizeAntiBot(antiBot) {
};
}
/**
* 按 profile 获取限流器
* @param {'standard'|'real'} profile
*/
function getLimiter(profile) {
return profile === 'real' ? realLimiter : standardLimiter;
}
/**
* 创建 Browser 会话
* @param {{ profile?: 'standard'|'real', extraArgs?: string[] }} options
* @returns {Promise<{ browser: import('puppeteer').Browser, page: import('puppeteer').Page|null, profile: string, cleanup: () => Promise<void> }>}
* 冷启动 Browser（供温池 launcher 使用）
* @param {{ profile?: 'standard'|'real', extraArgs?: string[], sessionKey?: string }} options
*/
async function createBrowserSession(options = {}) {
const profile = options.profile === 'real' ? 'real' : 'standard';
const extraArgs = options.extraArgs || [];
const sessionKey = options.sessionKey || '';
if (profile === 'real') {
const { browser, page } = await launchRealBrowser(extraArgs);
const { browser, page } = await launchRealBrowser(extraArgs, { sessionKey, profile: 'real' });
return {
browser,
page,
profile: 'real',
cleanup: async () => {
cleanup: async (destroy = false) => {
if (destroy || !sessionKey) {
try {
await browser.close();
} catch (_) {
/* ignore */
} catch (_) { /* ignore */ }
}
},
};
}
const browser = await launchStandardBrowser(extraArgs);
const browser = await launchStandardBrowser(extraArgs, { sessionKey, profile: 'standard' });
return {
browser,
page: null,
profile: 'standard',
cleanup: async () => {
cleanup: async (destroy = false) => {
if (destroy || !sessionKey) {
try {
await browser.close();
} catch (_) {
/* ignore */
} catch (_) { /* ignore */ }
}
},
};
}
/**
* 在并发槽位内执行 Browser 任务
* @template T
* @param {string} taskName
* @param {'standard'|'real'} profile
* @param {() => Promise<T>} fn
* @returns {Promise<T>}
* 从温池或冷启动获取 Browser
* @param {{ profile?: 'standard'|'real', extraArgs?: string[], sessionKey?: string }} options
*/
async function acquireBrowserSession(options = {}) {
const profile = options.profile === 'real' ? 'real' : 'standard';
const sessionKey = options.sessionKey || '';
const extraArgs = options.extraArgs || [];
return browserPool.acquire(sessionKey, profile, async () => createBrowserSession({
profile,
extraArgs,
sessionKey,
}));
}
async function runWithProfileLimit(taskName, profile, fn) {
const limiter = getLimiter(profile);
try {
@@ -109,23 +101,15 @@ async function runWithProfileLimit(taskName, profile, fn) {
}
}
/**
* 创建 Page 并设置 UA / viewport / cookies
* @param {import('puppeteer').Browser} browser
* @param {import('puppeteer').Page|null} existingPage
* @param {{ userAgent?: string, viewport?: object, cookies?: Array<object> }} [options]
*/
async function createManagedPage(browser, existingPage, options = {}) {
const page = existingPage || await browser.newPage();
page.setDefaultNavigationTimeout(30000);
page.setDefaultTimeout(15000);
page.setDefaultNavigationTimeout(NAVIGATION_TIMEOUT_MS);
page.setDefaultTimeout(PAGE_DEFAULT_TIMEOUT_MS);
const userAgent = options.userAgent || DEFAULT_UA;
await page.setUserAgent(userAgent);
if (options.viewport) {
await page.setViewport(options.viewport);
}
if (options.viewport) await page.setViewport(options.viewport);
if (options.cookies && options.cookies.length > 0) {
await page.setCookie(...options.cookies);
}
@@ -136,14 +120,12 @@ async function createManagedPage(browser, existingPage, options = {}) {
return page;
}
/**
* @returns {{ standard: object, real: object, realBrowserReady: boolean }}
*/
function getFactoryStats() {
return {
standard: standardLimiter.getStats(),
real: realLimiter.getStats(),
realBrowserReady: isRealBrowserAvailable(),
pool: browserPool.getStats(),
};
}
@@ -151,10 +133,12 @@ module.exports = {
normalizeAntiBot,
getLimiter,
createBrowserSession,
acquireBrowserSession,
runWithProfileLimit,
createManagedPage,
getFactoryStats,
standardLimiter,
realLimiter,
browserPool,
QUEUE_TIMEOUT_MS,
};
patches/puppeteer-api/lib/browser-pool.js
+184
View File
@@ -0,0 +1,184 @@
/**
* Browser 温池：按 profile:sessionKey 复用 Browser/Page，降低冷启动与 CF 触发
*/
const { POOL_IDLE_MS, MAX_POOLED_BROWSERS, POOL_ENABLED } = require('./constants');
class BrowserPool {
constructor() {
/** @type {Map<string, { browser: any, page: any|null, profile: string, sessionKey: string, lastUsedAt: number, inUse: boolean }>} */
this.entries = new Map();
this.hits = 0;
this.misses = 0;
this.evictions = 0;
this._sweepTimer = setInterval(() => this.evictIdle(), 30000);
this._sweepTimer.unref?.();
}
/**
* @param {string} sessionKey
* @param {'standard'|'real'} profile
*/
buildKey(sessionKey, profile) {
return `${profile}:${sessionKey}`;
}
/**
* @param {any} browser
*/
async isBrowserAlive(browser) {
if (!browser) return false;
try {
if (typeof browser.isConnected === 'function' && !browser.isConnected()) {
return false;
}
await browser.version();
return true;
} catch (_) {
return false;
}
}
/**
* @param {string} key
*/
async destroyEntry(key) {
const entry = this.entries.get(key);
if (!entry) return;
this.entries.delete(key);
try {
await entry.browser.close();
} catch (_) {
/* ignore */
}
this.evictions++;
}
enforceCapacity(excludeKey) {
if (this.entries.size < MAX_POOLED_BROWSERS) return;
const candidates = [...this.entries.entries()]
.filter(([k, e]) => k !== excludeKey && !e.inUse)
.sort((a, b) => a[1].lastUsedAt - b[1].lastUsedAt);
if (candidates.length === 0) return;
const [oldestKey] = candidates[0];
this.destroyEntry(oldestKey);
}
/**
* @param {string} sessionKey
* @param {'standard'|'real'} profile
* @param {() => Promise<{ browser: any, page: any|null, cleanup: (destroy?: boolean) => Promise<void> }>} launcher
*/
async acquire(sessionKey, profile, launcher) {
if (!POOL_ENABLED || !sessionKey) {
const launched = await launcher();
return {
browser: launched.browser,
page: launched.page,
profile,
pooled: false,
reused: false,
release: async (destroy = false) => {
await launched.cleanup(destroy);
},
};
}
const key = this.buildKey(sessionKey, profile);
const existing = this.entries.get(key);
if (existing && !existing.inUse && await this.isBrowserAlive(existing.browser)) {
existing.inUse = true;
existing.lastUsedAt = Date.now();
this.hits++;
return {
browser: existing.browser,
page: existing.page,
profile,
pooled: true,
reused: true,
release: async (destroy = false) => this.release(key, destroy),
};
}
if (existing) {
await this.destroyEntry(key);
}
this.enforceCapacity(key);
this.misses++;
const launched = await launcher();
this.entries.set(key, {
browser: launched.browser,
page: launched.page,
profile,
sessionKey,
lastUsedAt: Date.now(),
inUse: true,
});
return {
browser: launched.browser,
page: launched.page,
profile,
pooled: true,
reused: false,
release: async (destroy = false) => this.release(key, destroy),
};
}
/**
* @param {string} key
* @param {boolean} destroy
*/
async release(key, destroy = false) {
const entry = this.entries.get(key);
if (!entry) return;
entry.inUse = false;
entry.lastUsedAt = Date.now();
if (destroy || !(await this.isBrowserAlive(entry.browser))) {
await this.destroyEntry(key);
}
}
evictIdle() {
const now = Date.now();
for (const [key, entry] of this.entries.entries()) {
if (!entry.inUse && now - entry.lastUsedAt > POOL_IDLE_MS) {
this.destroyEntry(key);
}
}
}
async drain() {
clearInterval(this._sweepTimer);
const keys = [...this.entries.keys()];
for (const key of keys) {
await this.destroyEntry(key);
}
}
getStats() {
let idle = 0;
let active = 0;
for (const entry of this.entries.values()) {
if (entry.inUse) active++;
else idle++;
}
return {
enabled: POOL_ENABLED,
max: MAX_POOLED_BROWSERS,
idleMs: POOL_IDLE_MS,
size: this.entries.size,
idle,
active,
hits: this.hits,
misses: this.misses,
evictions: this.evictions,
};
}
}
const browserPool = new BrowserPool();
module.exports = {
BrowserPool,
browserPool,
};
patches/puppeteer-api/lib/cf-handler.js
+42 -22
View File
@@ -1,53 +1,75 @@
/**
* Cloudflare 挑战统一处理：检测 → 内置等待 → Captcha API 兜底
* Cloudflare 挑战统一处理：检测 → 会话快速通道 → 内置等待 → Captcha API 兜底
*/
const { detectCloudflare } = require('./cf-detector');
const { waitForTurnstile, extractTurnstileSitekey, injectTurnstileToken } = require('./turnstile-handler');
const { solveTurnstile, isCaptchaConfigured } = require('./captcha-solver');
const { isSessionLikelyValid } = require('./session-store');
/**
* @typedef {Object} CfHandleResult
* @property {boolean} success
* @property {string|null} code
* @property {string|null} challengeType
* @property {string|null} stage
* @property {boolean} solverUsed
* @property {number} elapsedMs
* @property {boolean} cfDetected
*/
/**
* 处理 Cloudflare / Turnstile 挑战（在 authActions 之前调用）
* @param {import('puppeteer').Page} page
* @param {{ turnstile?: boolean, solverFallback?: boolean, challengeTimeoutMs?: number }} antiBot
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
* @returns {Promise<CfHandleResult>}
* @param {import('./session-store').StoredSession|null} [storedSession]
*/
async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled) {
async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, storedSession = null) {
const started = Date.now();
const timeoutMs = antiBot.challengeTimeoutMs || 60000;
let cfState = await detectCloudflare(page);
if (!cfState.blocked && cfState.hasCfClearance) {
return {
success: true,
code: null,
challengeType: null,
stage: 'session_reuse',
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: false,
};
}
if (!cfState.blocked) {
return {
success: true,
code: null,
challengeType: null,
stage: null,
stage: storedSession && isSessionLikelyValid(storedSession) ? 'session_reuse' : null,
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: false,
};
}
if (storedSession && isSessionLikelyValid(storedSession)) {
console.log('[CF] 会话有效但仍被拦截，尝试 reload 一次');
try {
await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 45000) });
if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {});
cfState = await detectCloudflare(page);
if (!cfState.blocked) {
return {
success: true,
code: null,
challengeType: cfState.challengeType,
stage: 'session_reload',
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: true,
};
}
} catch (reloadErr) {
console.warn('[CF] reload 失败:', reloadErr.message);
}
}
console.log(`[CF] 检测到挑战 type=${cfState.challengeType} url=${cfState.url}`);
if (antiBot.turnstile !== false) {
const builtIn = await waitForTurnstile(page, { timeoutMs, useBuiltInClick: true });
if (builtIn.success) {
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {});
cfState = await detectCloudflare(page);
if (!cfState.blocked) {
return {
@@ -74,9 +96,7 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled) {
const apiWaitMs = Math.min(timeoutMs, 30000);
await waitForTurnstile(page, { timeoutMs: apiWaitMs });
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {});
cfState = await detectCloudflare(page);
if (!cfState.blocked) {
patches/puppeteer-api/lib/constants.js
+40 -28
View File
@@ -5,13 +5,9 @@ const fs = require('fs');
const path = require('path');
const puppeteer = require('puppeteer-extra');
/** 默认 Chrome 可执行文件路径（生产环境可通过环境变量覆盖） */
const DEFAULT_CHROME_EXECUTABLE = '/www/wwwroot/puppeteer-api/.cache/puppeteer/chrome/linux-149.0.7827.22/chrome-linux64/chrome';
/** 默认 User-Agent，与 standard / real 双轨保持一致 */
const DEFAULT_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
/** Chrome 启动基础参数：多实例 headless 场景下复用 */
const BASE_LAUNCH_ARGS = [
'--no-sandbox',
'--disable-setuid-sandbox',
@@ -26,24 +22,14 @@ const BASE_LAUNCH_ARGS = [
'--disable-crash-reporter',
];
/** puppeteer-api 专用运行时目录（与 server.js 同级 .runtime） */
const RUNTIME_DIR = path.join(__dirname, '..', '.runtime');
/**
* 确保 .runtime 子目录存在且 www 用户可写
* @param {string} name
* @returns {string}
*/
function ensureRuntimeSubdir(name) {
const dir = path.join(RUNTIME_DIR, name);
fs.mkdirSync(dir, { recursive: true, mode: 0o777 });
return dir;
}
/**
* 解析 Chrome 可执行文件路径
* @returns {string}
*/
function resolveChromeExecutable() {
const candidates = [
process.env.PUPPETEER_EXECUTABLE_PATH,
@@ -63,16 +49,12 @@ function resolveChromeExecutable() {
return bundled;
}
} catch (_) {
/* puppeteer 未内置浏览器时忽略 */
/* ignore */
}
return candidates[0] || DEFAULT_CHROME_EXECUTABLE;
}
/**
* 生产 Linux 常注入无效 DBUS 地址；删除而非设为 /dev/null
* @returns {NodeJS.ProcessEnv}
*/
function buildBrowserLaunchEnv() {
const env = { ...process.env };
delete env.DBUS_SESSION_BUS_ADDRESS;
@@ -87,10 +69,6 @@ function buildBrowserLaunchEnv() {
return env;
}
/**
* 校验 Chrome 可执行文件存在
* @returns {string}
*/
function assertChromeExecutable() {
const executablePath = resolveChromeExecutable();
if (!fs.existsSync(executablePath)) {
@@ -101,23 +79,48 @@ function assertChromeExecutable() {
return executablePath;
}
/** 并发与超时配置（可通过环境变量覆盖） */
const MAX_CONCURRENT_BROWSERS = Math.max(1, parseInt(process.env.MAX_CONCURRENT_BROWSERS || '4', 10));
const MAX_CONCURRENT_BROWSERS_REAL = Math.max(1, parseInt(process.env.MAX_CONCURRENT_BROWSERS_REAL || '2', 10));
const QUEUE_TIMEOUT_MS = Math.max(5000, parseInt(process.env.QUEUE_TIMEOUT_MS || '120000', 10));
const API_INTERCEPT_TIMEOUT_MS = Math.max(10000, parseInt(process.env.API_INTERCEPT_TIMEOUT_MS || '30000', 10));
const API_INTERCEPT_TIMEOUT_MS = Math.max(10000, parseInt(process.env.API_INTERCEPT_TIMEOUT_MS || '60000', 10));
const API_INTERCEPT_TIMEOUT_REAL_MS = Math.max(
API_INTERCEPT_TIMEOUT_MS,
parseInt(process.env.API_INTERCEPT_TIMEOUT_REAL_MS || '90000', 10)
);
/** Captcha API 配置 */
const NAVIGATION_TIMEOUT_MS = Math.max(15000, parseInt(process.env.NAVIGATION_TIMEOUT_MS || '60000', 10));
/** rebrowser-puppeteer-core / puppeteer-core 仅支持 load|domcontentloaded|networkidle0|networkidle2，不支持 commit */
const ALLOWED_NAVIGATION_WAIT_UNTIL = ['load', 'domcontentloaded', 'networkidle0', 'networkidle2'];
function resolveNavigationWaitUntil() {
const raw = (process.env.NAVIGATION_WAIT_UNTIL || 'domcontentloaded').trim();
if (ALLOWED_NAVIGATION_WAIT_UNTIL.includes(raw)) {
return raw;
}
if (raw === 'commit') {
console.warn('[constants] NAVIGATION_WAIT_UNTIL=commit 不被当前 Puppeteer 支持，已降级为 domcontentloaded');
return 'domcontentloaded';
}
console.warn(`[constants] 未知 NAVIGATION_WAIT_UNTIL=${raw}，已降级为 domcontentloaded`);
return 'domcontentloaded';
}
const NAVIGATION_WAIT_UNTIL = resolveNavigationWaitUntil();
const NAVIGATION_MAX_RETRIES = Math.max(0, parseInt(process.env.NAVIGATION_MAX_RETRIES || '1', 10));
const PAGE_DEFAULT_TIMEOUT_MS = Math.max(15000, parseInt(process.env.PAGE_DEFAULT_TIMEOUT_MS || '30000', 10));
const CAPTCHA_PROVIDER = (process.env.CAPTCHA_PROVIDER || 'capsolver').toLowerCase();
const CAPTCHA_API_KEY = process.env.CAPTCHA_API_KEY || '';
const CAPTCHA_MAX_WAIT_MS = Math.max(30000, parseInt(process.env.CAPTCHA_MAX_WAIT_MS || '120000', 10));
/** 会话 Cookie 持久化 TTL（毫秒） */
const SESSION_TTL_MS = Math.max(60000, parseInt(process.env.SESSION_TTL_MS || '1800000', 10));
const SESSION_TTL_MS = Math.max(60000, parseInt(process.env.SESSION_TTL_MS || '7200000', 10));
const PERSIST_BROWSER_PROFILE = process.env.PERSIST_BROWSER_PROFILE !== '0';
const PROFILE_MAX_AGE_MS = Math.max(0, parseInt(process.env.PROFILE_MAX_AGE_MS || String(7 * 24 * 3600 * 1000), 10));
const POOL_IDLE_MS = Math.max(60000, parseInt(process.env.POOL_IDLE_MS || '300000', 10));
const MAX_POOLED_BROWSERS = Math.max(0, parseInt(process.env.MAX_POOLED_BROWSERS || '4', 10));
const POOL_ENABLED = process.env.POOL_ENABLED !== '0' && MAX_POOLED_BROWSERS > 0;
module.exports = {
DEFAULT_CHROME_EXECUTABLE,
@@ -133,8 +136,17 @@ module.exports = {
QUEUE_TIMEOUT_MS,
API_INTERCEPT_TIMEOUT_MS,
API_INTERCEPT_TIMEOUT_REAL_MS,
NAVIGATION_TIMEOUT_MS,
NAVIGATION_WAIT_UNTIL,
NAVIGATION_MAX_RETRIES,
PAGE_DEFAULT_TIMEOUT_MS,
CAPTCHA_PROVIDER,
CAPTCHA_API_KEY,
CAPTCHA_MAX_WAIT_MS,
SESSION_TTL_MS,
PERSIST_BROWSER_PROFILE,
PROFILE_MAX_AGE_MS,
POOL_IDLE_MS,
MAX_POOLED_BROWSERS,
POOL_ENABLED,
};
patches/puppeteer-api/lib/launch-real-browser.js
+13 -17
View File
@@ -1,26 +1,17 @@
/**
* Real Browser 启动：puppeteer-real-browser（抗 Cloudflare Turnstile）
* 注意：不与 puppeteer-extra 混用在同一 Browser 实例
*/
const {
BASE_LAUNCH_ARGS,
resolveChromeExecutable,
} = require('./constants');
const { resolveProfileDir } = require('./profile-dir');
/** @type {boolean|null} */
let realBrowserModuleChecked = null;
/** @type {boolean} */
let realBrowserAvailable = false;
/**
* 检测 puppeteer-real-browser 是否已安装
* @returns {boolean}
*/
function isRealBrowserAvailable() {
if (realBrowserModuleChecked !== null) {
return realBrowserAvailable;
}
if (realBrowserModuleChecked !== null) return realBrowserAvailable;
try {
require.resolve('puppeteer-real-browser');
realBrowserAvailable = true;
@@ -32,11 +23,10 @@ function isRealBrowserAvailable() {
}
/**
* 启动 real profile Browser（headless:false + Xvfb + turnstile 内置点击）
* @param {string[]} [extraArgs]
* @returns {Promise<{ browser: import('puppeteer').Browser, page: import('puppeteer').Page }>}
* @param {{ sessionKey?: string, profile?: 'standard'|'real' }} [options]
*/
async function launchRealBrowser(extraArgs = []) {
async function launchRealBrowser(extraArgs = [], options = {}) {
if (!isRealBrowserAvailable()) {
throw new Error(
'puppeteer-real-browser 未安装。请在 puppeteer-api 目录执行: npm install puppeteer-real-browser@1.4.4'
@@ -45,22 +35,28 @@ async function launchRealBrowser(extraArgs = []) {
const { connect } = require('puppeteer-real-browser');
const chromePath = resolveChromeExecutable();
const profile = options.profile === 'real' ? 'real' : 'standard';
const profileDir = options.sessionKey ? resolveProfileDir(options.sessionKey, profile) : null;
const args = [...BASE_LAUNCH_ARGS, '--mute-audio', ...extraArgs];
if (profileDir) {
args.push(`--user-data-dir=${profileDir}`);
}
const result = await connect({
headless: false,
turnstile: true,
disableXvfb: false,
customConfig: { chromePath },
args: [...BASE_LAUNCH_ARGS, '--mute-audio', ...extraArgs],
args,
});
const browser = result.browser;
const page = result.page;
if (!browser || !page) {
throw new Error('puppeteer-real-browser connect 未返回 browser/page');
}
browser.__persistentProfile = !!profileDir;
browser.__userDataDir = profileDir || '';
return { browser, page };
}
patches/puppeteer-api/lib/launch-standard.js
+14 -9
View File
@@ -11,18 +11,21 @@ const {
buildBrowserLaunchEnv,
ensureRuntimeSubdir,
} = require('./constants');
const { resolveProfileDir } = require('./profile-dir');
// Stealth 必须在 launch 之前注册
puppeteer.use(StealthPlugin());
/**
* 启动 standard profile Browser
* @param {string[]} [extraArgs]
* @param {{ sessionKey?: string, profile?: 'standard'|'real' }} [options]
* @returns {Promise<import('puppeteer').Browser>}
*/
async function launchStandardBrowser(extraArgs = []) {
async function launchStandardBrowser(extraArgs = [], options = {}) {
const executablePath = assertChromeExecutable();
const userDataDir = fs.mkdtempSync(path.join(ensureRuntimeSubdir('chrome-profiles'), 'run-'));
const profile = options.profile === 'real' ? 'real' : 'standard';
const persistentDir = options.sessionKey ? resolveProfileDir(options.sessionKey, profile) : null;
const userDataDir = persistentDir || fs.mkdtempSync(path.join(ensureRuntimeSubdir('chrome-profiles'), 'run-'));
const persistent = !!persistentDir;
try {
const browser = await puppeteer.launch({
@@ -32,6 +35,7 @@ async function launchStandardBrowser(extraArgs = []) {
env: buildBrowserLaunchEnv(),
userDataDir,
});
if (!persistent) {
const originalClose = browser.close.bind(browser);
browser.close = async () => {
try {
@@ -39,17 +43,18 @@ async function launchStandardBrowser(extraArgs = []) {
} finally {
try {
fs.rmSync(userDataDir, { recursive: true, force: true });
} catch (_) {
/* 清理失败不影响主流程 */
}
} catch (_) { /* ignore */ }
}
};
}
browser.__userDataDir = userDataDir;
browser.__persistentProfile = persistent;
return browser;
} catch (err) {
if (!persistent) {
try {
fs.rmSync(userDataDir, { recursive: true, force: true });
} catch (_) {
/* ignore */
} catch (_) { /* ignore */ }
}
throw err;
}
patches/puppeteer-api/lib/profile-dir.js
+77
View File
@@ -0,0 +1,77 @@
/**
* 按 sessionKey 解析持久 Chrome Profile 目录
*/
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const { RUNTIME_DIR, PERSIST_BROWSER_PROFILE, PROFILE_MAX_AGE_MS } = require('./constants');
const PROFILES_DIR = path.join(RUNTIME_DIR, 'profiles');
/**
* @param {string} sessionKey
* @param {'standard'|'real'} profile
* @returns {string|null}
*/
function resolveProfileDir(sessionKey, profile = 'standard') {
if (!sessionKey || !PERSIST_BROWSER_PROFILE) {
return null;
}
const hash = crypto.createHash('md5').update(String(sessionKey)).digest('hex');
const dir = path.join(PROFILES_DIR, `${hash}-${profile}`);
fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
return dir;
}
function profileExists(sessionKey, profile = 'standard') {
const dir = resolveProfileDir(sessionKey, profile);
if (!dir) return false;
return fs.existsSync(dir);
}
function pruneStaleProfiles() {
if (!PERSIST_BROWSER_PROFILE || PROFILE_MAX_AGE_MS <= 0) {
return 0;
}
if (!fs.existsSync(PROFILES_DIR)) {
return 0;
}
const now = Date.now();
let removed = 0;
for (const name of fs.readdirSync(PROFILES_DIR)) {
const full = path.join(PROFILES_DIR, name);
try {
const stat = fs.statSync(full);
if (!stat.isDirectory()) continue;
if (now - stat.mtimeMs > PROFILE_MAX_AGE_MS) {
fs.rmSync(full, { recursive: true, force: true });
removed++;
}
} catch (_) {
/* ignore */
}
}
return removed;
}
function getProfileStats() {
if (!fs.existsSync(PROFILES_DIR)) {
return { profilesDir: PROFILES_DIR, count: 0 };
}
const count = fs.readdirSync(PROFILES_DIR).filter((name) => {
try {
return fs.statSync(path.join(PROFILES_DIR, name)).isDirectory();
} catch (_) {
return false;
}
}).length;
return { profilesDir: PROFILES_DIR, count };
}
module.exports = {
PROFILES_DIR,
resolveProfileDir,
profileExists,
pruneStaleProfiles,
getProfileStats,
};
patches/puppeteer-api/lib/session-context.js
+38
View File
@@ -0,0 +1,38 @@
/**
* 会话上下文：合并请求 cookies 与磁盘 session，供各 API 统一使用
*/
const {
loadSession,
sessionCookiesForPage,
mergeCookies,
touchSession,
} = require('./session-store');
const { SESSION_TTL_MS } = require('./constants');
/**
* @param {{ sessionKey?: string }} antiBot
* @param {string} pageUrl
* @param {object[]} [requestCookies]
*/
function resolveSessionContext(antiBot, pageUrl, requestCookies = []) {
const sessionKey = antiBot?.sessionKey || '';
const storedSession = sessionKey ? loadSession(sessionKey) : null;
const sessionCookies = storedSession ? sessionCookiesForPage(storedSession, pageUrl) : [];
const mergedCookies = mergeCookies(requestCookies, sessionCookies);
return { sessionKey, storedSession, mergedCookies };
}
/**
* 成功过盾后滑动续期 session TTL
* @param {string} sessionKey
*/
function touchSessionIfPresent(sessionKey) {
if (sessionKey) {
touchSession(sessionKey, SESSION_TTL_MS);
}
}
module.exports = {
resolveSessionContext,
touchSessionIfPresent,
};
patches/puppeteer-api/lib/session-store.js
+83 -48
View File
@@ -1,5 +1,5 @@
/**
* 会话 Cookie 持久化：按 sessionKey 保存 cf_clearance 等，降低重复过盾概率
* 会话 Cookie 持久化：按 sessionKey 保存完整 cookie jar，降低重复过盾概率
*/
const fs = require('fs');
const path = require('path');
@@ -8,42 +8,50 @@ const { RUNTIME_DIR, SESSION_TTL_MS } = require('./constants');
const SESSIONS_DIR = path.join(RUNTIME_DIR, 'sessions');
/**
* 确保 sessions 目录存在
*/
function ensureSessionsDir() {
fs.mkdirSync(SESSIONS_DIR, { recursive: true, mode: 0o777 });
}
/**
* sessionKey 转安全文件名
* @param {string} sessionKey
* @returns {string}
*/
function keyToFilename(sessionKey) {
const hash = crypto.createHash('md5').update(sessionKey).digest('hex');
return `${hash}.json`;
}
/**
* @typedef {Object} StoredSession
* @property {string} sessionKey
* @property {number} savedAt
* @property {number} expiresAt
* @property {Array<{name:string,value:string,domain?:string,path?:string}>} cookies
* @param {{name:string,domain?:string,path?:string}} cookie
* @returns {string}
*/
function cookieIdentity(cookie) {
return `${cookie.name}|${cookie.domain || ''}|${cookie.path || '/'}`;
}
function mergeCookies(requestCookies, sessionCookies) {
const map = new Map();
for (const cookie of sessionCookies || []) {
if (cookie && cookie.name) map.set(cookieIdentity(cookie), cookie);
}
for (const cookie of requestCookies || []) {
if (cookie && cookie.name) map.set(cookieIdentity(cookie), cookie);
}
return Array.from(map.values());
}
function isSessionLikelyValid(session) {
if (!session || !Array.isArray(session.cookies)) return false;
if (session.expiresAt && Date.now() > session.expiresAt) return false;
return session.cookies.some((c) => c.name === 'cf_clearance' && c.value);
}
function hasCfClearance(session) {
if (!session || !Array.isArray(session.cookies)) return false;
return session.cookies.some((c) => c.name === 'cf_clearance' && c.value);
}
/**
* 加载已保存的会话 cookies
* @param {string} sessionKey
* @returns {StoredSession|null}
*/
function loadSession(sessionKey) {
if (!sessionKey) return null;
ensureSessionsDir();
const filePath = path.join(SESSIONS_DIR, keyToFilename(sessionKey));
if (!fs.existsSync(filePath)) return null;
try {
const raw = JSON.parse(fs.readFileSync(filePath, 'utf8'));
if (!raw || !Array.isArray(raw.cookies)) return null;
@@ -57,45 +65,36 @@ function loadSession(sessionKey) {
}
}
/**
* 保存会话 cookies（优先保留 CF 相关 cookie）
* @param {string} sessionKey
* @param {Array<{name:string,value:string,domain?:string,path?:string}>} cookies
* @param {number} [ttlMs]
*/
function saveSession(sessionKey, cookies, ttlMs = SESSION_TTL_MS) {
function touchSession(sessionKey, ttlMs = SESSION_TTL_MS) {
const session = loadSession(sessionKey);
if (!session) return;
const now = Date.now();
session.savedAt = now;
session.expiresAt = now + ttlMs;
fs.writeFileSync(path.join(SESSIONS_DIR, keyToFilename(sessionKey)), JSON.stringify(session, null, 0), { mode: 0o666 });
}
function saveSession(sessionKey, cookies, ttlMs = SESSION_TTL_MS, meta = {}) {
if (!sessionKey || !Array.isArray(cookies) || cookies.length === 0) return;
ensureSessionsDir();
const cfRelated = cookies.filter((c) =>
['cf_clearance', '__cf_bm', 'cf_chl_2'].includes(c.name)
|| c.name.startsWith('__cf')
);
const toSave = cfRelated.length > 0 ? cfRelated : cookies;
const now = Date.now();
const payload = {
sessionKey,
savedAt: now,
lastSuccessAt: now,
expiresAt: now + ttlMs,
cookies: toSave.map((c) => ({
lastHost: meta.lastHost || '',
savedProfile: meta.savedProfile || '',
cookies: cookies.map((c) => ({
name: c.name,
value: c.value,
domain: c.domain,
path: c.path || '/',
})),
};
const filePath = path.join(SESSIONS_DIR, keyToFilename(sessionKey));
fs.writeFileSync(filePath, JSON.stringify(payload, null, 0), { mode: 0o666 });
fs.writeFileSync(path.join(SESSIONS_DIR, keyToFilename(sessionKey)), JSON.stringify(payload, null, 0), { mode: 0o666 });
}
/**
* 将会话 cookies 转为 puppeteer setCookie 格式
* @param {StoredSession|null} session
* @param {string} pageUrl
* @returns {Array<{name:string,value:string,domain?:string,path?:string,url?:string}>}
*/
function sessionCookiesForPage(session, pageUrl) {
if (!session || !Array.isArray(session.cookies)) return [];
let hostname = '';
@@ -104,18 +103,54 @@ function sessionCookiesForPage(session, pageUrl) {
} catch (_) {
return session.cookies;
}
return session.cookies.map((c) => {
const cookie = { ...c };
if (!cookie.domain && hostname) {
cookie.url = `https://${hostname}/`;
}
if (!cookie.domain && hostname) cookie.url = `https://${hostname}/`;
return cookie;
});
}
function getSessionDebugInfo(sessionKey) {
if (!sessionKey) return null;
const filePath = path.join(SESSIONS_DIR, keyToFilename(sessionKey));
const session = loadSession(sessionKey);
if (!session) {
return { sessionKey, exists: false, filePath };
}
let mtime = null;
try {
mtime = fs.statSync(filePath).mtime.toISOString();
} catch (_) { /* ignore */ }
return {
sessionKey,
exists: true,
filePath,
mtime,
savedAt: session.savedAt,
expiresAt: session.expiresAt,
lastSuccessAt: session.lastSuccessAt,
lastHost: session.lastHost,
savedProfile: session.savedProfile,
cookieCount: session.cookies.length,
hasCfClearance: hasCfClearance(session),
likelyValid: isSessionLikelyValid(session),
};
}
function getSessionStats() {
ensureSessionsDir();
const count = fs.readdirSync(SESSIONS_DIR).filter((f) => f.endsWith('.json')).length;
return { sessionsDir: SESSIONS_DIR, count };
}
module.exports = {
loadSession,
saveSession,
touchSession,
sessionCookiesForPage,
mergeCookies,
isSessionLikelyValid,
hasCfClearance,
getSessionDebugInfo,
getSessionStats,
};
patches/puppeteer-api/lib/ui-pagination-runner.js
+172
View File
@@ -0,0 +1,172 @@
/**
* UI 翻页抓取：供 ui-pagination 与 auth-intercept-and-paginate 共用
*/
const crypto = require('crypto');
function generateSafeHash(obj) {
if (!obj) return '';
const clone = JSON.parse(JSON.stringify(obj));
const removeDynamicKeys = (target) => {
if (typeof target !== 'object' || target === null) return;
['timestamp', 'time', 'serverTime', 'reqId', 'requestId', 'traceId'].forEach((k) => delete target[k]);
Object.values(target).forEach((val) => removeDynamicKeys(val));
};
removeDynamicKeys(clone);
return crypto.createHash('md5').update(JSON.stringify(clone)).digest('hex');
}
function classifyPuppeteerError(err) {
const msg = (err?.message || '').toLowerCase();
if (err?.name === 'TimeoutError' || msg.includes('timeout')) return 'timeout';
if (msg.includes('net::') || msg.includes('network') || msg.includes('err_connection')
|| msg.includes('err_address_unreachable') || msg.includes('err_name_not_resolved')) return 'network';
if (msg.includes('navigation') || msg.includes('frame was detached')) return 'navigation';
if (msg.includes('target closed') || msg.includes('session closed')) return 'target_closed';
return 'unknown';
}
/**
* @param {import('puppeteer').Page} page
* @param {{
* apiUrl: string,
* nextBtnSelector: string,
* clicksToPerform: number,
* waitMs?: number,
* firstPageData?: object,
* authActions?: object[],
* executeAuthActions?: (page: import('puppeteer').Page, actions: object[]) => Promise<void>,
* }} options
*/
async function runUiPagination(page, options) {
const {
apiUrl,
nextBtnSelector,
clicksToPerform,
waitMs = 2000,
firstPageData,
authActions,
executeAuthActions,
} = options;
const capturedData = [];
const debugLogs = [];
const log = (msg) => { console.log(msg); debugLogs.push(msg); };
const seenHashes = new Set();
if (firstPageData) {
seenHashes.add(generateSafeHash(firstPageData));
}
if (Array.isArray(authActions) && authActions.length > 0 && executeAuthActions) {
log('[状态同步] 正在为翻页引擎注入授权状态...');
await executeAuthActions(page, authActions);
log('[状态同步] 授权执行完毕，等待目标列表加载...');
const authSettled = await page.waitForResponse(
(resp) => resp.url().includes(apiUrl) && resp.request().method() !== 'OPTIONS' && resp.status() >= 200 && resp.status() < 400,
{ timeout: 5000 }
).catch(() => null);
if (!authSettled) {
log('[状态同步] 未捕获到目标 API 响应，降级为固定等待 3 秒...');
await new Promise((r) => setTimeout(r, 3000));
}
}
for (let i = 0; i < clicksToPerform; i++) {
const targetPageNum = i + 2;
let pageData = null;
let attempts = 0;
const maxAttempts = 3;
await page.evaluate(() => {
document.querySelectorAll('.el-loading-mask, .ant-spin-nested-loading, .el-loading-spinner').forEach((el) => el.remove());
});
await page.waitForFunction(
() => !document.querySelector('.el-loading-mask, .ant-spin-nested-loading, .el-loading-spinner'),
{ timeout: 1000, polling: 100 }
).catch(() => new Promise((r) => setTimeout(r, 500)));
while (attempts < maxAttempts && !pageData) {
attempts++;
log(`[抓取] 准备抓取第 ${targetPageNum} 页 (尝试 ${attempts}/${maxAttempts})...`);
let responseHandler = null;
let apiTimeoutId = null;
const waitForNewPageApi = new Promise((resolve) => {
const cleanupListener = () => {
if (apiTimeoutId) { clearTimeout(apiTimeoutId); apiTimeoutId = null; }
if (responseHandler) { page.off('response', responseHandler); responseHandler = null; }
};
responseHandler = async (response) => {
const status = response.status();
if (response.url().includes(apiUrl) && response.request().method() !== 'OPTIONS' && status >= 200 && status < 400) {
try {
const responseData = await response.json();
const currentHash = generateSafeHash(responseData);
if (!seenHashes.has(currentHash)) {
seenHashes.add(currentHash);
cleanupListener();
resolve(responseData);
} else {
log('[防轮询] 抛弃重复数据，继续监听...');
}
} catch (_) { /* 非 JSON 响应跳过 */ }
}
};
page.on('response', responseHandler);
apiTimeoutId = setTimeout(() => { cleanupListener(); resolve(null); }, 15000);
});
try {
await page.waitForSelector(nextBtnSelector, { timeout: 15000 });
const clickStatus = await page.evaluate((sel) => {
const btn = document.querySelector(sel);
if (!btn) return 'not_found';
if (btn.disabled || btn.classList.contains('is-disabled') || btn.classList.contains('disabled') || btn.getAttribute('aria-disabled') === 'true') {
return 'disabled';
}
const className = btn.className || '';
if (btn.disabled === true || className.includes('disabled') || btn.getAttribute('aria-disabled') === 'true') {
return 'disabled';
}
btn.scrollIntoView({ behavior: 'instant', block: 'center' });
btn.click();
btn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
return 'success';
}, nextBtnSelector);
if (clickStatus === 'disabled') {
log(`[提示] 第 ${targetPageNum} 页按钮已被禁用，说明到底了，抓取提前结束。`);
break;
}
log('[动作] 双引擎原生点击已执行！');
} catch (clickErr) {
log(`[错误] 寻找按钮异常 (${classifyPuppeteerError(clickErr)}): ${clickErr.message}`);
}
pageData = await waitForNewPageApi;
if (!pageData && attempts < maxAttempts) {
log('[重试警报] 雷达未扫描到新数据，准备重试...');
await new Promise((r) => setTimeout(r, 500 * Math.pow(2, attempts - 1)));
}
}
if (pageData) {
capturedData.push(pageData);
log(`[成功] 斩获第 ${targetPageNum} 页数据！`);
await new Promise((r) => setTimeout(r, waitMs));
} else {
log(`[致命异常] 连续 ${maxAttempts} 次尝试均失败，放弃后续翻页。`);
break;
}
}
return { data: capturedData, debugLogs };
}
module.exports = {
runUiPagination,
generateSafeHash,
};
patches/puppeteer-api/server.js
+272 -175
View File
@@ -1,6 +1,5 @@
const express = require('express');
const fs = require('fs');
const crypto = require('crypto');
const {
rewriteHttpsToHttpForLiteralIp,
attachHttpIpRewriteInterceptor,
@@ -17,19 +16,35 @@ const {
MAX_CONCURRENT_BROWSERS_REAL,
API_INTERCEPT_TIMEOUT_MS,
API_INTERCEPT_TIMEOUT_REAL_MS,
NAVIGATION_TIMEOUT_MS,
NAVIGATION_WAIT_UNTIL,
NAVIGATION_MAX_RETRIES,
SESSION_TTL_MS,
PERSIST_BROWSER_PROFILE,
POOL_IDLE_MS,
MAX_POOLED_BROWSERS,
} = require('./lib/constants');
const {
normalizeAntiBot,
createBrowserSession,
acquireBrowserSession,
runWithProfileLimit,
createManagedPage,
getFactoryStats,
standardLimiter,
browserPool,
} = require('./lib/browser-factory');
const { launchStandardBrowser } = require('./lib/launch-standard');
const { isRealBrowserAvailable } = require('./lib/launch-real-browser');
const { handleCloudflareChallenge, isCaptchaConfigured } = require('./lib/cf-handler');
const { loadSession, saveSession, sessionCookiesForPage } = require('./lib/session-store');
const {
saveSession,
getSessionDebugInfo,
getSessionStats,
} = require('./lib/session-store');
const { resolveSessionContext, touchSessionIfPresent } = require('./lib/session-context');
const { getProfileStats, profileExists } = require('./lib/profile-dir');
const { runUiPagination } = require('./lib/ui-pagination-runner');
const app = express();
app.use(express.json({ limit: '50mb' }));
@@ -63,16 +78,11 @@ function sendTaskError(res, error, extra = {}) {
return res.status(500).json({ success: false, error: error.message, ...extra });
}
function generateSafeHash(obj) {
if (!obj) return '';
const clone = JSON.parse(JSON.stringify(obj));
const removeDynamicKeys = (target) => {
if (typeof target !== 'object' || target === null) return;
['timestamp', 'time', 'serverTime', 'reqId', 'requestId', 'traceId'].forEach((k) => delete target[k]);
Object.values(target).forEach((val) => removeDynamicKeys(val));
};
removeDynamicKeys(clone);
return crypto.createHash('md5').update(JSON.stringify(clone)).digest('hex');
async function navigateToPage(page, url) {
await withExponentialBackoff(
() => page.goto(url, { waitUntil: NAVIGATION_WAIT_UNTIL, timeout: NAVIGATION_TIMEOUT_MS }),
{ maxRetries: NAVIGATION_MAX_RETRIES, baseDelayMs: 1000 }
);
}
async function safeCloseBrowser(browser) {
@@ -347,6 +357,9 @@ app.get('/api/stats', (_req, res) => {
shuttingDown,
queue: factoryStats.standard,
queueReal: factoryStats.real,
pool: factoryStats.pool,
sessions: getSessionStats(),
profiles: getProfileStats(),
realBrowserReady: factoryStats.realBrowserReady,
captchaConfigured: isCaptchaConfigured(),
chrome: {
@@ -358,10 +371,35 @@ app.get('/api/stats', (_req, res) => {
maxConcurrentBrowsersReal: MAX_CONCURRENT_BROWSERS_REAL,
apiInterceptTimeoutMs: API_INTERCEPT_TIMEOUT_MS,
apiInterceptTimeoutRealMs: API_INTERCEPT_TIMEOUT_REAL_MS,
navigationTimeoutMs: NAVIGATION_TIMEOUT_MS,
navigationWaitUntil: NAVIGATION_WAIT_UNTIL,
sessionTtlMs: SESSION_TTL_MS,
persistBrowserProfile: PERSIST_BROWSER_PROFILE,
poolIdleMs: POOL_IDLE_MS,
maxPooledBrowsers: MAX_POOLED_BROWSERS,
},
});
});
/** 运维调试：查看指定 sessionKey 的磁盘会话与 profile 状态 */
app.get('/api/session/:sessionKey', (req, res) => {
const sessionKey = decodeURIComponent(req.params.sessionKey || '');
if (!sessionKey) {
return res.status(400).json({ success: false, error: 'sessionKey 不能为空' });
}
const info = getSessionDebugInfo(sessionKey);
const profileStandard = getProfileStats();
res.json({
success: true,
session: info,
profileExists: {
standard: profileExists(sessionKey, 'standard'),
real: profileExists(sessionKey, 'real'),
},
profilesDir: profileStandard.profilesDir,
});
});
/** 运维自检：实际尝试 launch 一次 Browser（profile=standard|real） */
app.get('/api/browser-launch-test', async (req, res) => {
const profile = req.query.profile === 'real' ? 'real' : 'standard';
@@ -410,31 +448,28 @@ app.post('/api/auth-and-intercept', async (req, res) => {
const antiBot = normalizeAntiBot(rawAntiBot);
const profile = antiBot.profile || 'standard';
const interceptTimeout = resolveInterceptTimeout(antiBot);
const cfLog = { profile, cfDetected: false, turnstileStage: null, solverUsed: false, elapsedMs: 0 };
const cfLog = { profile, cfDetected: false, turnstileStage: null, solverUsed: false, elapsedMs: 0, browserReused: false };
try {
await runBrowserTask('auth-and-intercept', async () => {
let browser;
let sessionCleanup = null;
let browserSession = null;
let interceptorCleanup = null;
const taskStarted = Date.now();
let cfDestroyBrowser = false;
try {
const session = await createBrowserSession({
browserSession = await acquireBrowserSession({
profile,
extraArgs: ['--mute-audio'],
sessionKey: antiBot.sessionKey || '',
});
browser = session.browser;
sessionCleanup = session.cleanup;
cfLog.browserReused = !!browserSession.reused;
const sessionData = antiBot.sessionKey ? loadSession(antiBot.sessionKey) : null;
const sessionCookies = sessionData
? sessionCookiesForPage(sessionData, pageUrl)
: [];
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl);
const page = await createManagedPage(browser, session.page, {
const page = await createManagedPage(browserSession.browser, browserSession.page, {
userAgent: DEFAULT_UA,
cookies: sessionCookies,
cookies: mergedCookies,
});
const httpResolvedUrl = await resolveHttpRedirects(pageUrl).catch(() => pageUrl);
@@ -452,10 +487,7 @@ app.post('/api/auth-and-intercept', async (req, res) => {
}
await attachHttpIpRewriteInterceptor(page, 'auth-and-intercept', { shareToken });
await withExponentialBackoff(
() => page.goto(pageUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }),
{ maxRetries: 2, baseDelayMs: 1000 }
);
await navigateToPage(page, pageUrl);
let finalPageUrl = await waitForUrlSettled(page);
if (finalPageUrl !== pageUrl) {
@@ -463,13 +495,14 @@ app.post('/api/auth-and-intercept', async (req, res) => {
}
if (antiBot.enabled) {
const cfResult = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled);
const cfResult = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled, storedSession);
cfLog.cfDetected = cfResult.cfDetected;
cfLog.turnstileStage = cfResult.stage;
cfLog.solverUsed = cfResult.solverUsed;
cfLog.elapsedMs = cfResult.elapsedMs;
if (!cfResult.success) {
cfDestroyBrowser = true;
res.status(422).json({
success: false,
code: cfResult.code || 'CF_TURNSTILE_FAILED',
@@ -482,6 +515,7 @@ app.post('/api/auth-and-intercept', async (req, res) => {
}
finalPageUrl = page.url();
touchSessionIfPresent(antiBot.sessionKey);
}
const finalShareToken = resolveShareToken(finalPageUrl);
@@ -502,7 +536,14 @@ app.post('/api/auth-and-intercept', async (req, res) => {
}));
if (antiBot.sessionKey) {
saveSession(antiBot.sessionKey, cookiePayload);
let lastHost = '';
try {
lastHost = new URL(finalPageUrl).hostname;
} catch (_) { /* ignore */ }
saveSession(antiBot.sessionKey, cookiePayload, SESSION_TTL_MS, {
lastHost,
savedProfile: profile,
});
}
res.json({
@@ -513,14 +554,13 @@ app.post('/api/auth-and-intercept', async (req, res) => {
cookies: cookiePayload,
cf: cfLog,
profile,
browserReused: browserSession.reused,
elapsedMs: Date.now() - taskStarted,
});
} finally {
if (interceptorCleanup) interceptorCleanup();
if (sessionCleanup) {
await sessionCleanup();
} else {
await safeCloseBrowser(browser);
if (browserSession) {
await browserSession.release(cfDestroyBrowser);
}
}
}, profile);
@@ -541,18 +581,19 @@ app.post('/api/batch-fetch', async (req, res) => {
try {
await runBrowserTask('batch-fetch', async () => {
let browser;
let sessionCleanup = null;
let browserSession = null;
try {
const session = await createBrowserSession({
browserSession = await acquireBrowserSession({
profile,
extraArgs: ['--disable-web-security'],
sessionKey: antiBot.sessionKey || '',
});
browser = session.browser;
sessionCleanup = session.cleanup;
const page = await createManagedPage(browser, session.page, { cookies });
const firstOrigin = new URL(tasks[0].fullUrl).origin;
const firstUrl = tasks[0].fullUrl;
const { mergedCookies } = resolveSessionContext(antiBot, firstUrl, cookies || []);
const page = await createManagedPage(browserSession.browser, browserSession.page, { cookies: mergedCookies });
const firstOrigin = new URL(firstUrl).origin;
await page.setRequestInterception(true);
page.on('request', (reqObj) => {
@@ -570,10 +611,7 @@ app.post('/api/batch-fetch', async (req, res) => {
}
});
await withExponentialBackoff(
() => page.goto(firstOrigin, { waitUntil: 'domcontentloaded', timeout: 30000 }),
{ maxRetries: 2, baseDelayMs: 800 }
);
await navigateToPage(page, firstOrigin);
const FETCH_CONCURRENCY = Math.max(1, parseInt(process.env.FETCH_CONCURRENCY || '5', 10));
const FETCH_RETRY = 2;
@@ -636,12 +674,10 @@ app.post('/api/batch-fetch', async (req, res) => {
return out;
}, tasks, FETCH_CONCURRENCY, FETCH_RETRY);
res.json({ success: true, results: fetchResults, profile });
res.json({ success: true, results: fetchResults, profile, browserReused: browserSession.reused });
} finally {
if (sessionCleanup) {
await sessionCleanup();
} else {
await safeCloseBrowser(browser);
if (browserSession) {
await browserSession.release(false);
}
}
}, profile);
@@ -670,27 +706,25 @@ app.post('/api/ui-pagination', async (req, res) => {
try {
await runBrowserTask('ui-pagination', async () => {
let browser;
let sessionCleanup = null;
const capturedData = [];
let browserSession = null;
const debugLogs = [];
const log = (msg) => { console.log(msg); debugLogs.push(msg); };
const seenHashes = new Set();
if (firstPageData) { seenHashes.add(generateSafeHash(firstPageData)); }
let cfDestroyBrowser = false;
try {
const session = await createBrowserSession({
browserSession = await acquireBrowserSession({
profile,
extraArgs: ['--window-size=1920,1080'],
sessionKey: antiBot.sessionKey || '',
});
browser = session.browser;
sessionCleanup = session.cleanup;
const page = await createManagedPage(browser, session.page, {
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, navigationUrl, cookies || []);
const page = await createManagedPage(browserSession.browser, browserSession.page, {
viewport: { width: 1920, height: 1080 },
cookies,
cookies: mergedCookies,
});
const log = (msg) => { console.log(msg); debugLogs.push(msg); };
log(`[启动] 准备访问页面: ${navigationUrl}`);
if (finalPageUrl && finalPageUrl !== pageUrl) {
log(`[URL解析] 使用 auth 阶段落地地址，跳过短链重定向: ${pageUrl} -> ${finalPageUrl}`);
@@ -700,10 +734,7 @@ app.post('/api/ui-pagination', async (req, res) => {
await seedShareTokenCookie(page, navigationUrl);
await attachHttpIpRewriteInterceptor(page, 'ui-pagination', { shareToken });
await withExponentialBackoff(
() => page.goto(navigationUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }),
{ maxRetries: 2, baseDelayMs: 1000 }
);
await navigateToPage(page, navigationUrl);
const settledPageUrl = await waitForUrlSettled(page);
if (settledPageUrl !== navigationUrl) {
@@ -711,8 +742,9 @@ app.post('/api/ui-pagination', async (req, res) => {
}
if (antiBot.enabled) {
const cfResult = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled);
const cfResult = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled, storedSession);
if (!cfResult.success) {
cfDestroyBrowser = true;
res.status(422).json({
success: false,
code: cfResult.code || 'CF_TURNSTILE_FAILED',
@@ -723,125 +755,33 @@ app.post('/api/ui-pagination', async (req, res) => {
});
return;
}
touchSessionIfPresent(antiBot.sessionKey);
}
if (Array.isArray(authActions) && authActions.length > 0) {
log('[状态同步] 正在为翻页引擎注入授权状态...');
await executeAuthActions(page, authActions);
log('[状态同步] 授权执行完毕，等待目标列表加载...');
const authSettled = await page.waitForResponse(
(resp) => resp.url().includes(apiUrl) && resp.request().method() !== 'OPTIONS' && resp.status() >= 200 && resp.status() < 400,
{ timeout: 5000 }
).catch(() => null);
if (!authSettled) {
log('[状态同步] 未捕获到目标 API 响应，降级为固定等待 3 秒...');
await new Promise((r) => setTimeout(r, 3000));
}
}
for (let i = 0; i < clicksToPerform; i++) {
const targetPageNum = i + 2;
let pageData = null;
let attempts = 0;
const maxAttempts = 3;
await page.evaluate(() => {
document.querySelectorAll('.el-loading-mask, .ant-spin-nested-loading, .el-loading-spinner').forEach((el) => el.remove());
const paginationResult = await runUiPagination(page, {
apiUrl,
nextBtnSelector,
clicksToPerform,
waitMs,
firstPageData,
authActions,
executeAuthActions,
});
await page.waitForFunction(
() => !document.querySelector('.el-loading-mask, .ant-spin-nested-loading, .el-loading-spinner'),
{ timeout: 1000, polling: 100 }
).catch(() => new Promise((r) => setTimeout(r, 500)));
while (attempts < maxAttempts && !pageData) {
attempts++;
log(`[抓取] 准备抓取第 ${targetPageNum} 页 (尝试 ${attempts}/${maxAttempts})...`);
let responseHandler = null;
let apiTimeoutId = null;
const waitForNewPageApi = new Promise((resolve) => {
const cleanupListener = () => {
if (apiTimeoutId) { clearTimeout(apiTimeoutId); apiTimeoutId = null; }
if (responseHandler) { page.off('response', responseHandler); responseHandler = null; }
};
responseHandler = async (response) => {
const status = response.status();
if (response.url().includes(apiUrl) && response.request().method() !== 'OPTIONS' && status >= 200 && status < 400) {
try {
const responseData = await response.json();
const currentHash = generateSafeHash(responseData);
if (!seenHashes.has(currentHash)) {
seenHashes.add(currentHash);
cleanupListener();
resolve(responseData);
} else {
log('[防轮询] 抛弃重复数据，继续监听...');
}
} catch (_) { /* 非 JSON 响应跳过 */ }
}
};
page.on('response', responseHandler);
apiTimeoutId = setTimeout(() => { cleanupListener(); resolve(null); }, 15000);
res.json({
success: true,
data: paginationResult.data,
debug: [...debugLogs, ...paginationResult.debugLogs],
profile,
browserReused: browserSession.reused,
});
try {
await page.waitForSelector(nextBtnSelector, { timeout: 15000 });
const clickStatus = await page.evaluate((sel) => {
const btn = document.querySelector(sel);
if (!btn) return 'not_found';
if (btn.disabled || btn.classList.contains('is-disabled') || btn.classList.contains('disabled') || btn.getAttribute('aria-disabled') === 'true') {
return 'disabled';
}
const className = btn.className || '';
if (btn.disabled === true || className.includes('disabled') || btn.getAttribute('aria-disabled') === 'true') {
return 'disabled';
}
btn.scrollIntoView({ behavior: 'instant', block: 'center' });
btn.click();
btn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
return 'success';
}, nextBtnSelector);
if (clickStatus === 'disabled') {
log(`[提示] 第 ${targetPageNum} 页按钮已被禁用，说明到底了，抓取提前结束。`);
break;
}
log('[动作] 双引擎原生点击已执行！');
} catch (clickErr) {
log(`[错误] 寻找按钮异常 (${classifyPuppeteerError(clickErr)}): ${clickErr.message}`);
}
pageData = await waitForNewPageApi;
if (!pageData && attempts < maxAttempts) {
log('[重试警报] 雷达未扫描到新数据，准备重试...');
await new Promise((r) => setTimeout(r, 500 * Math.pow(2, attempts - 1)));
}
}
if (pageData) {
capturedData.push(pageData);
log(`[成功] 斩获第 ${targetPageNum} 页数据！`);
await new Promise((r) => setTimeout(r, waitMs));
} else {
log(`[致命异常] 连续 ${maxAttempts} 次尝试均失败，放弃后续翻页。`);
break;
}
}
res.json({ success: true, data: capturedData, debug: debugLogs, profile });
} catch (error) {
if (!res.headersSent) {
sendTaskError(res, error, { debug: debugLogs, profile });
}
} finally {
if (sessionCleanup) {
await sessionCleanup();
} else {
await safeCloseBrowser(browser);
if (browserSession) {
await browserSession.release(cfDestroyBrowser);
}
}
}, profile);
@@ -850,6 +790,160 @@ app.post('/api/ui-pagination', async (req, res) => {
}
});
/** 接口四：单 Browser 内 auth + 拦截 + UI 翻页（降低 CF 二次触发） */
app.post('/api/auth-intercept-and-paginate', async (req, res) => {
const {
pageUrl,
apiUrls,
authActions,
antiBot: rawAntiBot,
pagination,
} = req.body;
if (!pageUrl || !Array.isArray(apiUrls) || !pagination) {
return res.status(400).json({ success: false, error: '参数错误：需要 pageUrl、apiUrls、pagination' });
}
const {
apiUrl,
nextBtnSelector,
clicksToPerform = 0,
waitMs = 2000,
firstPageData,
} = pagination;
const antiBot = normalizeAntiBot(rawAntiBot);
const profile = antiBot.profile || 'standard';
const interceptTimeout = resolveInterceptTimeout(antiBot);
const cfLog = { profile, cfDetected: false, turnstileStage: null, solverUsed: false, elapsedMs: 0, browserReused: false };
try {
await runBrowserTask('auth-intercept-and-paginate', async () => {
let browserSession = null;
let interceptorCleanup = null;
const taskStarted = Date.now();
let cfDestroyBrowser = false;
try {
browserSession = await acquireBrowserSession({
profile,
extraArgs: ['--mute-audio', '--window-size=1920,1080'],
sessionKey: antiBot.sessionKey || '',
});
cfLog.browserReused = !!browserSession.reused;
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl);
const page = await createManagedPage(browserSession.browser, browserSession.page, {
userAgent: DEFAULT_UA,
viewport: { width: 1920, height: 1080 },
cookies: mergedCookies,
});
const httpResolvedUrl = await resolveHttpRedirects(pageUrl).catch(() => pageUrl);
const interceptor = createApiInterceptor(page, apiUrls, interceptTimeout);
interceptorCleanup = interceptor.cleanup;
let shareToken = resolveShareToken(pageUrl, httpResolvedUrl);
await seedShareTokenCookie(page, pageUrl);
await attachHttpIpRewriteInterceptor(page, 'auth-intercept-and-paginate', { shareToken });
await navigateToPage(page, pageUrl);
let finalPageUrl = await waitForUrlSettled(page);
if (antiBot.enabled) {
const cfResult = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled, storedSession);
cfLog.cfDetected = cfResult.cfDetected;
cfLog.turnstileStage = cfResult.stage;
cfLog.solverUsed = cfResult.solverUsed;
cfLog.elapsedMs = cfResult.elapsedMs;
if (!cfResult.success) {
cfDestroyBrowser = true;
res.status(422).json({
success: false,
code: cfResult.code || 'CF_TURNSTILE_FAILED',
challengeType: cfResult.challengeType || 'turnstile',
stage: cfResult.stage || 'timeout',
error: 'Cloudflare Turnstile 验证失败',
cf: cfLog,
});
return;
}
finalPageUrl = page.url();
touchSessionIfPresent(antiBot.sessionKey);
}
await executeAuthActions(page, authActions);
await interceptor.waitForApis(interceptTimeout);
const rawCookies = await page.cookies();
const cookiePayload = rawCookies.map((c) => ({
name: c.name,
value: c.value,
domain: c.domain,
path: c.path,
}));
if (antiBot.sessionKey) {
let lastHost = '';
try {
lastHost = new URL(finalPageUrl).hostname;
} catch (_) { /* ignore */ }
saveSession(antiBot.sessionKey, cookiePayload, SESSION_TTL_MS, {
lastHost,
savedProfile: profile,
});
}
let extraPages = [];
let paginationDebug = [];
if (clicksToPerform > 0 && apiUrl && nextBtnSelector) {
let effectiveFirstPageData = firstPageData;
if (!effectiveFirstPageData) {
const matchedKey = Object.keys(interceptor.interceptedApis).find((k) => k === apiUrl || k.includes(apiUrl));
if (matchedKey) {
effectiveFirstPageData = interceptor.interceptedApis[matchedKey].data;
}
}
const paginationResult = await runUiPagination(page, {
apiUrl,
nextBtnSelector,
clicksToPerform,
waitMs,
firstPageData: effectiveFirstPageData,
authActions,
executeAuthActions,
});
extraPages = paginationResult.data;
paginationDebug = paginationResult.debugLogs;
}
res.json({
success: true,
pageUrl,
finalPageUrl,
interceptedApis: interceptor.interceptedApis,
cookies: cookiePayload,
extraPages,
debug: paginationDebug,
cf: cfLog,
profile,
browserReused: browserSession.reused,
elapsedMs: Date.now() - taskStarted,
});
} finally {
if (interceptorCleanup) interceptorCleanup();
if (browserSession) {
await browserSession.release(cfDestroyBrowser);
}
}
}, profile);
} catch (error) {
if (!res.headersSent) sendTaskError(res, error, { cf: cfLog, profile });
}
});
const PORT = parseInt(process.env.PORT || '3001', 10);
const server = app.listen(PORT, '127.0.0.1', () => {
const chromePath = resolveChromeExecutable();
@@ -874,6 +968,9 @@ async function gracefulShutdown(signal) {
await new Promise((r) => setTimeout(r, 1000));
}
console.log('[shutdown] 排空 Browser 温池...');
await browserPool.drain();
const remaining = standardLimiter.getStats();
if (remaining.active > 0) {
console.warn(`[shutdown] 超时，仍有 ${remaining.active} 个 Browser 在运行，强制退出`);
public/assets/js/backend/split/ticket.js
+50 -3
View File
@@ -99,6 +99,12 @@ define(['jquery', 'bootstrap', 'backend', 'table', 'form'], function ($, undefin
operate: false,
formatter: Controller.api.formatter.syncDisplay
},
{
field: 'sync_success_text',
title: __('Sync success time'),
operate: false,
formatter: Controller.api.formatter.syncSuccessDisplay
},
$.extend(
{field: 'status', title: __('Status'), searchList: Config.statusList, formatter: Table.api.formatter.toggle, yes: 'normal', no: 'hidden'},
searchSelectMeta(false)
@@ -137,6 +143,7 @@ define(['jquery', 'bootstrap', 'backend', 'table', 'form'], function ($, undefin
table.on('load-success.bs.table', function (e, res) {
Controller.api.renderSummaryRow(res);
Controller.api.initSyncTooltips(table);
var pendingIds = window.__splitTicketPendingPostAddSyncIds;
if (!pendingIds || !pendingIds.length) {
return;
@@ -310,11 +317,26 @@ define(['jquery', 'bootstrap', 'backend', 'table', 'form'], function ($, undefin
*/
finishTicketsSync: function (table) {
Controller.api.syncingTicketIds = [];
table.bootstrapTable('refresh');
table.bootstrapTable('refresh', {
silent: true
});
Controller.api.initSyncTooltips(table);
var ids = Table.api.selectedids(table);
$('.btn-sync').toggleClass('btn-disabled disabled', ids.length === 0);
},
/**
* 初始化同步状态列 tooltip（完整异常原因）
*/
initSyncTooltips: function (table) {
var $container = table.closest('.bootstrap-table');
$container.find('[data-sync-tooltip="1"]').tooltip({
container: 'body',
placement: 'auto top',
html: false
});
},
formatter: {
/**
* 操作列：编辑 → 拷贝 → 删除
@@ -408,13 +430,38 @@ define(['jquery', 'bootstrap', 'backend', 'table', 'form'], function ($, undefin
return Controller.api.formatter.syncingDisplayHtml();
}
var text = value || '';
var tooltip = row.sync_tooltip_text ? String(row.sync_tooltip_text).trim() : '';
if (tooltip !== '') {
tooltip = tooltip.replace(/\r?\n/g, '；');
}
var hasTooltip = tooltip !== '';
var color = 'danger';
if (row.sync_status === 'success') {
var extraClass = '';
if (row.sync_auto_paused) {
color = 'warning';
extraClass = ' split-ticket-sync-status-paused';
} else if (row.sync_status === 'success') {
color = 'success';
} else if (row.sync_status === 'pending') {
color = 'muted';
} else if (row.sync_status === 'error') {
extraClass = ' split-ticket-sync-status-error';
}
return '<span class="text-' + color + '">' + Fast.api.escape(text) + '</span>';
var attrs = '';
if (hasTooltip) {
attrs = ' data-sync-tooltip="1" data-toggle="tooltip" title="' + Fast.api.escape(tooltip) + '"';
}
return '<span class="text-' + color + extraClass + '"' + attrs + '>' + Fast.api.escape(text) + '</span>';
},
syncSuccessDisplay: function (value, row) {
var text = value == null ? '' : String(value).trim();
if (text === '') {
return '<span class="text-muted">-</span>';
}
var colorClass = row.status === 'normal'
? 'split-sync-success-normal'
: 'split-sync-success-stopped';
return '<span class="' + colorClass + '">' + Fast.api.escape(text) + '</span>';
},
syncingDisplayHtml: function () {
var label = (typeof Config.syncInProgressMsg !== 'undefined' && Config.syncInProgressMsg)