Compare commits

..

2 Commits

Author SHA1 Message Date
root c3223f026e 修复工单PHP CLI同步问题 系统设置可以设置并发阈值 2026-07-02 19:03:06 +08:00
root 638b3c4032 提交同步状态为异步 2026-07-02 03:39:29 +08:00
28 changed files with 1807 additions and 283 deletions
+1 -1
View File
@@ -172,7 +172,7 @@ class Huojian extends AbstractScrmSpider
{ {
$unifiedData = $this->unifiedData; $unifiedData = $this->unifiedData;
$unifiedData->todayNewCount = (int) ($allListPagesData[0]['data']['counterWorker']['newTodayFriend'] ?? 0); $unifiedData->todayNewCount = (int) ($allListPagesData[0]['data']['counterWorker']['newTodayFriend'] ?? 0) + (int) ($allListPagesData[0]['data']['counterWorker']['newRemovedTodayFriend'] ?? 0);
$count = 0; $count = 0;
foreach ($allListPagesData as $pageRaw) { foreach ($allListPagesData as $pageRaw) {
+1 -1
View File
@@ -85,7 +85,7 @@ try {
// 长链:快速验证业务页拦截(推荐先测通) // 长链:快速验证业务页拦截(推荐先测通)
// $pageUrl = 'https://user.a2c.chat/visitors/counter/share?id=1b2cf9a91e2647c185b252e723c871de'; // $pageUrl = 'https://user.a2c.chat/visitors/counter/share?id=1b2cf9a91e2647c185b252e723c871de';
// 短链:与客户后台一致,上线前必须回归 // 短链:与客户后台一致,上线前必须回归
$pageUrl = 'https://yyk.ink/91SEWrL'; $pageUrl = 'https://yyk.ink/3xj1B2';
$username = ""; $username = "";
$password = ""; $password = "";
$spider = (new A2c($pageUrl, $username, $password))->setVerbose(true); $spider = (new A2c($pageUrl, $username, $password))->setVerbose(true);
@@ -0,0 +1,19 @@
-- 双通道定时同步:Node 通道上限与候选池配置(已有环境增量执行)
SET NAMES utf8mb4;
INSERT INTO `fa_config` (`name`, `group`, `title`, `tip`, `type`, `visible`, `value`, `content`, `rule`, `extend`, `setting`)
SELECT 'split_sync_max_per_cron', 'split', 'Node通道单次同步上限', '每分钟 cron 在 Node 通道最多处理几条到期工单(仅影响需调用 Node 的类型);纯 PHP 直连类型不受此限制', 'number', '', '5', '', '', '', NULL
FROM DUAL WHERE NOT EXISTS (SELECT 1 FROM `fa_config` WHERE `name` = 'split_sync_max_per_cron' LIMIT 1);
INSERT INTO `fa_config` (`name`, `group`, `title`, `tip`, `type`, `visible`, `value`, `content`, `rule`, `extend`, `setting`)
SELECT 'split_sync_node_candidate_multiplier', 'split', 'Node通道候选池倍数', 'Node 候选工单数 = 单次上限 × 本倍数,用于公平轮转', 'number', '', '10', '', '', '', NULL
FROM DUAL WHERE NOT EXISTS (SELECT 1 FROM `fa_config` WHERE `name` = 'split_sync_node_candidate_multiplier' LIMIT 1);
UPDATE `fa_config`
SET `title` = 'Node通道单次同步上限',
`tip` = '每分钟 cron 在 Node 通道最多处理几条到期工单(仅影响需调用 Node 的类型);纯 PHP 直连类型不受此限制'
WHERE `name` = 'split_sync_max_per_cron';
UPDATE `fa_config`
SET `value` = '5'
WHERE `name` = 'split_sync_max_per_cron' AND (`value` = '' OR `value` = '2');
View File
+74 -20
View File
@@ -8,8 +8,8 @@ use app\admin\model\split\Link as LinkModel;
use app\admin\model\split\Ticket as TicketModel; use app\admin\model\split\Ticket as TicketModel;
use app\common\controller\Backend; use app\common\controller\Backend;
use app\common\service\SplitTicketRuleService; use app\common\service\SplitTicketRuleService;
use app\common\service\SplitTicketSyncDispatchService;
use app\common\service\SplitTicketSyncLogger; use app\common\service\SplitTicketSyncLogger;
use app\common\service\SplitTicketSyncService;
use think\Db; use think\Db;
use think\Lang; use think\Lang;
use think\Loader; use think\Loader;
@@ -39,7 +39,7 @@ class Ticket extends Backend
protected $modelSceneValidate = true; protected $modelSceneValidate = true;
/** @var string[] 无需鉴权的方法 */ /** @var string[] 无需鉴权的方法 */
protected $noNeedRight = ['script']; protected $noNeedRight = ['script', 'syncpolling'];
/** @var string patches 视图目录 */ /** @var string patches 视图目录 */
private const PATCH_VIEW_DIR = 'patches/application/admin/view/split/ticket/'; private const PATCH_VIEW_DIR = 'patches/application/admin/view/split/ticket/';
@@ -64,6 +64,7 @@ class Ticket extends Backend
'syncBackgroundStartedMsg' => __('Sync background started'), 'syncBackgroundStartedMsg' => __('Sync background started'),
'syncInProgressMsg' => __('Sync in progress'), 'syncInProgressMsg' => __('Sync in progress'),
'syncTicketStartedMsg' => __('Sync ticket started'), 'syncTicketStartedMsg' => __('Sync ticket started'),
'syncDoneMsg' => __('Sync done'),
]); ]);
$this->setupPatchFrontend(); $this->setupPatchFrontend();
@@ -284,7 +285,7 @@ class Ticket extends Backend
} }
/** /**
* 手动同步选中工单 * 手动同步选中工单(投递 CLI 后台执行,Web 请求立即返回)
*/ */
public function sync(): void public function sync(): void
{ {
@@ -307,31 +308,84 @@ class Ticket extends Backend
'appDebug' => SplitTicketSyncLogger::isEnabled(), 'appDebug' => SplitTicketSyncLogger::isEnabled(),
]); ]);
$service = new SplitTicketSyncService(); /** @var int[] $allowedIds */
$ok = 0; $allowedIds = [];
$fail = 0; $denied = 0;
$messages = [];
foreach ($list as $row) { foreach ($list as $row) {
if (is_array($adminIds) && !in_array((int) $row[$this->dataLimitField], $adminIds, true)) { if (is_array($adminIds) && !in_array((int) $row[$this->dataLimitField], $adminIds, true)) {
$fail++; $denied++;
$messages[] = '#' . $row['id'] . ': 无权限';
continue; continue;
} }
$result = $service->syncOne((int) $row['id'], true); $allowedIds[] = (int) $row['id'];
if ($result['success']) { }
$ok++;
} else { if ($allowedIds === []) {
$fail++; $this->error(__('You have no permission'));
$messages[] = '#' . $row['id'] . ': ' . ($result['message'] ?? '失败'); }
// 尽早释放 Session 锁,避免投递等待期间阻塞同账号其它后台请求(如编辑/新增弹窗)
if (function_exists('session_write_close')) {
session_write_close();
}
$dispatch = new SplitTicketSyncDispatchService();
$result = $dispatch->dispatchManual($allowedIds);
$queuedCount = count($result['queued']);
$skippedCount = count($result['skipped']);
$failedCount = count($result['failed']);
if ($queuedCount === 0) {
if ($skippedCount > 0) {
$this->error(__('Sync all skipped busy'));
}
$this->error(__('Sync dispatch failed'));
}
$summary = sprintf(__('Sync dispatch queued'), $queuedCount);
if ($skippedCount > 0) {
$summary .= sprintf(__('Sync dispatch skipped suffix'), $skippedCount);
}
if ($failedCount > 0) {
$summary .= sprintf(__('Sync dispatch failed suffix'), $failedCount);
}
if ($denied > 0) {
$summary .= sprintf(__('Sync dispatch denied suffix'), $denied);
}
$this->success($summary, null, [
'queued' => $result['queued'],
'syncing' => $result['queued'],
]);
}
/**
* 轮询手工同步进度(依据 runtime 文件锁)
*/
public function syncpolling(): void
{
$idsRaw = trim((string) $this->request->request('ids', ''));
if ($idsRaw === '') {
$this->success('', null, ['syncing' => []]);
}
/** @var int[] $ticketIds */
$ticketIds = [];
foreach (explode(',', $idsRaw) as $part) {
$part = trim($part);
if ($part !== '' && ctype_digit($part)) {
$ticketIds[] = (int) $part;
} }
} }
$summary = sprintf('成功 %d 条,失败 %d 条', $ok, $fail); if ($ticketIds === []) {
if ($fail > 0) { $this->success('', null, ['syncing' => []]);
$summary .= '' . implode('', array_slice($messages, 0, 5));
} }
$this->success($summary);
$dispatch = new SplitTicketSyncDispatchService();
$syncing = $dispatch->filterSyncingIds($ticketIds);
$this->success('', null, ['syncing' => $syncing]);
} }
/** /**
@@ -31,6 +31,12 @@ return [
'Sync in progress' => '同步中', 'Sync in progress' => '同步中',
'Sync ticket started' => '正在同步工单', 'Sync ticket started' => '正在同步工单',
'Sync done' => '同步完成', 'Sync done' => '同步完成',
'Sync dispatch queued' => '已提交 %d 条同步任务到后台执行',
'Sync dispatch skipped suffix' => '%d 条已在同步中已跳过',
'Sync dispatch failed suffix' => '%d 条启动失败',
'Sync dispatch denied suffix' => '%d 条无权限未提交',
'Sync all skipped busy' => '所选工单均在同步中,请稍后再试',
'Sync dispatch failed' => '未能启动同步任务,请检查服务器是否允许后台执行 CLIexec/shell_exec',
'Sync status success' => '同步成功', 'Sync status success' => '同步成功',
'Sync status error' => '同步异常', 'Sync status error' => '同步异常',
'Sync status pending' => '待同步', 'Sync status pending' => '待同步',
View File
View File
@@ -80,7 +80,7 @@ class HuojianSpider extends AbstractScrmSpider
protected function parseToUnifiedData($detailData, array $allListPagesData): UnifiedScrmData protected function parseToUnifiedData($detailData, array $allListPagesData): UnifiedScrmData
{ {
$unifiedData = $this->unifiedData; $unifiedData = $this->unifiedData;
$unifiedData->todayNewCount = (int) ($allListPagesData[0]['data']['counterWorker']['newTodayFriend'] ?? 0); $unifiedData->todayNewCount = (int) ($allListPagesData[0]['data']['counterWorker']['newTodayFriend'] ?? 0) + (int) ($allListPagesData[0]['data']['counterWorker']['newRemovedTodayFriend'] ?? 0);
$count = 0; $count = 0;
foreach ($allListPagesData as $pageRaw) { foreach ($allListPagesData as $pageRaw) {
$records = $pageRaw['data']['counterCsAccountVo'] ?? []; $records = $pageRaw['data']['counterCsAccountVo'] ?? [];
View File
@@ -4,6 +4,7 @@ declare(strict_types=1);
namespace app\common\service; namespace app\common\service;
use app\common\library\scrm\AbstractScrmSpider;
use app\common\library\scrm\ScrmSpiderInterface; use app\common\library\scrm\ScrmSpiderInterface;
use app\common\library\scrm\spider\A2cSpider; use app\common\library\scrm\spider\A2cSpider;
use app\common\library\scrm\spider\ChatknowSpider; use app\common\library\scrm\spider\ChatknowSpider;
@@ -59,6 +60,49 @@ class SplitScrmSpiderFactory
return array_keys(self::MAP); return array_keys(self::MAP);
} }
/**
* 同步是否依赖 Node Headless(纯 PHP cURL 等为 false
*/
public static function requiresNode(string $ticketType): bool
{
$class = self::resolveClass($ticketType);
if ($class === null) {
return true;
}
return is_subclass_of($class, AbstractScrmSpider::class);
}
/**
* @return list<string>
*/
public static function listDirectSyncTypes(): array
{
$types = [];
foreach (self::listSupportedTypes() as $ticketType) {
if (!self::requiresNode($ticketType)) {
$types[] = $ticketType;
}
}
return $types;
}
/**
* @return list<string>
*/
public static function listNodeSyncTypes(): array
{
$types = [];
foreach (self::listSupportedTypes() as $ticketType) {
if (self::requiresNode($ticketType)) {
$types[] = $ticketType;
}
}
return $types;
}
/** /**
* @return ScrmSpiderInterface|null * @return ScrmSpiderInterface|null
*/ */
@@ -57,11 +57,23 @@ class SplitSyncConfigService
{ {
$value = self::getConfigValue('split_sync_max_per_cron'); $value = self::getConfigValue('split_sync_max_per_cron');
if ($value === '') { if ($value === '') {
return 2; return 5;
} }
return max(1, min(20, (int) $value)); return max(1, min(20, (int) $value));
} }
/**
* Node 通道候选池倍数(候选数 = maxPerRun × 本值)
*/
public static function getNodeCandidatePoolMultiplier(): int
{
$value = self::getConfigValue('split_sync_node_candidate_multiplier');
if ($value === '') {
return 10;
}
return max(3, min(50, (int) $value));
}
/** /**
* 全局 cron 锁过期秒数,防止异常退出后永久占锁 * 全局 cron 锁过期秒数,防止异常退出后永久占锁
*/ */
@@ -86,6 +98,53 @@ class SplitSyncConfigService
return max(0, (int) $value); return max(0, (int) $value);
} }
/**
* 后台投递 / Cron 使用的 PHP CLI 可执行文件路径
*
* Web(FPM) 环境下 PHP_BINARY 指向 php-fpm,不能用于 think 命令。
*/
public static function getCliPhpBinary(): string
{
$configured = trim(self::getConfigValue('split_sync_cli_php'));
if ($configured !== '' && self::isUsableCliPhp($configured)) {
return $configured;
}
if (defined('PHP_BINDIR') && PHP_BINDIR !== '') {
$candidate = rtrim(PHP_BINDIR, '/\\') . DIRECTORY_SEPARATOR . 'php';
if (self::isUsableCliPhp($candidate)) {
return $candidate;
}
}
if (PHP_SAPI === 'cli' && defined('PHP_BINARY') && PHP_BINARY !== '' && self::isUsableCliPhp(PHP_BINARY)) {
return PHP_BINARY;
}
foreach (['/usr/bin/php', '/usr/local/bin/php'] as $candidate) {
if (self::isUsableCliPhp($candidate)) {
return $candidate;
}
}
return 'php';
}
/**
* 判断路径是否为可用的 PHP CLI(排除 php-fpm
*/
private static function isUsableCliPhp(string $path): bool
{
if (stripos($path, 'fpm') !== false) {
return false;
}
if ($path === 'php') {
return true;
}
return is_file($path) && is_executable($path);
}
private static function getConfigValue(string $name): string private static function getConfigValue(string $name): string
{ {
$site = Config::get('site.' . $name); $site = Config::get('site.' . $name);
@@ -0,0 +1,224 @@
<?php
declare(strict_types=1);
namespace app\common\service;
/**
* 工单手工同步 CLI 后台投递
*
* Web 请求仅负责鉴权与投递,实际 Spider 在独立 CLI 进程中执行,
* 避免长时间占用 PHP-FPM 与 Session 锁导致后台其它页面无法打开。
*/
class SplitTicketSyncDispatchService
{
/**
* 投递手工同步任务到后台 CLI
*
* @param int[] $ticketIds 已通过权限校验的工单 ID
* @return array{queued:int[],skipped:int[],failed:int[]}
*/
public function dispatchManual(array $ticketIds): array
{
$queued = [];
$skipped = [];
$failed = [];
$php = $this->resolvePhpBinary();
$think = $this->resolveThinkScript();
$root = $this->resolveRootPath();
$lockService = new SplitTicketSyncLockService();
foreach ($ticketIds as $rawId) {
$ticketId = (int) $rawId;
if ($ticketId <= 0) {
continue;
}
if ($lockService->isLocked($ticketId)) {
$skipped[] = $ticketId;
continue;
}
if ($this->spawnCli($php, $think, $root, $ticketId)) {
$queued[] = $ticketId;
} else {
$failed[] = $ticketId;
}
}
SplitTicketSyncLogger::log('web', 'manual sync dispatched', [
'queued' => $queued,
'skipped' => $skipped,
'failed' => $failed,
'phpCli' => $php,
]);
return [
'queued' => $queued,
'skipped' => $skipped,
'failed' => $failed,
];
}
/**
* 查询仍在同步中的工单 ID(依据 runtime 文件锁)
*
* @param int[] $ticketIds
* @return int[]
*/
public function filterSyncingIds(array $ticketIds): array
{
$lockService = new SplitTicketSyncLockService();
$syncing = [];
foreach ($ticketIds as $rawId) {
$ticketId = (int) $rawId;
if ($ticketId > 0 && $lockService->isLocked($ticketId)) {
$syncing[] = $ticketId;
}
}
return $syncing;
}
/**
* 后台启动 split:sync-tickets CLI
*/
private function spawnCli(string $php, string $think, string $root, int $ticketId): bool
{
$logDir = $this->resolveLogDir();
$logFile = $logDir . 'cli_' . $ticketId . '_' . date('YmdHis') . '.log';
$inner = sprintf(
'%s %s split:sync-tickets --ticket=%d >> %s 2>&1',
escapeshellarg($php),
escapeshellarg($think),
$ticketId,
escapeshellarg($logFile)
);
if ($this->isWindows()) {
$command = sprintf('start /B cmd /C %s', $inner);
} else {
$command = sprintf('cd %s && nohup %s &', escapeshellarg($root), $inner);
}
if ($this->canUseExec()) {
@exec($command, $output, $exitCode);
SplitTicketSyncLogger::log('web', 'cli spawn exec', [
'ticketId' => $ticketId,
'command' => $command,
'exitCode' => $exitCode,
]);
return true;
}
if ($this->canUseShellExec()) {
@shell_exec($command);
SplitTicketSyncLogger::log('web', 'cli spawn shell_exec', [
'ticketId' => $ticketId,
'command' => $command,
]);
return true;
}
SplitTicketSyncLogger::log('web', 'cli spawn failed: exec disabled', [
'ticketId' => $ticketId,
]);
return false;
}
private function resolvePhpBinary(): string
{
if (method_exists(SplitSyncConfigService::class, 'getCliPhpBinary')) {
return SplitSyncConfigService::getCliPhpBinary();
}
return $this->resolvePhpBinaryFallback();
}
/**
* 当 SplitSyncConfigService 未部署 getCliPhpBinary 时的兜底解析
*/
private function resolvePhpBinaryFallback(): string
{
if (defined('PHP_BINDIR') && PHP_BINDIR !== '') {
$candidate = rtrim(PHP_BINDIR, '/\\') . DIRECTORY_SEPARATOR . 'php';
if ($this->isUsableCliPhp($candidate)) {
return $candidate;
}
}
foreach (['/usr/bin/php', '/usr/local/bin/php'] as $candidate) {
if ($this->isUsableCliPhp($candidate)) {
return $candidate;
}
}
return 'php';
}
private function isUsableCliPhp(string $path): bool
{
if (stripos($path, 'fpm') !== false) {
return false;
}
if ($path === 'php') {
return true;
}
return is_file($path) && is_executable($path);
}
private function resolveThinkScript(): string
{
$root = $this->resolveRootPath();
return rtrim($root, '/\\') . DIRECTORY_SEPARATOR . 'think';
}
private function resolveRootPath(): string
{
if (defined('ROOT_PATH') && ROOT_PATH !== '') {
return rtrim(ROOT_PATH, '/\\') . DIRECTORY_SEPARATOR;
}
return rtrim(dirname(__DIR__, 3), '/\\') . DIRECTORY_SEPARATOR;
}
private function resolveLogDir(): string
{
$runtime = defined('RUNTIME_PATH') ? RUNTIME_PATH : ($this->resolveRootPath() . 'runtime' . DIRECTORY_SEPARATOR);
$dir = rtrim($runtime, '/\\') . DIRECTORY_SEPARATOR . 'split_ticket_sync' . DIRECTORY_SEPARATOR;
if (!is_dir($dir)) {
@mkdir($dir, 0755, true);
}
return $dir;
}
private function isWindows(): bool
{
return strtoupper(substr(PHP_OS, 0, 3)) === 'WIN';
}
/**
* @return string[]
*/
private function disabledFunctions(): array
{
$raw = (string) ini_get('disable_functions');
if ($raw === '') {
return [];
}
return array_filter(array_map('trim', explode(',', $raw)));
}
private function canUseExec(): bool
{
return function_exists('exec') && !in_array('exec', $this->disabledFunctions(), true);
}
private function canUseShellExec(): bool
{
return function_exists('shell_exec') && !in_array('shell_exec', $this->disabledFunctions(), true);
}
}
@@ -43,6 +43,20 @@ class SplitTicketSyncLockService
} }
} }
/**
* 工单是否处于同步中(锁文件存在且未过期)
*/
public function isLocked(int $ticketId): bool
{
$path = $this->lockPath($ticketId);
if ($this->isStaleLock($path)) {
@unlink($path);
return false;
}
return is_file($path);
}
private function lockPath(int $ticketId): string private function lockPath(int $ticketId): string
{ {
$runtime = defined('RUNTIME_PATH') ? RUNTIME_PATH : (dirname(__DIR__, 3) . '/runtime/'); $runtime = defined('RUNTIME_PATH') ? RUNTIME_PATH : (dirname(__DIR__, 3) . '/runtime/');
@@ -80,131 +80,49 @@ class SplitTicketSyncService
} }
/** /**
* 扫描到期工单并同步(限流 + Node 队列感知 * 扫描到期工单并同步(直连 PHP 与 Node 双通道
*/ */
public function syncDueTickets(): int public function syncDueTickets(): int
{ {
$maxPerRun = SplitSyncConfigService::getMaxTicketsPerCronRun();
$autoSyncTypes = $this->resolveAutoSyncTicketTypes(); $autoSyncTypes = $this->resolveAutoSyncTicketTypes();
$failThreshold = SplitSyncConfigService::getFailPauseThreshold();
if ($autoSyncTypes === []) { if ($autoSyncTypes === []) {
SplitTicketSyncLogger::logCron('scan skip', ['reason' => 'no auto-sync ticket types']); SplitTicketSyncLogger::logCron('scan skip', ['reason' => 'no auto-sync ticket types']);
return 0; return 0;
} }
if (!SplitNodeHealthService::canAcceptWork()) { $failThreshold = SplitSyncConfigService::getFailPauseThreshold();
SplitTicketSyncLogger::logCron('scan skip', [ $directTypes = array_values(array_intersect(
'reason' => 'node queue busy', $autoSyncTypes,
'queue' => SplitNodeHealthService::getQueueSnapshot(), SplitScrmSpiderFactory::listDirectSyncTypes()
]); ));
return 0; $nodeTypes = array_values(array_intersect(
} $autoSyncTypes,
SplitScrmSpiderFactory::listNodeSyncTypes()
));
$query = Ticket::where('status', 'normal')->whereIn('ticket_type', $autoSyncTypes);
if ($failThreshold > 0) {
$query->where('sync_fail_count', '<', $failThreshold);
}
$list = $query->order('sync_time', 'asc')->order('id', 'asc')
->limit($maxPerRun * 5)
->select();
$list = $this->sortTicketsBySessionKey($list);
$candidateCount = count($list);
SplitTicketSyncLogger::logCron('scan start', [ SplitTicketSyncLogger::logCron('scan start', [
'candidateCount' => $candidateCount,
'maxPerRun' => $maxPerRun,
'autoSyncTypes' => $autoSyncTypes, 'autoSyncTypes' => $autoSyncTypes,
'directTypes' => $directTypes,
'nodeTypes' => $nodeTypes,
'failThreshold' => $failThreshold, 'failThreshold' => $failThreshold,
'nodeQueue' => SplitNodeHealthService::getQueueSnapshot(), 'nodeQueue' => SplitNodeHealthService::getQueueSnapshot(),
'groupedBy' => 'sessionKey',
]);
SplitTicketSyncLogger::log('cron', 'scan start', [
'candidateCount' => $candidateCount,
'maxPerRun' => $maxPerRun,
'autoSyncTypes' => $autoSyncTypes,
'nodeQueue' => SplitNodeHealthService::getQueueSnapshot(),
]); ]);
/** @var list<array<string, mixed>> $processed */ $directResult = $this->runDirectCronChannel($directTypes, $failThreshold);
$processed = []; $nodeResult = $this->runNodeCronChannel($nodeTypes, $failThreshold);
/** @var list<array<string, mixed>> $skipped */
$skipped = [];
$count = 0;
$stoppedByNodeBusy = false;
foreach ($list as $index => $ticket) { $processedCount = $directResult['processedCount'] + $nodeResult['processedCount'];
$ticketId = (int) $ticket['id']; $processed = array_merge($directResult['processed'], $nodeResult['processed']);
$ticketUrl = (string) $ticket['ticket_url']; $skipped = array_merge($directResult['skipped'], $nodeResult['skipped']);
$ticketType = (string) $ticket['ticket_type']; $candidateIds = array_merge($directResult['candidateIds'], $nodeResult['candidateIds']);
$sessionKey = AntiBotConfigBuilder::resolveSessionKey($ticketType, $ticketUrl);
if ($count >= $maxPerRun) {
$reason = '本轮处理上限已满';
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $reason);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $reason);
continue;
}
$skip = $this->shouldSkip($ticket);
if ($skip !== null) {
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $skip);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $skip);
continue;
}
if (!SplitNodeHealthService::canAcceptWork()) {
$reason = 'Node 队列繁忙,停止本轮后续同步';
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $reason);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $reason);
$stoppedByNodeBusy = true;
for ($i = $index + 1; $i < $candidateCount; $i++) {
$remain = $list[$i];
$remainReason = 'Node 队列繁忙,本轮未执行';
$skipped[] = $this->buildCronTicketEntry(
(int) $remain['id'],
(string) $remain['ticket_type'],
(string) $remain['ticket_url'],
$remainReason
);
}
SplitTicketSyncLogger::log('cron', 'node busy, stop batch', [
'processed' => $count,
'queue' => SplitNodeHealthService::getQueueSnapshot(),
]);
break;
}
$result = $this->syncOne($ticketId, false, true, 'auto');
if (!empty($result['skipped'])) {
$skipReason = (string) ($result['message'] ?? '已跳过');
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $skipReason);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $skipReason);
continue;
}
$processed[] = [
'ticketId' => $ticketId,
'ticketType' => $ticketType,
'ticketUrl' => $ticketUrl,
'sessionKey' => $sessionKey,
'success' => !empty($result['success']),
'message' => (string) ($result['message'] ?? ''),
];
$count++;
}
$candidateIds = array_map(static function ($row) {
return (int) (is_array($row) ? ($row['id'] ?? 0) : ($row['id'] ?? 0));
}, $list);
$openNotSynced = $this->buildOpenNotSyncedAudit( $openNotSynced = $this->buildOpenNotSyncedAudit(
$autoSyncTypes, $autoSyncTypes,
$failThreshold, $failThreshold,
$maxPerRun, SplitSyncConfigService::getMaxTicketsPerCronRun(),
$candidateIds, $candidateIds,
$processed, $processed,
$skipped, $skipped,
$stoppedByNodeBusy $nodeResult['stoppedByNodeBusy']
); );
$successCount = count(array_filter($processed, static function (array $row): bool { $successCount = count(array_filter($processed, static function (array $row): bool {
@@ -213,21 +131,362 @@ class SplitTicketSyncService
$failedCount = count($processed) - $successCount; $failedCount = count($processed) - $successCount;
SplitTicketSyncLogger::logCron('scan end', [ SplitTicketSyncLogger::logCron('scan end', [
'processedCount' => $count, 'processedCount' => $processedCount,
'successCount' => $successCount, 'successCount' => $successCount,
'failedCount' => $failedCount, 'failedCount' => $failedCount,
'skippedCount' => count($skipped), 'skippedCount' => count($skipped),
'directChannel' => $directResult['summary'],
'nodeChannel' => $nodeResult['summary'],
'processed' => $processed, 'processed' => $processed,
'skipped' => $skipped, 'skipped' => $skipped,
'openNotSynced' => $openNotSynced, 'openNotSynced' => $openNotSynced,
]); ]);
SplitTicketSyncLogger::log('cron', 'scan end', [ SplitTicketSyncLogger::log('cron', 'scan end', [
'processedCount' => $count, 'processedCount' => $processedCount,
'successCount' => $successCount, 'successCount' => $successCount,
'failedCount' => $failedCount, 'failedCount' => $failedCount,
]); ]);
return $count; return $processedCount;
}
/**
* 直连 PHP 通道:按周期同步,不占 Node 名额
*
* @param list<string> $directTypes
* @return array{
* processedCount:int,
* processed:list<array<string,mixed>>,
* skipped:list<array<string,mixed>>,
* candidateIds:list<int>,
* summary:array<string,mixed>
* }
*/
private function runDirectCronChannel(array $directTypes, int $failThreshold): array
{
if ($directTypes === []) {
return $this->emptyCronChannelResult('direct');
}
$list = $this->loadOpenTicketsForTypes($directTypes, $failThreshold, 0);
$candidateIds = $this->extractTicketIds($list);
SplitTicketSyncLogger::logCron('direct channel start', [
'types' => $directTypes,
'candidateCount' => count($list),
]);
/** @var list<array<string, mixed>> $processed */
$processed = [];
/** @var list<array<string, mixed>> $skipped */
$skipped = [];
$count = 0;
foreach ($list as $ticket) {
$ticketId = (int) $ticket['id'];
$ticketType = (string) $ticket['ticket_type'];
$ticketUrl = (string) $ticket['ticket_url'];
$skip = $this->shouldSkip($ticket);
if ($skip !== null) {
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $skip);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $skip, 'direct');
continue;
}
$result = $this->syncOne($ticketId, false, true, 'auto');
if (!empty($result['skipped'])) {
$skipReason = (string) ($result['message'] ?? '已跳过');
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $skipReason);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $skipReason, 'direct');
continue;
}
$processed[] = [
'ticketId' => $ticketId,
'ticketType' => $ticketType,
'ticketUrl' => $ticketUrl,
'channel' => 'direct',
'success' => !empty($result['success']),
'message' => (string) ($result['message'] ?? ''),
];
$count++;
}
return [
'processedCount' => $count,
'processed' => $processed,
'skipped' => $skipped,
'candidateIds' => $candidateIds,
'summary' => [
'channel' => 'direct',
'processedCount' => $count,
'skippedCount' => count($skipped),
],
'stoppedByNodeBusy' => false,
];
}
/**
* Node 通道:限流 + 类型公平轮转
*
* @param list<string> $nodeTypes
* @return array{
* processedCount:int,
* processed:list<array<string,mixed>>,
* skipped:list<array<string,mixed>>,
* candidateIds:list<int>,
* summary:array<string,mixed>,
* stoppedByNodeBusy:bool
* }
*/
private function runNodeCronChannel(array $nodeTypes, int $failThreshold): array
{
if ($nodeTypes === []) {
return $this->emptyCronChannelResult('node');
}
$maxPerRun = SplitSyncConfigService::getMaxTicketsPerCronRun();
$poolMultiplier = SplitSyncConfigService::getNodeCandidatePoolMultiplier();
$candidateLimit = $maxPerRun * $poolMultiplier;
$pool = $this->loadOpenTicketsForTypes($nodeTypes, $failThreshold, $candidateLimit);
$candidateIds = $this->extractTicketIds($pool);
$duePool = [];
foreach ($pool as $ticket) {
if ($this->shouldSkip($ticket) === null) {
$duePool[] = $ticket;
}
}
$picked = $this->pickFairNodeTickets($duePool, $maxPerRun);
$picked = $this->sortTicketsBySessionKey($picked);
$pickedIdMap = array_fill_keys($this->extractTicketIds($picked), true);
SplitTicketSyncLogger::logCron('node channel start', [
'types' => $nodeTypes,
'maxPerRun' => $maxPerRun,
'poolCount' => count($pool),
'dueCount' => count($duePool),
'pickedCount' => count($picked),
'nodeQueue' => SplitNodeHealthService::getQueueSnapshot(),
]);
if (!SplitNodeHealthService::canAcceptWork()) {
SplitTicketSyncLogger::logCron('node channel skip', [
'reason' => 'node queue busy',
'queue' => SplitNodeHealthService::getQueueSnapshot(),
]);
$skipped = [];
foreach ($picked as $ticket) {
$skipped[] = $this->buildCronTicketEntry(
(int) $ticket['id'],
(string) $ticket['ticket_type'],
(string) $ticket['ticket_url'],
'Node 队列繁忙,本轮未执行'
);
}
foreach ($pool as $ticket) {
$ticketId = (int) $ticket['id'];
if (isset($pickedIdMap[$ticketId])) {
continue;
}
$skip = $this->shouldSkip($ticket);
if ($skip !== null) {
continue;
}
$skipped[] = $this->buildCronTicketEntry(
$ticketId,
(string) $ticket['ticket_type'],
(string) $ticket['ticket_url'],
sprintf('未进入本轮 Node 候选(前%d条到期工单公平轮转)', $candidateLimit)
);
}
return [
'processedCount' => 0,
'processed' => [],
'skipped' => $skipped,
'candidateIds' => $candidateIds,
'summary' => [
'channel' => 'node',
'processedCount' => 0,
'skippedCount' => count($skipped),
'reason' => 'node queue busy',
],
'stoppedByNodeBusy' => true,
];
}
/** @var list<array<string, mixed>> $processed */
$processed = [];
/** @var list<array<string, mixed>> $skipped */
$skipped = [];
$count = 0;
$stoppedByNodeBusy = false;
foreach ($pool as $ticket) {
$ticketId = (int) $ticket['id'];
$ticketType = (string) $ticket['ticket_type'];
$ticketUrl = (string) $ticket['ticket_url'];
if (!isset($pickedIdMap[$ticketId])) {
$skip = $this->shouldSkip($ticket);
if ($skip !== null) {
continue;
}
$skipped[] = $this->buildCronTicketEntry(
$ticketId,
$ticketType,
$ticketUrl,
sprintf('未进入本轮 Node 候选(前%d条到期工单公平轮转)', $candidateLimit)
);
continue;
}
if (!SplitNodeHealthService::canAcceptWork()) {
$stoppedByNodeBusy = true;
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, 'Node 队列繁忙,停止本轮后续同步');
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, 'Node 队列繁忙,停止本轮后续同步', 'node');
continue;
}
$sessionKey = AntiBotConfigBuilder::resolveSessionKey($ticketType, $ticketUrl);
$result = $this->syncOne($ticketId, false, true, 'auto');
if (!empty($result['skipped'])) {
$skipReason = (string) ($result['message'] ?? '已跳过');
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $skipReason);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $skipReason, 'node');
continue;
}
$processed[] = [
'ticketId' => $ticketId,
'ticketType' => $ticketType,
'ticketUrl' => $ticketUrl,
'sessionKey' => $sessionKey,
'channel' => 'node',
'success' => !empty($result['success']),
'message' => (string) ($result['message'] ?? ''),
];
$count++;
}
return [
'processedCount' => $count,
'processed' => $processed,
'skipped' => $skipped,
'candidateIds' => $candidateIds,
'summary' => [
'channel' => 'node',
'processedCount' => $count,
'skippedCount' => count($skipped),
'maxPerRun' => $maxPerRun,
],
'stoppedByNodeBusy' => $stoppedByNodeBusy,
];
}
/**
* @param list<string> $types
* @return list<Ticket>
*/
private function loadOpenTicketsForTypes(array $types, int $failThreshold, int $limit): array
{
if ($types === []) {
return [];
}
$query = Ticket::where('status', 'normal')->whereIn('ticket_type', $types);
if ($failThreshold > 0) {
$query->where('sync_fail_count', '<', $failThreshold);
}
$query->order('sync_time', 'asc')->order('id', 'asc');
if ($limit > 0) {
$query->limit($limit);
}
$list = $query->select();
return $list instanceof \think\Collection ? $list->all() : (array) $list;
}
/**
* Node 通道:按工单类型公平轮转选取(每轮每类型优先取 1 条)
*
* @param list<Ticket> $dueTickets
* @return list<Ticket>
*/
private function pickFairNodeTickets(array $dueTickets, int $maxPerRun): array
{
if ($dueTickets === [] || $maxPerRun <= 0) {
return [];
}
/** @var array<string, list<Ticket>> $byType */
$byType = [];
foreach ($dueTickets as $ticket) {
$type = (string) $ticket['ticket_type'];
$byType[$type][] = $ticket;
}
$typeKeys = array_keys($byType);
sort($typeKeys, SORT_STRING);
/** @var list<Ticket> $picked */
$picked = [];
while (count($picked) < $maxPerRun) {
$added = false;
foreach ($typeKeys as $type) {
if (count($picked) >= $maxPerRun) {
break;
}
if ($byType[$type] === []) {
continue;
}
$picked[] = array_shift($byType[$type]);
$added = true;
}
if (!$added) {
break;
}
}
return $picked;
}
/**
* @param list<Ticket|array<string,mixed>> $list
* @return list<int>
*/
private function extractTicketIds(array $list): array
{
$ids = [];
foreach ($list as $row) {
$ids[] = (int) (is_array($row) ? ($row['id'] ?? 0) : ($row['id'] ?? 0));
}
return $ids;
}
/**
* @return array{
* processedCount:int,
* processed:list<array<string,mixed>>,
* skipped:list<array<string,mixed>>,
* candidateIds:list<int>,
* summary:array<string,mixed>,
* stoppedByNodeBusy:bool
* }
*/
private function emptyCronChannelResult(string $channel): array
{
return [
'processedCount' => 0,
'processed' => [],
'skipped' => [],
'candidateIds' => [],
'summary' => ['channel' => $channel, 'processedCount' => 0, 'skippedCount' => 0],
'stoppedByNodeBusy' => false,
];
} }
/** /**
@@ -466,13 +725,14 @@ class SplitTicketSyncService
]; ];
} }
private function logCronCandidateSkipped(int $ticketId, string $ticketType, string $ticketUrl, string $reason): void private function logCronCandidateSkipped(int $ticketId, string $ticketType, string $ticketUrl, string $reason, string $channel = 'legacy'): void
{ {
$ctx = [ $ctx = [
'ticketId' => $ticketId, 'ticketId' => $ticketId,
'ticketType' => $ticketType, 'ticketType' => $ticketType,
'ticketUrl' => $ticketUrl, 'ticketUrl' => $ticketUrl,
'reason' => $reason, 'reason' => $reason,
'channel' => $channel,
]; ];
SplitTicketSyncLogger::logCron('candidate skipped', $ctx); SplitTicketSyncLogger::logCron('candidate skipped', $ctx);
SplitTicketSyncLogger::log('cron', 'candidate skipped', $ctx); SplitTicketSyncLogger::log('cron', 'candidate skipped', $ctx);
@@ -515,14 +775,15 @@ class SplitTicketSyncService
->select(); ->select();
$result = []; $result = [];
foreach ($openList as $ticket) { foreach ($openList as $ticketRow) {
$ticketId = (int) $ticket['id']; $ticket = $ticketRow instanceof Ticket ? $ticketRow->toArray() : (array) $ticketRow;
if (isset($handledIds[$ticketId])) { $ticketId = (int) ($ticket['id'] ?? 0);
if ($ticketId <= 0 || isset($handledIds[$ticketId])) {
continue; continue;
} }
$ticketType = (string) $ticket['ticket_type']; $ticketType = (string) ($ticket['ticket_type'] ?? '');
$ticketUrl = (string) $ticket['ticket_url']; $ticketUrl = (string) ($ticket['ticket_url'] ?? '');
$reason = $this->resolveOpenNotSyncedReason( $reason = $this->resolveOpenNotSyncedReason(
$ticket, $ticket,
$autoSyncTypes, $autoSyncTypes,
@@ -568,7 +829,13 @@ class SplitTicketSyncService
return '该类型未配置自动同步周期'; return '该类型未配置自动同步周期';
} }
if (!isset($candidateIdMap[(int) ($ticket['id'] ?? 0)])) { if (!isset($candidateIdMap[(int) ($ticket['id'] ?? 0)])) {
return sprintf('未进入本轮候选队列(仅扫描最久未同步的前%d条)', $maxPerRun * 5); if (SplitScrmSpiderFactory::requiresNode($ticketType)) {
$poolSize = SplitSyncConfigService::getMaxTicketsPerCronRun()
* SplitSyncConfigService::getNodeCandidatePoolMultiplier();
return sprintf('未进入本轮 Node 候选(前%d条到期工单公平轮转)', $poolSize);
}
return '直连通道:未到同步周期或本轮已处理';
} }
if ($stoppedByNodeBusy) { if ($stoppedByNodeBusy) {
return 'Node 队列繁忙,本轮未执行'; return 'Node 队列繁忙,本轮未执行';
+1
View File
@@ -58,5 +58,6 @@ return array (
'split_sync_interval_whatshub' => '5', 'split_sync_interval_whatshub' => '5',
'split_sync_interval_sihai' => '0', 'split_sync_interval_sihai' => '0',
'split_sync_fail_pause_threshold' => '6', 'split_sync_fail_pause_threshold' => '6',
'split_sync_cli_php' => '',
'split_sync_interval_chatknow' => '5', 'split_sync_interval_chatknow' => '5',
); );
@@ -0,0 +1,19 @@
-- 双通道定时同步:Node 通道上限与候选池配置(已有环境增量执行)
SET NAMES utf8mb4;
INSERT INTO `fa_config` (`name`, `group`, `title`, `tip`, `type`, `visible`, `value`, `content`, `rule`, `extend`, `setting`)
SELECT 'split_sync_max_per_cron', 'split', 'Node通道单次同步上限', '每分钟 cron 在 Node 通道最多处理几条到期工单(仅影响需调用 Node 的类型);纯 PHP 直连类型不受此限制', 'number', '', '5', '', '', '', NULL
FROM DUAL WHERE NOT EXISTS (SELECT 1 FROM `fa_config` WHERE `name` = 'split_sync_max_per_cron' LIMIT 1);
INSERT INTO `fa_config` (`name`, `group`, `title`, `tip`, `type`, `visible`, `value`, `content`, `rule`, `extend`, `setting`)
SELECT 'split_sync_node_candidate_multiplier', 'split', 'Node通道候选池倍数', 'Node 候选工单数 = 单次上限 × 本倍数,用于公平轮转', 'number', '', '10', '', '', '', NULL
FROM DUAL WHERE NOT EXISTS (SELECT 1 FROM `fa_config` WHERE `name` = 'split_sync_node_candidate_multiplier' LIMIT 1);
UPDATE `fa_config`
SET `title` = 'Node通道单次同步上限',
`tip` = '每分钟 cron 在 Node 通道最多处理几条到期工单(仅影响需调用 Node 的类型);纯 PHP 直连类型不受此限制'
WHERE `name` = 'split_sync_max_per_cron';
UPDATE `fa_config`
SET `value` = '5'
WHERE `name` = 'split_sync_max_per_cron' AND (`value` = '' OR `value` = '2');
@@ -58,9 +58,13 @@ SELECT 'split_sync_interval_sihai', 'split', '四海云控同步周期(分钟)',
FROM DUAL WHERE NOT EXISTS (SELECT 1 FROM `fa_config` WHERE `name` = 'split_sync_interval_sihai' LIMIT 1); FROM DUAL WHERE NOT EXISTS (SELECT 1 FROM `fa_config` WHERE `name` = 'split_sync_interval_sihai' LIMIT 1);
INSERT INTO `fa_config` (`name`, `group`, `title`, `tip`, `type`, `visible`, `value`, `content`, `rule`, `extend`, `setting`) INSERT INTO `fa_config` (`name`, `group`, `title`, `tip`, `type`, `visible`, `value`, `content`, `rule`, `extend`, `setting`)
SELECT 'split_sync_max_per_cron', 'split', '单次定时同步上限', '每分钟 cron 最多处理几条到期工单,防止打满 Node Browser 槽位', 'number', '', '2', '', '', '', NULL SELECT 'split_sync_max_per_cron', 'split', 'Node通道单次同步上限', '每分钟 cron 在 Node 通道最多处理几条到期工单(仅影响需调用 Node 的类型);纯 PHP 直连类型不受此限制', 'number', '', '5', '', '', '', NULL
FROM DUAL WHERE NOT EXISTS (SELECT 1 FROM `fa_config` WHERE `name` = 'split_sync_max_per_cron' LIMIT 1); FROM DUAL WHERE NOT EXISTS (SELECT 1 FROM `fa_config` WHERE `name` = 'split_sync_max_per_cron' LIMIT 1);
INSERT INTO `fa_config` (`name`, `group`, `title`, `tip`, `type`, `visible`, `value`, `content`, `rule`, `extend`, `setting`)
SELECT 'split_sync_node_candidate_multiplier', 'split', 'Node通道候选池倍数', 'Node 候选工单数 = 单次上限 × 本倍数,用于公平轮转', 'number', '', '10', '', '', '', NULL
FROM DUAL WHERE NOT EXISTS (SELECT 1 FROM `fa_config` WHERE `name` = 'split_sync_node_candidate_multiplier' LIMIT 1);
INSERT INTO `fa_config` (`name`, `group`, `title`, `tip`, `type`, `visible`, `value`, `content`, `rule`, `extend`, `setting`) INSERT INTO `fa_config` (`name`, `group`, `title`, `tip`, `type`, `visible`, `value`, `content`, `rule`, `extend`, `setting`)
SELECT 'split_sync_cron_lock_ttl', 'split', '定时同步全局锁TTL(秒)', '异常退出后自动释放全局锁,避免后续 cron 永久跳过', 'number', '', '1800', '', '', '', NULL SELECT 'split_sync_cron_lock_ttl', 'split', '定时同步全局锁TTL(秒)', '异常退出后自动释放全局锁,避免后续 cron 永久跳过', 'number', '', '1800', '', '', '', NULL
FROM DUAL WHERE NOT EXISTS (SELECT 1 FROM `fa_config` WHERE `name` = 'split_sync_cron_lock_ttl' LIMIT 1); FROM DUAL WHERE NOT EXISTS (SELECT 1 FROM `fa_config` WHERE `name` = 'split_sync_cron_lock_ttl' LIMIT 1);
@@ -8,8 +8,8 @@ use app\admin\model\split\Link as LinkModel;
use app\admin\model\split\Ticket as TicketModel; use app\admin\model\split\Ticket as TicketModel;
use app\common\controller\Backend; use app\common\controller\Backend;
use app\common\service\SplitTicketRuleService; use app\common\service\SplitTicketRuleService;
use app\common\service\SplitTicketSyncDispatchService;
use app\common\service\SplitTicketSyncLogger; use app\common\service\SplitTicketSyncLogger;
use app\common\service\SplitTicketSyncService;
use think\Db; use think\Db;
use think\Lang; use think\Lang;
use think\Loader; use think\Loader;
@@ -39,7 +39,7 @@ class Ticket extends Backend
protected $modelSceneValidate = true; protected $modelSceneValidate = true;
/** @var string[] 无需鉴权的方法 */ /** @var string[] 无需鉴权的方法 */
protected $noNeedRight = ['script']; protected $noNeedRight = ['script', 'syncpolling'];
/** @var string patches 视图目录 */ /** @var string patches 视图目录 */
private const PATCH_VIEW_DIR = 'patches/application/admin/view/split/ticket/'; private const PATCH_VIEW_DIR = 'patches/application/admin/view/split/ticket/';
@@ -64,6 +64,7 @@ class Ticket extends Backend
'syncBackgroundStartedMsg' => __('Sync background started'), 'syncBackgroundStartedMsg' => __('Sync background started'),
'syncInProgressMsg' => __('Sync in progress'), 'syncInProgressMsg' => __('Sync in progress'),
'syncTicketStartedMsg' => __('Sync ticket started'), 'syncTicketStartedMsg' => __('Sync ticket started'),
'syncDoneMsg' => __('Sync done'),
]); ]);
$this->setupPatchFrontend(); $this->setupPatchFrontend();
@@ -284,7 +285,7 @@ class Ticket extends Backend
} }
/** /**
* 手动同步选中工单 * 手动同步选中工单(投递 CLI 后台执行,Web 请求立即返回)
*/ */
public function sync(): void public function sync(): void
{ {
@@ -307,31 +308,84 @@ class Ticket extends Backend
'appDebug' => SplitTicketSyncLogger::isEnabled(), 'appDebug' => SplitTicketSyncLogger::isEnabled(),
]); ]);
$service = new SplitTicketSyncService(); /** @var int[] $allowedIds */
$ok = 0; $allowedIds = [];
$fail = 0; $denied = 0;
$messages = [];
foreach ($list as $row) { foreach ($list as $row) {
if (is_array($adminIds) && !in_array((int) $row[$this->dataLimitField], $adminIds, true)) { if (is_array($adminIds) && !in_array((int) $row[$this->dataLimitField], $adminIds, true)) {
$fail++; $denied++;
$messages[] = '#' . $row['id'] . ': 无权限';
continue; continue;
} }
$result = $service->syncOne((int) $row['id'], true); $allowedIds[] = (int) $row['id'];
if ($result['success']) { }
$ok++;
} else { if ($allowedIds === []) {
$fail++; $this->error(__('You have no permission'));
$messages[] = '#' . $row['id'] . ': ' . ($result['message'] ?? '失败'); }
// 尽早释放 Session 锁,避免投递等待期间阻塞同账号其它后台请求(如编辑/新增弹窗)
if (function_exists('session_write_close')) {
session_write_close();
}
$dispatch = new SplitTicketSyncDispatchService();
$result = $dispatch->dispatchManual($allowedIds);
$queuedCount = count($result['queued']);
$skippedCount = count($result['skipped']);
$failedCount = count($result['failed']);
if ($queuedCount === 0) {
if ($skippedCount > 0) {
$this->error(__('Sync all skipped busy'));
}
$this->error(__('Sync dispatch failed'));
}
$summary = sprintf(__('Sync dispatch queued'), $queuedCount);
if ($skippedCount > 0) {
$summary .= sprintf(__('Sync dispatch skipped suffix'), $skippedCount);
}
if ($failedCount > 0) {
$summary .= sprintf(__('Sync dispatch failed suffix'), $failedCount);
}
if ($denied > 0) {
$summary .= sprintf(__('Sync dispatch denied suffix'), $denied);
}
$this->success($summary, null, [
'queued' => $result['queued'],
'syncing' => $result['queued'],
]);
}
/**
* 轮询手工同步进度(依据 runtime 文件锁)
*/
public function syncpolling(): void
{
$idsRaw = trim((string) $this->request->request('ids', ''));
if ($idsRaw === '') {
$this->success('', null, ['syncing' => []]);
}
/** @var int[] $ticketIds */
$ticketIds = [];
foreach (explode(',', $idsRaw) as $part) {
$part = trim($part);
if ($part !== '' && ctype_digit($part)) {
$ticketIds[] = (int) $part;
} }
} }
$summary = sprintf('成功 %d 条,失败 %d 条', $ok, $fail); if ($ticketIds === []) {
if ($fail > 0) { $this->success('', null, ['syncing' => []]);
$summary .= '' . implode('', array_slice($messages, 0, 5));
} }
$this->success($summary);
$dispatch = new SplitTicketSyncDispatchService();
$syncing = $dispatch->filterSyncingIds($ticketIds);
$this->success('', null, ['syncing' => $syncing]);
} }
/** /**
@@ -31,6 +31,12 @@ return [
'Sync in progress' => '同步中', 'Sync in progress' => '同步中',
'Sync ticket started' => '正在同步工单', 'Sync ticket started' => '正在同步工单',
'Sync done' => '同步完成', 'Sync done' => '同步完成',
'Sync dispatch queued' => '已提交 %d 条同步任务到后台执行',
'Sync dispatch skipped suffix' => '%d 条已在同步中已跳过',
'Sync dispatch failed suffix' => '%d 条启动失败',
'Sync dispatch denied suffix' => '%d 条无权限未提交',
'Sync all skipped busy' => '所选工单均在同步中,请稍后再试',
'Sync dispatch failed' => '未能启动同步任务,请检查服务器是否允许后台执行 CLIexec/shell_exec',
'Sync status success' => '同步成功', 'Sync status success' => '同步成功',
'Sync status error' => '同步异常', 'Sync status error' => '同步异常',
'Sync status pending' => '待同步', 'Sync status pending' => '待同步',
@@ -4,6 +4,7 @@ declare(strict_types=1);
namespace app\common\service; namespace app\common\service;
use app\common\library\scrm\AbstractScrmSpider;
use app\common\library\scrm\ScrmSpiderInterface; use app\common\library\scrm\ScrmSpiderInterface;
use app\common\library\scrm\spider\A2cSpider; use app\common\library\scrm\spider\A2cSpider;
use app\common\library\scrm\spider\ChatknowSpider; use app\common\library\scrm\spider\ChatknowSpider;
@@ -59,6 +60,49 @@ class SplitScrmSpiderFactory
return array_keys(self::MAP); return array_keys(self::MAP);
} }
/**
* 同步是否依赖 Node Headless(纯 PHP cURL 等为 false
*/
public static function requiresNode(string $ticketType): bool
{
$class = self::resolveClass($ticketType);
if ($class === null) {
return true;
}
return is_subclass_of($class, AbstractScrmSpider::class);
}
/**
* @return list<string>
*/
public static function listDirectSyncTypes(): array
{
$types = [];
foreach (self::listSupportedTypes() as $ticketType) {
if (!self::requiresNode($ticketType)) {
$types[] = $ticketType;
}
}
return $types;
}
/**
* @return list<string>
*/
public static function listNodeSyncTypes(): array
{
$types = [];
foreach (self::listSupportedTypes() as $ticketType) {
if (self::requiresNode($ticketType)) {
$types[] = $ticketType;
}
}
return $types;
}
/** /**
* @return ScrmSpiderInterface|null * @return ScrmSpiderInterface|null
*/ */
@@ -57,11 +57,23 @@ class SplitSyncConfigService
{ {
$value = self::getConfigValue('split_sync_max_per_cron'); $value = self::getConfigValue('split_sync_max_per_cron');
if ($value === '') { if ($value === '') {
return 2; return 5;
} }
return max(1, min(20, (int) $value)); return max(1, min(20, (int) $value));
} }
/**
* Node 通道候选池倍数(候选数 = maxPerRun × 本值)
*/
public static function getNodeCandidatePoolMultiplier(): int
{
$value = self::getConfigValue('split_sync_node_candidate_multiplier');
if ($value === '') {
return 10;
}
return max(3, min(50, (int) $value));
}
/** /**
* 全局 cron 锁过期秒数,防止异常退出后永久占锁 * 全局 cron 锁过期秒数,防止异常退出后永久占锁
*/ */
@@ -86,6 +98,53 @@ class SplitSyncConfigService
return max(0, (int) $value); return max(0, (int) $value);
} }
/**
* 后台投递 / Cron 使用的 PHP CLI 可执行文件路径
*
* Web(FPM) 环境下 PHP_BINARY 指向 php-fpm,不能用于 think 命令。
*/
public static function getCliPhpBinary(): string
{
$configured = trim(self::getConfigValue('split_sync_cli_php'));
if ($configured !== '' && self::isUsableCliPhp($configured)) {
return $configured;
}
if (defined('PHP_BINDIR') && PHP_BINDIR !== '') {
$candidate = rtrim(PHP_BINDIR, '/\\') . DIRECTORY_SEPARATOR . 'php';
if (self::isUsableCliPhp($candidate)) {
return $candidate;
}
}
if (PHP_SAPI === 'cli' && defined('PHP_BINARY') && PHP_BINARY !== '' && self::isUsableCliPhp(PHP_BINARY)) {
return PHP_BINARY;
}
foreach (['/usr/bin/php', '/usr/local/bin/php'] as $candidate) {
if (self::isUsableCliPhp($candidate)) {
return $candidate;
}
}
return 'php';
}
/**
* 判断路径是否为可用的 PHP CLI(排除 php-fpm
*/
private static function isUsableCliPhp(string $path): bool
{
if (stripos($path, 'fpm') !== false) {
return false;
}
if ($path === 'php') {
return true;
}
return is_file($path) && is_executable($path);
}
private static function getConfigValue(string $name): string private static function getConfigValue(string $name): string
{ {
$site = Config::get('site.' . $name); $site = Config::get('site.' . $name);
@@ -0,0 +1,224 @@
<?php
declare(strict_types=1);
namespace app\common\service;
/**
* 工单手工同步 CLI 后台投递
*
* Web 请求仅负责鉴权与投递,实际 Spider 在独立 CLI 进程中执行,
* 避免长时间占用 PHP-FPM 与 Session 锁导致后台其它页面无法打开。
*/
class SplitTicketSyncDispatchService
{
/**
* 投递手工同步任务到后台 CLI
*
* @param int[] $ticketIds 已通过权限校验的工单 ID
* @return array{queued:int[],skipped:int[],failed:int[]}
*/
public function dispatchManual(array $ticketIds): array
{
$queued = [];
$skipped = [];
$failed = [];
$php = $this->resolvePhpBinary();
$think = $this->resolveThinkScript();
$root = $this->resolveRootPath();
$lockService = new SplitTicketSyncLockService();
foreach ($ticketIds as $rawId) {
$ticketId = (int) $rawId;
if ($ticketId <= 0) {
continue;
}
if ($lockService->isLocked($ticketId)) {
$skipped[] = $ticketId;
continue;
}
if ($this->spawnCli($php, $think, $root, $ticketId)) {
$queued[] = $ticketId;
} else {
$failed[] = $ticketId;
}
}
SplitTicketSyncLogger::log('web', 'manual sync dispatched', [
'queued' => $queued,
'skipped' => $skipped,
'failed' => $failed,
'phpCli' => $php,
]);
return [
'queued' => $queued,
'skipped' => $skipped,
'failed' => $failed,
];
}
/**
* 查询仍在同步中的工单 ID(依据 runtime 文件锁)
*
* @param int[] $ticketIds
* @return int[]
*/
public function filterSyncingIds(array $ticketIds): array
{
$lockService = new SplitTicketSyncLockService();
$syncing = [];
foreach ($ticketIds as $rawId) {
$ticketId = (int) $rawId;
if ($ticketId > 0 && $lockService->isLocked($ticketId)) {
$syncing[] = $ticketId;
}
}
return $syncing;
}
/**
* 后台启动 split:sync-tickets CLI
*/
private function spawnCli(string $php, string $think, string $root, int $ticketId): bool
{
$logDir = $this->resolveLogDir();
$logFile = $logDir . 'cli_' . $ticketId . '_' . date('YmdHis') . '.log';
$inner = sprintf(
'%s %s split:sync-tickets --ticket=%d >> %s 2>&1',
escapeshellarg($php),
escapeshellarg($think),
$ticketId,
escapeshellarg($logFile)
);
if ($this->isWindows()) {
$command = sprintf('start /B cmd /C %s', $inner);
} else {
$command = sprintf('cd %s && nohup %s &', escapeshellarg($root), $inner);
}
if ($this->canUseExec()) {
@exec($command, $output, $exitCode);
SplitTicketSyncLogger::log('web', 'cli spawn exec', [
'ticketId' => $ticketId,
'command' => $command,
'exitCode' => $exitCode,
]);
return true;
}
if ($this->canUseShellExec()) {
@shell_exec($command);
SplitTicketSyncLogger::log('web', 'cli spawn shell_exec', [
'ticketId' => $ticketId,
'command' => $command,
]);
return true;
}
SplitTicketSyncLogger::log('web', 'cli spawn failed: exec disabled', [
'ticketId' => $ticketId,
]);
return false;
}
private function resolvePhpBinary(): string
{
if (method_exists(SplitSyncConfigService::class, 'getCliPhpBinary')) {
return SplitSyncConfigService::getCliPhpBinary();
}
return $this->resolvePhpBinaryFallback();
}
/**
* 当 SplitSyncConfigService 未部署 getCliPhpBinary 时的兜底解析
*/
private function resolvePhpBinaryFallback(): string
{
if (defined('PHP_BINDIR') && PHP_BINDIR !== '') {
$candidate = rtrim(PHP_BINDIR, '/\\') . DIRECTORY_SEPARATOR . 'php';
if ($this->isUsableCliPhp($candidate)) {
return $candidate;
}
}
foreach (['/usr/bin/php', '/usr/local/bin/php'] as $candidate) {
if ($this->isUsableCliPhp($candidate)) {
return $candidate;
}
}
return 'php';
}
private function isUsableCliPhp(string $path): bool
{
if (stripos($path, 'fpm') !== false) {
return false;
}
if ($path === 'php') {
return true;
}
return is_file($path) && is_executable($path);
}
private function resolveThinkScript(): string
{
$root = $this->resolveRootPath();
return rtrim($root, '/\\') . DIRECTORY_SEPARATOR . 'think';
}
private function resolveRootPath(): string
{
if (defined('ROOT_PATH') && ROOT_PATH !== '') {
return rtrim(ROOT_PATH, '/\\') . DIRECTORY_SEPARATOR;
}
return rtrim(dirname(__DIR__, 3), '/\\') . DIRECTORY_SEPARATOR;
}
private function resolveLogDir(): string
{
$runtime = defined('RUNTIME_PATH') ? RUNTIME_PATH : ($this->resolveRootPath() . 'runtime' . DIRECTORY_SEPARATOR);
$dir = rtrim($runtime, '/\\') . DIRECTORY_SEPARATOR . 'split_ticket_sync' . DIRECTORY_SEPARATOR;
if (!is_dir($dir)) {
@mkdir($dir, 0755, true);
}
return $dir;
}
private function isWindows(): bool
{
return strtoupper(substr(PHP_OS, 0, 3)) === 'WIN';
}
/**
* @return string[]
*/
private function disabledFunctions(): array
{
$raw = (string) ini_get('disable_functions');
if ($raw === '') {
return [];
}
return array_filter(array_map('trim', explode(',', $raw)));
}
private function canUseExec(): bool
{
return function_exists('exec') && !in_array('exec', $this->disabledFunctions(), true);
}
private function canUseShellExec(): bool
{
return function_exists('shell_exec') && !in_array('shell_exec', $this->disabledFunctions(), true);
}
}
@@ -43,6 +43,20 @@ class SplitTicketSyncLockService
} }
} }
/**
* 工单是否处于同步中(锁文件存在且未过期)
*/
public function isLocked(int $ticketId): bool
{
$path = $this->lockPath($ticketId);
if ($this->isStaleLock($path)) {
@unlink($path);
return false;
}
return is_file($path);
}
private function lockPath(int $ticketId): string private function lockPath(int $ticketId): string
{ {
$runtime = defined('RUNTIME_PATH') ? RUNTIME_PATH : (dirname(__DIR__, 3) . '/runtime/'); $runtime = defined('RUNTIME_PATH') ? RUNTIME_PATH : (dirname(__DIR__, 3) . '/runtime/');
@@ -80,131 +80,49 @@ class SplitTicketSyncService
} }
/** /**
* 扫描到期工单并同步(限流 + Node 队列感知 * 扫描到期工单并同步(直连 PHP 与 Node 双通道
*/ */
public function syncDueTickets(): int public function syncDueTickets(): int
{ {
$maxPerRun = SplitSyncConfigService::getMaxTicketsPerCronRun();
$autoSyncTypes = $this->resolveAutoSyncTicketTypes(); $autoSyncTypes = $this->resolveAutoSyncTicketTypes();
$failThreshold = SplitSyncConfigService::getFailPauseThreshold();
if ($autoSyncTypes === []) { if ($autoSyncTypes === []) {
SplitTicketSyncLogger::logCron('scan skip', ['reason' => 'no auto-sync ticket types']); SplitTicketSyncLogger::logCron('scan skip', ['reason' => 'no auto-sync ticket types']);
return 0; return 0;
} }
if (!SplitNodeHealthService::canAcceptWork()) { $failThreshold = SplitSyncConfigService::getFailPauseThreshold();
SplitTicketSyncLogger::logCron('scan skip', [ $directTypes = array_values(array_intersect(
'reason' => 'node queue busy', $autoSyncTypes,
'queue' => SplitNodeHealthService::getQueueSnapshot(), SplitScrmSpiderFactory::listDirectSyncTypes()
]); ));
return 0; $nodeTypes = array_values(array_intersect(
} $autoSyncTypes,
SplitScrmSpiderFactory::listNodeSyncTypes()
));
$query = Ticket::where('status', 'normal')->whereIn('ticket_type', $autoSyncTypes);
if ($failThreshold > 0) {
$query->where('sync_fail_count', '<', $failThreshold);
}
$list = $query->order('sync_time', 'asc')->order('id', 'asc')
->limit($maxPerRun * 5)
->select();
$list = $this->sortTicketsBySessionKey($list);
$candidateCount = count($list);
SplitTicketSyncLogger::logCron('scan start', [ SplitTicketSyncLogger::logCron('scan start', [
'candidateCount' => $candidateCount,
'maxPerRun' => $maxPerRun,
'autoSyncTypes' => $autoSyncTypes, 'autoSyncTypes' => $autoSyncTypes,
'directTypes' => $directTypes,
'nodeTypes' => $nodeTypes,
'failThreshold' => $failThreshold, 'failThreshold' => $failThreshold,
'nodeQueue' => SplitNodeHealthService::getQueueSnapshot(), 'nodeQueue' => SplitNodeHealthService::getQueueSnapshot(),
'groupedBy' => 'sessionKey',
]);
SplitTicketSyncLogger::log('cron', 'scan start', [
'candidateCount' => $candidateCount,
'maxPerRun' => $maxPerRun,
'autoSyncTypes' => $autoSyncTypes,
'nodeQueue' => SplitNodeHealthService::getQueueSnapshot(),
]); ]);
/** @var list<array<string, mixed>> $processed */ $directResult = $this->runDirectCronChannel($directTypes, $failThreshold);
$processed = []; $nodeResult = $this->runNodeCronChannel($nodeTypes, $failThreshold);
/** @var list<array<string, mixed>> $skipped */
$skipped = [];
$count = 0;
$stoppedByNodeBusy = false;
foreach ($list as $index => $ticket) { $processedCount = $directResult['processedCount'] + $nodeResult['processedCount'];
$ticketId = (int) $ticket['id']; $processed = array_merge($directResult['processed'], $nodeResult['processed']);
$ticketUrl = (string) $ticket['ticket_url']; $skipped = array_merge($directResult['skipped'], $nodeResult['skipped']);
$ticketType = (string) $ticket['ticket_type']; $candidateIds = array_merge($directResult['candidateIds'], $nodeResult['candidateIds']);
$sessionKey = AntiBotConfigBuilder::resolveSessionKey($ticketType, $ticketUrl);
if ($count >= $maxPerRun) {
$reason = '本轮处理上限已满';
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $reason);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $reason);
continue;
}
$skip = $this->shouldSkip($ticket);
if ($skip !== null) {
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $skip);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $skip);
continue;
}
if (!SplitNodeHealthService::canAcceptWork()) {
$reason = 'Node 队列繁忙,停止本轮后续同步';
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $reason);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $reason);
$stoppedByNodeBusy = true;
for ($i = $index + 1; $i < $candidateCount; $i++) {
$remain = $list[$i];
$remainReason = 'Node 队列繁忙,本轮未执行';
$skipped[] = $this->buildCronTicketEntry(
(int) $remain['id'],
(string) $remain['ticket_type'],
(string) $remain['ticket_url'],
$remainReason
);
}
SplitTicketSyncLogger::log('cron', 'node busy, stop batch', [
'processed' => $count,
'queue' => SplitNodeHealthService::getQueueSnapshot(),
]);
break;
}
$result = $this->syncOne($ticketId, false, true, 'auto');
if (!empty($result['skipped'])) {
$skipReason = (string) ($result['message'] ?? '已跳过');
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $skipReason);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $skipReason);
continue;
}
$processed[] = [
'ticketId' => $ticketId,
'ticketType' => $ticketType,
'ticketUrl' => $ticketUrl,
'sessionKey' => $sessionKey,
'success' => !empty($result['success']),
'message' => (string) ($result['message'] ?? ''),
];
$count++;
}
$candidateIds = array_map(static function ($row) {
return (int) (is_array($row) ? ($row['id'] ?? 0) : ($row['id'] ?? 0));
}, $list);
$openNotSynced = $this->buildOpenNotSyncedAudit( $openNotSynced = $this->buildOpenNotSyncedAudit(
$autoSyncTypes, $autoSyncTypes,
$failThreshold, $failThreshold,
$maxPerRun, SplitSyncConfigService::getMaxTicketsPerCronRun(),
$candidateIds, $candidateIds,
$processed, $processed,
$skipped, $skipped,
$stoppedByNodeBusy $nodeResult['stoppedByNodeBusy']
); );
$successCount = count(array_filter($processed, static function (array $row): bool { $successCount = count(array_filter($processed, static function (array $row): bool {
@@ -213,21 +131,362 @@ class SplitTicketSyncService
$failedCount = count($processed) - $successCount; $failedCount = count($processed) - $successCount;
SplitTicketSyncLogger::logCron('scan end', [ SplitTicketSyncLogger::logCron('scan end', [
'processedCount' => $count, 'processedCount' => $processedCount,
'successCount' => $successCount, 'successCount' => $successCount,
'failedCount' => $failedCount, 'failedCount' => $failedCount,
'skippedCount' => count($skipped), 'skippedCount' => count($skipped),
'directChannel' => $directResult['summary'],
'nodeChannel' => $nodeResult['summary'],
'processed' => $processed, 'processed' => $processed,
'skipped' => $skipped, 'skipped' => $skipped,
'openNotSynced' => $openNotSynced, 'openNotSynced' => $openNotSynced,
]); ]);
SplitTicketSyncLogger::log('cron', 'scan end', [ SplitTicketSyncLogger::log('cron', 'scan end', [
'processedCount' => $count, 'processedCount' => $processedCount,
'successCount' => $successCount, 'successCount' => $successCount,
'failedCount' => $failedCount, 'failedCount' => $failedCount,
]); ]);
return $count; return $processedCount;
}
/**
* 直连 PHP 通道:按周期同步,不占 Node 名额
*
* @param list<string> $directTypes
* @return array{
* processedCount:int,
* processed:list<array<string,mixed>>,
* skipped:list<array<string,mixed>>,
* candidateIds:list<int>,
* summary:array<string,mixed>
* }
*/
private function runDirectCronChannel(array $directTypes, int $failThreshold): array
{
if ($directTypes === []) {
return $this->emptyCronChannelResult('direct');
}
$list = $this->loadOpenTicketsForTypes($directTypes, $failThreshold, 0);
$candidateIds = $this->extractTicketIds($list);
SplitTicketSyncLogger::logCron('direct channel start', [
'types' => $directTypes,
'candidateCount' => count($list),
]);
/** @var list<array<string, mixed>> $processed */
$processed = [];
/** @var list<array<string, mixed>> $skipped */
$skipped = [];
$count = 0;
foreach ($list as $ticket) {
$ticketId = (int) $ticket['id'];
$ticketType = (string) $ticket['ticket_type'];
$ticketUrl = (string) $ticket['ticket_url'];
$skip = $this->shouldSkip($ticket);
if ($skip !== null) {
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $skip);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $skip, 'direct');
continue;
}
$result = $this->syncOne($ticketId, false, true, 'auto');
if (!empty($result['skipped'])) {
$skipReason = (string) ($result['message'] ?? '已跳过');
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $skipReason);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $skipReason, 'direct');
continue;
}
$processed[] = [
'ticketId' => $ticketId,
'ticketType' => $ticketType,
'ticketUrl' => $ticketUrl,
'channel' => 'direct',
'success' => !empty($result['success']),
'message' => (string) ($result['message'] ?? ''),
];
$count++;
}
return [
'processedCount' => $count,
'processed' => $processed,
'skipped' => $skipped,
'candidateIds' => $candidateIds,
'summary' => [
'channel' => 'direct',
'processedCount' => $count,
'skippedCount' => count($skipped),
],
'stoppedByNodeBusy' => false,
];
}
/**
* Node 通道:限流 + 类型公平轮转
*
* @param list<string> $nodeTypes
* @return array{
* processedCount:int,
* processed:list<array<string,mixed>>,
* skipped:list<array<string,mixed>>,
* candidateIds:list<int>,
* summary:array<string,mixed>,
* stoppedByNodeBusy:bool
* }
*/
private function runNodeCronChannel(array $nodeTypes, int $failThreshold): array
{
if ($nodeTypes === []) {
return $this->emptyCronChannelResult('node');
}
$maxPerRun = SplitSyncConfigService::getMaxTicketsPerCronRun();
$poolMultiplier = SplitSyncConfigService::getNodeCandidatePoolMultiplier();
$candidateLimit = $maxPerRun * $poolMultiplier;
$pool = $this->loadOpenTicketsForTypes($nodeTypes, $failThreshold, $candidateLimit);
$candidateIds = $this->extractTicketIds($pool);
$duePool = [];
foreach ($pool as $ticket) {
if ($this->shouldSkip($ticket) === null) {
$duePool[] = $ticket;
}
}
$picked = $this->pickFairNodeTickets($duePool, $maxPerRun);
$picked = $this->sortTicketsBySessionKey($picked);
$pickedIdMap = array_fill_keys($this->extractTicketIds($picked), true);
SplitTicketSyncLogger::logCron('node channel start', [
'types' => $nodeTypes,
'maxPerRun' => $maxPerRun,
'poolCount' => count($pool),
'dueCount' => count($duePool),
'pickedCount' => count($picked),
'nodeQueue' => SplitNodeHealthService::getQueueSnapshot(),
]);
if (!SplitNodeHealthService::canAcceptWork()) {
SplitTicketSyncLogger::logCron('node channel skip', [
'reason' => 'node queue busy',
'queue' => SplitNodeHealthService::getQueueSnapshot(),
]);
$skipped = [];
foreach ($picked as $ticket) {
$skipped[] = $this->buildCronTicketEntry(
(int) $ticket['id'],
(string) $ticket['ticket_type'],
(string) $ticket['ticket_url'],
'Node 队列繁忙,本轮未执行'
);
}
foreach ($pool as $ticket) {
$ticketId = (int) $ticket['id'];
if (isset($pickedIdMap[$ticketId])) {
continue;
}
$skip = $this->shouldSkip($ticket);
if ($skip !== null) {
continue;
}
$skipped[] = $this->buildCronTicketEntry(
$ticketId,
(string) $ticket['ticket_type'],
(string) $ticket['ticket_url'],
sprintf('未进入本轮 Node 候选(前%d条到期工单公平轮转)', $candidateLimit)
);
}
return [
'processedCount' => 0,
'processed' => [],
'skipped' => $skipped,
'candidateIds' => $candidateIds,
'summary' => [
'channel' => 'node',
'processedCount' => 0,
'skippedCount' => count($skipped),
'reason' => 'node queue busy',
],
'stoppedByNodeBusy' => true,
];
}
/** @var list<array<string, mixed>> $processed */
$processed = [];
/** @var list<array<string, mixed>> $skipped */
$skipped = [];
$count = 0;
$stoppedByNodeBusy = false;
foreach ($pool as $ticket) {
$ticketId = (int) $ticket['id'];
$ticketType = (string) $ticket['ticket_type'];
$ticketUrl = (string) $ticket['ticket_url'];
if (!isset($pickedIdMap[$ticketId])) {
$skip = $this->shouldSkip($ticket);
if ($skip !== null) {
continue;
}
$skipped[] = $this->buildCronTicketEntry(
$ticketId,
$ticketType,
$ticketUrl,
sprintf('未进入本轮 Node 候选(前%d条到期工单公平轮转)', $candidateLimit)
);
continue;
}
if (!SplitNodeHealthService::canAcceptWork()) {
$stoppedByNodeBusy = true;
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, 'Node 队列繁忙,停止本轮后续同步');
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, 'Node 队列繁忙,停止本轮后续同步', 'node');
continue;
}
$sessionKey = AntiBotConfigBuilder::resolveSessionKey($ticketType, $ticketUrl);
$result = $this->syncOne($ticketId, false, true, 'auto');
if (!empty($result['skipped'])) {
$skipReason = (string) ($result['message'] ?? '已跳过');
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $skipReason);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $skipReason, 'node');
continue;
}
$processed[] = [
'ticketId' => $ticketId,
'ticketType' => $ticketType,
'ticketUrl' => $ticketUrl,
'sessionKey' => $sessionKey,
'channel' => 'node',
'success' => !empty($result['success']),
'message' => (string) ($result['message'] ?? ''),
];
$count++;
}
return [
'processedCount' => $count,
'processed' => $processed,
'skipped' => $skipped,
'candidateIds' => $candidateIds,
'summary' => [
'channel' => 'node',
'processedCount' => $count,
'skippedCount' => count($skipped),
'maxPerRun' => $maxPerRun,
],
'stoppedByNodeBusy' => $stoppedByNodeBusy,
];
}
/**
* @param list<string> $types
* @return list<Ticket>
*/
private function loadOpenTicketsForTypes(array $types, int $failThreshold, int $limit): array
{
if ($types === []) {
return [];
}
$query = Ticket::where('status', 'normal')->whereIn('ticket_type', $types);
if ($failThreshold > 0) {
$query->where('sync_fail_count', '<', $failThreshold);
}
$query->order('sync_time', 'asc')->order('id', 'asc');
if ($limit > 0) {
$query->limit($limit);
}
$list = $query->select();
return $list instanceof \think\Collection ? $list->all() : (array) $list;
}
/**
* Node 通道:按工单类型公平轮转选取(每轮每类型优先取 1 条)
*
* @param list<Ticket> $dueTickets
* @return list<Ticket>
*/
private function pickFairNodeTickets(array $dueTickets, int $maxPerRun): array
{
if ($dueTickets === [] || $maxPerRun <= 0) {
return [];
}
/** @var array<string, list<Ticket>> $byType */
$byType = [];
foreach ($dueTickets as $ticket) {
$type = (string) $ticket['ticket_type'];
$byType[$type][] = $ticket;
}
$typeKeys = array_keys($byType);
sort($typeKeys, SORT_STRING);
/** @var list<Ticket> $picked */
$picked = [];
while (count($picked) < $maxPerRun) {
$added = false;
foreach ($typeKeys as $type) {
if (count($picked) >= $maxPerRun) {
break;
}
if ($byType[$type] === []) {
continue;
}
$picked[] = array_shift($byType[$type]);
$added = true;
}
if (!$added) {
break;
}
}
return $picked;
}
/**
* @param list<Ticket|array<string,mixed>> $list
* @return list<int>
*/
private function extractTicketIds(array $list): array
{
$ids = [];
foreach ($list as $row) {
$ids[] = (int) (is_array($row) ? ($row['id'] ?? 0) : ($row['id'] ?? 0));
}
return $ids;
}
/**
* @return array{
* processedCount:int,
* processed:list<array<string,mixed>>,
* skipped:list<array<string,mixed>>,
* candidateIds:list<int>,
* summary:array<string,mixed>,
* stoppedByNodeBusy:bool
* }
*/
private function emptyCronChannelResult(string $channel): array
{
return [
'processedCount' => 0,
'processed' => [],
'skipped' => [],
'candidateIds' => [],
'summary' => ['channel' => $channel, 'processedCount' => 0, 'skippedCount' => 0],
'stoppedByNodeBusy' => false,
];
} }
/** /**
@@ -466,13 +725,14 @@ class SplitTicketSyncService
]; ];
} }
private function logCronCandidateSkipped(int $ticketId, string $ticketType, string $ticketUrl, string $reason): void private function logCronCandidateSkipped(int $ticketId, string $ticketType, string $ticketUrl, string $reason, string $channel = 'legacy'): void
{ {
$ctx = [ $ctx = [
'ticketId' => $ticketId, 'ticketId' => $ticketId,
'ticketType' => $ticketType, 'ticketType' => $ticketType,
'ticketUrl' => $ticketUrl, 'ticketUrl' => $ticketUrl,
'reason' => $reason, 'reason' => $reason,
'channel' => $channel,
]; ];
SplitTicketSyncLogger::logCron('candidate skipped', $ctx); SplitTicketSyncLogger::logCron('candidate skipped', $ctx);
SplitTicketSyncLogger::log('cron', 'candidate skipped', $ctx); SplitTicketSyncLogger::log('cron', 'candidate skipped', $ctx);
@@ -515,14 +775,15 @@ class SplitTicketSyncService
->select(); ->select();
$result = []; $result = [];
foreach ($openList as $ticket) { foreach ($openList as $ticketRow) {
$ticketId = (int) $ticket['id']; $ticket = $ticketRow instanceof Ticket ? $ticketRow->toArray() : (array) $ticketRow;
if (isset($handledIds[$ticketId])) { $ticketId = (int) ($ticket['id'] ?? 0);
if ($ticketId <= 0 || isset($handledIds[$ticketId])) {
continue; continue;
} }
$ticketType = (string) $ticket['ticket_type']; $ticketType = (string) ($ticket['ticket_type'] ?? '');
$ticketUrl = (string) $ticket['ticket_url']; $ticketUrl = (string) ($ticket['ticket_url'] ?? '');
$reason = $this->resolveOpenNotSyncedReason( $reason = $this->resolveOpenNotSyncedReason(
$ticket, $ticket,
$autoSyncTypes, $autoSyncTypes,
@@ -568,7 +829,13 @@ class SplitTicketSyncService
return '该类型未配置自动同步周期'; return '该类型未配置自动同步周期';
} }
if (!isset($candidateIdMap[(int) ($ticket['id'] ?? 0)])) { if (!isset($candidateIdMap[(int) ($ticket['id'] ?? 0)])) {
return sprintf('未进入本轮候选队列(仅扫描最久未同步的前%d条)', $maxPerRun * 5); if (SplitScrmSpiderFactory::requiresNode($ticketType)) {
$poolSize = SplitSyncConfigService::getMaxTicketsPerCronRun()
* SplitSyncConfigService::getNodeCandidatePoolMultiplier();
return sprintf('未进入本轮 Node 候选(前%d条到期工单公平轮转)', $poolSize);
}
return '直连通道:未到同步周期或本轮已处理';
} }
if ($stoppedByNodeBusy) { if ($stoppedByNodeBusy) {
return 'Node 队列繁忙,本轮未执行'; return 'Node 队列繁忙,本轮未执行';
@@ -244,6 +244,9 @@ define(['jquery', 'bootstrap', 'backend', 'table', 'form'], function ($, undefin
/** @type {number[]} 正在手动同步的工单 ID */ /** @type {number[]} 正在手动同步的工单 ID */
syncingTicketIds: [], syncingTicketIds: [],
syncPollTimer: null,
syncPollStartedAt: 0,
syncPollGraceMs: 8000,
/** /**
* 渲染筛选结果汇总行(全量筛选数据,非当前页) * 渲染筛选结果汇总行(全量筛选数据,非当前页)
@@ -269,7 +272,7 @@ define(['jquery', 'bootstrap', 'backend', 'table', 'form'], function ($, undefin
}, },
/** /**
* 后台同步:标记「同步中」并请求 sync 接口 * 后台同步:标记「同步中」、投递 CLI,并轮询直至锁释放
* *
* @param {object} table bootstrapTable 实例 * @param {object} table bootstrapTable 实例
* @param {number[]} ids 工单 ID * @param {number[]} ids 工单 ID
@@ -292,13 +295,81 @@ define(['jquery', 'bootstrap', 'backend', 'table', 'form'], function ($, undefin
url: 'split.ticket/sync', url: 'split.ticket/sync',
data: {ids: ids.join(',')}, data: {ids: ids.join(',')},
loading: false loading: false
}, function () { }, function (data, ret) {
var queued = (ret.data && ret.data.queued) ? ret.data.queued : ids;
Controller.api.syncingTicketIds = (queued || []).map(function (id) {
return parseInt(id, 10);
}).filter(function (id) {
return !isNaN(id) && id > 0;
});
if (Controller.api.syncingTicketIds.length) {
Controller.api.startSyncPolling(table);
} else {
Controller.api.finishTicketsSync(table); Controller.api.finishTicketsSync(table);
}
}, function () { }, function () {
Controller.api.finishTicketsSync(table); Controller.api.finishTicketsSync(table);
}); });
}, },
/**
* 轮询同步锁状态,全部完成后刷新列表
*/
startSyncPolling: function (table) {
Controller.api.stopSyncPolling();
if (!Controller.api.syncingTicketIds.length) {
return;
}
Controller.api.syncPollStartedAt = Date.now();
Controller.api.syncPollTimer = setInterval(function () {
if (!Controller.api.syncingTicketIds.length) {
Controller.api.stopSyncPolling();
return;
}
Fast.api.ajax({
url: 'split.ticket/syncpolling',
data: {ids: Controller.api.syncingTicketIds.join(',')},
loading: false
}, function (data, ret) {
var still = (ret.data && ret.data.syncing) ? ret.data.syncing : [];
still = (still || []).map(function (id) {
return parseInt(id, 10);
}).filter(function (id) {
return !isNaN(id) && id > 0;
});
var elapsed = Date.now() - (Controller.api.syncPollStartedAt || 0);
var graceMs = Controller.api.syncPollGraceMs || 8000;
if (still.length === 0 && elapsed < graceMs) {
table.bootstrapTable('refresh', {silent: true});
return false;
}
if (still.length === 0) {
Controller.api.stopSyncPolling();
var doneMsg = (typeof Config.syncDoneMsg !== 'undefined' && Config.syncDoneMsg)
? Config.syncDoneMsg
: '同步完成';
Toastr.success(doneMsg);
Controller.api.finishTicketsSync(table);
return false;
}
if (still.join(',') !== Controller.api.syncingTicketIds.join(',')) {
Controller.api.syncingTicketIds = still;
var rowData = table.bootstrapTable('getData');
table.bootstrapTable('load', rowData);
}
table.bootstrapTable('refresh', {silent: true});
return false;
});
}, 3000);
},
stopSyncPolling: function () {
if (Controller.api.syncPollTimer) {
clearInterval(Controller.api.syncPollTimer);
Controller.api.syncPollTimer = null;
}
},
/** /**
* 将选中工单标记为「同步中」并刷新列表展示(不请求后端) * 将选中工单标记为「同步中」并刷新列表展示(不请求后端)
*/ */
@@ -316,6 +387,7 @@ define(['jquery', 'bootstrap', 'backend', 'table', 'form'], function ($, undefin
* 同步结束:清除标记并刷新列表数据 * 同步结束:清除标记并刷新列表数据
*/ */
finishTicketsSync: function (table) { finishTicketsSync: function (table) {
Controller.api.stopSyncPolling();
Controller.api.syncingTicketIds = []; Controller.api.syncingTicketIds = [];
table.bootstrapTable('refresh', { table.bootstrapTable('refresh', {
silent: true silent: true
+1
View File
@@ -60,6 +60,7 @@ const {
isHuojianShortEntryUrl, isHuojianShortEntryUrl,
applyHuojianBusinessHostHint, applyHuojianBusinessHostHint,
} = require('./lib/huojian-url'); } = require('./lib/huojian-url');
const { runUiPagination } = require('./lib/ui-pagination-runner');
const app = express(); const app = express();
app.use(express.json({ limit: '50mb' })); app.use(express.json({ limit: '50mb' }));
+74 -2
View File
@@ -244,6 +244,9 @@ define(['jquery', 'bootstrap', 'backend', 'table', 'form'], function ($, undefin
/** @type {number[]} 正在手动同步的工单 ID */ /** @type {number[]} 正在手动同步的工单 ID */
syncingTicketIds: [], syncingTicketIds: [],
syncPollTimer: null,
syncPollStartedAt: 0,
syncPollGraceMs: 8000,
/** /**
* 渲染筛选结果汇总行(全量筛选数据,非当前页) * 渲染筛选结果汇总行(全量筛选数据,非当前页)
@@ -269,7 +272,7 @@ define(['jquery', 'bootstrap', 'backend', 'table', 'form'], function ($, undefin
}, },
/** /**
* 后台同步:标记「同步中」并请求 sync 接口 * 后台同步:标记「同步中」、投递 CLI,并轮询直至锁释放
* *
* @param {object} table bootstrapTable 实例 * @param {object} table bootstrapTable 实例
* @param {number[]} ids 工单 ID * @param {number[]} ids 工单 ID
@@ -292,13 +295,81 @@ define(['jquery', 'bootstrap', 'backend', 'table', 'form'], function ($, undefin
url: 'split.ticket/sync', url: 'split.ticket/sync',
data: {ids: ids.join(',')}, data: {ids: ids.join(',')},
loading: false loading: false
}, function () { }, function (data, ret) {
var queued = (ret.data && ret.data.queued) ? ret.data.queued : ids;
Controller.api.syncingTicketIds = (queued || []).map(function (id) {
return parseInt(id, 10);
}).filter(function (id) {
return !isNaN(id) && id > 0;
});
if (Controller.api.syncingTicketIds.length) {
Controller.api.startSyncPolling(table);
} else {
Controller.api.finishTicketsSync(table); Controller.api.finishTicketsSync(table);
}
}, function () { }, function () {
Controller.api.finishTicketsSync(table); Controller.api.finishTicketsSync(table);
}); });
}, },
/**
* 轮询同步锁状态,全部完成后刷新列表
*/
startSyncPolling: function (table) {
Controller.api.stopSyncPolling();
if (!Controller.api.syncingTicketIds.length) {
return;
}
Controller.api.syncPollStartedAt = Date.now();
Controller.api.syncPollTimer = setInterval(function () {
if (!Controller.api.syncingTicketIds.length) {
Controller.api.stopSyncPolling();
return;
}
Fast.api.ajax({
url: 'split.ticket/syncpolling',
data: {ids: Controller.api.syncingTicketIds.join(',')},
loading: false
}, function (data, ret) {
var still = (ret.data && ret.data.syncing) ? ret.data.syncing : [];
still = (still || []).map(function (id) {
return parseInt(id, 10);
}).filter(function (id) {
return !isNaN(id) && id > 0;
});
var elapsed = Date.now() - (Controller.api.syncPollStartedAt || 0);
var graceMs = Controller.api.syncPollGraceMs || 8000;
if (still.length === 0 && elapsed < graceMs) {
table.bootstrapTable('refresh', {silent: true});
return false;
}
if (still.length === 0) {
Controller.api.stopSyncPolling();
var doneMsg = (typeof Config.syncDoneMsg !== 'undefined' && Config.syncDoneMsg)
? Config.syncDoneMsg
: '同步完成';
Toastr.success(doneMsg);
Controller.api.finishTicketsSync(table);
return false;
}
if (still.join(',') !== Controller.api.syncingTicketIds.join(',')) {
Controller.api.syncingTicketIds = still;
var rowData = table.bootstrapTable('getData');
table.bootstrapTable('load', rowData);
}
table.bootstrapTable('refresh', {silent: true});
return false;
});
}, 3000);
},
stopSyncPolling: function () {
if (Controller.api.syncPollTimer) {
clearInterval(Controller.api.syncPollTimer);
Controller.api.syncPollTimer = null;
}
},
/** /**
* 将选中工单标记为「同步中」并刷新列表展示(不请求后端) * 将选中工单标记为「同步中」并刷新列表展示(不请求后端)
*/ */
@@ -316,6 +387,7 @@ define(['jquery', 'bootstrap', 'backend', 'table', 'form'], function ($, undefin
* 同步结束:清除标记并刷新列表数据 * 同步结束:清除标记并刷新列表数据
*/ */
finishTicketsSync: function (table) { finishTicketsSync: function (table) {
Controller.api.stopSyncPolling();
Controller.api.syncingTicketIds = []; Controller.api.syncingTicketIds = [];
table.bootstrapTable('refresh', { table.bootstrapTable('refresh', {
silent: true silent: true