damon/links
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复工单PHP CLI同步问题 系统设置可以设置并发阈值

Browse Source
This commit is contained in:
root
2026-07-02 19:03:06 +08:00
parent 638b3c4032
commit c3223f026e
20 changed files with 1087 additions and 241 deletions
Download Patch File Download Diff File Expand all files Collapse all files
patches/application/common/service/SplitTicketSyncService.php
+382 -115
View File
@@ -80,131 +80,49 @@ class SplitTicketSyncService
}
/**
* 扫描到期工单并同步（限流 + Node 队列感知）
* 扫描到期工单并同步（直连 PHP 与 Node 双通道）
*/
public function syncDueTickets(): int
{
$maxPerRun = SplitSyncConfigService::getMaxTicketsPerCronRun();
$autoSyncTypes = $this->resolveAutoSyncTicketTypes();
$failThreshold = SplitSyncConfigService::getFailPauseThreshold();
if ($autoSyncTypes === []) {
SplitTicketSyncLogger::logCron('scan skip', ['reason' => 'no auto-sync ticket types']);
return 0;
}
if (!SplitNodeHealthService::canAcceptWork()) {
SplitTicketSyncLogger::logCron('scan skip', [
'reason' => 'node queue busy',
'queue' => SplitNodeHealthService::getQueueSnapshot(),
]);
return 0;
}
$failThreshold = SplitSyncConfigService::getFailPauseThreshold();
$directTypes = array_values(array_intersect(
$autoSyncTypes,
SplitScrmSpiderFactory::listDirectSyncTypes()
));
$nodeTypes = array_values(array_intersect(
$autoSyncTypes,
SplitScrmSpiderFactory::listNodeSyncTypes()
));
$query = Ticket::where('status', 'normal')->whereIn('ticket_type', $autoSyncTypes);
if ($failThreshold > 0) {
$query->where('sync_fail_count', '<', $failThreshold);
}
$list = $query->order('sync_time', 'asc')->order('id', 'asc')
->limit($maxPerRun * 5)
->select();
$list = $this->sortTicketsBySessionKey($list);
$candidateCount = count($list);
SplitTicketSyncLogger::logCron('scan start', [
'candidateCount' => $candidateCount,
'maxPerRun' => $maxPerRun,
'autoSyncTypes' => $autoSyncTypes,
'failThreshold' => $failThreshold,
'nodeQueue' => SplitNodeHealthService::getQueueSnapshot(),
'groupedBy' => 'sessionKey',
]);
SplitTicketSyncLogger::log('cron', 'scan start', [
'candidateCount' => $candidateCount,
'maxPerRun' => $maxPerRun,
'autoSyncTypes' => $autoSyncTypes,
'nodeQueue' => SplitNodeHealthService::getQueueSnapshot(),
'autoSyncTypes' => $autoSyncTypes,
'directTypes' => $directTypes,
'nodeTypes' => $nodeTypes,
'failThreshold' => $failThreshold,
'nodeQueue' => SplitNodeHealthService::getQueueSnapshot(),
]);
/** @var list<array<string, mixed>> $processed */
$processed = [];
/** @var list<array<string, mixed>> $skipped */
$skipped = [];
$count = 0;
$stoppedByNodeBusy = false;
$directResult = $this->runDirectCronChannel($directTypes, $failThreshold);
$nodeResult = $this->runNodeCronChannel($nodeTypes, $failThreshold);
foreach ($list as $index => $ticket) {
$ticketId = (int) $ticket['id'];
$ticketUrl = (string) $ticket['ticket_url'];
$ticketType = (string) $ticket['ticket_type'];
$sessionKey = AntiBotConfigBuilder::resolveSessionKey($ticketType, $ticketUrl);
if ($count >= $maxPerRun) {
$reason = '本轮处理上限已满';
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $reason);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $reason);
continue;
}
$skip = $this->shouldSkip($ticket);
if ($skip !== null) {
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $skip);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $skip);
continue;
}
if (!SplitNodeHealthService::canAcceptWork()) {
$reason = 'Node 队列繁忙，停止本轮后续同步';
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $reason);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $reason);
$stoppedByNodeBusy = true;
for ($i = $index + 1; $i < $candidateCount; $i++) {
$remain = $list[$i];
$remainReason = 'Node 队列繁忙，本轮未执行';
$skipped[] = $this->buildCronTicketEntry(
(int) $remain['id'],
(string) $remain['ticket_type'],
(string) $remain['ticket_url'],
$remainReason
);
}
SplitTicketSyncLogger::log('cron', 'node busy, stop batch', [
'processed' => $count,
'queue' => SplitNodeHealthService::getQueueSnapshot(),
]);
break;
}
$result = $this->syncOne($ticketId, false, true, 'auto');
if (!empty($result['skipped'])) {
$skipReason = (string) ($result['message'] ?? '已跳过');
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $skipReason);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $skipReason);
continue;
}
$processed[] = [
'ticketId' => $ticketId,
'ticketType' => $ticketType,
'ticketUrl' => $ticketUrl,
'sessionKey' => $sessionKey,
'success' => !empty($result['success']),
'message' => (string) ($result['message'] ?? ''),
];
$count++;
}
$candidateIds = array_map(static function ($row) {
return (int) (is_array($row) ? ($row['id'] ?? 0) : ($row['id'] ?? 0));
}, $list);
$processedCount = $directResult['processedCount'] + $nodeResult['processedCount'];
$processed = array_merge($directResult['processed'], $nodeResult['processed']);
$skipped = array_merge($directResult['skipped'], $nodeResult['skipped']);
$candidateIds = array_merge($directResult['candidateIds'], $nodeResult['candidateIds']);
$openNotSynced = $this->buildOpenNotSyncedAudit(
$autoSyncTypes,
$failThreshold,
$maxPerRun,
SplitSyncConfigService::getMaxTicketsPerCronRun(),
$candidateIds,
$processed,
$skipped,
$stoppedByNodeBusy
$nodeResult['stoppedByNodeBusy']
);
$successCount = count(array_filter($processed, static function (array $row): bool {
@@ -213,21 +131,362 @@ class SplitTicketSyncService
$failedCount = count($processed) - $successCount;
SplitTicketSyncLogger::logCron('scan end', [
'processedCount' => $count,
'processedCount' => $processedCount,
'successCount' => $successCount,
'failedCount' => $failedCount,
'skippedCount' => count($skipped),
'directChannel' => $directResult['summary'],
'nodeChannel' => $nodeResult['summary'],
'processed' => $processed,
'skipped' => $skipped,
'openNotSynced' => $openNotSynced,
]);
SplitTicketSyncLogger::log('cron', 'scan end', [
'processedCount' => $count,
'processedCount' => $processedCount,
'successCount' => $successCount,
'failedCount' => $failedCount,
]);
return $count;
return $processedCount;
}
/**
* 直连 PHP 通道：按周期同步，不占 Node 名额
*
* @param list<string> $directTypes
* @return array{
* processedCount:int,
* processed:list<array<string,mixed>>,
* skipped:list<array<string,mixed>>,
* candidateIds:list<int>,
* summary:array<string,mixed>
* }
*/
private function runDirectCronChannel(array $directTypes, int $failThreshold): array
{
if ($directTypes === []) {
return $this->emptyCronChannelResult('direct');
}
$list = $this->loadOpenTicketsForTypes($directTypes, $failThreshold, 0);
$candidateIds = $this->extractTicketIds($list);
SplitTicketSyncLogger::logCron('direct channel start', [
'types' => $directTypes,
'candidateCount' => count($list),
]);
/** @var list<array<string, mixed>> $processed */
$processed = [];
/** @var list<array<string, mixed>> $skipped */
$skipped = [];
$count = 0;
foreach ($list as $ticket) {
$ticketId = (int) $ticket['id'];
$ticketType = (string) $ticket['ticket_type'];
$ticketUrl = (string) $ticket['ticket_url'];
$skip = $this->shouldSkip($ticket);
if ($skip !== null) {
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $skip);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $skip, 'direct');
continue;
}
$result = $this->syncOne($ticketId, false, true, 'auto');
if (!empty($result['skipped'])) {
$skipReason = (string) ($result['message'] ?? '已跳过');
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $skipReason);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $skipReason, 'direct');
continue;
}
$processed[] = [
'ticketId' => $ticketId,
'ticketType' => $ticketType,
'ticketUrl' => $ticketUrl,
'channel' => 'direct',
'success' => !empty($result['success']),
'message' => (string) ($result['message'] ?? ''),
];
$count++;
}
return [
'processedCount' => $count,
'processed' => $processed,
'skipped' => $skipped,
'candidateIds' => $candidateIds,
'summary' => [
'channel' => 'direct',
'processedCount' => $count,
'skippedCount' => count($skipped),
],
'stoppedByNodeBusy' => false,
];
}
/**
* Node 通道：限流 + 类型公平轮转
*
* @param list<string> $nodeTypes
* @return array{
* processedCount:int,
* processed:list<array<string,mixed>>,
* skipped:list<array<string,mixed>>,
* candidateIds:list<int>,
* summary:array<string,mixed>,
* stoppedByNodeBusy:bool
* }
*/
private function runNodeCronChannel(array $nodeTypes, int $failThreshold): array
{
if ($nodeTypes === []) {
return $this->emptyCronChannelResult('node');
}
$maxPerRun = SplitSyncConfigService::getMaxTicketsPerCronRun();
$poolMultiplier = SplitSyncConfigService::getNodeCandidatePoolMultiplier();
$candidateLimit = $maxPerRun * $poolMultiplier;
$pool = $this->loadOpenTicketsForTypes($nodeTypes, $failThreshold, $candidateLimit);
$candidateIds = $this->extractTicketIds($pool);
$duePool = [];
foreach ($pool as $ticket) {
if ($this->shouldSkip($ticket) === null) {
$duePool[] = $ticket;
}
}
$picked = $this->pickFairNodeTickets($duePool, $maxPerRun);
$picked = $this->sortTicketsBySessionKey($picked);
$pickedIdMap = array_fill_keys($this->extractTicketIds($picked), true);
SplitTicketSyncLogger::logCron('node channel start', [
'types' => $nodeTypes,
'maxPerRun' => $maxPerRun,
'poolCount' => count($pool),
'dueCount' => count($duePool),
'pickedCount' => count($picked),
'nodeQueue' => SplitNodeHealthService::getQueueSnapshot(),
]);
if (!SplitNodeHealthService::canAcceptWork()) {
SplitTicketSyncLogger::logCron('node channel skip', [
'reason' => 'node queue busy',
'queue' => SplitNodeHealthService::getQueueSnapshot(),
]);
$skipped = [];
foreach ($picked as $ticket) {
$skipped[] = $this->buildCronTicketEntry(
(int) $ticket['id'],
(string) $ticket['ticket_type'],
(string) $ticket['ticket_url'],
'Node 队列繁忙，本轮未执行'
);
}
foreach ($pool as $ticket) {
$ticketId = (int) $ticket['id'];
if (isset($pickedIdMap[$ticketId])) {
continue;
}
$skip = $this->shouldSkip($ticket);
if ($skip !== null) {
continue;
}
$skipped[] = $this->buildCronTicketEntry(
$ticketId,
(string) $ticket['ticket_type'],
(string) $ticket['ticket_url'],
sprintf('未进入本轮 Node 候选(前%d条到期工单公平轮转)', $candidateLimit)
);
}
return [
'processedCount' => 0,
'processed' => [],
'skipped' => $skipped,
'candidateIds' => $candidateIds,
'summary' => [
'channel' => 'node',
'processedCount' => 0,
'skippedCount' => count($skipped),
'reason' => 'node queue busy',
],
'stoppedByNodeBusy' => true,
];
}
/** @var list<array<string, mixed>> $processed */
$processed = [];
/** @var list<array<string, mixed>> $skipped */
$skipped = [];
$count = 0;
$stoppedByNodeBusy = false;
foreach ($pool as $ticket) {
$ticketId = (int) $ticket['id'];
$ticketType = (string) $ticket['ticket_type'];
$ticketUrl = (string) $ticket['ticket_url'];
if (!isset($pickedIdMap[$ticketId])) {
$skip = $this->shouldSkip($ticket);
if ($skip !== null) {
continue;
}
$skipped[] = $this->buildCronTicketEntry(
$ticketId,
$ticketType,
$ticketUrl,
sprintf('未进入本轮 Node 候选(前%d条到期工单公平轮转)', $candidateLimit)
);
continue;
}
if (!SplitNodeHealthService::canAcceptWork()) {
$stoppedByNodeBusy = true;
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, 'Node 队列繁忙，停止本轮后续同步');
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, 'Node 队列繁忙，停止本轮后续同步', 'node');
continue;
}
$sessionKey = AntiBotConfigBuilder::resolveSessionKey($ticketType, $ticketUrl);
$result = $this->syncOne($ticketId, false, true, 'auto');
if (!empty($result['skipped'])) {
$skipReason = (string) ($result['message'] ?? '已跳过');
$skipped[] = $this->buildCronTicketEntry($ticketId, $ticketType, $ticketUrl, $skipReason);
$this->logCronCandidateSkipped($ticketId, $ticketType, $ticketUrl, $skipReason, 'node');
continue;
}
$processed[] = [
'ticketId' => $ticketId,
'ticketType' => $ticketType,
'ticketUrl' => $ticketUrl,
'sessionKey' => $sessionKey,
'channel' => 'node',
'success' => !empty($result['success']),
'message' => (string) ($result['message'] ?? ''),
];
$count++;
}
return [
'processedCount' => $count,
'processed' => $processed,
'skipped' => $skipped,
'candidateIds' => $candidateIds,
'summary' => [
'channel' => 'node',
'processedCount' => $count,
'skippedCount' => count($skipped),
'maxPerRun' => $maxPerRun,
],
'stoppedByNodeBusy' => $stoppedByNodeBusy,
];
}
/**
* @param list<string> $types
* @return list<Ticket>
*/
private function loadOpenTicketsForTypes(array $types, int $failThreshold, int $limit): array
{
if ($types === []) {
return [];
}
$query = Ticket::where('status', 'normal')->whereIn('ticket_type', $types);
if ($failThreshold > 0) {
$query->where('sync_fail_count', '<', $failThreshold);
}
$query->order('sync_time', 'asc')->order('id', 'asc');
if ($limit > 0) {
$query->limit($limit);
}
$list = $query->select();
return $list instanceof \think\Collection ? $list->all() : (array) $list;
}
/**
* Node 通道：按工单类型公平轮转选取（每轮每类型优先取 1 条）
*
* @param list<Ticket> $dueTickets
* @return list<Ticket>
*/
private function pickFairNodeTickets(array $dueTickets, int $maxPerRun): array
{
if ($dueTickets === [] || $maxPerRun <= 0) {
return [];
}
/** @var array<string, list<Ticket>> $byType */
$byType = [];
foreach ($dueTickets as $ticket) {
$type = (string) $ticket['ticket_type'];
$byType[$type][] = $ticket;
}
$typeKeys = array_keys($byType);
sort($typeKeys, SORT_STRING);
/** @var list<Ticket> $picked */
$picked = [];
while (count($picked) < $maxPerRun) {
$added = false;
foreach ($typeKeys as $type) {
if (count($picked) >= $maxPerRun) {
break;
}
if ($byType[$type] === []) {
continue;
}
$picked[] = array_shift($byType[$type]);
$added = true;
}
if (!$added) {
break;
}
}
return $picked;
}
/**
* @param list<Ticket|array<string,mixed>> $list
* @return list<int>
*/
private function extractTicketIds(array $list): array
{
$ids = [];
foreach ($list as $row) {
$ids[] = (int) (is_array($row) ? ($row['id'] ?? 0) : ($row['id'] ?? 0));
}
return $ids;
}
/**
* @return array{
* processedCount:int,
* processed:list<array<string,mixed>>,
* skipped:list<array<string,mixed>>,
* candidateIds:list<int>,
* summary:array<string,mixed>,
* stoppedByNodeBusy:bool
* }
*/
private function emptyCronChannelResult(string $channel): array
{
return [
'processedCount' => 0,
'processed' => [],
'skipped' => [],
'candidateIds' => [],
'summary' => ['channel' => $channel, 'processedCount' => 0, 'skippedCount' => 0],
'stoppedByNodeBusy' => false,
];
}
/**
@@ -466,13 +725,14 @@ class SplitTicketSyncService
];
}
private function logCronCandidateSkipped(int $ticketId, string $ticketType, string $ticketUrl, string $reason): void
private function logCronCandidateSkipped(int $ticketId, string $ticketType, string $ticketUrl, string $reason, string $channel = 'legacy'): void
{
$ctx = [
'ticketId' => $ticketId,
'ticketType' => $ticketType,
'ticketUrl' => $ticketUrl,
'reason' => $reason,
'channel' => $channel,
];
SplitTicketSyncLogger::logCron('candidate skipped', $ctx);
SplitTicketSyncLogger::log('cron', 'candidate skipped', $ctx);
@@ -515,14 +775,15 @@ class SplitTicketSyncService
->select();
$result = [];
foreach ($openList as $ticket) {
$ticketId = (int) $ticket['id'];
if (isset($handledIds[$ticketId])) {
foreach ($openList as $ticketRow) {
$ticket = $ticketRow instanceof Ticket ? $ticketRow->toArray() : (array) $ticketRow;
$ticketId = (int) ($ticket['id'] ?? 0);
if ($ticketId <= 0 || isset($handledIds[$ticketId])) {
continue;
}
$ticketType = (string) $ticket['ticket_type'];
$ticketUrl = (string) $ticket['ticket_url'];
$ticketType = (string) ($ticket['ticket_type'] ?? '');
$ticketUrl = (string) ($ticket['ticket_url'] ?? '');
$reason = $this->resolveOpenNotSyncedReason(
$ticket,
$autoSyncTypes,
@@ -568,7 +829,13 @@ class SplitTicketSyncService
return '该类型未配置自动同步周期';
}
if (!isset($candidateIdMap[(int) ($ticket['id'] ?? 0)])) {
return sprintf('未进入本轮候选队列(仅扫描最久未同步的前%d条)', $maxPerRun * 5);
if (SplitScrmSpiderFactory::requiresNode($ticketType)) {
$poolSize = SplitSyncConfigService::getMaxTicketsPerCronRun()
* SplitSyncConfigService::getNodeCandidatePoolMultiplier();
return sprintf('未进入本轮 Node 候选(前%d条到期工单公平轮转)', $poolSize);
}
return '直连通道：未到同步周期或本轮已处理';
}
if ($stoppedByNodeBusy) {
return 'Node 队列繁忙，本轮未执行';