damon/links
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化爬虫，增加缓存，线程池，保存浏览器用户信息

Browse Source
This commit is contained in:
root
2026-06-20 15:49:40 +08:00
parent e95f4a227c
commit 09bfa9ae01
26 changed files with 1584 additions and 454 deletions
Download Patch File Download Diff File Expand all files Collapse all files
patches/application/common/service/SplitTicketSyncService.php
+38 -2
View File
@@ -5,6 +5,7 @@ declare(strict_types=1);
namespace app\common\service;
use app\admin\model\split\Ticket;
use app\common\library\scrm\AntiBotConfigBuilder;
use app\common\library\scrm\UnifiedScrmData;
use think\Db;
use think\Exception;
@@ -107,6 +108,7 @@ class SplitTicketSyncService
$list = $query->order('sync_time', 'asc')->order('id', 'asc')
->limit($maxPerRun * 5)
->select();
$list = $this->sortTicketsBySessionKey($list);
$candidateCount = count($list);
SplitTicketSyncLogger::logCron('scan start', [
@@ -115,6 +117,7 @@ class SplitTicketSyncService
'autoSyncTypes' => $autoSyncTypes,
'failThreshold' => $failThreshold,
'nodeQueue' => SplitNodeHealthService::getQueueSnapshot(),
'groupedBy' => 'sessionKey',
]);
SplitTicketSyncLogger::log('cron', 'scan start', [
'candidateCount' => $candidateCount,
@@ -134,6 +137,7 @@ class SplitTicketSyncService
$ticketId = (int) $ticket['id'];
$ticketUrl = (string) $ticket['ticket_url'];
$ticketType = (string) $ticket['ticket_type'];
$sessionKey = AntiBotConfigBuilder::resolveSessionKey($ticketType, $ticketUrl);
if ($count >= $maxPerRun) {
$reason = '本轮处理上限已满';
@@ -183,6 +187,7 @@ class SplitTicketSyncService
'ticketId' => $ticketId,
'ticketType' => $ticketType,
'ticketUrl' => $ticketUrl,
'sessionKey' => $sessionKey,
'success' => !empty($result['success']),
'message' => (string) ($result['message'] ?? ''),
];
@@ -190,8 +195,8 @@ class SplitTicketSyncService
}
$candidateIds = array_map(static function ($row) {
return (int) $row['id'];
}, $list instanceof \think\Collection ? $list->all() : (array) $list);
return (int) (is_array($row) ? ($row['id'] ?? 0) : ($row['id'] ?? 0));
}, $list);
$openNotSynced = $this->buildOpenNotSyncedAudit(
$autoSyncTypes,
$failThreshold,
@@ -573,4 +578,35 @@ class SplitTicketSyncService
$skip = $this->shouldSkip(Ticket::get((int) $ticket['id']) ?: new Ticket($ticket));
return $skip ?? '未知原因，未进入执行队列';
}
/**
* 按 sessionKey（ticketType:host）分组排序，使同域名工单连续执行以命中 Browser 温池
*
* @param \think\Collection|array<int, Ticket|array<string, mixed>> $list
* @return list<Ticket|array<string, mixed>>
*/
private function sortTicketsBySessionKey($list): array
{
$rows = $list instanceof \think\Collection ? $list->all() : (array) $list;
usort($rows, static function ($a, $b): int {
$typeA = (string) (is_array($a) ? ($a['ticket_type'] ?? '') : ($a['ticket_type'] ?? ''));
$urlA = (string) (is_array($a) ? ($a['ticket_url'] ?? '') : ($a['ticket_url'] ?? ''));
$typeB = (string) (is_array($b) ? ($b['ticket_type'] ?? '') : ($b['ticket_type'] ?? ''));
$urlB = (string) (is_array($b) ? ($b['ticket_url'] ?? '') : ($b['ticket_url'] ?? ''));
$keyA = AntiBotConfigBuilder::resolveSessionKey($typeA, $urlA);
$keyB = AntiBotConfigBuilder::resolveSessionKey($typeB, $urlB);
if ($keyA === $keyB) {
$timeA = (int) (is_array($a) ? ($a['sync_time'] ?? 0) : ($a['sync_time'] ?? 0));
$timeB = (int) (is_array($b) ? ($b['sync_time'] ?? 0) : ($b['sync_time'] ?? 0));
if ($timeA === $timeB) {
$idA = (int) (is_array($a) ? ($a['id'] ?? 0) : ($a['id'] ?? 0));
$idB = (int) (is_array($b) ? ($b['id'] ?? 0) : ($b['id'] ?? 0));
return $idA <=> $idB;
}
return $timeA <=> $timeB;
}
return strcmp($keyA, $keyB);
});
return $rows;
}
}