多个A2C任务时防止同时同步引起错误
This commit is contained in:
@@ -4,6 +4,7 @@ declare(strict_types=1);
|
|||||||
|
|
||||||
namespace app\common\library\scrm;
|
namespace app\common\library\scrm;
|
||||||
|
|
||||||
|
use app\common\service\SplitPageUrlResolver;
|
||||||
use think\Config;
|
use think\Config;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -26,6 +27,10 @@ class AntiBotConfigBuilder
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if ($ticketType === 'a2c' && $landingUrlHint === '') {
|
||||||
|
$landingUrlHint = SplitPageUrlResolver::resolveFinalUrl($pageUrl);
|
||||||
|
}
|
||||||
|
|
||||||
$sessionScope = self::resolveSessionScope($ticketType, $pageUrl, $landingUrlHint);
|
$sessionScope = self::resolveSessionScope($ticketType, $pageUrl, $landingUrlHint);
|
||||||
if ($sessionScope === '') {
|
if ($sessionScope === '') {
|
||||||
return null;
|
return null;
|
||||||
@@ -83,6 +88,10 @@ class AntiBotConfigBuilder
|
|||||||
*/
|
*/
|
||||||
public static function resolveSessionKey(string $ticketType, string $pageUrl, string $landingUrlHint = ''): string
|
public static function resolveSessionKey(string $ticketType, string $pageUrl, string $landingUrlHint = ''): string
|
||||||
{
|
{
|
||||||
|
if ($ticketType === 'a2c' && $landingUrlHint === '') {
|
||||||
|
$landingUrlHint = SplitPageUrlResolver::resolveFinalUrl($pageUrl);
|
||||||
|
}
|
||||||
|
|
||||||
$scope = self::resolveSessionScope($ticketType, $pageUrl, $landingUrlHint);
|
$scope = self::resolveSessionScope($ticketType, $pageUrl, $landingUrlHint);
|
||||||
|
|
||||||
return $ticketType . ':' . ($scope !== '' ? $scope : 'unknown');
|
return $ticketType . ':' . ($scope !== '' ? $scope : 'unknown');
|
||||||
@@ -244,18 +253,41 @@ class AntiBotConfigBuilder
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* A2C 会话 scope:业务域 + 分享 id(避免多工单共用温池 Browser 互相干扰)
|
* A2C 会话 scope:业务域 + 分享 id(避免多工单共用温池 Browser 互相干扰)
|
||||||
|
*
|
||||||
|
* 短链先 HTTP 预解析再提取 ?id=,确保不同分享页不会落到同一 sessionKey。
|
||||||
*/
|
*/
|
||||||
private static function resolveA2cSessionScope(string $pageUrl, string $landingUrlHint = ''): string
|
private static function resolveA2cSessionScope(string $pageUrl, string $landingUrlHint = ''): string
|
||||||
{
|
{
|
||||||
$host = self::resolveA2cSessionHost($pageUrl, $landingUrlHint);
|
$resolvedUrl = self::resolveA2cResolvedUrl($pageUrl, $landingUrlHint);
|
||||||
|
$host = self::resolveA2cSessionHost($pageUrl, $resolvedUrl);
|
||||||
$shareId = self::extractA2cShareId($pageUrl);
|
$shareId = self::extractA2cShareId($pageUrl);
|
||||||
if ($shareId === '' && $landingUrlHint !== '') {
|
if ($shareId === '') {
|
||||||
$shareId = self::extractA2cShareId($landingUrlHint);
|
$shareId = self::extractA2cShareId($resolvedUrl);
|
||||||
}
|
}
|
||||||
|
|
||||||
return $shareId !== '' ? $host . ':' . $shareId : $host;
|
return $shareId !== '' ? $host . ':' . $shareId : $host;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A2C 短链 HTTP 预解析:优先已有 landingUrlHint,否则跟随重定向得到长链
|
||||||
|
*/
|
||||||
|
private static function resolveA2cResolvedUrl(string $pageUrl, string $landingUrlHint = ''): string
|
||||||
|
{
|
||||||
|
if ($landingUrlHint !== '' && self::extractA2cShareId($landingUrlHint) !== '') {
|
||||||
|
return $landingUrlHint;
|
||||||
|
}
|
||||||
|
if (self::extractA2cShareId($pageUrl) !== '') {
|
||||||
|
return $pageUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
$resolved = SplitPageUrlResolver::resolveFinalUrl($pageUrl);
|
||||||
|
if ($resolved !== '') {
|
||||||
|
return $resolved;
|
||||||
|
}
|
||||||
|
|
||||||
|
return $landingUrlHint !== '' ? $landingUrlHint : $pageUrl;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 从 A2C 分享 URL 提取 id 查询参数
|
* 从 A2C 分享 URL 提取 id 查询参数
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -37,6 +37,8 @@ class SplitNodeHealthService
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 当前是否适合向 Node 提交新 Browser 任务
|
* 当前是否适合向 Node 提交新 Browser 任务
|
||||||
|
*
|
||||||
|
* A2C 等 antiBot 任务走 real profile,应参考 queueReal 而非 standard 队列。
|
||||||
*/
|
*/
|
||||||
public static function canAcceptWork(): bool
|
public static function canAcceptWork(): bool
|
||||||
{
|
{
|
||||||
@@ -47,33 +49,61 @@ class SplitNodeHealthService
|
|||||||
if (!empty($stats['shuttingDown'])) {
|
if (!empty($stats['shuttingDown'])) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
$queue = is_array($stats['queue'] ?? null) ? $stats['queue'] : [];
|
|
||||||
$config = is_array($stats['config'] ?? null) ? $stats['config'] : [];
|
$snapshot = self::resolveQueueSnapshot($stats);
|
||||||
$queued = (int) ($queue['queued'] ?? 0);
|
|
||||||
$active = (int) ($queue['active'] ?? 0);
|
|
||||||
$max = max(1, (int) ($config['maxConcurrentBrowsers'] ?? ($queue['max'] ?? 4)));
|
|
||||||
$threshold = SplitSyncConfigService::getNodeQueueSkipThreshold();
|
$threshold = SplitSyncConfigService::getNodeQueueSkipThreshold();
|
||||||
if ($queued >= $threshold) {
|
if ($snapshot['queued'] >= $threshold) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return $active < $max;
|
|
||||||
|
return $snapshot['active'] < $snapshot['max'];
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return array{queued:int,active:int,max:int}
|
* @return array{queued:int,active:int,max:int,profile:string}
|
||||||
*/
|
*/
|
||||||
public static function getQueueSnapshot(): array
|
public static function getQueueSnapshot(): array
|
||||||
{
|
{
|
||||||
$stats = self::fetchStats();
|
$stats = self::fetchStats();
|
||||||
if ($stats === null) {
|
if ($stats === null) {
|
||||||
return ['queued' => 0, 'active' => 0, 'max' => 0];
|
return ['queued' => 0, 'active' => 0, 'max' => 0, 'profile' => 'real'];
|
||||||
}
|
}
|
||||||
$queue = is_array($stats['queue'] ?? null) ? $stats['queue'] : [];
|
|
||||||
|
$snapshot = self::resolveQueueSnapshot($stats);
|
||||||
|
return [
|
||||||
|
'queued' => $snapshot['queued'],
|
||||||
|
'active' => $snapshot['active'],
|
||||||
|
'max' => $snapshot['max'],
|
||||||
|
'profile' => $snapshot['profile'],
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 优先使用 real 队列(antiBot / A2C),无数据时回退 standard
|
||||||
|
*
|
||||||
|
* @param array<string, mixed> $stats
|
||||||
|
* @return array{queued:int,active:int,max:int,profile:string}
|
||||||
|
*/
|
||||||
|
private static function resolveQueueSnapshot(array $stats): array
|
||||||
|
{
|
||||||
$config = is_array($stats['config'] ?? null) ? $stats['config'] : [];
|
$config = is_array($stats['config'] ?? null) ? $stats['config'] : [];
|
||||||
|
$queueReal = is_array($stats['queueReal'] ?? null) ? $stats['queueReal'] : [];
|
||||||
|
if ($queueReal !== []) {
|
||||||
|
return [
|
||||||
|
'queued' => (int) ($queueReal['queued'] ?? 0),
|
||||||
|
'active' => (int) ($queueReal['active'] ?? 0),
|
||||||
|
'max' => max(1, (int) ($config['maxConcurrentBrowsersReal'] ?? ($queueReal['max'] ?? 2))),
|
||||||
|
'profile' => 'real',
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
$queue = is_array($stats['queue'] ?? null) ? $stats['queue'] : [];
|
||||||
|
|
||||||
return [
|
return [
|
||||||
'queued' => (int) ($queue['queued'] ?? 0),
|
'queued' => (int) ($queue['queued'] ?? 0),
|
||||||
'active' => (int) ($queue['active'] ?? 0),
|
'active' => (int) ($queue['active'] ?? 0),
|
||||||
'max' => max(1, (int) ($config['maxConcurrentBrowsers'] ?? ($queue['max'] ?? 4))),
|
'max' => max(1, (int) ($config['maxConcurrentBrowsers'] ?? ($queue['max'] ?? 4))),
|
||||||
|
'profile' => 'standard',
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,11 +4,15 @@ declare(strict_types=1);
|
|||||||
|
|
||||||
namespace app\common\service;
|
namespace app\common\service;
|
||||||
|
|
||||||
|
use think\Db;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 工单手工同步 CLI 后台投递
|
* 工单手工同步 CLI 后台投递
|
||||||
*
|
*
|
||||||
* Web 请求仅负责鉴权与投递,实际 Spider 在独立 CLI 进程中执行,
|
* Web 请求仅负责鉴权与投递,实际 Spider 在独立 CLI 进程中执行,
|
||||||
* 避免长时间占用 PHP-FPM 与 Session 锁导致后台其它页面无法打开。
|
* 避免长时间占用 PHP-FPM 与 Session 锁导致后台其它页面无法打开。
|
||||||
|
*
|
||||||
|
* Node 类型(A2C / Whatshub 等)在同一 nohup 子 shell 内串行执行,避免并行 Chrome 压垮温池。
|
||||||
*/
|
*/
|
||||||
class SplitTicketSyncDispatchService
|
class SplitTicketSyncDispatchService
|
||||||
{
|
{
|
||||||
@@ -28,6 +32,13 @@ class SplitTicketSyncDispatchService
|
|||||||
$think = $this->resolveThinkScript();
|
$think = $this->resolveThinkScript();
|
||||||
$root = $this->resolveRootPath();
|
$root = $this->resolveRootPath();
|
||||||
$lockService = new SplitTicketSyncLockService();
|
$lockService = new SplitTicketSyncLockService();
|
||||||
|
$nodeTypeMap = array_flip(SplitScrmSpiderFactory::listNodeSyncTypes());
|
||||||
|
$typeMap = $this->loadTicketTypeMap($ticketIds);
|
||||||
|
|
||||||
|
/** @var int[] $nodeTicketIds */
|
||||||
|
$nodeTicketIds = [];
|
||||||
|
/** @var int[] $directTicketIds */
|
||||||
|
$directTicketIds = [];
|
||||||
|
|
||||||
foreach ($ticketIds as $rawId) {
|
foreach ($ticketIds as $rawId) {
|
||||||
$ticketId = (int) $rawId;
|
$ticketId = (int) $rawId;
|
||||||
@@ -40,6 +51,15 @@ class SplitTicketSyncDispatchService
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$ticketType = (string) ($typeMap[$ticketId] ?? '');
|
||||||
|
if ($ticketType !== '' && isset($nodeTypeMap[$ticketType])) {
|
||||||
|
$nodeTicketIds[] = $ticketId;
|
||||||
|
} else {
|
||||||
|
$directTicketIds[] = $ticketId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach ($directTicketIds as $ticketId) {
|
||||||
if ($this->spawnCli($php, $think, $root, $ticketId)) {
|
if ($this->spawnCli($php, $think, $root, $ticketId)) {
|
||||||
$queued[] = $ticketId;
|
$queued[] = $ticketId;
|
||||||
} else {
|
} else {
|
||||||
@@ -47,10 +67,25 @@ class SplitTicketSyncDispatchService
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if ($nodeTicketIds !== []) {
|
||||||
|
$spawned = $this->spawnCliSequential($php, $think, $root, $nodeTicketIds);
|
||||||
|
if ($spawned) {
|
||||||
|
foreach ($nodeTicketIds as $ticketId) {
|
||||||
|
$queued[] = $ticketId;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
foreach ($nodeTicketIds as $ticketId) {
|
||||||
|
$failed[] = $ticketId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
SplitTicketSyncLogger::log('web', 'manual sync dispatched', [
|
SplitTicketSyncLogger::log('web', 'manual sync dispatched', [
|
||||||
'queued' => $queued,
|
'queued' => $queued,
|
||||||
'skipped' => $skipped,
|
'skipped' => $skipped,
|
||||||
'failed' => $failed,
|
'failed' => $failed,
|
||||||
|
'nodeSerial' => $nodeTicketIds,
|
||||||
|
'directParallel' => $directTicketIds,
|
||||||
'phpCli' => $php,
|
'phpCli' => $php,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
@@ -82,7 +117,37 @@ class SplitTicketSyncDispatchService
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 后台启动 split:sync-tickets CLI
|
* @param int[] $ticketIds
|
||||||
|
* @return array<int, string> ticketId => ticket_type
|
||||||
|
*/
|
||||||
|
private function loadTicketTypeMap(array $ticketIds): array
|
||||||
|
{
|
||||||
|
$ids = [];
|
||||||
|
foreach ($ticketIds as $rawId) {
|
||||||
|
$ticketId = (int) $rawId;
|
||||||
|
if ($ticketId > 0) {
|
||||||
|
$ids[] = $ticketId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ($ids === []) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
$rows = Db::name('split_ticket')
|
||||||
|
->where('id', 'in', $ids)
|
||||||
|
->field('id,ticket_type')
|
||||||
|
->select();
|
||||||
|
|
||||||
|
$map = [];
|
||||||
|
foreach ($rows as $row) {
|
||||||
|
$map[(int) $row['id']] = (string) ($row['ticket_type'] ?? '');
|
||||||
|
}
|
||||||
|
|
||||||
|
return $map;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 后台启动 split:sync-tickets CLI(单工单,并行投递)
|
||||||
*/
|
*/
|
||||||
private function spawnCli(string $php, string $think, string $root, int $ticketId): bool
|
private function spawnCli(string $php, string $think, string $root, int $ticketId): bool
|
||||||
{
|
{
|
||||||
@@ -97,16 +162,61 @@ class SplitTicketSyncDispatchService
|
|||||||
escapeshellarg($logFile)
|
escapeshellarg($logFile)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
return $this->runBackgroundCommand($root, $inner, $ticketId, 'parallel');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Node 类型工单:单条 nohup 内串行执行多个 CLI,避免并行 Chrome
|
||||||
|
*
|
||||||
|
* @param int[] $ticketIds
|
||||||
|
*/
|
||||||
|
private function spawnCliSequential(string $php, string $think, string $root, array $ticketIds): bool
|
||||||
|
{
|
||||||
|
if ($ticketIds === []) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
$logDir = $this->resolveLogDir();
|
||||||
|
$parts = [];
|
||||||
|
foreach ($ticketIds as $ticketId) {
|
||||||
|
$ticketId = (int) $ticketId;
|
||||||
|
if ($ticketId <= 0) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
$logFile = $logDir . 'cli_' . $ticketId . '_' . date('YmdHis') . '.log';
|
||||||
|
$parts[] = sprintf(
|
||||||
|
'%s %s split:sync-tickets --ticket=%d >> %s 2>&1',
|
||||||
|
escapeshellarg($php),
|
||||||
|
escapeshellarg($think),
|
||||||
|
$ticketId,
|
||||||
|
escapeshellarg($logFile)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if ($parts === []) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
$inner = implode('; ', $parts);
|
||||||
|
|
||||||
|
return $this->runBackgroundCommand($root, $inner, $ticketIds, 'node-serial');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param int|int[] $ticketRef 日志用 ticketId 或 ID 列表
|
||||||
|
*/
|
||||||
|
private function runBackgroundCommand(string $root, string $inner, $ticketRef, string $mode): bool
|
||||||
|
{
|
||||||
if ($this->isWindows()) {
|
if ($this->isWindows()) {
|
||||||
$command = sprintf('start /B cmd /C %s', $inner);
|
$command = sprintf('start /B cmd /C %s', $inner);
|
||||||
} else {
|
} else {
|
||||||
$command = sprintf('cd %s && nohup %s &', escapeshellarg($root), $inner);
|
$command = sprintf('cd %s && nohup bash -c %s &', escapeshellarg($root), escapeshellarg($inner));
|
||||||
}
|
}
|
||||||
|
|
||||||
if ($this->canUseExec()) {
|
if ($this->canUseExec()) {
|
||||||
@exec($command, $output, $exitCode);
|
@exec($command, $output, $exitCode);
|
||||||
SplitTicketSyncLogger::log('web', 'cli spawn exec', [
|
SplitTicketSyncLogger::log('web', 'cli spawn exec', [
|
||||||
'ticketId' => $ticketId,
|
'ticketRef' => $ticketRef,
|
||||||
|
'mode' => $mode,
|
||||||
'command' => $command,
|
'command' => $command,
|
||||||
'exitCode' => $exitCode,
|
'exitCode' => $exitCode,
|
||||||
]);
|
]);
|
||||||
@@ -116,14 +226,16 @@ class SplitTicketSyncDispatchService
|
|||||||
if ($this->canUseShellExec()) {
|
if ($this->canUseShellExec()) {
|
||||||
@shell_exec($command);
|
@shell_exec($command);
|
||||||
SplitTicketSyncLogger::log('web', 'cli spawn shell_exec', [
|
SplitTicketSyncLogger::log('web', 'cli spawn shell_exec', [
|
||||||
'ticketId' => $ticketId,
|
'ticketRef' => $ticketRef,
|
||||||
|
'mode' => $mode,
|
||||||
'command' => $command,
|
'command' => $command,
|
||||||
]);
|
]);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
SplitTicketSyncLogger::log('web', 'cli spawn failed: exec disabled', [
|
SplitTicketSyncLogger::log('web', 'cli spawn failed: exec disabled', [
|
||||||
'ticketId' => $ticketId,
|
'ticketRef' => $ticketRef,
|
||||||
|
'mode' => $mode,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ declare(strict_types=1);
|
|||||||
|
|
||||||
namespace app\common\library\scrm;
|
namespace app\common\library\scrm;
|
||||||
|
|
||||||
|
use app\common\service\SplitPageUrlResolver;
|
||||||
use think\Config;
|
use think\Config;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -26,6 +27,10 @@ class AntiBotConfigBuilder
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if ($ticketType === 'a2c' && $landingUrlHint === '') {
|
||||||
|
$landingUrlHint = SplitPageUrlResolver::resolveFinalUrl($pageUrl);
|
||||||
|
}
|
||||||
|
|
||||||
$sessionScope = self::resolveSessionScope($ticketType, $pageUrl, $landingUrlHint);
|
$sessionScope = self::resolveSessionScope($ticketType, $pageUrl, $landingUrlHint);
|
||||||
if ($sessionScope === '') {
|
if ($sessionScope === '') {
|
||||||
return null;
|
return null;
|
||||||
@@ -83,6 +88,10 @@ class AntiBotConfigBuilder
|
|||||||
*/
|
*/
|
||||||
public static function resolveSessionKey(string $ticketType, string $pageUrl, string $landingUrlHint = ''): string
|
public static function resolveSessionKey(string $ticketType, string $pageUrl, string $landingUrlHint = ''): string
|
||||||
{
|
{
|
||||||
|
if ($ticketType === 'a2c' && $landingUrlHint === '') {
|
||||||
|
$landingUrlHint = SplitPageUrlResolver::resolveFinalUrl($pageUrl);
|
||||||
|
}
|
||||||
|
|
||||||
$scope = self::resolveSessionScope($ticketType, $pageUrl, $landingUrlHint);
|
$scope = self::resolveSessionScope($ticketType, $pageUrl, $landingUrlHint);
|
||||||
|
|
||||||
return $ticketType . ':' . ($scope !== '' ? $scope : 'unknown');
|
return $ticketType . ':' . ($scope !== '' ? $scope : 'unknown');
|
||||||
@@ -244,18 +253,41 @@ class AntiBotConfigBuilder
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* A2C 会话 scope:业务域 + 分享 id(避免多工单共用温池 Browser 互相干扰)
|
* A2C 会话 scope:业务域 + 分享 id(避免多工单共用温池 Browser 互相干扰)
|
||||||
|
*
|
||||||
|
* 短链先 HTTP 预解析再提取 ?id=,确保不同分享页不会落到同一 sessionKey。
|
||||||
*/
|
*/
|
||||||
private static function resolveA2cSessionScope(string $pageUrl, string $landingUrlHint = ''): string
|
private static function resolveA2cSessionScope(string $pageUrl, string $landingUrlHint = ''): string
|
||||||
{
|
{
|
||||||
$host = self::resolveA2cSessionHost($pageUrl, $landingUrlHint);
|
$resolvedUrl = self::resolveA2cResolvedUrl($pageUrl, $landingUrlHint);
|
||||||
|
$host = self::resolveA2cSessionHost($pageUrl, $resolvedUrl);
|
||||||
$shareId = self::extractA2cShareId($pageUrl);
|
$shareId = self::extractA2cShareId($pageUrl);
|
||||||
if ($shareId === '' && $landingUrlHint !== '') {
|
if ($shareId === '') {
|
||||||
$shareId = self::extractA2cShareId($landingUrlHint);
|
$shareId = self::extractA2cShareId($resolvedUrl);
|
||||||
}
|
}
|
||||||
|
|
||||||
return $shareId !== '' ? $host . ':' . $shareId : $host;
|
return $shareId !== '' ? $host . ':' . $shareId : $host;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A2C 短链 HTTP 预解析:优先已有 landingUrlHint,否则跟随重定向得到长链
|
||||||
|
*/
|
||||||
|
private static function resolveA2cResolvedUrl(string $pageUrl, string $landingUrlHint = ''): string
|
||||||
|
{
|
||||||
|
if ($landingUrlHint !== '' && self::extractA2cShareId($landingUrlHint) !== '') {
|
||||||
|
return $landingUrlHint;
|
||||||
|
}
|
||||||
|
if (self::extractA2cShareId($pageUrl) !== '') {
|
||||||
|
return $pageUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
$resolved = SplitPageUrlResolver::resolveFinalUrl($pageUrl);
|
||||||
|
if ($resolved !== '') {
|
||||||
|
return $resolved;
|
||||||
|
}
|
||||||
|
|
||||||
|
return $landingUrlHint !== '' ? $landingUrlHint : $pageUrl;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 从 A2C 分享 URL 提取 id 查询参数
|
* 从 A2C 分享 URL 提取 id 查询参数
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -37,6 +37,8 @@ class SplitNodeHealthService
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 当前是否适合向 Node 提交新 Browser 任务
|
* 当前是否适合向 Node 提交新 Browser 任务
|
||||||
|
*
|
||||||
|
* A2C 等 antiBot 任务走 real profile,应参考 queueReal 而非 standard 队列。
|
||||||
*/
|
*/
|
||||||
public static function canAcceptWork(): bool
|
public static function canAcceptWork(): bool
|
||||||
{
|
{
|
||||||
@@ -47,33 +49,61 @@ class SplitNodeHealthService
|
|||||||
if (!empty($stats['shuttingDown'])) {
|
if (!empty($stats['shuttingDown'])) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
$queue = is_array($stats['queue'] ?? null) ? $stats['queue'] : [];
|
|
||||||
$config = is_array($stats['config'] ?? null) ? $stats['config'] : [];
|
$snapshot = self::resolveQueueSnapshot($stats);
|
||||||
$queued = (int) ($queue['queued'] ?? 0);
|
|
||||||
$active = (int) ($queue['active'] ?? 0);
|
|
||||||
$max = max(1, (int) ($config['maxConcurrentBrowsers'] ?? ($queue['max'] ?? 4)));
|
|
||||||
$threshold = SplitSyncConfigService::getNodeQueueSkipThreshold();
|
$threshold = SplitSyncConfigService::getNodeQueueSkipThreshold();
|
||||||
if ($queued >= $threshold) {
|
if ($snapshot['queued'] >= $threshold) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return $active < $max;
|
|
||||||
|
return $snapshot['active'] < $snapshot['max'];
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return array{queued:int,active:int,max:int}
|
* @return array{queued:int,active:int,max:int,profile:string}
|
||||||
*/
|
*/
|
||||||
public static function getQueueSnapshot(): array
|
public static function getQueueSnapshot(): array
|
||||||
{
|
{
|
||||||
$stats = self::fetchStats();
|
$stats = self::fetchStats();
|
||||||
if ($stats === null) {
|
if ($stats === null) {
|
||||||
return ['queued' => 0, 'active' => 0, 'max' => 0];
|
return ['queued' => 0, 'active' => 0, 'max' => 0, 'profile' => 'real'];
|
||||||
}
|
}
|
||||||
$queue = is_array($stats['queue'] ?? null) ? $stats['queue'] : [];
|
|
||||||
|
$snapshot = self::resolveQueueSnapshot($stats);
|
||||||
|
return [
|
||||||
|
'queued' => $snapshot['queued'],
|
||||||
|
'active' => $snapshot['active'],
|
||||||
|
'max' => $snapshot['max'],
|
||||||
|
'profile' => $snapshot['profile'],
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 优先使用 real 队列(antiBot / A2C),无数据时回退 standard
|
||||||
|
*
|
||||||
|
* @param array<string, mixed> $stats
|
||||||
|
* @return array{queued:int,active:int,max:int,profile:string}
|
||||||
|
*/
|
||||||
|
private static function resolveQueueSnapshot(array $stats): array
|
||||||
|
{
|
||||||
$config = is_array($stats['config'] ?? null) ? $stats['config'] : [];
|
$config = is_array($stats['config'] ?? null) ? $stats['config'] : [];
|
||||||
|
$queueReal = is_array($stats['queueReal'] ?? null) ? $stats['queueReal'] : [];
|
||||||
|
if ($queueReal !== []) {
|
||||||
|
return [
|
||||||
|
'queued' => (int) ($queueReal['queued'] ?? 0),
|
||||||
|
'active' => (int) ($queueReal['active'] ?? 0),
|
||||||
|
'max' => max(1, (int) ($config['maxConcurrentBrowsersReal'] ?? ($queueReal['max'] ?? 2))),
|
||||||
|
'profile' => 'real',
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
$queue = is_array($stats['queue'] ?? null) ? $stats['queue'] : [];
|
||||||
|
|
||||||
return [
|
return [
|
||||||
'queued' => (int) ($queue['queued'] ?? 0),
|
'queued' => (int) ($queue['queued'] ?? 0),
|
||||||
'active' => (int) ($queue['active'] ?? 0),
|
'active' => (int) ($queue['active'] ?? 0),
|
||||||
'max' => max(1, (int) ($config['maxConcurrentBrowsers'] ?? ($queue['max'] ?? 4))),
|
'max' => max(1, (int) ($config['maxConcurrentBrowsers'] ?? ($queue['max'] ?? 4))),
|
||||||
|
'profile' => 'standard',
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,11 +4,15 @@ declare(strict_types=1);
|
|||||||
|
|
||||||
namespace app\common\service;
|
namespace app\common\service;
|
||||||
|
|
||||||
|
use think\Db;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 工单手工同步 CLI 后台投递
|
* 工单手工同步 CLI 后台投递
|
||||||
*
|
*
|
||||||
* Web 请求仅负责鉴权与投递,实际 Spider 在独立 CLI 进程中执行,
|
* Web 请求仅负责鉴权与投递,实际 Spider 在独立 CLI 进程中执行,
|
||||||
* 避免长时间占用 PHP-FPM 与 Session 锁导致后台其它页面无法打开。
|
* 避免长时间占用 PHP-FPM 与 Session 锁导致后台其它页面无法打开。
|
||||||
|
*
|
||||||
|
* Node 类型(A2C / Whatshub 等)在同一 nohup 子 shell 内串行执行,避免并行 Chrome 压垮温池。
|
||||||
*/
|
*/
|
||||||
class SplitTicketSyncDispatchService
|
class SplitTicketSyncDispatchService
|
||||||
{
|
{
|
||||||
@@ -28,6 +32,13 @@ class SplitTicketSyncDispatchService
|
|||||||
$think = $this->resolveThinkScript();
|
$think = $this->resolveThinkScript();
|
||||||
$root = $this->resolveRootPath();
|
$root = $this->resolveRootPath();
|
||||||
$lockService = new SplitTicketSyncLockService();
|
$lockService = new SplitTicketSyncLockService();
|
||||||
|
$nodeTypeMap = array_flip(SplitScrmSpiderFactory::listNodeSyncTypes());
|
||||||
|
$typeMap = $this->loadTicketTypeMap($ticketIds);
|
||||||
|
|
||||||
|
/** @var int[] $nodeTicketIds */
|
||||||
|
$nodeTicketIds = [];
|
||||||
|
/** @var int[] $directTicketIds */
|
||||||
|
$directTicketIds = [];
|
||||||
|
|
||||||
foreach ($ticketIds as $rawId) {
|
foreach ($ticketIds as $rawId) {
|
||||||
$ticketId = (int) $rawId;
|
$ticketId = (int) $rawId;
|
||||||
@@ -40,6 +51,15 @@ class SplitTicketSyncDispatchService
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$ticketType = (string) ($typeMap[$ticketId] ?? '');
|
||||||
|
if ($ticketType !== '' && isset($nodeTypeMap[$ticketType])) {
|
||||||
|
$nodeTicketIds[] = $ticketId;
|
||||||
|
} else {
|
||||||
|
$directTicketIds[] = $ticketId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach ($directTicketIds as $ticketId) {
|
||||||
if ($this->spawnCli($php, $think, $root, $ticketId)) {
|
if ($this->spawnCli($php, $think, $root, $ticketId)) {
|
||||||
$queued[] = $ticketId;
|
$queued[] = $ticketId;
|
||||||
} else {
|
} else {
|
||||||
@@ -47,10 +67,25 @@ class SplitTicketSyncDispatchService
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if ($nodeTicketIds !== []) {
|
||||||
|
$spawned = $this->spawnCliSequential($php, $think, $root, $nodeTicketIds);
|
||||||
|
if ($spawned) {
|
||||||
|
foreach ($nodeTicketIds as $ticketId) {
|
||||||
|
$queued[] = $ticketId;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
foreach ($nodeTicketIds as $ticketId) {
|
||||||
|
$failed[] = $ticketId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
SplitTicketSyncLogger::log('web', 'manual sync dispatched', [
|
SplitTicketSyncLogger::log('web', 'manual sync dispatched', [
|
||||||
'queued' => $queued,
|
'queued' => $queued,
|
||||||
'skipped' => $skipped,
|
'skipped' => $skipped,
|
||||||
'failed' => $failed,
|
'failed' => $failed,
|
||||||
|
'nodeSerial' => $nodeTicketIds,
|
||||||
|
'directParallel' => $directTicketIds,
|
||||||
'phpCli' => $php,
|
'phpCli' => $php,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
@@ -82,7 +117,37 @@ class SplitTicketSyncDispatchService
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 后台启动 split:sync-tickets CLI
|
* @param int[] $ticketIds
|
||||||
|
* @return array<int, string> ticketId => ticket_type
|
||||||
|
*/
|
||||||
|
private function loadTicketTypeMap(array $ticketIds): array
|
||||||
|
{
|
||||||
|
$ids = [];
|
||||||
|
foreach ($ticketIds as $rawId) {
|
||||||
|
$ticketId = (int) $rawId;
|
||||||
|
if ($ticketId > 0) {
|
||||||
|
$ids[] = $ticketId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ($ids === []) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
$rows = Db::name('split_ticket')
|
||||||
|
->where('id', 'in', $ids)
|
||||||
|
->field('id,ticket_type')
|
||||||
|
->select();
|
||||||
|
|
||||||
|
$map = [];
|
||||||
|
foreach ($rows as $row) {
|
||||||
|
$map[(int) $row['id']] = (string) ($row['ticket_type'] ?? '');
|
||||||
|
}
|
||||||
|
|
||||||
|
return $map;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 后台启动 split:sync-tickets CLI(单工单,并行投递)
|
||||||
*/
|
*/
|
||||||
private function spawnCli(string $php, string $think, string $root, int $ticketId): bool
|
private function spawnCli(string $php, string $think, string $root, int $ticketId): bool
|
||||||
{
|
{
|
||||||
@@ -97,16 +162,61 @@ class SplitTicketSyncDispatchService
|
|||||||
escapeshellarg($logFile)
|
escapeshellarg($logFile)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
return $this->runBackgroundCommand($root, $inner, $ticketId, 'parallel');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Node 类型工单:单条 nohup 内串行执行多个 CLI,避免并行 Chrome
|
||||||
|
*
|
||||||
|
* @param int[] $ticketIds
|
||||||
|
*/
|
||||||
|
private function spawnCliSequential(string $php, string $think, string $root, array $ticketIds): bool
|
||||||
|
{
|
||||||
|
if ($ticketIds === []) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
$logDir = $this->resolveLogDir();
|
||||||
|
$parts = [];
|
||||||
|
foreach ($ticketIds as $ticketId) {
|
||||||
|
$ticketId = (int) $ticketId;
|
||||||
|
if ($ticketId <= 0) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
$logFile = $logDir . 'cli_' . $ticketId . '_' . date('YmdHis') . '.log';
|
||||||
|
$parts[] = sprintf(
|
||||||
|
'%s %s split:sync-tickets --ticket=%d >> %s 2>&1',
|
||||||
|
escapeshellarg($php),
|
||||||
|
escapeshellarg($think),
|
||||||
|
$ticketId,
|
||||||
|
escapeshellarg($logFile)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if ($parts === []) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
$inner = implode('; ', $parts);
|
||||||
|
|
||||||
|
return $this->runBackgroundCommand($root, $inner, $ticketIds, 'node-serial');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param int|int[] $ticketRef 日志用 ticketId 或 ID 列表
|
||||||
|
*/
|
||||||
|
private function runBackgroundCommand(string $root, string $inner, $ticketRef, string $mode): bool
|
||||||
|
{
|
||||||
if ($this->isWindows()) {
|
if ($this->isWindows()) {
|
||||||
$command = sprintf('start /B cmd /C %s', $inner);
|
$command = sprintf('start /B cmd /C %s', $inner);
|
||||||
} else {
|
} else {
|
||||||
$command = sprintf('cd %s && nohup %s &', escapeshellarg($root), $inner);
|
$command = sprintf('cd %s && nohup bash -c %s &', escapeshellarg($root), escapeshellarg($inner));
|
||||||
}
|
}
|
||||||
|
|
||||||
if ($this->canUseExec()) {
|
if ($this->canUseExec()) {
|
||||||
@exec($command, $output, $exitCode);
|
@exec($command, $output, $exitCode);
|
||||||
SplitTicketSyncLogger::log('web', 'cli spawn exec', [
|
SplitTicketSyncLogger::log('web', 'cli spawn exec', [
|
||||||
'ticketId' => $ticketId,
|
'ticketRef' => $ticketRef,
|
||||||
|
'mode' => $mode,
|
||||||
'command' => $command,
|
'command' => $command,
|
||||||
'exitCode' => $exitCode,
|
'exitCode' => $exitCode,
|
||||||
]);
|
]);
|
||||||
@@ -116,14 +226,16 @@ class SplitTicketSyncDispatchService
|
|||||||
if ($this->canUseShellExec()) {
|
if ($this->canUseShellExec()) {
|
||||||
@shell_exec($command);
|
@shell_exec($command);
|
||||||
SplitTicketSyncLogger::log('web', 'cli spawn shell_exec', [
|
SplitTicketSyncLogger::log('web', 'cli spawn shell_exec', [
|
||||||
'ticketId' => $ticketId,
|
'ticketRef' => $ticketRef,
|
||||||
|
'mode' => $mode,
|
||||||
'command' => $command,
|
'command' => $command,
|
||||||
]);
|
]);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
SplitTicketSyncLogger::log('web', 'cli spawn failed: exec disabled', [
|
SplitTicketSyncLogger::log('web', 'cli spawn failed: exec disabled', [
|
||||||
'ticketId' => $ticketId,
|
'ticketRef' => $ticketRef,
|
||||||
|
'mode' => $mode,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
|
|||||||
+171
-13
@@ -514,16 +514,24 @@ async function ensureCaptchaProxyBrowserIfNeeded(browserCtx) {
|
|||||||
extraArgs,
|
extraArgs,
|
||||||
sessionKey: antiBot.sessionKey || '',
|
sessionKey: antiBot.sessionKey || '',
|
||||||
});
|
});
|
||||||
|
tagBrowserSession(browserSession, profile, antiBot.sessionKey || '', extraArgs);
|
||||||
browserSession.captchaProxyActive = true;
|
browserSession.captchaProxyActive = true;
|
||||||
|
|
||||||
|
const proxySessionRef = { current: browserSession };
|
||||||
page = await createTaskPage(
|
page = await createTaskPage(
|
||||||
browserSession.browser,
|
browserSession.browser,
|
||||||
browserSession.reused ? null : browserSession.page,
|
browserSession.reused ? null : browserSession.page,
|
||||||
navigationUrl,
|
navigationUrl,
|
||||||
antiBot,
|
antiBot,
|
||||||
mergedCookies.length > 0 ? mergedCookies : snapshotCookies,
|
mergedCookies.length > 0 ? mergedCookies : snapshotCookies,
|
||||||
{ ...pageExtraOpts, useCaptchaProxy: true }
|
{
|
||||||
|
...pageExtraOpts,
|
||||||
|
useCaptchaProxy: true,
|
||||||
|
browserSession,
|
||||||
|
browserSessionRef: proxySessionRef,
|
||||||
|
}
|
||||||
);
|
);
|
||||||
|
browserSession = proxySessionRef.current;
|
||||||
|
|
||||||
if (antiBot?.enabled) {
|
if (antiBot?.enabled) {
|
||||||
attachSitekeyCapture(page, browserSession.browser);
|
attachSitekeyCapture(page, browserSession.browser);
|
||||||
@@ -541,28 +549,95 @@ async function ensureCaptchaProxyBrowserIfNeeded(browserCtx) {
|
|||||||
browserCtx.page = page;
|
browserCtx.page = page;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 任务结束关闭本次 Tab(温池复用 Browser 时保留 Browser 实例) */
|
/**
|
||||||
|
* 为温池复用 / newPage 重试附加元数据(profile、sessionKey、启动参数)
|
||||||
|
* @param {object} browserSession
|
||||||
|
* @param {'standard'|'real'} profile
|
||||||
|
* @param {string} sessionKey
|
||||||
|
* @param {string[]} launchExtraArgs
|
||||||
|
*/
|
||||||
|
function tagBrowserSession(browserSession, profile, sessionKey, launchExtraArgs = []) {
|
||||||
|
browserSession.profile = profile;
|
||||||
|
browserSession.sessionKey = sessionKey || '';
|
||||||
|
browserSession.launchExtraArgs = launchExtraArgs;
|
||||||
|
return browserSession;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @param {Error|unknown} err */
|
||||||
|
function isNewTabOpenError(err) {
|
||||||
|
const msg = String(err?.message || err || '').toLowerCase();
|
||||||
|
return msg.includes('target.createtarget') || msg.includes('failed to open a new tab');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 温池 Browser 若已无任何 Tab,销毁并冷启动(避免后续 newPage 失败)
|
||||||
|
* @param {object|null} browserSession
|
||||||
|
*/
|
||||||
|
async function relaunchBrowserSessionIfNoTabs(browserSession) {
|
||||||
|
if (!browserSession?.pooled || !browserSession.browser) {
|
||||||
|
return browserSession;
|
||||||
|
}
|
||||||
|
let pages = [];
|
||||||
|
try {
|
||||||
|
pages = await browserSession.browser.pages();
|
||||||
|
} catch (_) {
|
||||||
|
pages = [];
|
||||||
|
}
|
||||||
|
if (pages.length > 0) {
|
||||||
|
return browserSession;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.warn('[createTaskPage] 温池 Browser pages=0,销毁并冷启动');
|
||||||
|
await browserSession.release(true);
|
||||||
|
const relaunched = await acquireBrowserSession({
|
||||||
|
profile: browserSession.profile || 'real',
|
||||||
|
extraArgs: browserSession.launchExtraArgs || ['--mute-audio'],
|
||||||
|
sessionKey: browserSession.sessionKey || '',
|
||||||
|
});
|
||||||
|
tagBrowserSession(
|
||||||
|
relaunched,
|
||||||
|
browserSession.profile || relaunched.profile || 'real',
|
||||||
|
browserSession.sessionKey || '',
|
||||||
|
browserSession.launchExtraArgs || ['--mute-audio']
|
||||||
|
);
|
||||||
|
return relaunched;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 任务结束关闭本次 Tab(温池复用 Browser 时保留至少一个 Tab) */
|
||||||
async function closeTaskPage(page) {
|
async function closeTaskPage(page) {
|
||||||
if (!page) {
|
if (!page) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
if (typeof page.isClosed === 'function' && !page.isClosed()) {
|
if (typeof page.isClosed === 'function' && page.isClosed()) {
|
||||||
await page.close();
|
return;
|
||||||
}
|
}
|
||||||
|
const browser = typeof page.browser === 'function' ? page.browser() : null;
|
||||||
|
if (browser) {
|
||||||
|
const pages = await browser.pages().catch(() => []);
|
||||||
|
if (pages.length <= 1) {
|
||||||
|
await page.goto('about:blank', {
|
||||||
|
waitUntil: 'domcontentloaded',
|
||||||
|
timeout: Math.min(NAVIGATION_TIMEOUT_MS, 15000),
|
||||||
|
}).catch(() => {});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await page.close();
|
||||||
} catch (_) {
|
} catch (_) {
|
||||||
/* ignore */
|
/* ignore */
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 任务页创建(含 Whatshub skipUserAgent)
|
* 配置任务页(UA / Cookie / 事件监听)
|
||||||
* 温池复用 Browser 时每次 newPage,冷启动 Real Browser 可使用 launch 返回的首个 Tab
|
* @param {import('puppeteer').Page} page
|
||||||
|
* @param {import('puppeteer').Browser} browser
|
||||||
|
* @param {object} opts
|
||||||
|
* @param {object} extraOpts
|
||||||
|
* @param {object|null|undefined} antiBot
|
||||||
*/
|
*/
|
||||||
async function createTaskPage(browser, warmPage, pageUrl, antiBot, mergedCookies, extraOpts = {}) {
|
async function configureTaskPage(page, browser, opts, extraOpts, antiBot) {
|
||||||
const opts = buildManagedPageOptions(pageUrl, antiBot, mergedCookies, extraOpts);
|
|
||||||
const useWarmPage = !!(warmPage && extraOpts.useWarmPage !== false && !extraOpts.forceNewPage);
|
|
||||||
const page = useWarmPage ? warmPage : await browser.newPage();
|
|
||||||
page.setDefaultNavigationTimeout(NAVIGATION_TIMEOUT_MS);
|
page.setDefaultNavigationTimeout(NAVIGATION_TIMEOUT_MS);
|
||||||
page.setDefaultTimeout(PAGE_DEFAULT_TIMEOUT_MS);
|
page.setDefaultTimeout(PAGE_DEFAULT_TIMEOUT_MS);
|
||||||
|
|
||||||
@@ -586,7 +661,67 @@ async function createTaskPage(browser, warmPage, pageUrl, antiBot, mergedCookies
|
|||||||
if (antiBot?.enabled) {
|
if (antiBot?.enabled) {
|
||||||
attachSitekeyCapture(page, browser);
|
attachSitekeyCapture(page, browser);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务页创建(含 Whatshub skipUserAgent)
|
||||||
|
* 温池复用 Browser 时每次 newPage;失败则销毁温池条目并冷启动重试
|
||||||
|
*
|
||||||
|
* extraOpts.browserSession:传入时可触发 pages=0 重建与 newPage 失败重试
|
||||||
|
* extraOpts.browserSessionRef:可选 { current },重试后回写最新 session
|
||||||
|
*/
|
||||||
|
async function createTaskPage(browser, warmPage, pageUrl, antiBot, mergedCookies, extraOpts = {}) {
|
||||||
|
const opts = buildManagedPageOptions(pageUrl, antiBot, mergedCookies, extraOpts);
|
||||||
|
let browserSession = extraOpts.browserSession || null;
|
||||||
|
let currentBrowser = browser;
|
||||||
|
let currentWarmPage = warmPage;
|
||||||
|
|
||||||
|
if (browserSession) {
|
||||||
|
browserSession = await relaunchBrowserSessionIfNoTabs(browserSession);
|
||||||
|
currentBrowser = browserSession.browser;
|
||||||
|
currentWarmPage = browserSession.reused ? null : browserSession.page;
|
||||||
|
if (extraOpts.browserSessionRef) {
|
||||||
|
extraOpts.browserSessionRef.current = browserSession;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let page = null;
|
||||||
|
for (let attempt = 0; attempt < 2; attempt++) {
|
||||||
|
const useWarmPage = !!(currentWarmPage && extraOpts.useWarmPage !== false && !extraOpts.forceNewPage);
|
||||||
|
try {
|
||||||
|
page = useWarmPage ? currentWarmPage : await currentBrowser.newPage();
|
||||||
|
break;
|
||||||
|
} catch (err) {
|
||||||
|
if (attempt === 0 && browserSession && isNewTabOpenError(err)) {
|
||||||
|
console.warn('[createTaskPage] newPage 失败,销毁温池并冷启动重试:', err.message);
|
||||||
|
await browserSession.release(true);
|
||||||
|
browserSession = await acquireBrowserSession({
|
||||||
|
profile: browserSession.profile || (antiBot?.profile === 'real' ? 'real' : 'standard'),
|
||||||
|
extraArgs: browserSession.launchExtraArgs || ['--mute-audio'],
|
||||||
|
sessionKey: browserSession.sessionKey || antiBot?.sessionKey || '',
|
||||||
|
});
|
||||||
|
tagBrowserSession(
|
||||||
|
browserSession,
|
||||||
|
browserSession.profile || (antiBot?.profile === 'real' ? 'real' : 'standard'),
|
||||||
|
browserSession.sessionKey || antiBot?.sessionKey || '',
|
||||||
|
browserSession.launchExtraArgs || ['--mute-audio']
|
||||||
|
);
|
||||||
|
if (extraOpts.browserSessionRef) {
|
||||||
|
extraOpts.browserSessionRef.current = browserSession;
|
||||||
|
}
|
||||||
|
currentBrowser = browserSession.browser;
|
||||||
|
currentWarmPage = browserSession.reused ? null : browserSession.page;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!page) {
|
||||||
|
throw new Error('createTaskPage: 未能打开任务 Tab');
|
||||||
|
}
|
||||||
|
|
||||||
|
await configureTaskPage(page, currentBrowser, opts, extraOpts, antiBot);
|
||||||
return page;
|
return page;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1554,18 +1689,22 @@ app.post('/api/auth-and-intercept', async (req, res) => {
|
|||||||
extraArgs: ['--mute-audio'],
|
extraArgs: ['--mute-audio'],
|
||||||
sessionKey: antiBot.sessionKey || '',
|
sessionKey: antiBot.sessionKey || '',
|
||||||
});
|
});
|
||||||
|
tagBrowserSession(browserSession, profile, antiBot.sessionKey || '', ['--mute-audio']);
|
||||||
cfLog.browserReused = !!browserSession.reused;
|
cfLog.browserReused = !!browserSession.reused;
|
||||||
|
|
||||||
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl);
|
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl);
|
||||||
const navigationUrl = resolveNavigationUrl(pageUrl, antiBot);
|
const navigationUrl = resolveNavigationUrl(pageUrl, antiBot);
|
||||||
|
|
||||||
|
const authSessionRef = { current: browserSession };
|
||||||
page = await createTaskPage(
|
page = await createTaskPage(
|
||||||
browserSession.browser,
|
browserSession.browser,
|
||||||
browserSession.reused ? null : browserSession.page,
|
browserSession.reused ? null : browserSession.page,
|
||||||
navigationUrl,
|
navigationUrl,
|
||||||
antiBot,
|
antiBot,
|
||||||
mergedCookies
|
mergedCookies,
|
||||||
|
{ browserSession, browserSessionRef: authSessionRef }
|
||||||
);
|
);
|
||||||
|
browserSession = authSessionRef.current;
|
||||||
|
|
||||||
const httpResolvedUrl = await resolveHttpRedirects(pageUrl).catch(() => pageUrl);
|
const httpResolvedUrl = await resolveHttpRedirects(pageUrl).catch(() => pageUrl);
|
||||||
if (httpResolvedUrl !== pageUrl) {
|
if (httpResolvedUrl !== pageUrl) {
|
||||||
@@ -1840,17 +1979,24 @@ app.post('/api/ui-pagination', async (req, res) => {
|
|||||||
extraArgs: ['--window-size=1920,1080'],
|
extraArgs: ['--window-size=1920,1080'],
|
||||||
sessionKey: antiBot.sessionKey || '',
|
sessionKey: antiBot.sessionKey || '',
|
||||||
});
|
});
|
||||||
|
tagBrowserSession(browserSession, profile, antiBot.sessionKey || '', ['--window-size=1920,1080']);
|
||||||
|
|
||||||
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, navigationUrl, cookies || []);
|
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, navigationUrl, cookies || []);
|
||||||
|
|
||||||
|
const uiSessionRef = { current: browserSession };
|
||||||
page = await createTaskPage(
|
page = await createTaskPage(
|
||||||
browserSession.browser,
|
browserSession.browser,
|
||||||
browserSession.reused ? null : browserSession.page,
|
browserSession.reused ? null : browserSession.page,
|
||||||
navigationUrl,
|
navigationUrl,
|
||||||
antiBot,
|
antiBot,
|
||||||
mergedCookies,
|
mergedCookies,
|
||||||
{ viewport: { width: 1920, height: 1080 } }
|
{
|
||||||
|
viewport: { width: 1920, height: 1080 },
|
||||||
|
browserSession,
|
||||||
|
browserSessionRef: uiSessionRef,
|
||||||
|
}
|
||||||
);
|
);
|
||||||
|
browserSession = uiSessionRef.current;
|
||||||
|
|
||||||
const log = (msg) => { console.log(msg); debugLogs.push(msg); };
|
const log = (msg) => { console.log(msg); debugLogs.push(msg); };
|
||||||
log(`[启动] 准备访问页面: ${navigationUrl}`);
|
log(`[启动] 准备访问页面: ${navigationUrl}`);
|
||||||
@@ -1986,19 +2132,31 @@ app.post('/api/auth-intercept-and-paginate', async (req, res) => {
|
|||||||
extraArgs: ['--mute-audio', '--window-size=1920,1080'],
|
extraArgs: ['--mute-audio', '--window-size=1920,1080'],
|
||||||
sessionKey: antiBot.sessionKey || '',
|
sessionKey: antiBot.sessionKey || '',
|
||||||
});
|
});
|
||||||
|
tagBrowserSession(
|
||||||
|
browserSession,
|
||||||
|
profile,
|
||||||
|
antiBot.sessionKey || '',
|
||||||
|
['--mute-audio', '--window-size=1920,1080']
|
||||||
|
);
|
||||||
cfLog.browserReused = !!browserSession.reused;
|
cfLog.browserReused = !!browserSession.reused;
|
||||||
|
|
||||||
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl);
|
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl);
|
||||||
const navigationUrl = resolveNavigationUrl(pageUrl, antiBot);
|
const navigationUrl = resolveNavigationUrl(pageUrl, antiBot);
|
||||||
|
|
||||||
|
const mergedSessionRef = { current: browserSession };
|
||||||
page = await createTaskPage(
|
page = await createTaskPage(
|
||||||
browserSession.browser,
|
browserSession.browser,
|
||||||
browserSession.reused ? null : browserSession.page,
|
browserSession.reused ? null : browserSession.page,
|
||||||
navigationUrl,
|
navigationUrl,
|
||||||
antiBot,
|
antiBot,
|
||||||
mergedCookies,
|
mergedCookies,
|
||||||
{ viewport: { width: 1920, height: 1080 } }
|
{
|
||||||
|
viewport: { width: 1920, height: 1080 },
|
||||||
|
browserSession,
|
||||||
|
browserSessionRef: mergedSessionRef,
|
||||||
|
}
|
||||||
);
|
);
|
||||||
|
browserSession = mergedSessionRef.current;
|
||||||
|
|
||||||
const httpResolvedUrl = await resolveHttpRedirects(pageUrl).catch(() => pageUrl);
|
const httpResolvedUrl = await resolveHttpRedirects(pageUrl).catch(() => pageUrl);
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user