damon/links
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

多个A2C任务时防止同时同步引起错误

Browse Source
This commit is contained in:
root
2026-07-03 20:51:56 +08:00
parent 8508d51b66
commit 455b6669e8
7 changed files with 577 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
patches/application/common/library/scrm/AntiBotConfigBuilder.php
+35 -3
View File
@@ -4,6 +4,7 @@ declare(strict_types=1);
namespace app\common\library\scrm;
use app\common\service\SplitPageUrlResolver;
use think\Config;
/**
@@ -26,6 +27,10 @@ class AntiBotConfigBuilder
return null;
}
if ($ticketType === 'a2c' && $landingUrlHint === '') {
$landingUrlHint = SplitPageUrlResolver::resolveFinalUrl($pageUrl);
}
$sessionScope = self::resolveSessionScope($ticketType, $pageUrl, $landingUrlHint);
if ($sessionScope === '') {
return null;
@@ -83,6 +88,10 @@ class AntiBotConfigBuilder
*/
public static function resolveSessionKey(string $ticketType, string $pageUrl, string $landingUrlHint = ''): string
{
if ($ticketType === 'a2c' && $landingUrlHint === '') {
$landingUrlHint = SplitPageUrlResolver::resolveFinalUrl($pageUrl);
}
$scope = self::resolveSessionScope($ticketType, $pageUrl, $landingUrlHint);
return $ticketType . ':' . ($scope !== '' ? $scope : 'unknown');
@@ -244,18 +253,41 @@ class AntiBotConfigBuilder
/**
* A2C 会话 scope：业务域 + 分享 id（避免多工单共用温池 Browser 互相干扰）
*
* 短链先 HTTP 预解析再提取 ?id=，确保不同分享页不会落到同一 sessionKey。
*/
private static function resolveA2cSessionScope(string $pageUrl, string $landingUrlHint = ''): string
{
$host = self::resolveA2cSessionHost($pageUrl, $landingUrlHint);
$resolvedUrl = self::resolveA2cResolvedUrl($pageUrl, $landingUrlHint);
$host = self::resolveA2cSessionHost($pageUrl, $resolvedUrl);
$shareId = self::extractA2cShareId($pageUrl);
if ($shareId === '' && $landingUrlHint !== '') {
$shareId = self::extractA2cShareId($landingUrlHint);
if ($shareId === '') {
$shareId = self::extractA2cShareId($resolvedUrl);
}
return $shareId !== '' ? $host . ':' . $shareId : $host;
}
/**
* A2C 短链 HTTP 预解析：优先已有 landingUrlHint，否则跟随重定向得到长链
*/
private static function resolveA2cResolvedUrl(string $pageUrl, string $landingUrlHint = ''): string
{
if ($landingUrlHint !== '' && self::extractA2cShareId($landingUrlHint) !== '') {
return $landingUrlHint;
}
if (self::extractA2cShareId($pageUrl) !== '') {
return $pageUrl;
}
$resolved = SplitPageUrlResolver::resolveFinalUrl($pageUrl);
if ($resolved !== '') {
return $resolved;
}
return $landingUrlHint !== '' ? $landingUrlHint : $pageUrl;
}
/**
* 从 A2C 分享 URL 提取 id 查询参数
*/
patches/application/common/service/SplitNodeHealthService.php
+44 -14
View File
@@ -37,6 +37,8 @@ class SplitNodeHealthService
/**
* 当前是否适合向 Node 提交新 Browser 任务
*
* A2C 等 antiBot 任务走 real profile，应参考 queueReal 而非 standard 队列。
*/
public static function canAcceptWork(): bool
{
@@ -47,33 +49,61 @@ class SplitNodeHealthService
if (!empty($stats['shuttingDown'])) {
return false;
}
$queue = is_array($stats['queue'] ?? null) ? $stats['queue'] : [];
$config = is_array($stats['config'] ?? null) ? $stats['config'] : [];
$queued = (int) ($queue['queued'] ?? 0);
$active = (int) ($queue['active'] ?? 0);
$max = max(1, (int) ($config['maxConcurrentBrowsers'] ?? ($queue['max'] ?? 4)));
$snapshot = self::resolveQueueSnapshot($stats);
$threshold = SplitSyncConfigService::getNodeQueueSkipThreshold();
if ($queued >= $threshold) {
if ($snapshot['queued'] >= $threshold) {
return false;
}
return $active < $max;
return $snapshot['active'] < $snapshot['max'];
}
/**
* @return array{queued:int,active:int,max:int}
* @return array{queued:int,active:int,max:int,profile:string}
*/
public static function getQueueSnapshot(): array
{
$stats = self::fetchStats();
if ($stats === null) {
return ['queued' => 0, 'active' => 0, 'max' => 0];
return ['queued' => 0, 'active' => 0, 'max' => 0, 'profile' => 'real'];
}
$queue = is_array($stats['queue'] ?? null) ? $stats['queue'] : [];
$config = is_array($stats['config'] ?? null) ? $stats['config'] : [];
$snapshot = self::resolveQueueSnapshot($stats);
return [
'queued' => (int) ($queue['queued'] ?? 0),
'active' => (int) ($queue['active'] ?? 0),
'max' => max(1, (int) ($config['maxConcurrentBrowsers'] ?? ($queue['max'] ?? 4))),
'queued' => $snapshot['queued'],
'active' => $snapshot['active'],
'max' => $snapshot['max'],
'profile' => $snapshot['profile'],
];
}
/**
* 优先使用 real 队列（antiBot / A2C），无数据时回退 standard
*
* @param array<string, mixed> $stats
* @return array{queued:int,active:int,max:int,profile:string}
*/
private static function resolveQueueSnapshot(array $stats): array
{
$config = is_array($stats['config'] ?? null) ? $stats['config'] : [];
$queueReal = is_array($stats['queueReal'] ?? null) ? $stats['queueReal'] : [];
if ($queueReal !== []) {
return [
'queued' => (int) ($queueReal['queued'] ?? 0),
'active' => (int) ($queueReal['active'] ?? 0),
'max' => max(1, (int) ($config['maxConcurrentBrowsersReal'] ?? ($queueReal['max'] ?? 2))),
'profile' => 'real',
];
}
$queue = is_array($stats['queue'] ?? null) ? $stats['queue'] : [];
return [
'queued' => (int) ($queue['queued'] ?? 0),
'active' => (int) ($queue['active'] ?? 0),
'max' => max(1, (int) ($config['maxConcurrentBrowsers'] ?? ($queue['max'] ?? 4))),
'profile' => 'standard',
];
}
patches/application/common/service/SplitTicketSyncDispatchService.php
+124 -12
View File
@@ -4,11 +4,15 @@ declare(strict_types=1);
namespace app\common\service;
use think\Db;
/**
* 工单手工同步 CLI 后台投递
*
* Web 请求仅负责鉴权与投递，实际 Spider 在独立 CLI 进程中执行，
* 避免长时间占用 PHP-FPM 与 Session 锁导致后台其它页面无法打开。
*
* Node 类型（A2C / Whatshub 等）在同一 nohup 子 shell 内串行执行，避免并行 Chrome 压垮温池。
*/
class SplitTicketSyncDispatchService
{
@@ -28,6 +32,13 @@ class SplitTicketSyncDispatchService
$think = $this->resolveThinkScript();
$root = $this->resolveRootPath();
$lockService = new SplitTicketSyncLockService();
$nodeTypeMap = array_flip(SplitScrmSpiderFactory::listNodeSyncTypes());
$typeMap = $this->loadTicketTypeMap($ticketIds);
/** @var int[] $nodeTicketIds */
$nodeTicketIds = [];
/** @var int[] $directTicketIds */
$directTicketIds = [];
foreach ($ticketIds as $rawId) {
$ticketId = (int) $rawId;
@@ -40,6 +51,15 @@ class SplitTicketSyncDispatchService
continue;
}
$ticketType = (string) ($typeMap[$ticketId] ?? '');
if ($ticketType !== '' && isset($nodeTypeMap[$ticketType])) {
$nodeTicketIds[] = $ticketId;
} else {
$directTicketIds[] = $ticketId;
}
}
foreach ($directTicketIds as $ticketId) {
if ($this->spawnCli($php, $think, $root, $ticketId)) {
$queued[] = $ticketId;
} else {
@@ -47,11 +67,26 @@ class SplitTicketSyncDispatchService
}
}
if ($nodeTicketIds !== []) {
$spawned = $this->spawnCliSequential($php, $think, $root, $nodeTicketIds);
if ($spawned) {
foreach ($nodeTicketIds as $ticketId) {
$queued[] = $ticketId;
}
} else {
foreach ($nodeTicketIds as $ticketId) {
$failed[] = $ticketId;
}
}
}
SplitTicketSyncLogger::log('web', 'manual sync dispatched', [
'queued' => $queued,
'skipped' => $skipped,
'failed' => $failed,
'phpCli' => $php,
'queued' => $queued,
'skipped' => $skipped,
'failed' => $failed,
'nodeSerial' => $nodeTicketIds,
'directParallel' => $directTicketIds,
'phpCli' => $php,
]);
return [
@@ -82,7 +117,37 @@ class SplitTicketSyncDispatchService
}
/**
* 后台启动 split:sync-tickets CLI
* @param int[] $ticketIds
* @return array<int, string> ticketId => ticket_type
*/
private function loadTicketTypeMap(array $ticketIds): array
{
$ids = [];
foreach ($ticketIds as $rawId) {
$ticketId = (int) $rawId;
if ($ticketId > 0) {
$ids[] = $ticketId;
}
}
if ($ids === []) {
return [];
}
$rows = Db::name('split_ticket')
->where('id', 'in', $ids)
->field('id,ticket_type')
->select();
$map = [];
foreach ($rows as $row) {
$map[(int) $row['id']] = (string) ($row['ticket_type'] ?? '');
}
return $map;
}
/**
* 后台启动 split:sync-tickets CLI（单工单，并行投递）
*/
private function spawnCli(string $php, string $think, string $root, int $ticketId): bool
{
@@ -97,18 +162,63 @@ class SplitTicketSyncDispatchService
escapeshellarg($logFile)
);
return $this->runBackgroundCommand($root, $inner, $ticketId, 'parallel');
}
/**
* Node 类型工单：单条 nohup 内串行执行多个 CLI，避免并行 Chrome
*
* @param int[] $ticketIds
*/
private function spawnCliSequential(string $php, string $think, string $root, array $ticketIds): bool
{
if ($ticketIds === []) {
return false;
}
$logDir = $this->resolveLogDir();
$parts = [];
foreach ($ticketIds as $ticketId) {
$ticketId = (int) $ticketId;
if ($ticketId <= 0) {
continue;
}
$logFile = $logDir . 'cli_' . $ticketId . '_' . date('YmdHis') . '.log';
$parts[] = sprintf(
'%s %s split:sync-tickets --ticket=%d >> %s 2>&1',
escapeshellarg($php),
escapeshellarg($think),
$ticketId,
escapeshellarg($logFile)
);
}
if ($parts === []) {
return false;
}
$inner = implode('; ', $parts);
return $this->runBackgroundCommand($root, $inner, $ticketIds, 'node-serial');
}
/**
* @param int|int[] $ticketRef 日志用 ticketId 或 ID 列表
*/
private function runBackgroundCommand(string $root, string $inner, $ticketRef, string $mode): bool
{
if ($this->isWindows()) {
$command = sprintf('start /B cmd /C %s', $inner);
} else {
$command = sprintf('cd %s && nohup %s &', escapeshellarg($root), $inner);
$command = sprintf('cd %s && nohup bash -c %s &', escapeshellarg($root), escapeshellarg($inner));
}
if ($this->canUseExec()) {
@exec($command, $output, $exitCode);
SplitTicketSyncLogger::log('web', 'cli spawn exec', [
'ticketId' => $ticketId,
'command' => $command,
'exitCode' => $exitCode,
'ticketRef' => $ticketRef,
'mode' => $mode,
'command' => $command,
'exitCode' => $exitCode,
]);
return true;
}
@@ -116,14 +226,16 @@ class SplitTicketSyncDispatchService
if ($this->canUseShellExec()) {
@shell_exec($command);
SplitTicketSyncLogger::log('web', 'cli spawn shell_exec', [
'ticketId' => $ticketId,
'command' => $command,
'ticketRef' => $ticketRef,
'mode' => $mode,
'command' => $command,
]);
return true;
}
SplitTicketSyncLogger::log('web', 'cli spawn failed: exec disabled', [
'ticketId' => $ticketId,
'ticketRef' => $ticketRef,
'mode' => $mode,
]);
return false;
patches/puppeteer-api/server.js
+171 -13
View File
@@ -514,16 +514,24 @@ async function ensureCaptchaProxyBrowserIfNeeded(browserCtx) {
extraArgs,
sessionKey: antiBot.sessionKey || '',
});
tagBrowserSession(browserSession, profile, antiBot.sessionKey || '', extraArgs);
browserSession.captchaProxyActive = true;
const proxySessionRef = { current: browserSession };
page = await createTaskPage(
browserSession.browser,
browserSession.reused ? null : browserSession.page,
navigationUrl,
antiBot,
mergedCookies.length > 0 ? mergedCookies : snapshotCookies,
{ ...pageExtraOpts, useCaptchaProxy: true }
{
...pageExtraOpts,
useCaptchaProxy: true,
browserSession,
browserSessionRef: proxySessionRef,
}
);
browserSession = proxySessionRef.current;
if (antiBot?.enabled) {
attachSitekeyCapture(page, browserSession.browser);
@@ -541,28 +549,95 @@ async function ensureCaptchaProxyBrowserIfNeeded(browserCtx) {
browserCtx.page = page;
}
/** 任务结束关闭本次 Tab（温池复用 Browser 时保留 Browser 实例） */
/**
* 为温池复用 / newPage 重试附加元数据（profile、sessionKey、启动参数）
* @param {object} browserSession
* @param {'standard'|'real'} profile
* @param {string} sessionKey
* @param {string[]} launchExtraArgs
*/
function tagBrowserSession(browserSession, profile, sessionKey, launchExtraArgs = []) {
browserSession.profile = profile;
browserSession.sessionKey = sessionKey || '';
browserSession.launchExtraArgs = launchExtraArgs;
return browserSession;
}
/** @param {Error|unknown} err */
function isNewTabOpenError(err) {
const msg = String(err?.message || err || '').toLowerCase();
return msg.includes('target.createtarget') || msg.includes('failed to open a new tab');
}
/**
* 温池 Browser 若已无任何 Tab，销毁并冷启动（避免后续 newPage 失败）
* @param {object|null} browserSession
*/
async function relaunchBrowserSessionIfNoTabs(browserSession) {
if (!browserSession?.pooled || !browserSession.browser) {
return browserSession;
}
let pages = [];
try {
pages = await browserSession.browser.pages();
} catch (_) {
pages = [];
}
if (pages.length > 0) {
return browserSession;
}
console.warn('[createTaskPage] 温池 Browser pages=0，销毁并冷启动');
await browserSession.release(true);
const relaunched = await acquireBrowserSession({
profile: browserSession.profile || 'real',
extraArgs: browserSession.launchExtraArgs || ['--mute-audio'],
sessionKey: browserSession.sessionKey || '',
});
tagBrowserSession(
relaunched,
browserSession.profile || relaunched.profile || 'real',
browserSession.sessionKey || '',
browserSession.launchExtraArgs || ['--mute-audio']
);
return relaunched;
}
/** 任务结束关闭本次 Tab（温池复用 Browser 时保留至少一个 Tab） */
async function closeTaskPage(page) {
if (!page) {
return;
}
try {
if (typeof page.isClosed === 'function' && !page.isClosed()) {
await page.close();
if (typeof page.isClosed === 'function' && page.isClosed()) {
return;
}
const browser = typeof page.browser === 'function' ? page.browser() : null;
if (browser) {
const pages = await browser.pages().catch(() => []);
if (pages.length <= 1) {
await page.goto('about:blank', {
waitUntil: 'domcontentloaded',
timeout: Math.min(NAVIGATION_TIMEOUT_MS, 15000),
}).catch(() => {});
return;
}
}
await page.close();
} catch (_) {
/* ignore */
}
}
/**
* 任务页创建（含 Whatshub skipUserAgent）
* 温池复用 Browser 时每次 newPage，冷启动 Real Browser 可使用 launch 返回的首个 Tab
* 配置任务页（UA / Cookie / 事件监听）
* @param {import('puppeteer').Page} page
* @param {import('puppeteer').Browser} browser
* @param {object} opts
* @param {object} extraOpts
* @param {object|null|undefined} antiBot
*/
async function createTaskPage(browser, warmPage, pageUrl, antiBot, mergedCookies, extraOpts = {}) {
const opts = buildManagedPageOptions(pageUrl, antiBot, mergedCookies, extraOpts);
const useWarmPage = !!(warmPage && extraOpts.useWarmPage !== false && !extraOpts.forceNewPage);
const page = useWarmPage ? warmPage : await browser.newPage();
async function configureTaskPage(page, browser, opts, extraOpts, antiBot) {
page.setDefaultNavigationTimeout(NAVIGATION_TIMEOUT_MS);
page.setDefaultTimeout(PAGE_DEFAULT_TIMEOUT_MS);
@@ -586,7 +661,67 @@ async function createTaskPage(browser, warmPage, pageUrl, antiBot, mergedCookies
if (antiBot?.enabled) {
attachSitekeyCapture(page, browser);
}
}
/**
* 任务页创建（含 Whatshub skipUserAgent）
* 温池复用 Browser 时每次 newPage；失败则销毁温池条目并冷启动重试
*
* extraOpts.browserSession：传入时可触发 pages=0 重建与 newPage 失败重试
* extraOpts.browserSessionRef：可选 { current }，重试后回写最新 session
*/
async function createTaskPage(browser, warmPage, pageUrl, antiBot, mergedCookies, extraOpts = {}) {
const opts = buildManagedPageOptions(pageUrl, antiBot, mergedCookies, extraOpts);
let browserSession = extraOpts.browserSession || null;
let currentBrowser = browser;
let currentWarmPage = warmPage;
if (browserSession) {
browserSession = await relaunchBrowserSessionIfNoTabs(browserSession);
currentBrowser = browserSession.browser;
currentWarmPage = browserSession.reused ? null : browserSession.page;
if (extraOpts.browserSessionRef) {
extraOpts.browserSessionRef.current = browserSession;
}
}
let page = null;
for (let attempt = 0; attempt < 2; attempt++) {
const useWarmPage = !!(currentWarmPage && extraOpts.useWarmPage !== false && !extraOpts.forceNewPage);
try {
page = useWarmPage ? currentWarmPage : await currentBrowser.newPage();
break;
} catch (err) {
if (attempt === 0 && browserSession && isNewTabOpenError(err)) {
console.warn('[createTaskPage] newPage 失败，销毁温池并冷启动重试:', err.message);
await browserSession.release(true);
browserSession = await acquireBrowserSession({
profile: browserSession.profile || (antiBot?.profile === 'real' ? 'real' : 'standard'),
extraArgs: browserSession.launchExtraArgs || ['--mute-audio'],
sessionKey: browserSession.sessionKey || antiBot?.sessionKey || '',
});
tagBrowserSession(
browserSession,
browserSession.profile || (antiBot?.profile === 'real' ? 'real' : 'standard'),
browserSession.sessionKey || antiBot?.sessionKey || '',
browserSession.launchExtraArgs || ['--mute-audio']
);
if (extraOpts.browserSessionRef) {
extraOpts.browserSessionRef.current = browserSession;
}
currentBrowser = browserSession.browser;
currentWarmPage = browserSession.reused ? null : browserSession.page;
continue;
}
throw err;
}
}
if (!page) {
throw new Error('createTaskPage: 未能打开任务 Tab');
}
await configureTaskPage(page, currentBrowser, opts, extraOpts, antiBot);
return page;
}
@@ -1554,18 +1689,22 @@ app.post('/api/auth-and-intercept', async (req, res) => {
extraArgs: ['--mute-audio'],
sessionKey: antiBot.sessionKey || '',
});
tagBrowserSession(browserSession, profile, antiBot.sessionKey || '', ['--mute-audio']);
cfLog.browserReused = !!browserSession.reused;
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl);
const navigationUrl = resolveNavigationUrl(pageUrl, antiBot);
const authSessionRef = { current: browserSession };
page = await createTaskPage(
browserSession.browser,
browserSession.reused ? null : browserSession.page,
navigationUrl,
antiBot,
mergedCookies
mergedCookies,
{ browserSession, browserSessionRef: authSessionRef }
);
browserSession = authSessionRef.current;
const httpResolvedUrl = await resolveHttpRedirects(pageUrl).catch(() => pageUrl);
if (httpResolvedUrl !== pageUrl) {
@@ -1840,17 +1979,24 @@ app.post('/api/ui-pagination', async (req, res) => {
extraArgs: ['--window-size=1920,1080'],
sessionKey: antiBot.sessionKey || '',
});
tagBrowserSession(browserSession, profile, antiBot.sessionKey || '', ['--window-size=1920,1080']);
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, navigationUrl, cookies || []);
const uiSessionRef = { current: browserSession };
page = await createTaskPage(
browserSession.browser,
browserSession.reused ? null : browserSession.page,
navigationUrl,
antiBot,
mergedCookies,
{ viewport: { width: 1920, height: 1080 } }
{
viewport: { width: 1920, height: 1080 },
browserSession,
browserSessionRef: uiSessionRef,
}
);
browserSession = uiSessionRef.current;
const log = (msg) => { console.log(msg); debugLogs.push(msg); };
log(`[启动] 准备访问页面: ${navigationUrl}`);
@@ -1986,19 +2132,31 @@ app.post('/api/auth-intercept-and-paginate', async (req, res) => {
extraArgs: ['--mute-audio', '--window-size=1920,1080'],
sessionKey: antiBot.sessionKey || '',
});
tagBrowserSession(
browserSession,
profile,
antiBot.sessionKey || '',
['--mute-audio', '--window-size=1920,1080']
);
cfLog.browserReused = !!browserSession.reused;
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl);
const navigationUrl = resolveNavigationUrl(pageUrl, antiBot);
const mergedSessionRef = { current: browserSession };
page = await createTaskPage(
browserSession.browser,
browserSession.reused ? null : browserSession.page,
navigationUrl,
antiBot,
mergedCookies,
{ viewport: { width: 1920, height: 1080 } }
{
viewport: { width: 1920, height: 1080 },
browserSession,
browserSessionRef: mergedSessionRef,
}
);
browserSession = mergedSessionRef.current;
const httpResolvedUrl = await resolveHttpRedirects(pageUrl).catch(() => pageUrl);