修复A2C工单蜘蛛

This commit is contained in:
root
2026-07-01 01:26:27 +08:00
parent 9f2e904fab
commit 54c13c54ef
22 changed files with 1614 additions and 272 deletions
+91 -36
View File
@@ -1,101 +1,156 @@
<?php
// A2c云控
// A2c云控 — 测试脚本(短链/长链 + Real Browser 拦截 API,对齐生产 A2cSpider
require_once __DIR__ . '/AbstractScrmSpider.class.php';
require_once __DIR__ . '/SplitPageUrlResolver.php';
class A2c extends AbstractScrmSpider
{
const API_LIST = '/api/talk/counter/share/record/list'; // 列表API地址
const API_DETAILS = '/api/talk/counter/share/detail'; // 详情API地址
const API_COUNT = ''; // 总数API地址
const DEFAULT_PER_PAGE_COUNT = 20; // List默认每页显示的数量
const API_LIST = '/api/talk/counter/share/record/list';
const API_DETAILS = '/api/talk/counter/share/detail';
const API_COUNT = '';
const DEFAULT_PER_PAGE_COUNT = 20;
/** @var string */
private $pageUrl;
/** @var string */
private $account;
/** @var string */
private $password;
/** @var UnifiedScrmData */
private $unifiedData;
// 实例化时动态传入账号和密码
/** @var string HTTP 预解析后的落地 URL */
private $landingUrl = '';
public function __construct($pageUrl, $account, $password, $nodeHost = 'http://127.0.0.1:3001')
{
parent::__construct($nodeHost);
$this->account = $account;
$this->password = $password;
$this->pageUrl = $pageUrl;
$this->landingUrl = SplitPageUrlResolver::resolveFinalUrl($pageUrl);
$this->unifiedData = new UnifiedScrmData();
}
protected function getSpiderConfig()
{
$antiBot = $this->buildAntiBotConfig();
$this->logDebug('a2c spider config', [
'pageUrl' => $this->pageUrl,
'landingUrl' => $this->landingUrl,
'sessionKey' => $antiBot['sessionKey'] ?? '',
'cfClearanceRequired' => $antiBot['cfClearanceRequired'] ?? null,
]);
return [
'pageUrl' => $this->pageUrl,
'apiUrls' => [self::API_LIST, self::API_DETAILS],
// 明确指派角色
'listApi' => self::API_LIST, // 必须
'detailApi' => self::API_DETAILS, // 选填
'listApi' => self::API_LIST,
'detailApi' => self::API_DETAILS,
'listMethod' => 'POST',
'paginationMode' => self::MODE_UI,
'authActions' => [
// ['type' => 'type', 'selector' => 'input[type="password"]', 'value' => $this->password],
// ['type' => 'type', 'selector' => '#username_input', 'value' => $this->account],
// ['type' => 'press', 'key' => 'Enter'],
['type' => 'wait', 'ms' => 2000]
]
['type' => 'wait', 'ms' => 2000],
],
'antiBot' => $antiBot,
];
}
// 只负责解析 List 的总页数
/**
* A2C:业务域无需 cf_clearance;短链仍作 pageUrlsessionKey 固定业务域
*
* @return array<string, mixed>
*/
private function buildAntiBotConfig()
{
$sessionHost = $this->resolveA2cSessionHost();
return [
'enabled' => true,
'profile' => 'real',
'turnstile' => true,
'solverFallback' => false,
'cfClearanceRequired' => false,
'sessionKey' => 'a2c:' . $sessionHost,
'challengeTimeoutMs' => 60000,
'landingUrlHint' => $this->landingUrl,
'businessHostHint' => $sessionHost,
'postCfReadyUrlContains' => '/visitors/counter/share',
'postCfReadySelector' => '.btn-next',
'postCfReadyTimeoutMs' => 30000,
'postCfSettleMs' => 500,
'postCfSpaWaitMs' => 4000,
];
}
/**
* A2C 会话域:HTTP 预解析无法穿透短链 CF 时,回退到业务域 user.a2c.chat
*/
private function resolveA2cSessionHost()
{
$candidates = [];
if ($this->landingUrl !== '') {
$candidates[] = (string) parse_url($this->landingUrl, PHP_URL_HOST);
}
$candidates[] = (string) parse_url($this->pageUrl, PHP_URL_HOST);
foreach ($candidates as $host) {
$host = strtolower(trim($host));
if ($host !== '' && strpos($host, 'a2c.chat') !== false) {
return $host;
}
}
return 'user.a2c.chat';
}
protected function extractListTotalPages($listFirstPageData, $countData = null)
{
$default_per_page_count = self::DEFAULT_PER_PAGE_COUNT;
$this->unifiedData->total = $listFirstPageData['data']['total'];
if($listFirstPageData['data']['total'] <= $default_per_page_count) {
if ($listFirstPageData['data']['total'] <= $default_per_page_count) {
return 1;
}
return ceil($listFirstPageData['data']['total']/$default_per_page_count);
return (int) ceil($listFirstPageData['data']['total'] / $default_per_page_count);
}
// 只负责组装 List 的翻页参数
protected function buildListPageParams($page)
{
return ['page' => $page, 'pageSize' => self::DEFAULT_PER_PAGE_COUNT];
}
// 提供 List 的下一页按钮信息
protected function getUiPaginationConfig()
{
return [
'nextBtnSelector' => '.btn-next',
'waitMs' => 2000
'waitMs' => 2000,
];
}
// 清爽至极的数据清洗:详情是详情,列表是列表
protected function parseToUnifiedData($detailData, $allListPagesData)
{
$unifiedData = $this->unifiedData;
// 1. 如果捕获到了详情数据,提取今日新增
if ($detailData) {
$unifiedData->todayNewCount = (int)($detailData['data']['newFollowersToday'] ?? 0);
$unifiedData->todayNewCount = (int) ($detailData['data']['newFollowersToday'] ?? 0);
}
// 2. 循环合并了所有页数的 List 数组
foreach ($allListPagesData as $pageRaw) {
$records = $pageRaw['data']['rows'] ?? [];
foreach ($records as $item) {
if(!empty($item['account'])) {
$number = $item['account'] ?? null;
$isOnline = (isset($item['numberStatus']) && $item['numberStatus'] == 1);
$unifiedData->addNumber($number, $isOnline, $item['newFollowersToday']);
if (!empty($item['account'])) {
$number = $item['account'];
$isOnline = (isset($item['numberStatus']) && (int) $item['numberStatus'] === 1);
$unifiedData->addNumber($number, $isOnline, (int) ($item['newFollowersToday'] ?? 0));
}
}
}
+131
View File
@@ -0,0 +1,131 @@
<?php
declare(strict_types=1);
/**
* 短链 HTTP 重定向预解析(与 puppeteer-api/url-resolve.js 行为对齐)
*
* 用于 antiBot.sessionKey 按落地业务域生成,工单 URL 仍可为 yyk.ink 等短链。
*/
final class SplitPageUrlResolver
{
private const REDIRECT_CODES = [301, 302, 303, 307, 308];
/**
* 跟随 HTTP 重定向,返回最终 URL;失败时返回原始 URL
*/
public static function resolveFinalUrl(string $pageUrl, int $maxRedirects = 10, int $timeoutSec = 12): string
{
$current = trim($pageUrl);
if ($current === '') {
return '';
}
for ($i = 0; $i < $maxRedirects; $i++) {
$response = self::fetchHeadOrGet($current, $timeoutSec);
if ($response === null) {
return $current;
}
$code = $response['code'];
$location = $response['location'];
if (!in_array($code, self::REDIRECT_CODES, true) || $location === '') {
return $current;
}
$current = self::resolveRelativeUrl($current, $location);
}
return $current;
}
/**
* @return array{code: int, location: string}|null
*/
private static function fetchHeadOrGet(string $url, int $timeoutSec): ?array
{
$result = self::curlOnce($url, true, $timeoutSec);
if ($result !== null) {
return $result;
}
return self::curlOnce($url, false, $timeoutSec);
}
/**
* @return array{code: int, location: string}|null
*/
private static function curlOnce(string $url, bool $headOnly, int $timeoutSec): ?array
{
if (!function_exists('curl_init')) {
return null;
}
$ch = curl_init($url);
if ($ch === false) {
return null;
}
curl_setopt_array($ch, [
CURLOPT_NOBODY => $headOnly,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => false,
CURLOPT_TIMEOUT => $timeoutSec,
CURLOPT_HEADER => true,
CURLOPT_USERAGENT => 'Mozilla/5.0 (compatible; SplitPageUrlResolver/1.0)',
]);
$body = curl_exec($ch);
if ($body === false) {
curl_close($ch);
return null;
}
$code = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
$redirectUrl = (string) curl_getinfo($ch, CURLINFO_REDIRECT_URL);
curl_close($ch);
$location = $redirectUrl;
if ($location === '' && is_string($body)) {
$location = self::parseLocationHeader($body);
}
return ['code' => $code, 'location' => trim($location)];
}
private static function parseLocationHeader(string $rawHeaders): string
{
foreach (preg_split('/\r\n|\n|\r/', $rawHeaders) as $line) {
if (stripos($line, 'Location:') === 0) {
return trim(substr($line, 9));
}
}
return '';
}
private static function resolveRelativeUrl(string $baseUrl, string $location): string
{
if (preg_match('#^https?://#i', $location)) {
return $location;
}
$base = parse_url($baseUrl);
if (!is_array($base)) {
return $location;
}
$scheme = (string) ($base['scheme'] ?? 'https');
$host = (string) ($base['host'] ?? '');
if ($host === '') {
return $location;
}
if (strpos($location, '/') === 0) {
return $scheme . '://' . $host . $location;
}
$path = (string) ($base['path'] ?? '/');
$dir = rtrim(str_replace(basename($path), '', $path), '/');
return $scheme . '://' . $host . ($dir !== '' ? $dir . '/' : '/') . $location;
}
}
+65 -62
View File
@@ -11,52 +11,52 @@ require_once __DIR__ . '/SsCustomer.php'; // SS云控(Customer)
require_once __DIR__ . '/Haiwang.php'; // 海王
require_once __DIR__ . '/Whatshub.php'; // Whatshub 工单平台
try {
/*
// try {
// /*
命令行可以测试Cloudflare防火墙
curl -s -X POST http://127.0.0.1:3001/api/auth-and-intercept \
-H 'Content-Type: application/json' \
-d '{
"pageUrl": "https://web.whatshub.cc/m/KMYBBInp2066/1",
"apiUrls": ["/api/whatshub-counter/workShare/open/detail"],
"authActions": [
{"type": "wait", "ms": 1000},
{"type": "vue_fill", "selector": ".el-input__wrapper > .el-input__inner", "value": "602066"},
{"type": "wait", "ms": 500},
{"type": "vue_click", "selector": ".vxe-button-group .theme--primary"},
{"type": "wait", "ms": 2200}
],
"antiBot": {
"enabled": true,
"profile": "real",
"turnstile": true,
"solverFallback": true,
"sessionKey": "whatshub:web.whatshub.cc",
"challengeTimeoutMs": 60000
}
}' | jq .
*/
// 命令行可以测试Cloudflare防火墙
// curl -s -X POST http://127.0.0.1:3001/api/auth-and-intercept \
// -H 'Content-Type: application/json' \
// -d '{
// "pageUrl": "https://web.whatshub.cc/m/KMYBBInp2066/1",
// "apiUrls": ["/api/whatshub-counter/workShare/open/detail"],
// "authActions": [
// {"type": "wait", "ms": 1000},
// {"type": "vue_fill", "selector": ".el-input__wrapper > .el-input__inner", "value": "602066"},
// {"type": "wait", "ms": 500},
// {"type": "vue_click", "selector": ".vxe-button-group .theme--primary"},
// {"type": "wait", "ms": 2200}
// ],
// "antiBot": {
// "enabled": true,
// "profile": "real",
// "turnstile": true,
// "solverFallback": true,
// "sessionKey": "whatshub:web.whatshub.cc",
// "challengeTimeoutMs": 60000
// }
// }' | jq .
// */
echo "🚀 开始执行<Whatshub 工单平台>抓取任务 (多引擎智能调度)...\n\r";
$pageUrl = 'https://web.whatshub.cc/m/KMYBBInp2066/1'; // PageUrl 入口授权页
$username = ""; // 登录账号
$password = "602066"; // 登录密码
$spider = new Whatshub($pageUrl, $username, $password);
$finalData = $spider->run();
// echo "🚀 开始执行<Whatshub 工单平台>抓取任务 (多引擎智能调度)...\n\r";
// $pageUrl = 'https://web.whatshub.cc/m/KMYBBInp2066/1'; // PageUrl 入口授权页
// $username = ""; // 登录账号
// $password = "602066"; // 登录密码
// $spider = new Whatshub($pageUrl, $username, $password);
// $finalData = $spider->run();
echo "✅ 任务完成!统一数据如下:\n\r";
echo "----------------------------------------\n\r";
echo "当日新增:{$finalData->todayNewCount}\n\r";
echo "在线号码:{$finalData->totalOnline}\n\r";
echo "离线号码:{$finalData->totalOffline}\n\r";
echo "Total" . $finalData->total . " 个号码\n\r";
echo "实际总共抓取:" . count($finalData->numbers) . " 个号码\n\r";
echo "号码列表:\n\r";
echo dd($finalData->numbers);
} catch (Exception $e) {
echo "🚨 抓取异常:" . $e->getMessage() . "\n\r";
}
// echo "✅ 任务完成!统一数据如下:\n\r";
// echo "----------------------------------------\n\r";
// echo "当日新增:{$finalData->todayNewCount} 人\n\r";
// echo "在线号码:{$finalData->totalOnline} 个\n\r";
// echo "离线号码:{$finalData->totalOffline} 个\n\r";
// echo "Total" . $finalData->total . " 个号码\n\r";
// echo "实际总共抓取:" . count($finalData->numbers) . " 个号码\n\r";
// echo "号码列表:\n\r";
// echo dd($finalData->numbers);
// } catch (Exception $e) {
// echo "🚨 抓取异常:" . $e->getMessage() . "\n\r";
// }
// try {
@@ -80,27 +80,30 @@ try {
// echo "🚨 抓取异常:" . $e->getMessage() . "\n\r";
// }
// try {
// echo "🚀 开始执行<A2c云控>抓取任务 (多引擎智能调度)...\n\r";
// $pageUrl = 'https://yyk.ink/52oXZRG'; // PageUrl 入口授权页
// $username = ""; // 登录账号
// $password = ""; // 登录密码
// $spider = new A2c($pageUrl, $username, $password);
// $finalData = $spider->run();
try {
echo "🚀 开始执行<A2c云控>抓取任务 (多引擎智能调度)...\n\r";
// 长链:快速验证业务页拦截(推荐先测通)
// $pageUrl = 'https://user.a2c.chat/visitors/counter/share?id=1b2cf9a91e2647c185b252e723c871de';
// 短链:与客户后台一致,上线前必须回归
$pageUrl = 'https://yyk.ink/91SEWrL';
$username = "";
$password = "";
$spider = (new A2c($pageUrl, $username, $password))->setVerbose(true);
$finalData = $spider->run();
// echo "✅ 任务完成!统一数据如下:\n\r";
// echo "----------------------------------------\n\r";
// echo "当日新增:{$finalData->todayNewCount} 人\n\r";
// echo "在线号码:{$finalData->totalOnline} 个\n\r";
// echo "离线号码:{$finalData->totalOffline} 个\n\r";
// echo "Total" . $finalData->total . " 个号码\n\r";
// echo "实际总共抓取:" . count($finalData->numbers) . " 个号码\n\r";
// echo "号码列表:\n\r";
// echo dd($finalData->numbers);
echo "✅ 任务完成!统一数据如下:\n\r";
echo "----------------------------------------\n\r";
echo "当日新增:{$finalData->todayNewCount}\n\r";
echo "在线号码:{$finalData->totalOnline}\n\r";
echo "离线号码:{$finalData->totalOffline}\n\r";
echo "Total" . $finalData->total . " 个号码\n\r";
echo "实际总共抓取:" . count($finalData->numbers) . " 个号码\n\r";
echo "号码列表:\n\r";
echo dd($finalData->numbers);
// } catch (Exception $e) {
// echo "🚨 抓取异常:" . $e->getMessage() . "\n\r";
// }
} catch (Exception $e) {
echo "🚨 抓取异常:" . $e->getMessage() . "\n\r";
}
// try {
// echo "🚀 开始执行<星河云控>抓取任务 (多引擎智能调度)...\n\r";
@@ -25,43 +25,46 @@ class AntiBotConfigBuilder
return null;
}
$host = (string) parse_url($pageUrl, PHP_URL_HOST);
if ($host === '') {
$sessionHost = self::resolveSessionHost($ticketType, $pageUrl);
if ($sessionHost === '') {
return null;
}
$sessionKey = $ticketType . ':' . $host;
$sessionKey = $ticketType . ':' . $sessionHost;
$ttlMinutes = self::getSessionTtlMinutes();
return [
'enabled' => true,
'profile' => 'real',
'turnstile' => true,
'solverFallback' => true,
'turnstile' => self::defaultTurnstile($ticketType),
'solverFallback' => self::defaultSolverFallback($ticketType),
'cfClearanceRequired'=> self::defaultCfClearanceRequired($ticketType),
'sessionKey' => $sessionKey,
'challengeTimeoutMs' => 60000,
'sessionTtlMinutes' => $ttlMinutes,
'businessHostHint' => self::defaultBusinessHostHint($ticketType, $sessionHost),
] + self::postCfReadyExtras($ticketType);
}
/**
* 按工单类型附加 CF 过盾后的业务页就绪等待(仅 Whatshub 等需要的类型)
*
* @return array<string, mixed>
* 会话域:A2C 短链(yyk.ink)仍用业务域 user.a2c.chat 复用 Real Browser 会话
*/
private static function postCfReadyExtras(string $ticketType): array
public static function resolveSessionHost(string $ticketType, string $pageUrl): string
{
if ($ticketType !== 'whatshub') {
return [];
$host = strtolower(trim((string) parse_url($pageUrl, PHP_URL_HOST)));
if ($host === '') {
return '';
}
return [
'postCfReadyUrlContains' => 'work-order-sharing',
'postCfReadySelector' => '.el-input__inner',
'postCfReadyTimeoutMs' => 30000,
'postCfSettleMs' => 500,
'postCfSpaWaitMs' => 4000,
];
if ($ticketType === 'a2c') {
if (strpos($host, 'a2c.chat') !== false) {
return $host;
}
return 'user.a2c.chat';
}
return $host;
}
/**
@@ -69,7 +72,8 @@ class AntiBotConfigBuilder
*/
public static function resolveSessionKey(string $ticketType, string $pageUrl): string
{
$host = (string) parse_url($pageUrl, PHP_URL_HOST);
$host = self::resolveSessionHost($ticketType, $pageUrl);
return $ticketType . ':' . ($host !== '' ? $host : 'unknown');
}
@@ -101,4 +105,66 @@ class AntiBotConfigBuilder
$minutes = (int) Config::get('site.split_scrm_session_ttl_minutes');
return $minutes > 0 ? $minutes : 120;
}
/**
* 按工单类型附加 CF 过盾后的业务页就绪等待
*
* @return array<string, mixed>
*/
private static function postCfReadyExtras(string $ticketType): array
{
if ($ticketType === 'whatshub') {
return [
'postCfReadyUrlContains' => 'work-order-sharing',
'postCfReadySelector' => '.el-input__inner',
'postCfReadyTimeoutMs' => 30000,
'postCfSettleMs' => 500,
'postCfSpaWaitMs' => 4000,
];
}
if ($ticketType === 'a2c') {
return [
'postCfReadyUrlContains' => '/visitors/counter/share',
'postCfReadySelector' => '.btn-next',
'postCfReadyTimeoutMs' => 30000,
'postCfSettleMs' => 500,
'postCfSpaWaitMs' => 4000,
];
}
return [];
}
private static function defaultTurnstile(string $ticketType): bool
{
return true;
}
private static function defaultSolverFallback(string $ticketType): bool
{
if ($ticketType === 'a2c') {
return false;
}
return true;
}
private static function defaultCfClearanceRequired(string $ticketType): bool
{
if ($ticketType === 'a2c') {
return false;
}
return true;
}
private static function defaultBusinessHostHint(string $ticketType, string $sessionHost): string
{
if ($ticketType === 'a2c') {
return $sessionHost;
}
return '';
}
}
@@ -9,7 +9,9 @@ use app\common\library\scrm\AntiBotConfigBuilder;
use app\common\library\scrm\UnifiedScrmData;
/**
* A2C 云控蜘蛛(Real Browser + Turnstile 过盾 + UI 翻页)
* A2C 云控蜘蛛(Real Browser 打开分享页 + 拦截加密 API + UI 翻页)
*
* 业务域 user.a2c.chat 无需 cf_clearance;短链 yyk.ink 仍由浏览器 follow 跳转。
*/
class A2cSpider extends AbstractScrmSpider
{
@@ -7,12 +7,16 @@ namespace app\common\service;
use app\admin\model\split\Link;
/**
* 分流链接随机打乱配置读取
* 分流链接随机打乱配置与落地页选号
*
* - random_shuffle=0:跨工单合并号码池,按 id 顺序严格轮转
* - random_shuffle=1:跨工单合并号码池,每次访问完全随机选号
* - 同步写入时 random_shuffle=1 还会打乱新号码 insert 顺序(影响 id 分布)
*/
class SplitNumberWeighService
{
/**
* 链接是否开启随机打乱(新号码按随机插入顺序写入,跳转按 id 顺序轮转)
* 链接是否开启随机打乱
*/
public static function isRandomShuffleEnabled(int $linkId): bool
{
@@ -21,4 +25,30 @@ class SplitNumberWeighService
}
return (int) Link::where('id', $linkId)->value('random_shuffle') === 1;
}
/**
* 解析本次访问应使用的号码下标(0 .. numberCount-1
*
* @param int $linkId 分流链接 ID
* @param int $numberCount 可用号码数量
* @param SplitRoundRobinStore $roundRobinStore 关闭随机打乱时的轮转计数器
*/
public static function resolvePickIndex(
int $linkId,
int $numberCount,
SplitRoundRobinStore $roundRobinStore
): int {
if ($numberCount <= 0) {
return 0;
}
if ($numberCount === 1) {
return 0;
}
if (self::isRandomShuffleEnabled($linkId)) {
return random_int(0, $numberCount - 1);
}
return $roundRobinStore->nextIndex($linkId, $numberCount);
}
}
@@ -9,7 +9,10 @@ use app\admin\model\split\Number;
use think\Collection;
/**
* 分流链接落地页:查链、轮转选号、拼接跳转 URL、访问计数
* 分流链接落地页:查链、跨工单合并选号、拼接跳转 URL、访问计数
*
* 号码池为同一 split_link_id 下全部 status=normal 的号码;
* random_shuffle 关闭时严格轮转,开启时每次访问随机选号。
*/
class SplitRedirectService
{
@@ -59,7 +62,7 @@ class SplitRedirectService
}
$linkId = (int) $link->getAttr('id');
$index = $this->roundRobinStore->nextIndex($linkId, $count);
$index = SplitNumberWeighService::resolvePickIndex($linkId, $count, $this->roundRobinStore);
$list = $numbers instanceof \think\Collection ? $numbers->all() : (array) $numbers;
$picked = $list[$index] ?? $list[0] ?? null;
if ($picked === null) {
@@ -11,11 +11,16 @@ use think\Db;
/**
* 工单同步结果写入号码表
*
* 同步策略(全量镜像):
* - 爬虫返回的号码:写入 platform_status / inbound_count,开关 status 与云控在线状态一致
* - 爬虫未返回且 manual_manage=0:物理删除
* - manual_manage=1:不删除,仅当仍在爬虫结果中时更新 platform_status
*/
class SplitTicketNumberSyncService
{
/**
* 将蜘蛛返回的号码列表同步到号码管理
* 将蜘蛛返回的号码列表同步到号码管理(全量镜像,以爬虫为准)
*/
public function syncFromUnifiedData(Ticket $ticket, UnifiedScrmData $data): void
{
@@ -61,7 +66,7 @@ class SplitTicketNumberSyncService
$newFollowers = (int) ($row['newFollowersToday'] ?? 0);
if (isset($existingMap[$number])) {
$this->updateExistingNumber($existingMap[$number], $platformStatus, $newFollowers);
$this->updateExistingNumber($ticket, $existingMap[$number], $platformStatus, $newFollowers);
continue;
}
@@ -87,6 +92,7 @@ class SplitTicketNumberSyncService
}
}
$deleteIds = [];
foreach ($existingMap as $number => $item) {
if (isset($syncedNumberSet[$number])) {
continue;
@@ -94,18 +100,58 @@ class SplitTicketNumberSyncService
if ((int) $item['manual_manage'] === 1) {
continue;
}
Number::where('id', (int) $item['id'])->update([
'status' => 'hidden',
$deleteIds[] = (int) $item['id'];
}
if ($deleteIds !== []) {
Number::where('id', 'in', $deleteIds)->delete();
}
}
/**
* 工单手动开启等非同步场景:按已存的 platform_status 对齐开关(不跑单号上限/下号比率)
*/
public function applyStatusFromPlatformSnapshot(Ticket $ticket): void
{
if ((string) ($ticket['status'] ?? 'hidden') !== 'normal') {
return;
}
$numbers = Number::where('admin_id', (int) $ticket['admin_id'])
->where('split_link_id', (int) $ticket['split_link_id'])
->where('ticket_name', (string) $ticket['ticket_name'])
->where('manual_manage', 0)
->select();
foreach ($numbers as $number) {
$platformStatus = (string) ($number['platform_status'] ?? 'offline');
Number::where('id', (int) $number['id'])->update([
'status' => self::resolveSwitchStatus($ticket, $platformStatus),
'updatetime' => time(),
]);
}
}
/**
* 云控在线 → 开启;离线 → 关闭;工单关闭时一律关闭
*/
public static function resolveSwitchStatus(Ticket $ticket, string $platformStatus): string
{
if ((string) ($ticket['status'] ?? 'hidden') !== 'normal') {
return 'hidden';
}
return $platformStatus === 'online' ? 'normal' : 'hidden';
}
/**
* @param Number $row
*/
private function updateExistingNumber($row, string $platformStatus, int $newFollowers): void
{
private function updateExistingNumber(
Ticket $ticket,
$row,
string $platformStatus,
int $newFollowers
): void {
$update = [
'platform_status' => $platformStatus,
'updatetime' => time(),
@@ -116,8 +162,8 @@ class SplitTicketNumberSyncService
return;
}
// 进线人数由同步写入,最终开关由 applyNumberRules 统一判定(单号上限/下号比率等)
$update['inbound_count'] = max(0, $newFollowers);
$update['status'] = self::resolveSwitchStatus($ticket, $platformStatus);
Number::where('id', (int) $row['id'])->update($update);
}
@@ -139,7 +185,7 @@ class SplitTicketNumberSyncService
'inbound_count' => max(0, $newFollowers),
'manual_manage' => 0,
'platform_status' => $platformStatus,
'status' => 'hidden',
'status' => self::resolveSwitchStatus($ticket, $platformStatus),
'createtime' => $now,
'updatetime' => $now,
];
@@ -148,10 +194,11 @@ class SplitTicketNumberSyncService
Db::name('split_number')->insert($data);
} catch (\Throwable $e) {
$exists = Number::where('split_link_id', (int) $ticket['split_link_id'])
->where('ticket_name', (string) $ticket['ticket_name'])
->where('number', $number)
->find();
if ($exists) {
$this->updateExistingNumber($exists, $platformStatus, $newFollowers);
$this->updateExistingNumber($ticket, $exists, $platformStatus, $newFollowers);
}
}
}
@@ -13,7 +13,7 @@ use app\admin\model\split\Ticket;
class SplitTicketRuleService
{
/**
* 同步后应用全部规则并写回工单/号码
* 同步后应用工单开关规则;号码开关以爬虫/platform 快照为准(不再跑单号上限/下号比率)
*/
public function applyAfterSync(Ticket $ticket, int $completeCount): void
{
@@ -22,8 +22,9 @@ class SplitTicketRuleService
if ($fresh) {
if ((string) $fresh['status'] === 'hidden') {
$this->cascadeTicketClosedToNumbers($fresh);
} else {
(new SplitTicketNumberSyncService())->applyStatusFromPlatformSnapshot($fresh);
}
$this->applyNumberRules($fresh);
}
}
@@ -69,7 +70,7 @@ class SplitTicketRuleService
}
/**
* 手动切换工单状态时,联动非手动管理的号码
* 手动切换工单状态时,非手动号码 platform_status 对齐开关
*/
public function syncNumbersWithTicketStatus(Ticket $ticket): void
{
@@ -78,17 +79,17 @@ class SplitTicketRuleService
$this->cascadeTicketClosedToNumbers($ticket);
return;
}
$this->applyNumberRules($ticket);
(new SplitTicketNumberSyncService())->applyStatusFromPlatformSnapshot($ticket);
}
/**
* 工单处于开启状态时,将云控在线的非手动号码设为开启
*
* @deprecated 请使用 applyNumberRules(会校验单号上限/下号比率
* @deprecated 请使用 applyStatusFromPlatformSnapshot(以 platform_status 为准
*/
public function syncOnlineNumbersWhenTicketOpen(Ticket $ticket): void
{
$this->applyNumberRules($ticket);
(new SplitTicketNumberSyncService())->applyStatusFromPlatformSnapshot($ticket);
}
/**
@@ -297,7 +297,6 @@ class SplitTicketSyncService
if ((string) $freshTicket['status'] === 'hidden') {
$this->ruleService->cascadeTicketClosedToNumbers($freshTicket);
}
$this->ruleService->applyNumberRules($freshTicket);
$ticket = $freshTicket;
$inboundCount = $this->numberSync->sumInboundForTicket($ticket);
@@ -25,43 +25,46 @@ class AntiBotConfigBuilder
return null;
}
$host = (string) parse_url($pageUrl, PHP_URL_HOST);
if ($host === '') {
$sessionHost = self::resolveSessionHost($ticketType, $pageUrl);
if ($sessionHost === '') {
return null;
}
$sessionKey = $ticketType . ':' . $host;
$sessionKey = $ticketType . ':' . $sessionHost;
$ttlMinutes = self::getSessionTtlMinutes();
return [
'enabled' => true,
'profile' => 'real',
'turnstile' => true,
'solverFallback' => true,
'turnstile' => self::defaultTurnstile($ticketType),
'solverFallback' => self::defaultSolverFallback($ticketType),
'cfClearanceRequired'=> self::defaultCfClearanceRequired($ticketType),
'sessionKey' => $sessionKey,
'challengeTimeoutMs' => 60000,
'sessionTtlMinutes' => $ttlMinutes,
'businessHostHint' => self::defaultBusinessHostHint($ticketType, $sessionHost),
] + self::postCfReadyExtras($ticketType);
}
/**
* 按工单类型附加 CF 过盾后的业务页就绪等待(仅 Whatshub 等需要的类型)
*
* @return array<string, mixed>
* 会话域:A2C 短链(yyk.ink)仍用业务域 user.a2c.chat 复用 Real Browser 会话
*/
private static function postCfReadyExtras(string $ticketType): array
public static function resolveSessionHost(string $ticketType, string $pageUrl): string
{
if ($ticketType !== 'whatshub') {
return [];
$host = strtolower(trim((string) parse_url($pageUrl, PHP_URL_HOST)));
if ($host === '') {
return '';
}
return [
'postCfReadyUrlContains' => 'work-order-sharing',
'postCfReadySelector' => '.el-input__inner',
'postCfReadyTimeoutMs' => 30000,
'postCfSettleMs' => 500,
'postCfSpaWaitMs' => 8000,
];
if ($ticketType === 'a2c') {
if (strpos($host, 'a2c.chat') !== false) {
return $host;
}
return 'user.a2c.chat';
}
return $host;
}
/**
@@ -69,7 +72,8 @@ class AntiBotConfigBuilder
*/
public static function resolveSessionKey(string $ticketType, string $pageUrl): string
{
$host = (string) parse_url($pageUrl, PHP_URL_HOST);
$host = self::resolveSessionHost($ticketType, $pageUrl);
return $ticketType . ':' . ($host !== '' ? $host : 'unknown');
}
@@ -101,4 +105,66 @@ class AntiBotConfigBuilder
$minutes = (int) Config::get('site.split_scrm_session_ttl_minutes');
return $minutes > 0 ? $minutes : 120;
}
/**
* 按工单类型附加 CF 过盾后的业务页就绪等待
*
* @return array<string, mixed>
*/
private static function postCfReadyExtras(string $ticketType): array
{
if ($ticketType === 'whatshub') {
return [
'postCfReadyUrlContains' => 'work-order-sharing',
'postCfReadySelector' => '.el-input__inner',
'postCfReadyTimeoutMs' => 30000,
'postCfSettleMs' => 500,
'postCfSpaWaitMs' => 4000,
];
}
if ($ticketType === 'a2c') {
return [
'postCfReadyUrlContains' => '/visitors/counter/share',
'postCfReadySelector' => '.btn-next',
'postCfReadyTimeoutMs' => 30000,
'postCfSettleMs' => 500,
'postCfSpaWaitMs' => 4000,
];
}
return [];
}
private static function defaultTurnstile(string $ticketType): bool
{
return true;
}
private static function defaultSolverFallback(string $ticketType): bool
{
if ($ticketType === 'a2c') {
return false;
}
return true;
}
private static function defaultCfClearanceRequired(string $ticketType): bool
{
if ($ticketType === 'a2c') {
return false;
}
return true;
}
private static function defaultBusinessHostHint(string $ticketType, string $sessionHost): string
{
if ($ticketType === 'a2c') {
return $sessionHost;
}
return '';
}
}
@@ -9,7 +9,9 @@ use app\common\library\scrm\AntiBotConfigBuilder;
use app\common\library\scrm\UnifiedScrmData;
/**
* A2C 云控蜘蛛(Real Browser + Turnstile 过盾 + UI 翻页)
* A2C 云控蜘蛛(Real Browser 打开分享页 + 拦截加密 API + UI 翻页)
*
* 业务域 user.a2c.chat 无需 cf_clearance;短链 yyk.ink 仍由浏览器 follow 跳转。
*/
class A2cSpider extends AbstractScrmSpider
{
@@ -7,12 +7,16 @@ namespace app\common\service;
use app\admin\model\split\Link;
/**
* 分流链接随机打乱配置读取
* 分流链接随机打乱配置与落地页选号
*
* - random_shuffle=0:跨工单合并号码池,按 id 顺序严格轮转
* - random_shuffle=1:跨工单合并号码池,每次访问完全随机选号
* - 同步写入时 random_shuffle=1 还会打乱新号码 insert 顺序(影响 id 分布)
*/
class SplitNumberWeighService
{
/**
* 链接是否开启随机打乱(新号码按随机插入顺序写入,跳转按 id 顺序轮转)
* 链接是否开启随机打乱
*/
public static function isRandomShuffleEnabled(int $linkId): bool
{
@@ -21,4 +25,30 @@ class SplitNumberWeighService
}
return (int) Link::where('id', $linkId)->value('random_shuffle') === 1;
}
/**
* 解析本次访问应使用的号码下标(0 .. numberCount-1
*
* @param int $linkId 分流链接 ID
* @param int $numberCount 可用号码数量
* @param SplitRoundRobinStore $roundRobinStore 关闭随机打乱时的轮转计数器
*/
public static function resolvePickIndex(
int $linkId,
int $numberCount,
SplitRoundRobinStore $roundRobinStore
): int {
if ($numberCount <= 0) {
return 0;
}
if ($numberCount === 1) {
return 0;
}
if (self::isRandomShuffleEnabled($linkId)) {
return random_int(0, $numberCount - 1);
}
return $roundRobinStore->nextIndex($linkId, $numberCount);
}
}
@@ -9,7 +9,10 @@ use app\admin\model\split\Number;
use think\Collection;
/**
* 分流链接落地页:查链、轮转选号、拼接跳转 URL、访问计数
* 分流链接落地页:查链、跨工单合并选号、拼接跳转 URL、访问计数
*
* 号码池为同一 split_link_id 下全部 status=normal 的号码;
* random_shuffle 关闭时严格轮转,开启时每次访问随机选号。
*/
class SplitRedirectService
{
@@ -59,7 +62,7 @@ class SplitRedirectService
}
$linkId = (int) $link->getAttr('id');
$index = $this->roundRobinStore->nextIndex($linkId, $count);
$index = SplitNumberWeighService::resolvePickIndex($linkId, $count, $this->roundRobinStore);
$list = $numbers instanceof \think\Collection ? $numbers->all() : (array) $numbers;
$picked = $list[$index] ?? $list[0] ?? null;
if ($picked === null) {
@@ -11,11 +11,16 @@ use think\Db;
/**
* 工单同步结果写入号码表
*
* 同步策略(全量镜像):
* - 爬虫返回的号码:写入 platform_status / inbound_count,开关 status 与云控在线状态一致
* - 爬虫未返回且 manual_manage=0:物理删除
* - manual_manage=1:不删除,仅当仍在爬虫结果中时更新 platform_status
*/
class SplitTicketNumberSyncService
{
/**
* 将蜘蛛返回的号码列表同步到号码管理
* 将蜘蛛返回的号码列表同步到号码管理(全量镜像,以爬虫为准)
*/
public function syncFromUnifiedData(Ticket $ticket, UnifiedScrmData $data): void
{
@@ -61,7 +66,7 @@ class SplitTicketNumberSyncService
$newFollowers = (int) ($row['newFollowersToday'] ?? 0);
if (isset($existingMap[$number])) {
$this->updateExistingNumber($existingMap[$number], $platformStatus, $newFollowers);
$this->updateExistingNumber($ticket, $existingMap[$number], $platformStatus, $newFollowers);
continue;
}
@@ -87,6 +92,7 @@ class SplitTicketNumberSyncService
}
}
$deleteIds = [];
foreach ($existingMap as $number => $item) {
if (isset($syncedNumberSet[$number])) {
continue;
@@ -94,18 +100,58 @@ class SplitTicketNumberSyncService
if ((int) $item['manual_manage'] === 1) {
continue;
}
Number::where('id', (int) $item['id'])->update([
'status' => 'hidden',
$deleteIds[] = (int) $item['id'];
}
if ($deleteIds !== []) {
Number::where('id', 'in', $deleteIds)->delete();
}
}
/**
* 工单手动开启等非同步场景:按已存的 platform_status 对齐开关(不跑单号上限/下号比率)
*/
public function applyStatusFromPlatformSnapshot(Ticket $ticket): void
{
if ((string) ($ticket['status'] ?? 'hidden') !== 'normal') {
return;
}
$numbers = Number::where('admin_id', (int) $ticket['admin_id'])
->where('split_link_id', (int) $ticket['split_link_id'])
->where('ticket_name', (string) $ticket['ticket_name'])
->where('manual_manage', 0)
->select();
foreach ($numbers as $number) {
$platformStatus = (string) ($number['platform_status'] ?? 'offline');
Number::where('id', (int) $number['id'])->update([
'status' => self::resolveSwitchStatus($ticket, $platformStatus),
'updatetime' => time(),
]);
}
}
/**
* 云控在线 开启;离线 关闭;工单关闭时一律关闭
*/
public static function resolveSwitchStatus(Ticket $ticket, string $platformStatus): string
{
if ((string) ($ticket['status'] ?? 'hidden') !== 'normal') {
return 'hidden';
}
return $platformStatus === 'online' ? 'normal' : 'hidden';
}
/**
* @param Number $row
*/
private function updateExistingNumber($row, string $platformStatus, int $newFollowers): void
{
private function updateExistingNumber(
Ticket $ticket,
$row,
string $platformStatus,
int $newFollowers
): void {
$update = [
'platform_status' => $platformStatus,
'updatetime' => time(),
@@ -116,8 +162,8 @@ class SplitTicketNumberSyncService
return;
}
// 进线人数由同步写入,最终开关由 applyNumberRules 统一判定(单号上限/下号比率等)
$update['inbound_count'] = max(0, $newFollowers);
$update['status'] = self::resolveSwitchStatus($ticket, $platformStatus);
Number::where('id', (int) $row['id'])->update($update);
}
@@ -139,7 +185,7 @@ class SplitTicketNumberSyncService
'inbound_count' => max(0, $newFollowers),
'manual_manage' => 0,
'platform_status' => $platformStatus,
'status' => 'hidden',
'status' => self::resolveSwitchStatus($ticket, $platformStatus),
'createtime' => $now,
'updatetime' => $now,
];
@@ -148,10 +194,11 @@ class SplitTicketNumberSyncService
Db::name('split_number')->insert($data);
} catch (\Throwable $e) {
$exists = Number::where('split_link_id', (int) $ticket['split_link_id'])
->where('ticket_name', (string) $ticket['ticket_name'])
->where('number', $number)
->find();
if ($exists) {
$this->updateExistingNumber($exists, $platformStatus, $newFollowers);
$this->updateExistingNumber($ticket, $exists, $platformStatus, $newFollowers);
}
}
}
@@ -13,7 +13,7 @@ use app\admin\model\split\Ticket;
class SplitTicketRuleService
{
/**
* 同步后应用全部规则并写回工单/号码
* 同步后应用工单开关规则;号码开关以爬虫/platform 快照为准(不再跑单号上限/下号比率)
*/
public function applyAfterSync(Ticket $ticket, int $completeCount): void
{
@@ -22,8 +22,9 @@ class SplitTicketRuleService
if ($fresh) {
if ((string) $fresh['status'] === 'hidden') {
$this->cascadeTicketClosedToNumbers($fresh);
} else {
(new SplitTicketNumberSyncService())->applyStatusFromPlatformSnapshot($fresh);
}
$this->applyNumberRules($fresh);
}
}
@@ -69,7 +70,7 @@ class SplitTicketRuleService
}
/**
* 手动切换工单状态时,联动非手动管理的号码
* 手动切换工单状态时,非手动号码 platform_status 对齐开关
*/
public function syncNumbersWithTicketStatus(Ticket $ticket): void
{
@@ -78,17 +79,17 @@ class SplitTicketRuleService
$this->cascadeTicketClosedToNumbers($ticket);
return;
}
$this->applyNumberRules($ticket);
(new SplitTicketNumberSyncService())->applyStatusFromPlatformSnapshot($ticket);
}
/**
* 工单处于开启状态时,将云控在线的非手动号码设为开启
*
* @deprecated 请使用 applyNumberRules(会校验单号上限/下号比率
* @deprecated 请使用 applyStatusFromPlatformSnapshot(以 platform_status 为准
*/
public function syncOnlineNumbersWhenTicketOpen(Ticket $ticket): void
{
$this->applyNumberRules($ticket);
(new SplitTicketNumberSyncService())->applyStatusFromPlatformSnapshot($ticket);
}
/**
@@ -297,7 +297,6 @@ class SplitTicketSyncService
if ((string) $freshTicket['status'] === 'hidden') {
$this->ruleService->cascadeTicketClosedToNumbers($freshTicket);
}
$this->ruleService->applyNumberRules($freshTicket);
$ticket = $freshTicket;
$inboundCount = $this->numberSync->sumInboundForTicket($ticket);
@@ -26,12 +26,16 @@ function normalizeAntiBot(antiBot) {
profile: antiBot.profile === 'real' ? 'real' : 'standard',
turnstile: antiBot.turnstile !== false,
solverFallback: antiBot.solverFallback !== false,
cfClearanceRequired: antiBot.cfClearanceRequired !== false,
sessionKey: antiBot.sessionKey || '',
challengeTimeoutMs: antiBot.challengeTimeoutMs || 60000,
postCfReadyUrlContains: antiBot.postCfReadyUrlContains || '',
postCfReadySelector: antiBot.postCfReadySelector || '',
postCfReadyTimeoutMs: antiBot.postCfReadyTimeoutMs || 0,
postCfSettleMs: antiBot.postCfSettleMs || 0,
postCfSpaWaitMs: antiBot.postCfSpaWaitMs || 0,
businessHostHint: antiBot.businessHostHint || '',
landingUrlHint: antiBot.landingUrlHint || '',
};
}
+31 -12
View File
@@ -1,5 +1,8 @@
/**
* Cloudflare / Turnstile 挑战页检测
*
* antiBot.cfClearanceRequired === falseA2C 业务域仅真实 CF 中间页算 blocked
* 不因页面引用 turnstile.js 或缺少 cf_clearance cookie 误判
*/
/**
@@ -17,6 +20,14 @@ function urlIndicatesCfChallenge(url) {
|| url.includes('/cdn-cgi/challenge');
}
/**
* 是否要求 cf_clearance 才视为过盾完成A2C user.a2c.chat false
* @param {object|null|undefined} antiBot
*/
function isCfClearanceRequired(antiBot) {
return antiBot?.cfClearanceRequired !== false;
}
/**
* @typedef {Object} CloudflareState
* @property {boolean} blocked 是否处于 CF 挑战中
@@ -29,9 +40,10 @@ function urlIndicatesCfChallenge(url) {
/**
* 检测页面是否被 Cloudflare 拦截
* @param {import('puppeteer').Page} page
* @param {object|null} [antiBot]
* @returns {Promise<CloudflareState>}
*/
async function detectCloudflare(page) {
async function detectCloudflare(page, antiBot = null) {
const url = page.url();
const title = await page.title().catch(() => '');
@@ -61,22 +73,28 @@ async function detectCloudflare(page) {
const urlChallenge = urlIndicatesCfChallenge(url);
const realChallenge = urlChallenge
|| domSignals.titleHasJustAMoment
|| domSignals.bodyHasJustAMoment
|| domSignals.hasChallengeRunning
|| title.includes('Just a moment');
const turnstileSignals = domSignals.hasTurnstileInput || domSignals.hasTurnstileWidget;
let challengeType = null;
if (domSignals.hasTurnstileInput || domSignals.hasTurnstileWidget || url.includes('__cf_chl')) {
if (turnstileSignals || url.includes('__cf_chl')) {
challengeType = 'turnstile';
} else if (urlChallenge || domSignals.titleHasJustAMoment || domSignals.bodyHasJustAMoment) {
} else if (realChallenge) {
challengeType = 'js_challenge';
}
const blocked = !hasCfClearance && (
urlChallenge
|| domSignals.titleHasJustAMoment
|| domSignals.bodyHasJustAMoment
|| domSignals.hasTurnstileInput
|| domSignals.hasTurnstileWidget
|| domSignals.hasChallengeRunning
|| title.includes('Just a moment')
);
const requireClearance = isCfClearanceRequired(antiBot);
let blocked;
if (requireClearance) {
blocked = !hasCfClearance && (realChallenge || turnstileSignals);
} else {
blocked = realChallenge;
}
return {
blocked,
@@ -110,4 +128,5 @@ module.exports = {
detectCloudflare,
isTurnstileSolved,
urlIndicatesCfChallenge,
isCfClearanceRequired,
};
+17 -10
View File
@@ -5,7 +5,7 @@
* 否则清除可能过期的 cf_clearance 并重新触发 TurnstileWhatshub 短链 work-order-sharing 场景
*/
const { detectCloudflare } = require('./cf-detector');
const { waitForTurnstile, extractTurnstileSitekey, injectTurnstileToken } = require('./turnstile-handler');
const { waitForTurnstile, extractTurnstileSitekey, injectTurnstileToken, waitForTurnstileSurface } = require('./turnstile-handler');
const { solveTurnstile, isCaptchaConfigured } = require('./captcha-solver');
const { isSessionLikelyValid } = require('./session-store');
@@ -83,14 +83,14 @@ async function tryPostCfRecoverViaReload(page, waitForUrlSettled, timeoutMs, ant
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
* @param {number} timeoutMs
*/
async function invalidateStaleCfAndReload(page, waitForUrlSettled, timeoutMs) {
async function invalidateStaleCfAndReload(page, waitForUrlSettled, timeoutMs, antiBot = null) {
console.log('[CF] 清除 cf_clearance 并 reload,重新触发 Turnstile');
await clearCfClearanceCookies(page);
await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 45000) });
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
return detectCloudflare(page);
return detectCloudflare(page, antiBot);
}
/**
@@ -146,7 +146,7 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
const started = Date.now();
const timeoutMs = antiBot.challengeTimeoutMs || 60000;
let cfState = await detectCloudflare(page);
let cfState = await detectCloudflare(page, antiBot);
if (!cfState.blocked && cfState.hasCfClearance) {
const fast = await trySessionReuseFastPath(
@@ -155,7 +155,7 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
if (fast) {
return fast;
}
cfState = await invalidateStaleCfAndReload(page, waitForUrlSettled, timeoutMs);
cfState = await invalidateStaleCfAndReload(page, waitForUrlSettled, timeoutMs, antiBot);
}
if (!cfState.blocked) {
@@ -188,7 +188,7 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
}
if (cfState.hasCfClearance) {
cfState = await invalidateStaleCfAndReload(page, waitForUrlSettled, timeoutMs);
cfState = await invalidateStaleCfAndReload(page, waitForUrlSettled, timeoutMs, antiBot);
} else {
return {
success: true,
@@ -209,7 +209,7 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
cfState = await detectCloudflare(page);
cfState = await detectCloudflare(page, antiBot);
if (!cfState.blocked) {
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
@@ -237,7 +237,7 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
cfState = await detectCloudflare(page);
cfState = await detectCloudflare(page, antiBot);
if (!cfState.blocked) {
return {
success: true,
@@ -254,8 +254,15 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
if (antiBot.solverFallback !== false && isCaptchaConfigured()) {
try {
const sitekey = await extractTurnstileSitekey(page);
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
let sitekey = await waitForTurnstileSurface(page, Math.min(timeoutMs, 25000));
if (!sitekey) {
sitekey = await extractTurnstileSitekey(page);
}
const pageUrl = page.url();
console.log(`[CF] Captcha API 求解 pageUrl=${pageUrl} sitekey=${sitekey ? 'ok' : 'missing'}`);
const { token, provider } = await solveTurnstile({ pageUrl, sitekey });
console.log(`[CF] Captcha API (${provider}) 返回 token,注入页面`);
await injectTurnstileToken(page, token);
@@ -267,7 +274,7 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
await waitForUrlSettled(page).catch(() => {});
}
cfState = await detectCloudflare(page);
cfState = await detectCloudflare(page, antiBot);
if (!cfState.blocked) {
return {
success: true,
+112 -21
View File
@@ -1,5 +1,5 @@
/**
* Turnstile 内置求解等待 widget 自动完成 + token 轮询
* Turnstile 内置求解等待 widget 自动完成 + token 轮询 + frame sitekey 提取
*/
const { detectCloudflare, isTurnstileSolved, urlIndicatesCfChallenge } = require('./cf-detector');
@@ -10,6 +10,115 @@ const { detectCloudflare, isTurnstileSolved, urlIndicatesCfChallenge } = require
* @property {number} elapsedMs
*/
/**
* 在单个 frame 内提取 sitekey
* @param {import('puppeteer').Frame} frame
* @returns {Promise<string|null>}
*/
async function extractSitekeyFromFrame(frame) {
return frame.evaluate(() => {
const widget = document.querySelector('.cf-turnstile[data-sitekey], [data-sitekey]');
if (widget) {
const key = widget.getAttribute('data-sitekey');
if (key) {
return key;
}
}
const iframes = document.querySelectorAll('iframe[src*="challenges.cloudflare.com"], iframe[src*="challenge-platform"]');
for (const iframe of iframes) {
const src = iframe.getAttribute('src') || '';
const match = src.match(/[?&]sitekey=([^&]+)/i);
if (match) {
return decodeURIComponent(match[1]);
}
}
const scripts = document.querySelectorAll('script');
for (const script of scripts) {
const text = script.textContent || '';
const match = text.match(/sitekey['"\s:=]+['"]([0-9x_a-zA-Z-]{10,})['"]/i)
|| text.match(/data-sitekey=['"]([0-9x_a-zA-Z-]{10,})['"]/i);
if (match) {
return match[1];
}
}
return null;
}).catch(() => null);
}
/**
* 从主文档及所有子 frame 提取 Turnstile sitekey
* @param {import('puppeteer').Page} page
* @returns {Promise<string|null>}
*/
async function extractTurnstileSitekey(page) {
const mainKey = await extractSitekeyFromFrame(page.mainFrame());
if (mainKey) {
return mainKey;
}
for (const frame of page.frames()) {
if (frame === page.mainFrame()) {
continue;
}
const frameKey = await extractSitekeyFromFrame(frame);
if (frameKey) {
return frameKey;
}
}
return null;
}
/**
* Captcha API 调用前等待 Turnstile 挑战面出现或 URL 进入 CF 挑战态
*
* @param {import('puppeteer').Page} page
* @param {number} [timeoutMs]
* @returns {Promise<string|null>} 若已提取到 sitekey 则返回否则 null
*/
async function waitForTurnstileSurface(page, timeoutMs = 20000) {
const started = Date.now();
const pollMs = 400;
while (Date.now() - started < timeoutMs) {
const sitekey = await extractTurnstileSitekey(page);
if (sitekey) {
console.log(`[CF] 已检测到 Turnstile sitekey (${sitekey.slice(0, 8)}...)`);
return sitekey;
}
const url = page.url();
if (urlIndicatesCfChallenge(url)) {
await new Promise((r) => setTimeout(r, pollMs));
continue;
}
const state = await detectCloudflare(page);
if (!state.blocked && state.hasCfClearance) {
return null;
}
const hasSurface = await page.evaluate(() => {
return !!document.querySelector(
'.cf-turnstile, [data-sitekey], [name="cf-turnstile-response"], '
+ 'iframe[src*="challenges.cloudflare.com"], iframe[src*="challenge-platform"]'
);
}).catch(() => false);
if (hasSurface) {
await new Promise((r) => setTimeout(r, pollMs));
continue;
}
await new Promise((r) => setTimeout(r, pollMs));
}
return null;
}
/**
* 等待 Turnstile 通过内置点击 / 自动跳转
* @param {import('puppeteer').Page} page
@@ -49,26 +158,6 @@ async function waitForTurnstile(page, options = {}) {
return { success: false, stage: 'timeout', elapsedMs: Date.now() - started };
}
/**
* 从页面提取 Turnstile sitekey
* @param {import('puppeteer').Page} page
* @returns {Promise<string|null>}
*/
async function extractTurnstileSitekey(page) {
return page.evaluate(() => {
const widget = document.querySelector('.cf-turnstile[data-sitekey], [data-sitekey]');
if (widget) {
return widget.getAttribute('data-sitekey');
}
const iframe = document.querySelector('iframe[src*="challenges.cloudflare.com"]');
if (iframe && iframe.src) {
const match = iframe.src.match(/[?&]sitekey=([^&]+)/);
if (match) return decodeURIComponent(match[1]);
}
return null;
}).catch(() => null);
}
/**
* Captcha API 返回的 token 注入页面
* @param {import('puppeteer').Page} page
@@ -91,6 +180,8 @@ async function injectTurnstileToken(page, token) {
module.exports = {
waitForTurnstile,
waitForTurnstileSurface,
extractTurnstileSitekey,
extractSitekeyFromFrame,
injectTurnstileToken,
};
+778 -42
View File
@@ -21,6 +21,7 @@ const {
NAVIGATION_TIMEOUT_MS,
NAVIGATION_WAIT_UNTIL,
NAVIGATION_MAX_RETRIES,
PAGE_DEFAULT_TIMEOUT_MS,
SESSION_TTL_MS,
PERSIST_BROWSER_PROFILE,
POOL_IDLE_MS,
@@ -39,18 +40,22 @@ const {
const { launchStandardBrowser } = require('./lib/launch-standard');
const { isRealBrowserAvailable } = require('./lib/launch-real-browser');
const { handleCloudflareChallenge, isCaptchaConfigured } = require('./lib/cf-handler');
const { detectCloudflare, urlIndicatesCfChallenge } = require('./lib/cf-detector');
const {
extractTurnstileSitekey,
injectTurnstileToken,
waitForTurnstileSurface,
} = require('./lib/turnstile-handler');
const { solveTurnstile } = require('./lib/captcha-solver');
const {
saveSession,
getSessionDebugInfo,
getSessionStats,
isSessionLikelyValid,
} = require('./lib/session-store');
const { resolveSessionContext, touchSessionIfPresent } = require('./lib/session-context');
const { getProfileStats, profileExists } = require('./lib/profile-dir');
const { runUiPagination } = require('./lib/ui-pagination-runner');
const {
awaitPostCfBusinessReady,
executeAuthActionsWithDiagnostics,
} = require('./lib/post-cf-ready');
const app = express();
app.use(express.json({ limit: '50mb' }));
@@ -170,6 +175,719 @@ async function waitForUrlSettled(page, { maxAttempts = 15, stableMs = 500, redir
return currentUrl;
}
/** 合并 antiBot 扩展字段(postCfReady* 由 PHP 按工单类型传入,未配置则跳过) */
function resolveAntiBot(rawAntiBot) {
const base = normalizeAntiBot(rawAntiBot);
if (!rawAntiBot || !base.enabled) {
return base;
}
return {
...base,
postCfReadyUrlContains: rawAntiBot.postCfReadyUrlContains || '',
postCfReadySelector: rawAntiBot.postCfReadySelector || '',
postCfReadyTimeoutMs: rawAntiBot.postCfReadyTimeoutMs || 0,
postCfSettleMs: rawAntiBot.postCfSettleMs || 0,
postCfSpaWaitMs: rawAntiBot.postCfSpaWaitMs || 0,
cfClearanceRequired: rawAntiBot.cfClearanceRequired !== false,
businessHostHint: rawAntiBot.businessHostHint || '',
landingUrlHint: rawAntiBot.landingUrlHint || '',
};
}
/**
* Whatshub 短链场景PHP 未传 postCfReady* Node 侧自动补全避免 session_reuse 15s auth 超时
* @param {object} antiBot
* @param {string} pageUrl
*/
function applyWhatshubPostCfDefaults(antiBot, pageUrl) {
if (!antiBot || !antiBot.enabled) {
return antiBot;
}
if ((antiBot.postCfReadyUrlContains || '').trim()) {
return antiBot;
}
try {
const parsed = new URL(pageUrl || '');
if (!parsed.hostname.includes('whatshub.cc') || !parsed.pathname.includes('/m/')) {
return antiBot;
}
console.log('[CF] Whatshub 短链自动启用 postCfReady 默认配置');
return {
...antiBot,
postCfReadyUrlContains: 'work-order-sharing',
postCfReadySelector: '.el-input__inner',
postCfReadyTimeoutMs: antiBot.postCfReadyTimeoutMs || 30000,
postCfSettleMs: antiBot.postCfSettleMs || 500,
postCfSpaWaitMs: antiBot.postCfSpaWaitMs || 8000,
};
} catch (_) {
return antiBot;
}
}
/**
* A2C 短链 user.a2c.chatPHP 未传 postCfReady* Node 侧自动补全
* @param {object} antiBot
* @param {string} pageUrl
*/
function applyA2cPostCfDefaults(antiBot, pageUrl) {
if (!antiBot || !antiBot.enabled) {
return antiBot;
}
if ((antiBot.postCfReadyUrlContains || '').trim()) {
return antiBot;
}
let shouldApply = false;
const sessionKey = String(antiBot.sessionKey || '').toLowerCase();
if (sessionKey.startsWith('a2c:')) {
shouldApply = true;
}
try {
const parsed = new URL(pageUrl || '');
if (parsed.hostname.includes('a2c.chat')) {
shouldApply = true;
}
} catch (_) {
/* ignore */
}
if (!shouldApply) {
return antiBot;
}
console.log('[CF] A2C 自动启用 postCfReady 默认配置');
return {
...antiBot,
postCfReadyUrlContains: '/visitors/counter/share',
postCfReadySelector: '.btn-next',
postCfReadyTimeoutMs: antiBot.postCfReadyTimeoutMs || 30000,
postCfSettleMs: antiBot.postCfSettleMs || 500,
postCfSpaWaitMs: antiBot.postCfSpaWaitMs || 4000,
cfClearanceRequired: antiBot.cfClearanceRequired === undefined ? false : antiBot.cfClearanceRequired,
solverFallback: antiBot.solverFallback === undefined ? false : antiBot.solverFallback,
};
}
/** 按 pageUrl 合并 antiBot(含 Whatshub / A2C 默认 postCf */
function resolveAntiBotForPage(rawAntiBot, pageUrl) {
return applyA2cPostCfDefaults(
applyWhatshubPostCfDefaults(resolveAntiBot(rawAntiBot), pageUrl),
pageUrl
);
}
function isBusinessUrlReady(page, urlNeedle) {
return !urlNeedle || page.url().includes(urlNeedle);
}
/** Whatshub 分享短链 /m/xxx/1 */
function isWhatshubShortLinkPath(url) {
try {
const parsed = new URL(url || '');
return parsed.hostname.includes('whatshub.cc') && parsed.pathname.includes('/m/');
} catch (_) {
return false;
}
}
/**
* Whatshub 短链 + Real Browser不覆盖 UA不预注入磁盘 cf_clearance
* @param {string} pageUrl
* @param {object} antiBot
* @param {object[]} mergedCookies
* @param {object} extraOpts
*/
function buildManagedPageOptions(pageUrl, antiBot, mergedCookies, extraOpts = {}) {
const opts = { ...extraOpts };
const isWhatshubShort = isWhatshubShortLinkPath(pageUrl);
const isRealProfile = antiBot?.profile === 'real';
if (isWhatshubShort && isRealProfile && antiBot?.enabled) {
opts.skipUserAgent = true;
const filtered = (mergedCookies || []).filter((c) => c.name !== 'cf_clearance');
if (filtered.length < (mergedCookies || []).length) {
console.log('[Whatshub] 短链首访:跳过磁盘 cf_clearance 预注入,避免 SPA 路由未触发');
}
opts.cookies = filtered;
return opts;
}
opts.cookies = mergedCookies;
if (!opts.skipUserAgent && !opts.userAgent) {
opts.userAgent = DEFAULT_UA;
}
return opts;
}
/**
* 任务页创建 Whatshub skipUserAgentbrowser-factory 无写权限时在 server 侧实现
*/
async function createTaskPage(browser, existingPage, pageUrl, antiBot, mergedCookies, extraOpts = {}) {
const opts = buildManagedPageOptions(pageUrl, antiBot, mergedCookies, extraOpts);
const page = existingPage || await browser.newPage();
page.setDefaultNavigationTimeout(NAVIGATION_TIMEOUT_MS);
page.setDefaultTimeout(PAGE_DEFAULT_TIMEOUT_MS);
if (!opts.skipUserAgent) {
await page.setUserAgent(opts.userAgent || DEFAULT_UA);
}
if (opts.viewport) {
await page.setViewport(opts.viewport);
}
if (opts.cookies && opts.cookies.length > 0) {
await page.setCookie(...opts.cookies);
}
page.on('error', (err) => console.error('[Page Error]', err.message));
page.on('pageerror', (err) => console.error('[Page JS Error]', err.message));
return page;
}
/** CF 已过但仍在短链、未到业务页(Whatshub / A2C yyk.ink 等) */
function isOnShortLinkNotBusiness(page, urlNeedle) {
const url = page.url();
if (urlNeedle && url.includes(urlNeedle)) {
return false;
}
if (isWhatshubShortLinkPath(url)) {
return true;
}
if (urlNeedle && urlNeedle.includes('/visitors/counter/share')) {
try {
return !new URL(url).hostname.includes('a2c.chat');
} catch (_) {
return false;
}
}
return false;
}
/**
* CF 完成后等待 Vue SPA 从短链跳转到 work-order-sharing对齐 debug 脚本 goto+8s
* Whatshub二次 goto 短链保留 clearance禁止 reload会重新触发 __cf_chl_rt_tk
*/
async function waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettledFn, pageUrl = '') {
const urlNeedle = (antiBot.postCfReadyUrlContains || '').trim();
const targetUrl = pageUrl || page.url();
if (!urlNeedle || !isOnShortLinkNotBusiness(page, urlNeedle)) {
return isBusinessUrlReady(page, urlNeedle);
}
if (!(await isCfTrulyComplete(page, antiBot))) {
return false;
}
const spaWaitMs = Math.max(8000, antiBot.postCfSpaWaitMs || 8000);
const isWhatshub = isWhatshubShortLinkPath(targetUrl);
console.log(
`[CF] CF 已完成,等待 SPA 跳转 settle=${spaWaitMs}ms current=${page.url()}`
);
await new Promise((resolve) => setTimeout(resolve, spaWaitMs));
if (typeof waitForUrlSettledFn === 'function') {
await waitForUrlSettledFn(page).catch(() => {});
}
await page.waitForNetworkIdle({ idleTime: 500, timeout: 15000 }).catch(() => {});
if (isBusinessUrlReady(page, urlNeedle)) {
console.log(`[CF] SPA 已跳转至业务页 url=${page.url()}`);
return true;
}
if (isWhatshub && isWhatshubShortLinkPath(targetUrl)) {
console.log(`[CF] Whatshub 二次 goto 触发 SPA 路由(保留 clearanceurl=${targetUrl}`);
await navigateToPage(page, targetUrl);
if (typeof waitForUrlSettledFn === 'function') {
await waitForUrlSettledFn(page).catch(() => {});
}
if (!(await isCfTrulyComplete(page, antiBot))) {
console.log('[CF] 二次 goto 后 CF 未完成,等待过盾...');
const cfWait = await waitForCfTrulyComplete(page, antiBot.challengeTimeoutMs || 60000, 500, antiBot);
if (!cfWait.success) {
console.log(`[CF] SPA 等待后仍未到业务页 url=${page.url()}`);
return false;
}
}
console.log(`[CF] Whatshub 二次 goto 后 SPA 等待 settle=${spaWaitMs}ms current=${page.url()}`);
await new Promise((resolve) => setTimeout(resolve, spaWaitMs));
await page.waitForNetworkIdle({ idleTime: 500, timeout: 15000 }).catch(() => {});
if (isBusinessUrlReady(page, urlNeedle)) {
console.log(`[CF] 二次 goto 后 SPA 已跳转 url=${page.url()}`);
return true;
}
}
console.log(`[CF] SPA 等待后仍未到业务页 url=${page.url()}`);
return false;
}
/** 非 Whatshub 场景:软 reloadWhatshub 短链改用二次 goto */
async function softReloadForSpa(page, waitForUrlSettledFn, antiBot, pageUrl = '') {
const targetUrl = pageUrl || page.url();
if (isWhatshubShortLinkPath(targetUrl)) {
console.log(`[CF] Whatshub 跳过 soft reload,改用二次 goto url=${targetUrl}`);
return waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettledFn, targetUrl);
}
console.log(`[CF] 尝试软 reload(保留 clearanceurl=${page.url()}`);
await page.reload({ waitUntil: 'domcontentloaded', timeout: NAVIGATION_TIMEOUT_MS }).catch(() => {});
if (typeof waitForUrlSettledFn === 'function') {
await waitForUrlSettledFn(page).catch(() => {});
}
return waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettledFn, targetUrl);
}
/** CF 是否真正完成:默认需 cf_clearanceA2C 等 cfClearanceRequired=false 时仅看真实挑战 */
async function isCfTrulyComplete(page, antiBot = null) {
if (urlIndicatesCfChallenge(page.url())) {
return false;
}
if (antiBot && antiBot.cfClearanceRequired === false) {
const state = await detectCloudflare(page, antiBot);
return !state.blocked;
}
const cookies = await page.cookies();
if (!cookies.some((c) => c.name === 'cf_clearance')) {
return false;
}
const state = await detectCloudflare(page, antiBot);
return !state.blocked;
}
/** 轮询直到 CF 真正完成(不依赖 turnstile-handler 的 early return */
async function waitForCfTrulyComplete(page, timeoutMs, pollMs = 500, antiBot = null) {
const started = Date.now();
while (Date.now() - started < timeoutMs) {
if (await isCfTrulyComplete(page, antiBot)) {
return { success: true, elapsedMs: Date.now() - started };
}
await new Promise((resolve) => setTimeout(resolve, pollMs));
}
return { success: false, elapsedMs: Date.now() - started };
}
/** 校验 cfResult,防止 handleCloudflareChallenge 假 success */
async function finalizeCfResult(page, cfResult, antiBot = null) {
if (!cfResult || !cfResult.success) {
return cfResult;
}
if (await isCfTrulyComplete(page, antiBot)) {
return cfResult;
}
console.log(`[CF] 过盾返回 success 但 CF 未真正完成 url=${page.url()}`);
return {
success: false,
code: 'CF_TURNSTILE_FAILED',
challengeType: cfResult.challengeType || 'turnstile',
stage: 'incomplete',
solverUsed: cfResult.solverUsed || false,
elapsedMs: cfResult.elapsedMs || 0,
cfDetected: true,
};
}
/**
* 完整过盾内置 Real Browser Turnstile 轮询 Captcha API 兜底
*/
async function runCfChallengeFlow(page, antiBot, waitForUrlSettled, pageUrl = '') {
const started = Date.now();
const timeoutMs = Math.min(Math.max(Number(antiBot.challengeTimeoutMs) || 60000, 5000), 120000);
const navUrl = pageUrl || page.url();
console.log(`[CF] 开始过盾 url=${page.url()}`);
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
if (antiBot.cfClearanceRequired === false) {
const preState = await detectCloudflare(page, antiBot);
if (!preState.blocked) {
console.log('[CF] 免 clearance 模式:无真实 CF 挑战,跳过 Turnstile/CapSolver');
await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettled, navUrl);
return {
success: true,
code: null,
challengeType: null,
stage: 'no_clearance_required',
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: false,
};
}
console.log(`[CF] 免 clearance 模式但检测到真实 CF 挑战 type=${preState.challengeType},继续过盾`);
}
if (antiBot.turnstile !== false) {
console.log(`[CF] 内置 Turnstile 轮询 timeout=${timeoutMs}ms`);
const builtIn = await waitForCfTrulyComplete(page, timeoutMs, 500, antiBot);
if (builtIn.success) {
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettled, navUrl);
return {
success: true,
code: null,
challengeType: 'turnstile',
stage: 'built_in',
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: true,
};
}
console.warn(`[CF] 内置 Turnstile 未在 ${builtIn.elapsedMs}ms 内完成,尝试 Captcha API`);
}
if (antiBot.solverFallback !== false && isCaptchaConfigured()) {
try {
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
let sitekey = await waitForTurnstileSurface(page, Math.min(timeoutMs, 25000));
if (!sitekey) {
sitekey = await extractTurnstileSitekey(page);
}
const pageUrlForSolver = page.url();
console.log(`[CF] Captcha API 求解 pageUrl=${pageUrlForSolver} sitekey=${sitekey ? 'ok' : 'missing'}`);
const { token, provider } = await solveTurnstile({ pageUrl: pageUrlForSolver, sitekey });
console.log(`[CF] Captcha API (${provider}) 返回 token,注入页面`);
await injectTurnstileToken(page, token);
const apiWait = await waitForCfTrulyComplete(page, Math.min(timeoutMs, 45000), 500, antiBot);
if (apiWait.success) {
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettled, navUrl);
return {
success: true,
code: null,
challengeType: 'turnstile',
stage: 'api',
solverUsed: true,
elapsedMs: Date.now() - started,
cfDetected: true,
};
}
console.warn('[CF] Captcha API token 注入后仍未获得 cf_clearance');
} catch (apiErr) {
console.error('[CF] Captcha API 兜底失败:', apiErr.message);
return {
success: false,
code: 'CF_TURNSTILE_FAILED',
challengeType: 'turnstile',
stage: 'api',
solverUsed: true,
elapsedMs: Date.now() - started,
cfDetected: true,
};
}
}
return {
success: false,
code: 'CF_TURNSTILE_FAILED',
challengeType: 'turnstile',
stage: antiBot.solverFallback !== false && !isCaptchaConfigured() ? 'built_in' : 'timeout',
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: true,
};
}
/** 安全 CF 处理:未真正完成则走完整过盾,禁止 cf-handler 假过关 */
async function handleCloudflareChallengeSafe(page, antiBot, waitForUrlSettled, storedSession) {
if (!(await isCfTrulyComplete(page, antiBot))) {
return finalizeCfResult(page, await runCfChallengeFlow(page, antiBot, waitForUrlSettled), antiBot);
}
const result = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled, storedSession);
return finalizeCfResult(page, result, antiBot);
}
/** 是否应 purge 陈旧 clearance(短链 SPA 未跳转时禁止 purge) */
function shouldPurgeStaleClearance(page, cfState, urlNeedle = '') {
if (urlIndicatesCfChallenge(page.url())) {
return false;
}
if (urlNeedle && isOnShortLinkNotBusiness(page, urlNeedle) && cfState.hasCfClearance) {
return false;
}
return cfState.hasCfClearance && !cfState.blocked;
}
/** 业务页等待前确保 CF 已完成 */
async function ensureCfCompleteBeforeBusinessReady(page, antiBot, waitForUrlSettledFn, storedSession) {
if (await isCfTrulyComplete(page, antiBot)) {
return;
}
const cfState = await detectCloudflare(page, antiBot);
console.log(`[CF] 业务页等待前 CF 未完成 blocked=${cfState.blocked} url=${page.url()}`);
const cfResult = await handleCloudflareChallengeSafe(page, antiBot, waitForUrlSettledFn, storedSession);
if (!cfResult.success) {
const err = new Error('Cloudflare Turnstile 验证失败(业务页等待前)');
err.code = 'CF_TURNSTILE_FAILED';
throw err;
}
if (typeof waitForUrlSettledFn === 'function') {
await waitForUrlSettledFn(page).catch(() => {});
}
}
/** 清除 cf_clearance(含 Chrome Profile / CDP 层) */
async function clearCfClearanceCookies(page) {
const cookies = await page.cookies();
for (const c of cookies) {
if (c.name !== 'cf_clearance') {
continue;
}
const host = (c.domain || '').replace(/^\./, '');
if (host) {
await page.deleteCookie({ name: c.name, url: `https://${host}/` }).catch(() => {});
}
await page.deleteCookie({
name: c.name,
domain: c.domain,
path: c.path || '/',
}).catch(() => {});
}
}
async function purgeCfClearanceFully(page, pageUrl) {
await clearCfClearanceCookies(page);
let hostname = '';
try {
hostname = new URL(pageUrl).hostname;
} catch (_) {
return;
}
try {
const cdp = await page.createCDPSession();
const domains = new Set([hostname, `.${hostname}`]);
const cookies = await page.cookies();
for (const c of cookies) {
if (c.domain) {
domains.add(c.domain);
}
}
for (const domain of domains) {
await cdp.send('Network.deleteCookies', { name: 'cf_clearance', domain }).catch(() => {});
}
} catch (err) {
console.warn('[CF] CDP 清除 cf_clearance 失败:', err.message);
}
}
/**
* Whatshub 短链首访清除 Chrome Profile 内陈旧 cf_clearance确保 Turnstile+SPA 在同一 goto 生命周期
*/
async function prepareWhatshubShortLinkBeforeNav(page, pageUrl, antiBot) {
if (!antiBot?.enabled || antiBot.profile !== 'real' || !isWhatshubShortLinkPath(pageUrl)) {
return;
}
const cookies = await page.cookies();
if (!cookies.some((c) => c.name === 'cf_clearance')) {
return;
}
console.log('[Whatshub] 短链首访:清除 Profile/页面内陈旧 cf_clearance');
await purgeCfClearanceFully(page, pageUrl);
}
/**
* CF 处理 + 业务 URL 守卫磁盘 session 陈旧 clearance purge__cf_chl 中间态只过盾不 purge
*/
async function runCloudflareWithBusinessGuard(page, antiBot, waitForUrlSettled, storedSession, pageUrl) {
const urlNeedle = (antiBot.postCfReadyUrlContains || '').trim();
const hasStoredSession = !!(storedSession && isSessionLikelyValid(storedSession));
async function reNavigateWithCfPurge(label) {
if (urlIndicatesCfChallenge(page.url())) {
console.log(`[CF] ${label}: URL 含 __cf_chl,跳过 purge,直接过盾`);
return handleCloudflareChallengeSafe(page, antiBot, waitForUrlSettled, storedSession);
}
console.log(`[CF] ${label}: 清除陈旧 CF 并重新 goto ${pageUrl} (current=${page.url()})`);
await purgeCfClearanceFully(page, pageUrl);
await navigateToPage(page, pageUrl);
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
return handleCloudflareChallengeSafe(page, antiBot, waitForUrlSettled, storedSession);
}
let cfResult;
if (urlNeedle && !isBusinessUrlReady(page, urlNeedle) && hasStoredSession) {
const pre = await detectCloudflare(page, antiBot);
if (shouldPurgeStaleClearance(page, pre, urlNeedle)) {
cfResult = await reNavigateWithCfPurge('预检(磁盘会话-陈旧clearance)');
} else if (await isCfTrulyComplete(page, antiBot)) {
await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettled, pageUrl);
}
}
if (!cfResult) {
cfResult = await handleCloudflareChallengeSafe(page, antiBot, waitForUrlSettled, storedSession);
}
// 短链跨域落地(如 yyk.ink → user.a2c.chat)后,业务域可能仍有 CF
if (cfResult.success && !(await isCfTrulyComplete(page, antiBot))) {
const crossState = await detectCloudflare(page, antiBot);
if (crossState.blocked || urlIndicatesCfChallenge(page.url())) {
console.log(`[CF] 跨域落地后 CF 未完成,二次过盾 url=${page.url()}`);
cfResult = await finalizeCfResult(
page,
await runCfChallengeFlow(page, antiBot, waitForUrlSettled, page.url()),
antiBot
);
}
}
if (cfResult.success && urlNeedle && !isBusinessUrlReady(page, urlNeedle)) {
await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettled, pageUrl);
}
let retries = 0;
while (
cfResult.success
&& urlNeedle
&& !isBusinessUrlReady(page, urlNeedle)
&& retries < 2
) {
retries += 1;
if (!(await isCfTrulyComplete(page, antiBot))) {
console.log(`[CF] 业务页未达且 CF 未完成,完整过盾 重试#${retries}`);
cfResult = await finalizeCfResult(
page,
await runCfChallengeFlow(page, antiBot, waitForUrlSettled, pageUrl),
antiBot
);
} else {
console.log(`[CF] CF 已完成但未到业务页,等待 SPA 跳转 重试#${retries}`);
const spaOk = await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettled, pageUrl);
if (!spaOk && retries >= 2) {
await softReloadForSpa(page, waitForUrlSettled, antiBot, pageUrl);
}
if (isBusinessUrlReady(page, urlNeedle)) {
break;
}
}
if (!cfResult.success) {
break;
}
}
return finalizeCfResult(page, cfResult, antiBot);
}
async function collectPageDiagnostics(page) {
const url = page.url();
const snapshot = await page.evaluate(() => ({
url: window.location.href,
title: document.title || '',
bodyText: (document.body && document.body.innerText ? document.body.innerText : '').slice(0, 500),
elInputInner: document.querySelectorAll('.el-input__inner').length,
})).catch(() => null);
return snapshot || { url, title: '', bodyText: '', elInputInner: 0 };
}
async function awaitPostCfBusinessReady(page, antiBot, waitForUrlSettledFn, storedSession = null, pageUrl = '') {
if (!antiBot || !antiBot.enabled) {
return;
}
const urlContains = (antiBot.postCfReadyUrlContains || '').trim();
const selector = (antiBot.postCfReadySelector || '').trim();
if (urlContains === '' && selector === '') {
return;
}
const navUrl = pageUrl || page.url();
await ensureCfCompleteBeforeBusinessReady(page, antiBot, waitForUrlSettledFn, storedSession);
await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettledFn, navUrl);
const timeoutMs = Math.max(5000, antiBot.postCfReadyTimeoutMs || 30000);
console.log(
`[CF] 等待业务页就绪 current=${page.url()} urlContains="${urlContains}" `
+ `selector="${selector}" timeout=${timeoutMs}ms`
);
try {
if (urlContains !== '') {
await page.waitForFunction(
(needle) => window.location.href.includes(needle),
{ timeout: timeoutMs, polling: 200 },
urlContains
);
}
if (selector !== '') {
await page.waitForSelector(selector, { timeout: timeoutMs, visible: true });
}
} catch (err) {
const retryMs = Math.min(15000, timeoutMs);
const needsRetry = urlContains !== '' && !page.url().includes(urlContains);
if (needsRetry) {
if (!(await isCfTrulyComplete(page, antiBot))) {
console.warn(`[CF] 业务页等待失败且 CF 未完成 url=${page.url()},完整过盾后重试`);
try {
await ensureCfCompleteBeforeBusinessReady(page, antiBot, waitForUrlSettledFn, storedSession);
if (urlContains !== '') {
await page.waitForFunction(
(needle) => window.location.href.includes(needle),
{ timeout: retryMs, polling: 200 },
urlContains
);
}
if (selector !== '') {
await page.waitForSelector(selector, { timeout: retryMs, visible: true });
}
} catch (retryErr) {
retryErr.pageDiagnostics = await collectPageDiagnostics(page);
throw retryErr;
}
} else {
console.warn(`[CF] 业务页等待失败 url=${page.url()}SPA 等待 + 二次 goto`);
try {
await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettledFn, navUrl);
if (!page.url().includes(urlContains)) {
await softReloadForSpa(page, waitForUrlSettledFn, antiBot, navUrl);
}
if (urlContains !== '') {
await page.waitForFunction(
(needle) => window.location.href.includes(needle),
{ timeout: retryMs, polling: 200 },
urlContains
);
}
if (selector !== '') {
await page.waitForSelector(selector, { timeout: retryMs, visible: true });
}
} catch (retryErr) {
retryErr.pageDiagnostics = await collectPageDiagnostics(page);
throw retryErr;
}
}
} else {
err.pageDiagnostics = await collectPageDiagnostics(page);
throw err;
}
}
const settleMs = antiBot.postCfSettleMs || 0;
if (settleMs > 0) {
await new Promise((resolve) => setTimeout(resolve, settleMs));
}
console.log(`[CF] 业务页就绪 url=${page.url()}`);
}
async function executeAuthActionsWithDiagnostics(page, authActions) {
try {
await executeAuthActions(page, authActions);
} catch (err) {
err.pageDiagnostics = await collectPageDiagnostics(page);
throw err;
}
}
async function executeAuthActions(page, authActions) {
if (!Array.isArray(authActions)) return;
@@ -455,7 +1173,7 @@ app.post('/api/auth-and-intercept', async (req, res) => {
return res.status(400).json({ success: false, error: '参数错误' });
}
const antiBot = normalizeAntiBot(rawAntiBot);
const antiBot = resolveAntiBotForPage(rawAntiBot, pageUrl);
const profile = antiBot.profile || 'standard';
const interceptTimeout = resolveInterceptTimeout(antiBot);
const cfLog = { profile, cfDetected: false, turnstileStage: null, solverUsed: false, elapsedMs: 0, browserReused: false };
@@ -477,26 +1195,26 @@ app.post('/api/auth-and-intercept', async (req, res) => {
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl);
const page = await createManagedPage(browserSession.browser, browserSession.page, {
userAgent: DEFAULT_UA,
cookies: mergedCookies,
});
const page = await createTaskPage(
browserSession.browser,
browserSession.page,
pageUrl,
antiBot,
mergedCookies
);
const httpResolvedUrl = await resolveHttpRedirects(pageUrl).catch(() => pageUrl);
if (httpResolvedUrl !== pageUrl) {
console.log(`[URL解析] HTTP 重定向: ${pageUrl} -> ${httpResolvedUrl}`);
}
const interceptor = createApiInterceptor(page, apiUrls, interceptTimeout);
interceptorCleanup = interceptor.cleanup;
let shareToken = resolveShareToken(pageUrl, httpResolvedUrl);
await seedShareTokenCookie(page, pageUrl);
if (httpResolvedUrl !== pageUrl) {
await seedShareTokenCookie(page, httpResolvedUrl);
}
await attachHttpIpRewriteInterceptor(page, 'auth-and-intercept', { shareToken });
await prepareWhatshubShortLinkBeforeNav(page, pageUrl, antiBot);
await navigateToPage(page, pageUrl);
let finalPageUrl = await waitForUrlSettled(page);
@@ -504,8 +1222,11 @@ app.post('/api/auth-and-intercept', async (req, res) => {
console.log(`[URL解析] 浏览器落地: ${pageUrl} -> ${finalPageUrl}`);
}
let interceptor;
if (antiBot.enabled) {
const cfResult = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled, storedSession);
const cfResult = await runCloudflareWithBusinessGuard(
page, antiBot, waitForUrlSettled, storedSession, pageUrl
);
cfLog.cfDetected = cfResult.cfDetected;
cfLog.turnstileStage = cfResult.stage;
cfLog.solverUsed = cfResult.solverUsed;
@@ -526,6 +1247,8 @@ app.post('/api/auth-and-intercept', async (req, res) => {
finalPageUrl = page.url();
touchSessionIfPresent(antiBot.sessionKey);
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled, storedSession, pageUrl);
}
const finalShareToken = resolveShareToken(finalPageUrl);
@@ -534,11 +1257,11 @@ app.post('/api/auth-and-intercept', async (req, res) => {
await seedShareTokenCookie(page, finalPageUrl);
}
if (antiBot.enabled) {
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled);
}
interceptor = createApiInterceptor(page, apiUrls, interceptTimeout);
interceptorCleanup = interceptor.cleanup;
await attachHttpIpRewriteInterceptor(page, 'auth-and-intercept', { shareToken });
await executeAuthActionsWithDiagnostics(page, authActions, executeAuthActions);
await executeAuthActionsWithDiagnostics(page, authActions);
await interceptor.waitForApis(interceptTimeout);
const rawCookies = await page.cookies();
@@ -715,7 +1438,7 @@ app.post('/api/ui-pagination', async (req, res) => {
antiBot: rawAntiBot,
} = req.body;
const navigationUrl = finalPageUrl || pageUrl;
const antiBot = normalizeAntiBot(rawAntiBot);
const antiBot = resolveAntiBotForPage(rawAntiBot, pageUrl || navigationUrl);
const profile = antiBot.profile || 'standard';
try {
@@ -733,10 +1456,14 @@ app.post('/api/ui-pagination', async (req, res) => {
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, navigationUrl, cookies || []);
const page = await createManagedPage(browserSession.browser, browserSession.page, {
viewport: { width: 1920, height: 1080 },
cookies: mergedCookies,
});
const page = await createTaskPage(
browserSession.browser,
browserSession.page,
navigationUrl,
antiBot,
mergedCookies,
{ viewport: { width: 1920, height: 1080 } }
);
const log = (msg) => { console.log(msg); debugLogs.push(msg); };
log(`[启动] 准备访问页面: ${navigationUrl}`);
@@ -746,8 +1473,8 @@ app.post('/api/ui-pagination', async (req, res) => {
const shareToken = resolveShareToken(finalPageUrl, pageUrl);
await seedShareTokenCookie(page, navigationUrl);
await attachHttpIpRewriteInterceptor(page, 'ui-pagination', { shareToken });
await prepareWhatshubShortLinkBeforeNav(page, navigationUrl, antiBot);
await navigateToPage(page, navigationUrl);
const settledPageUrl = await waitForUrlSettled(page);
@@ -756,7 +1483,9 @@ app.post('/api/ui-pagination', async (req, res) => {
}
if (antiBot.enabled) {
const cfResult = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled, storedSession);
const cfResult = await runCloudflareWithBusinessGuard(
page, antiBot, waitForUrlSettled, storedSession, navigationUrl
);
if (!cfResult.success) {
cfDestroyBrowser = true;
res.status(422).json({
@@ -770,10 +1499,11 @@ app.post('/api/ui-pagination', async (req, res) => {
return;
}
touchSessionIfPresent(antiBot.sessionKey);
}
if (antiBot.enabled) {
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled);
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled, storedSession, navigationUrl);
await attachHttpIpRewriteInterceptor(page, 'ui-pagination', { shareToken });
} else {
await attachHttpIpRewriteInterceptor(page, 'ui-pagination', { shareToken });
}
const paginationResult = await runUiPagination(page, {
@@ -830,7 +1560,7 @@ app.post('/api/auth-intercept-and-paginate', async (req, res) => {
firstPageData,
} = pagination;
const antiBot = normalizeAntiBot(rawAntiBot);
const antiBot = resolveAntiBotForPage(rawAntiBot, pageUrl);
const profile = antiBot.profile || 'standard';
const interceptTimeout = resolveInterceptTimeout(antiBot);
const cfLog = { profile, cfDetected: false, turnstileStage: null, solverUsed: false, elapsedMs: 0, browserReused: false };
@@ -852,25 +1582,29 @@ app.post('/api/auth-intercept-and-paginate', async (req, res) => {
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl);
const page = await createManagedPage(browserSession.browser, browserSession.page, {
userAgent: DEFAULT_UA,
viewport: { width: 1920, height: 1080 },
cookies: mergedCookies,
});
const page = await createTaskPage(
browserSession.browser,
browserSession.page,
pageUrl,
antiBot,
mergedCookies,
{ viewport: { width: 1920, height: 1080 } }
);
const httpResolvedUrl = await resolveHttpRedirects(pageUrl).catch(() => pageUrl);
const interceptor = createApiInterceptor(page, apiUrls, interceptTimeout);
interceptorCleanup = interceptor.cleanup;
let shareToken = resolveShareToken(pageUrl, httpResolvedUrl);
await seedShareTokenCookie(page, pageUrl);
await attachHttpIpRewriteInterceptor(page, 'auth-intercept-and-paginate', { shareToken });
await prepareWhatshubShortLinkBeforeNav(page, pageUrl, antiBot);
await navigateToPage(page, pageUrl);
let finalPageUrl = await waitForUrlSettled(page);
let interceptor;
if (antiBot.enabled) {
const cfResult = await handleCloudflareChallenge(page, antiBot, waitForUrlSettled, storedSession);
const cfResult = await runCloudflareWithBusinessGuard(
page, antiBot, waitForUrlSettled, storedSession, pageUrl
);
cfLog.cfDetected = cfResult.cfDetected;
cfLog.turnstileStage = cfResult.stage;
cfLog.solverUsed = cfResult.solverUsed;
@@ -890,13 +1624,15 @@ app.post('/api/auth-intercept-and-paginate', async (req, res) => {
}
finalPageUrl = page.url();
touchSessionIfPresent(antiBot.sessionKey);
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled, storedSession, pageUrl);
}
if (antiBot.enabled) {
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled);
}
interceptor = createApiInterceptor(page, apiUrls, interceptTimeout);
interceptorCleanup = interceptor.cleanup;
await attachHttpIpRewriteInterceptor(page, 'auth-intercept-and-paginate', { shareToken });
await executeAuthActionsWithDiagnostics(page, authActions, executeAuthActions);
await executeAuthActionsWithDiagnostics(page, authActions);
await interceptor.waitForApis(interceptTimeout);
const rawCookies = await page.cookies();