A2C工单修复
This commit is contained in:
+37
-9
@@ -64,34 +64,37 @@ class A2c extends AbstractScrmSpider
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A2C:业务域无需 cf_clearance;短链仍作 pageUrl,sessionKey 固定业务域
|
* A2C:业务域 + 分享 id 会话;短链仍作 pageUrl,landingUrlHint 供 Node 直达长链
|
||||||
*
|
*
|
||||||
* @return array<string, mixed>
|
* @return array<string, mixed>
|
||||||
*/
|
*/
|
||||||
private function buildAntiBotConfig()
|
private function buildAntiBotConfig()
|
||||||
{
|
{
|
||||||
$sessionHost = $this->resolveA2cSessionHost();
|
$sessionScope = $this->resolveA2cSessionScope();
|
||||||
|
|
||||||
return [
|
return [
|
||||||
'enabled' => true,
|
'enabled' => true,
|
||||||
'profile' => 'real',
|
'profile' => 'real',
|
||||||
'turnstile' => true,
|
'turnstile' => true,
|
||||||
'solverFallback' => false,
|
'solverFallback' => true,
|
||||||
'cfClearanceRequired' => false,
|
'cfClearanceRequired' => false,
|
||||||
'sessionKey' => 'a2c:' . $sessionHost,
|
'sessionKey' => 'a2c:' . $sessionScope,
|
||||||
'challengeTimeoutMs' => 60000,
|
'challengeTimeoutMs' => 60000,
|
||||||
'landingUrlHint' => $this->landingUrl,
|
'landingUrlHint' => $this->landingUrl,
|
||||||
'businessHostHint' => $sessionHost,
|
'cfCloudflareSolver' => true,
|
||||||
|
'cfChallengeMode' => 'managed',
|
||||||
|
'businessHostHint' => $this->resolveA2cSessionHost(),
|
||||||
'postCfReadyUrlContains' => '/visitors/counter/share',
|
'postCfReadyUrlContains' => '/visitors/counter/share',
|
||||||
'postCfReadySelector' => '.btn-next',
|
'postCfReadySelector' => '.el-table',
|
||||||
'postCfReadyTimeoutMs' => 30000,
|
'postCfReadyApiUrl' => '/api/talk/counter/share/record/list',
|
||||||
|
'postCfReadyTimeoutMs' => 60000,
|
||||||
'postCfSettleMs' => 500,
|
'postCfSettleMs' => 500,
|
||||||
'postCfSpaWaitMs' => 4000,
|
'postCfSpaWaitMs' => 8000,
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A2C 会话域:HTTP 预解析无法穿透短链 CF 时,回退到业务域 user.a2c.chat
|
* A2C 业务域 host
|
||||||
*/
|
*/
|
||||||
private function resolveA2cSessionHost()
|
private function resolveA2cSessionHost()
|
||||||
{
|
{
|
||||||
@@ -111,6 +114,31 @@ class A2c extends AbstractScrmSpider
|
|||||||
return 'user.a2c.chat';
|
return 'user.a2c.chat';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A2C 会话 scope:host + share id
|
||||||
|
*/
|
||||||
|
private function resolveA2cSessionScope()
|
||||||
|
{
|
||||||
|
$host = $this->resolveA2cSessionHost();
|
||||||
|
$shareId = $this->extractA2cShareId($this->pageUrl);
|
||||||
|
if ($shareId === '' && $this->landingUrl !== '') {
|
||||||
|
$shareId = $this->extractA2cShareId($this->landingUrl);
|
||||||
|
}
|
||||||
|
|
||||||
|
return $shareId !== '' ? $host . ':' . $shareId : $host;
|
||||||
|
}
|
||||||
|
|
||||||
|
private function extractA2cShareId($url)
|
||||||
|
{
|
||||||
|
$query = (string) parse_url($url, PHP_URL_QUERY);
|
||||||
|
if ($query === '') {
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
parse_str($query, $params);
|
||||||
|
|
||||||
|
return isset($params['id']) ? trim((string) $params['id']) : '';
|
||||||
|
}
|
||||||
|
|
||||||
protected function extractListTotalPages($listFirstPageData, $countData = null)
|
protected function extractListTotalPages($listFirstPageData, $countData = null)
|
||||||
{
|
{
|
||||||
$default_per_page_count = self::DEFAULT_PER_PAGE_COUNT;
|
$default_per_page_count = self::DEFAULT_PER_PAGE_COUNT;
|
||||||
|
|||||||
+1
-1
@@ -85,7 +85,7 @@ try {
|
|||||||
// 长链:快速验证业务页拦截(推荐先测通)
|
// 长链:快速验证业务页拦截(推荐先测通)
|
||||||
// $pageUrl = 'https://user.a2c.chat/visitors/counter/share?id=1b2cf9a91e2647c185b252e723c871de';
|
// $pageUrl = 'https://user.a2c.chat/visitors/counter/share?id=1b2cf9a91e2647c185b252e723c871de';
|
||||||
// 短链:与客户后台一致,上线前必须回归
|
// 短链:与客户后台一致,上线前必须回归
|
||||||
$pageUrl = 'https://yyk.ink/3xj1B2';
|
$pageUrl = 'https://yyk.ink/5H3O8b';
|
||||||
$username = "";
|
$username = "";
|
||||||
$password = "";
|
$password = "";
|
||||||
$spider = (new A2c($pageUrl, $username, $password))->setVerbose(true);
|
$spider = (new A2c($pageUrl, $username, $password))->setVerbose(true);
|
||||||
|
|||||||
Regular → Executable
@@ -47,16 +47,18 @@ class AntiBotConfigBuilder
|
|||||||
'sessionTtlMinutes' => $ttlMinutes,
|
'sessionTtlMinutes' => $ttlMinutes,
|
||||||
'businessHostHint' => $businessHostHint,
|
'businessHostHint' => $businessHostHint,
|
||||||
'landingUrlHint' => $landingUrlHint !== '' ? $landingUrlHint : '',
|
'landingUrlHint' => $landingUrlHint !== '' ? $landingUrlHint : '',
|
||||||
|
'cfCloudflareSolver' => self::defaultCfCloudflareSolver($ticketType),
|
||||||
|
'cfChallengeMode' => self::defaultCfChallengeMode($ticketType),
|
||||||
] + self::postCfReadyExtras($ticketType);
|
] + self::postCfReadyExtras($ticketType);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 会话 scope:A2C 短链固定业务域;火箭长链用动态 host、短链用 share.{token}
|
* 会话 scope:A2C 业务域+分享 id;火箭长链用动态 host、短链用 share.{token}
|
||||||
*/
|
*/
|
||||||
public static function resolveSessionScope(string $ticketType, string $pageUrl, string $landingUrlHint = ''): string
|
public static function resolveSessionScope(string $ticketType, string $pageUrl, string $landingUrlHint = ''): string
|
||||||
{
|
{
|
||||||
if ($ticketType === 'a2c') {
|
if ($ticketType === 'a2c') {
|
||||||
return self::resolveA2cSessionHost($pageUrl, $landingUrlHint);
|
return self::resolveA2cSessionScope($pageUrl, $landingUrlHint);
|
||||||
}
|
}
|
||||||
|
|
||||||
if ($ticketType === 'huojian') {
|
if ($ticketType === 'huojian') {
|
||||||
@@ -135,10 +137,11 @@ class AntiBotConfigBuilder
|
|||||||
if ($ticketType === 'a2c') {
|
if ($ticketType === 'a2c') {
|
||||||
return [
|
return [
|
||||||
'postCfReadyUrlContains' => '/visitors/counter/share',
|
'postCfReadyUrlContains' => '/visitors/counter/share',
|
||||||
'postCfReadySelector' => '.btn-next',
|
'postCfReadySelector' => '.el-table',
|
||||||
'postCfReadyTimeoutMs' => 30000,
|
'postCfReadyApiUrl' => '/api/talk/counter/share/record/list',
|
||||||
|
'postCfReadyTimeoutMs' => 60000,
|
||||||
'postCfSettleMs' => 500,
|
'postCfSettleMs' => 500,
|
||||||
'postCfSpaWaitMs' => 4000,
|
'postCfSpaWaitMs' => 8000,
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -162,10 +165,6 @@ class AntiBotConfigBuilder
|
|||||||
|
|
||||||
private static function defaultSolverFallback(string $ticketType): bool
|
private static function defaultSolverFallback(string $ticketType): bool
|
||||||
{
|
{
|
||||||
if ($ticketType === 'a2c') {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -178,6 +177,26 @@ class AntiBotConfigBuilder
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A2C 等为 CF Managed Challenge(5s 盾),非独立 Turnstile 挂件
|
||||||
|
*/
|
||||||
|
private static function defaultCfChallengeMode(string $ticketType): string
|
||||||
|
{
|
||||||
|
if ($ticketType === 'a2c') {
|
||||||
|
return 'managed';
|
||||||
|
}
|
||||||
|
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* sitekey 缺失时是否允许 CapSolver AntiCloudflareTask(需 Node 侧 CAPTCHA_PROXY)
|
||||||
|
*/
|
||||||
|
private static function defaultCfCloudflareSolver(string $ticketType): bool
|
||||||
|
{
|
||||||
|
return $ticketType === 'a2c';
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 火箭/A2C 业务域 hint:优先长链动态 host,其次 landing 预解析
|
* 火箭/A2C 业务域 hint:优先长链动态 host,其次 landing 预解析
|
||||||
*/
|
*/
|
||||||
@@ -223,6 +242,35 @@ class AntiBotConfigBuilder
|
|||||||
return 'user.a2c.chat';
|
return 'user.a2c.chat';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A2C 会话 scope:业务域 + 分享 id(避免多工单共用温池 Browser 互相干扰)
|
||||||
|
*/
|
||||||
|
private static function resolveA2cSessionScope(string $pageUrl, string $landingUrlHint = ''): string
|
||||||
|
{
|
||||||
|
$host = self::resolveA2cSessionHost($pageUrl, $landingUrlHint);
|
||||||
|
$shareId = self::extractA2cShareId($pageUrl);
|
||||||
|
if ($shareId === '' && $landingUrlHint !== '') {
|
||||||
|
$shareId = self::extractA2cShareId($landingUrlHint);
|
||||||
|
}
|
||||||
|
|
||||||
|
return $shareId !== '' ? $host . ':' . $shareId : $host;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 从 A2C 分享 URL 提取 id 查询参数
|
||||||
|
*/
|
||||||
|
private static function extractA2cShareId(string $url): string
|
||||||
|
{
|
||||||
|
$query = (string) parse_url($url, PHP_URL_QUERY);
|
||||||
|
if ($query === '') {
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
parse_str($query, $params);
|
||||||
|
$id = isset($params['id']) ? trim((string) $params['id']) : '';
|
||||||
|
|
||||||
|
return $id !== '' ? $id : '';
|
||||||
|
}
|
||||||
|
|
||||||
private static function resolveHuojianSessionScope(string $pageUrl, string $landingUrlHint = ''): string
|
private static function resolveHuojianSessionScope(string $pageUrl, string $landingUrlHint = ''): string
|
||||||
{
|
{
|
||||||
$scope = HuojianUrlHelper::resolveSessionScope($pageUrl);
|
$scope = HuojianUrlHelper::resolveSessionScope($pageUrl);
|
||||||
|
|||||||
@@ -7,11 +7,12 @@ namespace app\common\library\scrm\spider;
|
|||||||
use app\common\library\scrm\AbstractScrmSpider;
|
use app\common\library\scrm\AbstractScrmSpider;
|
||||||
use app\common\library\scrm\AntiBotConfigBuilder;
|
use app\common\library\scrm\AntiBotConfigBuilder;
|
||||||
use app\common\library\scrm\UnifiedScrmData;
|
use app\common\library\scrm\UnifiedScrmData;
|
||||||
|
use app\common\service\SplitPageUrlResolver;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A2C 云控蜘蛛(Real Browser 打开分享页 + 拦截加密 API + UI 翻页)
|
* A2C 云控蜘蛛(Real Browser 打开分享页 + 拦截加密 API + UI 翻页)
|
||||||
*
|
*
|
||||||
* 业务域 user.a2c.chat 无需 cf_clearance;短链 yyk.ink 仍由浏览器 follow 跳转。
|
* 业务域 user.a2c.chat 在 CF 挑战页启用 CapSolver 兜底;短链 yyk.ink 预解析为长链后直达业务页。
|
||||||
*/
|
*/
|
||||||
class A2cSpider extends AbstractScrmSpider
|
class A2cSpider extends AbstractScrmSpider
|
||||||
{
|
{
|
||||||
@@ -27,6 +28,9 @@ class A2cSpider extends AbstractScrmSpider
|
|||||||
|
|
||||||
private string $password;
|
private string $password;
|
||||||
|
|
||||||
|
/** @var string HTTP 预解析落地 URL(供 antiBot landingUrlHint) */
|
||||||
|
private string $landingUrlHint = '';
|
||||||
|
|
||||||
private UnifiedScrmData $unifiedData;
|
private UnifiedScrmData $unifiedData;
|
||||||
|
|
||||||
public function __construct(
|
public function __construct(
|
||||||
@@ -39,6 +43,7 @@ class A2cSpider extends AbstractScrmSpider
|
|||||||
$this->pageUrl = $pageUrl;
|
$this->pageUrl = $pageUrl;
|
||||||
$this->account = $account;
|
$this->account = $account;
|
||||||
$this->password = $password;
|
$this->password = $password;
|
||||||
|
$this->landingUrlHint = SplitPageUrlResolver::resolveFinalUrl($pageUrl);
|
||||||
$this->unifiedData = new UnifiedScrmData();
|
$this->unifiedData = new UnifiedScrmData();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -53,7 +58,7 @@ class A2cSpider extends AbstractScrmSpider
|
|||||||
'authActions' => [
|
'authActions' => [
|
||||||
['type' => 'wait', 'ms' => 2000],
|
['type' => 'wait', 'ms' => 2000],
|
||||||
],
|
],
|
||||||
'antiBot' => AntiBotConfigBuilder::build('a2c', $this->pageUrl),
|
'antiBot' => AntiBotConfigBuilder::build('a2c', $this->pageUrl, $this->landingUrlHint),
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -80,7 +80,7 @@ class HuojianSpider extends AbstractScrmSpider
|
|||||||
protected function parseToUnifiedData($detailData, array $allListPagesData): UnifiedScrmData
|
protected function parseToUnifiedData($detailData, array $allListPagesData): UnifiedScrmData
|
||||||
{
|
{
|
||||||
$unifiedData = $this->unifiedData;
|
$unifiedData = $this->unifiedData;
|
||||||
$unifiedData->todayNewCount = (int) ($allListPagesData[0]['data']['counterWorker']['newTodayFriend'] ?? 0) + (int) ($allListPagesData[0]['data']['counterWorker']['newRemovedTodayFriend'] ?? 0);
|
$unifiedData->todayNewCount = (int) ($allListPagesData[0]['data']['counterWorker']['newTodayFriend'] ?? 0);
|
||||||
$count = 0;
|
$count = 0;
|
||||||
foreach ($allListPagesData as $pageRaw) {
|
foreach ($allListPagesData as $pageRaw) {
|
||||||
$records = $pageRaw['data']['counterCsAccountVo'] ?? [];
|
$records = $pageRaw['data']['counterCsAccountVo'] ?? [];
|
||||||
|
|||||||
@@ -47,16 +47,18 @@ class AntiBotConfigBuilder
|
|||||||
'sessionTtlMinutes' => $ttlMinutes,
|
'sessionTtlMinutes' => $ttlMinutes,
|
||||||
'businessHostHint' => $businessHostHint,
|
'businessHostHint' => $businessHostHint,
|
||||||
'landingUrlHint' => $landingUrlHint !== '' ? $landingUrlHint : '',
|
'landingUrlHint' => $landingUrlHint !== '' ? $landingUrlHint : '',
|
||||||
|
'cfCloudflareSolver' => self::defaultCfCloudflareSolver($ticketType),
|
||||||
|
'cfChallengeMode' => self::defaultCfChallengeMode($ticketType),
|
||||||
] + self::postCfReadyExtras($ticketType);
|
] + self::postCfReadyExtras($ticketType);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 会话 scope:A2C 短链固定业务域;火箭长链用动态 host、短链用 share.{token}
|
* 会话 scope:A2C 业务域+分享 id;火箭长链用动态 host、短链用 share.{token}
|
||||||
*/
|
*/
|
||||||
public static function resolveSessionScope(string $ticketType, string $pageUrl, string $landingUrlHint = ''): string
|
public static function resolveSessionScope(string $ticketType, string $pageUrl, string $landingUrlHint = ''): string
|
||||||
{
|
{
|
||||||
if ($ticketType === 'a2c') {
|
if ($ticketType === 'a2c') {
|
||||||
return self::resolveA2cSessionHost($pageUrl, $landingUrlHint);
|
return self::resolveA2cSessionScope($pageUrl, $landingUrlHint);
|
||||||
}
|
}
|
||||||
|
|
||||||
if ($ticketType === 'huojian') {
|
if ($ticketType === 'huojian') {
|
||||||
@@ -135,10 +137,11 @@ class AntiBotConfigBuilder
|
|||||||
if ($ticketType === 'a2c') {
|
if ($ticketType === 'a2c') {
|
||||||
return [
|
return [
|
||||||
'postCfReadyUrlContains' => '/visitors/counter/share',
|
'postCfReadyUrlContains' => '/visitors/counter/share',
|
||||||
'postCfReadySelector' => '.btn-next',
|
'postCfReadySelector' => '.el-table',
|
||||||
'postCfReadyTimeoutMs' => 30000,
|
'postCfReadyApiUrl' => '/api/talk/counter/share/record/list',
|
||||||
|
'postCfReadyTimeoutMs' => 60000,
|
||||||
'postCfSettleMs' => 500,
|
'postCfSettleMs' => 500,
|
||||||
'postCfSpaWaitMs' => 4000,
|
'postCfSpaWaitMs' => 8000,
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -162,10 +165,6 @@ class AntiBotConfigBuilder
|
|||||||
|
|
||||||
private static function defaultSolverFallback(string $ticketType): bool
|
private static function defaultSolverFallback(string $ticketType): bool
|
||||||
{
|
{
|
||||||
if ($ticketType === 'a2c') {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -178,6 +177,26 @@ class AntiBotConfigBuilder
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A2C 等为 CF Managed Challenge(5s 盾),非独立 Turnstile 挂件
|
||||||
|
*/
|
||||||
|
private static function defaultCfChallengeMode(string $ticketType): string
|
||||||
|
{
|
||||||
|
if ($ticketType === 'a2c') {
|
||||||
|
return 'managed';
|
||||||
|
}
|
||||||
|
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* sitekey 缺失时是否允许 CapSolver AntiCloudflareTask(需 Node 侧 CAPTCHA_PROXY)
|
||||||
|
*/
|
||||||
|
private static function defaultCfCloudflareSolver(string $ticketType): bool
|
||||||
|
{
|
||||||
|
return $ticketType === 'a2c';
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 火箭/A2C 业务域 hint:优先长链动态 host,其次 landing 预解析
|
* 火箭/A2C 业务域 hint:优先长链动态 host,其次 landing 预解析
|
||||||
*/
|
*/
|
||||||
@@ -223,6 +242,35 @@ class AntiBotConfigBuilder
|
|||||||
return 'user.a2c.chat';
|
return 'user.a2c.chat';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A2C 会话 scope:业务域 + 分享 id(避免多工单共用温池 Browser 互相干扰)
|
||||||
|
*/
|
||||||
|
private static function resolveA2cSessionScope(string $pageUrl, string $landingUrlHint = ''): string
|
||||||
|
{
|
||||||
|
$host = self::resolveA2cSessionHost($pageUrl, $landingUrlHint);
|
||||||
|
$shareId = self::extractA2cShareId($pageUrl);
|
||||||
|
if ($shareId === '' && $landingUrlHint !== '') {
|
||||||
|
$shareId = self::extractA2cShareId($landingUrlHint);
|
||||||
|
}
|
||||||
|
|
||||||
|
return $shareId !== '' ? $host . ':' . $shareId : $host;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 从 A2C 分享 URL 提取 id 查询参数
|
||||||
|
*/
|
||||||
|
private static function extractA2cShareId(string $url): string
|
||||||
|
{
|
||||||
|
$query = (string) parse_url($url, PHP_URL_QUERY);
|
||||||
|
if ($query === '') {
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
parse_str($query, $params);
|
||||||
|
$id = isset($params['id']) ? trim((string) $params['id']) : '';
|
||||||
|
|
||||||
|
return $id !== '' ? $id : '';
|
||||||
|
}
|
||||||
|
|
||||||
private static function resolveHuojianSessionScope(string $pageUrl, string $landingUrlHint = ''): string
|
private static function resolveHuojianSessionScope(string $pageUrl, string $landingUrlHint = ''): string
|
||||||
{
|
{
|
||||||
$scope = HuojianUrlHelper::resolveSessionScope($pageUrl);
|
$scope = HuojianUrlHelper::resolveSessionScope($pageUrl);
|
||||||
|
|||||||
@@ -7,11 +7,12 @@ namespace app\common\library\scrm\spider;
|
|||||||
use app\common\library\scrm\AbstractScrmSpider;
|
use app\common\library\scrm\AbstractScrmSpider;
|
||||||
use app\common\library\scrm\AntiBotConfigBuilder;
|
use app\common\library\scrm\AntiBotConfigBuilder;
|
||||||
use app\common\library\scrm\UnifiedScrmData;
|
use app\common\library\scrm\UnifiedScrmData;
|
||||||
|
use app\common\service\SplitPageUrlResolver;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A2C 云控蜘蛛(Real Browser 打开分享页 + 拦截加密 API + UI 翻页)
|
* A2C 云控蜘蛛(Real Browser 打开分享页 + 拦截加密 API + UI 翻页)
|
||||||
*
|
*
|
||||||
* 业务域 user.a2c.chat 无需 cf_clearance;短链 yyk.ink 仍由浏览器 follow 跳转。
|
* 业务域 user.a2c.chat 在 CF 挑战页启用 CapSolver 兜底;短链 yyk.ink 预解析为长链后直达业务页。
|
||||||
*/
|
*/
|
||||||
class A2cSpider extends AbstractScrmSpider
|
class A2cSpider extends AbstractScrmSpider
|
||||||
{
|
{
|
||||||
@@ -27,6 +28,9 @@ class A2cSpider extends AbstractScrmSpider
|
|||||||
|
|
||||||
private string $password;
|
private string $password;
|
||||||
|
|
||||||
|
/** @var string HTTP 预解析落地 URL(供 antiBot landingUrlHint) */
|
||||||
|
private string $landingUrlHint = '';
|
||||||
|
|
||||||
private UnifiedScrmData $unifiedData;
|
private UnifiedScrmData $unifiedData;
|
||||||
|
|
||||||
public function __construct(
|
public function __construct(
|
||||||
@@ -39,6 +43,7 @@ class A2cSpider extends AbstractScrmSpider
|
|||||||
$this->pageUrl = $pageUrl;
|
$this->pageUrl = $pageUrl;
|
||||||
$this->account = $account;
|
$this->account = $account;
|
||||||
$this->password = $password;
|
$this->password = $password;
|
||||||
|
$this->landingUrlHint = SplitPageUrlResolver::resolveFinalUrl($pageUrl);
|
||||||
$this->unifiedData = new UnifiedScrmData();
|
$this->unifiedData = new UnifiedScrmData();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -53,7 +58,7 @@ class A2cSpider extends AbstractScrmSpider
|
|||||||
'authActions' => [
|
'authActions' => [
|
||||||
['type' => 'wait', 'ms' => 2000],
|
['type' => 'wait', 'ms' => 2000],
|
||||||
],
|
],
|
||||||
'antiBot' => AntiBotConfigBuilder::build('a2c', $this->pageUrl),
|
'antiBot' => AntiBotConfigBuilder::build('a2c', $this->pageUrl, $this->landingUrlHint),
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,38 @@
|
|||||||
|
# puppeteer-api 环境配置示例
|
||||||
|
# 复制为 .env 后按实际环境修改;pm2 restart server --update-env
|
||||||
|
|
||||||
|
# ========== CapSolver(仅 A2C 等 Managed 过盾时 Node 调用 AntiCloudflareTask)==========
|
||||||
|
CAPTCHA_PROVIDER=capsolver
|
||||||
|
CAPTCHA_API_KEY=
|
||||||
|
# 静态住宅代理 ip:port:user:pass;CapSolver 过盾必填
|
||||||
|
# Chrome 仅在「检测到 CF 挑战」时才会使用同一代理(见 server.js ensureCaptchaProxyBrowserIfNeeded)
|
||||||
|
CAPTCHA_PROXY=
|
||||||
|
CAPTCHA_MAX_WAIT_MS=120000
|
||||||
|
|
||||||
|
# ========== Standard 槽位(未走 antiBot / profile=standard 的蜘蛛)==========
|
||||||
|
MAX_CONCURRENT_BROWSERS=4
|
||||||
|
|
||||||
|
# ========== Real Browser 槽位(whatshub / chatknow / huojian / a2c 共用)==========
|
||||||
|
# 20~30 工单后台批量:建议 2~3(非 A2C 不走代理,可与 A2C 并行)
|
||||||
|
MAX_CONCURRENT_BROWSERS_REAL=2
|
||||||
|
|
||||||
|
# ========== 排队(cron 一轮多工单时避免 503)==========
|
||||||
|
QUEUE_TIMEOUT_MS=300000
|
||||||
|
|
||||||
|
# ========== 温池(后台批量强烈建议开启,按 sessionKey 复用 Browser)==========
|
||||||
|
POOL_ENABLED=1
|
||||||
|
MAX_POOLED_BROWSERS=8
|
||||||
|
POOL_IDLE_MS=300000
|
||||||
|
|
||||||
|
# ========== 超时 ==========
|
||||||
|
NAVIGATION_TIMEOUT_MS=90000
|
||||||
|
API_INTERCEPT_TIMEOUT_MS=60000
|
||||||
|
# Real Browser + A2C 过盾(CapSolver + SPA)建议 180s
|
||||||
|
API_INTERCEPT_TIMEOUT_REAL_MS=180000
|
||||||
|
|
||||||
|
# ========== 会话复用(减少 A2C 重复过盾)==========
|
||||||
|
SESSION_TTL_MS=7200000
|
||||||
|
PERSIST_BROWSER_PROFILE=1
|
||||||
|
|
||||||
|
# ========== 服务端口 ==========
|
||||||
|
PORT=3001
|
||||||
@@ -59,11 +59,12 @@ async function createBrowserSession(options = {}) {
|
|||||||
page,
|
page,
|
||||||
profile: 'real',
|
profile: 'real',
|
||||||
cleanup: async (destroy = false) => {
|
cleanup: async (destroy = false) => {
|
||||||
if (destroy || !sessionKey) {
|
if (!destroy) {
|
||||||
try {
|
return;
|
||||||
await browser.close();
|
|
||||||
} catch (_) { /* ignore */ }
|
|
||||||
}
|
}
|
||||||
|
try {
|
||||||
|
await browser.close();
|
||||||
|
} catch (_) { /* ignore */ }
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -74,11 +75,12 @@ async function createBrowserSession(options = {}) {
|
|||||||
page: null,
|
page: null,
|
||||||
profile: 'standard',
|
profile: 'standard',
|
||||||
cleanup: async (destroy = false) => {
|
cleanup: async (destroy = false) => {
|
||||||
if (destroy || !sessionKey) {
|
if (!destroy) {
|
||||||
try {
|
return;
|
||||||
await browser.close();
|
|
||||||
} catch (_) { /* ignore */ }
|
|
||||||
}
|
}
|
||||||
|
try {
|
||||||
|
await browser.close();
|
||||||
|
} catch (_) { /* ignore */ }
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,12 +1,16 @@
|
|||||||
/**
|
/**
|
||||||
* Browser 温池:按 profile:sessionKey 复用 Browser/Page,降低冷启动与 CF 触发
|
* Browser 温池:按 profile:sessionKey 复用 Browser,降低冷启动与 CF 触发
|
||||||
|
*
|
||||||
|
* 同一 key 的 acquire 在 inUse 时排队等待,禁止销毁正在使用的 Browser(修复 A2C 并发误杀)。
|
||||||
*/
|
*/
|
||||||
const { POOL_IDLE_MS, MAX_POOLED_BROWSERS, POOL_ENABLED } = require('./constants');
|
const { POOL_IDLE_MS, MAX_POOLED_BROWSERS, POOL_ENABLED, QUEUE_TIMEOUT_MS } = require('./constants');
|
||||||
|
|
||||||
class BrowserPool {
|
class BrowserPool {
|
||||||
constructor() {
|
constructor() {
|
||||||
/** @type {Map<string, { browser: any, page: any|null, profile: string, sessionKey: string, lastUsedAt: number, inUse: boolean }>} */
|
/** @type {Map<string, { browser: any, page: any|null, profile: string, sessionKey: string, lastUsedAt: number, inUse: boolean }>} */
|
||||||
this.entries = new Map();
|
this.entries = new Map();
|
||||||
|
/** @type {Map<string, Array<{ resolve: () => void, reject: (err: Error) => void, timer: NodeJS.Timeout|null }>>} */
|
||||||
|
this._waitQueues = new Map();
|
||||||
this.hits = 0;
|
this.hits = 0;
|
||||||
this.misses = 0;
|
this.misses = 0;
|
||||||
this.evictions = 0;
|
this.evictions = 0;
|
||||||
@@ -14,17 +18,10 @@ class BrowserPool {
|
|||||||
this._sweepTimer.unref?.();
|
this._sweepTimer.unref?.();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {string} sessionKey
|
|
||||||
* @param {'standard'|'real'} profile
|
|
||||||
*/
|
|
||||||
buildKey(sessionKey, profile) {
|
buildKey(sessionKey, profile) {
|
||||||
return `${profile}:${sessionKey}`;
|
return `${profile}:${sessionKey}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {any} browser
|
|
||||||
*/
|
|
||||||
async isBrowserAlive(browser) {
|
async isBrowserAlive(browser) {
|
||||||
if (!browser) return false;
|
if (!browser) return false;
|
||||||
try {
|
try {
|
||||||
@@ -38,12 +35,13 @@ class BrowserPool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {string} key
|
|
||||||
*/
|
|
||||||
async destroyEntry(key) {
|
async destroyEntry(key) {
|
||||||
const entry = this.entries.get(key);
|
const entry = this.entries.get(key);
|
||||||
if (!entry) return;
|
if (!entry) return;
|
||||||
|
if (entry.inUse) {
|
||||||
|
console.warn(`[BrowserPool] 跳过销毁 inUse 条目 key=${key}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
this.entries.delete(key);
|
this.entries.delete(key);
|
||||||
try {
|
try {
|
||||||
await entry.browser.close();
|
await entry.browser.close();
|
||||||
@@ -63,11 +61,43 @@ class BrowserPool {
|
|||||||
this.destroyEntry(oldestKey);
|
this.destroyEntry(oldestKey);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
waitForRelease(key, timeoutMs = QUEUE_TIMEOUT_MS) {
|
||||||
* @param {string} sessionKey
|
const entry = this.entries.get(key);
|
||||||
* @param {'standard'|'real'} profile
|
if (!entry || !entry.inUse) {
|
||||||
* @param {() => Promise<{ browser: any, page: any|null, cleanup: (destroy?: boolean) => Promise<void> }>} launcher
|
return Promise.resolve();
|
||||||
*/
|
}
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const waiter = {
|
||||||
|
resolve: () => {
|
||||||
|
if (waiter.timer) clearTimeout(waiter.timer);
|
||||||
|
resolve();
|
||||||
|
},
|
||||||
|
reject,
|
||||||
|
timer: timeoutMs > 0
|
||||||
|
? setTimeout(() => {
|
||||||
|
const queue = this._waitQueues.get(key) || [];
|
||||||
|
const idx = queue.indexOf(waiter);
|
||||||
|
if (idx >= 0) queue.splice(idx, 1);
|
||||||
|
reject(new Error('POOL_ACQUIRE_TIMEOUT'));
|
||||||
|
}, timeoutMs)
|
||||||
|
: null,
|
||||||
|
};
|
||||||
|
if (!this._waitQueues.has(key)) {
|
||||||
|
this._waitQueues.set(key, []);
|
||||||
|
}
|
||||||
|
this._waitQueues.get(key).push(waiter);
|
||||||
|
console.log(`[BrowserPool] key=${key} 正在使用,排队等待 (队列 ${this._waitQueues.get(key).length})`);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
_wakeNextWaiter(key) {
|
||||||
|
const queue = this._waitQueues.get(key);
|
||||||
|
if (!queue || queue.length === 0) return;
|
||||||
|
const next = queue.shift();
|
||||||
|
if (next?.timer) clearTimeout(next.timer);
|
||||||
|
next?.resolve();
|
||||||
|
}
|
||||||
|
|
||||||
async acquire(sessionKey, profile, launcher) {
|
async acquire(sessionKey, profile, launcher) {
|
||||||
if (!POOL_ENABLED || !sessionKey) {
|
if (!POOL_ENABLED || !sessionKey) {
|
||||||
const launched = await launcher();
|
const launched = await launcher();
|
||||||
@@ -77,29 +107,38 @@ class BrowserPool {
|
|||||||
profile,
|
profile,
|
||||||
pooled: false,
|
pooled: false,
|
||||||
reused: false,
|
reused: false,
|
||||||
release: async (destroy = false) => {
|
release: async () => {
|
||||||
await launched.cleanup(destroy);
|
await launched.cleanup(true);
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const key = this.buildKey(sessionKey, profile);
|
const key = this.buildKey(sessionKey, profile);
|
||||||
const existing = this.entries.get(key);
|
|
||||||
if (existing && !existing.inUse && await this.isBrowserAlive(existing.browser)) {
|
while (true) {
|
||||||
existing.inUse = true;
|
const existing = this.entries.get(key);
|
||||||
existing.lastUsedAt = Date.now();
|
if (!existing) {
|
||||||
this.hits++;
|
break;
|
||||||
return {
|
}
|
||||||
browser: existing.browser,
|
if (existing.inUse) {
|
||||||
page: existing.page,
|
await this.waitForRelease(key);
|
||||||
profile,
|
continue;
|
||||||
pooled: true,
|
}
|
||||||
reused: true,
|
if (await this.isBrowserAlive(existing.browser)) {
|
||||||
release: async (destroy = false) => this.release(key, destroy),
|
existing.inUse = true;
|
||||||
};
|
existing.lastUsedAt = Date.now();
|
||||||
}
|
this.hits++;
|
||||||
if (existing) {
|
return {
|
||||||
|
browser: existing.browser,
|
||||||
|
page: null,
|
||||||
|
profile,
|
||||||
|
pooled: true,
|
||||||
|
reused: true,
|
||||||
|
release: async (destroy = false) => this.release(key, destroy),
|
||||||
|
};
|
||||||
|
}
|
||||||
await this.destroyEntry(key);
|
await this.destroyEntry(key);
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
this.enforceCapacity(key);
|
this.enforceCapacity(key);
|
||||||
@@ -107,7 +146,7 @@ class BrowserPool {
|
|||||||
const launched = await launcher();
|
const launched = await launcher();
|
||||||
this.entries.set(key, {
|
this.entries.set(key, {
|
||||||
browser: launched.browser,
|
browser: launched.browser,
|
||||||
page: launched.page,
|
page: null,
|
||||||
profile,
|
profile,
|
||||||
sessionKey,
|
sessionKey,
|
||||||
lastUsedAt: Date.now(),
|
lastUsedAt: Date.now(),
|
||||||
@@ -124,18 +163,19 @@ class BrowserPool {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {string} key
|
|
||||||
* @param {boolean} destroy
|
|
||||||
*/
|
|
||||||
async release(key, destroy = false) {
|
async release(key, destroy = false) {
|
||||||
const entry = this.entries.get(key);
|
const entry = this.entries.get(key);
|
||||||
if (!entry) return;
|
if (!entry) {
|
||||||
|
this._wakeNextWaiter(key);
|
||||||
|
return;
|
||||||
|
}
|
||||||
entry.inUse = false;
|
entry.inUse = false;
|
||||||
entry.lastUsedAt = Date.now();
|
entry.lastUsedAt = Date.now();
|
||||||
|
entry.page = null;
|
||||||
if (destroy || !(await this.isBrowserAlive(entry.browser))) {
|
if (destroy || !(await this.isBrowserAlive(entry.browser))) {
|
||||||
await this.destroyEntry(key);
|
await this.destroyEntry(key);
|
||||||
}
|
}
|
||||||
|
this._wakeNextWaiter(key);
|
||||||
}
|
}
|
||||||
|
|
||||||
evictIdle() {
|
evictIdle() {
|
||||||
@@ -151,17 +191,24 @@ class BrowserPool {
|
|||||||
clearInterval(this._sweepTimer);
|
clearInterval(this._sweepTimer);
|
||||||
const keys = [...this.entries.keys()];
|
const keys = [...this.entries.keys()];
|
||||||
for (const key of keys) {
|
for (const key of keys) {
|
||||||
await this.destroyEntry(key);
|
const entry = this.entries.get(key);
|
||||||
|
if (entry && !entry.inUse) {
|
||||||
|
await this.destroyEntry(key);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
getStats() {
|
getStats() {
|
||||||
let idle = 0;
|
let idle = 0;
|
||||||
let active = 0;
|
let active = 0;
|
||||||
|
let waiting = 0;
|
||||||
for (const entry of this.entries.values()) {
|
for (const entry of this.entries.values()) {
|
||||||
if (entry.inUse) active++;
|
if (entry.inUse) active++;
|
||||||
else idle++;
|
else idle++;
|
||||||
}
|
}
|
||||||
|
for (const queue of this._waitQueues.values()) {
|
||||||
|
waiting += queue.length;
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
enabled: POOL_ENABLED,
|
enabled: POOL_ENABLED,
|
||||||
max: MAX_POOLED_BROWSERS,
|
max: MAX_POOLED_BROWSERS,
|
||||||
@@ -169,6 +216,7 @@ class BrowserPool {
|
|||||||
size: this.entries.size,
|
size: this.entries.size,
|
||||||
idle,
|
idle,
|
||||||
active,
|
active,
|
||||||
|
waiting,
|
||||||
hits: this.hits,
|
hits: this.hits,
|
||||||
misses: this.misses,
|
misses: this.misses,
|
||||||
evictions: this.evictions,
|
evictions: this.evictions,
|
||||||
|
|||||||
@@ -0,0 +1,118 @@
|
|||||||
|
/**
|
||||||
|
* CAPTCHA_PROXY 解析与 Chrome 代理注入(与 CapSolver AntiCloudflareTask 使用同一 IP)
|
||||||
|
*
|
||||||
|
* 仅在实际检测到 CF 挑战、需要 Managed 过盾时由 server.js 显式启用(useCaptchaProxy=true)。
|
||||||
|
* 格式:ip:port:username:password
|
||||||
|
*/
|
||||||
|
const CAPTCHA_PROXY_RAW = (process.env.CAPTCHA_PROXY || '').trim();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @returns {{
|
||||||
|
* host: string,
|
||||||
|
* port: string,
|
||||||
|
* username: string,
|
||||||
|
* password: string,
|
||||||
|
* chromeProxyServer: string,
|
||||||
|
* capsolverString: string,
|
||||||
|
* }|null}
|
||||||
|
*/
|
||||||
|
function parseCaptchaProxy(raw = CAPTCHA_PROXY_RAW) {
|
||||||
|
const value = (raw || '').trim();
|
||||||
|
if (value === '') {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const parts = value.split(':');
|
||||||
|
if (parts.length < 4) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const host = parts[0];
|
||||||
|
const port = parts[1];
|
||||||
|
const username = parts[2];
|
||||||
|
const password = parts.slice(3).join(':');
|
||||||
|
|
||||||
|
if (!host || !port || !username || !password) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
host,
|
||||||
|
port,
|
||||||
|
username,
|
||||||
|
password,
|
||||||
|
chromeProxyServer: `http://${host}:${port}`,
|
||||||
|
capsolverString: value,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @returns {boolean} */
|
||||||
|
function hasCaptchaProxy() {
|
||||||
|
return parseCaptchaProxy() !== null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 是否属于可能走 CapSolver Managed 的工单(A2C 等)
|
||||||
|
* 注意:仅表示候选,不代表本次任务一定挂代理
|
||||||
|
* @param {object|null|undefined} antiBot
|
||||||
|
*/
|
||||||
|
function isManagedCfCandidate(antiBot) {
|
||||||
|
if (!antiBot?.enabled) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (antiBot.cfChallengeMode === 'managed') {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (antiBot.cfCloudflareSolver === true) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @deprecated 请使用 isManagedCfCandidate + 显式 useCaptchaProxy 参数
|
||||||
|
*/
|
||||||
|
function shouldUseCaptchaProxyForAntiBot(antiBot) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Chrome 启动参数 --proxy-server(须 useCaptchaProxy=true 时调用)
|
||||||
|
* @returns {string[]}
|
||||||
|
*/
|
||||||
|
function getCaptchaProxyChromeArgs() {
|
||||||
|
const parsed = parseCaptchaProxy();
|
||||||
|
if (!parsed) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
return [
|
||||||
|
`--proxy-server=${parsed.chromeProxyServer}`,
|
||||||
|
'--proxy-bypass-list=<-loopback>,localhost,127.0.0.1',
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 为 Page 设置代理认证(须配合 --proxy-server)
|
||||||
|
* @param {import('puppeteer').Page} page
|
||||||
|
* @returns {Promise<boolean>}
|
||||||
|
*/
|
||||||
|
async function applyCaptchaProxyToPage(page) {
|
||||||
|
const parsed = parseCaptchaProxy();
|
||||||
|
if (!parsed || !page) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
await page.authenticate({
|
||||||
|
username: parsed.username,
|
||||||
|
password: parsed.password,
|
||||||
|
});
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
parseCaptchaProxy,
|
||||||
|
hasCaptchaProxy,
|
||||||
|
isManagedCfCandidate,
|
||||||
|
shouldUseCaptchaProxyForAntiBot,
|
||||||
|
getCaptchaProxyChromeArgs,
|
||||||
|
applyCaptchaProxyToPage,
|
||||||
|
};
|
||||||
@@ -6,6 +6,10 @@ const {
|
|||||||
CAPTCHA_API_KEY,
|
CAPTCHA_API_KEY,
|
||||||
CAPTCHA_MAX_WAIT_MS,
|
CAPTCHA_MAX_WAIT_MS,
|
||||||
} = require('./constants');
|
} = require('./constants');
|
||||||
|
const { sanitizePageUrlForCfSolver } = require('./cf-detector');
|
||||||
|
|
||||||
|
/** AntiCloudflareTask 所需静态代理(ip:port:user:pass),未配置则不可用 */
|
||||||
|
const CAPTCHA_PROXY = (process.env.CAPTCHA_PROXY || '').trim();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param {string} url
|
* @param {string} url
|
||||||
@@ -113,6 +117,147 @@ async function solveVia2Captcha({ pageUrl, sitekey, apiKey }) {
|
|||||||
throw new Error('2Captcha 求解超时');
|
throw new Error('2Captcha 求解超时');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* CapSolver Managed Challenge(返回 cf_clearance cookie,需 CAPTCHA_PROXY)
|
||||||
|
* @param {{ pageUrl: string, apiKey: string, proxy: string, html?: string, userAgent?: string }} params
|
||||||
|
* @returns {Promise<{ cookies: Record<string, string>, userAgent?: string }>}
|
||||||
|
*/
|
||||||
|
async function solveViaCapSolverCloudflare({ pageUrl, apiKey, proxy, html, userAgent }) {
|
||||||
|
const task = {
|
||||||
|
type: 'AntiCloudflareTask',
|
||||||
|
websiteURL: pageUrl,
|
||||||
|
proxy,
|
||||||
|
};
|
||||||
|
if (html) {
|
||||||
|
task.html = html.slice(0, 500000);
|
||||||
|
}
|
||||||
|
if (userAgent) {
|
||||||
|
task.userAgent = userAgent;
|
||||||
|
}
|
||||||
|
|
||||||
|
const create = await httpJsonPost('https://api.capsolver.com/createTask', {
|
||||||
|
clientKey: apiKey,
|
||||||
|
task,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (create.errorId !== 0 || !create.taskId) {
|
||||||
|
throw new Error(`CapSolver AntiCloudflare createTask 失败: ${create.errorDescription || create.errorCode || 'unknown'}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const deadline = Date.now() + CAPTCHA_MAX_WAIT_MS;
|
||||||
|
while (Date.now() < deadline) {
|
||||||
|
await new Promise((r) => setTimeout(r, 3000));
|
||||||
|
const result = await httpJsonPost('https://api.capsolver.com/getTaskResult', {
|
||||||
|
clientKey: apiKey,
|
||||||
|
taskId: create.taskId,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (result.status === 'ready' && result.solution) {
|
||||||
|
const cookies = result.solution.cookies || {};
|
||||||
|
if (typeof cookies === 'string') {
|
||||||
|
return { cookies: { cf_clearance: cookies }, userAgent: result.solution.userAgent };
|
||||||
|
}
|
||||||
|
if (result.solution.token && !cookies.cf_clearance) {
|
||||||
|
return {
|
||||||
|
cookies: { cf_clearance: result.solution.token },
|
||||||
|
userAgent: result.solution.userAgent,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
cookies,
|
||||||
|
userAgent: result.solution.userAgent,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (result.errorId !== 0) {
|
||||||
|
throw new Error(`CapSolver AntiCloudflare getTaskResult 失败: ${result.errorDescription || result.errorCode}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new Error('CapSolver AntiCloudflare 求解超时');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 统一 Cloudflare Managed Challenge 求解
|
||||||
|
* @param {{ pageUrl: string, provider?: string, apiKey?: string, proxy?: string, html?: string, userAgent?: string }} params
|
||||||
|
* @returns {Promise<{ cookies: Record<string, string>, userAgent?: string, provider: string }>}
|
||||||
|
*/
|
||||||
|
async function solveCloudflare({ pageUrl, provider, apiKey, proxy, html, userAgent }) {
|
||||||
|
const resolvedProvider = (provider || CAPTCHA_PROVIDER).toLowerCase();
|
||||||
|
const resolvedKey = apiKey || CAPTCHA_API_KEY;
|
||||||
|
const resolvedProxy = (proxy || CAPTCHA_PROXY).trim();
|
||||||
|
const cleanPageUrl = sanitizePageUrlForCfSolver(pageUrl);
|
||||||
|
|
||||||
|
if (!resolvedKey) {
|
||||||
|
throw new Error('未配置 CAPTCHA_API_KEY,无法使用 AntiCloudflareTask');
|
||||||
|
}
|
||||||
|
if (!resolvedProxy) {
|
||||||
|
throw new Error('AntiCloudflareTask 需要配置 CAPTCHA_PROXY(静态代理 ip:port:user:pass)');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (resolvedProvider !== 'capsolver') {
|
||||||
|
throw new Error('AntiCloudflareTask 当前仅支持 CapSolver');
|
||||||
|
}
|
||||||
|
|
||||||
|
const { cookies, userAgent: solvedUa } = await solveViaCapSolverCloudflare({
|
||||||
|
pageUrl: cleanPageUrl,
|
||||||
|
apiKey: resolvedKey,
|
||||||
|
proxy: resolvedProxy,
|
||||||
|
html,
|
||||||
|
userAgent,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!cookies || !cookies.cf_clearance) {
|
||||||
|
throw new Error('CapSolver AntiCloudflare 未返回 cf_clearance');
|
||||||
|
}
|
||||||
|
|
||||||
|
return { cookies, userAgent: solvedUa || userAgent, provider: resolvedProvider };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 将 CapSolver AntiCloudflare 返回的 cookie / UA 写入页面并 reload
|
||||||
|
* @param {import('puppeteer').Page} page
|
||||||
|
* @param {string} cleanUrl
|
||||||
|
* @param {{ cookies: Record<string, string>, userAgent?: string }} solution
|
||||||
|
*/
|
||||||
|
async function applyCfClearanceSolution(page, cleanUrl, solution) {
|
||||||
|
if (solution.userAgent) {
|
||||||
|
await page.setUserAgent(solution.userAgent).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
let host = '';
|
||||||
|
let path = '/';
|
||||||
|
try {
|
||||||
|
const parsed = new URL(cleanUrl);
|
||||||
|
host = parsed.hostname;
|
||||||
|
path = parsed.pathname || '/';
|
||||||
|
} catch (_) {
|
||||||
|
host = '';
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [name, value] of Object.entries(solution.cookies || {})) {
|
||||||
|
if (!value) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const cookieDomain = host.startsWith('.') ? host : `.${host}`;
|
||||||
|
await page.setCookie({
|
||||||
|
name,
|
||||||
|
value: String(value),
|
||||||
|
domain: cookieDomain,
|
||||||
|
path: '/',
|
||||||
|
secure: true,
|
||||||
|
sameSite: 'None',
|
||||||
|
}).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`[CF] cf_clearance 已注入,goto 业务页 url=${cleanUrl}`);
|
||||||
|
await page.goto(cleanUrl, {
|
||||||
|
waitUntil: 'domcontentloaded',
|
||||||
|
timeout: 60000,
|
||||||
|
}).catch(() => {});
|
||||||
|
await page.waitForNetworkIdle({ idleTime: 500, timeout: 30000 }).catch(() => {
|
||||||
|
console.warn('[CF] cf_clearance goto 后 networkidle 超时,继续后续流程');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 统一 Turnstile 求解入口
|
* 统一 Turnstile 求解入口
|
||||||
* @param {{ pageUrl: string, sitekey: string, provider?: string, apiKey?: string }} params
|
* @param {{ pageUrl: string, sitekey: string, provider?: string, apiKey?: string }} params
|
||||||
@@ -121,6 +266,7 @@ async function solveVia2Captcha({ pageUrl, sitekey, apiKey }) {
|
|||||||
async function solveTurnstile({ pageUrl, sitekey, provider, apiKey }) {
|
async function solveTurnstile({ pageUrl, sitekey, provider, apiKey }) {
|
||||||
const resolvedProvider = (provider || CAPTCHA_PROVIDER).toLowerCase();
|
const resolvedProvider = (provider || CAPTCHA_PROVIDER).toLowerCase();
|
||||||
const resolvedKey = apiKey || CAPTCHA_API_KEY;
|
const resolvedKey = apiKey || CAPTCHA_API_KEY;
|
||||||
|
const cleanPageUrl = sanitizePageUrlForCfSolver(pageUrl);
|
||||||
|
|
||||||
if (!sitekey) {
|
if (!sitekey) {
|
||||||
throw new Error('无法提取 Turnstile sitekey');
|
throw new Error('无法提取 Turnstile sitekey');
|
||||||
@@ -131,9 +277,9 @@ async function solveTurnstile({ pageUrl, sitekey, provider, apiKey }) {
|
|||||||
|
|
||||||
let token;
|
let token;
|
||||||
if (resolvedProvider === '2captcha') {
|
if (resolvedProvider === '2captcha') {
|
||||||
token = await solveVia2Captcha({ pageUrl, sitekey, apiKey: resolvedKey });
|
token = await solveVia2Captcha({ pageUrl: cleanPageUrl, sitekey, apiKey: resolvedKey });
|
||||||
} else {
|
} else {
|
||||||
token = await solveViaCapSolver({ pageUrl, sitekey, apiKey: resolvedKey });
|
token = await solveViaCapSolver({ pageUrl: cleanPageUrl, sitekey, apiKey: resolvedKey });
|
||||||
}
|
}
|
||||||
|
|
||||||
return { token, provider: resolvedProvider };
|
return { token, provider: resolvedProvider };
|
||||||
@@ -149,5 +295,7 @@ function isCaptchaConfigured() {
|
|||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
solveTurnstile,
|
solveTurnstile,
|
||||||
|
solveCloudflare,
|
||||||
|
applyCfClearanceSolution,
|
||||||
isCaptchaConfigured,
|
isCaptchaConfigured,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -0,0 +1,173 @@
|
|||||||
|
/**
|
||||||
|
* CapSolver / 2Captcha 过盾兜底:Turnstile token 注入 或 AntiCloudflareTask cf_clearance
|
||||||
|
*
|
||||||
|
* A2C 等业务域为 CF Managed Challenge(非独立 Turnstile 挂件),须优先 AntiCloudflareTask。
|
||||||
|
*/
|
||||||
|
const { sanitizePageUrlForCfSolver } = require('./cf-detector');
|
||||||
|
const {
|
||||||
|
injectTurnstileToken,
|
||||||
|
recoverAfterTokenInject,
|
||||||
|
extractTurnstileSitekey,
|
||||||
|
} = require('./turnstile-handler');
|
||||||
|
const {
|
||||||
|
solveTurnstile,
|
||||||
|
solveCloudflare,
|
||||||
|
applyCfClearanceSolution,
|
||||||
|
} = require('./captcha-solver');
|
||||||
|
const { getSitekeyCapture } = require('./sitekey-capture');
|
||||||
|
|
||||||
|
/** @returns {boolean} */
|
||||||
|
function hasCaptchaProxy() {
|
||||||
|
return (process.env.CAPTCHA_PROXY || '').trim() !== '';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 是否应使用 CapSolver AntiCloudflareTask(Managed Challenge / 5s 盾)
|
||||||
|
* @param {object|null|undefined} antiBot
|
||||||
|
*/
|
||||||
|
function shouldPreferManagedCloudflareSolver(antiBot) {
|
||||||
|
if (!antiBot || antiBot.cfCloudflareSolver === false) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (antiBot.cfChallengeMode === 'managed') {
|
||||||
|
return hasCaptchaProxy();
|
||||||
|
}
|
||||||
|
if (antiBot.cfCloudflareSolver === true && hasCaptchaProxy()) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 解析可用于 AntiTurnstileTask 的 sitekey(排除 A2C 配置的 challenge sitekey)
|
||||||
|
* @param {import('puppeteer').Page} page
|
||||||
|
* @param {object} antiBot
|
||||||
|
* @param {number} waitMs
|
||||||
|
* @returns {Promise<string|null>}
|
||||||
|
*/
|
||||||
|
async function resolveTurnstileWidgetSitekey(page, antiBot, waitMs) {
|
||||||
|
if (antiBot.cfChallengeMode === 'managed') {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const capture = getSitekeyCapture(page);
|
||||||
|
if (capture) {
|
||||||
|
const fromCapture = await capture.resolveForSolver(page, antiBot, waitMs);
|
||||||
|
if (fromCapture && antiBot.cfChallengeMode !== 'managed') {
|
||||||
|
const configured = String(antiBot.turnstileSitekey || '').trim();
|
||||||
|
if (configured && fromCapture === configured && antiBot.cfCloudflareSolver) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return fromCapture;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const dynamic = await extractTurnstileSitekey(page);
|
||||||
|
if (dynamic) {
|
||||||
|
return dynamic;
|
||||||
|
}
|
||||||
|
|
||||||
|
const configured = String(antiBot?.turnstileSitekey || '').trim();
|
||||||
|
if (configured && !antiBot.cfCloudflareSolver) {
|
||||||
|
return configured;
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* CapSolver AntiCloudflareTask 求解并注入 cf_clearance
|
||||||
|
* @param {import('puppeteer').Page} page
|
||||||
|
* @param {object} params
|
||||||
|
*/
|
||||||
|
async function runManagedCloudflareSolver(page, params) {
|
||||||
|
const { cleanSolverUrl, waitForUrlSettled, timeoutMs } = params;
|
||||||
|
|
||||||
|
if (!hasCaptchaProxy()) {
|
||||||
|
throw new Error('AntiCloudflareTask 需要配置 CAPTCHA_PROXY(静态代理 ip:port:user:pass)');
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('[CF] A2C/Managed Challenge:使用 CapSolver AntiCloudflareTask');
|
||||||
|
const html = await page.content().catch(() => '');
|
||||||
|
const userAgent = await page.evaluate(() => navigator.userAgent).catch(() => '');
|
||||||
|
const { cookies, userAgent: solvedUa, provider } = await solveCloudflare({
|
||||||
|
pageUrl: cleanSolverUrl,
|
||||||
|
html,
|
||||||
|
userAgent,
|
||||||
|
});
|
||||||
|
console.log(`[CF] CapSolver AntiCloudflareTask (${provider}) 返回 cf_clearance`);
|
||||||
|
await applyCfClearanceSolution(page, cleanSolverUrl, { cookies, userAgent: solvedUa || userAgent });
|
||||||
|
if (typeof waitForUrlSettled === 'function') {
|
||||||
|
await waitForUrlSettled(page).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
return { mode: 'cloudflare', provider };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {import('puppeteer').Page} page
|
||||||
|
* @param {object} options
|
||||||
|
* @param {object} options.antiBot
|
||||||
|
* @param {string} [options.navUrl]
|
||||||
|
* @param {(page: import('puppeteer').Page) => Promise<string>} [options.waitForUrlSettled]
|
||||||
|
* @param {number} [options.timeoutMs]
|
||||||
|
* @returns {Promise<{ mode: 'turnstile'|'cloudflare', provider: string }>}
|
||||||
|
*/
|
||||||
|
async function runCaptchaApiFallback(page, options) {
|
||||||
|
const {
|
||||||
|
antiBot,
|
||||||
|
navUrl = '',
|
||||||
|
waitForUrlSettled,
|
||||||
|
timeoutMs = 60000,
|
||||||
|
} = options;
|
||||||
|
|
||||||
|
const cleanSolverUrl = sanitizePageUrlForCfSolver(navUrl || page.url());
|
||||||
|
|
||||||
|
if (shouldPreferManagedCloudflareSolver(antiBot)) {
|
||||||
|
console.log(`[CF] Captcha API 求解 pageUrl=${cleanSolverUrl} mode=managed_challenge`);
|
||||||
|
return runManagedCloudflareSolver(page, { cleanSolverUrl, waitForUrlSettled, timeoutMs });
|
||||||
|
}
|
||||||
|
|
||||||
|
const sitekey = await resolveTurnstileWidgetSitekey(
|
||||||
|
page,
|
||||||
|
antiBot,
|
||||||
|
Math.min(timeoutMs, 25000)
|
||||||
|
);
|
||||||
|
console.log(`[CF] Captcha API 求解 pageUrl=${cleanSolverUrl} sitekey=${sitekey ? 'ok' : 'missing'}`);
|
||||||
|
|
||||||
|
if (sitekey) {
|
||||||
|
try {
|
||||||
|
const { token, provider } = await solveTurnstile({ pageUrl: cleanSolverUrl, sitekey });
|
||||||
|
console.log(`[CF] Captcha API Turnstile (${provider}) 返回 token,注入页面`);
|
||||||
|
await injectTurnstileToken(page, token);
|
||||||
|
await recoverAfterTokenInject(
|
||||||
|
page,
|
||||||
|
cleanSolverUrl,
|
||||||
|
waitForUrlSettled,
|
||||||
|
Math.min(timeoutMs, 45000)
|
||||||
|
);
|
||||||
|
return { mode: 'turnstile', provider };
|
||||||
|
} catch (turnstileErr) {
|
||||||
|
const msg = (turnstileErr?.message || '').toLowerCase();
|
||||||
|
const isChallengeMismatch = msg.includes('challenge, not turnstile')
|
||||||
|
|| msg.includes('not turnstile');
|
||||||
|
if (isChallengeMismatch && antiBot.cfCloudflareSolver !== false && hasCaptchaProxy()) {
|
||||||
|
console.warn(`[CF] Turnstile 任务失败(${turnstileErr.message}),降级 AntiCloudflareTask`);
|
||||||
|
return runManagedCloudflareSolver(page, { cleanSolverUrl, waitForUrlSettled, timeoutMs });
|
||||||
|
}
|
||||||
|
throw turnstileErr;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (antiBot?.cfCloudflareSolver !== false && hasCaptchaProxy()) {
|
||||||
|
return runManagedCloudflareSolver(page, { cleanSolverUrl, waitForUrlSettled, timeoutMs });
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error('无法过盾:请配置 CAPTCHA_PROXY(A2C Managed Challenge)或提供有效 Turnstile sitekey');
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
runCaptchaApiFallback,
|
||||||
|
shouldPreferManagedCloudflareSolver,
|
||||||
|
hasCaptchaProxy,
|
||||||
|
};
|
||||||
@@ -20,6 +20,34 @@ function urlIndicatesCfChallenge(url) {
|
|||||||
|| url.includes('/cdn-cgi/challenge');
|
|| url.includes('/cdn-cgi/challenge');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** CF 挑战态查询参数(CapSolver websiteURL 需剔除) */
|
||||||
|
const CF_CHALLENGE_QUERY_KEYS = [
|
||||||
|
'__cf_chl_rt_tk',
|
||||||
|
'__cf_chl_tk',
|
||||||
|
'__cf_chl_f_tk',
|
||||||
|
'__cf_chl_captcha_tk__',
|
||||||
|
];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 去掉 URL 中 CF 挑战临时参数,供 CapSolver / reload 使用
|
||||||
|
* @param {string} url
|
||||||
|
* @returns {string}
|
||||||
|
*/
|
||||||
|
function sanitizePageUrlForCfSolver(url) {
|
||||||
|
if (!url || typeof url !== 'string') {
|
||||||
|
return url || '';
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const parsed = new URL(url);
|
||||||
|
for (const key of CF_CHALLENGE_QUERY_KEYS) {
|
||||||
|
parsed.searchParams.delete(key);
|
||||||
|
}
|
||||||
|
return parsed.toString();
|
||||||
|
} catch (_) {
|
||||||
|
return url.split('?')[0] || url;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 是否要求 cf_clearance 才视为过盾完成(A2C user.a2c.chat 为 false)
|
* 是否要求 cf_clearance 才视为过盾完成(A2C user.a2c.chat 为 false)
|
||||||
* @param {object|null|undefined} antiBot
|
* @param {object|null|undefined} antiBot
|
||||||
@@ -61,7 +89,9 @@ async function detectCloudflare(page, antiBot = null) {
|
|||||||
hasTurnstileWidget,
|
hasTurnstileWidget,
|
||||||
hasChallengeRunning,
|
hasChallengeRunning,
|
||||||
titleHasJustAMoment: titleText.includes('Just a moment'),
|
titleHasJustAMoment: titleText.includes('Just a moment'),
|
||||||
bodyHasJustAMoment: bodyText.includes('Just a moment') || bodyText.includes('Checking your browser'),
|
bodyHasJustAMoment: bodyText.includes('Just a moment')
|
||||||
|
|| bodyText.includes('Checking your browser')
|
||||||
|
|| bodyText.includes('Performing security verification'),
|
||||||
};
|
};
|
||||||
}).catch(() => ({
|
}).catch(() => ({
|
||||||
hasTurnstileInput: false,
|
hasTurnstileInput: false,
|
||||||
@@ -77,7 +107,8 @@ async function detectCloudflare(page, antiBot = null) {
|
|||||||
|| domSignals.titleHasJustAMoment
|
|| domSignals.titleHasJustAMoment
|
||||||
|| domSignals.bodyHasJustAMoment
|
|| domSignals.bodyHasJustAMoment
|
||||||
|| domSignals.hasChallengeRunning
|
|| domSignals.hasChallengeRunning
|
||||||
|| title.includes('Just a moment');
|
|| title.includes('Just a moment')
|
||||||
|
|| title.includes('Performing security verification');
|
||||||
|
|
||||||
const turnstileSignals = domSignals.hasTurnstileInput || domSignals.hasTurnstileWidget;
|
const turnstileSignals = domSignals.hasTurnstileInput || domSignals.hasTurnstileWidget;
|
||||||
|
|
||||||
@@ -93,7 +124,7 @@ async function detectCloudflare(page, antiBot = null) {
|
|||||||
if (requireClearance) {
|
if (requireClearance) {
|
||||||
blocked = !hasCfClearance && (realChallenge || turnstileSignals);
|
blocked = !hasCfClearance && (realChallenge || turnstileSignals);
|
||||||
} else {
|
} else {
|
||||||
blocked = realChallenge;
|
blocked = realChallenge || (turnstileSignals && urlChallenge);
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -129,4 +160,5 @@ module.exports = {
|
|||||||
isTurnstileSolved,
|
isTurnstileSolved,
|
||||||
urlIndicatesCfChallenge,
|
urlIndicatesCfChallenge,
|
||||||
isCfClearanceRequired,
|
isCfClearanceRequired,
|
||||||
|
sanitizePageUrlForCfSolver,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -5,8 +5,9 @@
|
|||||||
* 否则清除可能过期的 cf_clearance 并重新触发 Turnstile(Whatshub 短链 → work-order-sharing 场景)。
|
* 否则清除可能过期的 cf_clearance 并重新触发 Turnstile(Whatshub 短链 → work-order-sharing 场景)。
|
||||||
*/
|
*/
|
||||||
const { detectCloudflare } = require('./cf-detector');
|
const { detectCloudflare } = require('./cf-detector');
|
||||||
const { waitForTurnstile, extractTurnstileSitekey, injectTurnstileToken, waitForTurnstileSurface } = require('./turnstile-handler');
|
const { waitForTurnstile } = require('./turnstile-handler');
|
||||||
const { solveTurnstile, isCaptchaConfigured } = require('./captcha-solver');
|
const { isCaptchaConfigured } = require('./captcha-solver');
|
||||||
|
const { runCaptchaApiFallback } = require('./cf-api-fallback');
|
||||||
const { isSessionLikelyValid } = require('./session-store');
|
const { isSessionLikelyValid } = require('./session-store');
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -257,15 +258,12 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
|
|||||||
if (waitForUrlSettled) {
|
if (waitForUrlSettled) {
|
||||||
await waitForUrlSettled(page).catch(() => {});
|
await waitForUrlSettled(page).catch(() => {});
|
||||||
}
|
}
|
||||||
let sitekey = await waitForTurnstileSurface(page, Math.min(timeoutMs, 25000));
|
await runCaptchaApiFallback(page, {
|
||||||
if (!sitekey) {
|
antiBot,
|
||||||
sitekey = await extractTurnstileSitekey(page);
|
navUrl: page.url(),
|
||||||
}
|
waitForUrlSettled,
|
||||||
const pageUrl = page.url();
|
timeoutMs,
|
||||||
console.log(`[CF] Captcha API 求解 pageUrl=${pageUrl} sitekey=${sitekey ? 'ok' : 'missing'}`);
|
});
|
||||||
const { token, provider } = await solveTurnstile({ pageUrl, sitekey });
|
|
||||||
console.log(`[CF] Captcha API (${provider}) 返回 token,注入页面`);
|
|
||||||
await injectTurnstileToken(page, token);
|
|
||||||
|
|
||||||
const apiWaitMs = Math.min(timeoutMs, 30000);
|
const apiWaitMs = Math.min(timeoutMs, 30000);
|
||||||
await waitForTurnstile(page, { timeoutMs: apiWaitMs });
|
await waitForTurnstile(page, { timeoutMs: apiWaitMs });
|
||||||
|
|||||||
@@ -85,7 +85,7 @@ const QUEUE_TIMEOUT_MS = Math.max(5000, parseInt(process.env.QUEUE_TIMEOUT_MS ||
|
|||||||
const API_INTERCEPT_TIMEOUT_MS = Math.max(10000, parseInt(process.env.API_INTERCEPT_TIMEOUT_MS || '60000', 10));
|
const API_INTERCEPT_TIMEOUT_MS = Math.max(10000, parseInt(process.env.API_INTERCEPT_TIMEOUT_MS || '60000', 10));
|
||||||
const API_INTERCEPT_TIMEOUT_REAL_MS = Math.max(
|
const API_INTERCEPT_TIMEOUT_REAL_MS = Math.max(
|
||||||
API_INTERCEPT_TIMEOUT_MS,
|
API_INTERCEPT_TIMEOUT_MS,
|
||||||
parseInt(process.env.API_INTERCEPT_TIMEOUT_REAL_MS || '90000', 10)
|
parseInt(process.env.API_INTERCEPT_TIMEOUT_REAL_MS || '180000', 10)
|
||||||
);
|
);
|
||||||
|
|
||||||
const NAVIGATION_TIMEOUT_MS = Math.max(15000, parseInt(process.env.NAVIGATION_TIMEOUT_MS || '60000', 10));
|
const NAVIGATION_TIMEOUT_MS = Math.max(15000, parseInt(process.env.NAVIGATION_TIMEOUT_MS || '60000', 10));
|
||||||
|
|||||||
@@ -0,0 +1,182 @@
|
|||||||
|
/**
|
||||||
|
* Turnstile sitekey 提前捕获:在 goto 前挂载监听,避免 60s 空等后 iframe 已销毁
|
||||||
|
*/
|
||||||
|
const {
|
||||||
|
extractSitekeyFromChallengeUrl,
|
||||||
|
extractSitekeyFromHtml,
|
||||||
|
extractTurnstileSitekey,
|
||||||
|
waitForTurnstileSurface,
|
||||||
|
} = require('./turnstile-handler');
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef {Object} SitekeyCaptureHandle
|
||||||
|
* @property {() => string|null} get
|
||||||
|
* @property {(page: import('puppeteer').Page, antiBot?: object) => Promise<string|null>} scanNow
|
||||||
|
* @property {(page: import('puppeteer').Page, antiBot?: object, waitMs?: number) => Promise<string|null>} resolveForSolver
|
||||||
|
* @property {() => void} detach
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 尝试从 URL 字符串解析并缓存 sitekey
|
||||||
|
* @param {string|null|undefined} url
|
||||||
|
* @param {{ sitekey: string|null, log: (msg: string) => void }} state
|
||||||
|
* @returns {string|null}
|
||||||
|
*/
|
||||||
|
function tryStoreSitekeyFromUrl(url, state) {
|
||||||
|
const key = extractSitekeyFromChallengeUrl(url || '');
|
||||||
|
if (key && !state.sitekey) {
|
||||||
|
state.sitekey = key;
|
||||||
|
state.log(`[CF] sitekey 捕获 (${key.slice(0, 12)}...) source=${url.slice(0, 80)}`);
|
||||||
|
}
|
||||||
|
return state.sitekey;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 在 page.goto 之前挂载 frame/response/target 监听
|
||||||
|
* @param {import('puppeteer').Page} page
|
||||||
|
* @param {import('puppeteer').Browser} [browser]
|
||||||
|
* @returns {SitekeyCaptureHandle}
|
||||||
|
*/
|
||||||
|
function attachSitekeyCapture(page, browser) {
|
||||||
|
const state = {
|
||||||
|
sitekey: null,
|
||||||
|
attached: false,
|
||||||
|
cleanups: [],
|
||||||
|
log: (msg) => console.log(msg),
|
||||||
|
};
|
||||||
|
|
||||||
|
if (page.__sitekeyCaptureHandle) {
|
||||||
|
return page.__sitekeyCaptureHandle;
|
||||||
|
}
|
||||||
|
|
||||||
|
const onFrameNavigated = (frame) => {
|
||||||
|
try {
|
||||||
|
tryStoreSitekeyFromUrl(frame.url(), state);
|
||||||
|
} catch (_) {
|
||||||
|
/* ignore */
|
||||||
|
}
|
||||||
|
};
|
||||||
|
page.on('framenavigated', onFrameNavigated);
|
||||||
|
state.cleanups.push(() => page.off('framenavigated', onFrameNavigated));
|
||||||
|
|
||||||
|
const onResponse = (response) => {
|
||||||
|
try {
|
||||||
|
tryStoreSitekeyFromUrl(response.url(), state);
|
||||||
|
} catch (_) {
|
||||||
|
/* ignore */
|
||||||
|
}
|
||||||
|
};
|
||||||
|
page.on('response', onResponse);
|
||||||
|
state.cleanups.push(() => page.off('response', onResponse));
|
||||||
|
|
||||||
|
if (browser && typeof browser.on === 'function') {
|
||||||
|
const onTargetCreated = (target) => {
|
||||||
|
try {
|
||||||
|
tryStoreSitekeyFromUrl(target.url(), state);
|
||||||
|
} catch (_) {
|
||||||
|
/* ignore */
|
||||||
|
}
|
||||||
|
};
|
||||||
|
browser.on('targetcreated', onTargetCreated);
|
||||||
|
state.cleanups.push(() => browser.off('targetcreated', onTargetCreated));
|
||||||
|
}
|
||||||
|
|
||||||
|
state.attached = true;
|
||||||
|
|
||||||
|
/** @type {SitekeyCaptureHandle} */
|
||||||
|
const handle = {
|
||||||
|
get: () => state.sitekey,
|
||||||
|
scanNow: async (scanPage, antiBot) => {
|
||||||
|
if (state.sitekey) {
|
||||||
|
return state.sitekey;
|
||||||
|
}
|
||||||
|
|
||||||
|
const frames = scanPage.frames();
|
||||||
|
for (const frame of frames) {
|
||||||
|
tryStoreSitekeyFromUrl(frame.url(), state);
|
||||||
|
if (state.sitekey) {
|
||||||
|
return state.sitekey;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const html = await scanPage.content().catch(() => '');
|
||||||
|
const fromHtml = extractSitekeyFromHtml(html);
|
||||||
|
if (fromHtml) {
|
||||||
|
state.sitekey = fromHtml;
|
||||||
|
state.log(`[CF] sitekey 从 HTML 提取 (${fromHtml.slice(0, 12)}...)`);
|
||||||
|
return fromHtml;
|
||||||
|
}
|
||||||
|
|
||||||
|
const browserRef = scanPage.browser();
|
||||||
|
if (browserRef) {
|
||||||
|
for (const target of browserRef.targets()) {
|
||||||
|
tryStoreSitekeyFromUrl(target.url(), state);
|
||||||
|
if (state.sitekey) {
|
||||||
|
return state.sitekey;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const configured = String(antiBot?.turnstileSitekey || '').trim();
|
||||||
|
if (configured && antiBot?.cfChallengeMode !== 'managed' && !antiBot?.cfCloudflareSolver) {
|
||||||
|
state.sitekey = configured;
|
||||||
|
state.log(`[CF] 使用 antiBot.turnstileSitekey (${configured.slice(0, 12)}...)`);
|
||||||
|
return configured;
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
},
|
||||||
|
resolveForSolver: async (scanPage, antiBot, waitMs = 25000) => {
|
||||||
|
let sitekey = await handle.scanNow(scanPage, antiBot);
|
||||||
|
if (sitekey) {
|
||||||
|
return sitekey;
|
||||||
|
}
|
||||||
|
|
||||||
|
sitekey = await waitForTurnstileSurface(scanPage, waitMs);
|
||||||
|
if (sitekey) {
|
||||||
|
state.sitekey = sitekey;
|
||||||
|
return sitekey;
|
||||||
|
}
|
||||||
|
|
||||||
|
sitekey = await extractTurnstileSitekey(scanPage);
|
||||||
|
if (sitekey) {
|
||||||
|
state.sitekey = sitekey;
|
||||||
|
return sitekey;
|
||||||
|
}
|
||||||
|
|
||||||
|
return handle.scanNow(scanPage, antiBot);
|
||||||
|
},
|
||||||
|
detach: () => {
|
||||||
|
for (const fn of state.cleanups) {
|
||||||
|
try {
|
||||||
|
fn();
|
||||||
|
} catch (_) {
|
||||||
|
/* ignore */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
state.cleanups = [];
|
||||||
|
state.attached = false;
|
||||||
|
if (page.__sitekeyCaptureHandle === handle) {
|
||||||
|
delete page.__sitekeyCaptureHandle;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
page.__sitekeyCaptureHandle = handle;
|
||||||
|
return handle;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 读取 page 上已挂载的 capture(无则 null)
|
||||||
|
* @param {import('puppeteer').Page} page
|
||||||
|
* @returns {SitekeyCaptureHandle|null}
|
||||||
|
*/
|
||||||
|
function getSitekeyCapture(page) {
|
||||||
|
return page?.__sitekeyCaptureHandle || null;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
attachSitekeyCapture,
|
||||||
|
getSitekeyCapture,
|
||||||
|
tryStoreSitekeyFromUrl,
|
||||||
|
};
|
||||||
@@ -1,22 +1,52 @@
|
|||||||
/**
|
/**
|
||||||
* Turnstile 内置求解:等待 widget 自动完成 + token 轮询 + 多 frame sitekey 提取
|
* Turnstile 内置求解:等待 widget 自动完成 + token 轮询 + 多 frame sitekey 提取
|
||||||
|
*
|
||||||
|
* 支持新版 challenge-platform iframe:sitekey 在 URL 路径 /rch/{id}/{sitekey}/ 中
|
||||||
*/
|
*/
|
||||||
const { detectCloudflare, isTurnstileSolved, urlIndicatesCfChallenge } = require('./cf-detector');
|
const { detectCloudflare, isTurnstileSolved, urlIndicatesCfChallenge } = require('./cf-detector');
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @typedef {Object} TurnstileResult
|
* 从 challenge-platform / Turnstile URL 解析 sitekey(Node 侧,可读 frame.url())
|
||||||
* @property {boolean} success
|
* @param {string} url
|
||||||
* @property {string} stage built_in|timeout|already_clear
|
* @returns {string|null}
|
||||||
* @property {number} elapsedMs
|
|
||||||
*/
|
*/
|
||||||
|
function extractSitekeyFromChallengeUrl(url) {
|
||||||
|
if (!url || typeof url !== 'string') {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const pathMatch = url.match(/\/rch\/[^/]+\/(0x4[A-Za-z0-9_-]+)\//i);
|
||||||
|
if (pathMatch) {
|
||||||
|
return pathMatch[1];
|
||||||
|
}
|
||||||
|
const queryMatch = url.match(/[?&]sitekey=([^&]+)/i);
|
||||||
|
if (queryMatch) {
|
||||||
|
return decodeURIComponent(queryMatch[1]);
|
||||||
|
}
|
||||||
|
const looseMatch = url.match(/\/(0x4[A-Za-z0-9_-]{10,})\//i);
|
||||||
|
if (looseMatch) {
|
||||||
|
return looseMatch[1];
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 在单个 frame 内提取 sitekey
|
* 在单个 frame 内提取 sitekey(主文档 DOM + iframe src 属性)
|
||||||
* @param {import('puppeteer').Frame} frame
|
* @param {import('puppeteer').Frame} frame
|
||||||
* @returns {Promise<string|null>}
|
* @returns {Promise<string|null>}
|
||||||
*/
|
*/
|
||||||
async function extractSitekeyFromFrame(frame) {
|
async function extractSitekeyFromFrame(frame) {
|
||||||
return frame.evaluate(() => {
|
return frame.evaluate(() => {
|
||||||
|
const extractFromSrc = (src) => {
|
||||||
|
if (!src) return null;
|
||||||
|
const pathMatch = src.match(/\/rch\/[^/]+\/(0x4[A-Za-z0-9_-]+)\//i);
|
||||||
|
if (pathMatch) return pathMatch[1];
|
||||||
|
const queryMatch = src.match(/[?&]sitekey=([^&]+)/i);
|
||||||
|
if (queryMatch) return decodeURIComponent(queryMatch[1]);
|
||||||
|
const looseMatch = src.match(/\/(0x4[A-Za-z0-9_-]{10,})\//i);
|
||||||
|
if (looseMatch) return looseMatch[1];
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
|
||||||
const widget = document.querySelector('.cf-turnstile[data-sitekey], [data-sitekey]');
|
const widget = document.querySelector('.cf-turnstile[data-sitekey], [data-sitekey]');
|
||||||
if (widget) {
|
if (widget) {
|
||||||
const key = widget.getAttribute('data-sitekey');
|
const key = widget.getAttribute('data-sitekey');
|
||||||
@@ -25,12 +55,13 @@ async function extractSitekeyFromFrame(frame) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const iframes = document.querySelectorAll('iframe[src*="challenges.cloudflare.com"], iframe[src*="challenge-platform"]');
|
const iframes = document.querySelectorAll(
|
||||||
|
'iframe[src*="challenges.cloudflare.com"], iframe[src*="challenge-platform"], iframe#cf-chl-widget'
|
||||||
|
);
|
||||||
for (const iframe of iframes) {
|
for (const iframe of iframes) {
|
||||||
const src = iframe.getAttribute('src') || '';
|
const fromSrc = extractFromSrc(iframe.getAttribute('src') || '');
|
||||||
const match = src.match(/[?&]sitekey=([^&]+)/i);
|
if (fromSrc) {
|
||||||
if (match) {
|
return fromSrc;
|
||||||
return decodeURIComponent(match[1]);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -38,7 +69,8 @@ async function extractSitekeyFromFrame(frame) {
|
|||||||
for (const script of scripts) {
|
for (const script of scripts) {
|
||||||
const text = script.textContent || '';
|
const text = script.textContent || '';
|
||||||
const match = text.match(/sitekey['"\s:=]+['"]([0-9x_a-zA-Z-]{10,})['"]/i)
|
const match = text.match(/sitekey['"\s:=]+['"]([0-9x_a-zA-Z-]{10,})['"]/i)
|
||||||
|| text.match(/data-sitekey=['"]([0-9x_a-zA-Z-]{10,})['"]/i);
|
|| text.match(/data-sitekey=['"]([0-9x_a-zA-Z-]{10,})['"]/i)
|
||||||
|
|| text.match(/\/(0x4[A-Za-z0-9_-]{10,})\//i);
|
||||||
if (match) {
|
if (match) {
|
||||||
return match[1];
|
return match[1];
|
||||||
}
|
}
|
||||||
@@ -48,6 +80,34 @@ async function extractSitekeyFromFrame(frame) {
|
|||||||
}).catch(() => null);
|
}).catch(() => null);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 从整页 HTML 文本提取 sitekey(应对 OOPIF 无法 enumerate 的场景)
|
||||||
|
* @param {string} html
|
||||||
|
* @returns {string|null}
|
||||||
|
*/
|
||||||
|
function extractSitekeyFromHtml(html) {
|
||||||
|
if (!html || typeof html !== 'string') {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const pathMatch = html.match(/\/rch\/[^/]+\/(0x4[A-Za-z0-9_-]+)\//i);
|
||||||
|
if (pathMatch) {
|
||||||
|
return pathMatch[1];
|
||||||
|
}
|
||||||
|
const dataMatch = html.match(/data-sitekey=["']([^"']+)["']/i);
|
||||||
|
if (dataMatch) {
|
||||||
|
return dataMatch[1];
|
||||||
|
}
|
||||||
|
const queryMatch = html.match(/[?&]sitekey=([^&"']+)/i);
|
||||||
|
if (queryMatch) {
|
||||||
|
return decodeURIComponent(queryMatch[1]);
|
||||||
|
}
|
||||||
|
const looseMatch = html.match(/\/(0x4[A-Za-z0-9_-]{10,})\//i);
|
||||||
|
if (looseMatch) {
|
||||||
|
return looseMatch[1];
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 从主文档及所有子 frame 提取 Turnstile sitekey
|
* 从主文档及所有子 frame 提取 Turnstile sitekey
|
||||||
* @param {import('puppeteer').Page} page
|
* @param {import('puppeteer').Page} page
|
||||||
@@ -60,6 +120,10 @@ async function extractTurnstileSitekey(page) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for (const frame of page.frames()) {
|
for (const frame of page.frames()) {
|
||||||
|
const fromFrameUrl = extractSitekeyFromChallengeUrl(frame.url());
|
||||||
|
if (fromFrameUrl) {
|
||||||
|
return fromFrameUrl;
|
||||||
|
}
|
||||||
if (frame === page.mainFrame()) {
|
if (frame === page.mainFrame()) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -69,15 +133,30 @@ async function extractTurnstileSitekey(page) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const html = await page.content().catch(() => '');
|
||||||
|
const fromHtml = extractSitekeyFromHtml(html);
|
||||||
|
if (fromHtml) {
|
||||||
|
return fromHtml;
|
||||||
|
}
|
||||||
|
|
||||||
|
const browser = page.browser();
|
||||||
|
if (browser) {
|
||||||
|
for (const target of browser.targets()) {
|
||||||
|
const fromTarget = extractSitekeyFromChallengeUrl(target.url());
|
||||||
|
if (fromTarget) {
|
||||||
|
return fromTarget;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Captcha API 调用前:等待 Turnstile 挑战面出现或 URL 进入 CF 挑战态
|
* Captcha API 调用前:等待 Turnstile 挑战面出现或 URL 进入 CF 挑战态
|
||||||
*
|
|
||||||
* @param {import('puppeteer').Page} page
|
* @param {import('puppeteer').Page} page
|
||||||
* @param {number} [timeoutMs]
|
* @param {number} [timeoutMs]
|
||||||
* @returns {Promise<string|null>} 若已提取到 sitekey 则返回,否则 null
|
* @returns {Promise<string|null>}
|
||||||
*/
|
*/
|
||||||
async function waitForTurnstileSurface(page, timeoutMs = 20000) {
|
async function waitForTurnstileSurface(page, timeoutMs = 20000) {
|
||||||
const started = Date.now();
|
const started = Date.now();
|
||||||
@@ -86,7 +165,7 @@ async function waitForTurnstileSurface(page, timeoutMs = 20000) {
|
|||||||
while (Date.now() - started < timeoutMs) {
|
while (Date.now() - started < timeoutMs) {
|
||||||
const sitekey = await extractTurnstileSitekey(page);
|
const sitekey = await extractTurnstileSitekey(page);
|
||||||
if (sitekey) {
|
if (sitekey) {
|
||||||
console.log(`[CF] 已检测到 Turnstile sitekey (${sitekey.slice(0, 8)}...)`);
|
console.log(`[CF] 已检测到 Turnstile sitekey (${sitekey.slice(0, 12)}...)`);
|
||||||
return sitekey;
|
return sitekey;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -104,7 +183,8 @@ async function waitForTurnstileSurface(page, timeoutMs = 20000) {
|
|||||||
const hasSurface = await page.evaluate(() => {
|
const hasSurface = await page.evaluate(() => {
|
||||||
return !!document.querySelector(
|
return !!document.querySelector(
|
||||||
'.cf-turnstile, [data-sitekey], [name="cf-turnstile-response"], '
|
'.cf-turnstile, [data-sitekey], [name="cf-turnstile-response"], '
|
||||||
+ 'iframe[src*="challenges.cloudflare.com"], iframe[src*="challenge-platform"]'
|
+ 'iframe[src*="challenges.cloudflare.com"], iframe[src*="challenge-platform"], '
|
||||||
|
+ 'iframe[id^="cf-chl-widget"]'
|
||||||
);
|
);
|
||||||
}).catch(() => false);
|
}).catch(() => false);
|
||||||
|
|
||||||
@@ -122,8 +202,7 @@ async function waitForTurnstileSurface(page, timeoutMs = 20000) {
|
|||||||
/**
|
/**
|
||||||
* 等待 Turnstile 通过(内置点击 / 自动跳转)
|
* 等待 Turnstile 通过(内置点击 / 自动跳转)
|
||||||
* @param {import('puppeteer').Page} page
|
* @param {import('puppeteer').Page} page
|
||||||
* @param {{ timeoutMs?: number, pollMs?: number, useBuiltInClick?: boolean }} [options]
|
* @param {{ timeoutMs?: number, pollMs?: number }} [options]
|
||||||
* @returns {Promise<TurnstileResult>}
|
|
||||||
*/
|
*/
|
||||||
async function waitForTurnstile(page, options = {}) {
|
async function waitForTurnstile(page, options = {}) {
|
||||||
const timeoutMs = options.timeoutMs ?? 60000;
|
const timeoutMs = options.timeoutMs ?? 60000;
|
||||||
@@ -148,7 +227,7 @@ async function waitForTurnstile(page, options = {}) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const state = await detectCloudflare(page);
|
const state = await detectCloudflare(page);
|
||||||
if (!state.blocked && !urlIndicatesCfChallenge(page.url()) && state.hasCfClearance) {
|
if (!state.blocked && !urlIndicatesCfChallenge(page.url())) {
|
||||||
return { success: true, stage: 'built_in', elapsedMs: Date.now() - started };
|
return { success: true, stage: 'built_in', elapsedMs: Date.now() - started };
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -159,7 +238,27 @@ async function waitForTurnstile(page, options = {}) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 将 Captcha API 返回的 token 注入页面
|
* 等待页面离开 CF 挑战 URL / 挑战 DOM
|
||||||
|
* @param {import('puppeteer').Page} page
|
||||||
|
* @param {number} timeoutMs
|
||||||
|
* @param {number} pollMs
|
||||||
|
*/
|
||||||
|
async function waitForChallengeUrlClear(page, timeoutMs = 45000, pollMs = 500) {
|
||||||
|
const started = Date.now();
|
||||||
|
while (Date.now() - started < timeoutMs) {
|
||||||
|
if (!urlIndicatesCfChallenge(page.url())) {
|
||||||
|
const state = await detectCloudflare(page);
|
||||||
|
if (!state.blocked) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await new Promise((r) => setTimeout(r, pollMs));
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 将 Captcha API 返回的 token 注入页面并尝试触发提交
|
||||||
* @param {import('puppeteer').Page} page
|
* @param {import('puppeteer').Page} page
|
||||||
* @param {string} token
|
* @param {string} token
|
||||||
*/
|
*/
|
||||||
@@ -178,10 +277,39 @@ async function injectTurnstileToken(page, token) {
|
|||||||
}, token);
|
}, token);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* token 注入后等待过盾;若仍停留在挑战页则 reload 干净业务 URL
|
||||||
|
* @param {import('puppeteer').Page} page
|
||||||
|
* @param {string} cleanUrl
|
||||||
|
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
|
||||||
|
* @param {number} [timeoutMs]
|
||||||
|
*/
|
||||||
|
async function recoverAfterTokenInject(page, cleanUrl, waitForUrlSettled, timeoutMs = 45000) {
|
||||||
|
if (await waitForChallengeUrlClear(page, Math.min(timeoutMs, 12000))) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!cleanUrl) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`[CF] token 注入后仍处挑战页,reload 干净 URL: ${cleanUrl}`);
|
||||||
|
await page.goto(cleanUrl, { waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 60000) }).catch(() => {});
|
||||||
|
if (typeof waitForUrlSettled === 'function') {
|
||||||
|
await waitForUrlSettled(page).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
return waitForChallengeUrlClear(page, timeoutMs);
|
||||||
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
waitForTurnstile,
|
waitForTurnstile,
|
||||||
waitForTurnstileSurface,
|
waitForTurnstileSurface,
|
||||||
extractTurnstileSitekey,
|
extractTurnstileSitekey,
|
||||||
extractSitekeyFromFrame,
|
extractSitekeyFromFrame,
|
||||||
|
extractSitekeyFromChallengeUrl,
|
||||||
|
extractSitekeyFromHtml,
|
||||||
injectTurnstileToken,
|
injectTurnstileToken,
|
||||||
|
waitForChallengeUrlClear,
|
||||||
|
recoverAfterTokenInject,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -20,7 +20,8 @@ function classifyPuppeteerError(err) {
|
|||||||
if (err?.name === 'TimeoutError' || msg.includes('timeout')) return 'timeout';
|
if (err?.name === 'TimeoutError' || msg.includes('timeout')) return 'timeout';
|
||||||
if (msg.includes('net::') || msg.includes('network') || msg.includes('err_connection')
|
if (msg.includes('net::') || msg.includes('network') || msg.includes('err_connection')
|
||||||
|| msg.includes('err_address_unreachable') || msg.includes('err_name_not_resolved')) return 'network';
|
|| msg.includes('err_address_unreachable') || msg.includes('err_name_not_resolved')) return 'network';
|
||||||
if (msg.includes('navigation') || msg.includes('frame was detached')) return 'navigation';
|
if (msg.includes('navigation') || msg.includes('frame was detached')
|
||||||
|
|| msg.includes('detached frame') || msg.includes('attempted to use detached')) return 'navigation';
|
||||||
if (msg.includes('target closed') || msg.includes('session closed')) return 'target_closed';
|
if (msg.includes('target closed') || msg.includes('session closed')) return 'target_closed';
|
||||||
return 'unknown';
|
return 'unknown';
|
||||||
}
|
}
|
||||||
@@ -85,7 +86,9 @@ async function runUiPagination(page, options) {
|
|||||||
{ timeout: 1000, polling: 100 }
|
{ timeout: 1000, polling: 100 }
|
||||||
).catch(() => new Promise((r) => setTimeout(r, 500)));
|
).catch(() => new Promise((r) => setTimeout(r, 500)));
|
||||||
|
|
||||||
while (attempts < maxAttempts && !pageData) {
|
let paginationComplete = false;
|
||||||
|
|
||||||
|
while (attempts < maxAttempts && !pageData && !paginationComplete) {
|
||||||
attempts++;
|
attempts++;
|
||||||
log(`[抓取] 准备抓取第 ${targetPageNum} 页 (尝试 ${attempts}/${maxAttempts})...`);
|
log(`[抓取] 准备抓取第 ${targetPageNum} 页 (尝试 ${attempts}/${maxAttempts})...`);
|
||||||
|
|
||||||
@@ -139,6 +142,7 @@ async function runUiPagination(page, options) {
|
|||||||
|
|
||||||
if (clickStatus === 'disabled') {
|
if (clickStatus === 'disabled') {
|
||||||
log(`[提示] 第 ${targetPageNum} 页按钮已被禁用,说明到底了,抓取提前结束。`);
|
log(`[提示] 第 ${targetPageNum} 页按钮已被禁用,说明到底了,抓取提前结束。`);
|
||||||
|
paginationComplete = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
log('[动作] 双引擎原生点击已执行!');
|
log('[动作] 双引擎原生点击已执行!');
|
||||||
@@ -153,6 +157,11 @@ async function runUiPagination(page, options) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (paginationComplete) {
|
||||||
|
log('[完成] 翻页已到底,正常结束。');
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
if (pageData) {
|
if (pageData) {
|
||||||
capturedData.push(pageData);
|
capturedData.push(pageData);
|
||||||
log(`[成功] 斩获第 ${targetPageNum} 页数据!`);
|
log(`[成功] 斩获第 ${targetPageNum} 页数据!`);
|
||||||
|
|||||||
+431
-52
@@ -1,5 +1,12 @@
|
|||||||
require('dotenv').config({ path: require('path').join(__dirname, '.env') });
|
require('dotenv').config({ path: require('path').join(__dirname, '.env') });
|
||||||
|
|
||||||
|
process.on('uncaughtException', (err) => {
|
||||||
|
console.error('[FATAL] uncaughtException:', err);
|
||||||
|
});
|
||||||
|
process.on('unhandledRejection', (reason) => {
|
||||||
|
console.error('[FATAL] unhandledRejection:', reason);
|
||||||
|
});
|
||||||
|
|
||||||
const express = require('express');
|
const express = require('express');
|
||||||
const fs = require('fs');
|
const fs = require('fs');
|
||||||
const {
|
const {
|
||||||
@@ -41,12 +48,14 @@ const { launchStandardBrowser } = require('./lib/launch-standard');
|
|||||||
const { isRealBrowserAvailable } = require('./lib/launch-real-browser');
|
const { isRealBrowserAvailable } = require('./lib/launch-real-browser');
|
||||||
const { handleCloudflareChallenge, isCaptchaConfigured } = require('./lib/cf-handler');
|
const { handleCloudflareChallenge, isCaptchaConfigured } = require('./lib/cf-handler');
|
||||||
const { detectCloudflare, urlIndicatesCfChallenge } = require('./lib/cf-detector');
|
const { detectCloudflare, urlIndicatesCfChallenge } = require('./lib/cf-detector');
|
||||||
|
const { runCaptchaApiFallback, shouldPreferManagedCloudflareSolver } = require('./lib/cf-api-fallback');
|
||||||
const {
|
const {
|
||||||
extractTurnstileSitekey,
|
getCaptchaProxyChromeArgs,
|
||||||
injectTurnstileToken,
|
applyCaptchaProxyToPage,
|
||||||
waitForTurnstileSurface,
|
parseCaptchaProxy,
|
||||||
} = require('./lib/turnstile-handler');
|
hasCaptchaProxy,
|
||||||
const { solveTurnstile } = require('./lib/captcha-solver');
|
} = require('./lib/captcha-proxy');
|
||||||
|
const { attachSitekeyCapture } = require('./lib/sitekey-capture');
|
||||||
const {
|
const {
|
||||||
saveSession,
|
saveSession,
|
||||||
getSessionDebugInfo,
|
getSessionDebugInfo,
|
||||||
@@ -78,7 +87,7 @@ async function runBrowserTask(taskName, handler, profile = 'standard') {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function sendTaskError(res, error, extra = {}) {
|
function sendTaskError(res, error, extra = {}) {
|
||||||
if (error.code === 'QUEUE_TIMEOUT') {
|
if (error.code === 'QUEUE_TIMEOUT' || error.message === 'POOL_ACQUIRE_TIMEOUT') {
|
||||||
return res.status(503).json({
|
return res.status(503).json({
|
||||||
success: false,
|
success: false,
|
||||||
error: '服务繁忙,排队超时,请稍后重试',
|
error: '服务繁忙,排队超时,请稍后重试',
|
||||||
@@ -124,7 +133,8 @@ function classifyPuppeteerError(err) {
|
|||||||
if (err?.name === 'TimeoutError' || msg.includes('timeout')) return 'timeout';
|
if (err?.name === 'TimeoutError' || msg.includes('timeout')) return 'timeout';
|
||||||
if (msg.includes('net::') || msg.includes('network') || msg.includes('err_connection')
|
if (msg.includes('net::') || msg.includes('network') || msg.includes('err_connection')
|
||||||
|| msg.includes('err_address_unreachable') || msg.includes('err_name_not_resolved')) return 'network';
|
|| msg.includes('err_address_unreachable') || msg.includes('err_name_not_resolved')) return 'network';
|
||||||
if (msg.includes('navigation') || msg.includes('frame was detached')) return 'navigation';
|
if (msg.includes('navigation') || msg.includes('frame was detached')
|
||||||
|
|| msg.includes('detached frame') || msg.includes('attempted to use detached')) return 'navigation';
|
||||||
if (msg.includes('target closed') || msg.includes('session closed')) return 'target_closed';
|
if (msg.includes('target closed') || msg.includes('session closed')) return 'target_closed';
|
||||||
return 'unknown';
|
return 'unknown';
|
||||||
}
|
}
|
||||||
@@ -190,12 +200,16 @@ function resolveAntiBot(rawAntiBot) {
|
|||||||
...base,
|
...base,
|
||||||
postCfReadyUrlContains: rawAntiBot.postCfReadyUrlContains || '',
|
postCfReadyUrlContains: rawAntiBot.postCfReadyUrlContains || '',
|
||||||
postCfReadySelector: rawAntiBot.postCfReadySelector || '',
|
postCfReadySelector: rawAntiBot.postCfReadySelector || '',
|
||||||
|
postCfReadyApiUrl: rawAntiBot.postCfReadyApiUrl || '',
|
||||||
postCfReadyTimeoutMs: rawAntiBot.postCfReadyTimeoutMs || 0,
|
postCfReadyTimeoutMs: rawAntiBot.postCfReadyTimeoutMs || 0,
|
||||||
postCfSettleMs: rawAntiBot.postCfSettleMs || 0,
|
postCfSettleMs: rawAntiBot.postCfSettleMs || 0,
|
||||||
postCfSpaWaitMs: rawAntiBot.postCfSpaWaitMs || 0,
|
postCfSpaWaitMs: rawAntiBot.postCfSpaWaitMs || 0,
|
||||||
cfClearanceRequired: rawAntiBot.cfClearanceRequired !== false,
|
cfClearanceRequired: rawAntiBot.cfClearanceRequired !== false,
|
||||||
businessHostHint: rawAntiBot.businessHostHint || '',
|
businessHostHint: rawAntiBot.businessHostHint || '',
|
||||||
landingUrlHint: rawAntiBot.landingUrlHint || '',
|
landingUrlHint: rawAntiBot.landingUrlHint || '',
|
||||||
|
turnstileSitekey: rawAntiBot.turnstileSitekey || '',
|
||||||
|
cfCloudflareSolver: rawAntiBot.cfCloudflareSolver !== false,
|
||||||
|
cfChallengeMode: rawAntiBot.cfChallengeMode || '',
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -265,12 +279,15 @@ function applyA2cPostCfDefaults(antiBot, pageUrl) {
|
|||||||
return {
|
return {
|
||||||
...antiBot,
|
...antiBot,
|
||||||
postCfReadyUrlContains: '/visitors/counter/share',
|
postCfReadyUrlContains: '/visitors/counter/share',
|
||||||
postCfReadySelector: '.btn-next',
|
postCfReadySelector: '.el-table',
|
||||||
postCfReadyTimeoutMs: antiBot.postCfReadyTimeoutMs || 30000,
|
postCfReadyApiUrl: antiBot.postCfReadyApiUrl || '/api/talk/counter/share/record/list',
|
||||||
|
postCfReadyTimeoutMs: antiBot.postCfReadyTimeoutMs || 60000,
|
||||||
postCfSettleMs: antiBot.postCfSettleMs || 500,
|
postCfSettleMs: antiBot.postCfSettleMs || 500,
|
||||||
postCfSpaWaitMs: antiBot.postCfSpaWaitMs || 4000,
|
postCfSpaWaitMs: antiBot.postCfSpaWaitMs || 8000,
|
||||||
cfClearanceRequired: antiBot.cfClearanceRequired === undefined ? false : antiBot.cfClearanceRequired,
|
cfClearanceRequired: antiBot.cfClearanceRequired === undefined ? false : antiBot.cfClearanceRequired,
|
||||||
solverFallback: antiBot.solverFallback === undefined ? false : antiBot.solverFallback,
|
solverFallback: antiBot.solverFallback === undefined ? true : antiBot.solverFallback,
|
||||||
|
cfCloudflareSolver: antiBot.cfCloudflareSolver !== false,
|
||||||
|
cfChallengeMode: antiBot.cfChallengeMode || 'managed',
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -326,6 +343,51 @@ function isBusinessUrlReady(page, urlNeedle) {
|
|||||||
return !urlNeedle || page.url().includes(urlNeedle);
|
return !urlNeedle || page.url().includes(urlNeedle);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** A2C 已直达 user.a2c.chat 分享页(非 yyk.ink 短链跳转场景) */
|
||||||
|
function isA2cDirectSharePage(page, urlNeedle = '') {
|
||||||
|
const needle = (urlNeedle || '').trim();
|
||||||
|
if (!needle.includes('/visitors/counter/share')) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const parsed = new URL(page.url() || '');
|
||||||
|
return parsed.hostname.includes('a2c.chat') && parsed.pathname.includes('/visitors/counter/share');
|
||||||
|
} catch (_) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 过盾后等待业务 list API 返回(A2C Vue 挂载信号) */
|
||||||
|
async function waitForPostCfListApi(page, antiBot, timeoutMs) {
|
||||||
|
const apiUrl = (antiBot.postCfReadyApiUrl || '').trim();
|
||||||
|
if (!apiUrl) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
console.log(`[CF] 等待业务 API 响应 api=${apiUrl} timeout=${timeoutMs}ms`);
|
||||||
|
const matched = await page.waitForResponse(
|
||||||
|
(resp) => {
|
||||||
|
const req = resp.request();
|
||||||
|
if (req.method() === 'OPTIONS') {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return resp.url().includes(apiUrl) && resp.status() >= 200 && resp.status() < 400;
|
||||||
|
},
|
||||||
|
{ timeout: timeoutMs }
|
||||||
|
).catch(() => null);
|
||||||
|
if (matched) {
|
||||||
|
console.log(`[CF] 业务 API 已响应 status=${matched.status()} url=${matched.url()}`);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
console.warn('[CF] 业务 API 等待超时,降级为 DOM selector 检测');
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 等待业务 DOM:先 attached 再 visible,避免 Vue 渲染中途误判 */
|
||||||
|
async function waitForPostCfSelector(page, selector, timeoutMs) {
|
||||||
|
await page.waitForSelector(selector, { timeout: timeoutMs, visible: false });
|
||||||
|
await page.waitForSelector(selector, { timeout: Math.min(15000, timeoutMs), visible: true });
|
||||||
|
}
|
||||||
|
|
||||||
/** Whatshub 分享短链 /m/xxx/1 */
|
/** Whatshub 分享短链 /m/xxx/1 */
|
||||||
function isWhatshubShortLinkPath(url) {
|
function isWhatshubShortLinkPath(url) {
|
||||||
try {
|
try {
|
||||||
@@ -366,11 +428,141 @@ function buildManagedPageOptions(pageUrl, antiBot, mergedCookies, extraOpts = {}
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 任务页创建(含 Whatshub skipUserAgent);browser-factory 无写权限时在 server 侧实现
|
* A2C:短链入口时优先导航至 PHP 预解析的长链,减少 yyk.ink 跳转竞态
|
||||||
|
* @param {string} pageUrl
|
||||||
|
* @param {object} antiBot
|
||||||
*/
|
*/
|
||||||
async function createTaskPage(browser, existingPage, pageUrl, antiBot, mergedCookies, extraOpts = {}) {
|
function resolveNavigationUrl(pageUrl, antiBot) {
|
||||||
|
const hint = String(antiBot?.landingUrlHint || '').trim();
|
||||||
|
if (hint === '' || !hint.includes('/visitors/counter/share')) {
|
||||||
|
return pageUrl;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const pageHost = new URL(pageUrl).hostname;
|
||||||
|
const hintHost = new URL(hint).hostname;
|
||||||
|
if (hintHost.includes('a2c.chat') && pageHost !== hintHost) {
|
||||||
|
console.log(`[A2C] 短链入口,直接导航至预解析长链: ${hint}`);
|
||||||
|
return hint;
|
||||||
|
}
|
||||||
|
} catch (_) {
|
||||||
|
/* ignore */
|
||||||
|
}
|
||||||
|
return pageUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Chrome 启动参数:仅当 useCaptchaProxy=true 时注入 CAPTCHA_PROXY(与 CapSolver 同源 IP)
|
||||||
|
* @param {object|null|undefined} antiBot
|
||||||
|
* @param {string[]} [baseArgs]
|
||||||
|
* @param {boolean} [useCaptchaProxy]
|
||||||
|
*/
|
||||||
|
function buildBrowserExtraArgs(antiBot, baseArgs = [], useCaptchaProxy = false) {
|
||||||
|
const args = [...baseArgs];
|
||||||
|
if (!useCaptchaProxy || !hasCaptchaProxy()) {
|
||||||
|
return args;
|
||||||
|
}
|
||||||
|
const proxyArgs = getCaptchaProxyChromeArgs();
|
||||||
|
if (proxyArgs.length > 0) {
|
||||||
|
const parsed = parseCaptchaProxy();
|
||||||
|
console.log(
|
||||||
|
`[Proxy] Chrome 使用 CAPTCHA_PROXY ${parsed.host}:${parsed.port}(与 CapSolver 同源 IP)`
|
||||||
|
);
|
||||||
|
args.push(...proxyArgs);
|
||||||
|
}
|
||||||
|
return args;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Managed A2C 等:仅在检测到真实 CF 挑战时切换为 CAPTCHA_PROXY Browser(无盾则直连)
|
||||||
|
* 会原地更新 browserCtx.browserSession / browserCtx.page
|
||||||
|
* @param {object} browserCtx
|
||||||
|
*/
|
||||||
|
async function ensureCaptchaProxyBrowserIfNeeded(browserCtx) {
|
||||||
|
const {
|
||||||
|
antiBot,
|
||||||
|
profile,
|
||||||
|
navigationUrl,
|
||||||
|
mergedCookies,
|
||||||
|
baseExtraArgs = ['--mute-audio'],
|
||||||
|
pageExtraOpts = {},
|
||||||
|
} = browserCtx;
|
||||||
|
let { browserSession, page } = browserCtx;
|
||||||
|
|
||||||
|
if (!antiBot?.enabled || browserSession?.captchaProxyActive) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!shouldPreferManagedCloudflareSolver(antiBot) || !hasCaptchaProxy()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const cfState = await detectCloudflare(page, antiBot);
|
||||||
|
const needsProxy = cfState.blocked || urlIndicatesCfChallenge(page.url());
|
||||||
|
if (!needsProxy) {
|
||||||
|
console.log('[Proxy] 未检测到 CF 挑战,Chrome 直连(不使用 CAPTCHA_PROXY)');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('[Proxy] 检测到 CF 挑战,切换 Browser 为 CAPTCHA_PROXY(与 CapSolver 同源 IP)');
|
||||||
|
|
||||||
|
const snapshotCookies = await page.cookies().catch(() => []);
|
||||||
|
await closeTaskPage(page);
|
||||||
|
await browserSession.release(true);
|
||||||
|
|
||||||
|
const extraArgs = buildBrowserExtraArgs(antiBot, baseExtraArgs, true);
|
||||||
|
browserSession = await acquireBrowserSession({
|
||||||
|
profile,
|
||||||
|
extraArgs,
|
||||||
|
sessionKey: antiBot.sessionKey || '',
|
||||||
|
});
|
||||||
|
browserSession.captchaProxyActive = true;
|
||||||
|
|
||||||
|
page = await createTaskPage(
|
||||||
|
browserSession.browser,
|
||||||
|
browserSession.reused ? null : browserSession.page,
|
||||||
|
navigationUrl,
|
||||||
|
antiBot,
|
||||||
|
mergedCookies.length > 0 ? mergedCookies : snapshotCookies,
|
||||||
|
{ ...pageExtraOpts, useCaptchaProxy: true }
|
||||||
|
);
|
||||||
|
|
||||||
|
if (antiBot?.enabled) {
|
||||||
|
attachSitekeyCapture(page, browserSession.browser);
|
||||||
|
}
|
||||||
|
|
||||||
|
await prepareWhatshubShortLinkBeforeNav(page, navigationUrl, antiBot);
|
||||||
|
await navigateToPage(page, navigationUrl);
|
||||||
|
await waitForUrlSettled(page).catch(() => {});
|
||||||
|
|
||||||
|
if (typeof browserCtx.afterRelaunchNav === 'function') {
|
||||||
|
await browserCtx.afterRelaunchNav(page);
|
||||||
|
}
|
||||||
|
|
||||||
|
browserCtx.browserSession = browserSession;
|
||||||
|
browserCtx.page = page;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 任务结束关闭本次 Tab(温池复用 Browser 时保留 Browser 实例) */
|
||||||
|
async function closeTaskPage(page) {
|
||||||
|
if (!page) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
if (typeof page.isClosed === 'function' && !page.isClosed()) {
|
||||||
|
await page.close();
|
||||||
|
}
|
||||||
|
} catch (_) {
|
||||||
|
/* ignore */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务页创建(含 Whatshub skipUserAgent)
|
||||||
|
* 温池复用 Browser 时每次 newPage,冷启动 Real Browser 可使用 launch 返回的首个 Tab
|
||||||
|
*/
|
||||||
|
async function createTaskPage(browser, warmPage, pageUrl, antiBot, mergedCookies, extraOpts = {}) {
|
||||||
const opts = buildManagedPageOptions(pageUrl, antiBot, mergedCookies, extraOpts);
|
const opts = buildManagedPageOptions(pageUrl, antiBot, mergedCookies, extraOpts);
|
||||||
const page = existingPage || await browser.newPage();
|
const useWarmPage = !!(warmPage && extraOpts.useWarmPage !== false && !extraOpts.forceNewPage);
|
||||||
|
const page = useWarmPage ? warmPage : await browser.newPage();
|
||||||
page.setDefaultNavigationTimeout(NAVIGATION_TIMEOUT_MS);
|
page.setDefaultNavigationTimeout(NAVIGATION_TIMEOUT_MS);
|
||||||
page.setDefaultTimeout(PAGE_DEFAULT_TIMEOUT_MS);
|
page.setDefaultTimeout(PAGE_DEFAULT_TIMEOUT_MS);
|
||||||
|
|
||||||
@@ -384,9 +576,17 @@ async function createTaskPage(browser, existingPage, pageUrl, antiBot, mergedCoo
|
|||||||
await page.setCookie(...opts.cookies);
|
await page.setCookie(...opts.cookies);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (extraOpts.useCaptchaProxy && hasCaptchaProxy()) {
|
||||||
|
await applyCaptchaProxyToPage(page);
|
||||||
|
}
|
||||||
|
|
||||||
page.on('error', (err) => console.error('[Page Error]', err.message));
|
page.on('error', (err) => console.error('[Page Error]', err.message));
|
||||||
page.on('pageerror', (err) => console.error('[Page JS Error]', err.message));
|
page.on('pageerror', (err) => console.error('[Page JS Error]', err.message));
|
||||||
|
|
||||||
|
if (antiBot?.enabled) {
|
||||||
|
attachSitekeyCapture(page, browser);
|
||||||
|
}
|
||||||
|
|
||||||
return page;
|
return page;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -419,25 +619,32 @@ function isOnShortLinkNotBusiness(page, urlNeedle) {
|
|||||||
async function waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettledFn, pageUrl = '') {
|
async function waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettledFn, pageUrl = '') {
|
||||||
const urlNeedle = (antiBot.postCfReadyUrlContains || '').trim();
|
const urlNeedle = (antiBot.postCfReadyUrlContains || '').trim();
|
||||||
const targetUrl = pageUrl || page.url();
|
const targetUrl = pageUrl || page.url();
|
||||||
if (!urlNeedle || !isOnShortLinkNotBusiness(page, urlNeedle)) {
|
const onA2cDirect = isA2cDirectSharePage(page, urlNeedle);
|
||||||
|
if (!urlNeedle || (!isOnShortLinkNotBusiness(page, urlNeedle) && !onA2cDirect)) {
|
||||||
return isBusinessUrlReady(page, urlNeedle);
|
return isBusinessUrlReady(page, urlNeedle);
|
||||||
}
|
}
|
||||||
if (!(await isCfTrulyComplete(page, antiBot))) {
|
if (!(await isCfTrulyComplete(page, antiBot))) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
const spaWaitMs = Math.max(8000, antiBot.postCfSpaWaitMs || 8000);
|
const spaWaitMs = onA2cDirect
|
||||||
|
? Math.max(8000, antiBot.postCfSpaWaitMs || 8000)
|
||||||
|
: Math.max(8000, antiBot.postCfSpaWaitMs || 8000);
|
||||||
const isWhatshub = isWhatshubShortLinkPath(targetUrl);
|
const isWhatshub = isWhatshubShortLinkPath(targetUrl);
|
||||||
|
|
||||||
console.log(
|
console.log(
|
||||||
`[CF] CF 已完成,等待 SPA 跳转 settle=${spaWaitMs}ms current=${page.url()}`
|
`[CF] CF 已完成,等待 SPA ${onA2cDirect ? '直链挂载' : '跳转'} settle=${spaWaitMs}ms current=${page.url()}`
|
||||||
);
|
);
|
||||||
await new Promise((resolve) => setTimeout(resolve, spaWaitMs));
|
await new Promise((resolve) => setTimeout(resolve, spaWaitMs));
|
||||||
|
|
||||||
if (typeof waitForUrlSettledFn === 'function') {
|
if (typeof waitForUrlSettledFn === 'function') {
|
||||||
await waitForUrlSettledFn(page).catch(() => {});
|
await waitForUrlSettledFn(page).catch(() => {});
|
||||||
}
|
}
|
||||||
await page.waitForNetworkIdle({ idleTime: 500, timeout: 15000 }).catch(() => {});
|
await page.waitForNetworkIdle({ idleTime: 500, timeout: 20000 }).catch(() => {});
|
||||||
|
|
||||||
|
if (onA2cDirect) {
|
||||||
|
await waitForPostCfListApi(page, antiBot, Math.max(15000, antiBot.postCfReadyTimeoutMs || 60000)).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
if (isBusinessUrlReady(page, urlNeedle)) {
|
if (isBusinessUrlReady(page, urlNeedle)) {
|
||||||
console.log(`[CF] SPA 已跳转至业务页 url=${page.url()}`);
|
console.log(`[CF] SPA 已跳转至业务页 url=${page.url()}`);
|
||||||
@@ -503,13 +710,18 @@ async function isCfTrulyComplete(page, antiBot = null) {
|
|||||||
return !state.blocked;
|
return !state.blocked;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 轮询直到 CF 真正完成(不依赖 turnstile-handler 的 early return) */
|
/** 轮询直到 CF 真正完成;并行扫描 sitekey 供 CapSolver 提前使用 */
|
||||||
async function waitForCfTrulyComplete(page, timeoutMs, pollMs = 500, antiBot = null) {
|
async function waitForCfTrulyComplete(page, timeoutMs, pollMs = 500, antiBot = null) {
|
||||||
|
const { getSitekeyCapture } = require('./lib/sitekey-capture');
|
||||||
|
const capture = getSitekeyCapture(page);
|
||||||
const started = Date.now();
|
const started = Date.now();
|
||||||
while (Date.now() - started < timeoutMs) {
|
while (Date.now() - started < timeoutMs) {
|
||||||
if (await isCfTrulyComplete(page, antiBot)) {
|
if (await isCfTrulyComplete(page, antiBot)) {
|
||||||
return { success: true, elapsedMs: Date.now() - started };
|
return { success: true, elapsedMs: Date.now() - started };
|
||||||
}
|
}
|
||||||
|
if (capture) {
|
||||||
|
await capture.scanNow(page, antiBot).catch(() => {});
|
||||||
|
}
|
||||||
await new Promise((resolve) => setTimeout(resolve, pollMs));
|
await new Promise((resolve) => setTimeout(resolve, pollMs));
|
||||||
}
|
}
|
||||||
return { success: false, elapsedMs: Date.now() - started };
|
return { success: false, elapsedMs: Date.now() - started };
|
||||||
@@ -566,6 +778,62 @@ async function runCfChallengeFlow(page, antiBot, waitForUrlSettled, pageUrl = ''
|
|||||||
console.log(`[CF] 免 clearance 模式但检测到真实 CF 挑战 type=${preState.challengeType},继续过盾`);
|
console.log(`[CF] 免 clearance 模式但检测到真实 CF 挑战 type=${preState.challengeType},继续过盾`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const preferManaged = shouldPreferManagedCloudflareSolver(antiBot);
|
||||||
|
|
||||||
|
if (preferManaged && antiBot.solverFallback !== false && isCaptchaConfigured()) {
|
||||||
|
console.log('[CF] Managed Challenge:跳过内置 60s 轮询,直接使用 CapSolver AntiCloudflareTask');
|
||||||
|
try {
|
||||||
|
if (waitForUrlSettled) {
|
||||||
|
await waitForUrlSettled(page).catch(() => {});
|
||||||
|
}
|
||||||
|
await runCaptchaApiFallback(page, {
|
||||||
|
antiBot,
|
||||||
|
navUrl,
|
||||||
|
waitForUrlSettled,
|
||||||
|
timeoutMs,
|
||||||
|
});
|
||||||
|
|
||||||
|
const apiWait = await waitForCfTrulyComplete(page, Math.min(timeoutMs, 60000), 500, antiBot);
|
||||||
|
if (apiWait.success) {
|
||||||
|
if (waitForUrlSettled) {
|
||||||
|
await waitForUrlSettled(page).catch(() => {});
|
||||||
|
}
|
||||||
|
await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettled, navUrl);
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
code: null,
|
||||||
|
challengeType: 'managed',
|
||||||
|
stage: 'api_managed',
|
||||||
|
solverUsed: true,
|
||||||
|
elapsedMs: Date.now() - started,
|
||||||
|
cfDetected: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
console.warn('[CF] Managed CapSolver 注入后仍未获得有效 cf_clearance');
|
||||||
|
} catch (apiErr) {
|
||||||
|
console.error('[CF] Managed Challenge CapSolver 失败:', apiErr.message);
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
code: 'CF_TURNSTILE_FAILED',
|
||||||
|
challengeType: 'managed',
|
||||||
|
stage: 'api_managed',
|
||||||
|
solverUsed: true,
|
||||||
|
elapsedMs: Date.now() - started,
|
||||||
|
cfDetected: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
code: 'CF_TURNSTILE_FAILED',
|
||||||
|
challengeType: 'managed',
|
||||||
|
stage: 'api_managed',
|
||||||
|
solverUsed: true,
|
||||||
|
elapsedMs: Date.now() - started,
|
||||||
|
cfDetected: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
if (antiBot.turnstile !== false) {
|
if (antiBot.turnstile !== false) {
|
||||||
console.log(`[CF] 内置 Turnstile 轮询 timeout=${timeoutMs}ms`);
|
console.log(`[CF] 内置 Turnstile 轮询 timeout=${timeoutMs}ms`);
|
||||||
const builtIn = await waitForCfTrulyComplete(page, timeoutMs, 500, antiBot);
|
const builtIn = await waitForCfTrulyComplete(page, timeoutMs, 500, antiBot);
|
||||||
@@ -592,15 +860,12 @@ async function runCfChallengeFlow(page, antiBot, waitForUrlSettled, pageUrl = ''
|
|||||||
if (waitForUrlSettled) {
|
if (waitForUrlSettled) {
|
||||||
await waitForUrlSettled(page).catch(() => {});
|
await waitForUrlSettled(page).catch(() => {});
|
||||||
}
|
}
|
||||||
let sitekey = await waitForTurnstileSurface(page, Math.min(timeoutMs, 25000));
|
await runCaptchaApiFallback(page, {
|
||||||
if (!sitekey) {
|
antiBot,
|
||||||
sitekey = await extractTurnstileSitekey(page);
|
navUrl,
|
||||||
}
|
waitForUrlSettled,
|
||||||
const pageUrlForSolver = page.url();
|
timeoutMs,
|
||||||
console.log(`[CF] Captcha API 求解 pageUrl=${pageUrlForSolver} sitekey=${sitekey ? 'ok' : 'missing'}`);
|
});
|
||||||
const { token, provider } = await solveTurnstile({ pageUrl: pageUrlForSolver, sitekey });
|
|
||||||
console.log(`[CF] Captcha API (${provider}) 返回 token,注入页面`);
|
|
||||||
await injectTurnstileToken(page, token);
|
|
||||||
|
|
||||||
const apiWait = await waitForCfTrulyComplete(page, Math.min(timeoutMs, 45000), 500, antiBot);
|
const apiWait = await waitForCfTrulyComplete(page, Math.min(timeoutMs, 45000), 500, antiBot);
|
||||||
if (apiWait.success) {
|
if (apiWait.success) {
|
||||||
@@ -744,10 +1009,19 @@ async function prepareWhatshubShortLinkBeforeNav(page, pageUrl, antiBot) {
|
|||||||
/**
|
/**
|
||||||
* CF 处理 + 业务 URL 守卫:磁盘 session 陈旧 clearance 时 purge;__cf_chl 中间态只过盾不 purge
|
* CF 处理 + 业务 URL 守卫:磁盘 session 陈旧 clearance 时 purge;__cf_chl 中间态只过盾不 purge
|
||||||
*/
|
*/
|
||||||
async function runCloudflareWithBusinessGuard(page, antiBot, waitForUrlSettled, storedSession, pageUrl) {
|
async function runCloudflareWithBusinessGuard(page, antiBot, waitForUrlSettled, storedSession, pageUrl, browserCtx = null) {
|
||||||
const urlNeedle = (antiBot.postCfReadyUrlContains || '').trim();
|
const urlNeedle = (antiBot.postCfReadyUrlContains || '').trim();
|
||||||
const hasStoredSession = !!(storedSession && isSessionLikelyValid(storedSession));
|
const hasStoredSession = !!(storedSession && isSessionLikelyValid(storedSession));
|
||||||
|
|
||||||
|
async function maybeEnsureProxy(currentPage) {
|
||||||
|
if (!browserCtx) {
|
||||||
|
return currentPage;
|
||||||
|
}
|
||||||
|
browserCtx.page = currentPage;
|
||||||
|
await ensureCaptchaProxyBrowserIfNeeded(browserCtx);
|
||||||
|
return browserCtx.page;
|
||||||
|
}
|
||||||
|
|
||||||
async function reNavigateWithCfPurge(label) {
|
async function reNavigateWithCfPurge(label) {
|
||||||
if (urlIndicatesCfChallenge(page.url())) {
|
if (urlIndicatesCfChallenge(page.url())) {
|
||||||
console.log(`[CF] ${label}: URL 含 __cf_chl,跳过 purge,直接过盾`);
|
console.log(`[CF] ${label}: URL 含 __cf_chl,跳过 purge,直接过盾`);
|
||||||
@@ -759,6 +1033,7 @@ async function runCloudflareWithBusinessGuard(page, antiBot, waitForUrlSettled,
|
|||||||
if (waitForUrlSettled) {
|
if (waitForUrlSettled) {
|
||||||
await waitForUrlSettled(page).catch(() => {});
|
await waitForUrlSettled(page).catch(() => {});
|
||||||
}
|
}
|
||||||
|
page = await maybeEnsureProxy(page);
|
||||||
return handleCloudflareChallengeSafe(page, antiBot, waitForUrlSettled, storedSession);
|
return handleCloudflareChallengeSafe(page, antiBot, waitForUrlSettled, storedSession);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -773,6 +1048,7 @@ async function runCloudflareWithBusinessGuard(page, antiBot, waitForUrlSettled,
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (!cfResult) {
|
if (!cfResult) {
|
||||||
|
page = await maybeEnsureProxy(page);
|
||||||
cfResult = await handleCloudflareChallengeSafe(page, antiBot, waitForUrlSettled, storedSession);
|
cfResult = await handleCloudflareChallengeSafe(page, antiBot, waitForUrlSettled, storedSession);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -781,6 +1057,7 @@ async function runCloudflareWithBusinessGuard(page, antiBot, waitForUrlSettled,
|
|||||||
const crossState = await detectCloudflare(page, antiBot);
|
const crossState = await detectCloudflare(page, antiBot);
|
||||||
if (crossState.blocked || urlIndicatesCfChallenge(page.url())) {
|
if (crossState.blocked || urlIndicatesCfChallenge(page.url())) {
|
||||||
console.log(`[CF] 跨域落地后 CF 未完成,二次过盾 url=${page.url()}`);
|
console.log(`[CF] 跨域落地后 CF 未完成,二次过盾 url=${page.url()}`);
|
||||||
|
page = await maybeEnsureProxy(page);
|
||||||
cfResult = await finalizeCfResult(
|
cfResult = await finalizeCfResult(
|
||||||
page,
|
page,
|
||||||
await runCfChallengeFlow(page, antiBot, waitForUrlSettled, page.url()),
|
await runCfChallengeFlow(page, antiBot, waitForUrlSettled, page.url()),
|
||||||
@@ -803,6 +1080,7 @@ async function runCloudflareWithBusinessGuard(page, antiBot, waitForUrlSettled,
|
|||||||
retries += 1;
|
retries += 1;
|
||||||
if (!(await isCfTrulyComplete(page, antiBot))) {
|
if (!(await isCfTrulyComplete(page, antiBot))) {
|
||||||
console.log(`[CF] 业务页未达且 CF 未完成,完整过盾 重试#${retries}`);
|
console.log(`[CF] 业务页未达且 CF 未完成,完整过盾 重试#${retries}`);
|
||||||
|
page = await maybeEnsureProxy(page);
|
||||||
cfResult = await finalizeCfResult(
|
cfResult = await finalizeCfResult(
|
||||||
page,
|
page,
|
||||||
await runCfChallengeFlow(page, antiBot, waitForUrlSettled, pageUrl),
|
await runCfChallengeFlow(page, antiBot, waitForUrlSettled, pageUrl),
|
||||||
@@ -823,6 +1101,10 @@ async function runCloudflareWithBusinessGuard(page, antiBot, waitForUrlSettled,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (browserCtx) {
|
||||||
|
browserCtx.page = page;
|
||||||
|
}
|
||||||
|
|
||||||
return finalizeCfResult(page, cfResult, antiBot);
|
return finalizeCfResult(page, cfResult, antiBot);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -833,6 +1115,9 @@ async function collectPageDiagnostics(page) {
|
|||||||
title: document.title || '',
|
title: document.title || '',
|
||||||
bodyText: (document.body && document.body.innerText ? document.body.innerText : '').slice(0, 500),
|
bodyText: (document.body && document.body.innerText ? document.body.innerText : '').slice(0, 500),
|
||||||
elInputInner: document.querySelectorAll('.el-input__inner').length,
|
elInputInner: document.querySelectorAll('.el-input__inner').length,
|
||||||
|
elTable: document.querySelectorAll('.el-table').length,
|
||||||
|
btnNext: document.querySelectorAll('.btn-next').length,
|
||||||
|
elPagination: document.querySelectorAll('.el-pagination').length,
|
||||||
})).catch(() => null);
|
})).catch(() => null);
|
||||||
return snapshot || { url, title: '', bodyText: '', elInputInner: 0 };
|
return snapshot || { url, title: '', bodyText: '', elInputInner: 0 };
|
||||||
}
|
}
|
||||||
@@ -851,7 +1136,20 @@ async function awaitPostCfBusinessReady(page, antiBot, waitForUrlSettledFn, stor
|
|||||||
|
|
||||||
await ensureCfCompleteBeforeBusinessReady(page, antiBot, waitForUrlSettledFn, storedSession);
|
await ensureCfCompleteBeforeBusinessReady(page, antiBot, waitForUrlSettledFn, storedSession);
|
||||||
|
|
||||||
await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettledFn, navUrl);
|
const onA2cDirect = isA2cDirectSharePage(page, urlContains);
|
||||||
|
let skipSpaWait = false;
|
||||||
|
if (onA2cDirect && await isCfTrulyComplete(page, antiBot)) {
|
||||||
|
const domReady = selector
|
||||||
|
? await page.$(selector).catch(() => null)
|
||||||
|
: true;
|
||||||
|
if (domReady) {
|
||||||
|
console.log('[CF] A2C 业务 DOM 已就绪,跳过重复 SPA 等待');
|
||||||
|
skipSpaWait = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!skipSpaWait) {
|
||||||
|
await waitForSpaBusinessNavigation(page, antiBot, waitForUrlSettledFn, navUrl);
|
||||||
|
}
|
||||||
|
|
||||||
const timeoutMs = Math.max(5000, antiBot.postCfReadyTimeoutMs || 30000);
|
const timeoutMs = Math.max(5000, antiBot.postCfReadyTimeoutMs || 30000);
|
||||||
console.log(
|
console.log(
|
||||||
@@ -867,12 +1165,14 @@ async function awaitPostCfBusinessReady(page, antiBot, waitForUrlSettledFn, stor
|
|||||||
urlContains
|
urlContains
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
await waitForPostCfListApi(page, antiBot, timeoutMs);
|
||||||
if (selector !== '') {
|
if (selector !== '') {
|
||||||
await page.waitForSelector(selector, { timeout: timeoutMs, visible: true });
|
await waitForPostCfSelector(page, selector, timeoutMs);
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const retryMs = Math.min(15000, timeoutMs);
|
const retryMs = Math.min(30000, timeoutMs);
|
||||||
const needsRetry = urlContains !== '' && !page.url().includes(urlContains);
|
const needsRetry = urlContains !== '' && !page.url().includes(urlContains);
|
||||||
|
const needsDomRetry = urlContains !== '' && page.url().includes(urlContains) && isA2cDirectSharePage(page, urlContains);
|
||||||
if (needsRetry) {
|
if (needsRetry) {
|
||||||
if (!(await isCfTrulyComplete(page, antiBot))) {
|
if (!(await isCfTrulyComplete(page, antiBot))) {
|
||||||
console.warn(`[CF] 业务页等待失败且 CF 未完成 url=${page.url()},完整过盾后重试`);
|
console.warn(`[CF] 业务页等待失败且 CF 未完成 url=${page.url()},完整过盾后重试`);
|
||||||
@@ -886,7 +1186,7 @@ async function awaitPostCfBusinessReady(page, antiBot, waitForUrlSettledFn, stor
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
if (selector !== '') {
|
if (selector !== '') {
|
||||||
await page.waitForSelector(selector, { timeout: retryMs, visible: true });
|
await waitForPostCfSelector(page, selector, retryMs);
|
||||||
}
|
}
|
||||||
} catch (retryErr) {
|
} catch (retryErr) {
|
||||||
retryErr.pageDiagnostics = await collectPageDiagnostics(page);
|
retryErr.pageDiagnostics = await collectPageDiagnostics(page);
|
||||||
@@ -906,14 +1206,28 @@ async function awaitPostCfBusinessReady(page, antiBot, waitForUrlSettledFn, stor
|
|||||||
urlContains
|
urlContains
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
await waitForPostCfListApi(page, antiBot, retryMs);
|
||||||
if (selector !== '') {
|
if (selector !== '') {
|
||||||
await page.waitForSelector(selector, { timeout: retryMs, visible: true });
|
await waitForPostCfSelector(page, selector, retryMs);
|
||||||
}
|
}
|
||||||
} catch (retryErr) {
|
} catch (retryErr) {
|
||||||
retryErr.pageDiagnostics = await collectPageDiagnostics(page);
|
retryErr.pageDiagnostics = await collectPageDiagnostics(page);
|
||||||
throw retryErr;
|
throw retryErr;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} else if (needsDomRetry) {
|
||||||
|
console.warn(`[CF] A2C URL 已就绪但 DOM 未挂载 url=${page.url()},soft reload 后重试`);
|
||||||
|
try {
|
||||||
|
await page.waitForNetworkIdle({ idleTime: 500, timeout: 20000 }).catch(() => {});
|
||||||
|
await softReloadForSpa(page, waitForUrlSettledFn, antiBot, navUrl);
|
||||||
|
await waitForPostCfListApi(page, antiBot, retryMs);
|
||||||
|
if (selector !== '') {
|
||||||
|
await waitForPostCfSelector(page, selector, retryMs);
|
||||||
|
}
|
||||||
|
} catch (retryErr) {
|
||||||
|
retryErr.pageDiagnostics = await collectPageDiagnostics(page);
|
||||||
|
throw retryErr;
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
err.pageDiagnostics = await collectPageDiagnostics(page);
|
err.pageDiagnostics = await collectPageDiagnostics(page);
|
||||||
throw err;
|
throw err;
|
||||||
@@ -1230,6 +1544,7 @@ app.post('/api/auth-and-intercept', async (req, res) => {
|
|||||||
await runBrowserTask('auth-and-intercept', async () => {
|
await runBrowserTask('auth-and-intercept', async () => {
|
||||||
let browserSession = null;
|
let browserSession = null;
|
||||||
let interceptorCleanup = null;
|
let interceptorCleanup = null;
|
||||||
|
let page = null;
|
||||||
const taskStarted = Date.now();
|
const taskStarted = Date.now();
|
||||||
let cfDestroyBrowser = false;
|
let cfDestroyBrowser = false;
|
||||||
|
|
||||||
@@ -1242,11 +1557,12 @@ app.post('/api/auth-and-intercept', async (req, res) => {
|
|||||||
cfLog.browserReused = !!browserSession.reused;
|
cfLog.browserReused = !!browserSession.reused;
|
||||||
|
|
||||||
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl);
|
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl);
|
||||||
|
const navigationUrl = resolveNavigationUrl(pageUrl, antiBot);
|
||||||
|
|
||||||
const page = await createTaskPage(
|
page = await createTaskPage(
|
||||||
browserSession.browser,
|
browserSession.browser,
|
||||||
browserSession.page,
|
browserSession.reused ? null : browserSession.page,
|
||||||
pageUrl,
|
navigationUrl,
|
||||||
antiBot,
|
antiBot,
|
||||||
mergedCookies
|
mergedCookies
|
||||||
);
|
);
|
||||||
@@ -1257,13 +1573,13 @@ app.post('/api/auth-and-intercept', async (req, res) => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let shareToken = resolveShareToken(pageUrl, httpResolvedUrl);
|
let shareToken = resolveShareToken(pageUrl, httpResolvedUrl);
|
||||||
await seedShareTokenCookie(page, pageUrl);
|
await seedShareTokenCookie(page, navigationUrl);
|
||||||
if (httpResolvedUrl !== pageUrl) {
|
if (httpResolvedUrl !== pageUrl) {
|
||||||
await seedShareTokenCookie(page, httpResolvedUrl);
|
await seedShareTokenCookie(page, httpResolvedUrl);
|
||||||
}
|
}
|
||||||
|
|
||||||
await prepareWhatshubShortLinkBeforeNav(page, pageUrl, antiBot);
|
await prepareWhatshubShortLinkBeforeNav(page, navigationUrl, antiBot);
|
||||||
await navigateToPage(page, pageUrl);
|
await navigateToPage(page, navigationUrl);
|
||||||
|
|
||||||
let finalPageUrl = await waitForUrlSettled(page);
|
let finalPageUrl = await waitForUrlSettled(page);
|
||||||
applyHuojianBusinessHostHint(antiBot, finalPageUrl);
|
applyHuojianBusinessHostHint(antiBot, finalPageUrl);
|
||||||
@@ -1271,11 +1587,30 @@ app.post('/api/auth-and-intercept', async (req, res) => {
|
|||||||
console.log(`[URL解析] 浏览器落地: ${pageUrl} -> ${finalPageUrl}`);
|
console.log(`[URL解析] 浏览器落地: ${pageUrl} -> ${finalPageUrl}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const browserCtx = {
|
||||||
|
browserSession,
|
||||||
|
page,
|
||||||
|
antiBot,
|
||||||
|
profile,
|
||||||
|
navigationUrl,
|
||||||
|
mergedCookies,
|
||||||
|
baseExtraArgs: ['--mute-audio'],
|
||||||
|
pageExtraOpts: {},
|
||||||
|
afterRelaunchNav: async (p) => {
|
||||||
|
await seedShareTokenCookie(p, navigationUrl);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
await ensureCaptchaProxyBrowserIfNeeded(browserCtx);
|
||||||
|
browserSession = browserCtx.browserSession;
|
||||||
|
page = browserCtx.page;
|
||||||
|
|
||||||
let interceptor;
|
let interceptor;
|
||||||
if (antiBot.enabled) {
|
if (antiBot.enabled) {
|
||||||
const cfResult = await runCloudflareWithBusinessGuard(
|
const cfResult = await runCloudflareWithBusinessGuard(
|
||||||
page, antiBot, waitForUrlSettled, storedSession, pageUrl
|
page, antiBot, waitForUrlSettled, storedSession, pageUrl, browserCtx
|
||||||
);
|
);
|
||||||
|
page = browserCtx.page;
|
||||||
|
browserSession = browserCtx.browserSession;
|
||||||
cfLog.cfDetected = cfResult.cfDetected;
|
cfLog.cfDetected = cfResult.cfDetected;
|
||||||
cfLog.turnstileStage = cfResult.stage;
|
cfLog.turnstileStage = cfResult.stage;
|
||||||
cfLog.solverUsed = cfResult.solverUsed;
|
cfLog.solverUsed = cfResult.solverUsed;
|
||||||
@@ -1346,6 +1681,7 @@ app.post('/api/auth-and-intercept', async (req, res) => {
|
|||||||
});
|
});
|
||||||
} finally {
|
} finally {
|
||||||
if (interceptorCleanup) interceptorCleanup();
|
if (interceptorCleanup) interceptorCleanup();
|
||||||
|
await closeTaskPage(page);
|
||||||
if (browserSession) {
|
if (browserSession) {
|
||||||
await browserSession.release(cfDestroyBrowser);
|
await browserSession.release(cfDestroyBrowser);
|
||||||
}
|
}
|
||||||
@@ -1495,6 +1831,7 @@ app.post('/api/ui-pagination', async (req, res) => {
|
|||||||
await runBrowserTask('ui-pagination', async () => {
|
await runBrowserTask('ui-pagination', async () => {
|
||||||
let browserSession = null;
|
let browserSession = null;
|
||||||
const debugLogs = [];
|
const debugLogs = [];
|
||||||
|
let page = null;
|
||||||
let cfDestroyBrowser = false;
|
let cfDestroyBrowser = false;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -1506,9 +1843,9 @@ app.post('/api/ui-pagination', async (req, res) => {
|
|||||||
|
|
||||||
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, navigationUrl, cookies || []);
|
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, navigationUrl, cookies || []);
|
||||||
|
|
||||||
const page = await createTaskPage(
|
page = await createTaskPage(
|
||||||
browserSession.browser,
|
browserSession.browser,
|
||||||
browserSession.page,
|
browserSession.reused ? null : browserSession.page,
|
||||||
navigationUrl,
|
navigationUrl,
|
||||||
antiBot,
|
antiBot,
|
||||||
mergedCookies,
|
mergedCookies,
|
||||||
@@ -1532,10 +1869,29 @@ app.post('/api/ui-pagination', async (req, res) => {
|
|||||||
log(`[URL解析] 翻页阶段落地: ${navigationUrl} -> ${settledPageUrl}`);
|
log(`[URL解析] 翻页阶段落地: ${navigationUrl} -> ${settledPageUrl}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const browserCtx = {
|
||||||
|
browserSession,
|
||||||
|
page,
|
||||||
|
antiBot,
|
||||||
|
profile,
|
||||||
|
navigationUrl,
|
||||||
|
mergedCookies,
|
||||||
|
baseExtraArgs: ['--window-size=1920,1080'],
|
||||||
|
pageExtraOpts: { viewport: { width: 1920, height: 1080 } },
|
||||||
|
afterRelaunchNav: async (p) => {
|
||||||
|
await seedShareTokenCookie(p, navigationUrl);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
await ensureCaptchaProxyBrowserIfNeeded(browserCtx);
|
||||||
|
browserSession = browserCtx.browserSession;
|
||||||
|
page = browserCtx.page;
|
||||||
|
|
||||||
if (antiBot.enabled) {
|
if (antiBot.enabled) {
|
||||||
const cfResult = await runCloudflareWithBusinessGuard(
|
const cfResult = await runCloudflareWithBusinessGuard(
|
||||||
page, antiBot, waitForUrlSettled, storedSession, navigationUrl
|
page, antiBot, waitForUrlSettled, storedSession, navigationUrl, browserCtx
|
||||||
);
|
);
|
||||||
|
page = browserCtx.page;
|
||||||
|
browserSession = browserCtx.browserSession;
|
||||||
if (!cfResult.success) {
|
if (!cfResult.success) {
|
||||||
cfDestroyBrowser = true;
|
cfDestroyBrowser = true;
|
||||||
res.status(422).json({
|
res.status(422).json({
|
||||||
@@ -1578,6 +1934,7 @@ app.post('/api/ui-pagination', async (req, res) => {
|
|||||||
sendTaskError(res, error, { debug: debugLogs, profile });
|
sendTaskError(res, error, { debug: debugLogs, profile });
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
|
await closeTaskPage(page);
|
||||||
if (browserSession) {
|
if (browserSession) {
|
||||||
await browserSession.release(cfDestroyBrowser);
|
await browserSession.release(cfDestroyBrowser);
|
||||||
}
|
}
|
||||||
@@ -1619,6 +1976,7 @@ app.post('/api/auth-intercept-and-paginate', async (req, res) => {
|
|||||||
await runBrowserTask('auth-intercept-and-paginate', async () => {
|
await runBrowserTask('auth-intercept-and-paginate', async () => {
|
||||||
let browserSession = null;
|
let browserSession = null;
|
||||||
let interceptorCleanup = null;
|
let interceptorCleanup = null;
|
||||||
|
let page = null;
|
||||||
const taskStarted = Date.now();
|
const taskStarted = Date.now();
|
||||||
let cfDestroyBrowser = false;
|
let cfDestroyBrowser = false;
|
||||||
|
|
||||||
@@ -1631,11 +1989,12 @@ app.post('/api/auth-intercept-and-paginate', async (req, res) => {
|
|||||||
cfLog.browserReused = !!browserSession.reused;
|
cfLog.browserReused = !!browserSession.reused;
|
||||||
|
|
||||||
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl);
|
const { storedSession, mergedCookies } = resolveSessionContext(antiBot, pageUrl);
|
||||||
|
const navigationUrl = resolveNavigationUrl(pageUrl, antiBot);
|
||||||
|
|
||||||
const page = await createTaskPage(
|
page = await createTaskPage(
|
||||||
browserSession.browser,
|
browserSession.browser,
|
||||||
browserSession.page,
|
browserSession.reused ? null : browserSession.page,
|
||||||
pageUrl,
|
navigationUrl,
|
||||||
antiBot,
|
antiBot,
|
||||||
mergedCookies,
|
mergedCookies,
|
||||||
{ viewport: { width: 1920, height: 1080 } }
|
{ viewport: { width: 1920, height: 1080 } }
|
||||||
@@ -1644,18 +2003,37 @@ app.post('/api/auth-intercept-and-paginate', async (req, res) => {
|
|||||||
const httpResolvedUrl = await resolveHttpRedirects(pageUrl).catch(() => pageUrl);
|
const httpResolvedUrl = await resolveHttpRedirects(pageUrl).catch(() => pageUrl);
|
||||||
|
|
||||||
let shareToken = resolveShareToken(pageUrl, httpResolvedUrl);
|
let shareToken = resolveShareToken(pageUrl, httpResolvedUrl);
|
||||||
await seedShareTokenCookie(page, pageUrl);
|
await seedShareTokenCookie(page, navigationUrl);
|
||||||
|
|
||||||
await prepareWhatshubShortLinkBeforeNav(page, pageUrl, antiBot);
|
await prepareWhatshubShortLinkBeforeNav(page, navigationUrl, antiBot);
|
||||||
await navigateToPage(page, pageUrl);
|
await navigateToPage(page, navigationUrl);
|
||||||
let finalPageUrl = await waitForUrlSettled(page);
|
let finalPageUrl = await waitForUrlSettled(page);
|
||||||
applyHuojianBusinessHostHint(antiBot, finalPageUrl);
|
applyHuojianBusinessHostHint(antiBot, finalPageUrl);
|
||||||
|
|
||||||
|
const browserCtx = {
|
||||||
|
browserSession,
|
||||||
|
page,
|
||||||
|
antiBot,
|
||||||
|
profile,
|
||||||
|
navigationUrl,
|
||||||
|
mergedCookies,
|
||||||
|
baseExtraArgs: ['--mute-audio', '--window-size=1920,1080'],
|
||||||
|
pageExtraOpts: { viewport: { width: 1920, height: 1080 } },
|
||||||
|
afterRelaunchNav: async (p) => {
|
||||||
|
await seedShareTokenCookie(p, navigationUrl);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
await ensureCaptchaProxyBrowserIfNeeded(browserCtx);
|
||||||
|
browserSession = browserCtx.browserSession;
|
||||||
|
page = browserCtx.page;
|
||||||
|
|
||||||
let interceptor;
|
let interceptor;
|
||||||
if (antiBot.enabled) {
|
if (antiBot.enabled) {
|
||||||
const cfResult = await runCloudflareWithBusinessGuard(
|
const cfResult = await runCloudflareWithBusinessGuard(
|
||||||
page, antiBot, waitForUrlSettled, storedSession, pageUrl
|
page, antiBot, waitForUrlSettled, storedSession, navigationUrl, browserCtx
|
||||||
);
|
);
|
||||||
|
page = browserCtx.page;
|
||||||
|
browserSession = browserCtx.browserSession;
|
||||||
cfLog.cfDetected = cfResult.cfDetected;
|
cfLog.cfDetected = cfResult.cfDetected;
|
||||||
cfLog.turnstileStage = cfResult.stage;
|
cfLog.turnstileStage = cfResult.stage;
|
||||||
cfLog.solverUsed = cfResult.solverUsed;
|
cfLog.solverUsed = cfResult.solverUsed;
|
||||||
@@ -1677,7 +2055,7 @@ app.post('/api/auth-intercept-and-paginate', async (req, res) => {
|
|||||||
applyHuojianBusinessHostHint(antiBot, finalPageUrl);
|
applyHuojianBusinessHostHint(antiBot, finalPageUrl);
|
||||||
touchSessionIfPresent(antiBot.sessionKey);
|
touchSessionIfPresent(antiBot.sessionKey);
|
||||||
|
|
||||||
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled, storedSession, pageUrl);
|
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled, storedSession, navigationUrl);
|
||||||
}
|
}
|
||||||
|
|
||||||
interceptor = createApiInterceptor(page, apiUrls, interceptTimeout);
|
interceptor = createApiInterceptor(page, apiUrls, interceptTimeout);
|
||||||
@@ -1744,6 +2122,7 @@ app.post('/api/auth-intercept-and-paginate', async (req, res) => {
|
|||||||
});
|
});
|
||||||
} finally {
|
} finally {
|
||||||
if (interceptorCleanup) interceptorCleanup();
|
if (interceptorCleanup) interceptorCleanup();
|
||||||
|
await closeTaskPage(page);
|
||||||
if (browserSession) {
|
if (browserSession) {
|
||||||
await browserSession.release(cfDestroyBrowser);
|
await browserSession.release(cfDestroyBrowser);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user