修改whatshub工单爬虫

This commit is contained in:
root
2026-06-29 04:54:41 +08:00
parent 6130125427
commit 9f2e904fab
14 changed files with 1254 additions and 147 deletions
+225 -16
View File
@@ -75,11 +75,23 @@ abstract class AbstractScrmSpider
const MODE_FETCH = 'fetch'; // 极速并发拉取 (推荐默认) const MODE_FETCH = 'fetch'; // 极速并发拉取 (推荐默认)
const MODE_UI = 'ui_click'; // 强制 UI 点击下一页 const MODE_UI = 'ui_click'; // 强制 UI 点击下一页
/** CLI 测试时输出 Node 请求/失败详情 */
protected $verbose = false;
protected $nodeHost; protected $nodeHost;
public function __construct($nodeHost = 'http://127.0.0.1:3001') public function __construct($nodeHost = 'http://127.0.0.1:3001')
{ {
$this->nodeHost = $nodeHost; $this->nodeHost = rtrim($nodeHost, '/');
}
/**
* 开启调试输出(测试脚本可调用)
*/
public function setVerbose($verbose = true)
{
$this->verbose = (bool) $verbose;
return $this;
} }
/** ================= 子类必须实现的契约 ================= **/ /** ================= 子类必须实现的契约 ================= **/
@@ -106,13 +118,14 @@ abstract class AbstractScrmSpider
$apiUrlsToIntercept = [$listApi]; $apiUrlsToIntercept = [$listApi];
if ($detailApi) { $apiUrlsToIntercept[] = $detailApi; } if ($detailApi) { $apiUrlsToIntercept[] = $detailApi; }
if ($countApi) { $apiUrlsToIntercept[] = $countApi; } if ($countApi) { $apiUrlsToIntercept[] = $countApi; }
// var_dump($apiUrlsToIntercept);
// dump([
// 'pageUrl' => $config['pageUrl'],
// 'apiUrls' => $apiUrlsToIntercept,
// 'authActions' => $config['authActions']
// ]);
$antiBot = $config['antiBot'] ?? null; $antiBot = $config['antiBot'] ?? null;
$mode = $config['paginationMode'] ?? self::MODE_FETCH;
// antiBot + UI 翻页:单 Browser 合并路径(与生产 SplitTicket 同步一致)
if ($this->shouldUseUnifiedBrowserPath($antiBot, $mode)) {
return $this->runUnifiedUiPath($config, $antiBot, $apiUrlsToIntercept, $listApi, $detailApi, $countApi);
}
// 【阶段一】:初始化并首屏拦截 // 【阶段一】:初始化并首屏拦截
$initResult = $this->requestNode('/api/auth-and-intercept', [ $initResult = $this->requestNode('/api/auth-and-intercept', [
@@ -123,17 +136,20 @@ abstract class AbstractScrmSpider
], $this->resolveAuthInterceptTimeout($antiBot)); ], $this->resolveAuthInterceptTimeout($antiBot));
if (empty($initResult['success'])) { if (empty($initResult['success'])) {
$cfCode = (string) ($initResult['code'] ?? ''); throw new Exception($this->formatNodeFailure('初始化失败', $initResult));
if ($cfCode === 'CF_TURNSTILE_FAILED') {
throw new Exception('Cloudflare Turnstile 验证失败: ' . ($initResult['stage'] ?? 'timeout'));
}
throw new Exception("初始化失败: " . ($initResult['error'] ?? '未知'));
} }
$interceptedApis = $initResult['interceptedApis']; $interceptedApis = $initResult['interceptedApis'];
$cookies = $initResult['cookies']; $cookies = $initResult['cookies'];
$finalPageUrl = $initResult['finalPageUrl'] ?? $config['pageUrl']; $finalPageUrl = $initResult['finalPageUrl'] ?? $config['pageUrl'];
$this->logDebug('auth-and-intercept ok', [
'finalPageUrl' => $finalPageUrl,
'intercepted' => array_keys($interceptedApis),
'cf' => $initResult['cf'] ?? null,
'browserReused' => $initResult['browserReused'] ?? null,
]);
// 必须拦截到 List 接口,否则无法继续 // 必须拦截到 List 接口,否则无法继续
if (!isset($interceptedApis[$listApi])) { if (!isset($interceptedApis[$listApi])) {
throw new Exception("致命错误:未能拦截到必须的列表接口 [{$listApi}]"); throw new Exception("致命错误:未能拦截到必须的列表接口 [{$listApi}]");
@@ -229,6 +245,140 @@ abstract class AbstractScrmSpider
return $unifiedData; return $unifiedData;
} }
/**
* 单 Browser 路径:auth + 拦截 + UI 翻页(需 antiBot.sessionKey
*/
protected function runUnifiedUiPath($config, $antiBot, $apiUrlsToIntercept, $listApi, $detailApi, $countApi)
{
$uiConfig = $this->getUiPaginationConfig();
$sessionKey = is_array($antiBot) ? (string) ($antiBot['sessionKey'] ?? '') : '';
$this->logDebug('unified browser path', [
'sessionKey' => $sessionKey,
'listApi' => $listApi,
]);
$mergedResult = $this->requestNode('/api/auth-intercept-and-paginate', [
'pageUrl' => $config['pageUrl'],
'apiUrls' => $apiUrlsToIntercept,
'authActions' => $config['authActions'] ?? [],
'antiBot' => $antiBot,
'pagination' => [
'apiUrl' => $listApi,
'nextBtnSelector' => $uiConfig['nextBtnSelector'] ?? '',
'waitMs' => $uiConfig['waitMs'] ?? 2000,
'clicksToPerform' => 9999,
],
], 1200);
if (empty($mergedResult['success'])) {
throw new Exception($this->formatNodeFailure('合并同步失败', $mergedResult));
}
$this->logDebug('auth-intercept-and-paginate ok', [
'intercepted' => array_keys($mergedResult['interceptedApis'] ?? []),
'extraPages' => count($mergedResult['extraPages'] ?? []),
'cf' => $mergedResult['cf'] ?? null,
'browserReused' => $mergedResult['browserReused'] ?? null,
'finalPageUrl' => $mergedResult['finalPageUrl'] ?? null,
]);
$interceptedApis = $mergedResult['interceptedApis'] ?? [];
if (!isset($interceptedApis[$listApi])) {
throw new Exception("致命错误:未能拦截到必须的列表接口 [{$listApi}]");
}
$detailData = $detailApi && isset($interceptedApis[$detailApi])
? $interceptedApis[$detailApi]['data'] : null;
$countData = $countApi && isset($interceptedApis[$countApi])
? $interceptedApis[$countApi]['data'] : null;
$listApiNode = $interceptedApis[$listApi];
$allListPagesData = [$listApiNode['data']];
if (!empty($mergedResult['extraPages']) && is_array($mergedResult['extraPages'])) {
foreach ($mergedResult['extraPages'] as $pageData) {
$allListPagesData[] = $pageData;
}
}
$this->extractListTotalPages($listApiNode['data'], $countData);
return $this->parseToUnifiedData($detailData, $allListPagesData);
}
/**
* antiBot 启用且配置了 sessionKey 时,UI 翻页走合并 Browser 路径
*/
protected function shouldUseUnifiedBrowserPath($antiBot, $mode)
{
if ($mode !== self::MODE_UI) {
return false;
}
if (!is_array($antiBot) || empty($antiBot['enabled'])) {
return false;
}
return !empty($antiBot['sessionKey']);
}
/**
* 格式化 Node 失败信息,附带 CF / 落地页等诊断字段
*/
protected function formatNodeFailure($prefix, $result)
{
$cfCode = (string) ($result['code'] ?? '');
if ($cfCode === 'CF_TURNSTILE_FAILED') {
return 'Cloudflare Turnstile 验证失败: ' . ($result['stage'] ?? 'timeout');
}
$parts = [$prefix . ': ' . ($result['error'] ?? '未知')];
if (!empty($result['finalPageUrl'])) {
$parts[] = 'finalPageUrl=' . $result['finalPageUrl'];
}
if (!empty($result['page']) && is_array($result['page'])) {
$page = $result['page'];
if (!empty($page['url'])) {
$parts[] = 'pageUrl=' . $page['url'];
}
if (isset($page['elInputInner'])) {
$parts[] = 'elInputInner=' . $page['elInputInner'];
}
}
if (!empty($result['cf']) && is_array($result['cf'])) {
$cf = $result['cf'];
$cfBits = [];
if (isset($cf['cfDetected'])) {
$cfBits[] = 'cfDetected=' . ($cf['cfDetected'] ? 'true' : 'false');
}
if (!empty($cf['turnstileStage'])) {
$cfBits[] = 'turnstileStage=' . $cf['turnstileStage'];
}
if (isset($cf['solverUsed'])) {
$cfBits[] = 'solverUsed=' . ($cf['solverUsed'] ? 'true' : 'false');
}
if (isset($cf['elapsedMs'])) {
$cfBits[] = 'cfElapsedMs=' . $cf['elapsedMs'];
}
if (!empty($cfBits)) {
$parts[] = implode(', ', $cfBits);
}
}
return implode(' | ', $parts);
}
protected function logDebug($message, $context = [])
{
if (!$this->verbose) {
return;
}
$line = '[Spider] ' . $message;
if (!empty($context)) {
$line .= ' ' . json_encode($context, JSON_UNESCAPED_UNICODE);
}
echo $line . "\n";
}
/** /**
* Real Browser + Turnstile 首屏耗时较长,单独放宽 auth-and-intercept 超时 * Real Browser + Turnstile 首屏耗时较长,单独放宽 auth-and-intercept 超时
* *
@@ -244,17 +394,76 @@ abstract class AbstractScrmSpider
private function requestNode($endpoint, $payload, $timeout = 60) private function requestNode($endpoint, $payload, $timeout = 60)
{ {
$ch = curl_init($this->nodeHost . $endpoint); $maxAttempts = 3;
$lastDecoded = [];
for ($attempt = 1; $attempt <= $maxAttempts; $attempt++) {
$lastDecoded = $this->doRequestNode($endpoint, $payload, $timeout, $attempt);
if (!empty($lastDecoded['success'])) {
return $lastDecoded;
}
$error = (string) ($lastDecoded['error'] ?? '');
if (!$this->shouldRetryNodeResponse($lastDecoded, $error) || $attempt >= $maxAttempts) {
return $lastDecoded;
}
$this->logDebug('node retry', [
'endpoint' => $endpoint,
'attempt' => $attempt + 1,
'error' => $error,
]);
sleep($attempt);
}
return $lastDecoded;
}
private function doRequestNode($endpoint, $payload, $timeout, $attempt)
{
$url = $this->nodeHost . $endpoint;
$this->logDebug('node request', [
'url' => $url,
'timeout' => $timeout,
'attempt' => $attempt,
]);
$ch = curl_init($url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_POST, true); curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($payload)); curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($payload, JSON_UNESCAPED_UNICODE));
curl_setopt($ch, CURLOPT_HTTPHEADER, ['Content-Type: application/json']); curl_setopt($ch, CURLOPT_HTTPHEADER, ['Content-Type: application/json']);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5); curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5);
curl_setopt($ch, CURLOPT_TIMEOUT, $timeout); curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
$response = curl_exec($ch); $response = curl_exec($ch);
if (curl_errno($ch)) throw new Exception(curl_error($ch)); $httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
if (curl_errno($ch)) {
$err = curl_error($ch);
curl_close($ch); curl_close($ch);
return json_decode($response, true); throw new Exception($err);
}
curl_close($ch);
$decoded = json_decode((string) $response, true);
if (!is_array($decoded)) {
return ['success' => false, 'error' => 'Node 返回非 JSON', 'httpCode' => $httpCode];
}
if ($httpCode === 503) {
$decoded['success'] = false;
$decoded['error'] = (string) ($decoded['error'] ?? '服务繁忙');
$decoded['httpCode'] = 503;
}
return $decoded;
}
private function shouldRetryNodeResponse($decoded, $error)
{
if ((int) ($decoded['httpCode'] ?? 0) === 503) {
return true;
}
$lower = strtolower($error);
foreach (['排队超时', 'queue_timeout', 'timeout', 'net::', 'navigation', 'connection'] as $needle) {
if ($needle !== '' && strpos($lower, $needle) !== false) {
return true;
}
}
return false;
} }
/** /**
+182 -14
View File
@@ -7,8 +7,12 @@ class Whatshub extends AbstractScrmSpider
{ {
const API_LIST = '/api/whatshub-counter/workShare/open/detail'; // 列表API地址 const API_LIST = '/api/whatshub-counter/workShare/open/detail'; // 列表API地址
const API_DETAILS = '/api/whatshub-counter/workShare/open/statistics'; // 详情API地址 const API_DETAILS = '/api/whatshub-counter/workShare/open/statistics'; // 详情API地址
/** shareCode + password 直连接口(一次返回全量号码,无需翻页) */
const API_DETAIL_BY_SHARE_CODE = '/api/whatshub-counter/workShare/open/detailByShareCode';
const API_COUNT = ''; // 总数API地址 const API_COUNT = ''; // 总数API地址
const DEFAULT_PER_PAGE_COUNT = 20; // List默认每页显示的数量 const DEFAULT_PER_PAGE_COUNT = 20; // List默认每页显示的数量
const API_FETCH_TIMEOUT_SEC = 20;
private $pageUrl; private $pageUrl;
private $account; private $account;
private $password; private $password;
@@ -25,6 +29,165 @@ class Whatshub extends AbstractScrmSpider
$this->unifiedData = new UnifiedScrmData(); $this->unifiedData = new UnifiedScrmData();
} }
/**
* API 优先:detailByShareCode 直抓;失败则回退 Node 爬虫
*/
public function run()
{
$apiResult = $this->tryFetchViaShareCodeApi();
if ($apiResult instanceof UnifiedScrmData) {
$this->logDebug('whatshub api path ok', ['count' => count($apiResult->numbers)]);
return $apiResult;
}
$this->logDebug('whatshub api skipped or failed, fallback node', []);
return parent::run();
}
/**
* 从短链 pageUrl 解析 shareCode,例如 /m/KMYBBInp2066/1 → KMYBBInp2066
*
* @return string|null
*/
private function extractShareCodeFromPageUrl()
{
$path = (string) parse_url($this->pageUrl, PHP_URL_PATH);
if ($path === '') {
return null;
}
if (preg_match('#/m/([^/]+)/#', $path, $matches)) {
return $matches[1];
}
return null;
}
/**
* 拼接 detailByShareCode 完整 GET URL
*
* @param string $shareCode
* @return string|null
*/
private function buildShareCodeApiUrl($shareCode)
{
$parsed = parse_url($this->pageUrl);
$scheme = isset($parsed['scheme']) ? $parsed['scheme'] : 'https';
$host = isset($parsed['host']) ? $parsed['host'] : '';
if ($host === '') {
return null;
}
$query = http_build_query([
'shareCode' => $shareCode,
'password' => $this->password,
]);
return $scheme . '://' . $host . self::API_DETAIL_BY_SHARE_CODE . '?' . $query;
}
/**
* 尝试通过 shareCode API 抓取;成功返回 UnifiedScrmData,否则返回 null
*
* @return UnifiedScrmData|null
*/
private function tryFetchViaShareCodeApi()
{
if ($this->password === '' || $this->password === null) {
$this->logDebug('whatshub api skip: empty password', []);
return null;
}
$shareCode = $this->extractShareCodeFromPageUrl();
if ($shareCode === null || $shareCode === '') {
$this->logDebug('whatshub api skip: cannot parse shareCode', ['pageUrl' => $this->pageUrl]);
return null;
}
$apiUrl = $this->buildShareCodeApiUrl($shareCode);
if ($apiUrl === null) {
$this->logDebug('whatshub api skip: invalid pageUrl host', ['pageUrl' => $this->pageUrl]);
return null;
}
$this->logDebug('whatshub api request', ['url' => preg_replace('/password=[^&]+/', 'password=***', $apiUrl)]);
$ch = curl_init($apiUrl);
curl_setopt_array($ch, [
CURLOPT_RETURNTRANSFER => true,
CURLOPT_CONNECTTIMEOUT => 5,
CURLOPT_TIMEOUT => self::API_FETCH_TIMEOUT_SEC,
CURLOPT_HTTPHEADER => [
'Accept: application/json',
'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
],
]);
$response = curl_exec($ch);
$httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
$curlErr = curl_error($ch);
curl_close($ch);
if ($response === false || $curlErr !== '') {
$this->logDebug('whatshub api curl failed', ['error' => $curlErr]);
return null;
}
if ($httpCode !== 200) {
$this->logDebug('whatshub api http error', ['httpCode' => $httpCode]);
return null;
}
$payload = json_decode($response, true);
if (!is_array($payload)) {
$this->logDebug('whatshub api invalid json', []);
return null;
}
if ((int) ($payload['code'] ?? 0) !== 200) {
$this->logDebug('whatshub api business error', [
'code' => $payload['code'] ?? null,
'msg' => $payload['msg'] ?? '',
]);
return null;
}
if (!isset($payload['data']) || !is_array($payload['data'])) {
$this->logDebug('whatshub api invalid data field', []);
return null;
}
return $this->parseShareCodeApiResponse($payload);
}
/**
* 将 detailByShareCode 响应映射为 UnifiedScrmData
*
* @param array<string, mixed> $payload
* @return UnifiedScrmData
*/
private function parseShareCodeApiResponse(array $payload)
{
$unifiedData = new UnifiedScrmData();
$todayNewSum = 0;
foreach ($payload['data'] as $item) {
if (!is_array($item) || empty($item['account'])) {
continue;
}
$number = (string) $item['account'];
$isOnline = isset($item['isOnline']) && (int) $item['isOnline'] === 1;
$dayNewFans = (int) ($item['dayNewFans'] ?? 0);
$unifiedData->addNumber($number, $isOnline, $dayNewFans);
$todayNewSum += $dayNewFans;
}
$unifiedData->todayNewCount = $todayNewSum;
$unifiedData->total = count($unifiedData->numbers);
return $unifiedData;
}
protected function getSpiderConfig() protected function getSpiderConfig()
{ {
$host = (string) parse_url($this->pageUrl, PHP_URL_HOST); $host = (string) parse_url($this->pageUrl, PHP_URL_HOST);
@@ -33,14 +196,14 @@ class Whatshub extends AbstractScrmSpider
'apiUrls' => [self::API_LIST, self::API_DETAILS], 'apiUrls' => [self::API_LIST, self::API_DETAILS],
// 明确指派角色 // 明确指派角色
'listApi' => self::API_LIST, // 必须 'listApi' => self::API_LIST,
'detailApi' => self::API_DETAILS,
'listMethod' => 'POST', 'listMethod' => 'POST',
'paginationMode' => self::MODE_UI, 'paginationMode' => self::MODE_UI,
'authActions' => [ 'authActions' => [
// 1. 填入密码:寻找 class 包含 el-message-box__input 下面的任意 input // Node antiBot.postCfReady* 会在 CF 成功后等待 work-order-sharing + .el-input__inner
// Node.js 会自动扫描所有匹配项,并只把密码强行注入到那个“肉眼可见”的框里
['type' => 'vue_fill', 'selector' => '.el-input__inner', 'value' => $this->password], ['type' => 'vue_fill', 'selector' => '.el-input__inner', 'value' => $this->password],
// 2. 停顿 500ms,让 Vue 绑定的 v-model 彻底反应过来 // 2. 停顿 500ms,让 Vue 绑定的 v-model 彻底反应过来
@@ -51,16 +214,21 @@ class Whatshub extends AbstractScrmSpider
['type' => 'vue_click', 'selector' => '.vxe-button-group .theme--primary'], ['type' => 'vue_click', 'selector' => '.vxe-button-group .theme--primary'],
// 4. 等待弹窗淡出,接口开始请求 // 4. 等待弹窗淡出,接口开始请求
['type' => 'wait', 'ms' => 2200] ['type' => 'wait', 'ms' => 2200],
], ],
// Whatshub 专用:Real Browser + Turnstile + Captcha API 兜底 + 会话复用 // Whatshub 专用:Real Browser + Turnstile + Captcha API 兜底 + 会话复用
// sessionKey 必须与 pageUrl 的 host 一致,例如 web.whatshub.cc
'antiBot' => [ 'antiBot' => [
'enabled' => true, 'enabled' => true,
'profile' => 'real', 'profile' => 'real',
'turnstile' => true, 'turnstile' => true,
'solverFallback' => true, 'solverFallback' => true,
'sessionKey' => 'whatshub:' . $host, 'challengeTimeoutMs' => 40000,
'challengeTimeoutMs' => 60000, 'postCfReadyUrlContains' => 'work-order-sharing',
'postCfReadySelector' => '.el-input__inner',
'postCfReadyTimeoutMs' => 30000,
'postCfSettleMs' => 500,
'postCfSpaWaitMs' => 3000,
], ],
]; ];
} }
@@ -69,13 +237,13 @@ class Whatshub extends AbstractScrmSpider
protected function extractListTotalPages($listFirstPageData, $countData = null) protected function extractListTotalPages($listFirstPageData, $countData = null)
{ {
$default_per_page_count = self::DEFAULT_PER_PAGE_COUNT; $default_per_page_count = self::DEFAULT_PER_PAGE_COUNT;
$this->unifiedData->total = $listFirstPageData['data']['total']; $this->unifiedData->total = $listFirstPageData['total'];
if($this->unifiedData->total <= $default_per_page_count) { if ($this->unifiedData->total <= $default_per_page_count) {
return 1; return 1;
} }
return ceil($this->unifiedData->total/$default_per_page_count); return ceil($this->unifiedData->total / $default_per_page_count);
} }
// 没有分页返回空数组 // 没有分页返回空数组
@@ -89,7 +257,7 @@ class Whatshub extends AbstractScrmSpider
{ {
return [ return [
'nextBtnSelector' => '.vxe-pager--next-btn', 'nextBtnSelector' => '.vxe-pager--next-btn',
'waitMs' => 1500 'waitMs' => 1500,
]; ];
} }
@@ -100,14 +268,14 @@ class Whatshub extends AbstractScrmSpider
// 1. 如果捕获到了详情数据,提取今日新增 // 1. 如果捕获到了详情数据,提取今日新增
if ($detailData) { if ($detailData) {
$unifiedData->todayNewCount = (int)($detailData['data']['dayNewFans'] ?? 0); $unifiedData->todayNewCount = (int) ($detailData['data']['dayNewFans'] ?? 0);
} }
// 2. 循环合并了所有页数的 List 数组 // 2. 循环合并了所有页数的 List 数组
foreach ($allListPagesData as $pageRaw) { foreach ($allListPagesData as $pageRaw) {
$records = $pageRaw['data']['rows'] ?? []; $records = $pageRaw['rows'] ?? [];
foreach ($records as $item) { foreach ($records as $item) {
if(!empty($item['account'])) { if (!empty($item['account'])) {
$number = $item['account'] ?? null; $number = $item['account'] ?? null;
$isOnline = (isset($item['isOnline']) && $item['isOnline'] == 1); $isOnline = (isset($item['isOnline']) && $item['isOnline'] == 1);
@@ -116,7 +284,7 @@ class Whatshub extends AbstractScrmSpider
} }
} }
// 🚀 3. 终极统计:所有翻页数据均已入库,此时再统计真实的 Total 总数 // 3. 终极统计:所有翻页数据均已入库,此时再统计真实的 Total 总数
$unifiedData->total = count($unifiedData->numbers); $unifiedData->total = count($unifiedData->numbers);
return $unifiedData; return $unifiedData;
+63 -61
View File
@@ -11,50 +11,52 @@ require_once __DIR__ . '/SsCustomer.php'; // SS云控(Customer)
require_once __DIR__ . '/Haiwang.php'; // 海王 require_once __DIR__ . '/Haiwang.php'; // 海王
require_once __DIR__ . '/Whatshub.php'; // Whatshub 工单平台 require_once __DIR__ . '/Whatshub.php'; // Whatshub 工单平台
// try { try {
// /* /*
// 命令行可以测试Cloudflare防火墙
// curl -s -X POST http://127.0.0.1:3001/api/auth-and-intercept \
// -H 'Content-Type: application/json' \
// -d '{
// "pageUrl": "https://web.whatshub.cc/m/iTYsWKQH5030/1",
// "apiUrls": ["/api/whatshub-counter/workShare/open/detail"],
// "authActions": [
// {"type": "vue_fill", "selector": ".el-input__inner", "value": "745030"},
// {"type": "wait", "ms": 500},
// {"type": "vue_click", "selector": ".vxe-button-group .theme--primary"},
// {"type": "wait", "ms": 2200}
// ],
// "antiBot": {
// "enabled": true,
// "profile": "real",
// "turnstile": true,
// "solverFallback": true,
// "sessionKey": "whatshub:t.flowerbells.top",
// "challengeTimeoutMs": 60000
// }
// }' | jq .
// */
// echo "🚀 开始执行<Whatshub 工单平台>抓取任务 (多引擎智能调度)...\n\r"; 命令行可以测试Cloudflare防火墙
// $pageUrl = 'https://web.whatshub.cc/m/iTYsWKQH5030/1'; // PageUrl 入口授权页 curl -s -X POST http://127.0.0.1:3001/api/auth-and-intercept \
// $username = ""; // 登录账号 -H 'Content-Type: application/json' \
// $password = "745030"; // 登录密码 -d '{
// $spider = new Whatshub($pageUrl, $username, $password); "pageUrl": "https://web.whatshub.cc/m/KMYBBInp2066/1",
// $finalData = $spider->run(); "apiUrls": ["/api/whatshub-counter/workShare/open/detail"],
"authActions": [
{"type": "wait", "ms": 1000},
{"type": "vue_fill", "selector": ".el-input__wrapper > .el-input__inner", "value": "602066"},
{"type": "wait", "ms": 500},
{"type": "vue_click", "selector": ".vxe-button-group .theme--primary"},
{"type": "wait", "ms": 2200}
],
"antiBot": {
"enabled": true,
"profile": "real",
"turnstile": true,
"solverFallback": true,
"sessionKey": "whatshub:web.whatshub.cc",
"challengeTimeoutMs": 60000
}
}' | jq .
*/
// echo "✅ 任务完成!统一数据如下:\n\r"; echo "🚀 开始执行<Whatshub 工单平台>抓取任务 (多引擎智能调度)...\n\r";
// echo "----------------------------------------\n\r"; $pageUrl = 'https://web.whatshub.cc/m/KMYBBInp2066/1'; // PageUrl 入口授权页
// echo "当日新增:{$finalData->todayNewCount} 人\n\r"; $username = ""; // 登录账号
// echo "在线号码:{$finalData->totalOnline} 个\n\r"; $password = "602066"; // 登录密码
// echo "离线号码:{$finalData->totalOffline} 个\n\r"; $spider = new Whatshub($pageUrl, $username, $password);
// echo "Total" . $finalData->total . " 个号码\n\r"; $finalData = $spider->run();
// echo "实际总共抓取:" . count($finalData->numbers) . " 个号码\n\r";
// echo "号码列表:\n\r"; echo "✅ 任务完成!统一数据如下:\n\r";
// echo dd($finalData->numbers); echo "----------------------------------------\n\r";
// } catch (Exception $e) { echo "当日新增:{$finalData->todayNewCount}\n\r";
// echo "🚨 抓取异常:" . $e->getMessage() . "\n\r"; echo "在线号码:{$finalData->totalOnline}\n\r";
// } echo "离线号码:{$finalData->totalOffline}\n\r";
echo "Total" . $finalData->total . " 个号码\n\r";
echo "实际总共抓取:" . count($finalData->numbers) . " 个号码\n\r";
echo "号码列表:\n\r";
echo dd($finalData->numbers);
} catch (Exception $e) {
echo "🚨 抓取异常:" . $e->getMessage() . "\n\r";
}
// try { // try {
@@ -78,27 +80,27 @@ require_once __DIR__ . '/Whatshub.php'; // Whatshub 工单平台
// echo "🚨 抓取异常:" . $e->getMessage() . "\n\r"; // echo "🚨 抓取异常:" . $e->getMessage() . "\n\r";
// } // }
try { // try {
echo "🚀 开始执行<A2c云控>抓取任务 (多引擎智能调度)...\n\r"; // echo "🚀 开始执行<A2c云控>抓取任务 (多引擎智能调度)...\n\r";
$pageUrl = 'https://yyk.ink/52oXZRG'; // PageUrl 入口授权页 // $pageUrl = 'https://yyk.ink/52oXZRG'; // PageUrl 入口授权页
$username = ""; // 登录账号 // $username = ""; // 登录账号
$password = ""; // 登录密码 // $password = ""; // 登录密码
$spider = new A2c($pageUrl, $username, $password); // $spider = new A2c($pageUrl, $username, $password);
$finalData = $spider->run(); // $finalData = $spider->run();
echo "✅ 任务完成!统一数据如下:\n\r"; // echo "✅ 任务完成!统一数据如下:\n\r";
echo "----------------------------------------\n\r"; // echo "----------------------------------------\n\r";
echo "当日新增:{$finalData->todayNewCount}\n\r"; // echo "当日新增:{$finalData->todayNewCount} 人\n\r";
echo "在线号码:{$finalData->totalOnline}\n\r"; // echo "在线号码:{$finalData->totalOnline} 个\n\r";
echo "离线号码:{$finalData->totalOffline}\n\r"; // echo "离线号码:{$finalData->totalOffline} 个\n\r";
echo "Total" . $finalData->total . " 个号码\n\r"; // echo "Total" . $finalData->total . " 个号码\n\r";
echo "实际总共抓取:" . count($finalData->numbers) . " 个号码\n\r"; // echo "实际总共抓取:" . count($finalData->numbers) . " 个号码\n\r";
echo "号码列表:\n\r"; // echo "号码列表:\n\r";
echo dd($finalData->numbers); // echo dd($finalData->numbers);
} catch (Exception $e) { // } catch (Exception $e) {
echo "🚨 抓取异常:" . $e->getMessage() . "\n\r"; // echo "🚨 抓取异常:" . $e->getMessage() . "\n\r";
} // }
// try { // try {
// echo "🚀 开始执行<星河云控>抓取任务 (多引擎智能调度)...\n\r"; // echo "🚀 开始执行<星河云控>抓取任务 (多引擎智能调度)...\n\r";
@@ -41,6 +41,26 @@ class AntiBotConfigBuilder
'sessionKey' => $sessionKey, 'sessionKey' => $sessionKey,
'challengeTimeoutMs' => 60000, 'challengeTimeoutMs' => 60000,
'sessionTtlMinutes' => $ttlMinutes, 'sessionTtlMinutes' => $ttlMinutes,
] + self::postCfReadyExtras($ticketType);
}
/**
* 按工单类型附加 CF 过盾后的业务页就绪等待(仅 Whatshub 等需要的类型)
*
* @return array<string, mixed>
*/
private static function postCfReadyExtras(string $ticketType): array
{
if ($ticketType !== 'whatshub') {
return [];
}
return [
'postCfReadyUrlContains' => 'work-order-sharing',
'postCfReadySelector' => '.el-input__inner',
'postCfReadyTimeoutMs' => 30000,
'postCfSettleMs' => 500,
'postCfSpaWaitMs' => 4000,
]; ];
} }
@@ -7,11 +7,12 @@ namespace app\common\library\scrm\spider;
use app\common\library\scrm\AbstractScrmSpider; use app\common\library\scrm\AbstractScrmSpider;
use app\common\library\scrm\AntiBotConfigBuilder; use app\common\library\scrm\AntiBotConfigBuilder;
use app\common\library\scrm\UnifiedScrmData; use app\common\library\scrm\UnifiedScrmData;
use app\common\service\SplitTicketSyncLogger;
/** /**
* Whatshub 云控蜘蛛Real Browser + Turnstile 过盾 + UI 翻页) * Whatshub 云控蜘蛛
* *
* 时序:Cloudflare Turnstile → 密码弹窗 authActions → API 拦截 → UI 翻页 * 时序:detailByShareCode API 直抓(优先)→ 失败则 Real Browser + Turnstile + UI 翻页
*/ */
class WhatshubSpider extends AbstractScrmSpider class WhatshubSpider extends AbstractScrmSpider
{ {
@@ -21,9 +22,14 @@ class WhatshubSpider extends AbstractScrmSpider
/** 详情/统计 API 路径 */ /** 详情/统计 API 路径 */
private const API_DETAILS = '/api/whatshub-counter/workShare/open/statistics'; private const API_DETAILS = '/api/whatshub-counter/workShare/open/statistics';
/** shareCode + password 直连接口(一次返回全量号码,无需翻页) */
private const API_DETAIL_BY_SHARE_CODE = '/api/whatshub-counter/workShare/open/detailByShareCode';
/** 默认每页条数 */ /** 默认每页条数 */
private const DEFAULT_PER_PAGE_COUNT = 20; private const DEFAULT_PER_PAGE_COUNT = 20;
private const API_FETCH_TIMEOUT_SEC = 20;
private string $pageUrl; private string $pageUrl;
private string $account; private string $account;
@@ -45,6 +51,178 @@ class WhatshubSpider extends AbstractScrmSpider
$this->unifiedData = new UnifiedScrmData(); $this->unifiedData = new UnifiedScrmData();
} }
/**
* API 优先:detailByShareCode 直抓;失败则回退 Node 爬虫
*/
public function run(): UnifiedScrmData
{
$apiResult = $this->tryFetchViaShareCodeApi();
if ($apiResult instanceof UnifiedScrmData) {
SplitTicketSyncLogger::log('spider', 'whatshub api path ok', [
'count' => count($apiResult->numbers),
]);
return $apiResult;
}
SplitTicketSyncLogger::log('spider', 'whatshub api skipped or failed, fallback node', [
'pageUrl' => $this->pageUrl,
]);
return parent::run();
}
/**
* 从短链 pageUrl 解析 shareCode,例如 /m/KMYBBInp2066/1 → KMYBBInp2066
*/
private function extractShareCodeFromPageUrl(): ?string
{
$path = (string) parse_url($this->pageUrl, PHP_URL_PATH);
if ($path === '') {
return null;
}
if (preg_match('#/m/([^/]+)/#', $path, $matches)) {
return $matches[1];
}
return null;
}
/**
* 拼接 detailByShareCode 完整 GET URL
*/
private function buildShareCodeApiUrl(string $shareCode): ?string
{
$parsed = parse_url($this->pageUrl);
$scheme = isset($parsed['scheme']) ? (string) $parsed['scheme'] : 'https';
$host = isset($parsed['host']) ? (string) $parsed['host'] : '';
if ($host === '') {
return null;
}
$query = http_build_query([
'shareCode' => $shareCode,
'password' => $this->password,
]);
return $scheme . '://' . $host . self::API_DETAIL_BY_SHARE_CODE . '?' . $query;
}
/**
* 尝试通过 shareCode API 抓取;成功返回 UnifiedScrmData,否则返回 null
*/
private function tryFetchViaShareCodeApi(): ?UnifiedScrmData
{
if ($this->password === '') {
SplitTicketSyncLogger::log('spider', 'whatshub api skip: empty password', []);
return null;
}
$shareCode = $this->extractShareCodeFromPageUrl();
if ($shareCode === null || $shareCode === '') {
SplitTicketSyncLogger::log('spider', 'whatshub api skip: cannot parse shareCode', [
'pageUrl' => $this->pageUrl,
]);
return null;
}
$apiUrl = $this->buildShareCodeApiUrl($shareCode);
if ($apiUrl === null) {
SplitTicketSyncLogger::log('spider', 'whatshub api skip: invalid pageUrl host', [
'pageUrl' => $this->pageUrl,
]);
return null;
}
SplitTicketSyncLogger::log('spider', 'whatshub api request', [
'url' => preg_replace('/password=[^&]+/', 'password=***', $apiUrl),
]);
$ch = curl_init($apiUrl);
if ($ch === false) {
SplitTicketSyncLogger::log('spider', 'whatshub api curl init failed', []);
return null;
}
curl_setopt_array($ch, [
CURLOPT_RETURNTRANSFER => true,
CURLOPT_CONNECTTIMEOUT => 5,
CURLOPT_TIMEOUT => self::API_FETCH_TIMEOUT_SEC,
CURLOPT_HTTPHEADER => [
'Accept: application/json',
'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
],
]);
$response = curl_exec($ch);
$httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
$curlErr = curl_error($ch);
curl_close($ch);
if ($response === false || $curlErr !== '') {
SplitTicketSyncLogger::log('spider', 'whatshub api curl failed', [
'error' => $curlErr,
]);
return null;
}
if ($httpCode !== 200) {
SplitTicketSyncLogger::log('spider', 'whatshub api http error', [
'httpCode' => $httpCode,
]);
return null;
}
$payload = json_decode((string) $response, true);
if (!is_array($payload)) {
SplitTicketSyncLogger::log('spider', 'whatshub api invalid json', []);
return null;
}
if ((int) ($payload['code'] ?? 0) !== 200) {
SplitTicketSyncLogger::log('spider', 'whatshub api business error', [
'code' => $payload['code'] ?? null,
'msg' => $payload['msg'] ?? '',
]);
return null;
}
if (!isset($payload['data']) || !is_array($payload['data'])) {
SplitTicketSyncLogger::log('spider', 'whatshub api invalid data field', []);
return null;
}
return $this->parseShareCodeApiResponse($payload);
}
/**
* 将 detailByShareCode 响应映射为 UnifiedScrmData
*
* @param array<string, mixed> $payload
*/
private function parseShareCodeApiResponse(array $payload): UnifiedScrmData
{
$unifiedData = new UnifiedScrmData();
$todayNewSum = 0;
foreach ($payload['data'] as $item) {
if (!is_array($item) || empty($item['account'])) {
continue;
}
$number = (string) $item['account'];
$isOnline = isset($item['isOnline']) && (int) $item['isOnline'] === 1;
$dayNewFans = (int) ($item['dayNewFans'] ?? 0);
$unifiedData->addNumber($number, $isOnline, $dayNewFans);
$todayNewSum += $dayNewFans;
}
$unifiedData->todayNewCount = $todayNewSum;
$unifiedData->total = count($unifiedData->numbers);
return $unifiedData;
}
/** @return array<string, mixed> */ /** @return array<string, mixed> */
protected function getSpiderConfig(): array protected function getSpiderConfig(): array
{ {
@@ -41,6 +41,26 @@ class AntiBotConfigBuilder
'sessionKey' => $sessionKey, 'sessionKey' => $sessionKey,
'challengeTimeoutMs' => 60000, 'challengeTimeoutMs' => 60000,
'sessionTtlMinutes' => $ttlMinutes, 'sessionTtlMinutes' => $ttlMinutes,
] + self::postCfReadyExtras($ticketType);
}
/**
* 按工单类型附加 CF 过盾后的业务页就绪等待(仅 Whatshub 等需要的类型)
*
* @return array<string, mixed>
*/
private static function postCfReadyExtras(string $ticketType): array
{
if ($ticketType !== 'whatshub') {
return [];
}
return [
'postCfReadyUrlContains' => 'work-order-sharing',
'postCfReadySelector' => '.el-input__inner',
'postCfReadyTimeoutMs' => 30000,
'postCfSettleMs' => 500,
'postCfSpaWaitMs' => 8000,
]; ];
} }
@@ -7,11 +7,12 @@ namespace app\common\library\scrm\spider;
use app\common\library\scrm\AbstractScrmSpider; use app\common\library\scrm\AbstractScrmSpider;
use app\common\library\scrm\AntiBotConfigBuilder; use app\common\library\scrm\AntiBotConfigBuilder;
use app\common\library\scrm\UnifiedScrmData; use app\common\library\scrm\UnifiedScrmData;
use app\common\service\SplitTicketSyncLogger;
/** /**
* Whatshub 云控蜘蛛Real Browser + Turnstile 过盾 + UI 翻页) * Whatshub 云控蜘蛛
* *
* 时序:Cloudflare Turnstile → 密码弹窗 authActions → API 拦截 → UI 翻页 * 时序:detailByShareCode API 直抓(优先)→ 失败则 Real Browser + Turnstile + UI 翻页
*/ */
class WhatshubSpider extends AbstractScrmSpider class WhatshubSpider extends AbstractScrmSpider
{ {
@@ -21,9 +22,14 @@ class WhatshubSpider extends AbstractScrmSpider
/** 详情/统计 API 路径 */ /** 详情/统计 API 路径 */
private const API_DETAILS = '/api/whatshub-counter/workShare/open/statistics'; private const API_DETAILS = '/api/whatshub-counter/workShare/open/statistics';
/** shareCode + password 直连接口(一次返回全量号码,无需翻页) */
private const API_DETAIL_BY_SHARE_CODE = '/api/whatshub-counter/workShare/open/detailByShareCode';
/** 默认每页条数 */ /** 默认每页条数 */
private const DEFAULT_PER_PAGE_COUNT = 20; private const DEFAULT_PER_PAGE_COUNT = 20;
private const API_FETCH_TIMEOUT_SEC = 20;
private string $pageUrl; private string $pageUrl;
private string $account; private string $account;
@@ -45,6 +51,178 @@ class WhatshubSpider extends AbstractScrmSpider
$this->unifiedData = new UnifiedScrmData(); $this->unifiedData = new UnifiedScrmData();
} }
/**
* API 优先:detailByShareCode 直抓;失败则回退 Node 爬虫
*/
public function run(): UnifiedScrmData
{
$apiResult = $this->tryFetchViaShareCodeApi();
if ($apiResult instanceof UnifiedScrmData) {
SplitTicketSyncLogger::log('spider', 'whatshub api path ok', [
'count' => count($apiResult->numbers),
]);
return $apiResult;
}
SplitTicketSyncLogger::log('spider', 'whatshub api skipped or failed, fallback node', [
'pageUrl' => $this->pageUrl,
]);
return parent::run();
}
/**
* 从短链 pageUrl 解析 shareCode,例如 /m/KMYBBInp2066/1 → KMYBBInp2066
*/
private function extractShareCodeFromPageUrl(): ?string
{
$path = (string) parse_url($this->pageUrl, PHP_URL_PATH);
if ($path === '') {
return null;
}
if (preg_match('#/m/([^/]+)/#', $path, $matches)) {
return $matches[1];
}
return null;
}
/**
* 拼接 detailByShareCode 完整 GET URL
*/
private function buildShareCodeApiUrl(string $shareCode): ?string
{
$parsed = parse_url($this->pageUrl);
$scheme = isset($parsed['scheme']) ? (string) $parsed['scheme'] : 'https';
$host = isset($parsed['host']) ? (string) $parsed['host'] : '';
if ($host === '') {
return null;
}
$query = http_build_query([
'shareCode' => $shareCode,
'password' => $this->password,
]);
return $scheme . '://' . $host . self::API_DETAIL_BY_SHARE_CODE . '?' . $query;
}
/**
* 尝试通过 shareCode API 抓取;成功返回 UnifiedScrmData,否则返回 null
*/
private function tryFetchViaShareCodeApi(): ?UnifiedScrmData
{
if ($this->password === '') {
SplitTicketSyncLogger::log('spider', 'whatshub api skip: empty password', []);
return null;
}
$shareCode = $this->extractShareCodeFromPageUrl();
if ($shareCode === null || $shareCode === '') {
SplitTicketSyncLogger::log('spider', 'whatshub api skip: cannot parse shareCode', [
'pageUrl' => $this->pageUrl,
]);
return null;
}
$apiUrl = $this->buildShareCodeApiUrl($shareCode);
if ($apiUrl === null) {
SplitTicketSyncLogger::log('spider', 'whatshub api skip: invalid pageUrl host', [
'pageUrl' => $this->pageUrl,
]);
return null;
}
SplitTicketSyncLogger::log('spider', 'whatshub api request', [
'url' => preg_replace('/password=[^&]+/', 'password=***', $apiUrl),
]);
$ch = curl_init($apiUrl);
if ($ch === false) {
SplitTicketSyncLogger::log('spider', 'whatshub api curl init failed', []);
return null;
}
curl_setopt_array($ch, [
CURLOPT_RETURNTRANSFER => true,
CURLOPT_CONNECTTIMEOUT => 5,
CURLOPT_TIMEOUT => self::API_FETCH_TIMEOUT_SEC,
CURLOPT_HTTPHEADER => [
'Accept: application/json',
'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
],
]);
$response = curl_exec($ch);
$httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
$curlErr = curl_error($ch);
curl_close($ch);
if ($response === false || $curlErr !== '') {
SplitTicketSyncLogger::log('spider', 'whatshub api curl failed', [
'error' => $curlErr,
]);
return null;
}
if ($httpCode !== 200) {
SplitTicketSyncLogger::log('spider', 'whatshub api http error', [
'httpCode' => $httpCode,
]);
return null;
}
$payload = json_decode((string) $response, true);
if (!is_array($payload)) {
SplitTicketSyncLogger::log('spider', 'whatshub api invalid json', []);
return null;
}
if ((int) ($payload['code'] ?? 0) !== 200) {
SplitTicketSyncLogger::log('spider', 'whatshub api business error', [
'code' => $payload['code'] ?? null,
'msg' => $payload['msg'] ?? '',
]);
return null;
}
if (!isset($payload['data']) || !is_array($payload['data'])) {
SplitTicketSyncLogger::log('spider', 'whatshub api invalid data field', []);
return null;
}
return $this->parseShareCodeApiResponse($payload);
}
/**
* 将 detailByShareCode 响应映射为 UnifiedScrmData
*
* @param array<string, mixed> $payload
*/
private function parseShareCodeApiResponse(array $payload): UnifiedScrmData
{
$unifiedData = new UnifiedScrmData();
$todayNewSum = 0;
foreach ($payload['data'] as $item) {
if (!is_array($item) || empty($item['account'])) {
continue;
}
$number = (string) $item['account'];
$isOnline = isset($item['isOnline']) && (int) $item['isOnline'] === 1;
$dayNewFans = (int) ($item['dayNewFans'] ?? 0);
$unifiedData->addNumber($number, $isOnline, $dayNewFans);
$todayNewSum += $dayNewFans;
}
$unifiedData->todayNewCount = $todayNewSum;
$unifiedData->total = count($unifiedData->numbers);
return $unifiedData;
}
/** @return array<string, mixed> */ /** @return array<string, mixed> */
protected function getSpiderConfig(): array protected function getSpiderConfig(): array
{ {
@@ -28,6 +28,10 @@ function normalizeAntiBot(antiBot) {
solverFallback: antiBot.solverFallback !== false, solverFallback: antiBot.solverFallback !== false,
sessionKey: antiBot.sessionKey || '', sessionKey: antiBot.sessionKey || '',
challengeTimeoutMs: antiBot.challengeTimeoutMs || 60000, challengeTimeoutMs: antiBot.challengeTimeoutMs || 60000,
postCfReadyUrlContains: antiBot.postCfReadyUrlContains || '',
postCfReadySelector: antiBot.postCfReadySelector || '',
postCfReadyTimeoutMs: antiBot.postCfReadyTimeoutMs || 0,
postCfSettleMs: antiBot.postCfSettleMs || 0,
}; };
} }
@@ -106,8 +110,10 @@ async function createManagedPage(browser, existingPage, options = {}) {
page.setDefaultNavigationTimeout(NAVIGATION_TIMEOUT_MS); page.setDefaultNavigationTimeout(NAVIGATION_TIMEOUT_MS);
page.setDefaultTimeout(PAGE_DEFAULT_TIMEOUT_MS); page.setDefaultTimeout(PAGE_DEFAULT_TIMEOUT_MS);
if (!options.skipUserAgent) {
const userAgent = options.userAgent || DEFAULT_UA; const userAgent = options.userAgent || DEFAULT_UA;
await page.setUserAgent(userAgent); await page.setUserAgent(userAgent);
}
if (options.viewport) await page.setViewport(options.viewport); if (options.viewport) await page.setViewport(options.viewport);
if (options.cookies && options.cookies.length > 0) { if (options.cookies && options.cookies.length > 0) {
+18 -2
View File
@@ -2,6 +2,21 @@
* Cloudflare / Turnstile 挑战页检测 * Cloudflare / Turnstile 挑战页检测
*/ */
/**
* URL 是否处于 CF Managed Challenge 中间态(Whatshub 短链常见 __cf_chl_rt_tk
* @param {string} url
*/
function urlIndicatesCfChallenge(url) {
if (!url || typeof url !== 'string') {
return false;
}
return url.includes('__cf_chl_rt_tk')
|| url.includes('__cf_chl_tk')
|| url.includes('__cf_chl_f_tk')
|| url.includes('/cdn-cgi/challenge-platform')
|| url.includes('/cdn-cgi/challenge');
}
/** /**
* @typedef {Object} CloudflareState * @typedef {Object} CloudflareState
* @property {boolean} blocked 是否处于 CF 挑战中 * @property {boolean} blocked 是否处于 CF 挑战中
@@ -44,10 +59,10 @@ async function detectCloudflare(page) {
bodyHasJustAMoment: false, bodyHasJustAMoment: false,
})); }));
const urlChallenge = url.includes('/cdn-cgi/challenge-platform') || url.includes('/cdn-cgi/challenge'); const urlChallenge = urlIndicatesCfChallenge(url);
let challengeType = null; let challengeType = null;
if (domSignals.hasTurnstileInput || domSignals.hasTurnstileWidget) { if (domSignals.hasTurnstileInput || domSignals.hasTurnstileWidget || url.includes('__cf_chl')) {
challengeType = 'turnstile'; challengeType = 'turnstile';
} else if (urlChallenge || domSignals.titleHasJustAMoment || domSignals.bodyHasJustAMoment) { } else if (urlChallenge || domSignals.titleHasJustAMoment || domSignals.bodyHasJustAMoment) {
challengeType = 'js_challenge'; challengeType = 'js_challenge';
@@ -94,4 +109,5 @@ async function isTurnstileSolved(page) {
module.exports = { module.exports = {
detectCloudflare, detectCloudflare,
isTurnstileSolved, isTurnstileSolved,
urlIndicatesCfChallenge,
}; };
+185 -13
View File
@@ -1,5 +1,8 @@
/** /**
* Cloudflare 挑战统一处理:检测 → 会话快速通道 → 内置等待 → Captcha API 兜底 * Cloudflare 挑战统一处理:检测 → 会话快速通道 → 内置等待 → Captcha API 兜底
*
* 当 antiBot.postCfReadyUrlContains 已配置时,session_reuse 仅在业务 URL 已到达时才走快速通道;
* 否则清除可能过期的 cf_clearance 并重新触发 TurnstileWhatshub 短链 → work-order-sharing 场景)。
*/ */
const { detectCloudflare } = require('./cf-detector'); const { detectCloudflare } = require('./cf-detector');
const { waitForTurnstile, extractTurnstileSitekey, injectTurnstileToken } = require('./turnstile-handler'); const { waitForTurnstile, extractTurnstileSitekey, injectTurnstileToken } = require('./turnstile-handler');
@@ -7,8 +10,135 @@ const { solveTurnstile, isCaptchaConfigured } = require('./captcha-solver');
const { isSessionLikelyValid } = require('./session-store'); const { isSessionLikelyValid } = require('./session-store');
/** /**
* 是否配置了 CF 后必须到达的业务 URL
* @param {object} antiBot
*/
function needsPostCfBusinessUrl(antiBot) {
return !!(antiBot?.postCfReadyUrlContains || '').trim();
}
/**
* 当前页面 URL 是否已包含业务路径片段
* @param {import('puppeteer').Page} page * @param {import('puppeteer').Page} page
* @param {{ turnstile?: boolean, solverFallback?: boolean, challengeTimeoutMs?: number }} antiBot * @param {object} antiBot
*/
function isPostCfBusinessUrlReady(page, antiBot) {
const needle = (antiBot?.postCfReadyUrlContains || '').trim();
if (!needle) {
return true;
}
return page.url().includes(needle);
}
/**
* 清除 cf_clearance,避免「有 cookie 但 SPA 未触发跳转」的假过关
* @param {import('puppeteer').Page} page
*/
async function clearCfClearanceCookies(page) {
const cookies = await page.cookies();
for (const c of cookies) {
if (c.name !== 'cf_clearance') {
continue;
}
const host = (c.domain || '').replace(/^\./, '');
if (host) {
await page.deleteCookie({ name: c.name, url: `https://${host}/` }).catch(() => {});
}
await page.deleteCookie({
name: c.name,
domain: c.domain,
path: c.path || '/',
}).catch(() => {});
}
}
/**
* reload 后检查业务 URL 是否就绪
* @param {import('puppeteer').Page} page
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
* @param {number} timeoutMs
* @param {object} antiBot
*/
async function tryPostCfRecoverViaReload(page, waitForUrlSettled, timeoutMs, antiBot) {
if (!needsPostCfBusinessUrl(antiBot) || isPostCfBusinessUrlReady(page, antiBot)) {
return isPostCfBusinessUrlReady(page, antiBot);
}
console.log(`[CF] 业务 URL 未就绪 current=${page.url()},尝试 reload`);
try {
await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 45000) });
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
return isPostCfBusinessUrlReady(page, antiBot);
} catch (reloadErr) {
console.warn('[CF] reload 失败:', reloadErr.message);
return false;
}
}
/**
* 清除 cf_clearance 并 reload,使 Turnstile 重新出现
* @param {import('puppeteer').Page} page
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
* @param {number} timeoutMs
*/
async function invalidateStaleCfAndReload(page, waitForUrlSettled, timeoutMs) {
console.log('[CF] 清除 cf_clearance 并 reload,重新触发 Turnstile');
await clearCfClearanceCookies(page);
await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 45000) });
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
return detectCloudflare(page);
}
/**
* session_reuse 快速返回前校验:有 postCf 要求时必须已到业务页
* @param {import('puppeteer').Page} page
* @param {object} antiBot
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
* @param {number} timeoutMs
* @param {number} started
* @param {boolean} cfDetected
*/
async function trySessionReuseFastPath(page, antiBot, waitForUrlSettled, timeoutMs, started, cfDetected) {
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
if (!needsPostCfBusinessUrl(antiBot) || isPostCfBusinessUrlReady(page, antiBot)) {
return {
success: true,
code: null,
challengeType: null,
stage: 'session_reuse',
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected,
};
}
console.log(`[CF] cf_clearance/无挑战但业务页未就绪 url=${page.url()},降级重试`);
if (await tryPostCfRecoverViaReload(page, waitForUrlSettled, timeoutMs, antiBot)) {
return {
success: true,
code: null,
challengeType: null,
stage: 'session_reuse_reload',
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected,
};
}
return null;
}
/**
* @param {import('puppeteer').Page} page
* @param {{ turnstile?: boolean, solverFallback?: boolean, challengeTimeoutMs?: number, postCfReadyUrlContains?: string }} antiBot
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled] * @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
* @param {import('./session-store').StoredSession|null} [storedSession] * @param {import('./session-store').StoredSession|null} [storedSession]
*/ */
@@ -19,18 +149,21 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
let cfState = await detectCloudflare(page); let cfState = await detectCloudflare(page);
if (!cfState.blocked && cfState.hasCfClearance) { if (!cfState.blocked && cfState.hasCfClearance) {
return { const fast = await trySessionReuseFastPath(
success: true, page, antiBot, waitForUrlSettled, timeoutMs, started, false
code: null, );
challengeType: null, if (fast) {
stage: 'session_reuse', return fast;
solverUsed: false, }
elapsedMs: Date.now() - started, cfState = await invalidateStaleCfAndReload(page, waitForUrlSettled, timeoutMs);
cfDetected: false,
};
} }
if (!cfState.blocked) { if (!cfState.blocked) {
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
if (!needsPostCfBusinessUrl(antiBot) || isPostCfBusinessUrlReady(page, antiBot)) {
return { return {
success: true, success: true,
code: null, code: null,
@@ -42,13 +175,45 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
}; };
} }
if (await tryPostCfRecoverViaReload(page, waitForUrlSettled, timeoutMs, antiBot)) {
return {
success: true,
code: null,
challengeType: null,
stage: 'post_nav_reload',
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: false,
};
}
if (cfState.hasCfClearance) {
cfState = await invalidateStaleCfAndReload(page, waitForUrlSettled, timeoutMs);
} else {
return {
success: true,
code: null,
challengeType: null,
stage: 'pending_business_nav',
solverUsed: false,
elapsedMs: Date.now() - started,
cfDetected: false,
};
}
}
if (storedSession && isSessionLikelyValid(storedSession)) { if (storedSession && isSessionLikelyValid(storedSession)) {
console.log('[CF] 会话有效但仍被拦截,尝试 reload 一次'); console.log('[CF] 会话有效但仍被拦截,尝试 reload 一次');
try { try {
await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 45000) }); await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 45000) });
if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {}); if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
cfState = await detectCloudflare(page); cfState = await detectCloudflare(page);
if (!cfState.blocked) { if (!cfState.blocked) {
if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
return { return {
success: true, success: true,
code: null, code: null,
@@ -69,7 +234,9 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
if (antiBot.turnstile !== false) { if (antiBot.turnstile !== false) {
const builtIn = await waitForTurnstile(page, { timeoutMs, useBuiltInClick: true }); const builtIn = await waitForTurnstile(page, { timeoutMs, useBuiltInClick: true });
if (builtIn.success) { if (builtIn.success) {
if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {}); if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
cfState = await detectCloudflare(page); cfState = await detectCloudflare(page);
if (!cfState.blocked) { if (!cfState.blocked) {
return { return {
@@ -96,7 +263,9 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
const apiWaitMs = Math.min(timeoutMs, 30000); const apiWaitMs = Math.min(timeoutMs, 30000);
await waitForTurnstile(page, { timeoutMs: apiWaitMs }); await waitForTurnstile(page, { timeoutMs: apiWaitMs });
if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {}); if (waitForUrlSettled) {
await waitForUrlSettled(page).catch(() => {});
}
cfState = await detectCloudflare(page); cfState = await detectCloudflare(page);
if (!cfState.blocked) { if (!cfState.blocked) {
@@ -138,4 +307,7 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
module.exports = { module.exports = {
handleCloudflareChallenge, handleCloudflareChallenge,
isCaptchaConfigured, isCaptchaConfigured,
needsPostCfBusinessUrl,
isPostCfBusinessUrlReady,
clearCfClearanceCookies,
}; };
+109
View File
@@ -0,0 +1,109 @@
/**
* CF 过盾后等待业务页就绪(可选,由 antiBot.postCfReady* 驱动,未配置则跳过)
*/
/**
* 采集当前页面诊断信息(auth 失败时便于排查)
* @param {import('puppeteer').Page} page
*/
async function collectPageDiagnostics(page) {
const url = page.url();
const snapshot = await page.evaluate(() => ({
url: window.location.href,
title: document.title || '',
bodyText: (document.body && document.body.innerText ? document.body.innerText : '').slice(0, 500),
elInputInner: document.querySelectorAll('.el-input__inner').length,
})).catch(() => null);
return snapshot || { url, title: '', bodyText: '', elInputInner: 0 };
}
/**
* CF 成功后等待最终业务 URL / DOM 出现(Whatshub 等 SPA 短链跳转场景)
*
* @param {import('puppeteer').Page} page
* @param {object} antiBot normalizeAntiBot 返回值
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
* @param {number} [navigationTimeoutMs]
*/
async function awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled, navigationTimeoutMs = 60000) {
if (!antiBot || !antiBot.enabled) {
return;
}
const urlContains = (antiBot.postCfReadyUrlContains || '').trim();
const selector = (antiBot.postCfReadySelector || '').trim();
if (urlContains === '' && selector === '') {
return;
}
const timeoutMs = Math.max(5000, antiBot.postCfReadyTimeoutMs || 30000);
console.log(
`[CF] 等待业务页就绪 current=${page.url()} urlContains="${urlContains}" `
+ `selector="${selector}" timeout=${timeoutMs}ms`
);
try {
if (urlContains !== '') {
await page.waitForFunction(
(needle) => window.location.href.includes(needle),
{ timeout: timeoutMs, polling: 200 },
urlContains
);
}
if (selector !== '') {
await page.waitForSelector(selector, { timeout: timeoutMs, visible: true });
}
} catch (err) {
const retryMs = Math.min(15000, timeoutMs);
const needsRetry = urlContains !== '' && !page.url().includes(urlContains);
if (needsRetry) {
console.warn(`[CF] 业务页等待失败 url=${page.url()},最后尝试 reload (${retryMs}ms)`);
await page.reload({ waitUntil: 'domcontentloaded', timeout: navigationTimeoutMs }).catch(() => {});
if (typeof waitForUrlSettled === 'function') {
await waitForUrlSettled(page).catch(() => {});
}
if (urlContains !== '') {
await page.waitForFunction(
(needle) => window.location.href.includes(needle),
{ timeout: retryMs, polling: 200 },
urlContains
);
}
if (selector !== '') {
await page.waitForSelector(selector, { timeout: retryMs, visible: true });
}
} else {
err.pageDiagnostics = await collectPageDiagnostics(page);
throw err;
}
}
const settleMs = antiBot.postCfSettleMs || 0;
if (settleMs > 0) {
await new Promise((resolve) => setTimeout(resolve, settleMs));
}
console.log(`[CF] 业务页就绪 url=${page.url()}`);
}
/**
* 执行 authActions,失败时附带 page 诊断
* @param {import('puppeteer').Page} page
* @param {object[]|undefined} authActions
* @param {Function} executeAuthActionsFn
*/
async function executeAuthActionsWithDiagnostics(page, authActions, executeAuthActionsFn) {
try {
await executeAuthActionsFn(page, authActions);
} catch (err) {
err.pageDiagnostics = await collectPageDiagnostics(page);
throw err;
}
}
module.exports = {
collectPageDiagnostics,
awaitPostCfBusinessReady,
executeAuthActionsWithDiagnostics,
};
+11 -5
View File
@@ -1,7 +1,7 @@
/** /**
* Turnstile 内置求解:等待 widget 自动完成 + token 轮询 * Turnstile 内置求解:等待 widget 自动完成 + token 轮询
*/ */
const { detectCloudflare, isTurnstileSolved } = require('./cf-detector'); const { detectCloudflare, isTurnstileSolved, urlIndicatesCfChallenge } = require('./cf-detector');
/** /**
* @typedef {Object} TurnstileResult * @typedef {Object} TurnstileResult
@@ -22,18 +22,24 @@ async function waitForTurnstile(page, options = {}) {
const started = Date.now(); const started = Date.now();
const initial = await detectCloudflare(page); const initial = await detectCloudflare(page);
if (!initial.blocked || initial.hasCfClearance) { const initialUrlChallenge = urlIndicatesCfChallenge(page.url());
if ((!initial.blocked && !initialUrlChallenge) || initial.hasCfClearance) {
if (initial.hasCfClearance && !initialUrlChallenge) {
return { success: true, stage: 'already_clear', elapsedMs: Date.now() - started }; return { success: true, stage: 'already_clear', elapsedMs: Date.now() - started };
} }
if (!initial.blocked && !initialUrlChallenge) {
return { success: true, stage: 'already_clear', elapsedMs: Date.now() - started };
}
}
// real browser 已开启 turnstile:true 内置点击;此处轮询等待结果
while (Date.now() - started < timeoutMs) { while (Date.now() - started < timeoutMs) {
if (await isTurnstileSolved(page)) { if (await isTurnstileSolved(page) && !urlIndicatesCfChallenge(page.url())) {
return { success: true, stage: 'built_in', elapsedMs: Date.now() - started }; return { success: true, stage: 'built_in', elapsedMs: Date.now() - started };
} }
const state = await detectCloudflare(page); const state = await detectCloudflare(page);
if (!state.blocked) { if (!state.blocked && !urlIndicatesCfChallenge(page.url()) && state.hasCfClearance) {
return { success: true, stage: 'built_in', elapsedMs: Date.now() - started }; return { success: true, stage: 'built_in', elapsedMs: Date.now() - started };
} }
+1
View File
@@ -7,6 +7,7 @@
"start": "node server.js" "start": "node server.js"
}, },
"dependencies": { "dependencies": {
"dotenv": "^16.4.7",
"express": "^4.18.2", "express": "^4.18.2",
"puppeteer": "^25.1.0", "puppeteer": "^25.1.0",
"puppeteer-extra": "^3.3.6", "puppeteer-extra": "^3.3.6",
+25 -3
View File
@@ -1,3 +1,5 @@
require('dotenv').config({ path: require('path').join(__dirname, '.env') });
const express = require('express'); const express = require('express');
const fs = require('fs'); const fs = require('fs');
const { const {
@@ -45,6 +47,10 @@ const {
const { resolveSessionContext, touchSessionIfPresent } = require('./lib/session-context'); const { resolveSessionContext, touchSessionIfPresent } = require('./lib/session-context');
const { getProfileStats, profileExists } = require('./lib/profile-dir'); const { getProfileStats, profileExists } = require('./lib/profile-dir');
const { runUiPagination } = require('./lib/ui-pagination-runner'); const { runUiPagination } = require('./lib/ui-pagination-runner');
const {
awaitPostCfBusinessReady,
executeAuthActionsWithDiagnostics,
} = require('./lib/post-cf-ready');
const app = express(); const app = express();
app.use(express.json({ limit: '50mb' })); app.use(express.json({ limit: '50mb' }));
@@ -75,7 +81,11 @@ function sendTaskError(res, error, extra = {}) {
} }
const errType = classifyPuppeteerError(error); const errType = classifyPuppeteerError(error);
console.error(`[${errType}]`, error.message); console.error(`[${errType}]`, error.message);
return res.status(500).json({ success: false, error: error.message, ...extra }); const payload = { success: false, error: error.message, ...extra };
if (error.pageDiagnostics) {
payload.page = error.pageDiagnostics;
}
return res.status(500).json(payload);
} }
async function navigateToPage(page, url) { async function navigateToPage(page, url) {
@@ -524,7 +534,11 @@ app.post('/api/auth-and-intercept', async (req, res) => {
await seedShareTokenCookie(page, finalPageUrl); await seedShareTokenCookie(page, finalPageUrl);
} }
await executeAuthActions(page, authActions); if (antiBot.enabled) {
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled);
}
await executeAuthActionsWithDiagnostics(page, authActions, executeAuthActions);
await interceptor.waitForApis(interceptTimeout); await interceptor.waitForApis(interceptTimeout);
const rawCookies = await page.cookies(); const rawCookies = await page.cookies();
@@ -758,6 +772,10 @@ app.post('/api/ui-pagination', async (req, res) => {
touchSessionIfPresent(antiBot.sessionKey); touchSessionIfPresent(antiBot.sessionKey);
} }
if (antiBot.enabled) {
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled);
}
const paginationResult = await runUiPagination(page, { const paginationResult = await runUiPagination(page, {
apiUrl, apiUrl,
nextBtnSelector, nextBtnSelector,
@@ -874,7 +892,11 @@ app.post('/api/auth-intercept-and-paginate', async (req, res) => {
touchSessionIfPresent(antiBot.sessionKey); touchSessionIfPresent(antiBot.sessionKey);
} }
await executeAuthActions(page, authActions); if (antiBot.enabled) {
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled);
}
await executeAuthActionsWithDiagnostics(page, authActions, executeAuthActions);
await interceptor.waitForApis(interceptTimeout); await interceptor.waitForApis(interceptTimeout);
const rawCookies = await page.cookies(); const rawCookies = await page.cookies();