修改whatshub工单爬虫
This commit is contained in:
@@ -75,11 +75,23 @@ abstract class AbstractScrmSpider
|
||||
const MODE_FETCH = 'fetch'; // 极速并发拉取 (推荐默认)
|
||||
const MODE_UI = 'ui_click'; // 强制 UI 点击下一页
|
||||
|
||||
/** CLI 测试时输出 Node 请求/失败详情 */
|
||||
protected $verbose = false;
|
||||
|
||||
protected $nodeHost;
|
||||
|
||||
public function __construct($nodeHost = 'http://127.0.0.1:3001')
|
||||
{
|
||||
$this->nodeHost = $nodeHost;
|
||||
$this->nodeHost = rtrim($nodeHost, '/');
|
||||
}
|
||||
|
||||
/**
|
||||
* 开启调试输出(测试脚本可调用)
|
||||
*/
|
||||
public function setVerbose($verbose = true)
|
||||
{
|
||||
$this->verbose = (bool) $verbose;
|
||||
return $this;
|
||||
}
|
||||
|
||||
/** ================= 子类必须实现的契约 ================= **/
|
||||
@@ -106,13 +118,14 @@ abstract class AbstractScrmSpider
|
||||
$apiUrlsToIntercept = [$listApi];
|
||||
if ($detailApi) { $apiUrlsToIntercept[] = $detailApi; }
|
||||
if ($countApi) { $apiUrlsToIntercept[] = $countApi; }
|
||||
// var_dump($apiUrlsToIntercept);
|
||||
// dump([
|
||||
// 'pageUrl' => $config['pageUrl'],
|
||||
// 'apiUrls' => $apiUrlsToIntercept,
|
||||
// 'authActions' => $config['authActions']
|
||||
// ]);
|
||||
|
||||
$antiBot = $config['antiBot'] ?? null;
|
||||
$mode = $config['paginationMode'] ?? self::MODE_FETCH;
|
||||
|
||||
// antiBot + UI 翻页:单 Browser 合并路径(与生产 SplitTicket 同步一致)
|
||||
if ($this->shouldUseUnifiedBrowserPath($antiBot, $mode)) {
|
||||
return $this->runUnifiedUiPath($config, $antiBot, $apiUrlsToIntercept, $listApi, $detailApi, $countApi);
|
||||
}
|
||||
|
||||
// 【阶段一】:初始化并首屏拦截
|
||||
$initResult = $this->requestNode('/api/auth-and-intercept', [
|
||||
@@ -123,17 +136,20 @@ abstract class AbstractScrmSpider
|
||||
], $this->resolveAuthInterceptTimeout($antiBot));
|
||||
|
||||
if (empty($initResult['success'])) {
|
||||
$cfCode = (string) ($initResult['code'] ?? '');
|
||||
if ($cfCode === 'CF_TURNSTILE_FAILED') {
|
||||
throw new Exception('Cloudflare Turnstile 验证失败: ' . ($initResult['stage'] ?? 'timeout'));
|
||||
}
|
||||
throw new Exception("初始化失败: " . ($initResult['error'] ?? '未知'));
|
||||
throw new Exception($this->formatNodeFailure('初始化失败', $initResult));
|
||||
}
|
||||
|
||||
$interceptedApis = $initResult['interceptedApis'];
|
||||
$cookies = $initResult['cookies'];
|
||||
$finalPageUrl = $initResult['finalPageUrl'] ?? $config['pageUrl'];
|
||||
|
||||
$this->logDebug('auth-and-intercept ok', [
|
||||
'finalPageUrl' => $finalPageUrl,
|
||||
'intercepted' => array_keys($interceptedApis),
|
||||
'cf' => $initResult['cf'] ?? null,
|
||||
'browserReused' => $initResult['browserReused'] ?? null,
|
||||
]);
|
||||
|
||||
// 必须拦截到 List 接口,否则无法继续
|
||||
if (!isset($interceptedApis[$listApi])) {
|
||||
throw new Exception("致命错误:未能拦截到必须的列表接口 [{$listApi}]");
|
||||
@@ -229,6 +245,140 @@ abstract class AbstractScrmSpider
|
||||
return $unifiedData;
|
||||
}
|
||||
|
||||
/**
|
||||
* 单 Browser 路径:auth + 拦截 + UI 翻页(需 antiBot.sessionKey)
|
||||
*/
|
||||
protected function runUnifiedUiPath($config, $antiBot, $apiUrlsToIntercept, $listApi, $detailApi, $countApi)
|
||||
{
|
||||
$uiConfig = $this->getUiPaginationConfig();
|
||||
$sessionKey = is_array($antiBot) ? (string) ($antiBot['sessionKey'] ?? '') : '';
|
||||
|
||||
$this->logDebug('unified browser path', [
|
||||
'sessionKey' => $sessionKey,
|
||||
'listApi' => $listApi,
|
||||
]);
|
||||
|
||||
$mergedResult = $this->requestNode('/api/auth-intercept-and-paginate', [
|
||||
'pageUrl' => $config['pageUrl'],
|
||||
'apiUrls' => $apiUrlsToIntercept,
|
||||
'authActions' => $config['authActions'] ?? [],
|
||||
'antiBot' => $antiBot,
|
||||
'pagination' => [
|
||||
'apiUrl' => $listApi,
|
||||
'nextBtnSelector' => $uiConfig['nextBtnSelector'] ?? '',
|
||||
'waitMs' => $uiConfig['waitMs'] ?? 2000,
|
||||
'clicksToPerform' => 9999,
|
||||
],
|
||||
], 1200);
|
||||
|
||||
if (empty($mergedResult['success'])) {
|
||||
throw new Exception($this->formatNodeFailure('合并同步失败', $mergedResult));
|
||||
}
|
||||
|
||||
$this->logDebug('auth-intercept-and-paginate ok', [
|
||||
'intercepted' => array_keys($mergedResult['interceptedApis'] ?? []),
|
||||
'extraPages' => count($mergedResult['extraPages'] ?? []),
|
||||
'cf' => $mergedResult['cf'] ?? null,
|
||||
'browserReused' => $mergedResult['browserReused'] ?? null,
|
||||
'finalPageUrl' => $mergedResult['finalPageUrl'] ?? null,
|
||||
]);
|
||||
|
||||
$interceptedApis = $mergedResult['interceptedApis'] ?? [];
|
||||
if (!isset($interceptedApis[$listApi])) {
|
||||
throw new Exception("致命错误:未能拦截到必须的列表接口 [{$listApi}]");
|
||||
}
|
||||
|
||||
$detailData = $detailApi && isset($interceptedApis[$detailApi])
|
||||
? $interceptedApis[$detailApi]['data'] : null;
|
||||
$countData = $countApi && isset($interceptedApis[$countApi])
|
||||
? $interceptedApis[$countApi]['data'] : null;
|
||||
|
||||
$listApiNode = $interceptedApis[$listApi];
|
||||
$allListPagesData = [$listApiNode['data']];
|
||||
if (!empty($mergedResult['extraPages']) && is_array($mergedResult['extraPages'])) {
|
||||
foreach ($mergedResult['extraPages'] as $pageData) {
|
||||
$allListPagesData[] = $pageData;
|
||||
}
|
||||
}
|
||||
|
||||
$this->extractListTotalPages($listApiNode['data'], $countData);
|
||||
|
||||
return $this->parseToUnifiedData($detailData, $allListPagesData);
|
||||
}
|
||||
|
||||
/**
|
||||
* antiBot 启用且配置了 sessionKey 时,UI 翻页走合并 Browser 路径
|
||||
*/
|
||||
protected function shouldUseUnifiedBrowserPath($antiBot, $mode)
|
||||
{
|
||||
if ($mode !== self::MODE_UI) {
|
||||
return false;
|
||||
}
|
||||
if (!is_array($antiBot) || empty($antiBot['enabled'])) {
|
||||
return false;
|
||||
}
|
||||
return !empty($antiBot['sessionKey']);
|
||||
}
|
||||
|
||||
/**
|
||||
* 格式化 Node 失败信息,附带 CF / 落地页等诊断字段
|
||||
*/
|
||||
protected function formatNodeFailure($prefix, $result)
|
||||
{
|
||||
$cfCode = (string) ($result['code'] ?? '');
|
||||
if ($cfCode === 'CF_TURNSTILE_FAILED') {
|
||||
return 'Cloudflare Turnstile 验证失败: ' . ($result['stage'] ?? 'timeout');
|
||||
}
|
||||
|
||||
$parts = [$prefix . ': ' . ($result['error'] ?? '未知')];
|
||||
|
||||
if (!empty($result['finalPageUrl'])) {
|
||||
$parts[] = 'finalPageUrl=' . $result['finalPageUrl'];
|
||||
}
|
||||
if (!empty($result['page']) && is_array($result['page'])) {
|
||||
$page = $result['page'];
|
||||
if (!empty($page['url'])) {
|
||||
$parts[] = 'pageUrl=' . $page['url'];
|
||||
}
|
||||
if (isset($page['elInputInner'])) {
|
||||
$parts[] = 'elInputInner=' . $page['elInputInner'];
|
||||
}
|
||||
}
|
||||
if (!empty($result['cf']) && is_array($result['cf'])) {
|
||||
$cf = $result['cf'];
|
||||
$cfBits = [];
|
||||
if (isset($cf['cfDetected'])) {
|
||||
$cfBits[] = 'cfDetected=' . ($cf['cfDetected'] ? 'true' : 'false');
|
||||
}
|
||||
if (!empty($cf['turnstileStage'])) {
|
||||
$cfBits[] = 'turnstileStage=' . $cf['turnstileStage'];
|
||||
}
|
||||
if (isset($cf['solverUsed'])) {
|
||||
$cfBits[] = 'solverUsed=' . ($cf['solverUsed'] ? 'true' : 'false');
|
||||
}
|
||||
if (isset($cf['elapsedMs'])) {
|
||||
$cfBits[] = 'cfElapsedMs=' . $cf['elapsedMs'];
|
||||
}
|
||||
if (!empty($cfBits)) {
|
||||
$parts[] = implode(', ', $cfBits);
|
||||
}
|
||||
}
|
||||
|
||||
return implode(' | ', $parts);
|
||||
}
|
||||
|
||||
protected function logDebug($message, $context = [])
|
||||
{
|
||||
if (!$this->verbose) {
|
||||
return;
|
||||
}
|
||||
$line = '[Spider] ' . $message;
|
||||
if (!empty($context)) {
|
||||
$line .= ' ' . json_encode($context, JSON_UNESCAPED_UNICODE);
|
||||
}
|
||||
echo $line . "\n";
|
||||
}
|
||||
|
||||
/**
|
||||
* Real Browser + Turnstile 首屏耗时较长,单独放宽 auth-and-intercept 超时
|
||||
*
|
||||
@@ -244,17 +394,76 @@ abstract class AbstractScrmSpider
|
||||
|
||||
private function requestNode($endpoint, $payload, $timeout = 60)
|
||||
{
|
||||
$ch = curl_init($this->nodeHost . $endpoint);
|
||||
$maxAttempts = 3;
|
||||
$lastDecoded = [];
|
||||
for ($attempt = 1; $attempt <= $maxAttempts; $attempt++) {
|
||||
$lastDecoded = $this->doRequestNode($endpoint, $payload, $timeout, $attempt);
|
||||
if (!empty($lastDecoded['success'])) {
|
||||
return $lastDecoded;
|
||||
}
|
||||
$error = (string) ($lastDecoded['error'] ?? '');
|
||||
if (!$this->shouldRetryNodeResponse($lastDecoded, $error) || $attempt >= $maxAttempts) {
|
||||
return $lastDecoded;
|
||||
}
|
||||
$this->logDebug('node retry', [
|
||||
'endpoint' => $endpoint,
|
||||
'attempt' => $attempt + 1,
|
||||
'error' => $error,
|
||||
]);
|
||||
sleep($attempt);
|
||||
}
|
||||
return $lastDecoded;
|
||||
}
|
||||
|
||||
private function doRequestNode($endpoint, $payload, $timeout, $attempt)
|
||||
{
|
||||
$url = $this->nodeHost . $endpoint;
|
||||
$this->logDebug('node request', [
|
||||
'url' => $url,
|
||||
'timeout' => $timeout,
|
||||
'attempt' => $attempt,
|
||||
]);
|
||||
|
||||
$ch = curl_init($url);
|
||||
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
||||
curl_setopt($ch, CURLOPT_POST, true);
|
||||
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($payload));
|
||||
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($payload, JSON_UNESCAPED_UNICODE));
|
||||
curl_setopt($ch, CURLOPT_HTTPHEADER, ['Content-Type: application/json']);
|
||||
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5);
|
||||
curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
|
||||
$response = curl_exec($ch);
|
||||
if (curl_errno($ch)) throw new Exception(curl_error($ch));
|
||||
$httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
if (curl_errno($ch)) {
|
||||
$err = curl_error($ch);
|
||||
curl_close($ch);
|
||||
return json_decode($response, true);
|
||||
throw new Exception($err);
|
||||
}
|
||||
curl_close($ch);
|
||||
|
||||
$decoded = json_decode((string) $response, true);
|
||||
if (!is_array($decoded)) {
|
||||
return ['success' => false, 'error' => 'Node 返回非 JSON', 'httpCode' => $httpCode];
|
||||
}
|
||||
if ($httpCode === 503) {
|
||||
$decoded['success'] = false;
|
||||
$decoded['error'] = (string) ($decoded['error'] ?? '服务繁忙');
|
||||
$decoded['httpCode'] = 503;
|
||||
}
|
||||
return $decoded;
|
||||
}
|
||||
|
||||
private function shouldRetryNodeResponse($decoded, $error)
|
||||
{
|
||||
if ((int) ($decoded['httpCode'] ?? 0) === 503) {
|
||||
return true;
|
||||
}
|
||||
$lower = strtolower($error);
|
||||
foreach (['排队超时', 'queue_timeout', 'timeout', 'net::', 'navigation', 'connection'] as $needle) {
|
||||
if ($needle !== '' && strpos($lower, $needle) !== false) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+178
-10
@@ -7,8 +7,12 @@ class Whatshub extends AbstractScrmSpider
|
||||
{
|
||||
const API_LIST = '/api/whatshub-counter/workShare/open/detail'; // 列表API地址
|
||||
const API_DETAILS = '/api/whatshub-counter/workShare/open/statistics'; // 详情API地址
|
||||
/** shareCode + password 直连接口(一次返回全量号码,无需翻页) */
|
||||
const API_DETAIL_BY_SHARE_CODE = '/api/whatshub-counter/workShare/open/detailByShareCode';
|
||||
const API_COUNT = ''; // 总数API地址
|
||||
const DEFAULT_PER_PAGE_COUNT = 20; // List默认每页显示的数量
|
||||
const API_FETCH_TIMEOUT_SEC = 20;
|
||||
|
||||
private $pageUrl;
|
||||
private $account;
|
||||
private $password;
|
||||
@@ -25,6 +29,165 @@ class Whatshub extends AbstractScrmSpider
|
||||
$this->unifiedData = new UnifiedScrmData();
|
||||
}
|
||||
|
||||
/**
|
||||
* API 优先:detailByShareCode 直抓;失败则回退 Node 爬虫
|
||||
*/
|
||||
public function run()
|
||||
{
|
||||
$apiResult = $this->tryFetchViaShareCodeApi();
|
||||
if ($apiResult instanceof UnifiedScrmData) {
|
||||
$this->logDebug('whatshub api path ok', ['count' => count($apiResult->numbers)]);
|
||||
return $apiResult;
|
||||
}
|
||||
|
||||
$this->logDebug('whatshub api skipped or failed, fallback node', []);
|
||||
return parent::run();
|
||||
}
|
||||
|
||||
/**
|
||||
* 从短链 pageUrl 解析 shareCode,例如 /m/KMYBBInp2066/1 → KMYBBInp2066
|
||||
*
|
||||
* @return string|null
|
||||
*/
|
||||
private function extractShareCodeFromPageUrl()
|
||||
{
|
||||
$path = (string) parse_url($this->pageUrl, PHP_URL_PATH);
|
||||
if ($path === '') {
|
||||
return null;
|
||||
}
|
||||
if (preg_match('#/m/([^/]+)/#', $path, $matches)) {
|
||||
return $matches[1];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 拼接 detailByShareCode 完整 GET URL
|
||||
*
|
||||
* @param string $shareCode
|
||||
* @return string|null
|
||||
*/
|
||||
private function buildShareCodeApiUrl($shareCode)
|
||||
{
|
||||
$parsed = parse_url($this->pageUrl);
|
||||
$scheme = isset($parsed['scheme']) ? $parsed['scheme'] : 'https';
|
||||
$host = isset($parsed['host']) ? $parsed['host'] : '';
|
||||
if ($host === '') {
|
||||
return null;
|
||||
}
|
||||
|
||||
$query = http_build_query([
|
||||
'shareCode' => $shareCode,
|
||||
'password' => $this->password,
|
||||
]);
|
||||
|
||||
return $scheme . '://' . $host . self::API_DETAIL_BY_SHARE_CODE . '?' . $query;
|
||||
}
|
||||
|
||||
/**
|
||||
* 尝试通过 shareCode API 抓取;成功返回 UnifiedScrmData,否则返回 null
|
||||
*
|
||||
* @return UnifiedScrmData|null
|
||||
*/
|
||||
private function tryFetchViaShareCodeApi()
|
||||
{
|
||||
if ($this->password === '' || $this->password === null) {
|
||||
$this->logDebug('whatshub api skip: empty password', []);
|
||||
return null;
|
||||
}
|
||||
|
||||
$shareCode = $this->extractShareCodeFromPageUrl();
|
||||
if ($shareCode === null || $shareCode === '') {
|
||||
$this->logDebug('whatshub api skip: cannot parse shareCode', ['pageUrl' => $this->pageUrl]);
|
||||
return null;
|
||||
}
|
||||
|
||||
$apiUrl = $this->buildShareCodeApiUrl($shareCode);
|
||||
if ($apiUrl === null) {
|
||||
$this->logDebug('whatshub api skip: invalid pageUrl host', ['pageUrl' => $this->pageUrl]);
|
||||
return null;
|
||||
}
|
||||
|
||||
$this->logDebug('whatshub api request', ['url' => preg_replace('/password=[^&]+/', 'password=***', $apiUrl)]);
|
||||
|
||||
$ch = curl_init($apiUrl);
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_CONNECTTIMEOUT => 5,
|
||||
CURLOPT_TIMEOUT => self::API_FETCH_TIMEOUT_SEC,
|
||||
CURLOPT_HTTPHEADER => [
|
||||
'Accept: application/json',
|
||||
'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
],
|
||||
]);
|
||||
|
||||
$response = curl_exec($ch);
|
||||
$httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
$curlErr = curl_error($ch);
|
||||
curl_close($ch);
|
||||
|
||||
if ($response === false || $curlErr !== '') {
|
||||
$this->logDebug('whatshub api curl failed', ['error' => $curlErr]);
|
||||
return null;
|
||||
}
|
||||
|
||||
if ($httpCode !== 200) {
|
||||
$this->logDebug('whatshub api http error', ['httpCode' => $httpCode]);
|
||||
return null;
|
||||
}
|
||||
|
||||
$payload = json_decode($response, true);
|
||||
if (!is_array($payload)) {
|
||||
$this->logDebug('whatshub api invalid json', []);
|
||||
return null;
|
||||
}
|
||||
|
||||
if ((int) ($payload['code'] ?? 0) !== 200) {
|
||||
$this->logDebug('whatshub api business error', [
|
||||
'code' => $payload['code'] ?? null,
|
||||
'msg' => $payload['msg'] ?? '',
|
||||
]);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!isset($payload['data']) || !is_array($payload['data'])) {
|
||||
$this->logDebug('whatshub api invalid data field', []);
|
||||
return null;
|
||||
}
|
||||
|
||||
return $this->parseShareCodeApiResponse($payload);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 detailByShareCode 响应映射为 UnifiedScrmData
|
||||
*
|
||||
* @param array<string, mixed> $payload
|
||||
* @return UnifiedScrmData
|
||||
*/
|
||||
private function parseShareCodeApiResponse(array $payload)
|
||||
{
|
||||
$unifiedData = new UnifiedScrmData();
|
||||
$todayNewSum = 0;
|
||||
|
||||
foreach ($payload['data'] as $item) {
|
||||
if (!is_array($item) || empty($item['account'])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$number = (string) $item['account'];
|
||||
$isOnline = isset($item['isOnline']) && (int) $item['isOnline'] === 1;
|
||||
$dayNewFans = (int) ($item['dayNewFans'] ?? 0);
|
||||
|
||||
$unifiedData->addNumber($number, $isOnline, $dayNewFans);
|
||||
$todayNewSum += $dayNewFans;
|
||||
}
|
||||
|
||||
$unifiedData->todayNewCount = $todayNewSum;
|
||||
$unifiedData->total = count($unifiedData->numbers);
|
||||
|
||||
return $unifiedData;
|
||||
}
|
||||
|
||||
protected function getSpiderConfig()
|
||||
{
|
||||
$host = (string) parse_url($this->pageUrl, PHP_URL_HOST);
|
||||
@@ -33,14 +196,14 @@ class Whatshub extends AbstractScrmSpider
|
||||
'apiUrls' => [self::API_LIST, self::API_DETAILS],
|
||||
|
||||
// 明确指派角色
|
||||
'listApi' => self::API_LIST, // 必须
|
||||
'listApi' => self::API_LIST,
|
||||
'detailApi' => self::API_DETAILS,
|
||||
|
||||
'listMethod' => 'POST',
|
||||
'paginationMode' => self::MODE_UI,
|
||||
|
||||
'authActions' => [
|
||||
// 1. 填入密码:寻找 class 包含 el-message-box__input 下面的任意 input
|
||||
// Node.js 会自动扫描所有匹配项,并只把密码强行注入到那个“肉眼可见”的框里
|
||||
// Node antiBot.postCfReady* 会在 CF 成功后等待 work-order-sharing + .el-input__inner
|
||||
['type' => 'vue_fill', 'selector' => '.el-input__inner', 'value' => $this->password],
|
||||
|
||||
// 2. 停顿 500ms,让 Vue 绑定的 v-model 彻底反应过来
|
||||
@@ -51,16 +214,21 @@ class Whatshub extends AbstractScrmSpider
|
||||
['type' => 'vue_click', 'selector' => '.vxe-button-group .theme--primary'],
|
||||
|
||||
// 4. 等待弹窗淡出,接口开始请求
|
||||
['type' => 'wait', 'ms' => 2200]
|
||||
['type' => 'wait', 'ms' => 2200],
|
||||
],
|
||||
// Whatshub 专用:Real Browser + Turnstile + Captcha API 兜底 + 会话复用
|
||||
// sessionKey 必须与 pageUrl 的 host 一致,例如 web.whatshub.cc
|
||||
'antiBot' => [
|
||||
'enabled' => true,
|
||||
'profile' => 'real',
|
||||
'turnstile' => true,
|
||||
'solverFallback' => true,
|
||||
'sessionKey' => 'whatshub:' . $host,
|
||||
'challengeTimeoutMs' => 60000,
|
||||
'challengeTimeoutMs' => 40000,
|
||||
'postCfReadyUrlContains' => 'work-order-sharing',
|
||||
'postCfReadySelector' => '.el-input__inner',
|
||||
'postCfReadyTimeoutMs' => 30000,
|
||||
'postCfSettleMs' => 500,
|
||||
'postCfSpaWaitMs' => 3000,
|
||||
],
|
||||
];
|
||||
}
|
||||
@@ -69,7 +237,7 @@ class Whatshub extends AbstractScrmSpider
|
||||
protected function extractListTotalPages($listFirstPageData, $countData = null)
|
||||
{
|
||||
$default_per_page_count = self::DEFAULT_PER_PAGE_COUNT;
|
||||
$this->unifiedData->total = $listFirstPageData['data']['total'];
|
||||
$this->unifiedData->total = $listFirstPageData['total'];
|
||||
|
||||
if ($this->unifiedData->total <= $default_per_page_count) {
|
||||
return 1;
|
||||
@@ -89,7 +257,7 @@ class Whatshub extends AbstractScrmSpider
|
||||
{
|
||||
return [
|
||||
'nextBtnSelector' => '.vxe-pager--next-btn',
|
||||
'waitMs' => 1500
|
||||
'waitMs' => 1500,
|
||||
];
|
||||
}
|
||||
|
||||
@@ -105,7 +273,7 @@ class Whatshub extends AbstractScrmSpider
|
||||
|
||||
// 2. 循环合并了所有页数的 List 数组
|
||||
foreach ($allListPagesData as $pageRaw) {
|
||||
$records = $pageRaw['data']['rows'] ?? [];
|
||||
$records = $pageRaw['rows'] ?? [];
|
||||
foreach ($records as $item) {
|
||||
if (!empty($item['account'])) {
|
||||
$number = $item['account'] ?? null;
|
||||
@@ -116,7 +284,7 @@ class Whatshub extends AbstractScrmSpider
|
||||
}
|
||||
}
|
||||
|
||||
// 🚀 3. 终极统计:所有翻页数据均已入库,此时再统计真实的 Total 总数
|
||||
// 3. 终极统计:所有翻页数据均已入库,此时再统计真实的 Total 总数
|
||||
$unifiedData->total = count($unifiedData->numbers);
|
||||
|
||||
return $unifiedData;
|
||||
|
||||
+63
-61
@@ -11,50 +11,52 @@ require_once __DIR__ . '/SsCustomer.php'; // SS云控(Customer)
|
||||
require_once __DIR__ . '/Haiwang.php'; // 海王
|
||||
require_once __DIR__ . '/Whatshub.php'; // Whatshub 工单平台
|
||||
|
||||
// try {
|
||||
// /*
|
||||
// 命令行可以测试Cloudflare防火墙
|
||||
// curl -s -X POST http://127.0.0.1:3001/api/auth-and-intercept \
|
||||
// -H 'Content-Type: application/json' \
|
||||
// -d '{
|
||||
// "pageUrl": "https://web.whatshub.cc/m/iTYsWKQH5030/1",
|
||||
// "apiUrls": ["/api/whatshub-counter/workShare/open/detail"],
|
||||
// "authActions": [
|
||||
// {"type": "vue_fill", "selector": ".el-input__inner", "value": "745030"},
|
||||
// {"type": "wait", "ms": 500},
|
||||
// {"type": "vue_click", "selector": ".vxe-button-group .theme--primary"},
|
||||
// {"type": "wait", "ms": 2200}
|
||||
// ],
|
||||
// "antiBot": {
|
||||
// "enabled": true,
|
||||
// "profile": "real",
|
||||
// "turnstile": true,
|
||||
// "solverFallback": true,
|
||||
// "sessionKey": "whatshub:t.flowerbells.top",
|
||||
// "challengeTimeoutMs": 60000
|
||||
// }
|
||||
// }' | jq .
|
||||
// */
|
||||
try {
|
||||
/*
|
||||
|
||||
// echo "🚀 开始执行<Whatshub 工单平台>抓取任务 (多引擎智能调度)...\n\r";
|
||||
// $pageUrl = 'https://web.whatshub.cc/m/iTYsWKQH5030/1'; // PageUrl 入口授权页
|
||||
// $username = ""; // 登录账号
|
||||
// $password = "745030"; // 登录密码
|
||||
// $spider = new Whatshub($pageUrl, $username, $password);
|
||||
// $finalData = $spider->run();
|
||||
命令行可以测试Cloudflare防火墙
|
||||
curl -s -X POST http://127.0.0.1:3001/api/auth-and-intercept \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"pageUrl": "https://web.whatshub.cc/m/KMYBBInp2066/1",
|
||||
"apiUrls": ["/api/whatshub-counter/workShare/open/detail"],
|
||||
"authActions": [
|
||||
{"type": "wait", "ms": 1000},
|
||||
{"type": "vue_fill", "selector": ".el-input__wrapper > .el-input__inner", "value": "602066"},
|
||||
{"type": "wait", "ms": 500},
|
||||
{"type": "vue_click", "selector": ".vxe-button-group .theme--primary"},
|
||||
{"type": "wait", "ms": 2200}
|
||||
],
|
||||
"antiBot": {
|
||||
"enabled": true,
|
||||
"profile": "real",
|
||||
"turnstile": true,
|
||||
"solverFallback": true,
|
||||
"sessionKey": "whatshub:web.whatshub.cc",
|
||||
"challengeTimeoutMs": 60000
|
||||
}
|
||||
}' | jq .
|
||||
*/
|
||||
|
||||
// echo "✅ 任务完成!统一数据如下:\n\r";
|
||||
// echo "----------------------------------------\n\r";
|
||||
// echo "当日新增:{$finalData->todayNewCount} 人\n\r";
|
||||
// echo "在线号码:{$finalData->totalOnline} 个\n\r";
|
||||
// echo "离线号码:{$finalData->totalOffline} 个\n\r";
|
||||
// echo "Total:" . $finalData->total . " 个号码\n\r";
|
||||
// echo "实际总共抓取:" . count($finalData->numbers) . " 个号码\n\r";
|
||||
// echo "号码列表:\n\r";
|
||||
// echo dd($finalData->numbers);
|
||||
// } catch (Exception $e) {
|
||||
// echo "🚨 抓取异常:" . $e->getMessage() . "\n\r";
|
||||
// }
|
||||
echo "🚀 开始执行<Whatshub 工单平台>抓取任务 (多引擎智能调度)...\n\r";
|
||||
$pageUrl = 'https://web.whatshub.cc/m/KMYBBInp2066/1'; // PageUrl 入口授权页
|
||||
$username = ""; // 登录账号
|
||||
$password = "602066"; // 登录密码
|
||||
$spider = new Whatshub($pageUrl, $username, $password);
|
||||
$finalData = $spider->run();
|
||||
|
||||
echo "✅ 任务完成!统一数据如下:\n\r";
|
||||
echo "----------------------------------------\n\r";
|
||||
echo "当日新增:{$finalData->todayNewCount} 人\n\r";
|
||||
echo "在线号码:{$finalData->totalOnline} 个\n\r";
|
||||
echo "离线号码:{$finalData->totalOffline} 个\n\r";
|
||||
echo "Total:" . $finalData->total . " 个号码\n\r";
|
||||
echo "实际总共抓取:" . count($finalData->numbers) . " 个号码\n\r";
|
||||
echo "号码列表:\n\r";
|
||||
echo dd($finalData->numbers);
|
||||
} catch (Exception $e) {
|
||||
echo "🚨 抓取异常:" . $e->getMessage() . "\n\r";
|
||||
}
|
||||
|
||||
|
||||
// try {
|
||||
@@ -78,27 +80,27 @@ require_once __DIR__ . '/Whatshub.php'; // Whatshub 工单平台
|
||||
// echo "🚨 抓取异常:" . $e->getMessage() . "\n\r";
|
||||
// }
|
||||
|
||||
try {
|
||||
echo "🚀 开始执行<A2c云控>抓取任务 (多引擎智能调度)...\n\r";
|
||||
$pageUrl = 'https://yyk.ink/52oXZRG'; // PageUrl 入口授权页
|
||||
$username = ""; // 登录账号
|
||||
$password = ""; // 登录密码
|
||||
$spider = new A2c($pageUrl, $username, $password);
|
||||
$finalData = $spider->run();
|
||||
// try {
|
||||
// echo "🚀 开始执行<A2c云控>抓取任务 (多引擎智能调度)...\n\r";
|
||||
// $pageUrl = 'https://yyk.ink/52oXZRG'; // PageUrl 入口授权页
|
||||
// $username = ""; // 登录账号
|
||||
// $password = ""; // 登录密码
|
||||
// $spider = new A2c($pageUrl, $username, $password);
|
||||
// $finalData = $spider->run();
|
||||
|
||||
echo "✅ 任务完成!统一数据如下:\n\r";
|
||||
echo "----------------------------------------\n\r";
|
||||
echo "当日新增:{$finalData->todayNewCount} 人\n\r";
|
||||
echo "在线号码:{$finalData->totalOnline} 个\n\r";
|
||||
echo "离线号码:{$finalData->totalOffline} 个\n\r";
|
||||
echo "Total:" . $finalData->total . " 个号码\n\r";
|
||||
echo "实际总共抓取:" . count($finalData->numbers) . " 个号码\n\r";
|
||||
echo "号码列表:\n\r";
|
||||
echo dd($finalData->numbers);
|
||||
// echo "✅ 任务完成!统一数据如下:\n\r";
|
||||
// echo "----------------------------------------\n\r";
|
||||
// echo "当日新增:{$finalData->todayNewCount} 人\n\r";
|
||||
// echo "在线号码:{$finalData->totalOnline} 个\n\r";
|
||||
// echo "离线号码:{$finalData->totalOffline} 个\n\r";
|
||||
// echo "Total:" . $finalData->total . " 个号码\n\r";
|
||||
// echo "实际总共抓取:" . count($finalData->numbers) . " 个号码\n\r";
|
||||
// echo "号码列表:\n\r";
|
||||
// echo dd($finalData->numbers);
|
||||
|
||||
} catch (Exception $e) {
|
||||
echo "🚨 抓取异常:" . $e->getMessage() . "\n\r";
|
||||
}
|
||||
// } catch (Exception $e) {
|
||||
// echo "🚨 抓取异常:" . $e->getMessage() . "\n\r";
|
||||
// }
|
||||
|
||||
// try {
|
||||
// echo "🚀 开始执行<星河云控>抓取任务 (多引擎智能调度)...\n\r";
|
||||
|
||||
@@ -41,6 +41,26 @@ class AntiBotConfigBuilder
|
||||
'sessionKey' => $sessionKey,
|
||||
'challengeTimeoutMs' => 60000,
|
||||
'sessionTtlMinutes' => $ttlMinutes,
|
||||
] + self::postCfReadyExtras($ticketType);
|
||||
}
|
||||
|
||||
/**
|
||||
* 按工单类型附加 CF 过盾后的业务页就绪等待(仅 Whatshub 等需要的类型)
|
||||
*
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
private static function postCfReadyExtras(string $ticketType): array
|
||||
{
|
||||
if ($ticketType !== 'whatshub') {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [
|
||||
'postCfReadyUrlContains' => 'work-order-sharing',
|
||||
'postCfReadySelector' => '.el-input__inner',
|
||||
'postCfReadyTimeoutMs' => 30000,
|
||||
'postCfSettleMs' => 500,
|
||||
'postCfSpaWaitMs' => 4000,
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
@@ -7,11 +7,12 @@ namespace app\common\library\scrm\spider;
|
||||
use app\common\library\scrm\AbstractScrmSpider;
|
||||
use app\common\library\scrm\AntiBotConfigBuilder;
|
||||
use app\common\library\scrm\UnifiedScrmData;
|
||||
use app\common\service\SplitTicketSyncLogger;
|
||||
|
||||
/**
|
||||
* Whatshub 云控蜘蛛(Real Browser + Turnstile 过盾 + UI 翻页)
|
||||
* Whatshub 云控蜘蛛
|
||||
*
|
||||
* 时序:Cloudflare Turnstile → 密码弹窗 authActions → API 拦截 → UI 翻页
|
||||
* 时序:detailByShareCode API 直抓(优先)→ 失败则 Real Browser + Turnstile + UI 翻页
|
||||
*/
|
||||
class WhatshubSpider extends AbstractScrmSpider
|
||||
{
|
||||
@@ -21,9 +22,14 @@ class WhatshubSpider extends AbstractScrmSpider
|
||||
/** 详情/统计 API 路径 */
|
||||
private const API_DETAILS = '/api/whatshub-counter/workShare/open/statistics';
|
||||
|
||||
/** shareCode + password 直连接口(一次返回全量号码,无需翻页) */
|
||||
private const API_DETAIL_BY_SHARE_CODE = '/api/whatshub-counter/workShare/open/detailByShareCode';
|
||||
|
||||
/** 默认每页条数 */
|
||||
private const DEFAULT_PER_PAGE_COUNT = 20;
|
||||
|
||||
private const API_FETCH_TIMEOUT_SEC = 20;
|
||||
|
||||
private string $pageUrl;
|
||||
|
||||
private string $account;
|
||||
@@ -45,6 +51,178 @@ class WhatshubSpider extends AbstractScrmSpider
|
||||
$this->unifiedData = new UnifiedScrmData();
|
||||
}
|
||||
|
||||
/**
|
||||
* API 优先:detailByShareCode 直抓;失败则回退 Node 爬虫
|
||||
*/
|
||||
public function run(): UnifiedScrmData
|
||||
{
|
||||
$apiResult = $this->tryFetchViaShareCodeApi();
|
||||
if ($apiResult instanceof UnifiedScrmData) {
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api path ok', [
|
||||
'count' => count($apiResult->numbers),
|
||||
]);
|
||||
return $apiResult;
|
||||
}
|
||||
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api skipped or failed, fallback node', [
|
||||
'pageUrl' => $this->pageUrl,
|
||||
]);
|
||||
|
||||
return parent::run();
|
||||
}
|
||||
|
||||
/**
|
||||
* 从短链 pageUrl 解析 shareCode,例如 /m/KMYBBInp2066/1 → KMYBBInp2066
|
||||
*/
|
||||
private function extractShareCodeFromPageUrl(): ?string
|
||||
{
|
||||
$path = (string) parse_url($this->pageUrl, PHP_URL_PATH);
|
||||
if ($path === '') {
|
||||
return null;
|
||||
}
|
||||
if (preg_match('#/m/([^/]+)/#', $path, $matches)) {
|
||||
return $matches[1];
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 拼接 detailByShareCode 完整 GET URL
|
||||
*/
|
||||
private function buildShareCodeApiUrl(string $shareCode): ?string
|
||||
{
|
||||
$parsed = parse_url($this->pageUrl);
|
||||
$scheme = isset($parsed['scheme']) ? (string) $parsed['scheme'] : 'https';
|
||||
$host = isset($parsed['host']) ? (string) $parsed['host'] : '';
|
||||
if ($host === '') {
|
||||
return null;
|
||||
}
|
||||
|
||||
$query = http_build_query([
|
||||
'shareCode' => $shareCode,
|
||||
'password' => $this->password,
|
||||
]);
|
||||
|
||||
return $scheme . '://' . $host . self::API_DETAIL_BY_SHARE_CODE . '?' . $query;
|
||||
}
|
||||
|
||||
/**
|
||||
* 尝试通过 shareCode API 抓取;成功返回 UnifiedScrmData,否则返回 null
|
||||
*/
|
||||
private function tryFetchViaShareCodeApi(): ?UnifiedScrmData
|
||||
{
|
||||
if ($this->password === '') {
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api skip: empty password', []);
|
||||
return null;
|
||||
}
|
||||
|
||||
$shareCode = $this->extractShareCodeFromPageUrl();
|
||||
if ($shareCode === null || $shareCode === '') {
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api skip: cannot parse shareCode', [
|
||||
'pageUrl' => $this->pageUrl,
|
||||
]);
|
||||
return null;
|
||||
}
|
||||
|
||||
$apiUrl = $this->buildShareCodeApiUrl($shareCode);
|
||||
if ($apiUrl === null) {
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api skip: invalid pageUrl host', [
|
||||
'pageUrl' => $this->pageUrl,
|
||||
]);
|
||||
return null;
|
||||
}
|
||||
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api request', [
|
||||
'url' => preg_replace('/password=[^&]+/', 'password=***', $apiUrl),
|
||||
]);
|
||||
|
||||
$ch = curl_init($apiUrl);
|
||||
if ($ch === false) {
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api curl init failed', []);
|
||||
return null;
|
||||
}
|
||||
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_CONNECTTIMEOUT => 5,
|
||||
CURLOPT_TIMEOUT => self::API_FETCH_TIMEOUT_SEC,
|
||||
CURLOPT_HTTPHEADER => [
|
||||
'Accept: application/json',
|
||||
'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
],
|
||||
]);
|
||||
|
||||
$response = curl_exec($ch);
|
||||
$httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
$curlErr = curl_error($ch);
|
||||
curl_close($ch);
|
||||
|
||||
if ($response === false || $curlErr !== '') {
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api curl failed', [
|
||||
'error' => $curlErr,
|
||||
]);
|
||||
return null;
|
||||
}
|
||||
|
||||
if ($httpCode !== 200) {
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api http error', [
|
||||
'httpCode' => $httpCode,
|
||||
]);
|
||||
return null;
|
||||
}
|
||||
|
||||
$payload = json_decode((string) $response, true);
|
||||
if (!is_array($payload)) {
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api invalid json', []);
|
||||
return null;
|
||||
}
|
||||
|
||||
if ((int) ($payload['code'] ?? 0) !== 200) {
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api business error', [
|
||||
'code' => $payload['code'] ?? null,
|
||||
'msg' => $payload['msg'] ?? '',
|
||||
]);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!isset($payload['data']) || !is_array($payload['data'])) {
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api invalid data field', []);
|
||||
return null;
|
||||
}
|
||||
|
||||
return $this->parseShareCodeApiResponse($payload);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 detailByShareCode 响应映射为 UnifiedScrmData
|
||||
*
|
||||
* @param array<string, mixed> $payload
|
||||
*/
|
||||
private function parseShareCodeApiResponse(array $payload): UnifiedScrmData
|
||||
{
|
||||
$unifiedData = new UnifiedScrmData();
|
||||
$todayNewSum = 0;
|
||||
|
||||
foreach ($payload['data'] as $item) {
|
||||
if (!is_array($item) || empty($item['account'])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$number = (string) $item['account'];
|
||||
$isOnline = isset($item['isOnline']) && (int) $item['isOnline'] === 1;
|
||||
$dayNewFans = (int) ($item['dayNewFans'] ?? 0);
|
||||
|
||||
$unifiedData->addNumber($number, $isOnline, $dayNewFans);
|
||||
$todayNewSum += $dayNewFans;
|
||||
}
|
||||
|
||||
$unifiedData->todayNewCount = $todayNewSum;
|
||||
$unifiedData->total = count($unifiedData->numbers);
|
||||
|
||||
return $unifiedData;
|
||||
}
|
||||
|
||||
/** @return array<string, mixed> */
|
||||
protected function getSpiderConfig(): array
|
||||
{
|
||||
|
||||
@@ -41,6 +41,26 @@ class AntiBotConfigBuilder
|
||||
'sessionKey' => $sessionKey,
|
||||
'challengeTimeoutMs' => 60000,
|
||||
'sessionTtlMinutes' => $ttlMinutes,
|
||||
] + self::postCfReadyExtras($ticketType);
|
||||
}
|
||||
|
||||
/**
|
||||
* 按工单类型附加 CF 过盾后的业务页就绪等待(仅 Whatshub 等需要的类型)
|
||||
*
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
private static function postCfReadyExtras(string $ticketType): array
|
||||
{
|
||||
if ($ticketType !== 'whatshub') {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [
|
||||
'postCfReadyUrlContains' => 'work-order-sharing',
|
||||
'postCfReadySelector' => '.el-input__inner',
|
||||
'postCfReadyTimeoutMs' => 30000,
|
||||
'postCfSettleMs' => 500,
|
||||
'postCfSpaWaitMs' => 8000,
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
@@ -7,11 +7,12 @@ namespace app\common\library\scrm\spider;
|
||||
use app\common\library\scrm\AbstractScrmSpider;
|
||||
use app\common\library\scrm\AntiBotConfigBuilder;
|
||||
use app\common\library\scrm\UnifiedScrmData;
|
||||
use app\common\service\SplitTicketSyncLogger;
|
||||
|
||||
/**
|
||||
* Whatshub 云控蜘蛛(Real Browser + Turnstile 过盾 + UI 翻页)
|
||||
* Whatshub 云控蜘蛛
|
||||
*
|
||||
* 时序:Cloudflare Turnstile → 密码弹窗 authActions → API 拦截 → UI 翻页
|
||||
* 时序:detailByShareCode API 直抓(优先)→ 失败则 Real Browser + Turnstile + UI 翻页
|
||||
*/
|
||||
class WhatshubSpider extends AbstractScrmSpider
|
||||
{
|
||||
@@ -21,9 +22,14 @@ class WhatshubSpider extends AbstractScrmSpider
|
||||
/** 详情/统计 API 路径 */
|
||||
private const API_DETAILS = '/api/whatshub-counter/workShare/open/statistics';
|
||||
|
||||
/** shareCode + password 直连接口(一次返回全量号码,无需翻页) */
|
||||
private const API_DETAIL_BY_SHARE_CODE = '/api/whatshub-counter/workShare/open/detailByShareCode';
|
||||
|
||||
/** 默认每页条数 */
|
||||
private const DEFAULT_PER_PAGE_COUNT = 20;
|
||||
|
||||
private const API_FETCH_TIMEOUT_SEC = 20;
|
||||
|
||||
private string $pageUrl;
|
||||
|
||||
private string $account;
|
||||
@@ -45,6 +51,178 @@ class WhatshubSpider extends AbstractScrmSpider
|
||||
$this->unifiedData = new UnifiedScrmData();
|
||||
}
|
||||
|
||||
/**
|
||||
* API 优先:detailByShareCode 直抓;失败则回退 Node 爬虫
|
||||
*/
|
||||
public function run(): UnifiedScrmData
|
||||
{
|
||||
$apiResult = $this->tryFetchViaShareCodeApi();
|
||||
if ($apiResult instanceof UnifiedScrmData) {
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api path ok', [
|
||||
'count' => count($apiResult->numbers),
|
||||
]);
|
||||
return $apiResult;
|
||||
}
|
||||
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api skipped or failed, fallback node', [
|
||||
'pageUrl' => $this->pageUrl,
|
||||
]);
|
||||
|
||||
return parent::run();
|
||||
}
|
||||
|
||||
/**
|
||||
* 从短链 pageUrl 解析 shareCode,例如 /m/KMYBBInp2066/1 → KMYBBInp2066
|
||||
*/
|
||||
private function extractShareCodeFromPageUrl(): ?string
|
||||
{
|
||||
$path = (string) parse_url($this->pageUrl, PHP_URL_PATH);
|
||||
if ($path === '') {
|
||||
return null;
|
||||
}
|
||||
if (preg_match('#/m/([^/]+)/#', $path, $matches)) {
|
||||
return $matches[1];
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 拼接 detailByShareCode 完整 GET URL
|
||||
*/
|
||||
private function buildShareCodeApiUrl(string $shareCode): ?string
|
||||
{
|
||||
$parsed = parse_url($this->pageUrl);
|
||||
$scheme = isset($parsed['scheme']) ? (string) $parsed['scheme'] : 'https';
|
||||
$host = isset($parsed['host']) ? (string) $parsed['host'] : '';
|
||||
if ($host === '') {
|
||||
return null;
|
||||
}
|
||||
|
||||
$query = http_build_query([
|
||||
'shareCode' => $shareCode,
|
||||
'password' => $this->password,
|
||||
]);
|
||||
|
||||
return $scheme . '://' . $host . self::API_DETAIL_BY_SHARE_CODE . '?' . $query;
|
||||
}
|
||||
|
||||
/**
|
||||
* 尝试通过 shareCode API 抓取;成功返回 UnifiedScrmData,否则返回 null
|
||||
*/
|
||||
private function tryFetchViaShareCodeApi(): ?UnifiedScrmData
|
||||
{
|
||||
if ($this->password === '') {
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api skip: empty password', []);
|
||||
return null;
|
||||
}
|
||||
|
||||
$shareCode = $this->extractShareCodeFromPageUrl();
|
||||
if ($shareCode === null || $shareCode === '') {
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api skip: cannot parse shareCode', [
|
||||
'pageUrl' => $this->pageUrl,
|
||||
]);
|
||||
return null;
|
||||
}
|
||||
|
||||
$apiUrl = $this->buildShareCodeApiUrl($shareCode);
|
||||
if ($apiUrl === null) {
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api skip: invalid pageUrl host', [
|
||||
'pageUrl' => $this->pageUrl,
|
||||
]);
|
||||
return null;
|
||||
}
|
||||
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api request', [
|
||||
'url' => preg_replace('/password=[^&]+/', 'password=***', $apiUrl),
|
||||
]);
|
||||
|
||||
$ch = curl_init($apiUrl);
|
||||
if ($ch === false) {
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api curl init failed', []);
|
||||
return null;
|
||||
}
|
||||
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_CONNECTTIMEOUT => 5,
|
||||
CURLOPT_TIMEOUT => self::API_FETCH_TIMEOUT_SEC,
|
||||
CURLOPT_HTTPHEADER => [
|
||||
'Accept: application/json',
|
||||
'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
],
|
||||
]);
|
||||
|
||||
$response = curl_exec($ch);
|
||||
$httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
$curlErr = curl_error($ch);
|
||||
curl_close($ch);
|
||||
|
||||
if ($response === false || $curlErr !== '') {
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api curl failed', [
|
||||
'error' => $curlErr,
|
||||
]);
|
||||
return null;
|
||||
}
|
||||
|
||||
if ($httpCode !== 200) {
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api http error', [
|
||||
'httpCode' => $httpCode,
|
||||
]);
|
||||
return null;
|
||||
}
|
||||
|
||||
$payload = json_decode((string) $response, true);
|
||||
if (!is_array($payload)) {
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api invalid json', []);
|
||||
return null;
|
||||
}
|
||||
|
||||
if ((int) ($payload['code'] ?? 0) !== 200) {
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api business error', [
|
||||
'code' => $payload['code'] ?? null,
|
||||
'msg' => $payload['msg'] ?? '',
|
||||
]);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!isset($payload['data']) || !is_array($payload['data'])) {
|
||||
SplitTicketSyncLogger::log('spider', 'whatshub api invalid data field', []);
|
||||
return null;
|
||||
}
|
||||
|
||||
return $this->parseShareCodeApiResponse($payload);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 detailByShareCode 响应映射为 UnifiedScrmData
|
||||
*
|
||||
* @param array<string, mixed> $payload
|
||||
*/
|
||||
private function parseShareCodeApiResponse(array $payload): UnifiedScrmData
|
||||
{
|
||||
$unifiedData = new UnifiedScrmData();
|
||||
$todayNewSum = 0;
|
||||
|
||||
foreach ($payload['data'] as $item) {
|
||||
if (!is_array($item) || empty($item['account'])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$number = (string) $item['account'];
|
||||
$isOnline = isset($item['isOnline']) && (int) $item['isOnline'] === 1;
|
||||
$dayNewFans = (int) ($item['dayNewFans'] ?? 0);
|
||||
|
||||
$unifiedData->addNumber($number, $isOnline, $dayNewFans);
|
||||
$todayNewSum += $dayNewFans;
|
||||
}
|
||||
|
||||
$unifiedData->todayNewCount = $todayNewSum;
|
||||
$unifiedData->total = count($unifiedData->numbers);
|
||||
|
||||
return $unifiedData;
|
||||
}
|
||||
|
||||
/** @return array<string, mixed> */
|
||||
protected function getSpiderConfig(): array
|
||||
{
|
||||
|
||||
@@ -28,6 +28,10 @@ function normalizeAntiBot(antiBot) {
|
||||
solverFallback: antiBot.solverFallback !== false,
|
||||
sessionKey: antiBot.sessionKey || '',
|
||||
challengeTimeoutMs: antiBot.challengeTimeoutMs || 60000,
|
||||
postCfReadyUrlContains: antiBot.postCfReadyUrlContains || '',
|
||||
postCfReadySelector: antiBot.postCfReadySelector || '',
|
||||
postCfReadyTimeoutMs: antiBot.postCfReadyTimeoutMs || 0,
|
||||
postCfSettleMs: antiBot.postCfSettleMs || 0,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -106,8 +110,10 @@ async function createManagedPage(browser, existingPage, options = {}) {
|
||||
page.setDefaultNavigationTimeout(NAVIGATION_TIMEOUT_MS);
|
||||
page.setDefaultTimeout(PAGE_DEFAULT_TIMEOUT_MS);
|
||||
|
||||
if (!options.skipUserAgent) {
|
||||
const userAgent = options.userAgent || DEFAULT_UA;
|
||||
await page.setUserAgent(userAgent);
|
||||
}
|
||||
|
||||
if (options.viewport) await page.setViewport(options.viewport);
|
||||
if (options.cookies && options.cookies.length > 0) {
|
||||
|
||||
@@ -2,6 +2,21 @@
|
||||
* Cloudflare / Turnstile 挑战页检测
|
||||
*/
|
||||
|
||||
/**
|
||||
* URL 是否处于 CF Managed Challenge 中间态(Whatshub 短链常见 __cf_chl_rt_tk)
|
||||
* @param {string} url
|
||||
*/
|
||||
function urlIndicatesCfChallenge(url) {
|
||||
if (!url || typeof url !== 'string') {
|
||||
return false;
|
||||
}
|
||||
return url.includes('__cf_chl_rt_tk')
|
||||
|| url.includes('__cf_chl_tk')
|
||||
|| url.includes('__cf_chl_f_tk')
|
||||
|| url.includes('/cdn-cgi/challenge-platform')
|
||||
|| url.includes('/cdn-cgi/challenge');
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef {Object} CloudflareState
|
||||
* @property {boolean} blocked 是否处于 CF 挑战中
|
||||
@@ -44,10 +59,10 @@ async function detectCloudflare(page) {
|
||||
bodyHasJustAMoment: false,
|
||||
}));
|
||||
|
||||
const urlChallenge = url.includes('/cdn-cgi/challenge-platform') || url.includes('/cdn-cgi/challenge');
|
||||
const urlChallenge = urlIndicatesCfChallenge(url);
|
||||
|
||||
let challengeType = null;
|
||||
if (domSignals.hasTurnstileInput || domSignals.hasTurnstileWidget) {
|
||||
if (domSignals.hasTurnstileInput || domSignals.hasTurnstileWidget || url.includes('__cf_chl')) {
|
||||
challengeType = 'turnstile';
|
||||
} else if (urlChallenge || domSignals.titleHasJustAMoment || domSignals.bodyHasJustAMoment) {
|
||||
challengeType = 'js_challenge';
|
||||
@@ -94,4 +109,5 @@ async function isTurnstileSolved(page) {
|
||||
module.exports = {
|
||||
detectCloudflare,
|
||||
isTurnstileSolved,
|
||||
urlIndicatesCfChallenge,
|
||||
};
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
/**
|
||||
* Cloudflare 挑战统一处理:检测 → 会话快速通道 → 内置等待 → Captcha API 兜底
|
||||
*
|
||||
* 当 antiBot.postCfReadyUrlContains 已配置时,session_reuse 仅在业务 URL 已到达时才走快速通道;
|
||||
* 否则清除可能过期的 cf_clearance 并重新触发 Turnstile(Whatshub 短链 → work-order-sharing 场景)。
|
||||
*/
|
||||
const { detectCloudflare } = require('./cf-detector');
|
||||
const { waitForTurnstile, extractTurnstileSitekey, injectTurnstileToken } = require('./turnstile-handler');
|
||||
@@ -7,8 +10,135 @@ const { solveTurnstile, isCaptchaConfigured } = require('./captcha-solver');
|
||||
const { isSessionLikelyValid } = require('./session-store');
|
||||
|
||||
/**
|
||||
* 是否配置了 CF 后必须到达的业务 URL
|
||||
* @param {object} antiBot
|
||||
*/
|
||||
function needsPostCfBusinessUrl(antiBot) {
|
||||
return !!(antiBot?.postCfReadyUrlContains || '').trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* 当前页面 URL 是否已包含业务路径片段
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @param {{ turnstile?: boolean, solverFallback?: boolean, challengeTimeoutMs?: number }} antiBot
|
||||
* @param {object} antiBot
|
||||
*/
|
||||
function isPostCfBusinessUrlReady(page, antiBot) {
|
||||
const needle = (antiBot?.postCfReadyUrlContains || '').trim();
|
||||
if (!needle) {
|
||||
return true;
|
||||
}
|
||||
return page.url().includes(needle);
|
||||
}
|
||||
|
||||
/**
|
||||
* 清除 cf_clearance,避免「有 cookie 但 SPA 未触发跳转」的假过关
|
||||
* @param {import('puppeteer').Page} page
|
||||
*/
|
||||
async function clearCfClearanceCookies(page) {
|
||||
const cookies = await page.cookies();
|
||||
for (const c of cookies) {
|
||||
if (c.name !== 'cf_clearance') {
|
||||
continue;
|
||||
}
|
||||
const host = (c.domain || '').replace(/^\./, '');
|
||||
if (host) {
|
||||
await page.deleteCookie({ name: c.name, url: `https://${host}/` }).catch(() => {});
|
||||
}
|
||||
await page.deleteCookie({
|
||||
name: c.name,
|
||||
domain: c.domain,
|
||||
path: c.path || '/',
|
||||
}).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* reload 后检查业务 URL 是否就绪
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
|
||||
* @param {number} timeoutMs
|
||||
* @param {object} antiBot
|
||||
*/
|
||||
async function tryPostCfRecoverViaReload(page, waitForUrlSettled, timeoutMs, antiBot) {
|
||||
if (!needsPostCfBusinessUrl(antiBot) || isPostCfBusinessUrlReady(page, antiBot)) {
|
||||
return isPostCfBusinessUrlReady(page, antiBot);
|
||||
}
|
||||
|
||||
console.log(`[CF] 业务 URL 未就绪 current=${page.url()},尝试 reload`);
|
||||
try {
|
||||
await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 45000) });
|
||||
if (waitForUrlSettled) {
|
||||
await waitForUrlSettled(page).catch(() => {});
|
||||
}
|
||||
return isPostCfBusinessUrlReady(page, antiBot);
|
||||
} catch (reloadErr) {
|
||||
console.warn('[CF] reload 失败:', reloadErr.message);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 清除 cf_clearance 并 reload,使 Turnstile 重新出现
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
|
||||
* @param {number} timeoutMs
|
||||
*/
|
||||
async function invalidateStaleCfAndReload(page, waitForUrlSettled, timeoutMs) {
|
||||
console.log('[CF] 清除 cf_clearance 并 reload,重新触发 Turnstile');
|
||||
await clearCfClearanceCookies(page);
|
||||
await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 45000) });
|
||||
if (waitForUrlSettled) {
|
||||
await waitForUrlSettled(page).catch(() => {});
|
||||
}
|
||||
return detectCloudflare(page);
|
||||
}
|
||||
|
||||
/**
|
||||
* session_reuse 快速返回前校验:有 postCf 要求时必须已到业务页
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @param {object} antiBot
|
||||
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
|
||||
* @param {number} timeoutMs
|
||||
* @param {number} started
|
||||
* @param {boolean} cfDetected
|
||||
*/
|
||||
async function trySessionReuseFastPath(page, antiBot, waitForUrlSettled, timeoutMs, started, cfDetected) {
|
||||
if (waitForUrlSettled) {
|
||||
await waitForUrlSettled(page).catch(() => {});
|
||||
}
|
||||
|
||||
if (!needsPostCfBusinessUrl(antiBot) || isPostCfBusinessUrlReady(page, antiBot)) {
|
||||
return {
|
||||
success: true,
|
||||
code: null,
|
||||
challengeType: null,
|
||||
stage: 'session_reuse',
|
||||
solverUsed: false,
|
||||
elapsedMs: Date.now() - started,
|
||||
cfDetected,
|
||||
};
|
||||
}
|
||||
|
||||
console.log(`[CF] cf_clearance/无挑战但业务页未就绪 url=${page.url()},降级重试`);
|
||||
|
||||
if (await tryPostCfRecoverViaReload(page, waitForUrlSettled, timeoutMs, antiBot)) {
|
||||
return {
|
||||
success: true,
|
||||
code: null,
|
||||
challengeType: null,
|
||||
stage: 'session_reuse_reload',
|
||||
solverUsed: false,
|
||||
elapsedMs: Date.now() - started,
|
||||
cfDetected,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @param {{ turnstile?: boolean, solverFallback?: boolean, challengeTimeoutMs?: number, postCfReadyUrlContains?: string }} antiBot
|
||||
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
|
||||
* @param {import('./session-store').StoredSession|null} [storedSession]
|
||||
*/
|
||||
@@ -19,18 +149,21 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
|
||||
let cfState = await detectCloudflare(page);
|
||||
|
||||
if (!cfState.blocked && cfState.hasCfClearance) {
|
||||
return {
|
||||
success: true,
|
||||
code: null,
|
||||
challengeType: null,
|
||||
stage: 'session_reuse',
|
||||
solverUsed: false,
|
||||
elapsedMs: Date.now() - started,
|
||||
cfDetected: false,
|
||||
};
|
||||
const fast = await trySessionReuseFastPath(
|
||||
page, antiBot, waitForUrlSettled, timeoutMs, started, false
|
||||
);
|
||||
if (fast) {
|
||||
return fast;
|
||||
}
|
||||
cfState = await invalidateStaleCfAndReload(page, waitForUrlSettled, timeoutMs);
|
||||
}
|
||||
|
||||
if (!cfState.blocked) {
|
||||
if (waitForUrlSettled) {
|
||||
await waitForUrlSettled(page).catch(() => {});
|
||||
}
|
||||
|
||||
if (!needsPostCfBusinessUrl(antiBot) || isPostCfBusinessUrlReady(page, antiBot)) {
|
||||
return {
|
||||
success: true,
|
||||
code: null,
|
||||
@@ -42,13 +175,45 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
|
||||
};
|
||||
}
|
||||
|
||||
if (await tryPostCfRecoverViaReload(page, waitForUrlSettled, timeoutMs, antiBot)) {
|
||||
return {
|
||||
success: true,
|
||||
code: null,
|
||||
challengeType: null,
|
||||
stage: 'post_nav_reload',
|
||||
solverUsed: false,
|
||||
elapsedMs: Date.now() - started,
|
||||
cfDetected: false,
|
||||
};
|
||||
}
|
||||
|
||||
if (cfState.hasCfClearance) {
|
||||
cfState = await invalidateStaleCfAndReload(page, waitForUrlSettled, timeoutMs);
|
||||
} else {
|
||||
return {
|
||||
success: true,
|
||||
code: null,
|
||||
challengeType: null,
|
||||
stage: 'pending_business_nav',
|
||||
solverUsed: false,
|
||||
elapsedMs: Date.now() - started,
|
||||
cfDetected: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (storedSession && isSessionLikelyValid(storedSession)) {
|
||||
console.log('[CF] 会话有效但仍被拦截,尝试 reload 一次');
|
||||
try {
|
||||
await page.reload({ waitUntil: 'domcontentloaded', timeout: Math.min(timeoutMs, 45000) });
|
||||
if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {});
|
||||
if (waitForUrlSettled) {
|
||||
await waitForUrlSettled(page).catch(() => {});
|
||||
}
|
||||
cfState = await detectCloudflare(page);
|
||||
if (!cfState.blocked) {
|
||||
if (waitForUrlSettled) {
|
||||
await waitForUrlSettled(page).catch(() => {});
|
||||
}
|
||||
return {
|
||||
success: true,
|
||||
code: null,
|
||||
@@ -69,7 +234,9 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
|
||||
if (antiBot.turnstile !== false) {
|
||||
const builtIn = await waitForTurnstile(page, { timeoutMs, useBuiltInClick: true });
|
||||
if (builtIn.success) {
|
||||
if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {});
|
||||
if (waitForUrlSettled) {
|
||||
await waitForUrlSettled(page).catch(() => {});
|
||||
}
|
||||
cfState = await detectCloudflare(page);
|
||||
if (!cfState.blocked) {
|
||||
return {
|
||||
@@ -96,7 +263,9 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
|
||||
const apiWaitMs = Math.min(timeoutMs, 30000);
|
||||
await waitForTurnstile(page, { timeoutMs: apiWaitMs });
|
||||
|
||||
if (waitForUrlSettled) await waitForUrlSettled(page).catch(() => {});
|
||||
if (waitForUrlSettled) {
|
||||
await waitForUrlSettled(page).catch(() => {});
|
||||
}
|
||||
|
||||
cfState = await detectCloudflare(page);
|
||||
if (!cfState.blocked) {
|
||||
@@ -138,4 +307,7 @@ async function handleCloudflareChallenge(page, antiBot, waitForUrlSettled, store
|
||||
module.exports = {
|
||||
handleCloudflareChallenge,
|
||||
isCaptchaConfigured,
|
||||
needsPostCfBusinessUrl,
|
||||
isPostCfBusinessUrlReady,
|
||||
clearCfClearanceCookies,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
/**
|
||||
* CF 过盾后等待业务页就绪(可选,由 antiBot.postCfReady* 驱动,未配置则跳过)
|
||||
*/
|
||||
|
||||
/**
|
||||
* 采集当前页面诊断信息(auth 失败时便于排查)
|
||||
* @param {import('puppeteer').Page} page
|
||||
*/
|
||||
async function collectPageDiagnostics(page) {
|
||||
const url = page.url();
|
||||
const snapshot = await page.evaluate(() => ({
|
||||
url: window.location.href,
|
||||
title: document.title || '',
|
||||
bodyText: (document.body && document.body.innerText ? document.body.innerText : '').slice(0, 500),
|
||||
elInputInner: document.querySelectorAll('.el-input__inner').length,
|
||||
})).catch(() => null);
|
||||
|
||||
return snapshot || { url, title: '', bodyText: '', elInputInner: 0 };
|
||||
}
|
||||
|
||||
/**
|
||||
* CF 成功后等待最终业务 URL / DOM 出现(Whatshub 等 SPA 短链跳转场景)
|
||||
*
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @param {object} antiBot normalizeAntiBot 返回值
|
||||
* @param {(page: import('puppeteer').Page) => Promise<string>} [waitForUrlSettled]
|
||||
* @param {number} [navigationTimeoutMs]
|
||||
*/
|
||||
async function awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled, navigationTimeoutMs = 60000) {
|
||||
if (!antiBot || !antiBot.enabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const urlContains = (antiBot.postCfReadyUrlContains || '').trim();
|
||||
const selector = (antiBot.postCfReadySelector || '').trim();
|
||||
if (urlContains === '' && selector === '') {
|
||||
return;
|
||||
}
|
||||
|
||||
const timeoutMs = Math.max(5000, antiBot.postCfReadyTimeoutMs || 30000);
|
||||
console.log(
|
||||
`[CF] 等待业务页就绪 current=${page.url()} urlContains="${urlContains}" `
|
||||
+ `selector="${selector}" timeout=${timeoutMs}ms`
|
||||
);
|
||||
|
||||
try {
|
||||
if (urlContains !== '') {
|
||||
await page.waitForFunction(
|
||||
(needle) => window.location.href.includes(needle),
|
||||
{ timeout: timeoutMs, polling: 200 },
|
||||
urlContains
|
||||
);
|
||||
}
|
||||
if (selector !== '') {
|
||||
await page.waitForSelector(selector, { timeout: timeoutMs, visible: true });
|
||||
}
|
||||
} catch (err) {
|
||||
const retryMs = Math.min(15000, timeoutMs);
|
||||
const needsRetry = urlContains !== '' && !page.url().includes(urlContains);
|
||||
if (needsRetry) {
|
||||
console.warn(`[CF] 业务页等待失败 url=${page.url()},最后尝试 reload (${retryMs}ms)`);
|
||||
await page.reload({ waitUntil: 'domcontentloaded', timeout: navigationTimeoutMs }).catch(() => {});
|
||||
if (typeof waitForUrlSettled === 'function') {
|
||||
await waitForUrlSettled(page).catch(() => {});
|
||||
}
|
||||
if (urlContains !== '') {
|
||||
await page.waitForFunction(
|
||||
(needle) => window.location.href.includes(needle),
|
||||
{ timeout: retryMs, polling: 200 },
|
||||
urlContains
|
||||
);
|
||||
}
|
||||
if (selector !== '') {
|
||||
await page.waitForSelector(selector, { timeout: retryMs, visible: true });
|
||||
}
|
||||
} else {
|
||||
err.pageDiagnostics = await collectPageDiagnostics(page);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
const settleMs = antiBot.postCfSettleMs || 0;
|
||||
if (settleMs > 0) {
|
||||
await new Promise((resolve) => setTimeout(resolve, settleMs));
|
||||
}
|
||||
|
||||
console.log(`[CF] 业务页就绪 url=${page.url()}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行 authActions,失败时附带 page 诊断
|
||||
* @param {import('puppeteer').Page} page
|
||||
* @param {object[]|undefined} authActions
|
||||
* @param {Function} executeAuthActionsFn
|
||||
*/
|
||||
async function executeAuthActionsWithDiagnostics(page, authActions, executeAuthActionsFn) {
|
||||
try {
|
||||
await executeAuthActionsFn(page, authActions);
|
||||
} catch (err) {
|
||||
err.pageDiagnostics = await collectPageDiagnostics(page);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
collectPageDiagnostics,
|
||||
awaitPostCfBusinessReady,
|
||||
executeAuthActionsWithDiagnostics,
|
||||
};
|
||||
@@ -1,7 +1,7 @@
|
||||
/**
|
||||
* Turnstile 内置求解:等待 widget 自动完成 + token 轮询
|
||||
*/
|
||||
const { detectCloudflare, isTurnstileSolved } = require('./cf-detector');
|
||||
const { detectCloudflare, isTurnstileSolved, urlIndicatesCfChallenge } = require('./cf-detector');
|
||||
|
||||
/**
|
||||
* @typedef {Object} TurnstileResult
|
||||
@@ -22,18 +22,24 @@ async function waitForTurnstile(page, options = {}) {
|
||||
const started = Date.now();
|
||||
|
||||
const initial = await detectCloudflare(page);
|
||||
if (!initial.blocked || initial.hasCfClearance) {
|
||||
const initialUrlChallenge = urlIndicatesCfChallenge(page.url());
|
||||
|
||||
if ((!initial.blocked && !initialUrlChallenge) || initial.hasCfClearance) {
|
||||
if (initial.hasCfClearance && !initialUrlChallenge) {
|
||||
return { success: true, stage: 'already_clear', elapsedMs: Date.now() - started };
|
||||
}
|
||||
if (!initial.blocked && !initialUrlChallenge) {
|
||||
return { success: true, stage: 'already_clear', elapsedMs: Date.now() - started };
|
||||
}
|
||||
}
|
||||
|
||||
// real browser 已开启 turnstile:true 内置点击;此处轮询等待结果
|
||||
while (Date.now() - started < timeoutMs) {
|
||||
if (await isTurnstileSolved(page)) {
|
||||
if (await isTurnstileSolved(page) && !urlIndicatesCfChallenge(page.url())) {
|
||||
return { success: true, stage: 'built_in', elapsedMs: Date.now() - started };
|
||||
}
|
||||
|
||||
const state = await detectCloudflare(page);
|
||||
if (!state.blocked) {
|
||||
if (!state.blocked && !urlIndicatesCfChallenge(page.url()) && state.hasCfClearance) {
|
||||
return { success: true, stage: 'built_in', elapsedMs: Date.now() - started };
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
"start": "node server.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"dotenv": "^16.4.7",
|
||||
"express": "^4.18.2",
|
||||
"puppeteer": "^25.1.0",
|
||||
"puppeteer-extra": "^3.3.6",
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
require('dotenv').config({ path: require('path').join(__dirname, '.env') });
|
||||
|
||||
const express = require('express');
|
||||
const fs = require('fs');
|
||||
const {
|
||||
@@ -45,6 +47,10 @@ const {
|
||||
const { resolveSessionContext, touchSessionIfPresent } = require('./lib/session-context');
|
||||
const { getProfileStats, profileExists } = require('./lib/profile-dir');
|
||||
const { runUiPagination } = require('./lib/ui-pagination-runner');
|
||||
const {
|
||||
awaitPostCfBusinessReady,
|
||||
executeAuthActionsWithDiagnostics,
|
||||
} = require('./lib/post-cf-ready');
|
||||
|
||||
const app = express();
|
||||
app.use(express.json({ limit: '50mb' }));
|
||||
@@ -75,7 +81,11 @@ function sendTaskError(res, error, extra = {}) {
|
||||
}
|
||||
const errType = classifyPuppeteerError(error);
|
||||
console.error(`[${errType}]`, error.message);
|
||||
return res.status(500).json({ success: false, error: error.message, ...extra });
|
||||
const payload = { success: false, error: error.message, ...extra };
|
||||
if (error.pageDiagnostics) {
|
||||
payload.page = error.pageDiagnostics;
|
||||
}
|
||||
return res.status(500).json(payload);
|
||||
}
|
||||
|
||||
async function navigateToPage(page, url) {
|
||||
@@ -524,7 +534,11 @@ app.post('/api/auth-and-intercept', async (req, res) => {
|
||||
await seedShareTokenCookie(page, finalPageUrl);
|
||||
}
|
||||
|
||||
await executeAuthActions(page, authActions);
|
||||
if (antiBot.enabled) {
|
||||
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled);
|
||||
}
|
||||
|
||||
await executeAuthActionsWithDiagnostics(page, authActions, executeAuthActions);
|
||||
await interceptor.waitForApis(interceptTimeout);
|
||||
|
||||
const rawCookies = await page.cookies();
|
||||
@@ -758,6 +772,10 @@ app.post('/api/ui-pagination', async (req, res) => {
|
||||
touchSessionIfPresent(antiBot.sessionKey);
|
||||
}
|
||||
|
||||
if (antiBot.enabled) {
|
||||
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled);
|
||||
}
|
||||
|
||||
const paginationResult = await runUiPagination(page, {
|
||||
apiUrl,
|
||||
nextBtnSelector,
|
||||
@@ -874,7 +892,11 @@ app.post('/api/auth-intercept-and-paginate', async (req, res) => {
|
||||
touchSessionIfPresent(antiBot.sessionKey);
|
||||
}
|
||||
|
||||
await executeAuthActions(page, authActions);
|
||||
if (antiBot.enabled) {
|
||||
await awaitPostCfBusinessReady(page, antiBot, waitForUrlSettled);
|
||||
}
|
||||
|
||||
await executeAuthActionsWithDiagnostics(page, authActions, executeAuthActions);
|
||||
await interceptor.waitForApis(interceptTimeout);
|
||||
|
||||
const rawCookies = await page.cookies();
|
||||
|
||||
Reference in New Issue
Block a user