修改whatshub工单爬虫
This commit is contained in:
@@ -75,11 +75,23 @@ abstract class AbstractScrmSpider
|
||||
const MODE_FETCH = 'fetch'; // 极速并发拉取 (推荐默认)
|
||||
const MODE_UI = 'ui_click'; // 强制 UI 点击下一页
|
||||
|
||||
/** CLI 测试时输出 Node 请求/失败详情 */
|
||||
protected $verbose = false;
|
||||
|
||||
protected $nodeHost;
|
||||
|
||||
public function __construct($nodeHost = 'http://127.0.0.1:3001')
|
||||
{
|
||||
$this->nodeHost = $nodeHost;
|
||||
$this->nodeHost = rtrim($nodeHost, '/');
|
||||
}
|
||||
|
||||
/**
|
||||
* 开启调试输出(测试脚本可调用)
|
||||
*/
|
||||
public function setVerbose($verbose = true)
|
||||
{
|
||||
$this->verbose = (bool) $verbose;
|
||||
return $this;
|
||||
}
|
||||
|
||||
/** ================= 子类必须实现的契约 ================= **/
|
||||
@@ -97,7 +109,7 @@ abstract class AbstractScrmSpider
|
||||
public function run()
|
||||
{
|
||||
$config = $this->getSpiderConfig();
|
||||
|
||||
|
||||
$listApi = $config['listApi'];
|
||||
$detailApi = $config['detailApi'] ?? null; // detail 接口是可选的
|
||||
$countApi = $config['countApi'] ?? null; // 有些在线客服的总数通过单独调用接口获取
|
||||
@@ -106,13 +118,14 @@ abstract class AbstractScrmSpider
|
||||
$apiUrlsToIntercept = [$listApi];
|
||||
if ($detailApi) { $apiUrlsToIntercept[] = $detailApi; }
|
||||
if ($countApi) { $apiUrlsToIntercept[] = $countApi; }
|
||||
// var_dump($apiUrlsToIntercept);
|
||||
// dump([
|
||||
// 'pageUrl' => $config['pageUrl'],
|
||||
// 'apiUrls' => $apiUrlsToIntercept,
|
||||
// 'authActions' => $config['authActions']
|
||||
// ]);
|
||||
|
||||
$antiBot = $config['antiBot'] ?? null;
|
||||
$mode = $config['paginationMode'] ?? self::MODE_FETCH;
|
||||
|
||||
// antiBot + UI 翻页:单 Browser 合并路径(与生产 SplitTicket 同步一致)
|
||||
if ($this->shouldUseUnifiedBrowserPath($antiBot, $mode)) {
|
||||
return $this->runUnifiedUiPath($config, $antiBot, $apiUrlsToIntercept, $listApi, $detailApi, $countApi);
|
||||
}
|
||||
|
||||
// 【阶段一】:初始化并首屏拦截
|
||||
$initResult = $this->requestNode('/api/auth-and-intercept', [
|
||||
@@ -123,17 +136,20 @@ abstract class AbstractScrmSpider
|
||||
], $this->resolveAuthInterceptTimeout($antiBot));
|
||||
|
||||
if (empty($initResult['success'])) {
|
||||
$cfCode = (string) ($initResult['code'] ?? '');
|
||||
if ($cfCode === 'CF_TURNSTILE_FAILED') {
|
||||
throw new Exception('Cloudflare Turnstile 验证失败: ' . ($initResult['stage'] ?? 'timeout'));
|
||||
}
|
||||
throw new Exception("初始化失败: " . ($initResult['error'] ?? '未知'));
|
||||
throw new Exception($this->formatNodeFailure('初始化失败', $initResult));
|
||||
}
|
||||
|
||||
$interceptedApis = $initResult['interceptedApis'];
|
||||
$cookies = $initResult['cookies'];
|
||||
$finalPageUrl = $initResult['finalPageUrl'] ?? $config['pageUrl'];
|
||||
|
||||
$this->logDebug('auth-and-intercept ok', [
|
||||
'finalPageUrl' => $finalPageUrl,
|
||||
'intercepted' => array_keys($interceptedApis),
|
||||
'cf' => $initResult['cf'] ?? null,
|
||||
'browserReused' => $initResult['browserReused'] ?? null,
|
||||
]);
|
||||
|
||||
// 必须拦截到 List 接口,否则无法继续
|
||||
if (!isset($interceptedApis[$listApi])) {
|
||||
throw new Exception("致命错误:未能拦截到必须的列表接口 [{$listApi}]");
|
||||
@@ -229,6 +245,140 @@ abstract class AbstractScrmSpider
|
||||
return $unifiedData;
|
||||
}
|
||||
|
||||
/**
|
||||
* 单 Browser 路径:auth + 拦截 + UI 翻页(需 antiBot.sessionKey)
|
||||
*/
|
||||
protected function runUnifiedUiPath($config, $antiBot, $apiUrlsToIntercept, $listApi, $detailApi, $countApi)
|
||||
{
|
||||
$uiConfig = $this->getUiPaginationConfig();
|
||||
$sessionKey = is_array($antiBot) ? (string) ($antiBot['sessionKey'] ?? '') : '';
|
||||
|
||||
$this->logDebug('unified browser path', [
|
||||
'sessionKey' => $sessionKey,
|
||||
'listApi' => $listApi,
|
||||
]);
|
||||
|
||||
$mergedResult = $this->requestNode('/api/auth-intercept-and-paginate', [
|
||||
'pageUrl' => $config['pageUrl'],
|
||||
'apiUrls' => $apiUrlsToIntercept,
|
||||
'authActions' => $config['authActions'] ?? [],
|
||||
'antiBot' => $antiBot,
|
||||
'pagination' => [
|
||||
'apiUrl' => $listApi,
|
||||
'nextBtnSelector' => $uiConfig['nextBtnSelector'] ?? '',
|
||||
'waitMs' => $uiConfig['waitMs'] ?? 2000,
|
||||
'clicksToPerform' => 9999,
|
||||
],
|
||||
], 1200);
|
||||
|
||||
if (empty($mergedResult['success'])) {
|
||||
throw new Exception($this->formatNodeFailure('合并同步失败', $mergedResult));
|
||||
}
|
||||
|
||||
$this->logDebug('auth-intercept-and-paginate ok', [
|
||||
'intercepted' => array_keys($mergedResult['interceptedApis'] ?? []),
|
||||
'extraPages' => count($mergedResult['extraPages'] ?? []),
|
||||
'cf' => $mergedResult['cf'] ?? null,
|
||||
'browserReused' => $mergedResult['browserReused'] ?? null,
|
||||
'finalPageUrl' => $mergedResult['finalPageUrl'] ?? null,
|
||||
]);
|
||||
|
||||
$interceptedApis = $mergedResult['interceptedApis'] ?? [];
|
||||
if (!isset($interceptedApis[$listApi])) {
|
||||
throw new Exception("致命错误:未能拦截到必须的列表接口 [{$listApi}]");
|
||||
}
|
||||
|
||||
$detailData = $detailApi && isset($interceptedApis[$detailApi])
|
||||
? $interceptedApis[$detailApi]['data'] : null;
|
||||
$countData = $countApi && isset($interceptedApis[$countApi])
|
||||
? $interceptedApis[$countApi]['data'] : null;
|
||||
|
||||
$listApiNode = $interceptedApis[$listApi];
|
||||
$allListPagesData = [$listApiNode['data']];
|
||||
if (!empty($mergedResult['extraPages']) && is_array($mergedResult['extraPages'])) {
|
||||
foreach ($mergedResult['extraPages'] as $pageData) {
|
||||
$allListPagesData[] = $pageData;
|
||||
}
|
||||
}
|
||||
|
||||
$this->extractListTotalPages($listApiNode['data'], $countData);
|
||||
|
||||
return $this->parseToUnifiedData($detailData, $allListPagesData);
|
||||
}
|
||||
|
||||
/**
|
||||
* antiBot 启用且配置了 sessionKey 时,UI 翻页走合并 Browser 路径
|
||||
*/
|
||||
protected function shouldUseUnifiedBrowserPath($antiBot, $mode)
|
||||
{
|
||||
if ($mode !== self::MODE_UI) {
|
||||
return false;
|
||||
}
|
||||
if (!is_array($antiBot) || empty($antiBot['enabled'])) {
|
||||
return false;
|
||||
}
|
||||
return !empty($antiBot['sessionKey']);
|
||||
}
|
||||
|
||||
/**
|
||||
* 格式化 Node 失败信息,附带 CF / 落地页等诊断字段
|
||||
*/
|
||||
protected function formatNodeFailure($prefix, $result)
|
||||
{
|
||||
$cfCode = (string) ($result['code'] ?? '');
|
||||
if ($cfCode === 'CF_TURNSTILE_FAILED') {
|
||||
return 'Cloudflare Turnstile 验证失败: ' . ($result['stage'] ?? 'timeout');
|
||||
}
|
||||
|
||||
$parts = [$prefix . ': ' . ($result['error'] ?? '未知')];
|
||||
|
||||
if (!empty($result['finalPageUrl'])) {
|
||||
$parts[] = 'finalPageUrl=' . $result['finalPageUrl'];
|
||||
}
|
||||
if (!empty($result['page']) && is_array($result['page'])) {
|
||||
$page = $result['page'];
|
||||
if (!empty($page['url'])) {
|
||||
$parts[] = 'pageUrl=' . $page['url'];
|
||||
}
|
||||
if (isset($page['elInputInner'])) {
|
||||
$parts[] = 'elInputInner=' . $page['elInputInner'];
|
||||
}
|
||||
}
|
||||
if (!empty($result['cf']) && is_array($result['cf'])) {
|
||||
$cf = $result['cf'];
|
||||
$cfBits = [];
|
||||
if (isset($cf['cfDetected'])) {
|
||||
$cfBits[] = 'cfDetected=' . ($cf['cfDetected'] ? 'true' : 'false');
|
||||
}
|
||||
if (!empty($cf['turnstileStage'])) {
|
||||
$cfBits[] = 'turnstileStage=' . $cf['turnstileStage'];
|
||||
}
|
||||
if (isset($cf['solverUsed'])) {
|
||||
$cfBits[] = 'solverUsed=' . ($cf['solverUsed'] ? 'true' : 'false');
|
||||
}
|
||||
if (isset($cf['elapsedMs'])) {
|
||||
$cfBits[] = 'cfElapsedMs=' . $cf['elapsedMs'];
|
||||
}
|
||||
if (!empty($cfBits)) {
|
||||
$parts[] = implode(', ', $cfBits);
|
||||
}
|
||||
}
|
||||
|
||||
return implode(' | ', $parts);
|
||||
}
|
||||
|
||||
protected function logDebug($message, $context = [])
|
||||
{
|
||||
if (!$this->verbose) {
|
||||
return;
|
||||
}
|
||||
$line = '[Spider] ' . $message;
|
||||
if (!empty($context)) {
|
||||
$line .= ' ' . json_encode($context, JSON_UNESCAPED_UNICODE);
|
||||
}
|
||||
echo $line . "\n";
|
||||
}
|
||||
|
||||
/**
|
||||
* Real Browser + Turnstile 首屏耗时较长,单独放宽 auth-and-intercept 超时
|
||||
*
|
||||
@@ -244,17 +394,76 @@ abstract class AbstractScrmSpider
|
||||
|
||||
private function requestNode($endpoint, $payload, $timeout = 60)
|
||||
{
|
||||
$ch = curl_init($this->nodeHost . $endpoint);
|
||||
$maxAttempts = 3;
|
||||
$lastDecoded = [];
|
||||
for ($attempt = 1; $attempt <= $maxAttempts; $attempt++) {
|
||||
$lastDecoded = $this->doRequestNode($endpoint, $payload, $timeout, $attempt);
|
||||
if (!empty($lastDecoded['success'])) {
|
||||
return $lastDecoded;
|
||||
}
|
||||
$error = (string) ($lastDecoded['error'] ?? '');
|
||||
if (!$this->shouldRetryNodeResponse($lastDecoded, $error) || $attempt >= $maxAttempts) {
|
||||
return $lastDecoded;
|
||||
}
|
||||
$this->logDebug('node retry', [
|
||||
'endpoint' => $endpoint,
|
||||
'attempt' => $attempt + 1,
|
||||
'error' => $error,
|
||||
]);
|
||||
sleep($attempt);
|
||||
}
|
||||
return $lastDecoded;
|
||||
}
|
||||
|
||||
private function doRequestNode($endpoint, $payload, $timeout, $attempt)
|
||||
{
|
||||
$url = $this->nodeHost . $endpoint;
|
||||
$this->logDebug('node request', [
|
||||
'url' => $url,
|
||||
'timeout' => $timeout,
|
||||
'attempt' => $attempt,
|
||||
]);
|
||||
|
||||
$ch = curl_init($url);
|
||||
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
||||
curl_setopt($ch, CURLOPT_POST, true);
|
||||
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($payload));
|
||||
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($payload, JSON_UNESCAPED_UNICODE));
|
||||
curl_setopt($ch, CURLOPT_HTTPHEADER, ['Content-Type: application/json']);
|
||||
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5);
|
||||
curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
|
||||
$response = curl_exec($ch);
|
||||
if (curl_errno($ch)) throw new Exception(curl_error($ch));
|
||||
$httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
if (curl_errno($ch)) {
|
||||
$err = curl_error($ch);
|
||||
curl_close($ch);
|
||||
throw new Exception($err);
|
||||
}
|
||||
curl_close($ch);
|
||||
return json_decode($response, true);
|
||||
|
||||
$decoded = json_decode((string) $response, true);
|
||||
if (!is_array($decoded)) {
|
||||
return ['success' => false, 'error' => 'Node 返回非 JSON', 'httpCode' => $httpCode];
|
||||
}
|
||||
if ($httpCode === 503) {
|
||||
$decoded['success'] = false;
|
||||
$decoded['error'] = (string) ($decoded['error'] ?? '服务繁忙');
|
||||
$decoded['httpCode'] = 503;
|
||||
}
|
||||
return $decoded;
|
||||
}
|
||||
|
||||
private function shouldRetryNodeResponse($decoded, $error)
|
||||
{
|
||||
if ((int) ($decoded['httpCode'] ?? 0) === 503) {
|
||||
return true;
|
||||
}
|
||||
$lower = strtolower($error);
|
||||
foreach (['排队超时', 'queue_timeout', 'timeout', 'net::', 'navigation', 'connection'] as $needle) {
|
||||
if ($needle !== '' && strpos($lower, $needle) !== false) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+198
-30
@@ -7,13 +7,17 @@ class Whatshub extends AbstractScrmSpider
|
||||
{
|
||||
const API_LIST = '/api/whatshub-counter/workShare/open/detail'; // 列表API地址
|
||||
const API_DETAILS = '/api/whatshub-counter/workShare/open/statistics'; // 详情API地址
|
||||
/** shareCode + password 直连接口(一次返回全量号码,无需翻页) */
|
||||
const API_DETAIL_BY_SHARE_CODE = '/api/whatshub-counter/workShare/open/detailByShareCode';
|
||||
const API_COUNT = ''; // 总数API地址
|
||||
const DEFAULT_PER_PAGE_COUNT = 20; // List默认每页显示的数量
|
||||
const API_FETCH_TIMEOUT_SEC = 20;
|
||||
|
||||
private $pageUrl;
|
||||
private $account;
|
||||
private $password;
|
||||
private $unifiedData;
|
||||
|
||||
|
||||
// 实例化时动态传入账号和密码
|
||||
public function __construct($pageUrl, $account, $password, $nodeHost = 'http://127.0.0.1:3001')
|
||||
{
|
||||
@@ -21,46 +25,210 @@ class Whatshub extends AbstractScrmSpider
|
||||
$this->account = $account;
|
||||
$this->password = $password;
|
||||
$this->pageUrl = $pageUrl;
|
||||
|
||||
|
||||
$this->unifiedData = new UnifiedScrmData();
|
||||
}
|
||||
|
||||
/**
|
||||
* API 优先:detailByShareCode 直抓;失败则回退 Node 爬虫
|
||||
*/
|
||||
public function run()
|
||||
{
|
||||
$apiResult = $this->tryFetchViaShareCodeApi();
|
||||
if ($apiResult instanceof UnifiedScrmData) {
|
||||
$this->logDebug('whatshub api path ok', ['count' => count($apiResult->numbers)]);
|
||||
return $apiResult;
|
||||
}
|
||||
|
||||
$this->logDebug('whatshub api skipped or failed, fallback node', []);
|
||||
return parent::run();
|
||||
}
|
||||
|
||||
/**
|
||||
* 从短链 pageUrl 解析 shareCode,例如 /m/KMYBBInp2066/1 → KMYBBInp2066
|
||||
*
|
||||
* @return string|null
|
||||
*/
|
||||
private function extractShareCodeFromPageUrl()
|
||||
{
|
||||
$path = (string) parse_url($this->pageUrl, PHP_URL_PATH);
|
||||
if ($path === '') {
|
||||
return null;
|
||||
}
|
||||
if (preg_match('#/m/([^/]+)/#', $path, $matches)) {
|
||||
return $matches[1];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 拼接 detailByShareCode 完整 GET URL
|
||||
*
|
||||
* @param string $shareCode
|
||||
* @return string|null
|
||||
*/
|
||||
private function buildShareCodeApiUrl($shareCode)
|
||||
{
|
||||
$parsed = parse_url($this->pageUrl);
|
||||
$scheme = isset($parsed['scheme']) ? $parsed['scheme'] : 'https';
|
||||
$host = isset($parsed['host']) ? $parsed['host'] : '';
|
||||
if ($host === '') {
|
||||
return null;
|
||||
}
|
||||
|
||||
$query = http_build_query([
|
||||
'shareCode' => $shareCode,
|
||||
'password' => $this->password,
|
||||
]);
|
||||
|
||||
return $scheme . '://' . $host . self::API_DETAIL_BY_SHARE_CODE . '?' . $query;
|
||||
}
|
||||
|
||||
/**
|
||||
* 尝试通过 shareCode API 抓取;成功返回 UnifiedScrmData,否则返回 null
|
||||
*
|
||||
* @return UnifiedScrmData|null
|
||||
*/
|
||||
private function tryFetchViaShareCodeApi()
|
||||
{
|
||||
if ($this->password === '' || $this->password === null) {
|
||||
$this->logDebug('whatshub api skip: empty password', []);
|
||||
return null;
|
||||
}
|
||||
|
||||
$shareCode = $this->extractShareCodeFromPageUrl();
|
||||
if ($shareCode === null || $shareCode === '') {
|
||||
$this->logDebug('whatshub api skip: cannot parse shareCode', ['pageUrl' => $this->pageUrl]);
|
||||
return null;
|
||||
}
|
||||
|
||||
$apiUrl = $this->buildShareCodeApiUrl($shareCode);
|
||||
if ($apiUrl === null) {
|
||||
$this->logDebug('whatshub api skip: invalid pageUrl host', ['pageUrl' => $this->pageUrl]);
|
||||
return null;
|
||||
}
|
||||
|
||||
$this->logDebug('whatshub api request', ['url' => preg_replace('/password=[^&]+/', 'password=***', $apiUrl)]);
|
||||
|
||||
$ch = curl_init($apiUrl);
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_CONNECTTIMEOUT => 5,
|
||||
CURLOPT_TIMEOUT => self::API_FETCH_TIMEOUT_SEC,
|
||||
CURLOPT_HTTPHEADER => [
|
||||
'Accept: application/json',
|
||||
'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
],
|
||||
]);
|
||||
|
||||
$response = curl_exec($ch);
|
||||
$httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
$curlErr = curl_error($ch);
|
||||
curl_close($ch);
|
||||
|
||||
if ($response === false || $curlErr !== '') {
|
||||
$this->logDebug('whatshub api curl failed', ['error' => $curlErr]);
|
||||
return null;
|
||||
}
|
||||
|
||||
if ($httpCode !== 200) {
|
||||
$this->logDebug('whatshub api http error', ['httpCode' => $httpCode]);
|
||||
return null;
|
||||
}
|
||||
|
||||
$payload = json_decode($response, true);
|
||||
if (!is_array($payload)) {
|
||||
$this->logDebug('whatshub api invalid json', []);
|
||||
return null;
|
||||
}
|
||||
|
||||
if ((int) ($payload['code'] ?? 0) !== 200) {
|
||||
$this->logDebug('whatshub api business error', [
|
||||
'code' => $payload['code'] ?? null,
|
||||
'msg' => $payload['msg'] ?? '',
|
||||
]);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!isset($payload['data']) || !is_array($payload['data'])) {
|
||||
$this->logDebug('whatshub api invalid data field', []);
|
||||
return null;
|
||||
}
|
||||
|
||||
return $this->parseShareCodeApiResponse($payload);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 detailByShareCode 响应映射为 UnifiedScrmData
|
||||
*
|
||||
* @param array<string, mixed> $payload
|
||||
* @return UnifiedScrmData
|
||||
*/
|
||||
private function parseShareCodeApiResponse(array $payload)
|
||||
{
|
||||
$unifiedData = new UnifiedScrmData();
|
||||
$todayNewSum = 0;
|
||||
|
||||
foreach ($payload['data'] as $item) {
|
||||
if (!is_array($item) || empty($item['account'])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$number = (string) $item['account'];
|
||||
$isOnline = isset($item['isOnline']) && (int) $item['isOnline'] === 1;
|
||||
$dayNewFans = (int) ($item['dayNewFans'] ?? 0);
|
||||
|
||||
$unifiedData->addNumber($number, $isOnline, $dayNewFans);
|
||||
$todayNewSum += $dayNewFans;
|
||||
}
|
||||
|
||||
$unifiedData->todayNewCount = $todayNewSum;
|
||||
$unifiedData->total = count($unifiedData->numbers);
|
||||
|
||||
return $unifiedData;
|
||||
}
|
||||
|
||||
protected function getSpiderConfig()
|
||||
{
|
||||
$host = (string) parse_url($this->pageUrl, PHP_URL_HOST);
|
||||
return [
|
||||
'pageUrl' => $this->pageUrl,
|
||||
'apiUrls' => [self::API_LIST, self::API_DETAILS],
|
||||
|
||||
|
||||
// 明确指派角色
|
||||
'listApi' => self::API_LIST, // 必须
|
||||
|
||||
'listApi' => self::API_LIST,
|
||||
'detailApi' => self::API_DETAILS,
|
||||
|
||||
'listMethod' => 'POST',
|
||||
'paginationMode' => self::MODE_UI,
|
||||
|
||||
|
||||
'authActions' => [
|
||||
// 1. 填入密码:寻找 class 包含 el-message-box__input 下面的任意 input
|
||||
// Node.js 会自动扫描所有匹配项,并只把密码强行注入到那个“肉眼可见”的框里
|
||||
// Node antiBot.postCfReady* 会在 CF 成功后等待 work-order-sharing + .el-input__inner
|
||||
['type' => 'vue_fill', 'selector' => '.el-input__inner', 'value' => $this->password],
|
||||
|
||||
|
||||
// 2. 停顿 500ms,让 Vue 绑定的 v-model 彻底反应过来
|
||||
['type' => 'wait', 'ms' => 500],
|
||||
|
||||
|
||||
// 3. 点击确认:寻找 MessageBox 底部的蓝色 primary 确认按钮
|
||||
// 同样利用 vue_click 的可见性过滤,无视隐藏的旧弹窗按钮
|
||||
['type' => 'vue_click', 'selector' => '.vxe-button-group .theme--primary'],
|
||||
|
||||
|
||||
// 4. 等待弹窗淡出,接口开始请求
|
||||
['type' => 'wait', 'ms' => 2200]
|
||||
['type' => 'wait', 'ms' => 2200],
|
||||
],
|
||||
// Whatshub 专用:Real Browser + Turnstile + Captcha API 兜底 + 会话复用
|
||||
'antiBot' => [
|
||||
// sessionKey 必须与 pageUrl 的 host 一致,例如 web.whatshub.cc
|
||||
'antiBot' => [
|
||||
'enabled' => true,
|
||||
'profile' => 'real',
|
||||
'turnstile' => true,
|
||||
'solverFallback' => true,
|
||||
'sessionKey' => 'whatshub:' . $host,
|
||||
'challengeTimeoutMs' => 60000,
|
||||
'challengeTimeoutMs' => 40000,
|
||||
'postCfReadyUrlContains' => 'work-order-sharing',
|
||||
'postCfReadySelector' => '.el-input__inner',
|
||||
'postCfReadyTimeoutMs' => 30000,
|
||||
'postCfSettleMs' => 500,
|
||||
'postCfSpaWaitMs' => 3000,
|
||||
],
|
||||
];
|
||||
}
|
||||
@@ -69,13 +237,13 @@ class Whatshub extends AbstractScrmSpider
|
||||
protected function extractListTotalPages($listFirstPageData, $countData = null)
|
||||
{
|
||||
$default_per_page_count = self::DEFAULT_PER_PAGE_COUNT;
|
||||
$this->unifiedData->total = $listFirstPageData['data']['total'];
|
||||
|
||||
if($this->unifiedData->total <= $default_per_page_count) {
|
||||
$this->unifiedData->total = $listFirstPageData['total'];
|
||||
|
||||
if ($this->unifiedData->total <= $default_per_page_count) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
return ceil($this->unifiedData->total/$default_per_page_count);
|
||||
|
||||
return ceil($this->unifiedData->total / $default_per_page_count);
|
||||
}
|
||||
|
||||
// 没有分页返回空数组
|
||||
@@ -88,8 +256,8 @@ class Whatshub extends AbstractScrmSpider
|
||||
protected function getUiPaginationConfig()
|
||||
{
|
||||
return [
|
||||
'nextBtnSelector' => '.vxe-pager--next-btn',
|
||||
'waitMs' => 1500
|
||||
'nextBtnSelector' => '.vxe-pager--next-btn',
|
||||
'waitMs' => 1500,
|
||||
];
|
||||
}
|
||||
|
||||
@@ -97,28 +265,28 @@ class Whatshub extends AbstractScrmSpider
|
||||
protected function parseToUnifiedData($detailData, $allListPagesData)
|
||||
{
|
||||
$unifiedData = $this->unifiedData;
|
||||
|
||||
|
||||
// 1. 如果捕获到了详情数据,提取今日新增
|
||||
if ($detailData) {
|
||||
$unifiedData->todayNewCount = (int)($detailData['data']['dayNewFans'] ?? 0);
|
||||
$unifiedData->todayNewCount = (int) ($detailData['data']['dayNewFans'] ?? 0);
|
||||
}
|
||||
|
||||
// 2. 循环合并了所有页数的 List 数组
|
||||
foreach ($allListPagesData as $pageRaw) {
|
||||
$records = $pageRaw['data']['rows'] ?? [];
|
||||
$records = $pageRaw['rows'] ?? [];
|
||||
foreach ($records as $item) {
|
||||
if(!empty($item['account'])) {
|
||||
if (!empty($item['account'])) {
|
||||
$number = $item['account'] ?? null;
|
||||
$isOnline = (isset($item['isOnline']) && $item['isOnline'] == 1);
|
||||
|
||||
|
||||
$unifiedData->addNumber($number, $isOnline, $item['dayNewFans']);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 🚀 3. 终极统计:所有翻页数据均已入库,此时再统计真实的 Total 总数
|
||||
|
||||
// 3. 终极统计:所有翻页数据均已入库,此时再统计真实的 Total 总数
|
||||
$unifiedData->total = count($unifiedData->numbers);
|
||||
|
||||
return $unifiedData;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+63
-61
@@ -11,50 +11,52 @@ require_once __DIR__ . '/SsCustomer.php'; // SS云控(Customer)
|
||||
require_once __DIR__ . '/Haiwang.php'; // 海王
|
||||
require_once __DIR__ . '/Whatshub.php'; // Whatshub 工单平台
|
||||
|
||||
// try {
|
||||
// /*
|
||||
// 命令行可以测试Cloudflare防火墙
|
||||
// curl -s -X POST http://127.0.0.1:3001/api/auth-and-intercept \
|
||||
// -H 'Content-Type: application/json' \
|
||||
// -d '{
|
||||
// "pageUrl": "https://web.whatshub.cc/m/iTYsWKQH5030/1",
|
||||
// "apiUrls": ["/api/whatshub-counter/workShare/open/detail"],
|
||||
// "authActions": [
|
||||
// {"type": "vue_fill", "selector": ".el-input__inner", "value": "745030"},
|
||||
// {"type": "wait", "ms": 500},
|
||||
// {"type": "vue_click", "selector": ".vxe-button-group .theme--primary"},
|
||||
// {"type": "wait", "ms": 2200}
|
||||
// ],
|
||||
// "antiBot": {
|
||||
// "enabled": true,
|
||||
// "profile": "real",
|
||||
// "turnstile": true,
|
||||
// "solverFallback": true,
|
||||
// "sessionKey": "whatshub:t.flowerbells.top",
|
||||
// "challengeTimeoutMs": 60000
|
||||
// }
|
||||
// }' | jq .
|
||||
// */
|
||||
try {
|
||||
/*
|
||||
|
||||
// echo "🚀 开始执行<Whatshub 工单平台>抓取任务 (多引擎智能调度)...\n\r";
|
||||
// $pageUrl = 'https://web.whatshub.cc/m/iTYsWKQH5030/1'; // PageUrl 入口授权页
|
||||
// $username = ""; // 登录账号
|
||||
// $password = "745030"; // 登录密码
|
||||
// $spider = new Whatshub($pageUrl, $username, $password);
|
||||
// $finalData = $spider->run();
|
||||
命令行可以测试Cloudflare防火墙
|
||||
curl -s -X POST http://127.0.0.1:3001/api/auth-and-intercept \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"pageUrl": "https://web.whatshub.cc/m/KMYBBInp2066/1",
|
||||
"apiUrls": ["/api/whatshub-counter/workShare/open/detail"],
|
||||
"authActions": [
|
||||
{"type": "wait", "ms": 1000},
|
||||
{"type": "vue_fill", "selector": ".el-input__wrapper > .el-input__inner", "value": "602066"},
|
||||
{"type": "wait", "ms": 500},
|
||||
{"type": "vue_click", "selector": ".vxe-button-group .theme--primary"},
|
||||
{"type": "wait", "ms": 2200}
|
||||
],
|
||||
"antiBot": {
|
||||
"enabled": true,
|
||||
"profile": "real",
|
||||
"turnstile": true,
|
||||
"solverFallback": true,
|
||||
"sessionKey": "whatshub:web.whatshub.cc",
|
||||
"challengeTimeoutMs": 60000
|
||||
}
|
||||
}' | jq .
|
||||
*/
|
||||
|
||||
echo "🚀 开始执行<Whatshub 工单平台>抓取任务 (多引擎智能调度)...\n\r";
|
||||
$pageUrl = 'https://web.whatshub.cc/m/KMYBBInp2066/1'; // PageUrl 入口授权页
|
||||
$username = ""; // 登录账号
|
||||
$password = "602066"; // 登录密码
|
||||
$spider = new Whatshub($pageUrl, $username, $password);
|
||||
$finalData = $spider->run();
|
||||
|
||||
// echo "✅ 任务完成!统一数据如下:\n\r";
|
||||
// echo "----------------------------------------\n\r";
|
||||
// echo "当日新增:{$finalData->todayNewCount} 人\n\r";
|
||||
// echo "在线号码:{$finalData->totalOnline} 个\n\r";
|
||||
// echo "离线号码:{$finalData->totalOffline} 个\n\r";
|
||||
// echo "Total:" . $finalData->total . " 个号码\n\r";
|
||||
// echo "实际总共抓取:" . count($finalData->numbers) . " 个号码\n\r";
|
||||
// echo "号码列表:\n\r";
|
||||
// echo dd($finalData->numbers);
|
||||
// } catch (Exception $e) {
|
||||
// echo "🚨 抓取异常:" . $e->getMessage() . "\n\r";
|
||||
// }
|
||||
echo "✅ 任务完成!统一数据如下:\n\r";
|
||||
echo "----------------------------------------\n\r";
|
||||
echo "当日新增:{$finalData->todayNewCount} 人\n\r";
|
||||
echo "在线号码:{$finalData->totalOnline} 个\n\r";
|
||||
echo "离线号码:{$finalData->totalOffline} 个\n\r";
|
||||
echo "Total:" . $finalData->total . " 个号码\n\r";
|
||||
echo "实际总共抓取:" . count($finalData->numbers) . " 个号码\n\r";
|
||||
echo "号码列表:\n\r";
|
||||
echo dd($finalData->numbers);
|
||||
} catch (Exception $e) {
|
||||
echo "🚨 抓取异常:" . $e->getMessage() . "\n\r";
|
||||
}
|
||||
|
||||
|
||||
// try {
|
||||
@@ -78,27 +80,27 @@ require_once __DIR__ . '/Whatshub.php'; // Whatshub 工单平台
|
||||
// echo "🚨 抓取异常:" . $e->getMessage() . "\n\r";
|
||||
// }
|
||||
|
||||
try {
|
||||
echo "🚀 开始执行<A2c云控>抓取任务 (多引擎智能调度)...\n\r";
|
||||
$pageUrl = 'https://yyk.ink/52oXZRG'; // PageUrl 入口授权页
|
||||
$username = ""; // 登录账号
|
||||
$password = ""; // 登录密码
|
||||
$spider = new A2c($pageUrl, $username, $password);
|
||||
$finalData = $spider->run();
|
||||
// try {
|
||||
// echo "🚀 开始执行<A2c云控>抓取任务 (多引擎智能调度)...\n\r";
|
||||
// $pageUrl = 'https://yyk.ink/52oXZRG'; // PageUrl 入口授权页
|
||||
// $username = ""; // 登录账号
|
||||
// $password = ""; // 登录密码
|
||||
// $spider = new A2c($pageUrl, $username, $password);
|
||||
// $finalData = $spider->run();
|
||||
|
||||
echo "✅ 任务完成!统一数据如下:\n\r";
|
||||
echo "----------------------------------------\n\r";
|
||||
echo "当日新增:{$finalData->todayNewCount} 人\n\r";
|
||||
echo "在线号码:{$finalData->totalOnline} 个\n\r";
|
||||
echo "离线号码:{$finalData->totalOffline} 个\n\r";
|
||||
echo "Total:" . $finalData->total . " 个号码\n\r";
|
||||
echo "实际总共抓取:" . count($finalData->numbers) . " 个号码\n\r";
|
||||
echo "号码列表:\n\r";
|
||||
echo dd($finalData->numbers);
|
||||
// echo "✅ 任务完成!统一数据如下:\n\r";
|
||||
// echo "----------------------------------------\n\r";
|
||||
// echo "当日新增:{$finalData->todayNewCount} 人\n\r";
|
||||
// echo "在线号码:{$finalData->totalOnline} 个\n\r";
|
||||
// echo "离线号码:{$finalData->totalOffline} 个\n\r";
|
||||
// echo "Total:" . $finalData->total . " 个号码\n\r";
|
||||
// echo "实际总共抓取:" . count($finalData->numbers) . " 个号码\n\r";
|
||||
// echo "号码列表:\n\r";
|
||||
// echo dd($finalData->numbers);
|
||||
|
||||
} catch (Exception $e) {
|
||||
echo "🚨 抓取异常:" . $e->getMessage() . "\n\r";
|
||||
}
|
||||
// } catch (Exception $e) {
|
||||
// echo "🚨 抓取异常:" . $e->getMessage() . "\n\r";
|
||||
// }
|
||||
|
||||
// try {
|
||||
// echo "🚀 开始执行<星河云控>抓取任务 (多引擎智能调度)...\n\r";
|
||||
|
||||
Reference in New Issue
Block a user