修改whatshub工单爬虫
This commit is contained in:
root
+198
-30
@@ -7,13 +7,17 @@ class Whatshub extends AbstractScrmSpider
|
||||
{
|
||||
const API_LIST = '/api/whatshub-counter/workShare/open/detail'; // 列表API地址
|
||||
const API_DETAILS = '/api/whatshub-counter/workShare/open/statistics'; // 详情API地址
|
||||
/** shareCode + password 直连接口（一次返回全量号码，无需翻页） */
|
||||
const API_DETAIL_BY_SHARE_CODE = '/api/whatshub-counter/workShare/open/detailByShareCode';
|
||||
const API_COUNT = ''; // 总数API地址
|
||||
const DEFAULT_PER_PAGE_COUNT = 20; // List默认每页显示的数量
|
||||
const API_FETCH_TIMEOUT_SEC = 20;
|
||||
|
||||
private $pageUrl;
|
||||
private $account;
|
||||
private $password;
|
||||
private $unifiedData;
|
||||
|
||||
|
||||
// 实例化时动态传入账号和密码
|
||||
public function __construct($pageUrl, $account, $password, $nodeHost = 'http://127.0.0.1:3001')
|
||||
{
|
||||
@@ -21,46 +25,210 @@ class Whatshub extends AbstractScrmSpider
|
||||
$this->account = $account;
|
||||
$this->password = $password;
|
||||
$this->pageUrl = $pageUrl;
|
||||
|
||||
|
||||
$this->unifiedData = new UnifiedScrmData();
|
||||
}
|
||||
|
||||
/**
|
||||
* API 优先：detailByShareCode 直抓；失败则回退 Node 爬虫
|
||||
*/
|
||||
public function run()
|
||||
{
|
||||
$apiResult = $this->tryFetchViaShareCodeApi();
|
||||
if ($apiResult instanceof UnifiedScrmData) {
|
||||
$this->logDebug('whatshub api path ok', ['count' => count($apiResult->numbers)]);
|
||||
return $apiResult;
|
||||
}
|
||||
|
||||
$this->logDebug('whatshub api skipped or failed, fallback node', []);
|
||||
return parent::run();
|
||||
}
|
||||
|
||||
/**
|
||||
* 从短链 pageUrl 解析 shareCode，例如 /m/KMYBBInp2066/1 → KMYBBInp2066
|
||||
*
|
||||
* @return string|null
|
||||
*/
|
||||
private function extractShareCodeFromPageUrl()
|
||||
{
|
||||
$path = (string) parse_url($this->pageUrl, PHP_URL_PATH);
|
||||
if ($path === '') {
|
||||
return null;
|
||||
}
|
||||
if (preg_match('#/m/([^/]+)/#', $path, $matches)) {
|
||||
return $matches[1];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 拼接 detailByShareCode 完整 GET URL
|
||||
*
|
||||
* @param string $shareCode
|
||||
* @return string|null
|
||||
*/
|
||||
private function buildShareCodeApiUrl($shareCode)
|
||||
{
|
||||
$parsed = parse_url($this->pageUrl);
|
||||
$scheme = isset($parsed['scheme']) ? $parsed['scheme'] : 'https';
|
||||
$host = isset($parsed['host']) ? $parsed['host'] : '';
|
||||
if ($host === '') {
|
||||
return null;
|
||||
}
|
||||
|
||||
$query = http_build_query([
|
||||
'shareCode' => $shareCode,
|
||||
'password' => $this->password,
|
||||
]);
|
||||
|
||||
return $scheme . '://' . $host . self::API_DETAIL_BY_SHARE_CODE . '?' . $query;
|
||||
}
|
||||
|
||||
/**
|
||||
* 尝试通过 shareCode API 抓取；成功返回 UnifiedScrmData，否则返回 null
|
||||
*
|
||||
* @return UnifiedScrmData|null
|
||||
*/
|
||||
private function tryFetchViaShareCodeApi()
|
||||
{
|
||||
if ($this->password === '' || $this->password === null) {
|
||||
$this->logDebug('whatshub api skip: empty password', []);
|
||||
return null;
|
||||
}
|
||||
|
||||
$shareCode = $this->extractShareCodeFromPageUrl();
|
||||
if ($shareCode === null || $shareCode === '') {
|
||||
$this->logDebug('whatshub api skip: cannot parse shareCode', ['pageUrl' => $this->pageUrl]);
|
||||
return null;
|
||||
}
|
||||
|
||||
$apiUrl = $this->buildShareCodeApiUrl($shareCode);
|
||||
if ($apiUrl === null) {
|
||||
$this->logDebug('whatshub api skip: invalid pageUrl host', ['pageUrl' => $this->pageUrl]);
|
||||
return null;
|
||||
}
|
||||
|
||||
$this->logDebug('whatshub api request', ['url' => preg_replace('/password=[^&]+/', 'password=***', $apiUrl)]);
|
||||
|
||||
$ch = curl_init($apiUrl);
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_CONNECTTIMEOUT => 5,
|
||||
CURLOPT_TIMEOUT => self::API_FETCH_TIMEOUT_SEC,
|
||||
CURLOPT_HTTPHEADER => [
|
||||
'Accept: application/json',
|
||||
'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
],
|
||||
]);
|
||||
|
||||
$response = curl_exec($ch);
|
||||
$httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
$curlErr = curl_error($ch);
|
||||
curl_close($ch);
|
||||
|
||||
if ($response === false || $curlErr !== '') {
|
||||
$this->logDebug('whatshub api curl failed', ['error' => $curlErr]);
|
||||
return null;
|
||||
}
|
||||
|
||||
if ($httpCode !== 200) {
|
||||
$this->logDebug('whatshub api http error', ['httpCode' => $httpCode]);
|
||||
return null;
|
||||
}
|
||||
|
||||
$payload = json_decode($response, true);
|
||||
if (!is_array($payload)) {
|
||||
$this->logDebug('whatshub api invalid json', []);
|
||||
return null;
|
||||
}
|
||||
|
||||
if ((int) ($payload['code'] ?? 0) !== 200) {
|
||||
$this->logDebug('whatshub api business error', [
|
||||
'code' => $payload['code'] ?? null,
|
||||
'msg' => $payload['msg'] ?? '',
|
||||
]);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!isset($payload['data']) || !is_array($payload['data'])) {
|
||||
$this->logDebug('whatshub api invalid data field', []);
|
||||
return null;
|
||||
}
|
||||
|
||||
return $this->parseShareCodeApiResponse($payload);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 detailByShareCode 响应映射为 UnifiedScrmData
|
||||
*
|
||||
* @param array<string, mixed> $payload
|
||||
* @return UnifiedScrmData
|
||||
*/
|
||||
private function parseShareCodeApiResponse(array $payload)
|
||||
{
|
||||
$unifiedData = new UnifiedScrmData();
|
||||
$todayNewSum = 0;
|
||||
|
||||
foreach ($payload['data'] as $item) {
|
||||
if (!is_array($item) || empty($item['account'])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$number = (string) $item['account'];
|
||||
$isOnline = isset($item['isOnline']) && (int) $item['isOnline'] === 1;
|
||||
$dayNewFans = (int) ($item['dayNewFans'] ?? 0);
|
||||
|
||||
$unifiedData->addNumber($number, $isOnline, $dayNewFans);
|
||||
$todayNewSum += $dayNewFans;
|
||||
}
|
||||
|
||||
$unifiedData->todayNewCount = $todayNewSum;
|
||||
$unifiedData->total = count($unifiedData->numbers);
|
||||
|
||||
return $unifiedData;
|
||||
}
|
||||
|
||||
protected function getSpiderConfig()
|
||||
{
|
||||
$host = (string) parse_url($this->pageUrl, PHP_URL_HOST);
|
||||
return [
|
||||
'pageUrl' => $this->pageUrl,
|
||||
'apiUrls' => [self::API_LIST, self::API_DETAILS],
|
||||
|
||||
|
||||
// 明确指派角色
|
||||
'listApi' => self::API_LIST, // 必须
|
||||
|
||||
'listApi' => self::API_LIST,
|
||||
'detailApi' => self::API_DETAILS,
|
||||
|
||||
'listMethod' => 'POST',
|
||||
'paginationMode' => self::MODE_UI,
|
||||
|
||||
|
||||
'authActions' => [
|
||||
// 1. 填入密码：寻找 class 包含 el-message-box__input 下面的任意 input
|
||||
// Node.js 会自动扫描所有匹配项，并只把密码强行注入到那个“肉眼可见”的框里
|
||||
// Node antiBot.postCfReady* 会在 CF 成功后等待 work-order-sharing + .el-input__inner
|
||||
['type' => 'vue_fill', 'selector' => '.el-input__inner', 'value' => $this->password],
|
||||
|
||||
|
||||
// 2. 停顿 500ms，让 Vue 绑定的 v-model 彻底反应过来
|
||||
['type' => 'wait', 'ms' => 500],
|
||||
|
||||
|
||||
// 3. 点击确认：寻找 MessageBox 底部的蓝色 primary 确认按钮
|
||||
// 同样利用 vue_click 的可见性过滤，无视隐藏的旧弹窗按钮
|
||||
['type' => 'vue_click', 'selector' => '.vxe-button-group .theme--primary'],
|
||||
|
||||
|
||||
// 4. 等待弹窗淡出，接口开始请求
|
||||
['type' => 'wait', 'ms' => 2200]
|
||||
['type' => 'wait', 'ms' => 2200],
|
||||
],
|
||||
// Whatshub 专用：Real Browser + Turnstile + Captcha API 兜底 + 会话复用
|
||||
'antiBot' => [
|
||||
// sessionKey 必须与 pageUrl 的 host 一致，例如 web.whatshub.cc
|
||||
'antiBot' => [
|
||||
'enabled' => true,
|
||||
'profile' => 'real',
|
||||
'turnstile' => true,
|
||||
'solverFallback' => true,
|
||||
'sessionKey' => 'whatshub:' . $host,
|
||||
'challengeTimeoutMs' => 60000,
|
||||
'challengeTimeoutMs' => 40000,
|
||||
'postCfReadyUrlContains' => 'work-order-sharing',
|
||||
'postCfReadySelector' => '.el-input__inner',
|
||||
'postCfReadyTimeoutMs' => 30000,
|
||||
'postCfSettleMs' => 500,
|
||||
'postCfSpaWaitMs' => 3000,
|
||||
],
|
||||
];
|
||||
}
|
||||
@@ -69,13 +237,13 @@ class Whatshub extends AbstractScrmSpider
|
||||
protected function extractListTotalPages($listFirstPageData, $countData = null)
|
||||
{
|
||||
$default_per_page_count = self::DEFAULT_PER_PAGE_COUNT;
|
||||
$this->unifiedData->total = $listFirstPageData['data']['total'];
|
||||
|
||||
if($this->unifiedData->total <= $default_per_page_count) {
|
||||
$this->unifiedData->total = $listFirstPageData['total'];
|
||||
|
||||
if ($this->unifiedData->total <= $default_per_page_count) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
return ceil($this->unifiedData->total/$default_per_page_count);
|
||||
|
||||
return ceil($this->unifiedData->total / $default_per_page_count);
|
||||
}
|
||||
|
||||
// 没有分页返回空数组
|
||||
@@ -88,8 +256,8 @@ class Whatshub extends AbstractScrmSpider
|
||||
protected function getUiPaginationConfig()
|
||||
{
|
||||
return [
|
||||
'nextBtnSelector' => '.vxe-pager--next-btn',
|
||||
'waitMs' => 1500
|
||||
'nextBtnSelector' => '.vxe-pager--next-btn',
|
||||
'waitMs' => 1500,
|
||||
];
|
||||
}
|
||||
|
||||
@@ -97,28 +265,28 @@ class Whatshub extends AbstractScrmSpider
|
||||
protected function parseToUnifiedData($detailData, $allListPagesData)
|
||||
{
|
||||
$unifiedData = $this->unifiedData;
|
||||
|
||||
|
||||
// 1. 如果捕获到了详情数据，提取今日新增
|
||||
if ($detailData) {
|
||||
$unifiedData->todayNewCount = (int)($detailData['data']['dayNewFans'] ?? 0);
|
||||
$unifiedData->todayNewCount = (int) ($detailData['data']['dayNewFans'] ?? 0);
|
||||
}
|
||||
|
||||
// 2. 循环合并了所有页数的 List 数组
|
||||
foreach ($allListPagesData as $pageRaw) {
|
||||
$records = $pageRaw['data']['rows'] ?? [];
|
||||
$records = $pageRaw['rows'] ?? [];
|
||||
foreach ($records as $item) {
|
||||
if(!empty($item['account'])) {
|
||||
if (!empty($item['account'])) {
|
||||
$number = $item['account'] ?? null;
|
||||
$isOnline = (isset($item['isOnline']) && $item['isOnline'] == 1);
|
||||
|
||||
|
||||
$unifiedData->addNumber($number, $isOnline, $item['dayNewFans']);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 🚀 3. 终极统计：所有翻页数据均已入库，此时再统计真实的 Total 总数
|
||||
|
||||
// 3. 终极统计：所有翻页数据均已入库，此时再统计真实的 Total 总数
|
||||
$unifiedData->total = count($unifiedData->numbers);
|
||||
|
||||
return $unifiedData;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user