root
421 lines
15 KiB
PHP
Executable File
421 lines
15 KiB
PHP
Executable File
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
namespace app\common\library\scrm\spider;
|
|
|
|
use app\common\library\scrm\AbstractScrmSpider;
|
|
use app\common\library\scrm\AntiBotConfigBuilder;
|
|
use app\common\library\scrm\UnifiedScrmData;
|
|
use app\common\service\SplitTicketSyncLogger;
|
|
|
|
/**
|
|
* Whatshub 云控蜘蛛
|
|
*
|
|
* 时序：cURL detailByShareCode 拉号码 → Node 监听 statistics 算完成量 → 失败则完整 Node 爬虫
|
|
*/
|
|
class WhatshubSpider extends AbstractScrmSpider
|
|
{
|
|
/** 列表 API 路径 */
|
|
private const API_LIST = '/api/whatshub-counter/workShare/open/detail';
|
|
|
|
/** 详情/统计 API 路径 */
|
|
private const API_DETAILS = '/api/whatshub-counter/workShare/open/statistics';
|
|
|
|
/** shareCode + password 直连接口（一次返回全量号码，无需翻页） */
|
|
private const API_DETAIL_BY_SHARE_CODE = '/api/whatshub-counter/workShare/open/detailByShareCode';
|
|
|
|
/** 默认每页条数 */
|
|
private const DEFAULT_PER_PAGE_COUNT = 20;
|
|
|
|
private const API_FETCH_TIMEOUT_SEC = 20;
|
|
|
|
private string $pageUrl;
|
|
|
|
private string $account;
|
|
|
|
private string $password;
|
|
|
|
private UnifiedScrmData $unifiedData;
|
|
|
|
public function __construct(
|
|
string $pageUrl,
|
|
string $account = '',
|
|
string $password = '',
|
|
string $nodeHost = 'http://127.0.0.1:3001'
|
|
) {
|
|
parent::__construct($nodeHost);
|
|
$this->pageUrl = $pageUrl;
|
|
$this->account = $account;
|
|
$this->password = $password;
|
|
$this->unifiedData = new UnifiedScrmData();
|
|
}
|
|
|
|
/**
|
|
* 混合同步：cURL 拉号码 + Node 监听 statistics 算完成量
|
|
* statistics 失败时清 session 重试；仍失败则 fallback 完整 Node 仅取完成量
|
|
*/
|
|
public function run(): UnifiedScrmData
|
|
{
|
|
$apiResult = $this->tryFetchViaShareCodeApi();
|
|
$statsCount = $this->tryFetchStatisticsViaNode(false);
|
|
|
|
if ($apiResult instanceof UnifiedScrmData && $statsCount !== null) {
|
|
$apiResult->todayNewCount = $statsCount;
|
|
SplitTicketSyncLogger::log('spider', 'whatshub hybrid path ok', [
|
|
'numberCount' => count($apiResult->numbers),
|
|
'todayNewCount' => $statsCount,
|
|
]);
|
|
return $apiResult;
|
|
}
|
|
|
|
// cURL 已成功：清 session 再试 statistics；仍失败则完整 Node 只取完成量
|
|
if ($apiResult instanceof UnifiedScrmData) {
|
|
SplitTicketSyncLogger::log('spider', 'whatshub stats retry after purge session', [
|
|
'numberCount' => count($apiResult->numbers),
|
|
]);
|
|
$statsCount = $this->tryFetchStatisticsViaNode(true);
|
|
if ($statsCount !== null) {
|
|
$apiResult->todayNewCount = $statsCount;
|
|
SplitTicketSyncLogger::log('spider', 'whatshub hybrid path ok (retry)', [
|
|
'numberCount' => count($apiResult->numbers),
|
|
'todayNewCount' => $statsCount,
|
|
]);
|
|
return $apiResult;
|
|
}
|
|
|
|
SplitTicketSyncLogger::log('spider', 'whatshub stats fallback full node for complete count', [
|
|
'pageUrl' => $this->pageUrl,
|
|
'numberCount' => count($apiResult->numbers),
|
|
]);
|
|
try {
|
|
$fullResult = parent::run();
|
|
$apiResult->todayNewCount = $fullResult->todayNewCount;
|
|
SplitTicketSyncLogger::log('spider', 'whatshub hybrid path ok (full node stats)', [
|
|
'numberCount' => count($apiResult->numbers),
|
|
'todayNewCount' => $apiResult->todayNewCount,
|
|
]);
|
|
return $apiResult;
|
|
} catch (\Throwable $e) {
|
|
SplitTicketSyncLogger::log('spider', 'whatshub stats fallback full node failed', [
|
|
'error' => $e->getMessage(),
|
|
]);
|
|
}
|
|
|
|
throw new \RuntimeException('Whatshub statistics 同步失败（号码已通过 API 获取，无法更新完成量）');
|
|
}
|
|
|
|
SplitTicketSyncLogger::log('spider', 'whatshub hybrid incomplete, fallback full node', [
|
|
'pageUrl' => $this->pageUrl,
|
|
'apiOk' => false,
|
|
'statisticsOk' => $statsCount !== null,
|
|
]);
|
|
|
|
return parent::run();
|
|
}
|
|
|
|
/** Whatshub vxe 密码表单：placeholder 含「密码」+ 确认按钮，与业务页搜索框区分 */
|
|
private const AUTH_PASSWORD_INPUT = '.vxe-form .el-input__inner[placeholder*="密码"]';
|
|
|
|
private const AUTH_CONFIRM_BUTTON = '.vxe-form .vxe-button-group button.theme--primary[type="submit"]';
|
|
|
|
/**
|
|
* Node 监听 statistics API，返回 dayNewFans + reDayNewFans 总和；失败返回 null
|
|
*
|
|
* @param bool $isRetry 是否为清 session 后的二次尝试
|
|
*/
|
|
private function tryFetchStatisticsViaNode(bool $isRetry = false): ?int
|
|
{
|
|
$config = $this->getSpiderConfig();
|
|
$antiBot = $config['antiBot'] ?? null;
|
|
if (is_array($antiBot)) {
|
|
// statistics-only：早挂拦截器、识别 vxe 密码框、session 有效时 skip auth
|
|
$antiBot = array_merge($antiBot, [
|
|
'skipPostCfReadyApi' => true,
|
|
'authSkipIfNoPasswordDialog' => true,
|
|
'authSelectorTimeoutMs' => 30000,
|
|
'earlyApiIntercept' => true,
|
|
]);
|
|
if ($isRetry) {
|
|
$antiBot['purgeSession'] = true;
|
|
}
|
|
}
|
|
|
|
SplitTicketSyncLogger::log('spider', 'whatshub stats node request', [
|
|
'pageUrl' => $this->pageUrl,
|
|
'api' => self::API_DETAILS,
|
|
'isRetry' => $isRetry,
|
|
]);
|
|
|
|
$result = $this->requestNode('/api/auth-and-intercept', [
|
|
'pageUrl' => $config['pageUrl'],
|
|
'apiUrls' => [self::API_DETAILS],
|
|
'authActions' => $config['authActions'] ?? [],
|
|
'antiBot' => $antiBot,
|
|
], $this->resolveAuthInterceptTimeout(is_array($antiBot) ? $antiBot : null));
|
|
|
|
if (empty($result['success'])) {
|
|
SplitTicketSyncLogger::log('spider', 'whatshub stats node failed', [
|
|
'error' => $result['error'] ?? '未知',
|
|
'code' => $result['code'] ?? null,
|
|
'page' => $result['page'] ?? null,
|
|
]);
|
|
return null;
|
|
}
|
|
|
|
$intercepted = $result['interceptedApis'] ?? [];
|
|
if (!isset($intercepted[self::API_DETAILS]['data']) || !is_array($intercepted[self::API_DETAILS]['data'])) {
|
|
SplitTicketSyncLogger::log('spider', 'whatshub stats node missing payload', [
|
|
'intercepted' => array_keys(is_array($intercepted) ? $intercepted : []),
|
|
'page' => $result['page'] ?? null,
|
|
]);
|
|
return null;
|
|
}
|
|
|
|
$detailData = $intercepted[self::API_DETAILS]['data'];
|
|
$dayNewFans = (int) ($detailData['data']['dayNewFans'] ?? 0);
|
|
$reDayNewFans = (int) ($detailData['data']['reDayNewFans'] ?? 0);
|
|
$total = $dayNewFans + $reDayNewFans;
|
|
|
|
SplitTicketSyncLogger::log('spider', 'whatshub stats node ok', [
|
|
'dayNewFans' => $dayNewFans,
|
|
'reDayNewFans' => $reDayNewFans,
|
|
'total' => $total,
|
|
]);
|
|
|
|
return $total;
|
|
}
|
|
|
|
/**
|
|
* 从短链 pageUrl 解析 shareCode，例如 /m/KMYBBInp2066/1 → KMYBBInp2066
|
|
*/
|
|
private function extractShareCodeFromPageUrl(): ?string
|
|
{
|
|
$path = (string) parse_url($this->pageUrl, PHP_URL_PATH);
|
|
if ($path === '') {
|
|
return null;
|
|
}
|
|
if (preg_match('#/m/([^/]+)/#', $path, $matches)) {
|
|
return $matches[1];
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* 拼接 detailByShareCode 完整 GET URL
|
|
*/
|
|
private function buildShareCodeApiUrl(string $shareCode): ?string
|
|
{
|
|
$parsed = parse_url($this->pageUrl);
|
|
$scheme = isset($parsed['scheme']) ? (string) $parsed['scheme'] : 'https';
|
|
$host = isset($parsed['host']) ? (string) $parsed['host'] : '';
|
|
if ($host === '') {
|
|
return null;
|
|
}
|
|
|
|
$query = http_build_query([
|
|
'shareCode' => $shareCode,
|
|
'password' => $this->password,
|
|
]);
|
|
|
|
return $scheme . '://' . $host . self::API_DETAIL_BY_SHARE_CODE . '?' . $query;
|
|
}
|
|
|
|
/**
|
|
* 尝试通过 shareCode API 抓取；成功返回 UnifiedScrmData，否则返回 null
|
|
*/
|
|
private function tryFetchViaShareCodeApi(): ?UnifiedScrmData
|
|
{
|
|
if ($this->password === '') {
|
|
SplitTicketSyncLogger::log('spider', 'whatshub api skip: empty password', []);
|
|
return null;
|
|
}
|
|
|
|
$shareCode = $this->extractShareCodeFromPageUrl();
|
|
if ($shareCode === null || $shareCode === '') {
|
|
SplitTicketSyncLogger::log('spider', 'whatshub api skip: cannot parse shareCode', [
|
|
'pageUrl' => $this->pageUrl,
|
|
]);
|
|
return null;
|
|
}
|
|
|
|
$apiUrl = $this->buildShareCodeApiUrl($shareCode);
|
|
if ($apiUrl === null) {
|
|
SplitTicketSyncLogger::log('spider', 'whatshub api skip: invalid pageUrl host', [
|
|
'pageUrl' => $this->pageUrl,
|
|
]);
|
|
return null;
|
|
}
|
|
|
|
SplitTicketSyncLogger::log('spider', 'whatshub api request', [
|
|
'url' => preg_replace('/password=[^&]+/', 'password=***', $apiUrl),
|
|
]);
|
|
|
|
$ch = curl_init($apiUrl);
|
|
if ($ch === false) {
|
|
SplitTicketSyncLogger::log('spider', 'whatshub api curl init failed', []);
|
|
return null;
|
|
}
|
|
|
|
curl_setopt_array($ch, [
|
|
CURLOPT_RETURNTRANSFER => true,
|
|
CURLOPT_CONNECTTIMEOUT => 5,
|
|
CURLOPT_TIMEOUT => self::API_FETCH_TIMEOUT_SEC,
|
|
CURLOPT_HTTPHEADER => [
|
|
'Accept: application/json',
|
|
'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
],
|
|
]);
|
|
|
|
$response = curl_exec($ch);
|
|
$httpCode = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
|
$curlErr = curl_error($ch);
|
|
curl_close($ch);
|
|
|
|
if ($response === false || $curlErr !== '') {
|
|
SplitTicketSyncLogger::log('spider', 'whatshub api curl failed', [
|
|
'error' => $curlErr,
|
|
]);
|
|
return null;
|
|
}
|
|
|
|
if ($httpCode !== 200) {
|
|
SplitTicketSyncLogger::log('spider', 'whatshub api http error', [
|
|
'httpCode' => $httpCode,
|
|
]);
|
|
return null;
|
|
}
|
|
|
|
$payload = json_decode((string) $response, true);
|
|
if (!is_array($payload)) {
|
|
SplitTicketSyncLogger::log('spider', 'whatshub api invalid json', []);
|
|
return null;
|
|
}
|
|
|
|
if ((int) ($payload['code'] ?? 0) !== 200) {
|
|
SplitTicketSyncLogger::log('spider', 'whatshub api business error', [
|
|
'code' => $payload['code'] ?? null,
|
|
'msg' => $payload['msg'] ?? '',
|
|
]);
|
|
return null;
|
|
}
|
|
|
|
if (!isset($payload['data']) || !is_array($payload['data'])) {
|
|
SplitTicketSyncLogger::log('spider', 'whatshub api invalid data field', []);
|
|
return null;
|
|
}
|
|
|
|
return $this->parseShareCodeApiResponse($payload);
|
|
}
|
|
|
|
/**
|
|
* 将 detailByShareCode 响应映射为 UnifiedScrmData
|
|
*
|
|
* @param array<string, mixed> $payload
|
|
*/
|
|
private function parseShareCodeApiResponse(array $payload): UnifiedScrmData
|
|
{
|
|
$unifiedData = new UnifiedScrmData();
|
|
|
|
foreach ($payload['data'] as $item) {
|
|
if (!is_array($item) || empty($item['account'])) {
|
|
continue;
|
|
}
|
|
|
|
$number = (string) $item['account'];
|
|
$isOnline = isset($item['isOnline']) && (int) $item['isOnline'] === 1;
|
|
$dayNewFans = (int) ($item['dayNewFans'] ?? 0);
|
|
|
|
// 号码同步仅使用 dayNewFans，不含 reDayNewFans；完成量由 Node statistics 提供
|
|
$unifiedData->addNumber($number, $isOnline, $dayNewFans);
|
|
}
|
|
|
|
$unifiedData->total = count($unifiedData->numbers);
|
|
|
|
return $unifiedData;
|
|
}
|
|
|
|
/** @return array<string, mixed> */
|
|
protected function getSpiderConfig(): array
|
|
{
|
|
return [
|
|
'pageUrl' => $this->pageUrl,
|
|
'listApi' => self::API_LIST,
|
|
'detailApi' => self::API_DETAILS,
|
|
'listMethod' => 'POST',
|
|
'paginationMode' => self::MODE_UI,
|
|
'authActions' => [
|
|
// Whatshub 密码 UI 为 vxe-form + Element input，限定在表单内避免误填搜索框
|
|
['type' => 'vue_fill', 'selector' => self::AUTH_PASSWORD_INPUT, 'value' => $this->password],
|
|
['type' => 'wait', 'ms' => 500],
|
|
['type' => 'vue_click', 'selector' => self::AUTH_CONFIRM_BUTTON],
|
|
['type' => 'wait', 'ms' => 2200],
|
|
],
|
|
'antiBot' => AntiBotConfigBuilder::build('whatshub', $this->pageUrl),
|
|
];
|
|
}
|
|
|
|
/**
|
|
* @param array<string, mixed>|null $listFirstPageData
|
|
* @param array<string, mixed>|null $countData
|
|
*/
|
|
protected function extractListTotalPages($listFirstPageData, $countData = null)
|
|
{
|
|
$defaultPerPage = self::DEFAULT_PER_PAGE_COUNT;
|
|
$this->unifiedData->total = (int) ($listFirstPageData['data']['total'] ?? 0);
|
|
|
|
if ($this->unifiedData->total <= $defaultPerPage) {
|
|
return 1;
|
|
}
|
|
|
|
return (int) ceil($this->unifiedData->total / $defaultPerPage);
|
|
}
|
|
|
|
/** @return array<string, mixed> */
|
|
protected function buildListPageParams(int $page): array
|
|
{
|
|
return ['pageNum' => $page, 'pageSize' => self::DEFAULT_PER_PAGE_COUNT];
|
|
}
|
|
|
|
/** @return array<string, mixed> */
|
|
protected function getUiPaginationConfig(): array
|
|
{
|
|
return [
|
|
'nextBtnSelector' => '.vxe-pager--next-btn',
|
|
'waitMs' => 1500,
|
|
];
|
|
}
|
|
|
|
protected function parseToUnifiedData($detailData, array $allListPagesData): UnifiedScrmData
|
|
{
|
|
$unifiedData = $this->unifiedData;
|
|
|
|
if (is_array($detailData)) {
|
|
$unifiedData->todayNewCount = (int) ($detailData['data']['dayNewFans'] ?? 0) + (int) ($detailData['data']['reDayNewFans'] ?? 0);
|
|
}
|
|
|
|
foreach ($allListPagesData as $pageRaw) {
|
|
if (!is_array($pageRaw)) {
|
|
continue;
|
|
}
|
|
$records = $pageRaw['data']['rows'] ?? [];
|
|
if (!is_array($records)) {
|
|
continue;
|
|
}
|
|
foreach ($records as $item) {
|
|
if (!is_array($item) || empty($item['account'])) {
|
|
continue;
|
|
}
|
|
$number = (string) $item['account'];
|
|
$isOnline = isset($item['isOnline']) && (int) $item['isOnline'] === 1;
|
|
$unifiedData->addNumber($number, $isOnline, (int) ($item['dayNewFans'] ?? 0));
|
|
}
|
|
}
|
|
|
|
$unifiedData->total = count($unifiedData->numbers);
|
|
|
|
return $unifiedData;
|
|
}
|
|
}
|