You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
191 lines
9.2 KiB
PHP
191 lines
9.2 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\User;
|
|
use App\Models\CloudTrafficUsageSettlementBucket;
|
|
use Illuminate\Database\Connection;
|
|
|
|
/** Read-only bounded summary for traffic-storage compaction health. */
|
|
final class CloudTrafficStorageHealthService
|
|
{
|
|
private const WARNING_LAG_SECONDS = 3600;
|
|
private const ACCOUNTING_WARNING_SECONDS = 300;
|
|
private const RECENT_FAILURE_SECONDS = 3600;
|
|
private const INSTANCE_SCAN_LIMIT = 2000;
|
|
|
|
/**
|
|
* @return array{
|
|
* key:string,label:string,status:string,message:string,checked_at:int,
|
|
* metrics:array<string,int|bool>
|
|
* }
|
|
*/
|
|
public static function snapshot(?int $now = null): array
|
|
{
|
|
$now ??= time();
|
|
if ($now <= max(
|
|
CloudTrafficSettlementRetention::RETENTION_SECONDS,
|
|
CloudTrafficCheckpointRetention::RETENTION_SECONDS
|
|
)) {
|
|
throw new \InvalidArgumentException(
|
|
'traffic_storage_health_clock_invalid'
|
|
);
|
|
}
|
|
$main = User::getConnectionResolver()->connection();
|
|
$log = User::getConnectionResolver()->connection('log');
|
|
foreach ([
|
|
'cloud_traffic_usage_settlement_buckets',
|
|
'cloud_traffic_ledger_events', 'cloud_traffic_instance_cursors',
|
|
'cloud_instances', 'cloud_service_periods',
|
|
'cloud_traffic_pool_groups', 'cloud_traffic_pools', 'cloud_traffic_scan_failures',
|
|
] as $table) {
|
|
if (!$main->getSchemaBuilder()->hasTable($table)) {
|
|
throw new \RuntimeException(
|
|
'traffic_storage_health_schema_missing'
|
|
);
|
|
}
|
|
}
|
|
if (!$log->getSchemaBuilder()->hasTable(
|
|
'cloud_vm_traffic_checkpoints'
|
|
)) {
|
|
throw new \RuntimeException('traffic_storage_health_schema_missing');
|
|
}
|
|
return CloudTrafficConsistentRead::run(static fn (): array => self::readMetrics($main, $log, $now));
|
|
}
|
|
|
|
private static function readMetrics(Connection $main, Connection $log, int $now): array
|
|
{
|
|
$settlementCutoff = $now
|
|
- CloudTrafficSettlementRetention::RETENTION_SECONDS;
|
|
$checkpointCutoff = $now
|
|
- CloudTrafficCheckpointRetention::RETENTION_SECONDS;
|
|
$settlementQuery = $main->table(
|
|
'cloud_traffic_usage_settlement_buckets'
|
|
)->where('state', 'sealed')->where('updated_at', '<=', $settlementCutoff);
|
|
$settlementCount = (int) (clone $settlementQuery)->count();
|
|
$oldestSettlement = $settlementCount === 0 ? null
|
|
: (int) (clone $settlementQuery)->min('updated_at');
|
|
$checkpointQuery = $log->table('cloud_vm_traffic_checkpoints')
|
|
->where('instance_id', '>', 0)
|
|
->where('first_observed_at', '<=', $checkpointCutoff);
|
|
$checkpointCount = (int) (clone $checkpointQuery)->count();
|
|
$oldestCheckpoint = $checkpointCount === 0 ? null
|
|
: (int) (clone $checkpointQuery)->min('first_observed_at');
|
|
$dueInstanceCount = (int) (clone $checkpointQuery)
|
|
->distinct()->count('instance_id');
|
|
$attentionCount = $main->table('cloud_traffic_instance_cursors')->where('state', 'attention')->count();
|
|
$openQuery = CloudTrafficUsageSettlementBucket::where('state', 'open')->where('seal_at', '<=', $now);
|
|
$openBucketCount = (int) (clone $openQuery)->count();
|
|
$oldestOpenSealAt = $openBucketCount === 0 ? null
|
|
: (int) (clone $openQuery)->min('seal_at');
|
|
$openBucketRows = (clone $openQuery)->orderBy('seal_at')
|
|
->orderBy('id')->limit(self::INSTANCE_SCAN_LIMIT + 1)
|
|
->with('pool')->get();
|
|
$accountingScanTruncated = $openBucketRows->count()
|
|
> self::INSTANCE_SCAN_LIMIT;
|
|
if ($accountingScanTruncated) {
|
|
$openBucketRows = $openBucketRows->take(
|
|
self::INSTANCE_SCAN_LIMIT
|
|
);
|
|
}
|
|
$identityConflicts = count(CloudTrafficSettlementEnvelope::conflictingBucketIds($openBucketRows));
|
|
$unsettledQuery = CloudTrafficAccountingEvidence::pendingInstances();
|
|
$unsettledInstances = (int) (clone $unsettledQuery)
|
|
->distinct()->count('cloud_instances.id');
|
|
$cutoff = $now - self::ACCOUNTING_WARNING_SECONDS;
|
|
$staleCursorInstances = (clone $unsettledQuery)->where(static function ($query) use ($cutoff): void {
|
|
$query->where('traffic_cursor.last_observed_at', '<=', $cutoff)
|
|
->orWhere(static function ($missing) use ($cutoff): void {
|
|
$missing->whereNull('traffic_cursor.instance_id')->where('cloud_instances.traffic_synced_at', '<=', $cutoff);
|
|
});
|
|
})->count();
|
|
$staleSamples = $main->table('cloud_instances as vm')->where('vm.status', 'active')
|
|
->where('vm.traffic_synced_at', '<=', $cutoff)->where('vm.created_at', '<=', $cutoff)
|
|
->whereExists(static function ($query) use ($now): void {
|
|
$query->selectRaw('1')->from('cloud_service_periods as period')
|
|
->join('cloud_traffic_pool_groups as grp', 'grp.id', '=', 'period.traffic_pool_group_id')
|
|
->whereColumn('period.instance_id', 'vm.id')->whereIn('grp.mode', ['shadow', 'enforced'])
|
|
->whereIn('period.status', ['active', 'exhausted'])->where('period.traffic_quota_mode', 'finite')
|
|
->where('period.starts_at', '<=', $now)->where('period.ends_at', '>', $now);
|
|
})->count();
|
|
$recentAccountingFailures = (int) $main->table(
|
|
'cloud_traffic_scan_failures'
|
|
)->where('last_failed_at', '>=',
|
|
$now - self::RECENT_FAILURE_SECONDS)->count();
|
|
$oldestOpenLag = $oldestOpenSealAt === null
|
|
? 0 : max(0, $now - $oldestOpenSealAt);
|
|
$oldestLag = max(
|
|
$oldestSettlement === null
|
|
? 0 : $settlementCutoff - $oldestSettlement,
|
|
$oldestCheckpoint === null
|
|
? 0 : $checkpointCutoff - $oldestCheckpoint
|
|
);
|
|
$metrics = [
|
|
'settlement_buckets_due' => $settlementCount,
|
|
'hourly_checkpoints_due' => $checkpointCount,
|
|
'hourly_checkpoint_instances_due' => $dueInstanceCount,
|
|
'cursor_attention_instances' => $attentionCount,
|
|
'open_buckets_due' => $openBucketCount,
|
|
'oldest_open_bucket_lag_seconds' => $oldestOpenLag,
|
|
'settlement_identity_conflicts' => $identityConflicts,
|
|
'unsettled_instances' => $unsettledInstances,
|
|
'stale_cursor_instances' => $staleCursorInstances,
|
|
'stale_sample_instances' => $staleSamples,
|
|
'recent_accounting_failures' => $recentAccountingFailures,
|
|
'oldest_due_lag_seconds' => max(0, $oldestLag),
|
|
'instance_scan_truncated' => false, // Checkpoint instance count uses an exact SQL aggregate.
|
|
'accounting_scan_truncated' => $accountingScanTruncated,
|
|
];
|
|
$status = $attentionCount > 0 || $staleSamples > 0
|
|
|| $accountingScanTruncated
|
|
|| $oldestLag > self::WARNING_LAG_SECONDS
|
|
|| $oldestOpenLag > self::ACCOUNTING_WARNING_SECONDS
|
|
|| $identityConflicts > 0 || $staleCursorInstances > 0
|
|
|| $recentAccountingFailures > 0
|
|
? 'warning' : 'ok';
|
|
if ($settlementCount === 0 && $checkpointCount === 0
|
|
&& $openBucketCount === 0 && $unsettledInstances === 0
|
|
&& $status === 'ok') {
|
|
$message = '流量结算与存储保留队列均处于正常窗口内。';
|
|
} else {
|
|
$message = '到期未封账 ' . $openBucketCount . ' 个(最长延迟 '
|
|
. self::duration($oldestOpenLag) . '),未结算实例 '
|
|
. $unsettledInstances . ' 个,采样滞后 ' . $staleSamples . ' 个,结算游标滞后 '
|
|
. $staleCursorInstances . ' 个,身份冲突 '
|
|
. $identityConflicts . ' 个,近一小时失败 '
|
|
. $recentAccountingFailures . ' 个;待压缩结算桶 '
|
|
. $settlementCount
|
|
. ' 个,小时检查点 ' . $checkpointCount . ' 行(涉及 '
|
|
. $dueInstanceCount . ' 个实例);最旧超出窗口 '
|
|
. self::duration(max(0, $oldestLag)) . ',其中流量游标异常 '
|
|
. $attentionCount . ' 个实例。';
|
|
if ($accountingScanTruncated) {
|
|
$message .= ' 实例计数已达到有界扫描上限。';
|
|
}
|
|
if ($identityConflicts > 0
|
|
|| $oldestOpenLag > self::WARNING_LAG_SECONDS) {
|
|
$message .= ' 结算账务需立即核对。';
|
|
}
|
|
}
|
|
return [
|
|
'key' => 'traffic_storage',
|
|
'label' => '流量结算与存储',
|
|
'status' => $status,
|
|
'message' => $message,
|
|
'checked_at' => $now,
|
|
'metrics' => $metrics,
|
|
];
|
|
}
|
|
|
|
private static function duration(int $seconds): string
|
|
{
|
|
if ($seconds >= 86400) {
|
|
return intdiv($seconds, 86400) . ' 天';
|
|
}
|
|
if ($seconds >= 3600) {
|
|
return intdiv($seconds, 3600) . ' 小时';
|
|
}
|
|
return intdiv($seconds, 60) . ' 分钟';
|
|
}
|
|
}
|