You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudTrafficStorageHealthSe...

191 lines
9.2 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\User;
use App\Models\CloudTrafficUsageSettlementBucket;
use Illuminate\Database\Connection;
/** Read-only bounded summary for traffic-storage compaction health. */
final class CloudTrafficStorageHealthService
{
private const WARNING_LAG_SECONDS = 3600;
private const ACCOUNTING_WARNING_SECONDS = 300;
private const RECENT_FAILURE_SECONDS = 3600;
private const INSTANCE_SCAN_LIMIT = 2000;
/**
* @return array{
* key:string,label:string,status:string,message:string,checked_at:int,
* metrics:array<string,int|bool>
* }
*/
public static function snapshot(?int $now = null): array
{
$now ??= time();
if ($now <= max(
CloudTrafficSettlementRetention::RETENTION_SECONDS,
CloudTrafficCheckpointRetention::RETENTION_SECONDS
)) {
throw new \InvalidArgumentException(
'traffic_storage_health_clock_invalid'
);
}
$main = User::getConnectionResolver()->connection();
$log = User::getConnectionResolver()->connection('log');
foreach ([
'cloud_traffic_usage_settlement_buckets',
'cloud_traffic_ledger_events', 'cloud_traffic_instance_cursors',
'cloud_instances', 'cloud_service_periods',
'cloud_traffic_pool_groups', 'cloud_traffic_pools', 'cloud_traffic_scan_failures',
] as $table) {
if (!$main->getSchemaBuilder()->hasTable($table)) {
throw new \RuntimeException(
'traffic_storage_health_schema_missing'
);
}
}
if (!$log->getSchemaBuilder()->hasTable(
'cloud_vm_traffic_checkpoints'
)) {
throw new \RuntimeException('traffic_storage_health_schema_missing');
}
return CloudTrafficConsistentRead::run(static fn (): array => self::readMetrics($main, $log, $now));
}
private static function readMetrics(Connection $main, Connection $log, int $now): array
{
$settlementCutoff = $now
- CloudTrafficSettlementRetention::RETENTION_SECONDS;
$checkpointCutoff = $now
- CloudTrafficCheckpointRetention::RETENTION_SECONDS;
$settlementQuery = $main->table(
'cloud_traffic_usage_settlement_buckets'
)->where('state', 'sealed')->where('updated_at', '<=', $settlementCutoff);
$settlementCount = (int) (clone $settlementQuery)->count();
$oldestSettlement = $settlementCount === 0 ? null
: (int) (clone $settlementQuery)->min('updated_at');
$checkpointQuery = $log->table('cloud_vm_traffic_checkpoints')
->where('instance_id', '>', 0)
->where('first_observed_at', '<=', $checkpointCutoff);
$checkpointCount = (int) (clone $checkpointQuery)->count();
$oldestCheckpoint = $checkpointCount === 0 ? null
: (int) (clone $checkpointQuery)->min('first_observed_at');
$dueInstanceCount = (int) (clone $checkpointQuery)
->distinct()->count('instance_id');
$attentionCount = $main->table('cloud_traffic_instance_cursors')->where('state', 'attention')->count();
$openQuery = CloudTrafficUsageSettlementBucket::where('state', 'open')->where('seal_at', '<=', $now);
$openBucketCount = (int) (clone $openQuery)->count();
$oldestOpenSealAt = $openBucketCount === 0 ? null
: (int) (clone $openQuery)->min('seal_at');
$openBucketRows = (clone $openQuery)->orderBy('seal_at')
->orderBy('id')->limit(self::INSTANCE_SCAN_LIMIT + 1)
->with('pool')->get();
$accountingScanTruncated = $openBucketRows->count()
> self::INSTANCE_SCAN_LIMIT;
if ($accountingScanTruncated) {
$openBucketRows = $openBucketRows->take(
self::INSTANCE_SCAN_LIMIT
);
}
$identityConflicts = count(CloudTrafficSettlementEnvelope::conflictingBucketIds($openBucketRows));
$unsettledQuery = CloudTrafficAccountingEvidence::pendingInstances();
$unsettledInstances = (int) (clone $unsettledQuery)
->distinct()->count('cloud_instances.id');
$cutoff = $now - self::ACCOUNTING_WARNING_SECONDS;
$staleCursorInstances = (clone $unsettledQuery)->where(static function ($query) use ($cutoff): void {
$query->where('traffic_cursor.last_observed_at', '<=', $cutoff)
->orWhere(static function ($missing) use ($cutoff): void {
$missing->whereNull('traffic_cursor.instance_id')->where('cloud_instances.traffic_synced_at', '<=', $cutoff);
});
})->count();
$staleSamples = $main->table('cloud_instances as vm')->where('vm.status', 'active')
->where('vm.traffic_synced_at', '<=', $cutoff)->where('vm.created_at', '<=', $cutoff)
->whereExists(static function ($query) use ($now): void {
$query->selectRaw('1')->from('cloud_service_periods as period')
->join('cloud_traffic_pool_groups as grp', 'grp.id', '=', 'period.traffic_pool_group_id')
->whereColumn('period.instance_id', 'vm.id')->whereIn('grp.mode', ['shadow', 'enforced'])
->whereIn('period.status', ['active', 'exhausted'])->where('period.traffic_quota_mode', 'finite')
->where('period.starts_at', '<=', $now)->where('period.ends_at', '>', $now);
})->count();
$recentAccountingFailures = (int) $main->table(
'cloud_traffic_scan_failures'
)->where('last_failed_at', '>=',
$now - self::RECENT_FAILURE_SECONDS)->count();
$oldestOpenLag = $oldestOpenSealAt === null
? 0 : max(0, $now - $oldestOpenSealAt);
$oldestLag = max(
$oldestSettlement === null
? 0 : $settlementCutoff - $oldestSettlement,
$oldestCheckpoint === null
? 0 : $checkpointCutoff - $oldestCheckpoint
);
$metrics = [
'settlement_buckets_due' => $settlementCount,
'hourly_checkpoints_due' => $checkpointCount,
'hourly_checkpoint_instances_due' => $dueInstanceCount,
'cursor_attention_instances' => $attentionCount,
'open_buckets_due' => $openBucketCount,
'oldest_open_bucket_lag_seconds' => $oldestOpenLag,
'settlement_identity_conflicts' => $identityConflicts,
'unsettled_instances' => $unsettledInstances,
'stale_cursor_instances' => $staleCursorInstances,
'stale_sample_instances' => $staleSamples,
'recent_accounting_failures' => $recentAccountingFailures,
'oldest_due_lag_seconds' => max(0, $oldestLag),
'instance_scan_truncated' => false, // Checkpoint instance count uses an exact SQL aggregate.
'accounting_scan_truncated' => $accountingScanTruncated,
];
$status = $attentionCount > 0 || $staleSamples > 0
|| $accountingScanTruncated
|| $oldestLag > self::WARNING_LAG_SECONDS
|| $oldestOpenLag > self::ACCOUNTING_WARNING_SECONDS
|| $identityConflicts > 0 || $staleCursorInstances > 0
|| $recentAccountingFailures > 0
? 'warning' : 'ok';
if ($settlementCount === 0 && $checkpointCount === 0
&& $openBucketCount === 0 && $unsettledInstances === 0
&& $status === 'ok') {
$message = '流量结算与存储保留队列均处于正常窗口内。';
} else {
$message = '到期未封账 ' . $openBucketCount . ' 个(最长延迟 '
. self::duration($oldestOpenLag) . '),未结算实例 '
. $unsettledInstances . ' 个,采样滞后 ' . $staleSamples . ' 个,结算游标滞后 '
. $staleCursorInstances . ' 个,身份冲突 '
. $identityConflicts . ' 个,近一小时失败 '
. $recentAccountingFailures . ' 个;待压缩结算桶 '
. $settlementCount
. ' 个,小时检查点 ' . $checkpointCount . ' 行(涉及 '
. $dueInstanceCount . ' 个实例);最旧超出窗口 '
. self::duration(max(0, $oldestLag)) . ',其中流量游标异常 '
. $attentionCount . ' 个实例。';
if ($accountingScanTruncated) {
$message .= ' 实例计数已达到有界扫描上限。';
}
if ($identityConflicts > 0
|| $oldestOpenLag > self::WARNING_LAG_SECONDS) {
$message .= ' 结算账务需立即核对。';
}
}
return [
'key' => 'traffic_storage',
'label' => '流量结算与存储',
'status' => $status,
'message' => $message,
'checked_at' => $now,
'metrics' => $metrics,
];
}
private static function duration(int $seconds): string
{
if ($seconds >= 86400) {
return intdiv($seconds, 86400) . ' 天';
}
if ($seconds >= 3600) {
return intdiv($seconds, 3600) . ' 小时';
}
return intdiv($seconds, 60) . ' 分钟';
}
}