|
|
|
|
@ -8,6 +8,8 @@ use App\Models\User;
|
|
|
|
|
final class CloudTrafficStorageHealthService
|
|
|
|
|
{
|
|
|
|
|
private const WARNING_LAG_SECONDS = 3600;
|
|
|
|
|
private const ACCOUNTING_WARNING_SECONDS = 300;
|
|
|
|
|
private const RECENT_FAILURE_SECONDS = 3600;
|
|
|
|
|
private const INSTANCE_SCAN_LIMIT = 2000;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
@ -29,13 +31,21 @@ final class CloudTrafficStorageHealthService
|
|
|
|
|
}
|
|
|
|
|
$main = User::getConnectionResolver()->connection();
|
|
|
|
|
$log = User::getConnectionResolver()->connection('log');
|
|
|
|
|
if (!$main->getSchemaBuilder()->hasTable(
|
|
|
|
|
'cloud_traffic_usage_settlement_buckets'
|
|
|
|
|
) || !$main->getSchemaBuilder()->hasTable(
|
|
|
|
|
'cloud_traffic_instance_cursors'
|
|
|
|
|
) || !$log->getSchemaBuilder()->hasTable(
|
|
|
|
|
'cloud_vm_traffic_checkpoints'
|
|
|
|
|
)) {
|
|
|
|
|
foreach ([
|
|
|
|
|
'cloud_traffic_usage_settlement_buckets',
|
|
|
|
|
'cloud_traffic_ledger_events', 'cloud_traffic_instance_cursors',
|
|
|
|
|
'cloud_instances', 'cloud_service_periods',
|
|
|
|
|
'cloud_traffic_pool_groups', 'cloud_traffic_scan_failures',
|
|
|
|
|
] as $table) {
|
|
|
|
|
if (!$main->getSchemaBuilder()->hasTable($table)) {
|
|
|
|
|
throw new \RuntimeException(
|
|
|
|
|
'traffic_storage_health_schema_missing'
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (!$log->getSchemaBuilder()->hasTable(
|
|
|
|
|
'cloud_vm_traffic_checkpoints'
|
|
|
|
|
)) {
|
|
|
|
|
throw new \RuntimeException('traffic_storage_health_schema_missing');
|
|
|
|
|
}
|
|
|
|
|
$settlementCutoff = $now
|
|
|
|
|
@ -72,6 +82,78 @@ final class CloudTrafficStorageHealthService
|
|
|
|
|
: (int) $main->table('cloud_traffic_instance_cursors')
|
|
|
|
|
->whereIn('instance_id', $dueInstanceIds)
|
|
|
|
|
->where('state', 'attention')->count();
|
|
|
|
|
$openQuery = $main->table(
|
|
|
|
|
'cloud_traffic_usage_settlement_buckets'
|
|
|
|
|
)->where('state', 'open')->where('seal_at', '<=', $now);
|
|
|
|
|
$openBucketCount = (int) (clone $openQuery)->count();
|
|
|
|
|
$oldestOpenSealAt = $openBucketCount === 0 ? null
|
|
|
|
|
: (int) (clone $openQuery)->min('seal_at');
|
|
|
|
|
$openBucketRows = (clone $openQuery)->orderBy('seal_at')
|
|
|
|
|
->orderBy('id')->limit(self::INSTANCE_SCAN_LIMIT + 1)
|
|
|
|
|
->get(['id', 'event_identity']);
|
|
|
|
|
$accountingScanTruncated = $openBucketRows->count()
|
|
|
|
|
> self::INSTANCE_SCAN_LIMIT;
|
|
|
|
|
if ($accountingScanTruncated) {
|
|
|
|
|
$openBucketRows = $openBucketRows->take(
|
|
|
|
|
self::INSTANCE_SCAN_LIMIT
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
$eventKeys = [];
|
|
|
|
|
$identityConflicts = 0;
|
|
|
|
|
foreach ($openBucketRows as $row) {
|
|
|
|
|
$identity = (string) ($row->event_identity ?? '');
|
|
|
|
|
if (strlen($identity)
|
|
|
|
|
!== CloudTrafficSettlementEnvelope::IDENTITY_BYTES) {
|
|
|
|
|
$identityConflicts++;
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
$eventKeys[] = 'traffic-settlement:v2:' . bin2hex($identity);
|
|
|
|
|
}
|
|
|
|
|
if ($eventKeys !== []) {
|
|
|
|
|
$identityConflicts += (int) $main->table(
|
|
|
|
|
'cloud_traffic_ledger_events'
|
|
|
|
|
)->whereIn('event_key', $eventKeys)->count();
|
|
|
|
|
}
|
|
|
|
|
$unsettledQuery = $main->table('cloud_instances as health_instance')
|
|
|
|
|
->leftJoin(
|
|
|
|
|
'cloud_traffic_instance_cursors as health_cursor',
|
|
|
|
|
'health_cursor.instance_id', '=', 'health_instance.id'
|
|
|
|
|
)->whereExists(function ($eligible): void {
|
|
|
|
|
$eligible->selectRaw('1')
|
|
|
|
|
->from('cloud_service_periods as health_period')
|
|
|
|
|
->join('cloud_traffic_pool_groups as health_group',
|
|
|
|
|
'health_group.id', '=',
|
|
|
|
|
'health_period.traffic_pool_group_id')
|
|
|
|
|
->whereColumn('health_period.instance_id',
|
|
|
|
|
'health_instance.id')
|
|
|
|
|
->whereIn('health_group.mode', ['shadow', 'enforced']);
|
|
|
|
|
})->where(function ($query): void {
|
|
|
|
|
$query->where(function ($missing): void {
|
|
|
|
|
$missing->whereNull('health_cursor.instance_id')
|
|
|
|
|
->where('health_instance.traffic_synced_at', '>', 0);
|
|
|
|
|
})->orWhere('health_cursor.state', 'ready');
|
|
|
|
|
})->where(function ($query): void {
|
|
|
|
|
$query->whereNull('health_cursor.instance_id')
|
|
|
|
|
->orWhereColumn('health_instance.lifetime_download_bytes',
|
|
|
|
|
'!=', 'health_cursor.accounted_download_bytes')
|
|
|
|
|
->orWhereColumn('health_instance.lifetime_upload_bytes',
|
|
|
|
|
'!=', 'health_cursor.accounted_upload_bytes')
|
|
|
|
|
->orWhereColumn('health_instance.traffic_synced_at', '>',
|
|
|
|
|
'health_cursor.last_observed_at');
|
|
|
|
|
});
|
|
|
|
|
$unsettledInstances = (int) (clone $unsettledQuery)
|
|
|
|
|
->distinct()->count('health_instance.id');
|
|
|
|
|
$staleCursorInstances = (int) (clone $unsettledQuery)
|
|
|
|
|
->where('health_instance.traffic_synced_at', '>', 0)
|
|
|
|
|
->where('health_instance.traffic_synced_at', '<=',
|
|
|
|
|
$now - self::ACCOUNTING_WARNING_SECONDS)
|
|
|
|
|
->distinct()->count('health_instance.id');
|
|
|
|
|
$recentAccountingFailures = (int) $main->table(
|
|
|
|
|
'cloud_traffic_scan_failures'
|
|
|
|
|
)->where('last_failed_at', '>=',
|
|
|
|
|
$now - self::RECENT_FAILURE_SECONDS)->count();
|
|
|
|
|
$oldestOpenLag = $oldestOpenSealAt === null
|
|
|
|
|
? 0 : max(0, $now - $oldestOpenSealAt);
|
|
|
|
|
$oldestLag = max(
|
|
|
|
|
$oldestSettlement === null
|
|
|
|
|
? 0 : $settlementCutoff - $oldestSettlement,
|
|
|
|
|
@ -83,27 +165,50 @@ final class CloudTrafficStorageHealthService
|
|
|
|
|
'hourly_checkpoints_due' => $checkpointCount,
|
|
|
|
|
'hourly_checkpoint_instances_due' => $dueInstanceCount,
|
|
|
|
|
'cursor_attention_instances' => $attentionCount,
|
|
|
|
|
'open_buckets_due' => $openBucketCount,
|
|
|
|
|
'oldest_open_bucket_lag_seconds' => $oldestOpenLag,
|
|
|
|
|
'settlement_identity_conflicts' => $identityConflicts,
|
|
|
|
|
'unsettled_instances' => $unsettledInstances,
|
|
|
|
|
'stale_cursor_instances' => $staleCursorInstances,
|
|
|
|
|
'recent_accounting_failures' => $recentAccountingFailures,
|
|
|
|
|
'oldest_due_lag_seconds' => max(0, $oldestLag),
|
|
|
|
|
'instance_scan_truncated' => $scanTruncated,
|
|
|
|
|
'accounting_scan_truncated' => $accountingScanTruncated,
|
|
|
|
|
];
|
|
|
|
|
$status = $attentionCount > 0 || $scanTruncated
|
|
|
|
|
|| $accountingScanTruncated
|
|
|
|
|
|| $oldestLag > self::WARNING_LAG_SECONDS
|
|
|
|
|
|| $oldestOpenLag > self::ACCOUNTING_WARNING_SECONDS
|
|
|
|
|
|| $identityConflicts > 0 || $staleCursorInstances > 0
|
|
|
|
|
|| $recentAccountingFailures > 0
|
|
|
|
|
? 'warning' : 'ok';
|
|
|
|
|
if ($settlementCount === 0 && $checkpointCount === 0) {
|
|
|
|
|
$message = '结算桶和小时检查点均处于既定保留窗口内。';
|
|
|
|
|
if ($settlementCount === 0 && $checkpointCount === 0
|
|
|
|
|
&& $openBucketCount === 0 && $unsettledInstances === 0
|
|
|
|
|
&& $recentAccountingFailures === 0) {
|
|
|
|
|
$message = '流量结算与存储保留队列均处于正常窗口内。';
|
|
|
|
|
} else {
|
|
|
|
|
$message = '待压缩结算桶 ' . $settlementCount
|
|
|
|
|
$message = '到期未封账 ' . $openBucketCount . ' 个(最长延迟 '
|
|
|
|
|
. self::duration($oldestOpenLag) . '),未结算实例 '
|
|
|
|
|
. $unsettledInstances . ' 个,滞后游标 '
|
|
|
|
|
. $staleCursorInstances . ' 个,身份冲突 '
|
|
|
|
|
. $identityConflicts . ' 个,近一小时失败 '
|
|
|
|
|
. $recentAccountingFailures . ' 个;待压缩结算桶 '
|
|
|
|
|
. $settlementCount
|
|
|
|
|
. ' 个,小时检查点 ' . $checkpointCount . ' 行(涉及 '
|
|
|
|
|
. $dueInstanceCount . ' 个实例);最旧超出窗口 '
|
|
|
|
|
. self::duration(max(0, $oldestLag)) . ',其中流量游标异常 '
|
|
|
|
|
. $attentionCount . ' 个实例。';
|
|
|
|
|
if ($scanTruncated) {
|
|
|
|
|
if ($scanTruncated || $accountingScanTruncated) {
|
|
|
|
|
$message .= ' 实例计数已达到有界扫描上限。';
|
|
|
|
|
}
|
|
|
|
|
if ($identityConflicts > 0
|
|
|
|
|
|| $oldestOpenLag > self::WARNING_LAG_SECONDS) {
|
|
|
|
|
$message .= ' 结算账务需立即核对。';
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return [
|
|
|
|
|
'key' => 'traffic_storage',
|
|
|
|
|
'label' => '流量存储压缩',
|
|
|
|
|
'label' => '流量结算与存储',
|
|
|
|
|
'status' => $status,
|
|
|
|
|
'message' => $message,
|
|
|
|
|
'checked_at' => $now,
|
|
|
|
|
|