diff --git a/docs/TRAFFIC_POOL_ROLLOUT.md b/docs/TRAFFIC_POOL_ROLLOUT.md index 434edfbf..1692ff07 100644 --- a/docs/TRAFFIC_POOL_ROLLOUT.md +++ b/docs/TRAFFIC_POOL_ROLLOUT.md @@ -412,14 +412,16 @@ policy decision. Deploy it as a clean schema/code boundary: base counters need no anchor; an open period with null bases deliberately blocks deletion until the normal lifecycle path freezes real evidence. 6. Deploy the matching code and restart lifecycle. Open the administrator - task page and run its read-only health check. The “流量存储压缩” row exposes - due settlement buckets, due hourly rows, involved instances, cursor - attention count and oldest lag without exposing IDs or payloads. + task page and run its read-only health check. The “流量结算与存储” row exposes + overdue open buckets, longest accounting delay, event-identity conflicts, + unsettled and stale-cursor instances, recent accounting failures, due + compaction rows and cursor attention without exposing IDs or payloads. 7. Keep the pool in its existing rollout state while observing at least one complete collection, accounting, settlement-compaction and checkpoint-compaction pass. A backlog less than one hour beyond its - retention window remains informational; a larger lag, scan truncation or - attention cursor is a warning that must be investigated. + retention window remains informational. An open settlement delay over five + minutes is a warning; a delay over one hour, event-identity conflict, scan + truncation or attention cursor requires immediate investigation. Acceptance requires all of the following: diff --git a/web/app/Services/Cloud/CloudRuntimeHealthService.php b/web/app/Services/Cloud/CloudRuntimeHealthService.php index c28d5715..043e7d82 100644 --- a/web/app/Services/Cloud/CloudRuntimeHealthService.php +++ b/web/app/Services/Cloud/CloudRuntimeHealthService.php @@ -84,7 +84,7 @@ final class CloudRuntimeHealthService ? $this->trafficStorageCheck() : $this->row( 'traffic_storage', - '流量存储压缩', + '流量结算与存储', 'unknown', '数据库短超时探测未通过,未继续执行流量存储统计。' ); @@ -93,9 +93,9 @@ final class CloudRuntimeHealthService '未启用日志数据库;虚拟化指标依赖该配置。'); $checks[] = $this->row( 'traffic_storage', - '流量存储压缩', + '流量结算与存储', 'disabled', - '日志数据库未启用,未执行流量存储压缩检查。' + '日志数据库未启用,未执行流量结算与存储检查。' ); } $checks[] = $this->check('agents', '宿主机 Agent 配置', @@ -288,6 +288,9 @@ final class CloudRuntimeHealthService 'settlement_buckets_due', 'hourly_checkpoints_due', 'hourly_checkpoint_instances_due', 'cursor_attention_instances', 'oldest_due_lag_seconds', + 'open_buckets_due', 'oldest_open_bucket_lag_seconds', + 'settlement_identity_conflicts', 'unsettled_instances', + 'stale_cursor_instances', 'recent_accounting_failures', ] as $key) { $value = $result['metrics'][$key] ?? null; if (!is_int($value) || $value < 0) { @@ -300,18 +303,21 @@ final class CloudRuntimeHealthService $metrics['instance_scan_truncated'] = ($result['metrics']['instance_scan_truncated'] ?? null) === true; + $metrics['accounting_scan_truncated'] = + ($result['metrics']['accounting_scan_truncated'] ?? null) + === true; return $this->row( 'traffic_storage', - '流量存储压缩', + '流量结算与存储', (string) $result['status'], (string) $result['message'] ) + ['metrics' => $metrics]; } catch (\Throwable $exception) { return $this->row( 'traffic_storage', - '流量存储压缩', + '流量结算与存储', 'error', - '流量存储压缩状态读取失败,请检查数据库结构与生命周期日志。' + '流量结算与存储状态读取失败,请检查数据库结构与生命周期日志。' ); } } diff --git a/web/app/Services/Cloud/CloudTrafficStorageHealthService.php b/web/app/Services/Cloud/CloudTrafficStorageHealthService.php index efd6e661..6e57cbe2 100644 --- a/web/app/Services/Cloud/CloudTrafficStorageHealthService.php +++ b/web/app/Services/Cloud/CloudTrafficStorageHealthService.php @@ -8,6 +8,8 @@ use App\Models\User; final class CloudTrafficStorageHealthService { private const WARNING_LAG_SECONDS = 3600; + private const ACCOUNTING_WARNING_SECONDS = 300; + private const RECENT_FAILURE_SECONDS = 3600; private const INSTANCE_SCAN_LIMIT = 2000; /** @@ -29,13 +31,21 @@ final class CloudTrafficStorageHealthService } $main = User::getConnectionResolver()->connection(); $log = User::getConnectionResolver()->connection('log'); - if (!$main->getSchemaBuilder()->hasTable( - 'cloud_traffic_usage_settlement_buckets' - ) || !$main->getSchemaBuilder()->hasTable( - 'cloud_traffic_instance_cursors' - ) || !$log->getSchemaBuilder()->hasTable( - 'cloud_vm_traffic_checkpoints' - )) { + foreach ([ + 'cloud_traffic_usage_settlement_buckets', + 'cloud_traffic_ledger_events', 'cloud_traffic_instance_cursors', + 'cloud_instances', 'cloud_service_periods', + 'cloud_traffic_pool_groups', 'cloud_traffic_scan_failures', + ] as $table) { + if (!$main->getSchemaBuilder()->hasTable($table)) { + throw new \RuntimeException( + 'traffic_storage_health_schema_missing' + ); + } + } + if (!$log->getSchemaBuilder()->hasTable( + 'cloud_vm_traffic_checkpoints' + )) { throw new \RuntimeException('traffic_storage_health_schema_missing'); } $settlementCutoff = $now @@ -72,6 +82,78 @@ final class CloudTrafficStorageHealthService : (int) $main->table('cloud_traffic_instance_cursors') ->whereIn('instance_id', $dueInstanceIds) ->where('state', 'attention')->count(); + $openQuery = $main->table( + 'cloud_traffic_usage_settlement_buckets' + )->where('state', 'open')->where('seal_at', '<=', $now); + $openBucketCount = (int) (clone $openQuery)->count(); + $oldestOpenSealAt = $openBucketCount === 0 ? null + : (int) (clone $openQuery)->min('seal_at'); + $openBucketRows = (clone $openQuery)->orderBy('seal_at') + ->orderBy('id')->limit(self::INSTANCE_SCAN_LIMIT + 1) + ->get(['id', 'event_identity']); + $accountingScanTruncated = $openBucketRows->count() + > self::INSTANCE_SCAN_LIMIT; + if ($accountingScanTruncated) { + $openBucketRows = $openBucketRows->take( + self::INSTANCE_SCAN_LIMIT + ); + } + $eventKeys = []; + $identityConflicts = 0; + foreach ($openBucketRows as $row) { + $identity = (string) ($row->event_identity ?? ''); + if (strlen($identity) + !== CloudTrafficSettlementEnvelope::IDENTITY_BYTES) { + $identityConflicts++; + continue; + } + $eventKeys[] = 'traffic-settlement:v2:' . bin2hex($identity); + } + if ($eventKeys !== []) { + $identityConflicts += (int) $main->table( + 'cloud_traffic_ledger_events' + )->whereIn('event_key', $eventKeys)->count(); + } + $unsettledQuery = $main->table('cloud_instances as health_instance') + ->leftJoin( + 'cloud_traffic_instance_cursors as health_cursor', + 'health_cursor.instance_id', '=', 'health_instance.id' + )->whereExists(function ($eligible): void { + $eligible->selectRaw('1') + ->from('cloud_service_periods as health_period') + ->join('cloud_traffic_pool_groups as health_group', + 'health_group.id', '=', + 'health_period.traffic_pool_group_id') + ->whereColumn('health_period.instance_id', + 'health_instance.id') + ->whereIn('health_group.mode', ['shadow', 'enforced']); + })->where(function ($query): void { + $query->where(function ($missing): void { + $missing->whereNull('health_cursor.instance_id') + ->where('health_instance.traffic_synced_at', '>', 0); + })->orWhere('health_cursor.state', 'ready'); + })->where(function ($query): void { + $query->whereNull('health_cursor.instance_id') + ->orWhereColumn('health_instance.lifetime_download_bytes', + '!=', 'health_cursor.accounted_download_bytes') + ->orWhereColumn('health_instance.lifetime_upload_bytes', + '!=', 'health_cursor.accounted_upload_bytes') + ->orWhereColumn('health_instance.traffic_synced_at', '>', + 'health_cursor.last_observed_at'); + }); + $unsettledInstances = (int) (clone $unsettledQuery) + ->distinct()->count('health_instance.id'); + $staleCursorInstances = (int) (clone $unsettledQuery) + ->where('health_instance.traffic_synced_at', '>', 0) + ->where('health_instance.traffic_synced_at', '<=', + $now - self::ACCOUNTING_WARNING_SECONDS) + ->distinct()->count('health_instance.id'); + $recentAccountingFailures = (int) $main->table( + 'cloud_traffic_scan_failures' + )->where('last_failed_at', '>=', + $now - self::RECENT_FAILURE_SECONDS)->count(); + $oldestOpenLag = $oldestOpenSealAt === null + ? 0 : max(0, $now - $oldestOpenSealAt); $oldestLag = max( $oldestSettlement === null ? 0 : $settlementCutoff - $oldestSettlement, @@ -83,27 +165,50 @@ final class CloudTrafficStorageHealthService 'hourly_checkpoints_due' => $checkpointCount, 'hourly_checkpoint_instances_due' => $dueInstanceCount, 'cursor_attention_instances' => $attentionCount, + 'open_buckets_due' => $openBucketCount, + 'oldest_open_bucket_lag_seconds' => $oldestOpenLag, + 'settlement_identity_conflicts' => $identityConflicts, + 'unsettled_instances' => $unsettledInstances, + 'stale_cursor_instances' => $staleCursorInstances, + 'recent_accounting_failures' => $recentAccountingFailures, 'oldest_due_lag_seconds' => max(0, $oldestLag), 'instance_scan_truncated' => $scanTruncated, + 'accounting_scan_truncated' => $accountingScanTruncated, ]; $status = $attentionCount > 0 || $scanTruncated + || $accountingScanTruncated || $oldestLag > self::WARNING_LAG_SECONDS + || $oldestOpenLag > self::ACCOUNTING_WARNING_SECONDS + || $identityConflicts > 0 || $staleCursorInstances > 0 + || $recentAccountingFailures > 0 ? 'warning' : 'ok'; - if ($settlementCount === 0 && $checkpointCount === 0) { - $message = '结算桶和小时检查点均处于既定保留窗口内。'; + if ($settlementCount === 0 && $checkpointCount === 0 + && $openBucketCount === 0 && $unsettledInstances === 0 + && $recentAccountingFailures === 0) { + $message = '流量结算与存储保留队列均处于正常窗口内。'; } else { - $message = '待压缩结算桶 ' . $settlementCount + $message = '到期未封账 ' . $openBucketCount . ' 个(最长延迟 ' + . self::duration($oldestOpenLag) . '),未结算实例 ' + . $unsettledInstances . ' 个,滞后游标 ' + . $staleCursorInstances . ' 个,身份冲突 ' + . $identityConflicts . ' 个,近一小时失败 ' + . $recentAccountingFailures . ' 个;待压缩结算桶 ' + . $settlementCount . ' 个,小时检查点 ' . $checkpointCount . ' 行(涉及 ' . $dueInstanceCount . ' 个实例);最旧超出窗口 ' . self::duration(max(0, $oldestLag)) . ',其中流量游标异常 ' . $attentionCount . ' 个实例。'; - if ($scanTruncated) { + if ($scanTruncated || $accountingScanTruncated) { $message .= ' 实例计数已达到有界扫描上限。'; } + if ($identityConflicts > 0 + || $oldestOpenLag > self::WARNING_LAG_SECONDS) { + $message .= ' 结算账务需立即核对。'; + } } return [ 'key' => 'traffic_storage', - 'label' => '流量存储压缩', + 'label' => '流量结算与存储', 'status' => $status, 'message' => $message, 'checked_at' => $now, diff --git a/web/public/cloud/admin-operations-health.js b/web/public/cloud/admin-operations-health.js index 835b2fbd..c3fc9338 100644 --- a/web/public/cloud/admin-operations-health.js +++ b/web/public/cloud/admin-operations-health.js @@ -16,6 +16,34 @@ export function mountOperationsHealth({ transport, presentation }) { node.className = `fs-13 mb-0 ${error ? 'text-danger' : 'text-muted'}`; } + function duration(seconds) { + const value = Math.max(0, Number(seconds) || 0); + if (value >= 86400) return `${Math.floor(value / 86400)} 天`; + if (value >= 3600) return `${Math.floor(value / 3600)} 小时`; + if (value >= 60) return `${Math.floor(value / 60)} 分钟`; + return `${Math.floor(value)} 秒`; + } + + function trafficAccountingMetrics(metrics) { + if (!metrics || typeof metrics !== 'object') return null; + const list = el('dl', 'row g-1 mb-0 mt-2 fs-12'); + const values = [ + ['到期未封账', metrics.open_buckets_due], + ['最长结算延迟', duration(metrics.oldest_open_bucket_lag_seconds)], + ['身份冲突', metrics.settlement_identity_conflicts], + ['未结算实例', metrics.unsettled_instances], + ['滞后游标', metrics.stale_cursor_instances], + ['近一小时失败', metrics.recent_accounting_failures], + ]; + for (const [label, value] of values) { + list.append( + el('dt', 'col-7 text-muted fw-normal', label), + el('dd', 'col-5 mb-0 text-end fw-semibold', value), + ); + } + return list; + } + function render(payload) { if (!Array.isArray(payload?.checks) || !payload.checked_at) { throw new Error('运行状态数据格式无效。'); @@ -39,6 +67,10 @@ export function mountOperationsHealth({ transport, presentation }) { title, el('p', 'text-muted fs-13 mb-1 text-break', operationText(check.message)), ); + if (check.key === 'traffic_storage') { + const metrics = trafficAccountingMetrics(check.metrics); + if (metrics) column.append(metrics); + } if (check.last_seen_at) { column.append(el('p', 'text-muted fs-12 mb-0', `最近活动 ${operationDate(check.last_seen_at)}`)); diff --git a/web/tests/AdminOperationsCancelBehaviorTest.mjs b/web/tests/AdminOperationsCancelBehaviorTest.mjs index b1008e0e..aeff98d8 100644 --- a/web/tests/AdminOperationsCancelBehaviorTest.mjs +++ b/web/tests/AdminOperationsCancelBehaviorTest.mjs @@ -95,6 +95,10 @@ assert.match(entry, /mountOperationsRecovery\(/); assert.doesNotMatch(entry, /\/force-delete|\/repair-vpc|\/resolve-effect|\/retire-vpc-host/); assert.match(directory, /export function mountOperationsDirectory/); assert.match(health, /export function mountOperationsHealth/); +assert.match(health, /trafficAccountingMetrics/); +assert.match(health, /metrics\.open_buckets_due/); +assert.match(health, /metrics\.settlement_identity_conflicts/); +assert.match(health, /metrics\.unsettled_instances/); assert.match(recovery, /export function mountOperationsRecovery/); assert.match(presentation, /export function presentOperation/); diff --git a/web/tests/CloudRuntimeHealthTest.php b/web/tests/CloudRuntimeHealthTest.php index 99ab28ef..2ae2c912 100644 --- a/web/tests/CloudRuntimeHealthTest.php +++ b/web/tests/CloudRuntimeHealthTest.php @@ -50,8 +50,15 @@ $service = new CloudRuntimeHealthService( 'hourly_checkpoints_due' => 3, 'hourly_checkpoint_instances_due' => 1, 'cursor_attention_instances' => 0, + 'open_buckets_due' => 4, + 'oldest_open_bucket_lag_seconds' => 600, + 'settlement_identity_conflicts' => 1, + 'unsettled_instances' => 5, + 'stale_cursor_instances' => 3, + 'recent_accounting_failures' => 2, 'oldest_due_lag_seconds' => 120, 'instance_scan_truncated' => false, + 'accounting_scan_truncated' => false, ], ]; } @@ -80,6 +87,8 @@ $assert($checks['agents']['status'] === 'ok' && $checks['worker']['status'] === && $checks['abuse_monitor']['status'] === 'unknown' && $checks['traffic_storage']['status'] === 'warning' && $checks['traffic_storage']['metrics']['settlement_buckets_due'] === 2 + && $checks['traffic_storage']['metrics']['open_buckets_due'] === 4 + && $checks['traffic_storage']['metrics']['unsettled_instances'] === 5 && $checks['lifecycle']['status'] === 'warning', 'Health or heartbeat semantics changed.'); $assert(!str_contains(json_encode($payload), 'MUST_NOT_EXPOSE'), 'Heartbeat internal fields leaked.'); diff --git a/web/tests/CloudTrafficStorageHealthServiceTest.php b/web/tests/CloudTrafficStorageHealthServiceTest.php index c049ec2e..72c4f354 100644 --- a/web/tests/CloudTrafficStorageHealthServiceTest.php +++ b/web/tests/CloudTrafficStorageHealthServiceTest.php @@ -28,12 +28,44 @@ $main->create('cloud_traffic_usage_settlement_buckets', static function (Blueprint $table): void { $table->increments('id'); $table->string('state'); + $table->integer('seal_at'); + $table->binary('event_identity'); $table->integer('updated_at'); }); +$main->create('cloud_traffic_ledger_events', + static function (Blueprint $table): void { + $table->increments('id'); + $table->string('event_key')->unique(); + }); $main->create('cloud_traffic_instance_cursors', static function (Blueprint $table): void { $table->integer('instance_id')->primary(); $table->string('state'); + $table->integer('accounted_download_bytes')->default(0); + $table->integer('accounted_upload_bytes')->default(0); + $table->integer('last_observed_at')->default(0); + }); +$main->create('cloud_instances', static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('lifetime_download_bytes')->default(0); + $table->integer('lifetime_upload_bytes')->default(0); + $table->integer('traffic_synced_at')->default(0); +}); +$main->create('cloud_traffic_pool_groups', + static function (Blueprint $table): void { + $table->increments('id'); + $table->string('mode'); + }); +$main->create('cloud_service_periods', + static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('instance_id'); + $table->integer('traffic_pool_group_id'); + }); +$main->create('cloud_traffic_scan_failures', + static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('last_failed_at'); }); $capsule->schema('log')->create('cloud_vm_traffic_checkpoints', static function (Blueprint $table): void { @@ -45,6 +77,8 @@ $now = 20 * 86400; $healthy = CloudTrafficStorageHealthService::snapshot($now); $assert($healthy['status'] === 'ok' && $healthy['metrics']['settlement_buckets_due'] === 0 + && $healthy['metrics']['open_buckets_due'] === 0 + && $healthy['metrics']['unsettled_instances'] === 0 && $healthy['metrics']['hourly_checkpoints_due'] === 0 && $healthy['metrics']['cursor_attention_instances'] === 0, 'Empty compaction queues did not report a healthy bounded state.'); @@ -55,9 +89,19 @@ $checkpointCutoff = $now - CloudTrafficCheckpointRetention::RETENTION_SECONDS; $capsule->connection('default') ->table('cloud_traffic_usage_settlement_buckets')->insert([ - ['state' => 'sealed', 'updated_at' => $settlementCutoff - 7200], - ['state' => 'open', 'updated_at' => $settlementCutoff - 86400], + ['state' => 'sealed', 'seal_at' => $now - 86400, + 'event_identity' => random_bytes(16), + 'updated_at' => $settlementCutoff - 7200], + ['state' => 'open', 'seal_at' => $now - 7200, + 'event_identity' => str_repeat('a', 16), + 'updated_at' => $now - 7200], ]); +$capsule->connection('default')->table( + 'cloud_traffic_ledger_events' +)->insert([ + 'event_key' => 'traffic-settlement:v2:' + . bin2hex(str_repeat('a', 16)), +]); $capsule->connection('log')->table('cloud_vm_traffic_checkpoints')->insert([ ['instance_id' => 3, 'first_observed_at' => $checkpointCutoff - 7200], ['instance_id' => 4, 'first_observed_at' => $checkpointCutoff + 60], @@ -65,17 +109,44 @@ $capsule->connection('log')->table('cloud_vm_traffic_checkpoints')->insert([ $capsule->connection('default')->table( 'cloud_traffic_instance_cursors' )->insert([ - 'instance_id' => 3, - 'state' => 'attention', + ['instance_id' => 3, 'state' => 'attention', + 'accounted_download_bytes' => 0, 'accounted_upload_bytes' => 0, + 'last_observed_at' => 0], + ['instance_id' => 5, 'state' => 'ready', + 'accounted_download_bytes' => 10, 'accounted_upload_bytes' => 2, + 'last_observed_at' => $now - 1200], +]); +$capsule->connection('default')->table('cloud_instances')->insert([ + ['id' => 3, 'lifetime_download_bytes' => 0, + 'lifetime_upload_bytes' => 0, 'traffic_synced_at' => 0], + ['id' => 5, 'lifetime_download_bytes' => 20, + 'lifetime_upload_bytes' => 2, 'traffic_synced_at' => $now - 600], ]); +$capsule->connection('default')->table( + 'cloud_traffic_pool_groups' +)->insert(['id' => 1, 'mode' => 'enforced']); +$capsule->connection('default')->table('cloud_service_periods')->insert([ + ['id' => 1, 'instance_id' => 5, 'traffic_pool_group_id' => 1], +]); +$capsule->connection('default')->table( + 'cloud_traffic_scan_failures' +)->insert(['last_failed_at' => $now - 60]); $warning = CloudTrafficStorageHealthService::snapshot($now); $assert($warning['status'] === 'warning' && $warning['metrics']['settlement_buckets_due'] === 1 + && $warning['metrics']['open_buckets_due'] === 1 + && $warning['metrics']['oldest_open_bucket_lag_seconds'] === 7200 + && $warning['metrics']['settlement_identity_conflicts'] === 1 + && $warning['metrics']['unsettled_instances'] === 1 + && $warning['metrics']['stale_cursor_instances'] === 1 + && $warning['metrics']['recent_accounting_failures'] === 1 && $warning['metrics']['hourly_checkpoints_due'] === 1 && $warning['metrics']['hourly_checkpoint_instances_due'] === 1 && $warning['metrics']['cursor_attention_instances'] === 1 && $warning['metrics']['oldest_due_lag_seconds'] === 7200 - && str_contains($warning['message'], '结算桶 1 个') + && str_contains($warning['message'], '到期未封账 1 个') + && str_contains($warning['message'], '身份冲突 1 个') + && str_contains($warning['message'], '结算账务需立即核对') && str_contains($warning['message'], '小时检查点 1 行'), 'Compaction backlog or attention state was hidden from operator health.');