From 2bb9bb78b6e06287e1b0dde443368531d845e352 Mon Sep 17 00:00:00 2001 From: codex Date: Mon, 16 Nov 2026 23:19:49 +0800 Subject: [PATCH] feat(traffic): expose storage compaction health --- docs/DEPLOYMENT.md | 2 +- docs/TRAFFIC_POOL_ROLLOUT.md | 59 +++++++++ .../Cloud/CloudRuntimeHealthService.php | 62 ++++++++- .../Services/Cloud/CloudRuntimeHeartbeat.php | 1 + .../CloudTrafficStorageHealthService.php | 122 ++++++++++++++++++ web/tests/CloudRuntimeHealthTest.php | 29 ++++- .../CloudTrafficStorageHealthServiceTest.php | 82 ++++++++++++ 7 files changed, 350 insertions(+), 7 deletions(-) create mode 100644 web/app/Services/Cloud/CloudTrafficStorageHealthService.php create mode 100644 web/tests/CloudTrafficStorageHealthServiceTest.php diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 9d4df719..16b54e82 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -160,7 +160,7 @@ libvirt 的持久 pool 定义就是 Agent 后续使用的映射,不另写存 公网网络接入在宿主机的“网络接入”中按资源池线路独立配置,可使用 `bridge:br0`、其他 Linux 网桥或 `network:default` 这样的 libvirt network。名称以字母或数字开头,后续只能包含字母、数字、点、下划线和短横线,bridge 名称为 1~15 字符,libvirt network 名称为 1~64 字符。宿主机上的公网网桥或 libvirt network 必须由运维预先准备。Agent 只发现、验证和使用现有公网网络,不创建公网网桥、不启动 libvirt network,也不修改公网地址、上游网关或挂载。`cloud_hosts.default_network` 仅保留为兼容展示值,不参与 Agent 就绪、调度、恢复、迁移或换 IP 决策;真正的运行时权威是资源池线路、当前 Agent 代次已验证的网络接入,以及绑定该接入的公网地址池。VPC 使用下述独立的 Agent 受管 bridge 和 L3 路由,不改变这一公网边界。 -当前完整 schema 版本为 `go-agent-wss-v55-traffic-checkpoint-tiers`。v48 将滥用监测拆分为 72 小时原始证据、短期终态投影、7 天去重凭据以及 5 分钟/小时趋势汇总;v49 为实例性能图表增加 5 分钟和小时汇总,并将一分钟原始点限制为 24 小时、小时趋势限制为 35 天;v50 保持分钟级 FEFO 扣减和游标更新不变,将同一小时、实例和额度的用量累积到持久结算桶,小时结束、额度耗尽、到期、债务结算或异常栅栏时再生成不可变账本事件。迟到采样使用新的桶修订,绝不改写已封存事件;v51 将性能历史改为逻辑实例主键,删除每行重复的宿主机 hash 和文本 UUID,将永久流量总数/检查点的 UUID 与 hash 改为二进制并删除未被读取路径使用的宽索引;v52 保持实时 FEFO 扣减不变,将新用量累积到 UTC 每日结算窗口,并通过 `bucket_seconds` 与 `seal_at` 继续识别和封存升级前的小时窗口;v53 增加紧凑结算凭据,冻结已经安全压缩的窗口身份、最大修订、账本事件和载荷哈希,让迟到采样无需依赖完整旧桶也能从下一修订继续;v54 启用7天安全窗后的完整桶压缩,只有账本身份、金额、载荷和哈希全部吻合时,才在同一主库事务中更新凭据并删除完整桶;v55 为每个逻辑实例写入一条 UTC 每日累计检查点,并在服务周期首次确定基准时按周期冻结精确边界锚点,维护迁移后仍沿用同一逻辑实例历史。部署既有数据库时必须先停止生命周期与滥用监测 worker,按当前 `cloud.sql`、`cloud_log.sql` 的新增表和字段制作一次性迁移并完成回填,再启动新代码;v51 涉及日志表主键与二进制身份重建,不能在采集 worker 运行时原地改列,也不能让 v50 代码与 v51 日志表混跑;v52 必须先为既有结算桶回填 `bucket_seconds = 3600` 与 `seal_at = bucket_start + 3600`,再替换唯一索引、到期索引和检查约束;v53 必须先创建 `cloud_traffic_settlement_receipts`;v54 必须增加 `cloud_traffic_settlement_retention` 索引后才能启动清理,验证失败的桶保持原样且不得人工跳过;v55 必须先给小时检查点回填可确认的逻辑 `instance_id`,再创建每日检查点和周期边界锚点表;无法唯一对应实例的旧检查点保持原样并禁止自动清理。不得让缺少汇总表、结算桶窗口字段、紧凑凭据表、每日检查点、周期锚点或 `instance_id` 的旧数据库直接进入采集或流量结算。v47 为 `cloud_offers` 增加季度、年度及对应自动备份价格;既有方案升级时必须在恢复销售前显式回填 90/365 天价格,不能让数据库默认零价进入购买目录。从 v44 升级时,必须先停止滥用观测与投影 worker,为既有 finding 回填 `occurrence_generation = 1` 与 `transition_revision = 1`,再创建 `cloud_abuse_alert_events` 和 `cloud_abuse_alert_event_items`;从 v45 升级还必须停止通知 worker,创建 `cloud_abuse_alert_deliveries`、`administrator_telegram_delivery_gate` 并写入唯一 gate 行。历史 finding 不反向合成事件;事件只从升级后主库事务内真实发生的 `opened`、`escalated` 和 `evidence_clear` 解决转移产生。完成列、约束、索引、外键和 gate 核对后,先恢复滥用观测,再恢复通知 worker。 +当前完整 schema 版本为 `go-agent-wss-v55-traffic-checkpoint-tiers`。v48 将滥用监测拆分为 72 小时原始证据、短期终态投影、7 天去重凭据以及 5 分钟/小时趋势汇总;v49 为实例性能图表增加 5 分钟和小时汇总,并将一分钟原始点限制为 24 小时、小时趋势限制为 35 天;v50 保持分钟级 FEFO 扣减和游标更新不变,将同一小时、实例和额度的用量累积到持久结算桶,小时结束、额度耗尽、到期、债务结算或异常栅栏时再生成不可变账本事件。迟到采样使用新的桶修订,绝不改写已封存事件;v51 将性能历史改为逻辑实例主键,删除每行重复的宿主机 hash 和文本 UUID,将永久流量总数/检查点的 UUID 与 hash 改为二进制并删除未被读取路径使用的宽索引;v52 保持实时 FEFO 扣减不变,将新用量累积到 UTC 每日结算窗口,并通过 `bucket_seconds` 与 `seal_at` 继续识别和封存升级前的小时窗口;v53 增加紧凑结算凭据,冻结已经安全压缩的窗口身份、最大修订、账本事件和载荷哈希,让迟到采样无需依赖完整旧桶也能从下一修订继续;v54 启用7天安全窗后的完整桶压缩,只有账本身份、金额、载荷和哈希全部吻合时,才在同一主库事务中更新凭据并删除完整桶;v55 为每个逻辑实例写入一条 UTC 每日累计检查点,并在服务周期首次确定基准时按周期冻结精确边界锚点,维护迁移后仍沿用同一逻辑实例历史。部署既有数据库时必须先停止生命周期与滥用监测 worker,按当前 `cloud.sql`、`cloud_log.sql` 的新增表和字段制作一次性迁移并完成回填,再启动新代码;v51 涉及日志表主键与二进制身份重建,不能在采集 worker 运行时原地改列,也不能让 v50 代码与 v51 日志表混跑;v52 必须先为既有结算桶回填 `bucket_seconds = 3600` 与 `seal_at = bucket_start + 3600`,再替换唯一索引、到期索引和检查约束;v53 必须先创建 `cloud_traffic_settlement_receipts`;v54 必须增加 `cloud_traffic_settlement_retention` 索引后才能启动清理,验证失败的桶保持原样且不得人工跳过;v55 必须先给小时检查点回填可确认的逻辑 `instance_id`,再创建每日检查点和周期边界锚点表;无法唯一对应实例的旧检查点写入保留哨兵 `instance_id = 0`,不得伪造归属,自动清理也会忽略这些行。不得让缺少汇总表、结算桶窗口字段、紧凑凭据表、每日检查点、周期锚点或 `instance_id` 的旧数据库直接进入采集或流量结算。v47 为 `cloud_offers` 增加季度、年度及对应自动备份价格;既有方案升级时必须在恢复销售前显式回填 90/365 天价格,不能让数据库默认零价进入购买目录。从 v44 升级时,必须先停止滥用观测与投影 worker,为既有 finding 回填 `occurrence_generation = 1` 与 `transition_revision = 1`,再创建 `cloud_abuse_alert_events` 和 `cloud_abuse_alert_event_items`;从 v45 升级还必须停止通知 worker,创建 `cloud_abuse_alert_deliveries`、`administrator_telegram_delivery_gate` 并写入唯一 gate 行。历史 finding 不反向合成事件;事件只从升级后主库事务内真实发生的 `opened`、`escalated` 和 `evidence_clear` 解决转移产生。完成列、约束、索引、外键和 gate 核对后,先恢复滥用观测,再恢复通知 worker。 事件 header 与 projection receipt 在同一个主库事务中写入,且事件先于 receipt;同一实例同一采样桶的多规则、双协议族转移合并为一个 header 和多个 item。`continued` 不产生事件,`policy_replaced` 与 `owner_changed` 只关闭 finding,不能伪造“恢复”事件。item 已冻结当时的聚合 evidence、规则身份和 finding 代次,后续投递不得回读会变化的 finding。v46 为每个事件建立至多一条 Telegram 投递决策;远端调用开始后的中断或回执不确定统一进入 `unknown`,禁止自动重放。删除用户时按事件冻结的 `user_id` 显式删除 header,item 与投递账本按当前外键和账户清理规则收敛,避免实例转交后遗留原租户数据。 diff --git a/docs/TRAFFIC_POOL_ROLLOUT.md b/docs/TRAFFIC_POOL_ROLLOUT.md index ece0c8f4..6b2db233 100644 --- a/docs/TRAFFIC_POOL_ROLLOUT.md +++ b/docs/TRAFFIC_POOL_ROLLOUT.md @@ -358,3 +358,62 @@ Before marking rollout complete, retain: If live Agent/network acceptance was not completed, report only that static and local tests passed; do not claim the shared traffic pool is production ready. + +## 7. Traffic-storage compaction rollout + +Traffic-storage compaction changes retention only. It does not change the +minute-level cumulative counter, FEFO allocation, pool projection or network +policy decision. Deploy it as a clean schema/code boundary: + +1. Stop lifecycle before changing either database. Keep ordinary and recovery + workers running unless the same change also modifies their schema. +2. Back up the business and log databases. Record row counts and oldest + timestamps for settlement buckets, ledger events, hourly checkpoints, + daily traffic rows and service periods. +3. Apply the exact reviewed v55 migration. Create the compact settlement + receipt table, settlement retention index, daily checkpoint table and + service-period boundary-anchor table. Add `instance_id` and its index to + hourly checkpoints. +4. Backfill an hourly checkpoint's logical instance only from one exact, + unambiguous current or retained placement identity. Set an unmappable old + row to the reserved value `instance_id = 0`; never infer ownership from a + user, name, address or nearby timestamp. Those rows are retained for manual + review and are excluded from automatic deletion. +5. For each mapped instance/day, a reviewed one-time migration may derive one + daily row from the earliest and latest existing hourly cumulative rows. + Do not synthesize service-period anchors. Existing periods with persisted + base counters need no anchor; an open period with null bases deliberately + blocks deletion until the normal lifecycle path freezes real evidence. +6. Deploy the matching code and restart lifecycle. Open the administrator + task page and run its read-only health check. The “流量存储压缩” row exposes + due settlement buckets, due hourly rows, involved instances, cursor + attention count and oldest lag without exposing IDs or payloads. +7. Keep the pool in its existing rollout state while observing at least one + complete collection, accounting, settlement-compaction and + checkpoint-compaction pass. A backlog less than one hour beyond its + retention window remains informational; a larger lag, scan truncation or + attention cursor is a warning that must be investigated. + +Acceptance requires all of the following: + +- a repeated cumulative observation has no second accounting effect; +- a sealed daily bucket older than seven days is replaced by one matching + compact receipt while its immutable ledger event remains unchanged; +- a late observation starts the next bucket revision after detail compaction; +- an hourly checkpoint older than 72 hours is removed only when its daily row + contains the exact cumulative values and every relevant open period has + persisted bases or a valid anchor; +- missing daily evidence, a malformed anchor or an attention cursor retains + the hourly row and reports a warning; +- deleting an hourly row does not change the activated period baseline; +- 36-day performance-hour rows are pruned while 34-day rows and the complete + 30-day chart remain readable; +- account deletion removes the user's totals, checkpoints, daily history, + anchors and performance history. + +Rollback is a database restore plus matching source rollback performed while +lifecycle is stopped. Do not run old code against the v55 log schema, remove +receipts to recreate full settlement buckets, delete blockers, manufacture +anchors or mark an attention cursor ready. A retention incident is fail-safe: +leave the extra rows in place, repair the evidence or schema, and rerun the +same bounded lifecycle stage. diff --git a/web/app/Services/Cloud/CloudRuntimeHealthService.php b/web/app/Services/Cloud/CloudRuntimeHealthService.php index 5c812381..c93c99e4 100644 --- a/web/app/Services/Cloud/CloudRuntimeHealthService.php +++ b/web/app/Services/Cloud/CloudRuntimeHealthService.php @@ -16,9 +16,11 @@ final class CloudRuntimeHealthService private \Closure $agentProbe; private \Closure $redisProbe; private \Closure $heartbeatProbe; + private \Closure $trafficStorageProbe; public function __construct(?callable $databaseProbe = null, ?callable $agentProbe = null, - ?callable $redisProbe = null, ?callable $heartbeatProbe = null) + ?callable $redisProbe = null, ?callable $heartbeatProbe = null, + ?callable $trafficStorageProbe = null) { $this->databaseProbe = \Closure::fromCallable($databaseProbe ?? [self::class, 'probeDatabase']); $this->agentProbe = \Closure::fromCallable($agentProbe ?? static function ( @@ -53,6 +55,10 @@ final class CloudRuntimeHealthService }); $this->heartbeatProbe = \Closure::fromCallable($heartbeatProbe ?? [CloudRuntimeHeartbeat::class, 'snapshot']); + $this->trafficStorageProbe = \Closure::fromCallable( + $trafficStorageProbe + ?? [CloudTrafficStorageHealthService::class, 'snapshot'] + ); } public function snapshot(array $actor): array @@ -67,9 +73,16 @@ final class CloudRuntimeHealthService $checks[] = $this->databaseCheck('log', 'log_database', '日志数据库', '已完成只读查询;不代表指标采集程序正在运行。', '日志数据库读取失败,请检查连接和服务日志。'); + $checks[] = $this->trafficStorageCheck(); } else { $checks[] = $this->row('log_database', '日志数据库', 'disabled', '未启用日志数据库;虚拟化指标依赖该配置。'); + $checks[] = $this->row( + 'traffic_storage', + '流量存储压缩', + 'disabled', + '日志数据库未启用,未执行流量存储压缩检查。' + ); } $checks[] = $this->check('agents', '宿主机 Agent 配置', fn (): bool => ($this->agentProbe)($actor) === true, @@ -242,6 +255,53 @@ final class CloudRuntimeHealthService return $this->row($key, $label, $ok ? 'ok' : 'error', $ok ? $success : $failure); } + private function trafficStorageCheck(): array + { + try { + $result = ($this->trafficStorageProbe)(); + if (!is_array($result) + || !in_array($result['status'] ?? null, + ['ok', 'warning'], true) + || !is_string($result['message'] ?? null) + || strlen($result['message']) > 500 + || preg_match('/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/', + $result['message']) === 1 + || !is_array($result['metrics'] ?? null)) { + throw new \RuntimeException('traffic_storage_health_invalid'); + } + $metrics = []; + foreach ([ + 'settlement_buckets_due', 'hourly_checkpoints_due', + 'hourly_checkpoint_instances_due', + 'cursor_attention_instances', 'oldest_due_lag_seconds', + ] as $key) { + $value = $result['metrics'][$key] ?? null; + if (!is_int($value) || $value < 0) { + throw new \RuntimeException( + 'traffic_storage_health_invalid' + ); + } + $metrics[$key] = $value; + } + $metrics['instance_scan_truncated'] = + ($result['metrics']['instance_scan_truncated'] ?? null) + === true; + return $this->row( + 'traffic_storage', + '流量存储压缩', + (string) $result['status'], + (string) $result['message'] + ) + ['metrics' => $metrics]; + } catch (\Throwable $exception) { + return $this->row( + 'traffic_storage', + '流量存储压缩', + 'error', + '流量存储压缩状态读取失败,请检查数据库结构与生命周期日志。' + ); + } + } + private function row(string $key, string $label, string $status, string $message): array { return compact('key', 'label', 'status', 'message') + ['checked_at' => time()]; diff --git a/web/app/Services/Cloud/CloudRuntimeHeartbeat.php b/web/app/Services/Cloud/CloudRuntimeHeartbeat.php index d0d8ca67..04886a90 100644 --- a/web/app/Services/Cloud/CloudRuntimeHeartbeat.php +++ b/web/app/Services/Cloud/CloudRuntimeHeartbeat.php @@ -35,6 +35,7 @@ final class CloudRuntimeHeartbeat 'abuse_monitor_retention' => '网络观测保留清理', 'traffic_sync' => '流量同步', 'traffic_settlement_compaction' => '流量结算压缩', + 'traffic_checkpoint_compaction' => '流量检查点压缩', 'runtime_status_sync' => '实例状态同步', 'instance_reconcile' => '实例策略核对', 'idle' => '等待下一轮', 'stopped' => '已退出']; diff --git a/web/app/Services/Cloud/CloudTrafficStorageHealthService.php b/web/app/Services/Cloud/CloudTrafficStorageHealthService.php new file mode 100644 index 00000000..8b5725c3 --- /dev/null +++ b/web/app/Services/Cloud/CloudTrafficStorageHealthService.php @@ -0,0 +1,122 @@ + + * } + */ + public static function snapshot(?int $now = null): array + { + $now ??= time(); + if ($now <= max( + CloudTrafficSettlementRetention::RETENTION_SECONDS, + CloudTrafficCheckpointRetention::RETENTION_SECONDS + )) { + throw new \InvalidArgumentException( + 'traffic_storage_health_clock_invalid' + ); + } + $main = User::getConnectionResolver()->connection(); + $log = User::getConnectionResolver()->connection('log'); + if (!$main->getSchemaBuilder()->hasTable( + 'cloud_traffic_usage_settlement_buckets' + ) || !$main->getSchemaBuilder()->hasTable( + 'cloud_traffic_instance_cursors' + ) || !$log->getSchemaBuilder()->hasTable( + 'cloud_vm_traffic_checkpoints' + )) { + throw new \RuntimeException('traffic_storage_health_schema_missing'); + } + $settlementCutoff = $now + - CloudTrafficSettlementRetention::RETENTION_SECONDS; + $checkpointCutoff = $now + - CloudTrafficCheckpointRetention::RETENTION_SECONDS; + $settlementQuery = $main->table( + 'cloud_traffic_usage_settlement_buckets' + )->where('state', 'sealed')->where('updated_at', '<=', $settlementCutoff); + $settlementCount = (int) (clone $settlementQuery)->count(); + $oldestSettlement = $settlementCount === 0 ? null + : (int) (clone $settlementQuery)->min('updated_at'); + $checkpointQuery = $log->table('cloud_vm_traffic_checkpoints') + ->where('instance_id', '>', 0) + ->where('first_observed_at', '<=', $checkpointCutoff); + $checkpointCount = (int) (clone $checkpointQuery)->count(); + $oldestCheckpoint = $checkpointCount === 0 ? null + : (int) (clone $checkpointQuery)->min('first_observed_at'); + $dueInstanceIds = (clone $checkpointQuery) + ->select('instance_id')->distinct()->orderBy('instance_id') + ->limit(self::INSTANCE_SCAN_LIMIT + 1)->pluck('instance_id') + ->map(static fn ($id): int => (int) $id) + ->filter(static fn (int $id): bool => $id > 0) + ->values()->all(); + $scanTruncated = count($dueInstanceIds) > self::INSTANCE_SCAN_LIMIT; + if ($scanTruncated) { + $dueInstanceIds = array_slice( + $dueInstanceIds, 0, self::INSTANCE_SCAN_LIMIT + ); + } + $attentionCount = $dueInstanceIds === [] ? 0 + : (int) $main->table('cloud_traffic_instance_cursors') + ->whereIn('instance_id', $dueInstanceIds) + ->where('state', 'attention')->count(); + $oldestLag = max( + $oldestSettlement === null + ? 0 : $settlementCutoff - $oldestSettlement, + $oldestCheckpoint === null + ? 0 : $checkpointCutoff - $oldestCheckpoint + ); + $metrics = [ + 'settlement_buckets_due' => $settlementCount, + 'hourly_checkpoints_due' => $checkpointCount, + 'hourly_checkpoint_instances_due' => count($dueInstanceIds), + 'cursor_attention_instances' => $attentionCount, + 'oldest_due_lag_seconds' => max(0, $oldestLag), + 'instance_scan_truncated' => $scanTruncated, + ]; + $status = $attentionCount > 0 || $scanTruncated + || $oldestLag > self::WARNING_LAG_SECONDS + ? 'warning' : 'ok'; + if ($settlementCount === 0 && $checkpointCount === 0) { + $message = '结算桶和小时检查点均处于既定保留窗口内。'; + } else { + $message = '待压缩结算桶 ' . $settlementCount + . ' 个,小时检查点 ' . $checkpointCount . ' 行(涉及 ' + . count($dueInstanceIds) . ' 个实例);最旧超出窗口 ' + . self::duration(max(0, $oldestLag)) . ',其中流量游标异常 ' + . $attentionCount . ' 个实例。'; + if ($scanTruncated) { + $message .= ' 实例计数已达到有界扫描上限。'; + } + } + return [ + 'key' => 'traffic_storage', + 'label' => '流量存储压缩', + 'status' => $status, + 'message' => $message, + 'checked_at' => $now, + 'metrics' => $metrics, + ]; + } + + private static function duration(int $seconds): string + { + if ($seconds >= 86400) { + return intdiv($seconds, 86400) . ' 天'; + } + if ($seconds >= 3600) { + return intdiv($seconds, 3600) . ' 小时'; + } + return intdiv($seconds, 60) . ' 分钟'; + } +} diff --git a/web/tests/CloudRuntimeHealthTest.php b/web/tests/CloudRuntimeHealthTest.php index dfcf91cb..775189e1 100644 --- a/web/tests/CloudRuntimeHealthTest.php +++ b/web/tests/CloudRuntimeHealthTest.php @@ -39,7 +39,22 @@ $service = new CloudRuntimeHealthService( $calls[] = 'agents'; return true; }, static function () use (&$calls): bool { $calls[] = 'redis'; return true; }, - static fn (): array => $heartbeats + static fn (): array => $heartbeats, + static function () use (&$calls): array { + $calls[] = 'traffic_storage'; + return [ + 'status' => 'warning', + 'message' => '待压缩结算桶 2 个。', + 'metrics' => [ + 'settlement_buckets_due' => 2, + 'hourly_checkpoints_due' => 3, + 'hourly_checkpoint_instances_due' => 1, + 'cursor_attention_instances' => 0, + 'oldest_due_lag_seconds' => 120, + 'instance_scan_truncated' => false, + ], + ]; + } ); foreach ([['user_id' => 42, 'is_admin' => false], ['user_id' => 0, 'is_admin' => true], []] as $denied) { try { @@ -52,8 +67,8 @@ foreach ([['user_id' => 42, 'is_admin' => false], ['user_id' => 0, 'is_admin' => $assert($calls === [], 'Unauthorized requests performed probes.'); $payload = $service->snapshot($actor); $checks = array_column($payload['checks'], null, 'key'); -$assert(count($checks) === 14 - && $calls === ['default', 'log', 'agents', 'redis'], +$assert(count($checks) === 15 + && $calls === ['default', 'log', 'traffic_storage', 'agents', 'redis'], 'Missing or duplicate probes.'); $assert($checks['agents']['status'] === 'ok' && $checks['worker']['status'] === 'busy' && $checks['policy']['status'] === 'unknown' @@ -63,6 +78,8 @@ $assert($checks['agents']['status'] === 'ok' && $checks['worker']['status'] === && $checks['storage_publication']['status'] === 'unknown' && $checks['storage_health']['status'] === 'unknown' && $checks['abuse_monitor']['status'] === 'unknown' + && $checks['traffic_storage']['status'] === 'warning' + && $checks['traffic_storage']['metrics']['settlement_buckets_due'] === 2 && $checks['lifecycle']['status'] === 'warning', 'Health or heartbeat semantics changed.'); $assert(!str_contains(json_encode($payload), 'MUST_NOT_EXPOSE'), 'Heartbeat internal fields leaked.'); @@ -87,7 +104,9 @@ $calls = []; $optional = $service->snapshot($actor); $optionalChecks = array_column($optional['checks'], null, 'key'); $assert($calls === ['default', 'agents'] && $optionalChecks['redis']['status'] === 'disabled' - && $optionalChecks['log_database']['status'] === 'disabled', 'Disabled services were probed.'); + && $optionalChecks['log_database']['status'] === 'disabled' + && $optionalChecks['traffic_storage']['status'] === 'disabled', + 'Disabled services were probed.'); $falseProbe = new CloudRuntimeHealthService(static fn () => false, static fn () => false, static fn () => false, static fn () => []); $assert(array_column($falseProbe->snapshot($actor)['checks'], 'status', 'key')['agents'] === 'error', @@ -187,7 +206,7 @@ $response = (new CloudAdminRuntimeController())->index($request, new Response()) $assert($response->getStatusCode() === 200 && $response->getHeaderLine('Cache-Control') === 'private, no-store', 'Runtime response cached or unavailable.'); $decoded = json_decode((string) $response->getBody(), true); -$assert(count($decoded['checks']) === 14 +$assert(count($decoded['checks']) === 15 && array_column($decoded['checks'], 'status', 'key')['agents'] === 'error', 'Partial failure lost other checks or fabricated Agent configuration health.'); $routes = file_get_contents(dirname(__DIR__) . '/config/routes_cloud.php'); diff --git a/web/tests/CloudTrafficStorageHealthServiceTest.php b/web/tests/CloudTrafficStorageHealthServiceTest.php new file mode 100644 index 00000000..c049ec2e --- /dev/null +++ b/web/tests/CloudTrafficStorageHealthServiceTest.php @@ -0,0 +1,82 @@ +addConnection([ + 'driver' => 'sqlite', 'database' => ':memory:', +], 'default'); +$capsule->addConnection([ + 'driver' => 'sqlite', 'database' => ':memory:', +], 'log'); +$capsule->setAsGlobal(); +$capsule->bootEloquent(); + +$main = $capsule->schema('default'); +$main->create('cloud_traffic_usage_settlement_buckets', + static function (Blueprint $table): void { + $table->increments('id'); + $table->string('state'); + $table->integer('updated_at'); + }); +$main->create('cloud_traffic_instance_cursors', + static function (Blueprint $table): void { + $table->integer('instance_id')->primary(); + $table->string('state'); + }); +$capsule->schema('log')->create('cloud_vm_traffic_checkpoints', + static function (Blueprint $table): void { + $table->integer('instance_id'); + $table->integer('first_observed_at'); + }); + +$now = 20 * 86400; +$healthy = CloudTrafficStorageHealthService::snapshot($now); +$assert($healthy['status'] === 'ok' + && $healthy['metrics']['settlement_buckets_due'] === 0 + && $healthy['metrics']['hourly_checkpoints_due'] === 0 + && $healthy['metrics']['cursor_attention_instances'] === 0, + 'Empty compaction queues did not report a healthy bounded state.'); + +$settlementCutoff = $now + - CloudTrafficSettlementRetention::RETENTION_SECONDS; +$checkpointCutoff = $now + - CloudTrafficCheckpointRetention::RETENTION_SECONDS; +$capsule->connection('default') + ->table('cloud_traffic_usage_settlement_buckets')->insert([ + ['state' => 'sealed', 'updated_at' => $settlementCutoff - 7200], + ['state' => 'open', 'updated_at' => $settlementCutoff - 86400], + ]); +$capsule->connection('log')->table('cloud_vm_traffic_checkpoints')->insert([ + ['instance_id' => 3, 'first_observed_at' => $checkpointCutoff - 7200], + ['instance_id' => 4, 'first_observed_at' => $checkpointCutoff + 60], +]); +$capsule->connection('default')->table( + 'cloud_traffic_instance_cursors' +)->insert([ + 'instance_id' => 3, + 'state' => 'attention', +]); +$warning = CloudTrafficStorageHealthService::snapshot($now); +$assert($warning['status'] === 'warning' + && $warning['metrics']['settlement_buckets_due'] === 1 + && $warning['metrics']['hourly_checkpoints_due'] === 1 + && $warning['metrics']['hourly_checkpoint_instances_due'] === 1 + && $warning['metrics']['cursor_attention_instances'] === 1 + && $warning['metrics']['oldest_due_lag_seconds'] === 7200 + && str_contains($warning['message'], '结算桶 1 个') + && str_contains($warning['message'], '小时检查点 1 行'), + 'Compaction backlog or attention state was hidden from operator health.'); + +echo "Traffic storage health summary contracts passed.\n";