refactor(metrics): compact telemetry storage

go
codex 24 hours ago
parent 067370adc1
commit df2e77f1a8

@ -160,7 +160,7 @@ libvirt 的持久 pool 定义就是 Agent 后续使用的映射,不另写存
公网网络接入在宿主机的“网络接入”中按资源池线路独立配置,可使用 `bridge:br0`、其他 Linux 网桥或 `network:default` 这样的 libvirt network。名称以字母或数字开头,后续只能包含字母、数字、点、下划线和短横线,bridge 名称为 1~15 字符,libvirt network 名称为 1~64 字符。宿主机上的公网网桥或 libvirt network 必须由运维预先准备。Agent 只发现、验证和使用现有公网网络,不创建公网网桥、不启动 libvirt network,也不修改公网地址、上游网关或挂载。`cloud_hosts.default_network` 仅保留为兼容展示值,不参与 Agent 就绪、调度、恢复、迁移或换 IP 决策;真正的运行时权威是资源池线路、当前 Agent 代次已验证的网络接入,以及绑定该接入的公网地址池。VPC 使用下述独立的 Agent 受管 bridge 和 L3 路由,不改变这一公网边界。
当前完整 schema 版本为 `go-agent-wss-v50-traffic-settlement-buckets`。v48 将滥用监测拆分为 72 小时原始证据、短期终态投影、7 天去重凭据以及 5 分钟/小时趋势汇总;v49 为实例性能图表增加 5 分钟和小时汇总,并将一分钟原始点限制为 24 小时;v50 保持分钟级 FEFO 扣减和游标更新不变,将同一小时、实例和额度的用量累积到持久结算桶,小时结束、额度耗尽、到期、债务结算或异常栅栏时再生成不可变账本事件。迟到采样使用新的桶修订,绝不改写已封存事件。部署既有数据库时必须先停止生命周期与滥用监测 worker,按当前 `cloud.sql`、`cloud_log.sql` 的新增表和字段制作一次性迁移并完成回填,再启动新代码;不得让缺少汇总表、结算桶或 `instance_id` 的旧数据库直接进入采集或流量结算。v47 为 `cloud_offers` 增加季度、年度及对应自动备份价格;既有方案升级时必须在恢复销售前显式回填 90/365 天价格,不能让数据库默认零价进入购买目录。从 v44 升级时,必须先停止滥用观测与投影 worker,为既有 finding 回填 `occurrence_generation = 1` 与 `transition_revision = 1`,再创建 `cloud_abuse_alert_events` 和 `cloud_abuse_alert_event_items`;从 v45 升级还必须停止通知 worker,创建 `cloud_abuse_alert_deliveries`、`administrator_telegram_delivery_gate` 并写入唯一 gate 行。历史 finding 不反向合成事件;事件只从升级后主库事务内真实发生的 `opened`、`escalated` 和 `evidence_clear` 解决转移产生。完成列、约束、索引、外键和 gate 核对后,先恢复滥用观测,再恢复通知 worker。
当前完整 schema 版本为 `go-agent-wss-v51-compact-telemetry-storage`。v48 将滥用监测拆分为 72 小时原始证据、短期终态投影、7 天去重凭据以及 5 分钟/小时趋势汇总;v49 为实例性能图表增加 5 分钟和小时汇总,并将一分钟原始点限制为 24 小时;v50 保持分钟级 FEFO 扣减和游标更新不变,将同一小时、实例和额度的用量累积到持久结算桶,小时结束、额度耗尽、到期、债务结算或异常栅栏时再生成不可变账本事件。迟到采样使用新的桶修订,绝不改写已封存事件;v51 将性能历史改为逻辑实例主键,删除每行重复的宿主机 hash 和文本 UUID,将永久流量总数/检查点的 UUID 与 hash 改为二进制并删除未被读取路径使用的宽索引。部署既有数据库时必须先停止生命周期与滥用监测 worker,按当前 `cloud.sql`、`cloud_log.sql` 的新增表和字段制作一次性迁移并完成回填,再启动新代码;v51 涉及日志表主键与二进制身份重建,不能在采集 worker 运行时原地改列,也不能让 v50 代码与 v51 日志表混跑。不得让缺少汇总表、结算桶或 `instance_id` 的旧数据库直接进入采集或流量结算。v47 为 `cloud_offers` 增加季度、年度及对应自动备份价格;既有方案升级时必须在恢复销售前显式回填 90/365 天价格,不能让数据库默认零价进入购买目录。从 v44 升级时,必须先停止滥用观测与投影 worker,为既有 finding 回填 `occurrence_generation = 1` 与 `transition_revision = 1`,再创建 `cloud_abuse_alert_events` 和 `cloud_abuse_alert_event_items`;从 v45 升级还必须停止通知 worker,创建 `cloud_abuse_alert_deliveries`、`administrator_telegram_delivery_gate` 并写入唯一 gate 行。历史 finding 不反向合成事件;事件只从升级后主库事务内真实发生的 `opened`、`escalated` 和 `evidence_clear` 解决转移产生。完成列、约束、索引、外键和 gate 核对后,先恢复滥用观测,再恢复通知 worker。
事件 header 与 projection receipt 在同一个主库事务中写入,且事件先于 receipt;同一实例同一采样桶的多规则、双协议族转移合并为一个 header 和多个 item。`continued` 不产生事件,`policy_replaced` 与 `owner_changed` 只关闭 finding,不能伪造“恢复”事件。item 已冻结当时的聚合 evidence、规则身份和 finding 代次,后续投递不得回读会变化的 finding。v46 为每个事件建立至多一条 Telegram 投递决策;远端调用开始后的中断或回执不确定统一进入 `unknown`,禁止自动重放。删除用户时按事件冻结的 `user_id` 显式删除 header,item 与投递账本按当前外键和账户清理规则收敛,避免实例转交后遗留原租户数据。

@ -47,7 +47,7 @@
## 部署
全新数据库必须由当前 `CloudSetup` 创建并验证 `go-agent-wss-v50-traffic-settlement-buckets` 完整 schema;其中补偿模块包括:
全新数据库必须由当前 `CloudSetup` 创建并验证 `go-agent-wss-v51-compact-telemetry-storage` 完整 schema;其中补偿模块包括:
- `cloud_service_compensations`;
- `cloud_service_compensation_items`;

@ -8,7 +8,7 @@ use App\Services\Config;
/** Initializes only the clean-break PHP business and Agent log schemas. */
final class CloudSetup
{
private const SCHEMA_VERSION = 'go-agent-wss-v50-traffic-settlement-buckets';
private const SCHEMA_VERSION = 'go-agent-wss-v51-compact-telemetry-storage';
/** Tables installed by ss.sql before cloudInit and owned by this schema. */
private const PREINSTALLED_MAIN_TABLES = [
'ui_copy_locales', 'ui_copy_entries', 'ui_copy_drafts',
@ -663,15 +663,14 @@ final class CloudSetup
];
private const REQUIRED_LOG_COLUMNS = [
'cloud_vm_metric_samples' => [
'instance_id', 'host_scope', 'vm_uuid', 'host_id', 'user_id',
'sampled_at',
'instance_id', 'user_id', 'sampled_at',
'cpu_percent', 'memory_used_mib', 'memory_total_mib',
'disk_read_bps', 'disk_write_bps', 'network_rx_bps',
'network_tx_bps', 'network_rx_bytes', 'network_tx_bytes',
],
'cloud_vm_metric_rollups' => [
'instance_id', 'host_scope', 'vm_uuid', 'host_id', 'user_id',
'interval_seconds', 'bucket_start', 'sample_count',
'instance_id', 'user_id', 'interval_seconds', 'bucket_start',
'sample_count',
'last_sampled_at', 'cpu_percent_sum', 'cpu_percent_count',
'memory_used_mib_sum', 'memory_used_mib_count',
'memory_total_mib_sum', 'memory_total_mib_count',
@ -681,14 +680,14 @@ final class CloudSetup
'network_tx_bps_sum', 'network_tx_bps_count',
],
'cloud_vm_traffic_totals' => [
'host_scope', 'vm_uuid', 'host_id', 'user_id',
'host_id', 'vm_uuid', 'user_id',
'last_observed_at', 'network_signature',
'network_baseline_json', 'download_rx_bytes',
'upload_tx_bytes', 'updated_at',
],
'cloud_vm_traffic_checkpoints' => [
'host_scope', 'vm_uuid', 'checkpoint_hour', 'host_id',
'user_id', 'first_observed_at', 'download_rx_bytes',
'host_id', 'vm_uuid', 'checkpoint_hour', 'user_id',
'first_observed_at', 'download_rx_bytes',
'upload_tx_bytes',
],
'cloud_vm_daily_traffic_cursors' => [
@ -1526,12 +1525,17 @@ final class CloudSetup
];
private const REQUIRED_LOG_INDEXES = [
'cloud_vm_metric_samples' => [
'PRIMARY', 'vm_metric_retention', 'vm_metric_instance',
'vm_metric_owner',
'PRIMARY', 'vm_metric_retention', 'vm_metric_owner',
],
'cloud_vm_metric_rollups' => [
'PRIMARY', 'vm_metric_rollup_identity',
'vm_metric_rollup_retention', 'vm_metric_rollup_owner',
'PRIMARY', 'vm_metric_rollup_retention',
'vm_metric_rollup_owner',
],
'cloud_vm_traffic_totals' => [
'PRIMARY', 'vm_traffic_uuid', 'vm_traffic_owner',
],
'cloud_vm_traffic_checkpoints' => [
'PRIMARY', 'vm_traffic_checkpoint_owner',
],
'cloud_vm_network_observation_rollups' => [
'PRIMARY', 'cloud_network_rollup_owner',
@ -1540,8 +1544,6 @@ final class CloudSetup
'cloud_vm_network_observation_samples' => [
'PRIMARY', 'cloud_network_observation_source',
'cloud_network_observation_instance',
'cloud_network_observation_placement',
'cloud_network_observation_host',
'cloud_network_observation_retention',
],
'cloud_vm_network_projection_outbox' => [
@ -1958,20 +1960,17 @@ final class CloudSetup
'cloud_vm_metric_samples' => [
'PRIMARY' => [
'non_unique' => 0,
'columns' => ['host_scope', 'vm_uuid', 'sampled_at'],
'columns' => ['instance_id', 'sampled_at'],
],
'vm_metric_instance' => [
'vm_metric_owner' => [
'non_unique' => 1,
'columns' => ['instance_id', 'sampled_at'],
'columns' => ['user_id', 'sampled_at', 'instance_id'],
],
],
'cloud_vm_metric_rollups' => [
'PRIMARY' => [
'non_unique' => 0,
'columns' => [
'instance_id', 'host_scope', 'vm_uuid',
'interval_seconds', 'bucket_start',
],
'columns' => ['instance_id', 'interval_seconds', 'bucket_start'],
],
'vm_metric_rollup_retention' => [
'non_unique' => 1,
@ -1980,6 +1979,30 @@ final class CloudSetup
],
],
],
'cloud_vm_traffic_totals' => [
'PRIMARY' => [
'non_unique' => 0,
'columns' => ['host_id', 'vm_uuid'],
],
'vm_traffic_owner' => [
'non_unique' => 1,
'columns' => ['user_id', 'host_id'],
],
'vm_traffic_uuid' => [
'non_unique' => 1,
'columns' => ['vm_uuid', 'host_id'],
],
],
'cloud_vm_traffic_checkpoints' => [
'PRIMARY' => [
'non_unique' => 0,
'columns' => ['host_id', 'vm_uuid', 'checkpoint_hour'],
],
'vm_traffic_checkpoint_owner' => [
'non_unique' => 1,
'columns' => ['user_id', 'host_id', 'first_observed_at'],
],
],
'cloud_vm_network_observation_rollups' => [
'PRIMARY' => [
'non_unique' => 0,
@ -2016,16 +2039,6 @@ final class CloudSetup
'non_unique' => 1,
'columns' => ['instance_id', 'bucket_start'],
],
'cloud_network_observation_placement' => [
'non_unique' => 1,
'columns' => [
'instance_id', 'placement_key', 'bucket_start',
],
],
'cloud_network_observation_host' => [
'non_unique' => 1,
'columns' => ['host_id', 'bucket_start'],
],
'cloud_network_observation_retention' => [
'non_unique' => 1,
'columns' => ['bucket_start'],

@ -0,0 +1,41 @@
<?php
namespace App\Services\Cloud;
/** Compact, lossless identities used only at the log-database boundary. */
final class CloudLogIdentity
{
public static function uuid(string $uuid): string
{
$uuid = strtolower(trim($uuid));
if (preg_match(
'/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/D',
$uuid
) !== 1) {
throw new \InvalidArgumentException('Invalid VM UUID');
}
$binary = hex2bin(str_replace('-', '', $uuid));
if (!is_string($binary) || strlen($binary) !== 16) {
throw new \RuntimeException('Unable to encode VM UUID');
}
return $binary;
}
public static function uuidKey(string $uuid): string
{
return bin2hex(self::uuid($uuid));
}
public static function storedUuidKey(string $binary): string
{
if (strlen($binary) !== 16) {
throw new \RuntimeException('Stored VM UUID is invalid');
}
return bin2hex($binary);
}
public static function sha256(string $value): string
{
return hash('sha256', $value, true);
}
}

@ -46,9 +46,6 @@ final class CloudMetricHistoryStore
public static function series(
object $log,
int $instanceId,
string $hostScope,
string $vmUuid,
string $hostId,
int $userId,
string $range,
int $now
@ -67,9 +64,6 @@ final class CloudMetricHistoryStore
$rows = self::rawSeries(
$log,
$instanceId,
$hostScope,
$vmUuid,
$hostId,
$userId,
$now - $seconds - $outputInterval,
$outputInterval
@ -78,9 +72,6 @@ final class CloudMetricHistoryStore
$rows = self::rollupSeries(
$log,
$instanceId,
$hostScope,
$vmUuid,
$hostId,
$userId,
$now - $seconds - $outputInterval,
$outputInterval,
@ -137,17 +128,12 @@ final class CloudMetricHistoryStore
* $interval;
$query = $log->table('cloud_vm_metric_rollups')
->where('instance_id', (int) $sample['instance_id'])
->where('host_scope', (string) $sample['host_scope'])
->where('vm_uuid', (string) $sample['vm_uuid'])
->where('interval_seconds', $interval)
->where('bucket_start', $bucketStart);
$existing = $query->lockForUpdate()->first();
if ($existing === null) {
$values = [
'instance_id' => (int) $sample['instance_id'],
'host_scope' => (string) $sample['host_scope'],
'vm_uuid' => (string) $sample['vm_uuid'],
'host_id' => (string) $sample['host_id'],
'user_id' => (int) $sample['user_id'],
'interval_seconds' => $interval,
'bucket_start' => $bucketStart,
@ -162,10 +148,9 @@ final class CloudMetricHistoryStore
$log->table('cloud_vm_metric_rollups')->insert($values);
return;
}
if ((string) $existing->host_id !== (string) $sample['host_id']
|| (int) $existing->user_id !== (int) $sample['user_id']) {
if ((int) $existing->user_id !== (int) $sample['user_id']) {
throw new \RuntimeException(
'Metric rollup ownership changed inside one placement bucket'
'Metric rollup ownership changed inside one logical bucket'
);
}
$values = [
@ -192,9 +177,6 @@ final class CloudMetricHistoryStore
private static function rawSeries(
object $log,
int $instanceId,
string $hostScope,
string $vmUuid,
string $hostId,
int $userId,
int $cutoff,
int $interval
@ -207,9 +189,6 @@ final class CloudMetricHistoryStore
}
return $log->table('cloud_vm_metric_samples')
->where('instance_id', $instanceId)
->where('host_scope', $hostScope)
->where('vm_uuid', $vmUuid)
->where('host_id', $hostId)
->where('user_id', $userId)
->where('sampled_at', '>=', $cutoff)
->selectRaw($select)->groupByRaw($bucket)
@ -220,9 +199,6 @@ final class CloudMetricHistoryStore
private static function rollupSeries(
object $log,
int $instanceId,
string $hostScope,
string $vmUuid,
string $hostId,
int $userId,
int $cutoff,
int $outputInterval,
@ -239,9 +215,6 @@ final class CloudMetricHistoryStore
}
return $log->table('cloud_vm_metric_rollups')
->where('instance_id', $instanceId)
->where('host_scope', $hostScope)
->where('vm_uuid', $vmUuid)
->where('host_id', $hostId)
->where('user_id', $userId)
->where('interval_seconds', $sourceInterval)
->where('bucket_start', '>=', $cutoff)
@ -264,12 +237,11 @@ final class CloudMetricHistoryStore
$rows = $log->table('cloud_vm_metric_samples')
->where('sampled_at', '<', $cutoff)
->orderBy('sampled_at')->limit(self::MAX_PRUNE_ROWS)
->get(['host_scope', 'vm_uuid', 'sampled_at']);
->get(['instance_id', 'sampled_at']);
$deleted = 0;
foreach ($rows as $row) {
$deleted += $log->table('cloud_vm_metric_samples')
->where('host_scope', (string) $row->host_scope)
->where('vm_uuid', (string) $row->vm_uuid)
->where('instance_id', (int) $row->instance_id)
->where('sampled_at', (int) $row->sampled_at)
->where('sampled_at', '<', $cutoff)->delete();
}
@ -285,13 +257,11 @@ final class CloudMetricHistoryStore
->where('interval_seconds', $interval)
->where('bucket_start', '<', $cutoff)
->orderBy('bucket_start')->limit(self::MAX_PRUNE_ROWS)
->get(['instance_id', 'host_scope', 'vm_uuid', 'bucket_start']);
->get(['instance_id', 'bucket_start']);
$deleted = 0;
foreach ($rows as $row) {
$deleted += $log->table('cloud_vm_metric_rollups')
->where('instance_id', (int) $row->instance_id)
->where('host_scope', (string) $row->host_scope)
->where('vm_uuid', (string) $row->vm_uuid)
->where('interval_seconds', $interval)
->where('bucket_start', (int) $row->bucket_start)
->where('bucket_start', '<', $cutoff)->delete();

@ -79,11 +79,21 @@ final class CloudTrafficService
if ($attempted >= $limit) {
break;
}
$scope = self::hostScope((int) $hostId);
try {
$totals = $log->table('cloud_vm_traffic_totals')
->where('host_scope', $scope)->whereIn('vm_uuid',
array_column($owned, 'uuid'))->get()->keyBy('vm_uuid');
$storedUuids = array_map(
static fn (array $item): string =>
CloudLogIdentity::uuid($item['uuid']),
$owned
);
$totalRows = $log->table('cloud_vm_traffic_totals')
->where('host_id', (int) $hostId)
->whereIn('vm_uuid', $storedUuids)->get();
$totals = [];
foreach ($totalRows as $row) {
$totals[CloudLogIdentity::storedUuidKey(
(string) $row->vm_uuid
)] = $row;
}
} catch (\Throwable $error) {
$result['errors'][] = 'traffic_collection_failed';
continue;
@ -98,7 +108,9 @@ final class CloudTrafficService
$result['skipped']++;
continue;
}
$previous = $totals->get($item['uuid']);
$previous = $totals[
CloudLogIdentity::uuidKey($item['uuid'])
] ?? null;
$lastObservedAt = $previous === null ? 0
: (int) ($previous->last_observed_at ?? 0);
if ($lastObservedAt <= $now
@ -191,9 +203,6 @@ final class CloudTrafficService
return CloudMetricHistoryStore::series(
$log,
(int) $instance->id,
self::hostScope((int) $instance->host_id),
$uuid,
(string) (int) $instance->host_id,
(int) $instance->user_id,
$range,
time()
@ -224,12 +233,29 @@ final class CloudTrafficService
'rx_bytes' => self::counter($metrics, 'rx_bytes'),
'tx_bytes' => self::counter($metrics, 'tx_bytes'),
];
$scope = self::hostScope((int) $instance->host_id);
$storedUuid = CloudLogIdentity::uuid($uuid);
$spec = json_decode((string) $instance->spec_json, true);
$spec = is_array($spec) && !array_is_list($spec) ? $spec : [];
$generation = $spec['traffic_counter_generation'] ?? null;
$generation = is_array($generation) && !array_is_list($generation)
&& (int) ($generation['host_id'] ?? 0)
=== (int) $instance->host_id
&& hash_equals(
strtolower((string) ($generation['uuid'] ?? '')),
$uuid
)
&& is_int($generation['download_base_bytes'] ?? null)
&& $generation['download_base_bytes'] >= 0
&& is_int($generation['upload_base_bytes'] ?? null)
&& $generation['upload_base_bytes'] >= 0
? $generation : null;
$log->transaction(static function () use (
$log, $instance, $uuid, $collectedAt, $current, $scope
$log, $instance, $storedUuid, $collectedAt, $current, $spec,
$generation
): void {
$previous = $log->table('cloud_vm_traffic_totals')
->where('host_scope', $scope)->where('vm_uuid', $uuid)
->where('host_id', (int) $instance->host_id)
->where('vm_uuid', $storedUuid)
->lockForUpdate()->first();
if ($previous !== null) {
if (($previous->user_id !== null
@ -239,6 +265,15 @@ final class CloudTrafficService
!== (string) (int) $instance->host_id)) {
throw new CloudError('性能采样归属与实例不一致', 409);
}
} elseif ($generation === null
&& $log->table('cloud_vm_traffic_totals')
->where('vm_uuid', $storedUuid)
->where('host_id', '<>', (int) $instance->host_id)
->lockForUpdate()->first(['host_id']) !== null) {
// A cross-host UUID collision without a frozen migration
// generation is ambiguous. Do not create a second billing
// counter identity from potentially foreign data.
throw new CloudError('性能采样归属与实例不一致', 409);
}
$baseline = $previous === null ? [] : json_decode(
@ -255,8 +290,6 @@ final class CloudTrafficService
// timestamp is not suitable for chart ordering or rate math.
$elapsed = $orderedSample && $previous !== null
? $collectedAt - $previousObservedAt : 0;
$spec = json_decode((string) $instance->spec_json, true);
$spec = is_array($spec) && !array_is_list($spec) ? $spec : [];
$vcpu = filter_var($spec['vcpu'] ?? ($spec['cpu_cores'] ?? null),
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1, 'max_range' => 64]]);
$memoryTotal = filter_var($spec['memory_mib'] ?? null,
@ -299,19 +332,6 @@ final class CloudTrafficService
// first PHP observation they are already the authoritative lifetime
// total (including after a panel log-database restore), while there
// is deliberately no interval from which to derive a rate.
$generation = $spec['traffic_counter_generation'] ?? null;
$generation = is_array($generation) && !array_is_list($generation)
&& (int) ($generation['host_id'] ?? 0)
=== (int) $instance->host_id
&& hash_equals(
strtolower((string) ($generation['uuid'] ?? '')),
$uuid
)
&& is_int($generation['download_base_bytes'] ?? null)
&& $generation['download_base_bytes'] >= 0
&& is_int($generation['upload_base_bytes'] ?? null)
&& $generation['upload_base_bytes'] >= 0
? $generation : null;
$baseRx = $previous === null && $generation !== null
? $generation['download_base_bytes'] : 0;
$baseTx = $previous === null && $generation !== null
@ -340,9 +360,6 @@ final class CloudTrafficService
}
$sample = [
'instance_id' => (int) $instance->id,
'host_scope' => $scope,
'vm_uuid' => $uuid,
'host_id' => (string) (int) $instance->host_id,
'user_id' => (int) $instance->user_id,
'sampled_at' => $collectedAt,
'cpu_percent' => $cpu,
@ -368,10 +385,9 @@ final class CloudTrafficService
CloudMetricHistoryStore::recordRollups($log, $sample);
}
$log->table('cloud_vm_traffic_checkpoints')->insertOrIgnore([
'host_scope' => $scope,
'vm_uuid' => $uuid,
'host_id' => (int) $instance->host_id,
'vm_uuid' => $storedUuid,
'checkpoint_hour' => intdiv($collectedAt, 3600) * 3600,
'host_id' => (string) (int) $instance->host_id,
'user_id' => (int) $instance->user_id,
'first_observed_at' => $collectedAt,
'download_rx_bytes' => $panelRx,
@ -389,11 +405,11 @@ final class CloudTrafficService
],
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES);
$values = [
'host_id' => (string) (int) $instance->host_id,
'user_id' => (int) $instance->user_id,
'last_observed_at' => max($previousObservedAt, $collectedAt),
'network_signature' => hash('sha256',
'virt-agent-durable-traffic/v1'),
'network_signature' => CloudLogIdentity::sha256(
'virt-agent-durable-traffic/v1'
),
'network_baseline_json' => $encoded,
'download_rx_bytes' => $panelRx,
'upload_tx_bytes' => $panelTx,
@ -401,11 +417,13 @@ final class CloudTrafficService
];
if ($previous === null) {
$log->table('cloud_vm_traffic_totals')->insert($values + [
'host_scope' => $scope, 'vm_uuid' => $uuid,
'host_id' => (int) $instance->host_id,
'vm_uuid' => $storedUuid,
]);
} else {
$log->table('cloud_vm_traffic_totals')
->where('host_scope', $scope)->where('vm_uuid', $uuid)
->where('host_id', (int) $instance->host_id)
->where('vm_uuid', $storedUuid)
->update($values);
}
}, 3);
@ -450,11 +468,6 @@ final class CloudTrafficService
return $timestamp;
}
private static function hostScope(int $hostId): string
{
return hash('sha256', 'virt-cloud/host/' . $hostId);
}
private static function syncBatch($instances): int
{
if ($instances->isEmpty()) {
@ -472,16 +485,24 @@ final class CloudTrafficService
'cloud_vm_daily_traffic'
);
foreach ($groups as $hostId => $owned) {
$scope = self::hostScope($hostId);
$uuids = array_map(static fn ($instance) => (string) $instance->uuid, $owned);
$uuids = array_map(
static fn ($instance): string => CloudLogIdentity::uuid(
(string) $instance->uuid
),
$owned
);
$rows = $log->table('cloud_vm_traffic_totals')
->where('host_scope', $scope)->whereIn('vm_uuid', $uuids)->get();
->where('host_id', $hostId)->whereIn('vm_uuid', $uuids)->get();
$byUuid = [];
foreach ($rows as $row) {
$byUuid[(string) $row->vm_uuid] = $row;
$byUuid[CloudLogIdentity::storedUuidKey(
(string) $row->vm_uuid
)] = $row;
}
foreach ($owned as $instance) {
$row = $byUuid[(string) $instance->uuid] ?? null;
$row = $byUuid[CloudLogIdentity::uuidKey(
(string) $instance->uuid
)] ?? null;
// An absent or changed owner is unknown usage, not grounds to
// stop another tenant's VM. The next successful sample repairs it.
if ($row === null || $row->user_id === null
@ -541,12 +562,12 @@ final class CloudTrafficService
return null;
}
$log = User::getConnectionResolver()->connection('log');
$scope = self::hostScope((int) $instance->host_id);
$storedUuid = CloudLogIdentity::uuid((string) $instance->uuid);
if ($log->getSchemaBuilder()->hasTable('cloud_vm_traffic_checkpoints')) {
$checkpoint = $log->table('cloud_vm_traffic_checkpoints')
->where('host_scope', $scope)->where('vm_uuid', (string) $instance->uuid)
->where('host_id', (int) $instance->host_id)
->where('vm_uuid', $storedUuid)
->where('user_id', (int) $instance->user_id)
->where('host_id', (string) (int) $instance->host_id)
->where('first_observed_at', '>', $startsAt)
->where('first_observed_at', '<=', (int) $instance->traffic_synced_at)
->orderBy('first_observed_at')->first([
@ -570,9 +591,8 @@ final class CloudTrafficService
return null;
}
$rows = $log->table('cloud_vm_metric_samples')
->where('host_scope', $scope)->where('vm_uuid', (string) $instance->uuid)
->where('instance_id', (int) $instance->id)
->where('user_id', (int) $instance->user_id)
->where('host_id', (string) (int) $instance->host_id)
->where('sampled_at', '>', $startsAt)
->where('sampled_at', '<=', (int) $instance->traffic_synced_at)
->orderBy('sampled_at')->get([

@ -2,10 +2,7 @@
-- counters; PHP owns sampling, billing totals, chart retention and all writes.
CREATE TABLE cloud_vm_metric_samples (
instance_id BIGINT UNSIGNED NOT NULL,
host_scope CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
vm_uuid CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
host_id VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NULL,
user_id BIGINT UNSIGNED NULL,
user_id BIGINT UNSIGNED NOT NULL,
sampled_at BIGINT UNSIGNED NOT NULL,
cpu_percent DOUBLE NULL,
memory_used_mib DOUBLE NULL,
@ -16,10 +13,9 @@ CREATE TABLE cloud_vm_metric_samples (
network_tx_bps DOUBLE NULL,
network_rx_bytes BIGINT UNSIGNED NULL,
network_tx_bytes BIGINT UNSIGNED NULL,
PRIMARY KEY (host_scope, vm_uuid, sampled_at),
PRIMARY KEY (instance_id, sampled_at),
KEY vm_metric_retention (sampled_at),
KEY vm_metric_instance (instance_id, sampled_at),
KEY vm_metric_owner (user_id, host_id, sampled_at)
KEY vm_metric_owner (user_id, sampled_at, instance_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Hierarchical chart history. Every accepted raw sample contributes once to
@ -27,9 +23,6 @@ CREATE TABLE cloud_vm_metric_samples (
-- averages when a larger display bucket merges several stored rows.
CREATE TABLE cloud_vm_metric_rollups (
instance_id BIGINT UNSIGNED NOT NULL,
host_scope CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
vm_uuid CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
host_id VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
user_id BIGINT UNSIGNED NOT NULL,
interval_seconds SMALLINT UNSIGNED NOT NULL,
bucket_start BIGINT UNSIGNED NOT NULL,
@ -49,12 +42,7 @@ CREATE TABLE cloud_vm_metric_rollups (
network_rx_bps_count SMALLINT UNSIGNED NOT NULL,
network_tx_bps_sum DOUBLE NULL,
network_tx_bps_count SMALLINT UNSIGNED NOT NULL,
PRIMARY KEY (
instance_id, host_scope, vm_uuid, interval_seconds, bucket_start
),
KEY vm_metric_rollup_identity (
host_scope, vm_uuid, host_id, user_id, interval_seconds, bucket_start
),
PRIMARY KEY (instance_id, interval_seconds, bucket_start),
KEY vm_metric_rollup_retention (
interval_seconds, bucket_start, instance_id
),
@ -78,33 +66,33 @@ CREATE TABLE cloud_vm_metric_rollups (
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
CREATE TABLE cloud_vm_traffic_totals (
host_scope CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
vm_uuid CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
host_id VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NULL,
user_id BIGINT UNSIGNED NULL,
host_id BIGINT UNSIGNED NOT NULL,
vm_uuid BINARY(16) NOT NULL,
user_id BIGINT UNSIGNED NOT NULL,
last_observed_at BIGINT UNSIGNED NULL,
network_signature CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
network_signature BINARY(32) NOT NULL,
network_baseline_json JSON NOT NULL,
download_rx_bytes BIGINT UNSIGNED NOT NULL DEFAULT 0,
upload_tx_bytes BIGINT UNSIGNED NOT NULL DEFAULT 0,
updated_at BIGINT UNSIGNED NOT NULL,
PRIMARY KEY (host_scope, vm_uuid),
KEY vm_traffic_owner (user_id, host_id),
KEY vm_traffic_host (host_id, vm_uuid)
PRIMARY KEY (host_id, vm_uuid),
KEY vm_traffic_uuid (vm_uuid, host_id),
KEY vm_traffic_owner (user_id, host_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Compact, durable billing ledger. Chart samples below are pruned after 30
-- days; the first cumulative counter observed in every hour is not.
-- Compact, durable billing ledger. Raw chart samples are pruned after 24
-- hours and rollups are bounded; the first cumulative counter observed in
-- every hour is not. Host IDs remain numeric and UUIDs use their native
-- 16-byte representation because these rows grow for the life of a VM.
CREATE TABLE cloud_vm_traffic_checkpoints (
host_scope CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
vm_uuid CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
host_id BIGINT UNSIGNED NOT NULL,
vm_uuid BINARY(16) NOT NULL,
checkpoint_hour BIGINT UNSIGNED NOT NULL,
host_id VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NULL,
user_id BIGINT UNSIGNED NULL,
user_id BIGINT UNSIGNED NOT NULL,
first_observed_at BIGINT UNSIGNED NOT NULL,
download_rx_bytes BIGINT UNSIGNED NOT NULL,
upload_tx_bytes BIGINT UNSIGNED NOT NULL,
PRIMARY KEY (host_scope, vm_uuid, checkpoint_hour),
PRIMARY KEY (host_id, vm_uuid, checkpoint_hour),
KEY vm_traffic_checkpoint_owner (user_id, host_id, first_observed_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
@ -167,12 +155,6 @@ CREATE TABLE cloud_vm_network_observation_samples (
KEY cloud_network_observation_instance (
instance_id, bucket_start
),
KEY cloud_network_observation_placement (
instance_id, placement_key, bucket_start
),
KEY cloud_network_observation_host (
host_id, bucket_start
),
KEY cloud_network_observation_retention (bucket_start),
CONSTRAINT chk_cloud_network_observation_contract CHECK (
contract_version = 1

@ -208,15 +208,21 @@ foreach ([
foreach ([
'cloud_network_observation_source',
'cloud_network_observation_instance',
'cloud_network_observation_host',
'cloud_network_observation_retention',
'cloud_network_observation_placement',
] as $index) {
$assert(str_contains(
$definitions['cloud_vm_network_observation_samples'], $index
) && str_contains($setup, "'" . $index . "'"),
'Missing observation index: ' . $index);
}
$assert(!str_contains(
$definitions['cloud_vm_network_observation_samples'],
'cloud_network_observation_placement'
) && !str_contains(
$definitions['cloud_vm_network_observation_samples'],
'cloud_network_observation_host'
),
'Observation storage retained redundant wide placement/host indexes.');
foreach ([
'cloud_network_projection_due',
'cloud_network_projection_instance',
@ -409,7 +415,7 @@ $assert(str_contains(
$assert(str_contains(
$setup,
"SCHEMA_VERSION = 'go-agent-wss-v50-traffic-settlement-buckets'"
"SCHEMA_VERSION = 'go-agent-wss-v51-compact-telemetry-storage'"
), 'CloudSetup schema version was not advanced for abuse observation.');
foreach (array_keys($mainTables + $logTables) as $table) {

@ -96,7 +96,7 @@ foreach ($defaults->rules() as $code => $rule) {
'The fresh schema rule seed drifted from PHP: ' . $code);
}
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v50-traffic-settlement-buckets'")
"SCHEMA_VERSION = 'go-agent-wss-v51-compact-telemetry-storage'")
&& str_contains($service, "->where('active_slot', 1)->lockForUpdate()")
&& str_contains($service, "->where('active_slot', 1)->update")
&& str_contains($service, 'insertGetId'),

@ -66,10 +66,12 @@ $assert(str_contains($sample, 'metrics_json JSON NOT NULL')
'Observation storage can retain a raw Agent body instead of allowlisted metrics.');
$assert(str_contains($sample,
'cloud_network_observation_placement')
'placement_key CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL')
&& str_contains($sample,
'instance_id, placement_key, bucket_start'),
'Observation sample lacks migration-cutover placement evidence.');
'cloud_network_observation_instance')
&& !str_contains($sample, 'cloud_network_observation_placement')
&& !str_contains($sample, 'cloud_network_observation_host'),
'Observation sample lost placement evidence or retained redundant indexes.');
$assert(str_contains($outbox, 'PRIMARY KEY (sample_key)')
&& str_contains($outbox, 'instance_id BIGINT UNSIGNED NOT NULL')
@ -258,7 +260,7 @@ $assert(str_contains($projection, '->snapshotByIdentity($revision, $hash)')
&& !str_contains($projection, '->activeSnapshot()'),
'Projection can drift from the outbox identity to the current policy.');
$assert(str_contains($deployment,
'`go-agent-wss-v50-traffic-settlement-buckets`')
'`go-agent-wss-v51-compact-telemetry-storage`')
&& str_contains($deployment,
'`occurrence_generation = 1` 与 `transition_revision = 1`')
&& str_contains($deployment,
@ -302,8 +304,6 @@ foreach ([
foreach ([
"'cloud_network_observation_source'=>['non_unique'=>0,'columns'=>['host_id','collector_generation','window_id','vm_uuid',]",
"'cloud_network_observation_instance'=>['non_unique'=>1,'columns'=>['instance_id','bucket_start'],]",
"'cloud_network_observation_placement'=>['non_unique'=>1,'columns'=>['instance_id','placement_key','bucket_start',]",
"'cloud_network_observation_host'=>['non_unique'=>1,'columns'=>['host_id','bucket_start'],]",
"'cloud_network_observation_retention'=>['non_unique'=>1,'columns'=>['bucket_start'],]",
"'cloud_network_projection_due'=>['non_unique'=>1,'columns'=>['state','available_at','bucket_start','sample_key',]",
"'cloud_network_projection_instance'=>['non_unique'=>1,'columns'=>['instance_id','state','bucket_start','sample_key',]",

@ -136,6 +136,32 @@ $schema->create('cloud_traffic_ledger_events', static function (Blueprint $t): v
$t->text('payload_json');
$t->integer('created_at');
});
$schema->create('cloud_traffic_usage_settlement_buckets', static function (Blueprint $t): void {
$t->increments('id');
$t->integer('pool_id');
$t->integer('user_id');
$t->integer('instance_id');
$t->integer('service_period_id');
$t->integer('entitlement_id')->nullable();
$t->string('allocation_kind');
$t->integer('allocation_target_id');
$t->integer('bucket_start');
$t->integer('bucket_revision');
$t->integer('amount_bytes');
$t->integer('download_bytes');
$t->integer('upload_bytes');
$t->integer('sample_count');
$t->integer('first_observed_at');
$t->integer('last_observed_at');
$t->string('state');
$t->integer('sealed_event_id')->nullable();
$t->integer('created_at');
$t->integer('updated_at');
$t->unique([
'pool_id', 'instance_id', 'service_period_id', 'allocation_kind',
'allocation_target_id', 'bucket_start', 'bucket_revision',
], 'traffic_settlement_identity');
});
$schema->create('cloud_traffic_instance_cursors', static function (Blueprint $t): void {
$t->integer('instance_id')->primary();
$t->integer('user_id');

@ -64,7 +64,7 @@ foreach ([
'CloudSetup omits backup preparation contract: ' . $required);
}
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v50-traffic-settlement-buckets'")
"SCHEMA_VERSION = 'go-agent-wss-v51-compact-telemetry-storage'")
&& str_contains($setup,
"'columns' => [\n 'create_request_actor_id',\n 'create_request_idempotency_key',")
&& str_contains($setup, "'non_unique' => 0"),

@ -51,6 +51,7 @@ namespace {
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudInvoiceService;
use App\Services\Cloud\CloudLifecycleService;
use App\Services\Cloud\CloudLogIdentity;
use App\Services\Cloud\CloudRenewalService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
@ -337,26 +338,24 @@ namespace {
$logSchema = $capsule->schema('log');
$logSchema->create('cloud_vm_traffic_totals', static function (Blueprint $table): void {
$table->string('host_scope');
$table->string('vm_uuid');
$table->string('host_id')->nullable();
$table->integer('user_id')->nullable();
$table->integer('host_id');
$table->binary('vm_uuid');
$table->integer('user_id');
$table->integer('last_observed_at')->nullable();
$table->integer('download_rx_bytes')->default(0);
$table->integer('upload_tx_bytes')->default(0);
$table->integer('updated_at')->default(0);
$table->primary(['host_scope', 'vm_uuid']);
$table->primary(['host_id', 'vm_uuid']);
});
$logSchema->create('cloud_vm_traffic_checkpoints', static function (Blueprint $table): void {
$table->string('host_scope');
$table->string('vm_uuid');
$table->integer('host_id');
$table->binary('vm_uuid');
$table->integer('checkpoint_hour');
$table->string('host_id')->nullable();
$table->integer('user_id')->nullable();
$table->integer('user_id');
$table->integer('first_observed_at');
$table->integer('download_rx_bytes');
$table->integer('upload_tx_bytes');
$table->primary(['host_scope', 'vm_uuid', 'checkpoint_hour']);
$table->primary(['host_id', 'vm_uuid', 'checkpoint_hour']);
});
$user = new User();
@ -402,11 +401,9 @@ namespace {
$instance->current_period_id = $period->id;
$instance->save();
$scope = hash('sha256', 'virt-cloud/host/1');
$capsule->connection('log')->table('cloud_vm_traffic_totals')->insert([
'host_scope' => $scope,
'vm_uuid' => $instance->uuid,
'host_id' => '1',
'host_id' => 1,
'vm_uuid' => CloudLogIdentity::uuid((string) $instance->uuid),
'user_id' => 1,
'last_observed_at' => $now,
'download_rx_bytes' => 700,
@ -485,7 +482,9 @@ namespace {
$instance->network_status = 'active';
$instance->save();
$capsule->connection('log')->table('cloud_vm_traffic_totals')
->where('host_scope', $scope)->where('vm_uuid', $instance->uuid)->update([
->where('host_id', 1)
->where('vm_uuid', CloudLogIdentity::uuid((string) $instance->uuid))
->update([
'last_observed_at' => $now + 1,
'download_rx_bytes' => 800,
'upload_tx_bytes' => 450,
@ -776,9 +775,8 @@ namespace {
$stale->network_status = 'active';
$stale->save();
$capsule->connection('log')->table('cloud_vm_traffic_totals')->insert([
'host_scope' => $scope,
'vm_uuid' => $stale->uuid,
'host_id' => '1',
'host_id' => 1,
'vm_uuid' => CloudLogIdentity::uuid((string) $stale->uuid),
'user_id' => 1,
'last_observed_at' => $now + 2,
'download_rx_bytes' => 8000,
@ -794,7 +792,9 @@ namespace {
'The first post-boundary observation did not establish a fail-open baseline.');
$capsule->connection('log')->table('cloud_vm_traffic_totals')
->where('host_scope', $scope)->where('vm_uuid', $stale->uuid)->update([
->where('host_id', 1)
->where('vm_uuid', CloudLogIdentity::uuid((string) $stale->uuid))
->update([
'last_observed_at' => $now + 3,
'download_rx_bytes' => 8070,
'upload_tx_bytes' => 4040,
@ -1155,9 +1155,8 @@ namespace {
$rollback->current_period_id = $rollbackPeriod->id;
$rollback->save();
$capsule->connection('log')->table('cloud_vm_traffic_totals')->insert([
'host_scope' => $scope,
'vm_uuid' => $rollback->uuid,
'host_id' => '1',
'host_id' => 1,
'vm_uuid' => CloudLogIdentity::uuid((string) $rollback->uuid),
'user_id' => 1,
'last_observed_at' => $now + 100,
'download_rx_bytes' => 100,
@ -1189,10 +1188,9 @@ namespace {
$checkpointPeriod = $makePeriod($checkpointed, $now - 3600, $now + 3600,
'scheduled', 10000);
$capsule->connection('log')->table('cloud_vm_traffic_checkpoints')->insert([
'host_scope' => $scope,
'vm_uuid' => $checkpointed->uuid,
'host_id' => 1,
'vm_uuid' => CloudLogIdentity::uuid((string) $checkpointed->uuid),
'checkpoint_hour' => intdiv($now - 3540, 3600),
'host_id' => '1',
'user_id' => 1,
'first_observed_at' => $now - 3540,
'download_rx_bytes' => 400,

@ -27,10 +27,10 @@ $assert(stripos($schema, 'compatibility import') === false
$assert(!preg_match('/\bengine_[A-Za-z0-9_]*\b/', $schema),
'cloud.sql retained an engine-prefixed identifier.');
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v50-traffic-settlement-buckets'")
"SCHEMA_VERSION = 'go-agent-wss-v51-compact-telemetry-storage'")
&& !str_contains($setup, "SCHEMA_VERSION = 'go-v1'")
&& str_contains($deployment, '`go-agent-wss-v50-traffic-settlement-buckets`')
&& str_contains($compensation, '`go-agent-wss-v50-traffic-settlement-buckets`'),
&& str_contains($deployment, '`go-agent-wss-v51-compact-telemetry-storage`')
&& str_contains($compensation, '`go-agent-wss-v51-compact-telemetry-storage`'),
'Outbound Agent schema must reject the incompatible earlier Go database.');
$assert(str_contains($schema,
"timezone VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT 'UTC'")

@ -0,0 +1,36 @@
<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Services\Cloud\CloudLogIdentity;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$uuid = '123e4567-e89b-42d3-a456-426614174000';
$stored = CloudLogIdentity::uuid($uuid);
$assert(strlen($stored) === 16
&& CloudLogIdentity::uuidKey($uuid) === str_replace('-', '', $uuid)
&& CloudLogIdentity::storedUuidKey($stored)
=== str_replace('-', '', $uuid)
&& strlen(CloudLogIdentity::sha256('telemetry')) === 32,
'Compact log identities are not lossless fixed-width values.');
try {
CloudLogIdentity::uuid('not-a-uuid');
throw new RuntimeException('Malformed UUID was accepted.');
} catch (InvalidArgumentException) {
}
try {
CloudLogIdentity::storedUuidKey('short');
throw new RuntimeException('Malformed stored UUID was accepted.');
} catch (RuntimeException $error) {
$assert($error->getMessage() === 'Stored VM UUID is invalid',
'Stored UUID validation returned the wrong error.');
}
echo "Compact cloud log identity checks passed.\n";

@ -56,7 +56,7 @@ foreach ([
}
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v50-traffic-settlement-buckets'"),
"SCHEMA_VERSION = 'go-agent-wss-v51-compact-telemetry-storage'"),
'CloudSetup schema version was not advanced for maintenance notices.');
echo "Maintenance notice schema contracts passed.\n";

@ -32,9 +32,6 @@ $logSchema->create(
'cloud_vm_metric_samples',
static function (Blueprint $table): void {
$table->integer('instance_id');
$table->string('host_scope');
$table->string('vm_uuid');
$table->string('host_id');
$table->integer('user_id');
$table->integer('sampled_at');
foreach (['cpu_percent', 'memory_used_mib', 'memory_total_mib',
@ -44,16 +41,13 @@ $logSchema->create(
}
$table->integer('network_rx_bytes')->nullable();
$table->integer('network_tx_bytes')->nullable();
$table->primary(['host_scope', 'vm_uuid', 'sampled_at']);
$table->primary(['instance_id', 'sampled_at']);
}
);
$logSchema->create(
'cloud_vm_metric_rollups',
static function (Blueprint $table): void {
$table->integer('instance_id');
$table->string('host_scope');
$table->string('vm_uuid');
$table->string('host_id');
$table->integer('user_id');
$table->integer('interval_seconds');
$table->integer('bucket_start');
@ -65,10 +59,7 @@ $logSchema->create(
$table->double($field . '_sum')->nullable();
$table->integer($field . '_count');
}
$table->primary([
'instance_id', 'host_scope', 'vm_uuid',
'interval_seconds', 'bucket_start',
]);
$table->primary(['instance_id', 'interval_seconds', 'bucket_start']);
}
);
@ -76,14 +67,9 @@ $main = $capsule->connection('default');
$log = $capsule->connection('log');
$main->table('cloud_instances')->insert(['id' => 1, 'status' => 'active']);
$now = 20_000_000;
$scope = hash('sha256', 'metric-history-test');
$uuid = '123e4567-e89b-42d3-a456-426614174000';
$sample = static function (int $at, ?float $cpu) use ($scope, $uuid): array {
$sample = static function (int $at, ?float $cpu): array {
return [
'instance_id' => 1,
'host_scope' => $scope,
'vm_uuid' => $uuid,
'host_id' => '7',
'user_id' => 9,
'sampled_at' => $at,
'cpu_percent' => $cpu,
@ -107,9 +93,6 @@ foreach ([$sample($at, 10.0), $sample($at + 60, null),
$day = CloudMetricHistoryStore::series(
$log,
1,
$scope,
$uuid,
'7',
9,
'24h',
$now
@ -117,9 +100,6 @@ $day = CloudMetricHistoryStore::series(
$month = CloudMetricHistoryStore::series(
$log,
1,
$scope,
$uuid,
'7',
9,
'30d',
$now
@ -135,7 +115,6 @@ $oldRaw = $sample(
$now - CloudMetricHistoryStore::RAW_RETENTION_SECONDS - 60,
1.0
);
$oldRaw['vm_uuid'] = '223e4567-e89b-42d3-a456-426614174000';
$log->table('cloud_vm_metric_samples')->insert($oldRaw + [
'network_rx_bytes' => 1, 'network_tx_bytes' => 1,
]);
@ -143,13 +122,9 @@ $rollupFixture = static function (
int $instanceId,
int $interval,
int $bucket
) use ($scope, $uuid): array {
): array {
$values = [
'instance_id' => $instanceId,
'host_scope' => $scope,
'vm_uuid' => $instanceId === 1 ? $uuid
: '323e4567-e89b-42d3-a456-426614174000',
'host_id' => '7',
'user_id' => 9,
'interval_seconds' => $interval,
'bucket_start' => $bucket,
@ -179,7 +154,6 @@ $log->table('cloud_vm_metric_rollups')->insert([
]);
$deletedSample = $sample($now - 60, 1.0);
$deletedSample['instance_id'] = 2;
$deletedSample['vm_uuid'] = '323e4567-e89b-42d3-a456-426614174000';
$log->table('cloud_vm_metric_samples')->insert($deletedSample + [
'network_rx_bytes' => 1, 'network_tx_bytes' => 1,
]);

@ -53,7 +53,7 @@ $assert(is_string($schema)
'Public-pool replacement lacks durable batch, child, or claim schema.');
$assert(is_string($setup)
&& str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v50-traffic-settlement-buckets'")
"SCHEMA_VERSION = 'go-agent-wss-v51-compact-telemetry-storage'")
&& str_contains($setup,
"'cloud_public_ip_pool_replacement_batches'")
&& str_contains($setup,

@ -170,7 +170,7 @@ $assert(str_contains($setup,
'CloudSetup does not verify index uniqueness and ordered columns.');
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v50-traffic-settlement-buckets'")
"SCHEMA_VERSION = 'go-agent-wss-v51-compact-telemetry-storage'")
&& str_contains($setup, "'cloud_public_network_transitions'")
&& str_contains($setup, "'cloud_public_network_transition_ips'")
&& str_contains($setup, "'reservation_ip_claim_id'")

@ -57,7 +57,7 @@ $assert($replica !== ''
&& str_contains($replica, 'cleanup_resolved_at BIGINT UNSIGNED'),
'Rescue replica cannot prove exact identity or retain cleanup debt.');
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v50-traffic-settlement-buckets'")
"SCHEMA_VERSION = 'go-agent-wss-v51-compact-telemetry-storage'")
&& str_contains($setup, "'cloud_rescue_artifact_delete_due'")
&& str_contains($setup, "'cloud_rescue_replica_cleanup'"),
'CloudSetup version or rescue catalog indexes are incomplete.');

@ -218,7 +218,7 @@ foreach ([
'CloudSetup omits restore/migration ledger contract: ' . $required);
}
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v50-traffic-settlement-buckets'")
"SCHEMA_VERSION = 'go-agent-wss-v51-compact-telemetry-storage'")
&& str_contains($attemptModel,
"protected \$table = 'cloud_backup_restore_attempts';")
&& str_contains($attemptModel, "'active_attempt' => 'int'")

@ -105,7 +105,7 @@ $assert(str_contains($setup,
'CloudSetup lost the open-compensation fence index contract.');
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v50-traffic-settlement-buckets'"),
"SCHEMA_VERSION = 'go-agent-wss-v51-compact-telemetry-storage'"),
'CloudSetup schema version was not advanced for service compensation.');
$appliedShape = preg_match(

@ -138,7 +138,7 @@ $assert(is_string($publicationModel)
&& str_contains($artifactModel, "'maximum_size_bytes' => 'int'"),
'Storage publication/artifact model contracts are incomplete.');
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v50-traffic-settlement-buckets'"),
"SCHEMA_VERSION = 'go-agent-wss-v51-compact-telemetry-storage'"),
'CloudSetup schema version was not advanced for durable publications.');
echo "Storage publication schema contracts passed.\n";

@ -63,7 +63,7 @@ $assert(str_contains($schema,
&& str_contains($schema,
'CONSTRAINT chk_cloud_template_import_completion CHECK (')
&& str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v50-traffic-settlement-buckets'"),
"SCHEMA_VERSION = 'go-agent-wss-v51-compact-telemetry-storage'"),
'Template imports are missing durable identity or terminal-state constraints.');
$assert(str_contains($routes, "'/admin/cloud-template-imports'")
&& str_contains($routes, "'/admin/cloud-template-imports/{id}'")

@ -25,6 +25,7 @@ namespace {
use App\Models\CloudInstance;
use App\Services\Cloud\CloudAgentClient;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudLogIdentity;
use App\Services\Cloud\CloudTrafficService;
use GuzzleHttp\Client;
use GuzzleHttp\Handler\MockHandler;
@ -81,10 +82,7 @@ namespace {
$logSchema->create('cloud_vm_metric_samples',
static function (Blueprint $table): void {
$table->integer('instance_id');
$table->string('host_scope');
$table->string('vm_uuid');
$table->string('host_id')->nullable();
$table->integer('user_id')->nullable();
$table->integer('user_id');
$table->integer('sampled_at');
foreach (['cpu_percent', 'memory_used_mib', 'memory_total_mib',
'disk_read_bps', 'disk_write_bps', 'network_rx_bps',
@ -93,14 +91,11 @@ namespace {
}
$table->integer('network_rx_bytes')->nullable();
$table->integer('network_tx_bytes')->nullable();
$table->primary(['host_scope', 'vm_uuid', 'sampled_at']);
$table->primary(['instance_id', 'sampled_at']);
});
$logSchema->create('cloud_vm_metric_rollups',
static function (Blueprint $table): void {
$table->integer('instance_id');
$table->string('host_scope');
$table->string('vm_uuid');
$table->string('host_id');
$table->integer('user_id');
$table->integer('interval_seconds');
$table->integer('bucket_start');
@ -113,35 +108,32 @@ namespace {
$table->integer($field . '_count');
}
$table->primary([
'instance_id', 'host_scope', 'vm_uuid',
'interval_seconds', 'bucket_start',
'instance_id', 'interval_seconds', 'bucket_start',
]);
});
$logSchema->create('cloud_vm_traffic_totals',
static function (Blueprint $table): void {
$table->string('host_scope');
$table->string('vm_uuid');
$table->string('host_id')->nullable();
$table->integer('user_id')->nullable();
$table->integer('host_id');
$table->binary('vm_uuid');
$table->integer('user_id');
$table->integer('last_observed_at')->nullable();
$table->string('network_signature');
$table->binary('network_signature');
$table->text('network_baseline_json');
$table->integer('download_rx_bytes')->default(0);
$table->integer('upload_tx_bytes')->default(0);
$table->integer('updated_at');
$table->primary(['host_scope', 'vm_uuid']);
$table->primary(['host_id', 'vm_uuid']);
});
$logSchema->create('cloud_vm_traffic_checkpoints',
static function (Blueprint $table): void {
$table->string('host_scope');
$table->string('vm_uuid');
$table->integer('host_id');
$table->binary('vm_uuid');
$table->integer('checkpoint_hour');
$table->string('host_id')->nullable();
$table->integer('user_id')->nullable();
$table->integer('user_id');
$table->integer('first_observed_at');
$table->integer('download_rx_bytes');
$table->integer('upload_tx_bytes');
$table->primary(['host_scope', 'vm_uuid', 'checkpoint_hour']);
$table->primary(['host_id', 'vm_uuid', 'checkpoint_hour']);
});
$logSchema->create('cloud_vm_daily_traffic_cursors',
static function (Blueprint $table): void {
@ -178,7 +170,6 @@ namespace {
}
$resetAt = $secondAt + 60;
$uuid = '11111111-1111-4111-8111-111111111111';
$scope = hash('sha256', 'virt-cloud/host/7');
$instance = new CloudInstance();
$instance->host_id = 7;
$instance->user_id = 17;
@ -221,9 +212,6 @@ namespace {
$log = $capsule->connection('log');
$log->table('cloud_vm_metric_samples')->insert([
'instance_id' => (int) $instance->id,
'host_scope' => $scope,
'vm_uuid' => $uuid,
'host_id' => '7',
'user_id' => 17,
'sampled_at' => $now - 31 * 86400,
]);
@ -414,9 +402,6 @@ namespace {
$log->table('cloud_vm_metric_samples')->insert([
'instance_id' => (int) $instance->id,
'host_scope' => $scope,
'vm_uuid' => $uuid,
'host_id' => '7',
'user_id' => 999,
'sampled_at' => $now - 10,
'cpu_percent' => 99,
@ -467,7 +452,7 @@ namespace {
$migratedMetrics['uuid'] = $migratedUuid;
$store->invoke(null, $log, $instance, $migratedMetrics);
$migratedTotal = $log->table('cloud_vm_traffic_totals')
->where('vm_uuid', $migratedUuid)->first();
->where('vm_uuid', CloudLogIdentity::uuid($migratedUuid))->first();
$migrationSyncCount = CloudTrafficService::sync(10);
$migratedInstance = $instance->fresh();
$assert($migratedTotal !== null
@ -479,7 +464,12 @@ namespace {
&& (int) $migratedInstance->lifetime_upload_bytes === 57_450,
'A maintenance migration reset or double-counted the lifetime traffic generation: '
. json_encode([
'total' => $migratedTotal,
'total' => $migratedTotal === null ? null : [
'host_id' => (int) $migratedTotal->host_id,
'vm_uuid' => bin2hex((string) $migratedTotal->vm_uuid),
'download' => (int) $migratedTotal->download_rx_bytes,
'upload' => (int) $migratedTotal->upload_tx_bytes,
],
'sync_count' => $migrationSyncCount,
'instance_download' =>
(int) $migratedInstance->lifetime_download_bytes,
@ -493,7 +483,8 @@ namespace {
$logSchema->drop('cloud_vm_daily_traffic');
$logSchema->drop('cloud_vm_daily_traffic_cursors');
$log->table('cloud_vm_traffic_totals')
->where('vm_uuid', $migratedUuid)->update(['last_observed_at' => 0]);
->where('vm_uuid', CloudLogIdentity::uuid($migratedUuid))
->update(['last_observed_at' => 0]);
$withoutDailyMetrics = $metrics(
$now + 1, 110_000_000, 524_288, 110, 55, 1_350, 800
);
@ -511,7 +502,7 @@ namespace {
$withoutDaily = CloudTrafficService::collect(1, $withoutDailyAgent);
$withoutDailySyncCount = CloudTrafficService::sync(1);
$withoutDailyTotal = $log->table('cloud_vm_traffic_totals')
->where('vm_uuid', $migratedUuid)->first();
->where('vm_uuid', CloudLogIdentity::uuid($migratedUuid))->first();
$assert($withoutDaily === [
'observed' => 1, 'unavailable' => 0, 'skipped' => 0,
'errors' => [],

@ -76,7 +76,7 @@ $assert(str_contains($schema,
'Service-period traffic-pool identity is not frozen or indexed.');
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v50-traffic-settlement-buckets'")
"SCHEMA_VERSION = 'go-agent-wss-v51-compact-telemetry-storage'")
&& str_contains($setup, "'chk_cloud_traffic_entitlement_balance'")
&& str_contains($setup, "'cloud_traffic_ledger_sequence'")
&& str_contains($setup, "'chk_cloud_traffic_settlement_state'")

@ -101,7 +101,7 @@ $assert(str_contains($jobs, 'vpc_id BIGINT UNSIGNED DEFAULT NULL')
'Cloud jobs cannot be correlated to VPC or VPC-port work.');
$assert(str_contains($setup,
"private const SCHEMA_VERSION = 'go-agent-wss-v50-traffic-settlement-buckets'")
"private const SCHEMA_VERSION = 'go-agent-wss-v51-compact-telemetry-storage'")
&& str_contains($setup, "'name' => 'cloud_vpc_enabled'")
&& str_contains($setup, "'value' => 'false'")
&& str_contains($setup,

@ -104,7 +104,7 @@ foreach ($models as $class => $modelTable) {
}
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v50-traffic-settlement-buckets'")
"SCHEMA_VERSION = 'go-agent-wss-v51-compact-telemetry-storage'")
&& str_contains($setup,
"self::schema('ss.sql') . \"\\0\" . self::schema('cloud.sql')")
&& str_contains($setup,

@ -15,12 +15,11 @@ $assert = static function ($condition, string $message): void {
foreach ([
'cloud_vm_metric_samples' => [
'instance_id BIGINT UNSIGNED', 'host_scope CHAR(64)',
'vm_uuid CHAR(36)', 'host_id VARCHAR(64)',
'user_id BIGINT UNSIGNED', 'network_rx_bytes BIGINT UNSIGNED',
'instance_id BIGINT UNSIGNED', 'user_id BIGINT UNSIGNED',
'PRIMARY KEY (instance_id, sampled_at)',
'network_rx_bytes BIGINT UNSIGNED',
'network_tx_bytes BIGINT UNSIGNED',
'vm_metric_instance (instance_id, sampled_at)',
'vm_metric_owner (user_id, host_id, sampled_at)',
'vm_metric_owner (user_id, sampled_at, instance_id)',
],
'cloud_vm_metric_rollups' => [
'instance_id BIGINT UNSIGNED', 'interval_seconds SMALLINT UNSIGNED',
@ -30,12 +29,17 @@ foreach ([
'chk_vm_metric_rollup_counts',
],
'cloud_vm_traffic_totals' => [
'network_signature CHAR(64)', 'network_baseline_json JSON',
'host_id BIGINT UNSIGNED', 'vm_uuid BINARY(16)',
'network_signature BINARY(32)', 'network_baseline_json JSON',
'download_rx_bytes BIGINT UNSIGNED', 'upload_tx_bytes BIGINT UNSIGNED',
'PRIMARY KEY (host_id, vm_uuid)',
'vm_traffic_uuid (vm_uuid, host_id)',
'vm_traffic_owner (user_id, host_id)',
],
'cloud_vm_traffic_checkpoints' => [
'host_id BIGINT UNSIGNED', 'vm_uuid BINARY(16)',
'checkpoint_hour BIGINT UNSIGNED', 'first_observed_at BIGINT UNSIGNED',
'PRIMARY KEY (host_id, vm_uuid, checkpoint_hour)',
'vm_traffic_checkpoint_owner (user_id, host_id, first_observed_at)',
],
'cloud_vm_daily_traffic_cursors' => [
@ -62,6 +66,21 @@ foreach ([
}
}
foreach (['cloud_vm_metric_samples', 'cloud_vm_metric_rollups'] as $table) {
preg_match('/CREATE TABLE ' . preg_quote($table, '/')
. '\s*\((.*?)\) ENGINE=/s', $sql, $match);
$assert(!str_contains($match[1], 'host_scope')
&& !str_contains($match[1], 'vm_uuid')
&& !str_contains($match[1], 'host_id'),
$table . ' retained per-row placement text in logical history.');
}
$assert(!str_contains($sql, 'vm_metric_instance')
&& !str_contains($sql, 'vm_metric_rollup_identity')
&& !str_contains($sql, 'vm_traffic_host')
&& !str_contains($sql, 'cloud_network_observation_placement')
&& !str_contains($sql, 'cloud_network_observation_host'),
'Fresh log schema retained redundant high-frequency indexes.');
$assert(preg_match('/\b(?:ALTER|DROP)\s+TABLE\b/i', $sql) !== 1,
'Fresh cloud log schema contains migration/destructive DDL.');
$assert(!str_contains($sql, 'engine_vm_'),

Loading…
Cancel
Save