fix(traffic): report overdue settlement accounting

go
codex 15 hours ago
parent 69e4f0fca8
commit 9cc0f9dca9

@ -412,14 +412,16 @@ policy decision. Deploy it as a clean schema/code boundary:
base counters need no anchor; an open period with null bases deliberately
blocks deletion until the normal lifecycle path freezes real evidence.
6. Deploy the matching code and restart lifecycle. Open the administrator
task page and run its read-only health check. The “流量存储压缩” row exposes
due settlement buckets, due hourly rows, involved instances, cursor
attention count and oldest lag without exposing IDs or payloads.
task page and run its read-only health check. The “流量结算与存储” row exposes
overdue open buckets, longest accounting delay, event-identity conflicts,
unsettled and stale-cursor instances, recent accounting failures, due
compaction rows and cursor attention without exposing IDs or payloads.
7. Keep the pool in its existing rollout state while observing at least one
complete collection, accounting, settlement-compaction and
checkpoint-compaction pass. A backlog less than one hour beyond its
retention window remains informational; a larger lag, scan truncation or
attention cursor is a warning that must be investigated.
retention window remains informational. An open settlement delay over five
minutes is a warning; a delay over one hour, event-identity conflict, scan
truncation or attention cursor requires immediate investigation.
Acceptance requires all of the following:

@ -84,7 +84,7 @@ final class CloudRuntimeHealthService
? $this->trafficStorageCheck()
: $this->row(
'traffic_storage',
'流量存储压缩',
'流量结算与存储',
'unknown',
'数据库短超时探测未通过,未继续执行流量存储统计。'
);
@ -93,9 +93,9 @@ final class CloudRuntimeHealthService
'未启用日志数据库;虚拟化指标依赖该配置。');
$checks[] = $this->row(
'traffic_storage',
'流量存储压缩',
'流量结算与存储',
'disabled',
'日志数据库未启用,未执行流量存储压缩检查。'
'日志数据库未启用,未执行流量结算与存储检查。'
);
}
$checks[] = $this->check('agents', '宿主机 Agent 配置',
@ -288,6 +288,9 @@ final class CloudRuntimeHealthService
'settlement_buckets_due', 'hourly_checkpoints_due',
'hourly_checkpoint_instances_due',
'cursor_attention_instances', 'oldest_due_lag_seconds',
'open_buckets_due', 'oldest_open_bucket_lag_seconds',
'settlement_identity_conflicts', 'unsettled_instances',
'stale_cursor_instances', 'recent_accounting_failures',
] as $key) {
$value = $result['metrics'][$key] ?? null;
if (!is_int($value) || $value < 0) {
@ -300,18 +303,21 @@ final class CloudRuntimeHealthService
$metrics['instance_scan_truncated'] =
($result['metrics']['instance_scan_truncated'] ?? null)
=== true;
$metrics['accounting_scan_truncated'] =
($result['metrics']['accounting_scan_truncated'] ?? null)
=== true;
return $this->row(
'traffic_storage',
'流量存储压缩',
'流量结算与存储',
(string) $result['status'],
(string) $result['message']
) + ['metrics' => $metrics];
} catch (\Throwable $exception) {
return $this->row(
'traffic_storage',
'流量存储压缩',
'流量结算与存储',
'error',
'流量存储压缩状态读取失败,请检查数据库结构与生命周期日志。'
'流量结算与存储状态读取失败,请检查数据库结构与生命周期日志。'
);
}
}

@ -8,6 +8,8 @@ use App\Models\User;
final class CloudTrafficStorageHealthService
{
private const WARNING_LAG_SECONDS = 3600;
private const ACCOUNTING_WARNING_SECONDS = 300;
private const RECENT_FAILURE_SECONDS = 3600;
private const INSTANCE_SCAN_LIMIT = 2000;
/**
@ -29,13 +31,21 @@ final class CloudTrafficStorageHealthService
}
$main = User::getConnectionResolver()->connection();
$log = User::getConnectionResolver()->connection('log');
if (!$main->getSchemaBuilder()->hasTable(
'cloud_traffic_usage_settlement_buckets'
) || !$main->getSchemaBuilder()->hasTable(
'cloud_traffic_instance_cursors'
) || !$log->getSchemaBuilder()->hasTable(
'cloud_vm_traffic_checkpoints'
)) {
foreach ([
'cloud_traffic_usage_settlement_buckets',
'cloud_traffic_ledger_events', 'cloud_traffic_instance_cursors',
'cloud_instances', 'cloud_service_periods',
'cloud_traffic_pool_groups', 'cloud_traffic_scan_failures',
] as $table) {
if (!$main->getSchemaBuilder()->hasTable($table)) {
throw new \RuntimeException(
'traffic_storage_health_schema_missing'
);
}
}
if (!$log->getSchemaBuilder()->hasTable(
'cloud_vm_traffic_checkpoints'
)) {
throw new \RuntimeException('traffic_storage_health_schema_missing');
}
$settlementCutoff = $now
@ -72,6 +82,78 @@ final class CloudTrafficStorageHealthService
: (int) $main->table('cloud_traffic_instance_cursors')
->whereIn('instance_id', $dueInstanceIds)
->where('state', 'attention')->count();
$openQuery = $main->table(
'cloud_traffic_usage_settlement_buckets'
)->where('state', 'open')->where('seal_at', '<=', $now);
$openBucketCount = (int) (clone $openQuery)->count();
$oldestOpenSealAt = $openBucketCount === 0 ? null
: (int) (clone $openQuery)->min('seal_at');
$openBucketRows = (clone $openQuery)->orderBy('seal_at')
->orderBy('id')->limit(self::INSTANCE_SCAN_LIMIT + 1)
->get(['id', 'event_identity']);
$accountingScanTruncated = $openBucketRows->count()
> self::INSTANCE_SCAN_LIMIT;
if ($accountingScanTruncated) {
$openBucketRows = $openBucketRows->take(
self::INSTANCE_SCAN_LIMIT
);
}
$eventKeys = [];
$identityConflicts = 0;
foreach ($openBucketRows as $row) {
$identity = (string) ($row->event_identity ?? '');
if (strlen($identity)
!== CloudTrafficSettlementEnvelope::IDENTITY_BYTES) {
$identityConflicts++;
continue;
}
$eventKeys[] = 'traffic-settlement:v2:' . bin2hex($identity);
}
if ($eventKeys !== []) {
$identityConflicts += (int) $main->table(
'cloud_traffic_ledger_events'
)->whereIn('event_key', $eventKeys)->count();
}
$unsettledQuery = $main->table('cloud_instances as health_instance')
->leftJoin(
'cloud_traffic_instance_cursors as health_cursor',
'health_cursor.instance_id', '=', 'health_instance.id'
)->whereExists(function ($eligible): void {
$eligible->selectRaw('1')
->from('cloud_service_periods as health_period')
->join('cloud_traffic_pool_groups as health_group',
'health_group.id', '=',
'health_period.traffic_pool_group_id')
->whereColumn('health_period.instance_id',
'health_instance.id')
->whereIn('health_group.mode', ['shadow', 'enforced']);
})->where(function ($query): void {
$query->where(function ($missing): void {
$missing->whereNull('health_cursor.instance_id')
->where('health_instance.traffic_synced_at', '>', 0);
})->orWhere('health_cursor.state', 'ready');
})->where(function ($query): void {
$query->whereNull('health_cursor.instance_id')
->orWhereColumn('health_instance.lifetime_download_bytes',
'!=', 'health_cursor.accounted_download_bytes')
->orWhereColumn('health_instance.lifetime_upload_bytes',
'!=', 'health_cursor.accounted_upload_bytes')
->orWhereColumn('health_instance.traffic_synced_at', '>',
'health_cursor.last_observed_at');
});
$unsettledInstances = (int) (clone $unsettledQuery)
->distinct()->count('health_instance.id');
$staleCursorInstances = (int) (clone $unsettledQuery)
->where('health_instance.traffic_synced_at', '>', 0)
->where('health_instance.traffic_synced_at', '<=',
$now - self::ACCOUNTING_WARNING_SECONDS)
->distinct()->count('health_instance.id');
$recentAccountingFailures = (int) $main->table(
'cloud_traffic_scan_failures'
)->where('last_failed_at', '>=',
$now - self::RECENT_FAILURE_SECONDS)->count();
$oldestOpenLag = $oldestOpenSealAt === null
? 0 : max(0, $now - $oldestOpenSealAt);
$oldestLag = max(
$oldestSettlement === null
? 0 : $settlementCutoff - $oldestSettlement,
@ -83,27 +165,50 @@ final class CloudTrafficStorageHealthService
'hourly_checkpoints_due' => $checkpointCount,
'hourly_checkpoint_instances_due' => $dueInstanceCount,
'cursor_attention_instances' => $attentionCount,
'open_buckets_due' => $openBucketCount,
'oldest_open_bucket_lag_seconds' => $oldestOpenLag,
'settlement_identity_conflicts' => $identityConflicts,
'unsettled_instances' => $unsettledInstances,
'stale_cursor_instances' => $staleCursorInstances,
'recent_accounting_failures' => $recentAccountingFailures,
'oldest_due_lag_seconds' => max(0, $oldestLag),
'instance_scan_truncated' => $scanTruncated,
'accounting_scan_truncated' => $accountingScanTruncated,
];
$status = $attentionCount > 0 || $scanTruncated
|| $accountingScanTruncated
|| $oldestLag > self::WARNING_LAG_SECONDS
|| $oldestOpenLag > self::ACCOUNTING_WARNING_SECONDS
|| $identityConflicts > 0 || $staleCursorInstances > 0
|| $recentAccountingFailures > 0
? 'warning' : 'ok';
if ($settlementCount === 0 && $checkpointCount === 0) {
$message = '结算桶和小时检查点均处于既定保留窗口内。';
if ($settlementCount === 0 && $checkpointCount === 0
&& $openBucketCount === 0 && $unsettledInstances === 0
&& $recentAccountingFailures === 0) {
$message = '流量结算与存储保留队列均处于正常窗口内。';
} else {
$message = '待压缩结算桶 ' . $settlementCount
$message = '到期未封账 ' . $openBucketCount . ' 个(最长延迟 '
. self::duration($oldestOpenLag) . '),未结算实例 '
. $unsettledInstances . ' 个,滞后游标 '
. $staleCursorInstances . ' 个,身份冲突 '
. $identityConflicts . ' 个,近一小时失败 '
. $recentAccountingFailures . ' 个;待压缩结算桶 '
. $settlementCount
. ' 个,小时检查点 ' . $checkpointCount . ' 行(涉及 '
. $dueInstanceCount . ' 个实例);最旧超出窗口 '
. self::duration(max(0, $oldestLag)) . ',其中流量游标异常 '
. $attentionCount . ' 个实例。';
if ($scanTruncated) {
if ($scanTruncated || $accountingScanTruncated) {
$message .= ' 实例计数已达到有界扫描上限。';
}
if ($identityConflicts > 0
|| $oldestOpenLag > self::WARNING_LAG_SECONDS) {
$message .= ' 结算账务需立即核对。';
}
}
return [
'key' => 'traffic_storage',
'label' => '流量存储压缩',
'label' => '流量结算与存储',
'status' => $status,
'message' => $message,
'checked_at' => $now,

@ -16,6 +16,34 @@ export function mountOperationsHealth({ transport, presentation }) {
node.className = `fs-13 mb-0 ${error ? 'text-danger' : 'text-muted'}`;
}
function duration(seconds) {
const value = Math.max(0, Number(seconds) || 0);
if (value >= 86400) return `${Math.floor(value / 86400)} 天`;
if (value >= 3600) return `${Math.floor(value / 3600)} 小时`;
if (value >= 60) return `${Math.floor(value / 60)} 分钟`;
return `${Math.floor(value)} 秒`;
}
function trafficAccountingMetrics(metrics) {
if (!metrics || typeof metrics !== 'object') return null;
const list = el('dl', 'row g-1 mb-0 mt-2 fs-12');
const values = [
['到期未封账', metrics.open_buckets_due],
['最长结算延迟', duration(metrics.oldest_open_bucket_lag_seconds)],
['身份冲突', metrics.settlement_identity_conflicts],
['未结算实例', metrics.unsettled_instances],
['滞后游标', metrics.stale_cursor_instances],
['近一小时失败', metrics.recent_accounting_failures],
];
for (const [label, value] of values) {
list.append(
el('dt', 'col-7 text-muted fw-normal', label),
el('dd', 'col-5 mb-0 text-end fw-semibold', value),
);
}
return list;
}
function render(payload) {
if (!Array.isArray(payload?.checks) || !payload.checked_at) {
throw new Error('运行状态数据格式无效。');
@ -39,6 +67,10 @@ export function mountOperationsHealth({ transport, presentation }) {
title,
el('p', 'text-muted fs-13 mb-1 text-break', operationText(check.message)),
);
if (check.key === 'traffic_storage') {
const metrics = trafficAccountingMetrics(check.metrics);
if (metrics) column.append(metrics);
}
if (check.last_seen_at) {
column.append(el('p', 'text-muted fs-12 mb-0',
`最近活动 ${operationDate(check.last_seen_at)}`));

@ -95,6 +95,10 @@ assert.match(entry, /mountOperationsRecovery\(/);
assert.doesNotMatch(entry, /\/force-delete|\/repair-vpc|\/resolve-effect|\/retire-vpc-host/);
assert.match(directory, /export function mountOperationsDirectory/);
assert.match(health, /export function mountOperationsHealth/);
assert.match(health, /trafficAccountingMetrics/);
assert.match(health, /metrics\.open_buckets_due/);
assert.match(health, /metrics\.settlement_identity_conflicts/);
assert.match(health, /metrics\.unsettled_instances/);
assert.match(recovery, /export function mountOperationsRecovery/);
assert.match(presentation, /export function presentOperation/);

@ -50,8 +50,15 @@ $service = new CloudRuntimeHealthService(
'hourly_checkpoints_due' => 3,
'hourly_checkpoint_instances_due' => 1,
'cursor_attention_instances' => 0,
'open_buckets_due' => 4,
'oldest_open_bucket_lag_seconds' => 600,
'settlement_identity_conflicts' => 1,
'unsettled_instances' => 5,
'stale_cursor_instances' => 3,
'recent_accounting_failures' => 2,
'oldest_due_lag_seconds' => 120,
'instance_scan_truncated' => false,
'accounting_scan_truncated' => false,
],
];
}
@ -80,6 +87,8 @@ $assert($checks['agents']['status'] === 'ok' && $checks['worker']['status'] ===
&& $checks['abuse_monitor']['status'] === 'unknown'
&& $checks['traffic_storage']['status'] === 'warning'
&& $checks['traffic_storage']['metrics']['settlement_buckets_due'] === 2
&& $checks['traffic_storage']['metrics']['open_buckets_due'] === 4
&& $checks['traffic_storage']['metrics']['unsettled_instances'] === 5
&& $checks['lifecycle']['status'] === 'warning',
'Health or heartbeat semantics changed.');
$assert(!str_contains(json_encode($payload), 'MUST_NOT_EXPOSE'), 'Heartbeat internal fields leaked.');

@ -28,12 +28,44 @@ $main->create('cloud_traffic_usage_settlement_buckets',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('state');
$table->integer('seal_at');
$table->binary('event_identity');
$table->integer('updated_at');
});
$main->create('cloud_traffic_ledger_events',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('event_key')->unique();
});
$main->create('cloud_traffic_instance_cursors',
static function (Blueprint $table): void {
$table->integer('instance_id')->primary();
$table->string('state');
$table->integer('accounted_download_bytes')->default(0);
$table->integer('accounted_upload_bytes')->default(0);
$table->integer('last_observed_at')->default(0);
});
$main->create('cloud_instances', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('lifetime_download_bytes')->default(0);
$table->integer('lifetime_upload_bytes')->default(0);
$table->integer('traffic_synced_at')->default(0);
});
$main->create('cloud_traffic_pool_groups',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('mode');
});
$main->create('cloud_service_periods',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('instance_id');
$table->integer('traffic_pool_group_id');
});
$main->create('cloud_traffic_scan_failures',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('last_failed_at');
});
$capsule->schema('log')->create('cloud_vm_traffic_checkpoints',
static function (Blueprint $table): void {
@ -45,6 +77,8 @@ $now = 20 * 86400;
$healthy = CloudTrafficStorageHealthService::snapshot($now);
$assert($healthy['status'] === 'ok'
&& $healthy['metrics']['settlement_buckets_due'] === 0
&& $healthy['metrics']['open_buckets_due'] === 0
&& $healthy['metrics']['unsettled_instances'] === 0
&& $healthy['metrics']['hourly_checkpoints_due'] === 0
&& $healthy['metrics']['cursor_attention_instances'] === 0,
'Empty compaction queues did not report a healthy bounded state.');
@ -55,9 +89,19 @@ $checkpointCutoff = $now
- CloudTrafficCheckpointRetention::RETENTION_SECONDS;
$capsule->connection('default')
->table('cloud_traffic_usage_settlement_buckets')->insert([
['state' => 'sealed', 'updated_at' => $settlementCutoff - 7200],
['state' => 'open', 'updated_at' => $settlementCutoff - 86400],
['state' => 'sealed', 'seal_at' => $now - 86400,
'event_identity' => random_bytes(16),
'updated_at' => $settlementCutoff - 7200],
['state' => 'open', 'seal_at' => $now - 7200,
'event_identity' => str_repeat('a', 16),
'updated_at' => $now - 7200],
]);
$capsule->connection('default')->table(
'cloud_traffic_ledger_events'
)->insert([
'event_key' => 'traffic-settlement:v2:'
. bin2hex(str_repeat('a', 16)),
]);
$capsule->connection('log')->table('cloud_vm_traffic_checkpoints')->insert([
['instance_id' => 3, 'first_observed_at' => $checkpointCutoff - 7200],
['instance_id' => 4, 'first_observed_at' => $checkpointCutoff + 60],
@ -65,17 +109,44 @@ $capsule->connection('log')->table('cloud_vm_traffic_checkpoints')->insert([
$capsule->connection('default')->table(
'cloud_traffic_instance_cursors'
)->insert([
'instance_id' => 3,
'state' => 'attention',
['instance_id' => 3, 'state' => 'attention',
'accounted_download_bytes' => 0, 'accounted_upload_bytes' => 0,
'last_observed_at' => 0],
['instance_id' => 5, 'state' => 'ready',
'accounted_download_bytes' => 10, 'accounted_upload_bytes' => 2,
'last_observed_at' => $now - 1200],
]);
$capsule->connection('default')->table('cloud_instances')->insert([
['id' => 3, 'lifetime_download_bytes' => 0,
'lifetime_upload_bytes' => 0, 'traffic_synced_at' => 0],
['id' => 5, 'lifetime_download_bytes' => 20,
'lifetime_upload_bytes' => 2, 'traffic_synced_at' => $now - 600],
]);
$capsule->connection('default')->table(
'cloud_traffic_pool_groups'
)->insert(['id' => 1, 'mode' => 'enforced']);
$capsule->connection('default')->table('cloud_service_periods')->insert([
['id' => 1, 'instance_id' => 5, 'traffic_pool_group_id' => 1],
]);
$capsule->connection('default')->table(
'cloud_traffic_scan_failures'
)->insert(['last_failed_at' => $now - 60]);
$warning = CloudTrafficStorageHealthService::snapshot($now);
$assert($warning['status'] === 'warning'
&& $warning['metrics']['settlement_buckets_due'] === 1
&& $warning['metrics']['open_buckets_due'] === 1
&& $warning['metrics']['oldest_open_bucket_lag_seconds'] === 7200
&& $warning['metrics']['settlement_identity_conflicts'] === 1
&& $warning['metrics']['unsettled_instances'] === 1
&& $warning['metrics']['stale_cursor_instances'] === 1
&& $warning['metrics']['recent_accounting_failures'] === 1
&& $warning['metrics']['hourly_checkpoints_due'] === 1
&& $warning['metrics']['hourly_checkpoint_instances_due'] === 1
&& $warning['metrics']['cursor_attention_instances'] === 1
&& $warning['metrics']['oldest_due_lag_seconds'] === 7200
&& str_contains($warning['message'], '结算桶 1 个')
&& str_contains($warning['message'], '到期未封账 1 个')
&& str_contains($warning['message'], '身份冲突 1 个')
&& str_contains($warning['message'], '结算账务需立即核对')
&& str_contains($warning['message'], '小时检查点 1 行'),
'Compaction backlog or attention state was hidden from operator health.');

Loading…
Cancel
Save