From 0614ce69c1297813cbdf69465d14ee08dafe70ee Mon Sep 17 00:00:00 2001 From: codex Date: Wed, 18 Nov 2026 03:28:37 +0800 Subject: [PATCH] fix(traffic): read booked balances and accounting status consistently --- .../Cloud/CloudAdminTrafficPoolService.php | 9 ++- .../Cloud/CloudTrafficPoolFreshness.php | 53 ++++++++++++++- .../Cloud/CloudTrafficPoolReadModel.php | 62 ++++++++---------- web/tests/CloudTrafficPoolFreshnessTest.php | 12 +++- .../CloudTrafficPoolUserReadModelTest.php | 65 ++++++++++++++++--- 5 files changed, 151 insertions(+), 50 deletions(-) diff --git a/web/app/Services/Cloud/CloudAdminTrafficPoolService.php b/web/app/Services/Cloud/CloudAdminTrafficPoolService.php index 1162aff1..25ead0db 100644 --- a/web/app/Services/Cloud/CloudAdminTrafficPoolService.php +++ b/web/app/Services/Cloud/CloudAdminTrafficPoolService.php @@ -43,6 +43,11 @@ final class CloudAdminTrafficPoolService ]; public static function page(array $query = []): array + { + return CloudTrafficConsistentRead::run(static fn (): array => self::pageSnapshot()); + } + + private static function pageSnapshot(): array { $groups = CloudTrafficPoolGroup::orderBy('id')->get(); $generatedAt = time(); @@ -84,7 +89,7 @@ final class CloudAdminTrafficPoolService public static function read(int $groupId): array { - return ['item' => self::groupDto(self::findGroup($groupId))]; + return CloudTrafficConsistentRead::run(static fn (): array => ['item' => self::groupDto(self::findGroup($groupId))]); } public static function create(array $input): array @@ -227,7 +232,7 @@ final class CloudAdminTrafficPoolService int $groupId, ?int $now = null ): array { - return self::buildReadiness(self::findGroup($groupId), $now ?? time()); + return CloudTrafficConsistentRead::run(static fn (): array => self::buildReadiness(self::findGroup($groupId), $now ?? time())); } private static function assertModeTransition( diff --git a/web/app/Services/Cloud/CloudTrafficPoolFreshness.php b/web/app/Services/Cloud/CloudTrafficPoolFreshness.php index 52ae2562..5ca4688a 100644 --- a/web/app/Services/Cloud/CloudTrafficPoolFreshness.php +++ b/web/app/Services/Cloud/CloudTrafficPoolFreshness.php @@ -31,7 +31,12 @@ final class CloudTrafficPoolFreshness ?int $now = null, ?array $accountingSnapshots = null ): array { - $now ??= time(); + return CloudTrafficConsistentRead::run(static fn (): array => + self::readSnapshots($identities, $now ?? time(), $accountingSnapshots)); + } + + private static function readSnapshots(array $identities, int $now, ?array $accountingSnapshots): array + { $pairs = []; foreach ($identities as $identity) { $userId = (int) ($identity['user_id'] ?? 0); @@ -65,6 +70,16 @@ final class CloudTrafficPoolFreshness $factsByKey = []; if ($snapshotByKey !== []) { $facts = CloudServicePeriod::query() + ->leftJoin('cloud_traffic_entitlements as matching_entitlement', static function ($join): void { + $join->on('matching_entitlement.service_period_id', '=', 'cloud_service_periods.id') + ->on('matching_entitlement.user_id', '=', 'cloud_service_periods.user_id') + ->on('matching_entitlement.instance_id', '=', 'cloud_service_periods.instance_id') + ->on('matching_entitlement.starts_at', '=', 'cloud_service_periods.starts_at') + ->on('matching_entitlement.ends_at', '=', 'cloud_service_periods.ends_at') + ->on('matching_entitlement.granted_bytes', '=', 'cloud_service_periods.traffic_quota_bytes') + ->where('matching_entitlement.quota_mode', '=', 'finite') + ->whereIn('matching_entitlement.state', ['active', 'exhausted']); + }) ->leftJoin('cloud_instances as freshness_instance', 'freshness_instance.id', '=', 'cloud_service_periods.instance_id') @@ -111,6 +126,7 @@ final class CloudTrafficPoolFreshness ->where('cloud_service_periods.ends_at', '>', $now) ->orderBy('cloud_service_periods.id')->get([ 'cloud_service_periods.id as period_id', + 'matching_entitlement.pool_id as matched_pool_id', 'cloud_service_periods.instance_id', 'cloud_service_periods.user_id as period_user_id', 'cloud_service_periods.traffic_pool_group_id as group_id', @@ -165,12 +181,25 @@ final class CloudTrafficPoolFreshness $unsettled = []; $stale = []; $confirmed = []; + $latestSample = 0; + $pendingEntitlement = false; if ((string) $snapshot['state'] === 'attention') { $status = 'attention'; $code = 'pool_attention'; } foreach ($factsByKey[$key] ?? [] as $fact) { $instanceId = (int) $fact->instance_id; + if ((int) $fact->matched_pool_id !== (int) $snapshot['pool_id']) { + $pendingEntitlement = true; + if ($status === 'current') { + $status = 'delayed'; + $code = 'projection_pending'; + } + } + if ($fact->instance_status !== null && CloudTrafficAccountingEvidence::expectsSample($fact->instance_status) + && (int) $fact->instance_user_id === $pair['user_id']) { + $latestSample = max($latestSample, (int) $fact->traffic_synced_at); + } if ($fact->cursor_instance_id === null || (int) $fact->instance_user_id !== $pair['user_id'] || (int) $fact->cursor_pool_id @@ -240,6 +269,8 @@ final class CloudTrafficPoolFreshness $result[$key] = [ 'status' => $status, 'last_confirmed_at' => $lastConfirmed, + 'last_sample_at' => $latestSample ?: null, + 'projection_pending' => $pendingEntitlement, 'unsettled_instance_count' => count($unsettled), 'stale_cursor_count' => count($stale), 'overdue_bucket_count' => (int) $due['count'], @@ -264,7 +295,11 @@ final class CloudTrafficPoolFreshness array $trafficPoolGroupIds, ?int $now = null ): array { - $now ??= time(); + return CloudTrafficConsistentRead::run(static fn (): array => self::groupSnapshots($trafficPoolGroupIds, $now ?? time())); + } + + private static function groupSnapshots(array $trafficPoolGroupIds, int $now): array + { $groupIds = array_values(array_unique(array_filter(array_map( 'intval', $trafficPoolGroupIds @@ -281,6 +316,14 @@ final class CloudTrafficPoolFreshness (int) $pool->traffic_pool_group_id, ] )->all(); + $materialized = array_count_values(array_column($identities, 'traffic_pool_group_id')); + $current = CloudServicePeriod::whereIn('traffic_pool_group_id', $groupIds) + ->where('traffic_quota_mode', 'finite')->whereIn('status', ['active', 'exhausted']) + ->where('starts_at', '<=', $now)->where('ends_at', '>', $now) + ->distinct()->get(['user_id', 'traffic_pool_group_id']); + foreach ($current as $period) { + $identities[] = $period->only(['user_id', 'traffic_pool_group_id']); + } $groupByKey = []; foreach ($identities as $identity) { $groupByKey[(int) $identity['user_id'] . ':' @@ -300,6 +343,7 @@ final class CloudTrafficPoolFreshness $result[$groupId] = self::aggregate( $byGroup[$groupId] ?? [] ); + $result[$groupId]['pool_count'] = $materialized[$groupId] ?? 0; } return $result; } @@ -313,6 +357,7 @@ final class CloudTrafficPoolFreshness $rank = ['current' => 0, 'delayed' => 1, 'attention' => 2]; $status = 'current'; $lastConfirmed = []; + $latestSample = 0; $unsettled = 0; $stale = 0; $overdue = 0; @@ -320,6 +365,7 @@ final class CloudTrafficPoolFreshness if ($rank[$snapshot['status']] > $rank[$status]) { $status = $snapshot['status']; } + $latestSample = max($latestSample, (int) $snapshot['last_sample_at']); if ($snapshot['last_confirmed_at'] !== null) { $lastConfirmed[] = (int) $snapshot['last_confirmed_at']; } @@ -329,6 +375,7 @@ final class CloudTrafficPoolFreshness } return [ 'status' => $status, + 'last_sample_at' => $latestSample ?: null, 'last_confirmed_at' => count($lastConfirmed) === count($snapshots) ? min($lastConfirmed) : null, 'unsettled_instance_count' => $unsettled, @@ -346,6 +393,8 @@ final class CloudTrafficPoolFreshness return [ 'status' => $code === 'no_pools' ? 'current' : 'delayed', 'last_confirmed_at' => null, + 'last_sample_at' => null, + 'projection_pending' => $code === 'projection_pending', 'unsettled_instance_count' => 0, 'stale_cursor_count' => 0, 'overdue_bucket_count' => 0, diff --git a/web/app/Services/Cloud/CloudTrafficPoolReadModel.php b/web/app/Services/Cloud/CloudTrafficPoolReadModel.php index 41d32521..5f2042ae 100644 --- a/web/app/Services/Cloud/CloudTrafficPoolReadModel.php +++ b/web/app/Services/Cloud/CloudTrafficPoolReadModel.php @@ -15,7 +15,11 @@ final class CloudTrafficPoolReadModel { public static function forUser(int $userId, ?int $now = null): array { - $now ??= time(); + return CloudTrafficConsistentRead::run(static fn (): array => self::userSnapshot($userId, $now ?? time())); + } + + private static function userSnapshot(int $userId, int $now): array + { if ($userId < 1) { return [ 'line_pools' => [], @@ -92,34 +96,6 @@ final class CloudTrafficPoolReadModel $now, $snapshots ); - $entitlements = $currentPeriods === [] ? collect() - : CloudTrafficEntitlement::whereIn( - 'service_period_id', array_keys($currentPeriods) - )->get([ - 'pool_id', 'user_id', 'instance_id', 'service_period_id', - 'quota_mode', 'granted_bytes', 'state', 'starts_at', 'ends_at', - ])->keyBy('service_period_id'); - $pendingGroups = []; - foreach ($currentPeriods as $period) { - $groupId = (int) $period->traffic_pool_group_id; - $snapshot = $snapshots[$userId . ':' . $groupId] ?? null; - $entitlement = $entitlements->get((int) $period->id); - if ($snapshot === null || $snapshot['pool_id'] === null - || $entitlement === null - || (int) $entitlement->pool_id !== (int) $snapshot['pool_id'] - || (int) $entitlement->user_id !== $userId - || (int) $entitlement->instance_id - !== (int) $period->instance_id - || (string) $entitlement->quota_mode !== 'finite' - || (int) $entitlement->granted_bytes - !== (int) $period->traffic_quota_bytes - || !in_array((string) $entitlement->state, - ['active', 'exhausted'], true) - || (int) $entitlement->starts_at !== (int) $period->starts_at - || (int) $entitlement->ends_at !== (int) $period->ends_at) { - $pendingGroups[$groupId] = true; - } - } $poolStates = []; foreach ($visibleGroupIds as $groupId) { $poolStates[$groupId] = (string) ( @@ -137,7 +113,7 @@ final class CloudTrafficPoolReadModel $instancesByGroup[$groupId] ?? [], $networkStates[$groupId] ?? 'clear', $snapshots[$userId . ':' . $groupId], - isset($pendingGroups[$groupId]), + false, $freshness[$userId . ':' . $groupId] ?? self::pendingFreshness() ); @@ -168,14 +144,28 @@ final class CloudTrafficPoolReadModel /** Resolve many periods while projecting each user/line pool only once. */ public static function forPeriods(array $periods, ?int $now = null): array { - $now ??= time(); + return CloudTrafficConsistentRead::run(static fn (): array => self::periodSnapshots($periods, $now ?? time())); + } + + private static function periodSnapshots(array $input, int $now): array + { + // Callers may have loaded a period before a concurrent purchase or upgrade. + $owners = []; + foreach ($input as $period) { + if ($period instanceof CloudServicePeriod && $period->id > 0 && $period->user_id > 0) { + $owners[$period->id] = $period->user_id; + } + } + if ($owners === []) { + return []; + } + $periods = CloudServicePeriod::whereIn('id', array_keys($owners))->get(); $results = []; $finite = []; $groupIds = []; $identities = []; foreach ($periods as $period) { - if (!$period instanceof CloudServicePeriod || (int) $period->id < 1 - || (int) $period->user_id < 1) { + if ($owners[$period->id] !== $period->user_id) { continue; } $periodId = (int) $period->id; @@ -293,7 +283,7 @@ final class CloudTrafficPoolReadModel array $freshness ): array { $projectionReady = !$forcePreparing - && $snapshot['pool_id'] !== null; + && $snapshot['pool_id'] !== null && empty($freshness['projection_pending']); return [ 'group_id' => (int) $group->id, 'name' => (string) $group->name, @@ -312,7 +302,9 @@ final class CloudTrafficPoolReadModel 'last_observed_at' => $snapshot['last_observed_at'] === null ? null : (int) $snapshot['last_observed_at'], 'accounting_freshness' => [ - 'status' => (string) ($freshness['status'] ?? 'delayed'), + 'status' => !$projectionReady && ($freshness['status'] ?? '') === 'current' + ? 'delayed' : (string) ($freshness['status'] ?? 'delayed'), + 'last_sample_at' => $freshness['last_sample_at'] ?? null, 'last_confirmed_at' => ($freshness['last_confirmed_at'] ?? null) === null ? null : (int) $freshness['last_confirmed_at'], diff --git a/web/tests/CloudTrafficPoolFreshnessTest.php b/web/tests/CloudTrafficPoolFreshnessTest.php index 85db34d3..222a69b9 100644 --- a/web/tests/CloudTrafficPoolFreshnessTest.php +++ b/web/tests/CloudTrafficPoolFreshnessTest.php @@ -33,6 +33,9 @@ $schema->create('cloud_traffic_entitlements', static function (Blueprint $t): void { $t->increments('id'); $t->integer('pool_id'); + $t->integer('service_period_id'); + $t->integer('instance_id'); + $t->integer('user_id'); $t->string('quota_mode'); $t->string('state'); $t->integer('starts_at'); @@ -47,6 +50,7 @@ $schema->create('cloud_service_periods', static function (Blueprint $t): void { $t->integer('user_id'); $t->integer('traffic_pool_group_id'); $t->string('traffic_quota_mode'); + $t->integer('traffic_quota_bytes'); $t->string('status'); $t->integer('starts_at'); $t->integer('ends_at'); @@ -90,12 +94,12 @@ Capsule::table('cloud_traffic_pools')->insert([ ]); Capsule::table('cloud_service_periods')->insert([ 'id' => 10, 'instance_id' => 20, 'user_id' => 7, - 'traffic_pool_group_id' => 3, 'traffic_quota_mode' => 'finite', + 'traffic_pool_group_id' => 3, 'traffic_quota_mode' => 'finite', 'traffic_quota_bytes' => 1000, 'status' => 'active', 'starts_at' => $now - 1000, 'ends_at' => $now + 1000, ]); Capsule::table('cloud_traffic_entitlements')->insert([ - 'id' => 30, 'pool_id' => 1, 'quota_mode' => 'finite', + 'id' => 30, 'pool_id' => 1, 'quota_mode' => 'finite', 'service_period_id' => 10, 'instance_id' => 20, 'user_id' => 7, 'state' => 'active', 'starts_at' => $now - 1000, 'ends_at' => $now + 1000, 'granted_bytes' => 1000, 'consumed_bytes' => 120, 'forfeited_bytes' => 0, @@ -188,4 +192,8 @@ $assert($group['status'] === 'attention' && $group['overdue_bucket_count'] === 1, 'Administrator group projection drifted from the user freshness model.'); +Capsule::table('cloud_service_periods')->where('id', 10)->update(['traffic_pool_group_id' => 9]); +$preparingGroup = CloudTrafficPoolFreshness::forGroup(9, $now); +$assert($preparingGroup['pool_count'] === 0 && $preparingGroup['status'] === 'delayed', + 'A group waiting for its first entitlement was presented as current.'); echo "Traffic-pool freshness projection checks passed.\n"; diff --git a/web/tests/CloudTrafficPoolUserReadModelTest.php b/web/tests/CloudTrafficPoolUserReadModelTest.php index b293e8d5..6e33826b 100644 --- a/web/tests/CloudTrafficPoolUserReadModelTest.php +++ b/web/tests/CloudTrafficPoolUserReadModelTest.php @@ -29,12 +29,8 @@ $assertNoInternalFields = static function (array $payload, string $message) } }; -$capsule = new Capsule(); -$capsule->addConnection([ - 'driver' => 'sqlite', 'database' => ':memory:', -], 'default'); -$capsule->setAsGlobal(); -$capsule->bootEloquent(); +require __DIR__ . '/Support/TrafficTestDatabase.php'; +$capsule = trafficTestDatabase(); $schema = $capsule->schema('default'); $schema->create('cloud_traffic_pool_groups', static function (Blueprint $table): void { @@ -374,7 +370,8 @@ Capsule::table('cloud_instance_network_policy_states')->insert([ Capsule::connection('default')->flushQueryLog(); Capsule::connection('default')->enableQueryLog(); $payload = CloudTrafficPoolReadModel::forUser(1, $now); -$userProjectionQueries = count(Capsule::connection('default')->getQueryLog()); +$userProjectionQueries = count(array_filter(Capsule::connection('default')->getQueryLog(), + static fn ($query): bool => str_starts_with(strtolower($query['query']), 'select'))); Capsule::connection('default')->disableQueryLog(); $assert($userProjectionQueries <= 15, 'The line-pool projection added a query loop per pool or instance: ' @@ -452,6 +449,7 @@ $assert($lineA['active_entitlement_bytes'] === 1500 'User projection did not preserve the canonical FEFO snapshot.'); $assert($lineA['accounting_freshness'] === [ 'status' => 'delayed', + 'last_sample_at' => null, 'last_confirmed_at' => $now - 5, ], 'User projection did not expose the bounded actor-safe freshness state.'); @@ -486,9 +484,10 @@ $batchPeriods = [ Capsule::connection('default')->flushQueryLog(); Capsule::connection('default')->enableQueryLog(); $batch = CloudTrafficPoolReadModel::forPeriods($batchPeriods, $now); -$batchQueries = count(Capsule::connection('default')->getQueryLog()); +$batchQueries = count(array_filter(Capsule::connection('default')->getQueryLog(), + static fn ($query): bool => str_starts_with(strtolower($query['query']), 'select'))); Capsule::connection('default')->disableQueryLog(); -$assert($batchQueries <= 7 && count($batch) === 4, +$assert($batchQueries <= 8 && count($batch) === 4, 'Multi-user period projection regressed to a query loop per pool.'); $pooled = CloudTrafficPoolReadModel::forPeriod( @@ -542,4 +541,52 @@ $assert($upgradePending['traffic_accounting_mode'] === 'pooled' && $upgradePending['traffic_pool']['active_entitlement_bytes'] === 1500, 'A paid quota increase exposed the stale entitlement total as ready.'); +// With a recent sample and a fully accounted cursor, an unprojected upgrade is still pending. +$vm = Capsule::table('cloud_instances')->where('id', 1); +$cursor = Capsule::table('cloud_traffic_instance_cursors')->where('instance_id', 1); +$vm->update(['traffic_synced_at' => $now, 'lifetime_download_bytes' => 100]); +$cursor->update(['last_observed_at' => $now, 'accounted_download_bytes' => 100]); +$oldPeriod = CloudServicePeriod::find(101); +$oldPeriod->traffic_quota_bytes = 1000; +$pendingUpgrade = CloudTrafficPoolReadModel::forPeriod(1, $oldPeriod, $now)['traffic_pool']; +$assert(!$pendingUpgrade['projection_ready'] && $pendingUpgrade['accounting_freshness']['status'] === 'delayed', + 'A stale caller period or settled counters hid a pending paid upgrade.'); +Capsule::table('cloud_service_periods')->where('id', 101)->update(['traffic_quota_bytes' => 1000]); + +// Exactly 990 bytes are booked while a second connection commits 90 more bytes (900 left). +$ent = Capsule::table('cloud_traffic_entitlements')->where('id', 1); +$ent->update(['consumed_bytes' => 410]); +$vm->update(['lifetime_download_bytes' => 190]); +$race = true; +$capsule->getConnection()->beforeExecuting(static function ($sql) use ($capsule, &$race): void { + if ($race && str_starts_with(strtolower($sql), 'select') && str_contains($sql, 'freshness_instance')) { + $race = false; + $peer = $capsule->getConnection('peer'); + $peer->transaction(static function () use ($peer): void { + $peer->table('cloud_traffic_instance_cursors')->where('instance_id', 1)->update(['accounted_download_bytes' => 190]); + $peer->table('cloud_traffic_entitlements')->where('id', 1)->update(['consumed_bytes' => 500]); + }); + } +}); +$before = CloudTrafficPoolReadModel::forUser(1, $now)['line_pools'][0]; +$after = CloudTrafficPoolReadModel::forPeriod(1, $oldPeriod, $now)['traffic_pool']; +$assert(!$race && $before['available_bytes'] === 990 && $before['accounting_freshness']['status'] === 'delayed' + && $after['available_bytes'] === 900 && $after['accounting_freshness']['status'] === 'current' + && $after['accounting_freshness']['last_sample_at'] === $now, + 'Balance and freshness were taken from different database snapshots.'); +$ent->update(['consumed_bytes' => 410]); +$cursor->update(['accounted_download_bytes' => 100]); +$race = true; +$detailBefore = CloudTrafficPoolReadModel::forPeriod(1, $oldPeriod, $now)['traffic_pool']; +$assert(!$race && $detailBefore['available_bytes'] === 990 && $detailBefore['accounting_freshness']['status'] === 'delayed', + 'Instance detail mixed its period balance with a concurrent cursor advance.'); +$vm->update(['status' => 'deleted', 'lifetime_download_bytes' => 490]); +$retiredPool = CloudTrafficPoolReadModel::forUser(1, $now)['line_pools'][0]; +$assert($retiredPool['available_bytes'] === 900 && $retiredPool['accounting_freshness']['status'] === 'delayed' + && $retiredPool['accounting_freshness']['last_sample_at'] === null, 'The final 300 bytes were hidden by deletion.'); +$cursor->update(['accounted_download_bytes' => 490]); +$ent->update(['consumed_bytes' => 800]); +$retiredPool = CloudTrafficPoolReadModel::forUser(1, $now)['line_pools'][0]; +$assert($retiredPool['available_bytes'] === 600 && $retiredPool['accounting_freshness']['status'] === 'current', + 'Settled tail usage discarded a paid entitlement or kept freshness delayed.'); echo "Cloud traffic-pool user read-model tests passed.\n";