fix(traffic): read booked balances and accounting status consistently

go
codex 11 hours ago
parent 4a5688d88a
commit 0614ce69c1

@ -43,6 +43,11 @@ final class CloudAdminTrafficPoolService
];
public static function page(array $query = []): array
{
return CloudTrafficConsistentRead::run(static fn (): array => self::pageSnapshot());
}
private static function pageSnapshot(): array
{
$groups = CloudTrafficPoolGroup::orderBy('id')->get();
$generatedAt = time();
@ -84,7 +89,7 @@ final class CloudAdminTrafficPoolService
public static function read(int $groupId): array
{
return ['item' => self::groupDto(self::findGroup($groupId))];
return CloudTrafficConsistentRead::run(static fn (): array => ['item' => self::groupDto(self::findGroup($groupId))]);
}
public static function create(array $input): array
@ -227,7 +232,7 @@ final class CloudAdminTrafficPoolService
int $groupId,
?int $now = null
): array {
return self::buildReadiness(self::findGroup($groupId), $now ?? time());
return CloudTrafficConsistentRead::run(static fn (): array => self::buildReadiness(self::findGroup($groupId), $now ?? time()));
}
private static function assertModeTransition(

@ -31,7 +31,12 @@ final class CloudTrafficPoolFreshness
?int $now = null,
?array $accountingSnapshots = null
): array {
$now ??= time();
return CloudTrafficConsistentRead::run(static fn (): array =>
self::readSnapshots($identities, $now ?? time(), $accountingSnapshots));
}
private static function readSnapshots(array $identities, int $now, ?array $accountingSnapshots): array
{
$pairs = [];
foreach ($identities as $identity) {
$userId = (int) ($identity['user_id'] ?? 0);
@ -65,6 +70,16 @@ final class CloudTrafficPoolFreshness
$factsByKey = [];
if ($snapshotByKey !== []) {
$facts = CloudServicePeriod::query()
->leftJoin('cloud_traffic_entitlements as matching_entitlement', static function ($join): void {
$join->on('matching_entitlement.service_period_id', '=', 'cloud_service_periods.id')
->on('matching_entitlement.user_id', '=', 'cloud_service_periods.user_id')
->on('matching_entitlement.instance_id', '=', 'cloud_service_periods.instance_id')
->on('matching_entitlement.starts_at', '=', 'cloud_service_periods.starts_at')
->on('matching_entitlement.ends_at', '=', 'cloud_service_periods.ends_at')
->on('matching_entitlement.granted_bytes', '=', 'cloud_service_periods.traffic_quota_bytes')
->where('matching_entitlement.quota_mode', '=', 'finite')
->whereIn('matching_entitlement.state', ['active', 'exhausted']);
})
->leftJoin('cloud_instances as freshness_instance',
'freshness_instance.id', '=',
'cloud_service_periods.instance_id')
@ -111,6 +126,7 @@ final class CloudTrafficPoolFreshness
->where('cloud_service_periods.ends_at', '>', $now)
->orderBy('cloud_service_periods.id')->get([
'cloud_service_periods.id as period_id',
'matching_entitlement.pool_id as matched_pool_id',
'cloud_service_periods.instance_id',
'cloud_service_periods.user_id as period_user_id',
'cloud_service_periods.traffic_pool_group_id as group_id',
@ -165,12 +181,25 @@ final class CloudTrafficPoolFreshness
$unsettled = [];
$stale = [];
$confirmed = [];
$latestSample = 0;
$pendingEntitlement = false;
if ((string) $snapshot['state'] === 'attention') {
$status = 'attention';
$code = 'pool_attention';
}
foreach ($factsByKey[$key] ?? [] as $fact) {
$instanceId = (int) $fact->instance_id;
if ((int) $fact->matched_pool_id !== (int) $snapshot['pool_id']) {
$pendingEntitlement = true;
if ($status === 'current') {
$status = 'delayed';
$code = 'projection_pending';
}
}
if ($fact->instance_status !== null && CloudTrafficAccountingEvidence::expectsSample($fact->instance_status)
&& (int) $fact->instance_user_id === $pair['user_id']) {
$latestSample = max($latestSample, (int) $fact->traffic_synced_at);
}
if ($fact->cursor_instance_id === null
|| (int) $fact->instance_user_id !== $pair['user_id']
|| (int) $fact->cursor_pool_id
@ -240,6 +269,8 @@ final class CloudTrafficPoolFreshness
$result[$key] = [
'status' => $status,
'last_confirmed_at' => $lastConfirmed,
'last_sample_at' => $latestSample ?: null,
'projection_pending' => $pendingEntitlement,
'unsettled_instance_count' => count($unsettled),
'stale_cursor_count' => count($stale),
'overdue_bucket_count' => (int) $due['count'],
@ -264,7 +295,11 @@ final class CloudTrafficPoolFreshness
array $trafficPoolGroupIds,
?int $now = null
): array {
$now ??= time();
return CloudTrafficConsistentRead::run(static fn (): array => self::groupSnapshots($trafficPoolGroupIds, $now ?? time()));
}
private static function groupSnapshots(array $trafficPoolGroupIds, int $now): array
{
$groupIds = array_values(array_unique(array_filter(array_map(
'intval',
$trafficPoolGroupIds
@ -281,6 +316,14 @@ final class CloudTrafficPoolFreshness
(int) $pool->traffic_pool_group_id,
]
)->all();
$materialized = array_count_values(array_column($identities, 'traffic_pool_group_id'));
$current = CloudServicePeriod::whereIn('traffic_pool_group_id', $groupIds)
->where('traffic_quota_mode', 'finite')->whereIn('status', ['active', 'exhausted'])
->where('starts_at', '<=', $now)->where('ends_at', '>', $now)
->distinct()->get(['user_id', 'traffic_pool_group_id']);
foreach ($current as $period) {
$identities[] = $period->only(['user_id', 'traffic_pool_group_id']);
}
$groupByKey = [];
foreach ($identities as $identity) {
$groupByKey[(int) $identity['user_id'] . ':'
@ -300,6 +343,7 @@ final class CloudTrafficPoolFreshness
$result[$groupId] = self::aggregate(
$byGroup[$groupId] ?? []
);
$result[$groupId]['pool_count'] = $materialized[$groupId] ?? 0;
}
return $result;
}
@ -313,6 +357,7 @@ final class CloudTrafficPoolFreshness
$rank = ['current' => 0, 'delayed' => 1, 'attention' => 2];
$status = 'current';
$lastConfirmed = [];
$latestSample = 0;
$unsettled = 0;
$stale = 0;
$overdue = 0;
@ -320,6 +365,7 @@ final class CloudTrafficPoolFreshness
if ($rank[$snapshot['status']] > $rank[$status]) {
$status = $snapshot['status'];
}
$latestSample = max($latestSample, (int) $snapshot['last_sample_at']);
if ($snapshot['last_confirmed_at'] !== null) {
$lastConfirmed[] = (int) $snapshot['last_confirmed_at'];
}
@ -329,6 +375,7 @@ final class CloudTrafficPoolFreshness
}
return [
'status' => $status,
'last_sample_at' => $latestSample ?: null,
'last_confirmed_at' => count($lastConfirmed) === count($snapshots)
? min($lastConfirmed) : null,
'unsettled_instance_count' => $unsettled,
@ -346,6 +393,8 @@ final class CloudTrafficPoolFreshness
return [
'status' => $code === 'no_pools' ? 'current' : 'delayed',
'last_confirmed_at' => null,
'last_sample_at' => null,
'projection_pending' => $code === 'projection_pending',
'unsettled_instance_count' => 0,
'stale_cursor_count' => 0,
'overdue_bucket_count' => 0,

@ -15,7 +15,11 @@ final class CloudTrafficPoolReadModel
{
public static function forUser(int $userId, ?int $now = null): array
{
$now ??= time();
return CloudTrafficConsistentRead::run(static fn (): array => self::userSnapshot($userId, $now ?? time()));
}
private static function userSnapshot(int $userId, int $now): array
{
if ($userId < 1) {
return [
'line_pools' => [],
@ -92,34 +96,6 @@ final class CloudTrafficPoolReadModel
$now,
$snapshots
);
$entitlements = $currentPeriods === [] ? collect()
: CloudTrafficEntitlement::whereIn(
'service_period_id', array_keys($currentPeriods)
)->get([
'pool_id', 'user_id', 'instance_id', 'service_period_id',
'quota_mode', 'granted_bytes', 'state', 'starts_at', 'ends_at',
])->keyBy('service_period_id');
$pendingGroups = [];
foreach ($currentPeriods as $period) {
$groupId = (int) $period->traffic_pool_group_id;
$snapshot = $snapshots[$userId . ':' . $groupId] ?? null;
$entitlement = $entitlements->get((int) $period->id);
if ($snapshot === null || $snapshot['pool_id'] === null
|| $entitlement === null
|| (int) $entitlement->pool_id !== (int) $snapshot['pool_id']
|| (int) $entitlement->user_id !== $userId
|| (int) $entitlement->instance_id
!== (int) $period->instance_id
|| (string) $entitlement->quota_mode !== 'finite'
|| (int) $entitlement->granted_bytes
!== (int) $period->traffic_quota_bytes
|| !in_array((string) $entitlement->state,
['active', 'exhausted'], true)
|| (int) $entitlement->starts_at !== (int) $period->starts_at
|| (int) $entitlement->ends_at !== (int) $period->ends_at) {
$pendingGroups[$groupId] = true;
}
}
$poolStates = [];
foreach ($visibleGroupIds as $groupId) {
$poolStates[$groupId] = (string) (
@ -137,7 +113,7 @@ final class CloudTrafficPoolReadModel
$instancesByGroup[$groupId] ?? [],
$networkStates[$groupId] ?? 'clear',
$snapshots[$userId . ':' . $groupId],
isset($pendingGroups[$groupId]),
false,
$freshness[$userId . ':' . $groupId]
?? self::pendingFreshness()
);
@ -168,14 +144,28 @@ final class CloudTrafficPoolReadModel
/** Resolve many periods while projecting each user/line pool only once. */
public static function forPeriods(array $periods, ?int $now = null): array
{
$now ??= time();
return CloudTrafficConsistentRead::run(static fn (): array => self::periodSnapshots($periods, $now ?? time()));
}
private static function periodSnapshots(array $input, int $now): array
{
// Callers may have loaded a period before a concurrent purchase or upgrade.
$owners = [];
foreach ($input as $period) {
if ($period instanceof CloudServicePeriod && $period->id > 0 && $period->user_id > 0) {
$owners[$period->id] = $period->user_id;
}
}
if ($owners === []) {
return [];
}
$periods = CloudServicePeriod::whereIn('id', array_keys($owners))->get();
$results = [];
$finite = [];
$groupIds = [];
$identities = [];
foreach ($periods as $period) {
if (!$period instanceof CloudServicePeriod || (int) $period->id < 1
|| (int) $period->user_id < 1) {
if ($owners[$period->id] !== $period->user_id) {
continue;
}
$periodId = (int) $period->id;
@ -293,7 +283,7 @@ final class CloudTrafficPoolReadModel
array $freshness
): array {
$projectionReady = !$forcePreparing
&& $snapshot['pool_id'] !== null;
&& $snapshot['pool_id'] !== null && empty($freshness['projection_pending']);
return [
'group_id' => (int) $group->id,
'name' => (string) $group->name,
@ -312,7 +302,9 @@ final class CloudTrafficPoolReadModel
'last_observed_at' => $snapshot['last_observed_at'] === null
? null : (int) $snapshot['last_observed_at'],
'accounting_freshness' => [
'status' => (string) ($freshness['status'] ?? 'delayed'),
'status' => !$projectionReady && ($freshness['status'] ?? '') === 'current'
? 'delayed' : (string) ($freshness['status'] ?? 'delayed'),
'last_sample_at' => $freshness['last_sample_at'] ?? null,
'last_confirmed_at' =>
($freshness['last_confirmed_at'] ?? null) === null
? null : (int) $freshness['last_confirmed_at'],

@ -33,6 +33,9 @@ $schema->create('cloud_traffic_entitlements',
static function (Blueprint $t): void {
$t->increments('id');
$t->integer('pool_id');
$t->integer('service_period_id');
$t->integer('instance_id');
$t->integer('user_id');
$t->string('quota_mode');
$t->string('state');
$t->integer('starts_at');
@ -47,6 +50,7 @@ $schema->create('cloud_service_periods', static function (Blueprint $t): void {
$t->integer('user_id');
$t->integer('traffic_pool_group_id');
$t->string('traffic_quota_mode');
$t->integer('traffic_quota_bytes');
$t->string('status');
$t->integer('starts_at');
$t->integer('ends_at');
@ -90,12 +94,12 @@ Capsule::table('cloud_traffic_pools')->insert([
]);
Capsule::table('cloud_service_periods')->insert([
'id' => 10, 'instance_id' => 20, 'user_id' => 7,
'traffic_pool_group_id' => 3, 'traffic_quota_mode' => 'finite',
'traffic_pool_group_id' => 3, 'traffic_quota_mode' => 'finite', 'traffic_quota_bytes' => 1000,
'status' => 'active', 'starts_at' => $now - 1000,
'ends_at' => $now + 1000,
]);
Capsule::table('cloud_traffic_entitlements')->insert([
'id' => 30, 'pool_id' => 1, 'quota_mode' => 'finite',
'id' => 30, 'pool_id' => 1, 'quota_mode' => 'finite', 'service_period_id' => 10, 'instance_id' => 20, 'user_id' => 7,
'state' => 'active', 'starts_at' => $now - 1000,
'ends_at' => $now + 1000, 'granted_bytes' => 1000,
'consumed_bytes' => 120, 'forfeited_bytes' => 0,
@ -188,4 +192,8 @@ $assert($group['status'] === 'attention'
&& $group['overdue_bucket_count'] === 1,
'Administrator group projection drifted from the user freshness model.');
Capsule::table('cloud_service_periods')->where('id', 10)->update(['traffic_pool_group_id' => 9]);
$preparingGroup = CloudTrafficPoolFreshness::forGroup(9, $now);
$assert($preparingGroup['pool_count'] === 0 && $preparingGroup['status'] === 'delayed',
'A group waiting for its first entitlement was presented as current.');
echo "Traffic-pool freshness projection checks passed.\n";

@ -29,12 +29,8 @@ $assertNoInternalFields = static function (array $payload, string $message)
}
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
require __DIR__ . '/Support/TrafficTestDatabase.php';
$capsule = trafficTestDatabase();
$schema = $capsule->schema('default');
$schema->create('cloud_traffic_pool_groups', static function (Blueprint $table): void {
@ -374,7 +370,8 @@ Capsule::table('cloud_instance_network_policy_states')->insert([
Capsule::connection('default')->flushQueryLog();
Capsule::connection('default')->enableQueryLog();
$payload = CloudTrafficPoolReadModel::forUser(1, $now);
$userProjectionQueries = count(Capsule::connection('default')->getQueryLog());
$userProjectionQueries = count(array_filter(Capsule::connection('default')->getQueryLog(),
static fn ($query): bool => str_starts_with(strtolower($query['query']), 'select')));
Capsule::connection('default')->disableQueryLog();
$assert($userProjectionQueries <= 15,
'The line-pool projection added a query loop per pool or instance: '
@ -452,6 +449,7 @@ $assert($lineA['active_entitlement_bytes'] === 1500
'User projection did not preserve the canonical FEFO snapshot.');
$assert($lineA['accounting_freshness'] === [
'status' => 'delayed',
'last_sample_at' => null,
'last_confirmed_at' => $now - 5,
],
'User projection did not expose the bounded actor-safe freshness state.');
@ -486,9 +484,10 @@ $batchPeriods = [
Capsule::connection('default')->flushQueryLog();
Capsule::connection('default')->enableQueryLog();
$batch = CloudTrafficPoolReadModel::forPeriods($batchPeriods, $now);
$batchQueries = count(Capsule::connection('default')->getQueryLog());
$batchQueries = count(array_filter(Capsule::connection('default')->getQueryLog(),
static fn ($query): bool => str_starts_with(strtolower($query['query']), 'select')));
Capsule::connection('default')->disableQueryLog();
$assert($batchQueries <= 7 && count($batch) === 4,
$assert($batchQueries <= 8 && count($batch) === 4,
'Multi-user period projection regressed to a query loop per pool.');
$pooled = CloudTrafficPoolReadModel::forPeriod(
@ -542,4 +541,52 @@ $assert($upgradePending['traffic_accounting_mode'] === 'pooled'
&& $upgradePending['traffic_pool']['active_entitlement_bytes'] === 1500,
'A paid quota increase exposed the stale entitlement total as ready.');
// With a recent sample and a fully accounted cursor, an unprojected upgrade is still pending.
$vm = Capsule::table('cloud_instances')->where('id', 1);
$cursor = Capsule::table('cloud_traffic_instance_cursors')->where('instance_id', 1);
$vm->update(['traffic_synced_at' => $now, 'lifetime_download_bytes' => 100]);
$cursor->update(['last_observed_at' => $now, 'accounted_download_bytes' => 100]);
$oldPeriod = CloudServicePeriod::find(101);
$oldPeriod->traffic_quota_bytes = 1000;
$pendingUpgrade = CloudTrafficPoolReadModel::forPeriod(1, $oldPeriod, $now)['traffic_pool'];
$assert(!$pendingUpgrade['projection_ready'] && $pendingUpgrade['accounting_freshness']['status'] === 'delayed',
'A stale caller period or settled counters hid a pending paid upgrade.');
Capsule::table('cloud_service_periods')->where('id', 101)->update(['traffic_quota_bytes' => 1000]);
// Exactly 990 bytes are booked while a second connection commits 90 more bytes (900 left).
$ent = Capsule::table('cloud_traffic_entitlements')->where('id', 1);
$ent->update(['consumed_bytes' => 410]);
$vm->update(['lifetime_download_bytes' => 190]);
$race = true;
$capsule->getConnection()->beforeExecuting(static function ($sql) use ($capsule, &$race): void {
if ($race && str_starts_with(strtolower($sql), 'select') && str_contains($sql, 'freshness_instance')) {
$race = false;
$peer = $capsule->getConnection('peer');
$peer->transaction(static function () use ($peer): void {
$peer->table('cloud_traffic_instance_cursors')->where('instance_id', 1)->update(['accounted_download_bytes' => 190]);
$peer->table('cloud_traffic_entitlements')->where('id', 1)->update(['consumed_bytes' => 500]);
});
}
});
$before = CloudTrafficPoolReadModel::forUser(1, $now)['line_pools'][0];
$after = CloudTrafficPoolReadModel::forPeriod(1, $oldPeriod, $now)['traffic_pool'];
$assert(!$race && $before['available_bytes'] === 990 && $before['accounting_freshness']['status'] === 'delayed'
&& $after['available_bytes'] === 900 && $after['accounting_freshness']['status'] === 'current'
&& $after['accounting_freshness']['last_sample_at'] === $now,
'Balance and freshness were taken from different database snapshots.');
$ent->update(['consumed_bytes' => 410]);
$cursor->update(['accounted_download_bytes' => 100]);
$race = true;
$detailBefore = CloudTrafficPoolReadModel::forPeriod(1, $oldPeriod, $now)['traffic_pool'];
$assert(!$race && $detailBefore['available_bytes'] === 990 && $detailBefore['accounting_freshness']['status'] === 'delayed',
'Instance detail mixed its period balance with a concurrent cursor advance.');
$vm->update(['status' => 'deleted', 'lifetime_download_bytes' => 490]);
$retiredPool = CloudTrafficPoolReadModel::forUser(1, $now)['line_pools'][0];
$assert($retiredPool['available_bytes'] === 900 && $retiredPool['accounting_freshness']['status'] === 'delayed'
&& $retiredPool['accounting_freshness']['last_sample_at'] === null, 'The final 300 bytes were hidden by deletion.');
$cursor->update(['accounted_download_bytes' => 490]);
$ent->update(['consumed_bytes' => 800]);
$retiredPool = CloudTrafficPoolReadModel::forUser(1, $now)['line_pools'][0];
$assert($retiredPool['available_bytes'] === 600 && $retiredPool['accounting_freshness']['status'] === 'current',
'Settled tail usage discarded a paid entitlement or kept freshness delayed.');
echo "Cloud traffic-pool user read-model tests passed.\n";

Loading…
Cancel
Save