fix(traffic): require verified settlement before retiring or reporting recovery

go
codex 5 days ago
parent 0614ce69c1
commit 2b1f796766

@ -72,9 +72,24 @@ final class CloudTrafficScanRecovery
public static function clearIfResolved(?array $generation, int $now): bool
{
if (!$generation || !self::resolved($generation, $now)) {
if (!$generation) {
return false;
}
return CloudTrafficScanFailure::where($generation)->delete() === 1;
$connection = (new CloudTrafficScanFailure())->getConnection();
$connection->beginTransaction();
try {
// Claim this exact generation before taking a fresh view of its subject.
// An unresolved sample rolls the deletion back before it becomes visible.
if (CloudTrafficScanFailure::where($generation)->delete() !== 1
|| !self::resolved($generation, $now)) {
$connection->rollBack();
return false;
}
$connection->commit();
return true;
} catch (\Throwable $error) {
$connection->rollBack();
throw $error;
}
}
}

@ -183,10 +183,12 @@ final class CloudTrafficSettlementRecovery
$result = CloudTrafficPoolAccounting::reconcileServicePeriod(
$id, 'settlement-recovery:' . $planHash . ':period:' . $id, $now, $item
);
CloudTrafficRecoveryFacts::assertSame(true, $result['enabled']);
self::requireSettled($result);
break;
case 'unsettled_instances':
CloudTrafficPoolAccounting::settleInstanceUsage($id, $item['sample']['traffic_synced_at'], $item);
self::requireSettled(CloudTrafficPoolAccounting::settleInstanceUsage(
$id, $item['sample']['traffic_synced_at'], $item
));
break;
}
$completed[$kind][] = $id;
@ -319,6 +321,14 @@ final class CloudTrafficSettlementRecovery
])->all();
}
private static function requireSettled(array $result): void
{
CloudTrafficRecoveryFacts::assertSame(true, $result['enabled']);
if (($result['attention_code'] ?? null) !== null) {
throw new \RuntimeException($result['attention_code']);
}
}
private static function planHash(array $plan): string
{
return hash('sha256', json_encode(

@ -80,6 +80,15 @@ $schema->create('cloud_traffic_usage_settlement_buckets',
$t->integer('created_at');
$t->integer('updated_at');
});
$schema->create('cloud_traffic_settlement_receipts', static function (Blueprint $t): void {
$t->integer('pool_id');
$t->integer('instance_id');
$t->integer('service_period_id');
$t->string('allocation_kind');
$t->integer('allocation_target_id');
$t->integer('bucket_start');
$t->integer('max_revision');
});
$schema->create('cloud_traffic_instance_cursors',
static function (Blueprint $t): void {
$t->integer('instance_id')->primary();
@ -328,6 +337,57 @@ $assert(CloudTrafficSettlementRecovery::inspect()['counts']['scan_failures'] ===
&& !App\Services\Cloud\CloudTrafficScanRecovery::clearIfResolved($failure, $now),
'An attention cursor was treated as successfully recovered.');
Capsule::table('cloud_traffic_instance_cursors')->update(['state' => 'ready']);
$cleanup = CloudTrafficSettlementRecovery::inspect();
$changeOnCleanup = true;
$capsule->getConnection()->beforeExecuting(static function ($sql) use (&$changeOnCleanup, $capsule): void {
if ($changeOnCleanup && str_starts_with(strtolower($sql), 'delete from')) {
$changeOnCleanup = false;
$capsule->getConnection('peer')->table('cloud_instances')->where('id', 11)
->update(['lifetime_download_bytes' => 100]);
}
});
$result = CloudTrafficSettlementRecovery::apply($cleanup['plan_hash']);
$assert(!$changeOnCleanup && $result['status'] === 'stopped'
&& Capsule::table('cloud_traffic_scan_failures')->count() === 1,
'Cleanup erased a failure while a concurrently committed sample was unsettled.');
Capsule::table('cloud_traffic_pools')->where('id', 1)->update(['overage_bytes' => 0]);
$plan = CloudTrafficSettlementRecovery::inspect();
$expected = 100;
if ($capsule->getConnection()->getDriverName() === 'mysql') {
$changeDuringWrite = true;
$capsule->getConnection()->beforeExecuting(static function ($sql) use (&$changeDuringWrite, $capsule): void {
if ($changeDuringWrite && str_starts_with(strtolower($sql), 'update `cloud_traffic_instance_cursors`')) {
$changeDuringWrite = false;
$capsule->getConnection('peer')->table('cloud_instances')->where('id', 11)
->update(['lifetime_download_bytes' => 105]);
}
});
$stopped = CloudTrafficSettlementRecovery::apply($plan['plan_hash']);
$assert(!$changeDuringWrite && $stopped['failed']['reason'] === 'traffic_settlement_recovery_plan_changed'
&& (int) Capsule::table('cloud_traffic_entitlements')->value('consumed_bytes') === 0
&& (int) Capsule::table('cloud_traffic_instance_cursors')->value('accounted_download_bytes') === 90,
'A changed sample did not roll back the entire manual accounting write.');
$plan = CloudTrafficSettlementRecovery::inspect();
$expected = 105;
}
$result = CloudTrafficSettlementRecovery::apply($plan['plan_hash']);
$assert($result['status'] === 'applied' && $result['settled_instances'] === 1 && $result['cleared_scan_failures'] === 1
&& (int) Capsule::table('cloud_traffic_entitlements')->value('consumed_bytes') === $expected - 90
&& (int) Capsule::table('cloud_traffic_instance_cursors')->value('accounted_download_bytes') === $expected
&& (int) Capsule::table('cloud_traffic_usage_settlement_buckets')->where('entitlement_id', 41)->sum('amount_bytes') === $expected - 90,
'The reviewed sample was not booked exactly once with its failure retired: ' . json_encode($result));
$empty = CloudTrafficSettlementRecovery::inspect();
$assert(CloudTrafficSettlementRecovery::apply($empty['plan_hash'])['settled_instances'] === 0,
'A successfully booked sample was selected again.');
Capsule::table('cloud_instances')->where('id', 11)->update(['lifetime_download_bytes' => $expected - 1]);
$regression = CloudTrafficSettlementRecovery::inspect();
$result = CloudTrafficSettlementRecovery::apply($regression['plan_hash']);
$assert($result['status'] === 'stopped' && $result['failed']['reason'] === 'traffic_counter_regression'
&& $result['completed']['unsettled_instances'] === [] && $result['remaining']['unsettled_instances'] === [11]
&& Capsule::table('cloud_traffic_instance_cursors')->value('state') === 'attention'
&& (int) Capsule::table('cloud_traffic_entitlements')->value('consumed_bytes') === $expected - 90,
'A regressed counter was reported as settled or lost its durable attention marker.');
Capsule::table('cloud_traffic_usage_settlement_buckets')->insert(array_merge(
$bucket,
[

Loading…
Cancel
Save