diff --git a/web/app/Services/Cloud/CloudTrafficScanRecovery.php b/web/app/Services/Cloud/CloudTrafficScanRecovery.php index 163a89b1..855d4d52 100644 --- a/web/app/Services/Cloud/CloudTrafficScanRecovery.php +++ b/web/app/Services/Cloud/CloudTrafficScanRecovery.php @@ -72,9 +72,24 @@ final class CloudTrafficScanRecovery public static function clearIfResolved(?array $generation, int $now): bool { - if (!$generation || !self::resolved($generation, $now)) { + if (!$generation) { return false; } - return CloudTrafficScanFailure::where($generation)->delete() === 1; + $connection = (new CloudTrafficScanFailure())->getConnection(); + $connection->beginTransaction(); + try { + // Claim this exact generation before taking a fresh view of its subject. + // An unresolved sample rolls the deletion back before it becomes visible. + if (CloudTrafficScanFailure::where($generation)->delete() !== 1 + || !self::resolved($generation, $now)) { + $connection->rollBack(); + return false; + } + $connection->commit(); + return true; + } catch (\Throwable $error) { + $connection->rollBack(); + throw $error; + } } } diff --git a/web/app/Services/Cloud/CloudTrafficSettlementRecovery.php b/web/app/Services/Cloud/CloudTrafficSettlementRecovery.php index ccf7ea0a..81995208 100644 --- a/web/app/Services/Cloud/CloudTrafficSettlementRecovery.php +++ b/web/app/Services/Cloud/CloudTrafficSettlementRecovery.php @@ -183,10 +183,12 @@ final class CloudTrafficSettlementRecovery $result = CloudTrafficPoolAccounting::reconcileServicePeriod( $id, 'settlement-recovery:' . $planHash . ':period:' . $id, $now, $item ); - CloudTrafficRecoveryFacts::assertSame(true, $result['enabled']); + self::requireSettled($result); break; case 'unsettled_instances': - CloudTrafficPoolAccounting::settleInstanceUsage($id, $item['sample']['traffic_synced_at'], $item); + self::requireSettled(CloudTrafficPoolAccounting::settleInstanceUsage( + $id, $item['sample']['traffic_synced_at'], $item + )); break; } $completed[$kind][] = $id; @@ -319,6 +321,14 @@ final class CloudTrafficSettlementRecovery ])->all(); } + private static function requireSettled(array $result): void + { + CloudTrafficRecoveryFacts::assertSame(true, $result['enabled']); + if (($result['attention_code'] ?? null) !== null) { + throw new \RuntimeException($result['attention_code']); + } + } + private static function planHash(array $plan): string { return hash('sha256', json_encode( diff --git a/web/tests/CloudTrafficSettlementRecoveryTest.php b/web/tests/CloudTrafficSettlementRecoveryTest.php index 18deb426..3ba05082 100644 --- a/web/tests/CloudTrafficSettlementRecoveryTest.php +++ b/web/tests/CloudTrafficSettlementRecoveryTest.php @@ -80,6 +80,15 @@ $schema->create('cloud_traffic_usage_settlement_buckets', $t->integer('created_at'); $t->integer('updated_at'); }); +$schema->create('cloud_traffic_settlement_receipts', static function (Blueprint $t): void { + $t->integer('pool_id'); + $t->integer('instance_id'); + $t->integer('service_period_id'); + $t->string('allocation_kind'); + $t->integer('allocation_target_id'); + $t->integer('bucket_start'); + $t->integer('max_revision'); +}); $schema->create('cloud_traffic_instance_cursors', static function (Blueprint $t): void { $t->integer('instance_id')->primary(); @@ -328,6 +337,57 @@ $assert(CloudTrafficSettlementRecovery::inspect()['counts']['scan_failures'] === && !App\Services\Cloud\CloudTrafficScanRecovery::clearIfResolved($failure, $now), 'An attention cursor was treated as successfully recovered.'); Capsule::table('cloud_traffic_instance_cursors')->update(['state' => 'ready']); +$cleanup = CloudTrafficSettlementRecovery::inspect(); +$changeOnCleanup = true; +$capsule->getConnection()->beforeExecuting(static function ($sql) use (&$changeOnCleanup, $capsule): void { + if ($changeOnCleanup && str_starts_with(strtolower($sql), 'delete from')) { + $changeOnCleanup = false; + $capsule->getConnection('peer')->table('cloud_instances')->where('id', 11) + ->update(['lifetime_download_bytes' => 100]); + } +}); +$result = CloudTrafficSettlementRecovery::apply($cleanup['plan_hash']); +$assert(!$changeOnCleanup && $result['status'] === 'stopped' + && Capsule::table('cloud_traffic_scan_failures')->count() === 1, + 'Cleanup erased a failure while a concurrently committed sample was unsettled.'); + +Capsule::table('cloud_traffic_pools')->where('id', 1)->update(['overage_bytes' => 0]); +$plan = CloudTrafficSettlementRecovery::inspect(); +$expected = 100; +if ($capsule->getConnection()->getDriverName() === 'mysql') { + $changeDuringWrite = true; + $capsule->getConnection()->beforeExecuting(static function ($sql) use (&$changeDuringWrite, $capsule): void { + if ($changeDuringWrite && str_starts_with(strtolower($sql), 'update `cloud_traffic_instance_cursors`')) { + $changeDuringWrite = false; + $capsule->getConnection('peer')->table('cloud_instances')->where('id', 11) + ->update(['lifetime_download_bytes' => 105]); + } + }); + $stopped = CloudTrafficSettlementRecovery::apply($plan['plan_hash']); + $assert(!$changeDuringWrite && $stopped['failed']['reason'] === 'traffic_settlement_recovery_plan_changed' + && (int) Capsule::table('cloud_traffic_entitlements')->value('consumed_bytes') === 0 + && (int) Capsule::table('cloud_traffic_instance_cursors')->value('accounted_download_bytes') === 90, + 'A changed sample did not roll back the entire manual accounting write.'); + $plan = CloudTrafficSettlementRecovery::inspect(); + $expected = 105; +} +$result = CloudTrafficSettlementRecovery::apply($plan['plan_hash']); +$assert($result['status'] === 'applied' && $result['settled_instances'] === 1 && $result['cleared_scan_failures'] === 1 + && (int) Capsule::table('cloud_traffic_entitlements')->value('consumed_bytes') === $expected - 90 + && (int) Capsule::table('cloud_traffic_instance_cursors')->value('accounted_download_bytes') === $expected + && (int) Capsule::table('cloud_traffic_usage_settlement_buckets')->where('entitlement_id', 41)->sum('amount_bytes') === $expected - 90, + 'The reviewed sample was not booked exactly once with its failure retired: ' . json_encode($result)); +$empty = CloudTrafficSettlementRecovery::inspect(); +$assert(CloudTrafficSettlementRecovery::apply($empty['plan_hash'])['settled_instances'] === 0, + 'A successfully booked sample was selected again.'); +Capsule::table('cloud_instances')->where('id', 11)->update(['lifetime_download_bytes' => $expected - 1]); +$regression = CloudTrafficSettlementRecovery::inspect(); +$result = CloudTrafficSettlementRecovery::apply($regression['plan_hash']); +$assert($result['status'] === 'stopped' && $result['failed']['reason'] === 'traffic_counter_regression' + && $result['completed']['unsettled_instances'] === [] && $result['remaining']['unsettled_instances'] === [11] + && Capsule::table('cloud_traffic_instance_cursors')->value('state') === 'attention' + && (int) Capsule::table('cloud_traffic_entitlements')->value('consumed_bytes') === $expected - 90, + 'A regressed counter was reported as settled or lost its durable attention marker.'); Capsule::table('cloud_traffic_usage_settlement_buckets')->insert(array_merge( $bucket, [