schema('default'); $schema->create('cloud_traffic_pool_groups', static function (Blueprint $t): void { $t->increments('id'); $t->string('mode'); }); $schema->create('cloud_traffic_pools', static function (Blueprint $t): void { $t->increments('id'); $t->integer('user_id'); $t->integer('traffic_pool_group_id'); $t->string('state'); $t->integer('available_bytes')->default(0); $t->integer('overage_bytes')->default(0); $t->integer('version')->default(1); $t->integer('last_sequence')->default(0); $t->unique(['user_id', 'traffic_pool_group_id'], 'test_pool_identity'); $t->integer('last_event_id')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_traffic_ledger_events', static function (Blueprint $t): void { $t->increments('id'); $t->integer('pool_id'); $t->integer('user_id'); $t->integer('pool_sequence'); $t->string('event_key')->unique(); $t->string('event_type'); $t->string('source_type'); $t->string('source_id'); $t->integer('source_revision'); $t->string('payload_hash'); $t->integer('instance_id')->nullable(); $t->integer('service_period_id')->nullable(); $t->integer('entitlement_id')->nullable(); $t->integer('amount_bytes'); $t->integer('effective_at'); $t->integer('observed_at')->nullable(); $t->text('payload_json'); $t->integer('created_at'); $t->unique(['pool_id', 'pool_sequence']); }); $schema->create('cloud_traffic_usage_settlement_buckets', static function (Blueprint $t): void { $t->increments('id'); $t->integer('pool_id'); $t->integer('user_id'); $t->integer('instance_id'); $t->integer('service_period_id'); $t->integer('entitlement_id')->nullable(); $t->string('allocation_kind'); $t->integer('allocation_target_id'); $t->integer('bucket_start'); $t->integer('seal_at'); $t->integer('bucket_revision'); $t->binary('event_identity', 16)->unique('recovery_bucket_identity'); $t->integer('amount_bytes'); $t->integer('download_bytes'); $t->integer('upload_bytes'); $t->integer('sample_count'); $t->integer('first_observed_at'); $t->integer('last_observed_at'); $t->string('state'); $t->integer('sealed_event_id')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_traffic_settlement_receipts', static function (Blueprint $t): void { $t->integer('pool_id'); $t->integer('instance_id'); $t->integer('service_period_id'); $t->string('allocation_kind'); $t->integer('allocation_target_id'); $t->integer('bucket_start'); $t->integer('max_revision'); }); $schema->create('cloud_traffic_instance_cursors', static function (Blueprint $t): void { $t->integer('instance_id')->primary(); $t->integer('user_id'); $t->integer('pool_id'); $t->integer('accounted_download_bytes'); $t->integer('accounted_upload_bytes'); $t->integer('last_observed_at'); $t->integer('last_event_id')->nullable(); $t->string('state'); $t->string('attention_code'); $t->integer('version'); $t->integer('updated_at'); }); $schema->create('cloud_instances', static function (Blueprint $t): void { $t->increments('id'); $t->integer('user_id'); $t->integer('current_period_id')->nullable(); $t->integer('lifetime_download_bytes')->default(0); $t->integer('lifetime_upload_bytes')->default(0); $t->integer('traffic_synced_at')->default(0); $t->string('status')->default('active'); $t->integer('created_at')->default(0); $t->integer('updated_at')->default(0); }); $schema->create('cloud_service_periods', static function (Blueprint $t): void { $t->increments('id'); $t->integer('instance_id'); $t->integer('user_id'); $t->integer('traffic_pool_group_id')->nullable(); $t->string('traffic_quota_mode')->default('finite'); $t->integer('traffic_quota_bytes')->default(0); $t->integer('starts_at'); $t->integer('ends_at'); $t->string('status'); $t->integer('updated_at')->default(0); }); $schema->create('cloud_traffic_entitlements', static function (Blueprint $t): void { $t->increments('id'); $t->integer('pool_id'); $t->integer('user_id'); $t->integer('instance_id'); $t->integer('service_period_id'); $t->integer('starts_at'); $t->integer('ends_at'); $t->string('quota_mode'); $t->integer('granted_bytes'); $t->integer('consumed_bytes')->default(0); $t->integer('forfeited_bytes')->default(0); $t->string('state'); $t->integer('activated_at')->nullable(); $t->integer('closed_at')->nullable(); $t->integer('version')->default(1); $t->integer('created_at')->default(0); $t->integer('updated_at')->default(0); }); $schema->create('cloud_traffic_scan_failures', static function (Blueprint $t): void { $t->increments('id'); $t->string('scan_kind'); $t->integer('subject_id'); $t->integer('attempt_count'); $t->integer('last_failed_at'); $t->integer('first_failed_at'); $t->integer('retry_at'); $t->string('last_error_code'); }); $now = time(); Capsule::table('cloud_traffic_pool_groups')->insert([ 'id' => 1, 'mode' => 'enforced', ]); Capsule::table('cloud_traffic_pools')->insert([ 'id' => 1, 'user_id' => 7, 'traffic_pool_group_id' => 1, 'state' => 'available', 'available_bytes' => 0, 'overage_bytes' => 10, 'version' => 1, 'last_sequence' => 0, 'last_event_id' => null, 'created_at' => $now - 100, 'updated_at' => $now - 100, ]); $identity = random_bytes(16); $bucket = [ 'id' => 1, 'pool_id' => 1, 'user_id' => 7, 'instance_id' => 11, 'service_period_id' => 101, 'entitlement_id' => null, 'allocation_kind' => 'overage', 'allocation_target_id' => 0, 'bucket_start' => $now - 86_500, 'seal_at' => $now - 100, 'bucket_revision' => 1, 'event_identity' => $identity, 'amount_bytes' => 10, 'download_bytes' => 7, 'upload_bytes' => 3, 'sample_count' => 1, 'first_observed_at' => $now - 86_490, 'last_observed_at' => $now - 86_490, 'state' => 'open', 'sealed_event_id' => null, 'created_at' => $now - 100, 'updated_at' => $now - 100, ]; Capsule::table('cloud_traffic_usage_settlement_buckets')->insert($bucket); $plan = CloudTrafficSettlementRecovery::inspect(); $assert($plan['actionable'] === true && $plan['counts']['due_buckets'] === 1 && preg_match('/^[a-f0-9]{64}$/D', $plan['plan_hash']) === 1, 'Inspect did not produce a bounded actionable settlement plan: ' . json_encode($plan, JSON_THROW_ON_ERROR)); Capsule::table('cloud_traffic_usage_settlement_buckets')->where('id', 1) ->update(['amount_bytes' => 11, 'download_bytes' => 8]); try { CloudTrafficSettlementRecovery::apply($plan['plan_hash']); throw new RuntimeException('A changed recovery plan was applied.'); } catch (RuntimeException $error) { $assert($error->getMessage() === 'traffic_settlement_recovery_plan_changed', 'Changed plan failed with the wrong safety code.'); } Capsule::table('cloud_traffic_usage_settlement_buckets')->where('id', 1) ->update(['amount_bytes' => 10, 'download_bytes' => 7]); $fresh = CloudTrafficSettlementRecovery::inspect(); $applied = CloudTrafficSettlementRecovery::apply($fresh['plan_hash']); $assert($applied['sealed_buckets'] === 1 && Capsule::table('cloud_traffic_usage_settlement_buckets') ->where('id', 1)->value('state') === 'sealed' && Capsule::table('cloud_traffic_ledger_events') ->where('event_key', 'traffic-settlement:v2:' . bin2hex($identity)) ->count() === 1, 'The exact approved plan did not use the normal immutable seal path.'); $empty = CloudTrafficSettlementRecovery::inspect(); $repeat = CloudTrafficSettlementRecovery::apply($empty['plan_hash']); $assert($repeat['sealed_buckets'] === 0 && $repeat['reconciled_periods'] === 0 && $repeat['settled_instances'] === 0, 'A repeated empty recovery plan was not idempotent.'); // Health reads a bounded, coherent bucket/event view even as another connection seals. $capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'log'); $capsule->getConnection('log')->getSchemaBuilder()->create('cloud_vm_traffic_checkpoints', static function (Blueprint $t): void { $t->integer('instance_id'); $t->integer('first_observed_at'); }); $probe = static fn (): array => App\Services\Cloud\CloudTrafficStorageHealthService::snapshot($now)['metrics']; $bucketRow = Capsule::table('cloud_traffic_usage_settlement_buckets')->where('id', 1); $eventRow = Capsule::table('cloud_traffic_ledger_events')->where('id', 1); $event = (array) $eventRow->first(); $bucketRow->update(['state' => 'open', 'sealed_event_id' => null]); $assert($probe()['settlement_identity_conflicts'] === 0, 'Health rejected an exact immutable event.'); $eventRow->update(['amount_bytes' => 99]); $assert($probe()['settlement_identity_conflicts'] === 1, 'Health accepted a mismatched immutable event.'); $eventRow->delete(); $raceSeal = true; $capsule->getConnection()->beforeExecuting(static function ($sql) use ($event, $capsule, &$raceSeal): void { if ($raceSeal && str_starts_with(strtolower($sql), 'select * from') && str_contains($sql, 'cloud_traffic_ledger_events') && str_contains($sql, 'event_key')) { $raceSeal = false; $peer = $capsule->getConnection('peer'); $peer->transaction(static function () use ($peer, $event): void { $peer->table('cloud_traffic_ledger_events')->insert($event); $peer->table('cloud_traffic_usage_settlement_buckets')->where('id', 1) ->update(['state' => 'sealed', 'sealed_event_id' => 1]); }); } }); $assert($probe()['settlement_identity_conflicts'] === 0 && !$raceSeal && $probe()['open_buckets_due'] === 0, 'A concurrent valid seal produced an identity alarm.'); // Commit a peer update after plan validation but before the second approved action. foreach ([3, 4] as $id) { Capsule::table('cloud_traffic_usage_settlement_buckets')->insert(array_replace($bucket, [ 'id' => $id, 'event_identity' => random_bytes(16), ])); } $racePlan = CloudTrafficSettlementRecovery::inspect(); $countdown = 2; $capsule->getConnection()->beforeStartingTransaction(static function () use (&$countdown, $capsule): void { if (--$countdown === 0) { $capsule->getConnection('peer')->table('cloud_traffic_usage_settlement_buckets') ->where('id', 4)->update(['amount_bytes' => 90, 'download_bytes' => 87]); } }); $partial = CloudTrafficSettlementRecovery::apply($racePlan['plan_hash']); $assert($partial['status'] === 'stopped' && $partial['completed']['due_buckets'] === [3] && $partial['remaining']['due_buckets'] === [4] && $partial['failed']['reason'] === 'traffic_settlement_recovery_plan_changed' && Capsule::table('cloud_traffic_ledger_events')->count() === 2, 'A 10-byte approval consumed a concurrent 90-byte bucket or lost partial progress.'); $fresh = CloudTrafficSettlementRecovery::inspect(); $assert(CloudTrafficSettlementRecovery::apply($fresh['plan_hash'])['sealed_buckets'] === 1, 'A freshly reviewed plan did not resume from the remaining bucket.'); Capsule::table('cloud_instances')->insert(['id' => 11, 'user_id' => 7, 'traffic_synced_at' => $now, 'lifetime_download_bytes' => 10]); Capsule::table('cloud_service_periods')->insert(['id' => 101, 'instance_id' => 11, 'user_id' => 7, 'traffic_pool_group_id' => 1, 'status' => 'active', 'traffic_quota_bytes' => 1000, 'starts_at' => $now - 100, 'ends_at' => $now + 10000]); Capsule::table('cloud_traffic_entitlements')->insert(['id' => 41, 'pool_id' => 1, 'instance_id' => 11, 'user_id' => 7, 'service_period_id' => 101, 'quota_mode' => 'finite', 'granted_bytes' => 1000, 'state' => 'active', 'starts_at' => $now - 100, 'ends_at' => $now + 10000]); Capsule::table('cloud_traffic_instance_cursors')->insert(['instance_id' => 11, 'user_id' => 7, 'pool_id' => 1, 'accounted_download_bytes' => 0, 'accounted_upload_bytes' => 0, 'last_observed_at' => $now - 60, 'state' => 'ready', 'attention_code' => '', 'version' => 1, 'updated_at' => $now]); $plan = CloudTrafficSettlementRecovery::inspect(); $changeSample = true; $capsule->getConnection()->beforeStartingTransaction(static function () use (&$changeSample, $capsule): void { if ($changeSample) { $changeSample = false; $capsule->getConnection('peer')->table('cloud_instances')->where('id', 11)->update(['lifetime_download_bytes' => 90]); } }); $stopped = CloudTrafficSettlementRecovery::apply($plan['plan_hash']); $assert($stopped['failed']['kind'] === 'unsettled_instances' && $stopped['failed']['reason'] === 'traffic_settlement_recovery_plan_changed' && Capsule::table('cloud_traffic_instance_cursors')->value('accounted_download_bytes') == 0, 'The recovery writer advanced to an unapproved sample.'); Capsule::table('cloud_traffic_instance_cursors')->update(['accounted_download_bytes' => 90, 'last_observed_at' => $now]); Capsule::table('cloud_service_periods')->where('id', 101)->update(['traffic_quota_bytes' => 1100]); $plan = CloudTrafficSettlementRecovery::inspect(); $changeQuota = true; $capsule->getConnection()->beforeStartingTransaction(static function () use (&$changeQuota, $capsule): void { if ($changeQuota) { $changeQuota = false; $capsule->getConnection('peer')->table('cloud_service_periods')->where('id', 101)->update(['traffic_quota_bytes' => 2000]); } }); $stopped = CloudTrafficSettlementRecovery::apply($plan['plan_hash']); $assert($stopped['failed']['kind'] === 'unsettled_periods' && $stopped['failed']['reason'] === 'traffic_settlement_recovery_plan_changed' && Capsule::table('cloud_traffic_entitlements')->value('granted_bytes') == 1000, 'The recovery writer granted an unapproved quota.'); Capsule::table('cloud_service_periods')->where('id', 101)->update(['traffic_quota_bytes' => 1000]); $failure = ['id' => 1, 'scan_kind' => 'instance', 'subject_id' => 11, 'attempt_count' => 1, 'last_failed_at' => $now, 'first_failed_at' => $now, 'retry_at' => $now + 30, 'last_error_code' => 'Test']; Capsule::table('cloud_traffic_scan_failures')->insert($failure); $cleanup = CloudTrafficSettlementRecovery::inspect(); $assert($cleanup['counts']['scan_failures'] === 1 && !$cleanup['counts']['unsettled_instances'], 'Already settled failure has no manual recovery path.'); $race = true; $capsule->getConnection()->beforeExecuting(static function ($sql) use (&$race, $capsule): void { if ($race && str_starts_with(strtolower($sql), 'delete from')) { $race = false; $capsule->getConnection('peer')->table('cloud_traffic_scan_failures')->where('id', 1) ->update(['attempt_count' => 2]); } }); $stopped = CloudTrafficSettlementRecovery::apply($cleanup['plan_hash']); $assert($stopped['failed']['kind'] === 'scan_failures' && Capsule::table('cloud_traffic_scan_failures')->count() === 1, 'Cleanup erased a newer failure generation.'); $cleanup = CloudTrafficSettlementRecovery::inspect(); $assert(CloudTrafficSettlementRecovery::apply($cleanup['plan_hash'])['cleared_scan_failures'] === 1, 'Reviewed settled failure was left behind.'); Capsule::table('cloud_traffic_scan_failures')->insert($failure); Capsule::table('cloud_traffic_instance_cursors')->update(['state' => 'attention']); $assert(CloudTrafficSettlementRecovery::inspect()['counts']['scan_failures'] === 0 && !App\Services\Cloud\CloudTrafficScanRecovery::clearIfResolved($failure, $now), 'An attention cursor was treated as successfully recovered.'); Capsule::table('cloud_traffic_instance_cursors')->update(['state' => 'ready']); $cleanup = CloudTrafficSettlementRecovery::inspect(); $changeOnCleanup = true; $capsule->getConnection()->beforeExecuting(static function ($sql) use (&$changeOnCleanup, $capsule): void { if ($changeOnCleanup && str_starts_with(strtolower($sql), 'delete from')) { $changeOnCleanup = false; $capsule->getConnection('peer')->table('cloud_instances')->where('id', 11) ->update(['lifetime_download_bytes' => 100]); } }); $result = CloudTrafficSettlementRecovery::apply($cleanup['plan_hash']); $assert(!$changeOnCleanup && $result['status'] === 'stopped' && Capsule::table('cloud_traffic_scan_failures')->count() === 1, 'Cleanup erased a failure while a concurrently committed sample was unsettled.'); Capsule::table('cloud_traffic_pools')->where('id', 1)->update(['overage_bytes' => 0]); $plan = CloudTrafficSettlementRecovery::inspect(); $expected = 100; if ($capsule->getConnection()->getDriverName() === 'mysql') { $changeDuringWrite = true; $capsule->getConnection()->beforeExecuting(static function ($sql) use (&$changeDuringWrite, $capsule): void { if ($changeDuringWrite && str_starts_with(strtolower($sql), 'update `cloud_traffic_instance_cursors`')) { $changeDuringWrite = false; $capsule->getConnection('peer')->table('cloud_instances')->where('id', 11) ->update(['lifetime_download_bytes' => 105]); } }); $stopped = CloudTrafficSettlementRecovery::apply($plan['plan_hash']); $assert(!$changeDuringWrite && $stopped['failed']['reason'] === 'traffic_settlement_recovery_plan_changed' && (int) Capsule::table('cloud_traffic_entitlements')->value('consumed_bytes') === 0 && (int) Capsule::table('cloud_traffic_instance_cursors')->value('accounted_download_bytes') === 90, 'A changed sample did not roll back the entire manual accounting write.'); $plan = CloudTrafficSettlementRecovery::inspect(); $expected = 105; } $result = CloudTrafficSettlementRecovery::apply($plan['plan_hash']); $assert($result['status'] === 'applied' && $result['settled_instances'] === 1 && $result['cleared_scan_failures'] === 1 && (int) Capsule::table('cloud_traffic_entitlements')->value('consumed_bytes') === $expected - 90 && (int) Capsule::table('cloud_traffic_instance_cursors')->value('accounted_download_bytes') === $expected && (int) Capsule::table('cloud_traffic_usage_settlement_buckets')->where('entitlement_id', 41)->sum('amount_bytes') === $expected - 90, 'The reviewed sample was not booked exactly once with its failure retired: ' . json_encode($result)); $empty = CloudTrafficSettlementRecovery::inspect(); $assert(CloudTrafficSettlementRecovery::apply($empty['plan_hash'])['settled_instances'] === 0, 'A successfully booked sample was selected again.'); Capsule::table('cloud_instances')->where('id', 11)->update(['lifetime_download_bytes' => $expected - 1]); $regression = CloudTrafficSettlementRecovery::inspect(); $result = CloudTrafficSettlementRecovery::apply($regression['plan_hash']); $assert($result['status'] === 'stopped' && $result['failed']['reason'] === 'traffic_counter_regression' && $result['completed']['unsettled_instances'] === [] && $result['remaining']['unsettled_instances'] === [11] && Capsule::table('cloud_traffic_instance_cursors')->value('state') === 'attention' && (int) Capsule::table('cloud_traffic_entitlements')->value('consumed_bytes') === $expected - 90, 'A regressed counter was reported as settled or lost its durable attention marker.'); Capsule::table('cloud_traffic_usage_settlement_buckets')->insert(array_merge( $bucket, [ 'id' => 2, 'event_identity' => '', 'state' => 'open', 'sealed_event_id' => null, ] )); $blocked = CloudTrafficSettlementRecovery::inspect(); $assert($blocked['actionable'] === false && $blocked['blockers'][0]['code'] === 'settlement_event_identity_missing', 'A malformed persistent event identity did not block recovery.'); try { CloudTrafficSettlementRecovery::apply($blocked['plan_hash']); throw new RuntimeException('A blocked recovery plan was applied.'); } catch (RuntimeException $error) { $assert($error->getMessage() === 'traffic_settlement_recovery_blocked', 'Blocked plan failed with the wrong safety code.'); } $parse = new ReflectionMethod( CloudTrafficSettlementRecoveryCommand::class, 'parse' ); $assert($parse->invoke(null, [ 'xcat', 'cloudTrafficSettlementRecovery', 'inspect', ]) === ['action' => 'inspect'], 'CLI inspect arguments drifted.'); $hash = str_repeat('a', 64); $assert($parse->invoke(null, [ 'xcat', 'cloudTrafficSettlementRecovery', 'apply', '--plan-hash=' . $hash, '--apply', ]) === ['action' => 'apply', 'plan_hash' => $hash], 'CLI apply did not bind the exact reviewed plan hash.'); foreach ([ ['apply', '--plan-hash=' . $hash], ['apply', '--plan-hash=bad', '--apply'], ['apply', '--plan-hash=' . $hash, '--apply', '--force'], ['inspect', '--apply'], ] as $arguments) { try { $parse->invoke(null, array_merge([ 'xcat', 'cloudTrafficSettlementRecovery', ], $arguments)); throw new RuntimeException('Unsafe CLI arguments were accepted.'); } catch (ReflectionException $error) { throw $error; } catch (Throwable $error) { $actual = $error->getPrevious() ?? $error; $assert($actual instanceof InvalidArgumentException, 'Unsafe CLI arguments failed with the wrong exception.'); } } $service = file_get_contents(dirname(__DIR__) . '/app/Services/Cloud/CloudTrafficSettlementRecovery.php'); $xcat = file_get_contents(dirname(__DIR__) . '/app/Command/XCat.php'); $assert(is_string($service) && is_string($xcat) && !str_contains($service, '->delete(') && !str_contains($service, 'CloudAgentClient') && !str_contains($service, 'CloudJob') && str_contains($service, 'CloudTrafficPoolAccounting::sealPlannedUsageBuckets(') && str_contains($service, 'CloudTrafficPoolAccounting::reconcileServicePeriod(') && str_contains($service, 'CloudTrafficPoolAccounting::settleInstanceUsage(') && str_contains($xcat, "'cloudTrafficSettlementRecovery'") && str_contains($xcat, 'cloudTrafficSettlementRecovery apply --plan-hash=HASH --apply'), 'Recovery acquired a destructive or remote bypass path.'); echo "Traffic settlement recovery plan/apply checks passed.\n";