table('cloud_instances', static function (Blueprint $table): void { $table->text('spec_json')->default('{}'); }); $seedRecovery = static function (int $id, string $memberState = 'deleted') use ( $seedVpc, $seedInstance, $seedPair ): array { $octet = $id - 900; $vpc = $seedVpc($id, '10.98.' . $octet . '.0/24', '10.98.' . $octet . '.0/24', 10, 'deleted', 'active'); $instanceId = $id * 100; $memberId = $id * 1000; $privateId = $memberId + 1; $seedInstance($instanceId, 10); Capsule::table('cloud_instances')->where('id', $instanceId)->update([ 'spec_json' => json_encode(['region_id' => 1, 'region' => 'Test Region']), ]); $seedPair($memberId, $privateId, $vpc, $instanceId, 10, '10.98.' . $octet . '.1', $memberState); return $vpc + [ 'instance_id' => $instanceId, 'member_id' => $memberId, 'private_id' => $privateId, 'private_ip' => '10.98.' . $octet . '.1', ]; }; $attachment = static function (array $vpc) use ($actor, $assert): array { $rows = array_column(CloudVpcPageService::bootstrap($actor)['instances'], null, 'id'); $assert(isset($rows[$vpc['instance_id']]), 'Recovery instance is missing from the page bootstrap.'); return $rows[$vpc['instance_id']]['vpc_attachment']; }; $assertRetained = static function (array $vpc) use ($assert, $attachment): void { $assert(Capsule::table('cloud_vpc_members')->where('id', $vpc['member_id'])->count() === 1 && Capsule::table('cloud_vpc_private_ips')->where('id', $vpc['private_id'])->count() === 1 && $attachment($vpc)['state'] === 'cleanup_pending', 'An unproven or in-flight identity was released or became selectable.'); }; $recoverable = $seedRecovery(980); $beforeRoot = (array) Capsule::table('cloud_vpcs')->where('id', 980)->first(); $beforeOperation = (array) Capsule::table('cloud_vpc_operations') ->where('operation_id', $recoverable['operation_id'])->first(); CloudVpcService::show($recoverable['vpc_key'], $actor); $assertRetained($recoverable); foreach ([false, true] as $allowDeferredRetry) { $expectCloudError(static fn () => CloudVpcService::reconcile( $recoverable['vpc_key'], $actor, $allowDeferredRetry ), 409); $assertRetained($recoverable); } CloudVpcService::reconcile($recoverable['vpc_key'], $actor, true, true); $assert(Capsule::table('cloud_vpc_members')->where('id', $recoverable['member_id'])->count() === 0 && Capsule::table('cloud_vpc_private_ips')->where('id', $recoverable['private_id'])->count() === 0 && $attachment($recoverable) === ['state' => 'available', 'vpc_key' => null, 'vpc_name' => null] && (array) Capsule::table('cloud_vpcs')->where('id', 980)->first() === $beforeRoot && (array) Capsule::table('cloud_vpc_operations') ->where('operation_id', $recoverable['operation_id'])->first() === $beforeOperation && Capsule::table('cloud_vpc_operations')->where('vpc_id', 980)->count() === 1, 'Explicit proof-based recovery must release only the old pair and leave root/revision/operation audit unchanged.'); $expectCloudError(static fn () => CloudVpcService::reconcile( $recoverable['vpc_key'], $actor, true, true ), 409); // A later root marker does not erase exact historical absence evidence when // there is no operation to claim; the existing proof verifier remains owner. $advanced = $seedRecovery(981); Capsule::table('cloud_vpcs')->where('id', 981)->update([ 'desired_revision' => 2, 'desired_digest' => hash('sha256', 'recovery-newer-root'), 'state' => 'degraded', 'sync_state' => 'uncertain', 'last_operation_id' => $keyFor('recovery-newer-root', 981), ]); CloudVpcService::reconcile($advanced['vpc_key'], $actor, true, true); $assert($attachment($advanced)['state'] === 'available' && (int) Capsule::table('cloud_vpcs')->where('id', 981)->value('desired_revision') === 2 && Capsule::table('cloud_vpc_operations')->where('vpc_id', 981)->count() === 1, 'Recovery either lost valid historical proof or created a replacement operation.'); $missingProof = $seedRecovery(982); Capsule::table('cloud_vpc_operations')->where('operation_id', $missingProof['operation_id']) ->update(['observed_json' => null]); $deleting = $seedRecovery(983, 'deleting'); $referenced = $seedRecovery(984); $seedRoute(984000, $referenced, $referenced['member_id'], '10.99.0.1/32', $referenced['private_ip']); $rootDeleting = $seedRecovery(985); Capsule::table('cloud_vpcs')->where('id', 985)->update(['state' => 'deleting']); foreach ([$missingProof, $deleting, $referenced, $rootDeleting] as $blocked) { $expectCloudError(static fn () => CloudVpcService::reconcile( $blocked['vpc_key'], $actor, true, true ), 409); $assertRetained($blocked); $assert(Capsule::table('cloud_vpc_operations')->where('vpc_id', $blocked['id'])->count() === 1, 'A refused cleanup must not create an operation.'); } // A real pending target takes precedence even with a valid older removal // proof. Its live lease returns the ordinary document without GC or replay. $inFlight = $seedRecovery(986); $historical = (array) Capsule::table('cloud_vpc_operations') ->where('operation_id', $inFlight['operation_id'])->first(); $pendingDocument = json_decode($historical['payload_json'], true); unset($pendingDocument['operation_id'], $pendingDocument['desired_digest']); $pendingDocument['desired_revision'] = 2; $pendingDigest = hash('sha256', $canonicalJson($pendingDocument)); $pendingId = $keyFor('candidate-recovery-pending', 986); $pendingDocument['operation_id'] = $pendingId; $pendingDocument['desired_digest'] = $pendingDigest; $pending = $historical; unset($pending['id']); $pending = array_replace($pending, [ 'operation_id' => $pendingId, 'desired_revision' => 2, 'idempotency_key' => 'candidate-recovery-pending-986', 'request_hash' => hash('sha256', 'candidate-recovery-pending-986'), 'status' => 'running', 'payload_json' => json_encode($pendingDocument), 'observed_json' => null, 'finished_at' => null, 'lease_token' => str_repeat('a', 32), 'lease_until' => time() + 300, ]); Capsule::table('cloud_vpc_operations')->insert($pending); Capsule::table('cloud_vpcs')->where('id', 986)->update([ 'desired_revision' => 2, 'desired_digest' => $pendingDigest, 'state' => 'degraded', 'sync_state' => 'uncertain', 'last_operation_id' => $pendingId, ]); foreach (['queued', 'running', 'uncertain', 'waiting_observed'] as $status) { Capsule::table('cloud_vpc_operations')->where('operation_id', $pendingId)->update(['status' => $status]); CloudVpcService::reconcile($inFlight['vpc_key'], $actor, true, true); $assertRetained($inFlight); $assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $pendingId)->value('status') === $status && Capsule::table('cloud_vpc_operations')->where('vpc_id', 986)->count() === 2, 'A live operation was bypassed, replayed or replaced by candidate cleanup.'); } $assert(CloudEngineClient::$requests === 0, 'Candidate cleanup unexpectedly contacted the engine.'); echo "VPC member candidate recovery runtime passed.\n"; }