addConnection([ 'driver' => 'sqlite', 'database' => ':memory:', ], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('cloud_hosts', static function (Blueprint $table): void { $table->increments('id'); }); $schema->create('cloud_resource_pools', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('line_id'); }); $schema->create('cloud_host_network_attachments', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('host_id'); $table->integer('line_id'); $table->string('network_selector'); }); $schema->create('cloud_public_ip_pools', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('pool_key'); $table->integer('host_id'); $table->integer('network_attachment_id'); $table->integer('address_family'); $table->string('cidr'); $table->string('gateway'); $table->boolean('gateway_on_link'); $table->integer('guest_prefix_length'); $table->text('dns_servers_json'); $table->text('reserved_ips_json'); $table->string('allocation_start')->nullable(); $table->string('allocation_end')->nullable(); $table->string('allocation_cursor')->nullable(); $table->integer('max_allocations'); }); $schema->create('cloud_public_ips', static function (Blueprint $table): void { $table->increments('id'); $table->integer('host_id'); $table->integer('public_ip_pool_id'); $table->integer('address_family'); $table->string('address'); $table->binary('address_binary'); $table->integer('prefix_length'); $table->string('state'); $table->integer('reservation_id')->nullable(); $table->integer('reservation_ip_claim_id')->nullable(); $table->integer('instance_id')->nullable(); $table->string('operation_id')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_resource_reservation_ips', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('reservation_id'); $table->integer('address_family'); $table->integer('claim_generation')->default(1); $table->integer('active_claim')->nullable()->default(1); $table->integer('network_attachment_id'); $table->integer('public_ip_pool_id'); $table->integer('public_ip_id'); $table->string('operation_id'); $table->string('state'); $table->integer('released_at')->nullable(); $table->string('release_reason')->default(''); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_resource_reservations', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('reservation_key'); $table->string('operation_id'); $table->integer('order_id')->nullable(); $table->integer('backup_restore_id')->nullable(); $table->string('backup_restore_attempt_key')->nullable(); $table->integer('migration_source_debt_id')->nullable(); $table->integer('offer_id')->nullable(); $table->integer('resource_pool_id'); $table->integer('host_id'); $table->integer('user_id'); $table->integer('instance_id')->nullable(); $table->integer('network_attachment_id')->nullable(); $table->integer('public_ip_pool_id')->nullable(); $table->integer('public_ip_id')->nullable(); $table->integer('cpu_cores'); $table->integer('memory_mib'); $table->integer('disk_gib'); $table->string('state'); $table->integer('expires_at')->nullable(); $table->integer('released_at')->nullable(); $table->string('release_reason')->default(''); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_instances', static function (Blueprint $table): void { $table->increments('id'); $table->string('uuid'); $table->integer('host_id'); $table->integer('user_id'); $table->integer('order_id'); $table->integer('plan_id'); $table->integer('offer_id'); $table->integer('resource_pool_id'); $table->integer('reservation_id'); $table->integer('current_period_id'); $table->integer('expires_at'); $table->decimal('renewal_price', 12, 2); $table->integer('renewal_days'); $table->integer('renewal_traffic_bytes'); $table->integer('traffic_quota_bytes'); $table->integer('lifetime_download_bytes'); $table->integer('lifetime_upload_bytes'); $table->integer('traffic_synced_at')->default(0); $table->string('name'); $table->string('status'); $table->string('service_status'); $table->string('network_status'); $table->text('spec_json'); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_orders', static function (Blueprint $table): void { $table->increments('id'); $table->integer('user_id'); $table->integer('plan_id'); $table->integer('offer_id'); $table->integer('resource_pool_id'); $table->integer('host_id'); $table->integer('reservation_id'); $table->decimal('amount', 12, 2); $table->integer('instance_id'); $table->string('status'); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_backup_restores', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('target_host_id'); $table->integer('reservation_id')->nullable(); $table->integer('job_id')->nullable(); $table->string('operation_id'); $table->string('state'); $table->string('error_code')->default(''); $table->integer('ready_at')->nullable(); $table->integer('updated_at'); }); $schema->create('cloud_instance_migrations', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('migration_key'); $table->integer('instance_id'); $table->integer('restore_id'); $table->integer('source_host_id'); $table->string('source_uuid'); $table->integer('source_reservation_id'); $table->integer('target_host_id'); $table->string('target_uuid'); $table->integer('target_reservation_id'); $table->integer('vpc_port_id')->nullable(); $table->integer('source_prepare_job_id')->nullable(); $table->integer('target_receive_job_id')->nullable(); $table->integer('firewall_job_id')->nullable(); $table->integer('activate_job_id')->nullable(); $table->integer('rollback_job_id')->nullable(); $table->integer('cleanup_job_id')->nullable(); $table->string('cleanup_mode'); $table->string('state'); $table->string('recovery_action')->default(''); $table->integer('retry_generation')->default(0); $table->string('manifest_digest')->default(''); $table->boolean('source_was_running')->nullable(); $table->boolean('source_autostart_enabled')->nullable(); $table->string('source_service_status'); $table->string('source_network_status'); $table->string('error_code')->default(''); $table->integer('cutover_at')->nullable(); $table->integer('completed_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_peer_migration_artifacts', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('migration_id'); $table->string('artifact_id'); $table->string('role'); $table->integer('source_snapshot_id')->nullable(); $table->string('snapshot_key')->nullable(); $table->string('name')->default(''); $table->string('consistency'); $table->integer('object_size_bytes'); $table->integer('virtual_size_bytes'); $table->string('sha256'); $table->string('state'); $table->integer('snapshot_created_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_instance_runtime', static function ( Blueprint $table ): void { $table->integer('instance_id')->primary(); $table->integer('host_id'); $table->string('uuid'); $table->string('power_state'); $table->string('sync_state'); $table->integer('last_success_at'); $table->integer('last_attempt_at'); $table->integer('revision'); $table->string('last_error_code'); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_instance_migration_source_debts', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('debt_key'); $table->integer('migration_id'); $table->integer('source_reservation_id'); $table->integer('source_host_id'); $table->string('source_uuid'); $table->integer('source_network_attachment_id'); $table->text('source_identity_json'); $table->string('source_identity_sha256'); $table->string('source_claim_set_sha256'); $table->text('source_public_network_json'); $table->string('source_public_network_sha256'); $table->string('cleanup_operation_id')->nullable(); $table->integer('cleanup_job_id')->nullable(); $table->string('state'); $table->integer('active_debt')->nullable(); $table->string('last_error_code')->default(''); $table->integer('resolved_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $now = time(); Capsule::table('cloud_hosts')->insert([['id' => 1], ['id' => 2]]); Capsule::table('cloud_resource_pools')->insert(['id' => 10, 'line_id' => 7]); Capsule::table('cloud_host_network_attachments')->insert([ ['id' => 11, 'host_id' => 1, 'line_id' => 7, 'network_selector' => 'bridge:br0'], ['id' => 12, 'host_id' => 2, 'line_id' => 7, 'network_selector' => 'bridge:br0'], ]); $pool = static function (int $id, int $hostId, int $attachmentId, int $family, string $cidr, string $gateway): array { $network = explode('/', $cidr, 2)[0]; return [ 'id' => $id, 'pool_key' => md5('pool-' . $id), 'host_id' => $hostId, 'network_attachment_id' => $attachmentId, 'address_family' => $family, 'cidr' => $cidr, 'gateway' => $gateway, 'gateway_on_link' => 0, 'guest_prefix_length' => $family === 4 ? 24 : 64, 'dns_servers_json' => '["1.1.1.1"]', 'reserved_ips_json' => '[]', 'allocation_start' => $family === 6 ? $network . '100' : null, 'allocation_end' => $family === 6 ? $network . '1ff' : null, 'allocation_cursor' => null, 'max_allocations' => $family === 6 ? 100 : 0, ]; }; Capsule::table('cloud_public_ip_pools')->insert([ $pool(101, 1, 11, 4, '8.8.10.0/24', '8.8.10.1'), $pool(201, 1, 11, 6, '2404:c140:156:10::/64', '2404:c140:156:10::1'), $pool(102, 2, 12, 4, '8.8.20.0/24', '8.8.20.1'), $pool(202, 2, 12, 6, '2404:c140:156:20::/64', '2404:c140:156:20::1'), ]); $seed = static function (int $n) use ($now): array { $instanceId = 100 + $n; $orderId = 200 + $n; $restoreId = 300 + $n; $migrationId = 400 + $n; $sourceReservationId = 500 + ($n * 2); $targetReservationId = $sourceReservationId + 1; $sourceOperation = md5('source-' . $n); $targetOperation = md5('target-' . $n); $sourceUuid = sprintf( '11111111-1111-4111-8111-%012d', $instanceId ); $targetUuid = sprintf( '22222222-2222-4222-8222-%012d', $instanceId ); $sourceAddresses = [ 4 => '8.8.10.' . (20 + $n), 6 => '2404:c140:156:10::1' . dechex(20 + $n), ]; $targetAddresses = [ 4 => '8.8.20.' . (20 + $n), 6 => '2404:c140:156:20::1' . dechex(20 + $n), ]; $sourceNetwork = [ 'contract' => 1, 'network_selector' => 'bridge:br0', 'ipv4' => [ 'pool_key' => md5('pool-101'), 'pool_cidr' => '8.8.10.0/24', 'address_cidr' => $sourceAddresses[4] . '/24', 'gateway' => '8.8.10.1', 'gateway_on_link' => false, ], 'ipv6' => [ 'pool_key' => md5('pool-201'), 'pool_cidr' => '2404:c140:156:10::/64', 'address_cidr' => $sourceAddresses[6] . '/64', 'gateway' => '2404:c140:156:10::1', 'gateway_on_link' => false, ], 'dns_servers' => ['1.1.1.1'], ]; Capsule::table('cloud_instances')->insert([ 'id' => $instanceId, 'uuid' => $sourceUuid, 'host_id' => 1, 'user_id' => 9, 'order_id' => $orderId, 'plan_id' => 31, 'offer_id' => 41, 'resource_pool_id' => 10, 'reservation_id' => $sourceReservationId, 'current_period_id' => 700 + $n, 'expires_at' => $now + 86400, 'renewal_price' => '19.95', 'renewal_days' => 30, 'renewal_traffic_bytes' => 2000000, 'traffic_quota_bytes' => 1500000, 'lifetime_download_bytes' => 800000, 'lifetime_upload_bytes' => 300000, 'name' => 'vm-' . $n, 'status' => 'active', 'service_status' => 'active', 'network_status' => 'active', 'spec_json' => json_encode([ 'managed' => true, 'vcpu' => 2, 'cpu_cores' => 2, 'memory_mib' => 2048, 'ram_mib' => 2048, 'disk_gib' => 40, 'bandwidth_mbps' => 1000, 'public_network' => $sourceNetwork, ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES), 'created_at' => $now - 1000, 'updated_at' => $now, ]); Capsule::table('cloud_orders')->insert([ 'id' => $orderId, 'user_id' => 9, 'plan_id' => 31, 'offer_id' => 41, 'resource_pool_id' => 10, 'host_id' => 1, 'reservation_id' => $sourceReservationId, 'amount' => '12.34', 'instance_id' => $instanceId, 'status' => 'active', 'created_at' => $now - 1000, 'updated_at' => $now, ]); Capsule::table('cloud_backup_restores')->insert([ 'id' => $restoreId, 'target_host_id' => 2, 'reservation_id' => $targetReservationId, 'job_id' => null, 'operation_id' => $targetOperation, 'state' => 'restoring', 'error_code' => '', 'updated_at' => $now, ]); Capsule::table('cloud_resource_reservations')->insert([ [ 'id' => $sourceReservationId, 'reservation_key' => md5('source-reservation-' . $n), 'operation_id' => $sourceOperation, 'order_id' => $orderId, 'backup_restore_id' => null, 'backup_restore_attempt_key' => null, 'migration_source_debt_id' => null, 'offer_id' => 41, 'resource_pool_id' => 10, 'host_id' => 1, 'user_id' => 9, 'instance_id' => $instanceId, 'network_attachment_id' => null, 'public_ip_pool_id' => null, 'public_ip_id' => null, 'cpu_cores' => 2, 'memory_mib' => 2048, 'disk_gib' => 40, 'state' => 'committed', 'expires_at' => null, 'released_at' => null, 'release_reason' => '', 'created_at' => $now - 1000, 'updated_at' => $now, ], [ 'id' => $targetReservationId, 'reservation_key' => md5('target-reservation-' . $n), 'operation_id' => $targetOperation, 'order_id' => null, 'backup_restore_id' => $restoreId, 'backup_restore_attempt_key' => null, 'migration_source_debt_id' => null, 'offer_id' => null, 'resource_pool_id' => 10, 'host_id' => 2, 'user_id' => 9, 'instance_id' => null, 'network_attachment_id' => null, 'public_ip_pool_id' => null, 'public_ip_id' => null, 'cpu_cores' => 2, 'memory_mib' => 2048, 'disk_gib' => 40, 'state' => 'held', 'expires_at' => $now + 600, 'released_at' => null, 'release_reason' => '', 'created_at' => $now - 100, 'updated_at' => $now, ], ]); foreach ([ [$sourceReservationId, 1, 11, 101, 201, $sourceAddresses, $sourceOperation, 'committed', $instanceId], [$targetReservationId, 2, 12, 102, 202, $targetAddresses, $targetOperation, 'reserved', null], ] as [$reservationId, $hostId, $attachmentId, $v4Pool, $v6Pool, $addresses, $operation, $state, $ownerInstanceId]) { foreach ([4 => $v4Pool, 6 => $v6Pool] as $family => $poolId) { $address = $addresses[$family]; $ipId = Capsule::table('cloud_public_ips')->insertGetId([ 'host_id' => $hostId, 'public_ip_pool_id' => $poolId, 'address_family' => $family, 'address' => $address, 'address_binary' => inet_pton($address), 'prefix_length' => $family === 4 ? 24 : 64, 'state' => $state === 'committed' ? 'assigned' : 'reserved', 'reservation_id' => $reservationId, 'reservation_ip_claim_id' => null, 'instance_id' => $ownerInstanceId, 'operation_id' => $operation, 'created_at' => $now, 'updated_at' => $now, ]); $claimId = Capsule::table('cloud_resource_reservation_ips') ->insertGetId([ 'reservation_id' => $reservationId, 'address_family' => $family, 'claim_generation' => 1, 'active_claim' => 1, 'network_attachment_id' => $attachmentId, 'public_ip_pool_id' => $poolId, 'public_ip_id' => $ipId, 'operation_id' => $operation, 'state' => $state, 'released_at' => null, 'release_reason' => '', 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_public_ips')->where('id', $ipId)->update([ 'reservation_ip_claim_id' => $claimId, ]); } } Capsule::table('cloud_instance_migrations')->insert([ 'id' => $migrationId, 'migration_key' => md5('migration-' . $n), 'instance_id' => $instanceId, 'restore_id' => $restoreId, 'source_host_id' => 1, 'source_uuid' => $sourceUuid, 'source_reservation_id' => $sourceReservationId, 'target_host_id' => 2, 'target_uuid' => $targetUuid, 'target_reservation_id' => $targetReservationId, 'cleanup_job_id' => null, 'cleanup_mode' => 'delete', 'state' => 'transferring', 'recovery_action' => '', 'retry_generation' => 0, 'source_service_status' => 'active', 'source_network_status' => 'active', 'error_code' => '', 'created_at' => $now - 100, 'updated_at' => $now, ]); $sourceSet = CloudPublicNetworkAllocationService::currentClaimSet( $sourceReservationId, 'committed', $instanceId ); $targetSet = CloudPublicNetworkAllocationService::currentClaimSet( $targetReservationId, 'held' ); Capsule::table('cloud_instances')->where('id', $instanceId)->update([ 'spec_json' => json_encode([ 'managed' => true, 'vcpu' => 2, 'cpu_cores' => 2, 'memory_mib' => 2048, 'ram_mib' => 2048, 'disk_gib' => 40, 'bandwidth_mbps' => 1000, 'public_network' => $sourceSet->publicNetwork()->toArray(), ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES), ]); return compact( 'instanceId', 'orderId', 'restoreId', 'migrationId', 'sourceReservationId', 'targetReservationId', 'sourceOperation', 'targetOperation', 'sourceUuid', 'targetUuid', 'sourceSet', 'targetSet' ); }; $cutover = static function (array $case): CloudInstanceMigrationSourceDebt { $handoff = CloudPublicNetworkAllocationService::handoffMigrationBundles( $case['sourceReservationId'], $case['targetReservationId'], $case['instanceId'], $case['sourceOperation'], $case['targetOperation'] ); if (!hash_equals( $case['sourceSet']->digest(), $handoff['source']->digest() ) || !hash_equals( $case['targetSet']->digest(), $handoff['target']->digest() )) { throw new RuntimeException('Migration handoff changed claim identity.'); } Capsule::table('cloud_instances')->where('id', $case['instanceId']) ->update([ 'uuid' => $case['targetUuid'], 'host_id' => 2, 'reservation_id' => $case['targetReservationId'], 'spec_json' => json_encode([ 'managed' => true, 'public_network' => $handoff['target']->publicNetwork()->toArray(), ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES), 'updated_at' => time(), ]); $migration = CloudInstanceMigration::findOrFail($case['migrationId']); return CloudInstanceMigrationSettlement::settleCutoverLocked( $migration, $handoff['source'] ); }; $claimIdentity = static function (int $reservationId): array { return array_map( static fn (object $row): array => [ (int) $row->id, (int) $row->reservation_id, (int) $row->public_ip_id, (string) $row->operation_id, (int) $row->claim_generation, ], Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', $reservationId) ->orderBy('address_family')->orderBy('claim_generation') ->orderBy('id')->get()->all() ); }; $job = static function (array $case, int $id, string $operation): CloudJob { $job = new CloudJob(); $job->id = $id; $job->kind = 'instance_migration_cleanup'; $job->instance_id = $case['instanceId']; $job->host_id = 1; $job->backup_restore_id = $case['restoreId']; $job->operation_id = $operation; return $job; }; $migrationState = static function (array $case): array { $rows = static function ($query): array { return $query->orderBy('id')->get() ->map(static fn ($row): array => (array) $row)->all(); }; return [ 'instance' => (array) Capsule::table('cloud_instances') ->find($case['instanceId']), 'order' => (array) Capsule::table('cloud_orders') ->find($case['orderId']), 'restore' => (array) Capsule::table('cloud_backup_restores') ->find($case['restoreId']), 'migration' => (array) Capsule::table('cloud_instance_migrations') ->find($case['migrationId']), 'reservations' => $rows(Capsule::table('cloud_resource_reservations') ->whereIn('id', [ $case['sourceReservationId'], $case['targetReservationId'], ])), 'claims' => $rows(Capsule::table('cloud_resource_reservation_ips') ->whereIn('reservation_id', [ $case['sourceReservationId'], $case['targetReservationId'], ])), 'ips' => $rows(Capsule::table('cloud_public_ips') ->whereIn('reservation_id', [ $case['sourceReservationId'], $case['targetReservationId'], ])), 'debts' => $rows(Capsule::table( 'cloud_instance_migration_source_debts' )->where('migration_id', $case['migrationId'])), 'artifacts' => $rows(Capsule::table('cloud_peer_migration_artifacts') ->where('migration_id', $case['migrationId'])), 'runtime' => $rows(Capsule::table('cloud_instance_runtime') ->where('instance_id', $case['instanceId'])), ]; }; // Exercise the real private transfer finalizer under the same outer // transaction supplied by the worker. A deterministic failure at the runtime // cache write happens after network handoff, artifact sealing, placement // switch, commercial settlement, and restore finalization; every one of those // writes must roll back together. $atomic = $seed(5); $atomicArtifactId = md5('migration-root-' . $atomic['migrationId']); $atomicArtifactSha = hash('sha256', 'migration-root-bytes'); $atomicManifest = hash('sha256', 'migration-manifest'); $atomicJobId = 9105; Capsule::table('cloud_peer_migration_artifacts')->insert([ 'migration_id' => $atomic['migrationId'], 'artifact_id' => $atomicArtifactId, 'role' => 'root', 'source_snapshot_id' => null, 'snapshot_key' => null, 'name' => '', 'consistency' => 'crash', 'object_size_bytes' => 123456, 'virtual_size_bytes' => 40 * 1073741824, 'sha256' => $atomicArtifactSha, 'state' => 'ready', 'snapshot_created_at' => null, 'created_at' => $now - 20, 'updated_at' => $now, ]); Capsule::table('cloud_instance_migrations') ->where('id', $atomic['migrationId'])->update([ 'target_receive_job_id' => $atomicJobId, 'manifest_digest' => $atomicManifest, 'source_was_running' => 1, 'source_autostart_enabled' => 1, ]); $atomicJob = new CloudJob(); $atomicJob->id = $atomicJobId; $atomicJob->instance_id = $atomic['instanceId']; $atomicJob->backup_restore_id = $atomic['restoreId']; $atomicJob->host_id = 2; $atomicJob->user_id = 9; $atomicJob->operation_id = $atomic['targetOperation']; $atomicTargetNetwork = $atomic['targetSet']->publicNetwork()->toArray(); $atomicPayload = [ 'maintenance_migration_id' => $atomic['migrationId'], 'body' => [ 'vcpu' => 2, 'memory_mib' => 2048, 'disk_gib' => 40, 'bandwidth_mbps' => 1000, 'public_network' => $atomicTargetNetwork, 'vpc_port' => null, ], ]; $atomicResult = [ 'migration_id' => md5('migration-5'), 'manifest_digest' => $atomicManifest, 'uuid' => $atomic['targetUuid'], 'status' => 'shutoff', 'staged_powered_off' => true, 'vcpu' => 2, 'memory_mib' => 2048, 'disk_gib' => 40, 'bandwidth_mbps' => 1000, 'managed' => true, 'public_network' => $atomicTargetNetwork, 'vpc_port' => null, 'artifacts' => [[ 'id' => $atomicArtifactId, 'role' => 'root', 'snapshot_key' => '', 'name' => '', 'consistency' => 'crash', 'size_bytes' => 123456, 'virtual_size_bytes' => 40 * 1073741824, 'sha256' => $atomicArtifactSha, 'created_at' => $now - 20, ]], ]; $finishTransfer = new ReflectionMethod( CloudInstanceMigrationService::class, 'finishTransferLocked' ); $beforeAtomic = $migrationState($atomic); Capsule::statement( "CREATE TRIGGER fail_migration_runtime_seed\n" . "BEFORE INSERT ON cloud_instance_runtime\n" . "BEGIN SELECT RAISE(ABORT, 'forced runtime seed failure'); END" ); try { $atomicError = $captures( static fn () => Capsule::connection()->transaction( static fn () => $finishTransfer->invoke( null, CloudInstanceMigration::findOrFail($atomic['migrationId']), $atomicJob, 'succeeded', $atomicResult, $atomicPayload ) ), 'Runtime-cache failure did not abort the transfer finalizer.' ); } finally { Capsule::statement('DROP TRIGGER fail_migration_runtime_seed'); } $assert(str_contains($atomicError->getMessage(), 'forced runtime seed failure'), 'Transfer finalizer failed before the intended post-settlement fault.'); $assert($beforeAtomic === $migrationState($atomic), 'Outer transaction retained a partial transfer finalization.'); // A confirmed source rollback must restore the original service and release // the complete prepared target bundle. $rolledBackCase = $seed(6); $rollbackJobId = 9206; Capsule::table('cloud_instance_migrations') ->where('id', $rolledBackCase['migrationId'])->update([ 'target_receive_job_id' => 9196, 'rollback_job_id' => $rollbackJobId, 'state' => 'rollback_running', 'source_was_running' => 1, 'source_autostart_enabled' => 1, 'error_code' => 'migration_restore_failed', ]); $rollbackJob = new CloudJob(); $rollbackJob->id = $rollbackJobId; $rollbackJob->instance_id = $rolledBackCase['instanceId']; $rollbackJob->host_id = 1; $finishRollback = new ReflectionMethod( CloudInstanceMigrationService::class, 'finishRollbackLocked' ); $finishRollback->invoke( null, CloudInstanceMigration::findOrFail($rolledBackCase['migrationId']), $rollbackJob, 'succeeded', [ 'uuid' => $rolledBackCase['sourceUuid'], 'running' => true, 'autostart_enabled' => true, ], [] ); $rolledBack = CloudInstanceMigration::findOrFail( $rolledBackCase['migrationId'] ); $assert((string) $rolledBack->state === 'rolled_back' && (string) $rolledBack->error_code === 'migration_transfer_aborted' && $rolledBack->completed_at !== null && (int) CloudInstance::findOrFail( $rolledBackCase['instanceId'] )->host_id === 1 && Capsule::table('cloud_public_ips') ->where('reservation_id', $rolledBackCase['sourceReservationId']) ->where('state', 'assigned')->count() === 2 && Capsule::table('cloud_public_ips') ->where('state', 'available')->count() >= 2, 'Confirmed rollback did not preserve source or release target dual stack.'); // Activation must preserve a powered-off expired service exactly as frozen at // migration start, including the durable guest hostname. $poweredOffCase = $seed(7); $poweredOffDebt = $cutover($poweredOffCase); $poweredSpec = json_decode((string) CloudInstance::findOrFail( $poweredOffCase['instanceId'] )->spec_json, true, 16, JSON_THROW_ON_ERROR); $poweredSpec['guest_hostname'] = 'retained.example'; Capsule::table('cloud_instances') ->where('id', $poweredOffCase['instanceId'])->update([ 'service_status' => 'suspended_expired', 'network_status' => 'blocked_expired', 'spec_json' => json_encode($poweredSpec, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES), ]); $activationJobId = 9307; Capsule::table('cloud_instance_migrations') ->where('id', $poweredOffCase['migrationId'])->update([ 'activate_job_id' => $activationJobId, 'cleanup_mode' => 'retain', 'state' => 'activation_running', 'source_was_running' => 0, 'source_autostart_enabled' => 1, 'source_service_status' => 'suspended_expired', 'source_network_status' => 'blocked_expired', ]); $activationJob = new CloudJob(); $activationJob->id = $activationJobId; $activationJob->instance_id = $poweredOffCase['instanceId']; $activationJob->host_id = 2; $activationJob->user_id = 9; $finishActivation = new ReflectionMethod( CloudInstanceMigrationService::class, 'finishActivationLocked' ); $finishActivation->invoke( null, CloudInstanceMigration::findOrFail($poweredOffCase['migrationId']), $activationJob, 'succeeded', [ 'uuid' => $poweredOffCase['targetUuid'], 'running' => false, 'autostart_enabled' => true, ], [] ); $poweredOff = CloudInstance::findOrFail($poweredOffCase['instanceId']); $activated = CloudInstanceMigration::findOrFail( $poweredOffCase['migrationId'] ); $activatedDebt = CloudInstanceMigrationSourceDebt::findOrFail( (int) $poweredOffDebt->id ); $assert((string) $activated->state === 'source_retained' && (string) $activatedDebt->state === 'retained' && (string) $poweredOff->service_status === 'suspended_expired' && (string) $poweredOff->network_status === 'blocked_expired' && str_contains((string) $poweredOff->spec_json, 'retained.example'), 'Activation changed powered-off, expired-network, or hostname state.'); // Automatic cleanup: cutover creates prepared debt, then the exact cleanup // operation releases both source families without moving either claim set. $automatic = $seed(1); $historicalSourceOperation = md5('source-history-automatic'); $activeSourceClaims = Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', $automatic['sourceReservationId']) ->where('active_claim', 1)->orderBy('address_family')->get()->all(); foreach ($activeSourceClaims as $activeSourceClaim) { Capsule::table('cloud_resource_reservation_ips')->insert([ 'reservation_id' => $automatic['sourceReservationId'], 'address_family' => (int) $activeSourceClaim->address_family, 'claim_generation' => 1, 'active_claim' => null, 'network_attachment_id' => (int) $activeSourceClaim->network_attachment_id, 'public_ip_pool_id' => (int) $activeSourceClaim->public_ip_pool_id, 'public_ip_id' => (int) $activeSourceClaim->public_ip_id, 'operation_id' => $historicalSourceOperation, 'state' => 'released', 'released_at' => $now - 60, 'release_reason' => 'network_change_completed', 'created_at' => $now - 120, 'updated_at' => $now - 60, ]); Capsule::table('cloud_resource_reservation_ips') ->where('id', (int) $activeSourceClaim->id) ->update(['claim_generation' => 2]); } $automatic['sourceSet'] = CloudPublicNetworkAllocationService::currentClaimSet( $automatic['sourceReservationId'], 'committed', $automatic['instanceId'] ); $sourceIdentityBefore = $claimIdentity($automatic['sourceReservationId']); $targetIdentityBefore = $claimIdentity($automatic['targetReservationId']); $debt = $cutover($automatic); $migration = CloudInstanceMigration::findOrFail($automatic['migrationId']); $sameDebt = CloudInstanceMigrationSettlement::settleCutoverLocked( $migration, $automatic['sourceSet'] ); $assert((int) $sameDebt->id === (int) $debt->id, 'Cutover retry created a second source debt.'); $cleanup = $job($automatic, 9001, md5('cleanup-automatic')); $migration->cleanup_job_id = (int) $cleanup->id; $migration->save(); CloudInstanceMigrationSettlement::bindCleanupLocked($migration, $cleanup); CloudInstanceMigrationSettlement::releaseSourceLocked($migration, $cleanup); CloudInstanceMigrationSettlement::releaseSourceLocked($migration, $cleanup); $wrongCleanup = $job($automatic, 9010, md5('cleanup-wrong')); $rejects( static fn () => CloudInstanceMigrationSettlement::releaseSourceLocked( $migration, $wrongCleanup ), 'Released debt accepted a different cleanup generation.' ); $instance = Capsule::table('cloud_instances') ->find($automatic['instanceId']); $order = Capsule::table('cloud_orders')->find($automatic['orderId']); $source = Capsule::table('cloud_resource_reservations') ->find($automatic['sourceReservationId']); $target = Capsule::table('cloud_resource_reservations') ->find($automatic['targetReservationId']); $releasedDebt = Capsule::table('cloud_instance_migration_source_debts') ->find((int) $debt->id); $assert((int) $instance->reservation_id === $automatic['targetReservationId'] && (int) $order->reservation_id === $automatic['targetReservationId'] && (int) $target->order_id === $automatic['orderId'] && $target->backup_restore_id === null && (string) $target->state === 'committed' && (int) $source->migration_source_debt_id === (int) $debt->id && $source->order_id === null && (string) $source->state === 'released' && (string) $releasedDebt->state === 'released' && $releasedDebt->active_debt === null && $releasedDebt->resolved_at !== null, 'Migration did not settle target ownership and source cleanup debt.'); $assert($sourceIdentityBefore === $claimIdentity($automatic['sourceReservationId']) && $targetIdentityBefore === $claimIdentity($automatic['targetReservationId']) && (int) CloudInstanceMigration::findOrFail( $automatic['migrationId'] )->source_reservation_id === $automatic['sourceReservationId'] && (int) CloudInstanceMigration::findOrFail( $automatic['migrationId'] )->target_reservation_id === $automatic['targetReservationId'], 'Migration moved claims or swapped reservation identity.'); $assert(Capsule::table('cloud_public_ips') ->where('reservation_id', $automatic['sourceReservationId']) ->count() === 0 && Capsule::table('cloud_public_ips') ->whereIn('id', array_column($sourceIdentityBefore, 2)) ->where('state', 'available')->count() === 2 && Capsule::table('cloud_public_ips') ->where('reservation_id', $automatic['targetReservationId']) ->where('state', 'assigned')->count() === 2, 'Source cleanup did not release exactly its own dual-stack bundle.'); // A retained source may be cleaned later; an unknown cleanup keeps the exact // original pair fenced and cannot create a second debt or release one family. $retained = $seed(2); $retainedDebt = $cutover($retained); $retainedMigration = CloudInstanceMigration::findOrFail( $retained['migrationId'] ); CloudInstanceMigrationSettlement::markRetainedLocked($retainedMigration); $retainedJob = $job($retained, 9002, md5('cleanup-retained')); $retainedMigration->cleanup_job_id = (int) $retainedJob->id; $retainedMigration->save(); CloudInstanceMigrationSettlement::bindCleanupLocked( $retainedMigration, $retainedJob ); CloudInstanceMigrationSettlement::markCleanupAttentionLocked( $retainedMigration, $retainedJob, 'operation_effect_unknown' ); $attention = CloudInstanceMigrationSourceDebt::findOrFail( (int) $retainedDebt->id ); $assert((string) $attention->state === 'attention' && (int) $attention->active_debt === 1 && (int) $attention->cleanup_job_id === (int) $retainedJob->id && Capsule::table('cloud_public_ips') ->where('reservation_id', $retained['sourceReservationId']) ->where('state', 'reserved')->count() === 2, 'Unknown cleanup released or detached the retained source bundle.'); CloudInstanceMigrationSettlement::releaseSourceLocked( $retainedMigration, $retainedJob ); $lateSuccess = CloudInstanceMigrationSourceDebt::findOrFail( (int) $retainedDebt->id ); $assert((string) $lateSuccess->state === 'released' && $lateSuccess->active_debt === null && Capsule::table('cloud_public_ips') ->where('reservation_id', $retained['sourceReservationId']) ->count() === 0, 'Late confirmed cleanup success did not resolve attention debt.'); // A confirmed not-applied cleanup clears only its job claim, leaving the same // debt and bundle eligible for an explicit successor operation. $retryable = $seed(3); $retryableDebt = $cutover($retryable); $retryableMigration = CloudInstanceMigration::findOrFail( $retryable['migrationId'] ); $firstCleanup = $job($retryable, 9003, md5('cleanup-failed')); $retryableMigration->cleanup_job_id = (int) $firstCleanup->id; $retryableMigration->save(); CloudInstanceMigrationSettlement::bindCleanupLocked( $retryableMigration, $firstCleanup ); CloudInstanceMigrationSettlement::markCleanupFailedLocked( $retryableMigration, $firstCleanup, 'migration_cleanup_failed' ); $afterFailure = CloudInstanceMigrationSourceDebt::findOrFail( (int) $retryableDebt->id ); $assert((string) $afterFailure->state === 'retained' && $afterFailure->cleanup_job_id === null && $afterFailure->cleanup_operation_id === null && Capsule::table('cloud_public_ips') ->where('reservation_id', $retryable['sourceReservationId']) ->where('state', 'reserved')->count() === 2, 'Confirmed-not-applied cleanup did not retain a retryable source debt.'); $secondCleanup = $job($retryable, 9004, md5('cleanup-retry')); $retryableMigration->cleanup_job_id = (int) $secondCleanup->id; $retryableMigration->save(); CloudInstanceMigrationSettlement::bindCleanupLocked( $retryableMigration, $secondCleanup ); // Corrupt frozen evidence must fail closed without changing either family. Capsule::table('cloud_instance_migration_source_debts') ->where('id', (int) $retryableDebt->id) ->update(['source_identity_sha256' => str_repeat('0', 64)]); $beforeDrift = Capsule::table('cloud_public_ips') ->whereIn('reservation_id', [ $retryable['sourceReservationId'], $retryable['targetReservationId'], ])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all(); $rejects( static fn () => CloudInstanceMigrationSettlement::releaseSourceLocked( $retryableMigration, $secondCleanup ), 'Drifted source identity was released.' ); $afterDrift = Capsule::table('cloud_public_ips') ->whereIn('reservation_id', [ $retryable['sourceReservationId'], $retryable['targetReservationId'], ])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all(); $assert($beforeDrift === $afterDrift, 'Failed identity validation changed public-network ownership.'); // A second-family write failure rolls back the complete handoff, including // the already-updated IPv4 rows. $rollback = $seed(4); $beforeRollback = [ 'reservations' => Capsule::table('cloud_resource_reservations') ->whereIn('id', [ $rollback['sourceReservationId'], $rollback['targetReservationId'], ])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all(), 'claims' => Capsule::table('cloud_resource_reservation_ips') ->whereIn('reservation_id', [ $rollback['sourceReservationId'], $rollback['targetReservationId'], ])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all(), 'ips' => Capsule::table('cloud_public_ips') ->whereIn('reservation_id', [ $rollback['sourceReservationId'], $rollback['targetReservationId'], ])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all(), ]; Capsule::statement( "CREATE TRIGGER fail_migration_ipv6_handoff\n" . "BEFORE UPDATE ON cloud_public_ips\n" . "WHEN OLD.address_family = 6 AND NEW.state <> OLD.state\n" . "BEGIN SELECT RAISE(ABORT, 'forced ipv6 failure'); END" ); $rejects( static fn () => CloudPublicNetworkAllocationService:: handoffMigrationBundles( $rollback['sourceReservationId'], $rollback['targetReservationId'], $rollback['instanceId'], $rollback['sourceOperation'], $rollback['targetOperation'] ), 'Second-family fault did not abort migration handoff.' ); Capsule::statement('DROP TRIGGER fail_migration_ipv6_handoff'); $afterRollback = [ 'reservations' => Capsule::table('cloud_resource_reservations') ->whereIn('id', [ $rollback['sourceReservationId'], $rollback['targetReservationId'], ])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all(), 'claims' => Capsule::table('cloud_resource_reservation_ips') ->whereIn('reservation_id', [ $rollback['sourceReservationId'], $rollback['targetReservationId'], ])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all(), 'ips' => Capsule::table('cloud_public_ips') ->whereIn('reservation_id', [ $rollback['sourceReservationId'], $rollback['targetReservationId'], ])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all(), ]; $assert($beforeRollback === $afterRollback, 'Second-family failure left a partial migration handoff.'); // A released replay is still a ledger verification, not an unconditional // success. Tampering either the paid target authority or the frozen source // claim history must fail closed even though no allocator mutation remains. Capsule::table('cloud_resource_reservations') ->where('id', $automatic['targetReservationId']) ->update(['order_id' => null]); try { $rejects( static fn () => CloudInstanceMigrationSettlement::releaseSourceLocked( $migration, $cleanup ), 'Released replay accepted drifted target commercial authority.' ); } finally { Capsule::table('cloud_resource_reservations') ->where('id', $automatic['targetReservationId']) ->update(['order_id' => $automatic['orderId']]); } $releasedSourceClaim = Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', $automatic['sourceReservationId']) ->where('operation_id', $automatic['sourceOperation']) ->orderBy('address_family')->first(); $assert($releasedSourceClaim !== null, 'Released source claim history disappeared.'); Capsule::table('cloud_resource_reservation_ips') ->where('id', (int) $releasedSourceClaim->id) ->update(['operation_id' => md5('tampered-released-claim')]); try { $rejects( static fn () => CloudInstanceMigrationSettlement::releaseSourceLocked( $migration, $cleanup ), 'Released replay accepted drifted frozen source claim history.' ); } finally { Capsule::table('cloud_resource_reservation_ips') ->where('id', (int) $releasedSourceClaim->id) ->update(['operation_id' => $automatic['sourceOperation']]); } // Historical source generations remain immutable audit rows. Once cleanup is // released, the concrete addresses may be allocated to a new reservation and // claim generation; replay must validate the frozen source generation without // mistaking the new owner for stale source authority. $reallocatedReservationId = 9901; $reallocatedOperation = md5('reallocated-source-addresses'); Capsule::table('cloud_resource_reservations')->insert([ 'id' => $reallocatedReservationId, 'reservation_key' => md5('reallocated-reservation'), 'operation_id' => $reallocatedOperation, 'order_id' => null, 'backup_restore_id' => null, 'backup_restore_attempt_key' => null, 'migration_source_debt_id' => null, 'offer_id' => null, 'resource_pool_id' => 10, 'host_id' => 1, 'user_id' => 9, 'instance_id' => null, 'network_attachment_id' => null, 'public_ip_pool_id' => null, 'public_ip_id' => null, 'cpu_cores' => 2, 'memory_mib' => 2048, 'disk_gib' => 40, 'state' => 'held', 'expires_at' => $now + 600, 'released_at' => null, 'release_reason' => '', 'created_at' => $now, 'updated_at' => $now, ]); $releasedSourceClaims = Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', $automatic['sourceReservationId']) ->where('operation_id', $automatic['sourceOperation']) ->orderBy('address_family')->get()->keyBy('address_family'); $reallocatedClaims = []; foreach ([4, 6] as $family) { $releasedClaim = $releasedSourceClaims->get($family); $assert(is_object($releasedClaim), 'Released source generation is incomplete before reallocation.'); $newClaimId = Capsule::table('cloud_resource_reservation_ips') ->insertGetId([ 'reservation_id' => $reallocatedReservationId, 'address_family' => $family, 'claim_generation' => 1, 'active_claim' => 1, 'network_attachment_id' => 11, 'public_ip_pool_id' => (int) $releasedClaim->public_ip_pool_id, 'public_ip_id' => (int) $releasedClaim->public_ip_id, 'operation_id' => $reallocatedOperation, 'state' => 'reserved', 'released_at' => null, 'release_reason' => '', 'created_at' => $now, 'updated_at' => $now, ]); $updated = Capsule::table('cloud_public_ips') ->where('id', (int) $releasedClaim->public_ip_id) ->where('state', 'available') ->whereNull('reservation_id') ->whereNull('reservation_ip_claim_id') ->whereNull('operation_id') ->update([ 'state' => 'reserved', 'reservation_id' => $reallocatedReservationId, 'reservation_ip_claim_id' => $newClaimId, 'instance_id' => null, 'operation_id' => $reallocatedOperation, 'updated_at' => $now, ]); $assert($updated === 1, 'Released source address was not legally reallocated.'); $reallocatedClaims[$family] = [ 'public_ip_id' => (int) $releasedClaim->public_ip_id, 'source_claim_id' => (int) $releasedClaim->id, 'new_claim_id' => $newClaimId, ]; } $reallocatedSet = CloudPublicNetworkAllocationService::currentClaimSet( $reallocatedReservationId, 'held' ); $assert($reallocatedSet->reservationId() === $reallocatedReservationId, 'Reallocated addresses do not form a valid held dual-stack bundle.'); CloudInstanceMigrationSettlement::releaseSourceLocked($migration, $cleanup); $assert(Capsule::table('cloud_public_ips') ->where('reservation_id', $reallocatedReservationId) ->where('state', 'reserved')->count() === 2 && Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', $reallocatedReservationId) ->where('active_claim', 1)->count() === 2, 'Released replay rejected or changed a legal address reallocation.'); $reallocatedIpv4 = $reallocatedClaims[4]; foreach ([ 'reservation_id' => $automatic['sourceReservationId'], 'reservation_ip_claim_id' => $reallocatedIpv4['source_claim_id'], 'operation_id' => $automatic['sourceOperation'], ] as $field => $staleValue) { Capsule::table('cloud_public_ips') ->where('id', $reallocatedIpv4['public_ip_id']) ->update([$field => $staleValue]); try { $rejects( static fn () => CloudInstanceMigrationSettlement::releaseSourceLocked( $migration, $cleanup ), 'Released replay accepted a stale source ' . $field . ' pointer.' ); } finally { $replacement = match ($field) { 'reservation_id' => $reallocatedReservationId, 'reservation_ip_claim_id' => $reallocatedIpv4['new_claim_id'], 'operation_id' => $reallocatedOperation, }; Capsule::table('cloud_public_ips') ->where('id', $reallocatedIpv4['public_ip_id']) ->update([$field => $replacement]); } } CloudInstanceMigrationSettlement::releaseSourceLocked($migration, $cleanup); echo "Maintenance migration dual-stack source-debt settlement passed.\n";