addConnection([ 'driver' => 'sqlite', 'database' => ':memory:', ], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('cloud_instance_backups', static function (Blueprint $table): void { $table->increments('id'); $table->string('backup_key')->unique(); $table->integer('source_instance_id'); $table->integer('source_host_id'); $table->integer('user_id'); $table->string('source_uuid'); $table->string('name'); $table->string('purpose')->default('portable'); $table->string('state'); $table->string('consistency'); $table->string('object_key'); $table->integer('object_size_bytes')->default(0); $table->integer('virtual_size_bytes')->default(0); $table->string('sha256')->default(''); $table->string('object_etag')->default(''); $table->string('object_version_id')->default(''); $table->text('spec_json'); $table->text('sealed_storage_runtime'); $table->text('sealed_credential')->nullable(); $table->integer('storage_revision')->default(1); $table->integer('active_job_id')->nullable(); $table->string('last_operation_id')->nullable(); $table->string('error_code')->default(''); $table->integer('ready_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_backup_restores', static function (Blueprint $table): void { $table->increments('id'); $table->string('restore_key')->unique(); $table->integer('backup_id'); $table->integer('target_host_id'); $table->integer('target_instance_id')->nullable(); $table->integer('reservation_id')->nullable(); $table->integer('job_id')->nullable(); $table->string('operation_id')->unique(); $table->string('new_uuid'); $table->string('new_name'); $table->string('state'); $table->string('error_code')->default(''); $table->integer('ready_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_backup_recoveries', static function (Blueprint $table): void { $table->increments('id'); $table->string('recovery_key')->unique(); $table->integer('backup_id'); $table->integer('instance_id'); $table->integer('job_id')->nullable(); $table->string('operation_id')->unique(); $table->string('state'); $table->string('error_code')->default(''); $table->integer('completed_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('kind'); $table->integer('host_id'); $table->integer('instance_id')->nullable(); $table->integer('backup_id')->nullable(); $table->integer('backup_restore_id')->nullable(); $table->integer('backup_recovery_id')->nullable(); $table->integer('user_id')->nullable(); $table->string('operation_id')->unique(); }); $schema->create('cloud_instances', static function (Blueprint $table): void { $table->increments('id'); $table->string('uuid'); $table->integer('host_id'); $table->integer('user_id'); $table->string('status'); $table->string('service_status'); $table->string('network_status'); $table->text('spec_json'); $table->text('sealed_credential')->nullable(); $table->integer('updated_at'); }); $schema->create('cloud_hosts', static function (Blueprint $table): void { $table->increments('id'); }); $schema->create('cloud_resource_reservations', static function (Blueprint $table): void { $table->increments('id'); $table->integer('backup_restore_id'); $table->integer('host_id'); $table->string('operation_id'); $table->integer('public_ip_id'); $table->integer('instance_id')->nullable(); $table->string('state'); $table->integer('expires_at')->nullable(); $table->integer('released_at')->nullable(); $table->string('release_reason')->default(''); $table->integer('updated_at'); }); $schema->create('cloud_public_ips', static function (Blueprint $table): void { $table->increments('id'); $table->string('address'); $table->integer('prefix_length'); $table->string('state'); $table->integer('reservation_id')->nullable(); $table->integer('instance_id')->nullable(); $table->string('operation_id')->nullable(); $table->integer('updated_at'); }); $schema->create('cloud_instance_runtime', static function (Blueprint $table): void { $table->integer('instance_id')->primary(); $table->integer('host_id'); $table->string('uuid'); $table->string('power_state'); $table->string('sync_state'); $table->integer('last_success_at'); $table->integer('last_attempt_at'); $table->integer('revision'); $table->string('last_error_code'); $table->integer('created_at'); $table->integer('updated_at'); }); $now = time(); $sourceSpec = [ 'managed' => true, 'os_source' => 'cloud_template', 'os_name' => 'Debian 12', 'login_name' => 'root', 'guest_hostname' => 'backup-host', 'vcpu' => 2, 'memory_mib' => 2048, 'disk_gib' => 20, 'bandwidth_mbps' => 1000, 'public_ip_cidr' => '198.51.100.10/24', 'debian_12_network_fix' => true, ]; // A complete object remains ready and restorable even when only local // snapshot cleanup is pending. Re-uploading would be unsafe and unnecessary. $createOperation = str_repeat('1', 32); $capsule->table('cloud_instance_backups')->insert([ 'id' => 1, 'backup_key' => str_repeat('a', 32), 'source_instance_id' => 101, 'source_host_id' => 3, 'user_id' => 9, 'source_uuid' => '11111111-1111-4111-8111-111111111111', 'name' => 'cleanup-pending', 'state' => 'creating', 'consistency' => 'crash', 'object_key' => 'vm-backups/a/system.qcow2', 'spec_json' => json_encode($sourceSpec, JSON_THROW_ON_ERROR), 'sealed_storage_runtime' => 'sealed-runtime', 'sealed_credential' => 'sealed-source-credential', 'active_job_id' => 1, 'last_operation_id' => $createOperation, 'created_at' => $now, 'updated_at' => $now, ]); $capsule->table('cloud_jobs')->insert([ 'id' => 1, 'kind' => 'backup_create', 'host_id' => 3, 'instance_id' => 101, 'backup_id' => 1, 'backup_restore_id' => null, 'operation_id' => $createOperation, ]); $createJob = CloudJob::findOrFail(1); CloudBackupService::finishJobLocked($createJob, 'succeeded', [ 'backup_key' => str_repeat('a', 32), 'size_bytes' => 1024, 'virtual_size_bytes' => 20 * 1073741824, 'sha256' => str_repeat('b', 64), 'object_etag' => '"finalizer-etag"', 'object_version_id' => 'finalizer-version', 'consistency' => 'filesystem', 'cleanup_pending' => true, ], [ 'owner_id' => 9, 'backup_key' => str_repeat('a', 32), 'maximum_size_bytes' => 21 * 1073741824, ]); $createdBackup = CloudInstanceBackup::findOrFail(1); $assert((string) $createdBackup->state === 'ready' && (string) $createdBackup->error_code === 'backup_local_cleanup_pending' && (int) $createdBackup->object_size_bytes === 1024 && (string) $createdBackup->sha256 === str_repeat('b', 64) && (string) $createdBackup->object_etag === '"finalizer-etag"' && (string) $createdBackup->object_version_id === 'finalizer-version' && $createdBackup->active_job_id === null && $createdBackup->ready_at !== null, 'Cleanup-pending backup lost its ready/restorable object identity.'); $cleanupOperation = str_repeat('9', 32); $capsule->table('cloud_jobs')->insert([ 'id' => 2, 'kind' => 'backup_workspace_cleanup', 'host_id' => 3, 'instance_id' => 101, 'backup_id' => 1, 'backup_restore_id' => null, 'operation_id' => $cleanupOperation, ]); $cleanupJob = CloudJob::findOrFail(2); CloudBackupService::finishJobLocked($cleanupJob, 'succeeded', [ 'backup_key' => str_repeat('a', 32), 'cleaned' => true, 'workspace_absent' => true, 'metadata_absent' => true, ], [ 'owner_id' => 9, 'backup_key' => str_repeat('a', 32), 'prior_operation_id' => $createOperation, ]); $cleanedBackup = CloudInstanceBackup::findOrFail(1); $assert((string) $cleanedBackup->state === 'ready' && (string) $cleanedBackup->error_code === '' && (string) $cleanedBackup->sha256 === str_repeat('b', 64), 'Evidence-bound workspace cleanup did not preserve the remote backup identity.'); $fixture = static function (int $id, string $outcome) use ( $capsule, $sourceSpec, $now ): array { $hostId = 20 + $id; $backupId = 10 + $id; $restoreId = 20 + $id; $instanceId = 30 + $id; $reservationId = 40 + $id; $ipId = 50 + $id; $jobId = 60 + $id; $operationId = str_repeat((string) (($id % 8) + 2), 32); $backupKey = substr(hash('sha256', 'backup-' . $id), 0, 32); $restoreKey = substr(hash('sha256', 'restore-' . $id), 0, 32); $uuid = sprintf('00000000-0000-4000-8000-%012d', $id); $ip = '203.0.113.' . (10 + $id); $cidr = $ip . '/24'; $poolKey = substr(hash('sha256', 'pool-' . $id), 0, 32); $capsule->table('cloud_hosts')->insert(['id' => $hostId]); $capsule->table('cloud_instance_backups')->insert([ 'id' => $backupId, 'backup_key' => $backupKey, 'source_instance_id' => 100 + $id, 'source_host_id' => 3, 'user_id' => 9, 'source_uuid' => '11111111-1111-4111-8111-111111111111', 'name' => 'backup-' . $outcome, 'state' => 'ready', 'consistency' => 'crash', 'object_key' => 'vm-backups/' . $id . '/system.qcow2', 'object_size_bytes' => 1024, 'virtual_size_bytes' => 20 * 1073741824, 'sha256' => str_repeat('c', 64), 'spec_json' => json_encode($sourceSpec, JSON_THROW_ON_ERROR), 'sealed_storage_runtime' => 'sealed-runtime', 'sealed_credential' => 'sealed-source-credential', 'storage_revision' => 1, 'active_job_id' => null, 'last_operation_id' => null, 'error_code' => '', 'ready_at' => $now, 'created_at' => $now, 'updated_at' => $now, ]); $capsule->table('cloud_backup_restores')->insert([ 'id' => $restoreId, 'restore_key' => $restoreKey, 'backup_id' => $backupId, 'target_host_id' => $hostId, 'target_instance_id' => $instanceId, 'reservation_id' => $reservationId, 'job_id' => $jobId, 'operation_id' => $operationId, 'new_uuid' => $uuid, 'new_name' => 'restored-' . $outcome, 'state' => 'restoring', 'error_code' => '', 'ready_at' => null, 'created_at' => $now, 'updated_at' => $now, ]); $capsule->table('cloud_instances')->insert([ 'id' => $instanceId, 'uuid' => $uuid, 'host_id' => $hostId, 'user_id' => 9, 'status' => 'provisioning', 'service_status' => 'exempt', 'network_status' => 'pending', 'spec_json' => json_encode([ 'managed' => false, 'vcpu' => 2, 'memory_mib' => 2048, 'disk_gib' => 20, 'public_ip_cidr' => $cidr, ], JSON_THROW_ON_ERROR), 'sealed_credential' => 'sealed-target-credential', 'updated_at' => $now, ]); $capsule->table('cloud_public_ips')->insert([ 'id' => $ipId, 'address' => $ip, 'prefix_length' => 24, 'state' => 'reserved', 'reservation_id' => $reservationId, 'instance_id' => null, 'operation_id' => $operationId, 'updated_at' => $now, ]); $capsule->table('cloud_resource_reservations')->insert([ 'id' => $reservationId, 'backup_restore_id' => $restoreId, 'host_id' => $hostId, 'operation_id' => $operationId, 'public_ip_id' => $ipId, 'instance_id' => $instanceId, 'state' => 'held', 'expires_at' => null, 'released_at' => null, 'release_reason' => '', 'updated_at' => $now, ]); $capsule->table('cloud_jobs')->insert([ 'id' => $jobId, 'kind' => 'backup_restore', 'host_id' => $hostId, 'instance_id' => $instanceId, 'backup_id' => $backupId, 'backup_restore_id' => $restoreId, 'operation_id' => $operationId, ]); return [ 'job' => CloudJob::findOrFail($jobId), 'payload' => [ 'owner_id' => 9, 'backup_key' => $backupKey, 'restore_key' => $restoreKey, 'body' => [ 'public_ip_cidr' => $cidr, 'public_pool_key' => $poolKey, ], ], 'result' => [ 'backup_key' => $backupKey, 'sha256' => str_repeat('c', 64), 'vm' => [ 'uuid' => $uuid, 'public_ip_cidr' => $cidr, 'public_pool_key' => $poolKey, 'status' => 'running', 'managed' => true, 'os_source' => 'cloud_template', 'os_name' => 'Debian 12', 'login_name' => 'root', 'vcpu' => 2, 'memory_mib' => 2048, 'disk_gib' => 20, ], ], 'backup_id' => $backupId, 'restore_id' => $restoreId, 'instance_id' => $instanceId, 'reservation_id' => $reservationId, 'ip_id' => $ipId, 'cidr' => $cidr, 'uuid' => $uuid, ]; }; $success = $fixture(1, 'success'); CloudBackupService::finishJobLocked( $success['job'], 'succeeded', $success['result'], $success['payload'] ); $successRestore = CloudBackupRestore::findOrFail($success['restore_id']); $successInstance = CloudInstance::findOrFail($success['instance_id']); $successReservation = $capsule->table('cloud_resource_reservations') ->where('id', $success['reservation_id'])->first(); $successIp = $capsule->table('cloud_public_ips') ->where('id', $success['ip_id'])->first(); $successRuntime = $capsule->table('cloud_instance_runtime') ->where('instance_id', $success['instance_id'])->first(); $successSpec = json_decode((string) $successInstance->spec_json, true, flags: JSON_THROW_ON_ERROR); $assert((string) $successRestore->state === 'ready' && $successRestore->ready_at !== null && (string) $successReservation->state === 'committed' && (int) $successReservation->instance_id === $success['instance_id'] && (string) $successIp->state === 'assigned' && (int) $successIp->instance_id === $success['instance_id'] && (string) $successInstance->status === 'active' && (string) $successInstance->service_status === 'exempt' && (string) $successInstance->network_status === 'active' && ($successSpec['public_ip_cidr'] ?? null) === $success['cidr'] && ($successSpec['public_ip_cidr'] ?? null) !== $sourceSpec['public_ip_cidr'] && ($successSpec['restored_from_backup'] ?? null) === (string) CloudInstanceBackup::findOrFail( $success['backup_id'] )->backup_key && ($successSpec['managed'] ?? null) === true && (string) $successRuntime->power_state === 'running', 'Successful restore did not commit the fresh IP and activate an exempt independent instance.'); // CloudWorkerService may call failed only after recovery proves not_applied. $failed = $fixture(2, 'failed'); CloudBackupService::finishJobLocked( $failed['job'], 'failed', [], $failed['payload'] ); $failedRestore = CloudBackupRestore::findOrFail($failed['restore_id']); $failedInstance = CloudInstance::findOrFail($failed['instance_id']); $failedReservation = $capsule->table('cloud_resource_reservations') ->where('id', $failed['reservation_id'])->first(); $failedIp = $capsule->table('cloud_public_ips') ->where('id', $failed['ip_id'])->first(); $assert((string) $failedRestore->state === 'failed' && (string) $failedReservation->state === 'released' && (string) $failedReservation->release_reason === 'backup_restore_not_applied' && (string) $failedIp->state === 'available' && $failedIp->reservation_id === null && $failedIp->instance_id === null && $failedIp->operation_id === null && (string) $failedInstance->status === 'deleted' && (string) $failedInstance->network_status === 'inactive' && $failedInstance->sealed_credential === null, 'Proven-not-applied restore did not release capacity/IP and erase the failed target credential.'); $uncertain = $fixture(3, 'uncertain'); CloudBackupService::finishJobLocked( $uncertain['job'], 'uncertain', [], $uncertain['payload'] ); $uncertainRestore = CloudBackupRestore::findOrFail($uncertain['restore_id']); $uncertainInstance = CloudInstance::findOrFail($uncertain['instance_id']); $uncertainReservation = $capsule->table('cloud_resource_reservations') ->where('id', $uncertain['reservation_id'])->first(); $uncertainIp = $capsule->table('cloud_public_ips') ->where('id', $uncertain['ip_id'])->first(); $assert((string) $uncertainRestore->state === 'uncertain' && (string) $uncertainReservation->state === 'uncertain' && (int) $uncertainReservation->instance_id === $uncertain['instance_id'] && (string) $uncertainIp->state === 'uncertain' && (int) $uncertainIp->reservation_id === $uncertain['reservation_id'] && (string) $uncertainIp->operation_id === (string) $uncertain['job']->operation_id && (string) $uncertainInstance->status === 'provisioning' && (string) $uncertainInstance->sealed_credential === 'sealed-target-credential', 'Unknown restore effect released resources, IP or target credentials.'); // In-place recovery keeps the existing instance identity and converges only // from an exact backup/ledger/Agent result tuple. $recoveryOperation = str_repeat('e', 32); $recoveryKey = str_repeat('f', 32); $capsule->table('cloud_instances')->insert([ 'id' => 101, 'uuid' => '11111111-1111-4111-8111-111111111111', 'host_id' => 3, 'user_id' => 9, 'status' => 'active', 'service_status' => 'active', 'network_status' => 'active', 'spec_json' => json_encode(array_replace($sourceSpec, [ 'guest_hostname' => 'current-host', 'public_ip_cidr' => '198.51.100.99/24', ]), JSON_THROW_ON_ERROR), 'sealed_credential' => 'sealed-current-credential', 'updated_at' => $now, ]); $capsule->table('cloud_backup_recoveries')->insert([ 'id' => 200, 'recovery_key' => $recoveryKey, 'backup_id' => 1, 'instance_id' => 101, 'job_id' => 200, 'operation_id' => $recoveryOperation, 'state' => 'recovering', 'error_code' => '', 'completed_at' => null, 'created_at' => $now, 'updated_at' => $now, ]); $capsule->table('cloud_jobs')->insert([ 'id' => 200, 'kind' => 'backup_recover', 'host_id' => 3, 'instance_id' => 101, 'backup_id' => 1, 'backup_restore_id' => null, 'backup_recovery_id' => 200, 'user_id' => 9, 'operation_id' => $recoveryOperation, ]); $recoveryJob = CloudJob::findOrFail(200); CloudBackupService::finishJobLocked($recoveryJob, 'succeeded', [ 'backup_key' => str_repeat('a', 32), 'sha256' => str_repeat('b', 64), 'restored' => true, 'power_state' => 'running', 'active_volume_name' => 'managed-recovery-volume', ], [ 'owner_id' => 9, 'backup_key' => str_repeat('a', 32), 'recovery_key' => $recoveryKey, 'body' => [ 'backup_key' => str_repeat('a', 32), 'source_size_bytes' => 1024, 'source_sha256' => str_repeat('b', 64), ], ]); $recovery = \App\Models\CloudBackupRecovery::findOrFail(200); $recoveredInstance = CloudInstance::findOrFail(101); $recoveredSpec = json_decode( (string) $recoveredInstance->spec_json, true, 64, JSON_THROW_ON_ERROR ); $assert((string) $recovery->state === 'succeeded' && $recovery->completed_at !== null && (string) $recoveredInstance->uuid === '11111111-1111-4111-8111-111111111111' && (string) $recoveredInstance->status === 'active' && $recoveredInstance->sealed_credential === null && !array_key_exists('guest_hostname', $recoveredSpec) && !array_key_exists('hostname', $recoveredSpec) && ($recoveredSpec['public_ip_cidr'] ?? null) === '198.51.100.99/24' && ($recoveredSpec['restored_from_backup'] ?? null) === str_repeat('a', 32), 'In-place backup recovery did not preserve disk-authoritative guest identity and the current network.'); echo "Portable backup create/restore finalizer behavior tests passed.\n";