You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudPortableBackupFinalize...

529 lines
20 KiB
PHP

<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudBackupRestore;
use App\Models\CloudInstance;
use App\Models\CloudInstanceBackup;
use App\Models\CloudJob;
use App\Services\Cloud\CloudBackupService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_instance_backups',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('backup_key')->unique();
$table->integer('source_instance_id');
$table->integer('source_host_id');
$table->integer('user_id');
$table->string('source_uuid');
$table->string('name');
$table->string('purpose')->default('portable');
$table->string('state');
$table->string('consistency');
$table->string('object_key');
$table->integer('object_size_bytes')->default(0);
$table->integer('virtual_size_bytes')->default(0);
$table->string('sha256')->default('');
$table->text('spec_json');
$table->text('sealed_storage_runtime');
$table->text('sealed_credential')->nullable();
$table->integer('storage_revision')->default(1);
$table->integer('active_job_id')->nullable();
$table->string('last_operation_id')->nullable();
$table->string('error_code')->default('');
$table->integer('ready_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_backup_restores',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('restore_key')->unique();
$table->integer('backup_id');
$table->integer('target_host_id');
$table->integer('target_instance_id')->nullable();
$table->integer('reservation_id')->nullable();
$table->integer('job_id')->nullable();
$table->string('operation_id')->unique();
$table->string('new_uuid');
$table->string('new_name');
$table->string('state');
$table->string('error_code')->default('');
$table->integer('ready_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_backup_recoveries',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('recovery_key')->unique();
$table->integer('backup_id');
$table->integer('instance_id');
$table->integer('job_id')->nullable();
$table->string('operation_id')->unique();
$table->string('state');
$table->string('error_code')->default('');
$table->integer('completed_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('kind');
$table->integer('host_id');
$table->integer('instance_id')->nullable();
$table->integer('backup_id')->nullable();
$table->integer('backup_restore_id')->nullable();
$table->integer('backup_recovery_id')->nullable();
$table->integer('user_id')->nullable();
$table->string('operation_id')->unique();
});
$schema->create('cloud_instances', static function (Blueprint $table): void {
$table->increments('id');
$table->string('uuid');
$table->integer('host_id');
$table->integer('user_id');
$table->string('status');
$table->string('service_status');
$table->string('network_status');
$table->text('spec_json');
$table->text('sealed_credential')->nullable();
$table->integer('updated_at');
});
$schema->create('cloud_hosts', static function (Blueprint $table): void {
$table->increments('id');
});
$schema->create('cloud_resource_reservations',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('backup_restore_id');
$table->integer('host_id');
$table->string('operation_id');
$table->integer('public_ip_id');
$table->integer('instance_id')->nullable();
$table->string('state');
$table->integer('expires_at')->nullable();
$table->integer('released_at')->nullable();
$table->string('release_reason')->default('');
$table->integer('updated_at');
});
$schema->create('cloud_public_ips',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('address');
$table->integer('prefix_length');
$table->string('state');
$table->integer('reservation_id')->nullable();
$table->integer('instance_id')->nullable();
$table->string('operation_id')->nullable();
$table->integer('updated_at');
});
$schema->create('cloud_instance_runtime',
static function (Blueprint $table): void {
$table->integer('instance_id')->primary();
$table->integer('host_id');
$table->string('uuid');
$table->string('power_state');
$table->string('sync_state');
$table->integer('last_success_at');
$table->integer('last_attempt_at');
$table->integer('revision');
$table->string('last_error_code');
$table->integer('created_at');
$table->integer('updated_at');
});
$now = time();
$sourceSpec = [
'managed' => true,
'os_source' => 'cloud_template',
'os_name' => 'Debian 12',
'login_name' => 'root',
'guest_hostname' => 'backup-host',
'vcpu' => 2,
'memory_mib' => 2048,
'disk_gib' => 20,
'bandwidth_mbps' => 1000,
'public_ip_cidr' => '198.51.100.10/24',
'debian_12_network_fix' => true,
];
// A complete object remains ready and restorable even when only local
// snapshot cleanup is pending. Re-uploading would be unsafe and unnecessary.
$createOperation = str_repeat('1', 32);
$capsule->table('cloud_instance_backups')->insert([
'id' => 1,
'backup_key' => str_repeat('a', 32),
'source_instance_id' => 101,
'source_host_id' => 3,
'user_id' => 9,
'source_uuid' => '11111111-1111-4111-8111-111111111111',
'name' => 'cleanup-pending',
'state' => 'creating',
'consistency' => 'crash',
'object_key' => 'vm-backups/a/system.qcow2',
'spec_json' => json_encode($sourceSpec, JSON_THROW_ON_ERROR),
'sealed_storage_runtime' => 'sealed-runtime',
'sealed_credential' => 'sealed-source-credential',
'active_job_id' => 1,
'last_operation_id' => $createOperation,
'created_at' => $now,
'updated_at' => $now,
]);
$capsule->table('cloud_jobs')->insert([
'id' => 1,
'kind' => 'backup_create',
'host_id' => 3,
'instance_id' => 101,
'backup_id' => 1,
'backup_restore_id' => null,
'operation_id' => $createOperation,
]);
$createJob = CloudJob::findOrFail(1);
CloudBackupService::finishJobLocked($createJob, 'succeeded', [
'backup_key' => str_repeat('a', 32),
'size_bytes' => 1024,
'virtual_size_bytes' => 20 * 1073741824,
'sha256' => str_repeat('b', 64),
'consistency' => 'filesystem',
'cleanup_pending' => true,
], [
'owner_id' => 9,
'backup_key' => str_repeat('a', 32),
'maximum_size_bytes' => 21 * 1073741824,
]);
$createdBackup = CloudInstanceBackup::findOrFail(1);
$assert((string) $createdBackup->state === 'ready'
&& (string) $createdBackup->error_code
=== 'backup_local_cleanup_pending'
&& (int) $createdBackup->object_size_bytes === 1024
&& (string) $createdBackup->sha256 === str_repeat('b', 64)
&& $createdBackup->active_job_id === null
&& $createdBackup->ready_at !== null,
'Cleanup-pending backup lost its ready/restorable object identity.');
$fixture = static function (int $id, string $outcome) use (
$capsule, $sourceSpec, $now
): array {
$hostId = 20 + $id;
$backupId = 10 + $id;
$restoreId = 20 + $id;
$instanceId = 30 + $id;
$reservationId = 40 + $id;
$ipId = 50 + $id;
$jobId = 60 + $id;
$operationId = str_repeat((string) (($id % 8) + 2), 32);
$backupKey = substr(hash('sha256', 'backup-' . $id), 0, 32);
$restoreKey = substr(hash('sha256', 'restore-' . $id), 0, 32);
$uuid = sprintf('00000000-0000-4000-8000-%012d', $id);
$ip = '203.0.113.' . (10 + $id);
$cidr = $ip . '/24';
$poolKey = substr(hash('sha256', 'pool-' . $id), 0, 32);
$capsule->table('cloud_hosts')->insert(['id' => $hostId]);
$capsule->table('cloud_instance_backups')->insert([
'id' => $backupId,
'backup_key' => $backupKey,
'source_instance_id' => 100 + $id,
'source_host_id' => 3,
'user_id' => 9,
'source_uuid' => '11111111-1111-4111-8111-111111111111',
'name' => 'backup-' . $outcome,
'state' => 'ready',
'consistency' => 'crash',
'object_key' => 'vm-backups/' . $id . '/system.qcow2',
'object_size_bytes' => 1024,
'virtual_size_bytes' => 20 * 1073741824,
'sha256' => str_repeat('c', 64),
'spec_json' => json_encode($sourceSpec, JSON_THROW_ON_ERROR),
'sealed_storage_runtime' => 'sealed-runtime',
'sealed_credential' => 'sealed-source-credential',
'storage_revision' => 1,
'active_job_id' => null,
'last_operation_id' => null,
'error_code' => '',
'ready_at' => $now,
'created_at' => $now,
'updated_at' => $now,
]);
$capsule->table('cloud_backup_restores')->insert([
'id' => $restoreId,
'restore_key' => $restoreKey,
'backup_id' => $backupId,
'target_host_id' => $hostId,
'target_instance_id' => $instanceId,
'reservation_id' => $reservationId,
'job_id' => $jobId,
'operation_id' => $operationId,
'new_uuid' => $uuid,
'new_name' => 'restored-' . $outcome,
'state' => 'restoring',
'error_code' => '',
'ready_at' => null,
'created_at' => $now,
'updated_at' => $now,
]);
$capsule->table('cloud_instances')->insert([
'id' => $instanceId,
'uuid' => $uuid,
'host_id' => $hostId,
'user_id' => 9,
'status' => 'provisioning',
'service_status' => 'exempt',
'network_status' => 'pending',
'spec_json' => json_encode([
'managed' => false,
'vcpu' => 2,
'memory_mib' => 2048,
'disk_gib' => 20,
'public_ip_cidr' => $cidr,
], JSON_THROW_ON_ERROR),
'sealed_credential' => 'sealed-target-credential',
'updated_at' => $now,
]);
$capsule->table('cloud_public_ips')->insert([
'id' => $ipId,
'address' => $ip,
'prefix_length' => 24,
'state' => 'reserved',
'reservation_id' => $reservationId,
'instance_id' => null,
'operation_id' => $operationId,
'updated_at' => $now,
]);
$capsule->table('cloud_resource_reservations')->insert([
'id' => $reservationId,
'backup_restore_id' => $restoreId,
'host_id' => $hostId,
'operation_id' => $operationId,
'public_ip_id' => $ipId,
'instance_id' => $instanceId,
'state' => 'held',
'expires_at' => null,
'released_at' => null,
'release_reason' => '',
'updated_at' => $now,
]);
$capsule->table('cloud_jobs')->insert([
'id' => $jobId,
'kind' => 'backup_restore',
'host_id' => $hostId,
'instance_id' => $instanceId,
'backup_id' => $backupId,
'backup_restore_id' => $restoreId,
'operation_id' => $operationId,
]);
return [
'job' => CloudJob::findOrFail($jobId),
'payload' => [
'owner_id' => 9,
'backup_key' => $backupKey,
'restore_key' => $restoreKey,
'body' => [
'public_ip_cidr' => $cidr,
'public_pool_key' => $poolKey,
],
],
'result' => [
'backup_key' => $backupKey,
'sha256' => str_repeat('c', 64),
'vm' => [
'uuid' => $uuid,
'public_ip_cidr' => $cidr,
'public_pool_key' => $poolKey,
'status' => 'running',
'managed' => true,
'os_source' => 'cloud_template',
'os_name' => 'Debian 12',
'login_name' => 'root',
'vcpu' => 2,
'memory_mib' => 2048,
'disk_gib' => 20,
],
],
'backup_id' => $backupId,
'restore_id' => $restoreId,
'instance_id' => $instanceId,
'reservation_id' => $reservationId,
'ip_id' => $ipId,
'cidr' => $cidr,
'uuid' => $uuid,
];
};
$success = $fixture(1, 'success');
CloudBackupService::finishJobLocked(
$success['job'], 'succeeded', $success['result'], $success['payload']
);
$successRestore = CloudBackupRestore::findOrFail($success['restore_id']);
$successInstance = CloudInstance::findOrFail($success['instance_id']);
$successReservation = $capsule->table('cloud_resource_reservations')
->where('id', $success['reservation_id'])->first();
$successIp = $capsule->table('cloud_public_ips')
->where('id', $success['ip_id'])->first();
$successRuntime = $capsule->table('cloud_instance_runtime')
->where('instance_id', $success['instance_id'])->first();
$successSpec = json_decode((string) $successInstance->spec_json, true,
flags: JSON_THROW_ON_ERROR);
$assert((string) $successRestore->state === 'ready'
&& $successRestore->ready_at !== null
&& (string) $successReservation->state === 'committed'
&& (int) $successReservation->instance_id === $success['instance_id']
&& (string) $successIp->state === 'assigned'
&& (int) $successIp->instance_id === $success['instance_id']
&& (string) $successInstance->status === 'active'
&& (string) $successInstance->service_status === 'exempt'
&& (string) $successInstance->network_status === 'active'
&& ($successSpec['public_ip_cidr'] ?? null) === $success['cidr']
&& ($successSpec['public_ip_cidr'] ?? null)
!== $sourceSpec['public_ip_cidr']
&& ($successSpec['restored_from_backup'] ?? null)
=== (string) CloudInstanceBackup::findOrFail(
$success['backup_id']
)->backup_key
&& ($successSpec['managed'] ?? null) === true
&& (string) $successRuntime->power_state === 'running',
'Successful restore did not commit the fresh IP and activate an exempt independent instance.');
// CloudWorkerService may call failed only after recovery proves not_applied.
$failed = $fixture(2, 'failed');
CloudBackupService::finishJobLocked(
$failed['job'], 'failed', [], $failed['payload']
);
$failedRestore = CloudBackupRestore::findOrFail($failed['restore_id']);
$failedInstance = CloudInstance::findOrFail($failed['instance_id']);
$failedReservation = $capsule->table('cloud_resource_reservations')
->where('id', $failed['reservation_id'])->first();
$failedIp = $capsule->table('cloud_public_ips')
->where('id', $failed['ip_id'])->first();
$assert((string) $failedRestore->state === 'failed'
&& (string) $failedReservation->state === 'released'
&& (string) $failedReservation->release_reason
=== 'backup_restore_not_applied'
&& (string) $failedIp->state === 'available'
&& $failedIp->reservation_id === null
&& $failedIp->instance_id === null
&& $failedIp->operation_id === null
&& (string) $failedInstance->status === 'deleted'
&& (string) $failedInstance->network_status === 'inactive'
&& $failedInstance->sealed_credential === null,
'Proven-not-applied restore did not release capacity/IP and erase the failed target credential.');
$uncertain = $fixture(3, 'uncertain');
CloudBackupService::finishJobLocked(
$uncertain['job'], 'uncertain', [], $uncertain['payload']
);
$uncertainRestore = CloudBackupRestore::findOrFail($uncertain['restore_id']);
$uncertainInstance = CloudInstance::findOrFail($uncertain['instance_id']);
$uncertainReservation = $capsule->table('cloud_resource_reservations')
->where('id', $uncertain['reservation_id'])->first();
$uncertainIp = $capsule->table('cloud_public_ips')
->where('id', $uncertain['ip_id'])->first();
$assert((string) $uncertainRestore->state === 'uncertain'
&& (string) $uncertainReservation->state === 'uncertain'
&& (int) $uncertainReservation->instance_id === $uncertain['instance_id']
&& (string) $uncertainIp->state === 'uncertain'
&& (int) $uncertainIp->reservation_id === $uncertain['reservation_id']
&& (string) $uncertainIp->operation_id
=== (string) $uncertain['job']->operation_id
&& (string) $uncertainInstance->status === 'provisioning'
&& (string) $uncertainInstance->sealed_credential
=== 'sealed-target-credential',
'Unknown restore effect released resources, IP or target credentials.');
// In-place recovery keeps the existing instance identity and converges only
// from an exact backup/ledger/Agent result tuple.
$recoveryOperation = str_repeat('e', 32);
$recoveryKey = str_repeat('f', 32);
$capsule->table('cloud_instances')->insert([
'id' => 101,
'uuid' => '11111111-1111-4111-8111-111111111111',
'host_id' => 3,
'user_id' => 9,
'status' => 'active',
'service_status' => 'active',
'network_status' => 'active',
'spec_json' => json_encode(array_replace($sourceSpec, [
'guest_hostname' => 'current-host',
]), JSON_THROW_ON_ERROR),
'sealed_credential' => 'sealed-current-credential',
'updated_at' => $now,
]);
$capsule->table('cloud_backup_recoveries')->insert([
'id' => 200,
'recovery_key' => $recoveryKey,
'backup_id' => 1,
'instance_id' => 101,
'job_id' => 200,
'operation_id' => $recoveryOperation,
'state' => 'recovering',
'error_code' => '',
'completed_at' => null,
'created_at' => $now,
'updated_at' => $now,
]);
$capsule->table('cloud_jobs')->insert([
'id' => 200,
'kind' => 'backup_recover',
'host_id' => 3,
'instance_id' => 101,
'backup_id' => 1,
'backup_restore_id' => null,
'backup_recovery_id' => 200,
'user_id' => 9,
'operation_id' => $recoveryOperation,
]);
$recoveryJob = CloudJob::findOrFail(200);
CloudBackupService::finishJobLocked($recoveryJob, 'succeeded', [
'backup_key' => str_repeat('a', 32),
'sha256' => str_repeat('b', 64),
'restored' => true,
'power_state' => 'running',
'active_volume_name' => 'managed-recovery-volume',
], [
'owner_id' => 9,
'backup_key' => str_repeat('a', 32),
'recovery_key' => $recoveryKey,
'body' => [
'backup_key' => str_repeat('a', 32),
'source_size_bytes' => 1024,
'source_sha256' => str_repeat('b', 64),
],
]);
$recovery = \App\Models\CloudBackupRecovery::findOrFail(200);
$recoveredInstance = CloudInstance::findOrFail(101);
$recoveredSpec = json_decode(
(string) $recoveredInstance->spec_json,
true,
64,
JSON_THROW_ON_ERROR
);
$assert((string) $recovery->state === 'succeeded'
&& $recovery->completed_at !== null
&& (string) $recoveredInstance->uuid
=== '11111111-1111-4111-8111-111111111111'
&& (string) $recoveredInstance->status === 'active'
&& (string) $recoveredInstance->sealed_credential
=== 'sealed-source-credential'
&& ($recoveredSpec['guest_hostname'] ?? null) === 'backup-host'
&& ($recoveredSpec['restored_from_backup'] ?? null)
=== str_repeat('a', 32),
'In-place backup recovery did not preserve the business instance while restoring guest identity.');
echo "Portable backup create/restore finalizer behavior tests passed.\n";