You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudSnapshotWorkflowTest.php

397 lines
18 KiB
PHP

<?php
namespace App\Services\Cloud {
// Isolated test cipher only; production CloudSecrets is never loaded here.
final class CloudSecrets
{
private static array $values = [];
public static function encrypt(array $value): string
{
$key = bin2hex(random_bytes(16));
self::$values[$key] = $value;
return 'snapshot-test:' . $key;
}
public static function decrypt(string $value): array
{
return self::$values[substr($value, 14)];
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudInstance;
use App\Models\CloudInstanceSnapshot;
use App\Models\CloudJob;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudJobService;
use App\Services\Cloud\CloudOperationErrorCatalog;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\CloudSnapshotService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function ($condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$expect = static function (callable $callable, int $status,
string $message) use ($assert): void {
try {
$callable();
} catch (CloudError $error) {
$assert($error->httpStatus === $status,
$message . ' Unexpected status: ' . $error->httpStatus);
return;
}
throw new RuntimeException($message);
};
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_instances', static function (Blueprint $table): void {
$table->increments('id');
$table->string('uuid');
$table->integer('host_id');
$table->integer('user_id');
$table->string('name');
$table->string('status');
$table->string('service_status');
$table->string('network_status');
$table->integer('expires_at')->nullable();
$table->text('spec_json');
$table->text('credential_cipher')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_instance_snapshots', static function (Blueprint $table): void {
$table->increments('id');
$table->string('snapshot_key')->unique();
$table->integer('instance_id');
$table->integer('user_id');
$table->integer('host_id');
$table->string('name');
$table->string('state');
$table->string('consistency');
$table->integer('size_bytes')->nullable();
$table->string('last_operation_id')->nullable();
$table->integer('ready_at')->nullable();
$table->integer('last_restored_at')->nullable();
$table->integer('deleted_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_template_retirements', static function (
Blueprint $table): void {
$table->string('template_id')->primary();
$table->string('state');
$table->integer('template_created_at')->default(0);
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->integer('order_id')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('user_id');
$table->integer('host_id');
$table->string('kind');
$table->string('status');
$table->text('payload_cipher');
$table->integer('attempts')->default(0);
$table->integer('dispatched_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at');
$table->string('error_code')->nullable();
$table->text('result_json')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$instance = new CloudInstance();
$instance->uuid = '11111111-1111-4111-8111-111111111111';
$instance->host_id = 7;
$instance->user_id = 10;
$instance->name = 'vm-snap';
$instance->status = 'active';
$instance->service_status = 'exempt';
$instance->network_status = 'active';
$instance->expires_at = null;
$instance->spec_json = json_encode([
'template_id' => 'debian-12', 'os_source' => 'cloud_template',
'os_name' => 'Debian 12', 'login_name' => 'root',
'guest_hostname' => 'snapshot-old',
'public_ip_cidr' => '192.0.2.10/24', 'region' => 'HKG',
'vcpu' => 2, 'memory_mib' => 2048, 'disk_gib' => 40,
'engine_metadata_version' => 1,
], JSON_THROW_ON_ERROR);
$instance->credential_cipher = CloudSecrets::encrypt([
'password' => 'old-password', 'login_name' => 'root',
]);
$instance->created_at = $instance->updated_at = time();
$instance->save();
$actor = ['user_id' => 10, 'is_admin' => false,
'is_super_admin' => false, 'can_manage' => false];
$first = CloudSnapshotService::create($instance, $actor, [
'name' => '发布前', 'consistency_mode' => 'filesystem',
]);
$firstJob = $first['job'];
$firstPayload = CloudSecrets::decrypt((string) $firstJob->payload_cipher);
$firstKey = (string) $first['snapshot']['snapshot_key'];
$assert(preg_match('/^[a-f0-9]{32}$/D', $firstKey) === 1
&& $firstPayload['method'] === 'POST'
&& $firstPayload['resource'] === 'vms/' . $instance->uuid . '/snapshots'
&& $firstPayload['body'] === [
'snapshot_key' => $firstKey,
'name' => '发布前',
'consistency_mode' => 'filesystem',
]
&& (int) $firstJob->snapshot_id > 0,
'Snapshot create did not persist an opaque key and bounded engine contract.');
CloudSnapshotService::finishJobLocked($firstJob, 'succeeded', [
'snapshot_key' => $firstKey, 'created' => true,
'consistency' => 'filesystem', 'size_bytes' => 4096,
], $firstPayload);
$firstJob->status = 'succeeded';
$firstJob->save();
$listed = CloudSnapshotService::listing($instance, $actor);
$assert($listed['max_snapshots'] === 2 && $listed['used_slots'] === 1
&& $listed['can_create'] === true && $listed['can_mutate'] === true
&& count($listed['snapshots']) === 1
&& $listed['snapshots'][0]['state'] === 'ready'
&& $listed['snapshots'][0]['consistency'] === 'filesystem'
&& $listed['snapshots'][0]['size_bytes'] === 4096,
'Database-only snapshot listing did not expose the committed catalog state.');
$storedFirst = CloudInstanceSnapshot::find((int) $firstJob->snapshot_id);
$assert(!array_key_exists('credential_cipher', $storedFirst->getAttributes())
&& !array_key_exists('system_metadata_json', $storedFirst->getAttributes()),
'Snapshot catalog duplicated an instance credential or seed metadata.');
$settingsWithSnapshot = CloudJobService::enqueue($instance, $actor,
'instance_settings', 'POST', '/settings', [
'hostname' => 'current.example', 'confirm_restart' => true,
]);
$assert((string) $settingsWithSnapshot->status === 'queued',
'A ready snapshot incorrectly blocked a current-seed settings mutation.');
$settingsWithSnapshot->status = 'failed';
$settingsWithSnapshot->save();
$expect(static fn () => CloudJobService::enqueue($instance, $actor,
'reinstall', 'POST', '/reinstall', [
'template_id' => 'debian-12',
]), 409, 'A retained snapshot allowed a reinstall mutation.');
$second = CloudSnapshotService::create($instance, $actor, [
'name' => 'Crash point', 'consistency_mode' => 'crash',
]);
$secondJob = $second['job'];
$secondPayload = CloudSecrets::decrypt((string) $secondJob->payload_cipher);
CloudSnapshotService::finishJobLocked($secondJob, 'succeeded', [
'snapshot_key' => $second['snapshot']['snapshot_key'],
'created' => true, 'consistency' => 'crash',
], $secondPayload);
$secondJob->status = 'succeeded';
$secondJob->save();
$expect(static fn () => CloudSnapshotService::create($instance, $actor, [
'name' => 'third', 'consistency_mode' => 'filesystem',
]), 409, 'A third live snapshot bypassed the per-instance limit.');
$restore = CloudSnapshotService::restore($instance, $actor, $firstKey,
['confirmation_name' => 'vm-snap']);
$restoreJob = $restore['job'];
$restorePayload = CloudSecrets::decrypt((string) $restoreJob->payload_cipher);
$assert($restorePayload['method'] === 'POST'
&& $restorePayload['resource'] === 'vms/' . $instance->uuid
. '/snapshots/' . $firstKey . '/restore'
&& $restorePayload['body'] === [],
'Snapshot restore engine contract changed unexpectedly.');
CloudSnapshotService::finishJobLocked($restoreJob, 'uncertain', null,
$restorePayload);
$restoreJob->status = 'uncertain';
$restoreJob->save();
$uncertain = CloudSnapshotService::listing($instance, $actor);
$assert($uncertain['snapshots'][1]['state'] === 'uncertain'
|| $uncertain['snapshots'][0]['state'] === 'uncertain',
'An ambiguous restore did not keep a visible uncertain catalog state.');
$credentialBeforeRestore = (string) $instance->credential_cipher;
$specBeforeRestore = (string) $instance->spec_json;
CloudSnapshotService::finishJobLocked($restoreJob, 'succeeded', [
'snapshot_key' => $firstKey, 'restored' => true,
], $restorePayload);
$restoreJob->status = 'succeeded';
$restoreJob->save();
$instance->refresh();
$assert((string) $instance->credential_cipher === $credentialBeforeRestore
&& (string) $instance->spec_json === $specBeforeRestore,
'Snapshot restore rewrote panel credentials or seed metadata.');
$delete = CloudSnapshotService::delete($instance, $actor, $firstKey,
['confirmation_name' => 'vm-snap']);
$deleteJob = $delete['job'];
$deletePayload = CloudSecrets::decrypt((string) $deleteJob->payload_cipher);
$assert($deletePayload['method'] === 'DELETE'
&& $deletePayload['resource'] === 'vms/' . $instance->uuid
. '/snapshots/' . $firstKey
&& $deletePayload['body'] === [],
'Snapshot delete engine contract changed unexpectedly.');
CloudSnapshotService::finishJobLocked($deleteJob, 'succeeded', [
'snapshot_key' => $firstKey, 'deleted' => true,
], $deletePayload);
$deleteJob->status = 'succeeded';
$deleteJob->save();
$assert(CloudSnapshotService::listing($instance, $actor)['used_slots'] === 1,
'A confirmed deletion did not release exactly one snapshot slot.');
$failed = CloudSnapshotService::create($instance, $actor, [
'name' => 'will-fail', 'consistency_mode' => 'filesystem',
]);
$failedJob = $failed['job'];
CloudSnapshotService::finishJobLocked($failedJob, 'failed', null,
CloudSecrets::decrypt((string) $failedJob->payload_cipher));
$failedJob->status = 'failed';
$failedJob->save();
$assert(CloudInstanceSnapshot::find((int) $failedJob->snapshot_id)->state
=== 'failed'
&& CloudSnapshotService::listing($instance, $actor)['used_slots'] === 1,
'A definitive create failure kept a slot or lost its audit state.');
$expect(static fn () => CloudSnapshotService::create($instance,
['user_id' => 11, 'is_admin' => false, 'is_super_admin' => false,
'can_manage' => false],
['name' => 'foreign', 'consistency_mode' => 'crash']),
403, 'A different tenant created a snapshot for this instance.');
$expect(static fn () => CloudSnapshotService::listing($instance,
['user_id' => 11, 'is_admin' => false]), 404,
'A different tenant listed snapshots for this instance.');
$expect(static fn () => CloudSnapshotService::restore($instance, $actor,
(string) $second['snapshot']['snapshot_key'],
['confirmation_name' => 'wrong-name']), 422,
'Snapshot restore accepted a mismatched instance confirmation.');
$qga = CloudOperationErrorCatalog::workerSummary('snapshot_create',
'failed', ['error_code' => 'snapshot_guest_agent_unavailable']);
$assert($qga === 'snapshot_guest_agent_unavailable'
&& str_contains((string) CloudOperationErrorCatalog::userMessage(
'snapshot_create', 'failed', $qga), 'QEMU Guest Agent'),
'QGA unavailability was not preserved as a safe, actionable job error.');
$secondKey = (string) $second['snapshot']['snapshot_key'];
$restoreFailed = CloudSnapshotService::restore($instance, $actor,
$secondKey, ['confirmation_name' => 'vm-snap']);
$restoreFailedJob = $restoreFailed['job'];
CloudSnapshotService::finishJobLocked($restoreFailedJob, 'failed', null,
CloudSecrets::decrypt((string) $restoreFailedJob->payload_cipher));
$restoreFailedJob->status = 'failed';
$restoreFailedJob->save();
$assert(CloudInstanceSnapshot::find((int) $restoreFailedJob->snapshot_id)->state
=== 'ready',
'A definitive restore failure did not return the source snapshot to ready.');
$deleteUncertain = CloudSnapshotService::delete($instance, $actor,
$secondKey, ['confirmation_name' => 'vm-snap']);
$deleteUncertainJob = $deleteUncertain['job'];
$deleteUncertainPayload = CloudSecrets::decrypt(
(string) $deleteUncertainJob->payload_cipher
);
CloudSnapshotService::finishJobLocked($deleteUncertainJob, 'uncertain', null,
$deleteUncertainPayload);
$deleteUncertainJob->status = 'uncertain';
$deleteUncertainJob->save();
$assert(CloudInstanceSnapshot::find((int) $deleteUncertainJob->snapshot_id)->state
=== 'uncertain',
'An ambiguous delete did not preserve its uncertain snapshot slot.');
CloudSnapshotService::finishJobLocked($deleteUncertainJob, 'succeeded', [
'snapshot_key' => $secondKey, 'deleted' => true,
], $deleteUncertainPayload);
$deleteUncertainJob->status = 'succeeded';
$deleteUncertainJob->save();
$cascade = CloudSnapshotService::create($instance, $actor, [
'name' => 'delete-with-instance', 'consistency_mode' => 'crash',
]);
$cascadeJob = $cascade['job'];
$cascadeKey = (string) $cascade['snapshot']['snapshot_key'];
CloudSnapshotService::finishJobLocked($cascadeJob, 'succeeded', [
'snapshot_key' => $cascadeKey, 'created' => true,
'consistency' => 'crash',
], CloudSecrets::decrypt((string) $cascadeJob->payload_cipher));
$cascadeJob->status = 'succeeded';
$cascadeJob->save();
$deleteVm = new CloudJob();
$deleteVm->operation_id = str_repeat('d', 32);
$deleteVm->instance_id = $instance->id;
$deleteVm->kind = 'delete';
CloudSnapshotService::finishInstanceDeleteLocked($deleteVm, 'uncertain');
$assert(CloudInstanceSnapshot::where('snapshot_key', $cascadeKey)
->value('state') === 'uncertain',
'An ambiguous VM deletion did not preserve its snapshot cleanup uncertainty.');
CloudSnapshotService::finishInstanceDeleteLocked($deleteVm, 'failed');
$assert(CloudInstanceSnapshot::where('snapshot_key', $cascadeKey)
->value('state') === 'ready',
'A definitive VM-delete failure did not restore its snapshot catalog state.');
CloudSnapshotService::finishInstanceDeleteLocked($deleteVm, 'succeeded');
$assert(CloudInstanceSnapshot::where('instance_id', $instance->id)
->whereIn('state', ['creating', 'ready', 'restoring', 'deleting', 'uncertain'])
->count() === 0,
'Confirmed VM deletion did not cascade its PHP snapshot catalog.');
$delegated = CloudSnapshotService::create($instance, [
'user_id' => 99, 'is_admin' => true, 'is_super_admin' => true,
'can_manage' => true,
], ['name' => 'admin-audit', 'consistency_mode' => 'crash']);
$delegatedJob = $delegated['job'];
$delegatedSnapshot = CloudInstanceSnapshot::find(
(int) $delegatedJob->snapshot_id
);
$delegatedPayload = CloudSecrets::decrypt(
(string) $delegatedJob->payload_cipher
);
$assert((int) $delegatedJob->user_id === 99
&& (int) $delegatedSnapshot->user_id === (int) $instance->user_id
&& (int) $delegatedPayload['owner_id'] === (int) $instance->user_id,
'Administrator snapshot action lost actor/owner audit separation.');
$ownerView = CloudSnapshotService::listing($instance, $actor);
$adminView = CloudSnapshotService::listing($instance, [
'user_id' => 99, 'is_admin' => true, 'is_super_admin' => true,
'can_manage' => true,
]);
$assert($ownerView['snapshots'][0]['job_id'] === null
&& $ownerView['active_job'] !== null
&& !array_key_exists('id', $ownerView['active_job'])
&& (string) ($adminView['active_job']['id'] ?? '')
=== (string) $delegatedJob->id,
'Tenant listing exposed an administrator-scoped job as directly pollable.');
CloudSnapshotService::finishJobLocked($delegatedJob, 'failed', null,
$delegatedPayload);
$delegatedJob->status = 'failed';
$delegatedJob->save();
$assert(CloudInstanceSnapshot::find((int) $delegatedJob->snapshot_id)->state
=== 'failed',
'Administrator snapshot result could not reconcile against its owner.');
$settingsAfterDelete = CloudJobService::enqueue($instance, $actor,
'instance_settings', 'POST', '/settings', [
'hostname' => 'allowed.example', 'confirm_restart' => true,
]);
$assert((string) $settingsAfterDelete->status === 'queued',
'Deleting the final snapshot did not release instance settings admission.');
$settingsAfterDelete->status = 'failed';
$settingsAfterDelete->save();
$reinstallAfterDelete = CloudJobService::enqueue($instance, $actor,
'reinstall', 'POST', '/reinstall', [
'template_id' => 'debian-12',
]);
$assert((string) $reinstallAfterDelete->status === 'queued',
'Deleting the final snapshot did not release reinstall admission.');
echo "Cloud snapshot catalog, admission and worker state machine passed.\n";
}