You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudSnapshotWorkflowTest.php

497 lines
23 KiB
PHP

<?php
namespace App\Services\Cloud {
// Isolated test cipher only; production CloudSecrets is never loaded here.
final class CloudSecrets
{
private static array $values = [];
public static function encrypt(array $value, string $purpose): string
{
$key = bin2hex(random_bytes(16));
self::$values[$key] = $value;
return 'snapshot-test:' . $key;
}
public static function decrypt(string $value, string $purpose): array
{
return self::$values[substr($value, 14)];
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudInstance;
use App\Models\CloudInstanceSnapshot;
use App\Models\CloudJob;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudJobService;
use App\Services\Cloud\CloudOperationErrorCatalog;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\CloudSnapshotService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function ($condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$terminalize = static function (CloudJob $job, string $outcome,
bool $effect = false, bool $safe = false): void {
$now = time();
$job->phase = 'terminal';
$job->outcome = $outcome;
$job->status = $outcome;
$job->attempts = 1;
$job->dispatch_attempts = 1;
$job->dispatch_started_at ??= $now - 1;
$job->dispatched_at ??= $now - 1;
$job->agent_accepted_at ??= $now - 1;
$job->receipt_deadline_at ??= $now + 900;
$job->terminal_at = $now;
$job->effect_may_have_applied = $effect ? 1 : 0;
$job->safe_to_resubmit = $safe ? 1 : 0;
$job->effect_resolution = $effect ? 'unknown' : 'not_applicable';
$job->lease_token = null;
$job->lease_until = 0;
$job->available_at = 0;
$job->updated_at = $now;
$job->save();
};
$expect = static function (callable $callable, int $status,
string $message) use ($assert): void {
try {
$callable();
} catch (CloudError $error) {
$assert($error->httpStatus === $status,
$message . ' Unexpected status: ' . $error->httpStatus);
return;
}
throw new RuntimeException($message);
};
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_instances', static function (Blueprint $table): void {
$table->increments('id');
$table->string('uuid');
$table->integer('host_id');
$table->integer('user_id');
$table->string('name');
$table->string('status');
$table->string('service_status');
$table->string('network_status');
$table->integer('expires_at')->nullable();
$table->text('spec_json');
$table->text('sealed_credential')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_instance_snapshots', static function (Blueprint $table): void {
$table->increments('id');
$table->string('snapshot_key')->unique();
$table->integer('instance_id');
$table->integer('user_id');
$table->integer('host_id');
$table->string('name');
$table->string('state');
$table->string('consistency');
$table->integer('size_bytes')->nullable();
$table->string('last_operation_id')->nullable();
$table->integer('ready_at')->nullable();
$table->integer('last_restored_at')->nullable();
$table->integer('deleted_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_templates', static function (Blueprint $table): void {
$table->string('id')->primary();
$table->string('name');
$table->text('sealed_source_url');
$table->text('startup_script');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
});
$capsule->getConnection('default')->table('cloud_templates')->insert([
'id' => 'debian-12',
'name' => 'Debian 12',
'sealed_source_url' => CloudSecrets::encrypt([
'url' => 'https://example.invalid/debian-12.qcow2',
], 'template-source'),
'startup_script' => '',
'revision' => 1,
'generation_token' => str_repeat('a', 32),
'state' => 'active',
]);
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->string('request_idempotency_key');
$table->string('request_fingerprint');
$table->integer('parent_job_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('invoice_id')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('user_id');
$table->integer('host_id')->nullable();
$table->string('kind');
$table->string('execution_mode')->default('agent');
$table->string('phase')->default('accepted');
$table->string('outcome')->nullable();
$table->string('status');
$table->text('sealed_payload');
$table->integer('attempts')->default(0);
$table->integer('dispatch_attempts')->default(0);
$table->integer('dispatch_deadline_at')->default(0);
$table->integer('dispatch_started_at')->nullable();
$table->integer('agent_accepted_at')->nullable();
$table->integer('receipt_deadline_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution')->default('not_applicable');
$table->integer('effect_resolved_at')->nullable();
$table->integer('effect_resolved_by')->nullable();
$table->string('effect_resolution_reason')->default('');
$table->string('completion_event_id')->nullable();
$table->integer('read_at')->nullable();
$table->integer('dispatched_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at');
$table->string('error_code')->nullable();
$table->boolean('finalization_incident')->default(0);
$table->string('finalization_error_code')->nullable();
$table->text('result_json')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$instance = new CloudInstance();
$instance->uuid = '11111111-1111-4111-8111-111111111111';
$instance->host_id = 7;
$instance->user_id = 10;
$instance->name = 'vm-snap';
$instance->status = 'active';
$instance->service_status = 'exempt';
$instance->network_status = 'active';
$instance->expires_at = null;
$instance->spec_json = json_encode([
'template_id' => 'debian-12', 'os_source' => 'cloud_template',
'os_name' => 'Debian 12', 'login_name' => 'root',
'guest_hostname' => 'snapshot-old',
'public_ip_cidr' => '192.0.2.10/24', 'region' => 'HKG',
'vcpu' => 2, 'memory_mib' => 2048, 'disk_gib' => 40,
'agent_metadata_version' => 1,
], JSON_THROW_ON_ERROR);
$instance->sealed_credential = CloudSecrets::encrypt([
'password' => 'old-password', 'login_name' => 'root',
], 'instance-credential');
$instance->created_at = $instance->updated_at = time();
$instance->save();
$actor = ['user_id' => 10, 'is_admin' => false,
'is_super_admin' => false, 'can_manage' => false];
$first = CloudSnapshotService::create($instance, $actor, [
'name' => '发布前', 'consistency_mode' => 'filesystem',
]);
$firstJob = $first['job'];
$firstPayload = CloudSecrets::decrypt(
(string) $firstJob->sealed_payload, 'job-payload'
);
$firstKey = (string) $first['snapshot']['snapshot_key'];
$assert(preg_match('/^[a-f0-9]{32}$/D', $firstKey) === 1
&& $firstPayload['method'] === 'POST'
&& $firstPayload['request_target']
=== '/v1/vms/' . $instance->uuid . '/snapshots'
&& $firstPayload['body'] === [
'snapshot_key' => $firstKey,
'name' => '发布前',
'consistency_mode' => 'filesystem',
]
&& (int) $firstJob->snapshot_id > 0,
'Snapshot create did not persist an opaque key and bounded Agent contract.');
CloudSnapshotService::finishJobLocked($firstJob, 'succeeded', [
'snapshot_key' => $firstKey, 'created' => true,
'consistency' => 'crash', 'size_bytes' => 4096,
], $firstPayload);
$terminalize($firstJob, 'succeeded');
$listed = CloudSnapshotService::listing($instance, $actor);
$assert($listed['max_snapshots'] === 2 && $listed['used_slots'] === 1
&& $listed['can_create'] === true && $listed['can_mutate'] === true
&& count($listed['snapshots']) === 1
&& $listed['snapshots'][0]['state'] === 'ready'
&& $listed['snapshots'][0]['consistency'] === 'crash'
&& $listed['snapshots'][0]['size_bytes'] === 4096,
'Database-only snapshot listing did not expose the actual downgraded consistency.');
$storedFirst = CloudInstanceSnapshot::find((int) $firstJob->snapshot_id);
$assert(!array_key_exists('sealed_credential', $storedFirst->getAttributes())
&& !array_key_exists('system_metadata_json', $storedFirst->getAttributes()),
'Snapshot catalog duplicated an instance credential or seed metadata.');
$invalidUpgrade = CloudSnapshotService::create($instance, $actor, [
'name' => 'invalid-upgrade', 'consistency_mode' => 'crash',
]);
$invalidUpgradeJob = $invalidUpgrade['job'];
$invalidUpgradePayload = CloudSecrets::decrypt(
(string) $invalidUpgradeJob->sealed_payload, 'job-payload'
);
$expect(static fn () => CloudSnapshotService::finishJobLocked(
$invalidUpgradeJob, 'succeeded', [
'snapshot_key' => $invalidUpgrade['snapshot']['snapshot_key'],
'created' => true, 'consistency' => 'filesystem',
], $invalidUpgradePayload
), 502, 'A crash-consistent request accepted an Agent filesystem upgrade.');
CloudSnapshotService::finishJobLocked($invalidUpgradeJob, 'failed', null,
$invalidUpgradePayload);
$terminalize($invalidUpgradeJob, 'failed', false, true);
$settingsWithSnapshot = CloudJobService::enqueue($instance, $actor,
'instance_settings', 'POST', '/settings', [
'hostname' => 'current.example', 'confirm_restart' => true,
]);
$assert((string) $settingsWithSnapshot->status === 'queued',
'A ready snapshot incorrectly blocked a current-seed settings mutation.');
$terminalize($settingsWithSnapshot, 'failed', false, true);
$expect(static fn () => CloudJobService::enqueue($instance, $actor,
'reinstall', 'POST', '/reinstall', [
'template_id' => 'debian-12',
]), 409, 'A retained snapshot allowed a reinstall mutation.');
$second = CloudSnapshotService::create($instance, $actor, [
'name' => 'Crash point', 'consistency_mode' => 'crash',
]);
$secondJob = $second['job'];
$secondPayload = CloudSecrets::decrypt(
(string) $secondJob->sealed_payload, 'job-payload'
);
CloudSnapshotService::finishJobLocked($secondJob, 'succeeded', [
'snapshot_key' => $second['snapshot']['snapshot_key'],
'created' => true, 'consistency' => 'crash',
], $secondPayload);
$terminalize($secondJob, 'succeeded');
$expect(static fn () => CloudSnapshotService::create($instance, $actor, [
'name' => 'third', 'consistency_mode' => 'filesystem',
]), 409, 'A third live snapshot bypassed the per-instance limit.');
$restore = CloudSnapshotService::restore($instance, $actor, $firstKey,
['confirmation_name' => 'vm-snap']);
$restoreJob = $restore['job'];
$restorePayload = CloudSecrets::decrypt(
(string) $restoreJob->sealed_payload, 'job-payload'
);
$assert($restorePayload['method'] === 'POST'
&& $restorePayload['request_target'] === '/v1/vms/' . $instance->uuid
. '/snapshots/' . $firstKey . '/restore'
&& $restorePayload['body'] === [],
'Snapshot restore engine contract changed unexpectedly.');
CloudSnapshotService::finishJobLocked($restoreJob, 'uncertain', null,
$restorePayload);
$terminalize($restoreJob, 'failed', true, false);
$uncertain = CloudSnapshotService::listing($instance, $actor);
$assert($restoreJob->phase === 'terminal'
&& $restoreJob->outcome === 'failed'
&& $restoreJob->effect_resolution === 'unknown'
&& ($uncertain['snapshots'][1]['state'] === 'uncertain'
|| $uncertain['snapshots'][0]['state'] === 'uncertain'),
'An unknown-effect restore did not become a terminal failed task with a visible resource fence.');
$credentialBeforeRestore = (string) $instance->sealed_credential;
$specBeforeRestore = (string) $instance->spec_json;
// Simulate a later administrator inspection proving that the restore did
// apply. It resolves only the resource fence; the immutable task outcome
// remains failed and is never replayed or rewritten to success.
CloudInstanceSnapshot::where('id', (int) $restoreJob->snapshot_id)
->update(['state' => 'ready', 'last_restored_at' => time(),
'updated_at' => time()]);
$restoreJob->effect_resolution = 'applied';
$restoreJob->effect_resolved_at = time();
$restoreJob->effect_resolved_by = 99;
$restoreJob->effect_resolution_reason =
'Read-only host inspection confirmed snapshot restore effect.';
$restoreJob->save();
$instance->refresh();
$assert((string) $instance->sealed_credential === $credentialBeforeRestore
&& (string) $instance->spec_json === $specBeforeRestore,
'Snapshot restore rewrote panel credentials or seed metadata.');
$delete = CloudSnapshotService::delete($instance, $actor, $firstKey,
['confirmation_name' => 'vm-snap']);
$deleteJob = $delete['job'];
$deletePayload = CloudSecrets::decrypt(
(string) $deleteJob->sealed_payload, 'job-payload'
);
$assert($deletePayload['method'] === 'DELETE'
&& $deletePayload['request_target'] === '/v1/vms/' . $instance->uuid
. '/snapshots/' . $firstKey
&& $deletePayload['body'] === [],
'Snapshot delete engine contract changed unexpectedly.');
CloudSnapshotService::finishJobLocked($deleteJob, 'succeeded', [
'snapshot_key' => $firstKey, 'deleted' => true,
], $deletePayload);
$terminalize($deleteJob, 'succeeded');
$assert(CloudSnapshotService::listing($instance, $actor)['used_slots'] === 1,
'A confirmed deletion did not release exactly one snapshot slot.');
$failed = CloudSnapshotService::create($instance, $actor, [
'name' => 'will-fail', 'consistency_mode' => 'filesystem',
]);
$failedJob = $failed['job'];
CloudSnapshotService::finishJobLocked($failedJob, 'failed', null,
CloudSecrets::decrypt((string) $failedJob->sealed_payload,
'job-payload'));
$terminalize($failedJob, 'failed', false, true);
$assert(CloudInstanceSnapshot::find((int) $failedJob->snapshot_id)->state
=== 'failed'
&& CloudSnapshotService::listing($instance, $actor)['used_slots'] === 1,
'A definitive create failure kept a slot or lost its audit state.');
$expect(static fn () => CloudSnapshotService::create($instance,
['user_id' => 11, 'is_admin' => false, 'is_super_admin' => false,
'can_manage' => false],
['name' => 'foreign', 'consistency_mode' => 'crash']),
403, 'A different tenant created a snapshot for this instance.');
$expect(static fn () => CloudSnapshotService::listing($instance,
['user_id' => 11, 'is_admin' => false]), 404,
'A different tenant listed snapshots for this instance.');
$expect(static fn () => CloudSnapshotService::restore($instance, $actor,
(string) $second['snapshot']['snapshot_key'],
['confirmation_name' => 'wrong-name']), 422,
'Snapshot restore accepted a mismatched instance confirmation.');
$qga = CloudOperationErrorCatalog::workerSummary('snapshot_create',
'failed', ['error_code' => 'snapshot_guest_agent_unavailable']);
$assert($qga === 'snapshot_guest_agent_unavailable'
&& str_contains((string) CloudOperationErrorCatalog::userMessage(
'snapshot_create', 'failed', $qga), 'QEMU Guest Agent'),
'Legacy QGA unavailability was not preserved as a safe update diagnostic.');
$freeze = CloudOperationErrorCatalog::workerSummary('snapshot_create',
'failed', ['error_code' => 'snapshot_filesystem_freeze_failed']);
$assert($freeze === 'snapshot_filesystem_freeze_failed'
&& str_contains((string) CloudOperationErrorCatalog::userMessage(
'snapshot_create', 'failed', $freeze), '文件系统冻结失败'),
'A responsive-agent freeze failure was not preserved as a distinct safe error.');
$secondKey = (string) $second['snapshot']['snapshot_key'];
$restoreFailed = CloudSnapshotService::restore($instance, $actor,
$secondKey, ['confirmation_name' => 'vm-snap']);
$restoreFailedJob = $restoreFailed['job'];
CloudSnapshotService::finishJobLocked($restoreFailedJob, 'failed', null,
CloudSecrets::decrypt((string) $restoreFailedJob->sealed_payload,
'job-payload'));
$terminalize($restoreFailedJob, 'failed', false, true);
$assert(CloudInstanceSnapshot::find((int) $restoreFailedJob->snapshot_id)->state
=== 'ready',
'A definitive restore failure did not return the source snapshot to ready.');
$deleteUncertain = CloudSnapshotService::delete($instance, $actor,
$secondKey, ['confirmation_name' => 'vm-snap']);
$deleteUncertainJob = $deleteUncertain['job'];
$deleteUncertainPayload = CloudSecrets::decrypt(
(string) $deleteUncertainJob->sealed_payload,
'job-payload'
);
CloudSnapshotService::finishJobLocked($deleteUncertainJob, 'uncertain', null,
$deleteUncertainPayload);
$terminalize($deleteUncertainJob, 'failed', true, false);
$assert(CloudInstanceSnapshot::find((int) $deleteUncertainJob->snapshot_id)->state
=== 'uncertain'
&& $deleteUncertainJob->phase === 'terminal'
&& $deleteUncertainJob->outcome === 'failed',
'An unknown-effect delete did not preserve its resource fence as a terminal failure.');
CloudInstanceSnapshot::where('id', (int) $deleteUncertainJob->snapshot_id)
->update(['state' => 'ready', 'updated_at' => time()]);
$deleteUncertainJob->effect_resolution = 'not_applied';
$deleteUncertainJob->effect_resolved_at = time();
$deleteUncertainJob->effect_resolved_by = 99;
$deleteUncertainJob->effect_resolution_reason =
'Read-only host inspection confirmed snapshot still exists.';
$deleteUncertainJob->save();
$cascade = CloudSnapshotService::create($instance, $actor, [
'name' => 'delete-with-instance', 'consistency_mode' => 'crash',
]);
$cascadeJob = $cascade['job'];
$cascadeKey = (string) $cascade['snapshot']['snapshot_key'];
CloudSnapshotService::finishJobLocked($cascadeJob, 'succeeded', [
'snapshot_key' => $cascadeKey, 'created' => true,
'consistency' => 'crash',
], CloudSecrets::decrypt((string) $cascadeJob->sealed_payload,
'job-payload'));
$terminalize($cascadeJob, 'succeeded');
$deleteVm = new CloudJob();
$deleteVm->operation_id = str_repeat('d', 32);
$deleteVm->instance_id = $instance->id;
$deleteVm->kind = 'delete';
$deleteVm->execution_mode = 'agent';
CloudSnapshotService::finishInstanceDeleteLocked($deleteVm, 'uncertain');
$assert(CloudInstanceSnapshot::where('snapshot_key', $cascadeKey)
->value('state') === 'uncertain',
'An ambiguous VM deletion did not preserve its snapshot cleanup uncertainty.');
CloudSnapshotService::finishInstanceDeleteLocked($deleteVm, 'failed');
$assert(CloudInstanceSnapshot::where('snapshot_key', $cascadeKey)
->value('state') === 'ready',
'A definitive VM-delete failure did not restore its snapshot catalog state.');
CloudSnapshotService::finishInstanceDeleteLocked($deleteVm, 'succeeded');
$assert(CloudInstanceSnapshot::where('instance_id', $instance->id)
->whereIn('state', ['creating', 'ready', 'restoring', 'deleting', 'uncertain'])
->count() === 0,
'Confirmed VM deletion did not cascade its PHP snapshot catalog.');
$delegated = CloudSnapshotService::create($instance, [
'user_id' => 99, 'is_admin' => true, 'is_super_admin' => true,
'can_manage' => true,
], ['name' => 'admin-audit', 'consistency_mode' => 'crash']);
$delegatedJob = $delegated['job'];
$delegatedSnapshot = CloudInstanceSnapshot::find(
(int) $delegatedJob->snapshot_id
);
$delegatedPayload = CloudSecrets::decrypt(
(string) $delegatedJob->sealed_payload,
'job-payload'
);
$assert((int) $delegatedJob->user_id === 99
&& (int) $delegatedSnapshot->user_id === (int) $instance->user_id
&& (int) $delegatedPayload['owner_id'] === (int) $instance->user_id,
'Administrator snapshot action lost actor/owner audit separation.');
$ownerView = CloudSnapshotService::listing($instance, $actor);
$adminView = CloudSnapshotService::listing($instance, [
'user_id' => 99, 'is_admin' => true, 'is_super_admin' => true,
'can_manage' => true,
]);
$assert($ownerView['snapshots'][0]['task_id'] === null
&& $ownerView['active_job'] !== null
&& !array_key_exists('id', $ownerView['active_job'])
&& (string) ($adminView['active_job']['id'] ?? '')
=== (string) $delegatedJob->id,
'Tenant listing exposed an administrator-scoped job as directly pollable.');
CloudSnapshotService::finishJobLocked($delegatedJob, 'failed', null,
$delegatedPayload);
$terminalize($delegatedJob, 'failed', false, true);
$assert(CloudInstanceSnapshot::find((int) $delegatedJob->snapshot_id)->state
=== 'failed',
'Administrator snapshot result could not reconcile against its owner.');
$settingsAfterDelete = CloudJobService::enqueue($instance, $actor,
'instance_settings', 'POST', '/settings', [
'hostname' => 'allowed.example', 'confirm_restart' => true,
]);
$assert((string) $settingsAfterDelete->status === 'queued',
'Deleting the final snapshot did not release instance settings admission.');
$terminalize($settingsAfterDelete, 'failed', false, true);
$reinstallAfterDelete = CloudJobService::enqueue($instance, $actor,
'reinstall', 'POST', '/reinstall', [
'template_id' => 'debian-12',
]);
$assert((string) $reinstallAfterDelete->status === 'queued',
'Deleting the final snapshot did not release reinstall admission.');
echo "Cloud snapshot catalog, admission and worker state machine passed.\n";
}