You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudSnapshotWorkflowTest.php

700 lines
32 KiB
PHTML

<?php
namespace App\Services\Cloud {
// Isolated test cipher only; production CloudSecrets is never loaded here.
final class CloudSecrets
{
private static array $values = [];
public static function encrypt(array $value, string $purpose): string
{
$key = bin2hex(random_bytes(16));
self::$values[$key] = $value;
return 'snapshot-test:' . $key;
}
public static function decrypt(string $value, string $purpose): array
{
return self::$values[substr($value, 14)];
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudInstance;
use App\Models\CloudInstanceSnapshot;
use App\Models\CloudJob;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudInput;
use App\Services\Cloud\CloudJobService;
use App\Services\Cloud\CloudOperationCatalog;
use App\Services\Cloud\CloudOperationErrorCatalog;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\CloudSnapshotService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function ($condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$terminalize = static function (CloudJob $job, string $outcome,
bool $effect = false, bool $safe = false): void {
$now = time();
$job->phase = 'terminal';
$job->outcome = $outcome;
$job->status = $outcome;
$job->attempts = 1;
$job->dispatch_attempts = 1;
$job->dispatch_started_at ??= $now - 1;
$job->dispatched_at ??= $now - 1;
$job->agent_accepted_at ??= $now - 1;
$job->receipt_deadline_at ??= $now + 900;
$job->terminal_at = $now;
$job->effect_may_have_applied = $effect ? 1 : 0;
$job->safe_to_resubmit = $safe ? 1 : 0;
$job->effect_resolution = $effect ? 'unknown' : 'not_applicable';
$job->lease_token = null;
$job->lease_until = 0;
$job->available_at = 0;
$job->updated_at = $now;
$job->save();
};
$expect = static function (callable $callable, int $status,
string $message) use ($assert): void {
try {
$callable();
} catch (CloudError $error) {
$assert($error->httpStatus === $status,
$message . ' Unexpected status: ' . $error->httpStatus);
return;
}
throw new RuntimeException($message);
};
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_instances', static function (Blueprint $table): void {
$table->increments('id');
$table->string('uuid');
$table->integer('host_id');
$table->integer('user_id');
$table->string('name');
$table->string('status');
$table->string('service_status');
$table->string('network_status');
$table->integer('expires_at')->nullable();
$table->text('spec_json');
$table->text('sealed_credential')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_instance_snapshots', static function (Blueprint $table): void {
$table->increments('id');
$table->string('snapshot_key')->unique();
$table->integer('instance_id');
$table->integer('user_id');
$table->integer('host_id');
$table->string('name');
$table->string('state');
$table->string('consistency');
$table->integer('size_bytes')->nullable();
$table->string('last_operation_id')->nullable();
$table->integer('ready_at')->nullable();
$table->integer('last_restored_at')->nullable();
$table->integer('deleted_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_hosts', static function (Blueprint $table): void {
$table->integer('id')->primary();
$table->boolean('enabled');
$table->string('default_network');
});
$schema->create('cloud_host_agents', static function (Blueprint $table): void {
$table->integer('host_id')->primary();
$table->boolean('enabled');
$table->string('last_test_status');
$table->integer('revision');
$table->integer('tested_revision');
});
$schema->create('cloud_host_storage_configurations', static function (
Blueprint $table
): void {
$table->integer('host_id')->primary();
$table->string('state');
});
$schema->create('cloud_templates', static function (Blueprint $table): void {
$table->string('id')->primary();
$table->string('catalog_key')->default('test-catalog');
$table->string('version_label')->default('test-version');
$table->boolean('is_default')->default(true);
$table->string('name');
$table->integer('image_bytes');
$table->string('sha256');
$table->text('startup_script');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
});
$schema->create('cloud_template_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('template_id');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('image_bytes');
$table->string('sha256');
$table->string('operation_id');
$table->integer('job_id')->nullable();
$table->string('error_code');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'template_id']);
});
$capsule->getConnection('default')->table('cloud_hosts')->insert([
'id' => 7, 'enabled' => 1, 'default_network' => 'bridge:br0',
]);
$capsule->getConnection('default')->table('cloud_host_agents')->insert([
'host_id' => 7, 'enabled' => 1, 'last_test_status' => 'healthy',
'revision' => 1, 'tested_revision' => 1,
]);
$capsule->getConnection('default')->table(
'cloud_host_storage_configurations'
)->insert(['host_id' => 7, 'state' => 'applied']);
$capsule->getConnection('default')->table('cloud_templates')->insert([
'id' => 'debian-12',
'name' => 'Debian 12',
'image_bytes' => 4096,
'sha256' => hash('sha256', 'snapshot-template'),
'startup_script' => '',
'revision' => 1,
'generation_token' => str_repeat('a', 32),
'state' => 'active',
]);
$capsule->getConnection('default')->table(
'cloud_template_replicas'
)->insert([
'host_id' => 7,
'template_id' => 'debian-12',
'revision' => 1,
'generation_token' => str_repeat('a', 32),
'state' => 'active',
'image_bytes' => 4096,
'sha256' => hash('sha256', 'snapshot-template'),
'operation_id' => 'snapshot-fixture-template-import',
'job_id' => null,
'error_code' => '',
'created_at' => time(),
'updated_at' => time(),
]);
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->string('request_idempotency_key');
$table->string('request_fingerprint');
$table->integer('parent_job_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('invoice_id')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('user_id');
$table->integer('host_id')->nullable();
$table->string('kind');
$table->string('execution_mode')->default('agent');
$table->string('phase')->default('accepted');
$table->string('outcome')->nullable();
$table->string('status');
$table->text('sealed_payload');
$table->integer('attempts')->default(0);
$table->integer('dispatch_attempts')->default(0);
$table->integer('dispatch_deadline_at')->default(0);
$table->integer('dispatch_started_at')->nullable();
$table->integer('agent_accepted_at')->nullable();
$table->integer('receipt_deadline_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution')->default('not_applicable');
$table->integer('effect_resolved_at')->nullable();
$table->integer('effect_resolved_by')->nullable();
$table->string('effect_resolution_reason')->default('');
$table->string('completion_event_id')->nullable();
$table->integer('read_at')->nullable();
$table->integer('dispatched_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at');
$table->string('error_code')->nullable();
$table->boolean('finalization_incident')->default(0);
$table->string('finalization_error_code')->nullable();
$table->text('result_json')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$instance = new CloudInstance();
$instance->uuid = '11111111-1111-4111-8111-111111111111';
$instance->host_id = 7;
$instance->user_id = 10;
$instance->name = 'vm-snap';
$instance->status = 'active';
$instance->service_status = 'exempt';
$instance->network_status = 'active';
$instance->expires_at = null;
$instance->spec_json = json_encode([
'managed' => true,
'template_id' => 'debian-12', 'os_source' => 'cloud_template',
'os_name' => 'Debian 12', 'login_name' => 'root',
'guest_hostname' => 'snapshot-old',
'public_network' => [
'contract' => 1,
'network_selector' => 'bridge:br0',
'ipv4' => [
'pool_key' => str_repeat('1', 32),
'pool_cidr' => '8.8.8.0/24',
'address_cidr' => '8.8.8.10/24',
'gateway' => '8.8.8.1',
'gateway_on_link' => false,
],
'ipv6' => [
'pool_key' => str_repeat('2', 32),
'pool_cidr' => '2404:c140:156:3a::/64',
'address_cidr' => '2404:c140:156:3a::10/64',
'gateway' => '2404:c140:156::1',
'gateway_on_link' => true,
],
'dns_servers' => ['1.1.1.1', '2606:4700:4700::1111'],
],
'region' => 'HKG',
'vcpu' => 2, 'memory_mib' => 2048, 'disk_gib' => 40,
'agent_metadata_version' => 1,
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES);
$instance->sealed_credential = CloudSecrets::encrypt([
'password' => 'old-password', 'login_name' => 'root',
], 'instance-credential');
$instance->created_at = $instance->updated_at = time();
$instance->save();
$actor = ['user_id' => 10, 'is_admin' => false,
'is_super_admin' => false, 'can_manage' => false];
$first = CloudSnapshotService::create($instance, $actor, [
'name' => '发布前', 'consistency_mode' => 'filesystem',
]);
$firstJob = $first['job'];
$firstPayload = CloudSecrets::decrypt(
(string) $firstJob->sealed_payload, 'job-payload'
);
$firstKey = (string) $first['snapshot']['snapshot_key'];
$assert(preg_match('/^[a-f0-9]{32}$/D', $firstKey) === 1
&& $firstPayload['method'] === 'POST'
&& $firstPayload['request_target']
=== '/v1/vms/' . $instance->uuid . '/snapshots'
&& $firstPayload['body'] === [
'snapshot_key' => $firstKey,
'name' => '发布前',
'consistency_mode' => 'filesystem',
]
&& (int) $firstJob->snapshot_id > 0,
'Snapshot create did not persist an opaque key and bounded Agent contract.');
CloudSnapshotService::finishJobLocked($firstJob, 'succeeded', [
'snapshot_key' => $firstKey, 'created' => true,
'consistency' => 'crash', 'size_bytes' => 4096,
], $firstPayload);
$terminalize($firstJob, 'succeeded');
$listed = CloudSnapshotService::listing($instance, $actor);
$assert($listed['max_snapshots'] === 2 && $listed['used_slots'] === 1
&& $listed['can_create'] === true && $listed['can_mutate'] === true
&& count($listed['snapshots']) === 1
&& $listed['snapshots'][0]['state'] === 'ready'
&& $listed['snapshots'][0]['consistency'] === 'crash'
&& $listed['snapshots'][0]['size_bytes'] === 4096,
'Database-only snapshot listing did not expose the actual downgraded consistency.');
$storedFirst = CloudInstanceSnapshot::find((int) $firstJob->snapshot_id);
$assert(!array_key_exists('sealed_credential', $storedFirst->getAttributes())
&& !array_key_exists('system_metadata_json', $storedFirst->getAttributes()),
'Snapshot catalog duplicated an instance credential or seed metadata.');
$invalidUpgrade = CloudSnapshotService::create($instance, $actor, [
'name' => 'invalid-upgrade', 'consistency_mode' => 'crash',
]);
$invalidUpgradeJob = $invalidUpgrade['job'];
$invalidUpgradePayload = CloudSecrets::decrypt(
(string) $invalidUpgradeJob->sealed_payload, 'job-payload'
);
$expect(static fn () => CloudSnapshotService::finishJobLocked(
$invalidUpgradeJob, 'succeeded', [
'snapshot_key' => $invalidUpgrade['snapshot']['snapshot_key'],
'created' => true, 'consistency' => 'filesystem',
], $invalidUpgradePayload
), 502, 'A crash-consistent request accepted an Agent filesystem upgrade.');
CloudSnapshotService::finishJobLocked($invalidUpgradeJob, 'failed', null,
$invalidUpgradePayload);
$terminalize($invalidUpgradeJob, 'failed', false, true);
$settingsWithSnapshot = CloudJobService::enqueue($instance, $actor,
'instance_settings', 'POST', '/settings', [
'hostname' => 'current.example', 'confirm_restart' => true,
]);
$hostnameSettingsPayload = CloudSecrets::decrypt(
(string) $settingsWithSnapshot->sealed_payload, 'job-payload'
);
$assert((string) $settingsWithSnapshot->status === 'queued'
&& ($hostnameSettingsPayload['settings_fields'] ?? null)
=== ['hostname']
&& !array_key_exists('settings_fields',
$hostnameSettingsPayload['body'] ?? [])
&& CloudJobService::display($settingsWithSnapshot)['kind_label']
=== '修改主机名',
'A ready snapshot blocked settings or hostname task presentation was not persisted safely.');
$terminalize($settingsWithSnapshot, 'failed', false, true);
$legacyHostnamePayload = $hostnameSettingsPayload;
unset($legacyHostnamePayload['settings_fields']);
$legacyHostname = clone $settingsWithSnapshot;
$legacyHostname->sealed_payload = CloudSecrets::encrypt(
CloudInput::safeResult($legacyHostnamePayload), 'job-payload'
);
$assert(CloudJobService::display($legacyHostname)['kind_label']
=== '修改主机名',
'A completed legacy hostname task did not infer its safe detail label.');
$passwordSettings = CloudJobService::enqueue($instance, $actor,
'instance_settings', 'POST', '/settings', [
'root_password' => 'Aa1!Aa1!Aa1!', 'confirm_restart' => true,
]);
$passwordSettingsPayload = CloudSecrets::decrypt(
(string) $passwordSettings->sealed_payload, 'job-payload'
);
$passwordSafePayload = CloudInput::safeResult($passwordSettingsPayload);
$passwordSettings->sealed_payload = CloudSecrets::encrypt(
$passwordSafePayload, 'job-payload'
);
$terminalize($passwordSettings, 'failed', false, true);
$assert(($passwordSafePayload['settings_fields'] ?? null) === ['password']
&& !array_key_exists('root_password',
$passwordSafePayload['body'] ?? [])
&& CloudJobService::display($passwordSettings)['kind_label']
=== '重置 root 密码',
'Password settings lost their safe detail label or retained the password.');
$legacyPasswordPayload = $passwordSettingsPayload;
unset($legacyPasswordPayload['settings_fields']);
$legacyPassword = clone $passwordSettings;
$legacyPassword->phase = 'accepted';
$legacyPassword->outcome = null;
$legacyPassword->status = 'queued';
$legacyPassword->terminal_at = null;
$legacyPassword->sealed_payload = CloudSecrets::encrypt(
$legacyPasswordPayload, 'job-payload'
);
$legacyPasswordActiveLabel = CloudJobService::display(
$legacyPassword
)['kind_label'];
$legacyPassword->phase = 'terminal';
$legacyPassword->outcome = 'failed';
$legacyPassword->status = 'failed';
$legacyPassword->terminal_at = time();
$legacyPassword->sealed_payload = CloudSecrets::encrypt(
CloudInput::safeResult($legacyPasswordPayload), 'job-payload'
);
$assert($legacyPasswordActiveLabel === '重置 root 密码'
&& CloudJobService::display($legacyPassword)['kind_label']
=== '实例设置',
'Legacy settings labels were guessed after secret redaction.');
$sshBlob = pack('N', strlen('ssh-ed25519')) . 'ssh-ed25519'
. str_repeat("\x01", 32);
$sshSettings = CloudJobService::enqueue($instance, $actor,
'instance_settings', 'POST', '/settings', [
'ssh_public_key' => 'ssh-ed25519 ' . base64_encode($sshBlob),
'confirm_restart' => true,
]);
$sshSettingsPayload = CloudSecrets::decrypt(
(string) $sshSettings->sealed_payload, 'job-payload'
);
$sshSafePayload = CloudInput::safeResult($sshSettingsPayload);
$sshSettings->sealed_payload = CloudSecrets::encrypt(
$sshSafePayload, 'job-payload'
);
$terminalize($sshSettings, 'failed', false, true);
$assert(($sshSafePayload['settings_fields'] ?? null) === ['ssh_key']
&& !array_key_exists('ssh_public_key', $sshSafePayload['body'] ?? [])
&& CloudJobService::display($sshSettings)['kind_label']
=== '更新 SSH 公钥',
'SSH settings lost their safe detail label or retained the public key.');
$multiSettings = CloudJobService::enqueue($instance, $actor,
'instance_settings', 'POST', '/settings', [
'hostname' => 'multi.example',
'root_password' => 'Bb2@Bb2@Bb2@',
'ssh_public_key' => 'ssh-ed25519 ' . base64_encode($sshBlob),
'confirm_restart' => true,
]);
$multiSettingsPayload = CloudSecrets::decrypt(
(string) $multiSettings->sealed_payload, 'job-payload'
);
$assert(($multiSettingsPayload['settings_fields'] ?? null)
=== ['hostname', 'password', 'ssh_key']
&& CloudJobService::display($multiSettings)['kind_label']
=== '修改主机名、重置 root 密码、更新 SSH 公钥',
'Multi-field settings lost their stable order or detailed presentation.');
$terminalize($multiSettings, 'failed', false, true);
$expect(static fn () => CloudJobService::enqueue($instance, $actor,
'reinstall', 'POST', '/reinstall', [
'template_id' => 'debian-12',
]), 409, 'A retained snapshot allowed a reinstall mutation.');
$second = CloudSnapshotService::create($instance, $actor, [
'name' => 'Crash point', 'consistency_mode' => 'crash',
]);
$secondJob = $second['job'];
$secondPayload = CloudSecrets::decrypt(
(string) $secondJob->sealed_payload, 'job-payload'
);
CloudSnapshotService::finishJobLocked($secondJob, 'succeeded', [
'snapshot_key' => $second['snapshot']['snapshot_key'],
'created' => true, 'consistency' => 'crash',
], $secondPayload);
$terminalize($secondJob, 'succeeded');
$expect(static fn () => CloudSnapshotService::create($instance, $actor, [
'name' => 'third', 'consistency_mode' => 'filesystem',
]), 409, 'A third live snapshot bypassed the per-instance limit.');
$restore = CloudSnapshotService::restore($instance, $actor, $firstKey,
['confirmation_name' => 'vm-snap']);
$restoreJob = $restore['job'];
$restorePayload = CloudSecrets::decrypt(
(string) $restoreJob->sealed_payload, 'job-payload'
);
$assert($restorePayload['method'] === 'POST'
&& $restorePayload['request_target'] === '/v1/vms/' . $instance->uuid
. '/snapshots/' . $firstKey . '/restore'
&& $restorePayload['body'] === [],
'Snapshot restore engine contract changed unexpectedly.');
CloudSnapshotService::finishJobLocked($restoreJob, 'uncertain', null,
$restorePayload);
$terminalize($restoreJob, 'failed', true, false);
$uncertain = CloudSnapshotService::listing($instance, $actor);
$assert($restoreJob->phase === 'terminal'
&& $restoreJob->outcome === 'failed'
&& $restoreJob->effect_resolution === 'unknown'
&& ($uncertain['snapshots'][1]['state'] === 'uncertain'
|| $uncertain['snapshots'][0]['state'] === 'uncertain'),
'An unknown-effect restore did not become a terminal failed task with a visible resource fence.');
$credentialBeforeRestore = (string) $instance->sealed_credential;
$specBeforeRestore = (string) $instance->spec_json;
// Simulate a later administrator inspection proving that the restore did
// apply. It resolves only the resource fence; the immutable task outcome
// remains failed and is never replayed or rewritten to success.
CloudInstanceSnapshot::where('id', (int) $restoreJob->snapshot_id)
->update(['state' => 'ready', 'last_restored_at' => time(),
'updated_at' => time()]);
$restoreJob->effect_resolution = 'applied';
$restoreJob->effect_resolved_at = time();
$restoreJob->effect_resolved_by = 99;
$restoreJob->effect_resolution_reason =
'Read-only host inspection confirmed snapshot restore effect.';
$restoreJob->save();
$instance->refresh();
$assert((string) $instance->sealed_credential === $credentialBeforeRestore
&& (string) $instance->spec_json === $specBeforeRestore,
'Snapshot restore rewrote panel credentials or seed metadata.');
$delete = CloudSnapshotService::delete($instance, $actor, $firstKey,
['confirmation_name' => 'vm-snap']);
$deleteJob = $delete['job'];
$deletePayload = CloudSecrets::decrypt(
(string) $deleteJob->sealed_payload, 'job-payload'
);
$assert($deletePayload['method'] === 'DELETE'
&& $deletePayload['request_target'] === '/v1/vms/' . $instance->uuid
. '/snapshots/' . $firstKey
&& $deletePayload['body'] === [],
'Snapshot delete engine contract changed unexpectedly.');
CloudSnapshotService::finishJobLocked($deleteJob, 'succeeded', [
'snapshot_key' => $firstKey, 'deleted' => true,
], $deletePayload);
$terminalize($deleteJob, 'succeeded');
$assert(CloudSnapshotService::listing($instance, $actor)['used_slots'] === 1,
'A confirmed deletion did not release exactly one snapshot slot.');
$failed = CloudSnapshotService::create($instance, $actor, [
'name' => 'will-fail', 'consistency_mode' => 'filesystem',
]);
$failedJob = $failed['job'];
CloudSnapshotService::finishJobLocked($failedJob, 'failed', null,
CloudSecrets::decrypt((string) $failedJob->sealed_payload,
'job-payload'));
$terminalize($failedJob, 'failed', false, true);
$assert(CloudInstanceSnapshot::find((int) $failedJob->snapshot_id)->state
=== 'failed'
&& CloudSnapshotService::listing($instance, $actor)['used_slots'] === 1,
'A definitive create failure kept a slot or lost its audit state.');
$expect(static fn () => CloudSnapshotService::create($instance,
['user_id' => 11, 'is_admin' => false, 'is_super_admin' => false,
'can_manage' => false],
['name' => 'foreign', 'consistency_mode' => 'crash']),
403, 'A different tenant created a snapshot for this instance.');
$expect(static fn () => CloudSnapshotService::listing($instance,
['user_id' => 11, 'is_admin' => false]), 404,
'A different tenant listed snapshots for this instance.');
$expect(static fn () => CloudSnapshotService::restore($instance, $actor,
(string) $second['snapshot']['snapshot_key'],
['confirmation_name' => 'wrong-name']), 422,
'Snapshot restore accepted a mismatched instance confirmation.');
$qga = CloudOperationErrorCatalog::workerSummary('snapshot_create',
'failed', ['error_code' => 'snapshot_guest_agent_unavailable']);
$assert($qga === 'snapshot_guest_agent_unavailable'
&& str_contains((string) CloudOperationErrorCatalog::userMessage(
'snapshot_create', 'failed', $qga), 'QEMU Guest Agent'),
'Legacy QGA unavailability was not preserved as a safe update diagnostic.');
$freeze = CloudOperationErrorCatalog::workerSummary('snapshot_create',
'failed', ['error_code' => 'snapshot_filesystem_freeze_failed']);
$assert($freeze === 'snapshot_filesystem_freeze_failed'
&& str_contains((string) CloudOperationErrorCatalog::userMessage(
'snapshot_create', 'failed', $freeze), '文件系统冻结失败'),
'A responsive-agent freeze failure was not preserved as a distinct safe error.');
$secondKey = (string) $second['snapshot']['snapshot_key'];
$restoreFailed = CloudSnapshotService::restore($instance, $actor,
$secondKey, ['confirmation_name' => 'vm-snap']);
$restoreFailedJob = $restoreFailed['job'];
CloudSnapshotService::finishJobLocked($restoreFailedJob, 'failed', null,
CloudSecrets::decrypt((string) $restoreFailedJob->sealed_payload,
'job-payload'));
$terminalize($restoreFailedJob, 'failed', false, true);
$assert(CloudInstanceSnapshot::find((int) $restoreFailedJob->snapshot_id)->state
=== 'ready',
'A definitive restore failure did not return the source snapshot to ready.');
$deleteUncertain = CloudSnapshotService::delete($instance, $actor,
$secondKey, ['confirmation_name' => 'vm-snap']);
$deleteUncertainJob = $deleteUncertain['job'];
$deleteUncertainPayload = CloudSecrets::decrypt(
(string) $deleteUncertainJob->sealed_payload,
'job-payload'
);
CloudSnapshotService::finishJobLocked($deleteUncertainJob, 'uncertain', null,
$deleteUncertainPayload);
$terminalize($deleteUncertainJob, 'failed', true, false);
$assert(CloudInstanceSnapshot::find((int) $deleteUncertainJob->snapshot_id)->state
=== 'uncertain'
&& $deleteUncertainJob->phase === 'terminal'
&& $deleteUncertainJob->outcome === 'failed',
'An unknown-effect delete did not preserve its resource fence as a terminal failure.');
CloudInstanceSnapshot::where('id', (int) $deleteUncertainJob->snapshot_id)
->update(['state' => 'ready', 'updated_at' => time()]);
$deleteUncertainJob->effect_resolution = 'not_applied';
$deleteUncertainJob->effect_resolved_at = time();
$deleteUncertainJob->effect_resolved_by = 99;
$deleteUncertainJob->effect_resolution_reason =
'Read-only host inspection confirmed snapshot still exists.';
$deleteUncertainJob->save();
$cascade = CloudSnapshotService::create($instance, $actor, [
'name' => 'delete-with-instance', 'consistency_mode' => 'crash',
]);
$cascadeJob = $cascade['job'];
$cascadeKey = (string) $cascade['snapshot']['snapshot_key'];
CloudSnapshotService::finishJobLocked($cascadeJob, 'succeeded', [
'snapshot_key' => $cascadeKey, 'created' => true,
'consistency' => 'crash',
], CloudSecrets::decrypt((string) $cascadeJob->sealed_payload,
'job-payload'));
$terminalize($cascadeJob, 'succeeded');
$deleteVm = new CloudJob();
$deleteVm->operation_id = str_repeat('d', 32);
$deleteVm->instance_id = $instance->id;
$deleteVm->kind = 'delete';
$deleteVm->execution_mode = 'agent';
CloudSnapshotService::finishInstanceDeleteLocked($deleteVm, 'uncertain');
$assert(CloudInstanceSnapshot::where('snapshot_key', $cascadeKey)
->value('state') === 'uncertain',
'An ambiguous VM deletion did not preserve its snapshot cleanup uncertainty.');
CloudSnapshotService::finishInstanceDeleteLocked($deleteVm, 'failed');
$assert(CloudInstanceSnapshot::where('snapshot_key', $cascadeKey)
->value('state') === 'ready',
'A definitive VM-delete failure did not restore its snapshot catalog state.');
CloudSnapshotService::finishInstanceDeleteLocked($deleteVm, 'succeeded');
$assert(CloudInstanceSnapshot::where('instance_id', $instance->id)
->whereIn('state', ['creating', 'ready', 'restoring', 'deleting', 'uncertain'])
->count() === 0,
'Confirmed VM deletion did not cascade its PHP snapshot catalog.');
$delegated = CloudSnapshotService::create($instance, [
'user_id' => 99, 'is_admin' => true, 'is_super_admin' => true,
'can_manage' => true,
], ['name' => 'admin-audit', 'consistency_mode' => 'crash']);
$delegatedJob = $delegated['job'];
$delegatedSnapshot = CloudInstanceSnapshot::find(
(int) $delegatedJob->snapshot_id
);
$delegatedPayload = CloudSecrets::decrypt(
(string) $delegatedJob->sealed_payload,
'job-payload'
);
$assert((int) $delegatedJob->user_id === 99
&& (int) $delegatedSnapshot->user_id === (int) $instance->user_id
&& (int) $delegatedPayload['owner_id'] === (int) $instance->user_id,
'Administrator snapshot action lost actor/owner audit separation.');
$ownerView = CloudSnapshotService::listing($instance, $actor);
$adminView = CloudSnapshotService::listing($instance, [
'user_id' => 99, 'is_admin' => true, 'is_super_admin' => true,
'can_manage' => true,
]);
$assert($ownerView['snapshots'][0]['task_id'] === null
&& $ownerView['active_job'] !== null
&& !array_key_exists('id', $ownerView['active_job'])
&& (string) ($adminView['active_job']['id'] ?? '')
=== (string) $delegatedJob->id,
'Tenant listing exposed an administrator-scoped job as directly pollable.');
CloudSnapshotService::finishJobLocked($delegatedJob, 'failed', null,
$delegatedPayload);
$terminalize($delegatedJob, 'failed', false, true);
$assert(CloudInstanceSnapshot::find((int) $delegatedJob->snapshot_id)->state
=== 'failed',
'Administrator snapshot result could not reconcile against its owner.');
$settingsAfterDelete = CloudJobService::enqueue($instance, $actor,
'instance_settings', 'POST', '/settings', [
'hostname' => 'allowed.example', 'confirm_restart' => true,
]);
$assert((string) $settingsAfterDelete->status === 'queued',
'Deleting the final snapshot did not release instance settings admission.');
$terminalize($settingsAfterDelete, 'failed', false, true);
$reinstallAfterDelete = CloudJobService::enqueue($instance, $actor,
'reinstall', 'POST', '/reinstall', [
'template_id' => 'debian-12',
]);
$assert((string) $reinstallAfterDelete->status === 'queued',
'Deleting the final snapshot did not release reinstall admission.');
$operationLogIds = array_column(
CloudJobService::operationLogs($instance, 1, 50)['items'], 'id'
);
$assert(CloudOperationCatalog::instanceLogKinds() === [
'action', 'reinstall', 'instance_settings', 'network_change',
'instance_upgrade', 'firewall_apply', 'snapshot_create',
'snapshot_restore', 'snapshot_delete', 'backup_create',
'backup_workspace_cleanup', 'backup_restore', 'backup_recover',
'instance_migration_prepare',
'instance_migration_transfer', 'instance_migration_target_clean',
'instance_migration_cleanup',
'instance_migration_rollback', 'instance_migration_activate',
'vpc_host_apply', 'vpc_host_remove',
'vpc_port_attach', 'vpc_port_detach', 'rescue',
], 'The instance operation log does not cover every settings operation.');
$assert(in_array((string) $settingsAfterDelete->id, $operationLogIds, true),
'An instance-settings task was omitted from the instance operation log.');
echo "Cloud snapshot catalog, admission and worker state machine passed.\n";
}