You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
644 lines
30 KiB
PHP
644 lines
30 KiB
PHP
<?php
|
|
namespace App\Services\Cloud {
|
|
// Isolated test cipher only; production CloudSecrets is never loaded here.
|
|
final class CloudSecrets
|
|
{
|
|
private static array $values = [];
|
|
public static function encrypt(array $value, string $purpose): string
|
|
{
|
|
$key = bin2hex(random_bytes(16));
|
|
self::$values[$key] = $value;
|
|
return 'snapshot-test:' . $key;
|
|
}
|
|
public static function decrypt(string $value, string $purpose): array
|
|
{
|
|
return self::$values[substr($value, 14)];
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudInstanceSnapshot;
|
|
use App\Models\CloudJob;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\CloudInput;
|
|
use App\Services\Cloud\CloudJobService;
|
|
use App\Services\Cloud\CloudOperationCatalog;
|
|
use App\Services\Cloud\CloudOperationErrorCatalog;
|
|
use App\Services\Cloud\CloudSecrets;
|
|
use App\Services\Cloud\CloudSnapshotService;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function ($condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
$terminalize = static function (CloudJob $job, string $outcome,
|
|
bool $effect = false, bool $safe = false): void {
|
|
$now = time();
|
|
$job->phase = 'terminal';
|
|
$job->outcome = $outcome;
|
|
$job->status = $outcome;
|
|
$job->attempts = 1;
|
|
$job->dispatch_attempts = 1;
|
|
$job->dispatch_started_at ??= $now - 1;
|
|
$job->dispatched_at ??= $now - 1;
|
|
$job->agent_accepted_at ??= $now - 1;
|
|
$job->receipt_deadline_at ??= $now + 900;
|
|
$job->terminal_at = $now;
|
|
$job->effect_may_have_applied = $effect ? 1 : 0;
|
|
$job->safe_to_resubmit = $safe ? 1 : 0;
|
|
$job->effect_resolution = $effect ? 'unknown' : 'not_applicable';
|
|
$job->lease_token = null;
|
|
$job->lease_until = 0;
|
|
$job->available_at = 0;
|
|
$job->updated_at = $now;
|
|
$job->save();
|
|
};
|
|
$expect = static function (callable $callable, int $status,
|
|
string $message) use ($assert): void {
|
|
try {
|
|
$callable();
|
|
} catch (CloudError $error) {
|
|
$assert($error->httpStatus === $status,
|
|
$message . ' Unexpected status: ' . $error->httpStatus);
|
|
return;
|
|
}
|
|
throw new RuntimeException($message);
|
|
};
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
$schema->create('cloud_instances', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('uuid');
|
|
$table->integer('host_id');
|
|
$table->integer('user_id');
|
|
$table->string('name');
|
|
$table->string('status');
|
|
$table->string('service_status');
|
|
$table->string('network_status');
|
|
$table->integer('expires_at')->nullable();
|
|
$table->text('spec_json');
|
|
$table->text('sealed_credential')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_instance_snapshots', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('snapshot_key')->unique();
|
|
$table->integer('instance_id');
|
|
$table->integer('user_id');
|
|
$table->integer('host_id');
|
|
$table->string('name');
|
|
$table->string('state');
|
|
$table->string('consistency');
|
|
$table->integer('size_bytes')->nullable();
|
|
$table->string('last_operation_id')->nullable();
|
|
$table->integer('ready_at')->nullable();
|
|
$table->integer('last_restored_at')->nullable();
|
|
$table->integer('deleted_at')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_templates', static function (Blueprint $table): void {
|
|
$table->string('id')->primary();
|
|
$table->string('name');
|
|
$table->text('sealed_source_url');
|
|
$table->text('startup_script');
|
|
$table->integer('revision');
|
|
$table->string('generation_token');
|
|
$table->string('state');
|
|
});
|
|
$schema->create('cloud_template_replicas',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id');
|
|
$table->string('template_id');
|
|
$table->integer('revision');
|
|
$table->string('generation_token');
|
|
$table->string('state');
|
|
$table->integer('image_bytes');
|
|
$table->string('sha256');
|
|
$table->string('operation_id');
|
|
$table->integer('job_id')->nullable();
|
|
$table->string('error_code');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->primary(['host_id', 'template_id']);
|
|
});
|
|
$capsule->getConnection('default')->table('cloud_templates')->insert([
|
|
'id' => 'debian-12',
|
|
'name' => 'Debian 12',
|
|
'sealed_source_url' => CloudSecrets::encrypt([
|
|
'url' => 'https://example.invalid/debian-12.qcow2',
|
|
], 'template-source'),
|
|
'startup_script' => '',
|
|
'revision' => 1,
|
|
'generation_token' => str_repeat('a', 32),
|
|
'state' => 'active',
|
|
]);
|
|
$capsule->getConnection('default')->table(
|
|
'cloud_template_replicas'
|
|
)->insert([
|
|
'host_id' => 7,
|
|
'template_id' => 'debian-12',
|
|
'revision' => 1,
|
|
'generation_token' => str_repeat('a', 32),
|
|
'state' => 'active',
|
|
'image_bytes' => 4096,
|
|
'sha256' => hash('sha256', 'snapshot-template'),
|
|
'operation_id' => 'snapshot-fixture-template-import',
|
|
'job_id' => null,
|
|
'error_code' => '',
|
|
'created_at' => time(),
|
|
'updated_at' => time(),
|
|
]);
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('operation_id')->unique();
|
|
$table->string('request_idempotency_key');
|
|
$table->string('request_fingerprint');
|
|
$table->integer('parent_job_id')->nullable();
|
|
$table->integer('order_id')->nullable();
|
|
$table->integer('invoice_id')->nullable();
|
|
$table->integer('instance_id')->nullable();
|
|
$table->integer('snapshot_id')->nullable();
|
|
$table->integer('user_id');
|
|
$table->integer('host_id')->nullable();
|
|
$table->string('kind');
|
|
$table->string('execution_mode')->default('agent');
|
|
$table->string('phase')->default('accepted');
|
|
$table->string('outcome')->nullable();
|
|
$table->string('status');
|
|
$table->text('sealed_payload');
|
|
$table->integer('attempts')->default(0);
|
|
$table->integer('dispatch_attempts')->default(0);
|
|
$table->integer('dispatch_deadline_at')->default(0);
|
|
$table->integer('dispatch_started_at')->nullable();
|
|
$table->integer('agent_accepted_at')->nullable();
|
|
$table->integer('receipt_deadline_at')->nullable();
|
|
$table->integer('terminal_at')->nullable();
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
$table->string('effect_resolution')->default('not_applicable');
|
|
$table->integer('effect_resolved_at')->nullable();
|
|
$table->integer('effect_resolved_by')->nullable();
|
|
$table->string('effect_resolution_reason')->default('');
|
|
$table->string('completion_event_id')->nullable();
|
|
$table->integer('read_at')->nullable();
|
|
$table->integer('dispatched_at')->nullable();
|
|
$table->string('lease_token')->nullable();
|
|
$table->integer('lease_until')->default(0);
|
|
$table->integer('available_at');
|
|
$table->string('error_code')->nullable();
|
|
$table->boolean('finalization_incident')->default(0);
|
|
$table->string('finalization_error_code')->nullable();
|
|
$table->text('result_json')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
|
|
$instance = new CloudInstance();
|
|
$instance->uuid = '11111111-1111-4111-8111-111111111111';
|
|
$instance->host_id = 7;
|
|
$instance->user_id = 10;
|
|
$instance->name = 'vm-snap';
|
|
$instance->status = 'active';
|
|
$instance->service_status = 'exempt';
|
|
$instance->network_status = 'active';
|
|
$instance->expires_at = null;
|
|
$instance->spec_json = json_encode([
|
|
'template_id' => 'debian-12', 'os_source' => 'cloud_template',
|
|
'os_name' => 'Debian 12', 'login_name' => 'root',
|
|
'guest_hostname' => 'snapshot-old',
|
|
'public_ip_cidr' => '192.0.2.10/24', 'region' => 'HKG',
|
|
'vcpu' => 2, 'memory_mib' => 2048, 'disk_gib' => 40,
|
|
'agent_metadata_version' => 1,
|
|
], JSON_THROW_ON_ERROR);
|
|
$instance->sealed_credential = CloudSecrets::encrypt([
|
|
'password' => 'old-password', 'login_name' => 'root',
|
|
], 'instance-credential');
|
|
$instance->created_at = $instance->updated_at = time();
|
|
$instance->save();
|
|
$actor = ['user_id' => 10, 'is_admin' => false,
|
|
'is_super_admin' => false, 'can_manage' => false];
|
|
|
|
$first = CloudSnapshotService::create($instance, $actor, [
|
|
'name' => '发布前', 'consistency_mode' => 'filesystem',
|
|
]);
|
|
$firstJob = $first['job'];
|
|
$firstPayload = CloudSecrets::decrypt(
|
|
(string) $firstJob->sealed_payload, 'job-payload'
|
|
);
|
|
$firstKey = (string) $first['snapshot']['snapshot_key'];
|
|
$assert(preg_match('/^[a-f0-9]{32}$/D', $firstKey) === 1
|
|
&& $firstPayload['method'] === 'POST'
|
|
&& $firstPayload['request_target']
|
|
=== '/v1/vms/' . $instance->uuid . '/snapshots'
|
|
&& $firstPayload['body'] === [
|
|
'snapshot_key' => $firstKey,
|
|
'name' => '发布前',
|
|
'consistency_mode' => 'filesystem',
|
|
]
|
|
&& (int) $firstJob->snapshot_id > 0,
|
|
'Snapshot create did not persist an opaque key and bounded Agent contract.');
|
|
CloudSnapshotService::finishJobLocked($firstJob, 'succeeded', [
|
|
'snapshot_key' => $firstKey, 'created' => true,
|
|
'consistency' => 'crash', 'size_bytes' => 4096,
|
|
], $firstPayload);
|
|
$terminalize($firstJob, 'succeeded');
|
|
|
|
$listed = CloudSnapshotService::listing($instance, $actor);
|
|
$assert($listed['max_snapshots'] === 2 && $listed['used_slots'] === 1
|
|
&& $listed['can_create'] === true && $listed['can_mutate'] === true
|
|
&& count($listed['snapshots']) === 1
|
|
&& $listed['snapshots'][0]['state'] === 'ready'
|
|
&& $listed['snapshots'][0]['consistency'] === 'crash'
|
|
&& $listed['snapshots'][0]['size_bytes'] === 4096,
|
|
'Database-only snapshot listing did not expose the actual downgraded consistency.');
|
|
$storedFirst = CloudInstanceSnapshot::find((int) $firstJob->snapshot_id);
|
|
$assert(!array_key_exists('sealed_credential', $storedFirst->getAttributes())
|
|
&& !array_key_exists('system_metadata_json', $storedFirst->getAttributes()),
|
|
'Snapshot catalog duplicated an instance credential or seed metadata.');
|
|
|
|
$invalidUpgrade = CloudSnapshotService::create($instance, $actor, [
|
|
'name' => 'invalid-upgrade', 'consistency_mode' => 'crash',
|
|
]);
|
|
$invalidUpgradeJob = $invalidUpgrade['job'];
|
|
$invalidUpgradePayload = CloudSecrets::decrypt(
|
|
(string) $invalidUpgradeJob->sealed_payload, 'job-payload'
|
|
);
|
|
$expect(static fn () => CloudSnapshotService::finishJobLocked(
|
|
$invalidUpgradeJob, 'succeeded', [
|
|
'snapshot_key' => $invalidUpgrade['snapshot']['snapshot_key'],
|
|
'created' => true, 'consistency' => 'filesystem',
|
|
], $invalidUpgradePayload
|
|
), 502, 'A crash-consistent request accepted an Agent filesystem upgrade.');
|
|
CloudSnapshotService::finishJobLocked($invalidUpgradeJob, 'failed', null,
|
|
$invalidUpgradePayload);
|
|
$terminalize($invalidUpgradeJob, 'failed', false, true);
|
|
|
|
$settingsWithSnapshot = CloudJobService::enqueue($instance, $actor,
|
|
'instance_settings', 'POST', '/settings', [
|
|
'hostname' => 'current.example', 'confirm_restart' => true,
|
|
]);
|
|
$hostnameSettingsPayload = CloudSecrets::decrypt(
|
|
(string) $settingsWithSnapshot->sealed_payload, 'job-payload'
|
|
);
|
|
$assert((string) $settingsWithSnapshot->status === 'queued'
|
|
&& ($hostnameSettingsPayload['settings_fields'] ?? null)
|
|
=== ['hostname']
|
|
&& !array_key_exists('settings_fields',
|
|
$hostnameSettingsPayload['body'] ?? [])
|
|
&& CloudJobService::display($settingsWithSnapshot)['kind_label']
|
|
=== '修改主机名',
|
|
'A ready snapshot blocked settings or hostname task presentation was not persisted safely.');
|
|
$terminalize($settingsWithSnapshot, 'failed', false, true);
|
|
$legacyHostnamePayload = $hostnameSettingsPayload;
|
|
unset($legacyHostnamePayload['settings_fields']);
|
|
$legacyHostname = clone $settingsWithSnapshot;
|
|
$legacyHostname->sealed_payload = CloudSecrets::encrypt(
|
|
CloudInput::safeResult($legacyHostnamePayload), 'job-payload'
|
|
);
|
|
$assert(CloudJobService::display($legacyHostname)['kind_label']
|
|
=== '修改主机名',
|
|
'A completed legacy hostname task did not infer its safe detail label.');
|
|
|
|
$passwordSettings = CloudJobService::enqueue($instance, $actor,
|
|
'instance_settings', 'POST', '/settings', [
|
|
'root_password' => 'Aa1!Aa1!Aa1!', 'confirm_restart' => true,
|
|
]);
|
|
$passwordSettingsPayload = CloudSecrets::decrypt(
|
|
(string) $passwordSettings->sealed_payload, 'job-payload'
|
|
);
|
|
$passwordSafePayload = CloudInput::safeResult($passwordSettingsPayload);
|
|
$passwordSettings->sealed_payload = CloudSecrets::encrypt(
|
|
$passwordSafePayload, 'job-payload'
|
|
);
|
|
$terminalize($passwordSettings, 'failed', false, true);
|
|
$assert(($passwordSafePayload['settings_fields'] ?? null) === ['password']
|
|
&& !array_key_exists('root_password',
|
|
$passwordSafePayload['body'] ?? [])
|
|
&& CloudJobService::display($passwordSettings)['kind_label']
|
|
=== '重置 root 密码',
|
|
'Password settings lost their safe detail label or retained the password.');
|
|
$legacyPasswordPayload = $passwordSettingsPayload;
|
|
unset($legacyPasswordPayload['settings_fields']);
|
|
$legacyPassword = clone $passwordSettings;
|
|
$legacyPassword->phase = 'accepted';
|
|
$legacyPassword->outcome = null;
|
|
$legacyPassword->status = 'queued';
|
|
$legacyPassword->terminal_at = null;
|
|
$legacyPassword->sealed_payload = CloudSecrets::encrypt(
|
|
$legacyPasswordPayload, 'job-payload'
|
|
);
|
|
$legacyPasswordActiveLabel = CloudJobService::display(
|
|
$legacyPassword
|
|
)['kind_label'];
|
|
$legacyPassword->phase = 'terminal';
|
|
$legacyPassword->outcome = 'failed';
|
|
$legacyPassword->status = 'failed';
|
|
$legacyPassword->terminal_at = time();
|
|
$legacyPassword->sealed_payload = CloudSecrets::encrypt(
|
|
CloudInput::safeResult($legacyPasswordPayload), 'job-payload'
|
|
);
|
|
$assert($legacyPasswordActiveLabel === '重置 root 密码'
|
|
&& CloudJobService::display($legacyPassword)['kind_label']
|
|
=== '实例设置',
|
|
'Legacy settings labels were guessed after secret redaction.');
|
|
|
|
$sshBlob = pack('N', strlen('ssh-ed25519')) . 'ssh-ed25519'
|
|
. str_repeat("\x01", 32);
|
|
$sshSettings = CloudJobService::enqueue($instance, $actor,
|
|
'instance_settings', 'POST', '/settings', [
|
|
'ssh_public_key' => 'ssh-ed25519 ' . base64_encode($sshBlob),
|
|
'confirm_restart' => true,
|
|
]);
|
|
$sshSettingsPayload = CloudSecrets::decrypt(
|
|
(string) $sshSettings->sealed_payload, 'job-payload'
|
|
);
|
|
$sshSafePayload = CloudInput::safeResult($sshSettingsPayload);
|
|
$sshSettings->sealed_payload = CloudSecrets::encrypt(
|
|
$sshSafePayload, 'job-payload'
|
|
);
|
|
$terminalize($sshSettings, 'failed', false, true);
|
|
$assert(($sshSafePayload['settings_fields'] ?? null) === ['ssh_key']
|
|
&& !array_key_exists('ssh_public_key', $sshSafePayload['body'] ?? [])
|
|
&& CloudJobService::display($sshSettings)['kind_label']
|
|
=== '更新 SSH 公钥',
|
|
'SSH settings lost their safe detail label or retained the public key.');
|
|
|
|
$multiSettings = CloudJobService::enqueue($instance, $actor,
|
|
'instance_settings', 'POST', '/settings', [
|
|
'hostname' => 'multi.example',
|
|
'root_password' => 'Bb2@Bb2@Bb2@',
|
|
'ssh_public_key' => 'ssh-ed25519 ' . base64_encode($sshBlob),
|
|
'confirm_restart' => true,
|
|
]);
|
|
$multiSettingsPayload = CloudSecrets::decrypt(
|
|
(string) $multiSettings->sealed_payload, 'job-payload'
|
|
);
|
|
$assert(($multiSettingsPayload['settings_fields'] ?? null)
|
|
=== ['hostname', 'password', 'ssh_key']
|
|
&& CloudJobService::display($multiSettings)['kind_label']
|
|
=== '修改主机名、重置 root 密码、更新 SSH 公钥',
|
|
'Multi-field settings lost their stable order or detailed presentation.');
|
|
$terminalize($multiSettings, 'failed', false, true);
|
|
|
|
$expect(static fn () => CloudJobService::enqueue($instance, $actor,
|
|
'reinstall', 'POST', '/reinstall', [
|
|
'template_id' => 'debian-12',
|
|
]), 409, 'A retained snapshot allowed a reinstall mutation.');
|
|
|
|
$second = CloudSnapshotService::create($instance, $actor, [
|
|
'name' => 'Crash point', 'consistency_mode' => 'crash',
|
|
]);
|
|
$secondJob = $second['job'];
|
|
$secondPayload = CloudSecrets::decrypt(
|
|
(string) $secondJob->sealed_payload, 'job-payload'
|
|
);
|
|
CloudSnapshotService::finishJobLocked($secondJob, 'succeeded', [
|
|
'snapshot_key' => $second['snapshot']['snapshot_key'],
|
|
'created' => true, 'consistency' => 'crash',
|
|
], $secondPayload);
|
|
$terminalize($secondJob, 'succeeded');
|
|
$expect(static fn () => CloudSnapshotService::create($instance, $actor, [
|
|
'name' => 'third', 'consistency_mode' => 'filesystem',
|
|
]), 409, 'A third live snapshot bypassed the per-instance limit.');
|
|
|
|
$restore = CloudSnapshotService::restore($instance, $actor, $firstKey,
|
|
['confirmation_name' => 'vm-snap']);
|
|
$restoreJob = $restore['job'];
|
|
$restorePayload = CloudSecrets::decrypt(
|
|
(string) $restoreJob->sealed_payload, 'job-payload'
|
|
);
|
|
$assert($restorePayload['method'] === 'POST'
|
|
&& $restorePayload['request_target'] === '/v1/vms/' . $instance->uuid
|
|
. '/snapshots/' . $firstKey . '/restore'
|
|
&& $restorePayload['body'] === [],
|
|
'Snapshot restore engine contract changed unexpectedly.');
|
|
CloudSnapshotService::finishJobLocked($restoreJob, 'uncertain', null,
|
|
$restorePayload);
|
|
$terminalize($restoreJob, 'failed', true, false);
|
|
$uncertain = CloudSnapshotService::listing($instance, $actor);
|
|
$assert($restoreJob->phase === 'terminal'
|
|
&& $restoreJob->outcome === 'failed'
|
|
&& $restoreJob->effect_resolution === 'unknown'
|
|
&& ($uncertain['snapshots'][1]['state'] === 'uncertain'
|
|
|| $uncertain['snapshots'][0]['state'] === 'uncertain'),
|
|
'An unknown-effect restore did not become a terminal failed task with a visible resource fence.');
|
|
$credentialBeforeRestore = (string) $instance->sealed_credential;
|
|
$specBeforeRestore = (string) $instance->spec_json;
|
|
// Simulate a later administrator inspection proving that the restore did
|
|
// apply. It resolves only the resource fence; the immutable task outcome
|
|
// remains failed and is never replayed or rewritten to success.
|
|
CloudInstanceSnapshot::where('id', (int) $restoreJob->snapshot_id)
|
|
->update(['state' => 'ready', 'last_restored_at' => time(),
|
|
'updated_at' => time()]);
|
|
$restoreJob->effect_resolution = 'applied';
|
|
$restoreJob->effect_resolved_at = time();
|
|
$restoreJob->effect_resolved_by = 99;
|
|
$restoreJob->effect_resolution_reason =
|
|
'Read-only host inspection confirmed snapshot restore effect.';
|
|
$restoreJob->save();
|
|
$instance->refresh();
|
|
$assert((string) $instance->sealed_credential === $credentialBeforeRestore
|
|
&& (string) $instance->spec_json === $specBeforeRestore,
|
|
'Snapshot restore rewrote panel credentials or seed metadata.');
|
|
|
|
$delete = CloudSnapshotService::delete($instance, $actor, $firstKey,
|
|
['confirmation_name' => 'vm-snap']);
|
|
$deleteJob = $delete['job'];
|
|
$deletePayload = CloudSecrets::decrypt(
|
|
(string) $deleteJob->sealed_payload, 'job-payload'
|
|
);
|
|
$assert($deletePayload['method'] === 'DELETE'
|
|
&& $deletePayload['request_target'] === '/v1/vms/' . $instance->uuid
|
|
. '/snapshots/' . $firstKey
|
|
&& $deletePayload['body'] === [],
|
|
'Snapshot delete engine contract changed unexpectedly.');
|
|
CloudSnapshotService::finishJobLocked($deleteJob, 'succeeded', [
|
|
'snapshot_key' => $firstKey, 'deleted' => true,
|
|
], $deletePayload);
|
|
$terminalize($deleteJob, 'succeeded');
|
|
$assert(CloudSnapshotService::listing($instance, $actor)['used_slots'] === 1,
|
|
'A confirmed deletion did not release exactly one snapshot slot.');
|
|
|
|
$failed = CloudSnapshotService::create($instance, $actor, [
|
|
'name' => 'will-fail', 'consistency_mode' => 'filesystem',
|
|
]);
|
|
$failedJob = $failed['job'];
|
|
CloudSnapshotService::finishJobLocked($failedJob, 'failed', null,
|
|
CloudSecrets::decrypt((string) $failedJob->sealed_payload,
|
|
'job-payload'));
|
|
$terminalize($failedJob, 'failed', false, true);
|
|
$assert(CloudInstanceSnapshot::find((int) $failedJob->snapshot_id)->state
|
|
=== 'failed'
|
|
&& CloudSnapshotService::listing($instance, $actor)['used_slots'] === 1,
|
|
'A definitive create failure kept a slot or lost its audit state.');
|
|
|
|
$expect(static fn () => CloudSnapshotService::create($instance,
|
|
['user_id' => 11, 'is_admin' => false, 'is_super_admin' => false,
|
|
'can_manage' => false],
|
|
['name' => 'foreign', 'consistency_mode' => 'crash']),
|
|
403, 'A different tenant created a snapshot for this instance.');
|
|
$expect(static fn () => CloudSnapshotService::listing($instance,
|
|
['user_id' => 11, 'is_admin' => false]), 404,
|
|
'A different tenant listed snapshots for this instance.');
|
|
$expect(static fn () => CloudSnapshotService::restore($instance, $actor,
|
|
(string) $second['snapshot']['snapshot_key'],
|
|
['confirmation_name' => 'wrong-name']), 422,
|
|
'Snapshot restore accepted a mismatched instance confirmation.');
|
|
|
|
$qga = CloudOperationErrorCatalog::workerSummary('snapshot_create',
|
|
'failed', ['error_code' => 'snapshot_guest_agent_unavailable']);
|
|
$assert($qga === 'snapshot_guest_agent_unavailable'
|
|
&& str_contains((string) CloudOperationErrorCatalog::userMessage(
|
|
'snapshot_create', 'failed', $qga), 'QEMU Guest Agent'),
|
|
'Legacy QGA unavailability was not preserved as a safe update diagnostic.');
|
|
$freeze = CloudOperationErrorCatalog::workerSummary('snapshot_create',
|
|
'failed', ['error_code' => 'snapshot_filesystem_freeze_failed']);
|
|
$assert($freeze === 'snapshot_filesystem_freeze_failed'
|
|
&& str_contains((string) CloudOperationErrorCatalog::userMessage(
|
|
'snapshot_create', 'failed', $freeze), '文件系统冻结失败'),
|
|
'A responsive-agent freeze failure was not preserved as a distinct safe error.');
|
|
|
|
$secondKey = (string) $second['snapshot']['snapshot_key'];
|
|
$restoreFailed = CloudSnapshotService::restore($instance, $actor,
|
|
$secondKey, ['confirmation_name' => 'vm-snap']);
|
|
$restoreFailedJob = $restoreFailed['job'];
|
|
CloudSnapshotService::finishJobLocked($restoreFailedJob, 'failed', null,
|
|
CloudSecrets::decrypt((string) $restoreFailedJob->sealed_payload,
|
|
'job-payload'));
|
|
$terminalize($restoreFailedJob, 'failed', false, true);
|
|
$assert(CloudInstanceSnapshot::find((int) $restoreFailedJob->snapshot_id)->state
|
|
=== 'ready',
|
|
'A definitive restore failure did not return the source snapshot to ready.');
|
|
|
|
$deleteUncertain = CloudSnapshotService::delete($instance, $actor,
|
|
$secondKey, ['confirmation_name' => 'vm-snap']);
|
|
$deleteUncertainJob = $deleteUncertain['job'];
|
|
$deleteUncertainPayload = CloudSecrets::decrypt(
|
|
(string) $deleteUncertainJob->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
CloudSnapshotService::finishJobLocked($deleteUncertainJob, 'uncertain', null,
|
|
$deleteUncertainPayload);
|
|
$terminalize($deleteUncertainJob, 'failed', true, false);
|
|
$assert(CloudInstanceSnapshot::find((int) $deleteUncertainJob->snapshot_id)->state
|
|
=== 'uncertain'
|
|
&& $deleteUncertainJob->phase === 'terminal'
|
|
&& $deleteUncertainJob->outcome === 'failed',
|
|
'An unknown-effect delete did not preserve its resource fence as a terminal failure.');
|
|
CloudInstanceSnapshot::where('id', (int) $deleteUncertainJob->snapshot_id)
|
|
->update(['state' => 'ready', 'updated_at' => time()]);
|
|
$deleteUncertainJob->effect_resolution = 'not_applied';
|
|
$deleteUncertainJob->effect_resolved_at = time();
|
|
$deleteUncertainJob->effect_resolved_by = 99;
|
|
$deleteUncertainJob->effect_resolution_reason =
|
|
'Read-only host inspection confirmed snapshot still exists.';
|
|
$deleteUncertainJob->save();
|
|
|
|
$cascade = CloudSnapshotService::create($instance, $actor, [
|
|
'name' => 'delete-with-instance', 'consistency_mode' => 'crash',
|
|
]);
|
|
$cascadeJob = $cascade['job'];
|
|
$cascadeKey = (string) $cascade['snapshot']['snapshot_key'];
|
|
CloudSnapshotService::finishJobLocked($cascadeJob, 'succeeded', [
|
|
'snapshot_key' => $cascadeKey, 'created' => true,
|
|
'consistency' => 'crash',
|
|
], CloudSecrets::decrypt((string) $cascadeJob->sealed_payload,
|
|
'job-payload'));
|
|
$terminalize($cascadeJob, 'succeeded');
|
|
|
|
$deleteVm = new CloudJob();
|
|
$deleteVm->operation_id = str_repeat('d', 32);
|
|
$deleteVm->instance_id = $instance->id;
|
|
$deleteVm->kind = 'delete';
|
|
$deleteVm->execution_mode = 'agent';
|
|
CloudSnapshotService::finishInstanceDeleteLocked($deleteVm, 'uncertain');
|
|
$assert(CloudInstanceSnapshot::where('snapshot_key', $cascadeKey)
|
|
->value('state') === 'uncertain',
|
|
'An ambiguous VM deletion did not preserve its snapshot cleanup uncertainty.');
|
|
CloudSnapshotService::finishInstanceDeleteLocked($deleteVm, 'failed');
|
|
$assert(CloudInstanceSnapshot::where('snapshot_key', $cascadeKey)
|
|
->value('state') === 'ready',
|
|
'A definitive VM-delete failure did not restore its snapshot catalog state.');
|
|
CloudSnapshotService::finishInstanceDeleteLocked($deleteVm, 'succeeded');
|
|
$assert(CloudInstanceSnapshot::where('instance_id', $instance->id)
|
|
->whereIn('state', ['creating', 'ready', 'restoring', 'deleting', 'uncertain'])
|
|
->count() === 0,
|
|
'Confirmed VM deletion did not cascade its PHP snapshot catalog.');
|
|
|
|
$delegated = CloudSnapshotService::create($instance, [
|
|
'user_id' => 99, 'is_admin' => true, 'is_super_admin' => true,
|
|
'can_manage' => true,
|
|
], ['name' => 'admin-audit', 'consistency_mode' => 'crash']);
|
|
$delegatedJob = $delegated['job'];
|
|
$delegatedSnapshot = CloudInstanceSnapshot::find(
|
|
(int) $delegatedJob->snapshot_id
|
|
);
|
|
$delegatedPayload = CloudSecrets::decrypt(
|
|
(string) $delegatedJob->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
$assert((int) $delegatedJob->user_id === 99
|
|
&& (int) $delegatedSnapshot->user_id === (int) $instance->user_id
|
|
&& (int) $delegatedPayload['owner_id'] === (int) $instance->user_id,
|
|
'Administrator snapshot action lost actor/owner audit separation.');
|
|
$ownerView = CloudSnapshotService::listing($instance, $actor);
|
|
$adminView = CloudSnapshotService::listing($instance, [
|
|
'user_id' => 99, 'is_admin' => true, 'is_super_admin' => true,
|
|
'can_manage' => true,
|
|
]);
|
|
$assert($ownerView['snapshots'][0]['task_id'] === null
|
|
&& $ownerView['active_job'] !== null
|
|
&& !array_key_exists('id', $ownerView['active_job'])
|
|
&& (string) ($adminView['active_job']['id'] ?? '')
|
|
=== (string) $delegatedJob->id,
|
|
'Tenant listing exposed an administrator-scoped job as directly pollable.');
|
|
CloudSnapshotService::finishJobLocked($delegatedJob, 'failed', null,
|
|
$delegatedPayload);
|
|
$terminalize($delegatedJob, 'failed', false, true);
|
|
$assert(CloudInstanceSnapshot::find((int) $delegatedJob->snapshot_id)->state
|
|
=== 'failed',
|
|
'Administrator snapshot result could not reconcile against its owner.');
|
|
|
|
$settingsAfterDelete = CloudJobService::enqueue($instance, $actor,
|
|
'instance_settings', 'POST', '/settings', [
|
|
'hostname' => 'allowed.example', 'confirm_restart' => true,
|
|
]);
|
|
$assert((string) $settingsAfterDelete->status === 'queued',
|
|
'Deleting the final snapshot did not release instance settings admission.');
|
|
$terminalize($settingsAfterDelete, 'failed', false, true);
|
|
$reinstallAfterDelete = CloudJobService::enqueue($instance, $actor,
|
|
'reinstall', 'POST', '/reinstall', [
|
|
'template_id' => 'debian-12',
|
|
]);
|
|
$assert((string) $reinstallAfterDelete->status === 'queued',
|
|
'Deleting the final snapshot did not release reinstall admission.');
|
|
|
|
$operationLogIds = array_column(
|
|
CloudJobService::operationLogs($instance, 1, 50)['items'], 'id'
|
|
);
|
|
$assert(CloudOperationCatalog::instanceLogKinds() === [
|
|
'action', 'reinstall', 'instance_settings', 'network_change',
|
|
'instance_upgrade', 'firewall_apply', 'snapshot_create',
|
|
'snapshot_restore', 'snapshot_delete', 'backup_create',
|
|
'backup_restore', 'backup_recover', 'instance_migration_cleanup',
|
|
'instance_migration_rollback', 'instance_migration_activate', 'rescue',
|
|
], 'The instance operation log does not cover every settings operation.');
|
|
$assert(in_array((string) $settingsAfterDelete->id, $operationLogIds, true),
|
|
'An instance-settings task was omitted from the instance operation log.');
|
|
|
|
echo "Cloud snapshot catalog, admission and worker state machine passed.\n";
|
|
}
|