You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
397 lines
18 KiB
PHP
397 lines
18 KiB
PHP
<?php
|
|
namespace App\Services\Cloud {
|
|
// Isolated test cipher only; production CloudSecrets is never loaded here.
|
|
final class CloudSecrets
|
|
{
|
|
private static array $values = [];
|
|
public static function encrypt(array $value): string
|
|
{
|
|
$key = bin2hex(random_bytes(16));
|
|
self::$values[$key] = $value;
|
|
return 'snapshot-test:' . $key;
|
|
}
|
|
public static function decrypt(string $value): array
|
|
{
|
|
return self::$values[substr($value, 14)];
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudInstanceSnapshot;
|
|
use App\Models\CloudJob;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\CloudJobService;
|
|
use App\Services\Cloud\CloudOperationErrorCatalog;
|
|
use App\Services\Cloud\CloudSecrets;
|
|
use App\Services\Cloud\CloudSnapshotService;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function ($condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
$expect = static function (callable $callable, int $status,
|
|
string $message) use ($assert): void {
|
|
try {
|
|
$callable();
|
|
} catch (CloudError $error) {
|
|
$assert($error->httpStatus === $status,
|
|
$message . ' Unexpected status: ' . $error->httpStatus);
|
|
return;
|
|
}
|
|
throw new RuntimeException($message);
|
|
};
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
$schema->create('cloud_instances', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('uuid');
|
|
$table->integer('host_id');
|
|
$table->integer('user_id');
|
|
$table->string('name');
|
|
$table->string('status');
|
|
$table->string('service_status');
|
|
$table->string('network_status');
|
|
$table->integer('expires_at')->nullable();
|
|
$table->text('spec_json');
|
|
$table->text('credential_cipher')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_instance_snapshots', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('snapshot_key')->unique();
|
|
$table->integer('instance_id');
|
|
$table->integer('user_id');
|
|
$table->integer('host_id');
|
|
$table->string('name');
|
|
$table->string('state');
|
|
$table->string('consistency');
|
|
$table->integer('size_bytes')->nullable();
|
|
$table->string('last_operation_id')->nullable();
|
|
$table->integer('ready_at')->nullable();
|
|
$table->integer('last_restored_at')->nullable();
|
|
$table->integer('deleted_at')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_template_retirements', static function (
|
|
Blueprint $table): void {
|
|
$table->string('template_id')->primary();
|
|
$table->string('state');
|
|
$table->integer('template_created_at')->default(0);
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('operation_id')->unique();
|
|
$table->integer('order_id')->nullable();
|
|
$table->integer('instance_id')->nullable();
|
|
$table->integer('snapshot_id')->nullable();
|
|
$table->integer('user_id');
|
|
$table->integer('host_id');
|
|
$table->string('kind');
|
|
$table->string('status');
|
|
$table->text('payload_cipher');
|
|
$table->integer('attempts')->default(0);
|
|
$table->integer('dispatched_at')->nullable();
|
|
$table->string('lease_token')->nullable();
|
|
$table->integer('lease_until')->default(0);
|
|
$table->integer('available_at');
|
|
$table->string('error_code')->nullable();
|
|
$table->text('result_json')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
|
|
$instance = new CloudInstance();
|
|
$instance->uuid = '11111111-1111-4111-8111-111111111111';
|
|
$instance->host_id = 7;
|
|
$instance->user_id = 10;
|
|
$instance->name = 'vm-snap';
|
|
$instance->status = 'active';
|
|
$instance->service_status = 'exempt';
|
|
$instance->network_status = 'active';
|
|
$instance->expires_at = null;
|
|
$instance->spec_json = json_encode([
|
|
'template_id' => 'debian-12', 'os_source' => 'cloud_template',
|
|
'os_name' => 'Debian 12', 'login_name' => 'root',
|
|
'guest_hostname' => 'snapshot-old',
|
|
'public_ip_cidr' => '192.0.2.10/24', 'region' => 'HKG',
|
|
'vcpu' => 2, 'memory_mib' => 2048, 'disk_gib' => 40,
|
|
'engine_metadata_version' => 1,
|
|
], JSON_THROW_ON_ERROR);
|
|
$instance->credential_cipher = CloudSecrets::encrypt([
|
|
'password' => 'old-password', 'login_name' => 'root',
|
|
]);
|
|
$instance->created_at = $instance->updated_at = time();
|
|
$instance->save();
|
|
$actor = ['user_id' => 10, 'is_admin' => false,
|
|
'is_super_admin' => false, 'can_manage' => false];
|
|
|
|
$first = CloudSnapshotService::create($instance, $actor, [
|
|
'name' => '发布前', 'consistency_mode' => 'filesystem',
|
|
]);
|
|
$firstJob = $first['job'];
|
|
$firstPayload = CloudSecrets::decrypt((string) $firstJob->payload_cipher);
|
|
$firstKey = (string) $first['snapshot']['snapshot_key'];
|
|
$assert(preg_match('/^[a-f0-9]{32}$/D', $firstKey) === 1
|
|
&& $firstPayload['method'] === 'POST'
|
|
&& $firstPayload['resource'] === 'vms/' . $instance->uuid . '/snapshots'
|
|
&& $firstPayload['body'] === [
|
|
'snapshot_key' => $firstKey,
|
|
'name' => '发布前',
|
|
'consistency_mode' => 'filesystem',
|
|
]
|
|
&& (int) $firstJob->snapshot_id > 0,
|
|
'Snapshot create did not persist an opaque key and bounded engine contract.');
|
|
CloudSnapshotService::finishJobLocked($firstJob, 'succeeded', [
|
|
'snapshot_key' => $firstKey, 'created' => true,
|
|
'consistency' => 'filesystem', 'size_bytes' => 4096,
|
|
], $firstPayload);
|
|
$firstJob->status = 'succeeded';
|
|
$firstJob->save();
|
|
|
|
$listed = CloudSnapshotService::listing($instance, $actor);
|
|
$assert($listed['max_snapshots'] === 2 && $listed['used_slots'] === 1
|
|
&& $listed['can_create'] === true && $listed['can_mutate'] === true
|
|
&& count($listed['snapshots']) === 1
|
|
&& $listed['snapshots'][0]['state'] === 'ready'
|
|
&& $listed['snapshots'][0]['consistency'] === 'filesystem'
|
|
&& $listed['snapshots'][0]['size_bytes'] === 4096,
|
|
'Database-only snapshot listing did not expose the committed catalog state.');
|
|
$storedFirst = CloudInstanceSnapshot::find((int) $firstJob->snapshot_id);
|
|
$assert(!array_key_exists('credential_cipher', $storedFirst->getAttributes())
|
|
&& !array_key_exists('system_metadata_json', $storedFirst->getAttributes()),
|
|
'Snapshot catalog duplicated an instance credential or seed metadata.');
|
|
$settingsWithSnapshot = CloudJobService::enqueue($instance, $actor,
|
|
'instance_settings', 'POST', '/settings', [
|
|
'hostname' => 'current.example', 'confirm_restart' => true,
|
|
]);
|
|
$assert((string) $settingsWithSnapshot->status === 'queued',
|
|
'A ready snapshot incorrectly blocked a current-seed settings mutation.');
|
|
$settingsWithSnapshot->status = 'failed';
|
|
$settingsWithSnapshot->save();
|
|
$expect(static fn () => CloudJobService::enqueue($instance, $actor,
|
|
'reinstall', 'POST', '/reinstall', [
|
|
'template_id' => 'debian-12',
|
|
]), 409, 'A retained snapshot allowed a reinstall mutation.');
|
|
|
|
$second = CloudSnapshotService::create($instance, $actor, [
|
|
'name' => 'Crash point', 'consistency_mode' => 'crash',
|
|
]);
|
|
$secondJob = $second['job'];
|
|
$secondPayload = CloudSecrets::decrypt((string) $secondJob->payload_cipher);
|
|
CloudSnapshotService::finishJobLocked($secondJob, 'succeeded', [
|
|
'snapshot_key' => $second['snapshot']['snapshot_key'],
|
|
'created' => true, 'consistency' => 'crash',
|
|
], $secondPayload);
|
|
$secondJob->status = 'succeeded';
|
|
$secondJob->save();
|
|
$expect(static fn () => CloudSnapshotService::create($instance, $actor, [
|
|
'name' => 'third', 'consistency_mode' => 'filesystem',
|
|
]), 409, 'A third live snapshot bypassed the per-instance limit.');
|
|
|
|
$restore = CloudSnapshotService::restore($instance, $actor, $firstKey,
|
|
['confirmation_name' => 'vm-snap']);
|
|
$restoreJob = $restore['job'];
|
|
$restorePayload = CloudSecrets::decrypt((string) $restoreJob->payload_cipher);
|
|
$assert($restorePayload['method'] === 'POST'
|
|
&& $restorePayload['resource'] === 'vms/' . $instance->uuid
|
|
. '/snapshots/' . $firstKey . '/restore'
|
|
&& $restorePayload['body'] === [],
|
|
'Snapshot restore engine contract changed unexpectedly.');
|
|
CloudSnapshotService::finishJobLocked($restoreJob, 'uncertain', null,
|
|
$restorePayload);
|
|
$restoreJob->status = 'uncertain';
|
|
$restoreJob->save();
|
|
$uncertain = CloudSnapshotService::listing($instance, $actor);
|
|
$assert($uncertain['snapshots'][1]['state'] === 'uncertain'
|
|
|| $uncertain['snapshots'][0]['state'] === 'uncertain',
|
|
'An ambiguous restore did not keep a visible uncertain catalog state.');
|
|
$credentialBeforeRestore = (string) $instance->credential_cipher;
|
|
$specBeforeRestore = (string) $instance->spec_json;
|
|
CloudSnapshotService::finishJobLocked($restoreJob, 'succeeded', [
|
|
'snapshot_key' => $firstKey, 'restored' => true,
|
|
], $restorePayload);
|
|
$restoreJob->status = 'succeeded';
|
|
$restoreJob->save();
|
|
$instance->refresh();
|
|
$assert((string) $instance->credential_cipher === $credentialBeforeRestore
|
|
&& (string) $instance->spec_json === $specBeforeRestore,
|
|
'Snapshot restore rewrote panel credentials or seed metadata.');
|
|
|
|
$delete = CloudSnapshotService::delete($instance, $actor, $firstKey,
|
|
['confirmation_name' => 'vm-snap']);
|
|
$deleteJob = $delete['job'];
|
|
$deletePayload = CloudSecrets::decrypt((string) $deleteJob->payload_cipher);
|
|
$assert($deletePayload['method'] === 'DELETE'
|
|
&& $deletePayload['resource'] === 'vms/' . $instance->uuid
|
|
. '/snapshots/' . $firstKey
|
|
&& $deletePayload['body'] === [],
|
|
'Snapshot delete engine contract changed unexpectedly.');
|
|
CloudSnapshotService::finishJobLocked($deleteJob, 'succeeded', [
|
|
'snapshot_key' => $firstKey, 'deleted' => true,
|
|
], $deletePayload);
|
|
$deleteJob->status = 'succeeded';
|
|
$deleteJob->save();
|
|
$assert(CloudSnapshotService::listing($instance, $actor)['used_slots'] === 1,
|
|
'A confirmed deletion did not release exactly one snapshot slot.');
|
|
|
|
$failed = CloudSnapshotService::create($instance, $actor, [
|
|
'name' => 'will-fail', 'consistency_mode' => 'filesystem',
|
|
]);
|
|
$failedJob = $failed['job'];
|
|
CloudSnapshotService::finishJobLocked($failedJob, 'failed', null,
|
|
CloudSecrets::decrypt((string) $failedJob->payload_cipher));
|
|
$failedJob->status = 'failed';
|
|
$failedJob->save();
|
|
$assert(CloudInstanceSnapshot::find((int) $failedJob->snapshot_id)->state
|
|
=== 'failed'
|
|
&& CloudSnapshotService::listing($instance, $actor)['used_slots'] === 1,
|
|
'A definitive create failure kept a slot or lost its audit state.');
|
|
|
|
$expect(static fn () => CloudSnapshotService::create($instance,
|
|
['user_id' => 11, 'is_admin' => false, 'is_super_admin' => false,
|
|
'can_manage' => false],
|
|
['name' => 'foreign', 'consistency_mode' => 'crash']),
|
|
403, 'A different tenant created a snapshot for this instance.');
|
|
$expect(static fn () => CloudSnapshotService::listing($instance,
|
|
['user_id' => 11, 'is_admin' => false]), 404,
|
|
'A different tenant listed snapshots for this instance.');
|
|
$expect(static fn () => CloudSnapshotService::restore($instance, $actor,
|
|
(string) $second['snapshot']['snapshot_key'],
|
|
['confirmation_name' => 'wrong-name']), 422,
|
|
'Snapshot restore accepted a mismatched instance confirmation.');
|
|
|
|
$qga = CloudOperationErrorCatalog::workerSummary('snapshot_create',
|
|
'failed', ['error_code' => 'snapshot_guest_agent_unavailable']);
|
|
$assert($qga === 'snapshot_guest_agent_unavailable'
|
|
&& str_contains((string) CloudOperationErrorCatalog::userMessage(
|
|
'snapshot_create', 'failed', $qga), 'QEMU Guest Agent'),
|
|
'QGA unavailability was not preserved as a safe, actionable job error.');
|
|
|
|
$secondKey = (string) $second['snapshot']['snapshot_key'];
|
|
$restoreFailed = CloudSnapshotService::restore($instance, $actor,
|
|
$secondKey, ['confirmation_name' => 'vm-snap']);
|
|
$restoreFailedJob = $restoreFailed['job'];
|
|
CloudSnapshotService::finishJobLocked($restoreFailedJob, 'failed', null,
|
|
CloudSecrets::decrypt((string) $restoreFailedJob->payload_cipher));
|
|
$restoreFailedJob->status = 'failed';
|
|
$restoreFailedJob->save();
|
|
$assert(CloudInstanceSnapshot::find((int) $restoreFailedJob->snapshot_id)->state
|
|
=== 'ready',
|
|
'A definitive restore failure did not return the source snapshot to ready.');
|
|
|
|
$deleteUncertain = CloudSnapshotService::delete($instance, $actor,
|
|
$secondKey, ['confirmation_name' => 'vm-snap']);
|
|
$deleteUncertainJob = $deleteUncertain['job'];
|
|
$deleteUncertainPayload = CloudSecrets::decrypt(
|
|
(string) $deleteUncertainJob->payload_cipher
|
|
);
|
|
CloudSnapshotService::finishJobLocked($deleteUncertainJob, 'uncertain', null,
|
|
$deleteUncertainPayload);
|
|
$deleteUncertainJob->status = 'uncertain';
|
|
$deleteUncertainJob->save();
|
|
$assert(CloudInstanceSnapshot::find((int) $deleteUncertainJob->snapshot_id)->state
|
|
=== 'uncertain',
|
|
'An ambiguous delete did not preserve its uncertain snapshot slot.');
|
|
CloudSnapshotService::finishJobLocked($deleteUncertainJob, 'succeeded', [
|
|
'snapshot_key' => $secondKey, 'deleted' => true,
|
|
], $deleteUncertainPayload);
|
|
$deleteUncertainJob->status = 'succeeded';
|
|
$deleteUncertainJob->save();
|
|
|
|
$cascade = CloudSnapshotService::create($instance, $actor, [
|
|
'name' => 'delete-with-instance', 'consistency_mode' => 'crash',
|
|
]);
|
|
$cascadeJob = $cascade['job'];
|
|
$cascadeKey = (string) $cascade['snapshot']['snapshot_key'];
|
|
CloudSnapshotService::finishJobLocked($cascadeJob, 'succeeded', [
|
|
'snapshot_key' => $cascadeKey, 'created' => true,
|
|
'consistency' => 'crash',
|
|
], CloudSecrets::decrypt((string) $cascadeJob->payload_cipher));
|
|
$cascadeJob->status = 'succeeded';
|
|
$cascadeJob->save();
|
|
|
|
$deleteVm = new CloudJob();
|
|
$deleteVm->operation_id = str_repeat('d', 32);
|
|
$deleteVm->instance_id = $instance->id;
|
|
$deleteVm->kind = 'delete';
|
|
CloudSnapshotService::finishInstanceDeleteLocked($deleteVm, 'uncertain');
|
|
$assert(CloudInstanceSnapshot::where('snapshot_key', $cascadeKey)
|
|
->value('state') === 'uncertain',
|
|
'An ambiguous VM deletion did not preserve its snapshot cleanup uncertainty.');
|
|
CloudSnapshotService::finishInstanceDeleteLocked($deleteVm, 'failed');
|
|
$assert(CloudInstanceSnapshot::where('snapshot_key', $cascadeKey)
|
|
->value('state') === 'ready',
|
|
'A definitive VM-delete failure did not restore its snapshot catalog state.');
|
|
CloudSnapshotService::finishInstanceDeleteLocked($deleteVm, 'succeeded');
|
|
$assert(CloudInstanceSnapshot::where('instance_id', $instance->id)
|
|
->whereIn('state', ['creating', 'ready', 'restoring', 'deleting', 'uncertain'])
|
|
->count() === 0,
|
|
'Confirmed VM deletion did not cascade its PHP snapshot catalog.');
|
|
|
|
$delegated = CloudSnapshotService::create($instance, [
|
|
'user_id' => 99, 'is_admin' => true, 'is_super_admin' => true,
|
|
'can_manage' => true,
|
|
], ['name' => 'admin-audit', 'consistency_mode' => 'crash']);
|
|
$delegatedJob = $delegated['job'];
|
|
$delegatedSnapshot = CloudInstanceSnapshot::find(
|
|
(int) $delegatedJob->snapshot_id
|
|
);
|
|
$delegatedPayload = CloudSecrets::decrypt(
|
|
(string) $delegatedJob->payload_cipher
|
|
);
|
|
$assert((int) $delegatedJob->user_id === 99
|
|
&& (int) $delegatedSnapshot->user_id === (int) $instance->user_id
|
|
&& (int) $delegatedPayload['owner_id'] === (int) $instance->user_id,
|
|
'Administrator snapshot action lost actor/owner audit separation.');
|
|
$ownerView = CloudSnapshotService::listing($instance, $actor);
|
|
$adminView = CloudSnapshotService::listing($instance, [
|
|
'user_id' => 99, 'is_admin' => true, 'is_super_admin' => true,
|
|
'can_manage' => true,
|
|
]);
|
|
$assert($ownerView['snapshots'][0]['job_id'] === null
|
|
&& $ownerView['active_job'] !== null
|
|
&& !array_key_exists('id', $ownerView['active_job'])
|
|
&& (string) ($adminView['active_job']['id'] ?? '')
|
|
=== (string) $delegatedJob->id,
|
|
'Tenant listing exposed an administrator-scoped job as directly pollable.');
|
|
CloudSnapshotService::finishJobLocked($delegatedJob, 'failed', null,
|
|
$delegatedPayload);
|
|
$delegatedJob->status = 'failed';
|
|
$delegatedJob->save();
|
|
$assert(CloudInstanceSnapshot::find((int) $delegatedJob->snapshot_id)->state
|
|
=== 'failed',
|
|
'Administrator snapshot result could not reconcile against its owner.');
|
|
|
|
$settingsAfterDelete = CloudJobService::enqueue($instance, $actor,
|
|
'instance_settings', 'POST', '/settings', [
|
|
'hostname' => 'allowed.example', 'confirm_restart' => true,
|
|
]);
|
|
$assert((string) $settingsAfterDelete->status === 'queued',
|
|
'Deleting the final snapshot did not release instance settings admission.');
|
|
$settingsAfterDelete->status = 'failed';
|
|
$settingsAfterDelete->save();
|
|
$reinstallAfterDelete = CloudJobService::enqueue($instance, $actor,
|
|
'reinstall', 'POST', '/reinstall', [
|
|
'template_id' => 'debian-12',
|
|
]);
|
|
$assert((string) $reinstallAfterDelete->status === 'queued',
|
|
'Deleting the final snapshot did not release reinstall admission.');
|
|
|
|
echo "Cloud snapshot catalog, admission and worker state machine passed.\n";
|
|
}
|