You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudCatalogImportStateMach...

424 lines
17 KiB
PHP

<?php
declare(strict_types=1);
namespace App\Services\Cloud {
/** Test-only sealing stub; the state-machine assertions do not inspect ciphertext. */
final class CloudSecrets
{
public static function encrypt(array $payload, string $purpose): string
{
return 'test-sealed:' . $purpose;
}
}
final class CatalogImportFakeAgent
{
public array $requests = [];
/** @var null|callable */
public $beforeResponse = null;
public ?array $getResponse = null;
public function request(string $method, string $target, array $actor,
?array $payload = null): array
{
$this->requests[] = compact('method', 'target', 'actor', 'payload');
if (is_callable($this->beforeResponse)) {
($this->beforeResponse)($method, $target, $actor, $payload);
}
$operationId = (string) ($payload['operation_id']
?? $actor['operation_id'] ?? basename($target));
if ($method === 'GET') {
return ['operation_id' => $operationId]
+ ($this->getResponse ?? ['status' => 'running']);
}
if ($target === '/v1/templates') {
return [
'operation_id' => $operationId,
'status' => 'succeeded',
'result' => [
'id' => (string) $payload['id'],
'revision' => (int) $payload['revision'],
'generation_token' =>
(string) $payload['generation_token'],
'size_bytes' => 4096,
'sha256' => str_repeat('a', 64),
],
];
}
if ($target === '/v1/user-isos') {
return [
'operation_id' => $operationId,
'status' => 'succeeded',
'result' => [
'id' => (string) $payload['id'],
'owner_id' => (int) $payload['owner_id'],
'size_bytes' => 8192,
'sha256' => str_repeat('b', 64),
],
];
}
throw new \RuntimeException('Unexpected fake Agent request');
}
}
final class CloudAgentSettingsService
{
public static array $hostIds = [101];
public static ?CatalogImportFakeAgent $client = null;
public static function enabledHostIds(): array
{
return self::$hostIds;
}
public static function clientForHost(int $hostId,
float $timeoutSeconds = 20.0): CatalogImportFakeAgent
{
if (!in_array($hostId, self::$hostIds, true)
|| self::$client === null) {
throw new \RuntimeException('Missing fake Agent');
}
return self::$client;
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudTemplate;
use App\Models\CloudTemplateReplica;
use App\Models\CloudUserIso;
use App\Models\CloudUserIsoReplica;
use App\Services\Cloud\CatalogImportFakeAgent;
use App\Services\Cloud\CloudAgentSettingsService;
use App\Services\Cloud\CloudTemplateCatalogService;
use App\Services\Cloud\CloudUserIsoService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_templates', static function (Blueprint $table): void {
$table->string('id')->primary();
$table->string('name');
$table->string('os_family');
$table->string('os_version');
$table->string('source_type');
$table->text('sealed_source_url');
$table->integer('image_bytes')->default(0);
$table->string('sha256')->default('');
$table->text('startup_script')->default('');
$table->integer('revision');
$table->string('generation_token')->unique();
$table->string('state');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_template_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('template_id');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('image_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('operation_id')->default('');
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'template_id']);
});
$schema->create('cloud_orders', static function (Blueprint $table): void {
$table->increments('id');
$table->string('status');
$table->text('spec_json');
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('kind');
$table->string('status');
$table->text('sealed_payload');
});
$schema->create('cloud_user_isos', static function (Blueprint $table): void {
$table->string('id')->primary();
$table->integer('user_id');
$table->string('name');
$table->text('sealed_source_url');
$table->integer('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('state');
$table->string('import_operation_id');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_user_iso_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('user_iso_id');
$table->integer('user_id');
$table->string('state');
$table->integer('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('operation_id')->default('');
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'user_iso_id']);
});
$agent = new CatalogImportFakeAgent();
CloudAgentSettingsService::$client = $agent;
// An Agent success followed by a local metadata write failure is uncertain,
// never a definite dispatch failure. The transaction leaves the replica
// pending long enough for the guarded uncertainty transition to latch.
Capsule::statement("CREATE TRIGGER reject_template_digest "
. "BEFORE UPDATE OF sha256 ON cloud_templates "
. "WHEN NEW.sha256 <> OLD.sha256 BEGIN "
. "SELECT RAISE(ABORT, 'template persistence failure'); END");
$templateJob = CloudTemplateCatalogService::importRemote([
'name' => 'Debian 12 persistence test',
'url' => 'https://images.example/debian-12.qcow2',
'sha256' => '',
], ['user_id' => 1, 'can_manage' => true]);
$template = CloudTemplate::query()->where('id',
(string) $templateJob['job_id'])->first();
$templateReplica = CloudTemplateReplica::query()
->where('template_id', (string) $template->id)->first();
$assert($templateJob['status'] === 'uncertain'
&& (string) $template->state === 'error'
&& (string) $templateReplica->state === 'uncertain'
&& (string) $templateReplica->error_code === 'panel_persist_failed',
'Template Agent success was mislabeled after panel persistence failed.');
Capsule::statement('DROP TRIGGER reject_template_digest');
Capsule::statement("CREATE TRIGGER reject_iso_digest "
. "BEFORE UPDATE OF sha256 ON cloud_user_isos "
. "WHEN NEW.sha256 <> OLD.sha256 BEGIN "
. "SELECT RAISE(ABORT, 'ISO persistence failure'); END");
$isoJob = CloudUserIsoService::import([
'name' => 'Recovery ISO',
'url' => 'https://images.example/recovery.iso',
], ['user_id' => 42, 'can_manage' => false]);
$iso = CloudUserIso::query()->where('import_operation_id',
(string) $isoJob['job_id'])->first();
$isoReplica = CloudUserIsoReplica::query()
->where('user_iso_id', (string) $iso->id)->first();
$assert($isoJob['status'] === 'uncertain'
&& (string) $iso->state === 'failed'
&& (string) $isoReplica->state === 'uncertain'
&& (string) $isoReplica->error_code === 'panel_persist_failed',
'ISO Agent success was mislabeled after panel persistence failed.');
Capsule::statement('DROP TRIGGER reject_iso_digest');
// A process interruption after the last replica update must be repaired by
// status lookup even when there is no pending operation left to poll.
Capsule::table('cloud_templates')->where('id', (string) $template->id)
->update(['state' => 'importing']);
Capsule::table('cloud_template_replicas')
->where('template_id', (string) $template->id)->update([
'state' => 'active',
'image_bytes' => 4096,
'sha256' => str_repeat('a', 64),
'error_code' => '',
]);
$requestsBeforeTemplateRepair = count($agent->requests);
$repairedTemplate = CloudTemplateCatalogService::importJob(
(string) $template->id, ['user_id' => 1, 'can_manage' => true]
);
$assert($repairedTemplate['status'] === 'succeeded'
&& (string) CloudTemplate::find((string) $template->id)->state
=== 'active'
&& count($agent->requests) === $requestsBeforeTemplateRepair,
'Template aggregate state did not self-heal without pending replicas.');
Capsule::table('cloud_user_isos')->where('id', (string) $iso->id)
->update([
'state' => 'importing',
'size_bytes' => 8192,
'sha256' => str_repeat('b', 64),
]);
Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', (string) $iso->id)->update([
'state' => 'active',
'size_bytes' => 8192,
'sha256' => str_repeat('b', 64),
'error_code' => '',
]);
$requestsBeforeIsoRepair = count($agent->requests);
$repairedIso = CloudUserIsoService::job((string) $iso->import_operation_id,
['user_id' => 42, 'can_manage' => false]);
$assert($repairedIso['status'] === 'succeeded'
&& (string) CloudUserIso::find((string) $iso->id)->state === 'active'
&& count($agent->requests) === $requestsBeforeIsoRepair,
'ISO aggregate state did not self-heal without pending replicas.');
// Quota aggregation must use the bytes committed through the locked model,
// not the stale outer object that existed before the Agent response.
$quotaOwner = 77;
$quotaBytes = 10 * 1024 * 1024 * 1024;
Capsule::table('cloud_user_isos')->insert([
'id' => str_repeat('9', 32),
'user_id' => $quotaOwner,
'name' => 'Existing quota usage',
'sealed_source_url' => 'test-sealed:user-iso-source',
'size_bytes' => $quotaBytes - 4096,
'sha256' => str_repeat('9', 64),
'state' => 'active',
'import_operation_id' => str_repeat('8', 32),
'created_at' => time(),
'updated_at' => time(),
]);
$quotaJob = CloudUserIsoService::import([
'name' => 'Quota overflow ISO',
'url' => 'https://images.example/quota-overflow.iso',
], ['user_id' => $quotaOwner, 'can_manage' => false]);
$quotaIso = CloudUserIso::query()->where('import_operation_id',
(string) $quotaJob['job_id'])->first();
$assert($quotaJob['status'] === 'failed'
&& (int) $quotaIso->size_bytes === 8192
&& (string) $quotaIso->state === 'failed',
'ISO quota aggregation ignored the persisted Agent result size.');
// A successful stale template response must not overwrite a concurrently
// installed delete operation or move the parent out of deleting.
$staleTemplateId = 'debian-12-stale-import';
$oldTemplateOperation = str_repeat('c', 32);
$newTemplateOperation = str_repeat('d', 32);
$generation = str_repeat('e', 32);
$now = time();
Capsule::table('cloud_templates')->insert([
'id' => $staleTemplateId,
'name' => 'Debian 12 stale import',
'os_family' => 'Debian',
'os_version' => '12',
'source_type' => 'remote',
'sealed_source_url' => 'test-sealed:template-source',
'image_bytes' => 0,
'sha256' => '',
'startup_script' => '',
'revision' => 1,
'generation_token' => $generation,
'state' => 'importing',
'created_at' => $now,
'updated_at' => $now,
]);
Capsule::table('cloud_template_replicas')->insert([
'host_id' => 101,
'template_id' => $staleTemplateId,
'revision' => 1,
'generation_token' => $generation,
'state' => 'pending',
'image_bytes' => 0,
'sha256' => '',
'operation_id' => $oldTemplateOperation,
'error_code' => '',
'created_at' => $now,
'updated_at' => $now,
]);
$agent->beforeResponse = static function (string $method) use (
$staleTemplateId, $newTemplateOperation
): void {
if ($method !== 'GET') {
return;
}
Capsule::table('cloud_templates')->where('id', $staleTemplateId)
->update(['state' => 'deleting']);
Capsule::table('cloud_template_replicas')
->where('template_id', $staleTemplateId)->update([
'state' => 'delete_succeeded',
'operation_id' => $newTemplateOperation,
'error_code' => '',
]);
};
$agent->getResponse = [
'status' => 'succeeded',
'result' => [
'id' => $staleTemplateId,
'revision' => 1,
'generation_token' => $generation,
'size_bytes' => 4096,
'sha256' => str_repeat('a', 64),
],
];
CloudTemplateCatalogService::importJob($staleTemplateId,
['user_id' => 1, 'can_manage' => true]);
$staleTemplate = CloudTemplate::find($staleTemplateId);
$staleTemplateReplica = CloudTemplateReplica::query()
->where('template_id', $staleTemplateId)->first();
$assert((string) $staleTemplate->state === 'deleting'
&& (string) $staleTemplateReplica->state === 'delete_succeeded'
&& (string) $staleTemplateReplica->operation_id
=== $newTemplateOperation,
'Stale template polling rolled back a concurrent terminal delete.');
// The state predicate is independently required: even the same operation
// ID cannot move a terminal ISO replica back to pending.
$staleIsoId = str_repeat('f', 32);
$staleIsoJob = str_repeat('1', 32);
$staleIsoOperation = str_repeat('2', 32);
Capsule::table('cloud_user_isos')->insert([
'id' => $staleIsoId,
'user_id' => 42,
'name' => 'Stale ISO',
'sealed_source_url' => 'test-sealed:user-iso-source',
'size_bytes' => 0,
'sha256' => '',
'state' => 'importing',
'import_operation_id' => $staleIsoJob,
'created_at' => $now,
'updated_at' => $now,
]);
Capsule::table('cloud_user_iso_replicas')->insert([
'host_id' => 101,
'user_iso_id' => $staleIsoId,
'user_id' => 42,
'state' => 'pending',
'size_bytes' => 0,
'sha256' => '',
'operation_id' => $staleIsoOperation,
'error_code' => '',
'created_at' => $now,
'updated_at' => $now,
]);
$agent->beforeResponse = static function (string $method) use (
$staleIsoId
): void {
if ($method !== 'GET') {
return;
}
Capsule::table('cloud_user_isos')->where('id', $staleIsoId)
->update(['state' => 'active']);
Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $staleIsoId)->update([
'state' => 'active',
'error_code' => '',
]);
};
$agent->getResponse = null;
CloudUserIsoService::job($staleIsoJob,
['user_id' => 42, 'can_manage' => false]);
$staleIso = CloudUserIso::find($staleIsoId);
$staleIsoReplica = CloudUserIsoReplica::query()
->where('user_iso_id', $staleIsoId)->first();
$assert((string) $staleIso->state === 'active'
&& (string) $staleIsoReplica->state === 'active'
&& (string) $staleIsoReplica->operation_id === $staleIsoOperation,
'Stale ISO polling moved a terminal replica back to pending.');
echo "Catalog import persistence failures and stale-response CAS passed.\n";
}