You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudCatalogImportStateMach...

778 lines
34 KiB
PHTML

<?php
declare(strict_types=1);
namespace App\Services\Cloud {
/** Test-only sealing stub; the state-machine assertions do not inspect ciphertext. */
final class CloudSecrets
{
public static function encrypt(array $payload, string $purpose): string
{
return 'test-sealed:' . $purpose;
}
}
final class CatalogImportFakeAgent
{
public array $requests = [];
/** @var null|callable */
public $beforeResponse = null;
public ?array $getResponse = null;
public ?array $postResponse = null;
public ?array $reconcileResponse = null;
public bool $ambiguousReplay = false;
public bool $preDispatchBusy = false;
public function request(string $method, string $target, array $actor,
?array $payload = null): array
{
$this->requests[] = compact('method', 'target', 'actor', 'payload');
if (is_callable($this->beforeResponse)) {
($this->beforeResponse)($method, $target, $actor, $payload);
}
$operationId = (string) ($payload['operation_id']
?? $actor['operation_id'] ?? basename($target));
if (preg_match('#^/v1/operations/([A-Za-z0-9_-]+)/reconcile$#D',
$target, $match) === 1) {
$operationId = rawurldecode($match[1]);
}
if ($method === 'GET') {
return ['operation_id' => $operationId]
+ ($this->getResponse ?? ['status' => 'running']);
}
if ($method === 'POST'
&& preg_match('#^/v1/operations/[A-Za-z0-9_-]+/reconcile$#D',
$target) === 1) {
return ['operation_id' => $operationId]
+ ($this->reconcileResponse ?? $this->postResponse
?? ['status' => 'uncertain']);
}
if ($method === 'POST' && $this->preDispatchBusy
&& in_array($target,
['/v1/templates', '/v1/user-isos'], true)) {
throw new CloudAgentError('request was not dispatched', 503,
'agent_busy', true, 503, false, true);
}
if ($method === 'POST' && $this->ambiguousReplay) {
throw new CloudAgentError('nonce may have dispatched', 502,
'replayed_request', false, 401);
}
if ($this->postResponse !== null) {
return ['operation_id' => $operationId]
+ $this->postResponse;
}
if ($target === '/v1/templates') {
return [
'operation_id' => $operationId,
'status' => 'succeeded',
'result' => [
'id' => (string) $payload['id'],
'revision' => (int) $payload['revision'],
'generation_token' =>
(string) $payload['generation_token'],
'size_bytes' => 4096,
'sha256' => str_repeat('a', 64),
],
];
}
if ($target === '/v1/user-isos') {
return [
'operation_id' => $operationId,
'status' => 'succeeded',
'result' => [
'id' => (string) $payload['id'],
'owner_id' => (int) $payload['owner_id'],
'size_bytes' => 8192,
'sha256' => str_repeat('b', 64),
],
];
}
throw new \RuntimeException('Unexpected fake Agent request');
}
}
final class CloudAgentSettingsService
{
public static array $hostIds = [101];
public static ?CatalogImportFakeAgent $client = null;
public static function enabledHostIds(): array
{
return self::$hostIds;
}
public static function clientForHost(int $hostId,
float $timeoutSeconds = 20.0): CatalogImportFakeAgent
{
if (!in_array($hostId, self::$hostIds, true)
|| self::$client === null) {
throw new \RuntimeException('Missing fake Agent');
}
return self::$client;
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudTemplate;
use App\Models\CloudTemplateReplica;
use App\Models\CloudUserIso;
use App\Models\CloudUserIsoReplica;
use App\Services\Cloud\CatalogImportFakeAgent;
use App\Services\Cloud\CloudAgentSettingsService;
use App\Services\Cloud\CloudTemplateCatalogService;
use App\Services\Cloud\CloudUserIsoService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_templates', static function (Blueprint $table): void {
$table->string('id')->primary();
$table->string('name');
$table->string('os_family');
$table->string('os_version');
$table->string('source_type');
$table->text('sealed_source_url');
$table->integer('image_bytes')->default(0);
$table->string('sha256')->default('');
$table->text('startup_script')->default('');
$table->integer('revision');
$table->string('generation_token')->unique();
$table->string('state');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_template_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('template_id');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('image_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('operation_id')->default('');
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'template_id']);
});
$schema->create('cloud_orders', static function (Blueprint $table): void {
$table->increments('id');
$table->string('status');
$table->text('spec_json');
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('kind');
$table->string('status');
$table->text('sealed_payload');
});
$schema->create('cloud_user_isos', static function (Blueprint $table): void {
$table->string('id')->primary();
$table->integer('user_id');
$table->string('name');
$table->text('sealed_source_url');
$table->integer('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('state');
$table->string('import_operation_id');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_user_iso_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('user_iso_id');
$table->integer('user_id');
$table->string('state');
$table->integer('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('operation_id')->default('');
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'user_iso_id']);
});
$agent = new CatalogImportFakeAgent();
CloudAgentSettingsService::$client = $agent;
// An Agent success followed by a local metadata write failure is uncertain,
// never a definite dispatch failure. The transaction leaves the replica
// pending long enough for the guarded uncertainty transition to latch.
Capsule::statement("CREATE TRIGGER reject_template_digest "
. "BEFORE UPDATE OF sha256 ON cloud_templates "
. "WHEN NEW.sha256 <> OLD.sha256 BEGIN "
. "SELECT RAISE(ABORT, 'template persistence failure'); END");
$templateJob = CloudTemplateCatalogService::importRemote([
'name' => 'Debian 12 persistence test',
'url' => 'https://images.example/debian-12.qcow2',
'sha256' => '',
], ['user_id' => 1, 'can_manage' => true]);
$template = CloudTemplate::query()->where('id',
(string) $templateJob['job_id'])->first();
$templateReplica = CloudTemplateReplica::query()
->where('template_id', (string) $template->id)->first();
$assert($templateJob['status'] === 'uncertain'
&& (string) $template->state === 'error'
&& (string) $templateReplica->state === 'uncertain'
&& (string) $templateReplica->error_code === 'panel_persist_failed',
'Template Agent success was mislabeled after panel persistence failed.');
Capsule::statement('DROP TRIGGER reject_template_digest');
Capsule::statement("CREATE TRIGGER reject_iso_digest "
. "BEFORE UPDATE OF sha256 ON cloud_user_isos "
. "WHEN NEW.sha256 <> OLD.sha256 BEGIN "
. "SELECT RAISE(ABORT, 'ISO persistence failure'); END");
$isoJob = CloudUserIsoService::import([
'name' => 'Recovery ISO',
'url' => 'https://images.example/recovery.iso',
], ['user_id' => 42, 'can_manage' => false]);
$iso = CloudUserIso::query()->where('import_operation_id',
(string) $isoJob['job_id'])->first();
$isoReplica = CloudUserIsoReplica::query()
->where('user_iso_id', (string) $iso->id)->first();
$assert($isoJob['status'] === 'uncertain'
&& (string) $iso->state === 'failed'
&& (string) $isoReplica->state === 'uncertain'
&& (string) $isoReplica->error_code === 'panel_persist_failed',
'ISO Agent success was mislabeled after panel persistence failed.');
Capsule::statement('DROP TRIGGER reject_iso_digest');
// A process interruption after the last replica update must be repaired by
// status lookup even when there is no pending operation left to poll.
Capsule::table('cloud_templates')->where('id', (string) $template->id)
->update(['state' => 'importing']);
Capsule::table('cloud_template_replicas')
->where('template_id', (string) $template->id)->update([
'state' => 'active',
'image_bytes' => 4096,
'sha256' => str_repeat('a', 64),
'error_code' => '',
]);
$requestsBeforeTemplateRepair = count($agent->requests);
$repairedTemplate = CloudTemplateCatalogService::importJob(
(string) $template->id, ['user_id' => 1, 'can_manage' => true]
);
$assert($repairedTemplate['status'] === 'succeeded'
&& (string) CloudTemplate::find((string) $template->id)->state
=== 'active'
&& count($agent->requests) === $requestsBeforeTemplateRepair,
'Template aggregate state did not self-heal without pending replicas.');
Capsule::table('cloud_user_isos')->where('id', (string) $iso->id)
->update([
'state' => 'importing',
'size_bytes' => 8192,
'sha256' => str_repeat('b', 64),
]);
Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', (string) $iso->id)->update([
'state' => 'active',
'size_bytes' => 8192,
'sha256' => str_repeat('b', 64),
'error_code' => '',
]);
$requestsBeforeIsoRepair = count($agent->requests);
$repairedIso = CloudUserIsoService::job((string) $iso->import_operation_id,
['user_id' => 42, 'can_manage' => false]);
$assert($repairedIso['status'] === 'succeeded'
&& (string) CloudUserIso::find((string) $iso->id)->state === 'active'
&& count($agent->requests) === $requestsBeforeIsoRepair,
'ISO aggregate state did not self-heal without pending replicas.');
// `dispatching` is a normal durable Agent state. It must remain pollable
// rather than being surfaced as an ambiguous import result.
$agent->postResponse = ['status' => 'dispatching'];
$dispatchTemplateJob = CloudTemplateCatalogService::importRemote([
'name' => 'Debian 13 dispatching template',
'url' => 'https://images.example/debian-13.qcow2',
'sha256' => '',
], ['user_id' => 1, 'can_manage' => true]);
$dispatchTemplate = CloudTemplate::find(
(string) $dispatchTemplateJob['job_id']
);
$dispatchTemplateReplica = CloudTemplateReplica::query()
->where('template_id', (string) $dispatchTemplate->id)->first();
$assert($dispatchTemplateJob['status'] === 'running'
&& (string) $dispatchTemplate->state === 'importing'
&& (string) $dispatchTemplateReplica->state === 'pending'
&& (string) $dispatchTemplateReplica->error_code === '',
'Template Agent dispatching response was made uncertain.');
$dispatchIsoJob = CloudUserIsoService::import([
'name' => 'Dispatching ISO',
'url' => 'https://images.example/dispatching.iso',
], ['user_id' => 52, 'can_manage' => false]);
$dispatchIso = CloudUserIso::query()->where('import_operation_id',
(string) $dispatchIsoJob['job_id'])->first();
$dispatchIsoReplica = CloudUserIsoReplica::query()
->where('user_iso_id', (string) $dispatchIso->id)->first();
$assert($dispatchIsoJob['status'] === 'running'
&& (string) $dispatchIso->state === 'importing'
&& (string) $dispatchIsoReplica->state === 'pending'
&& (string) $dispatchIsoReplica->error_code === '',
'ISO Agent dispatching response was made uncertain.');
$agent->postResponse = null;
// Catalog imports intentionally get one POST only. Even exact
// not_dispatched backpressure is terminal so expiring upload/source
// capabilities are never reused by an automatic retry; an administrator
// must create a fresh import with a fresh operation and source snapshot.
$agent->preDispatchBusy = true;
$preDispatchTemplate = CloudTemplateCatalogService::importRemote([
'name' => 'Debian 12 pre-dispatch terminal template',
'url' => 'https://images.example/debian-12-predispatch.qcow2',
'sha256' => '',
], ['user_id' => 1, 'can_manage' => true]);
$preDispatchTemplateRow = CloudTemplate::find(
(string) $preDispatchTemplate['job_id']
);
$preDispatchTemplateReplica = CloudTemplateReplica::query()
->where('template_id', (string) $preDispatchTemplateRow->id)->first();
$preDispatchIso = CloudUserIsoService::import([
'name' => 'Pre-dispatch terminal ISO',
'url' => 'https://images.example/predispatch.iso',
], ['user_id' => 72, 'can_manage' => false]);
$preDispatchIsoRow = CloudUserIso::query()->where('import_operation_id',
(string) $preDispatchIso['job_id'])->first();
$preDispatchIsoReplica = CloudUserIsoReplica::query()
->where('user_iso_id', (string) $preDispatchIsoRow->id)->first();
$requestsAfterPreDispatch = count($agent->requests);
CloudTemplateCatalogService::importJob(
(string) $preDispatchTemplateRow->id,
['user_id' => 1, 'can_manage' => true]
);
CloudUserIsoService::job((string) $preDispatchIsoRow->import_operation_id,
['user_id' => 72, 'can_manage' => false]);
$assert($preDispatchTemplate['status'] === 'failed'
&& (string) $preDispatchTemplateReplica->state === 'failed'
&& (string) $preDispatchTemplateReplica->error_code === 'agent_busy'
&& $preDispatchIso['status'] === 'failed'
&& (string) $preDispatchIsoReplica->state === 'failed'
&& (string) $preDispatchIsoReplica->error_code === 'agent_busy'
&& count($agent->requests) === $requestsAfterPreDispatch,
'Definitive pre-dispatch catalog import was retried automatically.');
$agent->preDispatchBusy = false;
// A non-retryable response without explicit not-dispatched proof is still
// ambiguous. Retain both imports under their original operation IDs so a
// later status lookup can reconcile them; never label them failed here.
$agent->ambiguousReplay = true;
$replayedTemplateJob = CloudTemplateCatalogService::importRemote([
'name' => 'Debian 12 ambiguous replay template',
'url' => 'https://images.example/debian-12-ambiguous-replay.qcow2',
'sha256' => '',
], ['user_id' => 1, 'can_manage' => true]);
$replayedTemplate = CloudTemplate::find(
(string) $replayedTemplateJob['job_id']
);
$replayedTemplateReplica = CloudTemplateReplica::query()
->where('template_id', (string) $replayedTemplate->id)->first();
$replayedIsoJob = CloudUserIsoService::import([
'name' => 'Ambiguous replay ISO',
'url' => 'https://images.example/ambiguous-replay.iso',
], ['user_id' => 62, 'can_manage' => false]);
$replayedIso = CloudUserIso::query()->where('import_operation_id',
(string) $replayedIsoJob['job_id'])->first();
$replayedIsoReplica = CloudUserIsoReplica::query()
->where('user_iso_id', (string) $replayedIso->id)->first();
$assert($replayedTemplateJob['status'] === 'running'
&& (string) $replayedTemplateReplica->state === 'pending'
&& (string) $replayedTemplateReplica->error_code
=== 'replayed_request'
&& $replayedIsoJob['status'] === 'running'
&& (string) $replayedIsoReplica->state === 'pending'
&& (string) $replayedIsoReplica->error_code === 'replayed_request',
'Ambiguous non-retryable catalog import was made terminal.');
$agent->ambiguousReplay = false;
// An uncertain GET is inspected through the same operation's read-only
// reconcile route. Neither catalog import may be replayed.
$agent->postResponse = ['status' => 'dispatching'];
$reconcileTemplateJob = CloudTemplateCatalogService::importRemote([
'name' => 'Debian 12 template reconcile test',
'url' => 'https://images.example/debian-12-reconcile.qcow2',
'sha256' => '',
], ['user_id' => 1, 'can_manage' => true]);
$reconcileTemplate = CloudTemplate::find(
(string) $reconcileTemplateJob['job_id']
);
$reconcileTemplateReplica = CloudTemplateReplica::query()
->where('template_id', (string) $reconcileTemplate->id)->first();
$agent->postResponse = null;
$agent->getResponse = [
'status' => 'uncertain',
'error' => ['code' => 'operation_uncertain'],
];
$agent->reconcileResponse = [
'status' => 'succeeded',
'result' => [
'id' => (string) $reconcileTemplate->id,
'revision' => (int) $reconcileTemplate->revision,
'generation_token' =>
(string) $reconcileTemplate->generation_token,
'size_bytes' => 4096,
'sha256' => str_repeat('a', 64),
],
];
$templateRequestsBeforeReconcile = count($agent->requests);
$reconciledTemplate = CloudTemplateCatalogService::importJob(
(string) $reconcileTemplate->id,
['user_id' => 1, 'can_manage' => true]
);
$templateReconcileRequests = array_slice(
$agent->requests, $templateRequestsBeforeReconcile
);
$assert($reconciledTemplate['status'] === 'succeeded'
&& count($templateReconcileRequests) === 2
&& $templateReconcileRequests[0]['method'] === 'GET'
&& $templateReconcileRequests[0]['target']
=== '/v1/operations/'
. (string) $reconcileTemplateReplica->operation_id
&& $templateReconcileRequests[1]['method'] === 'POST'
&& $templateReconcileRequests[1]['target']
=== '/v1/operations/'
. (string) $reconcileTemplateReplica->operation_id
. '/reconcile',
'Template uncertainty did not reconcile the same operation without replay.');
$agent->postResponse = ['status' => 'dispatching'];
$reconcileIsoJob = CloudUserIsoService::import([
'name' => 'ISO reconcile test',
'url' => 'https://images.example/iso-reconcile.iso',
], ['user_id' => 54, 'can_manage' => false]);
$reconcileIso = CloudUserIso::query()->where('import_operation_id',
(string) $reconcileIsoJob['job_id'])->first();
$reconcileIsoReplica = CloudUserIsoReplica::query()
->where('user_iso_id', (string) $reconcileIso->id)->first();
$agent->postResponse = null;
$agent->reconcileResponse = [
'status' => 'succeeded',
'result' => [
'id' => (string) $reconcileIso->id,
'owner_id' => (int) $reconcileIso->user_id,
'size_bytes' => 8192,
'sha256' => str_repeat('b', 64),
],
];
$isoRequestsBeforeReconcile = count($agent->requests);
$reconciledIso = CloudUserIsoService::job(
(string) $reconcileIso->import_operation_id,
['user_id' => 54, 'can_manage' => false]
);
$isoReconcileRequests = array_slice(
$agent->requests, $isoRequestsBeforeReconcile
);
$assert($reconciledIso['status'] === 'succeeded'
&& count($isoReconcileRequests) === 2
&& $isoReconcileRequests[0]['method'] === 'GET'
&& $isoReconcileRequests[0]['target']
=== '/v1/operations/' . (string) $reconcileIsoReplica->operation_id
&& $isoReconcileRequests[1]['method'] === 'POST'
&& $isoReconcileRequests[1]['target']
=== '/v1/operations/' . (string) $reconcileIsoReplica->operation_id
. '/reconcile',
'User ISO uncertainty did not reconcile the same operation without replay.');
$agent->reconcileResponse = null;
// Repair only the historical false-positive marker. The old operation ID
// is polled exactly once; the import POST is never replayed.
$templateOperationId = (string) $dispatchTemplateReplica->operation_id;
Capsule::table('cloud_templates')->where('id',
(string) $dispatchTemplate->id)->update(['state' => 'error']);
Capsule::table('cloud_template_replicas')->where('template_id',
(string) $dispatchTemplate->id)->update([
'state' => 'uncertain',
'error_code' => 'agent_import_uncertain',
]);
$agent->getResponse = [
'status' => 'succeeded',
'result' => [
'id' => (string) $dispatchTemplate->id,
'revision' => (int) $dispatchTemplate->revision,
'generation_token' => (string) $dispatchTemplate->generation_token,
'size_bytes' => 4096,
'sha256' => str_repeat('a', 64),
],
];
$requestsBeforeTemplateRecovery = count($agent->requests);
$recoveredTemplate = CloudTemplateCatalogService::importJob(
(string) $dispatchTemplate->id,
['user_id' => 1, 'can_manage' => true]
);
$templateRecoveryRequest = $agent->requests[
$requestsBeforeTemplateRecovery
] ?? [];
$dispatchTemplateReplica->refresh();
$assert($recoveredTemplate['status'] === 'succeeded'
&& count($agent->requests) === $requestsBeforeTemplateRecovery + 1
&& ($templateRecoveryRequest['method'] ?? null) === 'GET'
&& ($templateRecoveryRequest['target'] ?? null)
=== '/v1/operations/' . $templateOperationId
&& (string) $dispatchTemplateReplica->operation_id
=== $templateOperationId
&& (string) $dispatchTemplateReplica->state === 'active',
'Historical template uncertainty did not reconcile its saved operation.');
$isoOperationId = (string) $dispatchIsoReplica->operation_id;
Capsule::table('cloud_user_isos')->where('id',
(string) $dispatchIso->id)->update(['state' => 'failed']);
Capsule::table('cloud_user_iso_replicas')->where('user_iso_id',
(string) $dispatchIso->id)->update([
'state' => 'uncertain',
'error_code' => 'agent_import_uncertain',
]);
$agent->getResponse = [
'status' => 'succeeded',
'result' => [
'id' => (string) $dispatchIso->id,
'owner_id' => (int) $dispatchIso->user_id,
'size_bytes' => 8192,
'sha256' => str_repeat('b', 64),
],
];
$requestsBeforeIsoRecovery = count($agent->requests);
$recoveredIso = CloudUserIsoService::job(
(string) $dispatchIso->import_operation_id,
['user_id' => 52, 'can_manage' => false]
);
$isoRecoveryRequest = $agent->requests[$requestsBeforeIsoRecovery] ?? [];
$dispatchIsoReplica->refresh();
$assert($recoveredIso['status'] === 'succeeded'
&& count($agent->requests) === $requestsBeforeIsoRecovery + 1
&& ($isoRecoveryRequest['method'] ?? null) === 'GET'
&& ($isoRecoveryRequest['target'] ?? null)
=== '/v1/operations/' . $isoOperationId
&& (string) $dispatchIsoReplica->operation_id === $isoOperationId
&& (string) $dispatchIsoReplica->state === 'active',
'Historical ISO uncertainty did not reconcile its saved operation.');
// A genuine Agent uncertainty gets a new marker and stays terminal. Only
// the exact legacy dispatching marker is eligible for recovery.
$agent->postResponse = [
'status' => 'uncertain',
'error' => ['code' => 'agent_import_uncertain'],
];
$lockedTemplateJob = CloudTemplateCatalogService::importRemote([
'name' => 'Debian 11 genuinely uncertain template',
'url' => 'https://images.example/debian-11.qcow2',
'sha256' => '',
], ['user_id' => 1, 'can_manage' => true]);
$lockedTemplate = CloudTemplate::find(
(string) $lockedTemplateJob['job_id']
);
$lockedTemplateReplica = CloudTemplateReplica::query()
->where('template_id', (string) $lockedTemplate->id)->first();
$agent->postResponse = null;
$requestsBeforeLockedTemplate = count($agent->requests);
$lockedTemplateAgain = CloudTemplateCatalogService::importJob(
(string) $lockedTemplate->id,
['user_id' => 1, 'can_manage' => true]
);
$lockedTemplateReplica->refresh();
$assert($lockedTemplateJob['status'] === 'uncertain'
&& $lockedTemplateAgain['status'] === 'uncertain'
&& count($agent->requests) === $requestsBeforeLockedTemplate
&& (string) $lockedTemplateReplica->state === 'uncertain'
&& (string) $lockedTemplateReplica->error_code
=== 'agent_operation_uncertain',
'A genuine template uncertainty was unlocked as legacy dispatching.');
$agent->postResponse = [
'status' => 'uncertain',
'error' => ['code' => 'agent_import_uncertain'],
];
$lockedIsoJob = CloudUserIsoService::import([
'name' => 'Genuinely uncertain ISO',
'url' => 'https://images.example/genuinely-uncertain.iso',
], ['user_id' => 53, 'can_manage' => false]);
$lockedIso = CloudUserIso::query()->where('import_operation_id',
(string) $lockedIsoJob['job_id'])->first();
$lockedIsoReplica = CloudUserIsoReplica::query()
->where('user_iso_id', (string) $lockedIso->id)->first();
$agent->postResponse = null;
$requestsBeforeLockedIso = count($agent->requests);
$lockedIsoAgain = CloudUserIsoService::job(
(string) $lockedIso->import_operation_id,
['user_id' => 53, 'can_manage' => false]
);
$lockedIsoReplica->refresh();
$assert($lockedIsoJob['status'] === 'uncertain'
&& $lockedIsoAgain['status'] === 'uncertain'
&& count($agent->requests) === $requestsBeforeLockedIso
&& (string) $lockedIsoReplica->state === 'uncertain'
&& (string) $lockedIsoReplica->error_code
=== 'agent_operation_uncertain',
'A genuine ISO uncertainty was unlocked as legacy dispatching.');
// Quota aggregation must use the bytes committed through the locked model,
// not the stale outer object that existed before the Agent response.
$quotaOwner = 77;
$quotaBytes = 10 * 1024 * 1024 * 1024;
Capsule::table('cloud_user_isos')->insert([
'id' => str_repeat('9', 32),
'user_id' => $quotaOwner,
'name' => 'Existing quota usage',
'sealed_source_url' => 'test-sealed:user-iso-source',
'size_bytes' => $quotaBytes - 4096,
'sha256' => str_repeat('9', 64),
'state' => 'active',
'import_operation_id' => str_repeat('8', 32),
'created_at' => time(),
'updated_at' => time(),
]);
$quotaJob = CloudUserIsoService::import([
'name' => 'Quota overflow ISO',
'url' => 'https://images.example/quota-overflow.iso',
], ['user_id' => $quotaOwner, 'can_manage' => false]);
$quotaIso = CloudUserIso::query()->where('import_operation_id',
(string) $quotaJob['job_id'])->first();
$assert($quotaJob['status'] === 'failed'
&& (int) $quotaIso->size_bytes === 8192
&& (string) $quotaIso->state === 'failed',
'ISO quota aggregation ignored the persisted Agent result size.');
// A successful stale template response must not overwrite a concurrently
// installed delete operation or move the parent out of deleting.
$staleTemplateId = 'debian-12-stale-import';
$oldTemplateOperation = str_repeat('c', 32);
$newTemplateOperation = str_repeat('d', 32);
$generation = str_repeat('e', 32);
$now = time();
Capsule::table('cloud_templates')->insert([
'id' => $staleTemplateId,
'name' => 'Debian 12 stale import',
'os_family' => 'Debian',
'os_version' => '12',
'source_type' => 'remote',
'sealed_source_url' => 'test-sealed:template-source',
'image_bytes' => 0,
'sha256' => '',
'startup_script' => '',
'revision' => 1,
'generation_token' => $generation,
'state' => 'importing',
'created_at' => $now,
'updated_at' => $now,
]);
Capsule::table('cloud_template_replicas')->insert([
'host_id' => 101,
'template_id' => $staleTemplateId,
'revision' => 1,
'generation_token' => $generation,
'state' => 'pending',
'image_bytes' => 0,
'sha256' => '',
'operation_id' => $oldTemplateOperation,
'error_code' => '',
'created_at' => $now,
'updated_at' => $now,
]);
$agent->beforeResponse = static function (string $method) use (
$staleTemplateId, $newTemplateOperation
): void {
if ($method !== 'GET') {
return;
}
Capsule::table('cloud_templates')->where('id', $staleTemplateId)
->update(['state' => 'deleting']);
Capsule::table('cloud_template_replicas')
->where('template_id', $staleTemplateId)->update([
'state' => 'delete_succeeded',
'operation_id' => $newTemplateOperation,
'error_code' => '',
]);
};
$agent->getResponse = [
'status' => 'succeeded',
'result' => [
'id' => $staleTemplateId,
'revision' => 1,
'generation_token' => $generation,
'size_bytes' => 4096,
'sha256' => str_repeat('a', 64),
],
];
CloudTemplateCatalogService::importJob($staleTemplateId,
['user_id' => 1, 'can_manage' => true]);
$staleTemplate = CloudTemplate::find($staleTemplateId);
$staleTemplateReplica = CloudTemplateReplica::query()
->where('template_id', $staleTemplateId)->first();
$assert((string) $staleTemplate->state === 'deleting'
&& (string) $staleTemplateReplica->state === 'delete_succeeded'
&& (string) $staleTemplateReplica->operation_id
=== $newTemplateOperation,
'Stale template polling rolled back a concurrent terminal delete.');
// The state predicate is independently required: even the same operation
// ID cannot move a terminal ISO replica back to pending.
$staleIsoId = str_repeat('f', 32);
$staleIsoJob = str_repeat('1', 32);
$staleIsoOperation = str_repeat('2', 32);
Capsule::table('cloud_user_isos')->insert([
'id' => $staleIsoId,
'user_id' => 42,
'name' => 'Stale ISO',
'sealed_source_url' => 'test-sealed:user-iso-source',
'size_bytes' => 0,
'sha256' => '',
'state' => 'importing',
'import_operation_id' => $staleIsoJob,
'created_at' => $now,
'updated_at' => $now,
]);
Capsule::table('cloud_user_iso_replicas')->insert([
'host_id' => 101,
'user_iso_id' => $staleIsoId,
'user_id' => 42,
'state' => 'pending',
'size_bytes' => 0,
'sha256' => '',
'operation_id' => $staleIsoOperation,
'error_code' => '',
'created_at' => $now,
'updated_at' => $now,
]);
$agent->beforeResponse = static function (string $method) use (
$staleIsoId
): void {
if ($method !== 'GET') {
return;
}
Capsule::table('cloud_user_isos')->where('id', $staleIsoId)
->update(['state' => 'active']);
Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $staleIsoId)->update([
'state' => 'active',
'error_code' => '',
]);
};
$agent->getResponse = null;
CloudUserIsoService::job($staleIsoJob,
['user_id' => 42, 'can_manage' => false]);
$staleIso = CloudUserIso::find($staleIsoId);
$staleIsoReplica = CloudUserIsoReplica::query()
->where('user_iso_id', $staleIsoId)->first();
$assert((string) $staleIso->state === 'active'
&& (string) $staleIsoReplica->state === 'active'
&& (string) $staleIsoReplica->operation_id === $staleIsoOperation,
'Stale ISO polling moved a terminal replica back to pending.');
echo "Catalog import persistence failures and stale-response CAS passed.\n";
}