requests[] = compact('method', 'target', 'actor', 'payload'); if (is_callable($this->beforeResponse)) { ($this->beforeResponse)($method, $target, $actor, $payload); } $operationId = (string) ($payload['operation_id'] ?? $actor['operation_id'] ?? basename($target)); if (preg_match('#^/v1/operations/([A-Za-z0-9_-]+)/reconcile$#D', $target, $match) === 1) { $operationId = rawurldecode($match[1]); } if ($method === 'GET') { return ['operation_id' => $operationId] + ($this->getResponse ?? ['status' => 'running']); } if ($method === 'POST' && preg_match('#^/v1/operations/[A-Za-z0-9_-]+/reconcile$#D', $target) === 1) { return ['operation_id' => $operationId] + ($this->reconcileResponse ?? $this->postResponse ?? ['status' => 'uncertain']); } if ($method === 'POST' && $this->preDispatchBusy && in_array($target, ['/v1/templates', '/v1/user-isos'], true)) { throw new CloudAgentError('request was not dispatched', 503, 'agent_busy', true, 503, false, true); } if ($method === 'POST' && $this->ambiguousReplay) { throw new CloudAgentError('nonce may have dispatched', 502, 'replayed_request', false, 401); } if ($this->postResponse !== null) { return ['operation_id' => $operationId] + $this->postResponse; } if ($target === '/v1/templates') { return [ 'operation_id' => $operationId, 'status' => 'succeeded', 'result' => [ 'id' => (string) $payload['id'], 'revision' => (int) $payload['revision'], 'generation_token' => (string) $payload['generation_token'], 'size_bytes' => 4096, 'sha256' => str_repeat('a', 64), ], ]; } if ($target === '/v1/user-isos') { return [ 'operation_id' => $operationId, 'status' => 'succeeded', 'result' => [ 'id' => (string) $payload['id'], 'owner_id' => (int) $payload['owner_id'], 'size_bytes' => 8192, 'sha256' => str_repeat('b', 64), ], ]; } throw new \RuntimeException('Unexpected fake Agent request'); } } final class CloudAgentSettingsService { public static array $hostIds = [101]; public static ?CatalogImportFakeAgent $client = null; public static function enabledHostIds(): array { return self::$hostIds; } public static function clientForHost(int $hostId, float $timeoutSeconds = 20.0): CatalogImportFakeAgent { if (!in_array($hostId, self::$hostIds, true) || self::$client === null) { throw new \RuntimeException('Missing fake Agent'); } return self::$client; } } } namespace { require dirname(__DIR__) . '/vendor/autoload.php'; use App\Models\CloudTemplate; use App\Models\CloudTemplateReplica; use App\Models\CloudUserIso; use App\Models\CloudUserIsoReplica; use App\Services\Cloud\CatalogImportFakeAgent; use App\Services\Cloud\CloudAgentSettingsService; use App\Services\Cloud\CloudTemplateCatalogService; use App\Services\Cloud\CloudUserIsoService; use Illuminate\Database\Capsule\Manager as Capsule; use Illuminate\Database\Schema\Blueprint; $assert = static function (bool $condition, string $message): void { if (!$condition) { throw new RuntimeException($message); } }; $capsule = new Capsule(); $capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('cloud_templates', static function (Blueprint $table): void { $table->string('id')->primary(); $table->string('name'); $table->string('os_family'); $table->string('os_version'); $table->string('source_type'); $table->text('sealed_source_url'); $table->integer('image_bytes')->default(0); $table->string('sha256')->default(''); $table->text('startup_script')->default(''); $table->integer('revision'); $table->string('generation_token')->unique(); $table->string('state'); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_template_replicas', static function (Blueprint $table): void { $table->integer('host_id'); $table->string('template_id'); $table->integer('revision'); $table->string('generation_token'); $table->string('state'); $table->integer('image_bytes')->default(0); $table->string('sha256')->default(''); $table->string('operation_id')->default(''); $table->string('error_code')->default(''); $table->integer('created_at'); $table->integer('updated_at'); $table->primary(['host_id', 'template_id']); }); $schema->create('cloud_orders', static function (Blueprint $table): void { $table->increments('id'); $table->string('status'); $table->text('spec_json'); }); $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('kind'); $table->string('status'); $table->text('sealed_payload'); }); $schema->create('cloud_user_isos', static function (Blueprint $table): void { $table->string('id')->primary(); $table->integer('user_id'); $table->string('name'); $table->text('sealed_source_url'); $table->integer('size_bytes')->default(0); $table->string('sha256')->default(''); $table->string('state'); $table->string('import_operation_id'); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_user_iso_replicas', static function (Blueprint $table): void { $table->integer('host_id'); $table->string('user_iso_id'); $table->integer('user_id'); $table->string('state'); $table->integer('size_bytes')->default(0); $table->string('sha256')->default(''); $table->string('operation_id')->default(''); $table->string('error_code')->default(''); $table->integer('created_at'); $table->integer('updated_at'); $table->primary(['host_id', 'user_iso_id']); }); $agent = new CatalogImportFakeAgent(); CloudAgentSettingsService::$client = $agent; // An Agent success followed by a local metadata write failure is uncertain, // never a definite dispatch failure. The transaction leaves the replica // pending long enough for the guarded uncertainty transition to latch. Capsule::statement("CREATE TRIGGER reject_template_digest " . "BEFORE UPDATE OF sha256 ON cloud_templates " . "WHEN NEW.sha256 <> OLD.sha256 BEGIN " . "SELECT RAISE(ABORT, 'template persistence failure'); END"); $templateJob = CloudTemplateCatalogService::importRemote([ 'name' => 'Debian 12 persistence test', 'url' => 'https://images.example/debian-12.qcow2', 'sha256' => '', ], ['user_id' => 1, 'can_manage' => true]); $template = CloudTemplate::query()->where('id', (string) $templateJob['job_id'])->first(); $templateReplica = CloudTemplateReplica::query() ->where('template_id', (string) $template->id)->first(); $assert($templateJob['status'] === 'uncertain' && (string) $template->state === 'error' && (string) $templateReplica->state === 'uncertain' && (string) $templateReplica->error_code === 'panel_persist_failed', 'Template Agent success was mislabeled after panel persistence failed.'); Capsule::statement('DROP TRIGGER reject_template_digest'); Capsule::statement("CREATE TRIGGER reject_iso_digest " . "BEFORE UPDATE OF sha256 ON cloud_user_isos " . "WHEN NEW.sha256 <> OLD.sha256 BEGIN " . "SELECT RAISE(ABORT, 'ISO persistence failure'); END"); $isoJob = CloudUserIsoService::import([ 'name' => 'Recovery ISO', 'url' => 'https://images.example/recovery.iso', ], ['user_id' => 42, 'can_manage' => false]); $iso = CloudUserIso::query()->where('import_operation_id', (string) $isoJob['job_id'])->first(); $isoReplica = CloudUserIsoReplica::query() ->where('user_iso_id', (string) $iso->id)->first(); $assert($isoJob['status'] === 'uncertain' && (string) $iso->state === 'failed' && (string) $isoReplica->state === 'uncertain' && (string) $isoReplica->error_code === 'panel_persist_failed', 'ISO Agent success was mislabeled after panel persistence failed.'); Capsule::statement('DROP TRIGGER reject_iso_digest'); // A process interruption after the last replica update must be repaired by // status lookup even when there is no pending operation left to poll. Capsule::table('cloud_templates')->where('id', (string) $template->id) ->update(['state' => 'importing']); Capsule::table('cloud_template_replicas') ->where('template_id', (string) $template->id)->update([ 'state' => 'active', 'image_bytes' => 4096, 'sha256' => str_repeat('a', 64), 'error_code' => '', ]); $requestsBeforeTemplateRepair = count($agent->requests); $repairedTemplate = CloudTemplateCatalogService::importJob( (string) $template->id, ['user_id' => 1, 'can_manage' => true] ); $assert($repairedTemplate['status'] === 'succeeded' && (string) CloudTemplate::find((string) $template->id)->state === 'active' && count($agent->requests) === $requestsBeforeTemplateRepair, 'Template aggregate state did not self-heal without pending replicas.'); Capsule::table('cloud_user_isos')->where('id', (string) $iso->id) ->update([ 'state' => 'importing', 'size_bytes' => 8192, 'sha256' => str_repeat('b', 64), ]); Capsule::table('cloud_user_iso_replicas') ->where('user_iso_id', (string) $iso->id)->update([ 'state' => 'active', 'size_bytes' => 8192, 'sha256' => str_repeat('b', 64), 'error_code' => '', ]); $requestsBeforeIsoRepair = count($agent->requests); $repairedIso = CloudUserIsoService::job((string) $iso->import_operation_id, ['user_id' => 42, 'can_manage' => false]); $assert($repairedIso['status'] === 'succeeded' && (string) CloudUserIso::find((string) $iso->id)->state === 'active' && count($agent->requests) === $requestsBeforeIsoRepair, 'ISO aggregate state did not self-heal without pending replicas.'); // `dispatching` is a normal durable Agent state. It must remain pollable // rather than being surfaced as an ambiguous import result. $agent->postResponse = ['status' => 'dispatching']; $dispatchTemplateJob = CloudTemplateCatalogService::importRemote([ 'name' => 'Debian 13 dispatching template', 'url' => 'https://images.example/debian-13.qcow2', 'sha256' => '', ], ['user_id' => 1, 'can_manage' => true]); $dispatchTemplate = CloudTemplate::find( (string) $dispatchTemplateJob['job_id'] ); $dispatchTemplateReplica = CloudTemplateReplica::query() ->where('template_id', (string) $dispatchTemplate->id)->first(); $assert($dispatchTemplateJob['status'] === 'running' && (string) $dispatchTemplate->state === 'importing' && (string) $dispatchTemplateReplica->state === 'pending' && (string) $dispatchTemplateReplica->error_code === '', 'Template Agent dispatching response was made uncertain.'); $dispatchIsoJob = CloudUserIsoService::import([ 'name' => 'Dispatching ISO', 'url' => 'https://images.example/dispatching.iso', ], ['user_id' => 52, 'can_manage' => false]); $dispatchIso = CloudUserIso::query()->where('import_operation_id', (string) $dispatchIsoJob['job_id'])->first(); $dispatchIsoReplica = CloudUserIsoReplica::query() ->where('user_iso_id', (string) $dispatchIso->id)->first(); $assert($dispatchIsoJob['status'] === 'running' && (string) $dispatchIso->state === 'importing' && (string) $dispatchIsoReplica->state === 'pending' && (string) $dispatchIsoReplica->error_code === '', 'ISO Agent dispatching response was made uncertain.'); $agent->postResponse = null; // Catalog imports intentionally get one POST only. Even exact // not_dispatched backpressure is terminal so expiring upload/source // capabilities are never reused by an automatic retry; an administrator // must create a fresh import with a fresh operation and source snapshot. $agent->preDispatchBusy = true; $preDispatchTemplate = CloudTemplateCatalogService::importRemote([ 'name' => 'Debian 12 pre-dispatch terminal template', 'url' => 'https://images.example/debian-12-predispatch.qcow2', 'sha256' => '', ], ['user_id' => 1, 'can_manage' => true]); $preDispatchTemplateRow = CloudTemplate::find( (string) $preDispatchTemplate['job_id'] ); $preDispatchTemplateReplica = CloudTemplateReplica::query() ->where('template_id', (string) $preDispatchTemplateRow->id)->first(); $preDispatchIso = CloudUserIsoService::import([ 'name' => 'Pre-dispatch terminal ISO', 'url' => 'https://images.example/predispatch.iso', ], ['user_id' => 72, 'can_manage' => false]); $preDispatchIsoRow = CloudUserIso::query()->where('import_operation_id', (string) $preDispatchIso['job_id'])->first(); $preDispatchIsoReplica = CloudUserIsoReplica::query() ->where('user_iso_id', (string) $preDispatchIsoRow->id)->first(); $requestsAfterPreDispatch = count($agent->requests); CloudTemplateCatalogService::importJob( (string) $preDispatchTemplateRow->id, ['user_id' => 1, 'can_manage' => true] ); CloudUserIsoService::job((string) $preDispatchIsoRow->import_operation_id, ['user_id' => 72, 'can_manage' => false]); $assert($preDispatchTemplate['status'] === 'failed' && (string) $preDispatchTemplateReplica->state === 'failed' && (string) $preDispatchTemplateReplica->error_code === 'agent_busy' && $preDispatchIso['status'] === 'failed' && (string) $preDispatchIsoReplica->state === 'failed' && (string) $preDispatchIsoReplica->error_code === 'agent_busy' && count($agent->requests) === $requestsAfterPreDispatch, 'Definitive pre-dispatch catalog import was retried automatically.'); $agent->preDispatchBusy = false; // A non-retryable response without explicit not-dispatched proof is still // ambiguous. Retain both imports under their original operation IDs so a // later status lookup can reconcile them; never label them failed here. $agent->ambiguousReplay = true; $replayedTemplateJob = CloudTemplateCatalogService::importRemote([ 'name' => 'Debian 12 ambiguous replay template', 'url' => 'https://images.example/debian-12-ambiguous-replay.qcow2', 'sha256' => '', ], ['user_id' => 1, 'can_manage' => true]); $replayedTemplate = CloudTemplate::find( (string) $replayedTemplateJob['job_id'] ); $replayedTemplateReplica = CloudTemplateReplica::query() ->where('template_id', (string) $replayedTemplate->id)->first(); $replayedIsoJob = CloudUserIsoService::import([ 'name' => 'Ambiguous replay ISO', 'url' => 'https://images.example/ambiguous-replay.iso', ], ['user_id' => 62, 'can_manage' => false]); $replayedIso = CloudUserIso::query()->where('import_operation_id', (string) $replayedIsoJob['job_id'])->first(); $replayedIsoReplica = CloudUserIsoReplica::query() ->where('user_iso_id', (string) $replayedIso->id)->first(); $assert($replayedTemplateJob['status'] === 'running' && (string) $replayedTemplateReplica->state === 'pending' && (string) $replayedTemplateReplica->error_code === 'replayed_request' && $replayedIsoJob['status'] === 'running' && (string) $replayedIsoReplica->state === 'pending' && (string) $replayedIsoReplica->error_code === 'replayed_request', 'Ambiguous non-retryable catalog import was made terminal.'); $agent->ambiguousReplay = false; // An uncertain GET is inspected through the same operation's read-only // reconcile route. Neither catalog import may be replayed. $agent->postResponse = ['status' => 'dispatching']; $reconcileTemplateJob = CloudTemplateCatalogService::importRemote([ 'name' => 'Debian 12 template reconcile test', 'url' => 'https://images.example/debian-12-reconcile.qcow2', 'sha256' => '', ], ['user_id' => 1, 'can_manage' => true]); $reconcileTemplate = CloudTemplate::find( (string) $reconcileTemplateJob['job_id'] ); $reconcileTemplateReplica = CloudTemplateReplica::query() ->where('template_id', (string) $reconcileTemplate->id)->first(); $agent->postResponse = null; $agent->getResponse = [ 'status' => 'uncertain', 'error' => ['code' => 'operation_uncertain'], ]; $agent->reconcileResponse = [ 'status' => 'succeeded', 'result' => [ 'id' => (string) $reconcileTemplate->id, 'revision' => (int) $reconcileTemplate->revision, 'generation_token' => (string) $reconcileTemplate->generation_token, 'size_bytes' => 4096, 'sha256' => str_repeat('a', 64), ], ]; $templateRequestsBeforeReconcile = count($agent->requests); $reconciledTemplate = CloudTemplateCatalogService::importJob( (string) $reconcileTemplate->id, ['user_id' => 1, 'can_manage' => true] ); $templateReconcileRequests = array_slice( $agent->requests, $templateRequestsBeforeReconcile ); $assert($reconciledTemplate['status'] === 'succeeded' && count($templateReconcileRequests) === 2 && $templateReconcileRequests[0]['method'] === 'GET' && $templateReconcileRequests[0]['target'] === '/v1/operations/' . (string) $reconcileTemplateReplica->operation_id && $templateReconcileRequests[1]['method'] === 'POST' && $templateReconcileRequests[1]['target'] === '/v1/operations/' . (string) $reconcileTemplateReplica->operation_id . '/reconcile', 'Template uncertainty did not reconcile the same operation without replay.'); $agent->postResponse = ['status' => 'dispatching']; $reconcileIsoJob = CloudUserIsoService::import([ 'name' => 'ISO reconcile test', 'url' => 'https://images.example/iso-reconcile.iso', ], ['user_id' => 54, 'can_manage' => false]); $reconcileIso = CloudUserIso::query()->where('import_operation_id', (string) $reconcileIsoJob['job_id'])->first(); $reconcileIsoReplica = CloudUserIsoReplica::query() ->where('user_iso_id', (string) $reconcileIso->id)->first(); $agent->postResponse = null; $agent->reconcileResponse = [ 'status' => 'succeeded', 'result' => [ 'id' => (string) $reconcileIso->id, 'owner_id' => (int) $reconcileIso->user_id, 'size_bytes' => 8192, 'sha256' => str_repeat('b', 64), ], ]; $isoRequestsBeforeReconcile = count($agent->requests); $reconciledIso = CloudUserIsoService::job( (string) $reconcileIso->import_operation_id, ['user_id' => 54, 'can_manage' => false] ); $isoReconcileRequests = array_slice( $agent->requests, $isoRequestsBeforeReconcile ); $assert($reconciledIso['status'] === 'succeeded' && count($isoReconcileRequests) === 2 && $isoReconcileRequests[0]['method'] === 'GET' && $isoReconcileRequests[0]['target'] === '/v1/operations/' . (string) $reconcileIsoReplica->operation_id && $isoReconcileRequests[1]['method'] === 'POST' && $isoReconcileRequests[1]['target'] === '/v1/operations/' . (string) $reconcileIsoReplica->operation_id . '/reconcile', 'User ISO uncertainty did not reconcile the same operation without replay.'); $agent->reconcileResponse = null; // Repair only the historical false-positive marker. The old operation ID // is polled exactly once; the import POST is never replayed. $templateOperationId = (string) $dispatchTemplateReplica->operation_id; Capsule::table('cloud_templates')->where('id', (string) $dispatchTemplate->id)->update(['state' => 'error']); Capsule::table('cloud_template_replicas')->where('template_id', (string) $dispatchTemplate->id)->update([ 'state' => 'uncertain', 'error_code' => 'agent_import_uncertain', ]); $agent->getResponse = [ 'status' => 'succeeded', 'result' => [ 'id' => (string) $dispatchTemplate->id, 'revision' => (int) $dispatchTemplate->revision, 'generation_token' => (string) $dispatchTemplate->generation_token, 'size_bytes' => 4096, 'sha256' => str_repeat('a', 64), ], ]; $requestsBeforeTemplateRecovery = count($agent->requests); $recoveredTemplate = CloudTemplateCatalogService::importJob( (string) $dispatchTemplate->id, ['user_id' => 1, 'can_manage' => true] ); $templateRecoveryRequest = $agent->requests[ $requestsBeforeTemplateRecovery ] ?? []; $dispatchTemplateReplica->refresh(); $assert($recoveredTemplate['status'] === 'succeeded' && count($agent->requests) === $requestsBeforeTemplateRecovery + 1 && ($templateRecoveryRequest['method'] ?? null) === 'GET' && ($templateRecoveryRequest['target'] ?? null) === '/v1/operations/' . $templateOperationId && (string) $dispatchTemplateReplica->operation_id === $templateOperationId && (string) $dispatchTemplateReplica->state === 'active', 'Historical template uncertainty did not reconcile its saved operation.'); $isoOperationId = (string) $dispatchIsoReplica->operation_id; Capsule::table('cloud_user_isos')->where('id', (string) $dispatchIso->id)->update(['state' => 'failed']); Capsule::table('cloud_user_iso_replicas')->where('user_iso_id', (string) $dispatchIso->id)->update([ 'state' => 'uncertain', 'error_code' => 'agent_import_uncertain', ]); $agent->getResponse = [ 'status' => 'succeeded', 'result' => [ 'id' => (string) $dispatchIso->id, 'owner_id' => (int) $dispatchIso->user_id, 'size_bytes' => 8192, 'sha256' => str_repeat('b', 64), ], ]; $requestsBeforeIsoRecovery = count($agent->requests); $recoveredIso = CloudUserIsoService::job( (string) $dispatchIso->import_operation_id, ['user_id' => 52, 'can_manage' => false] ); $isoRecoveryRequest = $agent->requests[$requestsBeforeIsoRecovery] ?? []; $dispatchIsoReplica->refresh(); $assert($recoveredIso['status'] === 'succeeded' && count($agent->requests) === $requestsBeforeIsoRecovery + 1 && ($isoRecoveryRequest['method'] ?? null) === 'GET' && ($isoRecoveryRequest['target'] ?? null) === '/v1/operations/' . $isoOperationId && (string) $dispatchIsoReplica->operation_id === $isoOperationId && (string) $dispatchIsoReplica->state === 'active', 'Historical ISO uncertainty did not reconcile its saved operation.'); // A genuine Agent uncertainty gets a new marker and stays terminal. Only // the exact legacy dispatching marker is eligible for recovery. $agent->postResponse = [ 'status' => 'uncertain', 'error' => ['code' => 'agent_import_uncertain'], ]; $lockedTemplateJob = CloudTemplateCatalogService::importRemote([ 'name' => 'Debian 11 genuinely uncertain template', 'url' => 'https://images.example/debian-11.qcow2', 'sha256' => '', ], ['user_id' => 1, 'can_manage' => true]); $lockedTemplate = CloudTemplate::find( (string) $lockedTemplateJob['job_id'] ); $lockedTemplateReplica = CloudTemplateReplica::query() ->where('template_id', (string) $lockedTemplate->id)->first(); $agent->postResponse = null; $requestsBeforeLockedTemplate = count($agent->requests); $lockedTemplateAgain = CloudTemplateCatalogService::importJob( (string) $lockedTemplate->id, ['user_id' => 1, 'can_manage' => true] ); $lockedTemplateReplica->refresh(); $assert($lockedTemplateJob['status'] === 'uncertain' && $lockedTemplateAgain['status'] === 'uncertain' && count($agent->requests) === $requestsBeforeLockedTemplate && (string) $lockedTemplateReplica->state === 'uncertain' && (string) $lockedTemplateReplica->error_code === 'agent_operation_uncertain', 'A genuine template uncertainty was unlocked as legacy dispatching.'); $agent->postResponse = [ 'status' => 'uncertain', 'error' => ['code' => 'agent_import_uncertain'], ]; $lockedIsoJob = CloudUserIsoService::import([ 'name' => 'Genuinely uncertain ISO', 'url' => 'https://images.example/genuinely-uncertain.iso', ], ['user_id' => 53, 'can_manage' => false]); $lockedIso = CloudUserIso::query()->where('import_operation_id', (string) $lockedIsoJob['job_id'])->first(); $lockedIsoReplica = CloudUserIsoReplica::query() ->where('user_iso_id', (string) $lockedIso->id)->first(); $agent->postResponse = null; $requestsBeforeLockedIso = count($agent->requests); $lockedIsoAgain = CloudUserIsoService::job( (string) $lockedIso->import_operation_id, ['user_id' => 53, 'can_manage' => false] ); $lockedIsoReplica->refresh(); $assert($lockedIsoJob['status'] === 'uncertain' && $lockedIsoAgain['status'] === 'uncertain' && count($agent->requests) === $requestsBeforeLockedIso && (string) $lockedIsoReplica->state === 'uncertain' && (string) $lockedIsoReplica->error_code === 'agent_operation_uncertain', 'A genuine ISO uncertainty was unlocked as legacy dispatching.'); // Quota aggregation must use the bytes committed through the locked model, // not the stale outer object that existed before the Agent response. $quotaOwner = 77; $quotaBytes = 10 * 1024 * 1024 * 1024; Capsule::table('cloud_user_isos')->insert([ 'id' => str_repeat('9', 32), 'user_id' => $quotaOwner, 'name' => 'Existing quota usage', 'sealed_source_url' => 'test-sealed:user-iso-source', 'size_bytes' => $quotaBytes - 4096, 'sha256' => str_repeat('9', 64), 'state' => 'active', 'import_operation_id' => str_repeat('8', 32), 'created_at' => time(), 'updated_at' => time(), ]); $quotaJob = CloudUserIsoService::import([ 'name' => 'Quota overflow ISO', 'url' => 'https://images.example/quota-overflow.iso', ], ['user_id' => $quotaOwner, 'can_manage' => false]); $quotaIso = CloudUserIso::query()->where('import_operation_id', (string) $quotaJob['job_id'])->first(); $assert($quotaJob['status'] === 'failed' && (int) $quotaIso->size_bytes === 8192 && (string) $quotaIso->state === 'failed', 'ISO quota aggregation ignored the persisted Agent result size.'); // A successful stale template response must not overwrite a concurrently // installed delete operation or move the parent out of deleting. $staleTemplateId = 'debian-12-stale-import'; $oldTemplateOperation = str_repeat('c', 32); $newTemplateOperation = str_repeat('d', 32); $generation = str_repeat('e', 32); $now = time(); Capsule::table('cloud_templates')->insert([ 'id' => $staleTemplateId, 'name' => 'Debian 12 stale import', 'os_family' => 'Debian', 'os_version' => '12', 'source_type' => 'remote', 'sealed_source_url' => 'test-sealed:template-source', 'image_bytes' => 0, 'sha256' => '', 'startup_script' => '', 'revision' => 1, 'generation_token' => $generation, 'state' => 'importing', 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_template_replicas')->insert([ 'host_id' => 101, 'template_id' => $staleTemplateId, 'revision' => 1, 'generation_token' => $generation, 'state' => 'pending', 'image_bytes' => 0, 'sha256' => '', 'operation_id' => $oldTemplateOperation, 'error_code' => '', 'created_at' => $now, 'updated_at' => $now, ]); $agent->beforeResponse = static function (string $method) use ( $staleTemplateId, $newTemplateOperation ): void { if ($method !== 'GET') { return; } Capsule::table('cloud_templates')->where('id', $staleTemplateId) ->update(['state' => 'deleting']); Capsule::table('cloud_template_replicas') ->where('template_id', $staleTemplateId)->update([ 'state' => 'delete_succeeded', 'operation_id' => $newTemplateOperation, 'error_code' => '', ]); }; $agent->getResponse = [ 'status' => 'succeeded', 'result' => [ 'id' => $staleTemplateId, 'revision' => 1, 'generation_token' => $generation, 'size_bytes' => 4096, 'sha256' => str_repeat('a', 64), ], ]; CloudTemplateCatalogService::importJob($staleTemplateId, ['user_id' => 1, 'can_manage' => true]); $staleTemplate = CloudTemplate::find($staleTemplateId); $staleTemplateReplica = CloudTemplateReplica::query() ->where('template_id', $staleTemplateId)->first(); $assert((string) $staleTemplate->state === 'deleting' && (string) $staleTemplateReplica->state === 'delete_succeeded' && (string) $staleTemplateReplica->operation_id === $newTemplateOperation, 'Stale template polling rolled back a concurrent terminal delete.'); // The state predicate is independently required: even the same operation // ID cannot move a terminal ISO replica back to pending. $staleIsoId = str_repeat('f', 32); $staleIsoJob = str_repeat('1', 32); $staleIsoOperation = str_repeat('2', 32); Capsule::table('cloud_user_isos')->insert([ 'id' => $staleIsoId, 'user_id' => 42, 'name' => 'Stale ISO', 'sealed_source_url' => 'test-sealed:user-iso-source', 'size_bytes' => 0, 'sha256' => '', 'state' => 'importing', 'import_operation_id' => $staleIsoJob, 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_user_iso_replicas')->insert([ 'host_id' => 101, 'user_iso_id' => $staleIsoId, 'user_id' => 42, 'state' => 'pending', 'size_bytes' => 0, 'sha256' => '', 'operation_id' => $staleIsoOperation, 'error_code' => '', 'created_at' => $now, 'updated_at' => $now, ]); $agent->beforeResponse = static function (string $method) use ( $staleIsoId ): void { if ($method !== 'GET') { return; } Capsule::table('cloud_user_isos')->where('id', $staleIsoId) ->update(['state' => 'active']); Capsule::table('cloud_user_iso_replicas') ->where('user_iso_id', $staleIsoId)->update([ 'state' => 'active', 'error_code' => '', ]); }; $agent->getResponse = null; CloudUserIsoService::job($staleIsoJob, ['user_id' => 42, 'can_manage' => false]); $staleIso = CloudUserIso::find($staleIsoId); $staleIsoReplica = CloudUserIsoReplica::query() ->where('user_iso_id', $staleIsoId)->first(); $assert((string) $staleIso->state === 'active' && (string) $staleIsoReplica->state === 'active' && (string) $staleIsoReplica->operation_id === $staleIsoOperation, 'Stale ISO polling moved a terminal replica back to pending.'); echo "Catalog import persistence failures and stale-response CAS passed.\n"; }