requests[] = compact('method', 'target', 'actor', 'payload'); if (is_callable($this->beforeResponse)) { ($this->beforeResponse)($method, $target, $actor, $payload); } $operationId = (string) ($payload['operation_id'] ?? $actor['operation_id'] ?? basename($target)); if ($method === 'GET') { return ['operation_id' => $operationId] + ($this->getResponse ?? ['status' => 'running']); } if ($target === '/v1/templates') { return [ 'operation_id' => $operationId, 'status' => 'succeeded', 'result' => [ 'id' => (string) $payload['id'], 'revision' => (int) $payload['revision'], 'generation_token' => (string) $payload['generation_token'], 'size_bytes' => 4096, 'sha256' => str_repeat('a', 64), ], ]; } if ($target === '/v1/user-isos') { return [ 'operation_id' => $operationId, 'status' => 'succeeded', 'result' => [ 'id' => (string) $payload['id'], 'owner_id' => (int) $payload['owner_id'], 'size_bytes' => 8192, 'sha256' => str_repeat('b', 64), ], ]; } throw new \RuntimeException('Unexpected fake Agent request'); } } final class CloudAgentSettingsService { public static array $hostIds = [101]; public static ?CatalogImportFakeAgent $client = null; public static function enabledHostIds(): array { return self::$hostIds; } public static function clientForHost(int $hostId, float $timeoutSeconds = 20.0): CatalogImportFakeAgent { if (!in_array($hostId, self::$hostIds, true) || self::$client === null) { throw new \RuntimeException('Missing fake Agent'); } return self::$client; } } } namespace { require dirname(__DIR__) . '/vendor/autoload.php'; use App\Models\CloudTemplate; use App\Models\CloudTemplateReplica; use App\Models\CloudUserIso; use App\Models\CloudUserIsoReplica; use App\Services\Cloud\CatalogImportFakeAgent; use App\Services\Cloud\CloudAgentSettingsService; use App\Services\Cloud\CloudTemplateCatalogService; use App\Services\Cloud\CloudUserIsoService; use Illuminate\Database\Capsule\Manager as Capsule; use Illuminate\Database\Schema\Blueprint; $assert = static function (bool $condition, string $message): void { if (!$condition) { throw new RuntimeException($message); } }; $capsule = new Capsule(); $capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('cloud_templates', static function (Blueprint $table): void { $table->string('id')->primary(); $table->string('name'); $table->string('os_family'); $table->string('os_version'); $table->string('source_type'); $table->text('sealed_source_url'); $table->integer('image_bytes')->default(0); $table->string('sha256')->default(''); $table->text('startup_script')->default(''); $table->integer('revision'); $table->string('generation_token')->unique(); $table->string('state'); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_template_replicas', static function (Blueprint $table): void { $table->integer('host_id'); $table->string('template_id'); $table->integer('revision'); $table->string('generation_token'); $table->string('state'); $table->integer('image_bytes')->default(0); $table->string('sha256')->default(''); $table->string('operation_id')->default(''); $table->string('error_code')->default(''); $table->integer('created_at'); $table->integer('updated_at'); $table->primary(['host_id', 'template_id']); }); $schema->create('cloud_orders', static function (Blueprint $table): void { $table->increments('id'); $table->string('status'); $table->text('spec_json'); }); $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('kind'); $table->string('status'); $table->text('sealed_payload'); }); $schema->create('cloud_user_isos', static function (Blueprint $table): void { $table->string('id')->primary(); $table->integer('user_id'); $table->string('name'); $table->text('sealed_source_url'); $table->integer('size_bytes')->default(0); $table->string('sha256')->default(''); $table->string('state'); $table->string('import_operation_id'); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_user_iso_replicas', static function (Blueprint $table): void { $table->integer('host_id'); $table->string('user_iso_id'); $table->integer('user_id'); $table->string('state'); $table->integer('size_bytes')->default(0); $table->string('sha256')->default(''); $table->string('operation_id')->default(''); $table->string('error_code')->default(''); $table->integer('created_at'); $table->integer('updated_at'); $table->primary(['host_id', 'user_iso_id']); }); $agent = new CatalogImportFakeAgent(); CloudAgentSettingsService::$client = $agent; // An Agent success followed by a local metadata write failure is uncertain, // never a definite dispatch failure. The transaction leaves the replica // pending long enough for the guarded uncertainty transition to latch. Capsule::statement("CREATE TRIGGER reject_template_digest " . "BEFORE UPDATE OF sha256 ON cloud_templates " . "WHEN NEW.sha256 <> OLD.sha256 BEGIN " . "SELECT RAISE(ABORT, 'template persistence failure'); END"); $templateJob = CloudTemplateCatalogService::importRemote([ 'name' => 'Debian 12 persistence test', 'url' => 'https://images.example/debian-12.qcow2', 'sha256' => '', ], ['user_id' => 1, 'can_manage' => true]); $template = CloudTemplate::query()->where('id', (string) $templateJob['job_id'])->first(); $templateReplica = CloudTemplateReplica::query() ->where('template_id', (string) $template->id)->first(); $assert($templateJob['status'] === 'uncertain' && (string) $template->state === 'error' && (string) $templateReplica->state === 'uncertain' && (string) $templateReplica->error_code === 'panel_persist_failed', 'Template Agent success was mislabeled after panel persistence failed.'); Capsule::statement('DROP TRIGGER reject_template_digest'); Capsule::statement("CREATE TRIGGER reject_iso_digest " . "BEFORE UPDATE OF sha256 ON cloud_user_isos " . "WHEN NEW.sha256 <> OLD.sha256 BEGIN " . "SELECT RAISE(ABORT, 'ISO persistence failure'); END"); $isoJob = CloudUserIsoService::import([ 'name' => 'Recovery ISO', 'url' => 'https://images.example/recovery.iso', ], ['user_id' => 42, 'can_manage' => false]); $iso = CloudUserIso::query()->where('import_operation_id', (string) $isoJob['job_id'])->first(); $isoReplica = CloudUserIsoReplica::query() ->where('user_iso_id', (string) $iso->id)->first(); $assert($isoJob['status'] === 'uncertain' && (string) $iso->state === 'failed' && (string) $isoReplica->state === 'uncertain' && (string) $isoReplica->error_code === 'panel_persist_failed', 'ISO Agent success was mislabeled after panel persistence failed.'); Capsule::statement('DROP TRIGGER reject_iso_digest'); // A process interruption after the last replica update must be repaired by // status lookup even when there is no pending operation left to poll. Capsule::table('cloud_templates')->where('id', (string) $template->id) ->update(['state' => 'importing']); Capsule::table('cloud_template_replicas') ->where('template_id', (string) $template->id)->update([ 'state' => 'active', 'image_bytes' => 4096, 'sha256' => str_repeat('a', 64), 'error_code' => '', ]); $requestsBeforeTemplateRepair = count($agent->requests); $repairedTemplate = CloudTemplateCatalogService::importJob( (string) $template->id, ['user_id' => 1, 'can_manage' => true] ); $assert($repairedTemplate['status'] === 'succeeded' && (string) CloudTemplate::find((string) $template->id)->state === 'active' && count($agent->requests) === $requestsBeforeTemplateRepair, 'Template aggregate state did not self-heal without pending replicas.'); Capsule::table('cloud_user_isos')->where('id', (string) $iso->id) ->update([ 'state' => 'importing', 'size_bytes' => 8192, 'sha256' => str_repeat('b', 64), ]); Capsule::table('cloud_user_iso_replicas') ->where('user_iso_id', (string) $iso->id)->update([ 'state' => 'active', 'size_bytes' => 8192, 'sha256' => str_repeat('b', 64), 'error_code' => '', ]); $requestsBeforeIsoRepair = count($agent->requests); $repairedIso = CloudUserIsoService::job((string) $iso->import_operation_id, ['user_id' => 42, 'can_manage' => false]); $assert($repairedIso['status'] === 'succeeded' && (string) CloudUserIso::find((string) $iso->id)->state === 'active' && count($agent->requests) === $requestsBeforeIsoRepair, 'ISO aggregate state did not self-heal without pending replicas.'); // Quota aggregation must use the bytes committed through the locked model, // not the stale outer object that existed before the Agent response. $quotaOwner = 77; $quotaBytes = 10 * 1024 * 1024 * 1024; Capsule::table('cloud_user_isos')->insert([ 'id' => str_repeat('9', 32), 'user_id' => $quotaOwner, 'name' => 'Existing quota usage', 'sealed_source_url' => 'test-sealed:user-iso-source', 'size_bytes' => $quotaBytes - 4096, 'sha256' => str_repeat('9', 64), 'state' => 'active', 'import_operation_id' => str_repeat('8', 32), 'created_at' => time(), 'updated_at' => time(), ]); $quotaJob = CloudUserIsoService::import([ 'name' => 'Quota overflow ISO', 'url' => 'https://images.example/quota-overflow.iso', ], ['user_id' => $quotaOwner, 'can_manage' => false]); $quotaIso = CloudUserIso::query()->where('import_operation_id', (string) $quotaJob['job_id'])->first(); $assert($quotaJob['status'] === 'failed' && (int) $quotaIso->size_bytes === 8192 && (string) $quotaIso->state === 'failed', 'ISO quota aggregation ignored the persisted Agent result size.'); // A successful stale template response must not overwrite a concurrently // installed delete operation or move the parent out of deleting. $staleTemplateId = 'debian-12-stale-import'; $oldTemplateOperation = str_repeat('c', 32); $newTemplateOperation = str_repeat('d', 32); $generation = str_repeat('e', 32); $now = time(); Capsule::table('cloud_templates')->insert([ 'id' => $staleTemplateId, 'name' => 'Debian 12 stale import', 'os_family' => 'Debian', 'os_version' => '12', 'source_type' => 'remote', 'sealed_source_url' => 'test-sealed:template-source', 'image_bytes' => 0, 'sha256' => '', 'startup_script' => '', 'revision' => 1, 'generation_token' => $generation, 'state' => 'importing', 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_template_replicas')->insert([ 'host_id' => 101, 'template_id' => $staleTemplateId, 'revision' => 1, 'generation_token' => $generation, 'state' => 'pending', 'image_bytes' => 0, 'sha256' => '', 'operation_id' => $oldTemplateOperation, 'error_code' => '', 'created_at' => $now, 'updated_at' => $now, ]); $agent->beforeResponse = static function (string $method) use ( $staleTemplateId, $newTemplateOperation ): void { if ($method !== 'GET') { return; } Capsule::table('cloud_templates')->where('id', $staleTemplateId) ->update(['state' => 'deleting']); Capsule::table('cloud_template_replicas') ->where('template_id', $staleTemplateId)->update([ 'state' => 'delete_succeeded', 'operation_id' => $newTemplateOperation, 'error_code' => '', ]); }; $agent->getResponse = [ 'status' => 'succeeded', 'result' => [ 'id' => $staleTemplateId, 'revision' => 1, 'generation_token' => $generation, 'size_bytes' => 4096, 'sha256' => str_repeat('a', 64), ], ]; CloudTemplateCatalogService::importJob($staleTemplateId, ['user_id' => 1, 'can_manage' => true]); $staleTemplate = CloudTemplate::find($staleTemplateId); $staleTemplateReplica = CloudTemplateReplica::query() ->where('template_id', $staleTemplateId)->first(); $assert((string) $staleTemplate->state === 'deleting' && (string) $staleTemplateReplica->state === 'delete_succeeded' && (string) $staleTemplateReplica->operation_id === $newTemplateOperation, 'Stale template polling rolled back a concurrent terminal delete.'); // The state predicate is independently required: even the same operation // ID cannot move a terminal ISO replica back to pending. $staleIsoId = str_repeat('f', 32); $staleIsoJob = str_repeat('1', 32); $staleIsoOperation = str_repeat('2', 32); Capsule::table('cloud_user_isos')->insert([ 'id' => $staleIsoId, 'user_id' => 42, 'name' => 'Stale ISO', 'sealed_source_url' => 'test-sealed:user-iso-source', 'size_bytes' => 0, 'sha256' => '', 'state' => 'importing', 'import_operation_id' => $staleIsoJob, 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_user_iso_replicas')->insert([ 'host_id' => 101, 'user_iso_id' => $staleIsoId, 'user_id' => 42, 'state' => 'pending', 'size_bytes' => 0, 'sha256' => '', 'operation_id' => $staleIsoOperation, 'error_code' => '', 'created_at' => $now, 'updated_at' => $now, ]); $agent->beforeResponse = static function (string $method) use ( $staleIsoId ): void { if ($method !== 'GET') { return; } Capsule::table('cloud_user_isos')->where('id', $staleIsoId) ->update(['state' => 'active']); Capsule::table('cloud_user_iso_replicas') ->where('user_iso_id', $staleIsoId)->update([ 'state' => 'active', 'error_code' => '', ]); }; $agent->getResponse = null; CloudUserIsoService::job($staleIsoJob, ['user_id' => 42, 'can_manage' => false]); $staleIso = CloudUserIso::find($staleIsoId); $staleIsoReplica = CloudUserIsoReplica::query() ->where('user_iso_id', $staleIsoId)->first(); $assert((string) $staleIso->state === 'active' && (string) $staleIsoReplica->state === 'active' && (string) $staleIsoReplica->operation_id === $staleIsoOperation, 'Stale ISO polling moved a terminal replica back to pending.'); echo "Catalog import persistence failures and stale-response CAS passed.\n"; }