requests[] = compact('method', 'target', 'actor', 'payload'); if ($method === 'DELETE' && in_array($this->mode, ['pre_dispatch_busy', 'pre_dispatch_draining'], true)) { $code = $this->mode === 'pre_dispatch_busy' ? 'agent_busy' : 'gateway_draining'; throw new CloudAgentError( 'request was not dispatched', 503, $code, true, 503, false, true ); } if ($method === 'DELETE') { $operationId = (string) ($payload['operation_id'] ?? ''); if (preg_match('#^/v1/templates/([^?]+)$#D', $target, $match)) { $resource = ['resource_type' => 'template', 'id' => rawurldecode($match[1]), 'owner_id' => 0]; } elseif (preg_match('#^/v1/user-isos/([^?]+)\?owner_id=([0-9]+)$#D', $target, $match)) { $resource = ['resource_type' => 'user_iso', 'id' => rawurldecode($match[1]), 'owner_id' => (int) $match[2]]; } elseif (preg_match('#^/v1/rescue-templates/([^?]+)$#D', $target, $match)) { $resource = ['resource_type' => 'rescue_template', 'id' => rawurldecode($match[1]), 'owner_id' => 0]; } else { throw new \RuntimeException('Unexpected fake Agent DELETE: ' . $target); } $this->operations[$operationId] = $resource; } elseif ($method === 'POST' && preg_match( '#^/v1/operations/([A-Za-z0-9_-]+)/reconcile$#D', $target, $match)) { $operationId = rawurldecode($match[1]); $resource = $this->operations[$operationId] ?? null; if (!is_array($resource)) { throw new CloudAgentError('operation missing', 404, 'operation_not_found', false, 404); } return $this->response($operationId, $resource, $this->reconcileMode ?? $this->mode); } elseif ($method === 'GET' && preg_match('#^/v1/operations/([A-Za-z0-9_-]+)$#D', $target, $match)) { $operationId = rawurldecode($match[1]); $resource = $this->operations[$operationId] ?? null; if (!is_array($resource)) { throw new CloudAgentError('operation missing', 404, 'operation_not_found', false, 404); } } else { throw new \RuntimeException('Unexpected fake Agent request: ' . $method . ' ' . $target); } return $this->response($operationId, $resource, $this->mode); } private function response(string $operationId, array $resource, string $mode): array { $notFoundCode = $resource['resource_type'] . '_not_found'; if ($mode === 'direct_not_found') { throw new CloudAgentError('Direct Agent 404 without operation', 502, $notFoundCode, false, 404); } if ($mode === 'durable_not_found') { return ['operation_id' => $operationId, 'status' => 'failed', 'retryable' => false, 'error' => ['code' => $notFoundCode]]; } if ($mode === 'running') { return ['operation_id' => $operationId, 'status' => 'running']; } if ($mode === 'failed') { return ['operation_id' => $operationId, 'status' => 'failed', 'error' => ['code' => 'resource_busy']]; } if ($mode === 'uncertain') { return ['operation_id' => $operationId, 'status' => 'uncertain', 'error' => ['code' => 'delete_uncertain']]; } if ($mode === 'missing_evidence') { return ['operation_id' => $operationId, 'status' => 'succeeded', 'result' => ['deleted' => true]]; } if (!in_array($mode, ['success', 'wrong_owner'], true)) { throw new \RuntimeException('Unknown fake Agent mode: ' . $mode); } $result = [ 'resource_type' => $resource['resource_type'], 'id' => $resource['id'], 'deleted' => true, 'managed_image_absent' => true, 'managed_metadata_absent' => true, 'managed_artifacts_absent' => true, ]; if ($resource['resource_type'] === 'user_iso') { $result['owner_id'] = $resource['owner_id'] + ($mode === 'wrong_owner' ? 1 : 0); } return ['operation_id' => $operationId, 'status' => 'succeeded', 'retryable' => false, 'result' => $result]; } } /** Replaces only Agent lookup in this isolated process. */ final class CloudAgentSettingsService { /** @var array */ public static array $clients = []; public static function clientForHost(int $hostId, float $timeoutSeconds = 20.0): CatalogDeletionFakeAgent { if (!isset(self::$clients[$hostId])) { throw new \RuntimeException('Missing fake Agent for host ' . $hostId); } return self::$clients[$hostId]; } } } namespace { require dirname(__DIR__) . '/vendor/autoload.php'; use App\Models\CloudRescueTemplateDeletion; use App\Models\CloudTemplate; use App\Models\CloudTemplateReplica; use App\Models\CloudUserIso; use App\Models\CloudUserIsoReplica; use App\Services\Cloud\CatalogDeletionFakeAgent; use App\Services\Cloud\CloudAgentCatalogDeletion; use App\Services\Cloud\CloudAgentSettingsService; use App\Services\Cloud\CloudError; use App\Services\Cloud\CloudRescueTemplateDeletionService; use App\Services\Cloud\CloudTemplateCatalogService; use App\Services\Cloud\CloudUserIsoService; use Illuminate\Database\Capsule\Manager as Capsule; use Illuminate\Database\Schema\Blueprint; $assert = static function (bool $condition, string $message): void { if (!$condition) { throw new \RuntimeException($message); } }; $expectCloudError = static function (callable $callable, int $status, string $message) use ($assert): void { try { $callable(); } catch (CloudError $error) { $assert($error->httpStatus === $status, $message . ' Unexpected status: ' . $error->httpStatus); return; } throw new \RuntimeException($message); }; $requestCount = static fn (CatalogDeletionFakeAgent $agent, string $method): int => count(array_filter($agent->requests, static fn (array $request): bool => $request['method'] === $method)); $proof = static function (string $type, string $id, ?int $ownerId = null): array { $result = [ 'resource_type' => $type, 'id' => $id, 'deleted' => true, 'managed_image_absent' => true, 'managed_metadata_absent' => true, 'managed_artifacts_absent' => true, ]; if ($ownerId !== null) { $result['owner_id'] = $ownerId; } return ['status' => 'succeeded', 'retryable' => false, 'result' => $result]; }; $capsule = new Capsule(); $capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('cloud_templates', static function (Blueprint $table): void { $table->string('id')->primary(); $table->string('name'); $table->string('os_family'); $table->string('os_version'); $table->string('source_type'); $table->text('sealed_source_url'); $table->integer('image_bytes')->default(0); $table->string('sha256')->default(''); $table->text('startup_script')->default(''); $table->integer('revision'); $table->string('generation_token')->unique(); $table->string('state'); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_template_replicas', static function (Blueprint $table): void { $table->integer('host_id'); $table->string('template_id'); $table->integer('revision'); $table->string('generation_token'); $table->string('state'); $table->integer('image_bytes')->default(0); $table->string('sha256')->default(''); $table->string('operation_id')->default(''); $table->string('error_code')->default(''); $table->integer('created_at'); $table->integer('updated_at'); $table->primary(['host_id', 'template_id']); }); $schema->create('cloud_orders', static function (Blueprint $table): void { $table->increments('id'); $table->string('status'); $table->text('spec_json'); }); $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('kind'); $table->string('status'); $table->text('sealed_payload'); }); $schema->create('cloud_user_isos', static function (Blueprint $table): void { $table->string('id')->primary(); $table->integer('user_id'); $table->string('name'); $table->text('sealed_source_url'); $table->integer('size_bytes')->default(0); $table->string('sha256')->default(''); $table->string('state'); $table->string('import_operation_id')->default(''); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_user_iso_replicas', static function (Blueprint $table): void { $table->integer('host_id'); $table->string('user_iso_id'); $table->integer('user_id'); $table->string('state'); $table->integer('size_bytes')->default(0); $table->string('sha256')->default(''); $table->string('operation_id')->default(''); $table->string('error_code')->default(''); $table->integer('created_at'); $table->integer('updated_at'); $table->primary(['host_id', 'user_iso_id']); }); $schema->create('cloud_rescue_template_deletions', static function (Blueprint $table): void { $table->integer('host_id'); $table->string('template_id'); $table->string('operation_id'); $table->string('state'); $table->string('error_code')->default(''); $table->integer('created_at'); $table->integer('updated_at'); $table->primary(['host_id', 'template_id']); $table->unique(['host_id', 'operation_id']); }); $seedTemplate = static function (string $id, int $hostId, string $generation): void { $now = time(); Capsule::table('cloud_templates')->insert([ 'id' => $id, 'name' => $id, 'os_family' => 'Debian', 'os_version' => '12', 'source_type' => 'remote', 'sealed_source_url' => 'test-only-sealed-source', 'image_bytes' => 1024, 'sha256' => str_repeat('a', 64), 'startup_script' => '', 'revision' => 1, 'generation_token' => $generation, 'state' => 'active', 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_template_replicas')->insert([ 'host_id' => $hostId, 'template_id' => $id, 'revision' => 1, 'generation_token' => $generation, 'state' => 'active', 'image_bytes' => 1024, 'sha256' => str_repeat('a', 64), 'operation_id' => bin2hex(random_bytes(16)), 'error_code' => '', 'created_at' => $now, 'updated_at' => $now, ]); }; $seedIso = static function (string $id, int $hostId, int $ownerId): void { $now = time(); Capsule::table('cloud_user_isos')->insert([ 'id' => $id, 'user_id' => $ownerId, 'name' => $id, 'sealed_source_url' => 'test-only-sealed-source', 'size_bytes' => 2048, 'sha256' => str_repeat('b', 64), 'state' => 'active', 'import_operation_id' => bin2hex(random_bytes(16)), 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_user_iso_replicas')->insert([ 'host_id' => $hostId, 'user_iso_id' => $id, 'user_id' => $ownerId, 'state' => 'active', 'size_bytes' => 2048, 'sha256' => str_repeat('b', 64), 'operation_id' => bin2hex(random_bytes(16)), 'error_code' => '', 'created_at' => $now, 'updated_at' => $now, ]); }; // Exact Agent cleanup evidence is fail-closed, including ISO ownership. $templateId = 'debian-12-delete-test'; $assert(CloudAgentCatalogDeletion::confirms( $proof('template', $templateId), 'template', $templateId ), 'Exact template cleanup evidence was rejected.'); $missing = $proof('template', $templateId); unset($missing['result']['managed_artifacts_absent']); $assert(!CloudAgentCatalogDeletion::confirms( $missing, 'template', $templateId ), 'Incomplete cleanup evidence was accepted.'); $isoProofId = str_repeat('1', 32); $assert(CloudAgentCatalogDeletion::confirms( $proof('user_iso', $isoProofId, 42), 'user_iso', $isoProofId, 42 ) && !CloudAgentCatalogDeletion::confirms( $proof('user_iso', $isoProofId, 43), 'user_iso', $isoProofId, 42 ), 'User ISO cleanup evidence was not bound to its exact owner.'); // Template: one destructive dispatch, persisted operation polling, and // terminal uncertain lock when a succeeded response lacks proof. $templateAgent = new CatalogDeletionFakeAgent(); CloudAgentSettingsService::$clients[10] = $templateAgent; $generation = str_repeat('a', 32); $seedTemplate($templateId, 10, $generation); $templateJob = CloudTemplateCatalogService::delete( $templateId, 1, $generation, ['user_id' => 1, 'can_manage' => true] ); $replica = CloudTemplateReplica::query()->where('host_id', 10) ->where('template_id', $templateId)->first(); $operationId = (string) $replica->operation_id; $assert($templateJob['deleted'] === false && $templateJob['status'] === 'running' && $templateJob['status_url'] === '/api/v1/admin/cloud-templates/delete-jobs/' . $templateId && preg_match('/^[a-f0-9]{32}$/D', $operationId) === 1 && $requestCount($templateAgent, 'DELETE') === 1, 'Template delete did not persist one durable operation before dispatch.'); CloudTemplateCatalogService::delete( $templateId, 1, $generation, ['user_id' => 1, 'can_manage' => true] ); $assert($requestCount($templateAgent, 'DELETE') === 1 && str_ends_with((string) end($templateAgent->requests)['target'], '/v1/operations/' . $operationId), 'Repeated template DELETE replayed destruction or changed operation ID.'); $templateAgent->mode = 'missing_evidence'; $uncertainTemplate = CloudTemplateCatalogService::deleteJob( $templateId, ['user_id' => 1, 'can_manage' => true] ); $requestsBeforeLockCheck = count($templateAgent->requests); $lockedTemplate = CloudTemplateCatalogService::deleteJob( $templateId, ['user_id' => 1, 'can_manage' => true] ); $replica->refresh(); $assert($uncertainTemplate['status'] === 'uncertain' && $lockedTemplate['status'] === 'uncertain' && (string) $replica->state === 'delete_uncertain' && (string) $replica->error_code === 'delete_evidence_missing' && count($templateAgent->requests) === $requestsBeforeLockCheck, 'Template missing-proof outcome was not latched without replay.'); // Exact not_dispatched backpressure is the only case where DELETE may be // sent again. It must retain and reuse the already-persisted operation ID. $busyTemplateId = 'debian-12-delete-busy'; $busyTemplateAgent = new CatalogDeletionFakeAgent(); $busyTemplateAgent->mode = 'pre_dispatch_busy'; CloudAgentSettingsService::$clients[17] = $busyTemplateAgent; $busyTemplateGeneration = str_repeat('1', 32); $seedTemplate($busyTemplateId, 17, $busyTemplateGeneration); $busyTemplate = CloudTemplateCatalogService::delete( $busyTemplateId, 1, $busyTemplateGeneration, ['user_id' => 1, 'can_manage' => true] ); $busyTemplateReplica = CloudTemplateReplica::query() ->where('template_id', $busyTemplateId)->first(); $busyTemplateOperation = (string) $busyTemplateReplica->operation_id; $assert($busyTemplate['status'] === 'running' && (string) $busyTemplateReplica->state === 'delete_dispatch' && $busyTemplateAgent->operations === [] && $requestCount($busyTemplateAgent, 'DELETE') === 1, 'Pre-dispatch template backpressure became a terminal deletion.'); $busyTemplateReplica->updated_at = time() - 6; $busyTemplateReplica->save(); $busyTemplateAgent->mode = 'success'; $retriedBusyTemplate = CloudTemplateCatalogService::deleteJob( $busyTemplateId, ['user_id' => 1, 'can_manage' => true] ); $busyTemplateDeletes = array_values(array_filter( $busyTemplateAgent->requests, static fn (array $request): bool => $request['method'] === 'DELETE' )); $assert($retriedBusyTemplate['status'] === 'succeeded' && count($busyTemplateDeletes) === 2 && ($busyTemplateDeletes[0]['payload']['operation_id'] ?? null) === $busyTemplateOperation && ($busyTemplateDeletes[1]['payload']['operation_id'] ?? null) === $busyTemplateOperation, 'Template backpressure retry changed operation ID or did not finish.'); $sendingTemplateId = 'debian-12-delete-sending'; $sendingTemplateAgent = new CatalogDeletionFakeAgent(); CloudAgentSettingsService::$clients[12] = $sendingTemplateAgent; $sendingGeneration = str_repeat('c', 32); $seedTemplate($sendingTemplateId, 12, $sendingGeneration); Capsule::table('cloud_templates')->where('id', $sendingTemplateId) ->update(['state' => 'deleting']); Capsule::table('cloud_template_replicas') ->where('template_id', $sendingTemplateId)->update([ 'state' => 'delete_sending', 'operation_id' => bin2hex(random_bytes(16)), 'updated_at' => time(), ]); $freshSendingTemplate = CloudTemplateCatalogService::deleteJob( $sendingTemplateId, ['user_id' => 1, 'can_manage' => true] ); $assert($freshSendingTemplate['status'] === 'running' && $sendingTemplateAgent->requests === [], 'Fresh template delete_sending claim was polled before dispatch settled.'); Capsule::table('cloud_template_replicas') ->where('template_id', $sendingTemplateId) ->update(['updated_at' => time() - 91]); $staleSendingTemplate = CloudTemplateCatalogService::deleteJob( $sendingTemplateId, ['user_id' => 1, 'can_manage' => true] ); $sendingTemplateReplica = CloudTemplateReplica::query() ->where('template_id', $sendingTemplateId)->first(); $assert($staleSendingTemplate['status'] === 'uncertain' && (string) $sendingTemplateReplica->error_code === 'operation_not_found' && count($sendingTemplateAgent->requests) === 1 && $sendingTemplateAgent->requests[0]['method'] === 'GET' && $sendingTemplateAgent->requests[0]['target'] === '/v1/operations/' . (string) $sendingTemplateReplica->operation_id && $requestCount($sendingTemplateAgent, 'DELETE') === 0, 'Stale template sending claim was not queried safely by operation ID.'); $reconcileTemplateId = 'debian-12-delete-reconcile'; $reconcileTemplateAgent = new CatalogDeletionFakeAgent(); CloudAgentSettingsService::$clients[15] = $reconcileTemplateAgent; $reconcileTemplateGeneration = str_repeat('f', 32); $seedTemplate($reconcileTemplateId, 15, $reconcileTemplateGeneration); CloudTemplateCatalogService::delete( $reconcileTemplateId, 1, $reconcileTemplateGeneration, ['user_id' => 1, 'can_manage' => true] ); $reconcileTemplateReplica = CloudTemplateReplica::query() ->where('template_id', $reconcileTemplateId)->first(); $reconcileTemplateOperation = (string) $reconcileTemplateReplica->operation_id; $reconcileTemplateAgent->mode = 'uncertain'; $reconcileTemplateAgent->reconcileMode = 'success'; $beforeTemplateReconcile = count($reconcileTemplateAgent->requests); $reconciledTemplateDelete = CloudTemplateCatalogService::deleteJob( $reconcileTemplateId, ['user_id' => 1, 'can_manage' => true] ); $templateReconcileRequests = array_slice( $reconcileTemplateAgent->requests, $beforeTemplateReconcile ); $assert($reconciledTemplateDelete['status'] === 'succeeded' && $requestCount($reconcileTemplateAgent, 'DELETE') === 1 && count($templateReconcileRequests) === 2 && $templateReconcileRequests[0]['method'] === 'GET' && $templateReconcileRequests[0]['target'] === '/v1/operations/' . $reconcileTemplateOperation && $templateReconcileRequests[1]['method'] === 'POST' && $templateReconcileRequests[1]['target'] === '/v1/operations/' . $reconcileTemplateOperation . '/reconcile', 'Template delete uncertainty replayed DELETE or changed operation ID.'); $trueUncertainTemplateId = 'debian-12-delete-uncertain'; $trueUncertainTemplateAgent = new CatalogDeletionFakeAgent(); CloudAgentSettingsService::$clients[16] = $trueUncertainTemplateAgent; $trueUncertainGeneration = str_repeat('0', 32); $seedTemplate( $trueUncertainTemplateId, 16, $trueUncertainGeneration ); CloudTemplateCatalogService::delete( $trueUncertainTemplateId, 1, $trueUncertainGeneration, ['user_id' => 1, 'can_manage' => true] ); $trueUncertainTemplateAgent->mode = 'uncertain'; $trueUncertainTemplateAgent->reconcileMode = 'uncertain'; $trueUncertainDelete = CloudTemplateCatalogService::deleteJob( $trueUncertainTemplateId, ['user_id' => 1, 'can_manage' => true] ); $requestsAtTrueUncertain = count($trueUncertainTemplateAgent->requests); $trueUncertainDeleteAgain = CloudTemplateCatalogService::deleteJob( $trueUncertainTemplateId, ['user_id' => 1, 'can_manage' => true] ); $assert($trueUncertainDelete['status'] === 'uncertain' && $trueUncertainDeleteAgain['status'] === 'uncertain' && $requestCount($trueUncertainTemplateAgent, 'DELETE') === 1 && count($trueUncertainTemplateAgent->requests) === $requestsAtTrueUncertain, 'A genuinely uncertain template delete was replayed or left pollable.'); $successfulTemplateId = 'debian-12-delete-success'; $successfulTemplateAgent = new CatalogDeletionFakeAgent(); $successfulTemplateAgent->mode = 'success'; CloudAgentSettingsService::$clients[11] = $successfulTemplateAgent; $successfulGeneration = str_repeat('b', 32); $seedTemplate($successfulTemplateId, 11, $successfulGeneration); $deletedTemplate = CloudTemplateCatalogService::delete( $successfulTemplateId, 1, $successfulGeneration, ['user_id' => 1, 'can_manage' => true] ); $assert($deletedTemplate['deleted'] === true && $deletedTemplate['status'] === 'succeeded' && (string) CloudTemplate::find($successfulTemplateId)->state === 'retired' && !CloudTemplateReplica::query()->where('template_id', $successfulTemplateId)->exists() && $requestCount($successfulTemplateAgent, 'DELETE') === 1, 'Exact template cleanup proof did not safely finalize retirement.'); $directTemplateId = 'debian-12-direct-404'; $directTemplateAgent = new CatalogDeletionFakeAgent(); $directTemplateAgent->mode = 'direct_not_found'; CloudAgentSettingsService::$clients[13] = $directTemplateAgent; $directTemplateGeneration = str_repeat('d', 32); $seedTemplate($directTemplateId, 13, $directTemplateGeneration); $directTemplate = CloudTemplateCatalogService::delete( $directTemplateId, 1, $directTemplateGeneration, ['user_id' => 1, 'can_manage' => true] ); $directTemplateReplica = CloudTemplateReplica::query() ->where('template_id', $directTemplateId)->first(); $assert($directTemplate['deleted'] === false && $directTemplate['status'] === 'uncertain' && (string) CloudTemplate::find($directTemplateId)->state === 'deleting' && (string) $directTemplateReplica->state === 'delete_uncertain' && (string) $directTemplateReplica->error_code === 'operation_not_found', 'Unmarked template 404 was made terminal without no-dispatch proof.'); $durableTemplateId = 'debian-12-durable-404'; $durableTemplateAgent = new CatalogDeletionFakeAgent(); $durableTemplateAgent->mode = 'durable_not_found'; CloudAgentSettingsService::$clients[14] = $durableTemplateAgent; $durableTemplateGeneration = str_repeat('e', 32); $seedTemplate($durableTemplateId, 14, $durableTemplateGeneration); $durableTemplate = CloudTemplateCatalogService::delete( $durableTemplateId, 1, $durableTemplateGeneration, ['user_id' => 1, 'can_manage' => true] ); $assert($durableTemplate['deleted'] === true && (string) CloudTemplate::find($durableTemplateId)->state === 'retired', 'Durable non-retryable template_not_found operation did not complete deletion.'); // User ISO: owner-scoped status plus strict owner evidence. $isoId = str_repeat('2', 32); $isoAgent = new CatalogDeletionFakeAgent(); CloudAgentSettingsService::$clients[20] = $isoAgent; $seedIso($isoId, 20, 42); $isoJob = CloudUserIsoService::delete($isoId, ['user_id' => 42, 'can_manage' => false]); $isoReplica = CloudUserIsoReplica::query()->where('host_id', 20) ->where('user_iso_id', $isoId)->first(); $isoOperation = (string) $isoReplica->operation_id; $assert($isoJob['status'] === 'running' && $isoJob['status_url'] === '/api/v1/user-isos/delete-jobs/' . $isoId && $requestCount($isoAgent, 'DELETE') === 1, 'User ISO delete did not return its durable owner-scoped job.'); $requestsBeforeWrongOwner = count($isoAgent->requests); $expectCloudError(static fn () => CloudUserIsoService::deleteJob( $isoId, ['user_id' => 43, 'can_manage' => false] ), 404, 'Another user read the ISO delete job.'); $assert(count($isoAgent->requests) === $requestsBeforeWrongOwner, 'Wrong-owner ISO status lookup contacted an Agent.'); CloudUserIsoService::delete($isoId, ['user_id' => 42, 'can_manage' => false]); $assert($requestCount($isoAgent, 'DELETE') === 1 && str_ends_with((string) end($isoAgent->requests)['target'], '/v1/operations/' . $isoOperation), 'Repeated ISO DELETE replayed destruction or changed operation ID.'); $isoAgent->mode = 'wrong_owner'; $uncertainIso = CloudUserIsoService::deleteJob( $isoId, ['user_id' => 42, 'can_manage' => false] ); $requestsBeforeIsoLockCheck = count($isoAgent->requests); $lockedIso = CloudUserIsoService::delete( $isoId, ['user_id' => 42, 'can_manage' => false] ); $isoReplica->refresh(); $assert($uncertainIso['status'] === 'uncertain' && $lockedIso['status'] === 'uncertain' && (string) $isoReplica->state === 'delete_uncertain' && (string) $isoReplica->error_code === 'delete_evidence_missing' && count($isoAgent->requests) === $requestsBeforeIsoLockCheck, 'ISO owner mismatch was not latched without destructive replay.'); $busyIsoId = str_repeat('b', 32); $busyIsoAgent = new CatalogDeletionFakeAgent(); $busyIsoAgent->mode = 'pre_dispatch_draining'; CloudAgentSettingsService::$clients[26] = $busyIsoAgent; $seedIso($busyIsoId, 26, 42); $busyIso = CloudUserIsoService::delete( $busyIsoId, ['user_id' => 42, 'can_manage' => false] ); $busyIsoReplica = CloudUserIsoReplica::query() ->where('user_iso_id', $busyIsoId)->first(); $busyIsoOperation = (string) $busyIsoReplica->operation_id; $assert($busyIso['status'] === 'running' && (string) $busyIsoReplica->state === 'delete_dispatch' && $busyIsoAgent->operations === [] && $requestCount($busyIsoAgent, 'DELETE') === 1, 'Pre-dispatch ISO backpressure became a terminal deletion.'); $busyIsoReplica->updated_at = time() - 6; $busyIsoReplica->save(); $busyIsoAgent->mode = 'success'; $retriedBusyIso = CloudUserIsoService::deleteJob( $busyIsoId, ['user_id' => 42, 'can_manage' => false] ); $busyIsoDeletes = array_values(array_filter( $busyIsoAgent->requests, static fn (array $request): bool => $request['method'] === 'DELETE' )); $assert($retriedBusyIso['status'] === 'succeeded' && count($busyIsoDeletes) === 2 && ($busyIsoDeletes[0]['payload']['operation_id'] ?? null) === $busyIsoOperation && ($busyIsoDeletes[1]['payload']['operation_id'] ?? null) === $busyIsoOperation, 'ISO backpressure retry changed operation ID or did not finish.'); $sendingIsoId = str_repeat('6', 32); $sendingIsoAgent = new CatalogDeletionFakeAgent(); CloudAgentSettingsService::$clients[22] = $sendingIsoAgent; $seedIso($sendingIsoId, 22, 42); Capsule::table('cloud_user_isos')->where('id', $sendingIsoId) ->update(['state' => 'deleting']); Capsule::table('cloud_user_iso_replicas') ->where('user_iso_id', $sendingIsoId)->update([ 'state' => 'delete_sending', 'operation_id' => bin2hex(random_bytes(16)), 'updated_at' => time(), ]); $freshSendingIso = CloudUserIsoService::deleteJob( $sendingIsoId, ['user_id' => 42, 'can_manage' => false] ); $assert($freshSendingIso['status'] === 'running' && $sendingIsoAgent->requests === [], 'Fresh ISO delete_sending claim was polled before dispatch settled.'); Capsule::table('cloud_user_iso_replicas') ->where('user_iso_id', $sendingIsoId) ->update(['updated_at' => time() - 91]); $staleSendingIso = CloudUserIsoService::deleteJob( $sendingIsoId, ['user_id' => 42, 'can_manage' => false] ); $sendingIsoReplica = CloudUserIsoReplica::query() ->where('user_iso_id', $sendingIsoId)->first(); $assert($staleSendingIso['status'] === 'uncertain' && (string) $sendingIsoReplica->error_code === 'operation_not_found' && count($sendingIsoAgent->requests) === 1 && $sendingIsoAgent->requests[0]['method'] === 'GET' && $sendingIsoAgent->requests[0]['target'] === '/v1/operations/' . (string) $sendingIsoReplica->operation_id && $requestCount($sendingIsoAgent, 'DELETE') === 0, 'Stale ISO sending claim was not queried safely by operation ID.'); $reconcileIsoId = str_repeat('d', 32); $reconcileIsoAgent = new CatalogDeletionFakeAgent(); CloudAgentSettingsService::$clients[25] = $reconcileIsoAgent; $seedIso($reconcileIsoId, 25, 42); CloudUserIsoService::delete( $reconcileIsoId, ['user_id' => 42, 'can_manage' => false] ); $reconcileIsoReplica = CloudUserIsoReplica::query() ->where('user_iso_id', $reconcileIsoId)->first(); $reconcileIsoOperation = (string) $reconcileIsoReplica->operation_id; $reconcileIsoAgent->mode = 'uncertain'; $reconcileIsoAgent->reconcileMode = 'success'; $beforeIsoReconcile = count($reconcileIsoAgent->requests); $reconciledIsoDelete = CloudUserIsoService::deleteJob( $reconcileIsoId, ['user_id' => 42, 'can_manage' => false] ); $isoReconcileRequests = array_slice( $reconcileIsoAgent->requests, $beforeIsoReconcile ); $assert($reconciledIsoDelete['status'] === 'succeeded' && $requestCount($reconcileIsoAgent, 'DELETE') === 1 && count($isoReconcileRequests) === 2 && $isoReconcileRequests[0]['method'] === 'GET' && $isoReconcileRequests[0]['target'] === '/v1/operations/' . $reconcileIsoOperation && $isoReconcileRequests[1]['method'] === 'POST' && $isoReconcileRequests[1]['target'] === '/v1/operations/' . $reconcileIsoOperation . '/reconcile', 'User ISO delete uncertainty replayed DELETE or changed operation ID.'); $successfulIsoId = str_repeat('3', 32); $successfulIsoAgent = new CatalogDeletionFakeAgent(); $successfulIsoAgent->mode = 'success'; CloudAgentSettingsService::$clients[21] = $successfulIsoAgent; $seedIso($successfulIsoId, 21, 42); $deletedIso = CloudUserIsoService::delete($successfulIsoId, ['user_id' => 42, 'can_manage' => false]); $successfulIsoTombstone = CloudUserIso::find($successfulIsoId); $polledDeletedIso = CloudUserIsoService::deleteJob($successfulIsoId, ['user_id' => 42, 'can_manage' => false]); $repeatedDeletedIso = CloudUserIsoService::delete($successfulIsoId, ['user_id' => 42, 'can_manage' => false]); $historicalImport = CloudUserIsoService::job( (string) $successfulIsoTombstone->import_operation_id, ['user_id' => 42, 'can_manage' => false] ); $listedAfterDelete = CloudUserIsoService::listing(42); $assert($deletedIso['deleted'] === true && $deletedIso['status'] === 'succeeded' && $polledDeletedIso['status'] === 'succeeded' && $repeatedDeletedIso['status'] === 'succeeded' && $historicalImport['status'] === 'succeeded' && $historicalImport['deleted'] === true && !in_array($successfulIsoId, array_column( $listedAfterDelete['isos'], 'id' ), true) && $successfulIsoTombstone !== null && (string) $successfulIsoTombstone->state === 'retired' && (string) $successfulIsoTombstone->name === '' && (string) $successfulIsoTombstone->sealed_source_url === '' && (string) $successfulIsoTombstone->sha256 === '' && (int) $successfulIsoTombstone->size_bytes === 0 && !CloudUserIsoReplica::query()->where('user_iso_id', $successfulIsoId)->exists() && $requestCount($successfulIsoAgent, 'DELETE') === 1, 'Exact ISO cleanup did not leave a pollable, data-free tombstone.'); $directIsoId = str_repeat('7', 32); $directIsoAgent = new CatalogDeletionFakeAgent(); $directIsoAgent->mode = 'direct_not_found'; CloudAgentSettingsService::$clients[23] = $directIsoAgent; $seedIso($directIsoId, 23, 42); $directIso = CloudUserIsoService::delete($directIsoId, ['user_id' => 42, 'can_manage' => false]); $directIsoReplica = CloudUserIsoReplica::query() ->where('user_iso_id', $directIsoId)->first(); $assert($directIso['deleted'] === false && $directIso['status'] === 'uncertain' && (string) CloudUserIso::find($directIsoId)->state === 'deleting' && (string) $directIsoReplica->state === 'delete_uncertain' && (string) $directIsoReplica->error_code === 'operation_not_found', 'Unmarked ISO 404 was made terminal without no-dispatch proof.'); $durableIsoId = str_repeat('8', 32); $durableIsoAgent = new CatalogDeletionFakeAgent(); $durableIsoAgent->mode = 'durable_not_found'; CloudAgentSettingsService::$clients[24] = $durableIsoAgent; $seedIso($durableIsoId, 24, 42); $durableIso = CloudUserIsoService::delete($durableIsoId, ['user_id' => 42, 'can_manage' => false]); $durableIsoTombstone = CloudUserIso::find($durableIsoId); $assert($durableIso['deleted'] === true && (string) $durableIsoTombstone->state === 'retired', 'Durable non-retryable user_iso_not_found operation did not complete deletion.'); // Rescue: terminal tombstones survive retries and prevent redispatch. $busyRescueId = str_repeat('c', 32); $busyRescueAgent = new CatalogDeletionFakeAgent(); $busyRescueAgent->mode = 'pre_dispatch_busy'; CloudAgentSettingsService::$clients[35] = $busyRescueAgent; $busyRescue = CloudRescueTemplateDeletionService::delete( 35, $busyRescueId, ['user_id' => 1, 'can_manage' => true] ); $busyRescueRecord = CloudRescueTemplateDeletion::query() ->where('host_id', 35)->where('template_id', $busyRescueId)->first(); $busyRescueOperation = (string) $busyRescueRecord->operation_id; $assert($busyRescue['status'] === 'running' && (string) $busyRescueRecord->state === 'dispatching' && $busyRescueAgent->operations === [] && $requestCount($busyRescueAgent, 'DELETE') === 1, 'Pre-dispatch rescue backpressure became a terminal deletion.'); // This model has a composite key and deliberately must not use save(). Capsule::table('cloud_rescue_template_deletions') ->where('host_id', 35)->where('template_id', $busyRescueId) ->where('operation_id', $busyRescueOperation) ->update(['updated_at' => time() - 6]); $busyRescueAgent->mode = 'success'; $retriedBusyRescue = CloudRescueTemplateDeletionService::job( 35, $busyRescueOperation, ['user_id' => 1, 'can_manage' => true] ); $busyRescueDeletes = array_values(array_filter( $busyRescueAgent->requests, static fn (array $request): bool => $request['method'] === 'DELETE' )); $assert($retriedBusyRescue['status'] === 'succeeded' && count($busyRescueDeletes) === 2 && ($busyRescueDeletes[0]['payload']['operation_id'] ?? null) === $busyRescueOperation && ($busyRescueDeletes[1]['payload']['operation_id'] ?? null) === $busyRescueOperation, 'Rescue backpressure retry changed operation ID or did not finish.'); $rescueId = str_repeat('4', 32); $rescueAgent = new CatalogDeletionFakeAgent(); CloudAgentSettingsService::$clients[30] = $rescueAgent; $rescueJob = CloudRescueTemplateDeletionService::delete( 30, $rescueId, ['user_id' => 1, 'can_manage' => true] ); $rescueRecord = CloudRescueTemplateDeletion::query() ->where('host_id', 30)->where('template_id', $rescueId)->first(); $rescueOperation = (string) $rescueRecord->operation_id; $assert($rescueJob['status'] === 'running' && $rescueJob['operation_id'] === $rescueOperation && $rescueJob['status_url'] === '/api/v1/admin/rescue-templates/delete-jobs/' . $rescueOperation . '?host_id=30' && $requestCount($rescueAgent, 'DELETE') === 1, 'Rescue delete did not persist its host-scoped operation tombstone.'); CloudRescueTemplateDeletionService::delete( 30, $rescueId, ['user_id' => 1, 'can_manage' => true] ); $assert($requestCount($rescueAgent, 'DELETE') === 1 && str_ends_with((string) end($rescueAgent->requests)['target'], '/v1/operations/' . $rescueOperation), 'Repeated rescue DELETE replayed destruction or changed operation ID.'); $rescueAgent->mode = 'missing_evidence'; $uncertainRescue = CloudRescueTemplateDeletionService::job( 30, $rescueOperation, ['user_id' => 1, 'can_manage' => true] ); $requestsBeforeRescueLockCheck = count($rescueAgent->requests); $lockedRescue = CloudRescueTemplateDeletionService::delete( 30, $rescueId, ['user_id' => 1, 'can_manage' => true] ); $rescueRecord = CloudRescueTemplateDeletion::query() ->where('host_id', 30)->where('template_id', $rescueId) ->where('operation_id', $rescueOperation)->first(); $assert($uncertainRescue['status'] === 'uncertain' && $lockedRescue['status'] === 'uncertain' && (string) $rescueRecord->state === 'uncertain' && (string) $rescueRecord->error_code === 'delete_evidence_missing' && count($rescueAgent->requests) === $requestsBeforeRescueLockCheck, 'Rescue missing-proof result was not retained as a no-replay tombstone.'); $reconcileRescueId = str_repeat('e', 32); $reconcileRescueAgent = new CatalogDeletionFakeAgent(); CloudAgentSettingsService::$clients[34] = $reconcileRescueAgent; $reconcileRescue = CloudRescueTemplateDeletionService::delete( 34, $reconcileRescueId, ['user_id' => 1, 'can_manage' => true] ); $reconcileRescueOperation = (string) $reconcileRescue['operation_id']; $reconcileRescueAgent->mode = 'uncertain'; $reconcileRescueAgent->reconcileMode = 'success'; $beforeRescueReconcile = count($reconcileRescueAgent->requests); $reconciledRescueDelete = CloudRescueTemplateDeletionService::job( 34, $reconcileRescueOperation, ['user_id' => 1, 'can_manage' => true] ); $rescueReconcileRequests = array_slice( $reconcileRescueAgent->requests, $beforeRescueReconcile ); $assert($reconciledRescueDelete['status'] === 'succeeded' && $requestCount($reconcileRescueAgent, 'DELETE') === 1 && count($rescueReconcileRequests) === 2 && $rescueReconcileRequests[0]['method'] === 'GET' && $rescueReconcileRequests[0]['target'] === '/v1/operations/' . $reconcileRescueOperation && $rescueReconcileRequests[1]['method'] === 'POST' && $rescueReconcileRequests[1]['target'] === '/v1/operations/' . $reconcileRescueOperation . '/reconcile', 'Rescue-template delete uncertainty replayed DELETE or changed operation ID.'); $successfulRescueId = str_repeat('5', 32); $successfulRescueAgent = new CatalogDeletionFakeAgent(); $successfulRescueAgent->mode = 'success'; CloudAgentSettingsService::$clients[31] = $successfulRescueAgent; $deletedRescue = CloudRescueTemplateDeletionService::delete( 31, $successfulRescueId, ['user_id' => 1, 'can_manage' => true] ); $requestsAfterRescueSuccess = count($successfulRescueAgent->requests); $rescueStatus = CloudRescueTemplateDeletionService::job( 31, (string) $deletedRescue['operation_id'], ['user_id' => 1, 'can_manage' => true] ); $rescueRetry = CloudRescueTemplateDeletionService::delete( 31, $successfulRescueId, ['user_id' => 1, 'can_manage' => true] ); $assert($deletedRescue['deleted'] === true && $rescueStatus['status'] === 'succeeded' && $rescueRetry['status'] === 'succeeded' && CloudRescueTemplateDeletion::query()->where('host_id', 31) ->where('template_id', $successfulRescueId)->exists() && $requestCount($successfulRescueAgent, 'DELETE') === 1 && count($successfulRescueAgent->requests) === $requestsAfterRescueSuccess, 'Successful rescue tombstone did not make every retry read-only.'); // A slower status poll may carry a stale pre-terminal model snapshot. // Its result must not move the durable success tombstone backwards. $staleRescueRecord = clone CloudRescueTemplateDeletion::query() ->where('host_id', 31)->where('template_id', $successfulRescueId) ->first(); $staleRescueRecord->state = 'running'; $markRescueDeletion = new ReflectionMethod( CloudRescueTemplateDeletionService::class, 'mark' ); $markRescueDeletion->invoke( null, $staleRescueRecord, 'running', 'late_poll_result' ); $successfulRescueRecord = CloudRescueTemplateDeletion::query() ->where('host_id', 31)->where('template_id', $successfulRescueId) ->first(); $assert((string) $successfulRescueRecord->state === 'succeeded' && (string) $successfulRescueRecord->error_code === '', 'A stale rescue deletion poll overwrote its terminal tombstone.'); $directRescueId = str_repeat('9', 32); $directRescueAgent = new CatalogDeletionFakeAgent(); $directRescueAgent->mode = 'direct_not_found'; CloudAgentSettingsService::$clients[32] = $directRescueAgent; $directRescue = CloudRescueTemplateDeletionService::delete( 32, $directRescueId, ['user_id' => 1, 'can_manage' => true] ); $directRescueRecord = CloudRescueTemplateDeletion::query() ->where('host_id', 32)->where('template_id', $directRescueId)->first(); $assert($directRescue['deleted'] === false && $directRescue['status'] === 'running' && (string) $directRescueRecord->state === 'running' && (string) $directRescueRecord->error_code === 'rescue_template_not_found', 'Unmarked rescue-template 404 was made terminal without no-dispatch proof.'); $durableRescueId = str_repeat('a', 32); $durableRescueAgent = new CatalogDeletionFakeAgent(); $durableRescueAgent->mode = 'durable_not_found'; CloudAgentSettingsService::$clients[33] = $durableRescueAgent; $durableRescue = CloudRescueTemplateDeletionService::delete( 33, $durableRescueId, ['user_id' => 1, 'can_manage' => true] ); $assert($durableRescue['deleted'] === true && $durableRescue['status'] === 'succeeded' && CloudRescueTemplateDeletion::query()->where('host_id', 33) ->where('template_id', $durableRescueId) ->where('state', 'succeeded')->exists(), 'Durable non-retryable rescue_template_not_found operation did not complete deletion.'); // The same rescue template ID may exist on several hosts. Every reload // must retain the complete host/template/operation identity. $sharedRescueId = str_repeat('b', 32); $otherOperation = str_repeat('c', 32); Capsule::table('cloud_rescue_template_deletions')->insert([ 'host_id' => 40, 'template_id' => $sharedRescueId, 'operation_id' => $otherOperation, 'state' => 'succeeded', 'error_code' => '', 'created_at' => time(), 'updated_at' => time(), ]); $sharedRescueAgent = new CatalogDeletionFakeAgent(); $sharedRescueAgent->mode = 'success'; CloudAgentSettingsService::$clients[41] = $sharedRescueAgent; $sharedHostDelete = CloudRescueTemplateDeletionService::delete( 41, $sharedRescueId, ['user_id' => 1, 'can_manage' => true] ); $otherHostRow = CloudRescueTemplateDeletion::query() ->where('host_id', 40)->where('template_id', $sharedRescueId) ->where('operation_id', $otherOperation)->first(); $targetHostRow = CloudRescueTemplateDeletion::query() ->where('host_id', 41)->where('template_id', $sharedRescueId) ->where('operation_id', (string) $sharedHostDelete['operation_id']) ->first(); $rescueServiceSource = file_get_contents(dirname(__DIR__) . '/app/Services/Cloud/CloudRescueTemplateDeletionService.php'); $assert($sharedHostDelete['status'] === 'succeeded' && $otherHostRow !== null && (string) $otherHostRow->state === 'succeeded' && (string) $otherHostRow->error_code === '' && $targetHostRow !== null && (string) $targetHostRow->state === 'succeeded' && $requestCount($sharedRescueAgent, 'DELETE') === 1 && is_string($rescueServiceSource) && !str_contains($rescueServiceSource, '->refresh()') && str_contains($rescueServiceSource, "->where('operation_id', (string) \$record->operation_id)"), 'Composite rescue-template identity crossed hosts during state reload.'); echo "Catalog delete operation persistence, exact cleanup evidence, owner binding and no-replay tombstones passed.\n"; }