addConnection([ 'driver' => 'sqlite', 'database' => ':memory:', ], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('operation_id'); $table->string('request_idempotency_key'); $table->string('request_fingerprint'); $table->integer('parent_job_id')->nullable(); $table->integer('user_id'); $table->integer('host_id')->nullable(); $table->string('kind'); $table->string('execution_mode'); $table->string('phase'); $table->string('outcome')->nullable(); $table->string('status'); $table->text('sealed_payload'); $table->integer('attempts')->default(0); $table->integer('dispatch_attempts')->default(0); $table->integer('dispatch_deadline_at')->default(1); $table->integer('dispatch_started_at')->nullable(); $table->integer('agent_accepted_at')->nullable(); $table->integer('dispatched_at')->nullable(); $table->integer('receipt_deadline_at')->nullable(); $table->integer('terminal_at')->nullable(); $table->boolean('effect_may_have_applied')->nullable(); $table->boolean('safe_to_resubmit')->nullable(); $table->string('effect_resolution')->default('not_applicable'); $table->integer('effect_resolved_at')->nullable(); $table->integer('effect_resolved_by')->nullable(); $table->string('effect_resolution_reason')->default(''); $table->string('completion_event_id')->nullable(); $table->integer('read_at')->nullable(); $table->string('lease_token')->nullable(); $table->integer('lease_until')->default(0); $table->integer('available_at')->default(0); $table->string('error_code')->nullable(); $table->boolean('finalization_incident')->default(0); $table->string('finalization_error_code')->nullable(); $table->text('result_json')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_templates', static function (Blueprint $table): void { $table->string('id')->primary(); $table->string('catalog_key'); $table->string('version_label'); $table->boolean('is_default')->default(false); $table->string('name'); $table->string('os_family'); $table->string('os_version'); $table->integer('image_bytes'); $table->string('sha256'); $table->text('startup_script'); $table->integer('revision'); $table->string('generation_token'); $table->string('state'); $table->integer('active_job_id')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_template_replicas', static function (Blueprint $table): void { $table->integer('host_id'); $table->string('template_id'); $table->integer('revision'); $table->string('generation_token'); $table->string('state'); $table->integer('image_bytes'); $table->string('sha256'); $table->string('operation_id'); $table->integer('job_id')->nullable(); $table->string('error_code')->default(''); $table->integer('created_at'); $table->integer('updated_at'); $table->primary(['host_id', 'template_id']); }); $schema->create('cloud_user_iso_replicas', static function (Blueprint $table): void { $table->integer('job_id')->nullable(); }); $schema->create('cloud_template_artifacts', static function (Blueprint $table): void { $table->string('template_id')->primary(); $table->integer('storage_revision'); $table->text('sealed_storage_runtime'); $table->string('object_key'); $table->string('object_version_id')->default(''); $table->string('object_etag'); $table->integer('size_bytes'); $table->string('sha256'); $table->string('state'); $table->string('error_code')->default(''); $table->string('delete_operation_id')->default(''); $table->integer('delete_generation')->default(0); $table->integer('delete_cycle_attempts')->default(0); $table->integer('delete_presence_confirmations')->default(0); $table->integer('delete_last_checked_at')->default(0); $table->integer('delete_next_check_at')->default(0); $table->integer('delete_attention_at')->nullable(); $table->integer('verified_at'); $table->integer('created_at'); $table->integer('updated_at'); }); $runtime = [ 'config' => [ 'endpoint' => 'https://objects.example.com', 'region' => 'ewr', 'bucket' => 'templates', 'prefix' => 'virt', 'path_style' => true, ], 'credentials' => [ 'access_key_id' => 'DELETEACCESSKEY', 'secret_access_key' => 'delete-secret', ], 'revision' => 1, ]; $now = time(); $fixture = static function (string $suffix) use ($runtime, $now): array { $parentId = (int) Capsule::table('cloud_jobs')->insertGetId([ 'operation_id' => md5('parent-' . $suffix), 'request_idempotency_key' => md5('parent-' . $suffix), 'request_fingerprint' => hash('sha256', 'parent-' . $suffix), 'parent_job_id' => null, 'user_id' => 1, 'host_id' => null, 'kind' => 'template_delete', 'execution_mode' => 'aggregate', 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'), 'attempts' => 0, 'dispatch_attempts' => 0, 'dispatch_deadline_at' => $now + 300, 'receipt_deadline_at' => $now + 3600, 'effect_resolution' => 'not_applicable', 'effect_resolution_reason' => '', 'lease_until' => 0, 'available_at' => $now, 'finalization_incident' => 0, 'created_at' => $now, 'updated_at' => $now, ]); $childOperation = md5('child-' . $suffix); $childId = (int) Capsule::table('cloud_jobs')->insertGetId([ 'operation_id' => $childOperation, 'request_idempotency_key' => $childOperation, 'request_fingerprint' => hash('sha256', 'child-' . $suffix), 'parent_job_id' => $parentId, 'user_id' => 1, 'host_id' => 7, 'kind' => 'template_delete_replica', 'execution_mode' => 'agent', 'phase' => 'terminal', 'outcome' => 'succeeded', 'status' => 'succeeded', 'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'), 'attempts' => 1, 'dispatch_attempts' => 1, 'dispatch_deadline_at' => $now - 20, 'dispatch_started_at' => $now - 19, 'agent_accepted_at' => $now - 18, 'dispatched_at' => $now - 17, 'receipt_deadline_at' => $now + 900, 'terminal_at' => $now - 1, 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => 'applied', 'effect_resolution_reason' => '', 'lease_until' => 0, 'available_at' => 0, 'finalization_incident' => 0, 'created_at' => $now - 20, 'updated_at' => $now - 1, ]); $templateId = 'delete-template-' . $suffix; $generation = md5('generation-' . $suffix); $sha = hash('sha256', 'image-' . $suffix); Capsule::table('cloud_templates')->insert([ 'id' => $templateId, 'catalog_key' => 'debian-13', 'version_label' => $suffix, 'is_default' => 0, 'name' => 'Delete ' . $suffix, 'os_family' => 'Debian', 'os_version' => '13', 'image_bytes' => 4096, 'sha256' => $sha, 'startup_script' => '', 'revision' => 1, 'generation_token' => $generation, 'state' => 'deleting', 'active_job_id' => $parentId, 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_template_replicas')->insert([ 'host_id' => 7, 'template_id' => $templateId, 'revision' => 1, 'generation_token' => $generation, 'state' => 'delete_succeeded', 'image_bytes' => 4096, 'sha256' => $sha, 'operation_id' => $childOperation, 'job_id' => $childId, 'error_code' => '', 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_template_artifacts')->insert([ 'template_id' => $templateId, 'storage_revision' => 1, 'sealed_storage_runtime' => CloudSecrets::encrypt( $runtime, 'template-artifact-runtime' ), 'object_key' => 'virt/cloud-templates/' . $templateId . '/' . $generation . '/system.qcow2', 'object_version_id' => 'version-' . $suffix, 'object_etag' => '"etag-' . $suffix . '"', 'size_bytes' => 4096, 'sha256' => $sha, 'state' => 'ready', 'error_code' => '', 'delete_operation_id' => '', 'delete_generation' => 0, 'delete_cycle_attempts' => 0, 'delete_presence_confirmations' => 0, 'delete_last_checked_at' => 0, 'delete_next_check_at' => 0, 'delete_attention_at' => null, 'verified_at' => $now, 'created_at' => $now, 'updated_at' => $now, ]); $template = CloudTemplate::findOrFail($templateId); $parent = CloudJob::findOrFail($parentId); $template->getConnection()->transaction( static function () use ($templateId, $parentId): void { $parent = CloudJob::where('id', $parentId) ->lockForUpdate()->firstOrFail(); $template = CloudTemplate::where('id', $templateId) ->lockForUpdate()->firstOrFail(); CloudTemplateDeletionService::armLocked($template, $parent); }, 3 ); return [$templateId, $parentId]; }; [$normalId, $normalParentId] = $fixture('normal'); $normalHistory = []; $normalStack = HandlerStack::create(new MockHandler([ new Response(200, [ 'Content-Length' => '4096', 'ETag' => '"etag-normal"', 'x-amz-version-id' => 'version-normal', ]), new Response(204), ])); $normalStack->push(Middleware::history($normalHistory)); $normalClient = new CloudObjectStorageS3Client(new Client([ 'handler' => $normalStack, ])); $normal = CloudTemplateDeletionService::processDue(1, $normalClient); $assert($normal['completed'] === 1 && CloudTemplateArtifact::find($normalId) === null && (string) CloudTemplate::findOrFail($normalId)->state === 'retired' && (int) CloudTemplate::findOrFail($normalId)->revision === 2 && (string) CloudJob::findOrFail($normalParentId)->outcome === 'succeeded' && array_map( static fn (array $item): string => $item['request']->getMethod(), $normalHistory ) === ['HEAD', 'DELETE'], 'Confirmed master deletion did not retire the exact catalog version.'); [$unknownId, $unknownParentId] = $fixture('unknown'); $unknownHistory = []; $unknownStack = HandlerStack::create(new MockHandler([ new Response(200, [ 'Content-Length' => '4096', 'ETag' => '"etag-unknown"', 'x-amz-version-id' => 'version-unknown', ]), new ConnectException('lost after delete', new Request( 'DELETE', 'https://objects.example.com' )), new Response(200, [ 'Content-Length' => '4096', 'ETag' => '"etag-unknown"', 'x-amz-version-id' => 'version-unknown', ]), new Response(200, [ 'Content-Length' => '4096', 'ETag' => '"etag-unknown"', 'x-amz-version-id' => 'version-unknown', ]), new Response(204), ])); $unknownStack->push(Middleware::history($unknownHistory)); $unknownClient = new CloudObjectStorageS3Client(new Client([ 'handler' => $unknownStack, ])); CloudTemplateDeletionService::processDue(1, $unknownClient); $unknownArtifact = CloudTemplateArtifact::findOrFail($unknownId); $assert((string) $unknownArtifact->state === 'delete_uncertain' && (int) $unknownArtifact->delete_generation === 1, 'Ambiguous DELETE was not fenced on its original generation.'); Capsule::table('cloud_template_artifacts')->where('template_id', $unknownId) ->update(['delete_next_check_at' => 0]); CloudTemplateDeletionService::processDue(1, $unknownClient); $unknownArtifact = CloudTemplateArtifact::findOrFail($unknownId); $assert((int) $unknownArtifact->delete_presence_confirmations === 1 && (int) $unknownArtifact->delete_generation === 1, 'One presence probe incorrectly replayed an ambiguous DELETE.'); Capsule::table('cloud_template_artifacts')->where('template_id', $unknownId) ->update([ 'delete_last_checked_at' => time() - 301, 'delete_next_check_at' => 0, ]); $converged = CloudTemplateDeletionService::processDue(1, $unknownClient); $assert($converged['completed'] === 1 && CloudTemplateArtifact::find($unknownId) === null && (string) CloudTemplate::findOrFail($unknownId)->state === 'retired' && (string) CloudJob::findOrFail($unknownParentId)->outcome === 'succeeded' && array_map( static fn (array $item): string => $item['request']->getMethod(), $unknownHistory ) === ['HEAD', 'DELETE', 'HEAD', 'HEAD', 'DELETE'], 'Stable two-probe evidence did not create exactly one fresh delete generation.'); [$attentionId, $attentionParentId] = $fixture('attention'); $attentionClient = new CloudObjectStorageS3Client(new Client([ 'handler' => HandlerStack::create(new MockHandler([ new Response(200, [ 'Content-Length' => '8192', 'ETag' => '"different"', 'x-amz-version-id' => 'version-attention', ]), ])), ])); $attention = CloudTemplateDeletionService::processDue(1, $attentionClient); $assert($attention['attention'] === 1 && (string) CloudTemplateArtifact::findOrFail($attentionId)->state === 'delete_attention' && (string) CloudTemplate::findOrFail($attentionId)->state === 'delete_failed' && (string) CloudJob::findOrFail($attentionParentId)->outcome === 'failed', 'Changed object identity was not fenced for explicit administrator review.'); echo "Cloud-template two-phase deletion and recovery behavior passed.\n"; }