addConnection([ 'driver' => 'sqlite', 'database' => ':memory:', ], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('cloud_hosts', static function (Blueprint $table): void { $table->integer('id')->primary(); $table->boolean('enabled'); $table->string('default_network'); }); $schema->create('cloud_host_agents', static function (Blueprint $table): void { $table->integer('host_id')->primary(); $table->boolean('enabled'); $table->string('last_test_status'); $table->integer('tested_revision'); $table->integer('revision'); }); $schema->create('cloud_host_storage_configurations', static function (Blueprint $table): void { $table->integer('host_id')->primary(); $table->string('state'); }); $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('operation_id')->unique(); $table->string('request_idempotency_key'); $table->string('request_fingerprint'); $table->integer('parent_job_id')->nullable(); $table->integer('user_id'); $table->integer('host_id')->nullable(); $table->string('kind'); $table->string('execution_mode'); $table->string('phase'); $table->string('outcome')->nullable(); $table->string('status'); $table->text('sealed_payload'); $table->integer('attempts')->default(0); $table->integer('dispatch_attempts')->default(0); $table->integer('dispatch_deadline_at')->default(1); $table->integer('dispatch_started_at')->nullable(); $table->integer('agent_accepted_at')->nullable(); $table->integer('dispatched_at')->nullable(); $table->integer('receipt_deadline_at')->nullable(); $table->integer('terminal_at')->nullable(); $table->boolean('effect_may_have_applied')->nullable(); $table->boolean('safe_to_resubmit')->nullable(); $table->string('effect_resolution')->default('not_applicable'); $table->integer('effect_resolved_at')->nullable(); $table->integer('effect_resolved_by')->nullable(); $table->string('effect_resolution_reason')->default(''); $table->boolean('finalization_incident')->default(false); $table->string('finalization_error_code')->nullable(); $table->string('completion_event_id')->nullable(); $table->integer('read_at')->nullable(); $table->string('lease_token')->nullable(); $table->integer('lease_until')->default(0); $table->integer('available_at')->default(0); $table->string('error_code')->nullable(); $table->text('result_json')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_rescue_templates', static function (Blueprint $table): void { $table->string('id')->primary(); $table->string('name'); $table->integer('size_bytes'); $table->string('sha256'); $table->integer('revision'); $table->string('generation_token'); $table->string('state'); $table->integer('active_job_id')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_rescue_template_artifacts', static function (Blueprint $table): void { $table->string('template_id')->primary(); $table->integer('storage_revision'); $table->text('sealed_storage_runtime'); $table->string('object_key'); $table->string('object_version_id'); $table->string('object_etag'); $table->integer('size_bytes'); $table->string('sha256'); $table->string('state'); $table->string('error_code')->default(''); $table->string('delete_operation_id')->default(''); $table->integer('delete_generation')->default(0); $table->integer('delete_cycle_attempts')->default(0); $table->integer('delete_presence_confirmations')->default(0); $table->integer('delete_last_checked_at')->default(0); $table->integer('delete_next_check_at')->default(0); $table->integer('delete_attention_at')->nullable(); $table->integer('verified_at')->default(0); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_rescue_template_replicas', static function (Blueprint $table): void { $table->integer('host_id'); $table->string('template_id'); $table->integer('revision'); $table->string('generation_token'); $table->string('state'); $table->integer('size_bytes'); $table->string('sha256'); $table->string('operation_id'); $table->integer('job_id')->nullable(); $table->string('error_code')->default(''); $table->string('observed_state')->default('never'); $table->integer('observed_revision')->default(0); $table->string('observed_generation_token')->default(''); $table->integer('observed_size_bytes')->default(0); $table->string('observed_sha256')->default(''); $table->integer('last_verified_at')->default(0); $table->integer('cleanup_resolved_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); $table->primary(['host_id', 'template_id']); }); $addHost = static function (int $hostId) use ($capsule): void { Capsule::table('cloud_hosts')->insert([ 'id' => $hostId, 'enabled' => 1, 'default_network' => 'bridge:br0', ]); Capsule::table('cloud_host_agents')->insert([ 'host_id' => $hostId, 'enabled' => 1, 'last_test_status' => 'healthy', 'tested_revision' => 3, 'revision' => 3, ]); Capsule::table('cloud_host_storage_configurations')->insert([ 'host_id' => $hostId, 'state' => 'applied', ]); }; $addHost(31); $addHost(32); $now = time(); $templateId = str_repeat('a', 32); $generation = str_repeat('b', 32); $sha256 = str_repeat('c', 64); $bytes = 2 * 1024 * 1024; Capsule::table('cloud_rescue_templates')->insert([ 'id' => $templateId, 'name' => 'SystemRescue', 'size_bytes' => $bytes, 'sha256' => $sha256, 'revision' => 1, 'generation_token' => $generation, 'state' => 'validating', 'active_job_id' => null, 'created_at' => $now, 'updated_at' => $now, ]); $runtime = [ 'config' => [ 'endpoint' => 'https://objects.example.com', 'region' => 'ewr', 'bucket' => 'rescue', 'prefix' => 'virt', 'path_style' => true, ], 'credentials' => [ 'access_key_id' => 'ACCESSKEY', 'secret_access_key' => 'secret', ], 'revision' => 1, ]; Capsule::table('cloud_rescue_template_artifacts')->insert([ 'template_id' => $templateId, 'storage_revision' => 1, 'sealed_storage_runtime' => CloudSecrets::encrypt( $runtime, 'rescue-artifact-runtime' ), 'object_key' => 'virt/rescue-isos/' . $templateId . '/' . $generation . '/rescue.iso', 'object_version_id' => 'version-1', 'object_etag' => '"etag-1"', 'size_bytes' => $bytes, 'sha256' => $sha256, 'state' => 'validating', 'created_at' => $now, 'updated_at' => $now, ]); $head = static fn (): Response => new Response(200, [ 'Content-Length' => (string) $bytes, 'ETag' => '"etag-1"', 'x-amz-version-id' => 'version-1', ]); $storageClient = new CloudObjectStorageS3Client(new Client([ 'handler' => HandlerStack::create(new MockHandler([ $head(), $head(), ])), ])); $actor = [ 'is_super_admin' => true, 'can_manage' => true, 'user_id' => 1, ]; $validation = CloudRescueTemplateReplicaCoordinator::beginValidation( $templateId, 31, $actor, [31, 32], $storageClient ); $assert(($validation['scheduled'] ?? false) === true, 'First-host validation was not scheduled.'); $validationJob = CloudJob::findOrFail((int) $validation['task_id']); $assert((string) $validationJob->kind === 'rescue_catalog_validate_replica' && (int) CloudRescueTemplate::findOrFail($templateId)->active_job_id === (int) $validationJob->id, 'Validation job did not own the quarantine fence.'); $result = [ 'id' => $templateId, 'name' => 'SystemRescue', 'size_bytes' => $bytes, 'sha256' => $sha256, 'revision' => 1, 'generation_token' => $generation, 'created_at' => gmdate(DATE_RFC3339), ]; CloudRescueTemplateReplicaCoordinator::finishJobLocked( $validationJob, 'succeeded', $result, ['rescue_template_id' => $templateId], ['effect_may_have_applied' => false, 'safe_to_resubmit' => false] ); $assert((string) CloudRescueTemplateArtifact::findOrFail($templateId)->state === 'ready' && (string) CloudRescueTemplate::findOrFail($templateId)->state === 'distributing' && CloudRescueTemplate::findOrFail($templateId)->active_job_id === null, 'Successful first-host validation did not promote the object.'); $distribution = CloudRescueTemplateReplicaCoordinator::distribute( $templateId, $actor, 'validation_complete', [31, 32], $storageClient ); $assert((int) $distribution['scheduled'] === 1 && (int) $distribution['tasks'][0]['host_id'] === 32, 'Distribution did not skip the exact validation host.'); $replicaJob = CloudJob::findOrFail( (int) $distribution['tasks'][0]['task_id'] ); CloudRescueTemplateReplicaCoordinator::finishJobLocked( $replicaJob, 'succeeded', $result, ['rescue_template_id' => $templateId], ['effect_may_have_applied' => false, 'safe_to_resubmit' => false] ); $assert((string) CloudRescueTemplate::findOrFail($templateId)->state === 'active' && CloudRescueTemplateReplica::query() ->where('template_id', $templateId) ->where('state', 'active')->count() === 2 && (int) CloudRescueTemplateReplica::query() ->where('template_id', $templateId)->where('host_id', 31) ->value('job_id') === (int) $validationJob->id && (int) CloudRescueTemplateReplica::query() ->where('template_id', $templateId)->where('host_id', 32) ->value('job_id') === (int) $replicaJob->id, 'All exact replicas did not activate the catalog.'); $publicReady = CloudRescueCatalogService::publicCatalogForHost(31); $publicUnavailable = CloudRescueCatalogService::publicCatalogForHost(99); $assert(($publicReady['status']['state'] ?? null) === 'ready' && ($publicReady['templates'][0]['id'] ?? null) === $templateId && array_keys($publicReady['templates'][0]) === ['id', 'name'] && ($publicUnavailable['status']['state'] ?? null) === 'host_unavailable' && $publicUnavailable['templates'] === [], 'Public rescue catalog exposed a non-exact or unavailable host copy.'); $addHost(33); $publicSyncing = CloudRescueCatalogService::publicCatalogForHost(33); $assert(($publicSyncing['status']['state'] ?? null) === 'syncing' && $publicSyncing['templates'] === [], 'Public rescue catalog did not distinguish a pending host replica.'); $adminCatalog = CloudRescueCatalogService::adminCatalog(); $adminEntry = $adminCatalog['templates'][0] ?? null; $missingHost = is_array($adminEntry) ? array_values(array_filter( $adminEntry['replicas'], static fn (array $replica): bool => (int) $replica['host_id'] === 33 ))[0] ?? null : null; $assert(count($adminCatalog['hosts']) === 3 && is_array($adminEntry) && (int) $adminEntry['replica_summary']['eligible'] === 3 && (int) $adminEntry['replica_summary']['ready'] === 2 && (int) $adminEntry['replica_summary']['attention'] === 1 && is_array($missingHost) && $missingHost['state'] === 'missing' && $missingHost['repair_action'] === 'import' && $missingHost['repair_reason'] === 'missing', 'Administrator rescue catalog did not expose the exact host matrix.'); $remoteExact = $result; $clientFactory = static fn (int $hostId): object => new class( $hostId, $remoteExact ) { public function __construct( private int $hostId, private array $exact ) { } public function request(string $method, string $path, array $actor): array { if ($method !== 'GET' || $path !== '/v1/rescue-templates') { throw new RuntimeException('Unexpected reconciliation request'); } return ['templates' => $this->hostId === 33 ? [] : [$this->exact]]; } }; $repairs = []; $reconciler = new CloudRescueTemplateReconciliationService( $clientFactory, static function ( string $id, int $hostId ) use (&$repairs): array { $repairs[] = [$id, $hostId]; return ['scheduled' => true]; } ); $report = $reconciler->reconcile($actor, 3, 3, 0); $assert($report['hosts_checked'] === 3 && $report['missing'] === 1 && $report['scheduled'] === 1 && $repairs === [[$templateId, 33]] && (string) CloudRescueTemplate::findOrFail($templateId)->state === 'degraded', 'Periodic reconciliation did not project and repair a missing host.'); $retirement = CloudRescueTemplateRetirementService::retire( $templateId, 1, $generation, $actor ); $assert((string) CloudRescueTemplate::findOrFail($templateId)->state === 'retiring' && (int) $retirement['cleanup_debt'] === 2 && count($retirement['tasks']) === 2 && (string) CloudRescueTemplateReplica::query() ->where('template_id', $templateId)->where('host_id', 33) ->value('state') === 'deleted', 'Logical retirement did not stop publication or resolve known absence.'); $deleteResult = [ 'resource_type' => 'rescue_template', 'id' => $templateId, 'deleted' => true, 'managed_image_absent' => true, 'managed_metadata_absent' => true, 'managed_artifacts_absent' => true, ]; foreach ($retirement['tasks'] as $task) { $job = CloudJob::findOrFail((int) $task['task_id']); CloudRescueTemplateRetirementService::finishJobLocked( $job, 'succeeded', $deleteResult, ['rescue_template_id' => $templateId], [ 'status' => 'succeeded', 'retryable' => false, 'result' => $deleteResult, 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, ] ); } $assert((string) CloudRescueTemplateArtifact::findOrFail($templateId)->state === 'delete_pending' && CloudRescueTemplateReplica::query() ->where('template_id', $templateId) ->where('state', '<>', 'deleted')->count() === 0, 'Canonical object cleanup was armed before all replicas proved absent.'); $deleteClient = new CloudObjectStorageS3Client(new Client([ 'handler' => HandlerStack::create(new MockHandler([ $head(), new Response(204), new Response(404), ])), ])); $cleanup = CloudRescueTemplateRetirementService::processDue( $actor, 8, $deleteClient ); $assert((int) $cleanup['completed'] === 1 && (string) CloudRescueTemplate::findOrFail($templateId)->state === 'retired' && (string) CloudRescueTemplateArtifact::findOrFail($templateId)->state === 'deleted' && CloudRescueTemplateReplica::query() ->where('template_id', $templateId)->count() === 3, 'Exact object deletion did not retain a settled retirement tombstone.'); $unknownId = str_repeat('d', 32); $unknownGeneration = str_repeat('e', 32); Capsule::table('cloud_rescue_templates')->insert([ 'id' => $unknownId, 'name' => 'Unknown delete rescue', 'size_bytes' => $bytes, 'sha256' => $sha256, 'revision' => 1, 'generation_token' => $unknownGeneration, 'state' => 'active', 'active_job_id' => null, 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_rescue_template_artifacts')->insert([ 'template_id' => $unknownId, 'storage_revision' => 1, 'sealed_storage_runtime' => CloudSecrets::encrypt( $runtime, 'rescue-artifact-runtime' ), 'object_key' => 'virt/rescue-isos/' . $unknownId . '/' . $unknownGeneration . '/rescue.iso', 'object_version_id' => 'version-1', 'object_etag' => '"etag-1"', 'size_bytes' => $bytes, 'sha256' => $sha256, 'state' => 'ready', 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_rescue_template_replicas')->insert([ 'host_id' => 31, 'template_id' => $unknownId, 'revision' => 1, 'generation_token' => $unknownGeneration, 'state' => 'active', 'size_bytes' => $bytes, 'sha256' => $sha256, 'operation_id' => '', 'job_id' => null, 'error_code' => '', 'observed_state' => 'exact', 'observed_revision' => 1, 'observed_generation_token' => $unknownGeneration, 'observed_size_bytes' => $bytes, 'observed_sha256' => $sha256, 'last_verified_at' => $now, 'cleanup_resolved_at' => null, 'created_at' => $now, 'updated_at' => $now, ]); $unknownRetirement = CloudRescueTemplateRetirementService::retire( $unknownId, 1, $unknownGeneration, $actor ); $unknownJob = CloudJob::findOrFail( (int) $unknownRetirement['tasks'][0]['task_id'] ); CloudRescueTemplateRetirementService::finishJobLocked( $unknownJob, 'failed', null, ['rescue_template_id' => $unknownId], [ 'effect_may_have_applied' => true, 'safe_to_resubmit' => false, 'cause_code' => 'agent_response_lost', ] ); $debt = CloudRescueTemplateRetirementService::cleanupState($unknownId); $assert(($debt['replica_states']['delete_unknown'] ?? 0) === 1 && (string) CloudRescueTemplateArtifact::findOrFail($unknownId)->state === 'ready', 'Unknown host deletion did not retain cleanup debt and the object.'); $absenceReconciler = new CloudRescueTemplateReconciliationService( static fn (int $hostId): object => new class { public function request( string $method, string $path, array $actor ): array { return ['templates' => []]; } }, static function (): array { throw new RuntimeException( 'Retiring rescue ISO attempted an import repair.' ); } ); $absenceReport = $absenceReconciler->reconcile($actor, 3, 3, 0); $assert($absenceReport['scheduled'] === 0 && (string) CloudRescueTemplateReplica::query() ->where('template_id', $unknownId)->where('host_id', 31) ->value('state') === 'deleted', 'Trusted host absence did not resolve unknown deletion debt.'); $unknownDeleteClient = new CloudObjectStorageS3Client(new Client([ 'handler' => HandlerStack::create(new MockHandler([ $head(), new Response(204), new Response(404), ])), ])); $unknownCleanup = CloudRescueTemplateRetirementService::processDue( $actor, 8, $unknownDeleteClient ); $assert((int) $unknownCleanup['completed'] === 1 && (string) CloudRescueTemplate::findOrFail($unknownId)->state === 'retired', 'Reconciled host absence did not release exact object cleanup.'); echo "Rescue replica validation, distribution and reconciliation passed.\n"; }