testedNetworkSelectors(); } public static function clientForConfiguration( int $hostId, float $timeoutSeconds = 120.0 ): never { self::$requests++; throw new \RuntimeException( 'Aggregate recovery must not contact an Agent.' ); } } } namespace { require dirname(__DIR__) . '/vendor/autoload.php'; use App\Models\CloudJob; use App\Services\Cloud\CloudAgentSettingsService; use App\Services\Cloud\CloudAutomaticOperationRecoveryService; use App\Services\Cloud\CloudError; use App\Services\Cloud\CloudJobService; use App\Services\Cloud\CloudOperationCompletionService; use App\Services\Cloud\CloudOperationRecoveryIdentity; use App\Services\Cloud\CloudRecoveryAggregateStateConflict; use App\Services\Cloud\CloudRecoveryInstanceStateConflict; use App\Services\Cloud\CloudRecoveryOperationIdentityConflict; use App\Services\Cloud\CloudSecrets; use App\Services\Cloud\CloudTemplateCatalogService; use App\Services\Cloud\CloudUserIsoService; use App\Services\Cloud\CloudWorkerService; use Illuminate\Database\Capsule\Manager as Capsule; use Illuminate\Database\Schema\Blueprint; $assert = static function (bool $condition, string $message): void { if (!$condition) { throw new RuntimeException($message); } }; $automaticSource = file_get_contents( dirname(__DIR__) . '/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php' ); $lockMethodStart = is_string($automaticSource) ? strpos($automaticSource, 'private static function resolveFenceAndCommit(') : false; $lockMethodEnd = $lockMethodStart === false ? false : strpos( $automaticSource, 'private static function assertFreshInspection(', $lockMethodStart ); $lockMethod = $lockMethodStart === false || $lockMethodEnd === false ? '' : substr( $automaticSource, $lockMethodStart, $lockMethodEnd - $lockMethodStart ); $parentLock = strpos( $lockMethod, '$aggregateParent = CloudJob::where(' ); $catalogLock = strpos( $lockMethod, 'CloudTemplateCatalogService::lockAggregateCatalogLocked(' ); $leafLock = strpos( $lockMethod, '$aggregateChildren = CloudJob::where(' ); $assert(str_contains($lockMethod, "->value('parent_job_id')") && $parentLock !== false && $catalogLock !== false && $leafLock !== false && $parentLock < $catalogLock && $catalogLock < $leafLock && str_contains( $lockMethod, "(int) (\$job->parent_job_id ?? 0)\n" . ' !== $aggregateParentId' ), 'Fence commit does not preserve aggregate parent-before-child lock order.'); $capsule = new Capsule(); $capsule->addConnection([ 'driver' => 'sqlite', 'database' => ':memory:', ], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('user', static function (Blueprint $table): void { $table->increments('id'); $table->integer('enable')->default(1); $table->integer('is_admin')->default(0); }); $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('operation_id')->unique(); $table->string('request_idempotency_key')->unique(); $table->string('request_fingerprint'); $table->integer('parent_job_id')->nullable(); $table->integer('order_id')->nullable(); $table->integer('invoice_id')->nullable(); $table->integer('instance_id')->nullable(); $table->integer('snapshot_id')->nullable(); $table->integer('public_network_transition_id')->nullable(); $table->integer('user_id'); $table->integer('host_id')->nullable(); $table->string('kind'); $table->string('execution_mode'); $table->string('phase'); $table->string('outcome')->nullable(); $table->string('status'); $table->text('sealed_payload'); $table->integer('attempts')->default(0); $table->integer('dispatch_attempts')->default(0); $table->integer('dispatch_deadline_at'); $table->integer('dispatch_started_at')->nullable(); $table->integer('agent_accepted_at')->nullable(); $table->integer('receipt_deadline_at')->nullable(); $table->integer('terminal_at')->nullable(); $table->boolean('effect_may_have_applied')->nullable(); $table->boolean('safe_to_resubmit')->nullable(); $table->string('effect_resolution'); $table->integer('effect_resolved_at')->nullable(); $table->integer('effect_resolved_by')->nullable(); $table->string('effect_resolution_reason')->default(''); $table->string('completion_event_id')->nullable(); $table->integer('read_at')->nullable(); $table->integer('dispatched_at')->nullable(); $table->string('lease_token')->nullable(); $table->integer('lease_until')->default(0); $table->integer('available_at')->default(0); $table->string('error_code')->nullable(); $table->boolean('finalization_incident')->default(0); $table->string('finalization_error_code')->nullable(); $table->text('result_json')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_backup_restore_attempts', static function (Blueprint $table): void { $table->increments('id'); $table->integer('job_id'); $table->string('state'); $table->string('effect_resolution'); $table->integer('active_attempt')->nullable(); $table->string('last_error_code')->default(''); }); $schema->create('cloud_public_network_transitions', static function (Blueprint $table): void { $table->increments('id'); $table->string('state'); $table->string('effect_resolution'); $table->string('last_error_code')->default(''); }); $schema->create('cloud_agent_completion_events', static function (Blueprint $table): void { $table->string('event_id')->primary(); $table->integer('host_id'); $table->string('operation_id'); $table->string('payload_sha256'); $table->string('outcome'); $table->boolean('effect_may_have_applied'); $table->boolean('safe_to_resubmit'); $table->string('error_code')->default(''); $table->text('result_json')->nullable(); $table->string('disposition')->default('received'); $table->integer('received_at'); $table->integer('applied_at')->nullable(); $table->unique(['host_id', 'operation_id']); }); $schema->create('cloud_orders', static function (Blueprint $table): void { $table->increments('id'); $table->string('status'); $table->text('spec_json'); }); $schema->create('cloud_templates', static function (Blueprint $table): void { $table->string('id')->primary(); $table->string('catalog_key')->default('test-catalog'); $table->string('version_label')->default('test-version'); $table->boolean('is_default')->default(true); $table->string('name'); $table->string('os_family'); $table->string('os_version'); $table->integer('image_bytes'); $table->string('sha256'); $table->text('startup_script'); $table->integer('revision'); $table->string('generation_token'); $table->string('state'); $table->integer('active_job_id')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_template_import_intents', static function (Blueprint $table): void { $table->string('id')->primary(); $table->integer('user_id'); $table->string('request_idempotency_key'); $table->string('request_fingerprint'); $table->string('source_kind'); $table->string('name'); $table->string('os_family'); $table->string('os_version'); $table->string('source_filename')->default(''); $table->integer('source_size_bytes')->default(0); $table->integer('source_last_modified')->default(0); $table->string('source_sha256')->default(''); $table->text('sealed_source'); $table->string('template_id')->unique(); $table->string('catalog_key'); $table->string('version_label'); $table->string('generation_token')->unique(); $table->integer('storage_revision'); $table->text('sealed_storage_runtime'); $table->string('object_key')->unique(); $table->string('multipart_upload_id')->default(''); $table->integer('multipart_part_size_bytes')->default(0); $table->integer('multipart_part_count')->default(0); $table->string('object_version_id')->default(''); $table->string('object_etag')->default(''); $table->string('state')->default('prepared'); $table->integer('active_job_id')->nullable(); $table->string('error_code')->default(''); $table->integer('expires_at'); $table->integer('completed_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); $table->unique(['user_id', 'request_idempotency_key']); }); $schema->create('cloud_template_import_parts', static function (Blueprint $table): void { $table->string('intent_id'); $table->integer('part_number'); $table->integer('size_bytes'); $table->string('etag'); $table->integer('created_at'); $table->integer('updated_at'); $table->primary(['intent_id', 'part_number']); }); $schema->create('cloud_template_artifacts', static function (Blueprint $table): void { $table->string('template_id')->primary(); $table->integer('storage_revision')->default(1); $table->text('sealed_storage_runtime'); $table->string('object_key'); $table->string('object_version_id')->default(''); $table->string('object_etag')->default('"test-etag"'); $table->integer('size_bytes'); $table->string('sha256'); $table->string('state')->default('ready'); $table->string('error_code')->default(''); $table->string('delete_operation_id')->default(''); $table->integer('delete_generation')->default(0); $table->integer('delete_cycle_attempts')->default(0); $table->integer('delete_presence_confirmations')->default(0); $table->integer('delete_last_checked_at')->default(0); $table->integer('delete_next_check_at')->default(0); $table->integer('delete_attention_at')->nullable(); $table->integer('verified_at')->default(0); $table->integer('created_at'); $table->integer('updated_at'); }); Capsule::connection('default')->statement(<<<'SQL' CREATE TRIGGER cloud_template_artifact_fixture AFTER INSERT ON cloud_templates BEGIN INSERT INTO cloud_template_artifacts ( template_id, storage_revision, sealed_storage_runtime, object_key, object_version_id, object_etag, size_bytes, sha256, state, error_code, delete_operation_id, delete_generation, delete_cycle_attempts, delete_presence_confirmations, delete_last_checked_at, delete_next_check_at, delete_attention_at, verified_at, created_at, updated_at ) VALUES ( NEW.id, 1, 'test-only', 'test/' || NEW.id, '', '"test-etag"', NEW.image_bytes, NEW.sha256, 'ready', '', '', 0, 0, 0, 0, 0, NULL, NEW.created_at, NEW.created_at, NEW.updated_at ); END SQL); $schema->create('cloud_template_replicas', static function (Blueprint $table): void { $table->integer('host_id'); $table->string('template_id'); $table->integer('revision'); $table->string('generation_token'); $table->string('state'); $table->integer('image_bytes'); $table->string('sha256'); $table->string('operation_id'); $table->integer('job_id')->nullable(); $table->string('error_code'); $table->integer('created_at'); $table->integer('updated_at'); $table->primary(['host_id', 'template_id']); }); $schema->create('configurations', static function (Blueprint $table): void { $table->increments('id'); $table->string('name')->unique(); $table->text('value'); }); $schema->create('shop', static function (Blueprint $table): void { $table->increments('id'); $table->integer('status'); }); $schema->create('cloud_regions', static function (Blueprint $table): void { $table->increments('id'); $table->string('status'); }); $schema->create('cloud_network_lines', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('region_id'); $table->integer('is_default'); $table->string('status'); }); $schema->create('cloud_resource_pools', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('line_id'); $table->string('status'); }); $schema->create('cloud_hosts', static function (Blueprint $table): void { $table->increments('id'); $table->integer('enabled'); $table->string('default_network'); }); $schema->create('cloud_host_agents', static function ( Blueprint $table ): void { $table->integer('host_id')->primary(); $table->integer('enabled'); $table->integer('revision'); $table->integer('tested_revision'); $table->string('last_test_status'); $table->text('network_selectors_json')->default('[]'); }); $schema->create('cloud_host_storage_configurations', static function ( Blueprint $table ): void { $table->integer('host_id')->primary(); $table->string('state'); }); $schema->create('cloud_host_resource_pool_memberships', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('host_id'); $table->integer('resource_pool_id'); $table->string('scheduler_state'); $table->integer('revision')->default(1); }); $schema->create('cloud_host_capacity_policies', static function ( Blueprint $table ): void { $table->integer('host_id')->primary(); $table->integer('configured_cpu_cores'); $table->integer('configured_memory_mib'); $table->integer('configured_disk_gib'); }); $schema->create('cloud_host_network_attachments', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('host_id'); $table->integer('line_id'); $table->string('network_selector'); $table->string('state'); $table->string('verification_state'); $table->integer('verified_agent_revision'); $table->integer('last_verified_at'); }); $schema->create('cloud_public_ip_pools', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('pool_key'); $table->integer('host_id'); $table->integer('network_attachment_id'); $table->string('cidr'); $table->string('gateway'); $table->integer('guest_prefix_length'); $table->text('dns_servers_json'); $table->text('reserved_ips_json'); $table->string('state'); $table->string('sync_state'); $table->integer('desired_revision'); $table->integer('observed_revision'); }); $schema->create('cloud_public_ips', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('host_id'); $table->integer('public_ip_pool_id'); $table->string('address'); $table->integer('prefix_length'); $table->string('state'); $table->integer('reservation_id')->nullable(); $table->integer('instance_id')->nullable(); $table->string('operation_id')->nullable(); }); $schema->create('cloud_offers', static function (Blueprint $table): void { $table->increments('id'); $table->integer('plan_id'); $table->integer('resource_pool_id'); $table->integer('status'); $table->string('currency'); }); $schema->create('cloud_user_isos', static function (Blueprint $table): void { $table->string('id')->primary(); $table->integer('user_id'); $table->string('name'); $table->text('sealed_source_url'); $table->integer('size_bytes')->default(0); $table->string('sha256')->default(''); $table->string('state')->default('importing'); $table->string('import_operation_id'); $table->integer('active_job_id')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_user_iso_replicas', static function (Blueprint $table): void { $table->integer('host_id'); $table->string('user_iso_id'); $table->integer('user_id'); $table->string('state')->default('pending'); $table->integer('size_bytes')->default(0); $table->string('sha256')->default(''); $table->string('operation_id'); $table->integer('job_id')->nullable(); $table->string('error_code')->default(''); $table->integer('created_at'); $table->integer('updated_at'); $table->primary(['host_id', 'user_iso_id']); }); Capsule::table('user')->insert([ 'id' => 1, 'enable' => 1, 'is_admin' => 0, ]); Capsule::table('configurations')->insert([ 'name' => 'cloud_network_lines_enabled', 'value' => 'false', ]); Capsule::table('shop')->insert(['id' => 1, 'status' => 1]); Capsule::table('cloud_regions')->insert([ 'id' => 1, 'status' => 'active', ]); Capsule::table('cloud_network_lines')->insert([ 'id' => 1, 'region_id' => 1, 'is_default' => 1, 'status' => 'active', ]); Capsule::table('cloud_resource_pools')->insert([ 'id' => 1, 'line_id' => 1, 'status' => 'active', ]); Capsule::table('cloud_offers')->insert([ 'id' => 1, 'plan_id' => 1, 'resource_pool_id' => 1, 'status' => 1, 'currency' => 'USD', ]); foreach ([7, 8] as $hostId) { Capsule::table('cloud_hosts')->insert([ 'id' => $hostId, 'enabled' => 1, 'default_network' => 'bridge:br0', ]); Capsule::table('cloud_host_agents')->insert([ 'host_id' => $hostId, 'enabled' => 1, 'revision' => 1, 'tested_revision' => 1, 'last_test_status' => 'healthy', 'network_selectors_json' => json_encode( ['bridge:public-' . $hostId], JSON_THROW_ON_ERROR ), ]); Capsule::table('cloud_host_storage_configurations')->insert([ 'host_id' => $hostId, 'state' => 'applied', ]); Capsule::table('cloud_host_resource_pool_memberships')->insert([ 'host_id' => $hostId, 'resource_pool_id' => 1, 'scheduler_state' => 'active', ]); Capsule::table('cloud_host_capacity_policies')->insert([ 'host_id' => $hostId, 'configured_cpu_cores' => 16, 'configured_memory_mib' => 32768, 'configured_disk_gib' => 512, ]); $attachmentId = 100 + $hostId; Capsule::table('cloud_host_network_attachments')->insert([ 'id' => $attachmentId, 'host_id' => $hostId, 'line_id' => 1, 'network_selector' => 'bridge:public-' . $hostId, 'state' => 'active', 'verification_state' => 'verified', 'verified_agent_revision' => 1, 'last_verified_at' => time(), ]); $publicPoolId = 200 + $hostId; Capsule::table('cloud_public_ip_pools')->insert([ 'id' => $publicPoolId, 'pool_key' => str_repeat((string) $hostId, 32), 'host_id' => $hostId, 'network_attachment_id' => $attachmentId, 'cidr' => '203.0.11' . $hostId . '.0/29', 'gateway' => '203.0.11' . $hostId . '.1', 'guest_prefix_length' => 29, 'dns_servers_json' => '[]', 'reserved_ips_json' => '[]', 'state' => 'active', 'sync_state' => 'applied', 'desired_revision' => 1, 'observed_revision' => 1, ]); Capsule::table('cloud_public_ips')->insert([ 'host_id' => $hostId, 'public_ip_pool_id' => $publicPoolId, 'address' => '203.0.11' . $hostId . '.2', 'prefix_length' => 29, 'state' => 'available', 'reservation_id' => null, 'instance_id' => null, 'operation_id' => null, ]); } $now = time(); $nextId = 1; $insertParent = static function ( string $kind = 'template_import', int $userId = 1 ) use (&$nextId, $now): int { $id = $nextId++; $operationId = md5('aggregate-parent:' . $id); Capsule::table('cloud_jobs')->insert([ 'id' => $id, 'operation_id' => $operationId, 'request_idempotency_key' => $operationId, 'request_fingerprint' => hash('sha256', $operationId), 'parent_job_id' => null, 'order_id' => null, 'invoice_id' => null, 'instance_id' => null, 'snapshot_id' => null, 'user_id' => $userId, 'host_id' => null, 'kind' => $kind, 'execution_mode' => 'aggregate', 'phase' => 'terminal', 'outcome' => 'failed', 'status' => 'failed', 'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'), 'attempts' => 0, 'dispatch_attempts' => 0, 'dispatch_deadline_at' => $now - 600, 'dispatch_started_at' => null, 'agent_accepted_at' => null, 'receipt_deadline_at' => $now - 1, 'terminal_at' => $now - 1, 'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0, 'effect_resolution' => 'unknown', 'effect_resolved_at' => null, 'effect_resolved_by' => null, 'effect_resolution_reason' => '', 'completion_event_id' => null, 'read_at' => $now - 1, 'dispatched_at' => null, 'lease_token' => str_repeat('a', 32), 'lease_until' => $now + 300, 'available_at' => $now + 300, 'error_code' => 'aggregate_deadline_exceeded', 'finalization_incident' => 1, 'finalization_error_code' => 'recovery_child_convergence', 'result_json' => null, 'created_at' => $now - 900, 'updated_at' => $now - 1, ]); return $id; }; $insertResolvedChild = static function ( int $parentId, string $resolution, string $kind = 'action', int $userId = 1, string $executionMode = 'agent' ) use (&$nextId, $now): int { $id = $nextId++; $operationId = md5('resolved-child:' . $id); Capsule::table('cloud_jobs')->insert([ 'id' => $id, 'operation_id' => $operationId, 'request_idempotency_key' => $operationId, 'request_fingerprint' => hash('sha256', $operationId), 'parent_job_id' => $parentId, 'order_id' => null, 'invoice_id' => null, 'instance_id' => null, 'snapshot_id' => null, 'user_id' => $userId, 'host_id' => 7, 'kind' => $kind, 'execution_mode' => $executionMode, 'phase' => 'terminal', 'outcome' => $resolution === 'applied' ? 'succeeded' : 'failed', 'status' => $resolution === 'applied' ? 'succeeded' : 'failed', 'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'), 'attempts' => 1, 'dispatch_attempts' => 1, 'dispatch_deadline_at' => $now - 600, 'dispatch_started_at' => $now - 500, 'agent_accepted_at' => $now - 490, 'receipt_deadline_at' => $now - 1, 'terminal_at' => $now - 1, 'effect_may_have_applied' => 0, 'safe_to_resubmit' => $resolution === 'not_applied' ? 1 : 0, 'effect_resolution' => $resolution, 'effect_resolved_at' => $now - 1, 'effect_resolved_by' => null, 'effect_resolution_reason' => 'normal_completion', 'completion_event_id' => null, 'read_at' => null, 'dispatched_at' => $now - 490, 'lease_token' => null, 'lease_until' => 0, 'available_at' => 0, 'error_code' => $resolution === 'not_applied' ? 'operation_rejected' : null, 'finalization_incident' => 0, 'finalization_error_code' => null, 'result_json' => null, 'created_at' => $now - 600, 'updated_at' => $now - 1, ]); return $id; }; $insertProjectionForParent = static function (int $parentId) use ($now): string { $parent = CloudJob::findOrFail($parentId); $children = CloudJob::where('parent_job_id', $parentId) ->orderBy('id')->get(); if ($children->isEmpty()) { throw new RuntimeException('Projection fixture requires children.'); } foreach ($children as $index => $child) { $hostId = 1000 + (int) $child->id; $networkSelector = 'bridge:b' . $hostId; Capsule::table('cloud_jobs')->where('id', (int) $child->id) ->update(['host_id' => $hostId]); Capsule::table('cloud_hosts')->insertOrIgnore([ 'id' => $hostId, 'enabled' => 1, 'default_network' => 'bridge:br0', ]); Capsule::table('cloud_host_agents')->insertOrIgnore([ 'host_id' => $hostId, 'enabled' => 1, 'revision' => 1, 'tested_revision' => 1, 'last_test_status' => 'healthy', 'network_selectors_json' => json_encode( [$networkSelector], JSON_THROW_ON_ERROR ), ]); Capsule::table('cloud_host_storage_configurations') ->insertOrIgnore([ 'host_id' => $hostId, 'state' => 'applied', ]); Capsule::table('cloud_host_resource_pool_memberships') ->insertOrIgnore([ 'host_id' => $hostId, 'resource_pool_id' => 1, 'scheduler_state' => 'active', ]); Capsule::table('cloud_host_capacity_policies')->insertOrIgnore([ 'host_id' => $hostId, 'configured_cpu_cores' => 16, 'configured_memory_mib' => 32768, 'configured_disk_gib' => 512, ]); $attachmentId = 100000 + $hostId; Capsule::table('cloud_host_network_attachments') ->insertOrIgnore([ 'id' => $attachmentId, 'host_id' => $hostId, 'line_id' => 1, 'network_selector' => $networkSelector, 'state' => 'active', 'verification_state' => 'verified', 'verified_agent_revision' => 1, 'last_verified_at' => $now, ]); $publicPoolId = 200000 + $hostId; $thirdOctet = intdiv($hostId, 30) % 250; $networkOctet = ($hostId % 30) * 8; $networkPrefix = '198.18.' . $thirdOctet . '.'; Capsule::table('cloud_public_ip_pools')->insertOrIgnore([ 'id' => $publicPoolId, 'pool_key' => md5('aggregate-network-pool:' . $hostId), 'host_id' => $hostId, 'network_attachment_id' => $attachmentId, 'cidr' => $networkPrefix . $networkOctet . '/29', 'gateway' => $networkPrefix . ($networkOctet + 1), 'guest_prefix_length' => 29, 'dns_servers_json' => '[]', 'reserved_ips_json' => '[]', 'state' => 'active', 'sync_state' => 'applied', 'desired_revision' => 1, 'observed_revision' => 1, ]); Capsule::table('cloud_public_ips')->insertOrIgnore([ 'host_id' => $hostId, 'public_ip_pool_id' => $publicPoolId, 'address' => $networkPrefix . ($networkOctet + 2), 'prefix_length' => 29, 'state' => 'available', 'reservation_id' => null, 'instance_id' => null, 'operation_id' => null, ]); $child->host_id = $hostId; } if (str_starts_with((string) $parent->kind, 'template_')) { $resourceId = 'aggregate-projection-' . $parentId; $generation = md5('projection-generation:' . $parentId); $sha256 = hash('sha256', 'projection-sha:' . $parentId); $state = match ((string) $parent->kind) { 'template_import' => 'degraded', 'template_metadata_sync' => 'active', 'template_delete' => 'delete_failed', }; Capsule::table('cloud_templates')->insert([ 'id' => $resourceId, 'name' => 'Aggregate projection ' . $parentId, 'os_family' => 'Debian', 'os_version' => '13', 'image_bytes' => 4096, 'sha256' => $sha256, 'startup_script' => '', 'revision' => (string) $parent->kind === 'template_metadata_sync' ? 2 : 1, 'generation_token' => $generation, 'state' => $state, 'active_job_id' => $parentId, 'created_at' => $now - 60, 'updated_at' => $now, ]); foreach ($children as $child) { $applied = (string) $child->effect_resolution === 'applied'; $replicaState = match ((string) $parent->kind) { 'template_import' => $applied ? 'active' : 'failed', 'template_metadata_sync' => $applied ? 'active' : 'stale', 'template_delete' => $applied ? 'delete_succeeded' : 'delete_failed', }; Capsule::table('cloud_template_replicas')->insert([ 'host_id' => (int) $child->host_id, 'template_id' => $resourceId, 'revision' => (string) $parent->kind === 'template_metadata_sync' ? ($applied ? 2 : 1) : 1, 'generation_token' => $generation, 'state' => $replicaState, 'image_bytes' => (string) $parent->kind === 'template_import' && !$applied ? 0 : 4096, 'sha256' => $sha256, 'operation_id' => (string) $child->operation_id, 'job_id' => (int) $child->id, 'error_code' => $applied ? '' : 'operation_not_applied', 'created_at' => $now - 60, 'updated_at' => $now, ]); } return $resourceId; } $resourceId = md5('aggregate-iso-projection:' . $parentId); $sha256 = hash('sha256', 'projection-iso-sha:' . $parentId); Capsule::table('cloud_user_isos')->insert([ 'id' => $resourceId, 'user_id' => (int) $parent->user_id, 'name' => 'Aggregate ISO ' . $parentId, 'sealed_source_url' => 'test-only', 'size_bytes' => 4096, 'sha256' => $sha256, 'state' => (string) $parent->kind === 'user_iso_import' ? 'import_failed' : 'delete_failed', 'import_operation_id' => (string) $parent->operation_id, 'active_job_id' => $parentId, 'created_at' => $now - 60, 'updated_at' => $now, ]); foreach ($children as $child) { $applied = (string) $child->effect_resolution === 'applied'; $replicaState = (string) $parent->kind === 'user_iso_import' ? ($applied ? 'active' : 'failed') : ($applied ? 'delete_succeeded' : 'delete_failed'); Capsule::table('cloud_user_iso_replicas')->insert([ 'host_id' => (int) $child->host_id, 'user_iso_id' => $resourceId, 'user_id' => (int) $parent->user_id, 'state' => $replicaState, 'size_bytes' => (string) $parent->kind === 'user_iso_delete' || $applied ? 4096 : 0, 'sha256' => $sha256, 'operation_id' => (string) $child->operation_id, 'job_id' => (int) $child->id, 'error_code' => $applied ? '' : 'operation_not_applied', 'created_at' => $now - 60, 'updated_at' => $now, ]); } return $resourceId; }; $settleLastChild = static function ( string $firstResolution ) use ( $insertParent, $insertResolvedChild, $insertProjectionForParent, $assert ): CloudJob { $parentId = $insertParent(); $insertResolvedChild( $parentId, $firstResolution, 'template_import_replica' ); $lastChildId = $insertResolvedChild( $parentId, 'not_applied', 'template_import_replica' ); $insertProjectionForParent($parentId); Capsule::table('cloud_jobs')->where('id', $lastChildId)->update([ 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolution_reason' => '', 'terminal_at' => null, ]); $before = CloudJob::find($parentId); $assert($before !== null && CloudJobService::recoveryPresentation($before, true) !== null, 'The aggregate fixture was not fenced before its last child.'); Capsule::table('cloud_jobs')->where('id', $lastChildId)->update([ 'phase' => 'terminal', 'outcome' => 'failed', 'status' => 'failed', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 1, 'effect_resolution' => 'not_applied', 'effect_resolved_at' => time(), 'effect_resolution_reason' => 'normal_completion', 'terminal_at' => time(), ]); $completed = CloudJob::find($lastChildId); CloudJobService::reconcileAggregateParentEffectLocked( $completed, null ); $assert((string) $completed->phase === 'terminal' && (string) $completed->outcome === 'failed' && (string) $completed->effect_resolution === 'not_applied', 'The last leaf did not enter its normal terminal state.'); return CloudJob::find($parentId); }; $allAbsent = $settleLastChild('not_applied'); $assert($allAbsent !== null && (string) $allAbsent->phase === 'terminal' && (string) $allAbsent->outcome === 'failed' && (string) $allAbsent->status === 'failed' && (string) $allAbsent->effect_resolution === 'not_applied' && (int) $allAbsent->effect_resolved_at > 0 && $allAbsent->effect_resolved_by === null && str_starts_with( (string) $allAbsent->effect_resolution_reason, 'child_effects_resolved:' ) && !(bool) $allAbsent->finalization_incident && $allAbsent->finalization_error_code === null && $allAbsent->lease_token === null && (int) $allAbsent->lease_until === 0 && (int) $allAbsent->available_at === 0 && $allAbsent->read_at === null && (string) Capsule::table('cloud_templates') ->where('active_job_id', (int) $allAbsent->id) ->value('state') === 'retired' && !Capsule::table('cloud_template_replicas') ->whereIn('template_id', static function ($query) use ($allAbsent) { $query->select('id')->from('cloud_templates') ->where('active_job_id', (int) $allAbsent->id); })->exists() && CloudJobService::recoveryPresentation($allAbsent, true) === null, 'All-not-applied import children did not retire their placeholder.'); $partial = $settleLastChild('applied'); $partialTemplateId = 'aggregate-projection-' . (int) $partial->id; $partialTemplate = Capsule::table('cloud_templates') ->where('id', $partialTemplateId)->first(); $partialReplicaStates = Capsule::table('cloud_template_replicas') ->where('template_id', $partialTemplateId)->orderBy('state') ->pluck('state')->all(); $partialTemplateDto = collect( CloudTemplateCatalogService::adminCatalog()['templates'] )->firstWhere('id', $partialTemplateId); $partialDisplay = CloudJobService::display($partial); $assert($partial !== null && (string) $partial->phase === 'terminal' && (string) $partial->outcome === 'failed' && (string) $partial->status === 'failed' && (string) $partial->effect_resolution === 'applied' && str_ends_with( (string) $partial->effect_resolution_reason, ':mixed' ) && !(bool) $partial->finalization_incident && $partial->finalization_error_code === null && $partial->lease_token === null && (int) $partial->available_at === 0 && $partialTemplate !== null && (string) $partialTemplate->state === 'degraded' && $partialTemplate->active_job_id === null && $partialReplicaStates === ['active', 'failed'] && is_array($partialTemplateDto) && ($partialTemplateDto['cleanup_reason'] ?? null) === 'partial_import' && ($partialDisplay['effect_scope'] ?? null) === 'partial' && CloudJobService::recoveryPresentation($partial, true) === null, 'A proven partial template import did not settle as failed-but-applied.'); $partialAppliedHost = (int) Capsule::table('cloud_template_replicas') ->where('template_id', $partialTemplateId) ->where('state', 'active')->value('host_id'); $partialOriginalOperations = CloudJob::where( 'parent_job_id', (int) $partial->id )->pluck('operation_id')->all(); $partialCleanup = CloudTemplateCatalogService::delete( $partialTemplateId, (int) $partialTemplate->revision, (string) $partialTemplate->generation_token, [ 'user_id' => 1, 'is_admin' => true, 'is_super_admin' => true, 'can_manage' => true, ] ); $partialCleanupId = (int) ($partialCleanup['task_id'] ?? 0); $partialCleanupParent = CloudJob::findOrFail($partialCleanupId); $partialCleanupChildren = CloudJob::where( 'parent_job_id', $partialCleanupId )->get(); $partialCleanupTemplate = Capsule::table('cloud_templates') ->where('id', $partialTemplateId)->first(); $partialCleanupReplicas = Capsule::table('cloud_template_replicas') ->where('template_id', $partialTemplateId)->get(); $assert($partialCleanupId > 0 && $partialCleanupId !== (int) $partial->id && (string) $partialCleanupParent->kind === 'template_delete' && (string) $partialCleanupParent->phase === 'awaiting_receipt' && $partialCleanupTemplate !== null && (string) $partialCleanupTemplate->state === 'deleting' && (int) $partialCleanupTemplate->active_job_id === $partialCleanupId && $partialCleanupChildren->count() === 1 && (int) $partialCleanupChildren->first()->host_id === $partialAppliedHost && !in_array( (string) $partialCleanupChildren->first()->operation_id, $partialOriginalOperations, true ) && $partialCleanupReplicas->count() === 1 && (string) $partialCleanupReplicas->first()->state === 'deleting', 'Partial template import could not start an independent cleanup.'); $nextId = max( $nextId, (int) Capsule::table('cloud_jobs')->max('id') + 1 ); $startMixedCleanup = static function ( string $parentKind, string $childKind ) use ( $insertParent, $insertResolvedChild, $insertProjectionForParent, $assert, &$nextId ): array { $parentId = $insertParent($parentKind); $appliedChildId = $insertResolvedChild( $parentId, 'applied', $childKind ); $insertResolvedChild($parentId, 'not_applied', $childKind); $resourceId = $insertProjectionForParent($parentId); CloudJobService::reconcileAggregateParentEffectLocked( CloudJob::findOrFail($appliedChildId), null ); $parent = CloudJob::findOrFail($parentId); $validReason = (string) $parent->effect_resolution_reason; $jobsBeforeMalformed = CloudJob::count(); $isTemplate = str_starts_with($parentKind, 'template_'); $replicaTable = $isTemplate ? 'cloud_template_replicas' : 'cloud_user_iso_replicas'; $replicaKey = $isTemplate ? 'template_id' : 'user_iso_id'; $replicasBeforeMalformed = Capsule::table($replicaTable) ->where($replicaKey, $resourceId)->orderBy('host_id')->get() ->map(static fn ($row): array => (array) $row)->all(); Capsule::table('cloud_jobs')->where('id', $parentId)->update([ 'effect_resolution_reason' => 'garbage:mixed', ]); $malformedRejected = false; try { if ($isTemplate) { $malformedTemplate = Capsule::table('cloud_templates') ->where('id', $resourceId)->first(); CloudTemplateCatalogService::delete( $resourceId, (int) $malformedTemplate->revision, (string) $malformedTemplate->generation_token, [ 'user_id' => 1, 'is_admin' => true, 'is_super_admin' => true, 'can_manage' => true, ] ); } else { $malformedDto = collect( CloudUserIsoService::listing(1)['isos'] )->firstWhere('id', $resourceId); $assert(is_array($malformedDto) && ($malformedDto['can_delete'] ?? true) === false && !array_key_exists('cleanup_reason', $malformedDto), 'Malformed mixed ISO marker exposed cleanup in its DTO.'); CloudUserIsoService::delete( $resourceId, ['user_id' => 1] ); } } catch (CloudError $error) { $malformedRejected = $error->httpStatus === 409; } $replicasAfterMalformed = Capsule::table($replicaTable) ->where($replicaKey, $resourceId)->orderBy('host_id')->get() ->map(static fn ($row): array => (array) $row)->all(); $malformedPointer = $isTemplate ? Capsule::table('cloud_templates')->where('id', $resourceId) ->value('active_job_id') : Capsule::table('cloud_user_isos')->where('id', $resourceId) ->value('active_job_id'); $assert($malformedRejected && CloudJob::count() === $jobsBeforeMalformed && $malformedPointer === null && $replicasAfterMalformed === $replicasBeforeMalformed, 'Malformed mixed marker crossed the cleanup mutation boundary: ' . $parentKind); Capsule::table('cloud_jobs')->where('id', $parentId)->update([ 'effect_resolution_reason' => $validReason, ]); $parent = CloudJob::findOrFail($parentId); $oldOperations = CloudJob::where('parent_job_id', $parentId) ->pluck('operation_id')->all(); if ($isTemplate) { $resource = Capsule::table('cloud_templates') ->where('id', $resourceId)->first(); $preCleanupDto = null; $cleanup = CloudTemplateCatalogService::delete( $resourceId, (int) $resource->revision, (string) $resource->generation_token, [ 'user_id' => 1, 'is_admin' => true, 'is_super_admin' => true, 'can_manage' => true, ] ); } else { $resource = Capsule::table('cloud_user_isos') ->where('id', $resourceId)->first(); $preCleanupDto = collect( CloudUserIsoService::listing(1)['isos'] )->firstWhere('id', $resourceId); $cleanup = CloudUserIsoService::delete( $resourceId, ['user_id' => 1] ); } $cleanupId = (int) ($cleanup['task_id'] ?? 0); $children = CloudJob::where('parent_job_id', $cleanupId)->get(); $nextId = max( $nextId, (int) Capsule::table('cloud_jobs')->max('id') + 1 ); return [ $parent, $resource, $resourceId, $cleanupId, $children, $oldOperations, $preCleanupDto, ]; }; [$partialDelete, $partialDeleteTemplate, $partialDeleteTemplateId, $partialDeleteCleanupId, $partialDeleteCleanupChildren, $partialDeleteOldOperations] = $startMixedCleanup( 'template_delete', 'template_delete_replica' ); $partialDeleteReplicas = Capsule::table('cloud_template_replicas') ->where('template_id', $partialDeleteTemplateId)->get(); $assert((string) $partialDelete->effect_resolution === 'applied' && str_ends_with( (string) $partialDelete->effect_resolution_reason, ':mixed' ) && (string) $partialDeleteTemplate->state === 'delete_failed' && $partialDeleteTemplate->active_job_id === null && $partialDeleteCleanupId > 0 && $partialDeleteCleanupId !== (int) $partialDelete->id && $partialDeleteCleanupChildren->count() === 1 && !in_array( (string) $partialDeleteCleanupChildren->first()->operation_id, $partialDeleteOldOperations, true ) && $partialDeleteReplicas->count() === 1 && (string) $partialDeleteReplicas->first()->state === 'deleting', 'Partial template delete did not preserve failure and retry survivors.'); [$partialIsoImport, $partialIsoImportResource, $partialIsoImportId, $partialIsoImportCleanupId, $partialIsoImportCleanupChildren, $partialIsoImportOldOperations, $partialIsoImportDto] = $startMixedCleanup( 'user_iso_import', 'user_iso_import_replica' ); $partialIsoImportReplicas = Capsule::table('cloud_user_iso_replicas') ->where('user_iso_id', $partialIsoImportId)->get(); $assert((string) $partialIsoImport->effect_resolution === 'applied' && str_ends_with( (string) $partialIsoImport->effect_resolution_reason, ':mixed' ) && (string) $partialIsoImportResource->state === 'import_failed' && $partialIsoImportResource->active_job_id === null && is_array($partialIsoImportDto) && ($partialIsoImportDto['can_delete'] ?? false) === true && ($partialIsoImportDto['cleanup_reason'] ?? null) === 'partial_import' && $partialIsoImportCleanupId > 0 && $partialIsoImportCleanupChildren->count() === 1 && !in_array( (string) $partialIsoImportCleanupChildren->first()->operation_id, $partialIsoImportOldOperations, true ) && $partialIsoImportReplicas->count() === 1 && (string) $partialIsoImportReplicas->first()->state === 'deleting', 'Partial ISO import did not expose an independent safe cleanup.'); [$partialIsoDelete, $partialIsoDeleteResource, $partialIsoDeleteId, $partialIsoDeleteCleanupId, $partialIsoDeleteCleanupChildren, $partialIsoDeleteOldOperations, $partialIsoDeleteDto] = $startMixedCleanup( 'user_iso_delete', 'user_iso_delete_replica' ); $partialIsoDeleteReplicas = Capsule::table('cloud_user_iso_replicas') ->where('user_iso_id', $partialIsoDeleteId)->get(); $assert((string) $partialIsoDelete->effect_resolution === 'applied' && str_ends_with( (string) $partialIsoDelete->effect_resolution_reason, ':mixed' ) && (string) $partialIsoDeleteResource->state === 'delete_failed' && $partialIsoDeleteResource->active_job_id === null && is_array($partialIsoDeleteDto) && ($partialIsoDeleteDto['can_delete'] ?? false) === true && ($partialIsoDeleteDto['cleanup_reason'] ?? null) === 'partial_delete' && $partialIsoDeleteCleanupId > 0 && $partialIsoDeleteCleanupChildren->count() === 1 && !in_array( (string) $partialIsoDeleteCleanupChildren->first()->operation_id, $partialIsoDeleteOldOperations, true ) && $partialIsoDeleteReplicas->count() === 1 && (string) $partialIsoDeleteReplicas->first()->state === 'deleting', 'Partial ISO delete did not expose an independent safe cleanup.'); // A zero-dispatch child can know its host effect is not-applied while its // PHP replica/catalog finalizer is still rolled back. Neither the realtime // sibling hook nor the legacy sweep may publish the parent until that // independent local incident has committed and cleared. $localIncidentParentId = $insertParent('template_import'); $localIncidentChildId = $insertResolvedChild( $localIncidentParentId, 'not_applied', 'template_import_replica' ); $insertProjectionForParent($localIncidentParentId); Capsule::table('cloud_jobs')->where( 'id', $localIncidentChildId )->update([ 'attempts' => 0, 'dispatch_attempts' => 0, 'dispatch_started_at' => null, 'agent_accepted_at' => null, 'dispatched_at' => null, 'receipt_deadline_at' => null, 'completion_event_id' => null, 'safe_to_resubmit' => 0, 'finalization_incident' => 1, 'finalization_error_code' => 'recovery_ledger_conflict', ]); CloudJobService::reconcileAggregateParentEffectLocked( CloudJob::findOrFail($localIncidentChildId), null ); $localIncidentParent = CloudJob::findOrFail($localIncidentParentId); $assert((string) $localIncidentParent->effect_resolution === 'unknown' && (bool) $localIncidentParent->finalization_incident, 'A local-finalizer child prematurely released its aggregate parent.'); Capsule::table('cloud_jobs')->where( 'id', $localIncidentChildId )->update([ 'safe_to_resubmit' => 1, 'finalization_incident' => 0, 'finalization_error_code' => null, 'effect_resolved_at' => time(), 'effect_resolution_reason' => 'automatic_local_not_dispatched', ]); CloudJobService::reconcileAggregateParentEffectLocked( CloudJob::findOrFail($localIncidentChildId), null ); $localIncidentParent->refresh(); $assert((string) $localIncidentParent->effect_resolution === 'not_applied' && !(bool) $localIncidentParent->finalization_incident, 'The aggregate parent stayed fenced after its local child converged.'); $makeLegacyEligible = static function (int $parentId): void { Capsule::table('cloud_jobs')->where('id', $parentId)->update([ 'lease_token' => null, 'lease_until' => 0, 'available_at' => 0, 'finalization_incident' => 1, 'finalization_error_code' => 'recovery_child_convergence', ]); }; $legacyLocalIncidentParentId = $insertParent('template_import'); $makeLegacyEligible($legacyLocalIncidentParentId); $legacyLocalIncidentChildId = $insertResolvedChild( $legacyLocalIncidentParentId, 'not_applied', 'template_import_replica' ); $insertProjectionForParent($legacyLocalIncidentParentId); Capsule::table('cloud_jobs')->where( 'id', $legacyLocalIncidentChildId )->update([ 'attempts' => 0, 'dispatch_attempts' => 0, 'dispatch_started_at' => null, 'agent_accepted_at' => null, 'dispatched_at' => null, 'receipt_deadline_at' => null, 'completion_event_id' => null, 'safe_to_resubmit' => 0, 'finalization_incident' => 1, 'finalization_error_code' => 'recovery_ledger_conflict', ]); $assert(CloudJobService::reconcileOneLegacyAggregateEffect(), 'The legacy sweep did not inspect a local-finalizer child incident.'); $legacyLocalIncidentParent = CloudJob::findOrFail( $legacyLocalIncidentParentId ); $assert((string) $legacyLocalIncidentParent->effect_resolution === 'unknown' && (string) $legacyLocalIncidentParent->finalization_error_code === 'recovery_child_convergence', 'The legacy sweep prematurely released a local-finalizer child.'); Capsule::table('cloud_jobs')->where( 'id', $legacyLocalIncidentChildId )->update([ 'safe_to_resubmit' => 1, 'finalization_incident' => 0, 'finalization_error_code' => null, 'effect_resolved_at' => time(), 'effect_resolution_reason' => 'automatic_local_not_dispatched', ]); $makeLegacyEligible($legacyLocalIncidentParentId); $assert(CloudJobService::reconcileOneLegacyAggregateEffect(), 'The legacy sweep did not resume after local finalizer convergence.'); $legacyLocalIncidentParent->refresh(); $assert((string) $legacyLocalIncidentParent->effect_resolution === 'not_applied' && !(bool) $legacyLocalIncidentParent->finalization_incident, 'The legacy parent stayed fenced after local finalizer convergence.'); // The realtime hook applies the same topology rules to both the child that // triggered it and every sibling. A wrong link is an incident, never // evidence that the aggregate effect can be published. $liveWrongCurrentParent = $insertParent('template_delete'); $makeLegacyEligible($liveWrongCurrentParent); $liveWrongCurrent = $insertResolvedChild( $liveWrongCurrentParent, 'applied', 'template_import_replica' ); $insertResolvedChild( $liveWrongCurrentParent, 'not_applied', 'template_delete_replica' ); CloudJobService::reconcileAggregateParentEffectLocked( CloudJob::find($liveWrongCurrent), null ); $liveWrongSiblingParent = $insertParent('user_iso_delete'); $makeLegacyEligible($liveWrongSiblingParent); $liveValidCurrent = $insertResolvedChild( $liveWrongSiblingParent, 'applied', 'user_iso_delete_replica' ); $insertResolvedChild( $liveWrongSiblingParent, 'not_applied', 'user_iso_import_replica' ); CloudJobService::reconcileAggregateParentEffectLocked( CloudJob::find($liveValidCurrent), null ); foreach ([ $liveWrongCurrentParent => 'current child', $liveWrongSiblingParent => 'sibling child', ] as $parentId => $description) { $conflict = CloudJob::find($parentId); $display = CloudJobService::recoveryPresentation($conflict, true); $assert((string) $conflict->effect_resolution === 'unknown' && (bool) $conflict->finalization_incident && (string) $conflict->finalization_error_code === 'recovery_aggregate_state_conflict' && (int) $conflict->available_at >= time() + 290 && $conflict->read_at === null && ($display['code'] ?? null) === 'recovery_aggregate_state_conflict', 'The live hook released a parent with a wrong ' . $description . ' relationship.'); } foreach ([ 'missing parent' => static function () use ( $insertResolvedChild ): CloudJob { return CloudJob::findOrFail($insertResolvedChild( 999999, 'applied', 'template_import_replica' )); }, 'non-aggregate parent' => static function () use ( $insertParent, $insertResolvedChild ): CloudJob { $parentId = $insertParent('template_import'); Capsule::table('cloud_jobs')->where('id', $parentId)->update([ 'execution_mode' => 'local', ]); return CloudJob::findOrFail($insertResolvedChild( $parentId, 'applied', 'template_import_replica' )); }, ] as $description => $fixture) { $thrown = false; try { CloudJobService::reconcileAggregateParentEffectLocked( $fixture(), null ); } catch (CloudRecoveryAggregateStateConflict $error) { $thrown = true; } $assert($thrown, 'A child with a ' . $description . ' could be silently acknowledged.'); } // A damaged aggregate relationship must be rejected before the recovery // coordinator writes the Agent journal fence. Cover both parent-envelope // and child-kind corruption at the private transaction boundary. $resolveFence = new ReflectionMethod( CloudAutomaticOperationRecoveryService::class, 'resolveFenceAndCommit' ); $preparedFence = [ 'resolution' => 'applied', 'journal_status' => 'failed', 'fence_resolution_required' => true, 'source' => 'host_inspection', 'operation' => [], ]; foreach ([ 'bad parent mode' => static function () use ( $insertParent, $insertResolvedChild ): CloudJob { $parentId = $insertParent('template_import'); $childId = $insertResolvedChild( $parentId, 'applied', 'template_import_replica' ); Capsule::table('cloud_jobs')->where('id', $parentId)->update([ 'execution_mode' => 'local', ]); Capsule::table('cloud_jobs')->where('id', $childId)->update([ 'effect_resolution' => 'unknown', 'lease_token' => str_repeat('f', 32), 'lease_until' => time() + 300, ]); return CloudJob::findOrFail($childId); }, 'wrong child kind' => static function () use ( $insertParent, $insertResolvedChild, $insertProjectionForParent ): CloudJob { $parentId = $insertParent('template_delete'); $childId = $insertResolvedChild( $parentId, 'applied', 'template_import_replica' ); $insertProjectionForParent($parentId); Capsule::table('cloud_jobs')->where('id', $childId)->update([ 'effect_resolution' => 'unknown', 'lease_token' => str_repeat('e', 32), 'lease_until' => time() + 300, ]); return CloudJob::findOrFail($childId); }, ] as $description => $fixture) { $beforeFenceRequests = CloudAgentSettingsService::$requests; $blocked = false; try { $resolveFence->invoke( null, $fixture(), $preparedFence, 'test-only', [] ); } catch (CloudRecoveryAggregateStateConflict $error) { $blocked = true; } $assert( $blocked && CloudAgentSettingsService::$requests === $beforeFenceRequests, 'A ' . $description . ' reached an Agent fence write.' ); } // Upgrade sweep: these parents were already terminal failed/unknown before // the realtime leaf hook existed. Each tick may converge exactly one local // parent from already-known children and must never need an Agent client. $legacyMappings = [ 'template_import' => 'template_import_replica', 'template_metadata_sync' => 'template_metadata_replica', 'template_delete' => 'template_delete_replica', 'user_iso_import' => 'user_iso_import_replica', 'user_iso_delete' => 'user_iso_delete_replica', ]; $legacyParents = []; foreach ($legacyMappings as $parentKind => $childKind) { $parentId = $insertParent($parentKind); $makeLegacyEligible($parentId); $firstResolution = $parentKind === 'template_import' ? 'not_applied' : 'applied'; $insertResolvedChild( $parentId, $firstResolution, $childKind ); $insertResolvedChild($parentId, $firstResolution, $childKind); $resourceId = $insertProjectionForParent($parentId); $legacyParents[$parentId] = [ 'kind' => $parentKind, 'resolution' => $firstResolution === 'applied' ? 'applied' : 'not_applied', 'resource_id' => $resourceId, ]; } $automatic = new CloudAutomaticOperationRecoveryService(); $resolvedLegacyCount = 0; foreach ($legacyParents as $parentId => $expectation) { $assert($automatic->tick(), 'The local legacy aggregate sweep did not claim a parent.'); $resolved = CloudJob::whereIn('id', array_keys($legacyParents)) ->whereIn('effect_resolution', ['applied', 'not_applied']) ->count(); $current = CloudJob::find($parentId); if ($expectation['kind'] === 'template_delete') { $resourceId = $expectation['resource_id']; $assert($resolved === $resolvedLegacyCount && $current !== null && (string) $current->effect_resolution === 'unknown' && (bool) $current->finalization_incident && (string) $current->finalization_error_code === 'recovery_child_convergence' && (string) Capsule::table('cloud_templates') ->where('id', $resourceId)->value('state') === 'delete_failed' && (string) Capsule::table('cloud_template_artifacts') ->where('template_id', $resourceId)->value('state') === 'delete_pending' && Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId)->exists(), 'Template delete recovery published applied before the exact ' . 'master object was confirmed absent.'); continue; } $resolvedLegacyCount++; $assert($resolved === $resolvedLegacyCount && $current !== null && (string) $current->effect_resolution === $expectation['resolution'] && (string) $current->outcome === 'failed' && !(bool) $current->finalization_incident && $current->finalization_error_code === null && CloudJobService::recoveryPresentation( $current, true ) === null, 'A tick handled multiple legacy parents or mapped children incorrectly.'); $resourceId = $expectation['resource_id']; $projectionOkay = match ($expectation['kind']) { 'template_import' => (string) Capsule::table('cloud_templates') ->where('id', $resourceId)->value('state') === 'retired' && !Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId)->exists(), 'template_metadata_sync' => (string) Capsule::table('cloud_templates') ->where('id', $resourceId)->value('state') === 'active' && !Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId) ->where('state', '<>', 'active')->exists(), 'user_iso_import' => (string) Capsule::table('cloud_user_isos') ->where('id', $resourceId)->value('state') === 'active', 'user_iso_delete' => (string) Capsule::table('cloud_user_isos') ->where('id', $resourceId)->value('state') === 'retired' && !Capsule::table('cloud_user_iso_replicas') ->where('user_iso_id', $resourceId)->exists(), }; $assert($projectionOkay, 'A legacy terminal parent did not recompute its resource projection: ' . $expectation['kind']); } $waitingParent = $insertParent('template_import'); $makeLegacyEligible($waitingParent); $waitingChild = $insertResolvedChild( $waitingParent, 'not_applied', 'template_import_replica' ); $insertProjectionForParent($waitingParent); Capsule::table('cloud_jobs')->where('id', $waitingChild)->update([ 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolution_reason' => '', ]); // Receipt handling must run before a legacy aggregate that can remain in // child-convergence for an arbitrarily long time. Use an invalid durable // event here so the scheduler must quarantine it without touching Agent // or catalog state; the aggregate remains immediately eligible for the // following tick instead of starving the receipt forever. $priorityChild = $insertResolvedChild( 0, 'not_applied', 'template_import_replica' ); Capsule::table('cloud_jobs')->where('id', $priorityChild)->update([ 'outcome' => 'failed', 'status' => 'failed', 'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0, 'effect_resolution' => 'unknown', 'effect_resolved_at' => null, 'effect_resolution_reason' => '', 'available_at' => 0, 'dispatched_at' => $now - 20, ]); $priorityOperation = (string) CloudJob::findOrFail( $priorityChild )->operation_id; Capsule::table('cloud_agent_completion_events')->insert([ 'event_id' => 'aggregate-priority-receipt', 'host_id' => 7, 'operation_id' => $priorityOperation, 'payload_sha256' => str_repeat('0', 64), 'outcome' => 'succeeded', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'error_code' => '', 'result_json' => null, 'disposition' => 'received', 'received_at' => $now, 'applied_at' => null, ]); $makeLegacyEligible($waitingParent); $assert($automatic->tick(), 'A durable receipt was not prioritised over a waiting aggregate.'); $priorityEvent = Capsule::table('cloud_agent_completion_events') ->where('event_id', 'aggregate-priority-receipt')->first(); $stillEligibleParent = CloudJob::findOrFail($waitingParent); $assert((string) $priorityEvent->disposition === 'rejected' && (int) $stillEligibleParent->available_at === 0, 'A waiting legacy aggregate starved receipt validation.'); $assert($automatic->tick(), 'A legacy aggregate with an active child was not inspected locally.'); $waiting = CloudJob::find($waitingParent); $waitingDisplay = CloudJobService::recoveryPresentation($waiting, true); $assert((string) $waiting->effect_resolution === 'unknown' && (string) $waiting->finalization_error_code === 'recovery_child_convergence' && (int) $waiting->available_at >= time() + 290 && ($waitingDisplay['code'] ?? null) === 'recovery_child_convergence', 'An unresolved but valid child released or corrupted its parent fence.'); // Corrupt aggregate relationships remain fenced and retryable. Cover every // structural invariant independently so a future relaxed query cannot turn // a wrong child into evidence for releasing its parent. $conflictFixtures = []; $missingChildParent = $insertParent('template_import'); $makeLegacyEligible($missingChildParent); $conflictFixtures[$missingChildParent] = 'missing child'; $wrongParentMode = $insertParent('template_import'); $makeLegacyEligible($wrongParentMode); $insertResolvedChild( $wrongParentMode, 'applied', 'template_import_replica' ); Capsule::table('cloud_jobs')->where('id', $wrongParentMode)->update([ 'execution_mode' => 'local', ]); $conflictFixtures[$wrongParentMode] = 'wrong parent execution mode'; $wrongParentKind = $insertParent('template_import'); $makeLegacyEligible($wrongParentKind); $insertResolvedChild( $wrongParentKind, 'applied', 'template_import_replica' ); Capsule::table('cloud_jobs')->where('id', $wrongParentKind)->update([ 'kind' => 'corrupt_aggregate_kind', ]); $conflictFixtures[$wrongParentKind] = 'wrong parent kind'; $wrongKindParent = $insertParent('template_delete'); $makeLegacyEligible($wrongKindParent); $insertResolvedChild( $wrongKindParent, 'applied', 'template_import_replica' ); $conflictFixtures[$wrongKindParent] = 'wrong child kind'; $wrongUserParent = $insertParent('user_iso_import'); $makeLegacyEligible($wrongUserParent); $insertResolvedChild( $wrongUserParent, 'applied', 'user_iso_import_replica', 2 ); $conflictFixtures[$wrongUserParent] = 'wrong child owner'; $wrongModeParent = $insertParent('user_iso_delete'); $makeLegacyEligible($wrongModeParent); $insertResolvedChild( $wrongModeParent, 'not_applied', 'user_iso_delete_replica', 1, 'local' ); $conflictFixtures[$wrongModeParent] = 'wrong child execution mode'; $nestedParent = $insertParent('template_metadata_sync'); $makeLegacyEligible($nestedParent); Capsule::table('cloud_jobs')->where('id', $nestedParent)->update([ // The parent only needs to be non-null to exercise the invalid // aggregate nesting invariant. Do not attach it to an earlier valid // aggregate fixture: doing so also (correctly) corrupts that parent's // already-published child topology and changes the next claimed row. 'parent_job_id' => 999999, ]); $insertResolvedChild( $nestedParent, 'applied', 'template_metadata_replica' ); $conflictFixtures[$nestedParent] = 'non-top-level aggregate parent'; // A malformed Agent child is itself a higher-priority PHP-only leaf // repair, so it may be claimed before an earlier aggregate parent. Drain // the bounded fixture set, then verify every parent retained its fence. $beforeConflictRequests = CloudAgentSettingsService::$requests; for ($attempt = 0; $attempt < count($conflictFixtures) * 3; $attempt++) { $remaining = CloudJob::whereIn('id', array_keys($conflictFixtures)) ->where(static function ($pending): void { $pending->where('finalization_incident', 0) ->orWhereNull('finalization_error_code') ->orWhere( 'finalization_error_code', '<>', 'recovery_aggregate_state_conflict' ); })->exists(); if (!$remaining) { break; } $assert($automatic->tick(), 'The aggregate conflict fixture queue stopped before convergence.'); } foreach ($conflictFixtures as $parentId => $description) { $conflict = CloudJob::find($parentId); $display = $conflict === null ? null : CloudJobService::recoveryPresentation($conflict, true); $assert($conflict !== null && (string) $conflict->effect_resolution === 'unknown' && (string) $conflict->outcome === 'failed' && (bool) $conflict->finalization_incident && (string) $conflict->finalization_error_code === 'recovery_aggregate_state_conflict' && (int) $conflict->available_at >= time() + 290 && $conflict->lease_token === null && (int) $conflict->lease_until === 0 && $conflict->read_at === null && ($display['code'] ?? null) === 'recovery_aggregate_state_conflict' && str_contains( (string) ($display['message'] ?? ''), '关系异常' ) && CloudAgentSettingsService::$requests === $beforeConflictRequests, 'A ' . $description . ' relationship released or concealed its aggregate fence.'); } // prepareRecoveredEffect updates a replica before the external Agent fence // write. Simulate that exact local prepare seam: the public parent must // remain visibly recovering until the child effect commit runs, then the // same transaction may publish the aggregate success. $preparedParentId = $nextId++; $preparedParentOperation = md5( 'aggregate-prepared-parent:' . $preparedParentId ); Capsule::table('cloud_jobs')->insert([ 'id' => $preparedParentId, 'operation_id' => $preparedParentOperation, 'request_idempotency_key' => $preparedParentOperation, 'request_fingerprint' => hash('sha256', $preparedParentOperation), 'parent_job_id' => null, 'user_id' => 1, 'host_id' => null, 'kind' => 'template_import', 'execution_mode' => 'aggregate', 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'), 'attempts' => 0, 'dispatch_attempts' => 0, 'dispatch_deadline_at' => $now + 300, 'receipt_deadline_at' => $now + 900, 'effect_resolution' => 'not_applicable', 'effect_resolution_reason' => '', 'lease_until' => 0, 'available_at' => $now, 'finalization_incident' => 0, 'created_at' => $now - 60, 'updated_at' => $now, ]); $preparedChildId = $insertResolvedChild( $preparedParentId, 'not_applied', 'template_import_replica' ); Capsule::table('cloud_jobs')->where('id', $preparedChildId)->update([ 'outcome' => 'failed', 'status' => 'failed', 'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0, 'effect_resolution' => 'unknown', 'effect_resolved_at' => null, 'effect_resolution_reason' => 'automatic_recovery:v1:1:1:prepared:test', 'finalization_incident' => 1, 'finalization_error_code' => 'recovery_payload_unavailable', ]); $preparedTemplateId = 'prepared-import-template'; $preparedGeneration = str_repeat('c', 32); $preparedSha = str_repeat('d', 64); Capsule::table('cloud_templates')->insert([ 'id' => $preparedTemplateId, 'name' => 'Prepared import', 'os_family' => 'Debian', 'os_version' => '13', 'image_bytes' => 4096, 'sha256' => $preparedSha, 'startup_script' => '', 'revision' => 1, 'generation_token' => $preparedGeneration, 'state' => 'importing', 'active_job_id' => $preparedParentId, 'created_at' => $now - 60, 'updated_at' => $now, ]); Capsule::table('cloud_template_replicas')->insert([ 'host_id' => 7, 'template_id' => $preparedTemplateId, 'revision' => 1, 'generation_token' => $preparedGeneration, 'state' => 'failed', 'image_bytes' => 0, 'sha256' => $preparedSha, 'operation_id' => (string) CloudJob::findOrFail( $preparedChildId )->operation_id, 'job_id' => $preparedChildId, 'error_code' => 'aggregate_deadline_exceeded', 'created_at' => $now - 60, 'updated_at' => $now, ]); $preparedOperation = [ 'operation_id' => (string) CloudJob::findOrFail( $preparedChildId )->operation_id, 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', ]; CloudTemplateCatalogService::finishJobLocked( CloudJob::findOrFail($preparedChildId), 'succeeded', [ 'id' => $preparedTemplateId, 'revision' => 1, 'generation_token' => $preparedGeneration, 'size_bytes' => 4096, 'sha256' => $preparedSha, ], ['template_id' => $preparedTemplateId], $preparedOperation, true ); $preparedParent = CloudJob::findOrFail($preparedParentId); $preparedParentDisplay = CloudJobService::display( $preparedParent, true ); $assert((string) $preparedParent->phase === 'awaiting_receipt' && (string) Capsule::table('cloud_templates') ->where('id', $preparedTemplateId)->value('state') === 'importing' && (string) Capsule::table('cloud_template_replicas') ->where('template_id', $preparedTemplateId) ->value('state') === 'active' && ($preparedParentDisplay['recovery']['pending'] ?? null) === true && ($preparedParentDisplay['recovery']['code'] ?? null) === 'recovery_payload_unavailable', 'A recovered child prepare prematurely completed or hid its parent.'); Capsule::table('cloud_jobs')->where('id', $preparedChildId)->update([ 'effect_resolution' => 'applied', 'effect_resolved_at' => time(), 'effect_resolution_reason' => 'agent_inspection:test', 'finalization_incident' => 0, 'finalization_error_code' => null, 'available_at' => 0, ]); CloudJobService::reconcileAggregateParentEffectLocked( CloudJob::findOrFail($preparedChildId), null ); $preparedParent->refresh(); $assert((string) $preparedParent->phase === 'terminal' && (string) $preparedParent->outcome === 'succeeded' && (string) $preparedParent->effect_resolution === 'applied' && (string) Capsule::table('cloud_templates') ->where('id', $preparedTemplateId)->value('state') === 'active' && CloudJobService::recoveryPresentation( $preparedParent, true ) === null, 'The aggregate parent did not settle after the child effect commit.'); $immutableParentAudit = static fn (CloudJob $job): array => [ 'phase' => (string) $job->phase, 'outcome' => $job->outcome === null ? null : (string) $job->outcome, 'status' => (string) $job->status, 'terminal_at' => $job->terminal_at === null ? null : (int) $job->terminal_at, 'error_code' => $job->error_code === null ? null : (string) $job->error_code, 'result_json' => $job->result_json === null ? null : (string) $job->result_json, ]; $markDeadlineAudit = static function (int $parentId): CloudJob { Capsule::table('cloud_jobs')->where('id', $parentId)->update([ 'result_json' => json_encode( ['deadline_parent' => $parentId], JSON_THROW_ON_ERROR ), ]); return CloudJob::findOrFail($parentId); }; // A parent deadline is an immutable failed audit, not the final catalog // projection. Once every late child resolves, imports become usable and // deletes become retired without rewriting that original failed task. $deadlineImportParentId = $insertParent('template_import'); $deadlineImportParent = $markDeadlineAudit($deadlineImportParentId); $deadlineImportAudit = $immutableParentAudit($deadlineImportParent); $deadlineImportChildId = $insertResolvedChild( $deadlineImportParentId, 'applied', 'template_import_replica' ); $deadlineImportTemplateId = 'deadline-import-template'; $deadlineImportGeneration = str_repeat('5', 32); $deadlineImportSha = str_repeat('6', 64); Capsule::table('cloud_templates')->insert([ 'id' => $deadlineImportTemplateId, 'name' => 'Deadline import', 'os_family' => 'Debian', 'os_version' => '13', 'image_bytes' => 4096, 'sha256' => $deadlineImportSha, 'startup_script' => '', 'revision' => 1, 'generation_token' => $deadlineImportGeneration, 'state' => 'error', 'active_job_id' => $deadlineImportParentId, 'created_at' => $now - 60, 'updated_at' => $now, ]); Capsule::table('cloud_template_replicas')->insert([ 'host_id' => 7, 'template_id' => $deadlineImportTemplateId, 'revision' => 1, 'generation_token' => $deadlineImportGeneration, 'state' => 'active', 'image_bytes' => 4096, 'sha256' => $deadlineImportSha, 'operation_id' => (string) CloudJob::findOrFail( $deadlineImportChildId )->operation_id, 'job_id' => $deadlineImportChildId, 'error_code' => '', 'created_at' => $now - 60, 'updated_at' => $now, ]); CloudJobService::reconcileAggregateParentEffectLocked( CloudJob::findOrFail($deadlineImportChildId), null ); $deadlineImportParent->refresh(); $deadlinePublicIds = array_column( CloudTemplateCatalogService::publicCatalog()['templates'], 'id' ); $assert($immutableParentAudit($deadlineImportParent) === $deadlineImportAudit && (string) $deadlineImportParent->effect_resolution === 'applied' && (string) Capsule::table('cloud_templates') ->where('id', $deadlineImportTemplateId)->value('state') === 'active' && in_array($deadlineImportTemplateId, $deadlinePublicIds, true), 'A recovered deadline import did not publish its catalog projection ' . 'without rewriting the failed task.'); $deadlineDeleteParentId = $insertParent('template_delete'); $deadlineDeleteParent = $markDeadlineAudit($deadlineDeleteParentId); $deadlineDeleteAudit = $immutableParentAudit($deadlineDeleteParent); $deadlineDeleteChildId = $insertResolvedChild( $deadlineDeleteParentId, 'applied', 'template_delete_replica' ); $deadlineDeleteTemplateId = 'deadline-delete-template'; $deadlineDeleteGeneration = str_repeat('7', 32); $deadlineDeleteSha = str_repeat('8', 64); Capsule::table('cloud_templates')->insert([ 'id' => $deadlineDeleteTemplateId, 'name' => 'Deadline delete', 'os_family' => 'Debian', 'os_version' => '13', 'image_bytes' => 4096, 'sha256' => $deadlineDeleteSha, 'startup_script' => '', 'revision' => 1, 'generation_token' => $deadlineDeleteGeneration, 'state' => 'delete_failed', 'active_job_id' => $deadlineDeleteParentId, 'created_at' => $now - 60, 'updated_at' => $now, ]); Capsule::table('cloud_template_replicas')->insert([ 'host_id' => 7, 'template_id' => $deadlineDeleteTemplateId, 'revision' => 1, 'generation_token' => $deadlineDeleteGeneration, 'state' => 'delete_succeeded', 'image_bytes' => 4096, 'sha256' => $deadlineDeleteSha, 'operation_id' => (string) CloudJob::findOrFail( $deadlineDeleteChildId )->operation_id, 'job_id' => $deadlineDeleteChildId, 'error_code' => '', 'created_at' => $now - 60, 'updated_at' => $now, ]); CloudJobService::reconcileAggregateParentEffectLocked( CloudJob::findOrFail($deadlineDeleteChildId), null ); $deadlineDeleteParent->refresh(); $assert($immutableParentAudit($deadlineDeleteParent) === $deadlineDeleteAudit && (string) $deadlineDeleteParent->effect_resolution === 'unknown' && (bool) $deadlineDeleteParent->finalization_incident && (string) $deadlineDeleteParent->finalization_error_code === 'recovery_child_convergence' && (string) Capsule::table('cloud_templates') ->where('id', $deadlineDeleteTemplateId)->value('state') === 'delete_failed' && (int) Capsule::table('cloud_templates') ->where('id', $deadlineDeleteTemplateId)->value('revision') === 1 && Capsule::table('cloud_template_replicas') ->where('template_id', $deadlineDeleteTemplateId)->exists() && (string) Capsule::table('cloud_template_artifacts') ->where('template_id', $deadlineDeleteTemplateId)->value('state') === 'delete_pending', 'A recovered deadline delete published applied before the master ' . 'object cleanup was confirmed.'); $deadlineIsoParentId = $insertParent('user_iso_import'); $deadlineIsoParent = $markDeadlineAudit($deadlineIsoParentId); $deadlineIsoAudit = $immutableParentAudit($deadlineIsoParent); $deadlineIsoChildId = $insertResolvedChild( $deadlineIsoParentId, 'applied', 'user_iso_import_replica' ); $deadlineIsoId = str_repeat('9', 32); $deadlineIsoSha = str_repeat('a', 64); Capsule::table('cloud_user_isos')->insert([ 'id' => $deadlineIsoId, 'user_id' => 1, 'name' => 'Deadline ISO', 'sealed_source_url' => 'test-only', 'size_bytes' => 4096, 'sha256' => $deadlineIsoSha, 'state' => 'import_failed', 'import_operation_id' => (string) $deadlineIsoParent->operation_id, 'active_job_id' => $deadlineIsoParentId, 'created_at' => $now - 60, 'updated_at' => $now, ]); Capsule::table('cloud_user_iso_replicas')->insert([ 'host_id' => 7, 'user_iso_id' => $deadlineIsoId, 'user_id' => 1, 'state' => 'active', 'size_bytes' => 4096, 'sha256' => $deadlineIsoSha, 'operation_id' => (string) CloudJob::findOrFail( $deadlineIsoChildId )->operation_id, 'job_id' => $deadlineIsoChildId, 'error_code' => '', 'created_at' => $now - 60, 'updated_at' => $now, ]); CloudJobService::reconcileAggregateParentEffectLocked( CloudJob::findOrFail($deadlineIsoChildId), null ); $deadlineIsoParent->refresh(); $assert($immutableParentAudit($deadlineIsoParent) === $deadlineIsoAudit && (string) $deadlineIsoParent->effect_resolution === 'applied' && (string) Capsule::table('cloud_user_isos') ->where('id', $deadlineIsoId)->value('state') === 'active', 'A recovered deadline ISO import did not publish its catalog ' . 'projection without rewriting the failed task.'); $settledPartialBefore = CloudJob::findOrFail((int) $partial->id); $settledPartialUpdatedAt = (int) $settledPartialBefore->updated_at; $automatic->tick(); $settledPartialAfter = CloudJob::findOrFail((int) $partial->id); $assert((int) $settledPartialAfter->updated_at === $settledPartialUpdatedAt && (int) $settledPartialAfter->available_at === 0 && (string) $settledPartialAfter->effect_resolution === 'applied' && !(bool) $settledPartialAfter->finalization_incident, 'A settled partial aggregate re-entered the recovery queue.'); $assert(CloudAgentSettingsService::$requests === 0, 'Aggregate recovery contacted an Agent instead of local ledgers.'); // A corrupt replica dispatch ledger must not be interpreted as an // undispatched/no-effect child. Doing so would fail the replica and // release the public template aggregate even though the Agent boundary is // unknown. Keep the catalog, child escrow and parent fence intact instead. $invalidDispatchParentId = $insertParent('template_import'); Capsule::table('cloud_jobs')->where( 'id', $invalidDispatchParentId )->update([ 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolution_reason' => '', 'finalization_incident' => 0, 'finalization_error_code' => null, 'lease_token' => null, 'lease_until' => 0, 'available_at' => 0, 'receipt_deadline_at' => $now + 600, ]); $invalidDispatchChildId = $insertResolvedChild( $invalidDispatchParentId, 'not_applied', 'template_import_replica' ); $invalidDispatchTemplateId = 'invalid-dispatch-template'; $invalidDispatchGeneration = str_repeat('e', 32); $invalidDispatchSha = str_repeat('f', 64); $invalidDispatchPayload = CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/templates/import', 'body' => [], 'template_id' => $invalidDispatchTemplateId, 'owner_id' => 1, 'can_manage' => true, ], 'job-payload'); Capsule::table('cloud_jobs')->where( 'id', $invalidDispatchChildId )->update([ 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'sealed_payload' => $invalidDispatchPayload, 'attempts' => 2, 'dispatch_attempts' => 2, 'dispatch_started_at' => $now - 30, 'agent_accepted_at' => $now - 29, 'receipt_deadline_at' => $now - 1, 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolution_reason' => '', 'completion_event_id' => null, 'dispatched_at' => $now - 29, 'lease_token' => null, 'lease_until' => 0, 'available_at' => 0, 'error_code' => null, 'finalization_incident' => 0, 'finalization_error_code' => null, ]); Capsule::table('cloud_templates')->insert([ 'id' => $invalidDispatchTemplateId, 'name' => 'Invalid dispatch template', 'os_family' => 'Debian', 'os_version' => '13', 'image_bytes' => 4096, 'sha256' => $invalidDispatchSha, 'startup_script' => '', 'revision' => 1, 'generation_token' => $invalidDispatchGeneration, 'state' => 'importing', 'active_job_id' => $invalidDispatchParentId, 'created_at' => $now - 60, 'updated_at' => $now, ]); Capsule::table('cloud_template_replicas')->insert([ 'host_id' => 7, 'template_id' => $invalidDispatchTemplateId, 'revision' => 1, 'generation_token' => $invalidDispatchGeneration, 'state' => 'importing', 'image_bytes' => 0, 'sha256' => $invalidDispatchSha, 'operation_id' => (string) CloudJob::findOrFail( $invalidDispatchChildId )->operation_id, 'job_id' => $invalidDispatchChildId, 'error_code' => '', 'created_at' => $now - 60, 'updated_at' => $now, ]); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $invalidDispatchChildId, 1 ) === 1, 'The invalid template child deadline was not claimed.'); $invalidDispatchChild = CloudJob::findOrFail( $invalidDispatchChildId ); $invalidDispatchParent = CloudJob::findOrFail( $invalidDispatchParentId ); $invalidDispatchParentRecovery = CloudJobService::recoveryPresentation( $invalidDispatchParent, true ); $assert((string) $invalidDispatchChild->effect_resolution === 'unknown' && (bool) $invalidDispatchChild->effect_may_have_applied && !(bool) $invalidDispatchChild->safe_to_resubmit && (bool) $invalidDispatchChild->finalization_incident && (string) $invalidDispatchChild->finalization_error_code === 'recovery_dispatch_ledger_conflict' && (string) $invalidDispatchChild->sealed_payload === $invalidDispatchPayload && (string) $invalidDispatchParent->effect_resolution !== 'not_applied' && (string) Capsule::table('cloud_templates') ->where('id', $invalidDispatchTemplateId)->value('state') === 'importing' && (string) Capsule::table('cloud_template_replicas') ->where('template_id', $invalidDispatchTemplateId) ->value('state') === 'importing' && ($invalidDispatchParentRecovery['code'] ?? null) === 'recovery_dispatch_ledger_conflict', 'An invalid template child released its catalog or aggregate as no-effect.'); // Ordinary receipt finalization runs before the currently completing child // is made terminal. That current transition is allowed, but a different // sibling with an invalid recovery ledger must keep the public catalog and // aggregate parent fenced. Once the sibling is repaired, the normal child // recovery hook publishes the already-prepared replicas. $normalGateParentId = $insertParent('template_import'); Capsule::table('cloud_jobs')->where('id', $normalGateParentId)->update([ 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolution_reason' => '', 'finalization_incident' => 0, 'finalization_error_code' => null, 'lease_token' => null, 'lease_until' => 0, 'available_at' => 0, 'receipt_deadline_at' => $now + 600, ]); $normalGateSiblingId = $insertResolvedChild( $normalGateParentId, 'applied', 'template_import_replica' ); Capsule::table('cloud_jobs')->where( 'id', $normalGateSiblingId )->update(['dispatch_attempts' => 2]); $normalGateCurrentId = $insertResolvedChild( $normalGateParentId, 'applied', 'template_import_replica' ); Capsule::table('cloud_jobs')->where('id', $normalGateCurrentId)->update([ 'host_id' => 8, 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolution_reason' => '', ]); $normalGateTemplateId = 'normal-settle-gated-template'; $normalGateGeneration = str_repeat('1', 32); $normalGateSha = str_repeat('2', 64); Capsule::table('cloud_templates')->insert([ 'id' => $normalGateTemplateId, 'name' => 'Normal settle gated', 'os_family' => 'Debian', 'os_version' => '13', 'image_bytes' => 8192, 'sha256' => $normalGateSha, 'startup_script' => '', 'revision' => 1, 'generation_token' => $normalGateGeneration, 'state' => 'importing', 'active_job_id' => $normalGateParentId, 'created_at' => $now - 60, 'updated_at' => $now, ]); foreach ([ [7, $normalGateSiblingId, 'active', 8192], [8, $normalGateCurrentId, 'importing', 0], ] as [$hostId, $jobId, $state, $imageBytes]) { Capsule::table('cloud_template_replicas')->insert([ 'host_id' => $hostId, 'template_id' => $normalGateTemplateId, 'revision' => 1, 'generation_token' => $normalGateGeneration, 'state' => $state, 'image_bytes' => $imageBytes, 'sha256' => $normalGateSha, 'operation_id' => (string) CloudJob::findOrFail($jobId) ->operation_id, 'job_id' => $jobId, 'error_code' => '', 'created_at' => $now - 60, 'updated_at' => $now, ]); } CloudTemplateCatalogService::finishJobLocked( CloudJob::findOrFail($normalGateCurrentId), 'succeeded', [ 'id' => $normalGateTemplateId, 'revision' => 1, 'generation_token' => $normalGateGeneration, 'size_bytes' => 8192, 'sha256' => $normalGateSha, ], ['template_id' => $normalGateTemplateId], [ 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', ] ); $normalGateParent = CloudJob::findOrFail($normalGateParentId); $assert((string) $normalGateParent->phase === 'awaiting_receipt' && (string) Capsule::table('cloud_templates') ->where('id', $normalGateTemplateId)->value('state') === 'importing' && Capsule::table('cloud_template_replicas') ->where('template_id', $normalGateTemplateId) ->where('state', 'active')->count() === 2, 'Normal completion published a parent with an unsafe sibling.'); Capsule::table('cloud_jobs')->where('id', $normalGateSiblingId)->update([ 'dispatch_attempts' => 1, ]); Capsule::table('cloud_jobs')->where('id', $normalGateCurrentId)->update([ 'phase' => 'terminal', 'outcome' => 'succeeded', 'status' => 'succeeded', 'terminal_at' => time(), 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => 'applied', 'effect_resolved_at' => time(), 'effect_resolution_reason' => 'normal_completion', ]); CloudJobService::reconcileAggregateParentEffectLocked( CloudJob::findOrFail($normalGateSiblingId), null ); $normalGateParent->refresh(); $assert((string) $normalGateParent->phase === 'terminal' && (string) $normalGateParent->outcome === 'succeeded' && (string) $normalGateParent->effect_resolution === 'applied' && (string) Capsule::table('cloud_templates') ->where('id', $normalGateTemplateId)->value('state') === 'active', 'The recovery hook did not publish a safe repaired aggregate.'); // The normal last-child receipt must use the same strict projection gate // as late recovery. Missing, extra or cross-wired sibling replicas may be // projected locally, but can never publish the aggregate parent. foreach ([ 'missing sibling replica' => [ 'mutation' => 'missing', 'exception' => CloudRecoveryAggregateStateConflict::class, ], 'extra sibling replica' => [ 'mutation' => 'extra', 'exception' => CloudRecoveryAggregateStateConflict::class, ], 'cross-wired sibling operation' => [ 'mutation' => 'mapping', 'exception' => CloudRecoveryOperationIdentityConflict::class, ], ] as $description => $fixture) { $parentId = $insertParent('template_import'); $siblingId = $insertResolvedChild( $parentId, 'applied', 'template_import_replica' ); $currentId = $insertResolvedChild( $parentId, 'applied', 'template_import_replica' ); $resourceId = $insertProjectionForParent($parentId); Capsule::table('cloud_jobs')->where('id', $parentId)->update([ 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolution_reason' => '', 'error_code' => null, 'finalization_incident' => 0, 'finalization_error_code' => null, 'lease_token' => null, 'lease_until' => 0, 'available_at' => 0, ]); Capsule::table('cloud_jobs')->where('id', $currentId)->update([ 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolution_reason' => '', ]); Capsule::table('cloud_templates')->where('id', $resourceId)->update([ 'state' => 'importing', ]); Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId) ->where('job_id', $currentId)->update([ 'state' => 'importing', 'image_bytes' => 0, ]); $template = Capsule::table('cloud_templates') ->where('id', $resourceId)->first(); CloudTemplateCatalogService::finishJobLocked( CloudJob::findOrFail($currentId), 'succeeded', [ 'id' => $resourceId, 'revision' => (int) $template->revision, 'generation_token' => (string) $template->generation_token, 'size_bytes' => (int) $template->image_bytes, 'sha256' => (string) $template->sha256, ], ['template_id' => $resourceId], [] ); if ($fixture['mutation'] === 'missing') { Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId) ->where('job_id', $siblingId)->delete(); } elseif ($fixture['mutation'] === 'extra') { $extra = (array) Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId) ->where('job_id', $siblingId)->first(); $extra['host_id'] = 900000 + $parentId; $extra['job_id'] = null; $extra['operation_id'] = md5('normal-extra:' . $parentId); Capsule::table('cloud_template_replicas')->insert($extra); } else { Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId) ->where('job_id', $siblingId)->update([ 'operation_id' => md5('normal-cross-wire:' . $parentId), ]); } Capsule::table('cloud_jobs')->where('id', $currentId)->update([ 'phase' => 'terminal', 'outcome' => 'succeeded', 'status' => 'succeeded', 'terminal_at' => time(), 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => 'applied', 'effect_resolved_at' => time(), 'effect_resolution_reason' => 'normal_completion', ]); try { CloudJobService::reconcileAggregateParentEffectLocked( CloudJob::findOrFail($currentId), null ); throw new RuntimeException( 'Normal aggregate accepted ' . $description . '.' ); } catch (CloudError $error) { $assert($error instanceof $fixture['exception'], 'Normal aggregate reported the wrong conflict for ' . $description . '.'); } $assert((string) CloudJob::findOrFail($parentId)->phase === 'awaiting_receipt' && (string) Capsule::table('cloud_templates') ->where('id', $resourceId)->value('state') === 'importing', 'Normal aggregate published after ' . $description . '.'); } // Exercise the actual ordinary completion order for all five catalog // aggregates: project the final failed replica, persist its terminal // no-effect conclusion, then let the shared strict reconciler restore the // safe resource boundary. foreach ([ 'template_import' => 'template_import_replica', 'template_metadata_sync' => 'template_metadata_replica', 'template_delete' => 'template_delete_replica', 'user_iso_import' => 'user_iso_import_replica', 'user_iso_delete' => 'user_iso_delete_replica', ] as $parentKind => $childKind) { $parentId = $insertParent($parentKind); $childId = $insertResolvedChild( $parentId, 'not_applied', $childKind ); $resourceId = $insertProjectionForParent($parentId); Capsule::table('cloud_jobs')->where('id', $parentId)->update([ 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolution_reason' => '', 'error_code' => null, 'finalization_incident' => 0, 'finalization_error_code' => null, 'lease_token' => null, 'lease_until' => 0, 'available_at' => 0, ]); Capsule::table('cloud_jobs')->where('id', $childId)->update([ 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolution_reason' => '', ]); if (str_starts_with($parentKind, 'template_')) { $resourceState = $parentKind === 'template_delete' ? 'deleting' : ($parentKind === 'template_import' ? 'importing' : 'active'); $replicaState = match ($parentKind) { 'template_import' => 'importing', 'template_metadata_sync' => 'syncing', 'template_delete' => 'deleting', }; Capsule::table('cloud_templates')->where('id', $resourceId) ->update(['state' => $resourceState]); Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId)->update([ 'state' => $replicaState, ]); $template = Capsule::table('cloud_templates') ->where('id', $resourceId)->first(); CloudTemplateCatalogService::finishJobLocked( CloudJob::findOrFail($childId), 'failed', null, ['template_id' => $resourceId], [ 'status' => 'failed', 'effect_may_have_applied' => false, 'safe_to_resubmit' => true, 'error_code' => 'operation_not_applied', ] ); } else { $resourceState = $parentKind === 'user_iso_delete' ? 'deleting' : 'importing'; $replicaState = $parentKind === 'user_iso_delete' ? 'deleting' : 'importing'; Capsule::table('cloud_user_isos')->where('id', $resourceId) ->update(['state' => $resourceState]); Capsule::table('cloud_user_iso_replicas') ->where('user_iso_id', $resourceId)->update([ 'state' => $replicaState, ]); CloudUserIsoService::finishJobLocked( CloudJob::findOrFail($childId), 'failed', null, ['user_iso_id' => $resourceId], [ 'status' => 'failed', 'effect_may_have_applied' => false, 'safe_to_resubmit' => true, 'error_code' => 'operation_not_applied', ] ); } Capsule::table('cloud_jobs')->where('id', $childId)->update([ 'phase' => 'terminal', 'outcome' => 'failed', 'status' => 'failed', 'terminal_at' => time(), 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 1, 'effect_resolution' => 'not_applied', 'effect_resolved_at' => time(), 'effect_resolution_reason' => 'normal_completion', 'error_code' => 'operation_rejected', ]); CloudJobService::reconcileAggregateParentEffectLocked( CloudJob::findOrFail($childId), null ); $parent = CloudJob::findOrFail($parentId); $projectionOkay = match ($parentKind) { 'template_import' => (string) Capsule::table('cloud_templates') ->where('id', $resourceId)->value('state') === 'retired' && !Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId)->exists(), 'template_metadata_sync' => (string) Capsule::table('cloud_templates') ->where('id', $resourceId)->value('state') === 'active' && (string) Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId)->value('state') === 'stale', 'template_delete' => (string) Capsule::table('cloud_templates') ->where('id', $resourceId)->value('state') === 'active' && (string) Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId)->value('state') === 'active', 'user_iso_import' => (string) Capsule::table('cloud_user_isos') ->where('id', $resourceId)->value('state') === 'retired' && !Capsule::table('cloud_user_iso_replicas') ->where('user_iso_id', $resourceId)->exists(), 'user_iso_delete' => (string) Capsule::table('cloud_user_isos') ->where('id', $resourceId)->value('state') === 'active' && (string) Capsule::table('cloud_user_iso_replicas') ->where('user_iso_id', $resourceId)->value('state') === 'active', }; if ($parentKind === 'template_metadata_sync') { $publicIds = array_column( CloudTemplateCatalogService::publicCatalog()['templates'], 'id' ); $projectionOkay = $projectionOkay && !in_array($resourceId, $publicIds, true); } $assert((string) $parent->phase === 'terminal' && (string) $parent->outcome === 'failed' && (string) $parent->effect_resolution === 'not_applied' && !(bool) $parent->finalization_incident && $projectionOkay, 'Normal all-not-applied completion did not restore ' . $parentKind . '.'); } // The ordinary last-child path must lock the one-and-only main catalog // record before publishing its aggregate parent. A duplicate binding is // data corruption, not permission to complete one arbitrary record. foreach ([ 'template_import' => 'template_import_replica', 'user_iso_import' => 'user_iso_import_replica', ] as $parentKind => $childKind) { $parentId = $insertParent($parentKind); $childId = $insertResolvedChild($parentId, 'applied', $childKind); $resourceId = $insertProjectionForParent($parentId); Capsule::table('cloud_jobs')->where('id', $parentId)->update([ 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolution_reason' => '', 'error_code' => null, 'finalization_incident' => 0, 'finalization_error_code' => null, ]); Capsule::table('cloud_jobs')->where('id', $childId)->update([ 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolution_reason' => '', ]); if ($parentKind === 'template_import') { Capsule::table('cloud_templates')->where('id', $resourceId) ->update(['state' => 'importing']); Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId)->update([ 'state' => 'importing', 'image_bytes' => 0, ]); $duplicate = (array) Capsule::table('cloud_templates') ->where('id', $resourceId)->first(); $duplicate['id'] = $resourceId . '-normal-duplicate'; Capsule::table('cloud_templates')->insert($duplicate); $template = Capsule::table('cloud_templates') ->where('id', $resourceId)->first(); CloudTemplateCatalogService::finishJobLocked( CloudJob::findOrFail($childId), 'succeeded', [ 'id' => $resourceId, 'revision' => (int) $template->revision, 'generation_token' => (string) $template->generation_token, 'size_bytes' => (int) $template->image_bytes, 'sha256' => (string) $template->sha256, ], ['template_id' => $resourceId], [] ); } else { Capsule::table('cloud_user_isos')->where('id', $resourceId) ->update(['state' => 'importing']); Capsule::table('cloud_user_iso_replicas') ->where('user_iso_id', $resourceId)->update([ 'state' => 'importing', 'size_bytes' => 0, ]); $duplicate = (array) Capsule::table('cloud_user_isos') ->where('id', $resourceId)->first(); $duplicate['id'] = md5('normal-duplicate:' . $parentId); Capsule::table('cloud_user_isos')->insert($duplicate); $iso = Capsule::table('cloud_user_isos') ->where('id', $resourceId)->first(); CloudUserIsoService::finishJobLocked( CloudJob::findOrFail($childId), 'succeeded', [ 'id' => $resourceId, 'owner_id' => (int) $iso->user_id, 'name' => (string) $iso->name, 'size_bytes' => (int) $iso->size_bytes, 'sha256' => (string) $iso->sha256, ], ['user_iso_id' => $resourceId], [] ); } Capsule::table('cloud_jobs')->where('id', $childId)->update([ 'phase' => 'terminal', 'outcome' => 'succeeded', 'status' => 'succeeded', 'terminal_at' => time(), 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => 'applied', 'effect_resolved_at' => time(), 'effect_resolution_reason' => 'normal_completion', ]); try { CloudJobService::reconcileAggregateParentEffectLocked( CloudJob::findOrFail($childId), null ); throw new RuntimeException( 'Normal catalog settle accepted duplicate main records.' ); } catch (CloudRecoveryAggregateStateConflict $expected) { // Expected: the child projection exists, but the parent remains // fenced until its one-and-only main record is repairable. } $assert((string) CloudJob::findOrFail($parentId)->phase === 'awaiting_receipt', 'A duplicate main record completed its normal aggregate parent.'); } foreach ([ 'template_delete' => [ CloudTemplateCatalogService::class, 'template_delete_replica', 'cloud_templates', ], 'user_iso_delete' => [ CloudUserIsoService::class, 'user_iso_delete_replica', 'cloud_user_isos', ], ] as $parentKind => [$service, $childKind, $table]) { $parentId = $insertParent($parentKind); $childId = $insertResolvedChild($parentId, 'applied', $childKind); $resourceId = $insertProjectionForParent($parentId); Capsule::table('cloud_jobs')->where('id', $parentId)->update([ 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolution_reason' => '', 'error_code' => null, 'finalization_incident' => 0, 'finalization_error_code' => null, ]); Capsule::table('cloud_jobs')->where('id', $childId)->update([ 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolution_reason' => '', ]); $duplicate = (array) Capsule::table($table) ->where('id', $resourceId)->first(); $duplicate['id'] = $parentKind === 'template_delete' ? $resourceId . '-normal-duplicate' : md5('normal-delete-duplicate:' . $parentId); Capsule::table($table)->insert($duplicate); $resource = $parentKind === 'template_delete' ? \App\Models\CloudTemplate::findOrFail($resourceId) : \App\Models\CloudUserIso::findOrFail($resourceId); try { if ($parentKind === 'template_delete') { CloudTemplateCatalogService::settleRecoveredParentLocked( CloudJob::findOrFail($parentId), 'applied' ); } else { $settleDelete = new ReflectionMethod( $service, 'settleDeleteParent' ); $settleDelete->invoke( null, $resource, $parentId, ['effect_may_have_applied' => true], $childId ); } throw new RuntimeException( 'Normal delete settle accepted duplicate main records.' ); } catch (CloudRecoveryAggregateStateConflict $expected) { // Expected. } $assert((string) CloudJob::findOrFail($parentId)->phase === 'awaiting_receipt', 'A duplicate main record completed its normal delete parent.'); } // ISO identity reconstruction must bind both import and delete children to // the expected aggregate root, owner and active main record before any // normal receipt can reach catalog finalization. $expectIsoIdentityConflict = static function ( CloudJob $child, array $payload ): void { try { CloudOperationRecoveryIdentity::expectedFromPayload( $child, $payload ); throw new RuntimeException( 'Cross-wired ISO aggregate identity was accepted.' ); } catch (CloudRecoveryInstanceStateConflict $expected) { // Expected. } }; $identityImportParent = $insertParent('user_iso_import'); $identityImportChild = $insertResolvedChild( $identityImportParent, 'applied', 'user_iso_import_replica' ); $identityImportIso = $insertProjectionForParent($identityImportParent); Capsule::table('cloud_user_isos')->where('id', $identityImportIso) ->update(['active_job_id' => $identityImportParent + 900000]); $identityImportRow = Capsule::table('cloud_user_isos') ->where('id', $identityImportIso)->first(); $expectIsoIdentityConflict( CloudJob::findOrFail($identityImportChild), [ 'method' => 'POST', 'request_target' => '/v1/user-isos', 'body' => [ 'id' => $identityImportIso, 'owner_id' => 1, 'name' => (string) $identityImportRow->name, ], 'user_iso_id' => $identityImportIso, 'owner_id' => 1, ] ); $identityDeleteParent = $insertParent('user_iso_delete'); $identityDeleteChild = $insertResolvedChild( $identityDeleteParent, 'applied', 'user_iso_delete_replica' ); $identityDeleteIso = $insertProjectionForParent($identityDeleteParent); Capsule::table('cloud_jobs')->where('id', $identityDeleteParent) ->update(['user_id' => 2]); $expectIsoIdentityConflict( CloudJob::findOrFail($identityDeleteChild), [ 'method' => 'DELETE', 'request_target' => '/v1/user-isos/' . rawurlencode($identityDeleteIso) . '?owner_id=1', 'body' => [], 'user_iso_id' => $identityDeleteIso, 'owner_id' => 1, ] ); // A historical fail-open parent can already say succeeded/applied while a // child remains unsafe. Such a template must disappear from the public // catalog and be rejected by every provisioning metadata boundary until // the child ledger is repaired. $historicalParentId = $insertParent('template_import'); Capsule::table('cloud_jobs')->where('id', $historicalParentId)->update([ 'outcome' => 'succeeded', 'status' => 'succeeded', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => 'applied', 'effect_resolved_at' => $now - 1, 'effect_resolution_reason' => 'historical_fail_open', 'error_code' => null, 'finalization_incident' => 0, 'finalization_error_code' => null, ]); $historicalChildId = $insertResolvedChild( $historicalParentId, 'applied', 'template_import_replica' ); Capsule::table('cloud_jobs')->where('id', $historicalChildId)->update([ 'dispatch_attempts' => 2, ]); $historicalTemplateId = 'historical-fail-open-template'; $historicalGeneration = str_repeat('3', 32); $historicalSha = str_repeat('4', 64); Capsule::table('cloud_templates')->insert([ 'id' => $historicalTemplateId, 'name' => 'Historical fail open', 'os_family' => 'Debian', 'os_version' => '13', 'image_bytes' => 4096, 'sha256' => $historicalSha, 'startup_script' => '', 'revision' => 1, 'generation_token' => $historicalGeneration, 'state' => 'active', 'active_job_id' => $historicalParentId, 'created_at' => $now - 60, 'updated_at' => $now, ]); Capsule::table('cloud_template_replicas')->insert([ 'host_id' => 7, 'template_id' => $historicalTemplateId, 'revision' => 1, 'generation_token' => $historicalGeneration, 'state' => 'active', 'image_bytes' => 4096, 'sha256' => $historicalSha, 'operation_id' => (string) CloudJob::findOrFail($historicalChildId) ->operation_id, 'job_id' => $historicalChildId, 'error_code' => '', 'created_at' => $now - 60, 'updated_at' => $now, ]); $publicIds = array_column( CloudTemplateCatalogService::publicCatalog()['templates'], 'id' ); $assert(!in_array($historicalTemplateId, $publicIds, true), 'A fail-open template remained visible in the public catalog.'); $expectUnsafeTemplate = static function ( callable $callable, string $message ) use ($assert): void { try { $callable(); } catch (CloudError $error) { $assert($error->httpStatus === 409, $message); return; } throw new RuntimeException($message); }; $expectUnsafeTemplate( static fn (): array => CloudTemplateCatalogService::applyProvisioningSnapshot([ 'template_id' => $historicalTemplateId, ]), 'Provisioning consumed a fail-open template.' ); $expectUnsafeTemplate( static fn (): array => CloudTemplateCatalogService::templateMetadataSnapshot([ 'template_id' => $historicalTemplateId, 'template_revision' => 1, 'template_generation_token' => $historicalGeneration, ]), 'Metadata validation consumed a fail-open template.' ); $expectUnsafeTemplate( static function () use ($historicalTemplateId): void { CloudTemplateCatalogService::assertProvisionable( $historicalTemplateId ); }, 'Provisionability accepted a fail-open template.' ); Capsule::table('cloud_jobs')->where('id', $historicalChildId)->update([ 'dispatch_attempts' => 1, ]); $publicIds = array_column( CloudTemplateCatalogService::publicCatalog()['templates'], 'id' ); $snapshot = CloudTemplateCatalogService::applyProvisioningSnapshot([ 'template_id' => $historicalTemplateId, ]); $metadata = CloudTemplateCatalogService::templateMetadataSnapshot([ 'template_id' => $historicalTemplateId, 'template_revision' => 1, 'template_generation_token' => $historicalGeneration, ]); CloudTemplateCatalogService::assertProvisionable($historicalTemplateId); $assert(in_array($historicalTemplateId, $publicIds, true) && ($snapshot['template_id'] ?? null) === $historicalTemplateId && ($metadata['template_id'] ?? null) === $historicalTemplateId, 'A safely repaired template remained unavailable.'); // Uniform not-applied evidence is a safe rollback boundary. Imports retire // their unused placeholders; deletes restore their prior usable records; // metadata keeps the panel revision while old host revisions remain stale. foreach ([ 'template_import' => 'template_import_replica', 'template_metadata_sync' => 'template_metadata_replica', 'template_delete' => 'template_delete_replica', 'user_iso_import' => 'user_iso_import_replica', 'user_iso_delete' => 'user_iso_delete_replica', ] as $parentKind => $childKind) { $parentId = $insertParent($parentKind); $childId = $insertResolvedChild( $parentId, 'not_applied', $childKind ); $resourceId = $insertProjectionForParent($parentId); $parent = $markDeadlineAudit($parentId); $audit = $immutableParentAudit($parent); CloudJobService::reconcileAggregateParentEffectLocked( CloudJob::findOrFail($childId), null ); $parent->refresh(); $projectionOkay = match ($parentKind) { 'template_import' => (string) Capsule::table('cloud_templates') ->where('id', $resourceId)->value('state') === 'retired' && !Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId)->exists(), 'template_metadata_sync' => (string) Capsule::table('cloud_templates') ->where('id', $resourceId)->value('state') === 'active' && (string) Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId)->value('state') === 'stale' && (int) Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId)->value('revision') < (int) Capsule::table('cloud_templates') ->where('id', $resourceId)->value('revision'), 'template_delete' => (string) Capsule::table('cloud_templates') ->where('id', $resourceId)->value('state') === 'active' && (string) Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId)->value('state') === 'active', 'user_iso_import' => (string) Capsule::table('cloud_user_isos') ->where('id', $resourceId)->value('state') === 'retired' && (int) Capsule::table('cloud_user_isos') ->where('id', $resourceId)->value('size_bytes') === 0 && (string) Capsule::table('cloud_user_isos') ->where('id', $resourceId)->value('name') === '' && !Capsule::table('cloud_user_iso_replicas') ->where('user_iso_id', $resourceId)->exists(), 'user_iso_delete' => (string) Capsule::table('cloud_user_isos') ->where('id', $resourceId)->value('state') === 'active' && (string) Capsule::table('cloud_user_iso_replicas') ->where('user_iso_id', $resourceId)->value('state') === 'active', }; if ($parentKind === 'template_metadata_sync') { $publicIds = array_column( CloudTemplateCatalogService::publicCatalog()['templates'], 'id' ); $snapshotBlocked = false; try { CloudTemplateCatalogService::applyProvisioningSnapshot([ 'template_id' => $resourceId, ]); } catch (CloudError $error) { $snapshotBlocked = $error->httpStatus === 409; } $assertBlocked = false; try { CloudTemplateCatalogService::assertProvisionable($resourceId); } catch (CloudError $error) { $assertBlocked = $error->httpStatus === 409; } $projectionOkay = $projectionOkay && !in_array($resourceId, $publicIds, true) && $snapshotBlocked && $assertBlocked; } $assert($immutableParentAudit($parent) === $audit && (string) $parent->effect_resolution === 'not_applied' && !(bool) $parent->finalization_incident && (int) $parent->available_at === 0 && $projectionOkay, 'Uniform not-applied recovery did not unlock ' . $parentKind . '.'); } $insertMixedMetadata = static function ( bool $terminal, bool $legacy ) use ( $insertParent, $insertResolvedChild, $insertProjectionForParent, $makeLegacyEligible, $now ): array { $parentId = $insertParent('template_metadata_sync'); if (!$terminal) { Capsule::table('cloud_jobs')->where('id', $parentId)->update([ 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolution_reason' => '', 'error_code' => null, 'finalization_incident' => 0, 'finalization_error_code' => null, 'lease_token' => null, 'lease_until' => 0, 'available_at' => $now, ]); } elseif ($legacy) { $makeLegacyEligible($parentId); } $appliedId = $insertResolvedChild( $parentId, 'applied', 'template_metadata_replica' ); $notAppliedId = $insertResolvedChild( $parentId, 'not_applied', 'template_metadata_replica' ); $resourceId = $insertProjectionForParent($parentId); return [$parentId, $appliedId, $notAppliedId, $resourceId]; }; $assertMixedMetadata = static function ( int $parentId, string $resourceId, string $context ) use ($assert): void { $parent = CloudJob::findOrFail($parentId); $replicas = Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId)->orderBy('host_id')->get(); $publicIds = array_column( CloudTemplateCatalogService::publicCatalog()['templates'], 'id' ); $snapshot = CloudTemplateCatalogService::applyProvisioningSnapshot([ 'template_id' => $resourceId, ]); CloudTemplateCatalogService::assertProvisionable($resourceId); $assert((string) $parent->phase === 'terminal' && (string) $parent->outcome === 'failed' && (string) $parent->effect_resolution === 'applied' && !(bool) $parent->finalization_incident && (string) Capsule::table('cloud_templates') ->where('id', $resourceId)->value('state') === 'active' && $replicas->count() === 2 && $replicas->contains(static fn ($replica): bool => (string) $replica->state === 'active' && (int) $replica->revision === 2) && $replicas->contains(static fn ($replica): bool => (string) $replica->state === 'stale' && (int) $replica->revision === 1) && in_array($resourceId, $publicIds, true) && ($snapshot['template_id'] ?? null) === $resourceId, 'Mixed metadata did not converge at the ' . $context . ' seam.'); }; [$realtimeMetadataParent, $realtimeMetadataApplied, $_realtimeMetadataAbsent, $realtimeMetadataResource] = $insertMixedMetadata(false, false); CloudJobService::reconcileAggregateParentEffectLocked( CloudJob::findOrFail($realtimeMetadataApplied), null ); $assertMixedMetadata( $realtimeMetadataParent, $realtimeMetadataResource, 'realtime' ); [$terminalMetadataParent, $terminalMetadataApplied, $_terminalMetadataAbsent, $terminalMetadataResource] = $insertMixedMetadata(true, false); $terminalMetadataAudit = $immutableParentAudit( $markDeadlineAudit($terminalMetadataParent) ); CloudJobService::reconcileAggregateParentEffectLocked( CloudJob::findOrFail($terminalMetadataApplied), null ); $assertMixedMetadata( $terminalMetadataParent, $terminalMetadataResource, 'terminal deadline' ); $assert($immutableParentAudit( CloudJob::findOrFail($terminalMetadataParent) ) === $terminalMetadataAudit, 'Mixed metadata recovery rewrote the deadline parent audit.'); [$legacyMetadataParent, $_legacyMetadataApplied, $_legacyMetadataAbsent, $legacyMetadataResource] = $insertMixedMetadata(true, true); $assert(CloudJobService::reconcileOneLegacyAggregateEffect(), 'Legacy mixed metadata was not inspected.'); $assertMixedMetadata( $legacyMetadataParent, $legacyMetadataResource, 'legacy sweep' ); // Historical partial imports/deletes use the same evidence-only exit. The // legacy sweep must publish the mixed result and release the catalog // pointer; it must not leave upgraded installations permanently fenced. foreach ([ 'template_import' => 'template_import_replica', 'template_delete' => 'template_delete_replica', 'user_iso_import' => 'user_iso_import_replica', 'user_iso_delete' => 'user_iso_delete_replica', ] as $parentKind => $childKind) { $legacyPartialId = $insertParent($parentKind); $makeLegacyEligible($legacyPartialId); $insertResolvedChild($legacyPartialId, 'applied', $childKind); $insertResolvedChild($legacyPartialId, 'not_applied', $childKind); $legacyPartialResourceId = $insertProjectionForParent( $legacyPartialId ); $assert(CloudJobService::reconcileOneLegacyAggregateEffect(), 'Legacy partial aggregate was not inspected: ' . $parentKind); $legacyPartial = CloudJob::findOrFail($legacyPartialId); $resource = str_starts_with($parentKind, 'template_') ? Capsule::table('cloud_templates') ->where('id', $legacyPartialResourceId)->first() : Capsule::table('cloud_user_isos') ->where('id', $legacyPartialResourceId)->first(); $expectedState = str_ends_with($parentKind, '_import') ? (str_starts_with($parentKind, 'template_') ? 'degraded' : 'import_failed') : 'delete_failed'; $assert((string) $legacyPartial->effect_resolution === 'applied' && str_ends_with( (string) $legacyPartial->effect_resolution_reason, ':mixed' ) && !(bool) $legacyPartial->finalization_incident && (int) $legacyPartial->available_at === 0 && $resource !== null && (string) $resource->state === $expectedState && $resource->active_job_id === null, 'Legacy partial aggregate stayed fenced: ' . $parentKind); } // An all-applied import can exceed quota only after host work finished. // Keep it unavailable, expose an explicit cleanup capability, and reserve // a brand-new delete aggregate instead of replaying an import child. $quotaExistingIsoId = md5('quota-existing-active-iso'); Capsule::table('cloud_user_isos')->insert([ 'id' => $quotaExistingIsoId, 'user_id' => 1, 'name' => 'Existing quota usage', 'sealed_source_url' => 'test-only', 'size_bytes' => 6 * 1024 * 1024 * 1024, 'sha256' => hash('sha256', 'existing-quota-iso'), 'state' => 'active', 'import_operation_id' => md5('existing-quota-operation'), 'active_job_id' => null, 'created_at' => $now - 120, 'updated_at' => $now, ]); $quotaParentId = $insertParent('user_iso_import'); Capsule::table('cloud_jobs')->where('id', $quotaParentId)->update([ 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolution_reason' => '', 'error_code' => null, 'finalization_incident' => 0, 'finalization_error_code' => null, 'lease_token' => null, 'lease_until' => 0, 'available_at' => $now, 'receipt_deadline_at' => $now + 600, ]); $quotaChildId = $insertResolvedChild( $quotaParentId, 'applied', 'user_iso_import_replica' ); $quotaIsoId = $insertProjectionForParent($quotaParentId); $quotaRecoveredBytes = 5 * 1024 * 1024 * 1024; Capsule::table('cloud_user_isos')->where('id', $quotaIsoId)->update([ 'size_bytes' => $quotaRecoveredBytes, ]); Capsule::table('cloud_user_iso_replicas') ->where('user_iso_id', $quotaIsoId)->update([ 'size_bytes' => $quotaRecoveredBytes, ]); CloudJobService::reconcileAggregateParentEffectLocked( CloudJob::findOrFail($quotaChildId), null ); $quotaParent = CloudJob::findOrFail($quotaParentId); $quotaIso = Capsule::table('cloud_user_isos') ->where('id', $quotaIsoId)->first(); $quotaDto = collect(CloudUserIsoService::listing(1)['isos']) ->firstWhere('id', $quotaIsoId); $assert((string) $quotaParent->phase === 'terminal' && (string) $quotaParent->outcome === 'failed' && (string) $quotaParent->error_code === 'user_iso_quota_exceeded' && (string) $quotaParent->effect_resolution === 'applied' && !(bool) $quotaParent->finalization_incident && $quotaIso !== null && (string) $quotaIso->state === 'import_failed' && is_array($quotaDto) && ($quotaDto['can_delete'] ?? false) === true && ($quotaDto['cleanup_reason'] ?? null) === 'quota_exceeded_after_import', 'Quota-rejected applied ISO import was not exposed for safe cleanup.'); $cleanup = CloudUserIsoService::delete($quotaIsoId, ['user_id' => 1]); $cleanupParentId = (int) ($cleanup['task_id'] ?? 0); $cleanupParent = CloudJob::findOrFail($cleanupParentId); $quotaIso = Capsule::table('cloud_user_isos') ->where('id', $quotaIsoId)->first(); $cleanupChild = CloudJob::where('parent_job_id', $cleanupParentId)->first(); $assert($cleanupParentId > 0 && $cleanupParentId !== $quotaParentId && (string) $cleanupParent->kind === 'user_iso_delete' && (string) $cleanupParent->phase === 'awaiting_receipt' && $quotaIso !== null && (string) $quotaIso->state === 'deleting' && (int) $quotaIso->active_job_id === $cleanupParentId && $cleanupChild !== null && (string) $cleanupChild->kind === 'user_iso_delete_replica' && (string) Capsule::table('cloud_user_iso_replicas') ->where('user_iso_id', $quotaIsoId)->value('state') === 'deleting', 'Quota cleanup did not reserve an independent delete operation.'); $nextId = max( $nextId, (int) Capsule::table('cloud_jobs')->max('id') + 1 ); // A deadline keeps its immutable aggregate_deadline_exceeded audit even // when every late host later proves the import applied. If quota prevented // publication, that exact all-active projection must still be removable // by a new delete operation without relying on the parent's old error. $deadlineQuotaParentId = $insertParent('user_iso_import'); $deadlineQuotaAudit = $immutableParentAudit( $markDeadlineAudit($deadlineQuotaParentId) ); $deadlineQuotaChildId = $insertResolvedChild( $deadlineQuotaParentId, 'applied', 'user_iso_import_replica' ); $deadlineQuotaIsoId = $insertProjectionForParent( $deadlineQuotaParentId ); Capsule::table('cloud_user_isos')->where( 'id', $deadlineQuotaIsoId )->update(['size_bytes' => $quotaRecoveredBytes]); Capsule::table('cloud_user_iso_replicas')->where( 'user_iso_id', $deadlineQuotaIsoId )->update(['size_bytes' => $quotaRecoveredBytes]); CloudJobService::reconcileAggregateParentEffectLocked( CloudJob::findOrFail($deadlineQuotaChildId), null ); $deadlineQuotaParent = CloudJob::findOrFail($deadlineQuotaParentId); $deadlineQuotaDto = collect(CloudUserIsoService::listing(1)['isos']) ->firstWhere('id', $deadlineQuotaIsoId); $assert($immutableParentAudit($deadlineQuotaParent) === $deadlineQuotaAudit && (string) $deadlineQuotaParent->error_code === 'aggregate_deadline_exceeded' && (string) $deadlineQuotaParent->effect_resolution === 'applied' && !(bool) $deadlineQuotaParent->finalization_incident && (string) Capsule::table('cloud_user_isos') ->where('id', $deadlineQuotaIsoId)->value('state') === 'import_failed' && is_array($deadlineQuotaDto) && ($deadlineQuotaDto['can_delete'] ?? false) === true && ($deadlineQuotaDto['cleanup_reason'] ?? null) === 'quota_exceeded_after_import', 'Deadline quota recovery lost its audit or safe cleanup capability.'); // Projection failure rolls back atomically and delays only that parent. A // later valid parent is claimed by the next tick instead of being starved. $missingCatalogParentId = $insertParent('template_import'); $missingCatalogChildId = $insertResolvedChild( $missingCatalogParentId, 'applied', 'template_import_replica' ); $makeLegacyEligible($missingCatalogParentId); $missingCatalogAudit = $immutableParentAudit( CloudJob::findOrFail($missingCatalogParentId) ); $validAfterDamageParentId = $insertParent('template_import'); $insertResolvedChild( $validAfterDamageParentId, 'applied', 'template_import_replica' ); $validAfterDamageResource = $insertProjectionForParent( $validAfterDamageParentId ); $makeLegacyEligible($validAfterDamageParentId); $assert($automatic->tick(), 'The damaged projection was not isolated by the legacy sweep.'); $missingCatalogParent = CloudJob::findOrFail($missingCatalogParentId); $assert($immutableParentAudit($missingCatalogParent) === $missingCatalogAudit && (string) $missingCatalogParent->effect_resolution === 'unknown' && (bool) $missingCatalogParent->finalization_incident && (string) $missingCatalogParent->finalization_error_code === 'recovery_instance_state_conflict' && (int) $missingCatalogParent->available_at >= time() + 290 && CloudJob::findOrFail($missingCatalogChildId)->effect_resolution === 'applied', 'A missing catalog escaped or rewrote the terminal parent audit.'); $assert($automatic->tick(), 'A damaged projection starved the following valid aggregate.'); $validAfterDamageParent = CloudJob::findOrFail( $validAfterDamageParentId ); $assert((string) $validAfterDamageParent->effect_resolution === 'applied' && !(bool) $validAfterDamageParent->finalization_incident && (string) Capsule::table('cloud_templates') ->where('id', $validAfterDamageResource)->value('state') === 'active', 'The valid aggregate after a damaged parent did not converge.'); $projectionConflictFixtures = []; foreach ([ 'missing replica' => 'missing', 'extra replica' => 'extra', 'residual importing replica' => 'pending', 'cross-wired operation' => 'mapping', ] as $description => $mutation) { $parentId = $insertParent('template_import'); $insertResolvedChild( $parentId, 'applied', 'template_import_replica' ); $resourceId = $insertProjectionForParent($parentId); if ($mutation === 'missing') { Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId)->delete(); } elseif ($mutation === 'extra') { $extra = (array) Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId)->first(); $extra['host_id'] = 999000 + $parentId; $extra['job_id'] = null; $extra['operation_id'] = md5('extra-replica:' . $parentId); Capsule::table('cloud_template_replicas')->insert($extra); } elseif ($mutation === 'pending') { Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId) ->update(['state' => 'importing']); } else { Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId)->update([ 'operation_id' => md5('wrong-operation:' . $parentId), ]); } $makeLegacyEligible($parentId); $projectionConflictFixtures[$parentId] = [ 'description' => $description, 'code' => in_array($mutation, ['missing', 'extra'], true) ? 'recovery_aggregate_state_conflict' : ($mutation === 'mapping' ? 'recovery_operation_identity_conflict' : 'recovery_instance_state_conflict'), ]; } foreach ([ 'template generation mismatch' => 'generation', 'template import content mismatch' => 'content', ] as $description => $mutation) { $parentId = $insertParent('template_import'); $insertResolvedChild( $parentId, 'applied', 'template_import_replica' ); $resourceId = $insertProjectionForParent($parentId); Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId)->update( $mutation === 'generation' ? ['generation_token' => str_repeat('f', 32)] : [ 'image_bytes' => 1, 'sha256' => str_repeat('0', 64), ] ); $makeLegacyEligible($parentId); $projectionConflictFixtures[$parentId] = [ 'description' => $description, 'code' => 'recovery_instance_state_conflict', ]; } $metadataContentParent = $insertParent('template_metadata_sync'); $insertResolvedChild( $metadataContentParent, 'applied', 'template_metadata_replica' ); $insertResolvedChild( $metadataContentParent, 'not_applied', 'template_metadata_replica' ); $metadataContentResource = $insertProjectionForParent( $metadataContentParent ); Capsule::table('cloud_template_replicas') ->where('template_id', $metadataContentResource) ->where('state', 'active')->update([ 'image_bytes' => 1, 'sha256' => str_repeat('0', 64), ]); $makeLegacyEligible($metadataContentParent); $projectionConflictFixtures[$metadataContentParent] = [ 'description' => 'template metadata mixed content mismatch', 'code' => 'recovery_instance_state_conflict', ]; $templateDeleteContentParent = $insertParent('template_delete'); $insertResolvedChild( $templateDeleteContentParent, 'not_applied', 'template_delete_replica' ); $templateDeleteContentResource = $insertProjectionForParent( $templateDeleteContentParent ); Capsule::table('cloud_template_replicas') ->where('template_id', $templateDeleteContentResource)->update([ 'image_bytes' => 1, 'sha256' => str_repeat('0', 64), ]); $makeLegacyEligible($templateDeleteContentParent); $projectionConflictFixtures[$templateDeleteContentParent] = [ 'description' => 'template delete not-applied content mismatch', 'code' => 'recovery_instance_state_conflict', ]; foreach ([ 'ISO import parent operation mismatch' => 'parent_operation', 'ISO import content mismatch' => 'content', 'ISO active job cross-user binding' => 'user', ] as $description => $mutation) { $parentId = $insertParent('user_iso_import'); $insertResolvedChild( $parentId, 'applied', 'user_iso_import_replica' ); $resourceId = $insertProjectionForParent($parentId); if ($mutation === 'parent_operation') { Capsule::table('cloud_user_isos')->where('id', $resourceId) ->update([ 'import_operation_id' => md5( 'wrong-import-parent:' . $parentId ), ]); } elseif ($mutation === 'content') { Capsule::table('cloud_user_iso_replicas') ->where('user_iso_id', $resourceId)->update([ 'size_bytes' => 1, 'sha256' => str_repeat('0', 64), ]); } else { Capsule::table('cloud_user_isos')->where('id', $resourceId) ->update(['user_id' => 2]); } $makeLegacyEligible($parentId); $projectionConflictFixtures[$parentId] = [ 'description' => $description, 'code' => in_array( $mutation, ['parent_operation', 'user'], true ) ? 'recovery_operation_identity_conflict' : 'recovery_instance_state_conflict', ]; } $isoDeleteContentParent = $insertParent('user_iso_delete'); $insertResolvedChild( $isoDeleteContentParent, 'not_applied', 'user_iso_delete_replica' ); $isoDeleteContentResource = $insertProjectionForParent( $isoDeleteContentParent ); Capsule::table('cloud_user_iso_replicas') ->where('user_iso_id', $isoDeleteContentResource)->update([ 'size_bytes' => 1, 'sha256' => str_repeat('0', 64), ]); $makeLegacyEligible($isoDeleteContentParent); $projectionConflictFixtures[$isoDeleteContentParent] = [ 'description' => 'ISO delete not-applied content mismatch', 'code' => 'recovery_instance_state_conflict', ]; foreach ([ 'template duplicate main binding' => 'template_import', 'ISO duplicate main binding' => 'user_iso_import', ] as $description => $parentKind) { $childKind = $parentKind === 'template_import' ? 'template_import_replica' : 'user_iso_import_replica'; $parentId = $insertParent($parentKind); $insertResolvedChild($parentId, 'applied', $childKind); $resourceId = $insertProjectionForParent($parentId); if ($parentKind === 'template_import') { $duplicate = (array) Capsule::table('cloud_templates') ->where('id', $resourceId)->first(); $duplicate['id'] = $resourceId . '-duplicate'; $duplicate['state'] = 'error'; Capsule::table('cloud_templates')->insert($duplicate); } else { $duplicate = (array) Capsule::table('cloud_user_isos') ->where('id', $resourceId)->first(); $duplicate['id'] = md5('duplicate-main:' . $parentId); $duplicate['state'] = 'retired'; Capsule::table('cloud_user_isos')->insert($duplicate); } $makeLegacyEligible($parentId); $projectionConflictFixtures[$parentId] = [ 'description' => $description, 'code' => 'recovery_aggregate_state_conflict', ]; } foreach ([ 'template mixed effects forged all-success' => [ 'parent' => 'template_import', 'child' => 'template_import_replica', ], 'ISO delete mixed effects forged all-success' => [ 'parent' => 'user_iso_delete', 'child' => 'user_iso_delete_replica', ], ] as $description => $kinds) { $parentId = $insertParent($kinds['parent']); $insertResolvedChild($parentId, 'applied', $kinds['child']); $insertResolvedChild($parentId, 'not_applied', $kinds['child']); $resourceId = $insertProjectionForParent($parentId); if ($kinds['parent'] === 'template_import') { Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId)->update([ 'state' => 'active', 'image_bytes' => 4096, ]); } else { Capsule::table('cloud_user_iso_replicas') ->where('user_iso_id', $resourceId)->update([ 'state' => 'delete_succeeded', ]); } $makeLegacyEligible($parentId); $projectionConflictFixtures[$parentId] = [ 'description' => $description, 'code' => 'recovery_instance_state_conflict', ]; } foreach ($projectionConflictFixtures as $parentId => $expectation) { $assert($automatic->tick(), 'The projection conflict was not inspected: ' . $expectation['description']); $parent = CloudJob::findOrFail($parentId); $assert((string) $parent->effect_resolution === 'unknown' && (bool) $parent->finalization_incident && (string) $parent->finalization_error_code === $expectation['code'] && (int) $parent->available_at >= time() + 290, 'The projection conflict escaped its recovery fence: ' . $expectation['description']); } $assert(CloudAgentSettingsService::$requests === 0, 'Aggregate projection recovery replayed an Agent mutation.'); // Published parents with no pre-existing incident are still audited while // they remain the catalog's current active_job_id. Damage introduced // after the original settlement must become a sticky local incident; the // immutable parent/child audit is not rewritten and Agent is never called. foreach ([ 'mapping' => 'recovery_operation_identity_conflict', 'content' => 'recovery_instance_state_conflict', 'state' => 'recovery_instance_state_conflict', ] as $mutation => $expectedCode) { $parentId = $insertParent('template_import'); $childId = $insertResolvedChild( $parentId, 'applied', 'template_import_replica' ); $resourceId = $insertProjectionForParent($parentId); CloudJobService::reconcileAggregateParentEffectLocked( CloudJob::findOrFail($childId), null ); $publishedAudit = $immutableParentAudit( CloudJob::findOrFail($parentId) ); $damage = match ($mutation) { 'mapping' => [ 'operation_id' => md5('published-wrong-op:' . $parentId), ], 'content' => [ 'image_bytes' => 1, 'sha256' => str_repeat('0', 64), ], 'state' => ['state' => 'stale'], }; Capsule::table('cloud_template_replicas') ->where('template_id', $resourceId)->update($damage); Capsule::table('cloud_jobs')->where('id', '<>', $parentId)->update([ 'available_at' => time() + 3600, 'lease_until' => time() + 3600, ]); Capsule::table('cloud_jobs')->where('id', $parentId)->update([ 'available_at' => 0, 'lease_until' => 0, ]); $beforePublishedAuditRequests = CloudAgentSettingsService::$requests; $assert($automatic->tick(), 'A current published projection was not audited: ' . $mutation); $damaged = CloudJob::findOrFail($parentId); $assert($immutableParentAudit($damaged) === $publishedAudit && (string) $damaged->effect_resolution === 'applied' && (bool) $damaged->finalization_incident && (string) $damaged->finalization_error_code === $expectedCode && (int) $damaged->available_at >= time() + 290 && CloudAgentSettingsService::$requests === $beforePublishedAuditRequests, 'Published projection damage rewrote audit, disappeared, or contacted Agent: ' . $mutation); } // A resolved child in a nonterminal phase is a PHP-ledger contradiction, // not ordinary child convergence. It stays in the same 300-second local // loop with the concrete aggregate-state code. $phaseConflictParentId = $insertParent('template_import'); $phaseConflictChildId = $insertResolvedChild( $phaseConflictParentId, 'applied', 'template_import_replica' ); $insertProjectionForParent($phaseConflictParentId); CloudJobService::reconcileAggregateParentEffectLocked( CloudJob::findOrFail($phaseConflictChildId), null ); Capsule::table('cloud_jobs')->where('id', $phaseConflictChildId)->update([ 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'terminal_at' => null, ]); Capsule::table('cloud_jobs')->where( 'id', '<>', $phaseConflictParentId )->update([ 'available_at' => time() + 3600, 'lease_until' => time() + 3600, ]); Capsule::table('cloud_jobs')->where('id', $phaseConflictParentId)->update([ 'available_at' => 0, 'lease_until' => 0, ]); $phaseBefore = CloudJobService::recoveryPresentation( CloudJob::findOrFail($phaseConflictParentId), true ); $beforePhaseRequests = CloudAgentSettingsService::$requests; $assert(($phaseBefore['code'] ?? null) === 'recovery_aggregate_state_conflict' && $automatic->tick(), 'A nonterminal child with a final effect was not retained locally.'); $phaseConflict = CloudJob::findOrFail($phaseConflictParentId); $assert((bool) $phaseConflict->finalization_incident && (string) $phaseConflict->finalization_error_code === 'recovery_aggregate_state_conflict' && (int) $phaseConflict->available_at >= time() + 290 && CloudAgentSettingsService::$requests === $beforePhaseRequests, 'A nonterminal/final child lost its concrete conflict or contacted Agent.'); // Ordinary leaf recovery has priority over healthy periodic catalog audit, // and a successful audit advances available_at so it cannot hot-loop. $healthyParentId = $insertParent('template_import'); $healthyChildId = $insertResolvedChild( $healthyParentId, 'applied', 'template_import_replica' ); $insertProjectionForParent($healthyParentId); CloudJobService::reconcileAggregateParentEffectLocked( CloudJob::findOrFail($healthyChildId), null ); $priorityLeafId = $insertResolvedChild( 0, 'applied', 'template_import_replica' ); Capsule::table('cloud_jobs')->where('id', $priorityLeafId)->update([ 'dispatch_attempts' => 0, 'dispatched_at' => null, 'phase' => 'terminal', 'outcome' => 'failed', 'status' => 'failed', 'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0, 'effect_resolution' => 'unknown', 'finalization_incident' => 0, 'finalization_error_code' => null, ]); Capsule::table('cloud_jobs')->update([ 'available_at' => time() + 3600, 'lease_until' => time() + 3600, ]); Capsule::table('cloud_jobs')->whereIn('id', [ $healthyParentId, $priorityLeafId, ])->update(['available_at' => 0, 'lease_until' => 0]); $assert($automatic->tick(), 'The ordinary invalid leaf was not preferred over healthy catalog audit.'); $priorityLeaf = CloudJob::findOrFail($priorityLeafId); $healthyBeforeAudit = CloudJob::findOrFail($healthyParentId); $assert((string) $priorityLeaf->finalization_error_code === 'recovery_instance_state_conflict' && !(bool) $healthyBeforeAudit->finalization_incident && (int) $healthyBeforeAudit->available_at === 0, 'Healthy aggregate audit starved an ordinary leaf recovery claim.'); $assert($automatic->tick(), 'The healthy current aggregate was not periodically audited.'); $healthyAfterAudit = CloudJob::findOrFail($healthyParentId); $assert(!(bool) $healthyAfterAudit->finalization_incident && (int) $healthyAfterAudit->available_at >= time() + 290, 'A healthy published aggregate audit hot-looped or created an incident.'); // A success receipt arriving after the immutable deadline failure must // converge the original delete generation in place. It may update the // effect/catalog projection, but it must not flip either failed audit to // succeeded or contact the destructive Agent endpoint again. $lateDeleteParentId = $insertParent('template_delete'); $lateDeleteChildId = $insertResolvedChild( $lateDeleteParentId, 'not_applied', 'template_delete_replica' ); $lateDeleteTemplateId = $insertProjectionForParent( $lateDeleteParentId ); $lateDeleteChild = CloudJob::findOrFail($lateDeleteChildId); Capsule::table('cloud_jobs')->where('id', $lateDeleteChildId)->update([ 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'DELETE', 'request_target' => '/v1/templates/' . rawurlencode($lateDeleteTemplateId), 'body' => [ 'operation_id' => (string) $lateDeleteChild->operation_id, ], 'owner_id' => 1, 'can_manage' => true, 'template_id' => $lateDeleteTemplateId, ], 'job-payload'), 'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0, 'effect_resolution' => 'unknown', 'effect_resolved_at' => null, 'effect_resolution_reason' => '', 'completion_event_id' => null, 'finalization_incident' => 1, 'finalization_error_code' => 'recovery_receipt_invalid', 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $lateDeleteChild = CloudJob::findOrFail($lateDeleteChildId); $lateDeleteIdentity = CloudOperationRecoveryIdentity::expected( $lateDeleteChild, true ); $beforeLateDeleteRequests = CloudAgentSettingsService::$requests; $lateDeleteAccepted = CloudOperationCompletionService::accept([ 'type' => 'completion', 'id' => 'aggregate-late-template-delete', 'agent_id' => (string) $lateDeleteChild->host_id, 'operation_id' => (string) $lateDeleteChild->operation_id, 'action' => $lateDeleteIdentity['action'], 'resource_id' => $lateDeleteIdentity['resource_id'], 'request_fingerprint' => $lateDeleteIdentity['request_fingerprint'], 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => [ 'resource_type' => 'template', 'id' => $lateDeleteTemplateId, 'deleted' => true, 'managed_image_absent' => true, 'managed_metadata_absent' => true, 'managed_artifacts_absent' => true, ], ]); $lateDeleteChild = CloudJob::findOrFail($lateDeleteChildId); $lateDeleteParent = CloudJob::findOrFail($lateDeleteParentId); $assert($lateDeleteAccepted === [ 'accepted' => true, 'disposition' => 'applied', ] && (string) $lateDeleteChild->outcome === 'failed' && (string) $lateDeleteChild->effect_resolution === 'applied' && !(bool) $lateDeleteChild->finalization_incident && (string) $lateDeleteParent->outcome === 'failed' && (string) $lateDeleteParent->error_code === 'aggregate_deadline_exceeded' && (string) $lateDeleteParent->effect_resolution === 'unknown' && (bool) $lateDeleteParent->finalization_incident && (string) $lateDeleteParent->finalization_error_code === 'recovery_child_convergence' && (string) Capsule::table('cloud_templates') ->where('id', $lateDeleteTemplateId)->value('state') === 'delete_failed' && Capsule::table('cloud_template_replicas') ->where('template_id', $lateDeleteTemplateId)->exists() && (string) Capsule::table('cloud_template_artifacts') ->where('template_id', $lateDeleteTemplateId)->value('state') === 'delete_pending' && CloudAgentSettingsService::$requests === $beforeLateDeleteRequests, 'A late template deletion receipt did not converge the original ' . 'host operation into the two-phase master cleanup fence.'); $lateIsoParentId = $insertParent('user_iso_delete'); $lateIsoChildId = $insertResolvedChild( $lateIsoParentId, 'not_applied', 'user_iso_delete_replica' ); $lateIsoId = $insertProjectionForParent($lateIsoParentId); $lateIsoChild = CloudJob::findOrFail($lateIsoChildId); Capsule::table('cloud_jobs')->where('id', $lateIsoChildId)->update([ 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'DELETE', 'request_target' => '/v1/user-isos/' . rawurlencode($lateIsoId) . '?owner_id=1', 'body' => [ 'operation_id' => (string) $lateIsoChild->operation_id, ], 'owner_id' => 1, 'can_manage' => false, 'user_iso_id' => $lateIsoId, ], 'job-payload'), 'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0, 'effect_resolution' => 'unknown', 'effect_resolved_at' => null, 'effect_resolution_reason' => '', 'completion_event_id' => null, 'finalization_incident' => 1, 'finalization_error_code' => 'recovery_receipt_invalid', 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $lateIsoChild = CloudJob::findOrFail($lateIsoChildId); $lateIsoIdentity = CloudOperationRecoveryIdentity::expected( $lateIsoChild, true ); $lateIsoAccepted = CloudOperationCompletionService::accept([ 'type' => 'completion', 'id' => 'aggregate-late-user-iso-delete', 'agent_id' => (string) $lateIsoChild->host_id, 'operation_id' => (string) $lateIsoChild->operation_id, 'action' => $lateIsoIdentity['action'], 'resource_id' => $lateIsoIdentity['resource_id'], 'request_fingerprint' => $lateIsoIdentity['request_fingerprint'], 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => [ 'resource_type' => 'user_iso', 'id' => $lateIsoId, 'owner_id' => 1, 'deleted' => true, 'managed_image_absent' => true, 'managed_metadata_absent' => true, 'managed_artifacts_absent' => true, ], ]); $lateIsoChild = CloudJob::findOrFail($lateIsoChildId); $lateIsoParent = CloudJob::findOrFail($lateIsoParentId); $lateIso = Capsule::table('cloud_user_isos') ->where('id', $lateIsoId)->first(); $assert($lateIsoAccepted === [ 'accepted' => true, 'disposition' => 'applied', ] && (string) $lateIsoChild->outcome === 'failed' && (string) $lateIsoChild->effect_resolution === 'applied' && !(bool) $lateIsoChild->finalization_incident && (string) $lateIsoParent->outcome === 'failed' && (string) $lateIsoParent->effect_resolution === 'applied' && !(bool) $lateIsoParent->finalization_incident && $lateIso !== null && (string) $lateIso->state === 'retired' && (string) $lateIso->name === '' && (int) $lateIso->size_bytes === 0 && !Capsule::table('cloud_user_iso_replicas') ->where('user_iso_id', $lateIsoId)->exists() && CloudAgentSettingsService::$requests === $beforeLateDeleteRequests, 'A late user-ISO deletion receipt did not converge its owner-bound ' . 'catalog ledgers without replay.'); echo "Cloud aggregate recovery convergence tests passed.\n"; }