httpStatus === 409, 'Unexpected recovery state-conflict status.'); return; } throw new RuntimeException( 'A dedicated recovery kind entered generic worker recovery.' ); }; $manifestPath = dirname(__DIR__, 2) . '/agent/operation/recovery_contract.json'; $manifest = json_decode( file_get_contents($manifestPath), true, 64, JSON_THROW_ON_ERROR ); $assert(($manifest['schema_version'] ?? null) === 1, 'Unsupported recovery contract schema version.'); $manifestOperations = []; foreach ($manifest['operations'] ?? [] as $entry) { $kind = $entry['kind'] ?? null; $assert(is_string($kind) && $kind !== '' && !array_key_exists($kind, $manifestOperations), 'Recovery contract contains a missing or duplicate kind.'); unset($entry['kind']); $manifestOperations[$kind] = $entry; } $assert(CloudOperationRecoveryContract::OPERATIONS === $manifestOperations, 'Generated PHP recovery contract is stale.'); $capsule = new Capsule(); $capsule->addConnection([ 'driver' => 'sqlite', 'database' => ':memory:', ], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('operation_id')->unique(); $table->string('request_idempotency_key')->unique(); $table->string('request_fingerprint'); $table->integer('parent_job_id')->nullable(); $table->integer('order_id')->nullable(); $table->integer('invoice_id')->nullable(); $table->integer('instance_id')->nullable(); $table->integer('snapshot_id')->nullable(); $table->integer('backup_id')->nullable(); $table->integer('backup_restore_id')->nullable(); $table->integer('user_id'); $table->integer('host_id')->nullable(); $table->string('kind'); $table->string('execution_mode'); $table->string('phase'); $table->string('outcome')->nullable(); $table->string('status'); $table->text('sealed_payload'); $table->integer('attempts')->default(1); $table->integer('dispatch_attempts')->default(1); $table->integer('dispatch_deadline_at')->nullable(); $table->integer('dispatch_started_at')->nullable(); $table->integer('agent_accepted_at')->nullable(); $table->integer('receipt_deadline_at')->nullable(); $table->integer('dispatched_at')->nullable(); $table->integer('terminal_at')->nullable(); $table->boolean('effect_may_have_applied')->nullable(); $table->boolean('safe_to_resubmit')->nullable(); $table->string('effect_resolution')->default('not_applicable'); $table->integer('effect_resolved_at')->nullable(); $table->integer('effect_resolved_by')->nullable(); $table->string('effect_resolution_reason')->default(''); $table->string('completion_event_id')->nullable(); $table->integer('read_at')->nullable(); $table->string('lease_token')->nullable(); $table->integer('lease_until')->default(0); $table->integer('available_at')->default(0); $table->string('error_code')->nullable(); $table->boolean('finalization_incident')->default(0); $table->string('finalization_error_code')->nullable(); $table->text('result_json')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_instances', static function (Blueprint $table): void { $table->increments('id'); $table->string('uuid'); $table->integer('host_id'); $table->integer('user_id'); $table->integer('reservation_id')->nullable(); $table->string('name'); $table->string('status'); $table->string('service_status')->default('exempt'); $table->string('network_status')->default('active'); $table->text('spec_json')->default('{}'); $table->text('sealed_credential')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_instance_snapshots', static function (Blueprint $table): void { $table->increments('id'); $table->string('snapshot_key')->unique(); $table->integer('instance_id'); $table->integer('user_id'); $table->integer('host_id'); $table->string('name'); $table->string('state'); $table->string('consistency'); $table->integer('size_bytes')->nullable(); $table->string('last_operation_id')->nullable(); $table->integer('ready_at')->nullable(); $table->integer('last_restored_at')->nullable(); $table->integer('deleted_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_instance_backups', static function (Blueprint $table): void { $table->increments('id'); $table->string('backup_key')->unique(); $table->integer('source_instance_id'); $table->integer('source_host_id'); $table->integer('user_id'); $table->string('source_uuid'); $table->string('name'); $table->string('purpose')->default('portable'); $table->string('state'); $table->string('consistency'); $table->string('object_key'); $table->integer('object_size_bytes')->default(0); $table->integer('virtual_size_bytes')->default(0); $table->string('sha256')->default(''); $table->text('spec_json'); $table->text('sealed_storage_runtime'); $table->text('sealed_credential')->nullable(); $table->integer('storage_revision')->default(1); $table->integer('active_job_id')->nullable(); $table->string('last_operation_id')->nullable(); $table->string('error_code')->default(''); $table->integer('ready_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_backup_restores', static function (Blueprint $table): void { $table->increments('id'); $table->string('restore_key')->unique(); $table->integer('backup_id'); $table->integer('target_host_id'); $table->integer('target_instance_id')->nullable(); $table->integer('reservation_id')->nullable(); $table->integer('job_id')->nullable(); $table->string('operation_id')->unique(); $table->string('new_uuid'); $table->string('new_name'); $table->string('state'); $table->string('error_code')->default(''); $table->integer('ready_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_instance_migrations', static function (Blueprint $table): void { $table->increments('id'); $table->string('migration_key')->unique(); $table->integer('instance_id'); $table->integer('backup_id'); $table->integer('restore_id')->nullable(); $table->integer('source_host_id'); $table->string('source_uuid'); $table->integer('source_reservation_id'); $table->integer('target_host_id'); $table->string('target_uuid'); $table->integer('target_reservation_id')->nullable(); $table->integer('backup_job_id'); $table->integer('restore_job_id')->nullable(); $table->integer('firewall_job_id')->nullable(); $table->integer('activate_job_id')->nullable(); $table->integer('rollback_job_id')->nullable(); $table->integer('cleanup_job_id')->nullable(); $table->string('cleanup_mode'); $table->string('state'); $table->boolean('source_was_running')->nullable(); $table->boolean('source_autostart_enabled')->nullable(); $table->string('source_service_status')->default('active'); $table->string('source_network_status')->default('active'); $table->string('error_code')->default(''); $table->integer('cutover_at')->nullable(); $table->integer('completed_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_resource_reservations', static function (Blueprint $table): void { $table->increments('id'); $table->integer('backup_restore_id'); $table->integer('host_id'); $table->integer('user_id'); $table->string('operation_id'); $table->integer('public_ip_id'); $table->integer('instance_id')->nullable(); $table->integer('cpu_cores'); $table->integer('memory_mib'); $table->integer('disk_gib'); $table->string('state'); $table->integer('expires_at')->nullable(); $table->integer('released_at')->nullable(); $table->string('release_reason')->default(''); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_public_ips', static function (Blueprint $table): void { $table->increments('id'); $table->integer('host_id'); $table->string('address'); $table->integer('prefix_length'); $table->string('state'); $table->integer('reservation_id')->nullable(); $table->integer('instance_id')->nullable(); $table->string('operation_id')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_instance_firewall_versions', static function (Blueprint $table): void { $table->increments('id'); $table->integer('instance_id'); $table->integer('revision'); $table->boolean('enabled')->default(1); $table->string('policy_digest'); $table->string('request_hash'); $table->string('idempotency_key'); $table->string('state'); $table->integer('job_id')->nullable(); $table->string('operation_id')->nullable(); $table->string('error_code')->nullable(); $table->integer('applied_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_templates', static function (Blueprint $table): void { $table->string('id')->primary(); $table->string('name'); $table->integer('revision'); $table->string('generation_token'); $table->string('state'); $table->integer('active_job_id')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_template_replicas', static function (Blueprint $table): void { $table->integer('host_id'); $table->string('template_id'); $table->integer('revision'); $table->string('generation_token'); $table->string('state'); $table->integer('image_bytes')->default(0); $table->string('sha256')->default(''); $table->string('operation_id'); $table->integer('job_id')->nullable(); $table->string('error_code')->default(''); $table->integer('created_at'); $table->integer('updated_at'); $table->primary(['host_id', 'template_id']); }); $schema->create('cloud_rescue_template_imports', static function (Blueprint $table): void { $table->string('operation_id')->primary(); $table->integer('host_id'); $table->string('template_id'); $table->integer('job_id'); $table->string('import_type'); $table->string('name'); $table->integer('size_bytes')->default(0); $table->string('sha256')->default(''); $table->string('state'); $table->string('error_code')->default(''); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_rescue_template_deletions', static function (Blueprint $table): void { $table->integer('host_id'); $table->string('template_id'); $table->string('operation_id'); $table->integer('job_id'); $table->string('state'); $table->string('error_code')->default(''); $table->integer('created_at'); $table->integer('updated_at'); $table->primary(['host_id', 'template_id']); }); $schema->create('cloud_user_isos', static function (Blueprint $table): void { $table->string('id')->primary(); $table->integer('user_id'); $table->string('name'); $table->string('state'); $table->integer('active_job_id')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_user_iso_replicas', static function (Blueprint $table): void { $table->integer('host_id'); $table->string('user_iso_id'); $table->integer('user_id'); $table->string('state'); $table->integer('size_bytes')->default(0); $table->string('sha256')->default(''); $table->string('operation_id'); $table->integer('job_id')->nullable(); $table->string('error_code')->default(''); $table->integer('created_at'); $table->integer('updated_at'); $table->primary(['host_id', 'user_iso_id']); }); $schema->create('cloud_host_storage_configurations', static function (Blueprint $table): void { $table->integer('host_id')->primary(); $table->integer('revision'); foreach (['managed', 'template', 'seed', 'user_iso', 'rescue'] as $role) { $table->text($role . '_directory'); } $table->string('state'); $table->string('operation_id')->nullable(); $table->integer('job_id')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $now = time(); $hostId = 7; $ownerId = 41; $instanceId = 1; $instanceUuid = '123e4567-e89b-42d3-a456-426614174000'; Capsule::table('cloud_instances')->insert([ 'id' => $instanceId, 'uuid' => $instanceUuid, 'host_id' => $hostId, 'user_id' => $ownerId, 'name' => 'matrix-vm', 'status' => 'active', 'service_status' => 'exempt', 'network_status' => 'active', 'spec_json' => '{}', 'created_at' => $now - 3600, 'updated_at' => $now - 3600, ]); $nextJobId = 1; /** * @return array{id:int,operation_id:string,job:CloudJob,payload:array} */ $insertJob = static function ( string $kind, callable $payloadFactory, ?int $boundInstanceId = null, ?int $snapshotId = null, ?int $parentJobId = null, ?int $orderId = null, string $executionMode = 'agent', int $userId = 41, ?int $backupId = null, ?int $backupRestoreId = null ) use (&$nextJobId, $hostId, $now): array { $id = $nextJobId++; $operationId = hash('md5', 'recovery-matrix-job:' . $id); $payload = $payloadFactory($id, $operationId); Capsule::table('cloud_jobs')->insert([ 'id' => $id, 'operation_id' => $operationId, 'request_idempotency_key' => 'recovery-matrix-' . $operationId, 'request_fingerprint' => hash('sha256', $operationId), 'parent_job_id' => $parentJobId, 'order_id' => $orderId, 'invoice_id' => null, 'instance_id' => $boundInstanceId, 'snapshot_id' => $snapshotId, 'backup_id' => $backupId, 'backup_restore_id' => $backupRestoreId, 'user_id' => $userId, 'host_id' => $hostId, 'kind' => $kind, 'execution_mode' => $executionMode, 'phase' => 'terminal', 'outcome' => 'failed', 'status' => 'failed', 'sealed_payload' => CloudSecrets::encrypt( $payload, 'job-payload' ), 'attempts' => 1, 'dispatch_attempts' => 1, 'dispatch_deadline_at' => $now - 120, 'dispatch_started_at' => $now - 110, 'agent_accepted_at' => $now - 100, 'receipt_deadline_at' => $now - 30, 'dispatched_at' => $now - 110, 'terminal_at' => $now - 20, 'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0, 'effect_resolution' => 'unknown', 'effect_resolution_reason' => '', 'lease_until' => 0, 'available_at' => 0, 'finalization_incident' => 0, 'created_at' => $now - 180, 'updated_at' => $now - 20, ]); return [ 'id' => $id, 'operation_id' => $operationId, 'job' => CloudJob::findOrFail($id), 'payload' => $payload, ]; }; $fixtures = []; $addFixture = static function ( string $kind, string $variant, array $record, string $expectedAction, string $expectedResource, array $projection ) use (&$fixtures): void { $fixtures[$kind][] = $record + [ 'variant' => $variant, 'expected_action' => $expectedAction, 'expected_resource' => $expectedResource, 'projection' => $projection, ]; }; $instanceFixture = static function ( string $kind, string $variant, string $method, string $suffix, array $body, string $expectedAction, array $projection, array $metadata = [] ) use ( $insertJob, $addFixture, $instanceId, $instanceUuid, $ownerId ): array { $record = $insertJob( $kind, static function (int $id, string $operationId) use ( $method, $suffix, $body, $metadata, $instanceUuid, $ownerId ): array { $resolvedBody = $body; foreach ($resolvedBody as $field => $value) { if ($value === '__OPERATION_ID__') { $resolvedBody[$field] = $operationId; } } return $metadata + [ 'method' => $method, 'request_target' => '/v1/vms/' . rawurlencode($instanceUuid) . $suffix, 'body' => $resolvedBody, 'owner_id' => $ownerId, 'can_manage' => false, ]; }, $instanceId ); $addFixture( $kind, $variant, $record, $expectedAction, $instanceUuid, $projection ); return $record; }; $powerProjection = ['uuid' => $instanceUuid, 'state' => 'running']; foreach ([ 'start' => 'power-start', 'shutdown' => 'power-shutdown', 'reboot' => 'power-reboot', 'reset' => 'power-reset', 'poweroff' => 'power-destroy', ] as $panelAction => $wireAction) { $instanceFixture( 'action', $panelAction, 'POST', '/actions', ['action' => $panelAction], $wireAction, $powerProjection ); } $instanceFixture( 'reinstall', 'default', 'POST', '/reinstall', [ 'template_id' => 'debian-13', 'template_revision' => 4, 'template_generation_token' => str_repeat('a', 32), 'preserve_data' => false, 'root_password' => 'MatrixPassword!123', ], 'reinstall-vm', ['uuid' => $instanceUuid, 'reinstall_committed' => true] ); $instanceFixture( 'rescue', 'enter', 'POST', '/rescue/enter', ['template_id' => 'rescue-debian-13'], 'enter-rescue', ['active' => true, 'staged' => true, 'network_suspended' => false] ); $instanceFixture( 'rescue', 'force-start', 'POST', '/rescue/force-start', ['ack_data_loss' => true], 'force-start-rescue', ['active' => true, 'staged' => true, 'network_suspended' => false] ); foreach ([ ['network_suspended' => false, 'state' => 'connected'], ['network_suspended' => true, 'state' => 'suspended'], ] as $rescueExit) { $networkSuspended = $rescueExit['network_suspended']; $state = $rescueExit['state']; $instanceFixture( 'rescue', 'exit-' . $state, 'POST', '/rescue/exit', [ 'network_suspended' => $networkSuspended, 'ack_data_loss' => true, ], 'exit-rescue:' . $state, [ 'active' => false, 'staged' => false, 'network_suspended' => $networkSuspended, ] ); } $deleteProjection = [ 'deleted' => true, 'managed_artifacts_absent' => true, 'forced' => false, ]; $instanceFixture( 'delete', 'user-delete', 'DELETE', '', ['remove_managed_disks' => true], 'delete-vm', $deleteProjection ); $instanceFixture( 'expiry_delete', 'expiry-delete', 'DELETE', '', ['remove_managed_disks' => true], 'delete-vm', $deleteProjection ); $priorOperationId = str_repeat('b', 32); $instanceFixture( 'delete_force', 'force-delete', 'POST', '/force-delete', [ 'operation_id' => '__OPERATION_ID__', 'prior_operation_id' => $priorOperationId, 'ack_data_loss' => true, 'remove_managed_disks' => true, ], 'force-delete-vm', array_replace($deleteProjection, ['forced' => true]), ['prior_operation_id' => $priorOperationId] ); $migrationCleanup = $insertJob( 'instance_migration_cleanup', static fn (int $id, string $operationId): array => [ 'method' => 'DELETE', 'request_target' => '/v1/vms/' . rawurlencode($instanceUuid), 'body' => ['remove_managed_disks' => true], 'owner_id' => $ownerId, 'can_manage' => true, 'maintenance_migration_id' => 1, ], $instanceId ); $migrationRollback = $insertJob( 'instance_migration_rollback', static fn (int $id, string $operationId): array => [ 'method' => 'POST', 'request_target' => '/v1/vms/' . rawurlencode($instanceUuid) . '/service/resume', 'body' => [ 'operation_id' => $operationId, 'reason' => 'maintenance_migration_rollback', 'start_if_suspended' => true, ], 'owner_id' => $ownerId, 'can_manage' => true, 'maintenance_migration_id' => 1, ], $instanceId ); $migrationTargetUuid = '123e4567-e89b-42d3-a456-426614174099'; $migrationActivate = $insertJob( 'instance_migration_activate', static fn (int $id, string $operationId): array => [ 'method' => 'POST', 'request_target' => '/v1/vms/' . rawurlencode($migrationTargetUuid) . '/service/resume', 'body' => [ 'operation_id' => $operationId, 'reason' => 'maintenance_migration_activate', 'start_if_suspended' => true, ], 'owner_id' => $ownerId, 'can_manage' => true, 'maintenance_migration_id' => 1, ], $instanceId ); Capsule::table('cloud_instance_migrations')->insert([ 'id' => 1, 'migration_key' => str_repeat('e', 32), 'instance_id' => $instanceId, 'backup_id' => 9001, 'restore_id' => 9002, 'source_host_id' => $hostId, 'source_uuid' => $instanceUuid, 'source_reservation_id' => 9003, 'target_host_id' => $hostId, 'target_uuid' => $migrationTargetUuid, 'target_reservation_id' => 9004, 'backup_job_id' => 9005, 'restore_job_id' => 9006, 'firewall_job_id' => null, 'activate_job_id' => $migrationActivate['id'], 'rollback_job_id' => $migrationRollback['id'], 'cleanup_job_id' => $migrationCleanup['id'], 'cleanup_mode' => 'delete', 'state' => 'cleanup_uncertain', 'source_was_running' => true, 'source_autostart_enabled' => true, 'source_service_status' => 'active', 'source_network_status' => 'active', 'error_code' => 'operation_effect_unknown', 'cutover_at' => $now - 60, 'completed_at' => null, 'created_at' => $now - 300, 'updated_at' => $now - 20, ]); $addFixture( 'instance_migration_cleanup', 'source-cleanup', $migrationCleanup, 'delete-vm', $instanceUuid, $deleteProjection ); $addFixture( 'instance_migration_rollback', 'source-rollback', $migrationRollback, 'service-resume', $instanceUuid, ['uuid' => $instanceUuid, 'state' => 'running'] ); $addFixture( 'instance_migration_activate', 'target-activate', $migrationActivate, 'service-resume', $migrationTargetUuid, ['uuid' => $migrationTargetUuid, 'state' => 'running'] ); $instanceFixture( 'network_change', 'public-ip-change', 'POST', '/network', [ 'network' => 'bridge:br0', 'public_pool_key' => str_repeat('c', 32), 'public_pool_cidr' => '203.0.113.0/24', 'public_ip_cidr' => '203.0.113.20/24', 'gateway' => '203.0.113.1', 'dns_servers' => ['1.1.1.1'], 'confirm_restart' => true, ], 'change-network', ['uuid' => $instanceUuid, 'network_changed' => true] ); $instanceFixture( 'instance_settings', 'hostname', 'PATCH', '/settings', [ 'hostname' => 'matrix.example', 'confirm_restart' => true, ], 'update-settings', ['uuid' => $instanceUuid, 'settings_changed' => true], ['settings_fields' => ['hostname'], 'credential' => null] ); $upgradeProjection = [ 'uuid' => $instanceUuid, 'from_vcpu' => 2, 'target_vcpu' => 4, 'applied_vcpu' => 4, 'from_memory_mib' => 2048, 'target_memory_mib' => 4096, 'applied_memory_mib' => 4096, 'from_disk_gib' => 40, 'target_disk_gib' => 80, 'applied_disk_gib' => 80, 'from_bandwidth_mbps' => 100, 'target_bandwidth_mbps' => 200, 'applied_bandwidth_mbps' => 200, 'status' => 'running', 'originally_running' => true, 'power_state_restored' => true, 'resources_resized' => true, ]; $upgradeRecord = $instanceFixture( 'instance_upgrade', 'grow-only', 'POST', '/resources', [ 'from_vcpu' => 2, 'target_vcpu' => 4, 'from_memory_mib' => 2048, 'target_memory_mib' => 4096, 'from_disk_gib' => 40, 'target_disk_gib' => 80, 'from_bandwidth_mbps' => 100, 'target_bandwidth_mbps' => 200, 'confirm_shutdown' => true, ], 'resize-vm', $upgradeProjection ); $snapshotFixture = static function ( string $kind, string $variant, string $method, string $suffix, array $body, string $expectedAction, array $projection, string $consistency = 'crash' ) use ( $insertJob, $addFixture, $instanceId, $instanceUuid, $ownerId, $hostId, $now ): void { static $snapshotId = 100; $id = $snapshotId++; $key = hash('md5', 'recovery-matrix-snapshot:' . $id); $record = $insertJob( $kind, static function (int $jobId, string $operationId) use ( $kind, $method, $suffix, $body, $key, $instanceUuid, $ownerId ): array { return [ 'method' => $method, 'request_target' => '/v1/vms/' . rawurlencode($instanceUuid) . '/snapshots' . ($kind === 'snapshot_create' ? '' : '/' . rawurlencode($key) . $suffix), 'body' => $body + ($body === [] ? [] : [ 'snapshot_key' => $key, ]), 'owner_id' => $ownerId, 'can_manage' => false, 'snapshot_key' => $key, 'snapshot_previous_state' => 'ready', ]; }, $instanceId, $id ); Capsule::table('cloud_instance_snapshots')->insert([ 'id' => $id, 'snapshot_key' => $key, 'instance_id' => $instanceId, 'user_id' => $ownerId, 'host_id' => $hostId, 'name' => 'matrix-snapshot-' . $id, 'state' => 'uncertain', 'consistency' => $consistency, 'size_bytes' => null, 'last_operation_id' => $record['operation_id'], 'ready_at' => $now - 300, 'last_restored_at' => null, 'deleted_at' => null, 'created_at' => $now - 600, 'updated_at' => $now - 20, ]); $resolvedAction = str_replace('__SNAPSHOT_KEY__', $key, $expectedAction); $resolvedProjection = array_map( static fn (mixed $value): mixed => $value === '__SNAPSHOT_KEY__' ? $key : $value, $projection ); $addFixture( $kind, $variant, $record, $resolvedAction, $instanceUuid, $resolvedProjection ); }; foreach (['filesystem', 'crash'] as $consistency) { $snapshotFixture( 'snapshot_create', $consistency, 'POST', '', [ 'name' => 'matrix-' . $consistency, 'description' => '', 'quiesce' => $consistency === 'filesystem', 'consistency_mode' => $consistency, ], 'create-snapshot:__SNAPSHOT_KEY__:' . $consistency, [ 'id' => 'matrix-id', 'snapshot_key' => '__SNAPSHOT_KEY__', 'created_at' => '2026-10-21T00:00:00Z', 'state' => 'ready', 'consistency' => $consistency, ], $consistency ); } $snapshotFixture( 'snapshot_restore', 'restore', 'POST', '/restore', [], 'restore-snapshot:__SNAPSHOT_KEY__', ['snapshot_key' => '__SNAPSHOT_KEY__', 'restored' => true] ); $snapshotFixture( 'snapshot_delete', 'delete', 'DELETE', '', [], 'delete-snapshot:__SNAPSHOT_KEY__', ['snapshot_key' => '__SNAPSHOT_KEY__', 'deleted' => true] ); foreach (['filesystem', 'crash'] as $consistency) { $backupId = $consistency === 'filesystem' ? 201 : 202; $backupKey = hash('md5', 'recovery-matrix-backup:' . $backupId); $record = $insertJob( 'backup_create', static function (int $jobId, string $operationId) use ( $instanceUuid, $ownerId, $backupKey, $consistency ): array { return [ 'method' => 'POST', 'request_target' => '/v1/vms/' . rawurlencode($instanceUuid) . '/backups', 'body' => [ 'operation_id' => $operationId, 'backup_key' => $backupKey, 'name' => 'matrix-' . $consistency, 'consistency_mode' => $consistency, 'object_upload' => [ 'part_size_bytes' => 8388608, 'maximum_size_bytes' => 42949672960, 'part_urls' => [ 'https://objects.invalid/part-1', ], 'complete_url' => 'https://objects.invalid/complete', 'abort_url' => 'https://objects.invalid/abort', 'verify_url' => 'https://objects.invalid/verify', ], ], 'owner_id' => $ownerId, 'can_manage' => true, 'backup_key' => $backupKey, 'maximum_size_bytes' => 42949672960, ]; }, $instanceId, null, null, null, 'agent', $ownerId, $backupId ); Capsule::table('cloud_instance_backups')->insert([ 'id' => $backupId, 'backup_key' => $backupKey, 'source_instance_id' => $instanceId, 'source_host_id' => $hostId, 'user_id' => $ownerId, 'source_uuid' => $instanceUuid, 'name' => 'matrix-' . $consistency, 'state' => 'uncertain', 'consistency' => $consistency, 'object_key' => 'vm-backups/' . $backupKey . '/system.qcow2', 'spec_json' => '{}', 'sealed_storage_runtime' => 'sealed-runtime', 'active_job_id' => $record['id'], 'last_operation_id' => $record['operation_id'], 'created_at' => $now - 180, 'updated_at' => $now - 20, ]); $addFixture( 'backup_create', $consistency, $record, 'create-backup:' . $backupKey . ':' . $consistency, $instanceUuid, [ 'backup_key' => $backupKey, 'size_bytes' => 1073741824, 'virtual_size_bytes' => 42949672960, 'sha256' => str_repeat('6', 64), 'consistency' => $consistency, ] ); } $firewallVersionId = 1; $firewallRevision = 9; $firewallDigest = str_repeat('d', 64); $firewallRecord = $instanceFixture( 'firewall_apply', 'apply', 'PUT', '/firewall', [ 'revision' => $firewallRevision, 'default_in' => 'drop', 'default_out' => 'accept', 'policy_digest' => $firewallDigest, 'rules' => [], ], 'apply-firewall', [ 'revision' => $firewallRevision, 'default_in' => 'drop', 'default_out' => 'accept', 'rules' => [], ], [ 'firewall_version_id' => $firewallVersionId, 'firewall_revision' => $firewallRevision, 'agent_firewall_policy_digest' => $firewallDigest, ] ); Capsule::table('cloud_instance_firewall_versions')->insert([ 'id' => $firewallVersionId, 'instance_id' => $instanceId, 'revision' => $firewallRevision, 'enabled' => 1, 'policy_digest' => $firewallDigest, 'request_hash' => str_repeat('e', 64), 'idempotency_key' => 'matrix-firewall-version', 'state' => 'uncertain', 'job_id' => $firewallRecord['id'], 'operation_id' => $firewallRecord['operation_id'], 'error_code' => '', 'created_at' => $now - 120, 'updated_at' => $now - 20, ]); $instanceFixture( 'service_suspend', 'purge', 'POST', '/service/suspend', ['reason' => 'purge'], 'service-suspend', ['uuid' => $instanceUuid, 'state' => 'shutoff'] ); $instanceFixture( 'network_suspend', 'traffic', 'POST', '/network/suspend', ['reason' => 'traffic'], 'network-suspend', ['network_enabled' => false] ); $instanceFixture( 'network_resume', 'resume', 'POST', '/network/resume', [], 'network-resume', ['network_enabled' => true] ); $insertAggregateParent = static function ( string $kind, int $userId = 41 ) use ($insertJob): array { return $insertJob( $kind, static fn (int $id, string $operationId): array => [ 'aggregate_kind' => $kind, ], null, null, null, null, 'aggregate', $userId ); }; $templateFixture = static function ( string $kind, string $parentKind, string $method, string $expectedAction, array $projection ) use ( $insertAggregateParent, $insertJob, $addFixture, $hostId, $ownerId, $now ): void { $parent = $insertAggregateParent($parentKind); $templateId = 'matrix-' . str_replace('_', '-', $kind); $revision = 5; $generationToken = hash('md5', 'generation:' . $kind); $record = $insertJob( $kind, static function (int $jobId, string $operationId) use ( $kind, $method, $templateId, $revision, $generationToken, $ownerId ): array { $target = $kind === 'template_import_replica' ? '/v1/templates' : '/v1/templates/' . rawurlencode($templateId); $body = ['operation_id' => $operationId]; if ($kind === 'template_import_replica') { $body += [ 'id' => $templateId, 'name' => 'Matrix Template', 'source_url' => 'https://example.invalid/template.qcow2', 'source_token' => 'matrix-token', 'source_size_bytes' => 2097152, 'os_family' => 'linux', 'version' => '13', 'architecture' => 'x86_64', 'sha256' => str_repeat('1', 64), 'revision' => $revision, 'generation_token' => $generationToken, ]; } elseif ($kind === 'template_metadata_replica') { $body += [ 'name' => 'Matrix Template Updated', 'revision' => $revision, 'generation_token' => $generationToken, ]; } return [ 'method' => $method, 'request_target' => $target, 'body' => $body, 'owner_id' => $ownerId, 'can_manage' => true, 'template_id' => $templateId, ]; }, null, null, $parent['id'] ); Capsule::table('cloud_templates')->insert([ 'id' => $templateId, 'name' => 'Matrix Template', 'revision' => $revision, 'generation_token' => $generationToken, 'state' => 'active', 'active_job_id' => $parent['id'], 'created_at' => $now - 300, 'updated_at' => $now - 20, ]); Capsule::table('cloud_template_replicas')->insert([ 'host_id' => $hostId, 'template_id' => $templateId, 'revision' => $revision, 'generation_token' => $generationToken, 'state' => 'uncertain', 'image_bytes' => 0, 'sha256' => '', 'operation_id' => $record['operation_id'], 'job_id' => $record['id'], 'error_code' => '', 'created_at' => $now - 120, 'updated_at' => $now - 20, ]); $addFixture( $kind, 'catalog', $record, $expectedAction, 'template:' . $templateId, $projection ); }; $templateFixture( 'template_import_replica', 'template_import', 'POST', 'import-template', [ 'id' => 'matrix-template-import-replica', 'generation_token' => hash('md5', 'generation:template_import_replica'), 'created_at' => '2026-10-21', ] ); $templateFixture( 'template_metadata_replica', 'template_metadata_sync', 'PATCH', 'update-template', [ 'id' => 'matrix-template-metadata-replica', 'name' => 'Matrix Template Updated', 'revision' => 5, 'generation_token' => hash('md5', 'generation:template_metadata_replica'), ] ); $templateFixture( 'template_delete_replica', 'template_delete', 'DELETE', 'delete-template', [ 'resource_type' => 'template', 'id' => 'matrix-template-delete-replica', 'deleted' => true, 'managed_image_absent' => true, 'managed_metadata_absent' => true, 'managed_artifacts_absent' => true, ] ); $rescueImportId = hash('md5', 'matrix-rescue-import'); $rescueImport = $insertJob( 'rescue_template_import', static fn (int $jobId, string $operationId): array => [ 'method' => 'POST', 'request_target' => '/v1/rescue-templates', 'body' => [ 'operation_id' => $operationId, 'id' => $rescueImportId, 'name' => 'Matrix Rescue', 'source_url' => 'https://example.invalid/rescue.iso', 'source_token' => 'matrix-token', 'source_size_bytes' => 2097152, 'sha256' => str_repeat('2', 64), ], 'owner_id' => $ownerId, 'can_manage' => true, 'rescue_template_id' => $rescueImportId, ] ); Capsule::table('cloud_rescue_template_imports')->insert([ 'operation_id' => $rescueImport['operation_id'], 'host_id' => $hostId, 'template_id' => $rescueImportId, 'job_id' => $rescueImport['id'], 'import_type' => 'url', 'name' => 'Matrix Rescue', 'size_bytes' => 0, 'sha256' => '', 'state' => 'uncertain', 'error_code' => '', 'created_at' => $now - 120, 'updated_at' => $now - 20, ]); $addFixture( 'rescue_template_import', 'catalog', $rescueImport, 'import-rescue-template', 'rescue-template:' . $rescueImportId, ['id' => $rescueImportId, 'created_at' => '2026-10-21'] ); $rescueDeleteId = hash('md5', 'matrix-rescue-delete'); $rescueDelete = $insertJob( 'rescue_template_delete', static fn (int $jobId, string $operationId): array => [ 'method' => 'DELETE', 'request_target' => '/v1/rescue-templates/' . rawurlencode($rescueDeleteId), 'body' => ['operation_id' => $operationId], 'owner_id' => $ownerId, 'can_manage' => true, 'rescue_template_id' => $rescueDeleteId, ] ); Capsule::table('cloud_rescue_template_deletions')->insert([ 'host_id' => $hostId, 'template_id' => $rescueDeleteId, 'operation_id' => $rescueDelete['operation_id'], 'job_id' => $rescueDelete['id'], 'state' => 'uncertain', 'error_code' => '', 'created_at' => $now - 120, 'updated_at' => $now - 20, ]); $addFixture( 'rescue_template_delete', 'catalog', $rescueDelete, 'delete-rescue-template', 'rescue-template:' . $rescueDeleteId, [ 'resource_type' => 'rescue_template', 'id' => $rescueDeleteId, 'deleted' => true, 'managed_image_absent' => true, 'managed_metadata_absent' => true, 'managed_artifacts_absent' => true, ] ); $isoFixture = static function ( string $kind, string $parentKind, string $method, string $expectedAction, array $projection ) use ( $insertAggregateParent, $insertJob, $addFixture, $hostId, $ownerId, $now ): void { $parent = $insertAggregateParent($parentKind, $ownerId); $isoId = hash('md5', 'matrix-iso:' . $kind); $record = $insertJob( $kind, static function (int $jobId, string $operationId) use ( $kind, $method, $isoId, $ownerId ): array { $target = $kind === 'user_iso_import_replica' ? '/v1/user-isos' : '/v1/user-isos/' . rawurlencode($isoId) . '?owner_id=' . $ownerId; $body = ['operation_id' => $operationId]; if ($kind === 'user_iso_import_replica') { $body += [ 'id' => $isoId, 'owner_id' => $ownerId, 'name' => 'Matrix ISO', 'source_url' => 'https://example.invalid/user.iso', 'source_token' => 'matrix-token', 'sha256' => str_repeat('3', 64), ]; } return [ 'method' => $method, 'request_target' => $target, 'body' => $body, 'owner_id' => $ownerId, 'can_manage' => false, 'user_iso_id' => $isoId, ]; }, null, null, $parent['id'], null, 'agent', $ownerId ); Capsule::table('cloud_user_isos')->insert([ 'id' => $isoId, 'user_id' => $ownerId, 'name' => 'Matrix ISO', 'state' => 'active', 'active_job_id' => $parent['id'], 'created_at' => $now - 300, 'updated_at' => $now - 20, ]); Capsule::table('cloud_user_iso_replicas')->insert([ 'host_id' => $hostId, 'user_iso_id' => $isoId, 'user_id' => $ownerId, 'state' => 'uncertain', 'size_bytes' => 0, 'sha256' => '', 'operation_id' => $record['operation_id'], 'job_id' => $record['id'], 'error_code' => '', 'created_at' => $now - 120, 'updated_at' => $now - 20, ]); $addFixture( $kind, 'catalog', $record, $expectedAction, 'user-iso:' . $ownerId . ':' . $isoId, $projection ); }; $isoFixture( 'user_iso_import_replica', 'user_iso_import', 'POST', 'import-user-iso', ['id' => hash('md5', 'matrix-iso:user_iso_import_replica'), 'created_at' => '2026-10-21'] ); $isoDeleteId = hash('md5', 'matrix-iso:user_iso_delete_replica'); $isoFixture( 'user_iso_delete_replica', 'user_iso_delete', 'DELETE', 'delete-user-iso', [ 'resource_type' => 'user_iso', 'id' => $isoDeleteId, 'deleted' => true, 'managed_image_absent' => true, 'managed_metadata_absent' => true, 'managed_artifacts_absent' => true, ] ); // Dedicated create/storage identities are covered without admitting them // to the generic worker/finalizer matrix. $dedicatedFixtures = []; $restoreBackupId = 203; $restoreId = 301; $restoreInstanceId = 2; $restoreReservationId = 401; $restoreIpId = 501; $restoreBackupKey = hash('md5', 'recovery-matrix-restore-backup'); $restoreKey = hash('md5', 'recovery-matrix-restore'); $restoreUuid = '923e4567-e89b-42d3-a456-426614174001'; $restoreName = 'matrix-restored-vm'; $restoreIp = '203.0.113.31'; $restorePoolKey = str_repeat('7', 32); $restoreSha256 = str_repeat('8', 64); $restoreSpec = [ 'managed' => true, 'os_source' => 'cloud_template', 'os_name' => 'Debian 12', 'login_name' => 'root', 'vcpu' => 2, 'memory_mib' => 2048, 'disk_gib' => 40, 'bandwidth_mbps' => 100, 'network' => 'bridge:br0', 'public_pool_key' => $restorePoolKey, 'public_ip_cidr' => $restoreIp . '/24', 'debian_12_network_fix' => true, ]; $restoreRecord = $insertJob( 'backup_restore', static function (int $jobId, string $operationId) use ( $restoreUuid, $restoreName, $restoreIp, $restorePoolKey, $restoreBackupKey, $restoreKey, $restoreReservationId, $restoreSha256, $ownerId ): array { return [ 'method' => 'POST', 'request_target' => '/v1/vms/' . rawurlencode($restoreUuid) . '/restore-backup', 'body' => [ 'operation_id' => $operationId, 'uuid' => $restoreUuid, 'source' => '', 'name' => $restoreName, 'description' => 'Recovery matrix restore', 'vcpu' => 2, 'memory_mib' => 2048, 'disk_gib' => 40, 'bandwidth_mbps' => 100, 'network' => 'bridge:br0', 'template_id' => 'debian-12', 'template_revision' => 1, 'template_generation_token' => str_repeat('9', 32), 'startup_script' => '', 'hostname' => $restoreName, 'guest_ssh_public_key' => '', 'root_password' => '', 'public_pool_key' => $restorePoolKey, 'public_pool_cidr' => '203.0.113.0/24', 'public_ip_cidr' => $restoreIp . '/24', 'gateway' => '203.0.113.1', 'dns_servers' => ['1.1.1.1'], 'platform_network_policy_snapshot' => [ 'schema_version' => 1, ], 'debian_12_network_fix' => true, 'backup_key' => $restoreBackupKey, 'source_url' => 'https://objects.invalid/matrix-restore.qcow2', 'source_size_bytes' => 1073741824, 'source_sha256' => $restoreSha256, ], 'owner_id' => $ownerId, 'can_manage' => true, 'backup_key' => $restoreBackupKey, 'restore_key' => $restoreKey, 'reservation_id' => $restoreReservationId, ]; }, $restoreInstanceId, null, null, null, 'agent', $ownerId, $restoreBackupId, $restoreId ); Capsule::table('cloud_instance_backups')->insert([ 'id' => $restoreBackupId, 'backup_key' => $restoreBackupKey, 'source_instance_id' => 999, 'source_host_id' => $hostId + 1, 'user_id' => $ownerId, 'source_uuid' => '123e4567-e89b-42d3-a456-426614174099', 'name' => 'matrix-restore-source', 'state' => 'ready', 'consistency' => 'crash', 'object_key' => 'vm-backups/' . $restoreBackupKey . '/system.qcow2', 'object_size_bytes' => 1073741824, 'virtual_size_bytes' => 42949672960, 'sha256' => $restoreSha256, 'spec_json' => json_encode($restoreSpec, JSON_THROW_ON_ERROR), 'sealed_storage_runtime' => 'sealed-runtime', 'sealed_credential' => 'sealed-credential', 'storage_revision' => 1, 'active_job_id' => null, 'last_operation_id' => null, 'error_code' => '', 'ready_at' => $now - 300, 'created_at' => $now - 600, 'updated_at' => $now - 20, ]); Capsule::table('cloud_instances')->insert([ 'id' => $restoreInstanceId, 'uuid' => $restoreUuid, 'host_id' => $hostId, 'user_id' => $ownerId, 'reservation_id' => $restoreReservationId, 'name' => $restoreName, 'status' => 'provisioning', 'service_status' => 'exempt', 'network_status' => 'pending', 'spec_json' => json_encode($restoreSpec, JSON_THROW_ON_ERROR), 'sealed_credential' => 'sealed-credential', 'created_at' => $now - 180, 'updated_at' => $now - 20, ]); Capsule::table('cloud_public_ips')->insert([ 'id' => $restoreIpId, 'host_id' => $hostId, 'address' => $restoreIp, 'prefix_length' => 24, 'state' => 'reserved', 'reservation_id' => $restoreReservationId, 'instance_id' => null, 'operation_id' => $restoreRecord['operation_id'], 'created_at' => $now - 180, 'updated_at' => $now - 20, ]); Capsule::table('cloud_resource_reservations')->insert([ 'id' => $restoreReservationId, 'backup_restore_id' => $restoreId, 'host_id' => $hostId, 'user_id' => $ownerId, 'operation_id' => $restoreRecord['operation_id'], 'public_ip_id' => $restoreIpId, 'instance_id' => $restoreInstanceId, 'cpu_cores' => 2, 'memory_mib' => 2048, 'disk_gib' => 40, 'state' => 'held', 'created_at' => $now - 180, 'updated_at' => $now - 20, ]); Capsule::table('cloud_backup_restores')->insert([ 'id' => $restoreId, 'restore_key' => $restoreKey, 'backup_id' => $restoreBackupId, 'target_host_id' => $hostId, 'target_instance_id' => $restoreInstanceId, 'reservation_id' => $restoreReservationId, 'job_id' => $restoreRecord['id'], 'operation_id' => $restoreRecord['operation_id'], 'new_uuid' => $restoreUuid, 'new_name' => $restoreName, 'state' => 'uncertain', 'created_at' => $now - 180, 'updated_at' => $now - 20, ]); $dedicatedFixtures['backup_restore'] = $restoreRecord + [ 'expected_action' => 'restore-backup:' . $restoreBackupKey, 'expected_resource' => $restoreUuid, ]; $createRecord = $insertJob( 'create', static fn (int $jobId, string $operationId): array => [ 'method' => 'POST', 'request_target' => '/v1/vms', 'body' => [ 'operation_id' => $operationId, 'source' => 'cloud', 'name' => 'matrix-create', 'vcpu' => 2, 'memory_mib' => 2048, 'disk_gib' => 40, 'bandwidth_mbps' => 100, 'network' => 'bridge:br0', 'template_id' => 'debian-13', 'template_revision' => 1, 'template_generation_token' => str_repeat('4', 32), 'root_password' => 'MatrixPassword!123', 'public_pool_key' => str_repeat('5', 32), 'public_pool_cidr' => '203.0.113.0/24', 'public_ip_cidr' => '203.0.113.30/24', 'gateway' => '203.0.113.1', 'dns_servers' => ['1.1.1.1'], 'platform_network_policy_snapshot' => [ 'schema_version' => 1, ], ], 'credential' => [ 'password' => 'MatrixPassword!123', 'login_name' => 'root', 'auth_mode' => 'password', ], 'template_metadata' => [ 'template_id' => 'debian-13', 'os_source' => 'cloud_template', 'os_name' => 'Debian 13', 'login_name' => 'root', 'debian_12_network_fix' => false, ], ], null, null, null, 9001 ); $dedicatedFixtures['create'] = $createRecord + [ 'expected_action' => 'create-vm', ]; $storageBody = [ 'managed_directory' => '/srv/virt/managed', 'template_directory' => '/srv/virt/templates', 'seed_directory' => '/srv/virt/seeds', 'user_iso_directory' => '/srv/virt/user-isos', 'rescue_directory' => '/srv/virt/rescue', ]; $storageRecord = $insertJob( 'host_storage_configure', static fn (int $jobId, string $operationId): array => [ 'method' => 'POST', 'request_target' => '/v1/storage/configuration', 'body' => $storageBody + ['operation_id' => $operationId], 'owner_id' => $ownerId, 'can_manage' => true, 'storage_revision' => 3, ] ); Capsule::table('cloud_host_storage_configurations')->insert([ 'host_id' => $hostId, 'revision' => 3, 'managed_directory' => $storageBody['managed_directory'], 'template_directory' => $storageBody['template_directory'], 'seed_directory' => $storageBody['seed_directory'], 'user_iso_directory' => $storageBody['user_iso_directory'], 'rescue_directory' => $storageBody['rescue_directory'], 'state' => 'uncertain', 'operation_id' => $storageRecord['operation_id'], 'job_id' => $storageRecord['id'], 'created_at' => $now - 120, 'updated_at' => $now - 20, ]); $dedicatedFixtures['host_storage_configure'] = $storageRecord + [ 'expected_action' => 'configure-storage', 'expected_resource' => 'host-storage', ]; $classificationOnlyKinds = [ 'payment_create' => 'payment', 'template_import' => 'aggregate', 'template_metadata_sync' => 'aggregate', 'template_delete' => 'aggregate', 'user_iso_import' => 'aggregate', 'user_iso_delete' => 'aggregate', ]; $genericKinds = array_keys(array_filter( $manifestOperations, static fn (array $operation): bool => $operation['recovery_mode'] === 'generic' )); $fixtureKinds = array_keys($fixtures); sort($genericKinds, SORT_STRING); sort($fixtureKinds, SORT_STRING); $assert($fixtureKinds === $genericKinds, 'Every generic recovery kind must have an executable fixture.'); $coveredKinds = array_merge( $fixtureKinds, array_keys($dedicatedFixtures), array_keys($classificationOnlyKinds) ); $contractKinds = array_keys($manifestOperations); sort($coveredKinds, SORT_STRING); sort($contractKinds, SORT_STRING); $assert($coveredKinds === $contractKinds, 'A recovery contract kind was added without a matrix fixture.'); $expectedAutomaticKinds = array_keys(array_filter( $manifestOperations, static fn (array $operation): bool => in_array( $operation['recovery_mode'], ['generic', 'create'], true ) )); $assert(CloudOperationRecoveryPolicy::kinds() === array_keys(array_filter( $manifestOperations, static fn (array $operation): bool => $operation['recovery_mode'] === 'generic' )) && CloudOperationRecoveryPolicy::automaticKinds() === $expectedAutomaticKinds, 'Policy kind lists diverged from the recovery modes.'); $projectionMethod = new ReflectionMethod( CloudAdminOperationRecoveryService::class, 'effectProjection' ); $expectInvalidProjection = static function ( array $projection, string $kind, string $message ) use ($projectionMethod, $assert): void { try { $projectionMethod->invoke(null, $projection, $kind, 'applied'); } catch (CloudAgentError $error) { $assert($error->agentCode === 'agent_invalid_inspection' && $error->agentHttpStatus === 200, 'Unexpected invalid-projection error for ' . $kind . '.'); return; } throw new RuntimeException($message); }; $upgradeIdentity = CloudOperationRecoveryIdentity::expectedFromPayload( $upgradeRecord['job'], $upgradeRecord['payload'] ); $inspectionMethod = new ReflectionMethod( CloudAdminOperationRecoveryService::class, 'inspection' ); $upgradeNotAppliedRejected = false; try { $inspectionMethod->invoke(null, [ 'operation_id' => $upgradeRecord['operation_id'], 'action' => $upgradeIdentity['action'], 'resource_id' => $upgradeIdentity['resource_id'], 'request_fingerprint' => $upgradeIdentity['request_fingerprint'], 'operation_status' => 'failed', 'checked_at' => time(), 'outcome' => 'failed', 'summary_code' => 'operation_failed', 'error_code' => 'resources_not_committed', 'effect_resolution' => 'not_applied', 'fence_state' => 'active', 'fence_resolution_required' => true, 'abort_job' => ['available' => false], 'cleanup' => ['available' => false], ], $upgradeRecord['operation_id'], 'instance_upgrade', true); } catch (CloudAgentError $error) { $assert($error->agentCode === 'agent_invalid_inspection' && $error->agentHttpStatus === 200, 'Unexpected fail-closed upgrade inspection error.'); $upgradeNotAppliedRejected = true; } $assert($upgradeNotAppliedRejected, 'Automatic recovery accepted unsafe not-applied upgrade evidence.'); $malformedUpgradeProjection = $upgradeProjection; unset($malformedUpgradeProjection['applied_disk_gib']); $expectInvalidProjection( $malformedUpgradeProjection, 'instance_upgrade', 'Incomplete instance-upgrade evidence crossed the projection boundary.' ); $unrestoredUpgradeProjection = $upgradeProjection; $unrestoredUpgradeProjection['power_state_restored'] = false; $expectInvalidProjection( $unrestoredUpgradeProjection, 'instance_upgrade', 'An upgrade without restored power state crossed the projection boundary.' ); $wrongPowerUpgradeProjection = $upgradeProjection; $wrongPowerUpgradeProjection['originally_running'] = false; $wrongPowerUpgradeProjection['status'] = 'running'; $expectInvalidProjection( $wrongPowerUpgradeProjection, 'instance_upgrade', 'An upgrade with the wrong final power state crossed the projection boundary.' ); foreach ($fixtures as $kind => $kindFixtures) { $entry = $manifestOperations[$kind]; $assert(CloudOperationRecoveryPolicy::supports($kind) && CloudOperationRecoveryPolicy::identityScope($kind) === $entry['identity_scope'] && CloudOperationRecoveryIdentity::supports($kind), 'Generic policy/identity support diverged for ' . $kind . '.'); $actualActions = []; foreach ($kindFixtures as $fixture) { $job = $fixture['job']; $payload = $fixture['payload']; $fromPayload = CloudOperationRecoveryIdentity::expectedFromPayload( $job, $payload ); $fromEncryptedJob = CloudOperationRecoveryIdentity::expected($job); $label = $kind . ':' . $fixture['variant']; $assert($fromPayload === $fromEncryptedJob, 'Encrypted identity reconstruction diverged for ' . $label); $assert($fromPayload['action'] === $fixture['expected_action'] && $fromPayload['resource_id'] === $fixture['expected_resource'], 'Identity action/resource mismatch for ' . $label); $assert(CloudOperationRecoveryPolicy::allowsAction( $kind, $fromPayload['action'] ), 'Contract rejected the rebuilt action for ' . $label); $wireBody = $payload['body']; if ($kind === 'action' && ($wireBody['action'] ?? null) === 'poweroff') { $wireBody['action'] = 'destroy'; } $encodedBody = json_encode( $wireBody === [] ? (object) [] : $wireBody, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE ); $expectedFingerprint = hash( 'sha256', strtoupper($payload['method']) . "\n" . $payload['request_target'] . "\n" . $encodedBody ); $assert(hash_equals( $expectedFingerprint, $fromPayload['request_fingerprint'] ), 'Wire fingerprint mismatch for ' . $label); $context = $kind === 'instance_settings' ? CloudWorkerService::settingsRecoveryContext($job) : CloudWorkerService::recoveryContext($job); $assert($context['payload'] === $payload, 'Worker lost the encrypted recovery payload for ' . $label); if ($entry['identity_scope'] === 'instance') { $assert(($context['instance']->id ?? null) === $instanceId, 'Worker lost the instance binding for ' . $label); } else { $assert($context['instance'] === null, 'Catalog recovery unexpectedly acquired an instance for ' . $label); } $projection = $projectionMethod->invoke( null, $fixture['projection'], $kind, 'applied' ); $assert($projection === $fixture['projection'], 'Evidence projection branch is missing for ' . $label); $actualActions[] = $fromPayload['action']; } foreach ($entry['actions'] as $action) { $assert(in_array($action, $actualActions, true), 'Literal action lacks a real identity fixture: ' . $kind . ':' . $action); } foreach ($entry['action_patterns'] as $pattern) { $matched = array_filter( $actualActions, static fn (string $action): bool => preg_match('~' . $pattern . '~D', $action) === 1 ); $assert($matched !== [], 'Pattern action lacks a real identity fixture: ' . $kind . ':' . $pattern); } foreach ($actualActions as $action) { $literal = in_array($action, $entry['actions'], true); $pattern = false; foreach ($entry['action_patterns'] as $candidate) { $pattern = $pattern || preg_match('~' . $candidate . '~D', $action) === 1; } $assert($literal || $pattern, 'Fixture produced an uncontracted action: ' . $kind . ':' . $action); } } $snapshotCreateActions = array_column( $fixtures['snapshot_create'], 'expected_action' ); $snapshotKeyPattern = '[a-f0-9]{32}'; $assert(count($snapshotCreateActions) === 2 && count(array_filter( $snapshotCreateActions, static fn (string $action): bool => preg_match( '~^create-snapshot:' . $snapshotKeyPattern . ':filesystem$~D', $action ) === 1 )) === 1 && count(array_filter( $snapshotCreateActions, static fn (string $action): bool => preg_match( '~^create-snapshot:' . $snapshotKeyPattern . ':crash$~D', $action ) === 1 )) === 1, 'Snapshot create must cover both filesystem and crash consistency.'); $validTemplateMetadataProjection = $fixtures['template_metadata_replica'][0]['projection']; $invalidGeneration = $validTemplateMetadataProjection; $invalidGeneration['generation_token'] = 'NOT-32-LOWER-HEX'; $expectInvalidProjection( $invalidGeneration, 'template_metadata_replica', 'An invalid template generation token crossed the projection boundary.' ); $expectInvalidProjection( [ 'id' => 'matrix-rescue', 'created_at' => '2026-10-21', 'generation_token' => str_repeat('a', 32), ], 'rescue_template_import', 'A generation token escaped its two template projection kinds.' ); $nestedToken = $validTemplateMetadataProjection; $nestedToken['generation_token'] = [ 'generation_token' => str_repeat('a', 32), ]; $expectInvalidProjection( $nestedToken, 'template_metadata_replica', 'A nested generation token bypassed sensitive-key filtering.' ); $accessToken = $validTemplateMetadataProjection; $accessToken['access_token'] = str_repeat('a', 32); $expectInvalidProjection( $accessToken, 'template_metadata_replica', 'An access token bypassed sensitive-key filtering.' ); foreach ($dedicatedFixtures as $kind => $fixture) { $identity = CloudOperationRecoveryIdentity::expectedFromPayload( $fixture['job'], $fixture['payload'] ); $assert($identity === CloudOperationRecoveryIdentity::expected( $fixture['job'] ) && $identity['action'] === $fixture['expected_action'] && CloudOperationRecoveryPolicy::allowsAction( $kind, $identity['action'] ) && !CloudOperationRecoveryPolicy::supports($kind), 'Dedicated identity contract diverged for ' . $kind . '.'); if ($kind === 'create') { $assert(CloudOperationRecoveryPolicy::identityScope($kind) === 'create' && preg_match( '/^[a-f0-9]{8}-[a-f0-9]{4}-5[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/D', $identity['resource_id'] ) === 1, 'Create identity lost its deterministic UUID scope.'); $projection = $projectionMethod->invoke(null, [ 'creation_absent' => true, 'managed_artifacts_absent' => true, ], 'create', 'not_applied'); $assert(($projection['creation_absent'] ?? null) === true && ($projection['managed_artifacts_absent'] ?? null) === true, 'Create absence projection is no longer complete.'); $expectStateConflict(static fn () => CloudWorkerService::recoveryContext($fixture['job'])); } elseif ($kind === 'backup_restore') { $assert(CloudOperationRecoveryPolicy::identityScope($kind) === 'instance' && $identity['resource_id'] === $fixture['expected_resource'], 'Backup restore identity lost its target VM scope.'); $context = CloudWorkerService::recoveryContext($fixture['job']); $assert(($context['instance']->id ?? null) === $restoreInstanceId && $context['payload'] === $fixture['payload'], 'Backup restore lost its fenced recovery context.'); $projection = $projectionMethod->invoke(null, [ 'creation_absent' => true, 'managed_artifacts_absent' => true, ], 'backup_restore', 'not_applied'); $assert(($projection['creation_absent'] ?? null) === true && ($projection['managed_artifacts_absent'] ?? null) === true, 'Backup restore absence projection is no longer complete.'); } else { $assert(CloudOperationRecoveryPolicy::identityScope($kind) === 'storage' && $identity['resource_id'] === $fixture['expected_resource'], 'Storage identity lost its dedicated scope.'); $assert($projectionMethod->invoke( null, ['configured' => true], 'host_storage_configure', 'applied' ) === ['configured' => true], 'Storage projection branch is missing.'); $expectStateConflict(static fn () => CloudWorkerService::recoveryContext($fixture['job'])); } } foreach ($classificationOnlyKinds as $kind => $mode) { $entry = $manifestOperations[$kind]; $assert($entry['recovery_mode'] === $mode && $entry['identity_scope'] === 'none' && $entry['actions'] === [] && $entry['action_patterns'] === [] && !CloudOperationRecoveryPolicy::supports($kind) && CloudOperationRecoveryPolicy::identityScope($kind) === null && !CloudOperationRecoveryIdentity::supports($kind), 'Aggregate/payment classification diverged for ' . $kind . '.'); } foreach ($manifestOperations as $kind => $entry) { foreach (['applied', 'not_applied', 'partial'] as $resolution) { $assert(CloudOperationRecoveryPolicy::allowsSystemResolution( $kind, $resolution ) === in_array( $resolution, $entry['system_resolutions'], true ), 'System resolution policy diverged for ' . $kind . ':' . $resolution); } $assert(!CloudOperationRecoveryPolicy::allowsAction( $kind, 'unlisted-operation' ), 'Recovery contract accepted an unlisted action for ' . $kind); } echo "Cloud operation recovery matrix tests passed\n"; }