httpStatus === $status, 'Unexpected CloudError status.'); return; } throw new RuntimeException('Expected CloudError was not thrown.'); }; $capsule = new Capsule(); $capsule->addConnection([ 'driver' => 'sqlite', 'database' => ':memory:', ], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->integer('host_id'); $table->integer('user_id'); $table->integer('order_id')->nullable(); $table->integer('instance_id')->nullable(); $table->integer('snapshot_id')->nullable(); $table->string('operation_id')->unique(); $table->string('execution_mode'); $table->string('kind'); $table->string('phase'); $table->string('outcome')->nullable(); $table->string('status'); $table->integer('dispatch_attempts')->default(1); $table->integer('terminal_at')->nullable(); $table->boolean('effect_may_have_applied')->nullable(); $table->boolean('safe_to_resubmit')->nullable(); $table->string('effect_resolution')->default('not_applicable'); $table->text('sealed_payload'); $table->string('completion_event_id')->nullable(); $table->boolean('finalization_incident')->default(0); $table->string('finalization_error_code')->nullable(); $table->integer('read_at')->nullable(); $table->string('lease_token')->nullable(); $table->integer('lease_until')->default(0); $table->integer('available_at')->default(0); $table->string('error_code')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_agent_completion_events', static function (Blueprint $table): void { $table->string('event_id')->primary(); $table->integer('host_id'); $table->string('operation_id'); $table->string('payload_sha256'); $table->string('outcome'); $table->boolean('effect_may_have_applied'); $table->boolean('safe_to_resubmit'); $table->string('error_code')->default(''); $table->text('result_json')->nullable(); $table->string('disposition')->default('received'); $table->integer('received_at'); $table->integer('applied_at')->nullable(); $table->unique(['host_id', 'operation_id']); }); $schema->create('cloud_host_storage_configurations', static function (Blueprint $table): void { $table->integer('host_id')->primary(); $table->integer('revision'); foreach (['managed', 'template', 'seed', 'user_iso', 'rescue'] as $role) { $table->text($role . '_directory'); } $table->string('state'); $table->string('operation_id')->nullable(); $table->integer('job_id')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_orders', static function (Blueprint $table): void { $table->increments('id'); $table->integer('user_id'); $table->integer('offer_id'); $table->integer('resource_pool_id'); $table->integer('host_id'); $table->integer('reservation_id'); $table->text('spec_json'); }); $schema->create('cloud_resource_reservations', static function (Blueprint $table): void { $table->increments('id'); $table->integer('order_id')->nullable(); $table->integer('backup_restore_id')->nullable(); $table->integer('offer_id')->nullable(); $table->integer('resource_pool_id'); $table->integer('host_id'); $table->integer('user_id'); $table->integer('instance_id')->nullable(); $table->string('operation_id'); $table->string('state'); }); $now = time(); $createBody = static function (string $operationId, int $suffix): array { return [ 'operation_id' => $operationId, 'source' => 'cloud', 'name' => 'identity-vm-' . $suffix, 'vcpu' => 2, 'memory_mib' => 2048, 'disk_gib' => 40, 'bandwidth_mbps' => 100, 'public_network' => [ 'contract' => 1, 'network_selector' => 'bridge:br0', 'ipv4' => [ 'pool_key' => str_repeat('2', 32), 'pool_cidr' => '8.8.8.0/24', 'address_cidr' => '8.8.8.' . $suffix . '/24', 'gateway' => '8.8.8.1', 'gateway_on_link' => false, ], 'ipv6' => [ 'pool_key' => str_repeat('3', 32), 'pool_cidr' => '2404:c140:156:3a::/64', 'address_cidr' => '2404:c140:156:3a::' . $suffix . '/64', 'gateway' => '2404:c140:156::1', 'gateway_on_link' => true, ], 'dns_servers' => [ '1.1.1.1', '2606:4700:4700::1111', ], ], 'template_id' => 'debian-13', 'template_revision' => 1, 'template_generation_token' => str_repeat('1', 32), 'root_password' => 'Identity-test9!Pass', 'platform_network_policy_snapshot' => ['schema_version' => 1], ]; }; $insertCreate = static function ( int $id, string $operationId ) use ($createBody, $now): void { $orderId = 1000 + $id; $reservationId = 2000 + $id; $body = $createBody($operationId, 20 + $id); Capsule::table('cloud_orders')->insert([ 'id' => $orderId, 'user_id' => 11, 'offer_id' => 1, 'resource_pool_id' => 1, 'host_id' => 7, 'reservation_id' => $reservationId, 'spec_json' => json_encode([ 'source' => 'cloud', 'public_network' => $body['public_network'], ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES), ]); Capsule::table('cloud_resource_reservations')->insert([ 'id' => $reservationId, 'order_id' => $orderId, 'backup_restore_id' => null, 'offer_id' => 1, 'resource_pool_id' => 1, 'host_id' => 7, 'user_id' => 11, 'instance_id' => null, 'operation_id' => $operationId, 'state' => 'held', ]); Capsule::table('cloud_jobs')->insert([ 'id' => $id, 'host_id' => 7, 'user_id' => 11, 'order_id' => $orderId, 'operation_id' => $operationId, 'execution_mode' => 'agent', 'kind' => 'create', 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'dispatch_attempts' => 1, 'effect_resolution' => 'not_applicable', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/vms', 'body' => $body, 'credential' => [ 'password' => 'Identity-test9!Pass', 'login_name' => 'root', 'auth_mode' => 'password', ], 'template_metadata' => [ 'template_id' => 'debian-13', 'os_source' => 'cloud_template', 'os_name' => 'Debian 13', 'login_name' => 'root', 'debian_12_network_fix' => false, ], ], 'job-payload'), 'created_at' => $now - 30, 'updated_at' => $now - 10, ]); }; $createOperation = str_repeat('1a', 16); $storageOperation = str_repeat('2b', 16); $invalidCreateOperation = str_repeat('3c', 16); $hashCreateOperation = str_repeat('4d', 16); $secretCreateOperation = str_repeat('5e', 16); $activeLedgerConflictOperation = str_repeat('6f', 16); $terminalLedgerConflictOperation = str_repeat('70', 16); $repairedDuringDeferOperation = str_repeat('81', 16); $insertCreate(1, $createOperation); $insertCreate(3, $invalidCreateOperation); $insertCreate(4, $hashCreateOperation); $insertCreate(5, $secretCreateOperation); $insertCreate(6, $activeLedgerConflictOperation); $insertCreate(7, $terminalLedgerConflictOperation); $insertCreate(8, $repairedDuringDeferOperation); $storageBody = [ 'operation_id' => $storageOperation, 'managed_directory' => '/srv/virt/managed', 'template_directory' => '/srv/virt/templates', 'seed_directory' => '/srv/virt/seeds', 'user_iso_directory' => '/srv/virt/user-isos', 'rescue_directory' => '/srv/virt/rescue', ]; Capsule::table('cloud_jobs')->insert([ 'id' => 2, 'host_id' => 8, 'user_id' => 11, 'operation_id' => $storageOperation, 'execution_mode' => 'agent', 'kind' => 'host_storage_configure', 'phase' => 'dispatching', 'outcome' => null, 'status' => 'running', 'dispatch_attempts' => 1, 'effect_resolution' => 'not_applicable', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/storage/configuration', 'body' => $storageBody, 'owner_id' => 11, 'can_manage' => true, 'storage_revision' => 4, ], 'job-payload'), 'created_at' => $now - 30, 'updated_at' => $now - 10, ]); Capsule::table('cloud_host_storage_configurations')->insert([ 'host_id' => 8, 'revision' => 4, 'managed_directory' => $storageBody['managed_directory'], 'template_directory' => $storageBody['template_directory'], 'seed_directory' => $storageBody['seed_directory'], 'user_iso_directory' => $storageBody['user_iso_directory'], 'rescue_directory' => $storageBody['rescue_directory'], 'state' => 'applying', 'operation_id' => $storageOperation, 'job_id' => 2, 'created_at' => $now - 30, 'updated_at' => $now - 10, ]); $createJob = CloudJob::findOrFail(1); $storageJob = CloudJob::findOrFail(2); $createIdentity = CloudOperationRecoveryIdentity::expected($createJob); $storageIdentity = CloudOperationRecoveryIdentity::expected($storageJob); $assert(CloudOperationRecoveryIdentity::supports('create') && CloudOperationRecoveryIdentity::supports( 'host_storage_configure' ) && !CloudOperationRecoveryPolicy::supports('create') && !CloudOperationRecoveryPolicy::supports( 'host_storage_configure' ) && $createIdentity['action'] === 'create-vm' && preg_match( '/^[a-f0-9]{8}-[a-f0-9]{4}-5[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/D', $createIdentity['resource_id'] ) === 1 && $storageIdentity['action'] === 'configure-storage' && $storageIdentity['resource_id'] === 'host-storage', 'Agent identity support was incorrectly coupled to recovery policy.'); $operationForJob = new ReflectionMethod( CloudWorkerService::class, 'operationForJob' ); $validSync = [ 'operation_id' => $createOperation, 'status' => 'succeeded', ] + $createIdentity; $assert($operationForJob->invoke(null, $createJob, $validSync) === $validSync, 'A correctly bound synchronous create response was rejected.'); $wrongSync = $validSync; $wrongSync['resource_id'] = 'different-create-resource'; try { $operationForJob->invoke(null, $createJob, $wrongSync); throw new RuntimeException( 'A cross-operation synchronous create response was accepted.' ); } catch (CloudRecoveryOperationIdentityConflict $expected) { // Expected: no result from the mismatched response can reach ledgers. } $validStorageSync = [ 'operation_id' => $storageOperation, 'status' => 'succeeded', ] + $storageIdentity; $wrongStorageSync = $validStorageSync; $wrongStorageSync['request_fingerprint'] = str_repeat('f', 64); try { $operationForJob->invoke(null, $storageJob, $wrongStorageSync); throw new RuntimeException( 'A cross-operation synchronous storage response was accepted.' ); } catch (CloudRecoveryOperationIdentityConflict $expected) { // Expected. } $receipt = static function ( string $eventId, int $hostId, string $operationId, array $identity ): array { return [ 'type' => 'completion', 'id' => $eventId, 'agent_id' => (string) $hostId, 'operation_id' => $operationId, 'action' => $identity['action'], 'resource_id' => $identity['resource_id'], 'request_fingerprint' => $identity['request_fingerprint'], 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => [], ]; }; // Completion ingestion may persist evidence for a corrupt dispatch // ledger, but it must not ACK/consume that evidence. Both an active task // and an apparently settled terminal task stay visibly repairable until // dispatch_attempts is restored to one of its only valid states (0/1). Capsule::table('cloud_jobs')->where('id', 6)->update([ 'dispatch_attempts' => 2, ]); $activeLedgerConflictJob = CloudJob::findOrFail(6); $activeLedgerConflictReceipt = $receipt( 'active-dispatch-ledger-conflict', 7, $activeLedgerConflictOperation, CloudOperationRecoveryIdentity::expected( $activeLedgerConflictJob, true ) ); $activeLedgerConflictAccepted = CloudOperationCompletionService::accept( $activeLedgerConflictReceipt ); $activeLedgerConflictJob->refresh(); $activeLedgerConflictEvent = Capsule::table( 'cloud_agent_completion_events' )->where( 'event_id', $activeLedgerConflictReceipt['id'] )->first(); $assert($activeLedgerConflictAccepted === [ 'accepted' => true, 'disposition' => 'received', ] && (string) $activeLedgerConflictJob->phase === 'terminal' && (string) $activeLedgerConflictJob->outcome === 'failed' && (bool) $activeLedgerConflictJob->effect_may_have_applied && !(bool) $activeLedgerConflictJob->safe_to_resubmit && (string) $activeLedgerConflictJob->effect_resolution === 'unknown' && (bool) $activeLedgerConflictJob->finalization_incident && (string) $activeLedgerConflictJob->finalization_error_code === 'recovery_dispatch_ledger_conflict' && (string) $activeLedgerConflictEvent->disposition === 'received' && $activeLedgerConflictEvent->applied_at === null, 'An active invalid dispatch ledger consumed its immutable completion: ' . json_encode([ 'accepted' => $activeLedgerConflictAccepted, 'job' => $activeLedgerConflictJob->toArray(), 'event' => (array) $activeLedgerConflictEvent, ], JSON_THROW_ON_ERROR)); $terminalLedgerConflictEventId = 'terminal-dispatch-ledger-conflict'; Capsule::table('cloud_jobs')->where('id', 7)->update([ 'phase' => 'terminal', 'outcome' => 'succeeded', 'status' => 'succeeded', 'dispatch_attempts' => 2, 'terminal_at' => time(), 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => 'applied', 'completion_event_id' => $terminalLedgerConflictEventId, ]); $terminalLedgerConflictJob = CloudJob::findOrFail(7); $terminalLedgerConflictTerminalAt = (int) $terminalLedgerConflictJob->terminal_at; $terminalLedgerConflictReceipt = $receipt( $terminalLedgerConflictEventId, 7, $terminalLedgerConflictOperation, CloudOperationRecoveryIdentity::expected( $terminalLedgerConflictJob, true ) ); $terminalLedgerConflictAccepted = CloudOperationCompletionService::accept( $terminalLedgerConflictReceipt ); $terminalLedgerConflictJob->refresh(); $terminalLedgerConflictEvent = \App\Models\CloudAgentCompletionEvent::where( 'event_id', $terminalLedgerConflictEventId )->firstOrFail(); $assert($terminalLedgerConflictAccepted === [ 'accepted' => true, 'disposition' => 'received', ] && (string) $terminalLedgerConflictJob->phase === 'terminal' && (string) $terminalLedgerConflictJob->outcome === 'succeeded' && (string) $terminalLedgerConflictJob->status === 'succeeded' && !(bool) $terminalLedgerConflictJob->effect_may_have_applied && !(bool) $terminalLedgerConflictJob->safe_to_resubmit && (string) $terminalLedgerConflictJob->effect_resolution === 'applied' && $terminalLedgerConflictJob->error_code === null && (int) $terminalLedgerConflictJob->terminal_at === $terminalLedgerConflictTerminalAt && (string) $terminalLedgerConflictJob->completion_event_id === $terminalLedgerConflictEventId && (bool) $terminalLedgerConflictJob->finalization_incident && (string) $terminalLedgerConflictJob->finalization_error_code === 'recovery_dispatch_ledger_conflict' && (string) $terminalLedgerConflictEvent->disposition === 'received' && $terminalLedgerConflictEvent->applied_at === null && !CloudOperationCompletionService::settledTaskWithEvent( $terminalLedgerConflictJob, $terminalLedgerConflictEvent ), 'A terminal invalid dispatch ledger ACKed a received completion.'); // Simulate an operator repairing the counter after a worker observed the // invalid value but before deferDispatchLedgerConflict acquired its lock. // The repaired value must remain sticky until background validation. Capsule::table('cloud_jobs')->where('id', 8)->update([ 'dispatch_attempts' => 2, ]); $repairedDuringDefer = CloudJob::findOrFail(8); Capsule::table('cloud_jobs')->where('id', 8)->update([ 'dispatch_attempts' => 1, ]); CloudOperationCompletionService::deferDispatchLedgerConflict( $repairedDuringDefer ); $repairedDuringDefer->refresh(); $assert((int) $repairedDuringDefer->dispatch_attempts === 1 && (string) $repairedDuringDefer->phase === 'terminal' && (string) $repairedDuringDefer->outcome === 'failed' && (bool) $repairedDuringDefer->effect_may_have_applied && (string) $repairedDuringDefer->effect_resolution === 'unknown' && (bool) $repairedDuringDefer->finalization_incident && (string) $repairedDuringDefer->finalization_error_code === 'recovery_dispatch_ledger_conflict', 'A lock-window dispatch repair bypassed the sticky safety incident.'); $wrongCreate = $receipt( 'create-wrong-identity-event', 7, $createOperation, $createIdentity ); $wrongCreate['resource_id'] = 'different-create-resource'; $createAccepted = CloudOperationCompletionService::accept($wrongCreate); $wrongStorage = $receipt( 'storage-wrong-identity-event', 8, $storageOperation, $storageIdentity ); $wrongStorage['request_fingerprint'] = str_repeat('f', 64); $storageAccepted = CloudOperationCompletionService::accept($wrongStorage); $createAfterWrong = CloudJob::findOrFail(1); $storageAfterWrong = CloudJob::findOrFail(2); $assert($createAccepted === [ 'accepted' => true, 'disposition' => 'rejected', ] && $storageAccepted === [ 'accepted' => true, 'disposition' => 'rejected', ] && (string) $createAfterWrong->phase === 'awaiting_receipt' && $createAfterWrong->outcome === null && (string) $createAfterWrong->effect_resolution === 'not_applicable' && (string) $storageAfterWrong->phase === 'dispatching' && $storageAfterWrong->outcome === null && (string) $storageAfterWrong->effect_resolution === 'not_applicable' && (string) Capsule::table('cloud_host_storage_configurations') ->where('host_id', 8)->value('state') === 'applying', 'A wrong identity bypassed create or storage deadline finalization.'); $invalidCreate = $receipt( 'create-invalid-receipt-event', 7, $invalidCreateOperation, CloudOperationRecoveryIdentity::expected(CloudJob::findOrFail(3)) ); unset($invalidCreate['action']); $expectCloudError( static fn () => CloudOperationCompletionService::accept( $invalidCreate ), 400 ); $invalidStorage = $receipt( 'storage-invalid-receipt-event', 8, $storageOperation, $storageIdentity ); unset($invalidStorage['action']); $expectCloudError( static fn () => CloudOperationCompletionService::accept( $invalidStorage ), 400 ); $invalidCreateJob = CloudJob::findOrFail(3); $invalidStorageJob = CloudJob::findOrFail(2); $assert((string) $invalidCreateJob->phase === 'awaiting_receipt' && $invalidCreateJob->outcome === null && (bool) $invalidCreateJob->finalization_incident && (string) $invalidCreateJob->finalization_error_code === 'recovery_receipt_invalid' && (string) $invalidStorageJob->phase === 'dispatching' && $invalidStorageJob->outcome === null && (bool) $invalidStorageJob->finalization_incident && (string) $invalidStorageJob->finalization_error_code === 'recovery_receipt_invalid' && (string) Capsule::table('cloud_host_storage_configurations') ->where('host_id', 8)->value('state') === 'applying', 'An attributable invalid receipt bypassed create/storage finalizers.'); $hashIdentity = CloudOperationRecoveryIdentity::expected( CloudJob::findOrFail(4) ); $hashReceipt = $receipt( 'create-hash-conflict-event', 7, $hashCreateOperation, $hashIdentity ); Capsule::table('cloud_agent_completion_events')->insert([ 'event_id' => $hashReceipt['id'], 'host_id' => 7, 'operation_id' => $hashCreateOperation, 'payload_sha256' => str_repeat('0', 64), 'outcome' => 'failed', 'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0, 'error_code' => 'old_conflicting_receipt', 'result_json' => null, 'disposition' => 'rejected', 'received_at' => $now - 5, 'applied_at' => null, ]); $expectCloudError( static fn () => CloudOperationCompletionService::accept($hashReceipt), 409 ); $hashJob = CloudJob::findOrFail(4); $assert((string) $hashJob->phase === 'awaiting_receipt' && $hashJob->outcome === null && (bool) $hashJob->finalization_incident && (string) $hashJob->finalization_error_code === 'recovery_operation_identity_conflict', 'A payload-hash conflict was invisible or bypassed create finalization.'); $secretReceipt = $receipt( 'create-secret-result-event', 7, $secretCreateOperation, CloudOperationRecoveryIdentity::expected( CloudJob::findOrFail(5), true ) ); $secretReceipt['result'] = [ 'vm' => ['ssh_private_key' => 'must-never-be-persisted'], ]; $expectCloudError( static fn () => CloudOperationCompletionService::accept( $secretReceipt ), 400 ); $secretJob = CloudJob::findOrFail(5); $assert(!Capsule::table('cloud_agent_completion_events') ->where('event_id', $secretReceipt['id'])->exists() && (string) $secretJob->phase === 'awaiting_receipt' && $secretJob->outcome === null && (bool) $secretJob->finalization_incident && (string) $secretJob->finalization_error_code === 'recovery_receipt_invalid', 'A create completion secret was persisted or bypassed its finalizer.'); $unknownSecretCases = [ 'unknown-generated-password' => [ 'generatedPassword' => 'must-never-persist', ], 'unknown-root-password' => [ 'nested' => ['root-password' => 'must-never-persist'], ], 'unknown-access-token' => [ 'access_token' => 'must-never-persist', ], 'unknown-passphrase' => [ 'diskPassphrase' => 'must-never-persist', ], 'unknown-plain-token' => [ 'token' => 'must-never-persist', ], ]; $unknownIndex = 0; foreach ($unknownSecretCases as $eventId => $secretResult) { $unknownIndex++; $unknownOperation = substr(hash( 'sha256', 'unknown-secret-operation:' . $unknownIndex ), 0, 32); $unknownReceipt = $receipt( $eventId . '-event', 77, $unknownOperation, $createIdentity ); $unknownReceipt['result'] = $secretResult; $expectCloudError( static fn () => CloudOperationCompletionService::accept( $unknownReceipt ), 400 ); $assert(!Capsule::table('cloud_agent_completion_events') ->where('event_id', $unknownReceipt['id'])->exists(), 'An unknown-operation secret completion was persisted.'); } echo "Agent mutation identity binding tests passed.\n"; }