requests[] = $request; return $this->respond($request); } public function systemRecoveryRequest( string $method, string $target, string $operationId, ?array $payload = null ): array { $actor = [ 'user_id' => 0, 'is_admin' => false, 'is_super_admin' => false, 'can_manage' => false, 'system_recovery' => true, 'operation_id' => $operationId, ]; $request = compact('method', 'target', 'actor', 'payload'); $request['system_recovery'] = true; $this->requests[] = $request; return $this->respond($request); } private function respond(array $request): array { if ($this->responses === []) { throw new \RuntimeException('Unexpected Agent request.'); } $response = array_shift($this->responses); if (is_callable($response)) { $response = $response($request); } if ($response instanceof \Throwable) { throw $response; } if (!is_array($response)) { throw new \RuntimeException('Invalid recovery test response.'); } return $response; } } final class CloudAgentSettingsService { public static ?RecoveryTestAgentClient $client = null; public static array $calls = []; public static function clientForConfiguration( int $hostId, float $timeoutSeconds = 120.0 ): RecoveryTestAgentClient { self::$calls[] = [$hostId, $timeoutSeconds]; return self::$client ?? throw new \RuntimeException('Missing recovery test client.'); } } } namespace { require dirname(__DIR__) . '/vendor/autoload.php'; use App\Services\Cloud\CloudAdminOperationRecoveryService as Recovery; use App\Services\Cloud\CloudAgentError; use App\Services\Cloud\CloudAgentSettingsService; use App\Services\Cloud\CloudError; use App\Services\Cloud\CloudOperationRecoveryIdentity; use App\Services\Cloud\CloudSecrets; use App\Services\Cloud\RecoveryTestAgentClient; use App\Models\CloudJob; use Illuminate\Database\Capsule\Manager as Capsule; use Illuminate\Database\Schema\Blueprint; $assert = static function (bool $condition, string $message): void { if (!$condition) { throw new RuntimeException($message); } }; $reject = static function (callable $callback, int $status, ?string $code = null) use ($assert): void { try { $callback(); } catch (CloudError $error) { $assert($error->httpStatus === $status, 'Unexpected HTTP status: ' . $error->httpStatus); if ($code !== null) { $assert($error instanceof CloudAgentError && $error->agentCode === $code, 'Unexpected Agent error code.'); } return; } throw new RuntimeException('Invalid recovery request was accepted.'); }; $capsule = new Capsule(); $capsule->addConnection([ 'driver' => 'sqlite', 'database' => ':memory:', ], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $capsule->schema('default')->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('operation_id'); $table->string('request_idempotency_key'); $table->string('request_fingerprint'); $table->integer('host_id'); $table->string('kind'); $table->string('execution_mode')->default('agent'); $table->string('phase'); $table->string('outcome')->nullable(); $table->string('status'); $table->integer('user_id')->default(99); $table->integer('instance_id')->nullable(); $table->integer('order_id')->nullable(); $table->integer('parent_job_id')->nullable(); $table->integer('snapshot_id')->nullable(); $table->integer('attempts')->default(1); $table->integer('dispatch_attempts')->default(1); $table->integer('dispatch_deadline_at')->default(1); $table->integer('dispatch_started_at')->nullable(); $table->integer('agent_accepted_at')->nullable(); $table->integer('dispatched_at')->nullable(); $table->integer('receipt_deadline_at')->nullable(); $table->string('completion_event_id')->nullable(); $table->integer('terminal_at')->nullable(); $table->boolean('effect_may_have_applied')->nullable(); $table->boolean('safe_to_resubmit')->nullable(); $table->string('effect_resolution')->default('not_applicable'); $table->string('effect_resolution_reason')->nullable(); $table->integer('available_at')->default(0); $table->integer('lease_until')->default(0); $table->string('error_code')->nullable(); $table->boolean('finalization_incident')->default(0); $table->string('finalization_error_code')->nullable(); $table->text('result_json')->nullable(); $table->text('sealed_payload')->default('forbidden-payload'); $table->string('lease_token')->nullable(); $table->integer('created_at')->nullable(); $table->integer('updated_at')->nullable(); }); $capsule->schema('default')->create('cloud_instances', static function (Blueprint $table): void { $table->increments('id'); $table->string('uuid'); $table->integer('host_id'); $table->integer('user_id'); $table->string('status'); }); $capsule->schema('default')->create('cloud_orders', static function (Blueprint $table): void { $table->increments('id'); $table->integer('user_id'); $table->integer('offer_id'); $table->integer('resource_pool_id'); $table->integer('host_id'); $table->integer('reservation_id'); $table->text('spec_json'); }); $capsule->schema('default')->create('cloud_resource_reservations', static function (Blueprint $table): void { $table->increments('id'); $table->integer('order_id')->nullable(); $table->integer('backup_restore_id')->nullable(); $table->integer('offer_id')->nullable(); $table->integer('resource_pool_id'); $table->integer('host_id'); $table->integer('user_id'); $table->integer('instance_id')->nullable(); $table->string('operation_id'); $table->string('state'); }); $now = time(); foreach ([ 41 => 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 42 => 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 43 => 'cccccccccccccccccccccccccccccccc', 44 => 'dddddddddddddddddddddddddddddddd', ] as $id => $operationId) { $publicNetwork = [ 'contract' => 1, 'network_selector' => 'bridge:br0', 'ipv4' => [ 'pool_key' => str_repeat('1', 32), 'pool_cidr' => '8.8.8.0/24', 'address_cidr' => '8.8.8.' . $id . '/24', 'gateway' => '8.8.8.1', 'gateway_on_link' => false, ], 'ipv6' => [ 'pool_key' => str_repeat('2', 32), 'pool_cidr' => '2404:c140:156:3a::/64', 'address_cidr' => '2404:c140:156:3a::' . $id . '/64', 'gateway' => '2404:c140:156::1', 'gateway_on_link' => true, ], 'dns_servers' => [ '1.1.1.1', '2606:4700:4700::1111', ], ]; $reservationId = 1000 + $id; Capsule::table('cloud_orders')->insert([ 'id' => $id, 'user_id' => 99, 'offer_id' => 1, 'resource_pool_id' => 1, 'host_id' => 7, 'reservation_id' => $reservationId, 'spec_json' => json_encode([ 'source' => 'cloud', 'public_network' => $publicNetwork, ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES), ]); Capsule::table('cloud_resource_reservations')->insert([ 'id' => $reservationId, 'order_id' => $id, 'backup_restore_id' => null, 'offer_id' => 1, 'resource_pool_id' => 1, 'host_id' => 7, 'user_id' => 99, 'instance_id' => null, 'operation_id' => $operationId, 'state' => 'uncertain', ]); Capsule::table('cloud_jobs')->insert([ 'id' => $id, 'operation_id' => $operationId, 'host_id' => 7, 'request_idempotency_key' => $operationId, 'request_fingerprint' => hash('sha256', $operationId), 'kind' => 'create', 'order_id' => $id, 'phase' => 'terminal', 'outcome' => 'failed', 'status' => 'failed', 'dispatched_at' => $now - 90, 'dispatch_started_at' => $now - 100, 'agent_accepted_at' => $now - 95, 'receipt_deadline_at' => $now + 30, 'terminal_at' => $now - 30, 'effect_may_have_applied' => true, 'safe_to_resubmit' => false, 'effect_resolution' => 'unknown', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/vms', 'body' => [ 'source' => 'cloud', 'name' => 'vm-recovery-' . $id, 'vcpu' => 2, 'memory_mib' => 2048, 'disk_gib' => 40, 'bandwidth_mbps' => 100, 'public_network' => $publicNetwork, 'template_id' => 'debian-13', 'template_revision' => 1, 'template_generation_token' => str_repeat('1', 32), 'root_password' => 'StrongPassword!123', 'platform_network_policy_snapshot' => [ 'schema_version' => 1, ], ], 'template_metadata' => [ 'template_id' => 'debian-13', 'os_source' => 'cloud_template', 'os_name' => 'Debian 13', 'login_name' => 'root', 'debian_12_network_fix' => false, ], ], 'job-payload'), 'created_at' => $now - 120, 'updated_at' => $now - 30, ]); } Capsule::table('cloud_instances')->insert([ 'id' => 45, 'uuid' => '123e4567-e89b-42d3-a456-426614174000', 'host_id' => 7, 'user_id' => 99, 'status' => 'active', ]); Capsule::table('cloud_jobs')->insert([ 'id' => 45, 'operation_id' => 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', 'request_idempotency_key' => 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', 'request_fingerprint' => hash( 'sha256', 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee' ), 'host_id' => 7, 'kind' => 'instance_settings', 'phase' => 'terminal', 'outcome' => 'failed', 'status' => 'failed', 'dispatched_at' => $now - 90, 'dispatch_started_at' => $now - 100, 'agent_accepted_at' => $now - 95, 'receipt_deadline_at' => $now + 30, 'terminal_at' => $now - 30, 'effect_may_have_applied' => true, 'safe_to_resubmit' => false, 'effect_resolution' => 'unknown', 'effect_resolution_reason' => 'automatic_recovery:v1:' . ($now - 60) . ':1:prepared:automatic_applied_failed_fence:' . str_repeat('a', 64) . ':' . str_repeat('b', 64), 'instance_id' => 45, 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/vms/' . '123e4567-e89b-42d3-a456-426614174000/settings', 'body' => [ 'operation_id' => 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', 'hostname' => 'safe.example', 'confirm_restart' => true, ], 'owner_id' => 99, 'can_manage' => false, 'settings_fields' => ['hostname'], 'credential' => null, ], 'job-payload'), 'created_at' => $now - 120, 'updated_at' => $now - 30, ]); $identityA = CloudOperationRecoveryIdentity::expected(CloudJob::find(41)); $identityB = CloudOperationRecoveryIdentity::expected(CloudJob::find(42)); $identityC = CloudOperationRecoveryIdentity::expected(CloudJob::find(43)); $identityD = CloudOperationRecoveryIdentity::expected(CloudJob::find(44)); $identityE = CloudOperationRecoveryIdentity::expected(CloudJob::find(45)); $admin = ['user_id' => 99, 'is_admin' => true, 'is_super_admin' => true, 'can_manage' => true]; $readonly = ['user_id' => 98, 'is_admin' => true, 'is_super_admin' => false, 'can_manage' => false]; $client = new RecoveryTestAgentClient(); CloudAgentSettingsService::$client = $client; $before = count($client->requests); $beforeClientBuilds = count(CloudAgentSettingsService::$calls); Capsule::table('cloud_jobs')->where('id', 41)->update([ 'execution_mode' => 'local', 'phase' => 'accepted', 'outcome' => null, 'status' => 'queued', ]); $reject(static fn () => Recovery::inspect($admin, '41'), 409); $assert(count($client->requests) === $before && count(CloudAgentSettingsService::$calls) === $beforeClientBuilds, 'A nonterminal supported Agent kind with a damaged execution mode reached the Agent client.'); Capsule::table('cloud_jobs')->where('id', 41)->update([ 'kind' => 'template_import', 'execution_mode' => 'agent', ]); $reject(static fn () => Recovery::inspect($admin, '41'), 409); $assert(count($client->requests) === $before && count(CloudAgentSettingsService::$calls) === $beforeClientBuilds, 'A malformed aggregate identity reached the Agent client.'); Capsule::table('cloud_jobs')->where('id', 41)->update([ 'kind' => 'create', 'execution_mode' => 'agent', 'phase' => 'terminal', 'outcome' => 'failed', 'status' => 'failed', ]); Capsule::table('cloud_jobs')->where('id', 45)->update([ 'receipt_deadline_at' => null, ]); $reject(static fn () => Recovery::inspect($admin, '45'), 409); $assert(count($client->requests) === $before && count(CloudAgentSettingsService::$calls) === $beforeClientBuilds, 'A terminal Agent task with a damaged immutable envelope reached the Agent client.'); Capsule::table('cloud_jobs')->where('id', 45)->update([ 'receipt_deadline_at' => $now + 30, 'outcome' => 'succeeded', 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'effect_resolution' => 'applied', 'finalization_incident' => true, 'finalization_error_code' => 'recovery_ledger_conflict', ]); $reject(static fn () => Recovery::inspect($admin, '45'), 409); $assert(count($client->requests) === $before && count(CloudAgentSettingsService::$calls) === $beforeClientBuilds, 'A resolved sticky recovery incident reached the Agent client.'); Capsule::table('cloud_jobs')->where('id', 45)->update([ 'outcome' => 'failed', 'status' => 'failed', 'effect_may_have_applied' => true, 'safe_to_resubmit' => false, 'effect_resolution' => 'unknown', 'finalization_incident' => false, 'finalization_error_code' => null, ]); $before = count($client->requests); Capsule::table('cloud_jobs')->where('id', 41)->update([ 'kind' => 'host_storage_configure', ]); $reject(static fn () => Recovery::inspectForRecovery('41'), 409); $reject(static fn () => Recovery::resolveFenceForRecovery( '41', 'not_applied', 'automatic recovery must reject unregistered task kinds', [ 'operation_id' => 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'checked_at' => $now, 'effect_resolution' => 'not_applied', 'fence_resolution_required' => true, 'fence_state' => 'active', ] ), 409); $reject(static fn () => Recovery::preparedFenceResolutionForRecovery( '41', 'not_applied', 'automatic recovery must reject unregistered task kinds' ), 409); $assert(count($client->requests) === $before, 'An unregistered task reached the system recovery Agent client.'); Capsule::table('cloud_jobs')->where('id', 41)->update([ 'kind' => 'create', ]); $operationA = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; $client->responses[] = $createInspectionResponse = [ 'operation_id' => $operationA, 'operation_status' => 'failed', 'checked_at' => $now, 'outcome' => 'failed', 'summary_code' => 'operation_failed', 'error_code' => 'creation_absent', 'effect_resolution' => 'not_applied', 'fence_resolution_required' => true, 'fence_state' => 'active', 'effect_projection' => [ 'creation_absent' => true, 'managed_artifacts_absent' => true, ], 'abort_job' => ['available' => false], 'cleanup' => ['available' => false], ] + $identityA; $inspection = Recovery::inspect($admin, '41'); $assert($inspection === [ 'operation_id' => $operationA, 'action' => $identityA['action'], 'resource_id' => $identityA['resource_id'], 'request_fingerprint' => $identityA['request_fingerprint'], 'journal_status' => 'failed', 'checked_at' => $now, 'outcome' => 'failed', 'effect_resolution' => 'not_applied', 'fence_state' => 'active', 'fence_resolution_required' => true, 'summary_code' => 'operation_failed', 'summary' => '只读证据与宿主机失败结果一致', 'error_code' => 'creation_absent', 'error' => '宿主机未发现虚拟机或面板管理资源', ] && !array_key_exists('effect_projection', $inspection), 'Read-only inspection was not reduced to the bounded DTO.'); $assert(CloudAgentSettingsService::$calls === [[7, 30.0]] && $client->requests[0]['method'] === 'GET' && $client->requests[0]['target'] === '/v1/operations/' . $operationA . '/inspect' && $client->requests[0]['payload'] === null, 'Inspection did not use exactly one signed read-only request.'); $assert(Capsule::table('cloud_jobs')->where('id', 41) ->value('effect_resolution') === 'unknown', 'Read-only inspection changed the task or its safety fence.'); $client->responses[] = $createInspectionResponse; $createInspection = Recovery::inspectForRecovery('41'); $createRequest = $client->requests[array_key_last($client->requests)]; $assert(($createInspection['error_code'] ?? null) === 'creation_absent' && ($createInspection['effect_projection']['creation_absent'] ?? null) === true && ($createInspection['effect_projection']['managed_artifacts_absent'] ?? null) === true && $createRequest['method'] === 'GET' && ($createRequest['system_recovery'] ?? false) === true && $createRequest['actor']['operation_id'] === $operationA && Capsule::table('cloud_jobs')->where('id', 41) ->value('effect_resolution') === 'unknown', 'Create inspection did not remain operation-bound and read-only.'); $before = count($client->requests); $reject(static fn () => Recovery::resolveFenceForRecovery( '41', 'not_applied', 'automatic create recovery requires durable financial intent', $createInspection ), 409); $reject(static fn () => Recovery::resolveFenceForRecovery( '41', 'applied', 'automatic create recovery cannot infer a successful instance', $createInspection ), 409); $reject(static fn () => Recovery::preparedFenceResolutionForRecovery( '41', 'not_applied', 'automatic create recovery requires durable financial intent' ), 409); $assert(count($client->requests) === $before, 'An unprepared create reached the system Agent fence routes.'); $client->responses[] = ['operation_id' => $operationA, 'operation_status' => 'uncertain', 'checked_at' => $now, 'outcome' => 'uncertain', 'summary_code' => 'evidence_incomplete']; $reject(static fn () => Recovery::inspect($admin, '41'), 502, 'agent_invalid_inspection'); $client->responses[] = [ 'operation_id' => $operationA, 'operation_status' => 'failed', 'checked_at' => $now, 'outcome' => 'failed', 'summary_code' => 'operation_failed', 'error_code' => 'creation_absent', 'effect_resolution' => 'not_applied', 'fence_resolution_required' => true, 'fence_state' => 'active', 'effect_projection' => [ 'creation_absent' => true, 'managed_artifacts_absent' => true, ], 'abort_job' => ['available' => false], 'cleanup' => ['available' => false], 'raw_error' => '/srv/private/secret.qcow2', ] + $identityA; $reject(static fn () => Recovery::inspect($admin, '41'), 502, 'agent_invalid_inspection'); $operationE = 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee'; $safeProjection = [ 'uuid' => '123e4567-e89b-42d3-a456-426614174000', 'hostname' => 'safe.example', 'settings_changed' => true, ]; $client->responses[] = [ 'operation_id' => $operationE, 'operation_status' => 'failed', 'checked_at' => $now, 'outcome' => 'succeeded', 'summary_code' => 'operation_succeeded', 'effect_resolution' => 'applied', 'fence_resolution_required' => true, 'fence_state' => 'active', 'effect_projection' => $safeProjection, 'abort_job' => ['available' => false], 'cleanup' => ['available' => false], ] + $identityE; $before = count($client->requests); $systemInspection = Recovery::inspectForRecovery('45'); $requests = array_slice($client->requests, $before); $assert(($systemInspection['effect_resolution'] ?? null) === 'applied' && ($systemInspection['effect_projection'] ?? null) === $safeProjection && count($requests) === 1 && ($requests[0]['system_recovery'] ?? false) === true && $requests[0]['actor'] === [ 'user_id' => 0, 'is_admin' => false, 'is_super_admin' => false, 'can_manage' => false, 'system_recovery' => true, 'operation_id' => $operationE, ], 'Internal inspection lost its bounded recovery evidence or actor.'); $client->responses[] = [ 'operation_id' => $operationE, 'operation_status' => 'failed', 'checked_at' => $now, 'outcome' => 'succeeded', 'summary_code' => 'operation_succeeded', 'effect_resolution' => 'applied', 'fence_resolution_required' => true, 'fence_state' => 'active', 'completion_delivery_state' => 'quarantined', 'completion_delivery_error_code' => 'completion_receipt_invalid', 'effect_projection' => $safeProjection, 'abort_job' => ['available' => false], 'cleanup' => ['available' => false], ] + $identityE; $poisonedInspection = Recovery::inspectForRecovery('45'); $assert(($poisonedInspection['completion_delivery_state'] ?? null) === 'quarantined' && ($poisonedInspection['completion_delivery_error_code'] ?? null) === 'completion_receipt_invalid' && str_contains( (string) ($poisonedInspection['completion_delivery_error'] ?? ''), '完成回执内容无效' ), 'A quarantined completion did not retain its bounded Chinese cause.'); $client->responses[] = [ 'operation_id' => $operationE, 'operation_status' => 'failed', 'checked_at' => $now, 'outcome' => 'succeeded', 'summary_code' => 'operation_succeeded', 'effect_resolution' => 'applied', 'fence_state' => 'resolved', 'fence_resolution_required' => false, 'effect_projection' => $safeProjection, 'abort_job' => ['available' => false], 'cleanup' => ['available' => false], ] + $identityE; $resolvedInspection = Recovery::inspectForRecovery('45'); $assert(($resolvedInspection['effect_resolution'] ?? null) === 'applied' && ($resolvedInspection['fence_state'] ?? null) === 'resolved' && ($resolvedInspection['fence_resolution_required'] ?? null) === false, 'A resolved Agent fence was collapsed into a never-fenced DTO.'); foreach ([ ['effect_resolution' => 'trusted', 'effect_projection' => $safeProjection], ['effect_resolution' => 'applied', 'effect_projection' => [ 'uuid' => '123e4567-e89b-42d3-a456-426614174000', 'root_password' => 'must-not-cross', 'settings_changed' => true, ]], ['effect_resolution' => 'applied', 'effect_projection' => [ 'uuid' => '123e4567-e89b-42d3-a456-426614174000', 'future_field' => true, 'settings_changed' => true, ]], ['effect_resolution' => 'applied', 'effect_projection' => [ 'uuid' => '123e4567-e89b-42d3-a456-426614174000', 'hostname' => str_repeat('x', 65537), 'settings_changed' => true, ]], ['effect_resolution' => 'applied', 'effect_projection' => $safeProjection, 'completion_delivery_state' => 'quarantined', 'completion_delivery_error_code' => 'future_poison_reason'], ['effect_resolution' => 'applied', 'effect_projection' => $safeProjection, 'completion_delivery_state' => 'quarantined'], ] as $unsafeEvidence) { $client->responses[] = [ 'operation_id' => $operationE, 'operation_status' => 'failed', 'checked_at' => $now, 'outcome' => 'succeeded', 'summary_code' => 'operation_succeeded', 'fence_resolution_required' => true, 'fence_state' => 'active', 'abort_job' => ['available' => false], 'cleanup' => ['available' => false], ] + $unsafeEvidence + $identityE; $reject(static fn () => Recovery::inspectForRecovery('45'), 502, 'agent_invalid_inspection'); } $reason = '已人工只读核对虚拟机、磁盘与网络状态,确认原操作效果已经发生。'; $reasonDigest = hash('sha256', $reason); $poisonedManualInspection = $inspection + [ 'completion_delivery_state' => 'quarantined', 'completion_delivery_error_code' => 'completion_operation_invalid', 'completion_delivery_error' => 'Agent 完成回执对应的原操作记录无效,当前结果不能采信', ]; $before = count($client->requests); $reject(static fn () => Recovery::resolveFence( $admin, '41', 'not_applied', $reason, $poisonedManualInspection ), 409); $assert(count($client->requests) === $before, 'A quarantined completion reached the Agent fence resolver.'); $oldReceipt = [ 'operation_id' => $operationA, 'resolution_id' => str_repeat('1', 32), 'resolution' => 'not_applied', 'inspection_checked_at' => $now - 60, 'reason_sha256' => $reasonDigest, 'resolved_at' => $now - 50, 'resolved_by' => 7, 'replayed' => false, ]; $before = count($client->requests); $client->responses[] = $oldReceipt; $recovered = Recovery::resolveFence( $admin, '41', 'not_applied', $reason, $inspection ); $requests = array_slice($client->requests, $before); $assert($recovered['resolution_id'] === str_repeat('1', 32) && count($requests) === 1 && $requests[0]['method'] === 'GET' && str_ends_with($requests[0]['target'], '/fence-resolution') && ($requests[0]['actor']['operation_id'] ?? null) === $operationA, 'A prior Agent commit was not recovered read-only after a PHP crash.'); $notFound = static fn () => new CloudAgentError( 'missing', 404, 'fence_resolution_not_found', false, 404 ); $automaticReason = 'Agent 实时只读检查确认实例设置已经提交,PHP 账本预提交完成。'; $client->responses[] = [ 'operation_id' => $operationE, 'resolution_id' => str_repeat('f', 32), 'resolution' => 'applied', 'inspection_checked_at' => $now, 'reason_sha256' => hash('sha256', $automaticReason), 'resolved_at' => $now, 'resolved_by' => 7, 'replayed' => false, ]; $before = count($client->requests); $reject(static fn () => Recovery::resolveFenceForRecovery( '45', 'applied', $automaticReason, $systemInspection ), 503, 'fence_resolution_read_failed'); $assert(count($client->requests) === $before + 1, 'System recovery accepted a positive human resolved_by value.'); $client->responses[] = $notFound(); $client->responses[] = new CloudAgentError( 'lost response', 503, 'agent_disconnected', true, 503 ); $client->responses[] = static function (array $request) use ( $client, $operationE, $now ): array { $post = $client->requests[count($client->requests) - 2]; return [ 'operation_id' => $operationE, 'resolution_id' => $post['payload']['resolution_id'], 'resolution' => $post['payload']['resolution'], 'inspection_checked_at' => $post['payload']['inspection_checked_at'], 'reason_sha256' => $post['payload']['reason_sha256'], 'resolved_at' => $now, 'resolved_by' => 0, 'replayed' => false, ]; }; $before = count($client->requests); $automaticFence = Recovery::resolveFenceForRecovery( '45', 'applied', $automaticReason, $systemInspection ); $requests = array_slice($client->requests, $before); $assert(array_column($requests, 'method') === ['GET', 'POST', 'GET'] && count(array_filter($requests, static fn (array $request): bool => $request['method'] === 'POST')) === 1 && count(array_filter($requests, static fn (array $request): bool => ($request['system_recovery'] ?? false) === true)) === 3 && $automaticFence['resolution'] === 'applied' && $automaticFence['resolved_by'] === '0', 'System recovery did not use one exact-operation fence write and read verification.'); $operationB = 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'; $inspectionB = [ 'operation_id' => $operationB, 'checked_at' => $now, 'outcome' => 'failed', 'summary_code' => 'operation_failed', 'error_code' => 'creation_absent', ] + $identityB; $client->responses[] = $notFound(); $client->responses[] = static function (array $request) use ( $operationB, $now ): array { return [ 'operation_id' => $operationB, 'resolution_id' => $request['payload']['resolution_id'], 'resolution' => $request['payload']['resolution'], 'inspection_checked_at' => $request['payload']['inspection_checked_at'], 'reason_sha256' => $request['payload']['reason_sha256'], 'resolved_at' => $now, 'resolved_by' => '7', 'replayed' => false, ]; }; $before = count($client->requests); $created = Recovery::resolveFence( $admin, '42', 'not_applied', $reason, $inspectionB ); $requests = array_slice($client->requests, $before); $assert(count($requests) === 2 && array_column($requests, 'method') === ['GET', 'POST'] && str_ends_with($requests[1]['target'], '/resolve-fence') && preg_match('/^[a-f0-9]{32}$/D', $requests[1]['payload']['resolution_id']) === 1 && $created['resolution'] === 'not_applied', 'New fence resolution was not dispatched exactly once.'); $operationC = 'cccccccccccccccccccccccccccccccc'; $inspectionC = [ 'operation_id' => $operationC, 'checked_at' => $now, 'outcome' => 'failed', 'summary_code' => 'operation_failed', 'error_code' => 'creation_absent', ] + $identityC; $client->responses[] = $notFound(); $client->responses[] = new CloudAgentError( 'lost response', 503, 'agent_disconnected', true, 503 ); $client->responses[] = static function (array $request) use ( $client, $operationC, $now ): array { $post = $client->requests[count($client->requests) - 2]; return [ 'operation_id' => $operationC, 'resolution_id' => $post['payload']['resolution_id'], 'resolution' => $post['payload']['resolution'], 'inspection_checked_at' => $post['payload']['inspection_checked_at'], 'reason_sha256' => $post['payload']['reason_sha256'], 'resolved_at' => $now, 'resolved_by' => 7, 'replayed' => false, ]; }; $before = count($client->requests); $verified = Recovery::resolveFence( $admin, '43', 'not_applied', $reason, $inspectionC ); $requests = array_slice($client->requests, $before); $assert(array_column($requests, 'method') === ['GET', 'POST', 'GET'] && count(array_filter($requests, static fn (array $request): bool => $request['method'] === 'POST')) === 1 && $verified['resolution'] === 'not_applied', 'Lost fence response caused a POST retry instead of one GET verification.'); $operationD = 'dddddddddddddddddddddddddddddddd'; $inspectionD = [ 'operation_id' => $operationD, 'checked_at' => $now, 'outcome' => 'failed', 'summary_code' => 'operation_failed', 'error_code' => 'creation_absent', ] + $identityD; $client->responses[] = array_replace($oldReceipt, [ 'operation_id' => $operationD, 'resolved_by' => 0, ]); $before = count($client->requests); $reject(static fn () => Recovery::resolveFence( $admin, '44', 'not_applied', $reason, $inspectionD ), 503, 'fence_resolution_read_failed'); $assert(count($client->requests) === $before + 1, 'A forged system actor ID reached the human fence write path.'); $client->responses[] = array_replace($oldReceipt, [ 'operation_id' => $operationD, 'resolution' => 'applied', ]); $before = count($client->requests); $reject(static fn () => Recovery::resolveFence( $admin, '44', 'not_applied', $reason, $inspectionD ), 409); $assert(count($client->requests) === $before + 1, 'Conflicting durable fence receipt triggered an Agent write.'); $before = count($client->requests); $reject(static fn () => Recovery::resolveFence( $readonly, '41', 'not_applied', $reason, $inspection ), 403); $reject(static fn () => Recovery::resolveFence( $admin, '41', 'not_applied', $reason, [ 'operation_id' => $operationA, 'checked_at' => $now - 121, 'outcome' => 'failed', 'summary_code' => 'operation_failed', 'error_code' => 'creation_absent', ] + $identityA ), 409); $assert(count($client->requests) === $before, 'Unauthorized or stale resolution reached the Agent.'); $source = file_get_contents(dirname(__DIR__) . '/app/Services/Cloud/CloudAdminOperationRecoveryService.php'); $assert(is_string($source) && !str_contains($source, 'function abortJob') && !str_contains($source, 'function cleanup') && !str_contains($source, 'confirm-inspection') && preg_match_all('/\$call\(\s*[\r\n ]*\'POST\'/m', $source) === 1, 'Retired recovery mutations or a second Agent POST remain reachable.'); echo "Administrator read-only inspection and single-write fence recovery tests passed.\n"; }