'POST', 'request_target' => '/v1/vms/test/actions', 'body' => ['action' => 'reboot'], 'owner_id' => 99, ]; } public static function encrypt(array $value, string $purpose): string { return 'sealed-terminal-audit'; } } final class RecoveryTestAgentClient { public array $requests = []; public array $responses = []; public function request( string $method, string $target, array $actor, ?array $payload = null ): array { $this->requests[] = compact('method', 'target', 'actor', 'payload'); if ($this->responses === []) { throw new \RuntimeException('Unexpected Agent request.'); } $response = array_shift($this->responses); if ($response instanceof \Throwable) { throw $response; } if (!is_array($response)) { throw new \RuntimeException('Invalid recovery test response.'); } return $response; } } final class CloudAgentSettingsService { public static ?RecoveryTestAgentClient $client = null; public static array $calls = []; public static function clientForConfiguration( int $hostId, float $timeoutSeconds = 120.0 ): RecoveryTestAgentClient { self::$calls[] = [$hostId, $timeoutSeconds]; return self::$client ?? throw new \RuntimeException('Missing recovery test client.'); } } } namespace { require dirname(__DIR__) . '/vendor/autoload.php'; use App\Services\Cloud\CloudAdminOperationRecoveryService as Recovery; use App\Services\Cloud\CloudAgentError; use App\Services\Cloud\CloudAgentSettingsService; use App\Services\Cloud\CloudError; use App\Services\Cloud\RecoveryTestAgentClient; use Illuminate\Database\Capsule\Manager as Capsule; use Illuminate\Database\Schema\Blueprint; $assert = static function ($condition, string $message): void { if (!$condition) { throw new \RuntimeException($message); } }; $reject = static function ( callable $callback, int $status, ?string $agentCode = null ) use ($assert): void { try { $callback(); } catch (CloudError $error) { $assert($error->httpStatus === $status, 'Unexpected rejection status: ' . $error->httpStatus); if ($agentCode !== null) { $assert($error instanceof CloudAgentError && $error->agentCode === $agentCode, 'Unexpected Agent rejection code.'); } return; } throw new \RuntimeException('Invalid recovery request was accepted.'); }; $capture = static function (callable $callback) use ($assert): CloudAgentError { try { $callback(); } catch (CloudAgentError $error) { return $error; } $assert(false, 'Expected a controlled Agent error.'); throw new \RuntimeException('Unreachable.'); }; $capsule = new Capsule(); $capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:']); $capsule->setAsGlobal(); $capsule->bootEloquent(); $capsule->schema()->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('operation_id'); $table->integer('host_id'); $table->string('kind'); $table->string('status'); $table->integer('user_id')->default(99); $table->integer('instance_id')->nullable(); $table->integer('order_id')->nullable(); $table->integer('snapshot_id')->nullable(); $table->integer('attempts')->default(1); $table->integer('dispatched_at')->nullable(); $table->integer('available_at')->default(0); $table->integer('lease_until')->default(0); $table->string('error_code')->nullable(); $table->text('result_json')->nullable(); $table->text('sealed_payload')->default('forbidden-payload'); $table->string('lease_token')->nullable(); $table->integer('created_at')->nullable(); $table->integer('updated_at')->nullable(); }); $capsule->schema()->create('user', static function (Blueprint $table): void { $table->increments('id'); $table->integer('enable')->default(1); }); Capsule::table('user')->insert(['id' => 99, 'enable' => 1]); Capsule::table('cloud_jobs')->insert([ 'id' => 41, 'operation_id' => 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'host_id' => 7, 'kind' => 'action', 'status' => 'uncertain', 'dispatched_at' => 1_799_999_900, ]); Capsule::table('cloud_jobs')->insert([ 'id' => 42, 'operation_id' => 'dddddddddddddddddddddddddddddddd', 'host_id' => 7, 'kind' => 'action', 'status' => 'needs_attention', 'error_code' => 'reconciliation_limit_reached', 'dispatched_at' => 1_799_999_900, ]); $admin = [ 'user_id' => 99, 'is_admin' => true, 'is_super_admin' => true, 'can_manage' => true, ]; $readonly = [ 'user_id' => 98, 'is_admin' => true, 'is_super_admin' => false, 'can_manage' => false, ]; $operationId = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; $token = str_repeat('b', 64); $planId = str_repeat('c', 64); $client = new RecoveryTestAgentClient(); CloudAgentSettingsService::$client = $client; $capsule->getConnection()->enableQueryLog(); $capsule->getConnection()->flushQueryLog(); $client->responses[] = [ 'operation_id' => $operationId, 'operation_status' => 'uncertain', 'checked_at' => 1_800_000_000, 'outcome' => 'uncertain', 'summary_code' => 'abortable_job_active', // Unknown codes and unrelated fields must not reach the browser DTO. 'error_code' => 'raw_libvirt_secret_path', 'raw_error' => '/srv/private/secret.qcow2', 'abort_job' => [ 'available' => true, 'kind' => 'backup', 'inspection_token' => $token, ], 'cleanup' => ['available' => false], 'confirmation' => ['available' => false], ]; $inspection = Recovery::inspect($admin, '41'); $inspectQueries = $capsule->getConnection()->getQueryLog(); $assert($inspection['operation_id'] === $operationId && $inspection['summary_code'] === 'abortable_job_active' && $inspection['abort_job'] === [ 'available' => true, 'kind' => 'backup', 'label' => '中止匹配的备份作业', 'inspection_token' => $token, ] && $inspection['cleanup']['available'] === false && !isset($inspection['error_code']) && !str_contains(json_encode($inspection), '/srv/private'), 'Read-only inspection was not strictly normalized.'); $assert(count($inspectQueries) === 1 && !str_contains(strtolower($inspectQueries[0]['query']), 'sealed_payload') && !str_contains(strtolower($inspectQueries[0]['query']), 'lease_token'), 'Explicit recovery read selected task secrets or lease tokens.'); $assert(CloudAgentSettingsService::$calls === [[7, 30.0]] && $client->requests[0]['method'] === 'GET' && $client->requests[0]['target'] === '/v1/operations/' . $operationId . '/inspect' && $client->requests[0]['payload'] === null, 'Inspection did not use the signed read-only Agent endpoint.'); $client->responses[] = [ 'operation_id' => $operationId, 'operation_status' => 'uncertain', 'checked_at' => 1_800_000_001, 'outcome' => 'uncertain', 'summary_code' => 'evidence_incomplete', 'abort_job' => ['available' => false], 'cleanup' => ['available' => false], 'confirmation' => ['available' => false], ]; $afterAbort = Recovery::abortJob($admin, '41', [ 'inspection_token' => $token, ]); $abortRequest = $client->requests[1]; $assert($abortRequest['method'] === 'POST' && str_ends_with($abortRequest['target'], '/abort-job') && $abortRequest['payload'] === ['inspection_token' => $token] && CloudAgentSettingsService::$calls[1] === [7, 60.0] && !$afterAbort['abort_job']['available'], 'Abort was not bound to the inspected job token.'); $client->responses[] = [ 'operation_id' => $operationId, 'operation_status' => 'uncertain', 'checked_at' => 1_800_000_002, 'outcome' => 'uncertain', 'summary_code' => 'cleanup_required', 'abort_job' => ['available' => false], 'cleanup' => [ 'available' => true, 'plan_id' => $planId, 'steps' => ['destroy_domain', 'delete_managed_volumes', 'remove_traffic_state', 'persist_cleanup_metadata'], ], 'confirmation' => ['available' => false], ]; $cleanup = Recovery::cleanup($admin, '41', ['plan_id' => $planId]); $cleanupRequest = $client->requests[2]; $assert($cleanupRequest['method'] === 'POST' && str_ends_with($cleanupRequest['target'], '/cleanup') && $cleanupRequest['payload'] === ['plan_id' => $planId] && CloudAgentSettingsService::$calls[2] === [7, 900.0] && $cleanup['cleanup']['step_codes'] === [ 'destroy_domain', 'delete_managed_volumes', 'remove_traffic_state', 'persist_cleanup_metadata', ] && $cleanup['cleanup']['steps'] === [ '停止残留虚拟机', '删除面板管理的残留磁盘', '删除残留流量计量状态', '清除已完成磁盘切换的待清理标记', ], 'Cleanup was not bound to the exact inspected plan.'); // Recovery errors arrive as CloudAgentError rather than operation records. // Only stable codes may survive; the Agent's free-form message, cause and // result must never be copied into the browser-facing exception. $rawAgentDetail = 'raw message cause=/srv/private/domain.xml ' . 'result={"secret":"never-expose"}'; $errorCases = [ [ 'operation_inspection_stale', false, static fn () => Recovery::abortJob($admin, '41', [ 'inspection_token' => $token, ]), '核对凭证已失效,请重新立即核对', ], [ 'cleanup_plan_stale', false, static fn () => Recovery::cleanup($admin, '41', [ 'plan_id' => $planId, ]), '宿主机状态已变化,清理计划已失效,请重新立即核对', ], [ 'job_abort_unconfirmed', true, static fn () => Recovery::abortJob($admin, '41', [ 'inspection_token' => $token, ]), 'libvirt 作业中止后的终态仍未确认,请稍后再次立即核对', ], [ 'operation_uncertain', true, static fn () => Recovery::cleanup($admin, '41', [ 'plan_id' => $planId, ]), '宿主机操作结果仍未确认,请稍后再次立即核对', ], [ 'external_snapshot_cleanup_required', false, static fn () => Recovery::inspect($admin, '41'), '检测到原生外部快照或回滚元数据,请先在宿主机安全完成清理,再重新立即核对', ], [ 'native_snapshot_delete_in_progress', false, static fn () => Recovery::inspect($admin, '41'), '原生快照删除尚未完成,请先在宿主机继续完成该删除,再重新立即核对', ], [ 'managed_disk_shared', false, static fn () => Recovery::inspect($admin, '41'), '目标受管磁盘仍被另一台虚拟机的当前配置、快照或保存状态引用,请先解除引用,再重新立即核对', ], [ 'managed_filter_shared', false, static fn () => Recovery::inspect($admin, '41'), '目标网络过滤器仍被另一台虚拟机或过滤器引用,请先解除引用,再重新立即核对', ], ]; foreach ($errorCases as [$code, $retryable, $call, $message]) { $client->responses[] = new CloudAgentError( $rawAgentDetail, 409, $code, $retryable, 409, true, true ); $mapped = $capture($call); $assert($mapped->httpStatus === 409 && $mapped->agentCode === $code && $mapped->retryable === $retryable && $mapped->getMessage() === $message && $mapped->safeValidationFailure === false && $mapped->safePreDispatchFailure === false && !str_contains($mapped->getMessage(), '/srv/private') && !str_contains($mapped->getMessage(), 'never-expose'), 'Recovery error was not safely mapped: ' . $code); } $client->responses[] = new CloudAgentError( $rawAgentDetail, 500, 'unexpected_private_backend_code', true, 500 ); $unknown = $capture(static fn () => Recovery::inspect($admin, '41')); $assert($unknown->httpStatus === 503 && $unknown->agentCode === 'operation_recovery_unconfirmed' && $unknown->retryable === true && $unknown->getMessage() === '宿主机状态暂时无法确认,请稍后重试' && !str_contains($unknown->getMessage(), '/srv/private') && !str_contains($unknown->getMessage(), 'never-expose'), 'Unknown Agent recovery error was exposed or misclassified.'); $beforeInvalid = count($client->requests); $reject(static fn () => Recovery::abortJob($admin, '41', [ 'inspection_token' => strtoupper($token), ]), 409); $reject(static fn () => Recovery::cleanup($admin, '41', [ 'plan_id' => $planId, 'confirm' => true, ]), 422); $reject(static fn () => Recovery::forceDelete($admin, '41', [ 'ack_data_loss' => false, ]), 409); $reject(static fn () => Recovery::forceDelete($admin, '41', [ 'ack_data_loss' => true, 'confirm' => true, ]), 422); $reject(static fn () => Recovery::forceDelete([ 'user_id' => 97, 'is_admin' => true, 'is_super_admin' => false, 'can_manage' => true, ], '41', ['ack_data_loss' => true]), 403); $reject(static fn () => Recovery::inspect($readonly, '41'), 403); $assert(count($client->requests) === $beforeInvalid, 'Invalid or unauthorized recovery request reached the Agent.'); $routeSource = file_get_contents( dirname(__DIR__) . '/config/routes_cloud.php' ); $assert(is_string($routeSource) && str_contains($routeSource, "/admin/operations/jobs/{id:[0-9]+}/force-delete"), 'Independent force-delete administrator route is missing.'); $controllerSource = file_get_contents( dirname(__DIR__) . '/app/Controllers/CloudAdminOperationsController.php' ); $assert(is_string($controllerSource) && preg_match( '/public function inspect\\(.*?public function abortJob/s', $controllerSource, $inspectController ) === 1 && strpos($inspectController[0], '$inspection = CloudAdminOperationRecoveryService::inspect(') !== false && strpos($inspectController[0], 'CloudAdminOperationsService::detail($actor, $id)') !== false && strpos($inspectController[0], '$inspection = CloudAdminOperationRecoveryService::inspect(') < strpos($inspectController[0], 'CloudAdminOperationsService::detail($actor, $id)'), 'Inspect response read stale panel state before token-confirmed settlement.'); $client->responses[] = [ 'operation_id' => $operationId, 'operation_status' => 'uncertain', 'checked_at' => 1_800_000_003, 'outcome' => 'uncertain', 'summary_code' => 'cleanup_required', 'abort_job' => [ 'available' => true, 'kind' => 'backup', 'inspection_token' => $token, ], 'cleanup' => [ 'available' => true, 'plan_id' => $planId, 'steps' => ['destroy_domain'], ], 'confirmation' => ['available' => false], ]; $reject(static fn () => Recovery::inspect($admin, '41'), 502, 'agent_invalid_inspection'); // A terminal observation must survive a second fresh read-only check // before the Agent journal and panel ledger can be settled. $confirmationToken = str_repeat('e', 64); $client->responses[] = [ 'operation_id' => $operationId, 'operation_status' => 'uncertain', 'checked_at' => 1_800_000_004, 'outcome' => 'failed', 'summary_code' => 'operation_failed', 'error_code' => 'snapshot_not_created', 'abort_job' => ['available' => false], 'cleanup' => ['available' => false], 'confirmation' => [ 'available' => true, 'inspection_token' => $confirmationToken, ], ]; $client->responses[] = [ 'operation' => [ 'operation_id' => $operationId, 'status' => 'failed', 'error' => ['code' => 'snapshot_not_created'], 'result' => [], ], 'inspection' => [ 'operation_id' => $operationId, 'operation_status' => 'failed', 'checked_at' => 1_800_000_005, 'outcome' => 'failed', 'summary_code' => 'operation_failed', 'error_code' => 'snapshot_not_created', 'abort_job' => ['available' => false], 'cleanup' => ['available' => false], 'confirmation' => [ 'available' => true, 'inspection_token' => $confirmationToken, ], ], ]; $failed = Recovery::inspect($admin, '41'); $confirmRequest = $client->requests[array_key_last($client->requests)]; $assert(($failed['error_code'] ?? null) === 'snapshot_not_created' && ($failed['error'] ?? null) === '宿主机证据确认快照没有创建' && !str_contains(json_encode($failed), $confirmationToken), 'Known inspection error code was not mapped to a safe Chinese action message.'); $assert($confirmRequest['method'] === 'POST' && str_ends_with($confirmRequest['target'], '/confirm-inspection') && $confirmRequest['payload'] === [ 'inspection_token' => $confirmationToken, ] && Capsule::table('cloud_jobs')->where('id', 41)->value('status') === 'failed', 'Confirmed failed inspection did not close the panel blocker.'); $successOperationId = 'dddddddddddddddddddddddddddddddd'; $successToken = str_repeat('f', 64); $successInspection = [ 'operation_id' => $successOperationId, 'operation_status' => 'needs_attention', 'checked_at' => 1_800_000_006, 'outcome' => 'succeeded', 'summary_code' => 'operation_succeeded', 'abort_job' => ['available' => false], 'cleanup' => ['available' => false], 'confirmation' => [ 'available' => true, 'inspection_token' => $successToken, ], ]; $client->responses[] = $successInspection; $client->responses[] = [ 'operation' => [ 'operation_id' => $successOperationId, 'status' => 'succeeded', 'result' => ['state' => 'running'], ], 'inspection' => array_replace($successInspection, [ 'operation_status' => 'succeeded', 'checked_at' => 1_800_000_007, ]), ]; $succeeded = Recovery::inspect($admin, '42'); $assert($succeeded['summary_code'] === 'operation_succeeded' && Capsule::table('cloud_jobs')->where('id', 42)->value('status') === 'succeeded', 'Confirmed successful inspection did not close needs_attention.'); Capsule::table('cloud_jobs')->where('id', 42)->update([ 'status' => 'needs_attention', 'error_code' => 'reconciliation_limit_reached', 'sealed_payload' => 'forbidden-payload', 'lease_token' => null, 'lease_until' => 0, ]); $client->responses[] = $successInspection; $client->responses[] = new CloudAgentError( 'raw changed host evidence', 409, 'inspection_inconclusive', false, 409 ); $inconclusive = $capture(static fn () => Recovery::inspect($admin, '42')); $assert($inconclusive->agentCode === 'inspection_inconclusive' && $inconclusive->getMessage() === '宿主机状态在两次只读核对之间发生变化,请重新立即核对' && Capsule::table('cloud_jobs')->where('id', 42)->value('status') === 'needs_attention', 'Inconclusive fresh evidence changed the panel blocker.'); $terminalResponse = static function (int $checkedAt) use ( $successInspection, $successOperationId, $successToken ): array { return [ 'operation' => [ 'operation_id' => $successOperationId, 'status' => 'succeeded', 'result' => ['state' => 'running'], ], 'inspection' => array_replace($successInspection, [ 'operation_status' => 'succeeded', 'checked_at' => $checkedAt, 'confirmation' => [ 'available' => true, 'inspection_token' => $successToken, ], ]), ]; }; $client->responses[] = array_replace($successInspection, [ 'checked_at' => 1_800_000_008, ]); $client->responses[] = $terminalResponse(1_800_000_009); Recovery::abortJob($admin, '42', ['inspection_token' => $token]); $lastTwo = array_slice($client->requests, -2); $assert(str_ends_with($lastTwo[0]['target'], '/abort-job') && str_ends_with($lastTwo[1]['target'], '/confirm-inspection') && Capsule::table('cloud_jobs')->where('id', 42)->value('status') === 'succeeded', 'Abort terminal response did not converge the panel ledger.'); Capsule::table('cloud_jobs')->where('id', 42)->update([ 'status' => 'needs_attention', 'error_code' => 'reconciliation_limit_reached', 'sealed_payload' => 'forbidden-payload', 'lease_token' => null, 'lease_until' => 0, ]); $client->responses[] = array_replace($successInspection, [ 'checked_at' => 1_800_000_010, ]); $client->responses[] = $terminalResponse(1_800_000_011); Recovery::cleanup($admin, '42', ['plan_id' => $planId]); $lastTwo = array_slice($client->requests, -2); $assert(str_ends_with($lastTwo[0]['target'], '/cleanup') && str_ends_with($lastTwo[1]['target'], '/confirm-inspection') && Capsule::table('cloud_jobs')->where('id', 42)->value('status') === 'succeeded', 'Cleanup terminal response did not converge the panel ledger.'); echo "Administrator inspect, token-confirmed settlement, recovery-error, abort and cleanup contracts passed.\n"; }