requests[] = compact('method', 'target', 'actor', 'payload'); if ($this->responses === []) { throw new \RuntimeException('Unexpected Agent request.'); } $response = array_shift($this->responses); if ($response instanceof \Throwable) { throw $response; } if (!is_array($response)) { throw new \RuntimeException('Invalid recovery test response.'); } return $response; } } final class CloudAgentSettingsService { public static ?RecoveryTestAgentClient $client = null; public static array $calls = []; public static function clientForConfiguration( int $hostId, float $timeoutSeconds = 120.0 ): RecoveryTestAgentClient { self::$calls[] = [$hostId, $timeoutSeconds]; return self::$client ?? throw new \RuntimeException('Missing recovery test client.'); } } } namespace { require dirname(__DIR__) . '/vendor/autoload.php'; use App\Services\Cloud\CloudAdminOperationRecoveryService as Recovery; use App\Services\Cloud\CloudAgentError; use App\Services\Cloud\CloudAgentSettingsService; use App\Services\Cloud\CloudError; use App\Services\Cloud\RecoveryTestAgentClient; use Illuminate\Database\Capsule\Manager as Capsule; use Illuminate\Database\Schema\Blueprint; $assert = static function ($condition, string $message): void { if (!$condition) { throw new \RuntimeException($message); } }; $reject = static function ( callable $callback, int $status, ?string $agentCode = null ) use ($assert): void { try { $callback(); } catch (CloudError $error) { $assert($error->httpStatus === $status, 'Unexpected rejection status: ' . $error->httpStatus); if ($agentCode !== null) { $assert($error instanceof CloudAgentError && $error->agentCode === $agentCode, 'Unexpected Agent rejection code.'); } return; } throw new \RuntimeException('Invalid recovery request was accepted.'); }; $capture = static function (callable $callback) use ($assert): CloudAgentError { try { $callback(); } catch (CloudAgentError $error) { return $error; } $assert(false, 'Expected a controlled Agent error.'); throw new \RuntimeException('Unreachable.'); }; $capsule = new Capsule(); $capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:']); $capsule->setAsGlobal(); $capsule->bootEloquent(); $capsule->schema()->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('operation_id'); $table->integer('host_id'); $table->string('kind'); $table->string('status'); $table->integer('dispatched_at')->nullable(); $table->text('sealed_payload')->default('forbidden-payload'); $table->string('lease_token')->nullable()->default('forbidden-lease'); $table->integer('created_at')->nullable(); $table->integer('updated_at')->nullable(); }); Capsule::table('cloud_jobs')->insert([ 'id' => 41, 'operation_id' => 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'host_id' => 7, 'kind' => 'snapshot_create', 'status' => 'uncertain', 'dispatched_at' => 1_799_999_900, ]); $admin = [ 'user_id' => 99, 'is_admin' => true, 'is_super_admin' => true, 'can_manage' => true, ]; $readonly = [ 'user_id' => 98, 'is_admin' => true, 'is_super_admin' => false, 'can_manage' => false, ]; $operationId = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; $token = str_repeat('b', 64); $planId = str_repeat('c', 64); $client = new RecoveryTestAgentClient(); CloudAgentSettingsService::$client = $client; $capsule->getConnection()->enableQueryLog(); $capsule->getConnection()->flushQueryLog(); $client->responses[] = [ 'operation_id' => $operationId, 'operation_status' => 'uncertain', 'checked_at' => 1_800_000_000, 'outcome' => 'uncertain', 'summary_code' => 'abortable_job_active', // Unknown codes and unrelated fields must not reach the browser DTO. 'error_code' => 'raw_libvirt_secret_path', 'raw_error' => '/srv/private/secret.qcow2', 'abort_job' => [ 'available' => true, 'kind' => 'backup', 'inspection_token' => $token, ], 'cleanup' => ['available' => false], ]; $inspection = Recovery::inspect($admin, '41'); $inspectQueries = $capsule->getConnection()->getQueryLog(); $assert($inspection['operation_id'] === $operationId && $inspection['summary_code'] === 'abortable_job_active' && $inspection['abort_job'] === [ 'available' => true, 'kind' => 'backup', 'label' => '中止匹配的备份作业', 'inspection_token' => $token, ] && $inspection['cleanup']['available'] === false && !isset($inspection['error_code']) && !str_contains(json_encode($inspection), '/srv/private'), 'Read-only inspection was not strictly normalized.'); $assert(count($inspectQueries) === 1 && !str_contains(strtolower($inspectQueries[0]['query']), 'sealed_payload') && !str_contains(strtolower($inspectQueries[0]['query']), 'lease_token'), 'Explicit recovery read selected task secrets or lease tokens.'); $assert(CloudAgentSettingsService::$calls === [[7, 30.0]] && $client->requests[0]['method'] === 'GET' && $client->requests[0]['target'] === '/v1/operations/' . $operationId . '/inspect' && $client->requests[0]['payload'] === null, 'Inspection did not use the signed read-only Agent endpoint.'); $client->responses[] = [ 'operation_id' => $operationId, 'operation_status' => 'uncertain', 'checked_at' => 1_800_000_001, 'outcome' => 'uncertain', 'summary_code' => 'evidence_incomplete', 'abort_job' => ['available' => false], 'cleanup' => ['available' => false], ]; $afterAbort = Recovery::abortJob($admin, '41', [ 'inspection_token' => $token, ]); $abortRequest = $client->requests[1]; $assert($abortRequest['method'] === 'POST' && str_ends_with($abortRequest['target'], '/abort-job') && $abortRequest['payload'] === ['inspection_token' => $token] && CloudAgentSettingsService::$calls[1] === [7, 60.0] && !$afterAbort['abort_job']['available'], 'Abort was not bound to the inspected job token.'); $client->responses[] = [ 'operation_id' => $operationId, 'operation_status' => 'uncertain', 'checked_at' => 1_800_000_002, 'outcome' => 'uncertain', 'summary_code' => 'cleanup_required', 'abort_job' => ['available' => false], 'cleanup' => [ 'available' => true, 'plan_id' => $planId, 'steps' => ['destroy_domain', 'delete_managed_volumes', 'remove_traffic_state', 'persist_cleanup_metadata'], ], ]; $cleanup = Recovery::cleanup($admin, '41', ['plan_id' => $planId]); $cleanupRequest = $client->requests[2]; $assert($cleanupRequest['method'] === 'POST' && str_ends_with($cleanupRequest['target'], '/cleanup') && $cleanupRequest['payload'] === ['plan_id' => $planId] && CloudAgentSettingsService::$calls[2] === [7, 900.0] && $cleanup['cleanup']['step_codes'] === [ 'destroy_domain', 'delete_managed_volumes', 'remove_traffic_state', 'persist_cleanup_metadata', ] && $cleanup['cleanup']['steps'] === [ '停止残留虚拟机', '删除面板管理的残留磁盘', '删除残留流量计量状态', '清除已完成磁盘切换的待清理标记', ], 'Cleanup was not bound to the exact inspected plan.'); // Recovery errors arrive as CloudAgentError rather than operation records. // Only stable codes may survive; the Agent's free-form message, cause and // result must never be copied into the browser-facing exception. $rawAgentDetail = 'raw message cause=/srv/private/domain.xml ' . 'result={"secret":"never-expose"}'; $errorCases = [ [ 'operation_inspection_stale', false, static fn () => Recovery::abortJob($admin, '41', [ 'inspection_token' => $token, ]), '核对凭证已失效,请重新立即核对', ], [ 'cleanup_plan_stale', false, static fn () => Recovery::cleanup($admin, '41', [ 'plan_id' => $planId, ]), '宿主机状态已变化,清理计划已失效,请重新立即核对', ], [ 'job_abort_unconfirmed', true, static fn () => Recovery::abortJob($admin, '41', [ 'inspection_token' => $token, ]), 'libvirt 作业中止后的终态仍未确认,请稍后再次立即核对', ], [ 'operation_uncertain', true, static fn () => Recovery::cleanup($admin, '41', [ 'plan_id' => $planId, ]), '宿主机操作结果仍未确认,请稍后再次立即核对', ], ]; foreach ($errorCases as [$code, $retryable, $call, $message]) { $client->responses[] = new CloudAgentError( $rawAgentDetail, 409, $code, $retryable, 409, true, true ); $mapped = $capture($call); $assert($mapped->httpStatus === 409 && $mapped->agentCode === $code && $mapped->retryable === $retryable && $mapped->getMessage() === $message && $mapped->safeValidationFailure === false && $mapped->safePreDispatchFailure === false && !str_contains($mapped->getMessage(), '/srv/private') && !str_contains($mapped->getMessage(), 'never-expose'), 'Recovery error was not safely mapped: ' . $code); } $client->responses[] = new CloudAgentError( $rawAgentDetail, 500, 'unexpected_private_backend_code', true, 500 ); $unknown = $capture(static fn () => Recovery::inspect($admin, '41')); $assert($unknown->httpStatus === 503 && $unknown->agentCode === 'operation_recovery_unconfirmed' && $unknown->retryable === true && $unknown->getMessage() === '宿主机状态暂时无法确认,请稍后重试' && !str_contains($unknown->getMessage(), '/srv/private') && !str_contains($unknown->getMessage(), 'never-expose'), 'Unknown Agent recovery error was exposed or misclassified.'); $beforeInvalid = count($client->requests); $reject(static fn () => Recovery::abortJob($admin, '41', [ 'inspection_token' => strtoupper($token), ]), 409); $reject(static fn () => Recovery::cleanup($admin, '41', [ 'plan_id' => $planId, 'confirm' => true, ]), 422); $reject(static fn () => Recovery::inspect($readonly, '41'), 403); $assert(count($client->requests) === $beforeInvalid, 'Invalid or unauthorized recovery request reached the Agent.'); $client->responses[] = [ 'operation_id' => $operationId, 'operation_status' => 'uncertain', 'checked_at' => 1_800_000_003, 'outcome' => 'uncertain', 'summary_code' => 'cleanup_required', 'abort_job' => [ 'available' => true, 'kind' => 'backup', 'inspection_token' => $token, ], 'cleanup' => [ 'available' => true, 'plan_id' => $planId, 'steps' => ['destroy_domain'], ], ]; $reject(static fn () => Recovery::inspect($admin, '41'), 502, 'agent_invalid_inspection'); $client->responses[] = [ 'operation_id' => $operationId, 'operation_status' => 'uncertain', 'checked_at' => 1_800_000_004, 'outcome' => 'failed', 'summary_code' => 'operation_failed', 'error_code' => 'snapshot_not_created', 'abort_job' => ['available' => false], 'cleanup' => ['available' => false], ]; $failed = Recovery::inspect($admin, '41'); $assert(($failed['error_code'] ?? null) === 'snapshot_not_created' && ($failed['error'] ?? null) === '宿主机证据确认快照没有创建', 'Known inspection error code was not mapped to a safe Chinese action message.'); echo "Administrator operation inspect, recovery-error, abort-token and cleanup-plan contracts passed.\n"; }