You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudAdminOperationRecovery...

338 lines
13 KiB
PHP

<?php
namespace App\Services\Cloud {
final class RecoveryTestAgentClient
{
public array $requests = [];
public array $responses = [];
public function request(
string $method,
string $target,
array $actor,
?array $payload = null
): array {
$this->requests[] = compact('method', 'target', 'actor', 'payload');
if ($this->responses === []) {
throw new \RuntimeException('Unexpected Agent request.');
}
$response = array_shift($this->responses);
if ($response instanceof \Throwable) {
throw $response;
}
if (!is_array($response)) {
throw new \RuntimeException('Invalid recovery test response.');
}
return $response;
}
}
final class CloudAgentSettingsService
{
public static ?RecoveryTestAgentClient $client = null;
public static array $calls = [];
public static function clientForConfiguration(
int $hostId,
float $timeoutSeconds = 120.0
): RecoveryTestAgentClient {
self::$calls[] = [$hostId, $timeoutSeconds];
return self::$client
?? throw new \RuntimeException('Missing recovery test client.');
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Services\Cloud\CloudAdminOperationRecoveryService as Recovery;
use App\Services\Cloud\CloudAgentError;
use App\Services\Cloud\CloudAgentSettingsService;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\RecoveryTestAgentClient;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function ($condition, string $message): void {
if (!$condition) {
throw new \RuntimeException($message);
}
};
$reject = static function (
callable $callback,
int $status,
?string $agentCode = null
) use ($assert): void {
try {
$callback();
} catch (CloudError $error) {
$assert($error->httpStatus === $status,
'Unexpected rejection status: ' . $error->httpStatus);
if ($agentCode !== null) {
$assert($error instanceof CloudAgentError
&& $error->agentCode === $agentCode,
'Unexpected Agent rejection code.');
}
return;
}
throw new \RuntimeException('Invalid recovery request was accepted.');
};
$capture = static function (callable $callback) use ($assert): CloudAgentError {
try {
$callback();
} catch (CloudAgentError $error) {
return $error;
}
$assert(false, 'Expected a controlled Agent error.');
throw new \RuntimeException('Unreachable.');
};
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
$capsule->setAsGlobal();
$capsule->bootEloquent();
$capsule->schema()->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id');
$table->integer('host_id');
$table->string('kind');
$table->string('status');
$table->integer('dispatched_at')->nullable();
$table->text('sealed_payload')->default('forbidden-payload');
$table->string('lease_token')->nullable()->default('forbidden-lease');
$table->integer('created_at')->nullable();
$table->integer('updated_at')->nullable();
});
Capsule::table('cloud_jobs')->insert([
'id' => 41,
'operation_id' => 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
'host_id' => 7,
'kind' => 'snapshot_create',
'status' => 'uncertain',
'dispatched_at' => 1_799_999_900,
]);
$admin = [
'user_id' => 99,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
];
$readonly = [
'user_id' => 98,
'is_admin' => true,
'is_super_admin' => false,
'can_manage' => false,
];
$operationId = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';
$token = str_repeat('b', 64);
$planId = str_repeat('c', 64);
$client = new RecoveryTestAgentClient();
CloudAgentSettingsService::$client = $client;
$capsule->getConnection()->enableQueryLog();
$capsule->getConnection()->flushQueryLog();
$client->responses[] = [
'operation_id' => $operationId,
'operation_status' => 'uncertain',
'checked_at' => 1_800_000_000,
'outcome' => 'uncertain',
'summary_code' => 'abortable_job_active',
// Unknown codes and unrelated fields must not reach the browser DTO.
'error_code' => 'raw_libvirt_secret_path',
'raw_error' => '/srv/private/secret.qcow2',
'abort_job' => [
'available' => true,
'kind' => 'backup',
'inspection_token' => $token,
],
'cleanup' => ['available' => false],
];
$inspection = Recovery::inspect($admin, '41');
$inspectQueries = $capsule->getConnection()->getQueryLog();
$assert($inspection['operation_id'] === $operationId
&& $inspection['summary_code'] === 'abortable_job_active'
&& $inspection['abort_job'] === [
'available' => true,
'kind' => 'backup',
'label' => '中止匹配的备份作业',
'inspection_token' => $token,
]
&& $inspection['cleanup']['available'] === false
&& !isset($inspection['error_code'])
&& !str_contains(json_encode($inspection), '/srv/private'),
'Read-only inspection was not strictly normalized.');
$assert(count($inspectQueries) === 1
&& !str_contains(strtolower($inspectQueries[0]['query']), 'sealed_payload')
&& !str_contains(strtolower($inspectQueries[0]['query']), 'lease_token'),
'Explicit recovery read selected task secrets or lease tokens.');
$assert(CloudAgentSettingsService::$calls === [[7, 30.0]]
&& $client->requests[0]['method'] === 'GET'
&& $client->requests[0]['target']
=== '/v1/operations/' . $operationId . '/inspect'
&& $client->requests[0]['payload'] === null,
'Inspection did not use the signed read-only Agent endpoint.');
$client->responses[] = [
'operation_id' => $operationId,
'operation_status' => 'uncertain',
'checked_at' => 1_800_000_001,
'outcome' => 'uncertain',
'summary_code' => 'evidence_incomplete',
'abort_job' => ['available' => false],
'cleanup' => ['available' => false],
];
$afterAbort = Recovery::abortJob($admin, '41', [
'inspection_token' => $token,
]);
$abortRequest = $client->requests[1];
$assert($abortRequest['method'] === 'POST'
&& str_ends_with($abortRequest['target'], '/abort-job')
&& $abortRequest['payload'] === ['inspection_token' => $token]
&& CloudAgentSettingsService::$calls[1] === [7, 60.0]
&& !$afterAbort['abort_job']['available'],
'Abort was not bound to the inspected job token.');
$client->responses[] = [
'operation_id' => $operationId,
'operation_status' => 'uncertain',
'checked_at' => 1_800_000_002,
'outcome' => 'uncertain',
'summary_code' => 'cleanup_required',
'abort_job' => ['available' => false],
'cleanup' => [
'available' => true,
'plan_id' => $planId,
'steps' => ['destroy_domain', 'delete_managed_volumes',
'remove_traffic_state', 'persist_cleanup_metadata'],
],
];
$cleanup = Recovery::cleanup($admin, '41', ['plan_id' => $planId]);
$cleanupRequest = $client->requests[2];
$assert($cleanupRequest['method'] === 'POST'
&& str_ends_with($cleanupRequest['target'], '/cleanup')
&& $cleanupRequest['payload'] === ['plan_id' => $planId]
&& CloudAgentSettingsService::$calls[2] === [7, 900.0]
&& $cleanup['cleanup']['step_codes'] === [
'destroy_domain', 'delete_managed_volumes', 'remove_traffic_state',
'persist_cleanup_metadata',
]
&& $cleanup['cleanup']['steps'] === [
'停止残留虚拟机', '删除面板管理的残留磁盘', '删除残留流量计量状态',
'清除已完成磁盘切换的待清理标记',
], 'Cleanup was not bound to the exact inspected plan.');
// Recovery errors arrive as CloudAgentError rather than operation records.
// Only stable codes may survive; the Agent's free-form message, cause and
// result must never be copied into the browser-facing exception.
$rawAgentDetail = 'raw message cause=/srv/private/domain.xml '
. 'result={"secret":"never-expose"}';
$errorCases = [
[
'operation_inspection_stale', false,
static fn () => Recovery::abortJob($admin, '41', [
'inspection_token' => $token,
]),
'核对凭证已失效,请重新立即核对',
],
[
'cleanup_plan_stale', false,
static fn () => Recovery::cleanup($admin, '41', [
'plan_id' => $planId,
]),
'宿主机状态已变化,清理计划已失效,请重新立即核对',
],
[
'job_abort_unconfirmed', true,
static fn () => Recovery::abortJob($admin, '41', [
'inspection_token' => $token,
]),
'libvirt 作业中止后的终态仍未确认,请稍后再次立即核对',
],
[
'operation_uncertain', true,
static fn () => Recovery::cleanup($admin, '41', [
'plan_id' => $planId,
]),
'宿主机操作结果仍未确认,请稍后再次立即核对',
],
];
foreach ($errorCases as [$code, $retryable, $call, $message]) {
$client->responses[] = new CloudAgentError(
$rawAgentDetail, 409, $code, $retryable, 409, true, true
);
$mapped = $capture($call);
$assert($mapped->httpStatus === 409
&& $mapped->agentCode === $code
&& $mapped->retryable === $retryable
&& $mapped->getMessage() === $message
&& $mapped->safeValidationFailure === false
&& $mapped->safePreDispatchFailure === false
&& !str_contains($mapped->getMessage(), '/srv/private')
&& !str_contains($mapped->getMessage(), 'never-expose'),
'Recovery error was not safely mapped: ' . $code);
}
$client->responses[] = new CloudAgentError(
$rawAgentDetail, 500, 'unexpected_private_backend_code', true, 500
);
$unknown = $capture(static fn () => Recovery::inspect($admin, '41'));
$assert($unknown->httpStatus === 503
&& $unknown->agentCode === 'operation_recovery_unconfirmed'
&& $unknown->retryable === true
&& $unknown->getMessage()
=== '宿主机状态暂时无法确认,请稍后重试'
&& !str_contains($unknown->getMessage(), '/srv/private')
&& !str_contains($unknown->getMessage(), 'never-expose'),
'Unknown Agent recovery error was exposed or misclassified.');
$beforeInvalid = count($client->requests);
$reject(static fn () => Recovery::abortJob($admin, '41', [
'inspection_token' => strtoupper($token),
]), 409);
$reject(static fn () => Recovery::cleanup($admin, '41', [
'plan_id' => $planId, 'confirm' => true,
]), 422);
$reject(static fn () => Recovery::inspect($readonly, '41'), 403);
$assert(count($client->requests) === $beforeInvalid,
'Invalid or unauthorized recovery request reached the Agent.');
$client->responses[] = [
'operation_id' => $operationId,
'operation_status' => 'uncertain',
'checked_at' => 1_800_000_003,
'outcome' => 'uncertain',
'summary_code' => 'cleanup_required',
'abort_job' => [
'available' => true,
'kind' => 'backup',
'inspection_token' => $token,
],
'cleanup' => [
'available' => true,
'plan_id' => $planId,
'steps' => ['destroy_domain'],
],
];
$reject(static fn () => Recovery::inspect($admin, '41'), 502,
'agent_invalid_inspection');
$client->responses[] = [
'operation_id' => $operationId,
'operation_status' => 'uncertain',
'checked_at' => 1_800_000_004,
'outcome' => 'failed',
'summary_code' => 'operation_failed',
'error_code' => 'snapshot_not_created',
'abort_job' => ['available' => false],
'cleanup' => ['available' => false],
];
$failed = Recovery::inspect($admin, '41');
$assert(($failed['error_code'] ?? null) === 'snapshot_not_created'
&& ($failed['error'] ?? null) === '宿主机证据确认快照没有创建',
'Known inspection error code was not mapped to a safe Chinese action message.');
echo "Administrator operation inspect, recovery-error, abort-token and cleanup-plan contracts passed.\n";
}