You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
338 lines
13 KiB
PHP
338 lines
13 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud {
|
|
final class RecoveryTestAgentClient
|
|
{
|
|
public array $requests = [];
|
|
public array $responses = [];
|
|
|
|
public function request(
|
|
string $method,
|
|
string $target,
|
|
array $actor,
|
|
?array $payload = null
|
|
): array {
|
|
$this->requests[] = compact('method', 'target', 'actor', 'payload');
|
|
if ($this->responses === []) {
|
|
throw new \RuntimeException('Unexpected Agent request.');
|
|
}
|
|
$response = array_shift($this->responses);
|
|
if ($response instanceof \Throwable) {
|
|
throw $response;
|
|
}
|
|
if (!is_array($response)) {
|
|
throw new \RuntimeException('Invalid recovery test response.');
|
|
}
|
|
return $response;
|
|
}
|
|
}
|
|
|
|
final class CloudAgentSettingsService
|
|
{
|
|
public static ?RecoveryTestAgentClient $client = null;
|
|
public static array $calls = [];
|
|
|
|
public static function clientForConfiguration(
|
|
int $hostId,
|
|
float $timeoutSeconds = 120.0
|
|
): RecoveryTestAgentClient {
|
|
self::$calls[] = [$hostId, $timeoutSeconds];
|
|
return self::$client
|
|
?? throw new \RuntimeException('Missing recovery test client.');
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Services\Cloud\CloudAdminOperationRecoveryService as Recovery;
|
|
use App\Services\Cloud\CloudAgentError;
|
|
use App\Services\Cloud\CloudAgentSettingsService;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\RecoveryTestAgentClient;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function ($condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new \RuntimeException($message);
|
|
}
|
|
};
|
|
$reject = static function (
|
|
callable $callback,
|
|
int $status,
|
|
?string $agentCode = null
|
|
) use ($assert): void {
|
|
try {
|
|
$callback();
|
|
} catch (CloudError $error) {
|
|
$assert($error->httpStatus === $status,
|
|
'Unexpected rejection status: ' . $error->httpStatus);
|
|
if ($agentCode !== null) {
|
|
$assert($error instanceof CloudAgentError
|
|
&& $error->agentCode === $agentCode,
|
|
'Unexpected Agent rejection code.');
|
|
}
|
|
return;
|
|
}
|
|
throw new \RuntimeException('Invalid recovery request was accepted.');
|
|
};
|
|
$capture = static function (callable $callback) use ($assert): CloudAgentError {
|
|
try {
|
|
$callback();
|
|
} catch (CloudAgentError $error) {
|
|
return $error;
|
|
}
|
|
$assert(false, 'Expected a controlled Agent error.');
|
|
throw new \RuntimeException('Unreachable.');
|
|
};
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$capsule->schema()->create('cloud_jobs', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('operation_id');
|
|
$table->integer('host_id');
|
|
$table->string('kind');
|
|
$table->string('status');
|
|
$table->integer('dispatched_at')->nullable();
|
|
$table->text('sealed_payload')->default('forbidden-payload');
|
|
$table->string('lease_token')->nullable()->default('forbidden-lease');
|
|
$table->integer('created_at')->nullable();
|
|
$table->integer('updated_at')->nullable();
|
|
});
|
|
Capsule::table('cloud_jobs')->insert([
|
|
'id' => 41,
|
|
'operation_id' => 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
|
|
'host_id' => 7,
|
|
'kind' => 'snapshot_create',
|
|
'status' => 'uncertain',
|
|
'dispatched_at' => 1_799_999_900,
|
|
]);
|
|
|
|
$admin = [
|
|
'user_id' => 99,
|
|
'is_admin' => true,
|
|
'is_super_admin' => true,
|
|
'can_manage' => true,
|
|
];
|
|
$readonly = [
|
|
'user_id' => 98,
|
|
'is_admin' => true,
|
|
'is_super_admin' => false,
|
|
'can_manage' => false,
|
|
];
|
|
$operationId = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';
|
|
$token = str_repeat('b', 64);
|
|
$planId = str_repeat('c', 64);
|
|
$client = new RecoveryTestAgentClient();
|
|
CloudAgentSettingsService::$client = $client;
|
|
$capsule->getConnection()->enableQueryLog();
|
|
$capsule->getConnection()->flushQueryLog();
|
|
|
|
$client->responses[] = [
|
|
'operation_id' => $operationId,
|
|
'operation_status' => 'uncertain',
|
|
'checked_at' => 1_800_000_000,
|
|
'outcome' => 'uncertain',
|
|
'summary_code' => 'abortable_job_active',
|
|
// Unknown codes and unrelated fields must not reach the browser DTO.
|
|
'error_code' => 'raw_libvirt_secret_path',
|
|
'raw_error' => '/srv/private/secret.qcow2',
|
|
'abort_job' => [
|
|
'available' => true,
|
|
'kind' => 'backup',
|
|
'inspection_token' => $token,
|
|
],
|
|
'cleanup' => ['available' => false],
|
|
];
|
|
$inspection = Recovery::inspect($admin, '41');
|
|
$inspectQueries = $capsule->getConnection()->getQueryLog();
|
|
$assert($inspection['operation_id'] === $operationId
|
|
&& $inspection['summary_code'] === 'abortable_job_active'
|
|
&& $inspection['abort_job'] === [
|
|
'available' => true,
|
|
'kind' => 'backup',
|
|
'label' => '中止匹配的备份作业',
|
|
'inspection_token' => $token,
|
|
]
|
|
&& $inspection['cleanup']['available'] === false
|
|
&& !isset($inspection['error_code'])
|
|
&& !str_contains(json_encode($inspection), '/srv/private'),
|
|
'Read-only inspection was not strictly normalized.');
|
|
$assert(count($inspectQueries) === 1
|
|
&& !str_contains(strtolower($inspectQueries[0]['query']), 'sealed_payload')
|
|
&& !str_contains(strtolower($inspectQueries[0]['query']), 'lease_token'),
|
|
'Explicit recovery read selected task secrets or lease tokens.');
|
|
$assert(CloudAgentSettingsService::$calls === [[7, 30.0]]
|
|
&& $client->requests[0]['method'] === 'GET'
|
|
&& $client->requests[0]['target']
|
|
=== '/v1/operations/' . $operationId . '/inspect'
|
|
&& $client->requests[0]['payload'] === null,
|
|
'Inspection did not use the signed read-only Agent endpoint.');
|
|
|
|
$client->responses[] = [
|
|
'operation_id' => $operationId,
|
|
'operation_status' => 'uncertain',
|
|
'checked_at' => 1_800_000_001,
|
|
'outcome' => 'uncertain',
|
|
'summary_code' => 'evidence_incomplete',
|
|
'abort_job' => ['available' => false],
|
|
'cleanup' => ['available' => false],
|
|
];
|
|
$afterAbort = Recovery::abortJob($admin, '41', [
|
|
'inspection_token' => $token,
|
|
]);
|
|
$abortRequest = $client->requests[1];
|
|
$assert($abortRequest['method'] === 'POST'
|
|
&& str_ends_with($abortRequest['target'], '/abort-job')
|
|
&& $abortRequest['payload'] === ['inspection_token' => $token]
|
|
&& CloudAgentSettingsService::$calls[1] === [7, 60.0]
|
|
&& !$afterAbort['abort_job']['available'],
|
|
'Abort was not bound to the inspected job token.');
|
|
|
|
$client->responses[] = [
|
|
'operation_id' => $operationId,
|
|
'operation_status' => 'uncertain',
|
|
'checked_at' => 1_800_000_002,
|
|
'outcome' => 'uncertain',
|
|
'summary_code' => 'cleanup_required',
|
|
'abort_job' => ['available' => false],
|
|
'cleanup' => [
|
|
'available' => true,
|
|
'plan_id' => $planId,
|
|
'steps' => ['destroy_domain', 'delete_managed_volumes',
|
|
'remove_traffic_state', 'persist_cleanup_metadata'],
|
|
],
|
|
];
|
|
$cleanup = Recovery::cleanup($admin, '41', ['plan_id' => $planId]);
|
|
$cleanupRequest = $client->requests[2];
|
|
$assert($cleanupRequest['method'] === 'POST'
|
|
&& str_ends_with($cleanupRequest['target'], '/cleanup')
|
|
&& $cleanupRequest['payload'] === ['plan_id' => $planId]
|
|
&& CloudAgentSettingsService::$calls[2] === [7, 900.0]
|
|
&& $cleanup['cleanup']['step_codes'] === [
|
|
'destroy_domain', 'delete_managed_volumes', 'remove_traffic_state',
|
|
'persist_cleanup_metadata',
|
|
]
|
|
&& $cleanup['cleanup']['steps'] === [
|
|
'停止残留虚拟机', '删除面板管理的残留磁盘', '删除残留流量计量状态',
|
|
'清除已完成磁盘切换的待清理标记',
|
|
], 'Cleanup was not bound to the exact inspected plan.');
|
|
|
|
// Recovery errors arrive as CloudAgentError rather than operation records.
|
|
// Only stable codes may survive; the Agent's free-form message, cause and
|
|
// result must never be copied into the browser-facing exception.
|
|
$rawAgentDetail = 'raw message cause=/srv/private/domain.xml '
|
|
. 'result={"secret":"never-expose"}';
|
|
$errorCases = [
|
|
[
|
|
'operation_inspection_stale', false,
|
|
static fn () => Recovery::abortJob($admin, '41', [
|
|
'inspection_token' => $token,
|
|
]),
|
|
'核对凭证已失效,请重新立即核对',
|
|
],
|
|
[
|
|
'cleanup_plan_stale', false,
|
|
static fn () => Recovery::cleanup($admin, '41', [
|
|
'plan_id' => $planId,
|
|
]),
|
|
'宿主机状态已变化,清理计划已失效,请重新立即核对',
|
|
],
|
|
[
|
|
'job_abort_unconfirmed', true,
|
|
static fn () => Recovery::abortJob($admin, '41', [
|
|
'inspection_token' => $token,
|
|
]),
|
|
'libvirt 作业中止后的终态仍未确认,请稍后再次立即核对',
|
|
],
|
|
[
|
|
'operation_uncertain', true,
|
|
static fn () => Recovery::cleanup($admin, '41', [
|
|
'plan_id' => $planId,
|
|
]),
|
|
'宿主机操作结果仍未确认,请稍后再次立即核对',
|
|
],
|
|
];
|
|
foreach ($errorCases as [$code, $retryable, $call, $message]) {
|
|
$client->responses[] = new CloudAgentError(
|
|
$rawAgentDetail, 409, $code, $retryable, 409, true, true
|
|
);
|
|
$mapped = $capture($call);
|
|
$assert($mapped->httpStatus === 409
|
|
&& $mapped->agentCode === $code
|
|
&& $mapped->retryable === $retryable
|
|
&& $mapped->getMessage() === $message
|
|
&& $mapped->safeValidationFailure === false
|
|
&& $mapped->safePreDispatchFailure === false
|
|
&& !str_contains($mapped->getMessage(), '/srv/private')
|
|
&& !str_contains($mapped->getMessage(), 'never-expose'),
|
|
'Recovery error was not safely mapped: ' . $code);
|
|
}
|
|
|
|
$client->responses[] = new CloudAgentError(
|
|
$rawAgentDetail, 500, 'unexpected_private_backend_code', true, 500
|
|
);
|
|
$unknown = $capture(static fn () => Recovery::inspect($admin, '41'));
|
|
$assert($unknown->httpStatus === 503
|
|
&& $unknown->agentCode === 'operation_recovery_unconfirmed'
|
|
&& $unknown->retryable === true
|
|
&& $unknown->getMessage()
|
|
=== '宿主机状态暂时无法确认,请稍后重试'
|
|
&& !str_contains($unknown->getMessage(), '/srv/private')
|
|
&& !str_contains($unknown->getMessage(), 'never-expose'),
|
|
'Unknown Agent recovery error was exposed or misclassified.');
|
|
|
|
$beforeInvalid = count($client->requests);
|
|
$reject(static fn () => Recovery::abortJob($admin, '41', [
|
|
'inspection_token' => strtoupper($token),
|
|
]), 409);
|
|
$reject(static fn () => Recovery::cleanup($admin, '41', [
|
|
'plan_id' => $planId, 'confirm' => true,
|
|
]), 422);
|
|
$reject(static fn () => Recovery::inspect($readonly, '41'), 403);
|
|
$assert(count($client->requests) === $beforeInvalid,
|
|
'Invalid or unauthorized recovery request reached the Agent.');
|
|
|
|
$client->responses[] = [
|
|
'operation_id' => $operationId,
|
|
'operation_status' => 'uncertain',
|
|
'checked_at' => 1_800_000_003,
|
|
'outcome' => 'uncertain',
|
|
'summary_code' => 'cleanup_required',
|
|
'abort_job' => [
|
|
'available' => true,
|
|
'kind' => 'backup',
|
|
'inspection_token' => $token,
|
|
],
|
|
'cleanup' => [
|
|
'available' => true,
|
|
'plan_id' => $planId,
|
|
'steps' => ['destroy_domain'],
|
|
],
|
|
];
|
|
$reject(static fn () => Recovery::inspect($admin, '41'), 502,
|
|
'agent_invalid_inspection');
|
|
|
|
$client->responses[] = [
|
|
'operation_id' => $operationId,
|
|
'operation_status' => 'uncertain',
|
|
'checked_at' => 1_800_000_004,
|
|
'outcome' => 'failed',
|
|
'summary_code' => 'operation_failed',
|
|
'error_code' => 'snapshot_not_created',
|
|
'abort_job' => ['available' => false],
|
|
'cleanup' => ['available' => false],
|
|
];
|
|
$failed = Recovery::inspect($admin, '41');
|
|
$assert(($failed['error_code'] ?? null) === 'snapshot_not_created'
|
|
&& ($failed['error'] ?? null) === '宿主机证据确认快照没有创建',
|
|
'Known inspection error code was not mapped to a safe Chinese action message.');
|
|
|
|
echo "Administrator operation inspect, recovery-error, abort-token and cleanup-plan contracts passed.\n";
|
|
}
|