You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudAdminOperationRecovery...

578 lines
23 KiB
PHP

<?php
namespace App\Services\Cloud {
final class CloudSecrets
{
public static function decrypt(string $value, string $purpose): array
{
return [
'method' => 'POST',
'request_target' => '/v1/vms/test/actions',
'body' => ['action' => 'reboot'],
'owner_id' => 99,
];
}
public static function encrypt(array $value, string $purpose): string
{
return 'sealed-terminal-audit';
}
}
final class RecoveryTestAgentClient
{
public array $requests = [];
public array $responses = [];
public function request(
string $method,
string $target,
array $actor,
?array $payload = null
): array {
$this->requests[] = compact('method', 'target', 'actor', 'payload');
if ($this->responses === []) {
throw new \RuntimeException('Unexpected Agent request.');
}
$response = array_shift($this->responses);
if ($response instanceof \Throwable) {
throw $response;
}
if (!is_array($response)) {
throw new \RuntimeException('Invalid recovery test response.');
}
return $response;
}
}
final class CloudAgentSettingsService
{
public static ?RecoveryTestAgentClient $client = null;
public static array $calls = [];
public static function clientForConfiguration(
int $hostId,
float $timeoutSeconds = 120.0
): RecoveryTestAgentClient {
self::$calls[] = [$hostId, $timeoutSeconds];
return self::$client
?? throw new \RuntimeException('Missing recovery test client.');
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Services\Cloud\CloudAdminOperationRecoveryService as Recovery;
use App\Services\Cloud\CloudAgentError;
use App\Services\Cloud\CloudAgentSettingsService;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\RecoveryTestAgentClient;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function ($condition, string $message): void {
if (!$condition) {
throw new \RuntimeException($message);
}
};
$reject = static function (
callable $callback,
int $status,
?string $agentCode = null
) use ($assert): void {
try {
$callback();
} catch (CloudError $error) {
$assert($error->httpStatus === $status,
'Unexpected rejection status: ' . $error->httpStatus);
if ($agentCode !== null) {
$assert($error instanceof CloudAgentError
&& $error->agentCode === $agentCode,
'Unexpected Agent rejection code.');
}
return;
}
throw new \RuntimeException('Invalid recovery request was accepted.');
};
$capture = static function (callable $callback) use ($assert): CloudAgentError {
try {
$callback();
} catch (CloudAgentError $error) {
return $error;
}
$assert(false, 'Expected a controlled Agent error.');
throw new \RuntimeException('Unreachable.');
};
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
$capsule->setAsGlobal();
$capsule->bootEloquent();
$capsule->schema()->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id');
$table->integer('host_id');
$table->string('kind');
$table->string('status');
$table->integer('user_id')->default(99);
$table->integer('instance_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('attempts')->default(1);
$table->integer('dispatched_at')->nullable();
$table->integer('available_at')->default(0);
$table->integer('lease_until')->default(0);
$table->string('error_code')->nullable();
$table->text('result_json')->nullable();
$table->text('sealed_payload')->default('forbidden-payload');
$table->string('lease_token')->nullable();
$table->integer('created_at')->nullable();
$table->integer('updated_at')->nullable();
});
$capsule->schema()->create('user', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('enable')->default(1);
});
Capsule::table('user')->insert(['id' => 99, 'enable' => 1]);
Capsule::table('cloud_jobs')->insert([
'id' => 41,
'operation_id' => 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
'host_id' => 7,
'kind' => 'action',
'status' => 'uncertain',
'dispatched_at' => 1_799_999_900,
]);
Capsule::table('cloud_jobs')->insert([
'id' => 42,
'operation_id' => 'dddddddddddddddddddddddddddddddd',
'host_id' => 7,
'kind' => 'action',
'status' => 'needs_attention',
'error_code' => 'reconciliation_limit_reached',
'dispatched_at' => 1_799_999_900,
]);
$admin = [
'user_id' => 99,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
];
$readonly = [
'user_id' => 98,
'is_admin' => true,
'is_super_admin' => false,
'can_manage' => false,
];
$operationId = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';
$token = str_repeat('b', 64);
$planId = str_repeat('c', 64);
$client = new RecoveryTestAgentClient();
CloudAgentSettingsService::$client = $client;
$capsule->getConnection()->enableQueryLog();
$capsule->getConnection()->flushQueryLog();
$client->responses[] = [
'operation_id' => $operationId,
'operation_status' => 'uncertain',
'checked_at' => 1_800_000_000,
'outcome' => 'uncertain',
'summary_code' => 'abortable_job_active',
// Unknown codes and unrelated fields must not reach the browser DTO.
'error_code' => 'raw_libvirt_secret_path',
'raw_error' => '/srv/private/secret.qcow2',
'abort_job' => [
'available' => true,
'kind' => 'backup',
'inspection_token' => $token,
],
'cleanup' => ['available' => false],
'confirmation' => ['available' => false],
];
$inspection = Recovery::inspect($admin, '41');
$inspectQueries = $capsule->getConnection()->getQueryLog();
$assert($inspection['operation_id'] === $operationId
&& $inspection['summary_code'] === 'abortable_job_active'
&& $inspection['abort_job'] === [
'available' => true,
'kind' => 'backup',
'label' => '中止匹配的备份作业',
'inspection_token' => $token,
]
&& $inspection['cleanup']['available'] === false
&& !isset($inspection['error_code'])
&& !str_contains(json_encode($inspection), '/srv/private'),
'Read-only inspection was not strictly normalized.');
$assert(count($inspectQueries) === 1
&& !str_contains(strtolower($inspectQueries[0]['query']), 'sealed_payload')
&& !str_contains(strtolower($inspectQueries[0]['query']), 'lease_token'),
'Explicit recovery read selected task secrets or lease tokens.');
$assert(CloudAgentSettingsService::$calls === [[7, 30.0]]
&& $client->requests[0]['method'] === 'GET'
&& $client->requests[0]['target']
=== '/v1/operations/' . $operationId . '/inspect'
&& $client->requests[0]['payload'] === null,
'Inspection did not use the signed read-only Agent endpoint.');
$client->responses[] = [
'operation_id' => $operationId,
'operation_status' => 'uncertain',
'checked_at' => 1_800_000_001,
'outcome' => 'uncertain',
'summary_code' => 'evidence_incomplete',
'abort_job' => ['available' => false],
'cleanup' => ['available' => false],
'confirmation' => ['available' => false],
];
$afterAbort = Recovery::abortJob($admin, '41', [
'inspection_token' => $token,
]);
$abortRequest = $client->requests[1];
$assert($abortRequest['method'] === 'POST'
&& str_ends_with($abortRequest['target'], '/abort-job')
&& $abortRequest['payload'] === ['inspection_token' => $token]
&& CloudAgentSettingsService::$calls[1] === [7, 60.0]
&& !$afterAbort['abort_job']['available'],
'Abort was not bound to the inspected job token.');
$client->responses[] = [
'operation_id' => $operationId,
'operation_status' => 'uncertain',
'checked_at' => 1_800_000_002,
'outcome' => 'uncertain',
'summary_code' => 'cleanup_required',
'abort_job' => ['available' => false],
'cleanup' => [
'available' => true,
'plan_id' => $planId,
'steps' => ['destroy_domain', 'delete_managed_volumes',
'remove_traffic_state', 'persist_cleanup_metadata'],
],
'confirmation' => ['available' => false],
];
$cleanup = Recovery::cleanup($admin, '41', ['plan_id' => $planId]);
$cleanupRequest = $client->requests[2];
$assert($cleanupRequest['method'] === 'POST'
&& str_ends_with($cleanupRequest['target'], '/cleanup')
&& $cleanupRequest['payload'] === ['plan_id' => $planId]
&& CloudAgentSettingsService::$calls[2] === [7, 900.0]
&& $cleanup['cleanup']['step_codes'] === [
'destroy_domain', 'delete_managed_volumes', 'remove_traffic_state',
'persist_cleanup_metadata',
]
&& $cleanup['cleanup']['steps'] === [
'停止残留虚拟机', '删除面板管理的残留磁盘', '删除残留流量计量状态',
'清除已完成磁盘切换的待清理标记',
], 'Cleanup was not bound to the exact inspected plan.');
// Recovery errors arrive as CloudAgentError rather than operation records.
// Only stable codes may survive; the Agent's free-form message, cause and
// result must never be copied into the browser-facing exception.
$rawAgentDetail = 'raw message cause=/srv/private/domain.xml '
. 'result={"secret":"never-expose"}';
$errorCases = [
[
'operation_inspection_stale', false,
static fn () => Recovery::abortJob($admin, '41', [
'inspection_token' => $token,
]),
'核对凭证已失效,请重新立即核对',
],
[
'cleanup_plan_stale', false,
static fn () => Recovery::cleanup($admin, '41', [
'plan_id' => $planId,
]),
'宿主机状态已变化,清理计划已失效,请重新立即核对',
],
[
'job_abort_unconfirmed', true,
static fn () => Recovery::abortJob($admin, '41', [
'inspection_token' => $token,
]),
'libvirt 作业中止后的终态仍未确认,请稍后再次立即核对',
],
[
'operation_uncertain', true,
static fn () => Recovery::cleanup($admin, '41', [
'plan_id' => $planId,
]),
'宿主机操作结果仍未确认,请稍后再次立即核对',
],
[
'external_snapshot_cleanup_required', false,
static fn () => Recovery::inspect($admin, '41'),
'检测到原生外部快照或回滚元数据,请先在宿主机安全完成清理,再重新立即核对',
],
[
'native_snapshot_delete_in_progress', false,
static fn () => Recovery::inspect($admin, '41'),
'原生快照删除尚未完成,请先在宿主机继续完成该删除,再重新立即核对',
],
[
'managed_disk_shared', false,
static fn () => Recovery::inspect($admin, '41'),
'目标受管磁盘仍被另一台虚拟机的当前配置、快照或保存状态引用,请先解除引用,再重新立即核对',
],
[
'managed_filter_shared', false,
static fn () => Recovery::inspect($admin, '41'),
'目标网络过滤器仍被另一台虚拟机或过滤器引用,请先解除引用,再重新立即核对',
],
];
foreach ($errorCases as [$code, $retryable, $call, $message]) {
$client->responses[] = new CloudAgentError(
$rawAgentDetail, 409, $code, $retryable, 409, true, true
);
$mapped = $capture($call);
$assert($mapped->httpStatus === 409
&& $mapped->agentCode === $code
&& $mapped->retryable === $retryable
&& $mapped->getMessage() === $message
&& $mapped->safeValidationFailure === false
&& $mapped->safePreDispatchFailure === false
&& !str_contains($mapped->getMessage(), '/srv/private')
&& !str_contains($mapped->getMessage(), 'never-expose'),
'Recovery error was not safely mapped: ' . $code);
}
$client->responses[] = new CloudAgentError(
$rawAgentDetail, 500, 'unexpected_private_backend_code', true, 500
);
$unknown = $capture(static fn () => Recovery::inspect($admin, '41'));
$assert($unknown->httpStatus === 503
&& $unknown->agentCode === 'operation_recovery_unconfirmed'
&& $unknown->retryable === true
&& $unknown->getMessage()
=== '宿主机状态暂时无法确认,请稍后重试'
&& !str_contains($unknown->getMessage(), '/srv/private')
&& !str_contains($unknown->getMessage(), 'never-expose'),
'Unknown Agent recovery error was exposed or misclassified.');
$beforeInvalid = count($client->requests);
$reject(static fn () => Recovery::abortJob($admin, '41', [
'inspection_token' => strtoupper($token),
]), 409);
$reject(static fn () => Recovery::cleanup($admin, '41', [
'plan_id' => $planId, 'confirm' => true,
]), 422);
$reject(static fn () => Recovery::forceDelete($admin, '41', [
'ack_data_loss' => false,
]), 409);
$reject(static fn () => Recovery::forceDelete($admin, '41', [
'ack_data_loss' => true, 'confirm' => true,
]), 422);
$reject(static fn () => Recovery::forceDelete([
'user_id' => 97, 'is_admin' => true, 'is_super_admin' => false,
'can_manage' => true,
], '41', ['ack_data_loss' => true]), 403);
$reject(static fn () => Recovery::inspect($readonly, '41'), 403);
$assert(count($client->requests) === $beforeInvalid,
'Invalid or unauthorized recovery request reached the Agent.');
$routeSource = file_get_contents(
dirname(__DIR__) . '/config/routes_cloud.php'
);
$assert(is_string($routeSource) && str_contains($routeSource,
"/admin/operations/jobs/{id:[0-9]+}/force-delete"),
'Independent force-delete administrator route is missing.');
$controllerSource = file_get_contents(
dirname(__DIR__) . '/app/Controllers/CloudAdminOperationsController.php'
);
$assert(is_string($controllerSource)
&& preg_match(
'/public function inspect\\(.*?public function abortJob/s',
$controllerSource,
$inspectController
) === 1
&& strpos($inspectController[0],
'$inspection = CloudAdminOperationRecoveryService::inspect(') !== false
&& strpos($inspectController[0],
'CloudAdminOperationsService::detail($actor, $id)') !== false
&& strpos($inspectController[0],
'$inspection = CloudAdminOperationRecoveryService::inspect(')
< strpos($inspectController[0],
'CloudAdminOperationsService::detail($actor, $id)'),
'Inspect response read stale panel state before token-confirmed settlement.');
$client->responses[] = [
'operation_id' => $operationId,
'operation_status' => 'uncertain',
'checked_at' => 1_800_000_003,
'outcome' => 'uncertain',
'summary_code' => 'cleanup_required',
'abort_job' => [
'available' => true,
'kind' => 'backup',
'inspection_token' => $token,
],
'cleanup' => [
'available' => true,
'plan_id' => $planId,
'steps' => ['destroy_domain'],
],
'confirmation' => ['available' => false],
];
$reject(static fn () => Recovery::inspect($admin, '41'), 502,
'agent_invalid_inspection');
// A terminal observation must survive a second fresh read-only check
// before the Agent journal and panel ledger can be settled.
$confirmationToken = str_repeat('e', 64);
$client->responses[] = [
'operation_id' => $operationId,
'operation_status' => 'uncertain',
'checked_at' => 1_800_000_004,
'outcome' => 'failed',
'summary_code' => 'operation_failed',
'error_code' => 'snapshot_not_created',
'abort_job' => ['available' => false],
'cleanup' => ['available' => false],
'confirmation' => [
'available' => true,
'inspection_token' => $confirmationToken,
],
];
$client->responses[] = [
'operation' => [
'operation_id' => $operationId,
'status' => 'failed',
'error' => ['code' => 'snapshot_not_created'],
'result' => [],
],
'inspection' => [
'operation_id' => $operationId,
'operation_status' => 'failed',
'checked_at' => 1_800_000_005,
'outcome' => 'failed',
'summary_code' => 'operation_failed',
'error_code' => 'snapshot_not_created',
'abort_job' => ['available' => false],
'cleanup' => ['available' => false],
'confirmation' => [
'available' => true,
'inspection_token' => $confirmationToken,
],
],
];
$failed = Recovery::inspect($admin, '41');
$confirmRequest = $client->requests[array_key_last($client->requests)];
$assert(($failed['error_code'] ?? null) === 'snapshot_not_created'
&& ($failed['error'] ?? null) === '宿主机证据确认快照没有创建'
&& !str_contains(json_encode($failed), $confirmationToken),
'Known inspection error code was not mapped to a safe Chinese action message.');
$assert($confirmRequest['method'] === 'POST'
&& str_ends_with($confirmRequest['target'], '/confirm-inspection')
&& $confirmRequest['payload'] === [
'inspection_token' => $confirmationToken,
]
&& Capsule::table('cloud_jobs')->where('id', 41)->value('status')
=== 'failed',
'Confirmed failed inspection did not close the panel blocker.');
$successOperationId = 'dddddddddddddddddddddddddddddddd';
$successToken = str_repeat('f', 64);
$successInspection = [
'operation_id' => $successOperationId,
'operation_status' => 'needs_attention',
'checked_at' => 1_800_000_006,
'outcome' => 'succeeded',
'summary_code' => 'operation_succeeded',
'abort_job' => ['available' => false],
'cleanup' => ['available' => false],
'confirmation' => [
'available' => true,
'inspection_token' => $successToken,
],
];
$client->responses[] = $successInspection;
$client->responses[] = [
'operation' => [
'operation_id' => $successOperationId,
'status' => 'succeeded',
'result' => ['state' => 'running'],
],
'inspection' => array_replace($successInspection, [
'operation_status' => 'succeeded',
'checked_at' => 1_800_000_007,
]),
];
$succeeded = Recovery::inspect($admin, '42');
$assert($succeeded['summary_code'] === 'operation_succeeded'
&& Capsule::table('cloud_jobs')->where('id', 42)->value('status')
=== 'succeeded',
'Confirmed successful inspection did not close needs_attention.');
Capsule::table('cloud_jobs')->where('id', 42)->update([
'status' => 'needs_attention',
'error_code' => 'reconciliation_limit_reached',
'sealed_payload' => 'forbidden-payload',
'lease_token' => null,
'lease_until' => 0,
]);
$client->responses[] = $successInspection;
$client->responses[] = new CloudAgentError(
'raw changed host evidence', 409, 'inspection_inconclusive', false, 409
);
$inconclusive = $capture(static fn () => Recovery::inspect($admin, '42'));
$assert($inconclusive->agentCode === 'inspection_inconclusive'
&& $inconclusive->getMessage()
=== '宿主机状态在两次只读核对之间发生变化,请重新立即核对'
&& Capsule::table('cloud_jobs')->where('id', 42)->value('status')
=== 'needs_attention',
'Inconclusive fresh evidence changed the panel blocker.');
$terminalResponse = static function (int $checkedAt) use (
$successInspection, $successOperationId, $successToken
): array {
return [
'operation' => [
'operation_id' => $successOperationId,
'status' => 'succeeded',
'result' => ['state' => 'running'],
],
'inspection' => array_replace($successInspection, [
'operation_status' => 'succeeded',
'checked_at' => $checkedAt,
'confirmation' => [
'available' => true,
'inspection_token' => $successToken,
],
]),
];
};
$client->responses[] = array_replace($successInspection, [
'checked_at' => 1_800_000_008,
]);
$client->responses[] = $terminalResponse(1_800_000_009);
Recovery::abortJob($admin, '42', ['inspection_token' => $token]);
$lastTwo = array_slice($client->requests, -2);
$assert(str_ends_with($lastTwo[0]['target'], '/abort-job')
&& str_ends_with($lastTwo[1]['target'], '/confirm-inspection')
&& Capsule::table('cloud_jobs')->where('id', 42)->value('status')
=== 'succeeded',
'Abort terminal response did not converge the panel ledger.');
Capsule::table('cloud_jobs')->where('id', 42)->update([
'status' => 'needs_attention',
'error_code' => 'reconciliation_limit_reached',
'sealed_payload' => 'forbidden-payload',
'lease_token' => null,
'lease_until' => 0,
]);
$client->responses[] = array_replace($successInspection, [
'checked_at' => 1_800_000_010,
]);
$client->responses[] = $terminalResponse(1_800_000_011);
Recovery::cleanup($admin, '42', ['plan_id' => $planId]);
$lastTwo = array_slice($client->requests, -2);
$assert(str_ends_with($lastTwo[0]['target'], '/cleanup')
&& str_ends_with($lastTwo[1]['target'], '/confirm-inspection')
&& Capsule::table('cloud_jobs')->where('id', 42)->value('status')
=== 'succeeded',
'Cleanup terminal response did not converge the panel ledger.');
echo "Administrator inspect, token-confirmed settlement, recovery-error, abort and cleanup contracts passed.\n";
}