You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudAdminOperationRecovery...

330 lines
14 KiB
PHTML

<?php
namespace App\Services\Cloud {
final class RecoveryTestAgentClient
{
public array $requests = [];
public array $responses = [];
public function request(
string $method,
string $target,
array $actor,
?array $payload = null
): array {
$request = compact('method', 'target', 'actor', 'payload');
$this->requests[] = $request;
if ($this->responses === []) {
throw new \RuntimeException('Unexpected Agent request.');
}
$response = array_shift($this->responses);
if (is_callable($response)) {
$response = $response($request);
}
if ($response instanceof \Throwable) {
throw $response;
}
if (!is_array($response)) {
throw new \RuntimeException('Invalid recovery test response.');
}
return $response;
}
}
final class CloudAgentSettingsService
{
public static ?RecoveryTestAgentClient $client = null;
public static array $calls = [];
public static function clientForConfiguration(
int $hostId,
float $timeoutSeconds = 120.0
): RecoveryTestAgentClient {
self::$calls[] = [$hostId, $timeoutSeconds];
return self::$client
?? throw new \RuntimeException('Missing recovery test client.');
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Services\Cloud\CloudAdminOperationRecoveryService as Recovery;
use App\Services\Cloud\CloudAgentError;
use App\Services\Cloud\CloudAgentSettingsService;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\RecoveryTestAgentClient;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$reject = static function (callable $callback, int $status,
?string $code = null) use ($assert): void {
try {
$callback();
} catch (CloudError $error) {
$assert($error->httpStatus === $status,
'Unexpected HTTP status: ' . $error->httpStatus);
if ($code !== null) {
$assert($error instanceof CloudAgentError
&& $error->agentCode === $code,
'Unexpected Agent error code.');
}
return;
}
throw new RuntimeException('Invalid recovery request was accepted.');
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$capsule->schema('default')->create('cloud_jobs',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id');
$table->integer('host_id');
$table->string('kind');
$table->string('execution_mode')->default('agent');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('status');
$table->integer('user_id')->default(99);
$table->integer('instance_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('attempts')->default(1);
$table->integer('dispatch_attempts')->default(1);
$table->integer('dispatch_deadline_at')->default(1);
$table->integer('dispatched_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution')->default('not_applicable');
$table->integer('available_at')->default(0);
$table->integer('lease_until')->default(0);
$table->string('error_code')->nullable();
$table->boolean('finalization_incident')->default(0);
$table->string('finalization_error_code')->nullable();
$table->text('result_json')->nullable();
$table->text('sealed_payload')->default('forbidden-payload');
$table->string('lease_token')->nullable();
$table->integer('created_at')->nullable();
$table->integer('updated_at')->nullable();
});
$now = time();
foreach ([
41 => 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
42 => 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb',
43 => 'cccccccccccccccccccccccccccccccc',
44 => 'dddddddddddddddddddddddddddddddd',
] as $id => $operationId) {
Capsule::table('cloud_jobs')->insert([
'id' => $id, 'operation_id' => $operationId, 'host_id' => 7,
'kind' => 'create', 'order_id' => 100,
'phase' => 'terminal', 'outcome' => 'failed',
'status' => 'failed', 'dispatched_at' => $now - 90,
'terminal_at' => $now - 30,
'effect_may_have_applied' => true,
'safe_to_resubmit' => false,
'effect_resolution' => 'unknown',
'created_at' => $now - 120, 'updated_at' => $now - 30,
]);
}
$admin = ['user_id' => 99, 'is_admin' => true,
'is_super_admin' => true, 'can_manage' => true];
$readonly = ['user_id' => 98, 'is_admin' => true,
'is_super_admin' => false, 'can_manage' => false];
$client = new RecoveryTestAgentClient();
CloudAgentSettingsService::$client = $client;
$operationA = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';
$client->responses[] = [
'operation_id' => $operationA,
'operation_status' => 'failed',
'checked_at' => $now,
'outcome' => 'failed',
'summary_code' => 'operation_failed',
'error_code' => 'creation_absent',
'raw_error' => '/srv/private/secret.qcow2',
'abort_job' => ['available' => true, 'token' => str_repeat('f', 64)],
];
$inspection = Recovery::inspect($admin, '41');
$assert($inspection === [
'operation_id' => $operationA,
'journal_status' => 'failed',
'checked_at' => $now,
'outcome' => 'failed',
'summary_code' => 'operation_failed',
'summary' => '只读证据与宿主机失败结果一致',
'error_code' => 'creation_absent',
'error' => '宿主机未发现虚拟机或面板管理资源',
] && !str_contains(json_encode($inspection), '/srv/private'),
'Read-only inspection was not reduced to the bounded DTO.');
$assert(CloudAgentSettingsService::$calls === [[7, 30.0]]
&& $client->requests[0]['method'] === 'GET'
&& $client->requests[0]['target']
=== '/v1/operations/' . $operationA . '/inspect'
&& $client->requests[0]['payload'] === null,
'Inspection did not use exactly one signed read-only request.');
$assert(Capsule::table('cloud_jobs')->where('id', 41)
->value('effect_resolution') === 'unknown',
'Read-only inspection changed the task or its safety fence.');
$client->responses[] = ['operation_id' => $operationA,
'operation_status' => 'uncertain', 'checked_at' => $now,
'outcome' => 'uncertain', 'summary_code' => 'evidence_incomplete'];
$reject(static fn () => Recovery::inspect($admin, '41'), 502,
'agent_invalid_inspection');
$reason = '已人工只读核对虚拟机、磁盘与网络状态,确认原操作效果已经发生。';
$reasonDigest = hash('sha256', $reason);
$oldReceipt = [
'operation_id' => $operationA,
'resolution_id' => str_repeat('1', 32),
'resolution' => 'not_applied',
'inspection_checked_at' => $now - 60,
'reason_sha256' => $reasonDigest,
'resolved_at' => $now - 50,
'resolved_by' => 7,
'replayed' => false,
];
$before = count($client->requests);
$client->responses[] = $oldReceipt;
$recovered = Recovery::resolveFence(
$admin, '41', 'not_applied', $reason, $inspection
);
$requests = array_slice($client->requests, $before);
$assert($recovered['resolution_id'] === str_repeat('1', 32)
&& count($requests) === 1 && $requests[0]['method'] === 'GET'
&& str_ends_with($requests[0]['target'], '/fence-resolution')
&& ($requests[0]['actor']['operation_id'] ?? null) === $operationA,
'A prior Agent commit was not recovered read-only after a PHP crash.');
$notFound = static fn () => new CloudAgentError(
'missing', 404, 'fence_resolution_not_found', false, 404
);
$operationB = 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb';
$inspectionB = [
'operation_id' => $operationB, 'checked_at' => $now,
'outcome' => 'failed', 'summary_code' => 'operation_failed',
'error_code' => 'creation_absent',
];
$client->responses[] = $notFound();
$client->responses[] = static function (array $request) use (
$operationB, $now
): array {
return [
'operation_id' => $operationB,
'resolution_id' => $request['payload']['resolution_id'],
'resolution' => $request['payload']['resolution'],
'inspection_checked_at' =>
$request['payload']['inspection_checked_at'],
'reason_sha256' => $request['payload']['reason_sha256'],
'resolved_at' => $now,
'resolved_by' => '7',
'replayed' => false,
];
};
$before = count($client->requests);
$created = Recovery::resolveFence(
$admin, '42', 'not_applied', $reason, $inspectionB
);
$requests = array_slice($client->requests, $before);
$assert(count($requests) === 2
&& array_column($requests, 'method') === ['GET', 'POST']
&& str_ends_with($requests[1]['target'], '/resolve-fence')
&& preg_match('/^[a-f0-9]{32}$/D',
$requests[1]['payload']['resolution_id']) === 1
&& $created['resolution'] === 'not_applied',
'New fence resolution was not dispatched exactly once.');
$operationC = 'cccccccccccccccccccccccccccccccc';
$inspectionC = [
'operation_id' => $operationC, 'checked_at' => $now,
'outcome' => 'failed', 'summary_code' => 'operation_failed',
'error_code' => 'creation_absent',
];
$client->responses[] = $notFound();
$client->responses[] = new CloudAgentError(
'lost response', 503, 'agent_disconnected', true, 503
);
$client->responses[] = static function (array $request) use (
$client, $operationC, $now
): array {
$post = $client->requests[count($client->requests) - 2];
return [
'operation_id' => $operationC,
'resolution_id' => $post['payload']['resolution_id'],
'resolution' => $post['payload']['resolution'],
'inspection_checked_at' =>
$post['payload']['inspection_checked_at'],
'reason_sha256' => $post['payload']['reason_sha256'],
'resolved_at' => $now, 'resolved_by' => 7,
'replayed' => false,
];
};
$before = count($client->requests);
$verified = Recovery::resolveFence(
$admin, '43', 'not_applied', $reason, $inspectionC
);
$requests = array_slice($client->requests, $before);
$assert(array_column($requests, 'method') === ['GET', 'POST', 'GET']
&& count(array_filter($requests,
static fn (array $request): bool => $request['method'] === 'POST'))
=== 1
&& $verified['resolution'] === 'not_applied',
'Lost fence response caused a POST retry instead of one GET verification.');
$operationD = 'dddddddddddddddddddddddddddddddd';
$inspectionD = [
'operation_id' => $operationD, 'checked_at' => $now,
'outcome' => 'failed', 'summary_code' => 'operation_failed',
'error_code' => 'creation_absent',
];
$client->responses[] = array_replace($oldReceipt, [
'operation_id' => $operationD,
'resolution' => 'applied',
]);
$before = count($client->requests);
$reject(static fn () => Recovery::resolveFence(
$admin, '44', 'not_applied', $reason, $inspectionD
), 409);
$assert(count($client->requests) === $before + 1,
'Conflicting durable fence receipt triggered an Agent write.');
$before = count($client->requests);
$reject(static fn () => Recovery::resolveFence(
$readonly, '41', 'not_applied', $reason, $inspection
), 403);
$reject(static fn () => Recovery::resolveFence(
$admin, '41', 'not_applied', $reason,
[
'operation_id' => $operationA, 'checked_at' => $now - 121,
'outcome' => 'failed', 'summary_code' => 'operation_failed',
'error_code' => 'creation_absent',
]
), 409);
$assert(count($client->requests) === $before,
'Unauthorized or stale resolution reached the Agent.');
$source = file_get_contents(dirname(__DIR__)
. '/app/Services/Cloud/CloudAdminOperationRecoveryService.php');
$assert(is_string($source)
&& !str_contains($source, 'function abortJob')
&& !str_contains($source, 'function cleanup')
&& !str_contains($source, 'confirm-inspection')
&& substr_count($source, "request('POST'") === 1,
'Retired recovery mutations or a second Agent POST remain reachable.');
echo "Administrator read-only inspection and single-write fence recovery tests passed.\n";
}