You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudRescueTemplateImportSt...

451 lines
20 KiB
PHP

<?php
declare(strict_types=1);
namespace App\Services\Cloud {
/** Deterministic in-process Agent for rescue import reconciliation tests. */
final class RescueImportFakeAgent
{
public array $requests = [];
public array $operations = [];
public string $mode = 'running';
public ?array $reconcileResponse = null;
public $onRequest = null;
public function request(string $method, string $target, array $actor,
?array $payload = null): array
{
$method = strtoupper($method);
$this->requests[] = compact('method', 'target', 'actor', 'payload');
if (is_callable($this->onRequest)) {
($this->onRequest)($method, $target, $actor, $payload);
}
if ($method === 'POST' && $target === '/v1/rescue-templates') {
$operationId = (string) ($payload['operation_id'] ?? '');
$templateId = (string) ($payload['id'] ?? '');
if ($this->mode === 'pre_dispatch_busy') {
throw new CloudAgentError('not dispatched', 503,
'agent_busy', true, 503, false, true);
}
$this->operations[$operationId] = $templateId;
if ($this->mode === 'ambiguous_replay') {
throw new CloudAgentError('nonce may have dispatched', 502,
'replayed_request', false, 401);
}
if ($this->mode === 'transport') {
throw new CloudAgentError('transport uncertain', 502,
'agent_transport_error', true);
}
} elseif ($method === 'GET'
&& preg_match('#^/v1/operations/([A-Za-z0-9_-]+)$#D',
$target, $match)) {
$operationId = rawurldecode($match[1]);
$templateId = $this->operations[$operationId] ?? '';
if ($this->mode === 'not_found' || $templateId === '') {
throw new CloudAgentError('operation missing', 404,
'operation_not_found', false, 404);
}
if ($this->mode === 'transport') {
throw new CloudAgentError('transport uncertain', 502,
'agent_transport_error', true);
}
} elseif ($method === 'POST'
&& preg_match(
'#^/v1/operations/([A-Za-z0-9_-]+)/reconcile$#D',
$target, $match)) {
$operationId = rawurldecode($match[1]);
$templateId = $this->operations[$operationId] ?? '';
if ($templateId === '') {
throw new CloudAgentError('operation missing', 404,
'operation_not_found', false, 404);
}
return ['operation_id' => $operationId]
+ ($this->reconcileResponse ?? [
'status' => 'uncertain',
'error' => ['code' => 'operation_uncertain'],
]);
} else {
throw new \RuntimeException('Unexpected fake Agent request: '
. $method . ' ' . $target);
}
if ($this->mode === 'wrong_operation') {
return ['operation_id' => str_repeat('f', 32),
'status' => 'running'];
}
if ($this->mode === 'failed') {
return ['operation_id' => $operationId, 'status' => 'failed',
'error' => ['code' => 'rescue_template_exists']];
}
if ($this->mode === 'success') {
return [
'operation_id' => $operationId,
'status' => 'succeeded',
'result' => [
'id' => $templateId,
'name' => 'Rescue',
'size_bytes' => 1048576,
'sha256' => str_repeat('a', 64),
],
];
}
if ($this->mode === 'uncertain') {
return [
'operation_id' => $operationId,
'status' => 'uncertain',
'error' => ['code' => 'operation_uncertain'],
];
}
return ['operation_id' => $operationId, 'status' => 'running'];
}
}
/** Replaces only Agent lookup in this isolated process. */
final class CloudAgentSettingsService
{
/** @var array<int, RescueImportFakeAgent> */
public static array $clients = [];
public static function clientForHost(int $hostId,
float $timeoutSeconds = 20.0): RescueImportFakeAgent
{
if (!isset(self::$clients[$hostId])) {
throw new \RuntimeException('Missing fake Agent for host '
. $hostId);
}
return self::$clients[$hostId];
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudRescueTemplateImport;
use App\Services\Cloud\CloudAgentSettingsService;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudRescueTemplateImportService;
use App\Services\Cloud\RescueImportFakeAgent;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new \RuntimeException($message);
}
};
$requestCount = static fn (RescueImportFakeAgent $agent,
string $method): int => count(array_filter($agent->requests,
static fn (array $request): bool => $request['method'] === $method));
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_rescue_template_imports',
static function (Blueprint $table): void {
$table->string('operation_id')->primary();
$table->integer('host_id');
$table->string('template_id');
$table->string('import_type');
$table->string('state');
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->unique(['host_id', 'template_id']);
});
$actor = ['user_id' => 1, 'is_admin' => true, 'can_manage' => true];
// The row must exist with its immutable identity before the first POST.
$remoteAgent = new RescueImportFakeAgent();
$remoteAgent->onRequest = static function (string $method, string $target,
array $requestActor, ?array $payload) use ($assert): void {
if ($method !== 'POST') {
return;
}
$row = CloudRescueTemplateImport::query()
->where('operation_id', (string) $payload['operation_id'])->first();
$assert($row !== null
&& (int) $row->host_id === 10
&& (string) $row->template_id === (string) $payload['id']
&& (string) $row->import_type === 'remote'
&& (string) $row->state === 'sending'
&& (string) ($requestActor['operation_id'] ?? '')
=== (string) $row->operation_id,
'Rescue import crossed dispatch before its durable identity existed.');
};
CloudAgentSettingsService::$clients[10] = $remoteAgent;
$remote = CloudRescueTemplateImportService::start(
10,
'remote',
'Rescue',
'https://images.example.com/rescue.iso',
'',
str_repeat('a', 64),
$actor
);
$remoteRow = CloudRescueTemplateImport::query()
->where('operation_id', (string) $remote['operation_id'])->first();
$assert($remote['status'] === 'running'
&& $remote['job_id'] === $remote['operation_id']
&& $remote['host_id'] === 10
&& $remote['template_id'] === (string) $remoteRow->template_id
&& $remote['import_type'] === 'remote'
&& $remote['status_url']
=== '/api/v1/admin/rescue-templates/import-jobs/'
. $remote['operation_id'] . '?host_id=10'
&& (string) $remoteRow->state === 'running'
&& $requestCount($remoteAgent, 'POST') === 1,
'Remote rescue import did not persist its running operation.');
// Authenticated capacity rejection is known to precede the Agent journal,
// so it is a deterministic failure rather than an ambiguous operation.
$busyAgent = new RescueImportFakeAgent();
$busyAgent->mode = 'pre_dispatch_busy';
CloudAgentSettingsService::$clients[11] = $busyAgent;
$busy = CloudRescueTemplateImportService::start(
11, 'remote', 'Busy rescue',
'https://images.example.com/busy.iso', '', '', $actor
);
$assert($busy['status'] === 'failed'
&& $busy['error_code'] === 'agent_busy'
&& $requestCount($busyAgent, 'POST') === 1
&& $busyAgent->operations === [],
'Definitive pre-dispatch rescue import was left for impossible reconciliation.');
$replayedAgent = new RescueImportFakeAgent();
$replayedAgent->mode = 'ambiguous_replay';
CloudAgentSettingsService::$clients[14] = $replayedAgent;
$replayed = CloudRescueTemplateImportService::start(
14, 'remote', 'Replay ambiguous rescue',
'https://images.example.com/replay-ambiguous.iso', '', '', $actor
);
$replayedRow = CloudRescueTemplateImport::query()
->where('operation_id', (string) $replayed['operation_id'])->first();
$assert($replayed['status'] === 'running'
&& (string) $replayedRow->state === 'running'
&& (string) $replayedRow->error_code === 'replayed_request'
&& $requestCount($replayedAgent, 'POST') === 1,
'Ambiguous non-retryable rescue import was made terminal.');
$replayedAgent->mode = 'success';
$replayedCompleted = CloudRescueTemplateImportService::job(
14, (string) $replayed['operation_id'], $actor
);
$assert($replayedCompleted['status'] === 'succeeded'
&& $requestCount($replayedAgent, 'POST') === 1,
'Ambiguous rescue import was replayed instead of polled by operation ID.');
// Reconciliation uses the exact saved operation and terminal rows are
// read-only; no later call is allowed to issue another import POST.
$remoteAgent->mode = 'success';
$completed = CloudRescueTemplateImportService::job(
10, (string) $remote['operation_id'], $actor
);
$requestsAfterSuccess = count($remoteAgent->requests);
$completedAgain = CloudRescueTemplateImportService::job(
10, (string) $remote['operation_id'], $actor
);
$lastPoll = $remoteAgent->requests[$requestsAfterSuccess - 1];
$assert($completed['status'] === 'succeeded'
&& $completedAgain['status'] === 'succeeded'
&& $requestCount($remoteAgent, 'POST') === 1
&& count($remoteAgent->requests) === $requestsAfterSuccess
&& $lastPoll['method'] === 'GET'
&& $lastPoll['target'] === '/v1/operations/'
. $remote['operation_id'],
'Rescue import reconciliation changed or replayed its operation.');
// GET uncertainty is resolved only through POST /reconcile for the same
// operation. The original rescue import POST remains single-shot.
$reconcileAgent = new RescueImportFakeAgent();
CloudAgentSettingsService::$clients[12] = $reconcileAgent;
$reconcileImport = CloudRescueTemplateImportService::start(
12, 'remote', 'Reconcile rescue',
'https://images.example.com/reconcile.iso', '', '', $actor
);
$reconcileAgent->mode = 'uncertain';
$reconcileAgent->reconcileResponse = [
'status' => 'succeeded',
'result' => [
'id' => (string) $reconcileImport['template_id'],
'name' => 'Reconcile rescue',
'size_bytes' => 1048576,
'sha256' => str_repeat('a', 64),
],
];
$beforeReconcilePoll = count($reconcileAgent->requests);
$reconcileCompleted = CloudRescueTemplateImportService::job(
12, (string) $reconcileImport['operation_id'], $actor
);
$reconcileRequests = array_slice(
$reconcileAgent->requests, $beforeReconcilePoll
);
$importPosts = array_filter($reconcileAgent->requests,
static fn (array $request): bool => $request['method'] === 'POST'
&& $request['target'] === '/v1/rescue-templates');
$assert($reconcileCompleted['status'] === 'succeeded'
&& count($importPosts) === 1
&& count($reconcileRequests) === 2
&& $reconcileRequests[0]['method'] === 'GET'
&& $reconcileRequests[0]['target'] === '/v1/operations/'
. $reconcileImport['operation_id']
&& $reconcileRequests[1]['method'] === 'POST'
&& $reconcileRequests[1]['target'] === '/v1/operations/'
. $reconcileImport['operation_id'] . '/reconcile',
'Rescue import uncertainty was replayed or reconciled under another operation.');
$trueUncertainAgent = new RescueImportFakeAgent();
CloudAgentSettingsService::$clients[13] = $trueUncertainAgent;
$trueUncertainImport = CloudRescueTemplateImportService::start(
13, 'remote', 'Uncertain rescue',
'https://images.example.com/uncertain.iso', '', '', $actor
);
$trueUncertainAgent->mode = 'uncertain';
$trueUncertainAgent->reconcileResponse = [
'status' => 'uncertain',
'error' => ['code' => 'operation_uncertain'],
];
$trueUncertain = CloudRescueTemplateImportService::job(
13, (string) $trueUncertainImport['operation_id'], $actor
);
$requestsAtUncertain = count($trueUncertainAgent->requests);
$trueUncertainAgain = CloudRescueTemplateImportService::job(
13, (string) $trueUncertainImport['operation_id'], $actor
);
$trueUncertainPosts = array_filter($trueUncertainAgent->requests,
static fn (array $request): bool => $request['method'] === 'POST'
&& $request['target'] === '/v1/rescue-templates');
$assert($trueUncertain['status'] === 'uncertain'
&& $trueUncertainAgain['status'] === 'uncertain'
&& count($trueUncertainPosts) === 1
&& count($trueUncertainAgent->requests) === $requestsAtUncertain,
'A genuinely uncertain rescue import was retried or left pollable.');
// A retryable transport failure may have crossed the boundary. Preserve
// the upload operation, then lock it uncertain if Agent has no record.
$uploadAgent = new RescueImportFakeAgent();
$uploadAgent->mode = 'transport';
CloudAgentSettingsService::$clients[20] = $uploadAgent;
$upload = CloudRescueTemplateImportService::start(
20,
'upload',
'Uploaded rescue',
'https://panel.example.com/api/internal/cloud/upload-staging/'
. str_repeat('b', 32),
str_repeat('c', 64),
str_repeat('d', 64),
$actor
);
$uploadRow = CloudRescueTemplateImport::query()
->where('operation_id', (string) $upload['operation_id'])->first();
$assert($upload['status'] === 'running'
&& $upload['import_type'] === 'upload'
&& (string) $uploadRow->state === 'running'
&& (string) $uploadRow->error_code === 'agent_transport_error'
&& $requestCount($uploadAgent, 'POST') === 1,
'Ambiguous upload dispatch was not retained for reconciliation.');
$uploadAgent->mode = 'not_found';
$uncertain = CloudRescueTemplateImportService::job(
20, (string) $upload['operation_id'], $actor
);
$requestsAfterUncertain = count($uploadAgent->requests);
$uncertainAgain = CloudRescueTemplateImportService::job(
20, (string) $upload['operation_id'], $actor
);
$assert($uncertain['status'] === 'uncertain'
&& $uncertainAgain['status'] === 'uncertain'
&& $uncertain['error_code'] === 'operation_not_found'
&& $requestCount($uploadAgent, 'POST') === 1
&& count($uploadAgent->requests) === $requestsAfterUncertain,
'Missing ambiguous operation was replayed instead of locked uncertain.');
// A mismatched Agent operation is terminally uncertain and host scoping
// prevents another host from learning or polling the job.
$mismatchAgent = new RescueImportFakeAgent();
$mismatchAgent->mode = 'wrong_operation';
CloudAgentSettingsService::$clients[30] = $mismatchAgent;
$mismatch = CloudRescueTemplateImportService::start(
30, 'remote', 'Mismatch', 'https://images.example.com/other.iso',
'', '', $actor
);
$assert($mismatch['status'] === 'uncertain'
&& $mismatch['error_code'] === 'operation_id_mismatch'
&& $requestCount($mismatchAgent, 'POST') === 1,
'Mismatched Agent operation identity was accepted.');
try {
CloudRescueTemplateImportService::job(
31, (string) $mismatch['operation_id'], $actor
);
throw new \RuntimeException('Cross-host rescue import lookup succeeded.');
} catch (CloudError $error) {
$assert($error->httpStatus === 404,
'Cross-host rescue import lookup returned the wrong status.');
}
// A process crash between durable reservation and completion cannot cause
// a later status request to issue the original POST.
$staleOperation = str_repeat('e', 32);
Capsule::table('cloud_rescue_template_imports')->insert([
'operation_id' => $staleOperation,
'host_id' => 40,
'template_id' => str_repeat('9', 32),
'import_type' => 'remote',
'state' => 'sending',
'error_code' => '',
'created_at' => time() - 180,
'updated_at' => time() - 180,
]);
$staleAgent = new RescueImportFakeAgent();
CloudAgentSettingsService::$clients[40] = $staleAgent;
$stale = CloudRescueTemplateImportService::job(
40, $staleOperation, $actor
);
$assert($stale['status'] === 'uncertain'
&& $stale['error_code'] === 'operation_not_found'
&& count($staleAgent->requests) === 1
&& $staleAgent->requests[0]['method'] === 'GET'
&& $staleAgent->requests[0]['target']
=== '/v1/operations/' . $staleOperation
&& $requestCount($staleAgent, 'POST') === 0,
'Stale sending boundary was not checked by operation ID without replay.');
// A response produced from a stale concurrent poll must not overwrite a
// terminal state committed while that request was in flight.
$raceAgent = new RescueImportFakeAgent();
CloudAgentSettingsService::$clients[50] = $raceAgent;
$race = CloudRescueTemplateImportService::start(
50, 'remote', 'Concurrent poll',
'https://images.example.com/concurrent.iso', '', '', $actor
);
$raceAgent->onRequest = static function (string $method, string $target)
use ($race): void {
if ($method === 'GET') {
CloudRescueTemplateImport::query()
->where('operation_id', (string) $race['operation_id'])
->update([
'state' => 'succeeded',
'error_code' => '',
'updated_at' => time(),
]);
}
};
$raced = CloudRescueTemplateImportService::job(
50, (string) $race['operation_id'], $actor
);
$assert($raced['status'] === 'succeeded'
&& (string) CloudRescueTemplateImport::find(
(string) $race['operation_id']
)->state === 'succeeded',
'A stale concurrent poll moved a terminal import state backwards.');
$columns = $schema->getColumnListing('cloud_rescue_template_imports');
$assert(!in_array('source_url', $columns, true)
&& !in_array('source_token', $columns, true),
'Durable rescue import table persisted source credentials.');
echo "Rescue import persistence, host binding and no-replay reconciliation passed.\n";
}