You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
451 lines
20 KiB
PHP
451 lines
20 KiB
PHP
<?php
|
|
declare(strict_types=1);
|
|
|
|
namespace App\Services\Cloud {
|
|
/** Deterministic in-process Agent for rescue import reconciliation tests. */
|
|
final class RescueImportFakeAgent
|
|
{
|
|
public array $requests = [];
|
|
public array $operations = [];
|
|
public string $mode = 'running';
|
|
public ?array $reconcileResponse = null;
|
|
public $onRequest = null;
|
|
|
|
public function request(string $method, string $target, array $actor,
|
|
?array $payload = null): array
|
|
{
|
|
$method = strtoupper($method);
|
|
$this->requests[] = compact('method', 'target', 'actor', 'payload');
|
|
if (is_callable($this->onRequest)) {
|
|
($this->onRequest)($method, $target, $actor, $payload);
|
|
}
|
|
|
|
if ($method === 'POST' && $target === '/v1/rescue-templates') {
|
|
$operationId = (string) ($payload['operation_id'] ?? '');
|
|
$templateId = (string) ($payload['id'] ?? '');
|
|
if ($this->mode === 'pre_dispatch_busy') {
|
|
throw new CloudAgentError('not dispatched', 503,
|
|
'agent_busy', true, 503, false, true);
|
|
}
|
|
$this->operations[$operationId] = $templateId;
|
|
if ($this->mode === 'ambiguous_replay') {
|
|
throw new CloudAgentError('nonce may have dispatched', 502,
|
|
'replayed_request', false, 401);
|
|
}
|
|
if ($this->mode === 'transport') {
|
|
throw new CloudAgentError('transport uncertain', 502,
|
|
'agent_transport_error', true);
|
|
}
|
|
} elseif ($method === 'GET'
|
|
&& preg_match('#^/v1/operations/([A-Za-z0-9_-]+)$#D',
|
|
$target, $match)) {
|
|
$operationId = rawurldecode($match[1]);
|
|
$templateId = $this->operations[$operationId] ?? '';
|
|
if ($this->mode === 'not_found' || $templateId === '') {
|
|
throw new CloudAgentError('operation missing', 404,
|
|
'operation_not_found', false, 404);
|
|
}
|
|
if ($this->mode === 'transport') {
|
|
throw new CloudAgentError('transport uncertain', 502,
|
|
'agent_transport_error', true);
|
|
}
|
|
} elseif ($method === 'POST'
|
|
&& preg_match(
|
|
'#^/v1/operations/([A-Za-z0-9_-]+)/reconcile$#D',
|
|
$target, $match)) {
|
|
$operationId = rawurldecode($match[1]);
|
|
$templateId = $this->operations[$operationId] ?? '';
|
|
if ($templateId === '') {
|
|
throw new CloudAgentError('operation missing', 404,
|
|
'operation_not_found', false, 404);
|
|
}
|
|
return ['operation_id' => $operationId]
|
|
+ ($this->reconcileResponse ?? [
|
|
'status' => 'uncertain',
|
|
'error' => ['code' => 'operation_uncertain'],
|
|
]);
|
|
} else {
|
|
throw new \RuntimeException('Unexpected fake Agent request: '
|
|
. $method . ' ' . $target);
|
|
}
|
|
|
|
if ($this->mode === 'wrong_operation') {
|
|
return ['operation_id' => str_repeat('f', 32),
|
|
'status' => 'running'];
|
|
}
|
|
if ($this->mode === 'failed') {
|
|
return ['operation_id' => $operationId, 'status' => 'failed',
|
|
'error' => ['code' => 'rescue_template_exists']];
|
|
}
|
|
if ($this->mode === 'success') {
|
|
return [
|
|
'operation_id' => $operationId,
|
|
'status' => 'succeeded',
|
|
'result' => [
|
|
'id' => $templateId,
|
|
'name' => 'Rescue',
|
|
'size_bytes' => 1048576,
|
|
'sha256' => str_repeat('a', 64),
|
|
],
|
|
];
|
|
}
|
|
if ($this->mode === 'uncertain') {
|
|
return [
|
|
'operation_id' => $operationId,
|
|
'status' => 'uncertain',
|
|
'error' => ['code' => 'operation_uncertain'],
|
|
];
|
|
}
|
|
return ['operation_id' => $operationId, 'status' => 'running'];
|
|
}
|
|
}
|
|
|
|
/** Replaces only Agent lookup in this isolated process. */
|
|
final class CloudAgentSettingsService
|
|
{
|
|
/** @var array<int, RescueImportFakeAgent> */
|
|
public static array $clients = [];
|
|
|
|
public static function clientForHost(int $hostId,
|
|
float $timeoutSeconds = 20.0): RescueImportFakeAgent
|
|
{
|
|
if (!isset(self::$clients[$hostId])) {
|
|
throw new \RuntimeException('Missing fake Agent for host '
|
|
. $hostId);
|
|
}
|
|
return self::$clients[$hostId];
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudRescueTemplateImport;
|
|
use App\Services\Cloud\CloudAgentSettingsService;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\CloudRescueTemplateImportService;
|
|
use App\Services\Cloud\RescueImportFakeAgent;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new \RuntimeException($message);
|
|
}
|
|
};
|
|
$requestCount = static fn (RescueImportFakeAgent $agent,
|
|
string $method): int => count(array_filter($agent->requests,
|
|
static fn (array $request): bool => $request['method'] === $method));
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
|
|
'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
$schema->create('cloud_rescue_template_imports',
|
|
static function (Blueprint $table): void {
|
|
$table->string('operation_id')->primary();
|
|
$table->integer('host_id');
|
|
$table->string('template_id');
|
|
$table->string('import_type');
|
|
$table->string('state');
|
|
$table->string('error_code')->default('');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->unique(['host_id', 'template_id']);
|
|
});
|
|
|
|
$actor = ['user_id' => 1, 'is_admin' => true, 'can_manage' => true];
|
|
|
|
// The row must exist with its immutable identity before the first POST.
|
|
$remoteAgent = new RescueImportFakeAgent();
|
|
$remoteAgent->onRequest = static function (string $method, string $target,
|
|
array $requestActor, ?array $payload) use ($assert): void {
|
|
if ($method !== 'POST') {
|
|
return;
|
|
}
|
|
$row = CloudRescueTemplateImport::query()
|
|
->where('operation_id', (string) $payload['operation_id'])->first();
|
|
$assert($row !== null
|
|
&& (int) $row->host_id === 10
|
|
&& (string) $row->template_id === (string) $payload['id']
|
|
&& (string) $row->import_type === 'remote'
|
|
&& (string) $row->state === 'sending'
|
|
&& (string) ($requestActor['operation_id'] ?? '')
|
|
=== (string) $row->operation_id,
|
|
'Rescue import crossed dispatch before its durable identity existed.');
|
|
};
|
|
CloudAgentSettingsService::$clients[10] = $remoteAgent;
|
|
$remote = CloudRescueTemplateImportService::start(
|
|
10,
|
|
'remote',
|
|
'Rescue',
|
|
'https://images.example.com/rescue.iso',
|
|
'',
|
|
str_repeat('a', 64),
|
|
$actor
|
|
);
|
|
$remoteRow = CloudRescueTemplateImport::query()
|
|
->where('operation_id', (string) $remote['operation_id'])->first();
|
|
$assert($remote['status'] === 'running'
|
|
&& $remote['job_id'] === $remote['operation_id']
|
|
&& $remote['host_id'] === 10
|
|
&& $remote['template_id'] === (string) $remoteRow->template_id
|
|
&& $remote['import_type'] === 'remote'
|
|
&& $remote['status_url']
|
|
=== '/api/v1/admin/rescue-templates/import-jobs/'
|
|
. $remote['operation_id'] . '?host_id=10'
|
|
&& (string) $remoteRow->state === 'running'
|
|
&& $requestCount($remoteAgent, 'POST') === 1,
|
|
'Remote rescue import did not persist its running operation.');
|
|
|
|
// Authenticated capacity rejection is known to precede the Agent journal,
|
|
// so it is a deterministic failure rather than an ambiguous operation.
|
|
$busyAgent = new RescueImportFakeAgent();
|
|
$busyAgent->mode = 'pre_dispatch_busy';
|
|
CloudAgentSettingsService::$clients[11] = $busyAgent;
|
|
$busy = CloudRescueTemplateImportService::start(
|
|
11, 'remote', 'Busy rescue',
|
|
'https://images.example.com/busy.iso', '', '', $actor
|
|
);
|
|
$assert($busy['status'] === 'failed'
|
|
&& $busy['error_code'] === 'agent_busy'
|
|
&& $requestCount($busyAgent, 'POST') === 1
|
|
&& $busyAgent->operations === [],
|
|
'Definitive pre-dispatch rescue import was left for impossible reconciliation.');
|
|
|
|
$replayedAgent = new RescueImportFakeAgent();
|
|
$replayedAgent->mode = 'ambiguous_replay';
|
|
CloudAgentSettingsService::$clients[14] = $replayedAgent;
|
|
$replayed = CloudRescueTemplateImportService::start(
|
|
14, 'remote', 'Replay ambiguous rescue',
|
|
'https://images.example.com/replay-ambiguous.iso', '', '', $actor
|
|
);
|
|
$replayedRow = CloudRescueTemplateImport::query()
|
|
->where('operation_id', (string) $replayed['operation_id'])->first();
|
|
$assert($replayed['status'] === 'running'
|
|
&& (string) $replayedRow->state === 'running'
|
|
&& (string) $replayedRow->error_code === 'replayed_request'
|
|
&& $requestCount($replayedAgent, 'POST') === 1,
|
|
'Ambiguous non-retryable rescue import was made terminal.');
|
|
$replayedAgent->mode = 'success';
|
|
$replayedCompleted = CloudRescueTemplateImportService::job(
|
|
14, (string) $replayed['operation_id'], $actor
|
|
);
|
|
$assert($replayedCompleted['status'] === 'succeeded'
|
|
&& $requestCount($replayedAgent, 'POST') === 1,
|
|
'Ambiguous rescue import was replayed instead of polled by operation ID.');
|
|
|
|
// Reconciliation uses the exact saved operation and terminal rows are
|
|
// read-only; no later call is allowed to issue another import POST.
|
|
$remoteAgent->mode = 'success';
|
|
$completed = CloudRescueTemplateImportService::job(
|
|
10, (string) $remote['operation_id'], $actor
|
|
);
|
|
$requestsAfterSuccess = count($remoteAgent->requests);
|
|
$completedAgain = CloudRescueTemplateImportService::job(
|
|
10, (string) $remote['operation_id'], $actor
|
|
);
|
|
$lastPoll = $remoteAgent->requests[$requestsAfterSuccess - 1];
|
|
$assert($completed['status'] === 'succeeded'
|
|
&& $completedAgain['status'] === 'succeeded'
|
|
&& $requestCount($remoteAgent, 'POST') === 1
|
|
&& count($remoteAgent->requests) === $requestsAfterSuccess
|
|
&& $lastPoll['method'] === 'GET'
|
|
&& $lastPoll['target'] === '/v1/operations/'
|
|
. $remote['operation_id'],
|
|
'Rescue import reconciliation changed or replayed its operation.');
|
|
|
|
// GET uncertainty is resolved only through POST /reconcile for the same
|
|
// operation. The original rescue import POST remains single-shot.
|
|
$reconcileAgent = new RescueImportFakeAgent();
|
|
CloudAgentSettingsService::$clients[12] = $reconcileAgent;
|
|
$reconcileImport = CloudRescueTemplateImportService::start(
|
|
12, 'remote', 'Reconcile rescue',
|
|
'https://images.example.com/reconcile.iso', '', '', $actor
|
|
);
|
|
$reconcileAgent->mode = 'uncertain';
|
|
$reconcileAgent->reconcileResponse = [
|
|
'status' => 'succeeded',
|
|
'result' => [
|
|
'id' => (string) $reconcileImport['template_id'],
|
|
'name' => 'Reconcile rescue',
|
|
'size_bytes' => 1048576,
|
|
'sha256' => str_repeat('a', 64),
|
|
],
|
|
];
|
|
$beforeReconcilePoll = count($reconcileAgent->requests);
|
|
$reconcileCompleted = CloudRescueTemplateImportService::job(
|
|
12, (string) $reconcileImport['operation_id'], $actor
|
|
);
|
|
$reconcileRequests = array_slice(
|
|
$reconcileAgent->requests, $beforeReconcilePoll
|
|
);
|
|
$importPosts = array_filter($reconcileAgent->requests,
|
|
static fn (array $request): bool => $request['method'] === 'POST'
|
|
&& $request['target'] === '/v1/rescue-templates');
|
|
$assert($reconcileCompleted['status'] === 'succeeded'
|
|
&& count($importPosts) === 1
|
|
&& count($reconcileRequests) === 2
|
|
&& $reconcileRequests[0]['method'] === 'GET'
|
|
&& $reconcileRequests[0]['target'] === '/v1/operations/'
|
|
. $reconcileImport['operation_id']
|
|
&& $reconcileRequests[1]['method'] === 'POST'
|
|
&& $reconcileRequests[1]['target'] === '/v1/operations/'
|
|
. $reconcileImport['operation_id'] . '/reconcile',
|
|
'Rescue import uncertainty was replayed or reconciled under another operation.');
|
|
|
|
$trueUncertainAgent = new RescueImportFakeAgent();
|
|
CloudAgentSettingsService::$clients[13] = $trueUncertainAgent;
|
|
$trueUncertainImport = CloudRescueTemplateImportService::start(
|
|
13, 'remote', 'Uncertain rescue',
|
|
'https://images.example.com/uncertain.iso', '', '', $actor
|
|
);
|
|
$trueUncertainAgent->mode = 'uncertain';
|
|
$trueUncertainAgent->reconcileResponse = [
|
|
'status' => 'uncertain',
|
|
'error' => ['code' => 'operation_uncertain'],
|
|
];
|
|
$trueUncertain = CloudRescueTemplateImportService::job(
|
|
13, (string) $trueUncertainImport['operation_id'], $actor
|
|
);
|
|
$requestsAtUncertain = count($trueUncertainAgent->requests);
|
|
$trueUncertainAgain = CloudRescueTemplateImportService::job(
|
|
13, (string) $trueUncertainImport['operation_id'], $actor
|
|
);
|
|
$trueUncertainPosts = array_filter($trueUncertainAgent->requests,
|
|
static fn (array $request): bool => $request['method'] === 'POST'
|
|
&& $request['target'] === '/v1/rescue-templates');
|
|
$assert($trueUncertain['status'] === 'uncertain'
|
|
&& $trueUncertainAgain['status'] === 'uncertain'
|
|
&& count($trueUncertainPosts) === 1
|
|
&& count($trueUncertainAgent->requests) === $requestsAtUncertain,
|
|
'A genuinely uncertain rescue import was retried or left pollable.');
|
|
|
|
// A retryable transport failure may have crossed the boundary. Preserve
|
|
// the upload operation, then lock it uncertain if Agent has no record.
|
|
$uploadAgent = new RescueImportFakeAgent();
|
|
$uploadAgent->mode = 'transport';
|
|
CloudAgentSettingsService::$clients[20] = $uploadAgent;
|
|
$upload = CloudRescueTemplateImportService::start(
|
|
20,
|
|
'upload',
|
|
'Uploaded rescue',
|
|
'https://panel.example.com/api/internal/cloud/upload-staging/'
|
|
. str_repeat('b', 32),
|
|
str_repeat('c', 64),
|
|
str_repeat('d', 64),
|
|
$actor
|
|
);
|
|
$uploadRow = CloudRescueTemplateImport::query()
|
|
->where('operation_id', (string) $upload['operation_id'])->first();
|
|
$assert($upload['status'] === 'running'
|
|
&& $upload['import_type'] === 'upload'
|
|
&& (string) $uploadRow->state === 'running'
|
|
&& (string) $uploadRow->error_code === 'agent_transport_error'
|
|
&& $requestCount($uploadAgent, 'POST') === 1,
|
|
'Ambiguous upload dispatch was not retained for reconciliation.');
|
|
$uploadAgent->mode = 'not_found';
|
|
$uncertain = CloudRescueTemplateImportService::job(
|
|
20, (string) $upload['operation_id'], $actor
|
|
);
|
|
$requestsAfterUncertain = count($uploadAgent->requests);
|
|
$uncertainAgain = CloudRescueTemplateImportService::job(
|
|
20, (string) $upload['operation_id'], $actor
|
|
);
|
|
$assert($uncertain['status'] === 'uncertain'
|
|
&& $uncertainAgain['status'] === 'uncertain'
|
|
&& $uncertain['error_code'] === 'operation_not_found'
|
|
&& $requestCount($uploadAgent, 'POST') === 1
|
|
&& count($uploadAgent->requests) === $requestsAfterUncertain,
|
|
'Missing ambiguous operation was replayed instead of locked uncertain.');
|
|
|
|
// A mismatched Agent operation is terminally uncertain and host scoping
|
|
// prevents another host from learning or polling the job.
|
|
$mismatchAgent = new RescueImportFakeAgent();
|
|
$mismatchAgent->mode = 'wrong_operation';
|
|
CloudAgentSettingsService::$clients[30] = $mismatchAgent;
|
|
$mismatch = CloudRescueTemplateImportService::start(
|
|
30, 'remote', 'Mismatch', 'https://images.example.com/other.iso',
|
|
'', '', $actor
|
|
);
|
|
$assert($mismatch['status'] === 'uncertain'
|
|
&& $mismatch['error_code'] === 'operation_id_mismatch'
|
|
&& $requestCount($mismatchAgent, 'POST') === 1,
|
|
'Mismatched Agent operation identity was accepted.');
|
|
try {
|
|
CloudRescueTemplateImportService::job(
|
|
31, (string) $mismatch['operation_id'], $actor
|
|
);
|
|
throw new \RuntimeException('Cross-host rescue import lookup succeeded.');
|
|
} catch (CloudError $error) {
|
|
$assert($error->httpStatus === 404,
|
|
'Cross-host rescue import lookup returned the wrong status.');
|
|
}
|
|
|
|
// A process crash between durable reservation and completion cannot cause
|
|
// a later status request to issue the original POST.
|
|
$staleOperation = str_repeat('e', 32);
|
|
Capsule::table('cloud_rescue_template_imports')->insert([
|
|
'operation_id' => $staleOperation,
|
|
'host_id' => 40,
|
|
'template_id' => str_repeat('9', 32),
|
|
'import_type' => 'remote',
|
|
'state' => 'sending',
|
|
'error_code' => '',
|
|
'created_at' => time() - 180,
|
|
'updated_at' => time() - 180,
|
|
]);
|
|
$staleAgent = new RescueImportFakeAgent();
|
|
CloudAgentSettingsService::$clients[40] = $staleAgent;
|
|
$stale = CloudRescueTemplateImportService::job(
|
|
40, $staleOperation, $actor
|
|
);
|
|
$assert($stale['status'] === 'uncertain'
|
|
&& $stale['error_code'] === 'operation_not_found'
|
|
&& count($staleAgent->requests) === 1
|
|
&& $staleAgent->requests[0]['method'] === 'GET'
|
|
&& $staleAgent->requests[0]['target']
|
|
=== '/v1/operations/' . $staleOperation
|
|
&& $requestCount($staleAgent, 'POST') === 0,
|
|
'Stale sending boundary was not checked by operation ID without replay.');
|
|
|
|
// A response produced from a stale concurrent poll must not overwrite a
|
|
// terminal state committed while that request was in flight.
|
|
$raceAgent = new RescueImportFakeAgent();
|
|
CloudAgentSettingsService::$clients[50] = $raceAgent;
|
|
$race = CloudRescueTemplateImportService::start(
|
|
50, 'remote', 'Concurrent poll',
|
|
'https://images.example.com/concurrent.iso', '', '', $actor
|
|
);
|
|
$raceAgent->onRequest = static function (string $method, string $target)
|
|
use ($race): void {
|
|
if ($method === 'GET') {
|
|
CloudRescueTemplateImport::query()
|
|
->where('operation_id', (string) $race['operation_id'])
|
|
->update([
|
|
'state' => 'succeeded',
|
|
'error_code' => '',
|
|
'updated_at' => time(),
|
|
]);
|
|
}
|
|
};
|
|
$raced = CloudRescueTemplateImportService::job(
|
|
50, (string) $race['operation_id'], $actor
|
|
);
|
|
$assert($raced['status'] === 'succeeded'
|
|
&& (string) CloudRescueTemplateImport::find(
|
|
(string) $race['operation_id']
|
|
)->state === 'succeeded',
|
|
'A stale concurrent poll moved a terminal import state backwards.');
|
|
|
|
$columns = $schema->getColumnListing('cloud_rescue_template_imports');
|
|
$assert(!in_array('source_url', $columns, true)
|
|
&& !in_array('source_token', $columns, true),
|
|
'Durable rescue import table persisted source credentials.');
|
|
|
|
echo "Rescue import persistence, host binding and no-replay reconciliation passed.\n";
|
|
}
|