You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
332 lines
14 KiB
PHP
332 lines
14 KiB
PHP
<?php
|
|
declare(strict_types=1);
|
|
|
|
namespace App\Services\Cloud {
|
|
/** Deterministic in-process Agent for rescue import reconciliation tests. */
|
|
final class RescueImportFakeAgent
|
|
{
|
|
public array $requests = [];
|
|
public array $operations = [];
|
|
public string $mode = 'running';
|
|
public $onRequest = null;
|
|
|
|
public function request(string $method, string $target, array $actor,
|
|
?array $payload = null): array
|
|
{
|
|
$method = strtoupper($method);
|
|
$this->requests[] = compact('method', 'target', 'actor', 'payload');
|
|
if (is_callable($this->onRequest)) {
|
|
($this->onRequest)($method, $target, $actor, $payload);
|
|
}
|
|
|
|
if ($method === 'POST' && $target === '/v1/rescue-templates') {
|
|
$operationId = (string) ($payload['operation_id'] ?? '');
|
|
$templateId = (string) ($payload['id'] ?? '');
|
|
if ($this->mode === 'pre_dispatch_busy') {
|
|
throw new CloudAgentError('not dispatched', 503,
|
|
'agent_busy', true, 503, false, true);
|
|
}
|
|
$this->operations[$operationId] = $templateId;
|
|
if ($this->mode === 'transport') {
|
|
throw new CloudAgentError('transport uncertain', 502,
|
|
'agent_transport_error', true);
|
|
}
|
|
} elseif ($method === 'GET'
|
|
&& preg_match('#^/v1/operations/([A-Za-z0-9_-]+)$#D',
|
|
$target, $match)) {
|
|
$operationId = rawurldecode($match[1]);
|
|
$templateId = $this->operations[$operationId] ?? '';
|
|
if ($this->mode === 'not_found' || $templateId === '') {
|
|
throw new CloudAgentError('operation missing', 404,
|
|
'operation_not_found', false, 404);
|
|
}
|
|
if ($this->mode === 'transport') {
|
|
throw new CloudAgentError('transport uncertain', 502,
|
|
'agent_transport_error', true);
|
|
}
|
|
} else {
|
|
throw new \RuntimeException('Unexpected fake Agent request: '
|
|
. $method . ' ' . $target);
|
|
}
|
|
|
|
if ($this->mode === 'wrong_operation') {
|
|
return ['operation_id' => str_repeat('f', 32),
|
|
'status' => 'running'];
|
|
}
|
|
if ($this->mode === 'failed') {
|
|
return ['operation_id' => $operationId, 'status' => 'failed',
|
|
'error' => ['code' => 'rescue_template_exists']];
|
|
}
|
|
if ($this->mode === 'success') {
|
|
return [
|
|
'operation_id' => $operationId,
|
|
'status' => 'succeeded',
|
|
'result' => [
|
|
'id' => $templateId,
|
|
'name' => 'Rescue',
|
|
'size_bytes' => 1048576,
|
|
'sha256' => str_repeat('a', 64),
|
|
],
|
|
];
|
|
}
|
|
return ['operation_id' => $operationId, 'status' => 'running'];
|
|
}
|
|
}
|
|
|
|
/** Replaces only Agent lookup in this isolated process. */
|
|
final class CloudAgentSettingsService
|
|
{
|
|
/** @var array<int, RescueImportFakeAgent> */
|
|
public static array $clients = [];
|
|
|
|
public static function clientForHost(int $hostId,
|
|
float $timeoutSeconds = 20.0): RescueImportFakeAgent
|
|
{
|
|
if (!isset(self::$clients[$hostId])) {
|
|
throw new \RuntimeException('Missing fake Agent for host '
|
|
. $hostId);
|
|
}
|
|
return self::$clients[$hostId];
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudRescueTemplateImport;
|
|
use App\Services\Cloud\CloudAgentSettingsService;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\CloudRescueTemplateImportService;
|
|
use App\Services\Cloud\RescueImportFakeAgent;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new \RuntimeException($message);
|
|
}
|
|
};
|
|
$requestCount = static fn (RescueImportFakeAgent $agent,
|
|
string $method): int => count(array_filter($agent->requests,
|
|
static fn (array $request): bool => $request['method'] === $method));
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
|
|
'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
$schema->create('cloud_rescue_template_imports',
|
|
static function (Blueprint $table): void {
|
|
$table->string('operation_id')->primary();
|
|
$table->integer('host_id');
|
|
$table->string('template_id');
|
|
$table->string('import_type');
|
|
$table->string('state');
|
|
$table->string('error_code')->default('');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->unique(['host_id', 'template_id']);
|
|
});
|
|
|
|
$actor = ['user_id' => 1, 'is_admin' => true, 'can_manage' => true];
|
|
|
|
// The row must exist with its immutable identity before the first POST.
|
|
$remoteAgent = new RescueImportFakeAgent();
|
|
$remoteAgent->onRequest = static function (string $method, string $target,
|
|
array $requestActor, ?array $payload) use ($assert): void {
|
|
if ($method !== 'POST') {
|
|
return;
|
|
}
|
|
$row = CloudRescueTemplateImport::query()
|
|
->where('operation_id', (string) $payload['operation_id'])->first();
|
|
$assert($row !== null
|
|
&& (int) $row->host_id === 10
|
|
&& (string) $row->template_id === (string) $payload['id']
|
|
&& (string) $row->import_type === 'remote'
|
|
&& (string) $row->state === 'sending'
|
|
&& (string) ($requestActor['operation_id'] ?? '')
|
|
=== (string) $row->operation_id,
|
|
'Rescue import crossed dispatch before its durable identity existed.');
|
|
};
|
|
CloudAgentSettingsService::$clients[10] = $remoteAgent;
|
|
$remote = CloudRescueTemplateImportService::start(
|
|
10,
|
|
'remote',
|
|
'Rescue',
|
|
'https://images.example.com/rescue.iso',
|
|
'',
|
|
str_repeat('a', 64),
|
|
$actor
|
|
);
|
|
$remoteRow = CloudRescueTemplateImport::query()
|
|
->where('operation_id', (string) $remote['operation_id'])->first();
|
|
$assert($remote['status'] === 'running'
|
|
&& $remote['job_id'] === $remote['operation_id']
|
|
&& $remote['host_id'] === 10
|
|
&& $remote['template_id'] === (string) $remoteRow->template_id
|
|
&& $remote['import_type'] === 'remote'
|
|
&& $remote['status_url']
|
|
=== '/api/v1/admin/rescue-templates/import-jobs/'
|
|
. $remote['operation_id'] . '?host_id=10'
|
|
&& (string) $remoteRow->state === 'running'
|
|
&& $requestCount($remoteAgent, 'POST') === 1,
|
|
'Remote rescue import did not persist its running operation.');
|
|
|
|
// Authenticated capacity rejection is known to precede the Agent journal,
|
|
// so it is a deterministic failure rather than an ambiguous operation.
|
|
$busyAgent = new RescueImportFakeAgent();
|
|
$busyAgent->mode = 'pre_dispatch_busy';
|
|
CloudAgentSettingsService::$clients[11] = $busyAgent;
|
|
$busy = CloudRescueTemplateImportService::start(
|
|
11, 'remote', 'Busy rescue',
|
|
'https://images.example.com/busy.iso', '', '', $actor
|
|
);
|
|
$assert($busy['status'] === 'failed'
|
|
&& $busy['error_code'] === 'agent_busy'
|
|
&& $requestCount($busyAgent, 'POST') === 1
|
|
&& $busyAgent->operations === [],
|
|
'Definitive pre-dispatch rescue import was left for impossible reconciliation.');
|
|
|
|
// Reconciliation uses the exact saved operation and terminal rows are
|
|
// read-only; no later call is allowed to issue another import POST.
|
|
$remoteAgent->mode = 'success';
|
|
$completed = CloudRescueTemplateImportService::job(
|
|
10, (string) $remote['operation_id'], $actor
|
|
);
|
|
$requestsAfterSuccess = count($remoteAgent->requests);
|
|
$completedAgain = CloudRescueTemplateImportService::job(
|
|
10, (string) $remote['operation_id'], $actor
|
|
);
|
|
$lastPoll = $remoteAgent->requests[$requestsAfterSuccess - 1];
|
|
$assert($completed['status'] === 'succeeded'
|
|
&& $completedAgain['status'] === 'succeeded'
|
|
&& $requestCount($remoteAgent, 'POST') === 1
|
|
&& count($remoteAgent->requests) === $requestsAfterSuccess
|
|
&& $lastPoll['method'] === 'GET'
|
|
&& $lastPoll['target'] === '/v1/operations/'
|
|
. $remote['operation_id'],
|
|
'Rescue import reconciliation changed or replayed its operation.');
|
|
|
|
// A retryable transport failure may have crossed the boundary. Preserve
|
|
// the upload operation, then lock it uncertain if Agent has no record.
|
|
$uploadAgent = new RescueImportFakeAgent();
|
|
$uploadAgent->mode = 'transport';
|
|
CloudAgentSettingsService::$clients[20] = $uploadAgent;
|
|
$upload = CloudRescueTemplateImportService::start(
|
|
20,
|
|
'upload',
|
|
'Uploaded rescue',
|
|
'https://panel.example.com/api/internal/cloud/upload-staging/'
|
|
. str_repeat('b', 32),
|
|
str_repeat('c', 64),
|
|
str_repeat('d', 64),
|
|
$actor
|
|
);
|
|
$uploadRow = CloudRescueTemplateImport::query()
|
|
->where('operation_id', (string) $upload['operation_id'])->first();
|
|
$assert($upload['status'] === 'running'
|
|
&& $upload['import_type'] === 'upload'
|
|
&& (string) $uploadRow->state === 'running'
|
|
&& (string) $uploadRow->error_code === 'agent_transport_error'
|
|
&& $requestCount($uploadAgent, 'POST') === 1,
|
|
'Ambiguous upload dispatch was not retained for reconciliation.');
|
|
$uploadAgent->mode = 'not_found';
|
|
$uncertain = CloudRescueTemplateImportService::job(
|
|
20, (string) $upload['operation_id'], $actor
|
|
);
|
|
$requestsAfterUncertain = count($uploadAgent->requests);
|
|
$uncertainAgain = CloudRescueTemplateImportService::job(
|
|
20, (string) $upload['operation_id'], $actor
|
|
);
|
|
$assert($uncertain['status'] === 'uncertain'
|
|
&& $uncertainAgain['status'] === 'uncertain'
|
|
&& $uncertain['error_code'] === 'operation_not_found'
|
|
&& $requestCount($uploadAgent, 'POST') === 1
|
|
&& count($uploadAgent->requests) === $requestsAfterUncertain,
|
|
'Missing ambiguous operation was replayed instead of locked uncertain.');
|
|
|
|
// A mismatched Agent operation is terminally uncertain and host scoping
|
|
// prevents another host from learning or polling the job.
|
|
$mismatchAgent = new RescueImportFakeAgent();
|
|
$mismatchAgent->mode = 'wrong_operation';
|
|
CloudAgentSettingsService::$clients[30] = $mismatchAgent;
|
|
$mismatch = CloudRescueTemplateImportService::start(
|
|
30, 'remote', 'Mismatch', 'https://images.example.com/other.iso',
|
|
'', '', $actor
|
|
);
|
|
$assert($mismatch['status'] === 'uncertain'
|
|
&& $mismatch['error_code'] === 'operation_id_mismatch'
|
|
&& $requestCount($mismatchAgent, 'POST') === 1,
|
|
'Mismatched Agent operation identity was accepted.');
|
|
try {
|
|
CloudRescueTemplateImportService::job(
|
|
31, (string) $mismatch['operation_id'], $actor
|
|
);
|
|
throw new \RuntimeException('Cross-host rescue import lookup succeeded.');
|
|
} catch (CloudError $error) {
|
|
$assert($error->httpStatus === 404,
|
|
'Cross-host rescue import lookup returned the wrong status.');
|
|
}
|
|
|
|
// A process crash between durable reservation and completion cannot cause
|
|
// a later status request to issue the original POST.
|
|
$staleOperation = str_repeat('e', 32);
|
|
Capsule::table('cloud_rescue_template_imports')->insert([
|
|
'operation_id' => $staleOperation,
|
|
'host_id' => 40,
|
|
'template_id' => str_repeat('9', 32),
|
|
'import_type' => 'remote',
|
|
'state' => 'sending',
|
|
'error_code' => '',
|
|
'created_at' => time() - 180,
|
|
'updated_at' => time() - 180,
|
|
]);
|
|
$staleAgent = new RescueImportFakeAgent();
|
|
CloudAgentSettingsService::$clients[40] = $staleAgent;
|
|
$stale = CloudRescueTemplateImportService::job(
|
|
40, $staleOperation, $actor
|
|
);
|
|
$assert($stale['status'] === 'uncertain'
|
|
&& $stale['error_code'] === 'import_dispatch_unconfirmed'
|
|
&& $staleAgent->requests === [],
|
|
'Stale dispatch boundary caused an unsafe import replay.');
|
|
|
|
// A response produced from a stale concurrent poll must not overwrite a
|
|
// terminal state committed while that request was in flight.
|
|
$raceAgent = new RescueImportFakeAgent();
|
|
CloudAgentSettingsService::$clients[50] = $raceAgent;
|
|
$race = CloudRescueTemplateImportService::start(
|
|
50, 'remote', 'Concurrent poll',
|
|
'https://images.example.com/concurrent.iso', '', '', $actor
|
|
);
|
|
$raceAgent->onRequest = static function (string $method, string $target)
|
|
use ($race): void {
|
|
if ($method === 'GET') {
|
|
CloudRescueTemplateImport::query()
|
|
->where('operation_id', (string) $race['operation_id'])
|
|
->update([
|
|
'state' => 'succeeded',
|
|
'error_code' => '',
|
|
'updated_at' => time(),
|
|
]);
|
|
}
|
|
};
|
|
$raced = CloudRescueTemplateImportService::job(
|
|
50, (string) $race['operation_id'], $actor
|
|
);
|
|
$assert($raced['status'] === 'succeeded'
|
|
&& (string) CloudRescueTemplateImport::find(
|
|
(string) $race['operation_id']
|
|
)->state === 'succeeded',
|
|
'A stale concurrent poll moved a terminal import state backwards.');
|
|
|
|
$columns = $schema->getColumnListing('cloud_rescue_template_imports');
|
|
$assert(!in_array('source_url', $columns, true)
|
|
&& !in_array('source_token', $columns, true),
|
|
'Durable rescue import table persisted source credentials.');
|
|
|
|
echo "Rescue import persistence, host binding and no-replay reconciliation passed.\n";
|
|
}
|