requests[] = compact('method', 'target', 'actor', 'payload'); if (is_callable($this->onRequest)) { ($this->onRequest)($method, $target, $actor, $payload); } if ($method === 'POST' && $target === '/v1/rescue-templates') { $operationId = (string) ($payload['operation_id'] ?? ''); $templateId = (string) ($payload['id'] ?? ''); if ($this->mode === 'pre_dispatch_busy') { throw new CloudAgentError('not dispatched', 503, 'agent_busy', true, 503, false, true); } $this->operations[$operationId] = $templateId; if ($this->mode === 'ambiguous_replay') { throw new CloudAgentError('nonce may have dispatched', 502, 'replayed_request', false, 401); } if ($this->mode === 'transport') { throw new CloudAgentError('transport uncertain', 502, 'agent_transport_error', true); } } elseif ($method === 'GET' && preg_match('#^/v1/operations/([A-Za-z0-9_-]+)$#D', $target, $match)) { $operationId = rawurldecode($match[1]); $templateId = $this->operations[$operationId] ?? ''; if ($this->mode === 'not_found' || $templateId === '') { throw new CloudAgentError('operation missing', 404, 'operation_not_found', false, 404); } if ($this->mode === 'transport') { throw new CloudAgentError('transport uncertain', 502, 'agent_transport_error', true); } } elseif ($method === 'POST' && preg_match( '#^/v1/operations/([A-Za-z0-9_-]+)/reconcile$#D', $target, $match)) { $operationId = rawurldecode($match[1]); $templateId = $this->operations[$operationId] ?? ''; if ($templateId === '') { throw new CloudAgentError('operation missing', 404, 'operation_not_found', false, 404); } return ['operation_id' => $operationId] + ($this->reconcileResponse ?? [ 'status' => 'uncertain', 'error' => ['code' => 'operation_uncertain'], ]); } else { throw new \RuntimeException('Unexpected fake Agent request: ' . $method . ' ' . $target); } if ($this->mode === 'wrong_operation') { return ['operation_id' => str_repeat('f', 32), 'status' => 'running']; } if ($this->mode === 'failed') { return ['operation_id' => $operationId, 'status' => 'failed', 'error' => ['code' => 'rescue_template_exists']]; } if ($this->mode === 'success') { return [ 'operation_id' => $operationId, 'status' => 'succeeded', 'result' => [ 'id' => $templateId, 'name' => 'Rescue', 'size_bytes' => 1048576, 'sha256' => str_repeat('a', 64), ], ]; } if ($this->mode === 'uncertain') { return [ 'operation_id' => $operationId, 'status' => 'uncertain', 'error' => ['code' => 'operation_uncertain'], ]; } return ['operation_id' => $operationId, 'status' => 'running']; } } /** Replaces only Agent lookup in this isolated process. */ final class CloudAgentSettingsService { /** @var array */ public static array $clients = []; public static function clientForHost(int $hostId, float $timeoutSeconds = 20.0): RescueImportFakeAgent { if (!isset(self::$clients[$hostId])) { throw new \RuntimeException('Missing fake Agent for host ' . $hostId); } return self::$clients[$hostId]; } } } namespace { require dirname(__DIR__) . '/vendor/autoload.php'; use App\Models\CloudRescueTemplateImport; use App\Services\Cloud\CloudAgentSettingsService; use App\Services\Cloud\CloudError; use App\Services\Cloud\CloudRescueTemplateImportService; use App\Services\Cloud\RescueImportFakeAgent; use Illuminate\Database\Capsule\Manager as Capsule; use Illuminate\Database\Schema\Blueprint; $assert = static function (bool $condition, string $message): void { if (!$condition) { throw new \RuntimeException($message); } }; $requestCount = static fn (RescueImportFakeAgent $agent, string $method): int => count(array_filter($agent->requests, static fn (array $request): bool => $request['method'] === $method)); $capsule = new Capsule(); $capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('cloud_rescue_template_imports', static function (Blueprint $table): void { $table->string('operation_id')->primary(); $table->integer('host_id'); $table->string('template_id'); $table->string('import_type'); $table->string('state'); $table->string('error_code')->default(''); $table->integer('created_at'); $table->integer('updated_at'); $table->unique(['host_id', 'template_id']); }); $actor = ['user_id' => 1, 'is_admin' => true, 'can_manage' => true]; // The row must exist with its immutable identity before the first POST. $remoteAgent = new RescueImportFakeAgent(); $remoteAgent->onRequest = static function (string $method, string $target, array $requestActor, ?array $payload) use ($assert): void { if ($method !== 'POST') { return; } $row = CloudRescueTemplateImport::query() ->where('operation_id', (string) $payload['operation_id'])->first(); $assert($row !== null && (int) $row->host_id === 10 && (string) $row->template_id === (string) $payload['id'] && (string) $row->import_type === 'remote' && (string) $row->state === 'sending' && (string) ($requestActor['operation_id'] ?? '') === (string) $row->operation_id, 'Rescue import crossed dispatch before its durable identity existed.'); }; CloudAgentSettingsService::$clients[10] = $remoteAgent; $remote = CloudRescueTemplateImportService::start( 10, 'remote', 'Rescue', 'https://images.example.com/rescue.iso', '', str_repeat('a', 64), $actor ); $remoteRow = CloudRescueTemplateImport::query() ->where('operation_id', (string) $remote['operation_id'])->first(); $assert($remote['status'] === 'running' && $remote['job_id'] === $remote['operation_id'] && $remote['host_id'] === 10 && $remote['template_id'] === (string) $remoteRow->template_id && $remote['import_type'] === 'remote' && $remote['status_url'] === '/api/v1/admin/rescue-templates/import-jobs/' . $remote['operation_id'] . '?host_id=10' && (string) $remoteRow->state === 'running' && $requestCount($remoteAgent, 'POST') === 1, 'Remote rescue import did not persist its running operation.'); // Authenticated capacity rejection is known to precede the Agent journal, // so it is a deterministic failure rather than an ambiguous operation. $busyAgent = new RescueImportFakeAgent(); $busyAgent->mode = 'pre_dispatch_busy'; CloudAgentSettingsService::$clients[11] = $busyAgent; $busy = CloudRescueTemplateImportService::start( 11, 'remote', 'Busy rescue', 'https://images.example.com/busy.iso', '', '', $actor ); $assert($busy['status'] === 'failed' && $busy['error_code'] === 'agent_busy' && $requestCount($busyAgent, 'POST') === 1 && $busyAgent->operations === [], 'Definitive pre-dispatch rescue import was left for impossible reconciliation.'); $replayedAgent = new RescueImportFakeAgent(); $replayedAgent->mode = 'ambiguous_replay'; CloudAgentSettingsService::$clients[14] = $replayedAgent; $replayed = CloudRescueTemplateImportService::start( 14, 'remote', 'Replay ambiguous rescue', 'https://images.example.com/replay-ambiguous.iso', '', '', $actor ); $replayedRow = CloudRescueTemplateImport::query() ->where('operation_id', (string) $replayed['operation_id'])->first(); $assert($replayed['status'] === 'running' && (string) $replayedRow->state === 'running' && (string) $replayedRow->error_code === 'replayed_request' && $requestCount($replayedAgent, 'POST') === 1, 'Ambiguous non-retryable rescue import was made terminal.'); $replayedAgent->mode = 'success'; $replayedCompleted = CloudRescueTemplateImportService::job( 14, (string) $replayed['operation_id'], $actor ); $assert($replayedCompleted['status'] === 'succeeded' && $requestCount($replayedAgent, 'POST') === 1, 'Ambiguous rescue import was replayed instead of polled by operation ID.'); // Reconciliation uses the exact saved operation and terminal rows are // read-only; no later call is allowed to issue another import POST. $remoteAgent->mode = 'success'; $completed = CloudRescueTemplateImportService::job( 10, (string) $remote['operation_id'], $actor ); $requestsAfterSuccess = count($remoteAgent->requests); $completedAgain = CloudRescueTemplateImportService::job( 10, (string) $remote['operation_id'], $actor ); $lastPoll = $remoteAgent->requests[$requestsAfterSuccess - 1]; $assert($completed['status'] === 'succeeded' && $completedAgain['status'] === 'succeeded' && $requestCount($remoteAgent, 'POST') === 1 && count($remoteAgent->requests) === $requestsAfterSuccess && $lastPoll['method'] === 'GET' && $lastPoll['target'] === '/v1/operations/' . $remote['operation_id'], 'Rescue import reconciliation changed or replayed its operation.'); // GET uncertainty is resolved only through POST /reconcile for the same // operation. The original rescue import POST remains single-shot. $reconcileAgent = new RescueImportFakeAgent(); CloudAgentSettingsService::$clients[12] = $reconcileAgent; $reconcileImport = CloudRescueTemplateImportService::start( 12, 'remote', 'Reconcile rescue', 'https://images.example.com/reconcile.iso', '', '', $actor ); $reconcileAgent->mode = 'uncertain'; $reconcileAgent->reconcileResponse = [ 'status' => 'succeeded', 'result' => [ 'id' => (string) $reconcileImport['template_id'], 'name' => 'Reconcile rescue', 'size_bytes' => 1048576, 'sha256' => str_repeat('a', 64), ], ]; $beforeReconcilePoll = count($reconcileAgent->requests); $reconcileCompleted = CloudRescueTemplateImportService::job( 12, (string) $reconcileImport['operation_id'], $actor ); $reconcileRequests = array_slice( $reconcileAgent->requests, $beforeReconcilePoll ); $importPosts = array_filter($reconcileAgent->requests, static fn (array $request): bool => $request['method'] === 'POST' && $request['target'] === '/v1/rescue-templates'); $assert($reconcileCompleted['status'] === 'succeeded' && count($importPosts) === 1 && count($reconcileRequests) === 2 && $reconcileRequests[0]['method'] === 'GET' && $reconcileRequests[0]['target'] === '/v1/operations/' . $reconcileImport['operation_id'] && $reconcileRequests[1]['method'] === 'POST' && $reconcileRequests[1]['target'] === '/v1/operations/' . $reconcileImport['operation_id'] . '/reconcile', 'Rescue import uncertainty was replayed or reconciled under another operation.'); $trueUncertainAgent = new RescueImportFakeAgent(); CloudAgentSettingsService::$clients[13] = $trueUncertainAgent; $trueUncertainImport = CloudRescueTemplateImportService::start( 13, 'remote', 'Uncertain rescue', 'https://images.example.com/uncertain.iso', '', '', $actor ); $trueUncertainAgent->mode = 'uncertain'; $trueUncertainAgent->reconcileResponse = [ 'status' => 'uncertain', 'error' => ['code' => 'operation_uncertain'], ]; $trueUncertain = CloudRescueTemplateImportService::job( 13, (string) $trueUncertainImport['operation_id'], $actor ); $requestsAtUncertain = count($trueUncertainAgent->requests); $trueUncertainAgain = CloudRescueTemplateImportService::job( 13, (string) $trueUncertainImport['operation_id'], $actor ); $trueUncertainPosts = array_filter($trueUncertainAgent->requests, static fn (array $request): bool => $request['method'] === 'POST' && $request['target'] === '/v1/rescue-templates'); $assert($trueUncertain['status'] === 'uncertain' && $trueUncertainAgain['status'] === 'uncertain' && count($trueUncertainPosts) === 1 && count($trueUncertainAgent->requests) === $requestsAtUncertain, 'A genuinely uncertain rescue import was retried or left pollable.'); // A retryable transport failure may have crossed the boundary. Preserve // the upload operation, then lock it uncertain if Agent has no record. $uploadAgent = new RescueImportFakeAgent(); $uploadAgent->mode = 'transport'; CloudAgentSettingsService::$clients[20] = $uploadAgent; $upload = CloudRescueTemplateImportService::start( 20, 'upload', 'Uploaded rescue', 'https://panel.example.com/api/internal/cloud/upload-staging/' . str_repeat('b', 32), str_repeat('c', 64), str_repeat('d', 64), $actor ); $uploadRow = CloudRescueTemplateImport::query() ->where('operation_id', (string) $upload['operation_id'])->first(); $assert($upload['status'] === 'running' && $upload['import_type'] === 'upload' && (string) $uploadRow->state === 'running' && (string) $uploadRow->error_code === 'agent_transport_error' && $requestCount($uploadAgent, 'POST') === 1, 'Ambiguous upload dispatch was not retained for reconciliation.'); $uploadAgent->mode = 'not_found'; $uncertain = CloudRescueTemplateImportService::job( 20, (string) $upload['operation_id'], $actor ); $requestsAfterUncertain = count($uploadAgent->requests); $uncertainAgain = CloudRescueTemplateImportService::job( 20, (string) $upload['operation_id'], $actor ); $assert($uncertain['status'] === 'uncertain' && $uncertainAgain['status'] === 'uncertain' && $uncertain['error_code'] === 'operation_not_found' && $requestCount($uploadAgent, 'POST') === 1 && count($uploadAgent->requests) === $requestsAfterUncertain, 'Missing ambiguous operation was replayed instead of locked uncertain.'); // A mismatched Agent operation is terminally uncertain and host scoping // prevents another host from learning or polling the job. $mismatchAgent = new RescueImportFakeAgent(); $mismatchAgent->mode = 'wrong_operation'; CloudAgentSettingsService::$clients[30] = $mismatchAgent; $mismatch = CloudRescueTemplateImportService::start( 30, 'remote', 'Mismatch', 'https://images.example.com/other.iso', '', '', $actor ); $assert($mismatch['status'] === 'uncertain' && $mismatch['error_code'] === 'operation_id_mismatch' && $requestCount($mismatchAgent, 'POST') === 1, 'Mismatched Agent operation identity was accepted.'); try { CloudRescueTemplateImportService::job( 31, (string) $mismatch['operation_id'], $actor ); throw new \RuntimeException('Cross-host rescue import lookup succeeded.'); } catch (CloudError $error) { $assert($error->httpStatus === 404, 'Cross-host rescue import lookup returned the wrong status.'); } // A process crash between durable reservation and completion cannot cause // a later status request to issue the original POST. $staleOperation = str_repeat('e', 32); Capsule::table('cloud_rescue_template_imports')->insert([ 'operation_id' => $staleOperation, 'host_id' => 40, 'template_id' => str_repeat('9', 32), 'import_type' => 'remote', 'state' => 'sending', 'error_code' => '', 'created_at' => time() - 180, 'updated_at' => time() - 180, ]); $staleAgent = new RescueImportFakeAgent(); CloudAgentSettingsService::$clients[40] = $staleAgent; $stale = CloudRescueTemplateImportService::job( 40, $staleOperation, $actor ); $assert($stale['status'] === 'uncertain' && $stale['error_code'] === 'operation_not_found' && count($staleAgent->requests) === 1 && $staleAgent->requests[0]['method'] === 'GET' && $staleAgent->requests[0]['target'] === '/v1/operations/' . $staleOperation && $requestCount($staleAgent, 'POST') === 0, 'Stale sending boundary was not checked by operation ID without replay.'); // A response produced from a stale concurrent poll must not overwrite a // terminal state committed while that request was in flight. $raceAgent = new RescueImportFakeAgent(); CloudAgentSettingsService::$clients[50] = $raceAgent; $race = CloudRescueTemplateImportService::start( 50, 'remote', 'Concurrent poll', 'https://images.example.com/concurrent.iso', '', '', $actor ); $raceAgent->onRequest = static function (string $method, string $target) use ($race): void { if ($method === 'GET') { CloudRescueTemplateImport::query() ->where('operation_id', (string) $race['operation_id']) ->update([ 'state' => 'succeeded', 'error_code' => '', 'updated_at' => time(), ]); } }; $raced = CloudRescueTemplateImportService::job( 50, (string) $race['operation_id'], $actor ); $assert($raced['status'] === 'succeeded' && (string) CloudRescueTemplateImport::find( (string) $race['operation_id'] )->state === 'succeeded', 'A stale concurrent poll moved a terminal import state backwards.'); $columns = $schema->getColumnListing('cloud_rescue_template_imports'); $assert(!in_array('source_url', $columns, true) && !in_array('source_token', $columns, true), 'Durable rescue import table persisted source credentials.'); echo "Rescue import persistence, host binding and no-replay reconciliation passed.\n"; }