httpStatus === $status, $message . ' Unexpected status: ' . $error->httpStatus); return; } throw new RuntimeException($message); }; $terminalize = static function (CloudJob $job, string $outcome, bool $effectMayHaveApplied = false): void { $now = time(); $job->phase = 'terminal'; $job->outcome = $outcome; $job->status = $outcome; $job->attempts = 1; $job->dispatch_attempts = 1; $job->dispatch_started_at = $now - 2; $job->dispatched_at = $now - 2; $job->agent_accepted_at = $now - 1; $job->receipt_deadline_at = $now + 900; $job->terminal_at = $now; $job->effect_may_have_applied = $effectMayHaveApplied ? 1 : 0; $job->safe_to_resubmit = 0; $job->effect_resolution = $outcome === 'succeeded' ? 'applied' : ($effectMayHaveApplied ? 'unknown' : 'not_applied'); $job->lease_token = null; $job->lease_until = 0; $job->available_at = 0; $job->updated_at = $now; $job->save(); }; $makeLegacyActive = static function (CloudJob $job, string $phase = 'awaiting_receipt'): void { $now = time(); $job->phase = $phase; $job->status = $phase === 'accepted' ? 'queued' : 'running'; $job->attempts = $phase === 'accepted' ? 0 : 1; $job->dispatch_attempts = $phase === 'accepted' ? 0 : 1; $job->created_at = $now - 181; $job->dispatch_deadline_at = $now + 119; $job->dispatch_started_at = $phase === 'accepted' ? null : $now - 181; $job->dispatched_at = $phase === 'accepted' ? null : $now - 181; $job->agent_accepted_at = $phase === 'accepted' ? null : $now - 180; $job->receipt_deadline_at = $phase === 'accepted' ? null : $now + 779; $job->lease_token = null; $job->lease_until = 0; $job->updated_at = $now; $job->save(); }; $capsule = new Capsule(); $capsule->addConnection([ 'driver' => 'sqlite', 'database' => ':memory:', ], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('user', static function (Blueprint $table): void { $table->increments('id'); }); $schema->create('cloud_hosts', static function (Blueprint $table): void { $table->increments('id'); $table->string('name'); $table->string('region'); $table->string('country_code', 2); $table->boolean('enabled'); $table->string('default_network'); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_host_agents', static function (Blueprint $table): void { $table->integer('host_id')->primary(); $table->text('sealed_api_secret'); $table->boolean('enabled')->default(false); $table->integer('revision')->default(1); $table->integer('tested_revision')->default(0); $table->string('last_test_status')->default('never'); $table->string('last_error_code')->default(''); $table->string('last_agent_version')->default(''); $table->text('network_selectors_json')->default('[]'); $table->integer('last_tested_at')->default(0); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_host_storage_configurations', static function (Blueprint $table): void { $table->integer('host_id')->primary(); $table->integer('revision')->default(0); foreach (['managed', 'template', 'seed', 'user_iso', 'rescue'] as $role) { $table->text($role . '_directory'); } $table->string('state')->default('applying'); $table->string('operation_id')->nullable()->unique(); $table->integer('job_id')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('operation_id')->unique(); $table->string('request_idempotency_key'); $table->string('request_fingerprint'); $table->integer('parent_job_id')->nullable(); $table->integer('order_id')->nullable(); $table->integer('invoice_id')->nullable(); $table->integer('instance_id')->nullable(); $table->integer('snapshot_id')->nullable(); $table->integer('user_id'); $table->integer('host_id')->nullable(); $table->string('kind'); $table->string('execution_mode'); $table->string('phase'); $table->string('outcome')->nullable(); $table->string('status'); $table->text('sealed_payload'); $table->integer('attempts'); $table->integer('dispatch_attempts'); $table->integer('dispatch_deadline_at'); $table->integer('dispatch_started_at')->nullable(); $table->integer('agent_accepted_at')->nullable(); $table->integer('receipt_deadline_at')->nullable(); $table->integer('terminal_at')->nullable(); $table->boolean('effect_may_have_applied')->nullable(); $table->boolean('safe_to_resubmit')->nullable(); $table->string('effect_resolution'); $table->integer('effect_resolved_at')->nullable(); $table->integer('effect_resolved_by')->nullable(); $table->text('effect_resolution_reason')->nullable(); $table->string('completion_event_id')->nullable(); $table->integer('read_at')->nullable(); $table->integer('dispatched_at')->nullable(); $table->string('lease_token')->nullable(); $table->integer('lease_until')->default(0); $table->integer('available_at'); $table->string('error_code')->nullable(); $table->boolean('finalization_incident')->default(0); $table->string('finalization_error_code')->nullable(); $table->text('result_json')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_agent_completion_events', static function (Blueprint $table): void { $table->string('event_id')->primary(); $table->integer('host_id'); $table->string('operation_id'); $table->string('payload_sha256'); $table->string('outcome'); $table->boolean('effect_may_have_applied'); $table->boolean('safe_to_resubmit'); $table->string('error_code')->default(''); $table->text('result_json')->nullable(); $table->string('disposition')->default('received'); $table->integer('received_at'); $table->integer('applied_at')->nullable(); $table->unique(['host_id', 'operation_id']); }); Capsule::table('user')->insert(['id' => 1]); foreach ([1, 2, 3, 4] as $hostId) { Capsule::table('cloud_hosts')->insert([ 'id' => $hostId, 'name' => 'Host ' . $hostId, 'region' => 'HongKong', 'country_code' => 'HK', 'enabled' => 1, 'default_network' => 'bridge:br0', 'created_at' => 1, 'updated_at' => 1, ]); Capsule::table('cloud_host_agents')->insert([ 'host_id' => $hostId, 'sealed_api_secret' => 'test-secret', 'enabled' => 1, 'revision' => 1, 'tested_revision' => 1, 'last_test_status' => 'healthy', 'last_error_code' => '', 'last_agent_version' => 'test', 'network_selectors_json' => '["bridge:br0"]', 'last_tested_at' => 1, 'created_at' => 1, 'updated_at' => 1, ]); } $actor = [ 'user_id' => 1, 'is_admin' => true, 'is_super_admin' => true, 'can_manage' => true, 'session_id' => 'storage-test', ]; $first = [ 'expected_revision' => 0, 'managed_directory' => '/home/libvirt/kvm', 'template_directory' => '/home/libvirt/images', 'seed_directory' => '/home/libvirt/seed', 'user_iso_directory' => '/home/libvirt/iso', 'rescue_directory' => '/home/libvirt/rescue-iso', ]; $empty = CloudHostStorageConfigurationService::read(1); $assert($empty['state'] === 'unconfigured' && $empty['revision'] === 0 && $empty['can_submit'] === true, 'An unconfigured host was not writable.'); $receipt1 = CloudHostStorageConfigurationService::save(1, $first, $actor); $job1 = CloudJob::find((int) $receipt1['task_id']); $row = CloudHostStorageConfiguration::find(1); $assert($job1 !== null && $row !== null && (string) $row->operation_id === (string) $job1->operation_id && (int) $row->job_id === (int) $job1->id && (int) $job1->dispatch_deadline_at - (int) $job1->created_at === 60 && (int) $row->revision === 1 && (string) $row->state === 'applying', 'Initial storage submission did not create one linked task.'); $activeView = CloudHostStorageConfigurationService::read(1); $assert($activeView['task_id'] === (string) $job1->id && $activeView['status_url'] === '/api/v1/tasks/' . (int) $job1->id, 'An active storage projection did not expose its bounded task observer.'); $sameActive = $first; $sameActive['expected_revision'] = 1; $activeReceipt = CloudHostStorageConfigurationService::save( 1, $sameActive, $actor ); $assert($activeReceipt['task_id'] === $receipt1['task_id'] && CloudJob::count() === 1, 'An active same-path request created another task.'); $changedActive = $sameActive; $changedActive['rescue_directory'] = '/home/libvirt/rescue-corrected'; $expect(static fn () => CloudHostStorageConfigurationService::save( 1, $changedActive, $actor ), 409, 'An active task allowed concurrent path replacement.'); CloudHostStorageConfigurationService::finishJobLocked( $job1, 'failed', null, ['effect_may_have_applied' => true] ); $terminalize($job1, 'failed', true); $failed = CloudHostStorageConfigurationService::read(1); $assert($failed['state'] === 'failed' && $failed['can_submit'] === true && $failed['operation_id'] === (string) $job1->operation_id, 'A terminal failure remained blocked.'); $receipt2 = CloudHostStorageConfigurationService::save( 1, $sameActive, $actor ); $job2 = CloudJob::find((int) $receipt2['task_id']); $row = CloudHostStorageConfiguration::find(1); $assert($job2 !== null && (int) $job2->id !== (int) $job1->id && (string) $job2->operation_id !== (string) $job1->operation_id && (int) $row->revision === 1 && (string) $row->operation_id === (string) $job2->operation_id && (string) $row->state === 'applying' && (string) $job1->phase === 'terminal' && (string) $job1->outcome === 'failed', 'A same-path correction did not create a new operation while preserving the failed audit.'); CloudHostStorageConfigurationService::finishJobLocked( $job1, 'succeeded', ['configured' => true], [] ); $row->refresh(); $assert((string) $row->operation_id === (string) $job2->operation_id && (string) $row->state === 'applying', 'A late superseded receipt overwrote the current generation.'); CloudHostStorageConfigurationService::finishJobLocked( $job2, 'failed', null, [] ); $terminalize($job2, 'failed'); $changedAfterFailure = $changedActive; $receipt3 = CloudHostStorageConfigurationService::save( 1, $changedAfterFailure, $actor ); $job3 = CloudJob::find((int) $receipt3['task_id']); $row = CloudHostStorageConfiguration::find(1); $assert($job3 !== null && (string) $job3->operation_id !== (string) $job2->operation_id && (int) $row->revision === 2 && (string) $row->rescue_directory === '/home/libvirt/rescue-corrected', 'A corrected path after terminal failure did not create the next revision.'); CloudHostStorageConfigurationService::finishJobLocked( $job3, 'succeeded', ['configured' => true], [] ); $terminalize($job3, 'succeeded'); $applied = CloudHostStorageConfigurationService::read(1); $assert($applied['state'] === 'applied' && $applied['configured'] === true && $applied['can_submit'] === false && $applied['requires_agent_retest'] === true, 'A confirmed storage layout was not locked as applied.'); $agentRevision = (int) Capsule::table('cloud_host_agents') ->where('host_id', 1)->value('revision'); Capsule::table('cloud_host_agents')->where('host_id', 1)->update([ 'tested_revision' => $agentRevision, 'last_test_status' => 'healthy', ]); $assert(CloudHostStorageConfigurationService::read(1)[ 'requires_agent_retest' ] === false, 'A current Agent retest did not satisfy storage readiness.'); Capsule::table('cloud_hosts')->where('id', 1)->update([ 'default_network' => 'retired-compatibility-value', ]); $assert(CloudHostStorageConfigurationService::read(1)[ 'requires_agent_retest' ] === false, 'Compatibility default incorrectly requires an Agent retest.'); $changedAfterFailure['expected_revision'] = 2; $sameApplied = CloudHostStorageConfigurationService::save( 1, $changedAfterFailure, $actor ); $assert($sameApplied['task_id'] === $receipt3['task_id'] && CloudJob::count() === 3, 'An exact applied request was not idempotent.'); $differentApplied = $changedAfterFailure; $differentApplied['managed_directory'] = '/home/libvirt/other-kvm'; $expect(static fn () => CloudHostStorageConfigurationService::save( 1, $differentApplied, $actor ), 409, 'An applied storage layout was mutable.'); // A worker-finalization incident must not turn a known terminal failure // into a permanent "applying" fence. $receipt4 = CloudHostStorageConfigurationService::save(2, $first, $actor); $job4 = CloudJob::find((int) $receipt4['task_id']); $terminalize($job4, 'failed', true); $staleProjection = CloudHostStorageConfigurationService::read(2); $assert($staleProjection['state'] === 'failed' && $staleProjection['can_submit'] === true, 'A terminally failed job remained visually locked behind stale applying state.'); $staleApplying = $first; $staleApplying['expected_revision'] = 1; $receipt5 = CloudHostStorageConfigurationService::save( 2, $staleApplying, $actor ); $job5 = CloudJob::find((int) $receipt5['task_id']); $row2 = CloudHostStorageConfiguration::find(2); $assert($job5 !== null && (int) $job5->id !== (int) $job4->id && (string) $row2->operation_id === (string) $job5->operation_id && (int) $row2->revision === 1, 'A linked terminal failure with a stale applying row remained blocked.'); // A reopened storage page locally closes a receipt-less task at the new // effective 180-second boundary, even when it persisted the old 960-second // deadline. This path performs no Agent request and no redispatch. $receipt6 = CloudHostStorageConfigurationService::save(3, $first, $actor); $job6 = CloudJob::find((int) $receipt6['task_id']); $makeLegacyActive($job6); $expiredRead = CloudHostStorageConfigurationService::read(3); $job6->refresh(); $row3 = CloudHostStorageConfiguration::find(3); $assert((string) $job6->phase === 'terminal' && (string) $job6->outcome === 'failed' && (string) $row3->state === 'failed' && $expiredRead['state'] === 'failed' && $expiredRead['can_submit'] === true && $expiredRead['task_id'] === null && $expiredRead['status_url'] === null, 'Storage read did not locally converge and reread an expired task.'); // Save has the same fallback. An accepted task carrying the former 300s // dispatch deadline is closed after 60s, then a new operation is created. $receipt7 = CloudHostStorageConfigurationService::save(4, $first, $actor); $job7 = CloudJob::find((int) $receipt7['task_id']); $makeLegacyActive($job7, 'accepted'); $sameLegacy = $first; $sameLegacy['expected_revision'] = 1; $receipt8 = CloudHostStorageConfigurationService::save( 4, $sameLegacy, $actor ); $job7->refresh(); $job8 = CloudJob::find((int) $receipt8['task_id']); $row4 = CloudHostStorageConfiguration::find(4); $assert((string) $job7->phase === 'terminal' && (string) $job7->outcome === 'failed' && $job8 !== null && (int) $job8->id !== (int) $job7->id && (string) $row4->operation_id === (string) $job8->operation_id && (string) $row4->state === 'applying', 'Storage save remained blocked by an expired accepted task.'); echo "Host storage failure supersession state machine passed.\n"; }