phase = 'awaiting_receipt'; $job->status = 'running'; $job->attempts = 1; $job->dispatch_attempts = 1; $job->dispatch_started_at = $now; $job->dispatched_at = $now; $job->agent_accepted_at = $now; $job->receipt_deadline_at = $now + 960; $job->save(); }; $terminalize = static function (CloudJob $job, string $outcome, bool $effect = false, bool $safe = false): void { $now = time(); $job->phase = 'terminal'; $job->outcome = $outcome; $job->status = $outcome; $job->attempts = max(1, (int) $job->attempts); $job->dispatch_attempts = max(1, (int) $job->dispatch_attempts); $job->dispatch_started_at ??= $now - 1; $job->dispatched_at ??= $now - 1; $job->agent_accepted_at ??= $now - 1; $job->receipt_deadline_at ??= $now + 900; $job->terminal_at = $now; $job->effect_may_have_applied = $effect ? 1 : 0; $job->safe_to_resubmit = $safe ? 1 : 0; $job->effect_resolution = $outcome === 'succeeded' ? 'applied' : ($effect ? 'unknown' : 'not_applied'); $job->lease_token = null; $job->lease_until = 0; $job->available_at = 0; $job->updated_at = $now; $job->save(); }; $setTestState = static function (CloudJob $job, string $state): void { $now = time(); if ($state === 'accepted') { $job->phase = 'accepted'; $job->outcome = null; $job->status = 'queued'; $job->attempts = 0; $job->dispatch_attempts = 0; $job->dispatch_started_at = null; $job->agent_accepted_at = null; $job->receipt_deadline_at = null; $job->terminal_at = null; $job->dispatched_at = null; $job->effect_may_have_applied = null; $job->safe_to_resubmit = null; $job->effect_resolution = 'not_applicable'; } elseif ($state === 'awaiting_receipt') { $job->phase = 'awaiting_receipt'; $job->outcome = null; $job->status = 'running'; $job->attempts = 1; $job->dispatch_attempts = 1; $job->dispatch_started_at = $now - 1; $job->agent_accepted_at = $now - 1; $job->receipt_deadline_at = $now + 900; $job->terminal_at = null; $job->dispatched_at = $now - 1; $job->effect_may_have_applied = null; $job->safe_to_resubmit = null; $job->effect_resolution = 'not_applicable'; } elseif ($state === 'failed_unknown') { $job->phase = 'terminal'; $job->outcome = 'failed'; $job->status = 'failed'; $job->attempts = 1; $job->dispatch_attempts = 1; $job->terminal_at = $now; $job->effect_may_have_applied = 1; $job->safe_to_resubmit = 0; $job->effect_resolution = 'unknown'; } else { throw new RuntimeException('Invalid test task state'); } $job->lease_token = null; $job->lease_until = 0; $job->save(); }; $expect = static function (callable $callable, int $status, string $message) use ($assert): void { try { $callable(); } catch (CloudError $error) { $assert($error->httpStatus === $status, $message . ' Unexpected status: ' . $error->httpStatus); return; } throw new RuntimeException($message); }; $capsule = new Capsule(); $capsule->addConnection([ 'driver' => 'sqlite', 'database' => ':memory:', ], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('user', static function (Blueprint $table): void { $table->increments('id'); $table->string('money')->default('0.00'); $table->integer('enable')->default(1); $table->integer('is_admin')->default(0); }); $schema->create('cloud_instances', static function (Blueprint $table): void { $table->increments('id'); $table->string('uuid'); $table->integer('host_id'); $table->integer('user_id'); $table->string('name'); $table->string('status'); $table->string('service_status'); $table->string('network_status'); $table->integer('expires_at')->nullable(); $table->text('spec_json'); $table->text('sealed_credential')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_instance_firewalls', static function ( Blueprint $table): void { $table->integer('instance_id')->primary(); $table->integer('user_id'); $table->integer('host_id'); $table->boolean('enabled'); $table->integer('desired_revision'); $table->integer('applied_revision'); $table->string('desired_digest')->nullable(); $table->string('applied_digest')->nullable(); $table->string('sync_state'); $table->string('last_operation_id')->nullable(); $table->integer('last_success_at'); $table->integer('last_failure_at'); $table->string('last_error_code')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_instance_firewall_versions', static function ( Blueprint $table): void { $table->increments('id'); $table->integer('instance_id'); $table->integer('revision'); $table->boolean('enabled'); $table->string('policy_digest'); $table->string('request_hash'); $table->string('idempotency_key'); $table->string('state'); $table->integer('job_id')->nullable(); $table->string('operation_id')->nullable(); $table->string('error_code')->nullable(); $table->integer('applied_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); $table->unique(['instance_id', 'revision']); $table->unique(['instance_id', 'idempotency_key']); }); $schema->create('cloud_instance_firewall_rules', static function ( Blueprint $table): void { $table->increments('id'); $table->integer('version_id'); $table->integer('sequence'); $table->string('direction'); $table->string('protocol'); $table->string('source_cidr'); $table->integer('port_from')->nullable(); $table->integer('port_to')->nullable(); $table->string('description'); $table->integer('created_at'); }); $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('operation_id')->unique(); $table->string('request_idempotency_key'); $table->string('request_fingerprint'); $table->integer('parent_job_id')->nullable(); $table->integer('order_id')->nullable(); $table->integer('invoice_id')->nullable(); $table->integer('instance_id')->nullable(); $table->integer('snapshot_id')->nullable(); $table->integer('user_id'); $table->integer('host_id')->nullable(); $table->string('kind'); $table->string('execution_mode'); $table->string('phase'); $table->string('outcome')->nullable(); $table->string('status'); $table->text('sealed_payload'); $table->integer('attempts'); $table->integer('dispatch_attempts'); $table->integer('dispatch_deadline_at'); $table->integer('dispatch_started_at')->nullable(); $table->integer('agent_accepted_at')->nullable(); $table->integer('receipt_deadline_at')->nullable(); $table->integer('terminal_at')->nullable(); $table->boolean('effect_may_have_applied')->nullable(); $table->boolean('safe_to_resubmit')->nullable(); $table->string('effect_resolution'); $table->integer('effect_resolved_at')->nullable(); $table->integer('effect_resolved_by')->nullable(); $table->text('effect_resolution_reason')->nullable(); $table->string('completion_event_id')->nullable(); $table->integer('read_at')->nullable(); $table->integer('dispatched_at')->nullable(); $table->string('lease_token')->nullable(); $table->integer('lease_until')->default(0); $table->integer('available_at'); $table->string('error_code')->nullable(); $table->boolean('finalization_incident')->default(0); $table->string('finalization_error_code')->nullable(); $table->text('result_json')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_agent_completion_events', static function ( Blueprint $table ): void { $table->string('event_id')->primary(); $table->integer('host_id'); $table->string('operation_id'); $table->string('payload_sha256'); $table->string('outcome'); $table->boolean('effect_may_have_applied'); $table->boolean('safe_to_resubmit'); $table->string('error_code')->default(''); $table->text('result_json')->nullable(); $table->string('disposition')->default('received'); $table->integer('received_at'); $table->integer('applied_at')->nullable(); $table->unique(['host_id', 'operation_id']); }); foreach ([10, 20, 30, 40] as $userId) { $user = new User(); $user->id = $userId; $user->money = '0.00'; $user->enable = 1; $user->is_admin = 0; $user->save(); } $newInstance = static function (int $owner, string $source = 'cloud_template'): CloudInstance { $instance = new CloudInstance(); $instance->uuid = sprintf('11111111-1111-4111-8111-%012d', $owner); $instance->host_id = 7; $instance->user_id = $owner; $instance->name = 'vm-firewall-' . $owner; $instance->status = 'active'; $instance->service_status = 'exempt'; $instance->network_status = 'active'; $instance->expires_at = null; $instance->spec_json = json_encode([ 'os_source' => $source, 'public_ip_cidr' => '192.0.2.10/24', ], JSON_THROW_ON_ERROR); $instance->created_at = $instance->updated_at = time(); $instance->save(); return $instance; }; $instance = $newInstance(10); $actor = ['user_id' => 10, 'is_admin' => false, 'is_super_admin' => false, 'can_manage' => false]; $empty = CloudFirewallService::listing($instance, $actor); $assert($empty === [ 'enabled' => false, 'desired_revision' => 0, 'applied_revision' => 0, 'sync_state' => 'disabled', 'last_success_at' => null, 'last_failure_at' => null, 'last_error_code' => null, 'can_mutate' => true, 'rules' => [], ], 'A fresh managed instance did not expose the disabled database-only policy.'); $request = [ 'enabled' => true, 'expected_revision' => 0, 'idempotency_key' => 'fw-request-00000001', // Deliberately unsorted; canonicalization is part of the digest spec. 'rules' => [[ 'protocol' => 'TCP', 'source_cidr' => '203.0.113.55', 'port_from' => 22, 'port_to' => 22, 'description' => 'SSH', ], [ 'protocol' => 'icmp', 'source_cidr' => '0.0.0.123/0', 'port_from' => null, 'port_to' => null, 'description' => 'Ping', ]], ]; $queued = CloudFirewallService::apply($instance, $actor, $request); $job = $queued['job']; $payload = CloudSecrets::decrypt( (string) $job->sealed_payload, 'job-payload' ); $expectedDigest = '46c4d40eef4410edaf998e17981ae91d161348e2fc6572b2bd96c8b5cbf9bfc3'; $expectedAgentDigest = '531f9f937370fc1e3f505eb51ef4f8a157f25ab5b137fbec2bf92d1b7f9e1e2f'; $assert($payload['method'] === 'PUT' && $payload['request_target'] === '/v1/vms/' . $instance->uuid . '/firewall' && $payload['body']['revision'] === 1 && $payload['body']['default_in'] === 'drop' && $payload['body']['default_out'] === 'accept' && $payload['firewall_policy_digest'] === $expectedDigest && $payload['agent_firewall_policy_digest'] === $expectedAgentDigest && $payload['body']['rules'][0] === [ 'direction' => 'ingress', 'action' => 'accept', 'protocol' => 'icmp', 'cidr' => '0.0.0.0/0', ] && $payload['body']['rules'][1]['cidr'] === '203.0.113.55/32', 'Firewall apply lost its typed Agent contract or fixed v1 digest vector.'); $assert($queued['policy']['sync_state'] === 'pending' && $queued['policy']['desired_revision'] === 1 && $queued['policy']['applied_revision'] === 0 && $queued['policy']['can_mutate'] === false, 'Queued desired state was presented as already applied.'); $retried = CloudFirewallService::apply($instance, $actor, $request); $assert((int) $retried['job']->id === (int) $job->id && CloudJob::count() === 1, 'An identical idempotent retry created a second Agent operation.'); $changedReuse = $request; $changedReuse['enabled'] = false; $expect(static fn () => CloudFirewallService::apply( $instance, $actor, $changedReuse ), 409, 'One idempotency key was accepted for two different requests.'); $nextRequest = [ 'enabled' => true, 'expected_revision' => 1, 'idempotency_key' => 'fw-request-00000002', 'rules' => [[ 'protocol' => 'tcp', 'source_cidr' => '198.51.100.0/24', 'port_from' => 443, 'port_to' => 443, 'description' => 'HTTPS', ]], ]; $expect(static fn () => CloudFirewallService::apply( $instance, $actor, $nextRequest ), 409, 'A queued firewall target accepted an unbounded replacement.'); $assert(CloudJob::count() === 1 && CloudInstanceFirewallVersion::count() === 1, 'A rejected queued replacement still amplified durable history.'); $markDispatched($job); $assert(CloudFirewallService::assertJobRunnable($job, $payload), 'The current firewall target was rejected before dispatch.'); CloudFirewallService::markJobRunning($job); CloudFirewallService::finishJobLocked($job, 'succeeded', [ 'revision' => 1, 'policy_digest' => $payload['agent_firewall_policy_digest'], ], $payload, [ 'operation_id' => (string) $job->operation_id, 'status' => 'succeeded', ]); $terminalize($job, 'succeeded'); $replacement = CloudFirewallService::apply( $instance, $actor, $nextRequest ); $replacementJob = $replacement['job']; $replacementPayload = CloudSecrets::decrypt( (string) $replacementJob->sealed_payload, 'job-payload' ); $assert((string) CloudInstanceFirewallVersion::where('revision', 1) ->value('state') === 'applied' && $replacement['policy']['desired_revision'] === 2 && CloudJob::where('status', 'queued')->count() === 1, 'A confirmed firewall target did not allow the next revision.'); $replacementJob->lease_token = str_repeat('a', 32); $replacementJob->lease_until = time() + 60; $replacementJob->save(); $expect(static fn () => CloudFirewallService::apply($instance, $actor, [ 'enabled' => false, 'expected_revision' => 2, 'idempotency_key' => 'fw-request-00000003', 'rules' => [], ]), 409, 'A leased firewall job was replaced under its worker.'); $replacementJob->lease_token = null; $replacementJob->lease_until = 0; $markDispatched($replacementJob); CloudFirewallService::assertJobRunnable($replacementJob, $replacementPayload); CloudFirewallService::markJobRunning($replacementJob); $expect(static fn () => CloudFirewallService::apply($instance, $actor, [ 'enabled' => false, 'expected_revision' => 2, 'idempotency_key' => 'fw-request-00000004', 'rules' => [], ]), 409, 'A running firewall operation accepted a competing target.'); $expect(static fn () => CloudFirewallService::finishJobLocked( $replacementJob, 'succeeded', [ 'revision' => 2, 'policy_digest' => $replacementPayload['agent_firewall_policy_digest'], ], $replacementPayload, [ 'operation_id' => str_repeat('f', 32), 'status' => 'succeeded', ] ), 502, 'A firewall result from another operation was accepted.'); CloudFirewallService::finishJobLocked($replacementJob, 'succeeded', [ 'revision' => 2, 'policy_digest' => $replacementPayload['agent_firewall_policy_digest'], ], $replacementPayload, [ 'operation_id' => (string) $replacementJob->operation_id, 'status' => 'succeeded', ]); $terminalize($replacementJob, 'succeeded'); $applied = CloudFirewallService::listing($instance, $actor); $assert($applied['sync_state'] === 'active' && $applied['desired_revision'] === 2 && $applied['applied_revision'] === 2 && $applied['last_success_at'] !== null && $applied['can_mutate'] === true, 'A confirmed result did not converge desired and applied revisions.'); $disable = CloudFirewallService::apply($instance, $actor, [ 'enabled' => false, 'expected_revision' => 2, 'idempotency_key' => 'fw-request-00000005', 'rules' => $applied['rules'], ]); $disableJob = $disable['job']; $disablePayload = CloudSecrets::decrypt( (string) $disableJob->sealed_payload, 'job-payload' ); $markDispatched($disableJob); CloudFirewallService::markJobUncertain($disableJob); $terminalize($disableJob, 'failed', true, false); $uncertain = CloudFirewallService::listing($instance, $actor); $assert($disableJob->phase === 'terminal' && $disableJob->outcome === 'failed' && $disableJob->effect_resolution === 'unknown' && $uncertain['sync_state'] === 'uncertain' && $uncertain['applied_revision'] === 2 && $uncertain['last_error_code'] === 'firewall_result_uncertain' && $uncertain['can_mutate'] === false, 'An unknown-effect receipt deadline did not end the task and preserve the resource fence.'); // Simulate an explicit read-only inspection proving that the requested // policy is active. The task remains failed; only its resource fence and // catalog projection are resolved. $disableJob->effect_resolution = 'applied'; $disableJob->effect_resolved_at = time(); $disableJob->effect_resolved_by = 99; $disableJob->effect_resolution_reason = 'Read-only host inspection confirmed firewall policy effect.'; $disableJob->save(); CloudFirewallService::finishJobLocked($disableJob, 'succeeded', [ 'revision' => 3, 'policy_digest' => $disablePayload['agent_firewall_policy_digest'], ], $disablePayload, [ 'operation_id' => (string) $disableJob->operation_id, 'status' => 'succeeded', ]); $disabled = CloudFirewallService::listing($instance, $actor); $assert($disabled['sync_state'] === 'disabled' && $disabled['enabled'] === false && $disabled['applied_revision'] === 3 && $disableJob->fresh()->outcome === 'failed', 'Explicit fence resolution rewrote the immutable task outcome or lost the inspected policy.'); $completedJobCount = CloudJob::count(); $completedVersionCount = CloudInstanceFirewallVersion::count(); $samePolicy = CloudFirewallService::apply($instance, $actor, [ 'enabled' => false, 'expected_revision' => 3, 'idempotency_key' => 'fw-request-same-policy-0001', 'rules' => $disabled['rules'], ]); $samePolicyJob = $samePolicy['job']; $samePolicyResult = json_decode( (string) $samePolicyJob->result_json, true, 512, JSON_THROW_ON_ERROR ); $assert($samePolicyJob instanceof CloudJob && $samePolicy['unchanged'] === true && (string) $samePolicyJob->kind === 'firewall_apply' && (string) $samePolicyJob->execution_mode === 'agent' && (string) $samePolicyJob->phase === 'terminal' && (string) $samePolicyJob->outcome === 'succeeded' && (string) $samePolicyJob->status === 'succeeded' && (int) $samePolicyJob->attempts === 0 && (int) $samePolicyJob->dispatch_attempts === 0 && $samePolicyJob->dispatch_started_at === null && $samePolicyJob->dispatched_at === null && $samePolicyJob->agent_accepted_at === null && $samePolicyJob->receipt_deadline_at === null && (int) $samePolicyJob->effect_may_have_applied === 0 && (int) $samePolicyJob->safe_to_resubmit === 0 && (string) $samePolicyJob->effect_resolution === 'applied' && $samePolicyResult === [ 'unchanged' => true, 'policy_digest' => (string) CloudInstanceFirewall::find( (int) $instance->id )->applied_digest, 'revision' => 3, ] && CloudSecrets::decrypt( (string) $samePolicyJob->sealed_payload, 'job-payload' ) === [] && CloudJob::count() === $completedJobCount + 1 && CloudInstanceFirewallVersion::count() === $completedVersionCount, 'An already-converged mutation did not receive an undispatched terminal task.'); $samePolicyReplay = CloudFirewallService::apply($instance, $actor, [ 'enabled' => false, 'expected_revision' => 3, 'idempotency_key' => 'fw-request-same-policy-0001', 'rules' => $disabled['rules'], ]); $assert((int) $samePolicyReplay['job']->id === (int) $samePolicyJob->id && $samePolicyReplay['unchanged'] === true && CloudJob::count() === $completedJobCount + 1 && CloudInstanceFirewallVersion::count() === $completedVersionCount, 'Replaying an already-converged mutation created a second task or policy version.'); $samePolicyHistory = []; $samePolicyStack = HandlerStack::create(new MockHandler([])); $samePolicyStack->push(Middleware::history($samePolicyHistory)); $samePolicyAgent = new CloudAgentClient( '1', 'test-secret-which-is-at-least-32-bytes', new Client(['handler' => $samePolicyStack]), 20.0, 'http://127.0.0.1:8001/internal/agent/request', 'test-gateway-secret-at-least-32-bytes' ); $assert((new CloudWorkerService($samePolicyAgent))->tick() === false && $samePolicyHistory === [] && (string) $samePolicyJob->fresh()->phase === 'terminal' && (int) $samePolicyJob->fresh()->dispatch_attempts === 0, 'An already-converged firewall task entered the worker or reached Agent.'); $expect(static fn () => CloudFirewallService::listing($instance, [ 'user_id' => 99, 'is_admin' => true, 'is_super_admin' => true, 'can_manage' => true, ]), 404, 'A non-owner received the tenant firewall policy.'); $iso = $newInstance(20, 'iso'); $expect(static fn () => CloudFirewallService::listing($iso, [ 'user_id' => 20, 'is_admin' => false, ]), 409, 'An ISO instance received unsupported firewall controls.'); $expect(static fn () => CloudFirewallService::apply($instance, $actor, [ 'enabled' => true, 'expected_revision' => 3, 'idempotency_key' => 'fw-request-invalid-1', 'rules' => [[ 'protocol' => 'icmp', 'source_cidr' => '::/0', 'port_from' => null, 'port_to' => null, 'description' => '', ]], ]), 422, 'An IPv6 rule bypassed the IPv4-only contract.'); // Network-only enforcement can converge around firewall work. Destructive // graceful purge and disk deletion retain the firewall row as a blocker so // they cannot race the same domain/filter resources. $overlap = CloudFirewallService::apply($instance, $actor, [ 'enabled' => true, 'expected_revision' => 3, 'idempotency_key' => 'fw-request-00000006', 'rules' => [[ 'protocol' => 'tcp', 'source_cidr' => '192.0.2.44/32', 'port_from' => 22, 'port_to' => 22, 'description' => 'Deletion overlap', ]], ]); $overlapJob = $overlap['job']; $overlapPayload = CloudSecrets::decrypt( (string) $overlapJob->sealed_payload, 'job-payload' ); $lifecycleConflict = new ReflectionMethod( CloudLifecycleService::class, 'hasConflictingJob' ); $lifecycle = new CloudLifecycleService(); foreach (['accepted', 'awaiting_receipt', 'failed_unknown'] as $firewallJobState) { $setTestState($overlapJob, $firewallJobState); foreach ([ ['active', 'blocking'], ['active', 'blocking_expired'], ['resume_pending', 'restore_pending'], ] as [$serviceState, $networkState]) { $instance->service_status = $serviceState; $instance->network_status = $networkState; $instance->save(); $assert($lifecycleConflict->invoke($lifecycle, $instance) === true, 'Active or unknown-effect firewall work did not serialize lifecycle mutation.'); } $instance->service_status = 'purge_stopping'; $instance->network_status = 'active'; $instance->save(); $assert($lifecycleConflict->invoke($lifecycle, $instance) === true, 'A firewall operation did not block destructive graceful purge.'); } $setTestState($overlapJob, 'failed_unknown'); $expect(static fn () => CloudJobService::enqueue( $instance, $actor, 'delete', 'DELETE', '', ['remove_managed_disks' => true] ), 409, 'An unresolved firewall operation did not block an owner-requested delete.'); $overlapJob->effect_resolution = 'not_applied'; $overlapJob->effect_resolved_at = time(); $overlapJob->effect_resolved_by = 99; $overlapJob->effect_resolution_reason = 'Read-only host inspection confirmed policy was not changed.'; $overlapJob->save(); $manualDelete = CloudJobService::enqueue( $instance, $actor, 'delete', 'DELETE', '', ['remove_managed_disks' => true] ); $manualDeletePayload = CloudSecrets::decrypt( (string) $manualDelete->sealed_payload, 'job-payload' ); $assert((string) $manualDelete->kind === 'delete' && (string) $manualDelete->status === 'queued' && ($manualDeletePayload['request_target'] ?? null) === '/v1/vms/' . $instance->uuid && ($manualDeletePayload['body'] ?? null) === ['remove_managed_disks' => true], 'A resolved firewall operation still blocked an owner-requested delete.'); $terminalize($manualDelete, 'failed', false, true); $instance->status = 'deleting'; $instance->service_status = 'purge_deleting'; $instance->network_status = 'blocked_expired'; $instance->save(); $overlapJob->effect_resolution = 'unknown'; $overlapJob->save(); $expect(static fn () => CloudJobService::enqueueExpiryDelete($instance), 409, 'An unresolved firewall operation did not block irreversible expiry cleanup.'); $overlapJob->effect_resolution = 'not_applied'; $overlapJob->effect_resolved_at = time(); $overlapJob->effect_resolved_by = 99; $overlapJob->save(); $expiryDelete = CloudJobService::enqueueExpiryDelete($instance); $assert((string) $expiryDelete->kind === 'expiry_delete' && (string) $expiryDelete->status === 'queued', 'A resolved firewall operation still blocked irreversible expiry cleanup.'); CloudFirewallService::finishInstanceDeleteLocked( $expiryDelete, 'succeeded' ); $instance->status = 'deleted'; $instance->save(); $terminalize($expiryDelete, 'succeeded'); $assert(CloudFirewallService::assertJobRunnable( $overlapJob, $overlapPayload ) === false, 'A firewall job remained runnable after confirmed instance deletion.'); $overlapJob->refresh(); $deletedPolicy = CloudInstanceFirewall::find((int) $instance->id); $assert((string) $overlapJob->phase === 'terminal' && (string) $overlapJob->outcome === 'failed' && (string) $deletedPolicy->sync_state === 'deleted' && (string) $deletedPolicy->last_operation_id === (string) $expiryDelete->operation_id && (string) CloudInstanceFirewallVersion::where('revision', 4) ->value('state') !== 'applied', 'A terminal firewall failure revived or obscured a deleted policy.'); // Once a mutation crossed the durable dispatch boundary, its absolute // receipt deadline closes it locally. The fallback never polls Agent and // never replays the original firewall mutation. $missingInstance = $newInstance(30); $missingActor = ['user_id' => 30, 'is_admin' => false, 'is_super_admin' => false, 'can_manage' => false]; $missingApply = CloudFirewallService::apply( $missingInstance, $missingActor, [ 'enabled' => true, 'expected_revision' => 0, 'idempotency_key' => 'fw-request-missing-0001', 'rules' => [[ 'protocol' => 'tcp', 'source_cidr' => '192.0.2.30/32', 'port_from' => 22, 'port_to' => 22, 'description' => 'Missing operation', ]], ] ); $missingJob = $missingApply['job']; $markDispatched($missingJob); $missingJob->receipt_deadline_at = time() - 1; $missingJob->lease_token = null; $missingJob->lease_until = 0; $missingJob->save(); $missingHistory = []; $missingStack = HandlerStack::create(new MockHandler([])); $missingStack->push(Middleware::history($missingHistory)); $assert(CloudWorkerService::expireDueTasksForOwner( 30, (int) $missingJob->id, 1 ) === 1, 'The firewall receipt deadline was not claimed locally.'); $missingJob->refresh(); $missingPolicy = CloudFirewallService::listing( $missingInstance, $missingActor ); $assert($missingHistory === [] && (string) $missingJob->phase === 'terminal' && (string) $missingJob->outcome === 'failed' && (string) $missingJob->effect_resolution === 'unknown' && (int) $missingJob->dispatch_attempts === 1 && $missingPolicy['sync_state'] === 'uncertain' && $missingPolicy['can_mutate'] === false, 'A firewall deadline contacted Agent, replayed the mutation, or remained active.'); // A strict Agent validation response carrying the exact not-dispatched // marker is terminal for firewall work as well. The panel-generated // failure remains bound to the original operation so the firewall // projection does not reject it and strand the job as uncertain. $validationInstance = $newInstance(40); $validationActor = ['user_id' => 40, 'is_admin' => false, 'is_super_admin' => false, 'can_manage' => false]; $validationApply = CloudFirewallService::apply( $validationInstance, $validationActor, [ 'enabled' => true, 'expected_revision' => 0, 'idempotency_key' => 'fw-request-validation-0001', 'rules' => [[ 'protocol' => 'tcp', 'source_cidr' => '192.0.2.40/32', 'port_from' => 22, 'port_to' => 22, 'description' => 'Strict validation failure', ]], ] ); $validationJob = $validationApply['job']; $validationHistory = []; $validationStack = HandlerStack::create(new MockHandler([ new Response(422, [], json_encode(['error' => [ 'code' => 'invalid_json', 'message' => 'request contains an unknown field', 'retryable' => false, 'dispatch_state' => 'not_dispatched', ]], JSON_THROW_ON_ERROR)), ])); $validationStack->push(Middleware::history($validationHistory)); $validationAgent = new CloudAgentClient( '1', 'test-secret-which-is-at-least-32-bytes', new Client(['handler' => $validationStack]), 20.0, 'http://127.0.0.1:8001/internal/agent/request', 'test-gateway-secret-at-least-32-bytes' ); (new CloudWorkerService($validationAgent))->tick(); $validationJob->refresh(); $validationPolicy = CloudFirewallService::listing( $validationInstance, $validationActor ); $assert(count($validationHistory) === 1 && (string) $validationJob->status === 'failed' && (string) $validationJob->error_code === 'firewall_apply_failed' && $validationPolicy['sync_state'] === 'failed' && $validationPolicy['can_mutate'] === true && (string) CloudInstanceFirewallVersion::where( 'instance_id', (int) $validationInstance->id )->value('state') === 'failed', 'A proven pre-dispatch firewall rejection was stranded as uncertain.'); $gatewayApply = CloudFirewallService::apply( $validationInstance, $validationActor, [ 'enabled' => false, 'expected_revision' => 1, 'idempotency_key' => 'fw-request-gateway-auth-0001', 'rules' => [], ] ); $gatewayJob = $gatewayApply['job']; $gatewayHistory = []; $gatewayStack = HandlerStack::create(new MockHandler([ new Response(403, [], json_encode(['error' => [ 'code' => 'gateway_forbidden', 'message' => 'Agent gateway request is forbidden', 'retryable' => false, 'dispatch_state' => 'not_dispatched', ]], JSON_THROW_ON_ERROR)), ])); $gatewayStack->push(Middleware::history($gatewayHistory)); $gatewayAgent = new CloudAgentClient( '1', 'test-secret-which-is-at-least-32-bytes', new Client(['handler' => $gatewayStack]), 20.0, 'http://127.0.0.1:8001/internal/agent/request', 'test-gateway-secret-at-least-32-bytes' ); (new CloudWorkerService($gatewayAgent))->tick(); $gatewayJob->refresh(); $gatewayPolicy = CloudFirewallService::listing( $validationInstance, $validationActor ); $assert(count($gatewayHistory) === 1 && (string) $gatewayJob->status === 'failed' && $gatewayPolicy['sync_state'] === 'failed' && $gatewayPolicy['can_mutate'] === true && (string) CloudInstanceFirewallVersion::where( 'instance_id', (int) $validationInstance->id )->where('revision', 2)->value('state') === 'failed', 'A pre-dispatch gateway rejection stranded firewall work as uncertain.'); $migrationInstance = $newInstance(50); $migrationActor = ['user_id' => 50, 'is_admin' => false, 'is_super_admin' => false, 'can_manage' => false]; $migrationApply = CloudFirewallService::apply( $migrationInstance, $migrationActor, [ 'enabled' => true, 'expected_revision' => 0, 'idempotency_key' => 'fw-request-migration-source-0001', 'rules' => [[ 'protocol' => 'tcp', 'source_cidr' => '192.0.2.50/32', 'port_from' => 22, 'port_to' => 22, 'description' => 'Preserved during migration', ]], ] ); $migrationSourceJob = $migrationApply['job']; $migrationSourcePayload = CloudSecrets::decrypt( (string) $migrationSourceJob->sealed_payload, 'job-payload' ); CloudFirewallService::finishJobLocked( $migrationSourceJob, 'succeeded', [ 'revision' => 1, 'policy_digest' => (string) $migrationSourcePayload['body']['policy_digest'], ], $migrationSourcePayload, ['operation_id' => (string) $migrationSourceJob->operation_id] ); CloudFirewallService::assertMaintenanceMigrationReady($migrationInstance); $migrationInstance->host_id = 8; $migrationInstance->save(); $migration = new CloudInstanceMigration(); $migration->id = 99; $migration->migration_key = str_repeat('a', 32); $migration->source_host_id = 7; $migration->target_host_id = 8; $firewallJob = CloudFirewallService:: queueMaintenanceMigrationReapplyLocked( $migrationInstance, $migration, 99 ); $firewallPayload = CloudSecrets::decrypt( (string) $firewallJob->sealed_payload, 'job-payload' ); $migratedPolicy = CloudInstanceFirewall::findOrFail( (int) $migrationInstance->id ); $assert($firewallJob !== null && (int) $firewallJob->host_id === 8 && (string) $firewallJob->kind === 'firewall_apply' && ($firewallPayload['maintenance_migration_id'] ?? null) === 99 && ($firewallPayload['request_target'] ?? null) === '/v1/vms/' . $migrationInstance->uuid . '/firewall' && count($firewallPayload['body']['rules'] ?? []) === 1 && (int) $migratedPolicy->host_id === 8 && (int) $migratedPolicy->desired_revision === 2 && (int) $migratedPolicy->applied_revision === 1 && (string) $migratedPolicy->sync_state === 'pending', 'Maintenance migration did not preserve and requeue the settled user firewall on the target host.'); echo "Cloud firewall desired/applied, idempotency and queue workflow passed.\n"; }