addConnection([ 'driver' => 'sqlite', 'database' => ':memory:', ], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('user', static function (Blueprint $table): void { $table->increments('id'); $table->decimal('money', 12, 2)->default(0); $table->integer('enable')->default(1); $table->integer('is_admin')->default(0); }); $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('operation_id')->unique(); $table->string('request_idempotency_key')->unique(); $table->string('request_fingerprint'); $table->integer('parent_job_id')->nullable(); $table->integer('order_id')->nullable(); $table->integer('invoice_id')->nullable(); $table->integer('instance_id')->nullable(); $table->integer('snapshot_id')->nullable(); $table->integer('vpc_repair_root_job_id')->nullable(); $table->integer('user_id'); $table->integer('host_id')->nullable(); $table->string('kind'); $table->string('execution_mode')->default('agent'); $table->string('phase'); $table->string('outcome')->nullable(); $table->string('status'); $table->text('sealed_payload'); $table->integer('attempts')->default(0); $table->integer('dispatch_attempts')->default(0); $table->integer('dispatch_deadline_at')->nullable(); $table->integer('dispatch_started_at')->nullable(); $table->integer('agent_accepted_at')->nullable(); $table->integer('receipt_deadline_at')->nullable(); $table->integer('terminal_at')->nullable(); $table->boolean('effect_may_have_applied')->nullable(); $table->boolean('safe_to_resubmit')->nullable(); $table->string('effect_resolution')->default('not_applicable'); $table->integer('effect_resolved_at')->nullable(); $table->integer('effect_resolved_by')->nullable(); $table->string('effect_resolution_reason')->default(''); $table->string('completion_event_id')->nullable(); $table->integer('read_at')->nullable(); $table->integer('dispatched_at')->nullable(); $table->string('lease_token')->nullable(); $table->integer('lease_until')->default(0); $table->integer('available_at')->default(0); $table->string('error_code')->nullable(); $table->boolean('finalization_incident')->default(0); $table->string('finalization_error_code')->nullable(); $table->text('result_json')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_instances', static function (Blueprint $table): void { $table->increments('id'); $table->string('uuid'); $table->integer('host_id'); $table->integer('user_id'); $table->integer('order_id')->nullable(); $table->integer('plan_id')->nullable(); $table->integer('offer_id')->nullable(); $table->integer('resource_pool_id')->nullable(); $table->integer('reservation_id')->nullable(); $table->integer('current_period_id')->nullable(); $table->string('name'); $table->string('status'); $table->string('service_status'); $table->string('network_status'); $table->integer('expires_at')->nullable(); $table->decimal('renewal_price', 12, 2)->nullable(); $table->integer('renewal_days')->default(0); $table->integer('renewal_traffic_bytes')->default(0); $table->integer('traffic_quota_bytes')->default(0); $table->integer('traffic_download_bytes')->default(0); $table->integer('traffic_upload_bytes')->default(0); $table->integer('lifetime_download_bytes')->default(0); $table->integer('lifetime_upload_bytes')->default(0); $table->integer('traffic_synced_at')->default(0); $table->integer('suspended_at')->nullable(); $table->integer('network_suspended_at')->nullable(); $table->text('spec_json'); $table->text('sealed_credential')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_host_storage_configurations', static function (Blueprint $table): void { $table->integer('host_id')->primary(); $table->integer('revision')->default(0); foreach (['managed', 'template', 'seed', 'user_iso', 'rescue'] as $role) { $table->text($role . '_directory'); } $table->string('state')->default('applying'); $table->string('operation_id')->nullable()->unique(); $table->integer('job_id')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_agent_completion_events', static function (Blueprint $table): void { $table->string('event_id')->primary(); $table->integer('host_id'); $table->string('operation_id'); $table->string('payload_sha256'); $table->string('outcome'); $table->boolean('effect_may_have_applied'); $table->boolean('safe_to_resubmit'); $table->string('error_code')->default(''); $table->text('result_json')->nullable(); $table->string('disposition')->default('received'); $table->integer('received_at'); $table->integer('applied_at')->nullable(); $table->unique(['host_id', 'operation_id']); }); $schema->create('cloud_templates', static function (Blueprint $table): void { $table->string('id')->primary(); $table->string('catalog_key')->default('test-catalog'); $table->string('version_label')->default('test-version'); $table->boolean('is_default')->default(true); $table->integer('active_job_id')->nullable(); $table->string('state')->default('active'); }); $schema->create('cloud_user_isos', static function (Blueprint $table): void { $table->string('id')->primary(); $table->integer('active_job_id')->nullable(); $table->string('state')->default('active'); }); $schema->create('cloud_template_replicas', static function (Blueprint $table): void { $table->integer('host_id'); $table->string('template_id'); $table->integer('revision')->default(0); $table->string('generation_token')->default(''); $table->string('state')->default('pending'); $table->integer('image_bytes')->default(0); $table->string('sha256')->default(''); $table->string('operation_id')->default(''); $table->integer('job_id')->nullable(); $table->string('error_code')->default(''); $table->integer('created_at')->default(0); $table->integer('updated_at')->default(0); $table->primary(['host_id', 'template_id']); }); $schema->create('cloud_user_iso_replicas', static function (Blueprint $table): void { $table->integer('host_id'); $table->string('user_iso_id'); $table->integer('user_id'); $table->string('state')->default('pending'); $table->integer('size_bytes')->default(0); $table->string('sha256')->default(''); $table->string('operation_id')->default(''); $table->integer('job_id')->nullable(); $table->string('error_code')->default(''); $table->integer('created_at')->default(0); $table->integer('updated_at')->default(0); $table->primary(['host_id', 'user_iso_id']); }); $schema->create('cloud_invoice_payments', static function (Blueprint $table): void { $table->increments('id'); $table->integer('invoice_id'); $table->integer('user_id'); $table->string('provider'); $table->string('provider_order_no'); $table->string('idempotency_key'); $table->decimal('amount', 12, 2)->default(0); $table->string('status'); $table->integer('created_at'); $table->integer('updated_at'); $table->unique(['user_id', 'idempotency_key']); }); Capsule::table('user')->insert([ ['id' => 1, 'money' => 0, 'enable' => 1, 'is_admin' => 0], ['id' => 2, 'money' => 0, 'enable' => 1, 'is_admin' => 0], ]); $now = time(); $insertJob = static function (int $userId, string $operationId, string $phase, int $dispatchAttempts) use ($now): int { return (int) Capsule::table('cloud_jobs')->insertGetId([ 'operation_id' => $operationId, 'request_idempotency_key' => $operationId, 'request_fingerprint' => hash('sha256', $operationId), 'parent_job_id' => null, 'order_id' => null, 'invoice_id' => null, 'instance_id' => null, 'snapshot_id' => null, 'user_id' => $userId, 'host_id' => 7, // An unrecognised kind exercises the generic terminal ledger path // without requiring a business-resource fixture. 'kind' => 'test_mutation', 'execution_mode' => 'agent', 'phase' => $phase, 'outcome' => null, 'status' => $phase === 'accepted' ? 'queued' : 'running', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/test', 'body' => ['operation_id' => $operationId], 'owner_id' => $userId, // Crossed-dispatch failures must retain one encrypted copy // until their immutable Agent receipt resolves the effect. 'credential' => [ 'username' => 'root', 'password' => 'deadline-recovery-secret', ], ], 'job-payload'), 'attempts' => $dispatchAttempts, 'dispatch_attempts' => $dispatchAttempts, 'dispatch_deadline_at' => $now - 20, 'dispatch_started_at' => $dispatchAttempts === 1 ? $now - 30 : null, 'agent_accepted_at' => null, 'receipt_deadline_at' => $dispatchAttempts === 1 ? $now - 10 : null, 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolved_by' => null, 'effect_resolution_reason' => '', 'completion_event_id' => null, 'read_at' => null, 'dispatched_at' => $dispatchAttempts === 1 ? $now - 30 : null, // Simulate a dead daemon whose stale lease extends past deadline. 'lease_token' => str_repeat('f', 32), 'lease_until' => $now + 600, 'available_at' => 0, 'error_code' => null, 'result_json' => null, 'created_at' => $now - 60, 'updated_at' => $now - 30, ]); }; $insertInstance = static function ( int $id, string $uuid, string $name ) use ($now): void { Capsule::table('cloud_instances')->insert([ 'id' => $id, 'uuid' => $uuid, 'host_id' => 7, 'user_id' => 1, 'order_id' => null, 'plan_id' => null, 'offer_id' => null, 'resource_pool_id' => null, 'reservation_id' => null, 'current_period_id' => null, 'name' => $name, 'status' => 'active', 'service_status' => 'exempt', 'network_status' => 'active', 'expires_at' => null, 'renewal_price' => null, 'renewal_days' => 0, 'renewal_traffic_bytes' => 0, 'traffic_quota_bytes' => 0, 'traffic_download_bytes' => 0, 'traffic_upload_bytes' => 0, 'lifetime_download_bytes' => 0, 'lifetime_upload_bytes' => 0, 'traffic_synced_at' => 0, 'suspended_at' => null, 'network_suspended_at' => null, 'spec_json' => json_encode( ['managed' => true], JSON_THROW_ON_ERROR ), 'sealed_credential' => null, 'created_at' => $now - 3600, 'updated_at' => $now - 3600, ]); }; $storagePayload = static function (string $operationId): string { return CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/storage/configuration', 'body' => [ 'managed_directory' => '/home/libvirt/kvm', 'template_directory' => '/home/libvirt/images', 'seed_directory' => '/home/libvirt/seed', 'user_iso_directory' => '/home/libvirt/iso', 'rescue_directory' => '/home/libvirt/rescue-iso', 'operation_id' => $operationId, ], 'owner_id' => 1, 'can_manage' => true, 'storage_revision' => 1, ], 'job-payload'); }; $identityFor = static function ( string $operationId, bool $applied ): array { $job = CloudJob::where('operation_id', $operationId)->firstOrFail(); if (CloudOperationRecoveryIdentity::supports((string) $job->kind)) { return CloudOperationRecoveryIdentity::expected($job, $applied); } return [ 'action' => 'test-mutation', 'resource_id' => 'operation-' . substr($operationId, 0, 16), 'request_fingerprint' => hash('sha256', $operationId), ]; }; $withIdentity = static function (array $receipt) use ( $identityFor ): array { return $receipt + $identityFor( (string) $receipt['operation_id'], ($receipt['status'] ?? null) === 'succeeded' ); }; $storedEvent = static function ( array $receipt, int $receivedAt, string $disposition = 'received' ): array { $canonical = [ 'event_id' => $receipt['id'], 'host_id' => (int) $receipt['agent_id'], 'operation_id' => $receipt['operation_id'], 'action' => $receipt['action'], 'resource_id' => $receipt['resource_id'], 'request_fingerprint' => $receipt['request_fingerprint'], 'status' => $receipt['status'], 'effect_may_have_applied' => $receipt['effect_may_have_applied'], 'safe_to_resubmit' => $receipt['safe_to_resubmit'], 'error_code' => $receipt['error_code'], 'result' => $receipt['result'], ]; return [ 'event_id' => $canonical['event_id'], 'host_id' => $canonical['host_id'], 'operation_id' => $canonical['operation_id'], 'payload_sha256' => hash('sha256', json_encode( $canonical, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE )), 'outcome' => $canonical['status'], 'effect_may_have_applied' => $canonical['effect_may_have_applied'] ? 1 : 0, 'safe_to_resubmit' => $canonical['safe_to_resubmit'] ? 1 : 0, 'error_code' => $canonical['error_code'], 'result_json' => json_encode([ 'schema' => 'virt_cloud_completion_event', 'version' => 1, 'identity' => [ 'action' => $canonical['action'], 'resource_id' => $canonical['resource_id'], 'request_fingerprint' => $canonical['request_fingerprint'], ], 'result' => $canonical['result'], ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE), 'disposition' => $disposition, 'received_at' => $receivedAt, 'applied_at' => null, ]; }; $receiptOperation = str_repeat('a', 32); $receiptJobId = $insertJob(1, $receiptOperation, 'dispatching', 1); $receiptIdentity = [ 'action' => 'test-mutation', 'resource_id' => 'test-resource', 'request_fingerprint' => str_repeat('1', 64), ]; $receipt = [ 'event_id' => str_repeat('e', 32), 'host_id' => 7, 'operation_id' => $receiptOperation, ] + $receiptIdentity + [ 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => ['confirmed' => true], ]; Capsule::table('cloud_agent_completion_events')->insert([ 'event_id' => $receipt['event_id'], 'host_id' => 7, 'operation_id' => $receiptOperation, 'payload_sha256' => hash('sha256', json_encode( $receipt, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE )), 'outcome' => 'succeeded', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'error_code' => '', 'result_json' => json_encode([ 'schema' => 'virt_cloud_completion_event', 'version' => 1, 'identity' => $receiptIdentity, 'result' => $receipt['result'], ], JSON_THROW_ON_ERROR), 'disposition' => 'received', 'received_at' => $now - 15, 'applied_at' => null, ]); $otherOperation = str_repeat('b', 32); $otherJobId = $insertJob(2, $otherOperation, 'dispatching', 1); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $receiptJobId, 1 ) === 1, 'The owner-scoped fallback did not claim the due task.'); $receiptJob = CloudJob::find($receiptJobId); $otherJob = CloudJob::find($otherJobId); $event = Capsule::table('cloud_agent_completion_events') ->where('event_id', str_repeat('e', 32))->first(); $assert($receiptJob !== null && (string) $receiptJob->phase === 'terminal' && (string) $receiptJob->outcome === 'succeeded' && (int) $receiptJob->dispatch_attempts === 1 && (string) $event->disposition === 'applied', 'A receipt saved during dispatch was replaced by deadline failure.'); $assert($otherJob !== null && (string) $otherJob->phase === 'dispatching', 'Owner-scoped expiry mutated another user\'s task.'); $failureOperation = str_repeat('c', 32); $failureJobId = $insertJob(1, $failureOperation, 'dispatching', 1); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $failureJobId, 1 ) === 1, 'The receipt-less due task was not closed.'); $failed = CloudJob::find($failureJobId); $failedRecoveryPayload = CloudSecrets::decrypt( (string) $failed->sealed_payload, 'job-payload' ); $assert($failed !== null && (string) $failed->phase === 'terminal' && (string) $failed->outcome === 'failed' && (bool) $failed->effect_may_have_applied && !(bool) $failed->safe_to_resubmit && !array_key_exists('credential', $failedRecoveryPayload), 'An unsupported unknown-effect kind retained secret recovery escrow.'); // dispatch_attempts is a durable 0/1 fence, not a retry counter. Values // outside that state machine must fail closed: neither deadline handling // nor its last-resort finalizer may reinterpret them as "not dispatched". $invalidLedgerUuid = '00000000-0000-4000-8000-000000000301'; $insertInstance(301, $invalidLedgerUuid, 'invalid-ledger-settings'); $invalidLedgerOperation = str_repeat('cd', 16); $invalidLedgerJobId = $insertJob( 1, $invalidLedgerOperation, 'dispatching', -1 ); $invalidLedgerPayload = CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/vms/' . $invalidLedgerUuid . '/settings', 'body' => [ 'hostname' => 'invalid-ledger.example', 'confirm_restart' => true, ], 'owner_id' => 1, 'can_manage' => false, 'settings_fields' => ['hostname'], ], 'job-payload'); Capsule::table('cloud_jobs')->where('id', $invalidLedgerJobId)->update([ 'kind' => 'instance_settings', 'instance_id' => 301, 'sealed_payload' => $invalidLedgerPayload, 'dispatch_started_at' => $now - 30, 'dispatched_at' => $now - 30, 'receipt_deadline_at' => $now - 10, ]); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $invalidLedgerJobId, 1 ) === 1, 'The invalid dispatch-ledger deadline was not claimed.'); $invalidLedgerJob = CloudJob::findOrFail($invalidLedgerJobId); $assert((string) $invalidLedgerJob->phase === 'terminal' && (string) $invalidLedgerJob->outcome === 'failed' && (int) $invalidLedgerJob->dispatch_attempts === -1 && (bool) $invalidLedgerJob->effect_may_have_applied && !(bool) $invalidLedgerJob->safe_to_resubmit && (string) $invalidLedgerJob->effect_resolution === 'unknown' && (bool) $invalidLedgerJob->finalization_incident && (string) $invalidLedgerJob->finalization_error_code === 'recovery_dispatch_ledger_conflict' && (string) $invalidLedgerJob->sealed_payload === $invalidLedgerPayload, 'A negative dispatch ledger was treated as a safe undispatched failure.'); // Exercise the same invariant through forceTerminalFailure(): the linked // instance is intentionally absent so the ordinary policy finalizer fails // and the last-resort path owns the durable incident write. $forcedLedgerOperation = str_repeat('ce', 16); $forcedLedgerJobId = $insertJob( 1, $forcedLedgerOperation, 'dispatching', 2 ); $forcedLedgerPayload = CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/vms/' . '00000000-0000-4000-8000-000000009999/suspend', 'body' => [], 'owner_id' => 1, 'can_manage' => false, ], 'job-payload'); Capsule::table('cloud_jobs')->where('id', $forcedLedgerJobId)->update([ 'kind' => 'service_suspend', 'instance_id' => 9999, 'sealed_payload' => $forcedLedgerPayload, 'dispatch_started_at' => $now - 30, 'dispatched_at' => $now - 30, 'receipt_deadline_at' => $now - 10, ]); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $forcedLedgerJobId, 1 ) === 1, 'The forced invalid dispatch-ledger deadline was not claimed.'); $forcedLedgerJob = CloudJob::findOrFail($forcedLedgerJobId); $assert((string) $forcedLedgerJob->phase === 'terminal' && (string) $forcedLedgerJob->outcome === 'failed' && (int) $forcedLedgerJob->dispatch_attempts === 2 && (bool) $forcedLedgerJob->effect_may_have_applied && !(bool) $forcedLedgerJob->safe_to_resubmit && (string) $forcedLedgerJob->effect_resolution === 'unknown' && (bool) $forcedLedgerJob->finalization_incident && (string) $forcedLedgerJob->finalization_error_code === 'recovery_dispatch_ledger_conflict' && (string) $forcedLedgerJob->sealed_payload === $forcedLedgerPayload, 'The forced finalizer concealed or unlocked an invalid dispatch ledger.'); // Deploying the shorter storage window must also bound tasks which already // persisted the old 960-second receipt deadline. No data rewrite is needed: // the effective deadline is the earlier of the two absolute boundaries. $legacyStorageOperation = str_repeat('3', 32); $legacyStorageJobId = $insertJob( 1, $legacyStorageOperation, 'awaiting_receipt', 1 ); Capsule::table('cloud_jobs')->where('id', $legacyStorageJobId)->update([ 'kind' => 'host_storage_configure', 'sealed_payload' => $storagePayload($legacyStorageOperation), 'dispatched_at' => $now - 181, 'dispatch_started_at' => $now - 181, 'receipt_deadline_at' => $now + 779, ]); Capsule::table('cloud_host_storage_configurations')->insert([ 'host_id' => 7, 'revision' => 1, 'managed_directory' => '/home/libvirt/kvm', 'template_directory' => '/home/libvirt/images', 'seed_directory' => '/home/libvirt/seed', 'user_iso_directory' => '/home/libvirt/iso', 'rescue_directory' => '/home/libvirt/rescue-iso', 'state' => 'applying', 'operation_id' => $legacyStorageOperation, 'job_id' => $legacyStorageJobId, 'created_at' => $now - 200, 'updated_at' => $now - 181, ]); $legacyStorage = CloudJob::find($legacyStorageJobId); $assert(CloudJobService::effectiveReceiptDeadline($legacyStorage) === $now - 1 && CloudJobService::receiptDeadlineReached($legacyStorage, $now), 'The persisted storage deadline was not bounded by dispatch plus 180 seconds.'); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $legacyStorageJobId, 1 ) === 1, 'An old 960-second storage task remained active after 180 seconds.'); $legacyStorage = CloudJob::find($legacyStorageJobId); $legacyStorageRow = Capsule::table( 'cloud_host_storage_configurations' )->where('host_id', 7)->first(); $assert((string) $legacyStorage->phase === 'terminal' && (string) $legacyStorage->outcome === 'failed' && (string) $legacyStorageRow->state === 'failed', 'Storage deadline fallback did not close the task and its projection.'); // Receipt classification must use exactly the same effective boundary as // the expiry query, otherwise a late success could revive a timed-out task. $lateStorageOperation = str_repeat('2', 32); $lateStorageJobId = $insertJob( 1, $lateStorageOperation, 'awaiting_receipt', 1 ); Capsule::table('cloud_jobs')->where('id', $lateStorageJobId)->update([ 'kind' => 'host_storage_configure', 'sealed_payload' => $storagePayload($lateStorageOperation), 'dispatched_at' => $now - 181, 'dispatch_started_at' => $now - 181, 'receipt_deadline_at' => $now + 779, ]); Capsule::table('cloud_host_storage_configurations') ->where('host_id', 7)->update([ 'state' => 'applying', 'operation_id' => $lateStorageOperation, 'job_id' => $lateStorageJobId, 'updated_at' => $now, ]); $lateStorageEventId = str_repeat('4', 32); $lateStorageReceipt = $withIdentity([ 'type' => 'completion', 'id' => $lateStorageEventId, 'agent_id' => '7', 'operation_id' => $lateStorageOperation, 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => ['configured' => true], ]); Capsule::table('cloud_agent_completion_events')->insert( $storedEvent($lateStorageReceipt, $now - 1) ); $lateStorage = (new CloudWorkerService())->settleCompletionReceipt( $lateStorageJobId, $lateStorageEventId ); $lateStorageEvent = Capsule::table('cloud_agent_completion_events') ->where('event_id', $lateStorageEventId)->first(); $assert((string) $lateStorage->phase === 'terminal' && (string) $lateStorage->outcome === 'failed' && (string) $lateStorageEvent->disposition === 'late', 'A storage receipt at the effective deadline was not classified as late.'); $acceptedStorageOperation = str_repeat('1', 32); $acceptedStorageJobId = $insertJob( 1, $acceptedStorageOperation, 'accepted', 0 ); Capsule::table('cloud_jobs')->where('id', $acceptedStorageJobId)->update([ 'kind' => 'host_storage_configure', 'sealed_payload' => $storagePayload($acceptedStorageOperation), 'created_at' => $now - 61, 'dispatch_deadline_at' => $now + 239, ]); Capsule::table('cloud_host_storage_configurations') ->where('host_id', 7)->update([ 'state' => 'applying', 'operation_id' => $acceptedStorageOperation, 'job_id' => $acceptedStorageJobId, 'updated_at' => $now, ]); $acceptedStorage = CloudJob::find($acceptedStorageJobId); $assert(CloudJobService::effectiveDispatchDeadline($acceptedStorage) === $now - 1 && CloudJobService::activeDeadlineReached($acceptedStorage, $now), 'The persisted storage dispatch deadline was not bounded to 60 seconds.'); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $acceptedStorageJobId, 1 ) === 1, 'An old 300-second accepted storage task remained active.'); $acceptedStorage = CloudJob::find($acceptedStorageJobId); $assert((string) $acceptedStorage->phase === 'terminal' && (string) $acceptedStorage->outcome === 'failed' && !(bool) $acceptedStorage->effect_may_have_applied && (bool) $acceptedStorage->safe_to_resubmit, 'An undispatched storage deadline did not become a safe failure.'); // The normal worker tick must run local deadline convergence before it can // claim an old storage task. Its injected Agent transport is a tripwire: // a request here would prove the former 300-second deadline was used. $tickStorageOperation = '0123456789abcdef0123456789abcdef'; $tickStorageJobId = $insertJob( 1, $tickStorageOperation, 'accepted', 0 ); Capsule::table('cloud_jobs')->where('id', $tickStorageJobId)->update([ 'kind' => 'host_storage_configure', 'sealed_payload' => $storagePayload($tickStorageOperation), 'created_at' => time() - 61, 'dispatch_deadline_at' => time() + 239, ]); Capsule::table('cloud_host_storage_configurations') ->where('host_id', 7)->update([ 'state' => 'applying', 'operation_id' => $tickStorageOperation, 'job_id' => $tickStorageJobId, 'updated_at' => time(), ]); $agentCalls = 0; $transport = new Client(['handler' => static function () use ( &$agentCalls ) { $agentCalls++; throw new RuntimeException('Expired storage task reached Agent'); }]); $tripwireAgent = new CloudAgentClient( '7', str_repeat('a', 32), $transport, 120.0, 'http://127.0.0.1:8001/internal/agent/request', str_repeat('b', 32) ); $assert((new CloudWorkerService($tripwireAgent))->tick() === true, 'Worker tick did not converge the expired storage task.'); $tickStorage = CloudJob::find($tickStorageJobId); $assert($agentCalls === 0 && (string) $tickStorage->phase === 'terminal' && (string) $tickStorage->outcome === 'failed' && (int) $tickStorage->dispatch_attempts === 0, 'Worker dispatched an old storage task after its effective 60-second boundary.'); $incidentOperation = str_repeat('5', 32); $incidentJobId = $insertJob( 1, $incidentOperation, 'awaiting_receipt', 1 ); Capsule::table('cloud_jobs')->where('id', $incidentJobId)->update([ 'sealed_payload' => 'deadline-test:missing-payload', ]); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $incidentJobId, 1 ) === 1, 'The finalization-incident fixture was not claimed.'); $incident = CloudJob::find($incidentJobId); $assert($incident !== null && (string) $incident->phase === 'terminal' && (string) $incident->outcome === 'failed' && (string) $incident->status === 'failed' && (bool) $incident->effect_may_have_applied && !(bool) $incident->safe_to_resubmit && (string) $incident->effect_resolution === 'unknown' && (string) $incident->error_code === 'local_finalization_failed' && (bool) $incident->finalization_incident && (string) $incident->finalization_error_code === 'recovery_payload_unavailable' && (string) $incident->sealed_payload === '', 'A local finalizer crash left the task active, leaked its payload, or concealed the incident.'); $redactedIncidentOperation = '8123456789abcdef8123456789abcdef'; $redactedIncidentJobId = $insertJob( 1, $redactedIncidentOperation, 'awaiting_receipt', 1 ); Capsule::table('cloud_jobs')->where( 'id', $redactedIncidentJobId )->update(['kind' => 'network_change']); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $redactedIncidentJobId, 1 ) === 1, 'The redacted finalization-incident fixture was not claimed.'); $redactedIncident = CloudJob::find($redactedIncidentJobId); $redactedIncidentPayload = CloudSecrets::decrypt( (string) $redactedIncident->sealed_payload, 'job-payload' ); $assert((bool) $redactedIncident->finalization_incident && (string) $redactedIncident->effect_resolution === 'unknown' && array_key_exists('credential', $redactedIncidentPayload), 'A recoverable finalization incident lost its encrypted recovery escrow.'); Capsule::table('cloud_jobs')->where( 'id', $redactedIncidentJobId )->update(['available_at' => time() + 3600]); $unreadableEnvelope = 'deadline-test:missing-policy-payload'; $unreadableIncidentJobId = $insertJob( 1, '823456789abcdef0823456789abcdef0', 'awaiting_receipt', 1 ); Capsule::table('cloud_jobs')->where( 'id', $unreadableIncidentJobId )->update([ 'kind' => 'network_change', 'sealed_payload' => $unreadableEnvelope, ]); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $unreadableIncidentJobId, 1 ) === 1, 'The unreadable recovery escrow fixture was not claimed.'); $unreadableIncident = CloudJob::findOrFail($unreadableIncidentJobId); $assert((bool) $unreadableIncident->finalization_incident && (string) $unreadableIncident->finalization_error_code === 'recovery_payload_unavailable' && (string) $unreadableIncident->sealed_payload === $unreadableEnvelope && (int) $unreadableIncident->available_at >= time() + 290, 'An unreadable recovery escrow was erased or left the retry queue.'); Capsule::table('cloud_jobs')->where( 'id', $unreadableIncidentJobId )->update(['available_at' => time() + 3600]); $lateOperation = str_repeat('7', 32); $lateJobId = $insertJob(1, $lateOperation, 'awaiting_receipt', 1); $lateEventId = str_repeat('6', 32); $lateReceipt = $withIdentity([ 'type' => 'completion', 'id' => $lateEventId, 'agent_id' => '7', 'operation_id' => $lateOperation, 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => ['confirmed' => true], ]); Capsule::table('cloud_agent_completion_events')->insert($storedEvent( $lateReceipt, // Equal to or later than the absolute boundary is audit-only. $now - 9 )); $lateJob = (new CloudWorkerService())->settleCompletionReceipt( $lateJobId, $lateEventId ); $lateEvent = Capsule::table('cloud_agent_completion_events') ->where('event_id', $lateEventId)->first(); $assert((string) $lateJob->phase === 'terminal' && (string) $lateJob->outcome === 'failed' && (string) $lateJob->error_code === 'operation_failed' && (bool) $lateJob->effect_may_have_applied && !(bool) $lateJob->safe_to_resubmit && !(bool) $lateJob->finalization_incident && (string) $lateEvent->disposition === 'late', 'A receipt persisted after the deadline revived or succeeded the task.'); // completion_event_id is reserved before the business finalizer commits. // If that finalizer fails, the matching on-time receipt must stay received // so the settings recovery worker can finish PHP without Agent replay. $onTimeRecoveryInstanceId = 90; $onTimeRecoveryUuid = '00000000-1111-4222-8333-444444444444'; $insertInstance( $onTimeRecoveryInstanceId, $onTimeRecoveryUuid, 'on-time-finalizer-recovery-vm' ); Capsule::table('cloud_instances')->where( 'id', $onTimeRecoveryInstanceId )->update(['spec_json' => '{invalid-json']); $onTimeRecoveryOperation = '0bcdef0123456789abcdef0123456789'; $onTimeRecoveryJobId = $insertJob( 1, $onTimeRecoveryOperation, 'awaiting_receipt', 1 ); Capsule::table('cloud_jobs')->where( 'id', $onTimeRecoveryJobId )->update([ 'instance_id' => $onTimeRecoveryInstanceId, 'kind' => 'instance_settings', 'receipt_deadline_at' => time() + 60, 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/vms/' . $onTimeRecoveryUuid . '/settings', 'body' => [ 'hostname' => 'on-time-recovered.example', 'confirm_restart' => true, ], 'owner_id' => 1, 'settings_fields' => ['hostname'], 'credential' => null, ], 'job-payload'), ]); $onTimeRecoveryReceipt = $withIdentity([ 'type' => 'completion', 'id' => 'settings-on-time-finalizer-0001', 'agent_id' => '7', 'operation_id' => $onTimeRecoveryOperation, 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => [ 'settings_changed' => true, 'hostname' => 'on-time-recovered.example', ], ]); $assert(CloudOperationCompletionService::accept( $onTimeRecoveryReceipt ) === ['accepted' => true, 'disposition' => 'received'], 'A failed on-time finalizer incorrectly acknowledged its receipt.'); $onTimeFailed = CloudJob::find($onTimeRecoveryJobId); $onTimeEvent = Capsule::table('cloud_agent_completion_events') ->where('event_id', $onTimeRecoveryReceipt['id'])->first(); $assert((string) $onTimeFailed->phase === 'terminal' && (string) $onTimeFailed->outcome === 'failed' && (string) $onTimeFailed->effect_resolution === 'unknown' && (string) $onTimeFailed->completion_event_id === $onTimeRecoveryReceipt['id'] && (string) $onTimeEvent->disposition === 'received' && CloudJobService::blockingQueryForInstance( $onTimeRecoveryInstanceId )->exists(), 'A failed on-time finalizer lost its recovery receipt or blocker.'); Capsule::table('cloud_instances')->where( 'id', $onTimeRecoveryInstanceId )->update([ 'spec_json' => json_encode( ['managed' => true], JSON_THROW_ON_ERROR ), ]); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The recovery worker did not resume the on-time receipt.'); $onTimeRecovered = CloudJob::find($onTimeRecoveryJobId); $onTimeEvent = Capsule::table('cloud_agent_completion_events') ->where('event_id', $onTimeRecoveryReceipt['id'])->first(); $onTimeSpec = json_decode( (string) Capsule::table('cloud_instances') ->where('id', $onTimeRecoveryInstanceId) ->value('spec_json'), true, flags: JSON_THROW_ON_ERROR ); $assert((string) $onTimeRecovered->outcome === 'failed' && (string) $onTimeRecovered->status === 'failed' && (string) $onTimeRecovered->effect_resolution === 'applied' && (string) $onTimeRecovered->completion_event_id === $onTimeRecoveryReceipt['id'] && (string) $onTimeEvent->disposition === 'applied' && ($onTimeSpec['guest_hostname'] ?? null) === 'on-time-recovered.example' && !CloudJobService::blockingQueryForInstance( $onTimeRecoveryInstanceId )->exists(), 'The on-time receipt did not finish PHP-only recovery exactly once.'); // A password reset can finish on the Agent after PHP has conservatively // failed its task. The immutable late receipt must converge the instance // ledger and release the blocker without rewriting the failed task. $settingsInstanceId = 91; $settingsUuid = '11111111-2222-4333-8444-555555555555'; $insertInstance($settingsInstanceId, $settingsUuid, 'late-settings-vm'); $settingsOperation = 'abcdef0123456789abcdef0123456789'; $settingsJobId = $insertJob( 1, $settingsOperation, 'awaiting_receipt', 1 ); Capsule::table('cloud_jobs')->where('id', $settingsJobId)->update([ 'instance_id' => $settingsInstanceId, 'kind' => 'instance_settings', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/vms/' . $settingsUuid . '/settings', 'body' => [ 'root_password' => 'New-late-password9!', 'confirm_restart' => true, ], 'owner_id' => 1, 'settings_fields' => ['password'], 'credential' => [ 'password' => 'New-late-password9!', 'login_name' => 'root', 'auth_mode' => 'password', ], ], 'job-payload'), ]); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $settingsJobId, 1 ) === 1, 'The settings fixture did not cross its receipt deadline.'); $timedOutSettings = CloudJob::find($settingsJobId); $originalTerminal = [ 'operation_id' => (string) $timedOutSettings->operation_id, 'kind' => (string) $timedOutSettings->kind, 'phase' => (string) $timedOutSettings->phase, 'outcome' => (string) $timedOutSettings->outcome, 'status' => (string) $timedOutSettings->status, 'dispatch_attempts' => (int) $timedOutSettings->dispatch_attempts, 'effect_may_have_applied' => (bool) $timedOutSettings->effect_may_have_applied, 'safe_to_resubmit' => (bool) $timedOutSettings->safe_to_resubmit, 'error_code' => (string) $timedOutSettings->error_code, 'result_json' => $timedOutSettings->result_json, 'terminal_at' => (int) $timedOutSettings->terminal_at, 'completion_event_id' => $timedOutSettings->completion_event_id, 'read_at' => $timedOutSettings->read_at, ]; $settingsReceipt = $withIdentity([ 'type' => 'completion', 'id' => 'settings-recovery-event-000001', 'agent_id' => '7', 'operation_id' => $settingsOperation, 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => [ 'settings_changed' => true, ], ]); $settingsAccepted = CloudOperationCompletionService::accept( $settingsReceipt ); $recoveredSettings = CloudJob::find($settingsJobId); $recoveredInstance = Capsule::table('cloud_instances') ->where('id', $settingsInstanceId)->first(); $recoveredSpec = json_decode( (string) $recoveredInstance->spec_json, true, flags: JSON_THROW_ON_ERROR ); $recoveredCredential = CloudSecrets::decrypt( (string) $recoveredInstance->sealed_credential, 'instance-credential' ); $redactedRecoveryPayload = CloudSecrets::decrypt( (string) $recoveredSettings->sealed_payload, 'job-payload' ); $settingsEvent = Capsule::table('cloud_agent_completion_events') ->where('event_id', $settingsReceipt['id'])->first(); $assert($settingsAccepted === [ 'accepted' => true, 'disposition' => 'applied', ] && (string) $settingsEvent->disposition === 'applied' && (string) $recoveredSettings->effect_resolution === 'applied' && (int) $recoveredSettings->effect_resolved_at > 0 && $originalTerminal === [ 'operation_id' => (string) $recoveredSettings->operation_id, 'kind' => (string) $recoveredSettings->kind, 'phase' => (string) $recoveredSettings->phase, 'outcome' => (string) $recoveredSettings->outcome, 'status' => (string) $recoveredSettings->status, 'dispatch_attempts' => (int) $recoveredSettings->dispatch_attempts, 'effect_may_have_applied' => (bool) $recoveredSettings->effect_may_have_applied, 'safe_to_resubmit' => (bool) $recoveredSettings->safe_to_resubmit, 'error_code' => (string) $recoveredSettings->error_code, 'result_json' => $recoveredSettings->result_json, 'terminal_at' => (int) $recoveredSettings->terminal_at, 'completion_event_id' => $recoveredSettings->completion_event_id, 'read_at' => $recoveredSettings->read_at, ] && $recoveredSpec === ['managed' => true] && ($recoveredCredential['password'] ?? null) === 'New-late-password9!' && !array_key_exists('credential', $redactedRecoveryPayload) && !CloudJobService::blockingQueryForInstance( $settingsInstanceId )->exists(), 'A matching late settings receipt did not safely converge the ledger.'); $recoveredUpdatedAt = (int) $recoveredSettings->updated_at; $recoveredCredentialEnvelope = (string) $recoveredInstance ->sealed_credential; $assert( CloudOperationCompletionService::accept($settingsReceipt) === [ 'accepted' => true, 'disposition' => 'applied', ] && (int) CloudJob::find($settingsJobId)->updated_at === $recoveredUpdatedAt && (string) Capsule::table('cloud_instances') ->where('id', $settingsInstanceId) ->value('sealed_credential') === $recoveredCredentialEnvelope, 'An identical late receipt was not idempotent after reconciliation.' ); $forgedSettingsReceipt = $settingsReceipt; $forgedSettingsReceipt['id'] = 'settings-recovery-event-forged01'; $forgedConflict = false; try { CloudOperationCompletionService::accept($forgedSettingsReceipt); } catch (\App\Services\Cloud\CloudError $error) { $forgedConflict = $error->httpStatus === 409; } $unknownOperationReceipt = $settingsReceipt; $unknownOperationReceipt['id'] = 'settings-recovery-event-unknown1'; $unknownOperationReceipt['operation_id'] = 'fedcba9876543210fedcba9876543210'; $assert($forgedConflict && CloudOperationCompletionService::accept( $unknownOperationReceipt ) === ['accepted' => true, 'disposition' => 'rejected'], 'A different event or operation was accepted as recovery evidence.'); // An exact terminal Agent failure which explicitly says no effect was // possible can also settle a timed-out settings task. Its transport error // vocabulary is normalized to the settings no-effect finalizer; it never // authorizes replay even when safe_to_resubmit=true. $notAppliedInstanceId = 93; $notAppliedUuid = 'bbbbbbbb-cccc-4ddd-8eee-ffffffffffff'; $insertInstance( $notAppliedInstanceId, $notAppliedUuid, 'not-applied-settings-vm' ); $notAppliedOperationId = '2234567890abcdef1234567890abcdef'; $notAppliedJobId = $insertJob( 1, $notAppliedOperationId, 'awaiting_receipt', 1 ); Capsule::table('cloud_jobs')->where('id', $notAppliedJobId)->update([ 'instance_id' => $notAppliedInstanceId, 'kind' => 'instance_settings', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/vms/' . $notAppliedUuid . '/settings', 'body' => [ 'hostname' => 'not-applied.example', 'confirm_restart' => true, ], 'owner_id' => 1, 'settings_fields' => ['hostname'], 'credential' => null, ], 'job-payload'), ]); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $notAppliedJobId, 1 ) === 1, 'The late no-effect fixture did not reach terminal failure.'); Capsule::table('cloud_jobs')->where('id', $notAppliedJobId)->update([ 'effect_resolution_reason' => 'automatic_recovery:v1:' . (time() - 21600) . ':12:pending', 'finalization_incident' => 1, 'finalization_error_code' => 'recovery_limit_reached', ]); $notAppliedTerminal = CloudJob::find($notAppliedJobId); $notAppliedAudit = [ 'phase' => (string) $notAppliedTerminal->phase, 'outcome' => (string) $notAppliedTerminal->outcome, 'status' => (string) $notAppliedTerminal->status, 'error_code' => (string) $notAppliedTerminal->error_code, 'result_json' => $notAppliedTerminal->result_json, 'terminal_at' => (int) $notAppliedTerminal->terminal_at, 'completion_event_id' => $notAppliedTerminal->completion_event_id, ]; $notAppliedReceipt = $withIdentity([ 'type' => 'completion', 'id' => 'settings-no-effect-event-000001', 'agent_id' => '7', 'operation_id' => $notAppliedOperationId, 'status' => 'failed', 'effect_may_have_applied' => false, 'safe_to_resubmit' => true, 'error_code' => 'guest_update_failed', 'result' => ['agent_detail' => 'bounded'], ]); $assert(CloudOperationCompletionService::accept( $notAppliedReceipt ) === ['accepted' => true, 'disposition' => 'applied'], 'An exact late no-effect settings receipt was not applied.'); $notAppliedRecovered = CloudJob::find($notAppliedJobId); $notAppliedPayload = CloudSecrets::decrypt( (string) $notAppliedRecovered->sealed_payload, 'job-payload' ); $assert((string) $notAppliedRecovered->effect_resolution === 'not_applied' && $notAppliedAudit === [ 'phase' => (string) $notAppliedRecovered->phase, 'outcome' => (string) $notAppliedRecovered->outcome, 'status' => (string) $notAppliedRecovered->status, 'error_code' => (string) $notAppliedRecovered->error_code, 'result_json' => $notAppliedRecovered->result_json, 'terminal_at' => (int) $notAppliedRecovered->terminal_at, 'completion_event_id' => $notAppliedRecovered->completion_event_id, ] && !array_key_exists('credential', $notAppliedPayload) && !(bool) $notAppliedRecovered->finalization_incident && $notAppliedRecovered->finalization_error_code === null && !CloudJobService::blockingQueryForInstance( $notAppliedInstanceId )->exists(), 'Late no-effect recovery rewrote task audit fields or retained its lock.'); // Receipt persistence and late-effect settlement are deliberately // separate transactions. If PHP dies between them, the recovery-only // worker must discover the exact received event through the job's indexed // host+operation key, even when inspection retries were already exhausted. $persistedEventInstanceId = 96; $persistedEventUuid = 'eeeeeeee-ffff-4000-8111-000000000003'; $insertInstance( $persistedEventInstanceId, $persistedEventUuid, 'persisted-event-settings-vm' ); $persistedEventOperationId = '5234567890abcdef1234567890abcdef'; $persistedEventJobId = $insertJob( 1, $persistedEventOperationId, 'awaiting_receipt', 1 ); Capsule::table('cloud_jobs')->where('id', $persistedEventJobId)->update([ 'instance_id' => $persistedEventInstanceId, 'kind' => 'instance_settings', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/vms/' . $persistedEventUuid . '/settings', 'body' => [ 'hostname' => 'persisted-event.example', 'confirm_restart' => true, ], 'owner_id' => 1, 'settings_fields' => ['hostname'], 'credential' => null, ], 'job-payload'), ]); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $persistedEventJobId, 1 ) === 1, 'The persisted-event crash fixture did not expire.'); Capsule::table('cloud_jobs')->where('id', $persistedEventJobId)->update([ 'effect_resolution_reason' => 'automatic_recovery:v1:' . (time() - 21600) . ':12:pending', 'finalization_incident' => 1, 'finalization_error_code' => 'recovery_limit_reached', // Exact immutable receipt evidence is stronger than an earlier // inconclusive/offline inspection backoff. 'available_at' => time() + 600, ]); $persistedEventId = 'settings-persisted-event-00001'; $persistedEventResult = ['agent_detail' => 'persisted-before-prepare']; $persistedEventReceipt = $withIdentity([ 'type' => 'completion', 'id' => $persistedEventId, 'agent_id' => '7', 'operation_id' => $persistedEventOperationId, 'status' => 'failed', 'effect_may_have_applied' => false, 'safe_to_resubmit' => true, 'error_code' => 'guest_update_failed', 'result' => $persistedEventResult, ]); Capsule::table('cloud_agent_completion_events')->insert( $storedEvent($persistedEventReceipt, time()) ); $persistedEventAudit = CloudJob::find($persistedEventJobId); $persistedEventAudit = [ 'phase' => (string) $persistedEventAudit->phase, 'outcome' => (string) $persistedEventAudit->outcome, 'status' => (string) $persistedEventAudit->status, 'error_code' => (string) $persistedEventAudit->error_code, 'result_json' => $persistedEventAudit->result_json, 'terminal_at' => (int) $persistedEventAudit->terminal_at, ]; $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The worker did not drain a persisted pre-prepare receipt.'); $persistedEventRecovered = CloudJob::find($persistedEventJobId); $persistedEventRow = Capsule::table('cloud_agent_completion_events') ->where('event_id', $persistedEventId)->first(); $assert((string) $persistedEventRecovered->effect_resolution === 'not_applied' && (string) $persistedEventRecovered->effect_resolution_reason === 'late_completion_receipt:' . $persistedEventId && !(bool) $persistedEventRecovered->finalization_incident && $persistedEventRecovered->finalization_error_code === null && (string) $persistedEventRow->disposition === 'applied' && $persistedEventAudit === [ 'phase' => (string) $persistedEventRecovered->phase, 'outcome' => (string) $persistedEventRecovered->outcome, 'status' => (string) $persistedEventRecovered->status, 'error_code' => (string) $persistedEventRecovered->error_code, 'result_json' => $persistedEventRecovered->result_json, 'terminal_at' => (int) $persistedEventRecovered->terminal_at, ] && !CloudJobService::blockingQueryForInstance( $persistedEventInstanceId )->exists(), 'Persisted pre-prepare receipt did not converge locally and exactly.'); // A row with the right indexed host+operation but a conflicting immutable // payload hash is not receipt authority. Reject only that event, surface // the concrete receipt error, and leave the job queued for fresh // inspection without contacting the Agent in the same tick. $conflictEventInstanceId = 97; $conflictEventUuid = 'ffffffff-0000-4111-8222-000000000004'; $insertInstance( $conflictEventInstanceId, $conflictEventUuid, 'conflict-event-settings-vm' ); $conflictEventOperationId = '6234567890abcdef1234567890abcdef'; $conflictEventJobId = $insertJob( 1, $conflictEventOperationId, 'awaiting_receipt', 1 ); Capsule::table('cloud_jobs')->where('id', $conflictEventJobId)->update([ 'instance_id' => $conflictEventInstanceId, 'kind' => 'instance_settings', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/vms/' . $conflictEventUuid . '/settings', 'body' => [ 'root_password' => 'Conflict-password9!', 'confirm_restart' => true, ], 'owner_id' => 1, 'settings_fields' => ['password'], 'credential' => [ 'password' => 'Conflict-password9!', 'login_name' => 'root', 'auth_mode' => 'password', ], ], 'job-payload'), ]); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $conflictEventJobId, 1 ) === 1, 'The conflicting-event fixture did not expire.'); Capsule::table('cloud_jobs')->where('id', $conflictEventJobId)->update([ 'effect_resolution_reason' => 'automatic_recovery:v1:' . (time() - 21600) . ':12:pending', 'finalization_incident' => 1, 'finalization_error_code' => 'recovery_state_invalid', ]); $conflictEventId = 'settings-conflict-event-000001'; Capsule::table('cloud_agent_completion_events')->insert([ 'event_id' => $conflictEventId, 'host_id' => 7, 'operation_id' => $conflictEventOperationId, 'payload_sha256' => str_repeat('0', 64), 'outcome' => 'failed', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 1, 'error_code' => 'guest_update_failed', 'result_json' => json_encode( ['agent_detail' => 'does-not-match-hash'], JSON_THROW_ON_ERROR ), 'disposition' => 'received', 'received_at' => time() + 1, 'applied_at' => null, ]); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The worker did not examine a conflicting persisted event.'); $conflictEventJob = CloudJob::find($conflictEventJobId); $conflictEventRow = Capsule::table('cloud_agent_completion_events') ->where('event_id', $conflictEventId)->first(); $assert((string) $conflictEventRow->disposition === 'rejected' && (string) $conflictEventJob->effect_resolution === 'unknown' && (bool) $conflictEventJob->finalization_incident && (string) $conflictEventJob->finalization_error_code === 'recovery_receipt_invalid' && (int) $conflictEventJob->available_at >= time() + 290 && CloudJobService::blockingQueryForInstance( $conflictEventInstanceId )->exists(), 'A conflicting persisted event altered the job or released its lock.'); // A self-consistent late receipt may still contradict an already prepared // conclusion. Such conflicts are deterministic: atomically isolate the // event once, retain the original marker/escrow/blocker, and schedule a // later fresh inspection without polling Agent in the same tick. $conflictEscrow = (string) $conflictEventJob->sealed_payload; $conflictIdentity = $identityFor($conflictEventOperationId, false); $noEffectOperation = $conflictIdentity + [ 'effect_may_have_applied' => false, 'error_code' => 'settings_not_committed', 'operation_id' => $conflictEventOperationId, 'result' => [], 'safe_to_resubmit' => false, 'status' => 'failed', ]; $canonicalNoEffectOperation = $noEffectOperation; ksort($canonicalNoEffectOperation, SORT_STRING); $noEffectOperationHash = hash( 'sha256', json_encode( $canonicalNoEffectOperation, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE ) ); $armConflictReceipt = static function ( string $status, bool $safeToResubmit, string $errorCode, array $result, string $marker ) use ( $conflictEventId, $conflictEventJobId, $conflictEventOperationId, $storedEvent, $withIdentity ): void { $receipt = $withIdentity([ 'type' => 'completion', 'id' => $conflictEventId, 'agent_id' => '7', 'operation_id' => $conflictEventOperationId, 'status' => $status, 'effect_may_have_applied' => false, 'safe_to_resubmit' => $safeToResubmit, 'error_code' => $errorCode, 'result' => $result, ]); Capsule::table('cloud_jobs')->where( 'id', $conflictEventJobId )->update([ 'effect_resolution_reason' => $marker, 'finalization_incident' => 0, 'finalization_error_code' => null, 'lease_token' => null, 'lease_until' => 0, 'available_at' => 0, ]); $eventRow = $storedEvent($receipt, time()); Capsule::table('cloud_agent_completion_events')->where( 'event_id', $conflictEventId )->update($eventRow); }; $assertPreparedConflict = static function ( string $marker, string $message ) use ( $assert, $conflictEscrow, $conflictEventId, $conflictEventInstanceId, $conflictEventJobId ): void { $assert((new CloudAutomaticOperationRecoveryService())->tick(), $message . ' was not examined.'); $job = CloudJob::find($conflictEventJobId); $event = Capsule::table('cloud_agent_completion_events') ->where('event_id', $conflictEventId)->first(); $updatedAt = (int) $job->updated_at; $assert((string) $event->disposition === 'late' && $event->applied_at === null && (string) $job->effect_resolution === 'unknown' && (string) $job->effect_resolution_reason === $marker && (bool) $job->finalization_incident && (string) $job->finalization_error_code === 'recovery_evidence_conflict' && (string) $job->sealed_payload === $conflictEscrow && $job->lease_token === null && (int) $job->lease_until === 0 && (int) $job->available_at >= time() + 290 && CloudJobService::blockingQueryForInstance( $conflictEventInstanceId )->exists(), $message . ' did not retain the fail-closed evidence fence.'); // Other newly recoverable kinds may be claimed here; the isolated job // itself must remain untouched until its own retry time arrives. (new CloudAutomaticOperationRecoveryService())->tick(); $afterSecondTick = CloudJob::find($conflictEventJobId); $afterSecondEvent = Capsule::table( 'cloud_agent_completion_events' )->where('event_id', $conflictEventId)->first(); $assert((int) $afterSecondTick->updated_at === $updatedAt && (string) $afterSecondTick->effect_resolution === 'unknown' && (string) $afterSecondTick->effect_resolution_reason === $marker && (bool) $afterSecondTick->finalization_incident && (string) $afterSecondTick->finalization_error_code === 'recovery_evidence_conflict' && (string) $afterSecondTick->sealed_payload === $conflictEscrow && (string) $afterSecondEvent->disposition === 'late' && $afterSecondEvent->applied_at === null && CloudJobService::blockingQueryForInstance( $conflictEventInstanceId )->exists(), $message . ' ignored its retry backoff.'); }; $directionConflictMarker = 'automatic_recovery:v1:' . (time() - 300) . ':2:prepared:automatic_applied_succeeded_nofence:' . str_repeat('1', 64) . ':' . $noEffectOperationHash; $armConflictReceipt( 'failed', true, 'guest_update_failed', ['agent_detail' => 'direction-conflict'], $directionConflictMarker ); $assertPreparedConflict( $directionConflictMarker, 'Opposite-direction late evidence' ); $hashConflictMarker = 'automatic_recovery:v1:' . (time() - 300) . ':2:prepared:automatic_not_applied_failed_nofence:' . str_repeat('2', 64) . ':' . str_repeat('3', 64); $armConflictReceipt( 'failed', true, 'guest_update_failed', ['agent_detail' => 'hash-conflict'], $hashConflictMarker ); $assertPreparedConflict( $hashConflictMarker, 'Different-operation-hash late evidence' ); $invalidStateConflictMarker = 'invalid-recovery-state'; $armConflictReceipt( 'failed', true, 'guest_update_failed', ['agent_detail' => 'invalid-state-conflict'], $invalidStateConflictMarker ); $assertPreparedConflict( $invalidStateConflictMarker, 'Invalid-state late evidence' ); $noEffectFenceConflictMarker = 'automatic_recovery:v1:' . (time() - 300) . ':2:prepared:automatic_not_applied_failed_fence:' . str_repeat('4', 64) . ':' . $noEffectOperationHash; $armConflictReceipt( 'failed', true, 'guest_update_failed', ['agent_detail' => 'no-effect-fence-conflict'], $noEffectFenceConflictMarker ); $assertPreparedConflict( $noEffectFenceConflictMarker, 'No-effect active-fence late evidence' ); $appliedReceiptResult = ['settings_changed' => true]; $appliedOperation = $conflictIdentity + [ 'effect_may_have_applied' => false, 'error_code' => '', 'operation_id' => $conflictEventOperationId, 'result' => $appliedReceiptResult, 'safe_to_resubmit' => false, 'status' => 'succeeded', ]; $canonicalAppliedOperation = $appliedOperation; ksort($canonicalAppliedOperation, SORT_STRING); $appliedOperationHash = hash( 'sha256', json_encode( $canonicalAppliedOperation, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE ) ); $appliedFenceConflictMarker = 'automatic_recovery:v1:' . (time() - 300) . ':2:prepared:automatic_applied_failed_fence:' . str_repeat('5', 64) . ':' . $appliedOperationHash; $armConflictReceipt( 'succeeded', false, '', $appliedReceiptResult, $appliedFenceConflictMarker ); $assertPreparedConflict( $appliedFenceConflictMarker, 'Applied active-fence late evidence' ); // A receipt can become durable while an automatic worker is outside the // database resolving an Agent fence. Its stale commit must yield before // publishing, so the next event-first turn checks compatibility. $pendingReceiptSource = 'automatic_not_applied_failed_nofence:' . str_repeat('6', 64); $pendingReceiptMarker = 'automatic_recovery:v1:' . (time() - 300) . ':2:prepared:' . $pendingReceiptSource . ':' . $noEffectOperationHash; $armConflictReceipt( 'failed', false, 'settings_not_committed', [], $pendingReceiptMarker ); $pendingReceiptCommitRejected = false; try { CloudOperationRecoveryService::commitNotApplied( $conflictEventJobId, $noEffectOperation, $pendingReceiptSource, 0 ); } catch (\App\Services\Cloud\CloudError $error) { $pendingReceiptCommitRejected = $error->httpStatus === 409; } $pendingReceiptJob = CloudJob::find($conflictEventJobId); $pendingReceiptEvent = Capsule::table( 'cloud_agent_completion_events' )->where('event_id', $conflictEventId)->first(); $assert($pendingReceiptCommitRejected && (string) $pendingReceiptJob->effect_resolution === 'unknown' && (string) $pendingReceiptJob->effect_resolution_reason === $pendingReceiptMarker && (string) $pendingReceiptEvent->disposition === 'received', 'An automatic commit ignored a newly durable completion receipt.'); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The compatible priority receipt was not claimed after commit yield.'); $pendingReceiptResolved = CloudJob::find($conflictEventJobId); $pendingReceiptEvent = Capsule::table( 'cloud_agent_completion_events' )->where('event_id', $conflictEventId)->first(); $assert((string) $pendingReceiptResolved->effect_resolution === 'not_applied' && (string) $pendingReceiptResolved->effect_resolution_reason === 'late_completion_receipt:' . $conflictEventId && (string) $pendingReceiptEvent->disposition === 'applied' && !CloudJobService::blockingQueryForInstance( $conflictEventInstanceId )->exists(), 'The priority receipt did not safely win the automatic commit race.'); // A receipt can crash after its PHP prepare transaction but before the // effect commit/event acknowledgement. Even if it replaced an exhausted // automatic prepared intent, the dedicated recovery worker must rebuild // the exact immutable event and finish PHP locally without an Agent // retransmission or host request. $lateCrashInstanceId = 95; $lateCrashUuid = 'dddddddd-eeee-4fff-8000-000000000002'; $insertInstance( $lateCrashInstanceId, $lateCrashUuid, 'late-crash-settings-vm' ); $lateCrashOperationId = '4234567890abcdef1234567890abcdef'; $lateCrashJobId = $insertJob( 1, $lateCrashOperationId, 'awaiting_receipt', 1 ); Capsule::table('cloud_jobs')->where('id', $lateCrashJobId)->update([ 'instance_id' => $lateCrashInstanceId, 'kind' => 'instance_settings', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/vms/' . $lateCrashUuid . '/settings', 'body' => [ 'hostname' => 'late-crash.example', 'confirm_restart' => true, ], 'owner_id' => 1, 'settings_fields' => ['hostname'], 'credential' => null, ], 'job-payload'), ]); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $lateCrashJobId, 1 ) === 1, 'The late-receipt crash fixture did not expire.'); $lateCrashOperation = $identityFor($lateCrashOperationId, false) + [ 'effect_may_have_applied' => false, 'error_code' => 'guest_update_failed', 'operation_id' => $lateCrashOperationId, 'result' => ['agent_detail' => 'bounded'], 'safe_to_resubmit' => true, 'status' => 'failed', ]; $canonicalLateCrashOperation = $lateCrashOperation; ksort($canonicalLateCrashOperation, SORT_STRING); $lateCrashOperationHash = hash( 'sha256', json_encode( $canonicalLateCrashOperation, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE ) ); Capsule::table('cloud_jobs')->where('id', $lateCrashJobId)->update([ 'effect_resolution_reason' => 'automatic_recovery:v1:' . (time() - 21600) . ':12:prepared:automatic_not_applied_failed_nofence:' . str_repeat('a', 64) . ':' . $lateCrashOperationHash, 'finalization_incident' => 1, 'finalization_error_code' => 'recovery_limit_reached', ]); $lateCrashEventId = 'settings-late-crash-event-0001'; $lateCrashReceipt = $withIdentity([ 'type' => 'completion', 'id' => $lateCrashEventId, 'agent_id' => '7', 'operation_id' => $lateCrashOperationId, 'status' => 'failed', 'effect_may_have_applied' => false, 'safe_to_resubmit' => true, 'error_code' => 'guest_update_failed', 'result' => ['agent_detail' => 'bounded'], ]); Capsule::table('cloud_agent_completion_events')->insert( $storedEvent($lateCrashReceipt, time()) ); CloudOperationRecoveryService::prepareNotApplied( $lateCrashJobId, $lateCrashOperation, 'late_completion_receipt:' . $lateCrashEventId ); $lateCrashPrepared = CloudJob::find($lateCrashJobId); $legacyLateCrashPreparedMarker = (string) $lateCrashPrepared->effect_resolution_reason; $assert(str_starts_with( $legacyLateCrashPreparedMarker, 'prepared:late_completion_receipt:' . $lateCrashEventId . ':' ) && !(bool) $lateCrashPrepared->finalization_incident && $lateCrashPrepared->finalization_error_code === null && (string) $lateCrashPrepared->effect_resolution === 'unknown', 'Exact late evidence did not replace exhausted matching prepare state.'); // Older crash windows could retain the stable late-receipt marker inside // the mutable automatic-attempt envelope. The exact same receipt must // upgrade that legacy wrapper without weakening operation-hash matching. Capsule::table('cloud_jobs')->where('id', $lateCrashJobId)->update([ 'effect_resolution_reason' => 'automatic_recovery:v1:' . (time() - 21600) . ':12:' . $legacyLateCrashPreparedMarker, ]); $lateCrashPrepared = CloudJob::find($lateCrashJobId); $lateCrashAudit = [ 'phase' => (string) $lateCrashPrepared->phase, 'outcome' => (string) $lateCrashPrepared->outcome, 'status' => (string) $lateCrashPrepared->status, 'error_code' => (string) $lateCrashPrepared->error_code, 'result_json' => $lateCrashPrepared->result_json, 'terminal_at' => (int) $lateCrashPrepared->terminal_at, 'completion_event_id' => $lateCrashPrepared->completion_event_id, ]; $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The recovery worker did not resume a prepared late receipt.'); $lateCrashRecovered = CloudJob::find($lateCrashJobId); $lateCrashEvent = Capsule::table('cloud_agent_completion_events') ->where('event_id', $lateCrashEventId)->first(); $assert((string) $lateCrashRecovered->effect_resolution === 'not_applied' && (string) $lateCrashRecovered->effect_resolution_reason === 'late_completion_receipt:' . $lateCrashEventId && (string) $lateCrashEvent->disposition === 'applied' && $lateCrashAudit === [ 'phase' => (string) $lateCrashRecovered->phase, 'outcome' => (string) $lateCrashRecovered->outcome, 'status' => (string) $lateCrashRecovered->status, 'error_code' => (string) $lateCrashRecovered->error_code, 'result_json' => $lateCrashRecovered->result_json, 'terminal_at' => (int) $lateCrashRecovered->terminal_at, 'completion_event_id' => $lateCrashRecovered->completion_event_id, ] && !CloudJobService::blockingQueryForInstance( $lateCrashInstanceId )->exists(), 'Prepared late receipt did not finish PHP-only convergence.'); // A hash-valid receipt can still violate the kind-specific finalizer // contract. This deterministic local error must become one incident, not // an unbounded received-event loop every five minutes. $invalidReceiptInstanceId = 98; $invalidReceiptUuid = '99999999-aaaa-4bbb-8ccc-dddddddddddd'; $insertInstance( $invalidReceiptInstanceId, $invalidReceiptUuid, 'invalid-receipt-settings-vm' ); $invalidReceiptOperationId = '7234567890abcdef1234567890abcdef'; $invalidReceiptJobId = $insertJob( 1, $invalidReceiptOperationId, 'awaiting_receipt', 1 ); Capsule::table('cloud_jobs')->where( 'id', $invalidReceiptJobId )->update([ 'instance_id' => $invalidReceiptInstanceId, 'kind' => 'instance_settings', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/vms/' . $invalidReceiptUuid . '/settings', 'body' => [ 'hostname' => 'invalid-receipt.example', 'confirm_restart' => true, ], 'owner_id' => 1, 'settings_fields' => ['hostname'], 'credential' => null, ], 'job-payload'), ]); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $invalidReceiptJobId, 1 ) === 1, 'The invalid-receipt fixture did not expire.'); $invalidReceipt = $withIdentity([ 'type' => 'completion', 'id' => 'settings-invalid-result-event-01', 'agent_id' => '7', 'operation_id' => $invalidReceiptOperationId, 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', // The generic receipt is valid, but instance_settings requires the // explicit settings_changed proof before touching its PHP ledger. 'result' => ['hostname' => 'invalid-receipt.example'], ]); $assert(CloudOperationCompletionService::accept( $invalidReceipt ) === ['accepted' => true, 'disposition' => 'late'], 'A kind-invalid exact receipt was not terminally parked.'); $invalidReceiptJob = CloudJob::find($invalidReceiptJobId); $invalidReceiptEvent = Capsule::table( 'cloud_agent_completion_events' )->where('event_id', $invalidReceipt['id'])->first(); $assert((string) $invalidReceiptJob->effect_resolution === 'unknown' && (bool) $invalidReceiptJob->finalization_incident && (string) $invalidReceiptJob->finalization_error_code === 'recovery_receipt_invalid' && (string) $invalidReceiptEvent->disposition === 'late' && (int) $invalidReceiptJob->available_at >= time() + 290 && CloudJobService::blockingQueryForInstance( $invalidReceiptInstanceId )->exists(), 'An invalid receipt altered the ledger or released its blocker.'); // The incident is diagnostic, not a permanent queue state. A later fresh // operation-bound inspection may prepare and publish a valid conclusion. $freshInspectionOperation = $identityFor( $invalidReceiptOperationId, true ) + [ 'operation_id' => $invalidReceiptOperationId, 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => [ 'settings_changed' => true, 'hostname' => 'invalid-receipt.example', ], ]; $freshInspectionSource = 'agent_inspection:fresh-after-receipt-incident'; CloudOperationRecoveryService::prepareApplied( $invalidReceiptJobId, $freshInspectionOperation, $freshInspectionSource ); CloudOperationRecoveryService::commitApplied( $invalidReceiptJobId, $freshInspectionOperation, $freshInspectionSource, 0 ); $invalidReceiptAfter = CloudJob::find($invalidReceiptJobId); $invalidReceiptSpec = json_decode( (string) Capsule::table('cloud_instances') ->where('id', $invalidReceiptInstanceId)->value('spec_json'), true, flags: JSON_THROW_ON_ERROR ); $assert((string) $invalidReceiptAfter->effect_resolution === 'applied' && (string) $invalidReceiptAfter->effect_resolution_reason === $freshInspectionSource && !(bool) $invalidReceiptAfter->finalization_incident && $invalidReceiptAfter->finalization_error_code === null && $invalidReceiptAfter->read_at === null && ($invalidReceiptSpec['guest_hostname'] ?? null) === 'invalid-receipt.example' && !CloudJobService::blockingQueryForInstance( $invalidReceiptInstanceId )->exists(), 'A fresh inspection could not converge after an invalid receipt.'); // A valid immutable receipt may be temporarily unable to update a damaged // local projection. Age is audit metadata only: keep it retryable until an // operator repairs PHP state, even beyond the former six-hour window. $receiptLimitInstanceId = 99; $receiptLimitUuid = 'aaaaaaaa-1111-4222-8333-bbbbbbbbbbbb'; $insertInstance( $receiptLimitInstanceId, $receiptLimitUuid, 'receipt-limit-settings-vm' ); Capsule::table('cloud_instances')->where( 'id', $receiptLimitInstanceId )->update(['spec_json' => '{invalid-json']); $receiptLimitOperationId = '8234567890abcdef1234567890abcdef'; $receiptLimitJobId = $insertJob( 1, $receiptLimitOperationId, 'awaiting_receipt', 1 ); Capsule::table('cloud_jobs')->where( 'id', $receiptLimitJobId )->update([ 'instance_id' => $receiptLimitInstanceId, 'kind' => 'instance_settings', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/vms/' . $receiptLimitUuid . '/settings', 'body' => [ 'hostname' => 'receipt-limit.example', 'confirm_restart' => true, ], 'owner_id' => 1, 'settings_fields' => ['hostname'], 'credential' => null, ], 'job-payload'), ]); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $receiptLimitJobId, 1 ) === 1, 'The receipt-limit fixture did not expire.'); $receiptLimitReceipt = $withIdentity([ 'type' => 'completion', 'id' => 'settings-receipt-limit-event-01', 'agent_id' => '7', 'operation_id' => $receiptLimitOperationId, 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => [ 'settings_changed' => true, 'hostname' => 'receipt-limit.example', ], ]); $assert(CloudOperationCompletionService::accept( $receiptLimitReceipt ) === ['accepted' => true, 'disposition' => 'received'], 'A repairable local finalizer error was parked immediately.'); Capsule::table('cloud_agent_completion_events')->where( 'event_id', $receiptLimitReceipt['id'] )->update(['received_at' => time() - 21599]); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The recovery worker did not claim the near-limit receipt.'); $nearLimitJob = CloudJob::find($receiptLimitJobId); $assert((string) Capsule::table('cloud_agent_completion_events') ->where('event_id', $receiptLimitReceipt['id']) ->value('disposition') === 'received' && (int) $nearLimitJob->available_at >= time() + 290, 'A repairable receipt did not retain its retry backoff.'); Capsule::table('cloud_agent_completion_events')->where( 'event_id', $receiptLimitReceipt['id'] )->update(['received_at' => time() - 21601]); Capsule::table('cloud_jobs')->where( 'id', $receiptLimitJobId )->update([ 'available_at' => 0, 'lease_until' => 0, 'lease_token' => null, ]); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The recovery worker did not claim the six-hour-old receipt.'); $receiptLimitJob = CloudJob::find($receiptLimitJobId); $receiptLimitEvent = CloudAgentCompletionEvent::find( $receiptLimitReceipt['id'] ); $assert((string) $receiptLimitJob->effect_resolution === 'unknown' && (string) $receiptLimitEvent->disposition === 'received' && (int) $receiptLimitJob->available_at >= time() + 290 && CloudJobService::blockingQueryForInstance( $receiptLimitInstanceId )->exists(), 'A six-hour-old receipt left the recovery queue.'); Capsule::table('cloud_instances')->where( 'id', $receiptLimitInstanceId )->update(['spec_json' => json_encode( ['managed' => true], JSON_THROW_ON_ERROR )]); Capsule::table('cloud_jobs')->where( 'id', $receiptLimitJobId )->update([ 'available_at' => 0, 'lease_until' => 0, 'lease_token' => null, ]); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The repaired six-hour-old receipt was not claimed.'); $receiptLimitJob = CloudJob::find($receiptLimitJobId); $receiptLimitEvent = CloudAgentCompletionEvent::find( $receiptLimitReceipt['id'] ); $receiptLimitSpec = json_decode( (string) Capsule::table('cloud_instances') ->where('id', $receiptLimitInstanceId)->value('spec_json'), true, flags: JSON_THROW_ON_ERROR ); $assert((string) $receiptLimitJob->effect_resolution === 'applied' && (string) $receiptLimitEvent->disposition === 'applied' && ($receiptLimitSpec['guest_hostname'] ?? null) === 'receipt-limit.example' && !CloudJobService::blockingQueryForInstance( $receiptLimitInstanceId )->exists(), 'A repaired six-hour-old receipt did not converge.'); // Decryptable escrow is not enough: a password operation whose retained // credential DTO is missing must never be treated as a complete ledger // finalization merely because the Agent receipt says settings_changed. $damagedEscrowInstanceId = 100; $damagedEscrowUuid = 'aaaaaaaa-2222-4333-8444-cccccccccccc'; $insertInstance( $damagedEscrowInstanceId, $damagedEscrowUuid, 'damaged-escrow-settings-vm' ); $damagedEscrowOperationId = '9234567890abcdef1234567890abcdef'; $damagedEscrowJobId = $insertJob( 1, $damagedEscrowOperationId, 'awaiting_receipt', 1 ); Capsule::table('cloud_jobs')->where( 'id', $damagedEscrowJobId )->update([ 'instance_id' => $damagedEscrowInstanceId, 'kind' => 'instance_settings', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/vms/' . $damagedEscrowUuid . '/settings', 'body' => [ 'root_password' => 'Escrow-damaged9!', 'confirm_restart' => true, ], 'owner_id' => 1, 'settings_fields' => ['password'], 'credential' => null, ], 'job-payload'), ]); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $damagedEscrowJobId, 1 ) === 1, 'The damaged-escrow fixture did not expire.'); $damagedEscrowPayload = (string) CloudJob::find( $damagedEscrowJobId )->sealed_payload; $damagedEscrowReceipt = $withIdentity([ 'type' => 'completion', 'id' => 'settings-damaged-escrow-event-01', 'agent_id' => '7', 'operation_id' => $damagedEscrowOperationId, 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => ['settings_changed' => true], ]); $assert(CloudOperationCompletionService::accept( $damagedEscrowReceipt ) === ['accepted' => true, 'disposition' => 'received'], 'Damaged recovery escrow falsely converged from a valid receipt.'); $damagedEscrowJob = CloudJob::find($damagedEscrowJobId); $assert((string) $damagedEscrowJob->effect_resolution === 'unknown' && (string) $damagedEscrowJob->sealed_payload === $damagedEscrowPayload && CloudJobService::blockingQueryForInstance( $damagedEscrowInstanceId )->exists(), 'Damaged escrow changed the ledger or released its blocker.'); Capsule::table('cloud_agent_completion_events')->where( 'event_id', $damagedEscrowReceipt['id'] )->update(['received_at' => time() - 21601]); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'Damaged escrow was not retried beyond six hours.'); $damagedEscrowJob = CloudJob::find($damagedEscrowJobId); $damagedEscrowEvent = CloudAgentCompletionEvent::find( $damagedEscrowReceipt['id'] ); $assert((string) $damagedEscrowJob->effect_resolution === 'unknown' && (string) $damagedEscrowEvent->disposition === 'received' && (int) $damagedEscrowJob->available_at >= time() + 290 && CloudJobService::blockingQueryForInstance( $damagedEscrowInstanceId )->exists(), 'Damaged escrow left the recovery queue or released its blocker.'); // A structurally valid credential DTO must still match the setting being // recovered. Otherwise a late receipt could overwrite credential metadata // for a hostname-only change or forget that a newly installed SSH key is // available. $sshBlob = pack('N', strlen('ssh-ed25519')) . 'ssh-ed25519' . pack('N', 32) . str_repeat("\x02", 32); $recoveryPublicKey = 'ssh-ed25519 ' . base64_encode($sshBlob); foreach ([ [ 'label' => 'hostname-only credential', 'instance_id' => 103, 'uuid' => 'aaaaaaaa-4444-4555-8666-eeeeeeeeeeee', 'operation_id' => 'd234567890abcdef1234567890abcdef', 'event_id' => 'settings-hostname-credential-damaged-01', 'body' => [ 'hostname' => 'credential-damaged.example', 'confirm_restart' => true, ], 'settings_fields' => ['hostname'], 'credential' => [ 'password' => 'Credential-damaged9!', 'login_name' => 'root', 'auth_mode' => 'password', ], 'result' => [ 'settings_changed' => true, 'hostname' => 'credential-damaged.example', ], ], [ 'label' => 'SSH-key credential mode', 'instance_id' => 104, 'uuid' => 'aaaaaaaa-5555-4666-8777-ffffffffffff', 'operation_id' => 'e234567890abcdef1234567890abcdef', 'event_id' => 'settings-ssh-credential-damaged-0001', 'body' => [ 'ssh_public_key' => $recoveryPublicKey, 'confirm_restart' => true, ], 'settings_fields' => ['ssh_key'], 'credential' => [ 'password' => 'Credential-damaged9!', 'login_name' => 'root', 'auth_mode' => 'password', ], 'result' => ['settings_changed' => true], ], ] as $credentialCase) { $insertInstance( $credentialCase['instance_id'], $credentialCase['uuid'], 'credential-semantics-settings-vm-' . $credentialCase['instance_id'] ); $credentialJobId = $insertJob( 1, $credentialCase['operation_id'], 'awaiting_receipt', 1 ); Capsule::table('cloud_jobs')->where('id', $credentialJobId)->update([ 'instance_id' => $credentialCase['instance_id'], 'kind' => 'instance_settings', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/vms/' . $credentialCase['uuid'] . '/settings', 'body' => $credentialCase['body'], 'owner_id' => 1, 'settings_fields' => $credentialCase['settings_fields'], 'credential' => $credentialCase['credential'], ], 'job-payload'), ]); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $credentialJobId, 1 ) === 1, $credentialCase['label'] . ' fixture did not expire.'); $credentialPayload = (string) CloudJob::find( $credentialJobId )->sealed_payload; $credentialReceipt = $withIdentity([ 'type' => 'completion', 'id' => $credentialCase['event_id'], 'agent_id' => '7', 'operation_id' => $credentialCase['operation_id'], 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => $credentialCase['result'], ]); $assert(CloudOperationCompletionService::accept( $credentialReceipt ) === ['accepted' => true, 'disposition' => 'received'], $credentialCase['label'] . ' corruption falsely converged from a valid receipt.'); $credentialJob = CloudJob::find($credentialJobId); $credentialEvent = CloudAgentCompletionEvent::find( $credentialCase['event_id'] ); $assert((string) $credentialJob->effect_resolution === 'unknown' && (string) $credentialJob->sealed_payload === $credentialPayload && (string) $credentialEvent->disposition === 'received' && Capsule::table('cloud_instances')->where( 'id', $credentialCase['instance_id'] )->value('sealed_credential') === null && CloudJobService::blockingQueryForInstance( $credentialCase['instance_id'] )->exists(), $credentialCase['label'] . ' corruption changed the ledger or released its blocker.'); Capsule::table('cloud_agent_completion_events')->where( 'event_id', $credentialCase['event_id'] )->update(['received_at' => time() - 21601]); $assert((new CloudAutomaticOperationRecoveryService())->tick(), $credentialCase['label'] . ' receipt was not retried beyond six hours.'); $credentialJob = CloudJob::find($credentialJobId); $credentialEvent = CloudAgentCompletionEvent::find( $credentialCase['event_id'] ); $assert((string) $credentialJob->effect_resolution === 'unknown' && (string) $credentialEvent->disposition === 'received' && (int) $credentialJob->available_at >= time() + 290 && CloudJobService::blockingQueryForInstance( $credentialCase['instance_id'] )->exists(), $credentialCase['label'] . ' corruption left the recovery queue or released its blocker.'); } // A stored event whose immutable fields no longer match its accepted hash // must be rejected before any settings projection or prepare marker moves. $corruptEventInstanceId = 101; $corruptEventUuid = 'aaaaaaaa-3333-4444-8555-dddddddddddd'; $insertInstance( $corruptEventInstanceId, $corruptEventUuid, 'corrupt-event-settings-vm' ); $corruptEventOperationId = 'a234567890abcdef1234567890abcdef'; $corruptEventJobId = $insertJob( 1, $corruptEventOperationId, 'awaiting_receipt', 1 ); Capsule::table('cloud_jobs')->where( 'id', $corruptEventJobId )->update([ 'instance_id' => $corruptEventInstanceId, 'kind' => 'instance_settings', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/vms/' . $corruptEventUuid . '/settings', 'body' => [ 'hostname' => 'corrupt-event.example', 'confirm_restart' => true, ], 'owner_id' => 1, 'settings_fields' => ['hostname'], 'credential' => null, ], 'job-payload'), ]); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $corruptEventJobId, 1 ) === 1, 'The corrupt-event fixture did not expire.'); $corruptEventId = 'settings-corrupt-stored-event-01'; $corruptAccepted = [ 'event_id' => $corruptEventId, 'host_id' => 7, 'operation_id' => $corruptEventOperationId, 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => [ 'settings_changed' => true, 'hostname' => 'corrupt-event.example', ], ]; Capsule::table('cloud_agent_completion_events')->insert([ 'event_id' => $corruptEventId, 'host_id' => 7, 'operation_id' => $corruptEventOperationId, 'payload_sha256' => hash('sha256', json_encode( $corruptAccepted, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE )), 'outcome' => 'succeeded', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'error_code' => '', // Simulate durable-row corruption after the original receipt hash was // accepted. This must fail the exact-event reconstruction. 'result_json' => json_encode([ 'settings_changed' => true, 'hostname' => 'tampered.example', ], JSON_THROW_ON_ERROR), 'disposition' => 'received', 'received_at' => time(), 'applied_at' => null, ]); $corruptEventJobBefore = CloudJob::find($corruptEventJobId); $corruptEventSpecBefore = (string) Capsule::table('cloud_instances') ->where('id', $corruptEventInstanceId)->value('spec_json'); $assert(CloudOperationRecoveryService::settleLateReceipt( $corruptEventJobBefore, CloudAgentCompletionEvent::find($corruptEventId) ) === 'rejected', 'A hash-inconsistent stored event was not rejected.'); $corruptEventJobAfter = CloudJob::find($corruptEventJobId); $assert((string) $corruptEventJobAfter->effect_resolution === 'unknown' && (string) $corruptEventJobAfter->effect_resolution_reason === (string) $corruptEventJobBefore->effect_resolution_reason && (string) $corruptEventJobAfter->sealed_payload === (string) $corruptEventJobBefore->sealed_payload && (string) Capsule::table('cloud_instances') ->where('id', $corruptEventInstanceId)->value('spec_json') === $corruptEventSpecBefore, 'A hash-inconsistent event changed PHP settings state.'); // Agent VM metadata may include a hostname even when the original setting // changed only a password. Preserve that exact bounded observation in both // recovery sources; the settings finalizer ignores it for password-only // requests instead of rewriting the receipt into a synthetic result. $extraHostnameInstanceId = 102; $extraHostnameUuid = 'aaaaaaaa-4444-4555-8666-eeeeeeeeeeee'; $insertInstance( $extraHostnameInstanceId, $extraHostnameUuid, 'extra-hostname-settings-vm' ); $extraHostnameOperationId = 'b234567890abcdef1234567890abcdef'; $extraHostnameJobId = $insertJob( 1, $extraHostnameOperationId, 'awaiting_receipt', 1 ); Capsule::table('cloud_jobs')->where( 'id', $extraHostnameJobId )->update([ 'instance_id' => $extraHostnameInstanceId, 'kind' => 'instance_settings', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/vms/' . $extraHostnameUuid . '/settings', 'body' => [ 'root_password' => 'Extra-hostname9!Pass', 'confirm_restart' => true, ], 'owner_id' => 1, 'settings_fields' => ['password'], 'credential' => [ 'password' => 'Extra-hostname9!Pass', 'login_name' => 'root', 'auth_mode' => 'password', ], ], 'job-payload'), ]); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $extraHostnameJobId, 1 ) === 1, 'The extra-hostname fixture did not expire.'); Capsule::table('cloud_jobs')->where( 'id', $extraHostnameJobId )->update([ 'effect_resolution_reason' => 'automatic_recovery:v1:' . (time() - 60) . ':1:pending', ]); $extraHostnameOperation = $identityFor( $extraHostnameOperationId, true ) + [ 'operation_id' => $extraHostnameOperationId, 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => [ 'uuid' => $extraHostnameUuid, 'settings_changed' => true, 'hostname' => 'metadata-only.example', ], ]; CloudOperationRecoveryService::prepareApplied( $extraHostnameJobId, $extraHostnameOperation, 'automatic_applied_succeeded_nofence:' . str_repeat('7', 64) ); $extraHostnameReceipt = $withIdentity([ 'type' => 'completion', 'id' => 'settings-extra-hostname-event-01', 'agent_id' => '7', 'operation_id' => $extraHostnameOperationId, 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => [ 'uuid' => $extraHostnameUuid, 'settings_changed' => true, 'hostname' => 'metadata-only.example', ], ]); $assert(CloudOperationCompletionService::accept( $extraHostnameReceipt ) === ['accepted' => true, 'disposition' => 'applied'], 'An unrelated receipt hostname conflicted with password recovery.'); $extraHostnameJob = CloudJob::find($extraHostnameJobId); $assert((string) $extraHostnameJob->effect_resolution === 'applied' && (string) $extraHostnameJob->effect_resolution_reason === 'late_completion_receipt:' . $extraHostnameReceipt['id'] && !(bool) $extraHostnameJob->finalization_incident && !CloudJobService::blockingQueryForInstance( $extraHostnameInstanceId )->exists(), 'Compatible password recovery did not release its blocker.'); // A failed receipt which still reports possible effect is not no-effect // evidence. It remains late and the original unknown-effect fence stays. $ambiguousInstanceId = 94; $ambiguousUuid = 'cccccccc-dddd-4eee-8fff-000000000001'; $insertInstance( $ambiguousInstanceId, $ambiguousUuid, 'ambiguous-settings-vm' ); $ambiguousOperationId = '3234567890abcdef1234567890abcdef'; $ambiguousJobId = $insertJob( 1, $ambiguousOperationId, 'awaiting_receipt', 1 ); Capsule::table('cloud_jobs')->where('id', $ambiguousJobId)->update([ 'instance_id' => $ambiguousInstanceId, 'kind' => 'instance_settings', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/vms/' . $ambiguousUuid . '/settings', 'body' => [ 'hostname' => 'ambiguous.example', 'confirm_restart' => true, ], 'owner_id' => 1, 'settings_fields' => ['hostname'], 'credential' => null, ], 'job-payload'), ]); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $ambiguousJobId, 1 ) === 1, 'The ambiguous late receipt fixture did not expire.'); $ambiguousReceipt = $withIdentity([ 'type' => 'completion', 'id' => 'settings-ambiguous-event-0001', 'agent_id' => '7', 'operation_id' => $ambiguousOperationId, 'status' => 'failed', 'effect_may_have_applied' => true, 'safe_to_resubmit' => false, 'error_code' => 'operation_failed', 'result' => null, ]); $assert(CloudOperationCompletionService::accept( $ambiguousReceipt ) === ['accepted' => true, 'disposition' => 'late'] && (string) CloudJob::find($ambiguousJobId)->effect_resolution === 'unknown' && CloudJobService::blockingQueryForInstance( $ambiguousInstanceId )->exists(), 'Ambiguous late failure incorrectly released the settings fence.'); // Live inspection uses the same PHP-only recovery seam in two phases: // first converge the ledger while preserving the blocker, then publish // the resolution only after the caller releases the Agent fence. $preparedInstanceId = 92; $preparedUuid = 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee'; $insertInstance( $preparedInstanceId, $preparedUuid, 'prepared-settings-vm' ); $preparedOperationId = '1234567890abcdef1234567890abcdef'; $preparedJobId = $insertJob( 1, $preparedOperationId, 'awaiting_receipt', 1 ); Capsule::table('cloud_jobs')->where('id', $preparedJobId)->update([ 'instance_id' => $preparedInstanceId, 'kind' => 'instance_settings', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'PATCH', 'request_target' => '/v1/vms/' . $preparedUuid . '/settings', 'body' => [ 'hostname' => 'prepared-recovered.example', 'root_password' => 'Prepared-password9!', 'confirm_restart' => true, ], 'owner_id' => 1, 'settings_fields' => ['hostname', 'password'], 'credential' => [ 'password' => 'Prepared-password9!', 'login_name' => 'root', 'auth_mode' => 'password', ], ], 'job-payload'), ]); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $preparedJobId, 1 ) === 1, 'The two-phase recovery fixture did not reach terminal failure.'); $preparedTerminal = CloudJob::find($preparedJobId); $preparedTerminalAt = (int) $preparedTerminal->terminal_at; $preparedError = (string) $preparedTerminal->error_code; $preparedResult = $preparedTerminal->result_json; $preparedCompletionEvent = $preparedTerminal->completion_event_id; $preparedReadAt = $preparedTerminal->read_at; $preparedOperation = $identityFor($preparedOperationId, true) + [ 'operation_id' => $preparedOperationId, 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => [ 'settings_changed' => true, 'hostname' => 'prepared-recovered.example', ], ]; $preparedEvidence = 'agent_inspection:settings-inspection-proof-0001'; $prepared = CloudOperationRecoveryService::prepareApplied( $preparedJobId, $preparedOperation, $preparedEvidence ); $preparedPayload = CloudSecrets::decrypt( (string) $prepared->sealed_payload, 'job-payload' ); $preparedInstance = Capsule::table('cloud_instances') ->where('id', $preparedInstanceId)->first(); $preparedSpec = json_decode( (string) $preparedInstance->spec_json, true, flags: JSON_THROW_ON_ERROR ); $preparedUpdatedAt = (int) $prepared->updated_at; $preparedAgain = CloudOperationRecoveryService::prepareApplied( $preparedJobId, $preparedOperation, $preparedEvidence ); $assert((string) $prepared->phase === 'terminal' && (string) $prepared->outcome === 'failed' && (string) $prepared->status === 'failed' && (bool) $prepared->effect_may_have_applied && !(bool) $prepared->safe_to_resubmit && (string) $prepared->effect_resolution === 'unknown' && str_starts_with( (string) $prepared->effect_resolution_reason, 'prepared:' . $preparedEvidence . ':' ) && (int) $prepared->terminal_at === $preparedTerminalAt && (string) $prepared->error_code === $preparedError && $prepared->result_json === $preparedResult && $prepared->completion_event_id === $preparedCompletionEvent && $prepared->read_at === $preparedReadAt && ($preparedPayload['credential']['password'] ?? null) === 'Prepared-password9!' && ($preparedSpec['guest_hostname'] ?? null) === 'prepared-recovered.example' && CloudJobService::blockingQueryForInstance( $preparedInstanceId )->exists() && (int) $preparedAgain->updated_at === $preparedUpdatedAt, 'Recovery prepare did not atomically converge while retaining fences.'); foreach (['commitApplied', 'commitNotApplied'] as $commitMethod) { $negativeResolverRejected = false; try { $commit = [CloudOperationRecoveryService::class, $commitMethod]; $commit( $preparedJobId, $preparedOperation, $preparedEvidence, -1 ); } catch (\App\Services\Cloud\CloudError $error) { $negativeResolverRejected = $error->httpStatus === 409; } $assert($negativeResolverRejected && (string) CloudJob::find($preparedJobId)->effect_resolution === 'unknown', $commitMethod . ' accepted a negative recovery audit actor.'); } $committed = CloudOperationRecoveryService::commitApplied( $preparedJobId, $preparedOperation, $preparedEvidence ); $committedPayload = CloudSecrets::decrypt( (string) $committed->sealed_payload, 'job-payload' ); $assert((string) $committed->phase === 'terminal' && (string) $committed->outcome === 'failed' && (string) $committed->status === 'failed' && (bool) $committed->effect_may_have_applied && !(bool) $committed->safe_to_resubmit && (string) $committed->effect_resolution === 'applied' && (string) $committed->effect_resolution_reason === $preparedEvidence && (int) $committed->effect_resolved_at > 0 && (int) $committed->terminal_at === $preparedTerminalAt && (string) $committed->error_code === $preparedError && $committed->result_json === $preparedResult && $committed->completion_event_id === $preparedCompletionEvent && $committed->read_at === $preparedReadAt && !array_key_exists('credential', $committedPayload) && !CloudJobService::blockingQueryForInstance( $preparedInstanceId )->exists(), 'Recovery commit did not publish the effect and clear its escrow.'); $committedAgain = CloudOperationRecoveryService::commitApplied( $preparedJobId, $preparedOperation, $preparedEvidence ); $mismatchedPreparedOperation = $preparedOperation; $mismatchedPreparedOperation['operation_id'] = '00000000000000000000000000000001'; $mismatchedCommitRejected = false; try { CloudOperationRecoveryService::commitApplied( $preparedJobId, $mismatchedPreparedOperation, $preparedEvidence ); } catch (\App\Services\Cloud\CloudError $error) { $mismatchedCommitRejected = $error->httpStatus === 409; } $assert((string) $committedAgain->effect_resolution === 'applied' && $mismatchedCommitRejected, 'Recovery commit was not idempotent or accepted another operation.'); $insertPaymentJob = static function (string $key, int $invoiceId) use ($now): int { $operationId = md5('payment-deadline:' . $key); return (int) Capsule::table('cloud_jobs')->insertGetId([ 'operation_id' => $operationId, 'request_idempotency_key' => $key, 'request_fingerprint' => hash('sha256', $key), 'parent_job_id' => null, 'order_id' => null, 'invoice_id' => $invoiceId, 'instance_id' => null, 'snapshot_id' => null, 'user_id' => 1, 'host_id' => null, 'kind' => 'payment_create', 'execution_mode' => 'local', 'phase' => 'dispatching', 'outcome' => null, 'status' => 'running', 'sealed_payload' => CloudSecrets::encrypt([ 'input' => ['idempotency_key' => $key, 'method' => 'paytaro'], ], 'job-payload'), 'attempts' => 1, 'dispatch_attempts' => 1, 'dispatch_deadline_at' => $now - 20, 'dispatch_started_at' => $now - 30, 'agent_accepted_at' => null, 'receipt_deadline_at' => $now - 10, 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolved_by' => null, 'effect_resolution_reason' => '', 'completion_event_id' => null, 'read_at' => null, 'dispatched_at' => $now - 30, 'lease_token' => bin2hex(random_bytes(16)), 'lease_until' => $now + 600, 'available_at' => 0, 'error_code' => null, 'result_json' => null, 'created_at' => $now - 60, 'updated_at' => $now - 30, ]); }; $missingPaymentKey = 'payment-missing-0001'; $pendingPaymentKey = 'payment-pending-0001'; $unknownPaymentKey = 'payment-unknown-0001'; Capsule::table('cloud_invoice_payments')->insert([ [ 'invoice_id' => 502, 'user_id' => 1, 'provider' => 'paytaro', 'provider_order_no' => 'provider-pending-0001', 'idempotency_key' => $pendingPaymentKey, 'amount' => 10, 'status' => 'pending', 'created_at' => $now - 30, 'updated_at' => $now - 30, ], [ 'invoice_id' => 503, 'user_id' => 1, 'provider' => 'paytaro', 'provider_order_no' => 'provider-unknown-0001', 'idempotency_key' => $unknownPaymentKey, 'amount' => 10, 'status' => 'creation_unknown', 'created_at' => $now - 30, 'updated_at' => $now - 30, ], ]); $missingPaymentJobId = $insertPaymentJob($missingPaymentKey, 501); $pendingPaymentJobId = $insertPaymentJob($pendingPaymentKey, 502); $unknownPaymentJobId = $insertPaymentJob($unknownPaymentKey, 503); foreach ([$missingPaymentJobId, $pendingPaymentJobId, $unknownPaymentJobId] as $paymentJobId) { $assert(CloudWorkerService::expireDueTasksForOwner( 1, $paymentJobId, 1 ) === 1, 'A due local payment task was not terminalized.'); } $missingPaymentJob = CloudJob::find($missingPaymentJobId); $pendingPaymentJob = CloudJob::find($pendingPaymentJobId); $unknownPaymentJob = CloudJob::find($unknownPaymentJobId); $pendingPaymentResult = json_decode( (string) $pendingPaymentJob->result_json, true, flags: JSON_THROW_ON_ERROR ); $assert($missingPaymentJob->phase === 'terminal' && $missingPaymentJob->outcome === 'failed' && $missingPaymentJob->effect_resolution === 'not_applied' && !(bool) $missingPaymentJob->effect_may_have_applied && (bool) $missingPaymentJob->safe_to_resubmit && $missingPaymentJob->error_code === 'payment_order_not_dispatched' && $pendingPaymentJob->phase === 'terminal' && $pendingPaymentJob->outcome === 'succeeded' && ($pendingPaymentResult['payment_id'] ?? null) === '1' && $unknownPaymentJob->phase === 'terminal' && $unknownPaymentJob->outcome === 'failed' && $unknownPaymentJob->effect_resolution === 'unknown' && (bool) $unknownPaymentJob->effect_may_have_applied && !(bool) $unknownPaymentJob->safe_to_resubmit && $unknownPaymentJob->error_code === 'payment_creation_unknown' && Capsule::table('cloud_invoice_payments')->count() === 2, 'Payment deadline fallback retried a provider POST or misclassified its durable attempt.'); $nullDispatchOperation = md5('null-dispatch-deadline-fixture'); $nullDispatchJobId = $insertJob( 1, $nullDispatchOperation, 'accepted', 0 ); Capsule::table('cloud_jobs')->where('id', $nullDispatchJobId)->update([ 'dispatch_deadline_at' => null, ]); $invalidReceiptOperation = md5('invalid-receipt-deadline-fixture'); $invalidReceiptJobId = $insertJob( 1, $invalidReceiptOperation, 'dispatching', 1 ); Capsule::table('cloud_jobs')->where('id', $invalidReceiptJobId)->update([ 'receipt_deadline_at' => 0, ]); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $nullDispatchJobId, 1 ) === 1, 'An accepted task with a NULL dispatch deadline stayed active.'); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $invalidReceiptJobId, 1 ) === 1, 'A dispatched task with an invalid receipt deadline stayed active.'); $nullDispatchJob = CloudJob::find($nullDispatchJobId); $invalidReceiptJob = CloudJob::find($invalidReceiptJobId); $assert($nullDispatchJob !== null && (string) $nullDispatchJob->phase === 'terminal' && (string) $nullDispatchJob->outcome === 'failed' && !(bool) $nullDispatchJob->effect_may_have_applied && (bool) $nullDispatchJob->safe_to_resubmit && $invalidReceiptJob !== null && (string) $invalidReceiptJob->phase === 'terminal' && (string) $invalidReceiptJob->outcome === 'failed' && (bool) $invalidReceiptJob->effect_may_have_applied && !(bool) $invalidReceiptJob->safe_to_resubmit, 'Malformed active deadlines did not fail closed by dispatch evidence.'); $assert(CloudJobService::dispatchDeadlineSeconds('action') === 300 && CloudJobService::dispatchDeadlineSeconds( 'host_storage_configure') === 60 && CloudJobService::receiptDeadlineSeconds('action') === 960 && CloudJobService::receiptDeadlineSeconds( 'host_storage_configure') === 180 && CloudJobService::receiptDeadlineSeconds( 'template_import_replica') === 8100 && CloudJobService::receiptDeadlineSeconds( 'rescue_template_import') === 8100 && CloudJobService::receiptDeadlineSeconds( 'user_iso_import_replica') === 8100 && CloudJobService::aggregateDeadlineSeconds( 'template_metadata_sync') === 1560 && CloudJobService::aggregateDeadlineSeconds( 'template_import') === 8700 && CloudJobService::aggregateDeadlineSeconds( 'user_iso_import') === 8700, 'Task deadlines lost dispatch, receipt, or aggregate finalization margin.'); $parentOperation = str_repeat('d', 32); $parentId = (int) Capsule::table('cloud_jobs')->insertGetId([ 'operation_id' => $parentOperation, 'request_idempotency_key' => $parentOperation, 'request_fingerprint' => hash('sha256', $parentOperation), 'parent_job_id' => null, 'order_id' => null, 'invoice_id' => null, 'instance_id' => null, 'snapshot_id' => null, 'user_id' => 1, 'host_id' => null, 'kind' => 'test_aggregate', 'execution_mode' => 'aggregate', 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'), 'attempts' => 0, 'dispatch_attempts' => 0, 'dispatch_deadline_at' => $now - 30, 'dispatch_started_at' => null, 'agent_accepted_at' => null, 'receipt_deadline_at' => $now - 1, 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolved_by' => null, 'effect_resolution_reason' => '', 'completion_event_id' => null, 'read_at' => null, 'dispatched_at' => null, 'lease_token' => str_repeat('9', 32), 'lease_until' => $now + 600, 'available_at' => 0, 'error_code' => null, 'result_json' => null, 'created_at' => $now - 100, 'updated_at' => $now - 50, ]); $childOperation = str_repeat('8', 32); $childId = (int) Capsule::table('cloud_jobs')->insertGetId([ 'operation_id' => $childOperation, 'request_idempotency_key' => $childOperation, 'request_fingerprint' => hash('sha256', $childOperation), 'parent_job_id' => $parentId, 'order_id' => null, 'invoice_id' => null, 'instance_id' => null, 'snapshot_id' => null, 'user_id' => 1, 'host_id' => 7, 'kind' => 'test_mutation', 'execution_mode' => 'agent', 'phase' => 'accepted', 'outcome' => null, 'status' => 'queued', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/test', 'body' => ['operation_id' => $childOperation], 'owner_id' => 1, ], 'job-payload'), 'attempts' => 0, 'dispatch_attempts' => 0, 'dispatch_deadline_at' => $now + 600, 'dispatch_started_at' => null, 'agent_accepted_at' => null, 'receipt_deadline_at' => null, 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolved_by' => null, 'effect_resolution_reason' => '', 'completion_event_id' => null, 'read_at' => null, 'dispatched_at' => null, 'lease_token' => null, 'lease_until' => 0, 'available_at' => 0, 'error_code' => null, 'result_json' => null, 'created_at' => $now - 80, 'updated_at' => $now - 40, ]); Capsule::unprepared('CREATE TRIGGER simulate_child_claim_cas_miss ' . 'BEFORE UPDATE OF lease_token ON cloud_jobs ' . 'WHEN OLD.id = ' . $childId . " AND OLD.phase <> 'terminal' " . 'BEGIN SELECT RAISE(IGNORE); END'); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $parentId, 10 ) === 1, 'Owner fallback did not claim an expired aggregate parent.'); $parent = CloudJob::find($parentId); $child = CloudJob::find($childId); $assert($parent !== null && $child !== null && (string) $parent->phase === 'awaiting_receipt' && $parent->outcome === null && $parent->lease_token === null && (int) $parent->lease_until === 0 && (string) $child->phase === 'accepted', 'A child claim CAS miss hid active work behind a terminal parent.'); Capsule::unprepared('DROP TRIGGER simulate_child_claim_cas_miss'); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $parentId, 10 ) === 1, 'A later fallback pass did not reclaim the aggregate parent.'); $parent = CloudJob::find($parentId); $child = CloudJob::find($childId); $assert($parent !== null && $child !== null && (string) $child->phase === 'terminal' && (string) $child->outcome === 'failed' && !(bool) $child->effect_may_have_applied && (bool) $child->safe_to_resubmit && (string) $parent->phase === 'terminal' && (string) $parent->outcome === 'failed', 'A later aggregate fallback did not terminalize children first.'); echo "Owner-scoped task deadline fallback tests passed.\n"; }