addConnection([ 'driver' => 'sqlite', 'database' => ':memory:', ], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('user', static function (Blueprint $table): void { $table->increments('id'); $table->string('money')->default('0.00'); $table->integer('enable')->default(1); $table->integer('is_admin')->default(0); }); $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('operation_id')->unique(); $table->string('request_idempotency_key')->unique(); $table->string('request_fingerprint'); $table->integer('parent_job_id')->nullable(); $table->integer('order_id')->nullable(); $table->integer('invoice_id')->nullable(); $table->integer('instance_id')->nullable(); $table->integer('snapshot_id')->nullable(); $table->integer('public_network_transition_id')->nullable(); $table->integer('vpc_repair_root_job_id')->nullable(); $table->integer('user_id'); $table->integer('host_id')->nullable(); $table->string('kind'); $table->string('execution_mode'); $table->string('phase'); $table->string('outcome')->nullable(); $table->string('status'); $table->text('sealed_payload'); $table->integer('attempts')->default(0); $table->integer('dispatch_attempts')->default(0); $table->integer('dispatch_deadline_at')->default(0); $table->integer('dispatch_started_at')->nullable(); $table->integer('agent_accepted_at')->nullable(); $table->integer('receipt_deadline_at')->nullable(); $table->integer('terminal_at')->nullable(); $table->boolean('effect_may_have_applied')->nullable(); $table->boolean('safe_to_resubmit')->nullable(); $table->string('effect_resolution'); $table->integer('effect_resolved_at')->nullable(); $table->integer('effect_resolved_by')->nullable(); $table->string('effect_resolution_reason')->default(''); $table->string('completion_event_id')->nullable(); $table->integer('read_at')->nullable(); $table->integer('dispatched_at')->nullable(); $table->string('lease_token')->nullable(); $table->integer('lease_until')->default(0); $table->integer('available_at')->default(0); $table->string('error_code')->nullable(); $table->boolean('finalization_incident')->default(0); $table->string('finalization_error_code')->nullable(); $table->text('result_json')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_backup_restore_attempts', static function (Blueprint $table): void { $table->increments('id'); $table->integer('job_id'); $table->string('state'); $table->string('effect_resolution'); $table->integer('active_attempt')->nullable(); $table->string('last_error_code')->default(''); }); $schema->create('cloud_public_network_transitions', static function (Blueprint $table): void { $table->increments('id'); $table->string('state'); $table->string('effect_resolution'); $table->string('last_error_code')->default(''); }); $schema->create('cloud_instances', static function (Blueprint $table): void { $table->increments('id'); $table->string('uuid'); $table->integer('host_id'); $table->integer('user_id'); $table->integer('order_id')->nullable(); $table->string('name'); $table->string('status'); $table->string('service_status'); $table->string('network_status'); $table->text('spec_json'); $table->text('sealed_credential')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_agent_completion_events', static function (Blueprint $table): void { $table->string('event_id')->primary(); $table->integer('host_id'); $table->string('operation_id'); $table->string('payload_sha256'); $table->string('outcome'); $table->boolean('effect_may_have_applied'); $table->boolean('safe_to_resubmit'); $table->string('error_code')->default(''); $table->text('result_json')->nullable(); $table->string('disposition')->default('received'); $table->integer('received_at'); $table->integer('applied_at')->nullable(); $table->unique(['host_id', 'operation_id']); }); $schema->create('cloud_templates', static function (Blueprint $table): void { $table->string('id')->primary(); $table->string('catalog_key')->default('test-catalog'); $table->string('version_label')->default('test-version'); $table->boolean('is_default')->default(true); $table->integer('active_job_id')->nullable(); $table->string('state')->default('active'); }); $schema->create('cloud_user_isos', static function (Blueprint $table): void { $table->string('id')->primary(); $table->integer('active_job_id')->nullable(); $table->string('state')->default('active'); }); $schema->create('cloud_template_replicas', static function (Blueprint $table): void { $table->integer('host_id'); $table->string('template_id'); $table->integer('revision')->default(0); $table->string('generation_token')->default(''); $table->string('state')->default('pending'); $table->integer('image_bytes')->default(0); $table->string('sha256')->default(''); $table->string('operation_id')->default(''); $table->integer('job_id')->nullable(); $table->string('error_code')->default(''); $table->integer('created_at')->default(0); $table->integer('updated_at')->default(0); $table->primary(['host_id', 'template_id']); }); $schema->create('cloud_user_iso_replicas', static function (Blueprint $table): void { $table->integer('host_id'); $table->string('user_iso_id'); $table->integer('user_id'); $table->string('state')->default('pending'); $table->integer('size_bytes')->default(0); $table->string('sha256')->default(''); $table->string('operation_id')->default(''); $table->integer('job_id')->nullable(); $table->string('error_code')->default(''); $table->integer('created_at')->default(0); $table->integer('updated_at')->default(0); $table->primary(['host_id', 'user_iso_id']); }); $schema->create('cloud_hosts', static function (Blueprint $table): void { $table->increments('id'); }); $schema->create('cloud_host_network_attachments', static function (Blueprint $table): void { $table->increments('id'); $table->integer('host_id'); $table->integer('line_id'); $table->string('network_selector'); $table->string('state'); $table->integer('priority')->default(0); }); $schema->create('cloud_public_ip_pools', static function (Blueprint $table): void { $table->increments('id'); $table->string('pool_key'); $table->integer('host_id'); $table->integer('network_attachment_id'); $table->integer('address_family'); $table->string('cidr'); $table->string('gateway'); $table->boolean('gateway_on_link'); $table->integer('guest_prefix_length'); $table->text('dns_servers_json'); $table->text('reserved_ips_json'); $table->string('allocation_start')->nullable(); $table->string('allocation_end')->nullable(); $table->string('allocation_cursor')->nullable(); $table->integer('max_allocations'); $table->string('state'); $table->integer('priority')->default(0); $table->integer('desired_revision'); $table->integer('observed_revision'); $table->string('sync_state'); }); $schema->create('cloud_offers', static function (Blueprint $table): void { $table->increments('id'); $table->integer('stock')->default(-1); $table->integer('updated_at')->default(0); }); $schema->create('cloud_orders', static function (Blueprint $table): void { $table->increments('id'); $table->integer('user_id'); $table->integer('plan_id'); $table->integer('offer_id'); $table->integer('resource_pool_id'); $table->integer('host_id')->nullable(); $table->integer('reservation_id')->nullable(); $table->string('idempotency_key'); $table->string('request_hash'); $table->string('amount'); $table->string('currency')->default('USD'); $table->string('status'); $table->text('spec_json'); $table->text('sealed_payload')->nullable(); $table->integer('instance_id')->nullable(); $table->integer('refunded_at')->nullable(); $table->integer('stock_reserved_at')->nullable(); $table->integer('stock_released_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_resource_reservations', static function (Blueprint $table): void { $table->increments('id'); $table->string('reservation_key'); $table->string('operation_id'); $table->integer('order_id'); $table->integer('backup_restore_id')->nullable(); $table->integer('offer_id'); $table->integer('resource_pool_id'); $table->integer('host_id'); $table->integer('user_id'); $table->integer('instance_id')->nullable(); $table->integer('network_attachment_id')->nullable(); $table->integer('public_ip_pool_id')->nullable(); $table->integer('public_ip_id')->nullable(); $table->integer('cpu_cores'); $table->integer('memory_mib'); $table->integer('disk_gib'); $table->string('state'); $table->integer('expires_at')->nullable(); $table->integer('released_at')->nullable(); $table->string('release_reason')->default(''); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_public_ips', static function (Blueprint $table): void { $table->increments('id'); $table->integer('host_id'); $table->integer('public_ip_pool_id')->nullable(); $table->integer('address_family'); $table->string('address'); $table->binary('address_binary'); $table->integer('prefix_length'); $table->string('state'); $table->integer('reservation_id')->nullable(); $table->integer('reservation_ip_claim_id')->nullable(); $table->integer('instance_id')->nullable(); $table->string('operation_id')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_resource_reservation_ips', static function (Blueprint $table): void { $table->increments('id'); $table->integer('reservation_id'); $table->integer('address_family'); $table->integer('claim_generation')->default(1); $table->integer('active_claim')->nullable()->default(1); $table->integer('network_attachment_id'); $table->integer('public_ip_pool_id'); $table->integer('public_ip_id'); $table->string('operation_id'); $table->string('state'); $table->integer('created_at'); $table->integer('updated_at'); $table->integer('released_at')->nullable(); $table->string('release_reason')->default(''); }); $schema->create('cloud_invoices', static function (Blueprint $table): void { $table->increments('id'); $table->string('invoice_no'); $table->string('purpose'); $table->integer('instance_id')->nullable(); $table->integer('order_id')->nullable(); $table->integer('user_id'); $table->integer('plan_id')->nullable(); $table->integer('offer_id')->nullable(); $table->integer('renewal_id')->nullable(); $table->integer('renews_expires_at')->nullable(); $table->integer('due_at'); $table->string('amount'); $table->string('currency')->default('USD'); $table->integer('validity_days'); $table->integer('traffic_quota_bytes'); $table->string('status'); $table->string('paid_via')->nullable(); $table->string('payment_reference')->nullable(); $table->integer('paid_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $now = time(); $operationId = str_repeat('ad', 16); $uuid = '00000000-0000-4000-8000-000000000501'; $payload = [ 'method' => 'POST', 'request_target' => '/v1/vms/' . $uuid . '/settings', 'body' => [ 'hostname' => 'local-finalizer.example', 'root_password' => 'Secret9!Password', 'confirm_restart' => true, ], 'owner_id' => 1, 'can_manage' => false, 'credential' => [ 'password' => 'Secret9!Password', 'login_name' => 'root', 'auth_mode' => 'password', ], 'settings_fields' => ['hostname', 'password'], ]; Capsule::table('user')->insert([ 'id' => 1, 'enable' => 1, 'is_admin' => 0, ]); Capsule::table('cloud_instances')->insert([ 'id' => 501, 'uuid' => $uuid, 'host_id' => 7, 'user_id' => 1, 'name' => 'local-finalizer', 'status' => 'active', 'service_status' => 'exempt', 'network_status' => 'active', 'spec_json' => json_encode( ['managed' => true], JSON_THROW_ON_ERROR ), 'sealed_credential' => null, 'created_at' => $now - 3600, 'updated_at' => $now - 3600, ]); $originalEnvelope = CloudSecrets::encrypt($payload, 'job-payload'); $originalResult = json_encode( ['cause_code' => 'local_finalization_failed'], JSON_THROW_ON_ERROR ); Capsule::table('cloud_jobs')->insert([ 'id' => 501, 'operation_id' => $operationId, 'request_idempotency_key' => $operationId, 'request_fingerprint' => hash('sha256', $operationId), 'parent_job_id' => null, 'order_id' => null, 'invoice_id' => null, 'instance_id' => 501, 'snapshot_id' => null, 'user_id' => 1, 'host_id' => 7, 'kind' => 'instance_settings', 'execution_mode' => 'agent', 'phase' => 'terminal', 'outcome' => 'failed', 'status' => 'failed', 'sealed_payload' => $originalEnvelope, 'attempts' => 0, 'dispatch_attempts' => 0, 'dispatch_deadline_at' => $now - 30, 'terminal_at' => $now - 20, 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => 'not_applied', 'effect_resolution_reason' => '', 'lease_until' => 0, 'available_at' => 0, 'error_code' => 'local_finalization_failed', 'finalization_incident' => 1, 'finalization_error_code' => 'recovery_ledger_conflict', 'result_json' => $originalResult, 'created_at' => $now - 30, 'updated_at' => $now - 20, ]); $before = CloudJob::findOrFail(501); $assert(CloudJobService::blockingQueryForInstance(501)->exists(), 'An undispatched local finalizer incident did not block mutations.'); $service = new CloudAutomaticOperationRecoveryService(); $assert($service->tick(), 'The undispatched PHP-only finalizer was not claimed.'); $after = CloudJob::findOrFail(501); $scrubbed = CloudSecrets::decrypt( (string) $after->sealed_payload, 'job-payload' ); $assert((string) $after->phase === (string) $before->phase && (string) $after->outcome === (string) $before->outcome && (string) $after->status === (string) $before->status && (int) $after->terminal_at === (int) $before->terminal_at && (string) $after->error_code === (string) $before->error_code && (string) $after->result_json === (string) $before->result_json && $after->completion_event_id === $before->completion_event_id && !(bool) $after->effect_may_have_applied && (bool) $after->safe_to_resubmit && (string) $after->effect_resolution === 'not_applied' && (string) $after->effect_resolution_reason === 'automatic_local_not_dispatched' && (int) $after->effect_resolved_at > 0 && !(bool) $after->finalization_incident && $after->finalization_error_code === null && $after->lease_token === null && (int) $after->lease_until === 0 && (int) $after->available_at === 0 && (string) $after->sealed_payload !== $originalEnvelope && !isset($scrubbed['credential']) && !isset($scrubbed['body']['root_password']) && !CloudJobService::blockingQueryForInstance(501)->exists(), 'The PHP-only finalizer changed terminal audit data or failed to converge safely.'); $updatedAt = (int) $after->updated_at; $assert(!$service->tick(), 'A converged PHP-only finalizer was claimed a second time.'); $afterSecondTick = CloudJob::findOrFail(501); $assert((int) $afterSecondTick->updated_at === $updatedAt, 'A converged PHP-only finalizer repeated its side effects.'); // A missing/corrupt encrypted escrow remains a concrete payload incident; // it must not degrade into a generic ledger-conflict message. Capsule::table('cloud_instances')->insert([ 'id' => 502, 'uuid' => '00000000-0000-4000-8000-000000000502', 'host_id' => 7, 'user_id' => 1, 'name' => 'local-finalizer-corrupt-payload', 'status' => 'active', 'service_status' => 'exempt', 'network_status' => 'active', 'spec_json' => json_encode(['managed' => true], JSON_THROW_ON_ERROR), 'sealed_credential' => null, 'created_at' => $now - 3600, 'updated_at' => $now - 3600, ]); $corruptOperationId = str_repeat('ae', 16); Capsule::table('cloud_jobs')->insert([ 'id' => 502, 'operation_id' => $corruptOperationId, 'request_idempotency_key' => $corruptOperationId, 'request_fingerprint' => hash('sha256', $corruptOperationId), 'parent_job_id' => null, 'order_id' => null, 'invoice_id' => null, 'instance_id' => 502, 'snapshot_id' => null, 'user_id' => 1, 'host_id' => 7, 'kind' => 'instance_settings', 'execution_mode' => 'agent', 'phase' => 'terminal', 'outcome' => 'failed', 'status' => 'failed', 'sealed_payload' => 'undispatched-finalizer-test:missing', 'attempts' => 0, 'dispatch_attempts' => 0, 'dispatch_deadline_at' => $now - 30, 'terminal_at' => $now - 20, 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => 'not_applied', 'effect_resolution_reason' => '', 'lease_until' => 0, 'available_at' => 0, 'error_code' => 'local_finalization_failed', 'finalization_incident' => 1, 'finalization_error_code' => 'recovery_payload_unavailable', 'result_json' => $originalResult, 'created_at' => $now - 30, 'updated_at' => $now - 20, ]); $assert($service->tick(), 'The corrupt undispatched recovery payload was not checked.'); $corruptPayloadJob = CloudJob::findOrFail(502); $assert((bool) $corruptPayloadJob->finalization_incident && (string) $corruptPayloadJob->finalization_error_code === 'recovery_payload_unavailable' && (string) $corruptPayloadJob->sealed_payload === 'undispatched-finalizer-test:missing' && (int) $corruptPayloadJob->lease_until >= time() + 290 && (int) $corruptPayloadJob->available_at >= time() + 290 && CloudJobService::blockingQueryForInstance(502)->exists(), 'A corrupt PHP-only recovery payload lost its concrete incident or resource fence.'); // A create that never crossed the dispatch boundary has definitive // no-effect proof, but its paid ledgers still require the create-specific // atomic finalizer. Temporary key or database damage must retain escrow // and every fence; once repaired, the worker refunds/releases exactly // once and never sends the original create request to an Agent. $createOperationId = str_repeat('af', 16); $createPayload = [ 'method' => 'POST', 'request_target' => '/v1/vms', 'body' => [ 'operation_id' => $createOperationId, 'source' => 'cloud', 'name' => 'undispatched-create', 'root_password' => 'Secret9!Password', ], 'credential' => [ 'password' => 'Secret9!Password', 'login_name' => 'root', 'auth_mode' => 'password', ], ]; $createEnvelope = CloudSecrets::encrypt( $createPayload, 'job-payload' ); Capsule::table('user')->where('id', 1)->update(['money' => '0.00']); Capsule::table('cloud_hosts')->insert(['id' => 7]); Capsule::table('cloud_host_network_attachments')->insert([ 'id' => 601, 'host_id' => 7, 'line_id' => 601, 'network_selector' => 'bridge:br0', 'state' => 'active', 'priority' => 0, ]); $publicPool = [ 'host_id' => 7, 'network_attachment_id' => 601, 'gateway_on_link' => 0, 'reserved_ips_json' => '[]', 'allocation_cursor' => null, 'state' => 'active', 'priority' => 0, 'desired_revision' => 1, 'observed_revision' => 1, 'sync_state' => 'applied', ]; Capsule::table('cloud_public_ip_pools')->insert([ array_merge($publicPool, [ 'id' => 601, 'pool_key' => str_repeat('1', 32), 'address_family' => 4, 'cidr' => '8.8.8.0/24', 'gateway' => '8.8.8.1', 'guest_prefix_length' => 24, 'dns_servers_json' => '["1.1.1.1"]', 'allocation_start' => null, 'allocation_end' => null, 'max_allocations' => 0, ]), array_merge($publicPool, [ 'id' => 602, 'pool_key' => str_repeat('2', 32), 'address_family' => 6, 'cidr' => '2404:c140:156:3a::/64', 'gateway' => '2404:c140:156::1', 'gateway_on_link' => 1, 'guest_prefix_length' => 64, 'dns_servers_json' => '["2606:4700:4700::1111"]', 'allocation_start' => '2404:c140:156:3a::10', 'allocation_end' => '2404:c140:156:3a::ff', 'max_allocations' => 8, ]), ]); Capsule::table('cloud_offers')->insert([ 'id' => 601, 'stock' => 0, 'updated_at' => $now - 30, ]); Capsule::table('cloud_orders')->insert([ 'id' => 601, 'user_id' => 1, 'plan_id' => 601, 'offer_id' => 601, 'resource_pool_id' => 601, 'host_id' => 7, 'reservation_id' => 601, 'idempotency_key' => 'undispatched-create-order', 'request_hash' => hash('sha256', 'undispatched-create-order'), 'amount' => '12.34', 'currency' => 'USD', 'status' => 'queued', 'spec_json' => '{}', 'sealed_payload' => null, 'instance_id' => null, 'refunded_at' => null, 'stock_reserved_at' => $now - 60, 'stock_released_at' => null, 'created_at' => $now - 120, 'updated_at' => $now - 30, ]); Capsule::table('cloud_resource_reservations')->insert([ 'id' => 601, 'reservation_key' => 'undispatched-create-reservation', 'operation_id' => $createOperationId, 'order_id' => 601, 'backup_restore_id' => null, 'offer_id' => 601, 'resource_pool_id' => 601, 'host_id' => 7, 'user_id' => 1, 'instance_id' => null, 'network_attachment_id' => null, 'public_ip_pool_id' => null, 'public_ip_id' => null, 'cpu_cores' => 1, 'memory_mib' => 512, 'disk_gib' => 10, 'state' => 'held', 'expires_at' => null, 'released_at' => null, 'release_reason' => '', 'created_at' => $now - 120, 'updated_at' => $now - 30, ]); Capsule::table('cloud_public_ips')->insert([ [ 'id' => 601, 'host_id' => 7, 'public_ip_pool_id' => 601, 'address_family' => 4, 'address' => '8.8.8.61', 'address_binary' => inet_pton('8.8.8.61'), 'prefix_length' => 24, 'state' => 'reserved', 'reservation_id' => 601, 'reservation_ip_claim_id' => 601, 'instance_id' => null, 'operation_id' => $createOperationId, 'created_at' => $now - 120, 'updated_at' => $now - 30, ], [ 'id' => 602, 'host_id' => 7, 'public_ip_pool_id' => 602, 'address_family' => 6, 'address' => '2404:c140:156:3a::61', 'address_binary' => inet_pton('2404:c140:156:3a::61'), 'prefix_length' => 64, 'state' => 'reserved', 'reservation_id' => 601, 'reservation_ip_claim_id' => 602, 'instance_id' => null, 'operation_id' => $createOperationId, 'created_at' => $now - 120, 'updated_at' => $now - 30, ], ]); Capsule::table('cloud_resource_reservation_ips')->insert([ [ 'id' => 601, 'reservation_id' => 601, 'address_family' => 4, 'claim_generation' => 1, 'active_claim' => 1, 'network_attachment_id' => 601, 'public_ip_pool_id' => 601, 'public_ip_id' => 601, 'operation_id' => $createOperationId, 'state' => 'reserved', 'created_at' => $now - 120, 'updated_at' => $now - 30, 'released_at' => null, 'release_reason' => '', ], [ 'id' => 602, 'reservation_id' => 601, 'address_family' => 6, 'claim_generation' => 1, 'active_claim' => 1, 'network_attachment_id' => 601, 'public_ip_pool_id' => 602, 'public_ip_id' => 602, 'operation_id' => $createOperationId, 'state' => 'reserved', 'created_at' => $now - 120, 'updated_at' => $now - 30, 'released_at' => null, 'release_reason' => '', ], ]); Capsule::table('cloud_jobs')->insert([ 'id' => 601, 'operation_id' => $createOperationId, 'request_idempotency_key' => $createOperationId, 'request_fingerprint' => hash('sha256', $createOperationId), 'parent_job_id' => null, 'order_id' => 601, 'invoice_id' => null, 'instance_id' => null, 'snapshot_id' => null, 'user_id' => 1, 'host_id' => 7, 'kind' => 'create', 'execution_mode' => 'agent', 'phase' => 'accepted', 'outcome' => null, 'status' => 'queued', 'sealed_payload' => $createEnvelope, 'attempts' => 0, 'dispatch_attempts' => 0, 'dispatch_deadline_at' => $now - 30, 'dispatch_started_at' => null, 'agent_accepted_at' => null, 'receipt_deadline_at' => null, 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolved_by' => null, 'effect_resolution_reason' => '', 'completion_event_id' => null, 'read_at' => null, 'dispatched_at' => null, 'lease_token' => str_repeat('6', 32), 'lease_until' => $now + 180, 'available_at' => 0, 'error_code' => null, 'finalization_incident' => 0, 'finalization_error_code' => null, 'result_json' => $originalResult, 'created_at' => $now - 30, 'updated_at' => $now - 20, ]); CloudSecrets::setAvailable(false); $forceTerminalFailure = new ReflectionMethod( CloudWorkerService::class, 'forceTerminalFailure' ); $forceTerminalFailure->invoke( null, 601, str_repeat('6', 32), false, true, 'dispatch_not_sent' ); $createAfterEmergency = CloudJob::findOrFail(601); $assert((string) $createAfterEmergency->phase === 'terminal' && (string) $createAfterEmergency->outcome === 'failed' && (int) $createAfterEmergency->dispatch_attempts === 0 && !(bool) $createAfterEmergency->effect_may_have_applied && !(bool) $createAfterEmergency->safe_to_resubmit && (string) $createAfterEmergency->effect_resolution === 'not_applied' && (bool) $createAfterEmergency->finalization_incident && (string) $createAfterEmergency->finalization_error_code === 'recovery_payload_unavailable' && (string) $createAfterEmergency->sealed_payload === $createEnvelope && (int) $createAfterEmergency->available_at >= time() + 290, 'Create emergency terminalization lost its generation-zero proof or escrow.'); Capsule::table('cloud_jobs')->where('id', 601)->update([ 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $createService = new CloudAutomaticOperationRecoveryService(); $assert($createService->tick(), 'An undispatched create with unavailable escrow was not claimed.'); $createAfterKeyFailure = CloudJob::findOrFail(601); $assert((bool) $createAfterKeyFailure->finalization_incident && (string) $createAfterKeyFailure->finalization_error_code === 'recovery_payload_unavailable' && (string) $createAfterKeyFailure->sealed_payload === $createEnvelope && (string) Capsule::table('user')->where('id', 1) ->value('money') === '0.00' && (string) Capsule::table('cloud_resource_reservations') ->where('id', 601)->value('state') === 'held' && CloudAgentSettingsService::$requests === 0, 'Unavailable create escrow released money/resources or contacted Agent.'); CloudSecrets::setAvailable(true); Capsule::table('cloud_jobs')->where('id', 601)->update([ 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $assert($createService->tick(), 'An undispatched create with a missing invoice was not retried.'); $createAfterLedgerFailure = CloudJob::findOrFail(601); $assert((bool) $createAfterLedgerFailure->finalization_incident && (string) Capsule::table('user')->where('id', 1) ->value('money') === '0.00' && (string) Capsule::table('cloud_resource_reservations') ->where('id', 601)->value('state') === 'held' && Capsule::table('cloud_public_ips') ->where('reservation_id', 601) ->where('state', 'reserved')->count() === 2 && CloudAgentSettingsService::$requests === 0, 'Missing create ledger data caused a partial refund/release or Agent call.'); Capsule::table('cloud_invoices')->insert([ 'id' => 601, 'invoice_no' => 'VC-UNDISPATCHED-601', 'purpose' => 'provision', 'instance_id' => null, 'order_id' => 601, 'user_id' => 1, 'plan_id' => 601, 'offer_id' => 601, 'renewal_id' => null, 'renews_expires_at' => null, 'due_at' => $now - 120, 'amount' => '12.34', 'currency' => 'USD', 'validity_days' => 30, 'traffic_quota_bytes' => 0, 'status' => 'paid', 'paid_via' => 'balance', 'payment_reference' => 'undispatched-create-payment', 'paid_at' => $now - 60, 'created_at' => $now - 120, 'updated_at' => $now - 60, ]); Capsule::table('cloud_jobs')->where('id', 601)->update([ 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $beforeCreateAudit = CloudJob::findOrFail(601); $assert($createService->tick(), 'A repaired undispatched create was not finalized locally.'); $createAfterRecovery = CloudJob::findOrFail(601); $createSafePayload = CloudSecrets::decrypt( (string) $createAfterRecovery->sealed_payload, 'job-payload' ); $assert((string) $createAfterRecovery->phase === (string) $beforeCreateAudit->phase && (string) $createAfterRecovery->outcome === (string) $beforeCreateAudit->outcome && (int) $createAfterRecovery->terminal_at === (int) $beforeCreateAudit->terminal_at && (string) $createAfterRecovery->error_code === (string) $beforeCreateAudit->error_code && !(bool) $createAfterRecovery->effect_may_have_applied && !(bool) $createAfterRecovery->safe_to_resubmit && (string) $createAfterRecovery->effect_resolution === 'not_applied' && (string) $createAfterRecovery->effect_resolution_reason === 'automatic_local_not_dispatched' && !(bool) $createAfterRecovery->finalization_incident && $createAfterRecovery->finalization_error_code === null && !isset($createSafePayload['credential']) && !isset($createSafePayload['body']['root_password']) && (string) Capsule::table('user')->where('id', 1) ->value('money') === '12.34' && (string) Capsule::table('cloud_invoices')->where('id', 601) ->value('status') === 'refunded' && Capsule::table('cloud_orders')->where('id', 601) ->value('refunded_at') !== null && Capsule::table('cloud_orders')->where('id', 601) ->value('stock_released_at') !== null && (int) Capsule::table('cloud_offers')->where('id', 601) ->value('stock') === 1 && (string) Capsule::table('cloud_resource_reservations') ->where('id', 601)->value('state') === 'released' && Capsule::table('cloud_public_ips') ->whereIn('id', [601, 602]) ->where('state', 'available')->count() === 2 && Capsule::table('cloud_public_ips') ->whereIn('id', [601, 602]) ->whereNotNull('reservation_id')->count() === 0 && Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', 601) ->where('state', 'released')->count() === 2 && CloudAgentSettingsService::$requests === 0, 'Recovered create did not atomically refund, release, scrub, and close.'); $createUpdatedAt = (int) $createAfterRecovery->updated_at; $assert(!$createService->tick(), 'A converged undispatched create was claimed a second time.'); $assert((string) Capsule::table('user')->where('id', 1) ->value('money') === '12.34' && (int) Capsule::table('cloud_offers')->where('id', 601) ->value('stock') === 1 && (int) CloudJob::findOrFail(601)->updated_at === $createUpdatedAt && CloudAgentSettingsService::$requests === 0, 'Undispatched create recovery repeated refund/release or contacted Agent.'); echo "Cloud undispatched finalizer recovery tests passed.\n"; }