httpStatus === 409, $message . ': HTTP ' . $error->httpStatus); return; } throw new RuntimeException($message . ': request was accepted'); }; $capsule = new Capsule(); $capsule->addConnection([ 'driver' => 'sqlite', 'database' => ':memory:', ], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('cloud_hosts', static function (Blueprint $table): void { $table->increments('id'); $table->string('name'); $table->string('region')->default(''); $table->boolean('enabled')->default(false); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_host_agents', static function (Blueprint $table): void { $table->integer('host_id')->primary(); $table->boolean('enabled')->default(false); }); $schema->create('cloud_host_resource_pool_memberships', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('host_id'); $table->integer('resource_pool_id'); $table->string('scheduler_state'); $table->integer('priority')->default(0); $table->integer('weight')->default(100); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_instances', static function (Blueprint $table): void { $table->increments('id'); $table->string('uuid'); $table->integer('host_id'); $table->integer('user_id'); $table->string('name'); $table->string('status'); $table->string('service_status'); $table->string('network_status'); $table->text('spec_json'); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_resource_reservations', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('host_id'); $table->string('state'); }); $schema->create('cloud_instance_migrations', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('instance_id'); $table->integer('source_host_id'); $table->integer('target_host_id'); $table->string('state'); }); $schema->create('cloud_vpcs', static function (Blueprint $table): void { $table->increments('id'); $table->string('vpc_key'); $table->integer('user_id'); $table->integer('region_id'); $table->integer('cidr_pool_id'); $table->string('name'); $table->string('cidr'); $table->integer('prefix_length'); $table->string('gateway'); $table->integer('active_claim')->nullable(); $table->string('state'); $table->integer('desired_revision'); $table->integer('deleted_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_vpc_ports', static function (Blueprint $table): void { $table->increments('id'); $table->string('port_key'); $table->integer('vpc_id'); $table->integer('instance_id'); $table->integer('migration_host_id')->nullable(); $table->integer('user_id'); $table->string('private_ip'); $table->string('mac_address'); $table->integer('active_claim')->nullable(); $table->string('state'); $table->integer('desired_revision'); $table->integer('applied_revision'); $table->string('last_operation_id')->nullable(); $table->string('last_error_code')->default(''); $table->integer('attached_at')->nullable(); $table->integer('released_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_vpc_host_states', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('vpc_id'); $table->integer('host_id'); $table->integer('desired_revision'); $table->integer('applied_revision'); $table->string('desired_digest'); $table->string('applied_digest')->nullable(); $table->string('sync_state'); $table->string('last_operation_id')->nullable(); $table->string('last_error_code')->default(''); $table->integer('last_success_at')->default(0); $table->integer('last_failure_at')->default(0); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_vpc_membership_batches', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('batch_key'); $table->integer('vpc_id'); $table->integer('user_id'); $table->integer('parent_job_id'); $table->string('kind'); $table->string('state'); $table->integer('desired_revision'); $table->integer('active_claim')->nullable(); $table->string('last_error_code')->default(''); $table->integer('terminal_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_vpc_membership_batch_items', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('batch_id'); $table->integer('user_id'); $table->integer('instance_id'); $table->integer('vpc_port_id'); $table->integer('host_id'); $table->integer('child_job_id')->nullable(); $table->string('state'); $table->string('last_error_code')->default(''); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('operation_id'); $table->string('request_idempotency_key'); $table->string('request_fingerprint'); $table->integer('parent_job_id')->nullable(); $table->integer('supersedes_job_id')->nullable(); $table->integer('vpc_repair_root_job_id')->nullable(); $table->integer('order_id')->nullable(); $table->integer('invoice_id')->nullable(); $table->integer('instance_id')->nullable(); $table->integer('snapshot_id')->nullable(); $table->integer('backup_id')->nullable(); $table->integer('backup_restore_id')->nullable(); $table->integer('backup_recovery_id')->nullable(); $table->integer('vpc_id')->nullable(); $table->integer('vpc_port_id')->nullable(); $table->integer('vpc_batch_id')->nullable(); $table->integer('user_id'); $table->integer('host_id')->nullable(); $table->string('kind'); $table->string('execution_mode'); $table->string('phase'); $table->string('outcome')->nullable(); $table->string('status'); $table->text('sealed_payload'); $table->integer('attempts'); $table->integer('dispatch_attempts'); $table->integer('dispatch_deadline_at'); $table->integer('dispatch_started_at')->nullable(); $table->integer('agent_accepted_at')->nullable(); $table->integer('receipt_deadline_at')->nullable(); $table->integer('terminal_at')->nullable(); $table->boolean('effect_may_have_applied')->nullable(); $table->boolean('safe_to_resubmit')->nullable(); $table->string('effect_resolution'); $table->integer('effect_resolved_at')->nullable(); $table->integer('effect_resolved_by')->nullable(); $table->text('effect_resolution_reason')->default(''); $table->boolean('finalization_incident')->default(false); $table->string('finalization_error_code')->nullable(); $table->string('completion_event_id')->nullable(); $table->integer('read_at')->nullable(); $table->integer('dispatched_at')->nullable(); $table->string('lease_token')->nullable(); $table->integer('lease_until')->default(0); $table->integer('available_at'); $table->string('error_code')->nullable(); $table->text('result_json')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $now = time(); $actor = [ 'user_id' => 99, 'is_admin' => true, 'is_super_admin' => true, 'can_manage' => true, ]; Capsule::table('cloud_hosts')->insert([ ['id' => 1, 'name' => 'retiring-host', 'region' => 'hk', 'enabled' => 1, 'created_at' => $now, 'updated_at' => $now], ['id' => 2, 'name' => 'surviving-host', 'region' => 'hk', 'enabled' => 1, 'created_at' => $now, 'updated_at' => $now], ]); Capsule::table('cloud_host_agents')->insert([ 'host_id' => 1, 'enabled' => 1, ]); Capsule::table('cloud_host_resource_pool_memberships')->insert([ 'id' => 1, 'host_id' => 1, 'resource_pool_id' => 1, 'scheduler_state' => 'active', 'priority' => 0, 'weight' => 100, 'created_at' => $now, 'updated_at' => $now, ]); $instanceRows = [ ['id' => 10, 'uuid' => '11111111-1111-4111-8111-111111111110', 'host_id' => 1, 'user_id' => 7, 'name' => 'retired-reference', 'status' => 'deleted'], ['id' => 11, 'uuid' => '11111111-1111-4111-8111-111111111111', 'host_id' => 1, 'user_id' => 7, 'name' => 'host-blocker', 'status' => 'active'], ]; foreach ($instanceRows as &$row) { $row += ['service_status' => 'active', 'network_status' => 'active', 'spec_json' => '{}', 'created_at' => $now, 'updated_at' => $now]; } unset($row); Capsule::table('cloud_instances')->insert($instanceRows); Capsule::table('cloud_resource_reservations')->insert([ 'id' => 1, 'host_id' => 1, 'state' => 'held', ]); Capsule::table('cloud_instance_migrations')->insert([ 'id' => 1, 'instance_id' => 11, 'source_host_id' => 1, 'target_host_id' => 2, 'state' => 'preparing_source', ]); Capsule::table('cloud_vpcs')->insert([ 'id' => 20, 'vpc_key' => str_repeat('a', 32), 'user_id' => 7, 'region_id' => 1, 'cidr_pool_id' => 1, 'name' => 'retire-test', 'cidr' => '10.10.0.0/24', 'prefix_length' => 24, 'gateway' => '10.10.0.1', 'active_claim' => 1, 'state' => 'provisioning', 'desired_revision' => 3, 'deleted_at' => null, 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_vpc_ports')->insert([ [ 'id' => 30, 'port_key' => str_repeat('b', 32), 'vpc_id' => 20, 'instance_id' => 10, 'migration_host_id' => null, 'user_id' => 7, 'private_ip' => '10.10.0.2', 'mac_address' => '02:00:00:00:00:02', 'active_claim' => 1, 'state' => 'release_pending', 'desired_revision' => 3, 'applied_revision' => 2, 'last_operation_id' => null, 'last_error_code' => '', 'attached_at' => $now - 100, 'released_at' => null, 'created_at' => $now, 'updated_at' => $now, ], [ 'id' => 31, 'port_key' => str_repeat('c', 32), 'vpc_id' => 20, 'instance_id' => 11, 'migration_host_id' => null, 'user_id' => 7, 'private_ip' => '10.10.0.3', 'mac_address' => '02:00:00:00:00:03', 'active_claim' => 1, 'state' => 'attached', 'desired_revision' => 2, 'applied_revision' => 2, 'last_operation_id' => null, 'last_error_code' => '', 'attached_at' => $now - 100, 'released_at' => null, 'created_at' => $now, 'updated_at' => $now, ], ]); $rootOperation = str_repeat('d', 32); $tipOperation = str_repeat('e', 32); Capsule::table('cloud_vpc_host_states')->insert([ 'id' => 40, 'vpc_id' => 20, 'host_id' => 1, 'desired_revision' => 3, 'applied_revision' => 2, 'desired_digest' => str_repeat('f', 64), 'applied_digest' => str_repeat('1', 64), 'sync_state' => 'unknown', 'last_operation_id' => $tipOperation, 'last_error_code' => 'operation_effect_unknown', 'last_success_at' => $now - 100, 'last_failure_at' => $now - 10, 'created_at' => $now, 'updated_at' => $now, ]); $job = static function ( int $id, string $operationId, ?int $supersedes, ?int $root ) use ($now): array { return [ 'id' => $id, 'operation_id' => $operationId, 'request_idempotency_key' => $operationId, 'request_fingerprint' => hash('sha256', $operationId), 'parent_job_id' => null, 'supersedes_job_id' => $supersedes, 'vpc_repair_root_job_id' => $root, 'order_id' => null, 'invoice_id' => null, 'instance_id' => 10, 'snapshot_id' => null, 'backup_id' => null, 'backup_restore_id' => null, 'backup_recovery_id' => null, 'vpc_id' => 20, 'vpc_port_id' => 30, 'vpc_batch_id' => null, 'user_id' => 7, 'host_id' => 1, 'kind' => 'vpc_host_remove', 'execution_mode' => 'agent', 'phase' => 'terminal', 'outcome' => 'failed', 'status' => 'failed', 'sealed_payload' => '{}', 'attempts' => 1, 'dispatch_attempts' => 1, 'dispatch_deadline_at' => $now - 200, 'dispatch_started_at' => $now - 190, 'agent_accepted_at' => $now - 185, 'receipt_deadline_at' => $now - 100, 'terminal_at' => $now - 20, 'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0, 'effect_resolution' => 'unknown', 'effect_resolved_at' => null, 'effect_resolved_by' => null, 'effect_resolution_reason' => '', 'finalization_incident' => 0, 'finalization_error_code' => null, 'completion_event_id' => null, 'read_at' => null, 'dispatched_at' => $now - 185, 'lease_token' => null, 'lease_until' => 0, 'available_at' => 0, 'error_code' => 'operation_effect_unknown', 'result_json' => null, 'created_at' => $now - 200, 'updated_at' => $now - 20, ]; }; Capsule::table('cloud_jobs')->insert([ $job(100, $rootOperation, null, null), $job(101, $tipOperation, 100, 100), array_replace( $job(102, str_repeat('9', 32), 101, 100), [ 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 1, 'effect_resolution' => 'not_applied', 'error_code' => 'agent_rejected', ] ), ]); $retire = static fn (): array => CloudVpcHostRetirementService::retire( $actor, 101, 'The physical node was decommissioned after all tenant data was removed.' ); $reject($retire, 'enabled host must block permanent retirement'); Capsule::table('cloud_hosts')->where('id', 1)->update(['enabled' => 0]); $reject($retire, 'enabled Agent must block permanent retirement'); Capsule::table('cloud_host_agents')->where('host_id', 1) ->update(['enabled' => 0]); $reject($retire, 'active scheduler membership must block retirement'); Capsule::table('cloud_host_resource_pool_memberships')->where('id', 1) ->update(['scheduler_state' => 'disabled']); $reject($retire, 'nondeleted instance must block permanent retirement'); Capsule::table('cloud_instances')->where('id', 11) ->update(['status' => 'deleted']); $reject($retire, 'active resource reservation must block retirement'); Capsule::table('cloud_resource_reservations')->where('id', 1) ->update(['state' => 'released']); $reject($retire, 'active maintenance migration must block retirement'); Capsule::table('cloud_instance_migrations')->where('id', 1) ->update(['state' => 'completed']); $reject($retire, 'attached VPC port must block permanent retirement'); Capsule::table('cloud_vpc_ports')->where('id', 31)->update([ 'active_claim' => null, 'state' => 'released', 'released_at' => $now, ]); Capsule::table('cloud_jobs')->where('id', 102)->update([ 'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0, 'effect_resolution' => 'unknown', ]); $reject($retire, 'parallel unknown repair generation must block retirement'); Capsule::table('cloud_jobs')->where('id', 102)->update([ 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 1, 'effect_resolution' => 'not_applied', ]); Capsule::table('cloud_jobs')->where('id', 101)->update([ 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 1, 'effect_resolution' => 'not_applied', ]); $reject($retire, 'known no-effect job must not be rewritten as applied'); Capsule::table('cloud_jobs')->where('id', 101)->update([ 'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0, 'effect_resolution' => 'unknown', ]); $result = $retire(); $assert($result['retired'] === true && $result['task_id'] === '101' && $result['host_id'] === '1' && $result['vpc_id'] === '20' && $result['resolved_generations'] === 2, 'retirement result must identify the exact converged chain'); $assert(Capsule::table('cloud_vpc_host_states')->where('id', 40) ->doesntExist(), 'retired host projection must be deleted'); $resolved = CloudJob::query()->whereIn('id', [100, 101]) ->orderBy('id')->get(); $assert($resolved->count() === 2, 'repair chain audit rows must be retained'); foreach ($resolved as $generation) { $assert((string) $generation->outcome === 'failed' && (string) $generation->status === 'failed' && (string) $generation->effect_resolution === 'applied' && (int) $generation->effect_resolved_by === 99 && str_starts_with( (string) $generation->effect_resolution_reason, 'host_permanently_retired:' ), 'each failed generation must retain outcome and gain audit proof'); } $knownNoEffectSibling = CloudJob::find(102); $assert($knownNoEffectSibling !== null && (string) $knownNoEffectSibling->effect_resolution === 'not_applied' && $knownNoEffectSibling->effect_resolved_by === null, 'known no-effect repair sibling must retain its original conclusion'); $port = Capsule::table('cloud_vpc_ports')->where('id', 30)->first(); $vpc = Capsule::table('cloud_vpcs')->where('id', 20)->first(); $assert($port !== null && $port->active_claim === null && $port->state === 'released' && (int) $port->released_at > 0, 'reference VPC port must finish its pending release'); $assert($vpc !== null && $vpc->state === 'active', 'VPC must converge after its last host projection is retired'); Capsule::table('cloud_instances')->insert([ 'id' => 12, 'uuid' => '11111111-1111-4111-8111-111111111112', 'host_id' => 1, 'user_id' => 7, 'name' => 'batch-reference', 'status' => 'deleted', 'service_status' => 'active', 'network_status' => 'active', 'spec_json' => '{}', 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_vpcs')->insert([ 'id' => 21, 'vpc_key' => str_repeat('4', 32), 'user_id' => 7, 'region_id' => 1, 'cidr_pool_id' => 1, 'name' => 'batch-retire-test', 'cidr' => '10.20.0.0/24', 'prefix_length' => 24, 'gateway' => '10.20.0.1', 'active_claim' => 1, 'state' => 'degraded', 'desired_revision' => 4, 'deleted_at' => null, 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_vpc_ports')->insert([ 'id' => 32, 'port_key' => str_repeat('5', 32), 'vpc_id' => 21, 'instance_id' => 12, 'migration_host_id' => null, 'user_id' => 7, 'private_ip' => '10.20.0.2', 'mac_address' => '02:00:00:00:00:04', 'active_claim' => 1, 'state' => 'release_pending', 'desired_revision' => 4, 'applied_revision' => 3, 'last_operation_id' => null, 'last_error_code' => '', 'attached_at' => $now - 100, 'released_at' => null, 'created_at' => $now, 'updated_at' => $now, ]); $batchOperation = str_repeat('6', 32); Capsule::table('cloud_vpc_host_states')->insert([ 'id' => 41, 'vpc_id' => 21, 'host_id' => 1, 'desired_revision' => 4, 'applied_revision' => 3, 'desired_digest' => str_repeat('7', 64), 'applied_digest' => str_repeat('8', 64), 'sync_state' => 'unknown', 'last_operation_id' => $batchOperation, 'last_error_code' => 'operation_effect_unknown', 'last_success_at' => $now - 100, 'last_failure_at' => $now - 10, 'created_at' => $now, 'updated_at' => $now, ]); $batchPrefix = CloudVpcPortService::batchProjectionRequestPrefix(50, 4); $parent = array_replace($job(200, str_repeat('2', 32), null, null), [ 'request_idempotency_key' => str_repeat('2', 32), 'instance_id' => null, 'vpc_id' => 21, 'vpc_port_id' => null, 'vpc_batch_id' => null, 'host_id' => null, 'kind' => 'vpc_membership_batch', 'execution_mode' => 'local', 'phase' => 'accepted', 'outcome' => null, 'status' => 'queued', 'attempts' => 0, 'dispatch_attempts' => 0, 'dispatch_deadline_at' => $now + 3600, 'dispatch_started_at' => null, 'agent_accepted_at' => null, 'receipt_deadline_at' => null, 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'dispatched_at' => null, 'error_code' => null, ]); $batchChild = array_replace( $job(201, $batchOperation, null, null), [ 'request_idempotency_key' => $batchPrefix . 'vpc_host_remove-1-' . $batchOperation, 'instance_id' => 12, 'vpc_id' => 21, 'vpc_port_id' => 32, 'vpc_batch_id' => 50, ] ); Capsule::table('cloud_jobs')->insert([$parent, $batchChild]); Capsule::table('cloud_vpc_membership_batches')->insert([ 'id' => 50, 'batch_key' => str_repeat('3', 32), 'vpc_id' => 21, 'user_id' => 7, 'parent_job_id' => 200, 'kind' => 'detach', 'state' => 'blocked', 'desired_revision' => 4, 'active_claim' => 1, 'last_error_code' => 'vpc_result_uncertain', 'terminal_at' => null, 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_vpc_membership_batch_items')->insert([ 'id' => 60, 'batch_id' => 50, 'user_id' => 7, 'instance_id' => 12, 'vpc_port_id' => 32, 'host_id' => 1, 'child_job_id' => null, 'state' => 'succeeded', 'last_error_code' => '', 'created_at' => $now, 'updated_at' => $now, ]); $batchResult = CloudVpcHostRetirementService::retire( $actor, 201, 'The second physical node projection was retired after the batch was audited.' ); $retiredBatch = Capsule::table('cloud_vpc_membership_batches') ->where('id', 50)->first(); $retiredParent = CloudJob::find(200); $retiredBatchPort = Capsule::table('cloud_vpc_ports') ->where('id', 32)->first(); $assert($batchResult['retired'] === true && $batchResult['resolved_generations'] === 1, 'blocked batch host retirement must resolve one exact child generation'); $assert($retiredBatch !== null && $retiredBatch->state === 'succeeded' && $retiredBatch->active_claim === null && (int) $retiredBatch->terminal_at > 0, 'blocked membership batch must reach a terminal success state'); $assert($retiredParent !== null && (string) $retiredParent->phase === 'terminal' && (string) $retiredParent->outcome === 'succeeded', 'membership batch parent task must converge with the retired child'); $assert($retiredBatchPort !== null && $retiredBatchPort->active_claim === null && $retiredBatchPort->state === 'released', 'batch release-pending port must be released after host retirement'); $root = dirname(__DIR__); $routes = file_get_contents($root . '/config/routes_cloud.php'); $controller = file_get_contents( $root . '/app/Controllers/CloudAdminOperationsController.php' ); $recovery = file_get_contents( $root . '/app/Services/Cloud/CloudAdminOperationRecoveryService.php' ); $script = file_get_contents( $root . '/public/cloud/admin-operations-recovery.js' ); $view = file_get_contents( $root . '/resources/views/venix/admin/cloud/operations.tpl' ); $assert(is_string($routes) && is_string($controller) && is_string($recovery) && is_string($script) && is_string($view) && str_contains($routes, '/retire-vpc-host') && str_contains($controller, 'function retireVpcHost(') && str_contains($recovery, 'function retireVpcHost(') && str_contains($script, 'async function retireVpcHost(') && str_contains($view, 'operation-detail-retire-vpc-host') && str_contains($view, 'operation-vpc-retirement-reason') && str_contains($script, 'reason: reason.value') && !str_contains($script, 'window.prompt('), 'permanent host-retirement route, controller, UI, or service wiring is incomplete'); echo "CloudVpcHostRetirementServiceTest passed\n"; }