httpStatus === $status, $message . ' (unexpected HTTP status)'); return; } throw new RuntimeException($message . ' (no CloudError)'); }; $capsule = new Capsule(); $capsule->addConnection([ 'driver' => 'sqlite', 'database' => ':memory:', ], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('cloud_hosts', static function (Blueprint $table): void { $table->increments('id'); $table->string('name'); $table->boolean('enabled')->default(true); $table->integer('created_at')->default(0); $table->integer('updated_at')->default(0); }); $schema->create('cloud_host_agents', static function ( Blueprint $table ): void { $table->integer('host_id')->primary(); $table->text('sealed_api_secret')->default('sealed'); $table->boolean('enabled'); $table->integer('revision'); $table->integer('tested_revision'); $table->string('last_test_status'); $table->string('last_error_code')->default(''); $table->text('network_selectors_json'); $table->integer('last_tested_at')->default(0); $table->integer('created_at')->default(0); $table->integer('updated_at')->default(0); }); $schema->create('cloud_host_network_attachments', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('host_id'); $table->integer('line_id'); $table->string('network_selector'); $table->string('state'); $table->integer('priority')->default(0); $table->string('verification_state'); $table->integer('verified_agent_revision'); $table->integer('last_verified_at'); $table->integer('created_at')->default(0); $table->integer('updated_at')->default(0); }); $schema->create('cloud_public_ip_pools', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('pool_key', 32)->unique(); $table->integer('host_id'); $table->integer('network_attachment_id'); $table->string('name'); $table->string('cidr'); $table->string('gateway'); $table->integer('guest_prefix_length'); $table->text('dns_servers_json'); $table->text('reserved_ips_json'); $table->string('state'); $table->integer('priority')->default(0); $table->integer('desired_revision'); $table->integer('observed_revision'); $table->string('sync_state'); $table->string('last_error_code')->default(''); $table->integer('created_at')->default(0); $table->integer('updated_at')->default(0); }); $schema->create('cloud_resource_pools', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('line_id'); }); $schema->create('cloud_resource_reservations', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('operation_id', 32)->unique(); $table->integer('resource_pool_id'); $table->integer('host_id'); $table->integer('user_id'); $table->integer('instance_id')->nullable(); $table->integer('network_attachment_id'); $table->integer('public_ip_pool_id'); $table->integer('public_ip_id'); $table->string('state'); $table->integer('created_at')->default(0); $table->integer('updated_at')->default(0); }); $schema->create('cloud_instances', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('uuid'); $table->string('name'); $table->integer('user_id'); $table->integer('host_id'); $table->integer('resource_pool_id'); $table->integer('reservation_id'); $table->string('status'); $table->text('spec_json'); $table->integer('created_at')->default(0); $table->integer('updated_at')->default(0); }); $schema->create('cloud_public_ips', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('host_id'); $table->integer('public_ip_pool_id'); $table->string('address'); $table->integer('prefix_length'); $table->string('state'); $table->integer('reservation_id')->nullable(); $table->integer('instance_id')->nullable(); $table->string('operation_id', 32)->nullable(); $table->integer('created_at')->default(0); $table->integer('updated_at')->default(0); }); $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('operation_id', 32)->unique(); $table->string('request_idempotency_key', 128); $table->string('request_fingerprint', 64); $table->integer('parent_job_id')->nullable(); $table->integer('supersedes_job_id')->nullable(); $table->integer('vpc_repair_root_job_id')->nullable(); $table->integer('order_id')->nullable(); $table->integer('invoice_id')->nullable(); $table->integer('instance_id')->nullable(); $table->integer('snapshot_id')->nullable(); $table->integer('backup_id')->nullable(); $table->integer('backup_restore_id')->nullable(); $table->integer('backup_recovery_id')->nullable(); $table->integer('vpc_id')->nullable(); $table->integer('vpc_port_id')->nullable(); $table->integer('vpc_batch_id')->nullable(); $table->integer('user_id'); $table->integer('host_id')->nullable(); $table->string('kind'); $table->string('execution_mode'); $table->string('phase'); $table->string('outcome')->nullable(); $table->string('status'); $table->text('sealed_payload'); $table->integer('attempts'); $table->integer('dispatch_attempts'); $table->integer('dispatch_deadline_at'); $table->integer('dispatch_started_at')->nullable(); $table->integer('agent_accepted_at')->nullable(); $table->integer('receipt_deadline_at')->nullable(); $table->integer('terminal_at')->nullable(); $table->boolean('effect_may_have_applied')->nullable(); $table->boolean('safe_to_resubmit')->nullable(); $table->string('effect_resolution'); $table->integer('effect_resolved_at')->nullable(); $table->integer('effect_resolved_by')->nullable(); $table->text('effect_resolution_reason')->nullable(); $table->boolean('finalization_incident')->default(false); $table->string('finalization_error_code')->nullable(); $table->string('completion_event_id')->nullable(); $table->integer('read_at')->nullable(); $table->integer('dispatched_at')->nullable(); $table->string('lease_token')->nullable(); $table->integer('lease_until')->default(0); $table->integer('available_at'); $table->string('error_code')->nullable(); $table->text('result_json')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); $table->unique(['user_id', 'request_idempotency_key']); }); $schema->create('cloud_public_ip_pool_replacement_batches', static function (Blueprint $table): void { $table->increments('id'); $table->string('batch_key', 32)->unique(); $table->integer('host_id'); $table->integer('source_public_ip_pool_id'); $table->integer('target_public_ip_pool_id'); $table->string('source_initial_state'); $table->integer('parent_job_id')->unique(); $table->integer('requested_by_user_id'); $table->string('state'); $table->integer('active_claim')->nullable(); $table->string('last_error_code')->default(''); $table->integer('terminal_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_public_ip_pool_replacement_batch_items', static function (Blueprint $table): void { $table->increments('id'); $table->integer('batch_id'); $table->integer('instance_id'); $table->integer('source_reservation_id'); $table->integer('source_public_ip_id'); $table->integer('target_public_ip_id'); $table->integer('source_network_attachment_id'); $table->integer('target_network_attachment_id'); $table->integer('child_job_id')->unique(); $table->string('state'); $table->string('last_error_code')->default(''); $table->integer('created_at'); $table->integer('updated_at'); $table->unique(['batch_id', 'target_public_ip_id']); }); $now = time(); Capsule::table('cloud_hosts')->insert([ 'id' => 1, 'name' => 'HKG-A', 'enabled' => 1, 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_host_agents')->insert([ 'host_id' => 1, 'enabled' => 1, 'revision' => 7, 'tested_revision' => 7, 'last_test_status' => 'healthy', 'network_selectors_json' => json_encode([ 'bridge:cross-line', 'bridge:other-source', 'bridge:source', 'bridge:target', ], JSON_THROW_ON_ERROR), 'last_tested_at' => $now, 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_host_network_attachments')->insert([ ['id' => 11, 'host_id' => 1, 'line_id' => 10, 'network_selector' => 'bridge:source', 'state' => 'active', 'priority' => 10, 'verification_state' => 'verified', 'verified_agent_revision' => 7, 'last_verified_at' => $now], ['id' => 12, 'host_id' => 1, 'line_id' => 10, 'network_selector' => 'bridge:target', 'state' => 'active', 'priority' => 20, 'verification_state' => 'verified', 'verified_agent_revision' => 7, 'last_verified_at' => $now], ['id' => 13, 'host_id' => 1, 'line_id' => 10, 'network_selector' => 'bridge:other-source', 'state' => 'active', 'priority' => 30, 'verification_state' => 'verified', 'verified_agent_revision' => 7, 'last_verified_at' => $now], ['id' => 14, 'host_id' => 1, 'line_id' => 20, 'network_selector' => 'bridge:cross-line', 'state' => 'active', 'priority' => 40, 'verification_state' => 'verified', 'verified_agent_revision' => 7, 'last_verified_at' => $now], ]); Capsule::table('cloud_resource_pools')->insert([ ['id' => 21, 'line_id' => 10], ]); $pool = static function ( int $id, int $attachmentId, string $keyDigit, string $name, string $cidr, string $gateway, int $revision ): array { return [ 'id' => $id, 'pool_key' => str_repeat($keyDigit, 32), 'host_id' => 1, 'network_attachment_id' => $attachmentId, 'name' => $name, 'cidr' => $cidr, 'gateway' => $gateway, 'guest_prefix_length' => 29, 'dns_servers_json' => json_encode(['1.1.1.1']), 'reserved_ips_json' => '[]', 'state' => 'active', 'priority' => $id, 'desired_revision' => $revision, 'observed_revision' => $revision, 'sync_state' => 'applied', 'last_error_code' => '', ]; }; Capsule::table('cloud_public_ip_pools')->insert([ $pool(31, 11, 'a', 'Source A', '198.51.100.0/29', '198.51.100.1', 7), $pool(32, 12, 'b', 'Target B', '198.51.100.8/29', '198.51.100.9', 9), $pool(33, 13, 'c', 'Source D', '198.51.100.16/29', '198.51.100.17', 4), $pool(34, 14, 'd', 'Cross Line', '198.51.100.24/29', '198.51.100.25', 3), ]); $reservationOperation = static fn (int $id): string => str_pad( dechex($id), 32, '0', STR_PAD_LEFT ); Capsule::table('cloud_resource_reservations')->insert([ ['id' => 41, 'operation_id' => $reservationOperation(41), 'resource_pool_id' => 21, 'host_id' => 1, 'user_id' => 501, 'instance_id' => 51, 'network_attachment_id' => 11, 'public_ip_pool_id' => 31, 'public_ip_id' => 61, 'state' => 'committed'], ['id' => 42, 'operation_id' => $reservationOperation(42), 'resource_pool_id' => 21, 'host_id' => 1, 'user_id' => 502, 'instance_id' => 52, 'network_attachment_id' => 11, 'public_ip_pool_id' => 31, 'public_ip_id' => 62, 'state' => 'committed'], ['id' => 43, 'operation_id' => $reservationOperation(43), 'resource_pool_id' => 21, 'host_id' => 1, 'user_id' => 503, 'instance_id' => 53, 'network_attachment_id' => 13, 'public_ip_pool_id' => 33, 'public_ip_id' => 63, 'state' => 'committed'], ]); $instanceSpec = static function (string $publicIp): string { return json_encode([ 'managed' => true, 'os_source' => 'cloud_template', 'public_ip_cidr' => $publicIp . '/29', ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES); }; Capsule::table('cloud_instances')->insert([ ['id' => 51, 'uuid' => '11111111-1111-4111-8111-111111111111', 'name' => 'vm-a1', 'user_id' => 501, 'host_id' => 1, 'resource_pool_id' => 21, 'reservation_id' => 41, 'status' => 'active', 'spec_json' => $instanceSpec('198.51.100.2')], ['id' => 52, 'uuid' => '22222222-2222-4222-8222-222222222222', 'name' => 'vm-a2', 'user_id' => 502, 'host_id' => 1, 'resource_pool_id' => 21, 'reservation_id' => 42, 'status' => 'active', 'spec_json' => $instanceSpec('198.51.100.3')], ['id' => 53, 'uuid' => '33333333-3333-4333-8333-333333333333', 'name' => 'vm-d1', 'user_id' => 503, 'host_id' => 1, 'resource_pool_id' => 21, 'reservation_id' => 43, 'status' => 'active', 'spec_json' => $instanceSpec('198.51.100.18')], ]); Capsule::table('cloud_public_ips')->insert([ ['id' => 61, 'host_id' => 1, 'public_ip_pool_id' => 31, 'address' => '198.51.100.2', 'prefix_length' => 29, 'state' => 'assigned', 'reservation_id' => 41, 'instance_id' => 51, 'operation_id' => $reservationOperation(41)], ['id' => 62, 'host_id' => 1, 'public_ip_pool_id' => 31, 'address' => '198.51.100.3', 'prefix_length' => 29, 'state' => 'assigned', 'reservation_id' => 42, 'instance_id' => 52, 'operation_id' => $reservationOperation(42)], ['id' => 63, 'host_id' => 1, 'public_ip_pool_id' => 33, 'address' => '198.51.100.18', 'prefix_length' => 29, 'state' => 'assigned', 'reservation_id' => 43, 'instance_id' => 53, 'operation_id' => $reservationOperation(43)], ['id' => 71, 'host_id' => 1, 'public_ip_pool_id' => 32, 'address' => '198.51.100.10', 'prefix_length' => 29, 'state' => 'available', 'reservation_id' => null, 'instance_id' => null, 'operation_id' => null], ['id' => 72, 'host_id' => 1, 'public_ip_pool_id' => 32, 'address' => '198.51.100.11', 'prefix_length' => 29, 'state' => 'available', 'reservation_id' => null, 'instance_id' => null, 'operation_id' => null], ['id' => 73, 'host_id' => 1, 'public_ip_pool_id' => 32, 'address' => '198.51.100.12', 'prefix_length' => 29, 'state' => 'available', 'reservation_id' => null, 'instance_id' => null, 'operation_id' => null], ['id' => 74, 'host_id' => 1, 'public_ip_pool_id' => 32, 'address' => '198.51.100.13', 'prefix_length' => 29, 'state' => 'available', 'reservation_id' => null, 'instance_id' => null, 'operation_id' => null], ['id' => 75, 'host_id' => 1, 'public_ip_pool_id' => 34, 'address' => '198.51.100.26', 'prefix_length' => 29, 'state' => 'available', 'reservation_id' => null, 'instance_id' => null, 'operation_id' => null], ]); $admin = [ 'user_id' => 99, 'is_admin' => true, 'is_super_admin' => true, 'can_manage' => true, ]; $readAdmin = [ 'user_id' => 98, 'is_admin' => true, 'is_super_admin' => false, 'can_manage' => false, ]; $sourceKey = str_repeat('a', 32); $targetKey = str_repeat('b', 32); $otherSourceKey = str_repeat('c', 32); $overview = CloudPublicIpPoolReplacementBatchService::overview( 1, $sourceKey, $readAdmin ); $assert($overview['source_pool']['expected_revision'] === 7 && $overview['source_pool']['assigned_count'] === 2 && $overview['source_pool']['replacement_locked'] === false && $overview['can_start'] === true, 'Read-only administrator overview lost source revision or occupancy.'); $targetOptions = array_column( $overview['target_options'], null, 'pool_key' ); $assert(isset($targetOptions[$targetKey]) && $targetOptions[$targetKey]['expected_revision'] === 9 && $targetOptions[$targetKey]['available_count'] === 4 && $targetOptions[$targetKey]['eligible'] === true && !isset($targetOptions[str_repeat('d', 32)]), 'Overview did not enforce same-line verified target selection.'); $expectCloudError( static fn () => CloudPublicIpPoolReplacementBatchService::overview( 1, $sourceKey, ['user_id' => 501, 'is_admin' => false] ), 403, 'Tenant must not read administrator replacement batches.' ); $staleInput = [ 'target_pool_key' => $targetKey, 'expected_source_revision' => 6, 'expected_target_revision' => 9, 'confirm_replacement' => true, 'request_key' => 'replacement-stale-0001', ]; $expectCloudError( static fn () => CloudPublicIpPoolReplacementBatchService::start( 1, $sourceKey, $admin, $staleInput ), 409, 'A stale source revision must fail after the locked recheck.' ); $assert(CloudJob::count() === 0, 'Stale revision rejection created a durable operation.'); $input = [ 'target_pool_key' => $targetKey, 'expected_source_revision' => 7, 'expected_target_revision' => 9, 'confirm_replacement' => true, 'request_key' => 'replacement-valid-0001', ]; $started = CloudPublicIpPoolReplacementBatchService::start( 1, $sourceKey, $admin, $input ); $assert($started['unchanged'] === false && $started['batch']['state'] === 'running' && $started['batch']['source_initial_state'] === 'active' && $started['batch']['counts']['total'] === 2, 'Valid replacement did not create one running durable batch.'); $parent = CloudJob::find((int) $started['job']->id); $children = CloudJob::where('parent_job_id', (int) $parent->id) ->orderBy('instance_id')->get(); $assert($children->count() === 2 && $children->every(static fn ($job): bool => (string) $job->kind === 'network_change' && (string) $job->execution_mode === 'agent') && Capsule::table('cloud_public_ips') ->whereIn('id', [71, 72])->where('state', 'changing') ->count() === 2 && Capsule::table('cloud_public_ip_pools')->where('id', 31) ->value('state') === 'draining', 'Batch did not atomically fence every target IP and drain the source.'); $runningChild = $children->first(); $runningChild->phase = 'dispatching'; $runningChild->status = 'running'; $runningChild->save(); $runningDetail = CloudPublicIpPoolReplacementBatchService::detail( 1, $sourceKey, $started['batch']['batch_key'], $readAdmin ); $assert($runningDetail['counts']['running'] === 1 && $runningDetail['counts']['queued'] === 1 && $runningDetail['items'][0]['state'] === 'running', 'Batch DTO did not project an executing child as running.'); $idempotent = CloudPublicIpPoolReplacementBatchService::start( 1, $sourceKey, $admin, $input ); $assert($idempotent['unchanged'] === true && (int) $idempotent['job']->id === (int) $parent->id && CloudJob::where('parent_job_id', (int) $parent->id)->count() === 2, 'Idempotent retry duplicated child mutations.'); $activeOverview = CloudPublicIpPoolReplacementBatchService::overview( 1, $sourceKey, $readAdmin ); $activeTargets = array_column( $activeOverview['target_options'], null, 'pool_key' ); $assert($activeOverview['source_pool']['replacement_locked'] === true && $activeOverview['can_start'] === false && $activeOverview['active_batch']['batch_key'] === $started['batch']['batch_key'] && $activeTargets[$targetKey]['replacement_locked'] === true && $activeTargets[$targetKey]['eligible'] === false, 'Overview did not publish source/target participation locks.'); $overlapInput = [ 'target_pool_key' => $targetKey, 'expected_source_revision' => 4, 'expected_target_revision' => 9, 'confirm_replacement' => true, 'request_key' => 'replacement-overlap-001', ]; $expectCloudError( static fn () => CloudPublicIpPoolReplacementBatchService::start( 1, $otherSourceKey, $admin, $overlapInput ), 409, 'An active target pool must not join a second concurrent batch.' ); $expectCloudError( static fn () => CloudPublicIpPoolReplacementBatchService::detail( 1, $otherSourceKey, $started['batch']['batch_key'], $readAdmin ), 404, 'Batch detail must bind the batch to its exact source pool.' ); $settleChild = static function ( CloudJob $child, string $outcome, bool $publish = true ): void { $item = Capsule::table( 'cloud_public_ip_pool_replacement_batch_items' )->where('child_job_id', (int) $child->id)->first(); $instance = CloudInstance::find((int) $child->instance_id); CloudPlacementService::finishNetworkChange( $instance, (string) $child->operation_id, (int) $item->source_public_ip_id, (int) $item->target_public_ip_id, (int) $item->source_network_attachment_id, (int) $item->target_network_attachment_id, (int) Capsule::table('cloud_public_ips') ->where('id', (int) $item->source_public_ip_id) ->value('public_ip_pool_id'), 32, $outcome ); if ($outcome === 'succeeded') { $target = Capsule::table('cloud_public_ips') ->where('id', (int) $item->target_public_ip_id)->first(); $spec = json_decode((string) $instance->spec_json, true); $spec['public_ip_cidr'] = (string) $target->address . '/29'; $instance->spec_json = json_encode( $spec, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES ); $instance->save(); } if ($publish) { CloudPublicIpPoolReplacementBatchService::finishChildLocked( $child, $outcome ); } }; $first = $children->first(); $settleChild($first, 'uncertain'); $blocked = CloudPublicIpPoolReplacementBatch::where( 'batch_key', $started['batch']['batch_key'] )->first(); $assert((string) $blocked->state === 'blocked' && (int) $blocked->active_claim === 1 && (string) $parent->fresh()->phase !== 'terminal', 'Unknown child effect did not retain the active claim and parent fence.'); $settleChild($first->fresh(), 'succeeded', false); $resolvedFirst = $first->fresh(); $resolvedFirst->phase = 'terminal'; $resolvedFirst->outcome = 'failed'; $resolvedFirst->status = 'failed'; $resolvedFirst->effect_may_have_applied = 1; $resolvedFirst->safe_to_resubmit = 0; $resolvedFirst->effect_resolution = 'applied'; $resolvedFirst->save(); CloudJobService::reconcileAggregateParentEffectLocked( $resolvedFirst, 99 ); $settleChild($children->last()->fresh(), 'succeeded'); $completed = $blocked->fresh(); $parent = $parent->fresh(); $assert((string) $completed->state === 'succeeded' && $completed->active_claim === null && (string) $parent->phase === 'terminal' && (string) $parent->outcome === 'succeeded' && Capsule::table('cloud_public_ip_pools')->where('id', 31) ->value('state') === 'disabled' && Capsule::table('cloud_public_ips')->where('public_ip_pool_id', 31) ->where('state', '<>', 'available')->count() === 0, 'Evidence-backed recovery did not converge the batch and disable the empty source.'); $completedOverview = CloudPublicIpPoolReplacementBatchService::overview( 1, $sourceKey, $readAdmin ); $assert($completedOverview['replacement_history_count'] === 1 && $completedOverview['has_replacement_history'] === true && $completedOverview['source_pool']['replacement_history_count'] === 1, 'Read-only overview did not expose terminal replacement history.'); Capsule::table('cloud_public_ip_pools')->where('id', 33)->update([ 'state' => 'draining', 'desired_revision' => 5, 'observed_revision' => 5, ]); $failureInput = [ 'target_pool_key' => $targetKey, 'expected_source_revision' => 5, 'expected_target_revision' => 9, 'confirm_replacement' => true, 'request_key' => 'replacement-failure-001', ]; $failure = CloudPublicIpPoolReplacementBatchService::start( 1, $otherSourceKey, $admin, $failureInput ); $failureParent = CloudJob::find((int) $failure['job']->id); $failureChild = CloudJob::where( 'parent_job_id', (int) $failureParent->id )->first(); $settleChild($failureChild, 'failed'); $resolvedFailureChild = $failureChild->fresh(); $resolvedFailureChild->phase = 'terminal'; $resolvedFailureChild->outcome = 'failed'; $resolvedFailureChild->status = 'failed'; $resolvedFailureChild->effect_may_have_applied = 0; $resolvedFailureChild->safe_to_resubmit = 1; $resolvedFailureChild->effect_resolution = 'not_applied'; $resolvedFailureChild->error_code = 'agent_network_not_applied'; $resolvedFailureChild->save(); CloudJobService::reconcileAggregateParentEffectLocked( $resolvedFailureChild, 99 ); $failedBatch = CloudPublicIpPoolReplacementBatch::where( 'batch_key', $failure['batch']['batch_key'] )->first(); $failureParent = $failureParent->fresh(); $assert((string) $failedBatch->state === 'failed' && (string) $failedBatch->source_initial_state === 'draining' && $failedBatch->active_claim === null && (string) $failureParent->outcome === 'failed' && (int) $failureParent->effect_may_have_applied === 0 && (int) $failureParent->safe_to_resubmit === 1 && Capsule::table('cloud_public_ip_pools')->where('id', 33) ->value('state') === 'draining' && Capsule::table('cloud_public_ips') ->where('id', (int) Capsule::table( 'cloud_public_ip_pool_replacement_batch_items' )->where('batch_id', (int) $failedBatch->id) ->value('target_public_ip_id')) ->value('state') === 'available' && Capsule::table( 'cloud_public_ip_pool_replacement_batch_items' )->where('batch_id', (int) $failedBatch->id) ->value('last_error_code') === 'agent_network_not_applied', 'All-not-applied failure did not restore the exact initial source state.'); echo "Cloud public IP pool replacement batch service tests passed\n"; }