'http://127.0.0.1:8000', 'cloud_engine_hmac_key' => base64_encode(str_repeat('k', 32)), ]; $assert = static function ($condition, string $message): void { if (!$condition) { throw new RuntimeException($message); } }; $assertCloudError = static function (callable $callback, int $status, string $message) use ($assert): CloudError { try { $callback(); } catch (CloudError $error) { $assert($error->httpStatus === $status, $message . ' (unexpected HTTP status ' . $error->httpStatus . ')'); return $error; } throw new RuntimeException($message . ' (no CloudError was raised)'); }; $response = static fn ($payload): Response => new Response(200, [ 'Content-Type' => 'application/json', ], json_encode($payload, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES)); $engine = static function (array $responses, array &$history): CloudEngineClient { $mock = new MockHandler($responses); $stack = HandlerStack::create($mock); $stack->push(Middleware::history($history)); return new CloudEngineClient(new Client(['handler' => $stack])); }; $findById = static function (array $rows, int $id): ?array { foreach ($rows as $row) { if ((int) ($row['region_id'] ?? 0) === $id) { return $row; } } return null; }; $hostById = static function (array $rows, int $id): ?array { foreach ($rows as $row) { if ((int) ($row['host_id'] ?? 0) === $id) { return $row; } } return null; }; $capsule = new Capsule(); $capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->getConnection('default')->getSchemaBuilder(); $schema->create('cloud_regions', static function (Blueprint $table): void { $table->increments('id'); $table->string('code'); $table->string('name'); $table->string('country_code', 2)->default(''); $table->string('status'); $table->integer('sort')->default(0); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_network_lines', static function (Blueprint $table): void { $table->increments('id'); $table->integer('region_id'); $table->string('code'); $table->string('name'); $table->boolean('is_default')->default(false); $table->string('status'); $table->integer('sort')->default(0); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_resource_pools', static function (Blueprint $table): void { $table->increments('id'); $table->integer('line_id'); $table->string('code'); $table->string('name'); $table->string('status'); $table->string('scheduling_policy')->default('balanced'); $table->integer('sort')->default(0); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_hosts', static function (Blueprint $table): void { $table->increments('id'); $table->string('name'); $table->string('region')->default(''); $table->string('country_code', 2)->default(''); $table->boolean('enabled')->default(true); $table->integer('resource_pool_id')->nullable(); $table->string('scheduler_state')->default('disabled'); $table->string('default_network')->default('bridge:br0'); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_host_resource_pool_memberships', static function (Blueprint $table): void { $table->increments('id'); $table->integer('host_id'); $table->integer('resource_pool_id'); $table->string('scheduler_state')->default('disabled'); $table->integer('priority')->default(0); $table->integer('weight')->default(100); $table->integer('created_at'); $table->integer('updated_at'); $table->unique(['host_id', 'resource_pool_id']); }); $schema->create('cloud_region_fabrics', static function (Blueprint $table): void { $table->increments('id'); $table->integer('region_id')->unique(); $table->string('fabric_id')->unique(); $table->string('fabric_region'); $table->integer('fabric_revision'); $table->string('state'); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_private_cidr_allocator_configs', static function (Blueprint $table): void { $table->integer('id')->primary(); $table->string('pool_cidr', 18); $table->integer('quantum_prefix_length'); $table->integer('min_prefix_length'); $table->integer('max_prefix_length'); $table->integer('revision'); $table->integer('updated_at'); }); $schema->create('cloud_instances', static function (Blueprint $table): void { $table->increments('id'); $table->integer('host_id'); $table->integer('resource_pool_id')->nullable(); $table->string('status'); }); $schema->create('cloud_vpcs', static function (Blueprint $table): void { $table->increments('id'); $table->integer('region_id'); $table->string('fabric_id')->default(''); $table->string('fabric_region')->default(''); $table->integer('fabric_revision')->default(0); $table->string('state'); }); $schema->create('cloud_vpc_host_deployments', static function (Blueprint $table): void { $table->increments('id'); $table->integer('vpc_id'); $table->integer('host_id'); $table->string('state'); }); $now = 1770000000; Capsule::table('cloud_regions')->insert([ ['id' => 1, 'code' => 'hk', 'name' => 'Hong Kong', 'country_code' => 'HK', 'status' => 'active', 'sort' => 10, 'created_at' => $now, 'updated_at' => $now], ['id' => 2, 'code' => 'sg', 'name' => 'Singapore', 'country_code' => 'SG', 'status' => 'active', 'sort' => 20, 'created_at' => $now, 'updated_at' => $now], ]); Capsule::table('cloud_private_cidr_allocator_configs')->insert([ 'id' => 1, 'pool_cidr' => '10.0.0.0/8', 'quantum_prefix_length' => 24, 'min_prefix_length' => 16, 'max_prefix_length' => 24, 'revision' => 1, 'updated_at' => $now, ]); Capsule::table('cloud_network_lines')->insert([ ['id' => 11, 'region_id' => 1, 'code' => 'hk-default', 'name' => 'HK default', 'is_default' => 1, 'status' => 'active', 'sort' => 10, 'created_at' => $now, 'updated_at' => $now], ['id' => 12, 'region_id' => 2, 'code' => 'sg-default', 'name' => 'SG default', 'is_default' => 1, 'status' => 'active', 'sort' => 20, 'created_at' => $now, 'updated_at' => $now], ]); Capsule::table('cloud_resource_pools')->insert([ ['id' => 21, 'line_id' => 11, 'code' => 'hk-general', 'name' => 'HK general', 'status' => 'active', 'scheduling_policy' => 'balanced', 'sort' => 10, 'created_at' => $now, 'updated_at' => $now], ['id' => 22, 'line_id' => 12, 'code' => 'sg-general', 'name' => 'SG general', 'status' => 'active', 'scheduling_policy' => 'balanced', 'sort' => 20, 'created_at' => $now, 'updated_at' => $now], ]); for ($id = 1; $id <= 6; $id++) { Capsule::table('cloud_hosts')->insert([ 'id' => $id, 'name' => 'host-' . $id, // host-1 remains in the eventual apply set and host-5 exercises the // larger readiness projection. Both use display labels different from // the sales-region code; canonical membership owns their placement. 'region' => in_array($id, [1, 5], true) ? 'Hong Kong' : 'hk', 'country_code' => 'HK', 'enabled' => 1, 'resource_pool_id' => 21, 'scheduler_state' => $id <= 5 ? 'active' : 'disabled', 'default_network' => 'bridge:br0', 'created_at' => $now, 'updated_at' => $now, ]); if ($id <= 5) { Capsule::table('cloud_host_resource_pool_memberships')->insert([ 'id' => 30 + $id, 'host_id' => $id, 'resource_pool_id' => 21, 'scheduler_state' => 'active', 'priority' => 0, 'weight' => 100, 'created_at' => $now, 'updated_at' => $now, ]); } } $actor = [ 'user_id' => 99, 'is_admin' => true, 'is_super_admin' => true, 'can_manage' => true, ]; $history = []; $busyClient = $engine([new Response(409, [ 'Content-Type' => 'application/json', ], json_encode(['error_code' => 'fabric_probe_busy'], JSON_THROW_ON_ERROR))], $history); $busy = $assertCloudError( static fn () => $busyClient->fabricProbe(['1'], $actor), 409, 'An overlapping engine readiness probe lost its stable 409 contract' ); $assert(str_contains($busy->getMessage(), '已有地区宿主机检测正在运行') && count($history) === 1, 'The reviewed probe-busy error was hidden or retried by PHP.'); // Engine outages keep platform inventory visible, but every workflow write // gate must fail closed until an authoritative Fabric snapshot is available. $history = []; $unavailable = CloudAdminFabricService::listing($actor, $engine([ new Response(503, ['Content-Type' => 'application/json'], json_encode([ 'error_code' => 'fabric_host_unavailable', ], JSON_THROW_ON_ERROR)), ], $history)); $unavailableRegion = $findById($unavailable['regions'], 1); $assert(!array_key_exists('fabrics', $unavailable) && $unavailable['engine_available'] === false && is_array($unavailableRegion) && !array_key_exists('fabric', $unavailableRegion) && ($unavailableRegion['workflow'] ?? null) === [ 'stage' => 'engine_unavailable', 'can_probe' => false, 'can_apply' => false, 'can_reconcile' => false, 'blocking_reasons' => ['engine_unavailable'], 'action_mode' => 'none', ] && $unavailableRegion['plan_preview'] === null, 'An engine outage did not fail every region workflow write gate closed.'); $overlay = static function (string $regionCode, int $offset = 0): string { $slot = (hexdec(substr(hash('sha256', $regionCode), 0, 8)) + $offset) % 16384; return '100.' . (64 + intdiv($slot, 256)) . '.' . ($slot % 256) . '.0/24'; }; $firstHkOverlay = $overlay('hk'); $hkOverlay = $overlay('hk', 1); $overlappingEndpoint = substr($firstHkOverlay, 0, -4) . '77'; $probeHost = static function (int $id, array $changes = []): array { $row = [ 'host_id' => (string) $id, 'configured' => true, 'reachable' => true, 'region' => 'hk', 'endpoint_ip' => '203.0.113.' . (10 + $id), 'endpoint_source' => 'ssh_host_ipv4', 'probed_at' => 1770000100 + $id, 'vpc_ready' => true, 'reasons' => [], 'capabilities' => [ 'os_id' => 'almalinux', 'os_version' => '10', 'network_manager' => 'networkmanager', 'network_manager_expected' => 'networkmanager', 'network_manager_supported' => true, 'deployment_mode' => 'networkmanager_nmcli', 'firewall_manager' => 'nftables', 'firewall_mode' => 'native_nftables', 'ip_forward' => 1, 'rp_filter' => 0, 'rp_filter_all' => 0, 'rp_filter_default' => 1, 'rp_filter_default_effective' => 1, 'rp_filter_underlay_raw' => 1, 'rp_filter_underlay_effective' => 1, 'rp_filter_fabric_raw' => null, 'rp_filter_fabric_effective' => null, 'wireguard_ready' => true, 'wireguard_owned' => false, 'nftables_ready' => true, 'nftables_active' => true, 'nftables_persistent' => true, 'public_selector' => 'bridge:br0', 'public_verified' => true, ], ]; if (isset($changes['capabilities'])) { $row['capabilities'] = array_replace( $row['capabilities'], $changes['capabilities'] ); unset($changes['capabilities']); } return array_replace($row, $changes); }; $fabricStatus = static function (array $hostIds) use ($hkOverlay): array { $digest = str_repeat('a', 64); $network = substr($hkOverlay, 0, -4); $nodes = []; foreach (array_values($hostIds) as $index => $hostId) { $nodes[] = [ 'host_id' => (string) $hostId, 'overlay_ip' => $network . ($index + 1), 'endpoint' => '203.0.113.' . (10 + (int) $hostId) . ':51820', 'listen_port' => 51820, 'public_key' => base64_encode(str_repeat(chr((int) $hostId), 32)), 'state' => 'active', 'transport_verified' => true, 'observed_revision' => 1, 'observed_plan_digest' => $digest, 'last_error_code' => null, 'observed' => [], ]; } return [ 'fabric_id' => 'fabric-hk', 'region' => 'hk', 'overlay_cidr' => $hkOverlay, 'revision' => 1, 'listen_port' => 51820, 'plan_digest' => $digest, 'interface' => 'wg-fabric0', 'state' => 'active', 'next_stage_allowed' => true, 'updated_at' => 1770000200, 'operation' => [ 'id' => str_repeat('b', 32), 'kind' => 'apply', 'status' => 'succeeded', 'started_at' => 1770000180, 'updated_at' => 1770000200, 'finished_at' => 1770000200, ], 'nodes' => $nodes, ]; }; $status = $fabricStatus([1, 2]); $status['nodes'][0]['endpoint'] = $overlappingEndpoint . ':51820'; $driftStatus = $fabricStatus([1, 2, 6]); $historicalStatus = $driftStatus; $historicalStatus['fabric_id'] = 'legacy-hk'; $historicalExactStatus = $fabricStatus([1, 2]); $historicalExactStatus['fabric_id'] = 'legacy-hk'; $historicalExactStatus['region'] = 'legacy-hong-kong'; $foreignRegionStatus = $fabricStatus([1]); $foreignRegionStatus['fabric_id'] = 'fabric-sg'; $foreignRegionStatus['region'] = 'sg'; // Broken placement relations must fail closed before any engine request. An // inner join must not silently shrink the required Fabric host union. Capsule::table('cloud_host_resource_pool_memberships')->insert([ 'id' => 99, 'host_id' => 1, 'resource_pool_id' => 999, 'scheduler_state' => 'active', 'priority' => 0, 'weight' => 100, 'created_at' => $now, 'updated_at' => $now, ]); $history = []; $client = $engine([], $history); $assertCloudError( static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client), 503, 'An active membership with a missing pool/line was silently omitted' ); $assert($history === [], 'A corrupt active membership was rejected only after an engine request.'); Capsule::table('cloud_host_resource_pool_memberships')->where('id', 99)->delete(); Capsule::table('cloud_host_resource_pool_memberships')->insert([ 'id' => 100, 'host_id' => 1, 'resource_pool_id' => 22, 'scheduler_state' => 'draining', 'priority' => 0, 'weight' => 100, 'created_at' => $now, 'updated_at' => $now, ]); $history = []; $client = $engine([], $history); $assertCloudError( static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client), 409, 'Concurrent active/draining region memberships were not treated as a conflict' ); $assert($history === [], 'A live membership conflict was rejected only after an engine request.'); Capsule::table('cloud_host_resource_pool_memberships')->where('id', 100)->delete(); Capsule::table('cloud_instances')->insert([ 'id' => 101, 'host_id' => 1, 'resource_pool_id' => 999, 'status' => 'running', ]); $history = []; $client = $engine([], $history); $assertCloudError( static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client), 503, 'A live instance with a missing pool/line was silently omitted' ); $assert($history === [], 'A corrupt instance placement was rejected only after an engine request.'); Capsule::table('cloud_instances')->where('id', 101)->delete(); Capsule::table('cloud_vpcs')->insert([ 'id' => 201, 'region_id' => 999, 'state' => 'active', ]); Capsule::table('cloud_vpc_host_deployments')->insert([ 'id' => 202, 'vpc_id' => 201, 'host_id' => 1, 'state' => 'active', ]); $history = []; $client = $engine([], $history); $assertCloudError( static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client), 503, 'A live VPC deployment with an unknown region was silently omitted' ); $assert($history === [], 'An invalid VPC deployment region was rejected only after an engine request.'); Capsule::table('cloud_vpc_host_deployments')->where('id', 202)->delete(); Capsule::table('cloud_vpcs')->where('id', 201)->delete(); Capsule::table('cloud_vpc_host_deployments')->insert([ 'id' => 203, 'vpc_id' => 999, 'host_id' => 1, 'state' => 'active', ]); $history = []; $client = $engine([], $history); $assertCloudError( static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client), 503, 'A live deployment whose VPC root is missing was silently omitted' ); $assert($history === [], 'An orphaned VPC deployment was rejected only after an engine request.'); Capsule::table('cloud_vpc_host_deployments')->where('id', 203)->delete(); // Workloads remain part of the exact required-host union even when the host // no longer has an active scheduler membership. Capsule::table('cloud_instances')->insert([ 'id' => 102, 'host_id' => 6, 'resource_pool_id' => 21, 'status' => 'running', ]); $history = []; $instanceListing = CloudAdminFabricService::listing($actor, $engine([ $response([]), ], $history)); $instanceRegion = $findById($instanceListing['regions'], 1); $instanceHost = is_array($instanceRegion) ? $hostById($instanceRegion['hosts'], 6) : null; $assert(is_array($instanceHost) && $instanceHost['required'] === true && in_array('instance', $instanceHost['required_sources'], true) && in_array('workload_membership_mismatch', $instanceHost['reasons'], true), 'A live instance host without membership disappeared from readiness.'); Capsule::table('cloud_instances')->where('id', 102)->delete(); Capsule::table('cloud_vpcs')->insert([ 'id' => 204, 'region_id' => 1, 'state' => 'active', ]); Capsule::table('cloud_vpc_host_deployments')->insert([ 'id' => 205, 'vpc_id' => 204, 'host_id' => 6, 'state' => 'active', ]); $history = []; $deploymentListing = CloudAdminFabricService::listing($actor, $engine([ $response([]), ], $history)); $deploymentRegion = $findById($deploymentListing['regions'], 1); $deploymentHost = is_array($deploymentRegion) ? $hostById($deploymentRegion['hosts'], 6) : null; $assert(is_array($deploymentHost) && $deploymentHost['required'] === true && in_array('vpc_deployment', $deploymentHost['required_sources'], true) && in_array('workload_membership_mismatch', $deploymentHost['reasons'], true), 'A live VPC deployment host without membership disappeared from readiness.'); Capsule::table('cloud_vpc_host_deployments')->where('id', 205)->delete(); Capsule::table('cloud_vpcs')->where('id', 204)->delete(); // Existing active Fabric: a disabled host that is not a Fabric member must // still be visible, and host-5 remains in HK solely by canonical membership. Capsule::table('cloud_region_fabrics')->insert([ 'id' => 1, 'region_id' => 1, 'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk', 'fabric_revision' => 1, 'state' => 'active', 'created_at' => $now, 'updated_at' => $now, ]); $history = []; $listing = CloudAdminFabricService::listing($actor, $engine([ $response([$driftStatus]), ], $history)); $hk = $findById($listing['regions'], 1); $sg = $findById($listing['regions'], 2); $assert(!array_key_exists('fabrics', $listing) && is_array($hk) && is_array($sg) && !array_key_exists('fabric', $hk) && !array_key_exists('fabric', $sg), 'The readiness inventory dropped a platform region.'); $assert(array_map(static fn (array $host): int => (int) $host['host_id'], $hk['hosts']) === [1, 2, 3, 4, 5, 6] && $sg['hosts'] === [], 'Host geography did not follow membership -> pool -> line -> region.'); $regionDrift = $hostById($hk['hosts'], 5); $assert(is_array($regionDrift) && $regionDrift['engine_region'] === 'Hong Kong' && $regionDrift['platform_ready'] === true && !in_array('panel_host_region_mismatch', $regionDrift['reasons'], true), 'A display-only host region label still blocked canonical placement.'); $notMember = $hostById($hk['hosts'], 3); $assert(is_array($notMember) && $notMember['fabric_member'] === false && in_array('fabric_membership_drift', $notMember['reasons'], true), 'A platform host outside the current Fabric disappeared from readiness.'); $orphan = $hostById($hk['hosts'], 6); $assert(is_array($orphan) && $orphan['fabric_member'] === true && in_array('fabric_membership_drift', $orphan['reasons'], true) && $hk['ready'] === false && $hk['host_set_matches'] === false, 'A scheduler/Fabric membership drift was hidden or promoted to ready.'); $legacyPhaseHost = $hostById($hk['hosts'], 1); $assert(is_array($legacyPhaseHost) && array_key_exists('last_error_phase', $legacyPhaseHost) && $legacyPhaseHost['last_error_phase'] === null, 'A mixed-version Fabric response without a failure phase was rejected.'); $phaseStatus = $driftStatus; foreach (['identity', 'observe', 'apply'] as $index => $phase) { $phaseStatus['nodes'][$index]['last_error_code'] = 'fabric_remote_command_failed'; $phaseStatus['nodes'][$index]['last_error_phase'] = $phase; } $phaseHistory = []; $phaseProjection = CloudAdminFabricService::listing($actor, $engine([ $response([$phaseStatus]), ], $phaseHistory)); $phaseRegion = $findById($phaseProjection['regions'], 1); $assert(is_array($phaseRegion) && ($hostById($phaseRegion['hosts'], 1)['last_error_phase'] ?? null) === 'identity' && ($hostById($phaseRegion['hosts'], 2)['last_error_phase'] ?? null) === 'observe' && ($hostById($phaseRegion['hosts'], 6)['last_error_phase'] ?? null) === 'apply', 'Reviewed Fabric failure phases were not projected to their host rows.'); $invalidPhaseStatus = $driftStatus; $invalidPhaseStatus['nodes'][0]['last_error_code'] = 'fabric_remote_command_failed'; $invalidPhaseStatus['nodes'][0]['last_error_phase'] = 'probe'; $invalidPhaseHistory = []; $assertCloudError( static fn () => CloudAdminFabricService::listing($actor, $engine([ $response([$invalidPhaseStatus]), ], $invalidPhaseHistory)), 502, 'An unreviewed Fabric failure phase escaped the Engine response boundary' ); $orphanPhaseStatus = $driftStatus; $orphanPhaseStatus['nodes'][0]['last_error_phase'] = 'apply'; $orphanPhaseHistory = []; $assertCloudError( static fn () => CloudAdminFabricService::listing($actor, $engine([ $response([$orphanPhaseStatus]), ], $orphanPhaseHistory)), 502, 'A Fabric failure phase without an error code escaped the response boundary' ); $assert(($hk['workflow']['stage'] ?? null) === 'migration_required' && ($hk['workflow']['can_apply'] ?? true) === false && ($hk['workflow']['can_reconcile'] ?? true) === false && in_array('fabric_membership_drift', $hk['workflow']['blocking_reasons'] ?? [], true) && ($hk['plan_preview']['mode'] ?? null) === 'existing' && !array_intersect(['nodes', 'endpoints', 'host_ids'], array_keys($hk['plan_preview'])) && ($sg['workflow']['stage'] ?? null) === 'no_hosts' && ($sg['workflow']['action_mode'] ?? null) === 'none', 'Membership drift/no-host inventory did not project safe wizard stages.'); // A fresh probe is an isolated per-host snapshot. One failed host must not // erase successful or independently blocked rows. CloudRegionFabric::query()->delete(); $probeRows = [ $probeHost(1, [ 'endpoint_source' => 'configured', ]), $probeHost(2, [ 'reachable' => false, 'vpc_ready' => false, 'error_code' => 'fabric_host_capability_probe_failed', ]), $probeHost(3, [ 'endpoint_ip' => '10.20.0.10', 'vpc_ready' => true, 'reasons' => [], 'capabilities' => [ 'firewall_manager' => 'firewalld', 'firewall_mode' => 'firewalld_nft', 'nftables_ready' => true, 'nftables_active' => false, 'nftables_persistent' => false, 'underlay_runtime_zone' => 'public', 'underlay_permanent_zone' => 'public', 'rp_filter_all' => 3, 'rp_filter_default' => '1', 'rp_filter_fabric_raw' => true, ], ]), $probeHost(4, ['endpoint_ip' => 'host.example.test']), $probeHost(5, [ 'region' => 'sg', 'endpoint_source' => 'untrusted-engine-value', ]), $probeHost(6), ]; $history = []; $probed = CloudAdminFabricService::probeRegion('1', $actor, $engine([ $response([$driftStatus]), $response(['hosts' => $probeRows]), ], $history)); $rows = $probed['region']['hosts']; $assert(count($rows) === 6 && ($hostById($rows, 1)['readiness_state'] ?? null) === 'ready', 'A single probe failure discarded healthy host rows.'); $assert(($hostById($rows, 1)['endpoint_source'] ?? null) === 'configured', 'An explicit effective Fabric endpoint source was not projected.'); $assert(($hostById($rows, 1)['capabilities']['firewall_mode'] ?? null) === 'native_nftables' && ($hostById($rows, 1)['capabilities']['firewall_manager'] ?? null) === 'nftables', 'A native nftables host was not preserved as ready.'); $rpFilterCapabilities = $hostById($rows, 1)['capabilities'] ?? []; $assert(($rpFilterCapabilities['rp_filter'] ?? null) === 0 && ($rpFilterCapabilities['rp_filter_all'] ?? null) === 0 && ($rpFilterCapabilities['rp_filter_default'] ?? null) === 1 && ($rpFilterCapabilities['rp_filter_default_effective'] ?? null) === 1 && ($rpFilterCapabilities['rp_filter_underlay_raw'] ?? null) === 1 && ($rpFilterCapabilities['rp_filter_underlay_effective'] ?? null) === 1 && array_key_exists('rp_filter_fabric_raw', $rpFilterCapabilities) && $rpFilterCapabilities['rp_filter_fabric_raw'] === null && array_key_exists('rp_filter_fabric_effective', $rpFilterCapabilities) && $rpFilterCapabilities['rp_filter_fabric_effective'] === null, 'The bounded raw/effective rp_filter evidence was dropped or Alma strict defaults were hidden.'); $failed = $hostById($rows, 2); $firewalld = $hostById($rows, 3); $nonIpv4 = $hostById($rows, 4); $wrongRegion = $hostById($rows, 5); $orphan = $hostById($rows, 6); $assert(is_array($failed) && $failed['probe_checked'] === true && $failed['reachable'] === false && in_array('fabric_host_capability_probe_failed', $failed['reasons'], true), 'The isolated failed-host result was dropped or promoted.'); $assert(is_array($firewalld) && ($firewalld['capabilities']['firewall_manager'] ?? null) === 'firewalld' && ($firewalld['capabilities']['firewall_mode'] ?? null) === 'firewalld_nft' && ($firewalld['vpc_ready'] ?? true) === false && !array_key_exists('underlay_runtime_zone', $firewalld['capabilities']) && !array_key_exists('underlay_permanent_zone', $firewalld['capabilities']) && !array_key_exists('rp_filter_all', $firewalld['capabilities']) && !array_key_exists('rp_filter_default', $firewalld['capabilities']) && !array_key_exists('rp_filter_fabric_raw', $firewalld['capabilities']) && in_array('firewalld_unsupported_requires_maintenance', $firewalld['reasons'], true) && in_array('fabric_endpoint_vpc_pool_conflict', $firewalld['reasons'], true), 'A legacy firewalld result was not failed closed with its endpoint blocker.'); $assert(is_array($nonIpv4) && $nonIpv4['endpoint_ip'] === null && $nonIpv4['endpoint_source'] === null && in_array('fabric_endpoint_unavailable', $nonIpv4['reasons'], true), 'A non-IPv4 WireGuard endpoint was accepted or not explained.'); $assert(is_array($wrongRegion) && $wrongRegion['endpoint_ip'] === null && $wrongRegion['endpoint_source'] === null && $wrongRegion['readiness_state'] === 'blocked' && in_array('fabric_endpoint_source_invalid', $wrongRegion['reasons'], true) && !in_array('fabric_region_mismatch', $wrongRegion['reasons'], true), 'An untrusted endpoint source or display-only probe region escaped the bounded projection.'); $assert(is_array($orphan) && $orphan['probe_checked'] === true && $orphan['fabric_member'] === true && in_array('fabric_membership_drift', $orphan['reasons'], true), 'An orphaned Fabric node escaped fresh host detection.'); $assert(array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history) === [ 'GET /internal/v1/fabric', 'POST /internal/v1/fabric/probe', ], 'Fresh detection did not freeze the engine membership snapshot before probing.'); $detectedIds = json_decode((string) $history[1]['request']->getBody(), true, 32, JSON_THROW_ON_ERROR)['host_ids'] ?? []; $assert($detectedIds === ['1', '2', '3', '4', '5', '6'], 'Fresh detection omitted a canonical member or orphaned Fabric node.'); // Retire the probe-only hosts before configuring the clean two-node target. CloudHostResourcePoolMembership::whereIn('host_id', [3, 4, 5])->update([ 'scheduler_state' => 'disabled', 'updated_at' => $now + 1, ]); // Inventory alone never opens apply: the current request must carry a fresh, // complete probe snapshot for every required host. $history = []; $needsProbeListing = CloudAdminFabricService::listing($actor, $engine([ $response([]), ], $history)); $needsProbeRegion = $findById($needsProbeListing['regions'], 1); $assert(is_array($needsProbeRegion) && ($needsProbeRegion['workflow'] ?? null) === [ 'stage' => 'needs_probe', 'can_probe' => true, 'can_apply' => false, 'can_reconcile' => false, 'blocking_reasons' => ['probe_required'], 'action_mode' => 'probe', ] && $needsProbeRegion['plan_preview'] === null, 'A stale inventory snapshot opened automatic Fabric apply without probing.'); $history = []; $blockedWizard = CloudAdminFabricService::probeRegion('1', $actor, $engine([ $response([]), $response(['hosts' => [ $probeHost(1, [ 'capabilities' => [ 'firewall_manager' => 'firewalld', 'firewall_mode' => 'firewalld_nft', ], ]), $probeHost(2), ]]), ], $history)); $blockedWorkflow = $blockedWizard['region']['workflow'] ?? null; $assert(($blockedWorkflow['stage'] ?? null) === 'blocked' && ($blockedWorkflow['can_probe'] ?? false) === true && ($blockedWorkflow['can_apply'] ?? true) === false && ($blockedWorkflow['can_reconcile'] ?? true) === false && ($blockedWorkflow['action_mode'] ?? null) === 'probe' && in_array('firewalld_unsupported_requires_maintenance', $blockedWorkflow['blocking_reasons'] ?? [], true), 'A legacy firewalld probe did not keep the wizard blocked.'); // A fresh successful detection is the only path to ready_to_apply. The // browser receives a bounded server-owned preview, never nodes or endpoints. $history = []; $readyToApply = CloudAdminFabricService::probeRegion('1', $actor, $engine([ $response([]), $response(['hosts' => [ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2), ]]), ], $history)); $readyWorkflow = $readyToApply['region']['workflow'] ?? null; $readyPreview = $readyToApply['region']['plan_preview'] ?? null; $assert(!array_key_exists('fabric', $readyToApply['region']) && $readyWorkflow === [ 'stage' => 'ready_to_apply', 'can_probe' => true, 'can_apply' => true, 'can_reconcile' => false, 'blocking_reasons' => [], 'action_mode' => 'apply', ] && $readyPreview === [ 'mode' => 'automatic', 'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk', 'revision' => 1, 'listen_port' => 51820, 'overlay_cidr' => $hkOverlay, 'host_count' => 2, ], 'Fresh readiness did not produce the bounded automatic plan preview.'); // Even an exact-host-set custom Fabric remains an explicit migration. The // wizard never adopts or replays an unmapped legacy identity. $history = []; $legacyExactListing = CloudAdminFabricService::listing($actor, $engine([ $response([$historicalExactStatus]), ], $history)); $legacyExactRegion = $findById($legacyExactListing['regions'], 1); $assert(is_array($legacyExactRegion) && ($legacyExactRegion['workflow']['stage'] ?? null) === 'migration_required' && ($legacyExactRegion['workflow']['can_apply'] ?? true) === false && ($legacyExactRegion['workflow']['can_reconcile'] ?? true) === false && array_column($legacyExactRegion['fabric_conflicts'], 'fabric_id') === ['legacy-hk'], 'An unmapped custom Fabric was projected as automatically adoptable.'); $history = []; $client = $engine([ $response(['hosts' => [$probeHost(1), $probeHost(2)]]), $response([$historicalExactStatus]), ], $history); $assertCloudError( static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client), 409, 'Automatic configuration adopted an exact unmapped custom Fabric' ); $assert(count($history) === 2 && $history[1]['request']->getUri()->getPath() === '/internal/v1/fabric', 'Custom Fabric rejection occurred after a host-writing request.'); // A historical custom-ID Fabric in the same engine region is not owned merely // because no database mapping exists. It must remain visible (including its // orphan host), while keeping the region unready and automatic writes closed. $history = []; $historicalListing = CloudAdminFabricService::listing($actor, $engine([ $response([$historicalStatus]), ], $history)); $historicalRegion = $findById($historicalListing['regions'], 1); $historicalOrphan = is_array($historicalRegion) ? $hostById($historicalRegion['hosts'], 6) : null; $assert(is_array($historicalRegion) && $historicalRegion['ready'] === false && $historicalRegion['host_set_matches'] === false && array_column($historicalRegion['fabric_conflicts'], 'fabric_id') === ['legacy-hk'] && array_map(static fn (array $host): int => (int) $host['host_id'], $historicalRegion['hosts']) === [1, 2, 6] && is_array($historicalOrphan) && $historicalOrphan['fabric_member'] === true && in_array('fabric_identity_conflict', $historicalOrphan['reasons'], true), 'An unmapped historical same-region Fabric was hidden or treated as owned.'); $assert(($historicalRegion['workflow']['stage'] ?? null) === 'migration_required' && in_array('fabric_identity_conflict', $historicalRegion['workflow']['blocking_reasons'] ?? [], true) && ($historicalRegion['workflow']['can_apply'] ?? true) === false && ($historicalRegion['workflow']['can_reconcile'] ?? true) === false, 'A historical Fabric identity conflict did not require manual migration.'); $history = []; $client = $engine([ $response(['hosts' => [ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2), ]]), $response([$historicalStatus]), ], $history); $assertCloudError( static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client), 409, 'Automatic configuration adopted an unmapped historical custom Fabric' ); $assert(array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history) === [ 'POST /internal/v1/fabric/probe', 'GET /internal/v1/fabric', ], 'Historical custom-Fabric ownership was checked after an engine write.'); // A different/legacy region label does not make a Fabric safe when it already // owns a host required by HK: wg-fabric0 is host-global. Keep the conflict // visible and stop automatic apply before either host-writing endpoint. $history = []; $foreignRegionListing = CloudAdminFabricService::listing($actor, $engine([ $response([$foreignRegionStatus]), ], $history)); $foreignRegionHk = $findById($foreignRegionListing['regions'], 1); $foreignRegionHost = is_array($foreignRegionHk) ? $hostById($foreignRegionHk['hosts'], 1) : null; $assert(is_array($foreignRegionHk) && $foreignRegionHk['ready'] === false && array_column($foreignRegionHk['fabric_conflicts'], 'fabric_id') === ['fabric-sg'] && is_array($foreignRegionHost) && in_array('fabric_identity_conflict', $foreignRegionHost['reasons'], true), 'A foreign-region Fabric owning an HK host was hidden from readiness.'); $history = []; $client = $engine([ $response(['hosts' => [ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2), ]]), $response([$foreignRegionStatus]), ], $history); $assertCloudError( static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client), 409, 'A foreign-region Fabric owning an HK host reached automatic apply' ); $paths = array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history); $assert($paths === [ 'POST /internal/v1/fabric/probe', 'GET /internal/v1/fabric', ] && !in_array('POST /internal/v1/fabric/apply', $paths, true) && !in_array('POST /internal/v1/fabric/fabric-hk/reconcile', $paths, true), 'Foreign-region host ownership was rejected only after an engine write.'); // Automatic configuration always repeats the fresh probe. Each unsafe probe // class must stop before status lookup or an engine mutation. $missingEndpointSource = $probeHost(1); unset($missingEndpointSource['endpoint_source']); $blockedProbeSets = [ 'legacy firewalld mode' => [ $probeHost(1, [ 'capabilities' => [ 'firewall_manager' => 'firewalld', 'firewall_mode' => 'firewalld_nft', ], ]), $probeHost(2), ], 'non-IPv4 endpoint' => [ $probeHost(1, ['endpoint_ip' => 'host.example.test']), $probeHost(2), ], 'non-routable endpoint' => [ $probeHost(1, ['endpoint_ip' => '0.0.0.0']), $probeHost(2), ], 'invalid endpoint source' => [ $probeHost(1, ['endpoint_source' => 'untrusted-engine-value']), $probeHost(2), ], 'missing endpoint source' => [ $missingEndpointSource, $probeHost(2), ], ]; foreach ($blockedProbeSets as $label => $items) { $history = []; $client = $engine([$response(['hosts' => $items])], $history); $assertCloudError( static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client), 409, 'Automatic Fabric configuration did not block ' . $label ); $assert(count($history) === 1 && $history[0]['request']->getUri()->getPath() === '/internal/v1/fabric/probe', 'The ' . $label . ' blocker reached an engine read/write after fresh probing.'); } // WireGuard endpoints are an engine-wide identity for the generated plan. // A duplicated fresh endpoint must fail before list/apply can mutate Fabric. $duplicateEndpoint = '203.0.113.250'; $history = []; $client = $engine([$response(['hosts' => [ $probeHost(1, ['endpoint_ip' => $duplicateEndpoint]), $probeHost(2, ['endpoint_ip' => $duplicateEndpoint]), ]])], $history); $assertCloudError( static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client), 409, 'Duplicate automatic WireGuard endpoints reached Fabric apply' ); $assert(array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history) === [ 'POST /internal/v1/fabric/probe', ], 'Duplicate endpoints were rejected only after an engine list/write request.'); // The deterministic ID is still a globally owned database identity. A // mapping belonging to another region must block both possible engine write // branches, even when the engine catalog is empty or contains a record whose // shape would otherwise be accepted for this region. Capsule::table('cloud_region_fabrics')->insert([ 'region_id' => 2, 'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk', 'fabric_revision' => 1, 'state' => 'disabled', 'created_at' => $now, 'updated_at' => $now, ]); $foreignOwnerCases = [ 'empty engine catalog' => [ [ $response(['hosts' => [ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2), ]]), $response([]), $response($status), ], '/internal/v1/fabric/apply', ], 'exact-shaped engine record' => [ [ $response(['hosts' => [ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2), ]]), $response([$status]), $response($status), ], '/internal/v1/fabric/fabric-hk/reconcile', ], ]; foreach ($foreignOwnerCases as $label => [$responses, $forbiddenPath]) { $history = []; $client = $engine($responses, $history); $assertCloudError( static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client), 409, 'A foreign region owner reached the ' . $label . ' write branch' ); $paths = array_map(static fn (array $entry): string => $entry['request']->getUri()->getPath(), $history); $assert($history === [] && !in_array($forbiddenPath, $paths, true), 'Foreign ownership was checked only after ' . $forbiddenPath . '.'); } CloudRegionFabric::where('region_id', 2)->delete(); // Automatic configuration: the browser supplies only the region action while // PHP deterministically creates the Fabric identity, overlay and sorted node // plan. Apply only admits one asynchronous operation and never publishes the // region mapping; explicit reconcile owns the later exact readback/activation. $queuedApply = $status; $queuedApply['state'] = 'applying'; $queuedApply['next_stage_allowed'] = false; $queuedApply['operation']['status'] = 'queued'; $queuedApply['operation']['finished_at'] = null; $history = []; $client = $engine([ $response(['hosts' => [ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2), ]]), $response([]), $response($queuedApply), ], $history); $accepted = CloudAdminFabricService::applyRegion('1', $actor, $client); $assert(($accepted['operation']['id'] ?? null) === str_repeat('b', 32) && ($accepted['operation']['kind'] ?? null) === 'apply' && ($accepted['operation']['status'] ?? null) === 'queued', 'Automatic configuration did not return the bounded async receipt.'); $assert(count($history) === 3, 'Automatic configuration performed an unexpected engine request sequence.'); $paths = array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history); $assert($paths === [ 'POST /internal/v1/fabric/probe', 'GET /internal/v1/fabric', 'POST /internal/v1/fabric/apply', ], 'Automatic configuration lost its probe -> plan -> async-admission ordering.'); $plan = json_decode((string) $history[2]['request']->getBody(), true, 32, JSON_THROW_ON_ERROR); $assert(array_keys($plan) === [ 'schema_version', 'fabric_id', 'region', 'overlay_cidr', 'revision', 'listen_port', 'nodes', ] && $plan['fabric_id'] === 'fabric-hk' && $plan['region'] === 'hk' && $plan['overlay_cidr'] === $hkOverlay && $plan['revision'] === 1 && $plan['listen_port'] === 51820 && array_column($plan['nodes'], 'host_id') === ['1', '2'] && $plan['nodes'][0]['endpoint'] === $overlappingEndpoint . ':51820' && $plan['overlay_cidr'] !== $firstHkOverlay && array_column($plan['nodes'], 'overlay_ip') === [ substr($hkOverlay, 0, -4) . '1', substr($hkOverlay, 0, -4) . '2', ], 'PHP did not generate the deterministic bounded Fabric plan.'); $browser = file_get_contents(dirname(__DIR__) . '/public/cloud/admin-fabric.js'); $assert(is_string($browser) && str_contains($browser, "'/api/v1/admin/fabric/regions/' + encodeURIComponent(regionId) + '/' + action") && str_contains($browser, "{ method: 'POST', body: '{}' }") && !str_contains($browser, 'fabric-apply-form'), 'The browser still accepts or assembles a manual Fabric plan.'); $assert(CloudRegionFabric::where('region_id', 1)->first() === null, 'A partial apply success published a mapping before exact active readback.'); // A later read-only listing can observe that the asynchronous root operation // (initial apply or recovery reconcile) already converged. It must expose // only the explicit reconcile confirmation action; // GET neither writes the missing mapping nor asks the browser to probe again. $completedReconcileStatus = $status; $completedReconcileStatus['operation']['kind'] = 'reconcile'; foreach ($completedReconcileStatus['nodes'] as &$completedReconcileNode) { $completedReconcileNode['operation'] = [ 'id' => str_repeat('b', 32), 'status' => 'skipped', 'phase' => 'observe', 'checkpoint' => 'complete', 'reason' => 'none', 'started_at' => 1770000180, 'updated_at' => 1770000200, 'finished_at' => 1770000200, ]; } unset($completedReconcileNode); $terminalSuccessCases = [ 'apply' => $status, 'reconcile' => $completedReconcileStatus, ]; foreach ($terminalSuccessCases as $operationKind => $terminalStatus) { $history = []; $terminalListing = CloudAdminFabricService::listing($actor, $engine([ $response([$terminalStatus]), ], $history)); $terminalRegion = $findById($terminalListing['regions'], 1); $assert(is_array($terminalRegion) && ($terminalRegion['workflow'] ?? null) === [ 'stage' => 'needs_reconcile', 'can_probe' => true, 'can_apply' => false, 'can_reconcile' => true, 'blocking_reasons' => ['fabric_reconcile_required'], 'action_mode' => 'reconcile', ] && ($terminalRegion['plan_preview']['mode'] ?? null) === 'existing' && array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history) === ['GET /internal/v1/fabric'] && CloudRegionFabric::where('region_id', 1)->first() === null, 'A completed unmapped ' . $operationKind . ' regressed to probe or mutated state during GET.'); } // Only a valid terminal success may bypass the extra browser-side probe. // In-flight operations remain operation-driven, while unsuccessful terminal // states still require a new detection before any confirmation is enabled. $nonSuccessfulOperations = [ 'queued' => [ 'stage' => 'operation_running', 'can_probe' => false, 'can_apply' => false, 'can_reconcile' => true, 'blocking_reasons' => ['fabric_operation_running'], 'action_mode' => 'reconcile', ], 'running' => [ 'stage' => 'operation_running', 'can_probe' => false, 'can_apply' => false, 'can_reconcile' => true, 'blocking_reasons' => ['fabric_operation_running'], 'action_mode' => 'reconcile', ], 'failed' => [ 'stage' => 'needs_probe', 'can_probe' => true, 'can_apply' => false, 'can_reconcile' => false, 'blocking_reasons' => ['probe_required'], 'action_mode' => 'probe', ], 'uncertain' => [ 'stage' => 'needs_probe', 'can_probe' => true, 'can_apply' => false, 'can_reconcile' => false, 'blocking_reasons' => ['probe_required'], 'action_mode' => 'probe', ], ]; foreach ($nonSuccessfulOperations as $operationStatus => $expectedWorkflow) { $nonSuccessfulStatus = $completedReconcileStatus; $nonSuccessfulStatus['operation']['status'] = $operationStatus; if (in_array($operationStatus, ['queued', 'running'], true)) { $nonSuccessfulStatus['operation']['finished_at'] = null; } $history = []; $nonSuccessfulListing = CloudAdminFabricService::listing($actor, $engine([ $response([$nonSuccessfulStatus]), ], $history)); $nonSuccessfulRegion = $findById($nonSuccessfulListing['regions'], 1); $assert(is_array($nonSuccessfulRegion) && ($nonSuccessfulRegion['workflow'] ?? null) === $expectedWorkflow && array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history) === ['GET /internal/v1/fabric'] && CloudRegionFabric::where('region_id', 1)->first() === null, 'A non-successful operation used the terminal confirmation shortcut: ' . $operationStatus); } $terminalIdentityCases = [ 'wrong-region' => ['region' => 'legacy-hong-kong'], 'wrong-revision' => ['revision' => 2], ]; foreach ($terminalIdentityCases as $label => $changes) { $wrongTerminalIdentity = array_replace( $completedReconcileStatus, $changes ); $history = []; $wrongIdentityListing = CloudAdminFabricService::listing($actor, $engine([ $response([$wrongTerminalIdentity]), ], $history)); $wrongIdentityRegion = $findById($wrongIdentityListing['regions'], 1); $assert(is_array($wrongIdentityRegion) && ($wrongIdentityRegion['workflow']['stage'] ?? null) === 'needs_probe' && ($wrongIdentityRegion['workflow']['can_reconcile'] ?? true) === false && CloudRegionFabric::where('region_id', 1)->first() === null, 'A succeeded automatic-ID Fabric with ' . $label . ' used the terminal confirmation shortcut.'); } // The reconcile boundary repeats the fresh host probe and exact automatic-plan // validation, then publishes only the mapping. It must not replay the host // reconcile after the successful root operation is already authoritative. $history = []; $terminalReceipt = CloudAdminFabricService::reconcileRegion( '1', $actor, $engine([ $response($completedReconcileStatus), $response([$completedReconcileStatus]), $response(['hosts' => [ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2), ]]), $response($completedReconcileStatus), ], $history) ); $terminalPaths = array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history); $terminalMapping = CloudRegionFabric::where('region_id', 1)->first(); $assert(($terminalReceipt['operation']['id'] ?? null) === str_repeat('b', 32) && ($terminalReceipt['operation']['kind'] ?? null) === 'reconcile' && ($terminalReceipt['operation']['status'] ?? null) === 'succeeded' && $terminalPaths === [ 'GET /internal/v1/fabric/fabric-hk', 'GET /internal/v1/fabric', 'POST /internal/v1/fabric/probe', 'GET /internal/v1/fabric/fabric-hk', ] && !in_array('POST /internal/v1/fabric/fabric-hk/reconcile', $terminalPaths, true) && $terminalMapping !== null && (string) $terminalMapping->fabric_id === 'fabric-hk' && (string) $terminalMapping->state === 'active', 'Terminal confirmation replayed host reconcile or failed to publish its pin.'); CloudRegionFabric::where('region_id', 1)->delete(); // A live VPC without its platform mapping is a migration, not an orphaned // desired plan that the wizard may adopt through reconcile. Capsule::table('cloud_vpcs')->insert([ 'id' => 299, 'region_id' => 1, 'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk', 'fabric_revision' => 1, 'state' => 'active', ]); $history = []; $missingVpcMapping = CloudAdminFabricService::listing($actor, $engine([ $response([$status]), ], $history)); $missingVpcMappingRegion = $findById($missingVpcMapping['regions'], 1); $assert(is_array($missingVpcMappingRegion) && ($missingVpcMappingRegion['workflow']['stage'] ?? null) === 'migration_required' && ($missingVpcMappingRegion['workflow']['can_apply'] ?? true) === false && ($missingVpcMappingRegion['workflow']['can_reconcile'] ?? true) === false && in_array('vpc_fabric_mapping_missing', $missingVpcMappingRegion['workflow']['blocking_reasons'] ?? [], true), 'A live VPC without a region mapping was projected as recoverable.'); $history = []; $client = $engine([], $history); $assertCloudError( static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client), 409, 'A live VPC without a mapping reached automatic Fabric apply' ); $assert($history === [] && CloudRegionFabric::where('region_id', 1)->first() === null, 'Missing VPC mapping validation ran after automatic apply I/O.'); $history = []; $client = $engine([ $response($status), $response([$status]), $response(['hosts' => [ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2), ]]), ], $history); $assertCloudError( static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client), 409, 'A live VPC without a mapping reached host-writing reconcile' ); $assert(array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history) === [ 'GET /internal/v1/fabric/fabric-hk', 'GET /internal/v1/fabric', 'POST /internal/v1/fabric/probe', ] && CloudRegionFabric::where('region_id', 1)->first() === null, 'Missing VPC mapping validation ran after an engine write or created a pin.'); Capsule::table('cloud_vpcs')->where('id', 299)->delete(); // A deterministic desired Fabric left by a partial apply is recoverable only // through the region reconcile action after another fresh detection. $uncertainStatus = $status; $uncertainStatus['state'] = 'uncertain'; $uncertainStatus['next_stage_allowed'] = false; foreach ($uncertainStatus['nodes'] as $uncertainIndex => &$uncertainNode) { $uncertainNode['state'] = 'uncertain'; $uncertainNode['transport_verified'] = false; $uncertainNode['observed_revision'] = null; $uncertainNode['observed_plan_digest'] = null; $uncertainNode['last_error_code'] = 'fabric_remote_command_failed'; $uncertainNode['last_error_phase'] = $uncertainIndex === 0 ? 'apply' : 'observe'; } unset($uncertainNode); $recoveryProbePayload = ['hosts' => [ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2), ]]; $runningStatus = $queuedApply; $runningStatus['operation']['status'] = 'running'; $history = []; $runningListing = CloudAdminFabricService::listing($actor, $engine([ $response([$runningStatus]), ], $history)); $runningRegion = $findById($runningListing['regions'], 1); $assert(is_array($runningRegion) && ($runningRegion['workflow'] ?? null) === [ 'stage' => 'operation_running', 'can_probe' => false, 'can_apply' => false, 'can_reconcile' => true, 'blocking_reasons' => ['fabric_operation_running'], 'action_mode' => 'reconcile', ], 'A running Fabric operation did not expose reconcile-only recovery.'); $history = []; $runningReceipt = CloudAdminFabricService::reconcileRegion( '1', $actor, $engine([ $response($runningStatus), $response([$runningStatus]), $response($recoveryProbePayload), $response($runningStatus), ], $history) ); $runningPaths = array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history); $assert(($runningReceipt['operation']['id'] ?? null) === str_repeat('b', 32) && ($runningReceipt['operation']['status'] ?? null) === 'running' && $runningPaths === [ 'GET /internal/v1/fabric/fabric-hk', 'GET /internal/v1/fabric', 'POST /internal/v1/fabric/probe', 'POST /internal/v1/fabric/fabric-hk/reconcile', ] && CloudRegionFabric::where('region_id', 1)->first() === null, 'Explicit recovery of a running operation short-circuited before Engine reconcile.'); $history = []; $partialRecovery = CloudAdminFabricService::probeRegion('1', $actor, $engine([ $response([$uncertainStatus]), $response($recoveryProbePayload), ], $history)); $partialWorkflow = $partialRecovery['region']['workflow'] ?? null; $partialHostOne = $hostById($partialRecovery['region']['hosts'] ?? [], 1); $partialHostTwo = $hostById($partialRecovery['region']['hosts'] ?? [], 2); $assert(($partialWorkflow['stage'] ?? null) === 'needs_reconcile' && ($partialWorkflow['can_apply'] ?? true) === false && ($partialWorkflow['can_reconcile'] ?? false) === true && ($partialWorkflow['action_mode'] ?? null) === 'reconcile' && in_array('fabric_reconcile_required', $partialWorkflow['blocking_reasons'] ?? [], true) && ($partialRecovery['region']['plan_preview']['mode'] ?? null) === 'existing', 'A partial automatic apply was not projected as a reconcile-only recovery.'); $assert(($partialHostOne['last_error_phase'] ?? null) === 'apply' && ($partialHostTwo['last_error_phase'] ?? null) === 'observe', 'A partial Fabric recovery lost the persisted failure phase diagnostic.'); $history = []; $client = $engine([ $response(['hosts' => [ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2), ]]), $response([$uncertainStatus]), ], $history); $assertCloudError( static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client), 409, 'An unmapped partial Fabric was accepted by the creation-only apply action' ); $partialApplyPaths = array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history); $assert($partialApplyPaths === [ 'POST /internal/v1/fabric/probe', 'GET /internal/v1/fabric', ] && !in_array('POST /internal/v1/fabric/apply', $partialApplyPaths, true) && !in_array('POST /internal/v1/fabric/fabric-hk/reconcile', $partialApplyPaths, true) && CloudRegionFabric::where('region_id', 1)->first() === null, 'Partial-success apply recovery reached an engine write or created a mapping.'); // Reconcile may safely replay only the fixed desired plan, but an uncertain // engine response and exact readback still cannot publish a platform mapping. $history = []; $client = $engine([ $response($uncertainStatus), $response([$uncertainStatus]), $response($recoveryProbePayload), $response($uncertainStatus), $response($uncertainStatus), ], $history); $assertCloudError( static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client), 409, 'An uncertain replay result published an active region mapping' ); $uncertainPaths = array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history); $assert($uncertainPaths === [ 'GET /internal/v1/fabric/fabric-hk', 'GET /internal/v1/fabric', 'POST /internal/v1/fabric/probe', 'POST /internal/v1/fabric/fabric-hk/reconcile', 'GET /internal/v1/fabric/fabric-hk', ] && !in_array('POST /internal/v1/fabric/apply', $uncertainPaths, true) && CloudRegionFabric::where('region_id', 1)->first() === null, 'Uncertain replay either repeated apply or created a platform mapping.'); $history = []; $client = $engine([ $response($uncertainStatus), $response([$uncertainStatus]), $response($recoveryProbePayload), new ConnectException('timeout', new Request( 'POST', 'http://127.0.0.1:8000/internal/v1/fabric/fabric-hk/reconcile' )), ], $history); $assertCloudError( static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client), 502, 'A lost replay response published an active region mapping' ); $unknownReplayPaths = array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history); $assert($unknownReplayPaths === [ 'GET /internal/v1/fabric/fabric-hk', 'GET /internal/v1/fabric', 'POST /internal/v1/fabric/probe', 'POST /internal/v1/fabric/fabric-hk/reconcile', ] && CloudRegionFabric::where('region_id', 1)->first() === null, 'Unknown replay outcome created a platform mapping.'); // A deterministic name is not proof of ownership. With no panel mapping, // recovery must reject every foreign desired-plan dimension before issuing // the host-writing reconcile request. $foreignRevision = $status; $foreignRevision['revision'] = 2; $foreignOverlay = $status; $foreignOverlay['overlay_cidr'] = $overlay('hk', 2); $foreignOverlayBase = substr($foreignOverlay['overlay_cidr'], 0, -4); $foreignOverlay['nodes'][0]['overlay_ip'] = $foreignOverlayBase . '1'; $foreignOverlay['nodes'][1]['overlay_ip'] = $foreignOverlayBase . '2'; $foreignEndpoint = $status; $foreignEndpoint['nodes'][0]['endpoint'] = '203.0.113.250:51820'; $foreignCases = [ 'revision' => [$foreignRevision, false], 'overlay' => [$foreignOverlay, true], 'endpoint' => [$foreignEndpoint, true], ]; foreach ($foreignCases as $label => [$foreign, $needsListAndProbe]) { $history = []; $responses = [$response($foreign)]; if ($needsListAndProbe) { $responses[] = $response([$foreign]); $responses[] = $response($recoveryProbePayload); } $client = $engine($responses, $history); $assertCloudError( static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client), 409, 'A foreign deterministic-ID Fabric with mismatched ' . $label . ' reached reconcile' ); $paths = array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history); $assert($paths === ($needsListAndProbe ? [ 'GET /internal/v1/fabric/fabric-hk', 'GET /internal/v1/fabric', 'POST /internal/v1/fabric/probe', ] : [ 'GET /internal/v1/fabric/fabric-hk', ]) && !in_array('POST /internal/v1/fabric/fabric-hk/reconcile', $paths, true), 'Foreign ' . $label . ' validation occurred after a reconcile write.'); $assert(CloudRegionFabric::where('region_id', 1)->first() === null, 'Foreign ' . $label . ' recovery created a platform mapping.'); } $history = []; $configured = CloudAdminFabricService::reconcileRegion('1', $actor, $engine([ $response($uncertainStatus), $response([$uncertainStatus]), $response(['hosts' => [ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2), ]]), $response($status), $response($status), $response([$status]), ], $history)); $assert(array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history) === [ 'GET /internal/v1/fabric/fabric-hk', 'GET /internal/v1/fabric', 'POST /internal/v1/fabric/probe', 'POST /internal/v1/fabric/fabric-hk/reconcile', 'GET /internal/v1/fabric/fabric-hk', 'GET /internal/v1/fabric', ], 'Partial-success recovery regenerated a plan or skipped exact readback.'); $mapping = CloudRegionFabric::where('region_id', 1)->first(); $configuredRegion = $findById($configured['regions'], 1); $assert(!array_key_exists('fabrics', $configured) && $mapping !== null && (string) $mapping->fabric_id === 'fabric-hk' && (string) $mapping->fabric_region === 'hk' && (int) $mapping->fabric_revision === 1 && (string) $mapping->state === 'active' && is_array($configuredRegion) && !array_key_exists('fabric', $configuredRegion) && $configuredRegion['ready'] === true && $configuredRegion['host_set_matches'] === true && array_map(static fn (array $host): int => (int) $host['host_id'], $configuredRegion['hosts']) === [1, 2], 'Exact active readback did not produce the matching region Fabric mapping.'); $assert(($configuredRegion['workflow'] ?? null) === [ 'stage' => 'active', 'can_probe' => true, 'can_apply' => false, 'can_reconcile' => true, 'blocking_reasons' => [], 'action_mode' => 'none', ] && ($configuredRegion['plan_preview'] ?? null) === [ 'mode' => 'existing', 'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk', 'revision' => 1, 'listen_port' => 51820, 'overlay_cidr' => $hkOverlay, 'host_count' => 2, ], 'An exact mapped Fabric did not converge to the active wizard stage.'); // A host-profile endpoint change does not mutate an existing Fabric plan. // Detection must expose the drift and require a versioned migration instead // of combining a newly displayed endpoint with the old active readback. $history = []; $endpointDriftProbe = CloudAdminFabricService::probeRegion( '1', $actor, $engine([ $response([$status]), $response(['hosts' => [ $probeHost(1, ['endpoint_ip' => '198.51.100.77']), $probeHost(2), ]]), ], $history) ); $endpointDriftRegion = $endpointDriftProbe['region']; $endpointDriftHost = $hostById($endpointDriftRegion['hosts'], 1); $assert(($endpointDriftRegion['ready'] ?? true) === false && ($endpointDriftRegion['workflow']['stage'] ?? null) === 'migration_required' && ($endpointDriftRegion['workflow']['can_apply'] ?? true) === false && ($endpointDriftRegion['workflow']['can_reconcile'] ?? true) === false && in_array('fabric_endpoint_mismatch', $endpointDriftRegion['workflow']['blocking_reasons'] ?? [], true) && is_array($endpointDriftHost) && in_array('fabric_endpoint_mismatch', $endpointDriftHost['reasons'] ?? [], true), 'A changed host endpoint was combined with an old active Fabric plan.'); // The public workflow projection is not an authorization boundary. Direct // callers of reconcileRegion must repeat every platform and fresh-probe gate // before the first host-writing engine request. Capsule::table('cloud_regions')->where('id', 1)->update([ 'status' => 'disabled', 'updated_at' => $now + 1, ]); $history = []; $client = $engine([], $history); $assertCloudError( static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client), 409, 'A disabled region reached mapped Fabric reconcile' ); $assert($history === [], 'Disabled-region reconcile was rejected only after an engine request.'); Capsule::table('cloud_regions')->where('id', 1)->update([ 'status' => 'active', 'updated_at' => $now + 2, ]); Capsule::table('cloud_hosts')->where('id', 2)->update([ 'enabled' => 0, 'updated_at' => $now + 3, ]); $history = []; $client = $engine([], $history); $assertCloudError( static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client), 409, 'A platform-blocked host reached mapped Fabric reconcile' ); $assert($history === [], 'Platform-blocked reconcile was rejected only after an engine request.'); Capsule::table('cloud_hosts')->where('id', 2)->update([ 'enabled' => 1, 'updated_at' => $now + 4, ]); $history = []; $client = $engine([ $response($status), $response([$status]), $response(['hosts' => [ $probeHost(1, [ 'vpc_ready' => false, 'reasons' => ['firewalld_unsupported_requires_maintenance'], 'capabilities' => [ 'firewall_manager' => 'firewalld', 'firewall_mode' => 'unsupported', ], ]), $probeHost(2), ]]), ], $history); $assertCloudError( static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client), 409, 'A failed fresh capability probe reached mapped Fabric reconcile' ); $reconcilePaths = array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history); $assert($reconcilePaths === [ 'GET /internal/v1/fabric/fabric-hk', 'GET /internal/v1/fabric', 'POST /internal/v1/fabric/probe', ] && !in_array('POST /internal/v1/fabric/fabric-hk/reconcile', $reconcilePaths, true), 'Fresh-probe failure was checked only after host-writing reconcile.'); $degradedStatus = $status; $degradedStatus['state'] = 'degraded'; $degradedStatus['next_stage_allowed'] = false; $degradedStatus['nodes'][0]['state'] = 'degraded'; $degradedStatus['nodes'][0]['transport_verified'] = false; $degradedStatus['nodes'][0]['last_error_code'] = 'fabric_host_unavailable'; $history = []; $mappedReconciled = CloudAdminFabricService::reconcileRegion( '1', $actor, $engine([ $response($degradedStatus), $response([$degradedStatus]), $response(['hosts' => [ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2), ]]), $response($status), $response([$status]), ], $history) ); $mappedRegion = $findById($mappedReconciled['regions'], 1); $assert(!array_key_exists('fabrics', $mappedReconciled) && is_array($mappedRegion) && !array_key_exists('fabric', $mappedRegion) && array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history) === [ 'GET /internal/v1/fabric/fabric-hk', 'GET /internal/v1/fabric', 'POST /internal/v1/fabric/probe', 'POST /internal/v1/fabric/fabric-hk/reconcile', 'GET /internal/v1/fabric', ] && ($mappedRegion['workflow']['stage'] ?? null) === 'active', 'A valid mapped degraded Fabric did not pass fresh-gated reconciliation.'); $history = []; $activeProbeFailure = CloudAdminFabricService::probeRegion('1', $actor, $engine([ $response([$status]), $response(['hosts' => [ $probeHost(1, [ 'endpoint_ip' => $overlappingEndpoint, 'vpc_ready' => false, 'reasons' => ['firewalld_unsupported_requires_maintenance'], 'capabilities' => [ 'firewall_manager' => 'firewalld', 'firewall_mode' => 'unsupported', ], ]), $probeHost(2), ]]), ], $history)); $activeBlockedWorkflow = $activeProbeFailure['region']['workflow'] ?? null; $assert(($activeBlockedWorkflow['stage'] ?? null) === 'blocked' && ($activeBlockedWorkflow['can_probe'] ?? false) === true && ($activeBlockedWorkflow['can_apply'] ?? true) === false && ($activeBlockedWorkflow['can_reconcile'] ?? true) === false && ($activeBlockedWorkflow['action_mode'] ?? null) === 'probe' && in_array('firewalld_unsupported_requires_maintenance', $activeBlockedWorkflow['blocking_reasons'] ?? [], true), 'A failed complete fresh probe left active Fabric reconcile enabled.'); // Apply is creation-only once a mapping or engine desired record exists. // Recovery/activation always uses the fresh-gated reconcile action. $history = []; $client = $engine([], $history); $assertCloudError( static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client), 409, 'An active mapping was accepted by the automatic-configuration action' ); $assert($history === [] && (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'active', 'Wrong active->apply action performed engine I/O or changed the mapping.'); CloudRegionFabric::where('region_id', 1)->update([ 'state' => 'disabled', 'updated_at' => $now + 5, ]); $history = []; $disabledListing = CloudAdminFabricService::listing($actor, $engine([ $response([$status]), ], $history)); $disabledListingRegion = $findById($disabledListing['regions'], 1); $assert(is_array($disabledListingRegion) && ($disabledListingRegion['workflow'] ?? null) === [ 'stage' => 'needs_reconcile', 'can_probe' => true, 'can_apply' => false, 'can_reconcile' => true, 'blocking_reasons' => ['fabric_reconcile_required'], 'action_mode' => 'reconcile', ] && array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history) === ['GET /internal/v1/fabric'] && (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'disabled', 'A completed disabled Fabric regressed to probe or was activated during GET.'); $history = []; $disabledProbe = CloudAdminFabricService::probeRegion('1', $actor, $engine([ $response([$status]), $response(['hosts' => [ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2), ]]), ], $history)); $disabledWorkflow = $disabledProbe['region']['workflow'] ?? null; $assert(($disabledWorkflow['stage'] ?? null) === 'needs_reconcile' && ($disabledWorkflow['can_apply'] ?? true) === false && ($disabledWorkflow['can_reconcile'] ?? false) === true && ($disabledWorkflow['action_mode'] ?? null) === 'reconcile', 'A fresh exact disabled mapping did not project reconcile-only activation.'); $history = []; $client = $engine([], $history); $assertCloudError( static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client), 409, 'A disabled mapping was accepted by the creation-only apply action' ); $assert($history === [] && (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'disabled', 'Wrong disabled->apply action performed engine I/O or changed the mapping.'); // A lost exact-readback response during control-plane confirmation leaves the // mapping disabled. A retry may activate only the same Fabric identity while // live VPCs retain their matching immutable pins; host reconcile is not replayed. Capsule::table('cloud_vpcs')->insert([ [ 'id' => 300, 'region_id' => 1, 'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk', 'fabric_revision' => 1, 'state' => 'active', ], [ 'id' => 301, 'region_id' => 1, 'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk', 'fabric_revision' => 1, 'state' => 'degraded', ], ]); $history = []; $client = $engine([ $response($status), $response([$status]), $response(['hosts' => [ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2), ]]), new ConnectException('timeout', new Request( 'GET', 'http://127.0.0.1:8000/internal/v1/fabric/fabric-hk' )), ], $history); $assertCloudError( static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client), 502, 'A lost activation readback did not surface an uncertain reconcile result' ); $unknownPaths = array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history); $assert($unknownPaths === [ 'GET /internal/v1/fabric/fabric-hk', 'GET /internal/v1/fabric', 'POST /internal/v1/fabric/probe', 'GET /internal/v1/fabric/fabric-hk', ] && (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'disabled', 'Unknown confirmation published a mapping or replayed host reconcile.'); $history = []; $activated = CloudAdminFabricService::reconcileRegion('1', $actor, $engine([ $response($status), $response([$status]), $response(['hosts' => [ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2), ]]), $response($status), ], $history)); $assert(($activated['operation']['id'] ?? null) === str_repeat('b', 32) && ($activated['operation']['status'] ?? null) === 'succeeded' && array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history) === [ 'GET /internal/v1/fabric/fabric-hk', 'GET /internal/v1/fabric', 'POST /internal/v1/fabric/probe', 'GET /internal/v1/fabric/fabric-hk', ] && (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'active' && !array_filter($history, static fn (array $entry): bool => $entry['request']->getUri()->getPath() === '/internal/v1/fabric/fabric-hk/reconcile'), 'A repeated confirmation replayed hosts or did not activate the exact mapping.'); // Legacy blank pins and any single-field drift must stop before the // host-writing reconcile request. The disabled mapping remains recoverable // after an operator completes an explicit VPC migration. $mismatchedVpcPins = [ 'legacy-empty' => [ 'fabric_id' => '', 'fabric_region' => '', 'fabric_revision' => 0, ], 'wrong-id' => [ 'fabric_id' => 'fabric-other', 'fabric_region' => 'hk', 'fabric_revision' => 1, ], 'wrong-region' => [ 'fabric_id' => 'fabric-hk', 'fabric_region' => 'sg', 'fabric_revision' => 1, ], 'wrong-revision' => [ 'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk', 'fabric_revision' => 2, ], ]; foreach ($mismatchedVpcPins as $label => $pin) { CloudRegionFabric::where('region_id', 1)->update([ 'state' => 'disabled', 'updated_at' => $now + 6, ]); Capsule::table('cloud_vpcs')->where('id', 300)->update($pin); if ($label === 'legacy-empty') { $projectionHistory = []; $projection = CloudAdminFabricService::listing($actor, $engine([ $response([$status]), ], $projectionHistory)); $projectedRegion = $findById($projection['regions'], 1); $assert(is_array($projectedRegion) && ($projectedRegion['workflow']['stage'] ?? null) === 'migration_required' && ($projectedRegion['workflow']['can_apply'] ?? true) === false && ($projectedRegion['workflow']['can_reconcile'] ?? true) === false && in_array('vpc_fabric_pin_mismatch', $projectedRegion['workflow']['blocking_reasons'] ?? [], true), 'A disabled mapping with legacy VPC pins was projected as actionable.'); } $history = []; $client = $engine([ $response($status), $response([$status]), $response(['hosts' => [ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2), ]]), ], $history); $assertCloudError( static fn () => CloudAdminFabricService::reconcileRegion( '1', $actor, $client ), 409, 'A disabled mapping accepted mismatched live-VPC pins: ' . $label ); $assert(array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history) === [ 'GET /internal/v1/fabric/fabric-hk', 'GET /internal/v1/fabric', 'POST /internal/v1/fabric/probe', ] && (string) CloudRegionFabric::where('region_id', 1) ->value('state') === 'disabled', 'A mismatched live-VPC pin reached an engine write or changed mapping: ' . $label); } // Active mappings use the same server-side pin gate. UI state must not be // able to invoke a host write while any live VPC still points elsewhere. CloudRegionFabric::where('region_id', 1)->update([ 'state' => 'active', 'updated_at' => $now + 6, ]); $history = []; $client = $engine([ $response($status), $response([$status]), $response(['hosts' => [ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2), ]]), ], $history); $assertCloudError( static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client), 409, 'An active mapping reconciled hosts despite a live-VPC pin mismatch' ); $assert(array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history) === [ 'GET /internal/v1/fabric/fabric-hk', 'GET /internal/v1/fabric', 'POST /internal/v1/fabric/probe', ] && (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'active', 'Active live-VPC mismatch validation ran after host-writing reconcile.'); CloudRegionFabric::where('region_id', 1)->update([ 'state' => 'disabled', 'updated_at' => $now + 6, ]); Capsule::table('cloud_vpcs')->where('id', 300)->update([ 'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk', 'fabric_revision' => 1, ]); // The VPC preflight is intentionally outside the exact engine-gate request. // If a pin changes before mapping commit, the transaction must catch the race // and refuse to publish active state without replaying hosts. $history = []; $client = $engine([ $response($status), $response([$status]), $response(['hosts' => [ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2), ]]), static function () use ($response, $status): Response { Capsule::table('cloud_vpcs')->where('id', 301)->update([ 'fabric_revision' => 2, ]); return $response($status); }, ], $history); $assertCloudError( static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client), 409, 'A live-VPC pin race was published after exact engine reconciliation' ); $assert(array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history) === [ 'GET /internal/v1/fabric/fabric-hk', 'GET /internal/v1/fabric', 'POST /internal/v1/fabric/probe', 'GET /internal/v1/fabric/fabric-hk', ] && (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'disabled', 'The transactional live-VPC pin guard did not preserve disabled state.'); Capsule::table('cloud_vpcs')->where('id', 301)->update([ 'fabric_revision' => 1, ]); CloudRegionFabric::where('region_id', 1)->update([ 'state' => 'active', 'updated_at' => $now + 7, ]); $unsafeMappingChanges = [ 'active-to-disabled' => [ 'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk', 'fabric_revision' => 1, 'state' => 'disabled', ], 'identity-change' => [ 'fabric_id' => 'fabric-other', 'fabric_region' => 'hk', 'fabric_revision' => 1, 'state' => 'disabled', ], 'revision-change' => [ 'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk', 'fabric_revision' => 2, 'state' => 'disabled', ], ]; foreach ($unsafeMappingChanges as $label => $change) { $assertCloudError( static fn () => CloudAdminResourceService::saveRegionFabric( '1', $change, $actor ), 409, 'Live VPC accepted unsafe Fabric mapping change: ' . $label ); } $mappingAfterBlockedChanges = CloudRegionFabric::where('region_id', 1)->first(); $assert($mappingAfterBlockedChanges !== null && (string) $mappingAfterBlockedChanges->fabric_id === 'fabric-hk' && (int) $mappingAfterBlockedChanges->fabric_revision === 1 && (string) $mappingAfterBlockedChanges->state === 'active', 'A live-VPC mapping guard partially persisted an unsafe change.'); Capsule::table('cloud_regions')->where('id', 1)->update([ 'status' => 'disabled', 'updated_at' => $now + 13, ]); $history = []; $client = $engine([$response($status)], $history); $assertCloudError( static fn () => CloudAdminResourceService::saveRegionFabric('1', [ 'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk', 'fabric_revision' => 1, 'state' => 'active', ], $actor, false, $client), 409, 'An active mapping was published while its locked region was disabled' ); $assert(array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history) === ['GET /internal/v1/fabric/fabric-hk'] && (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'active', 'Disabled-region linearization changed the mapping or skipped engine gate.'); Capsule::table('cloud_regions')->where('id', 1)->update([ 'status' => 'active', 'updated_at' => $now + 14, ]); Capsule::table('cloud_vpcs')->whereIn('id', [300, 301])->delete(); // The same exact disabled activation remains valid when the region has no // live VPCs; the migration guard must not manufacture a dependency. CloudRegionFabric::where('region_id', 1)->update([ 'state' => 'disabled', 'updated_at' => $now + 14, ]); $history = []; $withoutVpcs = CloudAdminFabricService::reconcileRegion('1', $actor, $engine([ $response($status), $response([$status]), $response(['hosts' => [ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2), ]]), $response($status), ], $history)); $assert(($withoutVpcs['operation']['status'] ?? null) === 'succeeded' && array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history) === [ 'GET /internal/v1/fabric/fabric-hk', 'GET /internal/v1/fabric', 'POST /internal/v1/fabric/probe', 'GET /internal/v1/fabric/fabric-hk', ] && (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'active' && !array_filter($history, static fn (array $entry): bool => $entry['request']->getUri()->getPath() === '/internal/v1/fabric/fabric-hk/reconcile'), 'A no-VPC disabled mapping replayed hosts or did not activate.'); // A disabled activation must re-read platform generation after the exact // engine gate. If the region changes before commit, keep the mapping disabled. CloudRegionFabric::where('region_id', 1)->update([ 'state' => 'disabled', 'updated_at' => $now + 15, ]); $history = []; $client = $engine([ $response($status), $response([$status]), $response(['hosts' => [ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2), ]]), static function () use ($response, $status, $now): Response { Capsule::table('cloud_regions')->where('id', 1)->update([ 'status' => 'disabled', 'updated_at' => $now + 16, ]); return $response($status); }, ], $history); $assertCloudError( static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client), 409, 'A disabled mapping was activated after its region changed during reconcile' ); $assert(array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history) === [ 'GET /internal/v1/fabric/fabric-hk', 'GET /internal/v1/fabric', 'POST /internal/v1/fabric/probe', 'GET /internal/v1/fabric/fabric-hk', ] && (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'disabled', 'Pre-commit generation drift published an active mapping.'); Capsule::table('cloud_regions')->where('id', 1)->update([ 'status' => 'active', 'updated_at' => $now + 17, ]); CloudRegionFabric::where('region_id', 1)->update([ 'state' => 'active', 'updated_at' => $now + 18, ]); // A mapping owns exactly one identity, not every desired record sharing its // region string. Reconcile must read both the mapped detail and full catalog, // then fail before touching hosts when an extra same-region Fabric exists. $history = []; $client = $engine([ $response($status), $response([$status, $historicalStatus]), ], $history); $assertCloudError( static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client), 409, 'Mapped-region reconcile ignored an additional same-region Fabric' ); $assert(array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history) === [ 'GET /internal/v1/fabric/fabric-hk', 'GET /internal/v1/fabric', ] && CloudRegionFabric::where('region_id', 1)->where('state', 'active')->exists(), 'Mapped conflict detection skipped detail/catalog reads or reached reconcile.'); // Once a mapping exists, a missing engine desired record cannot be recreated // from mutable inventory under the already pinned revision. CloudRegionFabric::where('region_id', 1)->update([ 'state' => 'disabled', 'updated_at' => $now + 6, ]); $history = []; $client = $engine([ new Response(404, ['Content-Type' => 'application/json'], json_encode([ 'error_code' => 'fabric_not_found', ], JSON_THROW_ON_ERROR)), ], $history); $assertCloudError( static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client), 404, 'A missing mapped engine desired plan reached reconcile' ); $assert(array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history) === [ 'GET /internal/v1/fabric/fabric-hk', ] && CloudRegionFabric::where('region_id', 1)->where('fabric_revision', 1) ->where('state', 'disabled')->exists(), 'Missing desired-plan recovery mutated the mapping or reached an engine write.'); CloudRegionFabric::where('region_id', 1)->update([ 'state' => 'active', 'updated_at' => $now + 7, ]); $history = []; $missingDesiredListing = CloudAdminFabricService::listing($actor, $engine([ $response([]), ], $history)); $missingDesiredRegion = $findById($missingDesiredListing['regions'], 1); $assert(is_array($missingDesiredRegion) && ($missingDesiredRegion['workflow']['stage'] ?? null) === 'recovery_required' && ($missingDesiredRegion['workflow']['can_apply'] ?? true) === false && ($missingDesiredRegion['workflow']['can_reconcile'] ?? true) === false && ($missingDesiredRegion['workflow']['action_mode'] ?? null) === 'none' && in_array('fabric_desired_missing', $missingDesiredRegion['workflow']['blocking_reasons'] ?? [], true) && $missingDesiredRegion['plan_preview'] === null, 'A mapped region with missing engine desired state was not failed closed.'); // A host can become required while an engine reconcile is in flight. The // second platform read must reject the stale two-host result before mapping // persistence is consulted. $membership = CloudHostResourcePoolMembership::where('host_id', 3)->first(); $history = []; $client = $engine([ $response($status), $response([$status]), static function () use ( $response, $probeHost, $overlappingEndpoint, $membership, $now ): Response { $membership->scheduler_state = 'active'; $membership->updated_at = $now + 2; $membership->save(); return $response(['hosts' => [ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2), ]]); }, ], $history); $assertCloudError( static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client), 409, 'A topology change during fresh-gated reconcile reached host writes' ); $assert(array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history) === [ 'GET /internal/v1/fabric/fabric-hk', 'GET /internal/v1/fabric', 'POST /internal/v1/fabric/probe', ], 'The topology generation gate ran after host-writing reconcile.'); $membership->scheduler_state = 'disabled'; $membership->updated_at = $now + 3; $membership->save(); // Likewise, a concurrent administrator may change the mapping after the // engine exact-readback request starts. The transaction must compare against // the pre-network snapshot rather than overwriting that winner. CloudRegionFabric::where('region_id', 1)->update([ 'state' => 'disabled', 'updated_at' => $now + 8, ]); $history = []; $client = $engine([ $response($status), $response([$status]), $response(['hosts' => [ $probeHost(1, ['endpoint_ip' => $overlappingEndpoint]), $probeHost(2), ]]), static function () use ($response, $status, $now): Response { CloudRegionFabric::where('region_id', 1)->update([ 'state' => 'active', 'updated_at' => $now + 9, ]); return $response($status); }, ], $history); $assertCloudError( static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client), 409, 'A concurrent mapping update was overwritten after engine readback' ); $assert(array_map(static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history) === [ 'GET /internal/v1/fabric/fabric-hk', 'GET /internal/v1/fabric', 'POST /internal/v1/fabric/probe', 'GET /internal/v1/fabric/fabric-hk', ] && (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'active', 'The mapping CAS did not preserve the concurrent writer.'); CloudRegionFabric::where('region_id', 1)->update([ 'state' => 'active', 'updated_at' => $now + 10, ]); // Adding a new required host is a topology migration, not an automatic apply. // The existing desired membership must stay fixed and no mutation may run. CloudRegionFabric::where('region_id', 1)->update([ 'state' => 'disabled', 'updated_at' => $now + 11, ]); $membership->scheduler_state = 'active'; $membership->updated_at = $now + 1; $membership->save(); $history = []; $client = $engine([ $response($status), ], $history); $assertCloudError( static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client), 409, 'Existing Fabric membership drift was automatically overwritten' ); $assert(count($history) === 1 && $history[0]['request']->getUri()->getPath() === '/internal/v1/fabric/fabric-hk' && !array_filter($history, static fn (array $entry): bool => in_array($entry['request']->getUri()->getPath(), [ '/internal/v1/fabric/apply', '/internal/v1/fabric/fabric-hk/reconcile', ], true)), 'Membership drift reached apply/reconcile instead of failing closed.'); $mapping->refresh(); $assert((int) $mapping->fabric_revision === 1 && (string) $mapping->state === 'disabled', 'Membership drift changed the existing region mapping.'); CloudRegionFabric::where('region_id', 1)->update([ 'state' => 'active', 'updated_at' => $now + 12, ]); // A readiness-only inventory may exceed the engine's 64-host Fabric size. // Detection must split it into bounded requests without dropping rows or // converting numeric database IDs into JSON numbers. $sgProbeRows = []; for ($id = 100; $id < 165; $id++) { Capsule::table('cloud_hosts')->insert([ 'id' => $id, 'name' => 'sg-host-' . $id, 'region' => 'sg', 'country_code' => 'SG', 'enabled' => 1, 'resource_pool_id' => 22, 'scheduler_state' => 'active', 'default_network' => 'bridge:br0', 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_host_resource_pool_memberships')->insert([ 'id' => 1000 + $id, 'host_id' => $id, 'resource_pool_id' => 22, 'scheduler_state' => 'active', 'priority' => 0, 'weight' => 100, 'created_at' => $now, 'updated_at' => $now, ]); $sgProbeRows[] = $probeHost($id, [ 'region' => 'sg', 'endpoint_ip' => '198.51.100.' . ($id - 99), ]); } $history = []; $sgProbe = CloudAdminFabricService::probeRegion('2', $actor, $engine([ $response([]), $response(['hosts' => array_slice($sgProbeRows, 0, 64)]), $response(['hosts' => array_slice($sgProbeRows, 64)]), ], $history)); $firstBatch = json_decode((string) $history[1]['request']->getBody(), true, 32, JSON_THROW_ON_ERROR)['host_ids'] ?? []; $secondBatch = json_decode((string) $history[2]['request']->getBody(), true, 32, JSON_THROW_ON_ERROR)['host_ids'] ?? []; $assert(count($sgProbe['region']['hosts']) === 65 && count($firstBatch) === 64 && count($secondBatch) === 1 && array_reduce(array_merge($firstBatch, $secondBatch), static fn (bool $carry, $hostId): bool => $carry && is_string($hostId), true) && $firstBatch[0] === '100' && $secondBatch[0] === '164', 'Readiness detection did not preserve all rows across bounded string-ID batches.'); for ($id = 165; $id <= 228; $id++) { Capsule::table('cloud_hosts')->insert([ 'id' => $id, 'name' => 'sg-host-' . $id, 'region' => 'sg', 'country_code' => 'SG', 'enabled' => 1, 'resource_pool_id' => 22, 'scheduler_state' => 'active', 'default_network' => 'bridge:br0', 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_host_resource_pool_memberships')->insert([ 'id' => 1000 + $id, 'host_id' => $id, 'resource_pool_id' => 22, 'scheduler_state' => 'active', 'priority' => 0, 'weight' => 100, 'created_at' => $now, 'updated_at' => $now, ]); } $history = []; $client = $engine([$response([])], $history); $assertCloudError( static fn () => CloudAdminFabricService::probeRegion('2', $actor, $client), 409, 'An unbounded readiness inventory exceeded the browser request budget' ); $assert(count($history) === 1 && $history[0]['request']->getUri()->getPath() === '/internal/v1/fabric', 'The oversized readiness inventory started a host probe before rejection.'); echo "Administrator Fabric readiness workflow passed.\n";