You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/AdminFabricReadinessWorkflo...

2277 lines
90 KiB
PHP

<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudHostResourcePoolMembership;
use App\Models\CloudRegionFabric;
use App\Services\Cloud\CloudAdminFabricService;
use App\Services\Cloud\CloudAdminResourceService;
use App\Services\Cloud\CloudEngineClient;
use App\Services\Cloud\CloudError;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\ConnectException;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Middleware;
use GuzzleHttp\Psr7\Request;
use GuzzleHttp\Psr7\Response;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$System_Config = [
'cloud_engine_url' => 'http://127.0.0.1:8000',
'cloud_engine_hmac_key' => base64_encode(str_repeat('k', 32)),
];
$assert = static function ($condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$assertCloudError = static function (callable $callback, int $status,
string $message) use ($assert): CloudError {
try {
$callback();
} catch (CloudError $error) {
$assert($error->httpStatus === $status,
$message . ' (unexpected HTTP status ' . $error->httpStatus . ')');
return $error;
}
throw new RuntimeException($message . ' (no CloudError was raised)');
};
$response = static fn ($payload): Response => new Response(200, [
'Content-Type' => 'application/json',
], json_encode($payload, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES));
$engine = static function (array $responses, array &$history): CloudEngineClient {
$mock = new MockHandler($responses);
$stack = HandlerStack::create($mock);
$stack->push(Middleware::history($history));
return new CloudEngineClient(new Client(['handler' => $stack]));
};
$findById = static function (array $rows, int $id): ?array {
foreach ($rows as $row) {
if ((int) ($row['region_id'] ?? 0) === $id) {
return $row;
}
}
return null;
};
$hostById = static function (array $rows, int $id): ?array {
foreach ($rows as $row) {
if ((int) ($row['host_id'] ?? 0) === $id) {
return $row;
}
}
return null;
};
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->getConnection('default')->getSchemaBuilder();
$schema->create('cloud_regions', static function (Blueprint $table): void {
$table->increments('id');
$table->string('code');
$table->string('name');
$table->string('country_code', 2)->default('');
$table->string('status');
$table->integer('sort')->default(0);
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_network_lines', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('region_id');
$table->string('code');
$table->string('name');
$table->boolean('is_default')->default(false);
$table->string('status');
$table->integer('sort')->default(0);
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_resource_pools', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('line_id');
$table->string('code');
$table->string('name');
$table->string('status');
$table->string('scheduling_policy')->default('balanced');
$table->integer('sort')->default(0);
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_hosts', static function (Blueprint $table): void {
$table->increments('id');
$table->string('name');
$table->string('region')->default('');
$table->string('country_code', 2)->default('');
$table->boolean('enabled')->default(true);
$table->integer('resource_pool_id')->nullable();
$table->string('scheduler_state')->default('disabled');
$table->string('default_network')->default('bridge:br0');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_host_resource_pool_memberships',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('host_id');
$table->integer('resource_pool_id');
$table->string('scheduler_state')->default('disabled');
$table->integer('priority')->default(0);
$table->integer('weight')->default(100);
$table->integer('created_at');
$table->integer('updated_at');
$table->unique(['host_id', 'resource_pool_id']);
});
$schema->create('cloud_region_fabrics', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('region_id')->unique();
$table->string('fabric_id')->unique();
$table->string('fabric_region');
$table->integer('fabric_revision');
$table->string('state');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_private_cidr_allocator_configs',
static function (Blueprint $table): void {
$table->integer('id')->primary();
$table->string('pool_cidr', 18);
$table->integer('quantum_prefix_length');
$table->integer('min_prefix_length');
$table->integer('max_prefix_length');
$table->integer('revision');
$table->integer('updated_at');
});
$schema->create('cloud_instances', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('host_id');
$table->integer('resource_pool_id')->nullable();
$table->string('status');
});
$schema->create('cloud_vpcs', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('region_id');
$table->string('fabric_id')->default('');
$table->string('fabric_region')->default('');
$table->integer('fabric_revision')->default(0);
$table->string('state');
});
$schema->create('cloud_vpc_host_deployments',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('vpc_id');
$table->integer('host_id');
$table->string('state');
});
$now = 1770000000;
Capsule::table('cloud_regions')->insert([
['id' => 1, 'code' => 'hk', 'name' => 'Hong Kong', 'country_code' => 'HK',
'status' => 'active', 'sort' => 10, 'created_at' => $now, 'updated_at' => $now],
['id' => 2, 'code' => 'sg', 'name' => 'Singapore', 'country_code' => 'SG',
'status' => 'active', 'sort' => 20, 'created_at' => $now, 'updated_at' => $now],
]);
Capsule::table('cloud_private_cidr_allocator_configs')->insert([
'id' => 1, 'pool_cidr' => '10.0.0.0/8',
'quantum_prefix_length' => 24, 'min_prefix_length' => 16,
'max_prefix_length' => 24, 'revision' => 1, 'updated_at' => $now,
]);
Capsule::table('cloud_network_lines')->insert([
['id' => 11, 'region_id' => 1, 'code' => 'hk-default', 'name' => 'HK default',
'is_default' => 1, 'status' => 'active', 'sort' => 10,
'created_at' => $now, 'updated_at' => $now],
['id' => 12, 'region_id' => 2, 'code' => 'sg-default', 'name' => 'SG default',
'is_default' => 1, 'status' => 'active', 'sort' => 20,
'created_at' => $now, 'updated_at' => $now],
]);
Capsule::table('cloud_resource_pools')->insert([
['id' => 21, 'line_id' => 11, 'code' => 'hk-general', 'name' => 'HK general',
'status' => 'active', 'scheduling_policy' => 'balanced', 'sort' => 10,
'created_at' => $now, 'updated_at' => $now],
['id' => 22, 'line_id' => 12, 'code' => 'sg-general', 'name' => 'SG general',
'status' => 'active', 'scheduling_policy' => 'balanced', 'sort' => 20,
'created_at' => $now, 'updated_at' => $now],
]);
for ($id = 1; $id <= 6; $id++) {
Capsule::table('cloud_hosts')->insert([
'id' => $id,
'name' => 'host-' . $id,
// host-1 remains in the eventual apply set and host-5 exercises the
// larger readiness projection. Both use display labels different from
// the sales-region code; canonical membership owns their placement.
'region' => in_array($id, [1, 5], true) ? 'Hong Kong' : 'hk',
'country_code' => 'HK',
'enabled' => 1,
'resource_pool_id' => 21,
'scheduler_state' => $id <= 5 ? 'active' : 'disabled',
'default_network' => 'bridge:br0',
'created_at' => $now,
'updated_at' => $now,
]);
if ($id <= 5) {
Capsule::table('cloud_host_resource_pool_memberships')->insert([
'id' => 30 + $id,
'host_id' => $id,
'resource_pool_id' => 21,
'scheduler_state' => 'active',
'priority' => 0,
'weight' => 100,
'created_at' => $now,
'updated_at' => $now,
]);
}
}
$actor = [
'user_id' => 99,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
];
$history = [];
$busyClient = $engine([new Response(409, [
'Content-Type' => 'application/json',
], json_encode(['error_code' => 'fabric_probe_busy'], JSON_THROW_ON_ERROR))],
$history);
$busy = $assertCloudError(
static fn () => $busyClient->fabricProbe(['1'], $actor),
409,
'An overlapping engine readiness probe lost its stable 409 contract'
);
$assert(str_contains($busy->getMessage(), '已有地区宿主机检测正在运行')
&& count($history) === 1,
'The reviewed probe-busy error was hidden or retried by PHP.');
// Engine outages keep platform inventory visible, but every workflow write
// gate must fail closed until an authoritative Fabric snapshot is available.
$history = [];
$unavailable = CloudAdminFabricService::listing($actor, $engine([
new Response(503, ['Content-Type' => 'application/json'], json_encode([
'error_code' => 'fabric_host_unavailable',
], JSON_THROW_ON_ERROR)),
], $history));
$unavailableRegion = $findById($unavailable['regions'], 1);
$assert(!array_key_exists('fabrics', $unavailable)
&& $unavailable['engine_available'] === false
&& is_array($unavailableRegion)
&& !array_key_exists('fabric', $unavailableRegion)
&& ($unavailableRegion['workflow'] ?? null) === [
'stage' => 'engine_unavailable',
'can_probe' => false,
'can_apply' => false,
'can_reconcile' => false,
'blocking_reasons' => ['engine_unavailable'],
'action_mode' => 'none',
]
&& $unavailableRegion['plan_preview'] === null,
'An engine outage did not fail every region workflow write gate closed.');
$overlay = static function (string $regionCode, int $offset = 0): string {
$slot = (hexdec(substr(hash('sha256', $regionCode), 0, 8)) + $offset)
% 16384;
return '100.' . (64 + intdiv($slot, 256)) . '.' . ($slot % 256) . '.0/24';
};
$firstHkOverlay = $overlay('hk');
$hkOverlay = $overlay('hk', 1);
$overlappingEndpoint = substr($firstHkOverlay, 0, -4) . '77';
$probeHost = static function (int $id, array $changes = []): array {
$row = [
'host_id' => (string) $id,
'configured' => true,
'reachable' => true,
'region' => 'hk',
'endpoint_ip' => '203.0.113.' . (10 + $id),
'endpoint_source' => 'ssh_host_ipv4',
'probed_at' => 1770000100 + $id,
'vpc_ready' => true,
'reasons' => [],
'capabilities' => [
'os_id' => 'almalinux',
'os_version' => '10',
'network_manager' => 'networkmanager',
'network_manager_expected' => 'networkmanager',
'network_manager_supported' => true,
'deployment_mode' => 'networkmanager_nmcli',
'firewall_manager' => 'nftables',
'firewall_mode' => 'native_nftables',
'ip_forward' => 1,
'rp_filter' => 0,
'rp_filter_all' => 0,
'rp_filter_default' => 1,
'rp_filter_default_effective' => 1,
'rp_filter_underlay_raw' => 1,
'rp_filter_underlay_effective' => 1,
'rp_filter_fabric_raw' => null,
'rp_filter_fabric_effective' => null,
'wireguard_ready' => true,
'wireguard_owned' => false,
'nftables_ready' => true,
'nftables_active' => true,
'nftables_persistent' => true,
'public_selector' => 'bridge:br0',
'public_verified' => true,
],
];
if (isset($changes['capabilities'])) {
$row['capabilities'] = array_replace(
$row['capabilities'], $changes['capabilities']
);
unset($changes['capabilities']);
}
return array_replace($row, $changes);
};
$fabricStatus = static function (array $hostIds) use ($hkOverlay): array {
$digest = str_repeat('a', 64);
$network = substr($hkOverlay, 0, -4);
$nodes = [];
foreach (array_values($hostIds) as $index => $hostId) {
$nodes[] = [
'host_id' => (string) $hostId,
'overlay_ip' => $network . ($index + 1),
'endpoint' => '203.0.113.' . (10 + (int) $hostId) . ':51820',
'listen_port' => 51820,
'public_key' => base64_encode(str_repeat(chr((int) $hostId), 32)),
'state' => 'active',
'transport_verified' => true,
'observed_revision' => 1,
'observed_plan_digest' => $digest,
'last_error_code' => null,
'observed' => [],
];
}
return [
'fabric_id' => 'fabric-hk',
'region' => 'hk',
'overlay_cidr' => $hkOverlay,
'revision' => 1,
'listen_port' => 51820,
'plan_digest' => $digest,
'interface' => 'wg-fabric0',
'state' => 'active',
'next_stage_allowed' => true,
'updated_at' => 1770000200,
'operation' => [
'id' => str_repeat('b', 32),
'kind' => 'apply',
'status' => 'succeeded',
'started_at' => 1770000180,
'updated_at' => 1770000200,
'finished_at' => 1770000200,
],
'nodes' => $nodes,
];
};
$status = $fabricStatus([1, 2]);
$status['nodes'][0]['endpoint'] = $overlappingEndpoint . ':51820';
$driftStatus = $fabricStatus([1, 2, 6]);
$historicalStatus = $driftStatus;
$historicalStatus['fabric_id'] = 'legacy-hk';
$historicalExactStatus = $fabricStatus([1, 2]);
$historicalExactStatus['fabric_id'] = 'legacy-hk';
$historicalExactStatus['region'] = 'legacy-hong-kong';
$foreignRegionStatus = $fabricStatus([1]);
$foreignRegionStatus['fabric_id'] = 'fabric-sg';
$foreignRegionStatus['region'] = 'sg';
// Broken placement relations must fail closed before any engine request. An
// inner join must not silently shrink the required Fabric host union.
Capsule::table('cloud_host_resource_pool_memberships')->insert([
'id' => 99,
'host_id' => 1,
'resource_pool_id' => 999,
'scheduler_state' => 'active',
'priority' => 0,
'weight' => 100,
'created_at' => $now,
'updated_at' => $now,
]);
$history = [];
$client = $engine([], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
503,
'An active membership with a missing pool/line was silently omitted'
);
$assert($history === [],
'A corrupt active membership was rejected only after an engine request.');
Capsule::table('cloud_host_resource_pool_memberships')->where('id', 99)->delete();
Capsule::table('cloud_host_resource_pool_memberships')->insert([
'id' => 100,
'host_id' => 1,
'resource_pool_id' => 22,
'scheduler_state' => 'draining',
'priority' => 0,
'weight' => 100,
'created_at' => $now,
'updated_at' => $now,
]);
$history = [];
$client = $engine([], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
409,
'Concurrent active/draining region memberships were not treated as a conflict'
);
$assert($history === [],
'A live membership conflict was rejected only after an engine request.');
Capsule::table('cloud_host_resource_pool_memberships')->where('id', 100)->delete();
Capsule::table('cloud_instances')->insert([
'id' => 101,
'host_id' => 1,
'resource_pool_id' => 999,
'status' => 'running',
]);
$history = [];
$client = $engine([], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
503,
'A live instance with a missing pool/line was silently omitted'
);
$assert($history === [],
'A corrupt instance placement was rejected only after an engine request.');
Capsule::table('cloud_instances')->where('id', 101)->delete();
Capsule::table('cloud_vpcs')->insert([
'id' => 201,
'region_id' => 999,
'state' => 'active',
]);
Capsule::table('cloud_vpc_host_deployments')->insert([
'id' => 202,
'vpc_id' => 201,
'host_id' => 1,
'state' => 'active',
]);
$history = [];
$client = $engine([], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
503,
'A live VPC deployment with an unknown region was silently omitted'
);
$assert($history === [],
'An invalid VPC deployment region was rejected only after an engine request.');
Capsule::table('cloud_vpc_host_deployments')->where('id', 202)->delete();
Capsule::table('cloud_vpcs')->where('id', 201)->delete();
Capsule::table('cloud_vpc_host_deployments')->insert([
'id' => 203,
'vpc_id' => 999,
'host_id' => 1,
'state' => 'active',
]);
$history = [];
$client = $engine([], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
503,
'A live deployment whose VPC root is missing was silently omitted'
);
$assert($history === [],
'An orphaned VPC deployment was rejected only after an engine request.');
Capsule::table('cloud_vpc_host_deployments')->where('id', 203)->delete();
// Workloads remain part of the exact required-host union even when the host
// no longer has an active scheduler membership.
Capsule::table('cloud_instances')->insert([
'id' => 102,
'host_id' => 6,
'resource_pool_id' => 21,
'status' => 'running',
]);
$history = [];
$instanceListing = CloudAdminFabricService::listing($actor, $engine([
$response([]),
], $history));
$instanceRegion = $findById($instanceListing['regions'], 1);
$instanceHost = is_array($instanceRegion)
? $hostById($instanceRegion['hosts'], 6) : null;
$assert(is_array($instanceHost)
&& $instanceHost['required'] === true
&& in_array('instance', $instanceHost['required_sources'], true)
&& in_array('workload_membership_mismatch', $instanceHost['reasons'], true),
'A live instance host without membership disappeared from readiness.');
Capsule::table('cloud_instances')->where('id', 102)->delete();
Capsule::table('cloud_vpcs')->insert([
'id' => 204,
'region_id' => 1,
'state' => 'active',
]);
Capsule::table('cloud_vpc_host_deployments')->insert([
'id' => 205,
'vpc_id' => 204,
'host_id' => 6,
'state' => 'active',
]);
$history = [];
$deploymentListing = CloudAdminFabricService::listing($actor, $engine([
$response([]),
], $history));
$deploymentRegion = $findById($deploymentListing['regions'], 1);
$deploymentHost = is_array($deploymentRegion)
? $hostById($deploymentRegion['hosts'], 6) : null;
$assert(is_array($deploymentHost)
&& $deploymentHost['required'] === true
&& in_array('vpc_deployment', $deploymentHost['required_sources'], true)
&& in_array('workload_membership_mismatch', $deploymentHost['reasons'], true),
'A live VPC deployment host without membership disappeared from readiness.');
Capsule::table('cloud_vpc_host_deployments')->where('id', 205)->delete();
Capsule::table('cloud_vpcs')->where('id', 204)->delete();
// Existing active Fabric: a disabled host that is not a Fabric member must
// still be visible, and host-5 remains in HK solely by canonical membership.
Capsule::table('cloud_region_fabrics')->insert([
'id' => 1,
'region_id' => 1,
'fabric_id' => 'fabric-hk',
'fabric_region' => 'hk',
'fabric_revision' => 1,
'state' => 'active',
'created_at' => $now,
'updated_at' => $now,
]);
$history = [];
$listing = CloudAdminFabricService::listing($actor, $engine([
$response([$driftStatus]),
], $history));
$hk = $findById($listing['regions'], 1);
$sg = $findById($listing['regions'], 2);
$assert(!array_key_exists('fabrics', $listing)
&& is_array($hk) && is_array($sg)
&& !array_key_exists('fabric', $hk)
&& !array_key_exists('fabric', $sg),
'The readiness inventory dropped a platform region.');
$assert(array_map(static fn (array $host): int => (int) $host['host_id'],
$hk['hosts']) === [1, 2, 3, 4, 5, 6]
&& $sg['hosts'] === [],
'Host geography did not follow membership -> pool -> line -> region.');
$regionDrift = $hostById($hk['hosts'], 5);
$assert(is_array($regionDrift)
&& $regionDrift['engine_region'] === 'Hong Kong'
&& $regionDrift['platform_ready'] === true
&& !in_array('panel_host_region_mismatch', $regionDrift['reasons'], true),
'A display-only host region label still blocked canonical placement.');
$notMember = $hostById($hk['hosts'], 3);
$assert(is_array($notMember)
&& $notMember['fabric_member'] === false
&& in_array('fabric_membership_drift', $notMember['reasons'], true),
'A platform host outside the current Fabric disappeared from readiness.');
$orphan = $hostById($hk['hosts'], 6);
$assert(is_array($orphan)
&& $orphan['fabric_member'] === true
&& in_array('fabric_membership_drift', $orphan['reasons'], true)
&& $hk['ready'] === false
&& $hk['host_set_matches'] === false,
'A scheduler/Fabric membership drift was hidden or promoted to ready.');
$legacyPhaseHost = $hostById($hk['hosts'], 1);
$assert(is_array($legacyPhaseHost)
&& array_key_exists('last_error_phase', $legacyPhaseHost)
&& $legacyPhaseHost['last_error_phase'] === null,
'A mixed-version Fabric response without a failure phase was rejected.');
$phaseStatus = $driftStatus;
foreach (['identity', 'observe', 'apply'] as $index => $phase) {
$phaseStatus['nodes'][$index]['last_error_code'] = 'fabric_remote_command_failed';
$phaseStatus['nodes'][$index]['last_error_phase'] = $phase;
}
$phaseHistory = [];
$phaseProjection = CloudAdminFabricService::listing($actor, $engine([
$response([$phaseStatus]),
], $phaseHistory));
$phaseRegion = $findById($phaseProjection['regions'], 1);
$assert(is_array($phaseRegion)
&& ($hostById($phaseRegion['hosts'], 1)['last_error_phase'] ?? null) === 'identity'
&& ($hostById($phaseRegion['hosts'], 2)['last_error_phase'] ?? null) === 'observe'
&& ($hostById($phaseRegion['hosts'], 6)['last_error_phase'] ?? null) === 'apply',
'Reviewed Fabric failure phases were not projected to their host rows.');
$invalidPhaseStatus = $driftStatus;
$invalidPhaseStatus['nodes'][0]['last_error_code'] = 'fabric_remote_command_failed';
$invalidPhaseStatus['nodes'][0]['last_error_phase'] = 'probe';
$invalidPhaseHistory = [];
$assertCloudError(
static fn () => CloudAdminFabricService::listing($actor, $engine([
$response([$invalidPhaseStatus]),
], $invalidPhaseHistory)),
502,
'An unreviewed Fabric failure phase escaped the Engine response boundary'
);
$orphanPhaseStatus = $driftStatus;
$orphanPhaseStatus['nodes'][0]['last_error_phase'] = 'apply';
$orphanPhaseHistory = [];
$assertCloudError(
static fn () => CloudAdminFabricService::listing($actor, $engine([
$response([$orphanPhaseStatus]),
], $orphanPhaseHistory)),
502,
'A Fabric failure phase without an error code escaped the response boundary'
);
$assert(($hk['workflow']['stage'] ?? null) === 'migration_required'
&& ($hk['workflow']['can_apply'] ?? true) === false
&& ($hk['workflow']['can_reconcile'] ?? true) === false
&& in_array('fabric_membership_drift',
$hk['workflow']['blocking_reasons'] ?? [], true)
&& ($hk['plan_preview']['mode'] ?? null) === 'existing'
&& !array_intersect(['nodes', 'endpoints', 'host_ids'],
array_keys($hk['plan_preview']))
&& ($sg['workflow']['stage'] ?? null) === 'no_hosts'
&& ($sg['workflow']['action_mode'] ?? null) === 'none',
'Membership drift/no-host inventory did not project safe wizard stages.');
// A fresh probe is an isolated per-host snapshot. One failed host must not
// erase successful or independently blocked rows.
CloudRegionFabric::query()->delete();
$probeRows = [
$probeHost(1, [
'endpoint_source' => 'configured',
]),
$probeHost(2, [
'reachable' => false,
'vpc_ready' => false,
'error_code' => 'fabric_host_capability_probe_failed',
]),
$probeHost(3, [
'endpoint_ip' => '10.20.0.10',
'vpc_ready' => true,
'reasons' => [],
'capabilities' => [
'firewall_manager' => 'firewalld',
'firewall_mode' => 'firewalld_nft',
'nftables_ready' => true,
'nftables_active' => false,
'nftables_persistent' => false,
'underlay_runtime_zone' => 'public',
'underlay_permanent_zone' => 'public',
'rp_filter_all' => 3,
'rp_filter_default' => '1',
'rp_filter_fabric_raw' => true,
],
]),
$probeHost(4, ['endpoint_ip' => 'host.example.test']),
$probeHost(5, [
'region' => 'sg',
'endpoint_source' => 'untrusted-engine-value',
]),
$probeHost(6),
];
$history = [];
$probed = CloudAdminFabricService::probeRegion('1', $actor, $engine([
$response([$driftStatus]),
$response(['hosts' => $probeRows]),
], $history));
$rows = $probed['region']['hosts'];
$assert(count($rows) === 6 && ($hostById($rows, 1)['readiness_state'] ?? null) === 'ready',
'A single probe failure discarded healthy host rows.');
$assert(($hostById($rows, 1)['endpoint_source'] ?? null) === 'configured',
'An explicit effective Fabric endpoint source was not projected.');
$assert(($hostById($rows, 1)['capabilities']['firewall_mode'] ?? null)
=== 'native_nftables'
&& ($hostById($rows, 1)['capabilities']['firewall_manager'] ?? null)
=== 'nftables',
'A native nftables host was not preserved as ready.');
$rpFilterCapabilities = $hostById($rows, 1)['capabilities'] ?? [];
$assert(($rpFilterCapabilities['rp_filter'] ?? null) === 0
&& ($rpFilterCapabilities['rp_filter_all'] ?? null) === 0
&& ($rpFilterCapabilities['rp_filter_default'] ?? null) === 1
&& ($rpFilterCapabilities['rp_filter_default_effective'] ?? null) === 1
&& ($rpFilterCapabilities['rp_filter_underlay_raw'] ?? null) === 1
&& ($rpFilterCapabilities['rp_filter_underlay_effective'] ?? null) === 1
&& array_key_exists('rp_filter_fabric_raw', $rpFilterCapabilities)
&& $rpFilterCapabilities['rp_filter_fabric_raw'] === null
&& array_key_exists('rp_filter_fabric_effective', $rpFilterCapabilities)
&& $rpFilterCapabilities['rp_filter_fabric_effective'] === null,
'The bounded raw/effective rp_filter evidence was dropped or Alma strict defaults were hidden.');
$failed = $hostById($rows, 2);
$firewalld = $hostById($rows, 3);
$nonIpv4 = $hostById($rows, 4);
$wrongRegion = $hostById($rows, 5);
$orphan = $hostById($rows, 6);
$assert(is_array($failed) && $failed['probe_checked'] === true
&& $failed['reachable'] === false
&& in_array('fabric_host_capability_probe_failed', $failed['reasons'], true),
'The isolated failed-host result was dropped or promoted.');
$assert(is_array($firewalld)
&& ($firewalld['capabilities']['firewall_manager'] ?? null) === 'firewalld'
&& ($firewalld['capabilities']['firewall_mode'] ?? null) === 'firewalld_nft'
&& ($firewalld['vpc_ready'] ?? true) === false
&& !array_key_exists('underlay_runtime_zone', $firewalld['capabilities'])
&& !array_key_exists('underlay_permanent_zone', $firewalld['capabilities'])
&& !array_key_exists('rp_filter_all', $firewalld['capabilities'])
&& !array_key_exists('rp_filter_default', $firewalld['capabilities'])
&& !array_key_exists('rp_filter_fabric_raw', $firewalld['capabilities'])
&& in_array('firewalld_unsupported_requires_maintenance',
$firewalld['reasons'], true)
&& in_array('fabric_endpoint_vpc_pool_conflict',
$firewalld['reasons'], true),
'A legacy firewalld result was not failed closed with its endpoint blocker.');
$assert(is_array($nonIpv4) && $nonIpv4['endpoint_ip'] === null
&& $nonIpv4['endpoint_source'] === null
&& in_array('fabric_endpoint_unavailable', $nonIpv4['reasons'], true),
'A non-IPv4 WireGuard endpoint was accepted or not explained.');
$assert(is_array($wrongRegion)
&& $wrongRegion['endpoint_ip'] === null
&& $wrongRegion['endpoint_source'] === null
&& $wrongRegion['readiness_state'] === 'blocked'
&& in_array('fabric_endpoint_source_invalid', $wrongRegion['reasons'], true)
&& !in_array('fabric_region_mismatch', $wrongRegion['reasons'], true),
'An untrusted endpoint source or display-only probe region escaped the bounded projection.');
$assert(is_array($orphan) && $orphan['probe_checked'] === true
&& $orphan['fabric_member'] === true
&& in_array('fabric_membership_drift', $orphan['reasons'], true),
'An orphaned Fabric node escaped fresh host detection.');
$assert(array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history) === [
'GET /internal/v1/fabric',
'POST /internal/v1/fabric/probe',
],
'Fresh detection did not freeze the engine membership snapshot before probing.');
$detectedIds = json_decode((string) $history[1]['request']->getBody(), true,
32, JSON_THROW_ON_ERROR)['host_ids'] ?? [];
$assert($detectedIds === ['1', '2', '3', '4', '5', '6'],
'Fresh detection omitted a canonical member or orphaned Fabric node.');
// Retire the probe-only hosts before configuring the clean two-node target.
CloudHostResourcePoolMembership::whereIn('host_id', [3, 4, 5])->update([
'scheduler_state' => 'disabled',
'updated_at' => $now + 1,
]);
// Inventory alone never opens apply: the current request must carry a fresh,
// complete probe snapshot for every required host.
$history = [];
$needsProbeListing = CloudAdminFabricService::listing($actor, $engine([
$response([]),
], $history));
$needsProbeRegion = $findById($needsProbeListing['regions'], 1);
$assert(is_array($needsProbeRegion)
&& ($needsProbeRegion['workflow'] ?? null) === [
'stage' => 'needs_probe',
'can_probe' => true,
'can_apply' => false,
'can_reconcile' => false,
'blocking_reasons' => ['probe_required'],
'action_mode' => 'probe',
]
&& $needsProbeRegion['plan_preview'] === null,
'A stale inventory snapshot opened automatic Fabric apply without probing.');
$history = [];
$blockedWizard = CloudAdminFabricService::probeRegion('1', $actor, $engine([
$response([]),
$response(['hosts' => [
$probeHost(1, [
'capabilities' => [
'firewall_manager' => 'firewalld',
'firewall_mode' => 'firewalld_nft',
],
]),
$probeHost(2),
]]),
], $history));
$blockedWorkflow = $blockedWizard['region']['workflow'] ?? null;
$assert(($blockedWorkflow['stage'] ?? null) === 'blocked'
&& ($blockedWorkflow['can_probe'] ?? false) === true
&& ($blockedWorkflow['can_apply'] ?? true) === false
&& ($blockedWorkflow['can_reconcile'] ?? true) === false
&& ($blockedWorkflow['action_mode'] ?? null) === 'probe'
&& in_array('firewalld_unsupported_requires_maintenance',
$blockedWorkflow['blocking_reasons'] ?? [], true),
'A legacy firewalld probe did not keep the wizard blocked.');
// A fresh successful detection is the only path to ready_to_apply. The
// browser receives a bounded server-owned preview, never nodes or endpoints.
$history = [];
$readyToApply = CloudAdminFabricService::probeRegion('1', $actor, $engine([
$response([]),
$response(['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]]),
], $history));
$readyWorkflow = $readyToApply['region']['workflow'] ?? null;
$readyPreview = $readyToApply['region']['plan_preview'] ?? null;
$assert(!array_key_exists('fabric', $readyToApply['region'])
&& $readyWorkflow === [
'stage' => 'ready_to_apply',
'can_probe' => true,
'can_apply' => true,
'can_reconcile' => false,
'blocking_reasons' => [],
'action_mode' => 'apply',
]
&& $readyPreview === [
'mode' => 'automatic',
'fabric_id' => 'fabric-hk',
'fabric_region' => 'hk',
'revision' => 1,
'listen_port' => 51820,
'overlay_cidr' => $hkOverlay,
'host_count' => 2,
], 'Fresh readiness did not produce the bounded automatic plan preview.');
// Even an exact-host-set custom Fabric remains an explicit migration. The
// wizard never adopts or replays an unmapped legacy identity.
$history = [];
$legacyExactListing = CloudAdminFabricService::listing($actor, $engine([
$response([$historicalExactStatus]),
], $history));
$legacyExactRegion = $findById($legacyExactListing['regions'], 1);
$assert(is_array($legacyExactRegion)
&& ($legacyExactRegion['workflow']['stage'] ?? null) === 'migration_required'
&& ($legacyExactRegion['workflow']['can_apply'] ?? true) === false
&& ($legacyExactRegion['workflow']['can_reconcile'] ?? true) === false
&& array_column($legacyExactRegion['fabric_conflicts'], 'fabric_id')
=== ['legacy-hk'],
'An unmapped custom Fabric was projected as automatically adoptable.');
$history = [];
$client = $engine([
$response(['hosts' => [$probeHost(1), $probeHost(2)]]),
$response([$historicalExactStatus]),
], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
409,
'Automatic configuration adopted an exact unmapped custom Fabric'
);
$assert(count($history) === 2
&& $history[1]['request']->getUri()->getPath() === '/internal/v1/fabric',
'Custom Fabric rejection occurred after a host-writing request.');
// A historical custom-ID Fabric in the same engine region is not owned merely
// because no database mapping exists. It must remain visible (including its
// orphan host), while keeping the region unready and automatic writes closed.
$history = [];
$historicalListing = CloudAdminFabricService::listing($actor, $engine([
$response([$historicalStatus]),
], $history));
$historicalRegion = $findById($historicalListing['regions'], 1);
$historicalOrphan = is_array($historicalRegion)
? $hostById($historicalRegion['hosts'], 6) : null;
$assert(is_array($historicalRegion)
&& $historicalRegion['ready'] === false
&& $historicalRegion['host_set_matches'] === false
&& array_column($historicalRegion['fabric_conflicts'], 'fabric_id')
=== ['legacy-hk']
&& array_map(static fn (array $host): int => (int) $host['host_id'],
$historicalRegion['hosts']) === [1, 2, 6]
&& is_array($historicalOrphan)
&& $historicalOrphan['fabric_member'] === true
&& in_array('fabric_identity_conflict',
$historicalOrphan['reasons'], true),
'An unmapped historical same-region Fabric was hidden or treated as owned.');
$assert(($historicalRegion['workflow']['stage'] ?? null) === 'migration_required'
&& in_array('fabric_identity_conflict',
$historicalRegion['workflow']['blocking_reasons'] ?? [], true)
&& ($historicalRegion['workflow']['can_apply'] ?? true) === false
&& ($historicalRegion['workflow']['can_reconcile'] ?? true) === false,
'A historical Fabric identity conflict did not require manual migration.');
$history = [];
$client = $engine([
$response(['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]]),
$response([$historicalStatus]),
], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
409,
'Automatic configuration adopted an unmapped historical custom Fabric'
);
$assert(array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history) === [
'POST /internal/v1/fabric/probe',
'GET /internal/v1/fabric',
],
'Historical custom-Fabric ownership was checked after an engine write.');
// A different/legacy region label does not make a Fabric safe when it already
// owns a host required by HK: wg-fabric0 is host-global. Keep the conflict
// visible and stop automatic apply before either host-writing endpoint.
$history = [];
$foreignRegionListing = CloudAdminFabricService::listing($actor, $engine([
$response([$foreignRegionStatus]),
], $history));
$foreignRegionHk = $findById($foreignRegionListing['regions'], 1);
$foreignRegionHost = is_array($foreignRegionHk)
? $hostById($foreignRegionHk['hosts'], 1) : null;
$assert(is_array($foreignRegionHk)
&& $foreignRegionHk['ready'] === false
&& array_column($foreignRegionHk['fabric_conflicts'], 'fabric_id')
=== ['fabric-sg']
&& is_array($foreignRegionHost)
&& in_array('fabric_identity_conflict',
$foreignRegionHost['reasons'], true),
'A foreign-region Fabric owning an HK host was hidden from readiness.');
$history = [];
$client = $engine([
$response(['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]]),
$response([$foreignRegionStatus]),
], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
409,
'A foreign-region Fabric owning an HK host reached automatic apply'
);
$paths = array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history);
$assert($paths === [
'POST /internal/v1/fabric/probe',
'GET /internal/v1/fabric',
]
&& !in_array('POST /internal/v1/fabric/apply', $paths, true)
&& !in_array('POST /internal/v1/fabric/fabric-hk/reconcile', $paths, true),
'Foreign-region host ownership was rejected only after an engine write.');
// Automatic configuration always repeats the fresh probe. Each unsafe probe
// class must stop before status lookup or an engine mutation.
$missingEndpointSource = $probeHost(1);
unset($missingEndpointSource['endpoint_source']);
$blockedProbeSets = [
'legacy firewalld mode' => [
$probeHost(1, [
'capabilities' => [
'firewall_manager' => 'firewalld',
'firewall_mode' => 'firewalld_nft',
],
]),
$probeHost(2),
],
'non-IPv4 endpoint' => [
$probeHost(1, ['endpoint_ip' => 'host.example.test']),
$probeHost(2),
],
'non-routable endpoint' => [
$probeHost(1, ['endpoint_ip' => '0.0.0.0']),
$probeHost(2),
],
'invalid endpoint source' => [
$probeHost(1, ['endpoint_source' => 'untrusted-engine-value']),
$probeHost(2),
],
'missing endpoint source' => [
$missingEndpointSource,
$probeHost(2),
],
];
foreach ($blockedProbeSets as $label => $items) {
$history = [];
$client = $engine([$response(['hosts' => $items])], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
409,
'Automatic Fabric configuration did not block ' . $label
);
$assert(count($history) === 1
&& $history[0]['request']->getUri()->getPath() === '/internal/v1/fabric/probe',
'The ' . $label . ' blocker reached an engine read/write after fresh probing.');
}
// WireGuard endpoints are an engine-wide identity for the generated plan.
// A duplicated fresh endpoint must fail before list/apply can mutate Fabric.
$duplicateEndpoint = '203.0.113.250';
$history = [];
$client = $engine([$response(['hosts' => [
$probeHost(1, ['endpoint_ip' => $duplicateEndpoint]),
$probeHost(2, ['endpoint_ip' => $duplicateEndpoint]),
]])], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
409,
'Duplicate automatic WireGuard endpoints reached Fabric apply'
);
$assert(array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history) === [
'POST /internal/v1/fabric/probe',
],
'Duplicate endpoints were rejected only after an engine list/write request.');
// The deterministic ID is still a globally owned database identity. A
// mapping belonging to another region must block both possible engine write
// branches, even when the engine catalog is empty or contains a record whose
// shape would otherwise be accepted for this region.
Capsule::table('cloud_region_fabrics')->insert([
'region_id' => 2,
'fabric_id' => 'fabric-hk',
'fabric_region' => 'hk',
'fabric_revision' => 1,
'state' => 'disabled',
'created_at' => $now,
'updated_at' => $now,
]);
$foreignOwnerCases = [
'empty engine catalog' => [
[
$response(['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]]),
$response([]),
$response($status),
],
'/internal/v1/fabric/apply',
],
'exact-shaped engine record' => [
[
$response(['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]]),
$response([$status]),
$response($status),
],
'/internal/v1/fabric/fabric-hk/reconcile',
],
];
foreach ($foreignOwnerCases as $label => [$responses, $forbiddenPath]) {
$history = [];
$client = $engine($responses, $history);
$assertCloudError(
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
409,
'A foreign region owner reached the ' . $label . ' write branch'
);
$paths = array_map(static fn (array $entry): string =>
$entry['request']->getUri()->getPath(), $history);
$assert($history === [] && !in_array($forbiddenPath, $paths, true),
'Foreign ownership was checked only after ' . $forbiddenPath . '.');
}
CloudRegionFabric::where('region_id', 2)->delete();
// Automatic configuration: the browser supplies only the region action while
// PHP deterministically creates the Fabric identity, overlay and sorted node
// plan. Apply only admits one asynchronous operation and never publishes the
// region mapping; explicit reconcile owns the later exact readback/activation.
$queuedApply = $status;
$queuedApply['state'] = 'applying';
$queuedApply['next_stage_allowed'] = false;
$queuedApply['operation']['status'] = 'queued';
$queuedApply['operation']['finished_at'] = null;
$history = [];
$client = $engine([
$response(['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]]),
$response([]),
$response($queuedApply),
], $history);
$accepted = CloudAdminFabricService::applyRegion('1', $actor, $client);
$assert(($accepted['operation']['id'] ?? null) === str_repeat('b', 32)
&& ($accepted['operation']['kind'] ?? null) === 'apply'
&& ($accepted['operation']['status'] ?? null) === 'queued',
'Automatic configuration did not return the bounded async receipt.');
$assert(count($history) === 3,
'Automatic configuration performed an unexpected engine request sequence.');
$paths = array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history);
$assert($paths === [
'POST /internal/v1/fabric/probe',
'GET /internal/v1/fabric',
'POST /internal/v1/fabric/apply',
], 'Automatic configuration lost its probe -> plan -> async-admission ordering.');
$plan = json_decode((string) $history[2]['request']->getBody(), true,
32, JSON_THROW_ON_ERROR);
$assert(array_keys($plan) === [
'schema_version', 'fabric_id', 'region', 'overlay_cidr',
'revision', 'listen_port', 'nodes',
]
&& $plan['fabric_id'] === 'fabric-hk'
&& $plan['region'] === 'hk'
&& $plan['overlay_cidr'] === $hkOverlay
&& $plan['revision'] === 1
&& $plan['listen_port'] === 51820
&& array_column($plan['nodes'], 'host_id') === ['1', '2']
&& $plan['nodes'][0]['endpoint'] === $overlappingEndpoint . ':51820'
&& $plan['overlay_cidr'] !== $firstHkOverlay
&& array_column($plan['nodes'], 'overlay_ip') === [
substr($hkOverlay, 0, -4) . '1',
substr($hkOverlay, 0, -4) . '2',
], 'PHP did not generate the deterministic bounded Fabric plan.');
$browser = file_get_contents(dirname(__DIR__) . '/public/cloud/admin-fabric.js');
$assert(is_string($browser)
&& str_contains($browser,
"'/api/v1/admin/fabric/regions/' + encodeURIComponent(regionId) + '/' + action")
&& str_contains($browser, "{ method: 'POST', body: '{}' }")
&& !str_contains($browser, 'fabric-apply-form'),
'The browser still accepts or assembles a manual Fabric plan.');
$assert(CloudRegionFabric::where('region_id', 1)->first() === null,
'A partial apply success published a mapping before exact active readback.');
// A live VPC without its platform mapping is a migration, not an orphaned
// desired plan that the wizard may adopt through reconcile.
Capsule::table('cloud_vpcs')->insert([
'id' => 299,
'region_id' => 1,
'fabric_id' => 'fabric-hk',
'fabric_region' => 'hk',
'fabric_revision' => 1,
'state' => 'active',
]);
$history = [];
$missingVpcMapping = CloudAdminFabricService::listing($actor, $engine([
$response([$status]),
], $history));
$missingVpcMappingRegion = $findById($missingVpcMapping['regions'], 1);
$assert(is_array($missingVpcMappingRegion)
&& ($missingVpcMappingRegion['workflow']['stage'] ?? null)
=== 'migration_required'
&& ($missingVpcMappingRegion['workflow']['can_apply'] ?? true) === false
&& ($missingVpcMappingRegion['workflow']['can_reconcile'] ?? true) === false
&& in_array('vpc_fabric_mapping_missing',
$missingVpcMappingRegion['workflow']['blocking_reasons'] ?? [], true),
'A live VPC without a region mapping was projected as recoverable.');
$history = [];
$client = $engine([], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
409,
'A live VPC without a mapping reached automatic Fabric apply'
);
$assert($history === []
&& CloudRegionFabric::where('region_id', 1)->first() === null,
'Missing VPC mapping validation ran after automatic apply I/O.');
$history = [];
$client = $engine([
$response($status),
$response([$status]),
$response(['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]]),
], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
409,
'A live VPC without a mapping reached host-writing reconcile'
);
$assert(array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history) === [
'GET /internal/v1/fabric/fabric-hk',
'GET /internal/v1/fabric',
'POST /internal/v1/fabric/probe',
]
&& CloudRegionFabric::where('region_id', 1)->first() === null,
'Missing VPC mapping validation ran after an engine write or created a pin.');
Capsule::table('cloud_vpcs')->where('id', 299)->delete();
// A deterministic desired Fabric left by a partial apply is recoverable only
// through the region reconcile action after another fresh detection.
$uncertainStatus = $status;
$uncertainStatus['state'] = 'uncertain';
$uncertainStatus['next_stage_allowed'] = false;
foreach ($uncertainStatus['nodes'] as $uncertainIndex => &$uncertainNode) {
$uncertainNode['state'] = 'uncertain';
$uncertainNode['transport_verified'] = false;
$uncertainNode['observed_revision'] = null;
$uncertainNode['observed_plan_digest'] = null;
$uncertainNode['last_error_code'] = 'fabric_remote_command_failed';
$uncertainNode['last_error_phase'] = $uncertainIndex === 0 ? 'apply' : 'observe';
}
unset($uncertainNode);
$recoveryProbePayload = ['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]];
$runningStatus = $queuedApply;
$runningStatus['operation']['status'] = 'running';
$history = [];
$runningListing = CloudAdminFabricService::listing($actor, $engine([
$response([$runningStatus]),
], $history));
$runningRegion = $findById($runningListing['regions'], 1);
$assert(is_array($runningRegion)
&& ($runningRegion['workflow'] ?? null) === [
'stage' => 'operation_running',
'can_probe' => false,
'can_apply' => false,
'can_reconcile' => true,
'blocking_reasons' => ['fabric_operation_running'],
'action_mode' => 'reconcile',
],
'A running Fabric operation did not expose reconcile-only recovery.');
$history = [];
$runningReceipt = CloudAdminFabricService::reconcileRegion(
'1', $actor, $engine([
$response($runningStatus),
$response([$runningStatus]),
$response($recoveryProbePayload),
$response($runningStatus),
], $history)
);
$runningPaths = array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history);
$assert(($runningReceipt['operation']['id'] ?? null) === str_repeat('b', 32)
&& ($runningReceipt['operation']['status'] ?? null) === 'running'
&& $runningPaths === [
'GET /internal/v1/fabric/fabric-hk',
'GET /internal/v1/fabric',
'POST /internal/v1/fabric/probe',
'POST /internal/v1/fabric/fabric-hk/reconcile',
]
&& CloudRegionFabric::where('region_id', 1)->first() === null,
'Explicit recovery of a running operation short-circuited before Engine reconcile.');
$history = [];
$partialRecovery = CloudAdminFabricService::probeRegion('1', $actor, $engine([
$response([$uncertainStatus]),
$response($recoveryProbePayload),
], $history));
$partialWorkflow = $partialRecovery['region']['workflow'] ?? null;
$partialHostOne = $hostById($partialRecovery['region']['hosts'] ?? [], 1);
$partialHostTwo = $hostById($partialRecovery['region']['hosts'] ?? [], 2);
$assert(($partialWorkflow['stage'] ?? null) === 'needs_reconcile'
&& ($partialWorkflow['can_apply'] ?? true) === false
&& ($partialWorkflow['can_reconcile'] ?? false) === true
&& ($partialWorkflow['action_mode'] ?? null) === 'reconcile'
&& in_array('fabric_reconcile_required',
$partialWorkflow['blocking_reasons'] ?? [], true)
&& ($partialRecovery['region']['plan_preview']['mode'] ?? null) === 'existing',
'A partial automatic apply was not projected as a reconcile-only recovery.');
$assert(($partialHostOne['last_error_phase'] ?? null) === 'apply'
&& ($partialHostTwo['last_error_phase'] ?? null) === 'observe',
'A partial Fabric recovery lost the persisted failure phase diagnostic.');
$history = [];
$client = $engine([
$response(['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]]),
$response([$uncertainStatus]),
], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
409,
'An unmapped partial Fabric was accepted by the creation-only apply action'
);
$partialApplyPaths = array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history);
$assert($partialApplyPaths === [
'POST /internal/v1/fabric/probe',
'GET /internal/v1/fabric',
]
&& !in_array('POST /internal/v1/fabric/apply', $partialApplyPaths, true)
&& !in_array('POST /internal/v1/fabric/fabric-hk/reconcile',
$partialApplyPaths, true)
&& CloudRegionFabric::where('region_id', 1)->first() === null,
'Partial-success apply recovery reached an engine write or created a mapping.');
// Reconcile may safely replay only the fixed desired plan, but an uncertain
// engine response and exact readback still cannot publish a platform mapping.
$history = [];
$client = $engine([
$response($uncertainStatus),
$response([$uncertainStatus]),
$response($recoveryProbePayload),
$response($uncertainStatus),
$response($uncertainStatus),
], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
409,
'An uncertain replay result published an active region mapping'
);
$uncertainPaths = array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history);
$assert($uncertainPaths === [
'GET /internal/v1/fabric/fabric-hk',
'GET /internal/v1/fabric',
'POST /internal/v1/fabric/probe',
'POST /internal/v1/fabric/fabric-hk/reconcile',
'GET /internal/v1/fabric/fabric-hk',
]
&& !in_array('POST /internal/v1/fabric/apply', $uncertainPaths, true)
&& CloudRegionFabric::where('region_id', 1)->first() === null,
'Uncertain replay either repeated apply or created a platform mapping.');
$history = [];
$client = $engine([
$response($uncertainStatus),
$response([$uncertainStatus]),
$response($recoveryProbePayload),
new ConnectException('timeout', new Request(
'POST', 'http://127.0.0.1:8000/internal/v1/fabric/fabric-hk/reconcile'
)),
], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
502,
'A lost replay response published an active region mapping'
);
$unknownReplayPaths = array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history);
$assert($unknownReplayPaths === [
'GET /internal/v1/fabric/fabric-hk',
'GET /internal/v1/fabric',
'POST /internal/v1/fabric/probe',
'POST /internal/v1/fabric/fabric-hk/reconcile',
]
&& CloudRegionFabric::where('region_id', 1)->first() === null,
'Unknown replay outcome created a platform mapping.');
// A deterministic name is not proof of ownership. With no panel mapping,
// recovery must reject every foreign desired-plan dimension before issuing
// the host-writing reconcile request.
$foreignRevision = $status;
$foreignRevision['revision'] = 2;
$foreignOverlay = $status;
$foreignOverlay['overlay_cidr'] = $overlay('hk', 2);
$foreignOverlayBase = substr($foreignOverlay['overlay_cidr'], 0, -4);
$foreignOverlay['nodes'][0]['overlay_ip'] = $foreignOverlayBase . '1';
$foreignOverlay['nodes'][1]['overlay_ip'] = $foreignOverlayBase . '2';
$foreignEndpoint = $status;
$foreignEndpoint['nodes'][0]['endpoint'] = '203.0.113.250:51820';
$foreignCases = [
'revision' => [$foreignRevision, false],
'overlay' => [$foreignOverlay, true],
'endpoint' => [$foreignEndpoint, true],
];
foreach ($foreignCases as $label => [$foreign, $needsListAndProbe]) {
$history = [];
$responses = [$response($foreign)];
if ($needsListAndProbe) {
$responses[] = $response([$foreign]);
$responses[] = $response($recoveryProbePayload);
}
$client = $engine($responses, $history);
$assertCloudError(
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
409,
'A foreign deterministic-ID Fabric with mismatched ' . $label
. ' reached reconcile'
);
$paths = array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history);
$assert($paths === ($needsListAndProbe ? [
'GET /internal/v1/fabric/fabric-hk',
'GET /internal/v1/fabric',
'POST /internal/v1/fabric/probe',
] : [
'GET /internal/v1/fabric/fabric-hk',
])
&& !in_array('POST /internal/v1/fabric/fabric-hk/reconcile', $paths, true),
'Foreign ' . $label . ' validation occurred after a reconcile write.');
$assert(CloudRegionFabric::where('region_id', 1)->first() === null,
'Foreign ' . $label . ' recovery created a platform mapping.');
}
$history = [];
$configured = CloudAdminFabricService::reconcileRegion('1', $actor, $engine([
$response($uncertainStatus),
$response([$uncertainStatus]),
$response(['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]]),
$response($status),
$response($status),
$response([$status]),
], $history));
$assert(array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history) === [
'GET /internal/v1/fabric/fabric-hk',
'GET /internal/v1/fabric',
'POST /internal/v1/fabric/probe',
'POST /internal/v1/fabric/fabric-hk/reconcile',
'GET /internal/v1/fabric/fabric-hk',
'GET /internal/v1/fabric',
],
'Partial-success recovery regenerated a plan or skipped exact readback.');
$mapping = CloudRegionFabric::where('region_id', 1)->first();
$configuredRegion = $findById($configured['regions'], 1);
$assert(!array_key_exists('fabrics', $configured)
&& $mapping !== null
&& (string) $mapping->fabric_id === 'fabric-hk'
&& (string) $mapping->fabric_region === 'hk'
&& (int) $mapping->fabric_revision === 1
&& (string) $mapping->state === 'active'
&& is_array($configuredRegion)
&& !array_key_exists('fabric', $configuredRegion)
&& $configuredRegion['ready'] === true
&& $configuredRegion['host_set_matches'] === true
&& array_map(static fn (array $host): int => (int) $host['host_id'],
$configuredRegion['hosts']) === [1, 2],
'Exact active readback did not produce the matching region Fabric mapping.');
$assert(($configuredRegion['workflow'] ?? null) === [
'stage' => 'active',
'can_probe' => true,
'can_apply' => false,
'can_reconcile' => true,
'blocking_reasons' => [],
'action_mode' => 'none',
]
&& ($configuredRegion['plan_preview'] ?? null) === [
'mode' => 'existing',
'fabric_id' => 'fabric-hk',
'fabric_region' => 'hk',
'revision' => 1,
'listen_port' => 51820,
'overlay_cidr' => $hkOverlay,
'host_count' => 2,
], 'An exact mapped Fabric did not converge to the active wizard stage.');
// A host-profile endpoint change does not mutate an existing Fabric plan.
// Detection must expose the drift and require a versioned migration instead
// of combining a newly displayed endpoint with the old active readback.
$history = [];
$endpointDriftProbe = CloudAdminFabricService::probeRegion(
'1', $actor, $engine([
$response([$status]),
$response(['hosts' => [
$probeHost(1, ['endpoint_ip' => '198.51.100.77']),
$probeHost(2),
]]),
], $history)
);
$endpointDriftRegion = $endpointDriftProbe['region'];
$endpointDriftHost = $hostById($endpointDriftRegion['hosts'], 1);
$assert(($endpointDriftRegion['ready'] ?? true) === false
&& ($endpointDriftRegion['workflow']['stage'] ?? null)
=== 'migration_required'
&& ($endpointDriftRegion['workflow']['can_apply'] ?? true) === false
&& ($endpointDriftRegion['workflow']['can_reconcile'] ?? true) === false
&& in_array('fabric_endpoint_mismatch',
$endpointDriftRegion['workflow']['blocking_reasons'] ?? [], true)
&& is_array($endpointDriftHost)
&& in_array('fabric_endpoint_mismatch',
$endpointDriftHost['reasons'] ?? [], true),
'A changed host endpoint was combined with an old active Fabric plan.');
// The public workflow projection is not an authorization boundary. Direct
// callers of reconcileRegion must repeat every platform and fresh-probe gate
// before the first host-writing engine request.
Capsule::table('cloud_regions')->where('id', 1)->update([
'status' => 'disabled',
'updated_at' => $now + 1,
]);
$history = [];
$client = $engine([], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
409,
'A disabled region reached mapped Fabric reconcile'
);
$assert($history === [],
'Disabled-region reconcile was rejected only after an engine request.');
Capsule::table('cloud_regions')->where('id', 1)->update([
'status' => 'active',
'updated_at' => $now + 2,
]);
Capsule::table('cloud_hosts')->where('id', 2)->update([
'enabled' => 0,
'updated_at' => $now + 3,
]);
$history = [];
$client = $engine([], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
409,
'A platform-blocked host reached mapped Fabric reconcile'
);
$assert($history === [],
'Platform-blocked reconcile was rejected only after an engine request.');
Capsule::table('cloud_hosts')->where('id', 2)->update([
'enabled' => 1,
'updated_at' => $now + 4,
]);
$history = [];
$client = $engine([
$response($status),
$response([$status]),
$response(['hosts' => [
$probeHost(1, [
'vpc_ready' => false,
'reasons' => ['firewalld_unsupported_requires_maintenance'],
'capabilities' => [
'firewall_manager' => 'firewalld',
'firewall_mode' => 'unsupported',
],
]),
$probeHost(2),
]]),
], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
409,
'A failed fresh capability probe reached mapped Fabric reconcile'
);
$reconcilePaths = array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history);
$assert($reconcilePaths === [
'GET /internal/v1/fabric/fabric-hk',
'GET /internal/v1/fabric',
'POST /internal/v1/fabric/probe',
] && !in_array('POST /internal/v1/fabric/fabric-hk/reconcile',
$reconcilePaths, true),
'Fresh-probe failure was checked only after host-writing reconcile.');
$degradedStatus = $status;
$degradedStatus['state'] = 'degraded';
$degradedStatus['next_stage_allowed'] = false;
$degradedStatus['nodes'][0]['state'] = 'degraded';
$degradedStatus['nodes'][0]['transport_verified'] = false;
$degradedStatus['nodes'][0]['last_error_code'] = 'fabric_host_unavailable';
$history = [];
$mappedReconciled = CloudAdminFabricService::reconcileRegion(
'1', $actor, $engine([
$response($degradedStatus),
$response([$degradedStatus]),
$response(['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]]),
$response($status),
$response([$status]),
], $history)
);
$mappedRegion = $findById($mappedReconciled['regions'], 1);
$assert(!array_key_exists('fabrics', $mappedReconciled)
&& is_array($mappedRegion)
&& !array_key_exists('fabric', $mappedRegion)
&& array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history) === [
'GET /internal/v1/fabric/fabric-hk',
'GET /internal/v1/fabric',
'POST /internal/v1/fabric/probe',
'POST /internal/v1/fabric/fabric-hk/reconcile',
'GET /internal/v1/fabric',
]
&& ($mappedRegion['workflow']['stage'] ?? null) === 'active',
'A valid mapped degraded Fabric did not pass fresh-gated reconciliation.');
$history = [];
$activeProbeFailure = CloudAdminFabricService::probeRegion('1', $actor, $engine([
$response([$status]),
$response(['hosts' => [
$probeHost(1, [
'endpoint_ip' => $overlappingEndpoint,
'vpc_ready' => false,
'reasons' => ['firewalld_unsupported_requires_maintenance'],
'capabilities' => [
'firewall_manager' => 'firewalld',
'firewall_mode' => 'unsupported',
],
]),
$probeHost(2),
]]),
], $history));
$activeBlockedWorkflow = $activeProbeFailure['region']['workflow'] ?? null;
$assert(($activeBlockedWorkflow['stage'] ?? null) === 'blocked'
&& ($activeBlockedWorkflow['can_probe'] ?? false) === true
&& ($activeBlockedWorkflow['can_apply'] ?? true) === false
&& ($activeBlockedWorkflow['can_reconcile'] ?? true) === false
&& ($activeBlockedWorkflow['action_mode'] ?? null) === 'probe'
&& in_array('firewalld_unsupported_requires_maintenance',
$activeBlockedWorkflow['blocking_reasons'] ?? [], true),
'A failed complete fresh probe left active Fabric reconcile enabled.');
// Apply is creation-only once a mapping or engine desired record exists.
// Recovery/activation always uses the fresh-gated reconcile action.
$history = [];
$client = $engine([], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
409,
'An active mapping was accepted by the automatic-configuration action'
);
$assert($history === []
&& (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'active',
'Wrong active->apply action performed engine I/O or changed the mapping.');
CloudRegionFabric::where('region_id', 1)->update([
'state' => 'disabled',
'updated_at' => $now + 5,
]);
$history = [];
$disabledProbe = CloudAdminFabricService::probeRegion('1', $actor, $engine([
$response([$status]),
$response(['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]]),
], $history));
$disabledWorkflow = $disabledProbe['region']['workflow'] ?? null;
$assert(($disabledWorkflow['stage'] ?? null) === 'needs_reconcile'
&& ($disabledWorkflow['can_apply'] ?? true) === false
&& ($disabledWorkflow['can_reconcile'] ?? false) === true
&& ($disabledWorkflow['action_mode'] ?? null) === 'reconcile',
'A fresh exact disabled mapping did not project reconcile-only activation.');
$history = [];
$client = $engine([], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
409,
'A disabled mapping was accepted by the creation-only apply action'
);
$assert($history === []
&& (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'disabled',
'Wrong disabled->apply action performed engine I/O or changed the mapping.');
// A lost response after host reconciliation leaves the mapping disabled. A
// repeated reconcile must be idempotent and may perform the exact
// disabled->active pin even while live VPCs retain the same Fabric identity.
Capsule::table('cloud_vpcs')->insert([
[
'id' => 300, 'region_id' => 1,
'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk',
'fabric_revision' => 1, 'state' => 'active',
],
[
'id' => 301, 'region_id' => 1,
'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk',
'fabric_revision' => 1, 'state' => 'degraded',
],
]);
$history = [];
$client = $engine([
$response($status),
$response([$status]),
$response(['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]]),
$response($status),
new ConnectException('timeout', new Request(
'GET', 'http://127.0.0.1:8000/internal/v1/fabric/fabric-hk'
)),
], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
502,
'A lost activation readback did not surface an uncertain reconcile result'
);
$unknownPaths = array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history);
$assert($unknownPaths === [
'GET /internal/v1/fabric/fabric-hk',
'GET /internal/v1/fabric',
'POST /internal/v1/fabric/probe',
'POST /internal/v1/fabric/fabric-hk/reconcile',
'GET /internal/v1/fabric/fabric-hk',
]
&& (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'disabled',
'Unknown reconcile published a mapping or skipped exact post-write readback.');
$history = [];
$activated = CloudAdminFabricService::reconcileRegion('1', $actor, $engine([
$response($status),
$response([$status]),
$response(['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]]),
$response($status),
$response($status),
$response([$status]),
], $history));
$activatedRegion = $findById($activated['regions'], 1);
$assert(!array_key_exists('fabrics', $activated)
&& is_array($activatedRegion)
&& !array_key_exists('fabric', $activatedRegion)
&& array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history) === [
'GET /internal/v1/fabric/fabric-hk',
'GET /internal/v1/fabric',
'POST /internal/v1/fabric/probe',
'POST /internal/v1/fabric/fabric-hk/reconcile',
'GET /internal/v1/fabric/fabric-hk',
'GET /internal/v1/fabric',
]
&& (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'active'
&& ($activatedRegion['workflow']['stage'] ?? null) === 'active',
'A repeated reconcile did not converge exact disabled state to active.');
// Legacy blank pins and any single-field drift must stop before the
// host-writing reconcile request. The disabled mapping remains recoverable
// after an operator completes an explicit VPC migration.
$mismatchedVpcPins = [
'legacy-empty' => [
'fabric_id' => '', 'fabric_region' => '', 'fabric_revision' => 0,
],
'wrong-id' => [
'fabric_id' => 'fabric-other', 'fabric_region' => 'hk',
'fabric_revision' => 1,
],
'wrong-region' => [
'fabric_id' => 'fabric-hk', 'fabric_region' => 'sg',
'fabric_revision' => 1,
],
'wrong-revision' => [
'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk',
'fabric_revision' => 2,
],
];
foreach ($mismatchedVpcPins as $label => $pin) {
CloudRegionFabric::where('region_id', 1)->update([
'state' => 'disabled',
'updated_at' => $now + 6,
]);
Capsule::table('cloud_vpcs')->where('id', 300)->update($pin);
if ($label === 'legacy-empty') {
$projectionHistory = [];
$projection = CloudAdminFabricService::listing($actor, $engine([
$response([$status]),
], $projectionHistory));
$projectedRegion = $findById($projection['regions'], 1);
$assert(is_array($projectedRegion)
&& ($projectedRegion['workflow']['stage'] ?? null)
=== 'migration_required'
&& ($projectedRegion['workflow']['can_apply'] ?? true) === false
&& ($projectedRegion['workflow']['can_reconcile'] ?? true) === false
&& in_array('vpc_fabric_pin_mismatch',
$projectedRegion['workflow']['blocking_reasons'] ?? [], true),
'A disabled mapping with legacy VPC pins was projected as actionable.');
}
$history = [];
$client = $engine([
$response($status),
$response([$status]),
$response(['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]]),
], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::reconcileRegion(
'1', $actor, $client
),
409,
'A disabled mapping accepted mismatched live-VPC pins: ' . $label
);
$assert(array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' '
. $entry['request']->getUri()->getPath(),
$history) === [
'GET /internal/v1/fabric/fabric-hk',
'GET /internal/v1/fabric',
'POST /internal/v1/fabric/probe',
]
&& (string) CloudRegionFabric::where('region_id', 1)
->value('state') === 'disabled',
'A mismatched live-VPC pin reached an engine write or changed mapping: '
. $label);
}
// Active mappings use the same server-side pin gate. UI state must not be
// able to invoke a host write while any live VPC still points elsewhere.
CloudRegionFabric::where('region_id', 1)->update([
'state' => 'active',
'updated_at' => $now + 6,
]);
$history = [];
$client = $engine([
$response($status),
$response([$status]),
$response(['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]]),
], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
409,
'An active mapping reconciled hosts despite a live-VPC pin mismatch'
);
$assert(array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history) === [
'GET /internal/v1/fabric/fabric-hk',
'GET /internal/v1/fabric',
'POST /internal/v1/fabric/probe',
]
&& (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'active',
'Active live-VPC mismatch validation ran after host-writing reconcile.');
CloudRegionFabric::where('region_id', 1)->update([
'state' => 'disabled',
'updated_at' => $now + 6,
]);
Capsule::table('cloud_vpcs')->where('id', 300)->update([
'fabric_id' => 'fabric-hk',
'fabric_region' => 'hk',
'fabric_revision' => 1,
]);
// The preflight is intentionally outside the long engine request. If a VPC
// pin changes after host reconciliation, the mapping transaction must catch
// the race and refuse to publish active state.
$history = [];
$client = $engine([
$response($status),
$response([$status]),
$response(['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]]),
$response($status),
static function () use ($response, $status): Response {
Capsule::table('cloud_vpcs')->where('id', 301)->update([
'fabric_revision' => 2,
]);
return $response($status);
},
], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
409,
'A live-VPC pin race was published after exact engine reconciliation'
);
$assert(array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history) === [
'GET /internal/v1/fabric/fabric-hk',
'GET /internal/v1/fabric',
'POST /internal/v1/fabric/probe',
'POST /internal/v1/fabric/fabric-hk/reconcile',
'GET /internal/v1/fabric/fabric-hk',
]
&& (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'disabled',
'The transactional live-VPC pin guard did not preserve disabled state.');
Capsule::table('cloud_vpcs')->where('id', 301)->update([
'fabric_revision' => 1,
]);
CloudRegionFabric::where('region_id', 1)->update([
'state' => 'active',
'updated_at' => $now + 7,
]);
$unsafeMappingChanges = [
'active-to-disabled' => [
'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk',
'fabric_revision' => 1, 'state' => 'disabled',
],
'identity-change' => [
'fabric_id' => 'fabric-other', 'fabric_region' => 'hk',
'fabric_revision' => 1, 'state' => 'disabled',
],
'revision-change' => [
'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk',
'fabric_revision' => 2, 'state' => 'disabled',
],
];
foreach ($unsafeMappingChanges as $label => $change) {
$assertCloudError(
static fn () => CloudAdminResourceService::saveRegionFabric(
'1', $change, $actor
),
409,
'Live VPC accepted unsafe Fabric mapping change: ' . $label
);
}
$mappingAfterBlockedChanges = CloudRegionFabric::where('region_id', 1)->first();
$assert($mappingAfterBlockedChanges !== null
&& (string) $mappingAfterBlockedChanges->fabric_id === 'fabric-hk'
&& (int) $mappingAfterBlockedChanges->fabric_revision === 1
&& (string) $mappingAfterBlockedChanges->state === 'active',
'A live-VPC mapping guard partially persisted an unsafe change.');
Capsule::table('cloud_regions')->where('id', 1)->update([
'status' => 'disabled',
'updated_at' => $now + 13,
]);
$history = [];
$client = $engine([$response($status)], $history);
$assertCloudError(
static fn () => CloudAdminResourceService::saveRegionFabric('1', [
'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk',
'fabric_revision' => 1, 'state' => 'active',
], $actor, false, $client),
409,
'An active mapping was published while its locked region was disabled'
);
$assert(array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history) === ['GET /internal/v1/fabric/fabric-hk']
&& (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'active',
'Disabled-region linearization changed the mapping or skipped engine gate.');
Capsule::table('cloud_regions')->where('id', 1)->update([
'status' => 'active',
'updated_at' => $now + 14,
]);
Capsule::table('cloud_vpcs')->whereIn('id', [300, 301])->delete();
// The same exact disabled activation remains valid when the region has no
// live VPCs; the migration guard must not manufacture a dependency.
CloudRegionFabric::where('region_id', 1)->update([
'state' => 'disabled',
'updated_at' => $now + 14,
]);
$history = [];
$withoutVpcs = CloudAdminFabricService::reconcileRegion('1', $actor, $engine([
$response($status),
$response([$status]),
$response(['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]]),
$response($status),
$response($status),
$response([$status]),
], $history));
$withoutVpcsRegion = $findById($withoutVpcs['regions'], 1);
$assert(array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history) === [
'GET /internal/v1/fabric/fabric-hk',
'GET /internal/v1/fabric',
'POST /internal/v1/fabric/probe',
'POST /internal/v1/fabric/fabric-hk/reconcile',
'GET /internal/v1/fabric/fabric-hk',
'GET /internal/v1/fabric',
]
&& (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'active'
&& ($withoutVpcsRegion['workflow']['stage'] ?? null) === 'active',
'A no-VPC disabled mapping did not converge through exact reconcile.');
// A disabled activation must re-read platform generation after the engine
// write. If the region changes during reconcile, keep the mapping disabled.
CloudRegionFabric::where('region_id', 1)->update([
'state' => 'disabled',
'updated_at' => $now + 15,
]);
$history = [];
$client = $engine([
$response($status),
$response([$status]),
$response(['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]]),
$response($status),
static function () use ($response, $status, $now): Response {
Capsule::table('cloud_regions')->where('id', 1)->update([
'status' => 'disabled',
'updated_at' => $now + 16,
]);
return $response($status);
},
], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
409,
'A disabled mapping was activated after its region changed during reconcile'
);
$assert(array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history) === [
'GET /internal/v1/fabric/fabric-hk',
'GET /internal/v1/fabric',
'POST /internal/v1/fabric/probe',
'POST /internal/v1/fabric/fabric-hk/reconcile',
'GET /internal/v1/fabric/fabric-hk',
]
&& (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'disabled',
'Post-reconcile generation drift published an active mapping.');
Capsule::table('cloud_regions')->where('id', 1)->update([
'status' => 'active',
'updated_at' => $now + 17,
]);
CloudRegionFabric::where('region_id', 1)->update([
'state' => 'active',
'updated_at' => $now + 18,
]);
// A mapping owns exactly one identity, not every desired record sharing its
// region string. Reconcile must read both the mapped detail and full catalog,
// then fail before touching hosts when an extra same-region Fabric exists.
$history = [];
$client = $engine([
$response($status),
$response([$status, $historicalStatus]),
], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
409,
'Mapped-region reconcile ignored an additional same-region Fabric'
);
$assert(array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history) === [
'GET /internal/v1/fabric/fabric-hk',
'GET /internal/v1/fabric',
]
&& CloudRegionFabric::where('region_id', 1)->where('state', 'active')->exists(),
'Mapped conflict detection skipped detail/catalog reads or reached reconcile.');
// Once a mapping exists, a missing engine desired record cannot be recreated
// from mutable inventory under the already pinned revision.
CloudRegionFabric::where('region_id', 1)->update([
'state' => 'disabled',
'updated_at' => $now + 6,
]);
$history = [];
$client = $engine([
new Response(404, ['Content-Type' => 'application/json'], json_encode([
'error_code' => 'fabric_not_found',
], JSON_THROW_ON_ERROR)),
], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
404,
'A missing mapped engine desired plan reached reconcile'
);
$assert(array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history) === [
'GET /internal/v1/fabric/fabric-hk',
]
&& CloudRegionFabric::where('region_id', 1)->where('fabric_revision', 1)
->where('state', 'disabled')->exists(),
'Missing desired-plan recovery mutated the mapping or reached an engine write.');
CloudRegionFabric::where('region_id', 1)->update([
'state' => 'active',
'updated_at' => $now + 7,
]);
$history = [];
$missingDesiredListing = CloudAdminFabricService::listing($actor, $engine([
$response([]),
], $history));
$missingDesiredRegion = $findById($missingDesiredListing['regions'], 1);
$assert(is_array($missingDesiredRegion)
&& ($missingDesiredRegion['workflow']['stage'] ?? null) === 'recovery_required'
&& ($missingDesiredRegion['workflow']['can_apply'] ?? true) === false
&& ($missingDesiredRegion['workflow']['can_reconcile'] ?? true) === false
&& ($missingDesiredRegion['workflow']['action_mode'] ?? null) === 'none'
&& in_array('fabric_desired_missing',
$missingDesiredRegion['workflow']['blocking_reasons'] ?? [], true)
&& $missingDesiredRegion['plan_preview'] === null,
'A mapped region with missing engine desired state was not failed closed.');
// A host can become required while an engine reconcile is in flight. The
// second platform read must reject the stale two-host result before mapping
// persistence is consulted.
$membership = CloudHostResourcePoolMembership::where('host_id', 3)->first();
$history = [];
$client = $engine([
$response($status),
$response([$status]),
static function () use (
$response, $probeHost, $overlappingEndpoint, $membership, $now
): Response {
$membership->scheduler_state = 'active';
$membership->updated_at = $now + 2;
$membership->save();
return $response(['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]]);
},
], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
409,
'A topology change during fresh-gated reconcile reached host writes'
);
$assert(array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history) === [
'GET /internal/v1/fabric/fabric-hk',
'GET /internal/v1/fabric',
'POST /internal/v1/fabric/probe',
],
'The topology generation gate ran after host-writing reconcile.');
$membership->scheduler_state = 'disabled';
$membership->updated_at = $now + 3;
$membership->save();
// Likewise, a concurrent administrator may change the mapping after the
// engine exact-readback request starts. The transaction must compare against
// the pre-network snapshot rather than overwriting that winner.
CloudRegionFabric::where('region_id', 1)->update([
'state' => 'disabled',
'updated_at' => $now + 8,
]);
$history = [];
$client = $engine([
$response($status),
$response([$status]),
$response(['hosts' => [
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
$probeHost(2),
]]),
$response($status),
static function () use ($response, $status, $now): Response {
CloudRegionFabric::where('region_id', 1)->update([
'state' => 'active',
'updated_at' => $now + 9,
]);
return $response($status);
},
], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
409,
'A concurrent mapping update was overwritten after engine readback'
);
$assert(array_map(static fn (array $entry): string =>
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
$history) === [
'GET /internal/v1/fabric/fabric-hk',
'GET /internal/v1/fabric',
'POST /internal/v1/fabric/probe',
'POST /internal/v1/fabric/fabric-hk/reconcile',
'GET /internal/v1/fabric/fabric-hk',
]
&& (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'active',
'The mapping CAS did not preserve the concurrent writer.');
CloudRegionFabric::where('region_id', 1)->update([
'state' => 'active',
'updated_at' => $now + 10,
]);
// Adding a new required host is a topology migration, not an automatic apply.
// The existing desired membership must stay fixed and no mutation may run.
CloudRegionFabric::where('region_id', 1)->update([
'state' => 'disabled',
'updated_at' => $now + 11,
]);
$membership->scheduler_state = 'active';
$membership->updated_at = $now + 1;
$membership->save();
$history = [];
$client = $engine([
$response($status),
], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
409,
'Existing Fabric membership drift was automatically overwritten'
);
$assert(count($history) === 1
&& $history[0]['request']->getUri()->getPath()
=== '/internal/v1/fabric/fabric-hk'
&& !array_filter($history, static fn (array $entry): bool =>
in_array($entry['request']->getUri()->getPath(), [
'/internal/v1/fabric/apply',
'/internal/v1/fabric/fabric-hk/reconcile',
], true)),
'Membership drift reached apply/reconcile instead of failing closed.');
$mapping->refresh();
$assert((int) $mapping->fabric_revision === 1
&& (string) $mapping->state === 'disabled',
'Membership drift changed the existing region mapping.');
CloudRegionFabric::where('region_id', 1)->update([
'state' => 'active',
'updated_at' => $now + 12,
]);
// A readiness-only inventory may exceed the engine's 64-host Fabric size.
// Detection must split it into bounded requests without dropping rows or
// converting numeric database IDs into JSON numbers.
$sgProbeRows = [];
for ($id = 100; $id < 165; $id++) {
Capsule::table('cloud_hosts')->insert([
'id' => $id,
'name' => 'sg-host-' . $id,
'region' => 'sg',
'country_code' => 'SG',
'enabled' => 1,
'resource_pool_id' => 22,
'scheduler_state' => 'active',
'default_network' => 'bridge:br0',
'created_at' => $now,
'updated_at' => $now,
]);
Capsule::table('cloud_host_resource_pool_memberships')->insert([
'id' => 1000 + $id,
'host_id' => $id,
'resource_pool_id' => 22,
'scheduler_state' => 'active',
'priority' => 0,
'weight' => 100,
'created_at' => $now,
'updated_at' => $now,
]);
$sgProbeRows[] = $probeHost($id, [
'region' => 'sg',
'endpoint_ip' => '198.51.100.' . ($id - 99),
]);
}
$history = [];
$sgProbe = CloudAdminFabricService::probeRegion('2', $actor, $engine([
$response([]),
$response(['hosts' => array_slice($sgProbeRows, 0, 64)]),
$response(['hosts' => array_slice($sgProbeRows, 64)]),
], $history));
$firstBatch = json_decode((string) $history[1]['request']->getBody(), true,
32, JSON_THROW_ON_ERROR)['host_ids'] ?? [];
$secondBatch = json_decode((string) $history[2]['request']->getBody(), true,
32, JSON_THROW_ON_ERROR)['host_ids'] ?? [];
$assert(count($sgProbe['region']['hosts']) === 65
&& count($firstBatch) === 64
&& count($secondBatch) === 1
&& array_reduce(array_merge($firstBatch, $secondBatch),
static fn (bool $carry, $hostId): bool => $carry && is_string($hostId), true)
&& $firstBatch[0] === '100'
&& $secondBatch[0] === '164',
'Readiness detection did not preserve all rows across bounded string-ID batches.');
for ($id = 165; $id <= 228; $id++) {
Capsule::table('cloud_hosts')->insert([
'id' => $id,
'name' => 'sg-host-' . $id,
'region' => 'sg',
'country_code' => 'SG',
'enabled' => 1,
'resource_pool_id' => 22,
'scheduler_state' => 'active',
'default_network' => 'bridge:br0',
'created_at' => $now,
'updated_at' => $now,
]);
Capsule::table('cloud_host_resource_pool_memberships')->insert([
'id' => 1000 + $id,
'host_id' => $id,
'resource_pool_id' => 22,
'scheduler_state' => 'active',
'priority' => 0,
'weight' => 100,
'created_at' => $now,
'updated_at' => $now,
]);
}
$history = [];
$client = $engine([$response([])], $history);
$assertCloudError(
static fn () => CloudAdminFabricService::probeRegion('2', $actor, $client),
409,
'An unbounded readiness inventory exceeded the browser request budget'
);
$assert(count($history) === 1
&& $history[0]['request']->getUri()->getPath() === '/internal/v1/fabric',
'The oversized readiness inventory started a host probe before rejection.');
echo "Administrator Fabric readiness workflow passed.\n";