You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2173 lines
85 KiB
PHP
2173 lines
85 KiB
PHP
<?php
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudHostResourcePoolMembership;
|
|
use App\Models\CloudRegionFabric;
|
|
use App\Services\Cloud\CloudAdminFabricService;
|
|
use App\Services\Cloud\CloudAdminResourceService;
|
|
use App\Services\Cloud\CloudEngineClient;
|
|
use App\Services\Cloud\CloudError;
|
|
use GuzzleHttp\Client;
|
|
use GuzzleHttp\Exception\ConnectException;
|
|
use GuzzleHttp\Handler\MockHandler;
|
|
use GuzzleHttp\HandlerStack;
|
|
use GuzzleHttp\Middleware;
|
|
use GuzzleHttp\Psr7\Request;
|
|
use GuzzleHttp\Psr7\Response;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$System_Config = [
|
|
'cloud_engine_url' => 'http://127.0.0.1:8000',
|
|
'cloud_engine_hmac_key' => base64_encode(str_repeat('k', 32)),
|
|
];
|
|
|
|
$assert = static function ($condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
$assertCloudError = static function (callable $callback, int $status,
|
|
string $message) use ($assert): CloudError {
|
|
try {
|
|
$callback();
|
|
} catch (CloudError $error) {
|
|
$assert($error->httpStatus === $status,
|
|
$message . ' (unexpected HTTP status ' . $error->httpStatus . ')');
|
|
return $error;
|
|
}
|
|
throw new RuntimeException($message . ' (no CloudError was raised)');
|
|
};
|
|
$response = static fn ($payload): Response => new Response(200, [
|
|
'Content-Type' => 'application/json',
|
|
], json_encode($payload, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES));
|
|
$engine = static function (array $responses, array &$history): CloudEngineClient {
|
|
$mock = new MockHandler($responses);
|
|
$stack = HandlerStack::create($mock);
|
|
$stack->push(Middleware::history($history));
|
|
return new CloudEngineClient(new Client(['handler' => $stack]));
|
|
};
|
|
$findById = static function (array $rows, int $id): ?array {
|
|
foreach ($rows as $row) {
|
|
if ((int) ($row['region_id'] ?? 0) === $id) {
|
|
return $row;
|
|
}
|
|
}
|
|
return null;
|
|
};
|
|
$hostById = static function (array $rows, int $id): ?array {
|
|
foreach ($rows as $row) {
|
|
if ((int) ($row['host_id'] ?? 0) === $id) {
|
|
return $row;
|
|
}
|
|
}
|
|
return null;
|
|
};
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->getConnection('default')->getSchemaBuilder();
|
|
|
|
$schema->create('cloud_regions', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('code');
|
|
$table->string('name');
|
|
$table->string('country_code', 2)->default('');
|
|
$table->string('status');
|
|
$table->integer('sort')->default(0);
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_network_lines', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('region_id');
|
|
$table->string('code');
|
|
$table->string('name');
|
|
$table->boolean('is_default')->default(false);
|
|
$table->string('status');
|
|
$table->integer('sort')->default(0);
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_resource_pools', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('line_id');
|
|
$table->string('code');
|
|
$table->string('name');
|
|
$table->string('status');
|
|
$table->string('scheduling_policy')->default('balanced');
|
|
$table->integer('sort')->default(0);
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_hosts', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('name');
|
|
$table->string('region')->default('');
|
|
$table->string('country_code', 2)->default('');
|
|
$table->boolean('enabled')->default(true);
|
|
$table->integer('resource_pool_id')->nullable();
|
|
$table->string('scheduler_state')->default('disabled');
|
|
$table->string('default_network')->default('bridge:br0');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_host_resource_pool_memberships',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('host_id');
|
|
$table->integer('resource_pool_id');
|
|
$table->string('scheduler_state')->default('disabled');
|
|
$table->integer('priority')->default(0);
|
|
$table->integer('weight')->default(100);
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->unique(['host_id', 'resource_pool_id']);
|
|
});
|
|
$schema->create('cloud_region_fabrics', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('region_id')->unique();
|
|
$table->string('fabric_id')->unique();
|
|
$table->string('fabric_region');
|
|
$table->integer('fabric_revision');
|
|
$table->string('state');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_private_cidr_allocator_configs',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('id')->primary();
|
|
$table->string('pool_cidr', 18);
|
|
$table->integer('quantum_prefix_length');
|
|
$table->integer('min_prefix_length');
|
|
$table->integer('max_prefix_length');
|
|
$table->integer('revision');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_instances', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('host_id');
|
|
$table->integer('resource_pool_id')->nullable();
|
|
$table->string('status');
|
|
});
|
|
$schema->create('cloud_vpcs', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('region_id');
|
|
$table->string('fabric_id')->default('');
|
|
$table->string('fabric_region')->default('');
|
|
$table->integer('fabric_revision')->default(0);
|
|
$table->string('state');
|
|
});
|
|
$schema->create('cloud_vpc_host_deployments',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('vpc_id');
|
|
$table->integer('host_id');
|
|
$table->string('state');
|
|
});
|
|
|
|
$now = 1770000000;
|
|
Capsule::table('cloud_regions')->insert([
|
|
['id' => 1, 'code' => 'hk', 'name' => 'Hong Kong', 'country_code' => 'HK',
|
|
'status' => 'active', 'sort' => 10, 'created_at' => $now, 'updated_at' => $now],
|
|
['id' => 2, 'code' => 'sg', 'name' => 'Singapore', 'country_code' => 'SG',
|
|
'status' => 'active', 'sort' => 20, 'created_at' => $now, 'updated_at' => $now],
|
|
]);
|
|
Capsule::table('cloud_private_cidr_allocator_configs')->insert([
|
|
'id' => 1, 'pool_cidr' => '10.0.0.0/8',
|
|
'quantum_prefix_length' => 24, 'min_prefix_length' => 16,
|
|
'max_prefix_length' => 24, 'revision' => 1, 'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_network_lines')->insert([
|
|
['id' => 11, 'region_id' => 1, 'code' => 'hk-default', 'name' => 'HK default',
|
|
'is_default' => 1, 'status' => 'active', 'sort' => 10,
|
|
'created_at' => $now, 'updated_at' => $now],
|
|
['id' => 12, 'region_id' => 2, 'code' => 'sg-default', 'name' => 'SG default',
|
|
'is_default' => 1, 'status' => 'active', 'sort' => 20,
|
|
'created_at' => $now, 'updated_at' => $now],
|
|
]);
|
|
Capsule::table('cloud_resource_pools')->insert([
|
|
['id' => 21, 'line_id' => 11, 'code' => 'hk-general', 'name' => 'HK general',
|
|
'status' => 'active', 'scheduling_policy' => 'balanced', 'sort' => 10,
|
|
'created_at' => $now, 'updated_at' => $now],
|
|
['id' => 22, 'line_id' => 12, 'code' => 'sg-general', 'name' => 'SG general',
|
|
'status' => 'active', 'scheduling_policy' => 'balanced', 'sort' => 20,
|
|
'created_at' => $now, 'updated_at' => $now],
|
|
]);
|
|
for ($id = 1; $id <= 6; $id++) {
|
|
Capsule::table('cloud_hosts')->insert([
|
|
'id' => $id,
|
|
'name' => 'host-' . $id,
|
|
// host-1 remains in the eventual apply set and host-5 exercises the
|
|
// larger readiness projection. Both use display labels different from
|
|
// the sales-region code; canonical membership owns their placement.
|
|
'region' => in_array($id, [1, 5], true) ? 'Hong Kong' : 'hk',
|
|
'country_code' => 'HK',
|
|
'enabled' => 1,
|
|
'resource_pool_id' => 21,
|
|
'scheduler_state' => $id <= 5 ? 'active' : 'disabled',
|
|
'default_network' => 'bridge:br0',
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
if ($id <= 5) {
|
|
Capsule::table('cloud_host_resource_pool_memberships')->insert([
|
|
'id' => 30 + $id,
|
|
'host_id' => $id,
|
|
'resource_pool_id' => 21,
|
|
'scheduler_state' => 'active',
|
|
'priority' => 0,
|
|
'weight' => 100,
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
}
|
|
}
|
|
|
|
$actor = [
|
|
'user_id' => 99,
|
|
'is_admin' => true,
|
|
'is_super_admin' => true,
|
|
'can_manage' => true,
|
|
];
|
|
$history = [];
|
|
$busyClient = $engine([new Response(409, [
|
|
'Content-Type' => 'application/json',
|
|
], json_encode(['error_code' => 'fabric_probe_busy'], JSON_THROW_ON_ERROR))],
|
|
$history);
|
|
$busy = $assertCloudError(
|
|
static fn () => $busyClient->fabricProbe(['1'], $actor),
|
|
409,
|
|
'An overlapping engine readiness probe lost its stable 409 contract'
|
|
);
|
|
$assert(str_contains($busy->getMessage(), '已有地区宿主机检测正在运行')
|
|
&& count($history) === 1,
|
|
'The reviewed probe-busy error was hidden or retried by PHP.');
|
|
|
|
// Engine outages keep platform inventory visible, but every workflow write
|
|
// gate must fail closed until an authoritative Fabric snapshot is available.
|
|
$history = [];
|
|
$unavailable = CloudAdminFabricService::listing($actor, $engine([
|
|
new Response(503, ['Content-Type' => 'application/json'], json_encode([
|
|
'error_code' => 'fabric_host_unavailable',
|
|
], JSON_THROW_ON_ERROR)),
|
|
], $history));
|
|
$unavailableRegion = $findById($unavailable['regions'], 1);
|
|
$assert(!array_key_exists('fabrics', $unavailable)
|
|
&& $unavailable['engine_available'] === false
|
|
&& is_array($unavailableRegion)
|
|
&& !array_key_exists('fabric', $unavailableRegion)
|
|
&& ($unavailableRegion['workflow'] ?? null) === [
|
|
'stage' => 'engine_unavailable',
|
|
'can_probe' => false,
|
|
'can_apply' => false,
|
|
'can_reconcile' => false,
|
|
'blocking_reasons' => ['engine_unavailable'],
|
|
'action_mode' => 'none',
|
|
]
|
|
&& $unavailableRegion['plan_preview'] === null,
|
|
'An engine outage did not fail every region workflow write gate closed.');
|
|
|
|
$overlay = static function (string $regionCode, int $offset = 0): string {
|
|
$slot = (hexdec(substr(hash('sha256', $regionCode), 0, 8)) + $offset)
|
|
% 16384;
|
|
return '100.' . (64 + intdiv($slot, 256)) . '.' . ($slot % 256) . '.0/24';
|
|
};
|
|
$firstHkOverlay = $overlay('hk');
|
|
$hkOverlay = $overlay('hk', 1);
|
|
$overlappingEndpoint = substr($firstHkOverlay, 0, -4) . '77';
|
|
$probeHost = static function (int $id, array $changes = []): array {
|
|
$row = [
|
|
'host_id' => (string) $id,
|
|
'configured' => true,
|
|
'reachable' => true,
|
|
'region' => 'hk',
|
|
'endpoint_ip' => '203.0.113.' . (10 + $id),
|
|
'endpoint_source' => 'ssh_host_ipv4',
|
|
'probed_at' => 1770000100 + $id,
|
|
'vpc_ready' => true,
|
|
'reasons' => [],
|
|
'capabilities' => [
|
|
'os_id' => 'almalinux',
|
|
'os_version' => '10',
|
|
'network_manager' => 'networkmanager',
|
|
'network_manager_expected' => 'networkmanager',
|
|
'network_manager_supported' => true,
|
|
'deployment_mode' => 'networkmanager_nmcli',
|
|
'firewall_manager' => 'nftables',
|
|
'firewall_mode' => 'native_nftables',
|
|
'ip_forward' => 1,
|
|
'rp_filter' => 0,
|
|
'rp_filter_all' => 0,
|
|
'rp_filter_default' => 1,
|
|
'rp_filter_default_effective' => 1,
|
|
'rp_filter_underlay_raw' => 1,
|
|
'rp_filter_underlay_effective' => 1,
|
|
'rp_filter_fabric_raw' => null,
|
|
'rp_filter_fabric_effective' => null,
|
|
'wireguard_ready' => true,
|
|
'wireguard_owned' => false,
|
|
'nftables_ready' => true,
|
|
'nftables_active' => true,
|
|
'nftables_persistent' => true,
|
|
'public_selector' => 'bridge:br0',
|
|
'public_verified' => true,
|
|
],
|
|
];
|
|
if (isset($changes['capabilities'])) {
|
|
$row['capabilities'] = array_replace(
|
|
$row['capabilities'], $changes['capabilities']
|
|
);
|
|
unset($changes['capabilities']);
|
|
}
|
|
return array_replace($row, $changes);
|
|
};
|
|
$fabricStatus = static function (array $hostIds) use ($hkOverlay): array {
|
|
$digest = str_repeat('a', 64);
|
|
$network = substr($hkOverlay, 0, -4);
|
|
$nodes = [];
|
|
foreach (array_values($hostIds) as $index => $hostId) {
|
|
$nodes[] = [
|
|
'host_id' => (string) $hostId,
|
|
'overlay_ip' => $network . ($index + 1),
|
|
'endpoint' => '203.0.113.' . (10 + (int) $hostId) . ':51820',
|
|
'listen_port' => 51820,
|
|
'public_key' => base64_encode(str_repeat(chr((int) $hostId), 32)),
|
|
'state' => 'active',
|
|
'transport_verified' => true,
|
|
'observed_revision' => 1,
|
|
'observed_plan_digest' => $digest,
|
|
'last_error_code' => null,
|
|
'observed' => [],
|
|
];
|
|
}
|
|
return [
|
|
'fabric_id' => 'fabric-hk',
|
|
'region' => 'hk',
|
|
'overlay_cidr' => $hkOverlay,
|
|
'revision' => 1,
|
|
'listen_port' => 51820,
|
|
'plan_digest' => $digest,
|
|
'interface' => 'wg-fabric0',
|
|
'state' => 'active',
|
|
'next_stage_allowed' => true,
|
|
'updated_at' => 1770000200,
|
|
'nodes' => $nodes,
|
|
];
|
|
};
|
|
$status = $fabricStatus([1, 2]);
|
|
$status['nodes'][0]['endpoint'] = $overlappingEndpoint . ':51820';
|
|
$driftStatus = $fabricStatus([1, 2, 6]);
|
|
$historicalStatus = $driftStatus;
|
|
$historicalStatus['fabric_id'] = 'legacy-hk';
|
|
$historicalExactStatus = $fabricStatus([1, 2]);
|
|
$historicalExactStatus['fabric_id'] = 'legacy-hk';
|
|
$historicalExactStatus['region'] = 'legacy-hong-kong';
|
|
$foreignRegionStatus = $fabricStatus([1]);
|
|
$foreignRegionStatus['fabric_id'] = 'fabric-sg';
|
|
$foreignRegionStatus['region'] = 'sg';
|
|
|
|
// Broken placement relations must fail closed before any engine request. An
|
|
// inner join must not silently shrink the required Fabric host union.
|
|
Capsule::table('cloud_host_resource_pool_memberships')->insert([
|
|
'id' => 99,
|
|
'host_id' => 1,
|
|
'resource_pool_id' => 999,
|
|
'scheduler_state' => 'active',
|
|
'priority' => 0,
|
|
'weight' => 100,
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
$history = [];
|
|
$client = $engine([], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
|
|
503,
|
|
'An active membership with a missing pool/line was silently omitted'
|
|
);
|
|
$assert($history === [],
|
|
'A corrupt active membership was rejected only after an engine request.');
|
|
Capsule::table('cloud_host_resource_pool_memberships')->where('id', 99)->delete();
|
|
|
|
Capsule::table('cloud_host_resource_pool_memberships')->insert([
|
|
'id' => 100,
|
|
'host_id' => 1,
|
|
'resource_pool_id' => 22,
|
|
'scheduler_state' => 'draining',
|
|
'priority' => 0,
|
|
'weight' => 100,
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
$history = [];
|
|
$client = $engine([], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
|
|
409,
|
|
'Concurrent active/draining region memberships were not treated as a conflict'
|
|
);
|
|
$assert($history === [],
|
|
'A live membership conflict was rejected only after an engine request.');
|
|
Capsule::table('cloud_host_resource_pool_memberships')->where('id', 100)->delete();
|
|
|
|
Capsule::table('cloud_instances')->insert([
|
|
'id' => 101,
|
|
'host_id' => 1,
|
|
'resource_pool_id' => 999,
|
|
'status' => 'running',
|
|
]);
|
|
$history = [];
|
|
$client = $engine([], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
|
|
503,
|
|
'A live instance with a missing pool/line was silently omitted'
|
|
);
|
|
$assert($history === [],
|
|
'A corrupt instance placement was rejected only after an engine request.');
|
|
Capsule::table('cloud_instances')->where('id', 101)->delete();
|
|
|
|
Capsule::table('cloud_vpcs')->insert([
|
|
'id' => 201,
|
|
'region_id' => 999,
|
|
'state' => 'active',
|
|
]);
|
|
Capsule::table('cloud_vpc_host_deployments')->insert([
|
|
'id' => 202,
|
|
'vpc_id' => 201,
|
|
'host_id' => 1,
|
|
'state' => 'active',
|
|
]);
|
|
$history = [];
|
|
$client = $engine([], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
|
|
503,
|
|
'A live VPC deployment with an unknown region was silently omitted'
|
|
);
|
|
$assert($history === [],
|
|
'An invalid VPC deployment region was rejected only after an engine request.');
|
|
Capsule::table('cloud_vpc_host_deployments')->where('id', 202)->delete();
|
|
Capsule::table('cloud_vpcs')->where('id', 201)->delete();
|
|
|
|
Capsule::table('cloud_vpc_host_deployments')->insert([
|
|
'id' => 203,
|
|
'vpc_id' => 999,
|
|
'host_id' => 1,
|
|
'state' => 'active',
|
|
]);
|
|
$history = [];
|
|
$client = $engine([], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
|
|
503,
|
|
'A live deployment whose VPC root is missing was silently omitted'
|
|
);
|
|
$assert($history === [],
|
|
'An orphaned VPC deployment was rejected only after an engine request.');
|
|
Capsule::table('cloud_vpc_host_deployments')->where('id', 203)->delete();
|
|
|
|
// Workloads remain part of the exact required-host union even when the host
|
|
// no longer has an active scheduler membership.
|
|
Capsule::table('cloud_instances')->insert([
|
|
'id' => 102,
|
|
'host_id' => 6,
|
|
'resource_pool_id' => 21,
|
|
'status' => 'running',
|
|
]);
|
|
$history = [];
|
|
$instanceListing = CloudAdminFabricService::listing($actor, $engine([
|
|
$response([]),
|
|
], $history));
|
|
$instanceRegion = $findById($instanceListing['regions'], 1);
|
|
$instanceHost = is_array($instanceRegion)
|
|
? $hostById($instanceRegion['hosts'], 6) : null;
|
|
$assert(is_array($instanceHost)
|
|
&& $instanceHost['required'] === true
|
|
&& in_array('instance', $instanceHost['required_sources'], true)
|
|
&& in_array('workload_membership_mismatch', $instanceHost['reasons'], true),
|
|
'A live instance host without membership disappeared from readiness.');
|
|
Capsule::table('cloud_instances')->where('id', 102)->delete();
|
|
|
|
Capsule::table('cloud_vpcs')->insert([
|
|
'id' => 204,
|
|
'region_id' => 1,
|
|
'state' => 'active',
|
|
]);
|
|
Capsule::table('cloud_vpc_host_deployments')->insert([
|
|
'id' => 205,
|
|
'vpc_id' => 204,
|
|
'host_id' => 6,
|
|
'state' => 'active',
|
|
]);
|
|
$history = [];
|
|
$deploymentListing = CloudAdminFabricService::listing($actor, $engine([
|
|
$response([]),
|
|
], $history));
|
|
$deploymentRegion = $findById($deploymentListing['regions'], 1);
|
|
$deploymentHost = is_array($deploymentRegion)
|
|
? $hostById($deploymentRegion['hosts'], 6) : null;
|
|
$assert(is_array($deploymentHost)
|
|
&& $deploymentHost['required'] === true
|
|
&& in_array('vpc_deployment', $deploymentHost['required_sources'], true)
|
|
&& in_array('workload_membership_mismatch', $deploymentHost['reasons'], true),
|
|
'A live VPC deployment host without membership disappeared from readiness.');
|
|
Capsule::table('cloud_vpc_host_deployments')->where('id', 205)->delete();
|
|
Capsule::table('cloud_vpcs')->where('id', 204)->delete();
|
|
|
|
// Existing active Fabric: a disabled host that is not a Fabric member must
|
|
// still be visible, and host-5 remains in HK solely by canonical membership.
|
|
Capsule::table('cloud_region_fabrics')->insert([
|
|
'id' => 1,
|
|
'region_id' => 1,
|
|
'fabric_id' => 'fabric-hk',
|
|
'fabric_region' => 'hk',
|
|
'fabric_revision' => 1,
|
|
'state' => 'active',
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
$history = [];
|
|
$listing = CloudAdminFabricService::listing($actor, $engine([
|
|
$response([$driftStatus]),
|
|
], $history));
|
|
$hk = $findById($listing['regions'], 1);
|
|
$sg = $findById($listing['regions'], 2);
|
|
$assert(!array_key_exists('fabrics', $listing)
|
|
&& is_array($hk) && is_array($sg)
|
|
&& !array_key_exists('fabric', $hk)
|
|
&& !array_key_exists('fabric', $sg),
|
|
'The readiness inventory dropped a platform region.');
|
|
$assert(array_map(static fn (array $host): int => (int) $host['host_id'],
|
|
$hk['hosts']) === [1, 2, 3, 4, 5, 6]
|
|
&& $sg['hosts'] === [],
|
|
'Host geography did not follow membership -> pool -> line -> region.');
|
|
$regionDrift = $hostById($hk['hosts'], 5);
|
|
$assert(is_array($regionDrift)
|
|
&& $regionDrift['engine_region'] === 'Hong Kong'
|
|
&& $regionDrift['platform_ready'] === true
|
|
&& !in_array('panel_host_region_mismatch', $regionDrift['reasons'], true),
|
|
'A display-only host region label still blocked canonical placement.');
|
|
$notMember = $hostById($hk['hosts'], 3);
|
|
$assert(is_array($notMember)
|
|
&& $notMember['fabric_member'] === false
|
|
&& in_array('fabric_membership_drift', $notMember['reasons'], true),
|
|
'A platform host outside the current Fabric disappeared from readiness.');
|
|
$orphan = $hostById($hk['hosts'], 6);
|
|
$assert(is_array($orphan)
|
|
&& $orphan['fabric_member'] === true
|
|
&& in_array('fabric_membership_drift', $orphan['reasons'], true)
|
|
&& $hk['ready'] === false
|
|
&& $hk['host_set_matches'] === false,
|
|
'A scheduler/Fabric membership drift was hidden or promoted to ready.');
|
|
$assert(($hk['workflow']['stage'] ?? null) === 'migration_required'
|
|
&& ($hk['workflow']['can_apply'] ?? true) === false
|
|
&& ($hk['workflow']['can_reconcile'] ?? true) === false
|
|
&& in_array('fabric_membership_drift',
|
|
$hk['workflow']['blocking_reasons'] ?? [], true)
|
|
&& ($hk['plan_preview']['mode'] ?? null) === 'existing'
|
|
&& !array_intersect(['nodes', 'endpoints', 'host_ids'],
|
|
array_keys($hk['plan_preview']))
|
|
&& ($sg['workflow']['stage'] ?? null) === 'no_hosts'
|
|
&& ($sg['workflow']['action_mode'] ?? null) === 'none',
|
|
'Membership drift/no-host inventory did not project safe wizard stages.');
|
|
|
|
// A fresh probe is an isolated per-host snapshot. One failed host must not
|
|
// erase successful or independently blocked rows.
|
|
CloudRegionFabric::query()->delete();
|
|
$probeRows = [
|
|
$probeHost(1, [
|
|
'endpoint_source' => 'configured',
|
|
'capabilities' => [
|
|
'firewall_manager' => 'firewalld',
|
|
'firewall_mode' => 'firewalld_nft',
|
|
'nftables_ready' => true,
|
|
'nftables_active' => false,
|
|
'nftables_persistent' => false,
|
|
],
|
|
]),
|
|
$probeHost(2, [
|
|
'reachable' => false,
|
|
'vpc_ready' => false,
|
|
'error_code' => 'fabric_host_capability_probe_failed',
|
|
]),
|
|
$probeHost(3, [
|
|
'endpoint_ip' => '10.20.0.10',
|
|
'vpc_ready' => false,
|
|
'reasons' => ['firewall_owner_not_nftables'],
|
|
'capabilities' => [
|
|
'firewall_manager' => 'firewalld',
|
|
'firewall_mode' => 'unsupported',
|
|
'nftables_ready' => false,
|
|
'nftables_active' => false,
|
|
'nftables_persistent' => false,
|
|
'rp_filter_all' => 3,
|
|
'rp_filter_default' => '1',
|
|
'rp_filter_fabric_raw' => true,
|
|
],
|
|
]),
|
|
$probeHost(4, ['endpoint_ip' => 'host.example.test']),
|
|
$probeHost(5, [
|
|
'region' => 'sg',
|
|
'endpoint_source' => 'untrusted-engine-value',
|
|
]),
|
|
$probeHost(6),
|
|
];
|
|
$history = [];
|
|
$probed = CloudAdminFabricService::probeRegion('1', $actor, $engine([
|
|
$response([$driftStatus]),
|
|
$response(['hosts' => $probeRows]),
|
|
], $history));
|
|
$rows = $probed['region']['hosts'];
|
|
$assert(count($rows) === 6 && ($hostById($rows, 1)['readiness_state'] ?? null) === 'ready',
|
|
'A single probe failure discarded healthy host rows.');
|
|
$assert(($hostById($rows, 1)['endpoint_source'] ?? null) === 'configured',
|
|
'An explicit effective Fabric endpoint source was not projected.');
|
|
$assert(($hostById($rows, 1)['capabilities']['firewall_mode'] ?? null)
|
|
=== 'firewalld_nft'
|
|
&& ($hostById($rows, 1)['capabilities']['firewall_manager'] ?? null)
|
|
=== 'firewalld',
|
|
'A supported firewalld+nft host was not preserved as ready.');
|
|
$rpFilterCapabilities = $hostById($rows, 1)['capabilities'] ?? [];
|
|
$assert(($rpFilterCapabilities['rp_filter'] ?? null) === 0
|
|
&& ($rpFilterCapabilities['rp_filter_all'] ?? null) === 0
|
|
&& ($rpFilterCapabilities['rp_filter_default'] ?? null) === 1
|
|
&& ($rpFilterCapabilities['rp_filter_default_effective'] ?? null) === 1
|
|
&& ($rpFilterCapabilities['rp_filter_underlay_raw'] ?? null) === 1
|
|
&& ($rpFilterCapabilities['rp_filter_underlay_effective'] ?? null) === 1
|
|
&& array_key_exists('rp_filter_fabric_raw', $rpFilterCapabilities)
|
|
&& $rpFilterCapabilities['rp_filter_fabric_raw'] === null
|
|
&& array_key_exists('rp_filter_fabric_effective', $rpFilterCapabilities)
|
|
&& $rpFilterCapabilities['rp_filter_fabric_effective'] === null,
|
|
'The bounded raw/effective rp_filter evidence was dropped or Alma strict defaults were hidden.');
|
|
$failed = $hostById($rows, 2);
|
|
$firewalld = $hostById($rows, 3);
|
|
$nonIpv4 = $hostById($rows, 4);
|
|
$wrongRegion = $hostById($rows, 5);
|
|
$orphan = $hostById($rows, 6);
|
|
$assert(is_array($failed) && $failed['probe_checked'] === true
|
|
&& $failed['reachable'] === false
|
|
&& in_array('fabric_host_capability_probe_failed', $failed['reasons'], true),
|
|
'The isolated failed-host result was dropped or promoted.');
|
|
$assert(is_array($firewalld)
|
|
&& ($firewalld['capabilities']['firewall_manager'] ?? null) === 'firewalld'
|
|
&& !array_key_exists('rp_filter_all', $firewalld['capabilities'])
|
|
&& !array_key_exists('rp_filter_default', $firewalld['capabilities'])
|
|
&& !array_key_exists('rp_filter_fabric_raw', $firewalld['capabilities'])
|
|
&& in_array('firewall_owner_not_nftables', $firewalld['reasons'], true)
|
|
&& in_array('fabric_endpoint_vpc_pool_conflict',
|
|
$firewalld['reasons'], true),
|
|
'A firewall or tenant-pool endpoint blocker was not preserved in readiness.');
|
|
$assert(is_array($nonIpv4) && $nonIpv4['endpoint_ip'] === null
|
|
&& $nonIpv4['endpoint_source'] === null
|
|
&& in_array('fabric_endpoint_unavailable', $nonIpv4['reasons'], true),
|
|
'A non-IPv4 WireGuard endpoint was accepted or not explained.');
|
|
$assert(is_array($wrongRegion)
|
|
&& $wrongRegion['endpoint_ip'] === null
|
|
&& $wrongRegion['endpoint_source'] === null
|
|
&& $wrongRegion['readiness_state'] === 'blocked'
|
|
&& in_array('fabric_endpoint_source_invalid', $wrongRegion['reasons'], true)
|
|
&& !in_array('fabric_region_mismatch', $wrongRegion['reasons'], true),
|
|
'An untrusted endpoint source or display-only probe region escaped the bounded projection.');
|
|
$assert(is_array($orphan) && $orphan['probe_checked'] === true
|
|
&& $orphan['fabric_member'] === true
|
|
&& in_array('fabric_membership_drift', $orphan['reasons'], true),
|
|
'An orphaned Fabric node escaped fresh host detection.');
|
|
$assert(array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history) === [
|
|
'GET /internal/v1/fabric',
|
|
'POST /internal/v1/fabric/probe',
|
|
],
|
|
'Fresh detection did not freeze the engine membership snapshot before probing.');
|
|
$detectedIds = json_decode((string) $history[1]['request']->getBody(), true,
|
|
32, JSON_THROW_ON_ERROR)['host_ids'] ?? [];
|
|
$assert($detectedIds === ['1', '2', '3', '4', '5', '6'],
|
|
'Fresh detection omitted a canonical member or orphaned Fabric node.');
|
|
|
|
// Retire the probe-only hosts before configuring the clean two-node target.
|
|
CloudHostResourcePoolMembership::whereIn('host_id', [3, 4, 5])->update([
|
|
'scheduler_state' => 'disabled',
|
|
'updated_at' => $now + 1,
|
|
]);
|
|
|
|
// Inventory alone never opens apply: the current request must carry a fresh,
|
|
// complete probe snapshot for every required host.
|
|
$history = [];
|
|
$needsProbeListing = CloudAdminFabricService::listing($actor, $engine([
|
|
$response([]),
|
|
], $history));
|
|
$needsProbeRegion = $findById($needsProbeListing['regions'], 1);
|
|
$assert(is_array($needsProbeRegion)
|
|
&& ($needsProbeRegion['workflow'] ?? null) === [
|
|
'stage' => 'needs_probe',
|
|
'can_probe' => true,
|
|
'can_apply' => false,
|
|
'can_reconcile' => false,
|
|
'blocking_reasons' => ['probe_required'],
|
|
'action_mode' => 'probe',
|
|
]
|
|
&& $needsProbeRegion['plan_preview'] === null,
|
|
'A stale inventory snapshot opened automatic Fabric apply without probing.');
|
|
|
|
$history = [];
|
|
$blockedWizard = CloudAdminFabricService::probeRegion('1', $actor, $engine([
|
|
$response([]),
|
|
$response(['hosts' => [
|
|
$probeHost(1, [
|
|
'vpc_ready' => false,
|
|
'reasons' => ['firewall_owner_not_nftables'],
|
|
'capabilities' => ['firewall_manager' => 'firewalld'],
|
|
]),
|
|
$probeHost(2),
|
|
]]),
|
|
], $history));
|
|
$blockedWorkflow = $blockedWizard['region']['workflow'] ?? null;
|
|
$assert(($blockedWorkflow['stage'] ?? null) === 'blocked'
|
|
&& ($blockedWorkflow['can_probe'] ?? false) === true
|
|
&& ($blockedWorkflow['can_apply'] ?? true) === false
|
|
&& ($blockedWorkflow['can_reconcile'] ?? true) === false
|
|
&& ($blockedWorkflow['action_mode'] ?? null) === 'probe'
|
|
&& in_array('firewall_owner_not_nftables',
|
|
$blockedWorkflow['blocking_reasons'] ?? [], true),
|
|
'A failed fresh host probe did not keep the wizard blocked.');
|
|
|
|
// A fresh successful detection is the only path to ready_to_apply. The
|
|
// browser receives a bounded server-owned preview, never nodes or endpoints.
|
|
$history = [];
|
|
$readyToApply = CloudAdminFabricService::probeRegion('1', $actor, $engine([
|
|
$response([]),
|
|
$response(['hosts' => [
|
|
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
|
|
$probeHost(2),
|
|
]]),
|
|
], $history));
|
|
$readyWorkflow = $readyToApply['region']['workflow'] ?? null;
|
|
$readyPreview = $readyToApply['region']['plan_preview'] ?? null;
|
|
$assert(!array_key_exists('fabric', $readyToApply['region'])
|
|
&& $readyWorkflow === [
|
|
'stage' => 'ready_to_apply',
|
|
'can_probe' => true,
|
|
'can_apply' => true,
|
|
'can_reconcile' => false,
|
|
'blocking_reasons' => [],
|
|
'action_mode' => 'apply',
|
|
]
|
|
&& $readyPreview === [
|
|
'mode' => 'automatic',
|
|
'fabric_id' => 'fabric-hk',
|
|
'fabric_region' => 'hk',
|
|
'revision' => 1,
|
|
'listen_port' => 51820,
|
|
'overlay_cidr' => $hkOverlay,
|
|
'host_count' => 2,
|
|
], 'Fresh readiness did not produce the bounded automatic plan preview.');
|
|
|
|
// Even an exact-host-set custom Fabric remains an explicit migration. The
|
|
// wizard never adopts or replays an unmapped legacy identity.
|
|
$history = [];
|
|
$legacyExactListing = CloudAdminFabricService::listing($actor, $engine([
|
|
$response([$historicalExactStatus]),
|
|
], $history));
|
|
$legacyExactRegion = $findById($legacyExactListing['regions'], 1);
|
|
$assert(is_array($legacyExactRegion)
|
|
&& ($legacyExactRegion['workflow']['stage'] ?? null) === 'migration_required'
|
|
&& ($legacyExactRegion['workflow']['can_apply'] ?? true) === false
|
|
&& ($legacyExactRegion['workflow']['can_reconcile'] ?? true) === false
|
|
&& array_column($legacyExactRegion['fabric_conflicts'], 'fabric_id')
|
|
=== ['legacy-hk'],
|
|
'An unmapped custom Fabric was projected as automatically adoptable.');
|
|
$history = [];
|
|
$client = $engine([
|
|
$response(['hosts' => [$probeHost(1), $probeHost(2)]]),
|
|
$response([$historicalExactStatus]),
|
|
], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
|
|
409,
|
|
'Automatic configuration adopted an exact unmapped custom Fabric'
|
|
);
|
|
$assert(count($history) === 2
|
|
&& $history[1]['request']->getUri()->getPath() === '/internal/v1/fabric',
|
|
'Custom Fabric rejection occurred after a host-writing request.');
|
|
|
|
// A historical custom-ID Fabric in the same engine region is not owned merely
|
|
// because no database mapping exists. It must remain visible (including its
|
|
// orphan host), while keeping the region unready and automatic writes closed.
|
|
$history = [];
|
|
$historicalListing = CloudAdminFabricService::listing($actor, $engine([
|
|
$response([$historicalStatus]),
|
|
], $history));
|
|
$historicalRegion = $findById($historicalListing['regions'], 1);
|
|
$historicalOrphan = is_array($historicalRegion)
|
|
? $hostById($historicalRegion['hosts'], 6) : null;
|
|
$assert(is_array($historicalRegion)
|
|
&& $historicalRegion['ready'] === false
|
|
&& $historicalRegion['host_set_matches'] === false
|
|
&& array_column($historicalRegion['fabric_conflicts'], 'fabric_id')
|
|
=== ['legacy-hk']
|
|
&& array_map(static fn (array $host): int => (int) $host['host_id'],
|
|
$historicalRegion['hosts']) === [1, 2, 6]
|
|
&& is_array($historicalOrphan)
|
|
&& $historicalOrphan['fabric_member'] === true
|
|
&& in_array('fabric_identity_conflict',
|
|
$historicalOrphan['reasons'], true),
|
|
'An unmapped historical same-region Fabric was hidden or treated as owned.');
|
|
$assert(($historicalRegion['workflow']['stage'] ?? null) === 'migration_required'
|
|
&& in_array('fabric_identity_conflict',
|
|
$historicalRegion['workflow']['blocking_reasons'] ?? [], true)
|
|
&& ($historicalRegion['workflow']['can_apply'] ?? true) === false
|
|
&& ($historicalRegion['workflow']['can_reconcile'] ?? true) === false,
|
|
'A historical Fabric identity conflict did not require manual migration.');
|
|
|
|
$history = [];
|
|
$client = $engine([
|
|
$response(['hosts' => [
|
|
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
|
|
$probeHost(2),
|
|
]]),
|
|
$response([$historicalStatus]),
|
|
], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
|
|
409,
|
|
'Automatic configuration adopted an unmapped historical custom Fabric'
|
|
);
|
|
$assert(array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history) === [
|
|
'POST /internal/v1/fabric/probe',
|
|
'GET /internal/v1/fabric',
|
|
],
|
|
'Historical custom-Fabric ownership was checked after an engine write.');
|
|
|
|
// A different/legacy region label does not make a Fabric safe when it already
|
|
// owns a host required by HK: wg-fabric0 is host-global. Keep the conflict
|
|
// visible and stop automatic apply before either host-writing endpoint.
|
|
$history = [];
|
|
$foreignRegionListing = CloudAdminFabricService::listing($actor, $engine([
|
|
$response([$foreignRegionStatus]),
|
|
], $history));
|
|
$foreignRegionHk = $findById($foreignRegionListing['regions'], 1);
|
|
$foreignRegionHost = is_array($foreignRegionHk)
|
|
? $hostById($foreignRegionHk['hosts'], 1) : null;
|
|
$assert(is_array($foreignRegionHk)
|
|
&& $foreignRegionHk['ready'] === false
|
|
&& array_column($foreignRegionHk['fabric_conflicts'], 'fabric_id')
|
|
=== ['fabric-sg']
|
|
&& is_array($foreignRegionHost)
|
|
&& in_array('fabric_identity_conflict',
|
|
$foreignRegionHost['reasons'], true),
|
|
'A foreign-region Fabric owning an HK host was hidden from readiness.');
|
|
|
|
$history = [];
|
|
$client = $engine([
|
|
$response(['hosts' => [
|
|
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
|
|
$probeHost(2),
|
|
]]),
|
|
$response([$foreignRegionStatus]),
|
|
], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
|
|
409,
|
|
'A foreign-region Fabric owning an HK host reached automatic apply'
|
|
);
|
|
$paths = array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history);
|
|
$assert($paths === [
|
|
'POST /internal/v1/fabric/probe',
|
|
'GET /internal/v1/fabric',
|
|
]
|
|
&& !in_array('POST /internal/v1/fabric/apply', $paths, true)
|
|
&& !in_array('POST /internal/v1/fabric/fabric-hk/reconcile', $paths, true),
|
|
'Foreign-region host ownership was rejected only after an engine write.');
|
|
|
|
// Automatic configuration always repeats the fresh probe. Each unsafe probe
|
|
// class must stop before status lookup or an engine mutation.
|
|
$missingEndpointSource = $probeHost(1);
|
|
unset($missingEndpointSource['endpoint_source']);
|
|
$blockedProbeSets = [
|
|
'firewalld' => [
|
|
$probeHost(1, [
|
|
'vpc_ready' => false,
|
|
'reasons' => ['firewall_owner_not_nftables'],
|
|
'capabilities' => ['firewall_manager' => 'firewalld'],
|
|
]),
|
|
$probeHost(2),
|
|
],
|
|
'non-IPv4 endpoint' => [
|
|
$probeHost(1, ['endpoint_ip' => 'host.example.test']),
|
|
$probeHost(2),
|
|
],
|
|
'non-routable endpoint' => [
|
|
$probeHost(1, ['endpoint_ip' => '0.0.0.0']),
|
|
$probeHost(2),
|
|
],
|
|
'invalid endpoint source' => [
|
|
$probeHost(1, ['endpoint_source' => 'untrusted-engine-value']),
|
|
$probeHost(2),
|
|
],
|
|
'missing endpoint source' => [
|
|
$missingEndpointSource,
|
|
$probeHost(2),
|
|
],
|
|
];
|
|
foreach ($blockedProbeSets as $label => $items) {
|
|
$history = [];
|
|
$client = $engine([$response(['hosts' => $items])], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
|
|
409,
|
|
'Automatic Fabric configuration did not block ' . $label
|
|
);
|
|
$assert(count($history) === 1
|
|
&& $history[0]['request']->getUri()->getPath() === '/internal/v1/fabric/probe',
|
|
'The ' . $label . ' blocker reached an engine read/write after fresh probing.');
|
|
}
|
|
|
|
// WireGuard endpoints are an engine-wide identity for the generated plan.
|
|
// A duplicated fresh endpoint must fail before list/apply can mutate Fabric.
|
|
$duplicateEndpoint = '203.0.113.250';
|
|
$history = [];
|
|
$client = $engine([$response(['hosts' => [
|
|
$probeHost(1, ['endpoint_ip' => $duplicateEndpoint]),
|
|
$probeHost(2, ['endpoint_ip' => $duplicateEndpoint]),
|
|
]])], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
|
|
409,
|
|
'Duplicate automatic WireGuard endpoints reached Fabric apply'
|
|
);
|
|
$assert(array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history) === [
|
|
'POST /internal/v1/fabric/probe',
|
|
],
|
|
'Duplicate endpoints were rejected only after an engine list/write request.');
|
|
|
|
// The deterministic ID is still a globally owned database identity. A
|
|
// mapping belonging to another region must block both possible engine write
|
|
// branches, even when the engine catalog is empty or contains a record whose
|
|
// shape would otherwise be accepted for this region.
|
|
Capsule::table('cloud_region_fabrics')->insert([
|
|
'region_id' => 2,
|
|
'fabric_id' => 'fabric-hk',
|
|
'fabric_region' => 'hk',
|
|
'fabric_revision' => 1,
|
|
'state' => 'disabled',
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
$foreignOwnerCases = [
|
|
'empty engine catalog' => [
|
|
[
|
|
$response(['hosts' => [
|
|
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
|
|
$probeHost(2),
|
|
]]),
|
|
$response([]),
|
|
$response($status),
|
|
],
|
|
'/internal/v1/fabric/apply',
|
|
],
|
|
'exact-shaped engine record' => [
|
|
[
|
|
$response(['hosts' => [
|
|
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
|
|
$probeHost(2),
|
|
]]),
|
|
$response([$status]),
|
|
$response($status),
|
|
],
|
|
'/internal/v1/fabric/fabric-hk/reconcile',
|
|
],
|
|
];
|
|
foreach ($foreignOwnerCases as $label => [$responses, $forbiddenPath]) {
|
|
$history = [];
|
|
$client = $engine($responses, $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
|
|
409,
|
|
'A foreign region owner reached the ' . $label . ' write branch'
|
|
);
|
|
$paths = array_map(static fn (array $entry): string =>
|
|
$entry['request']->getUri()->getPath(), $history);
|
|
$assert($history === [] && !in_array($forbiddenPath, $paths, true),
|
|
'Foreign ownership was checked only after ' . $forbiddenPath . '.');
|
|
}
|
|
CloudRegionFabric::where('region_id', 2)->delete();
|
|
|
|
// Automatic configuration: the browser supplies only the region action while
|
|
// PHP deterministically creates the Fabric identity, overlay and sorted node
|
|
// plan. Simulate apply success followed by a stale exact-readback gate: the
|
|
// mapping must remain absent until the explicit reconcile workflow recovers it.
|
|
$staleReadback = $status;
|
|
$staleReadback['next_stage_allowed'] = false;
|
|
$history = [];
|
|
$client = $engine([
|
|
$response(['hosts' => [
|
|
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
|
|
$probeHost(2),
|
|
]]),
|
|
$response([]),
|
|
$response($status),
|
|
$response($staleReadback),
|
|
], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
|
|
409,
|
|
'A non-exact post-apply readback incorrectly created the region mapping'
|
|
);
|
|
$assert(count($history) === 4,
|
|
'Automatic configuration performed an unexpected engine request sequence.');
|
|
$paths = array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history);
|
|
$assert($paths === [
|
|
'POST /internal/v1/fabric/probe',
|
|
'GET /internal/v1/fabric',
|
|
'POST /internal/v1/fabric/apply',
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
], 'Automatic configuration lost its probe -> plan -> exact-readback ordering.');
|
|
$plan = json_decode((string) $history[2]['request']->getBody(), true,
|
|
32, JSON_THROW_ON_ERROR);
|
|
$assert(array_keys($plan) === [
|
|
'schema_version', 'fabric_id', 'region', 'overlay_cidr',
|
|
'revision', 'listen_port', 'nodes',
|
|
]
|
|
&& $plan['fabric_id'] === 'fabric-hk'
|
|
&& $plan['region'] === 'hk'
|
|
&& $plan['overlay_cidr'] === $hkOverlay
|
|
&& $plan['revision'] === 1
|
|
&& $plan['listen_port'] === 51820
|
|
&& array_column($plan['nodes'], 'host_id') === ['1', '2']
|
|
&& $plan['nodes'][0]['endpoint'] === $overlappingEndpoint . ':51820'
|
|
&& $plan['overlay_cidr'] !== $firstHkOverlay
|
|
&& array_column($plan['nodes'], 'overlay_ip') === [
|
|
substr($hkOverlay, 0, -4) . '1',
|
|
substr($hkOverlay, 0, -4) . '2',
|
|
], 'PHP did not generate the deterministic bounded Fabric plan.');
|
|
$browser = file_get_contents(dirname(__DIR__) . '/public/cloud/admin-fabric.js');
|
|
$assert(is_string($browser)
|
|
&& str_contains($browser,
|
|
"'/api/v1/admin/fabric/regions/' + encodeURIComponent(regionId) + '/' + action")
|
|
&& str_contains($browser, "{ method: 'POST', body: '{}' }")
|
|
&& !str_contains($browser, 'fabric-apply-form'),
|
|
'The browser still accepts or assembles a manual Fabric plan.');
|
|
$assert(CloudRegionFabric::where('region_id', 1)->first() === null,
|
|
'A partial apply success published a mapping before exact active readback.');
|
|
|
|
// A live VPC without its platform mapping is a migration, not an orphaned
|
|
// desired plan that the wizard may adopt through reconcile.
|
|
Capsule::table('cloud_vpcs')->insert([
|
|
'id' => 299,
|
|
'region_id' => 1,
|
|
'fabric_id' => 'fabric-hk',
|
|
'fabric_region' => 'hk',
|
|
'fabric_revision' => 1,
|
|
'state' => 'active',
|
|
]);
|
|
$history = [];
|
|
$missingVpcMapping = CloudAdminFabricService::listing($actor, $engine([
|
|
$response([$status]),
|
|
], $history));
|
|
$missingVpcMappingRegion = $findById($missingVpcMapping['regions'], 1);
|
|
$assert(is_array($missingVpcMappingRegion)
|
|
&& ($missingVpcMappingRegion['workflow']['stage'] ?? null)
|
|
=== 'migration_required'
|
|
&& ($missingVpcMappingRegion['workflow']['can_apply'] ?? true) === false
|
|
&& ($missingVpcMappingRegion['workflow']['can_reconcile'] ?? true) === false
|
|
&& in_array('vpc_fabric_mapping_missing',
|
|
$missingVpcMappingRegion['workflow']['blocking_reasons'] ?? [], true),
|
|
'A live VPC without a region mapping was projected as recoverable.');
|
|
$history = [];
|
|
$client = $engine([], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
|
|
409,
|
|
'A live VPC without a mapping reached automatic Fabric apply'
|
|
);
|
|
$assert($history === []
|
|
&& CloudRegionFabric::where('region_id', 1)->first() === null,
|
|
'Missing VPC mapping validation ran after automatic apply I/O.');
|
|
$history = [];
|
|
$client = $engine([
|
|
$response($status),
|
|
$response([$status]),
|
|
$response(['hosts' => [
|
|
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
|
|
$probeHost(2),
|
|
]]),
|
|
], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
|
|
409,
|
|
'A live VPC without a mapping reached host-writing reconcile'
|
|
);
|
|
$assert(array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history) === [
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
'GET /internal/v1/fabric',
|
|
'POST /internal/v1/fabric/probe',
|
|
]
|
|
&& CloudRegionFabric::where('region_id', 1)->first() === null,
|
|
'Missing VPC mapping validation ran after an engine write or created a pin.');
|
|
Capsule::table('cloud_vpcs')->where('id', 299)->delete();
|
|
|
|
// A deterministic desired Fabric left by a partial apply is recoverable only
|
|
// through the region reconcile action after another fresh detection.
|
|
$uncertainStatus = $status;
|
|
$uncertainStatus['state'] = 'uncertain';
|
|
$uncertainStatus['next_stage_allowed'] = false;
|
|
foreach ($uncertainStatus['nodes'] as &$uncertainNode) {
|
|
$uncertainNode['state'] = 'uncertain';
|
|
$uncertainNode['transport_verified'] = false;
|
|
$uncertainNode['observed_revision'] = null;
|
|
$uncertainNode['observed_plan_digest'] = null;
|
|
$uncertainNode['last_error_code'] = 'fabric_remote_command_failed';
|
|
}
|
|
unset($uncertainNode);
|
|
$recoveryProbePayload = ['hosts' => [
|
|
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
|
|
$probeHost(2),
|
|
]];
|
|
$history = [];
|
|
$partialRecovery = CloudAdminFabricService::probeRegion('1', $actor, $engine([
|
|
$response([$uncertainStatus]),
|
|
$response($recoveryProbePayload),
|
|
], $history));
|
|
$partialWorkflow = $partialRecovery['region']['workflow'] ?? null;
|
|
$assert(($partialWorkflow['stage'] ?? null) === 'needs_reconcile'
|
|
&& ($partialWorkflow['can_apply'] ?? true) === false
|
|
&& ($partialWorkflow['can_reconcile'] ?? false) === true
|
|
&& ($partialWorkflow['action_mode'] ?? null) === 'reconcile'
|
|
&& in_array('fabric_reconcile_required',
|
|
$partialWorkflow['blocking_reasons'] ?? [], true)
|
|
&& ($partialRecovery['region']['plan_preview']['mode'] ?? null) === 'existing',
|
|
'A partial automatic apply was not projected as a reconcile-only recovery.');
|
|
|
|
$history = [];
|
|
$client = $engine([
|
|
$response(['hosts' => [
|
|
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
|
|
$probeHost(2),
|
|
]]),
|
|
$response([$uncertainStatus]),
|
|
], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
|
|
409,
|
|
'An unmapped partial Fabric was accepted by the creation-only apply action'
|
|
);
|
|
$partialApplyPaths = array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history);
|
|
$assert($partialApplyPaths === [
|
|
'POST /internal/v1/fabric/probe',
|
|
'GET /internal/v1/fabric',
|
|
]
|
|
&& !in_array('POST /internal/v1/fabric/apply', $partialApplyPaths, true)
|
|
&& !in_array('POST /internal/v1/fabric/fabric-hk/reconcile',
|
|
$partialApplyPaths, true)
|
|
&& CloudRegionFabric::where('region_id', 1)->first() === null,
|
|
'Partial-success apply recovery reached an engine write or created a mapping.');
|
|
|
|
// Reconcile may safely replay only the fixed desired plan, but an uncertain
|
|
// engine response and exact readback still cannot publish a platform mapping.
|
|
$history = [];
|
|
$client = $engine([
|
|
$response($uncertainStatus),
|
|
$response([$uncertainStatus]),
|
|
$response($recoveryProbePayload),
|
|
$response($uncertainStatus),
|
|
$response($uncertainStatus),
|
|
], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
|
|
409,
|
|
'An uncertain replay result published an active region mapping'
|
|
);
|
|
$uncertainPaths = array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history);
|
|
$assert($uncertainPaths === [
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
'GET /internal/v1/fabric',
|
|
'POST /internal/v1/fabric/probe',
|
|
'POST /internal/v1/fabric/fabric-hk/reconcile',
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
]
|
|
&& !in_array('POST /internal/v1/fabric/apply', $uncertainPaths, true)
|
|
&& CloudRegionFabric::where('region_id', 1)->first() === null,
|
|
'Uncertain replay either repeated apply or created a platform mapping.');
|
|
|
|
$history = [];
|
|
$client = $engine([
|
|
$response($uncertainStatus),
|
|
$response([$uncertainStatus]),
|
|
$response($recoveryProbePayload),
|
|
new ConnectException('timeout', new Request(
|
|
'POST', 'http://127.0.0.1:8000/internal/v1/fabric/fabric-hk/reconcile'
|
|
)),
|
|
], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
|
|
502,
|
|
'A lost replay response published an active region mapping'
|
|
);
|
|
$unknownReplayPaths = array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history);
|
|
$assert($unknownReplayPaths === [
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
'GET /internal/v1/fabric',
|
|
'POST /internal/v1/fabric/probe',
|
|
'POST /internal/v1/fabric/fabric-hk/reconcile',
|
|
]
|
|
&& CloudRegionFabric::where('region_id', 1)->first() === null,
|
|
'Unknown replay outcome created a platform mapping.');
|
|
|
|
// A deterministic name is not proof of ownership. With no panel mapping,
|
|
// recovery must reject every foreign desired-plan dimension before issuing
|
|
// the host-writing reconcile request.
|
|
$foreignRevision = $status;
|
|
$foreignRevision['revision'] = 2;
|
|
$foreignOverlay = $status;
|
|
$foreignOverlay['overlay_cidr'] = $overlay('hk', 2);
|
|
$foreignOverlayBase = substr($foreignOverlay['overlay_cidr'], 0, -4);
|
|
$foreignOverlay['nodes'][0]['overlay_ip'] = $foreignOverlayBase . '1';
|
|
$foreignOverlay['nodes'][1]['overlay_ip'] = $foreignOverlayBase . '2';
|
|
$foreignEndpoint = $status;
|
|
$foreignEndpoint['nodes'][0]['endpoint'] = '203.0.113.250:51820';
|
|
$foreignCases = [
|
|
'revision' => [$foreignRevision, false],
|
|
'overlay' => [$foreignOverlay, true],
|
|
'endpoint' => [$foreignEndpoint, true],
|
|
];
|
|
foreach ($foreignCases as $label => [$foreign, $needsListAndProbe]) {
|
|
$history = [];
|
|
$responses = [$response($foreign)];
|
|
if ($needsListAndProbe) {
|
|
$responses[] = $response([$foreign]);
|
|
$responses[] = $response($recoveryProbePayload);
|
|
}
|
|
$client = $engine($responses, $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
|
|
409,
|
|
'A foreign deterministic-ID Fabric with mismatched ' . $label
|
|
. ' reached reconcile'
|
|
);
|
|
$paths = array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history);
|
|
$assert($paths === ($needsListAndProbe ? [
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
'GET /internal/v1/fabric',
|
|
'POST /internal/v1/fabric/probe',
|
|
] : [
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
])
|
|
&& !in_array('POST /internal/v1/fabric/fabric-hk/reconcile', $paths, true),
|
|
'Foreign ' . $label . ' validation occurred after a reconcile write.');
|
|
$assert(CloudRegionFabric::where('region_id', 1)->first() === null,
|
|
'Foreign ' . $label . ' recovery created a platform mapping.');
|
|
}
|
|
|
|
$history = [];
|
|
$configured = CloudAdminFabricService::reconcileRegion('1', $actor, $engine([
|
|
$response($uncertainStatus),
|
|
$response([$uncertainStatus]),
|
|
$response(['hosts' => [
|
|
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
|
|
$probeHost(2),
|
|
]]),
|
|
$response($status),
|
|
$response($status),
|
|
$response([$status]),
|
|
], $history));
|
|
$assert(array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history) === [
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
'GET /internal/v1/fabric',
|
|
'POST /internal/v1/fabric/probe',
|
|
'POST /internal/v1/fabric/fabric-hk/reconcile',
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
'GET /internal/v1/fabric',
|
|
],
|
|
'Partial-success recovery regenerated a plan or skipped exact readback.');
|
|
$mapping = CloudRegionFabric::where('region_id', 1)->first();
|
|
$configuredRegion = $findById($configured['regions'], 1);
|
|
$assert(!array_key_exists('fabrics', $configured)
|
|
&& $mapping !== null
|
|
&& (string) $mapping->fabric_id === 'fabric-hk'
|
|
&& (string) $mapping->fabric_region === 'hk'
|
|
&& (int) $mapping->fabric_revision === 1
|
|
&& (string) $mapping->state === 'active'
|
|
&& is_array($configuredRegion)
|
|
&& !array_key_exists('fabric', $configuredRegion)
|
|
&& $configuredRegion['ready'] === true
|
|
&& $configuredRegion['host_set_matches'] === true
|
|
&& array_map(static fn (array $host): int => (int) $host['host_id'],
|
|
$configuredRegion['hosts']) === [1, 2],
|
|
'Exact active readback did not produce the matching region Fabric mapping.');
|
|
$assert(($configuredRegion['workflow'] ?? null) === [
|
|
'stage' => 'active',
|
|
'can_probe' => true,
|
|
'can_apply' => false,
|
|
'can_reconcile' => true,
|
|
'blocking_reasons' => [],
|
|
'action_mode' => 'none',
|
|
]
|
|
&& ($configuredRegion['plan_preview'] ?? null) === [
|
|
'mode' => 'existing',
|
|
'fabric_id' => 'fabric-hk',
|
|
'fabric_region' => 'hk',
|
|
'revision' => 1,
|
|
'listen_port' => 51820,
|
|
'overlay_cidr' => $hkOverlay,
|
|
'host_count' => 2,
|
|
], 'An exact mapped Fabric did not converge to the active wizard stage.');
|
|
|
|
// A host-profile endpoint change does not mutate an existing Fabric plan.
|
|
// Detection must expose the drift and require a versioned migration instead
|
|
// of combining a newly displayed endpoint with the old active readback.
|
|
$history = [];
|
|
$endpointDriftProbe = CloudAdminFabricService::probeRegion(
|
|
'1', $actor, $engine([
|
|
$response([$status]),
|
|
$response(['hosts' => [
|
|
$probeHost(1, ['endpoint_ip' => '198.51.100.77']),
|
|
$probeHost(2),
|
|
]]),
|
|
], $history)
|
|
);
|
|
$endpointDriftRegion = $endpointDriftProbe['region'];
|
|
$endpointDriftHost = $hostById($endpointDriftRegion['hosts'], 1);
|
|
$assert(($endpointDriftRegion['ready'] ?? true) === false
|
|
&& ($endpointDriftRegion['workflow']['stage'] ?? null)
|
|
=== 'migration_required'
|
|
&& ($endpointDriftRegion['workflow']['can_apply'] ?? true) === false
|
|
&& ($endpointDriftRegion['workflow']['can_reconcile'] ?? true) === false
|
|
&& in_array('fabric_endpoint_mismatch',
|
|
$endpointDriftRegion['workflow']['blocking_reasons'] ?? [], true)
|
|
&& is_array($endpointDriftHost)
|
|
&& in_array('fabric_endpoint_mismatch',
|
|
$endpointDriftHost['reasons'] ?? [], true),
|
|
'A changed host endpoint was combined with an old active Fabric plan.');
|
|
|
|
// The public workflow projection is not an authorization boundary. Direct
|
|
// callers of reconcileRegion must repeat every platform and fresh-probe gate
|
|
// before the first host-writing engine request.
|
|
Capsule::table('cloud_regions')->where('id', 1)->update([
|
|
'status' => 'disabled',
|
|
'updated_at' => $now + 1,
|
|
]);
|
|
$history = [];
|
|
$client = $engine([], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
|
|
409,
|
|
'A disabled region reached mapped Fabric reconcile'
|
|
);
|
|
$assert($history === [],
|
|
'Disabled-region reconcile was rejected only after an engine request.');
|
|
Capsule::table('cloud_regions')->where('id', 1)->update([
|
|
'status' => 'active',
|
|
'updated_at' => $now + 2,
|
|
]);
|
|
|
|
Capsule::table('cloud_hosts')->where('id', 2)->update([
|
|
'enabled' => 0,
|
|
'updated_at' => $now + 3,
|
|
]);
|
|
$history = [];
|
|
$client = $engine([], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
|
|
409,
|
|
'A platform-blocked host reached mapped Fabric reconcile'
|
|
);
|
|
$assert($history === [],
|
|
'Platform-blocked reconcile was rejected only after an engine request.');
|
|
Capsule::table('cloud_hosts')->where('id', 2)->update([
|
|
'enabled' => 1,
|
|
'updated_at' => $now + 4,
|
|
]);
|
|
|
|
$history = [];
|
|
$client = $engine([
|
|
$response($status),
|
|
$response([$status]),
|
|
$response(['hosts' => [
|
|
$probeHost(1, [
|
|
'vpc_ready' => false,
|
|
'reasons' => ['firewall_owner_not_nftables'],
|
|
'capabilities' => ['firewall_manager' => 'firewalld'],
|
|
]),
|
|
$probeHost(2),
|
|
]]),
|
|
], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
|
|
409,
|
|
'A failed fresh capability probe reached mapped Fabric reconcile'
|
|
);
|
|
$reconcilePaths = array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history);
|
|
$assert($reconcilePaths === [
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
'GET /internal/v1/fabric',
|
|
'POST /internal/v1/fabric/probe',
|
|
] && !in_array('POST /internal/v1/fabric/fabric-hk/reconcile',
|
|
$reconcilePaths, true),
|
|
'Fresh-probe failure was checked only after host-writing reconcile.');
|
|
|
|
$degradedStatus = $status;
|
|
$degradedStatus['state'] = 'degraded';
|
|
$degradedStatus['next_stage_allowed'] = false;
|
|
$degradedStatus['nodes'][0]['state'] = 'degraded';
|
|
$degradedStatus['nodes'][0]['transport_verified'] = false;
|
|
$degradedStatus['nodes'][0]['last_error_code'] = 'fabric_host_unavailable';
|
|
$history = [];
|
|
$mappedReconciled = CloudAdminFabricService::reconcileRegion(
|
|
'1', $actor, $engine([
|
|
$response($degradedStatus),
|
|
$response([$degradedStatus]),
|
|
$response(['hosts' => [
|
|
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
|
|
$probeHost(2),
|
|
]]),
|
|
$response($status),
|
|
$response([$status]),
|
|
], $history)
|
|
);
|
|
$mappedRegion = $findById($mappedReconciled['regions'], 1);
|
|
$assert(!array_key_exists('fabrics', $mappedReconciled)
|
|
&& is_array($mappedRegion)
|
|
&& !array_key_exists('fabric', $mappedRegion)
|
|
&& array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history) === [
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
'GET /internal/v1/fabric',
|
|
'POST /internal/v1/fabric/probe',
|
|
'POST /internal/v1/fabric/fabric-hk/reconcile',
|
|
'GET /internal/v1/fabric',
|
|
]
|
|
&& ($mappedRegion['workflow']['stage'] ?? null) === 'active',
|
|
'A valid mapped degraded Fabric did not pass fresh-gated reconciliation.');
|
|
|
|
$history = [];
|
|
$activeProbeFailure = CloudAdminFabricService::probeRegion('1', $actor, $engine([
|
|
$response([$status]),
|
|
$response(['hosts' => [
|
|
$probeHost(1, [
|
|
'endpoint_ip' => $overlappingEndpoint,
|
|
'vpc_ready' => false,
|
|
'reasons' => ['firewall_owner_not_nftables'],
|
|
'capabilities' => ['firewall_manager' => 'firewalld'],
|
|
]),
|
|
$probeHost(2),
|
|
]]),
|
|
], $history));
|
|
$activeBlockedWorkflow = $activeProbeFailure['region']['workflow'] ?? null;
|
|
$assert(($activeBlockedWorkflow['stage'] ?? null) === 'blocked'
|
|
&& ($activeBlockedWorkflow['can_probe'] ?? false) === true
|
|
&& ($activeBlockedWorkflow['can_apply'] ?? true) === false
|
|
&& ($activeBlockedWorkflow['can_reconcile'] ?? true) === false
|
|
&& ($activeBlockedWorkflow['action_mode'] ?? null) === 'probe'
|
|
&& in_array('firewall_owner_not_nftables',
|
|
$activeBlockedWorkflow['blocking_reasons'] ?? [], true),
|
|
'A failed complete fresh probe left active Fabric reconcile enabled.');
|
|
|
|
// Apply is creation-only once a mapping or engine desired record exists.
|
|
// Recovery/activation always uses the fresh-gated reconcile action.
|
|
$history = [];
|
|
$client = $engine([], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
|
|
409,
|
|
'An active mapping was accepted by the automatic-configuration action'
|
|
);
|
|
$assert($history === []
|
|
&& (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'active',
|
|
'Wrong active->apply action performed engine I/O or changed the mapping.');
|
|
|
|
CloudRegionFabric::where('region_id', 1)->update([
|
|
'state' => 'disabled',
|
|
'updated_at' => $now + 5,
|
|
]);
|
|
$history = [];
|
|
$disabledProbe = CloudAdminFabricService::probeRegion('1', $actor, $engine([
|
|
$response([$status]),
|
|
$response(['hosts' => [
|
|
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
|
|
$probeHost(2),
|
|
]]),
|
|
], $history));
|
|
$disabledWorkflow = $disabledProbe['region']['workflow'] ?? null;
|
|
$assert(($disabledWorkflow['stage'] ?? null) === 'needs_reconcile'
|
|
&& ($disabledWorkflow['can_apply'] ?? true) === false
|
|
&& ($disabledWorkflow['can_reconcile'] ?? false) === true
|
|
&& ($disabledWorkflow['action_mode'] ?? null) === 'reconcile',
|
|
'A fresh exact disabled mapping did not project reconcile-only activation.');
|
|
|
|
$history = [];
|
|
$client = $engine([], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::applyRegion('1', $actor, $client),
|
|
409,
|
|
'A disabled mapping was accepted by the creation-only apply action'
|
|
);
|
|
$assert($history === []
|
|
&& (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'disabled',
|
|
'Wrong disabled->apply action performed engine I/O or changed the mapping.');
|
|
|
|
// A lost response after host reconciliation leaves the mapping disabled. A
|
|
// repeated reconcile must be idempotent and may perform the exact
|
|
// disabled->active pin even while live VPCs retain the same Fabric identity.
|
|
Capsule::table('cloud_vpcs')->insert([
|
|
[
|
|
'id' => 300, 'region_id' => 1,
|
|
'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk',
|
|
'fabric_revision' => 1, 'state' => 'active',
|
|
],
|
|
[
|
|
'id' => 301, 'region_id' => 1,
|
|
'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk',
|
|
'fabric_revision' => 1, 'state' => 'degraded',
|
|
],
|
|
]);
|
|
$history = [];
|
|
$client = $engine([
|
|
$response($status),
|
|
$response([$status]),
|
|
$response(['hosts' => [
|
|
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
|
|
$probeHost(2),
|
|
]]),
|
|
$response($status),
|
|
new ConnectException('timeout', new Request(
|
|
'GET', 'http://127.0.0.1:8000/internal/v1/fabric/fabric-hk'
|
|
)),
|
|
], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
|
|
502,
|
|
'A lost activation readback did not surface an uncertain reconcile result'
|
|
);
|
|
$unknownPaths = array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history);
|
|
$assert($unknownPaths === [
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
'GET /internal/v1/fabric',
|
|
'POST /internal/v1/fabric/probe',
|
|
'POST /internal/v1/fabric/fabric-hk/reconcile',
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
]
|
|
&& (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'disabled',
|
|
'Unknown reconcile published a mapping or skipped exact post-write readback.');
|
|
|
|
$history = [];
|
|
$activated = CloudAdminFabricService::reconcileRegion('1', $actor, $engine([
|
|
$response($status),
|
|
$response([$status]),
|
|
$response(['hosts' => [
|
|
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
|
|
$probeHost(2),
|
|
]]),
|
|
$response($status),
|
|
$response($status),
|
|
$response([$status]),
|
|
], $history));
|
|
$activatedRegion = $findById($activated['regions'], 1);
|
|
$assert(!array_key_exists('fabrics', $activated)
|
|
&& is_array($activatedRegion)
|
|
&& !array_key_exists('fabric', $activatedRegion)
|
|
&& array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history) === [
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
'GET /internal/v1/fabric',
|
|
'POST /internal/v1/fabric/probe',
|
|
'POST /internal/v1/fabric/fabric-hk/reconcile',
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
'GET /internal/v1/fabric',
|
|
]
|
|
&& (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'active'
|
|
&& ($activatedRegion['workflow']['stage'] ?? null) === 'active',
|
|
'A repeated reconcile did not converge exact disabled state to active.');
|
|
|
|
// Legacy blank pins and any single-field drift must stop before the
|
|
// host-writing reconcile request. The disabled mapping remains recoverable
|
|
// after an operator completes an explicit VPC migration.
|
|
$mismatchedVpcPins = [
|
|
'legacy-empty' => [
|
|
'fabric_id' => '', 'fabric_region' => '', 'fabric_revision' => 0,
|
|
],
|
|
'wrong-id' => [
|
|
'fabric_id' => 'fabric-other', 'fabric_region' => 'hk',
|
|
'fabric_revision' => 1,
|
|
],
|
|
'wrong-region' => [
|
|
'fabric_id' => 'fabric-hk', 'fabric_region' => 'sg',
|
|
'fabric_revision' => 1,
|
|
],
|
|
'wrong-revision' => [
|
|
'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk',
|
|
'fabric_revision' => 2,
|
|
],
|
|
];
|
|
foreach ($mismatchedVpcPins as $label => $pin) {
|
|
CloudRegionFabric::where('region_id', 1)->update([
|
|
'state' => 'disabled',
|
|
'updated_at' => $now + 6,
|
|
]);
|
|
Capsule::table('cloud_vpcs')->where('id', 300)->update($pin);
|
|
if ($label === 'legacy-empty') {
|
|
$projectionHistory = [];
|
|
$projection = CloudAdminFabricService::listing($actor, $engine([
|
|
$response([$status]),
|
|
], $projectionHistory));
|
|
$projectedRegion = $findById($projection['regions'], 1);
|
|
$assert(is_array($projectedRegion)
|
|
&& ($projectedRegion['workflow']['stage'] ?? null)
|
|
=== 'migration_required'
|
|
&& ($projectedRegion['workflow']['can_apply'] ?? true) === false
|
|
&& ($projectedRegion['workflow']['can_reconcile'] ?? true) === false
|
|
&& in_array('vpc_fabric_pin_mismatch',
|
|
$projectedRegion['workflow']['blocking_reasons'] ?? [], true),
|
|
'A disabled mapping with legacy VPC pins was projected as actionable.');
|
|
}
|
|
$history = [];
|
|
$client = $engine([
|
|
$response($status),
|
|
$response([$status]),
|
|
$response(['hosts' => [
|
|
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
|
|
$probeHost(2),
|
|
]]),
|
|
], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::reconcileRegion(
|
|
'1', $actor, $client
|
|
),
|
|
409,
|
|
'A disabled mapping accepted mismatched live-VPC pins: ' . $label
|
|
);
|
|
$assert(array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' '
|
|
. $entry['request']->getUri()->getPath(),
|
|
$history) === [
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
'GET /internal/v1/fabric',
|
|
'POST /internal/v1/fabric/probe',
|
|
]
|
|
&& (string) CloudRegionFabric::where('region_id', 1)
|
|
->value('state') === 'disabled',
|
|
'A mismatched live-VPC pin reached an engine write or changed mapping: '
|
|
. $label);
|
|
}
|
|
|
|
// Active mappings use the same server-side pin gate. UI state must not be
|
|
// able to invoke a host write while any live VPC still points elsewhere.
|
|
CloudRegionFabric::where('region_id', 1)->update([
|
|
'state' => 'active',
|
|
'updated_at' => $now + 6,
|
|
]);
|
|
$history = [];
|
|
$client = $engine([
|
|
$response($status),
|
|
$response([$status]),
|
|
$response(['hosts' => [
|
|
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
|
|
$probeHost(2),
|
|
]]),
|
|
], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
|
|
409,
|
|
'An active mapping reconciled hosts despite a live-VPC pin mismatch'
|
|
);
|
|
$assert(array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history) === [
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
'GET /internal/v1/fabric',
|
|
'POST /internal/v1/fabric/probe',
|
|
]
|
|
&& (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'active',
|
|
'Active live-VPC mismatch validation ran after host-writing reconcile.');
|
|
CloudRegionFabric::where('region_id', 1)->update([
|
|
'state' => 'disabled',
|
|
'updated_at' => $now + 6,
|
|
]);
|
|
Capsule::table('cloud_vpcs')->where('id', 300)->update([
|
|
'fabric_id' => 'fabric-hk',
|
|
'fabric_region' => 'hk',
|
|
'fabric_revision' => 1,
|
|
]);
|
|
|
|
// The preflight is intentionally outside the long engine request. If a VPC
|
|
// pin changes after host reconciliation, the mapping transaction must catch
|
|
// the race and refuse to publish active state.
|
|
$history = [];
|
|
$client = $engine([
|
|
$response($status),
|
|
$response([$status]),
|
|
$response(['hosts' => [
|
|
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
|
|
$probeHost(2),
|
|
]]),
|
|
$response($status),
|
|
static function () use ($response, $status): Response {
|
|
Capsule::table('cloud_vpcs')->where('id', 301)->update([
|
|
'fabric_revision' => 2,
|
|
]);
|
|
return $response($status);
|
|
},
|
|
], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
|
|
409,
|
|
'A live-VPC pin race was published after exact engine reconciliation'
|
|
);
|
|
$assert(array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history) === [
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
'GET /internal/v1/fabric',
|
|
'POST /internal/v1/fabric/probe',
|
|
'POST /internal/v1/fabric/fabric-hk/reconcile',
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
]
|
|
&& (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'disabled',
|
|
'The transactional live-VPC pin guard did not preserve disabled state.');
|
|
Capsule::table('cloud_vpcs')->where('id', 301)->update([
|
|
'fabric_revision' => 1,
|
|
]);
|
|
CloudRegionFabric::where('region_id', 1)->update([
|
|
'state' => 'active',
|
|
'updated_at' => $now + 7,
|
|
]);
|
|
|
|
$unsafeMappingChanges = [
|
|
'active-to-disabled' => [
|
|
'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk',
|
|
'fabric_revision' => 1, 'state' => 'disabled',
|
|
],
|
|
'identity-change' => [
|
|
'fabric_id' => 'fabric-other', 'fabric_region' => 'hk',
|
|
'fabric_revision' => 1, 'state' => 'disabled',
|
|
],
|
|
'revision-change' => [
|
|
'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk',
|
|
'fabric_revision' => 2, 'state' => 'disabled',
|
|
],
|
|
];
|
|
foreach ($unsafeMappingChanges as $label => $change) {
|
|
$assertCloudError(
|
|
static fn () => CloudAdminResourceService::saveRegionFabric(
|
|
'1', $change, $actor
|
|
),
|
|
409,
|
|
'Live VPC accepted unsafe Fabric mapping change: ' . $label
|
|
);
|
|
}
|
|
$mappingAfterBlockedChanges = CloudRegionFabric::where('region_id', 1)->first();
|
|
$assert($mappingAfterBlockedChanges !== null
|
|
&& (string) $mappingAfterBlockedChanges->fabric_id === 'fabric-hk'
|
|
&& (int) $mappingAfterBlockedChanges->fabric_revision === 1
|
|
&& (string) $mappingAfterBlockedChanges->state === 'active',
|
|
'A live-VPC mapping guard partially persisted an unsafe change.');
|
|
|
|
Capsule::table('cloud_regions')->where('id', 1)->update([
|
|
'status' => 'disabled',
|
|
'updated_at' => $now + 13,
|
|
]);
|
|
$history = [];
|
|
$client = $engine([$response($status)], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminResourceService::saveRegionFabric('1', [
|
|
'fabric_id' => 'fabric-hk', 'fabric_region' => 'hk',
|
|
'fabric_revision' => 1, 'state' => 'active',
|
|
], $actor, false, $client),
|
|
409,
|
|
'An active mapping was published while its locked region was disabled'
|
|
);
|
|
$assert(array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history) === ['GET /internal/v1/fabric/fabric-hk']
|
|
&& (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'active',
|
|
'Disabled-region linearization changed the mapping or skipped engine gate.');
|
|
Capsule::table('cloud_regions')->where('id', 1)->update([
|
|
'status' => 'active',
|
|
'updated_at' => $now + 14,
|
|
]);
|
|
Capsule::table('cloud_vpcs')->whereIn('id', [300, 301])->delete();
|
|
|
|
// The same exact disabled activation remains valid when the region has no
|
|
// live VPCs; the migration guard must not manufacture a dependency.
|
|
CloudRegionFabric::where('region_id', 1)->update([
|
|
'state' => 'disabled',
|
|
'updated_at' => $now + 14,
|
|
]);
|
|
$history = [];
|
|
$withoutVpcs = CloudAdminFabricService::reconcileRegion('1', $actor, $engine([
|
|
$response($status),
|
|
$response([$status]),
|
|
$response(['hosts' => [
|
|
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
|
|
$probeHost(2),
|
|
]]),
|
|
$response($status),
|
|
$response($status),
|
|
$response([$status]),
|
|
], $history));
|
|
$withoutVpcsRegion = $findById($withoutVpcs['regions'], 1);
|
|
$assert(array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history) === [
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
'GET /internal/v1/fabric',
|
|
'POST /internal/v1/fabric/probe',
|
|
'POST /internal/v1/fabric/fabric-hk/reconcile',
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
'GET /internal/v1/fabric',
|
|
]
|
|
&& (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'active'
|
|
&& ($withoutVpcsRegion['workflow']['stage'] ?? null) === 'active',
|
|
'A no-VPC disabled mapping did not converge through exact reconcile.');
|
|
|
|
// A disabled activation must re-read platform generation after the engine
|
|
// write. If the region changes during reconcile, keep the mapping disabled.
|
|
CloudRegionFabric::where('region_id', 1)->update([
|
|
'state' => 'disabled',
|
|
'updated_at' => $now + 15,
|
|
]);
|
|
$history = [];
|
|
$client = $engine([
|
|
$response($status),
|
|
$response([$status]),
|
|
$response(['hosts' => [
|
|
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
|
|
$probeHost(2),
|
|
]]),
|
|
$response($status),
|
|
static function () use ($response, $status, $now): Response {
|
|
Capsule::table('cloud_regions')->where('id', 1)->update([
|
|
'status' => 'disabled',
|
|
'updated_at' => $now + 16,
|
|
]);
|
|
return $response($status);
|
|
},
|
|
], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
|
|
409,
|
|
'A disabled mapping was activated after its region changed during reconcile'
|
|
);
|
|
$assert(array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history) === [
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
'GET /internal/v1/fabric',
|
|
'POST /internal/v1/fabric/probe',
|
|
'POST /internal/v1/fabric/fabric-hk/reconcile',
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
]
|
|
&& (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'disabled',
|
|
'Post-reconcile generation drift published an active mapping.');
|
|
Capsule::table('cloud_regions')->where('id', 1)->update([
|
|
'status' => 'active',
|
|
'updated_at' => $now + 17,
|
|
]);
|
|
CloudRegionFabric::where('region_id', 1)->update([
|
|
'state' => 'active',
|
|
'updated_at' => $now + 18,
|
|
]);
|
|
|
|
// A mapping owns exactly one identity, not every desired record sharing its
|
|
// region string. Reconcile must read both the mapped detail and full catalog,
|
|
// then fail before touching hosts when an extra same-region Fabric exists.
|
|
$history = [];
|
|
$client = $engine([
|
|
$response($status),
|
|
$response([$status, $historicalStatus]),
|
|
], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
|
|
409,
|
|
'Mapped-region reconcile ignored an additional same-region Fabric'
|
|
);
|
|
$assert(array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history) === [
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
'GET /internal/v1/fabric',
|
|
]
|
|
&& CloudRegionFabric::where('region_id', 1)->where('state', 'active')->exists(),
|
|
'Mapped conflict detection skipped detail/catalog reads or reached reconcile.');
|
|
|
|
// Once a mapping exists, a missing engine desired record cannot be recreated
|
|
// from mutable inventory under the already pinned revision.
|
|
CloudRegionFabric::where('region_id', 1)->update([
|
|
'state' => 'disabled',
|
|
'updated_at' => $now + 6,
|
|
]);
|
|
$history = [];
|
|
$client = $engine([
|
|
new Response(404, ['Content-Type' => 'application/json'], json_encode([
|
|
'error_code' => 'fabric_not_found',
|
|
], JSON_THROW_ON_ERROR)),
|
|
], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
|
|
404,
|
|
'A missing mapped engine desired plan reached reconcile'
|
|
);
|
|
$assert(array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history) === [
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
]
|
|
&& CloudRegionFabric::where('region_id', 1)->where('fabric_revision', 1)
|
|
->where('state', 'disabled')->exists(),
|
|
'Missing desired-plan recovery mutated the mapping or reached an engine write.');
|
|
CloudRegionFabric::where('region_id', 1)->update([
|
|
'state' => 'active',
|
|
'updated_at' => $now + 7,
|
|
]);
|
|
|
|
$history = [];
|
|
$missingDesiredListing = CloudAdminFabricService::listing($actor, $engine([
|
|
$response([]),
|
|
], $history));
|
|
$missingDesiredRegion = $findById($missingDesiredListing['regions'], 1);
|
|
$assert(is_array($missingDesiredRegion)
|
|
&& ($missingDesiredRegion['workflow']['stage'] ?? null) === 'recovery_required'
|
|
&& ($missingDesiredRegion['workflow']['can_apply'] ?? true) === false
|
|
&& ($missingDesiredRegion['workflow']['can_reconcile'] ?? true) === false
|
|
&& ($missingDesiredRegion['workflow']['action_mode'] ?? null) === 'none'
|
|
&& in_array('fabric_desired_missing',
|
|
$missingDesiredRegion['workflow']['blocking_reasons'] ?? [], true)
|
|
&& $missingDesiredRegion['plan_preview'] === null,
|
|
'A mapped region with missing engine desired state was not failed closed.');
|
|
|
|
// A host can become required while an engine reconcile is in flight. The
|
|
// second platform read must reject the stale two-host result before mapping
|
|
// persistence is consulted.
|
|
$membership = CloudHostResourcePoolMembership::where('host_id', 3)->first();
|
|
$history = [];
|
|
$client = $engine([
|
|
$response($status),
|
|
$response([$status]),
|
|
static function () use (
|
|
$response, $probeHost, $overlappingEndpoint, $membership, $now
|
|
): Response {
|
|
$membership->scheduler_state = 'active';
|
|
$membership->updated_at = $now + 2;
|
|
$membership->save();
|
|
return $response(['hosts' => [
|
|
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
|
|
$probeHost(2),
|
|
]]);
|
|
},
|
|
], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
|
|
409,
|
|
'A topology change during fresh-gated reconcile reached host writes'
|
|
);
|
|
$assert(array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history) === [
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
'GET /internal/v1/fabric',
|
|
'POST /internal/v1/fabric/probe',
|
|
],
|
|
'The topology generation gate ran after host-writing reconcile.');
|
|
$membership->scheduler_state = 'disabled';
|
|
$membership->updated_at = $now + 3;
|
|
$membership->save();
|
|
|
|
// Likewise, a concurrent administrator may change the mapping after the
|
|
// engine exact-readback request starts. The transaction must compare against
|
|
// the pre-network snapshot rather than overwriting that winner.
|
|
CloudRegionFabric::where('region_id', 1)->update([
|
|
'state' => 'disabled',
|
|
'updated_at' => $now + 8,
|
|
]);
|
|
$history = [];
|
|
$client = $engine([
|
|
$response($status),
|
|
$response([$status]),
|
|
$response(['hosts' => [
|
|
$probeHost(1, ['endpoint_ip' => $overlappingEndpoint]),
|
|
$probeHost(2),
|
|
]]),
|
|
$response($status),
|
|
static function () use ($response, $status, $now): Response {
|
|
CloudRegionFabric::where('region_id', 1)->update([
|
|
'state' => 'active',
|
|
'updated_at' => $now + 9,
|
|
]);
|
|
return $response($status);
|
|
},
|
|
], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
|
|
409,
|
|
'A concurrent mapping update was overwritten after engine readback'
|
|
);
|
|
$assert(array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(),
|
|
$history) === [
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
'GET /internal/v1/fabric',
|
|
'POST /internal/v1/fabric/probe',
|
|
'POST /internal/v1/fabric/fabric-hk/reconcile',
|
|
'GET /internal/v1/fabric/fabric-hk',
|
|
]
|
|
&& (string) CloudRegionFabric::where('region_id', 1)->value('state') === 'active',
|
|
'The mapping CAS did not preserve the concurrent writer.');
|
|
CloudRegionFabric::where('region_id', 1)->update([
|
|
'state' => 'active',
|
|
'updated_at' => $now + 10,
|
|
]);
|
|
|
|
// Adding a new required host is a topology migration, not an automatic apply.
|
|
// The existing desired membership must stay fixed and no mutation may run.
|
|
CloudRegionFabric::where('region_id', 1)->update([
|
|
'state' => 'disabled',
|
|
'updated_at' => $now + 11,
|
|
]);
|
|
$membership->scheduler_state = 'active';
|
|
$membership->updated_at = $now + 1;
|
|
$membership->save();
|
|
$history = [];
|
|
$client = $engine([
|
|
$response($status),
|
|
], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::reconcileRegion('1', $actor, $client),
|
|
409,
|
|
'Existing Fabric membership drift was automatically overwritten'
|
|
);
|
|
$assert(count($history) === 1
|
|
&& $history[0]['request']->getUri()->getPath()
|
|
=== '/internal/v1/fabric/fabric-hk'
|
|
&& !array_filter($history, static fn (array $entry): bool =>
|
|
in_array($entry['request']->getUri()->getPath(), [
|
|
'/internal/v1/fabric/apply',
|
|
'/internal/v1/fabric/fabric-hk/reconcile',
|
|
], true)),
|
|
'Membership drift reached apply/reconcile instead of failing closed.');
|
|
$mapping->refresh();
|
|
$assert((int) $mapping->fabric_revision === 1
|
|
&& (string) $mapping->state === 'disabled',
|
|
'Membership drift changed the existing region mapping.');
|
|
CloudRegionFabric::where('region_id', 1)->update([
|
|
'state' => 'active',
|
|
'updated_at' => $now + 12,
|
|
]);
|
|
|
|
// A readiness-only inventory may exceed the engine's 64-host Fabric size.
|
|
// Detection must split it into bounded requests without dropping rows or
|
|
// converting numeric database IDs into JSON numbers.
|
|
$sgProbeRows = [];
|
|
for ($id = 100; $id < 165; $id++) {
|
|
Capsule::table('cloud_hosts')->insert([
|
|
'id' => $id,
|
|
'name' => 'sg-host-' . $id,
|
|
'region' => 'sg',
|
|
'country_code' => 'SG',
|
|
'enabled' => 1,
|
|
'resource_pool_id' => 22,
|
|
'scheduler_state' => 'active',
|
|
'default_network' => 'bridge:br0',
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_host_resource_pool_memberships')->insert([
|
|
'id' => 1000 + $id,
|
|
'host_id' => $id,
|
|
'resource_pool_id' => 22,
|
|
'scheduler_state' => 'active',
|
|
'priority' => 0,
|
|
'weight' => 100,
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
$sgProbeRows[] = $probeHost($id, [
|
|
'region' => 'sg',
|
|
'endpoint_ip' => '198.51.100.' . ($id - 99),
|
|
]);
|
|
}
|
|
$history = [];
|
|
$sgProbe = CloudAdminFabricService::probeRegion('2', $actor, $engine([
|
|
$response([]),
|
|
$response(['hosts' => array_slice($sgProbeRows, 0, 64)]),
|
|
$response(['hosts' => array_slice($sgProbeRows, 64)]),
|
|
], $history));
|
|
$firstBatch = json_decode((string) $history[1]['request']->getBody(), true,
|
|
32, JSON_THROW_ON_ERROR)['host_ids'] ?? [];
|
|
$secondBatch = json_decode((string) $history[2]['request']->getBody(), true,
|
|
32, JSON_THROW_ON_ERROR)['host_ids'] ?? [];
|
|
$assert(count($sgProbe['region']['hosts']) === 65
|
|
&& count($firstBatch) === 64
|
|
&& count($secondBatch) === 1
|
|
&& array_reduce(array_merge($firstBatch, $secondBatch),
|
|
static fn (bool $carry, $hostId): bool => $carry && is_string($hostId), true)
|
|
&& $firstBatch[0] === '100'
|
|
&& $secondBatch[0] === '164',
|
|
'Readiness detection did not preserve all rows across bounded string-ID batches.');
|
|
|
|
for ($id = 165; $id <= 228; $id++) {
|
|
Capsule::table('cloud_hosts')->insert([
|
|
'id' => $id,
|
|
'name' => 'sg-host-' . $id,
|
|
'region' => 'sg',
|
|
'country_code' => 'SG',
|
|
'enabled' => 1,
|
|
'resource_pool_id' => 22,
|
|
'scheduler_state' => 'active',
|
|
'default_network' => 'bridge:br0',
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_host_resource_pool_memberships')->insert([
|
|
'id' => 1000 + $id,
|
|
'host_id' => $id,
|
|
'resource_pool_id' => 22,
|
|
'scheduler_state' => 'active',
|
|
'priority' => 0,
|
|
'weight' => 100,
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
}
|
|
$history = [];
|
|
$client = $engine([$response([])], $history);
|
|
$assertCloudError(
|
|
static fn () => CloudAdminFabricService::probeRegion('2', $actor, $client),
|
|
409,
|
|
'An unbounded readiness inventory exceeded the browser request budget'
|
|
);
|
|
$assert(count($history) === 1
|
|
&& $history[0]['request']->getUri()->getPath() === '/internal/v1/fabric',
|
|
'The oversized readiness inventory started a host probe before rejection.');
|
|
|
|
echo "Administrator Fabric readiness workflow passed.\n";
|