diff --git a/web/app/Models/CloudPublicIp.php b/web/app/Models/CloudPublicIp.php index f78ea4e5..40b24f5c 100644 --- a/web/app/Models/CloudPublicIp.php +++ b/web/app/Models/CloudPublicIp.php @@ -17,4 +17,16 @@ final class CloudPublicIp extends Model 'created_at' => 'int', 'updated_at' => 'int', ]; + + /** New ledger rows must always belong to one validated public pool. */ + protected static function booted(): void + { + static::creating(static function (self $row): void { + if ((int) ($row->public_ip_pool_id ?? 0) < 1) { + throw new \LogicException( + 'New public IP rows require a public pool association.' + ); + } + }); + } } diff --git a/web/app/Services/Cloud/CloudAdminHostSchedulingService.php b/web/app/Services/Cloud/CloudAdminHostSchedulingService.php index dc032495..e36ffdcf 100644 --- a/web/app/Services/Cloud/CloudAdminHostSchedulingService.php +++ b/web/app/Services/Cloud/CloudAdminHostSchedulingService.php @@ -17,17 +17,31 @@ final class CloudAdminHostSchedulingService public static function read(int $hostId): array { - $host = CloudHost::find($hostId); - if ($host === null) { - throw new CloudError('宿主机不存在', 404); - } + [$host, $membership, $memberships] = (new CloudHost())->getConnection() + ->transaction(function () use ($hostId): array { + $host = CloudHost::where('id', $hostId)->lockForUpdate()->first(); + if ($host === null) { + throw new CloudError('宿主机不存在', 404); + } + $rows = CloudHostPoolMembershipService::lockedRows($host, time()); + $selected = CloudHostPoolMembershipService::selected($host, $rows); + $poolId = $host->resource_pool_id; + $state = (string) $host->scheduler_state; + CloudHostPoolMembershipService::mirrorLocked($host, $selected); + if ($poolId !== $host->resource_pool_id + || $state !== (string) $host->scheduler_state) { + $host->updated_at = time(); + $host->save(); + } + return [$host, $selected, $rows]; + }, 3); $policy = CloudHostCapacityPolicy::find($hostId); $snapshot = CloudHostCapacitySnapshot::find($hostId); $pools = self::poolOptions(); $selected = null; foreach ($pools as $pool) { - if ((int) $pool['id'] === (int) ($host->resource_pool_id ?? 0)) { + if ((int) $pool['id'] === (int) ($membership?->resource_pool_id ?? 0)) { $selected = $pool; break; } @@ -39,11 +53,12 @@ final class CloudAdminHostSchedulingService 'id' => (int) $host->id, 'name' => self::safeText((string) $host->name, 80), 'enabled' => (bool) $host->enabled, - 'resource_pool_id' => $host->resource_pool_id === null - ? null : (int) $host->resource_pool_id, - 'scheduler_state' => in_array((string) $host->scheduler_state, + 'resource_pool_id' => $membership === null + ? null : (int) $membership->resource_pool_id, + 'scheduler_state' => in_array((string) ($membership?->scheduler_state + ?? 'disabled'), self::SCHEDULER_STATES, true) - ? (string) $host->scheduler_state : 'disabled', + ? (string) ($membership?->scheduler_state ?? 'disabled') : 'disabled', 'health_state' => self::safeText((string) ($host->health_state ?? 'unknown'), 32), 'last_seen_at' => self::nullableInt($host->last_seen_at), 'health_checked_at' => self::nullableInt($host->health_checked_at), @@ -52,6 +67,13 @@ final class CloudAdminHostSchedulingService 'policy' => self::policyDto($policy), 'snapshot' => $snapshotDto, 'placement' => $selected, + 'memberships' => $memberships->map(static fn ($row): array => [ + 'id' => (int) $row->id, + 'resource_pool_id' => (int) $row->resource_pool_id, + 'scheduler_state' => (string) $row->scheduler_state, + 'priority' => (int) $row->priority, + 'weight' => (int) $row->weight, + ])->values()->all(), 'pools' => $pools, 'network_lines_enabled' => CloudNetworkLineFeature::enabled(), 'warnings' => self::warnings($host, $snapshotDto, $selected), @@ -85,6 +107,9 @@ final class CloudAdminHostSchedulingService if ($host === null) { throw new CloudError('宿主机不存在', 404); } + // Membership is the scheduling authority. Lock it directly + // after the host sentinel and before host-global capacity. + CloudHostPoolMembershipService::lockedRows($host, time()); $policy = CloudHostCapacityPolicy::where('host_id', $hostId) ->lockForUpdate()->first(); if ($policy === null) { @@ -104,8 +129,9 @@ final class CloudAdminHostSchedulingService } $now = time(); - $host->resource_pool_id = $poolId; - $host->scheduler_state = $schedulerState; + CloudHostPoolMembershipService::replaceLocked( + $host, $poolId, $schedulerState, $now + ); $host->updated_at = $now; $host->save(); foreach ($values as $field => $value) { diff --git a/web/app/Services/Cloud/CloudAdminResourceService.php b/web/app/Services/Cloud/CloudAdminResourceService.php index bedf8773..f169adad 100644 --- a/web/app/Services/Cloud/CloudAdminResourceService.php +++ b/web/app/Services/Cloud/CloudAdminResourceService.php @@ -5,6 +5,7 @@ namespace App\Services\Cloud; use App\Models\CloudHost; use App\Models\CloudHostCapacityPolicy; use App\Models\CloudHostCapacitySnapshot; +use App\Models\CloudHostResourcePoolMembership; use App\Models\CloudNetworkLine; use App\Models\CloudOffer; use App\Models\CloudPublicIp; @@ -240,7 +241,9 @@ final class CloudAdminResourceService throw new CloudError('所选线路不存在', 409); } if ($pool->exists && (int) $pool->line_id !== $lineId - && (CloudHost::where('resource_pool_id', (int) $pool->id)->exists() + && (CloudHostResourcePoolMembership::where( + 'resource_pool_id', (int) $pool->id)->exists() + || CloudHost::where('resource_pool_id', (int) $pool->id)->exists() || CloudOffer::where('resource_pool_id', (int) $pool->id)->exists() || CloudResourceReservation::where('resource_pool_id', (int) $pool->id)->exists())) { throw new CloudError('已被节点、销售方案或预留引用的资源池不能更换线路', 409); @@ -367,8 +370,14 @@ final class CloudAdminResourceService $offerCounts[(int) $row->resource_pool_id] = (int) $row->row_count; } - $hosts = CloudHost::whereNotNull('resource_pool_id')->orderBy('id')->get(); - $hostIds = $hosts->pluck('id')->map(static fn ($id): int => (int) $id)->all(); + $memberships = CloudHostResourcePoolMembership::orderBy('host_id') + ->orderBy('resource_pool_id')->orderBy('id')->get(); + $hostIds = $memberships->pluck('host_id')->map( + static fn ($id): int => (int) $id + )->unique()->values()->all(); + $hosts = $hostIds === [] ? collect() + : CloudHost::whereIn('id', $hostIds)->orderBy('id')->get(); + $hostById = self::keyById($hosts); $policies = $hostIds === [] ? collect() : CloudHostCapacityPolicy::whereIn('host_id', $hostIds)->get(); $snapshots = $hostIds === [] ? collect() : CloudHostCapacitySnapshot::whereIn('host_id', $hostIds)->get(); $policyByHost = self::keyBy($policies, 'host_id'); @@ -422,8 +431,18 @@ final class CloudAdminResourceService } $poolHosts = []; - foreach ($hosts as $host) { - $poolHosts[(int) $host->resource_pool_id][] = $host; + foreach ($memberships->groupBy('host_id') as $hostId => $rows) { + $host = $hostById[(int) $hostId] ?? null; + if ($host === null) { + continue; + } + $membership = CloudHostPoolMembershipService::selected($host, $rows); + if ($membership !== null) { + $poolHosts[(int) $membership->resource_pool_id][] = [ + 'host' => $host, + 'membership' => $membership, + ]; + } } $poolRows = []; foreach ($pools as $pool) { @@ -434,7 +453,9 @@ final class CloudAdminResourceService $schedulable = 0; $syncState = 'never'; $syncRank = 0; - foreach ($poolHosts[$poolId] ?? [] as $host) { + foreach ($poolHosts[$poolId] ?? [] as $placement) { + $host = $placement['host']; + $membership = $placement['membership']; $hostId = (int) $host->id; $policy = $policyByHost[$hostId] ?? null; $snapshot = $snapshotByHost[$hostId] ?? null; @@ -504,6 +525,7 @@ final class CloudAdminResourceService && $region !== null && (string) $region->status === 'active' && (bool) $host->enabled && (string) $host->scheduler_state === 'active' + && (string) $membership->scheduler_state === 'active' && (string) $host->health_state === 'healthy' && $state === 'fresh' && ($maxInstances <= 0 || $instanceCount < $maxInstances) diff --git a/web/app/Services/Cloud/CloudHostInventorySyncService.php b/web/app/Services/Cloud/CloudHostInventorySyncService.php index aa23e2f2..321b32d4 100644 --- a/web/app/Services/Cloud/CloudHostInventorySyncService.php +++ b/web/app/Services/Cloud/CloudHostInventorySyncService.php @@ -4,7 +4,10 @@ namespace App\Services\Cloud; use App\Models\CloudHost; use App\Models\CloudHostCapacitySnapshot; +use App\Models\CloudHostNetworkAttachment; +use App\Models\CloudPublicIpPool; use App\Models\CloudPublicIp; +use App\Models\CloudResourcePool; use App\Services\Config; /** @@ -19,7 +22,15 @@ final class CloudHostInventorySyncService { private const ENGINE_TIMEOUT_SECONDS = 3.0; private const MAX_HOSTS_PER_PASS = 10; + private const MAX_PUBLIC_POOLS = 32; + private const MAX_PUBLIC_IPS_PER_POOL = 254; + private const MAX_PUBLIC_IPS_PER_HOST = 8128; private const UINT32_MAX = 4294967295; + private const LEGACY_POOL_KEY = 'legacy-default'; + private const PUBLIC_POOL_STATES = ['active', 'draining', 'disabled']; + private const LOCAL_IP_HOLDS = [ + 'reserved', 'assigned', 'changing', 'uncertain', + ]; private const ERROR_UNAVAILABLE = 'host_inventory_unavailable'; private const ERROR_ENGINE_REJECTED = 'host_inventory_engine_rejected'; @@ -170,10 +181,9 @@ final class CloudHostInventorySyncService try { $network = self::network($networkPayload); - if ($capacity['public_ip_configured'] !== $network['configured'] - || $capacity['public_ip_available_count'] !== count($network['available'])) { - throw new \UnexpectedValueException('inconsistent public IP inventory'); - } + [$capacity, $pools] = self::reconcilePublicInventory( + $capacity, $network + ); } catch (\UnexpectedValueException $error) { $this->failed($hostId, ['degraded', self::ERROR_INVALID_NETWORK], $result, @@ -183,7 +193,7 @@ final class CloudHostInventorySyncService try { $applied = $this->apply( - $hostId, $capacity, $network['available'], $attemptedAt, + $hostId, $capacity, $pools, $attemptedAt, $attemptGeneration ); if (!$applied) { @@ -191,6 +201,10 @@ final class CloudHostInventorySyncService return; } $result['succeeded']++; + } catch (\UnexpectedValueException $error) { + $this->failed($hostId, + ['degraded', self::ERROR_INVALID_NETWORK], $result, + $attemptedAt, $attemptGeneration); } catch (\DomainException $error) { $this->failed($hostId, ['degraded', self::ERROR_INVENTORY_CONFLICT], $result, @@ -259,12 +273,12 @@ final class CloudHostInventorySyncService return ['degraded', self::ERROR_ENGINE_REJECTED]; } - private function apply(int $hostId, array $capacity, array $available, + private function apply(int $hostId, array $capacity, array $pools, int $attemptedAt, int $attemptGeneration): bool { $now = $this->now(); return (new CloudHost())->getConnection()->transaction( - function () use ($hostId, $capacity, $available, $now, $attemptedAt, + function () use ($hostId, $capacity, $pools, $now, $attemptedAt, $attemptGeneration): bool { $host = CloudHost::where('id', $hostId)->lockForUpdate()->first(); if ($host === null || !(bool) $host->enabled) { @@ -280,7 +294,211 @@ final class CloudHostInventorySyncService || (int) $snapshot->attempt_generation !== $attemptGeneration) { return false; } - $addresses = array_keys($available); + + // Older engines did not expose the selector in the scalar + // network-options response. Its one synthetic pool is tied + // to the already validated host default and cannot select a + // second attachment. + foreach ($pools as $poolKey => &$pool) { + if ($poolKey === self::LEGACY_POOL_KEY + && (string) $pool['network_selector'] === '') { + $pool['network_selector'] = (string) $host->default_network; + } + if (!hash_equals((string) $host->default_network, + (string) $pool['network_selector'])) { + throw new \UnexpectedValueException( + 'public pool selector differs from host default network' + ); + } + } + unset($pool); + + // The host sentinel also serializes membership changes. The + // snapshot lock remains second so an older observation can + // never regress the existing generation fence. + $memberships = CloudHostPoolMembershipService::lockedRows($host, $now); + $membership = CloudHostPoolMembershipService::selected( + $host, $memberships + ); + $lineId = null; + if ($pools !== []) { + $resourcePool = $membership === null ? null + : CloudResourcePool::find((int) $membership->resource_pool_id); + $lineId = $resourcePool === null ? null + : (int) $resourcePool->line_id; + if ($lineId === null || $lineId < 1) { + throw new \UnexpectedValueException( + 'public pools require one host resource-pool membership' + ); + } + } + + // Lock observed network identity before concrete addresses. + // All host-scoped inventory writers use host -> snapshot -> + // membership -> attachment -> public pool -> IP; rows within + // each group are ordered. Keeping snapshot second preserves + // the observation-generation fence used by beginAttempt(). + $attachmentRows = CloudHostNetworkAttachment::where('host_id', $hostId) + ->orderBy('id')->lockForUpdate()->get(); + $publicPoolRows = CloudPublicIpPool::where('host_id', $hostId) + ->orderBy('id')->lockForUpdate()->get(); + $ipRows = CloudPublicIp::where('host_id', $hostId) + ->orderBy('id')->lockForUpdate()->get(); + + $attachmentsByKey = []; + foreach ($attachmentRows as $row) { + $attachmentsByKey[(string) $row->attachment_key] = $row; + } + $publicPoolsByKey = []; + foreach ($publicPoolRows as $row) { + $publicPoolsByKey[(string) $row->pool_key] = $row; + } + + $observedAttachmentKeys = []; + $observedPoolKeys = []; + $poolModels = []; + foreach ($pools as $poolKey => $pool) { + $attachmentKey = self::attachmentKey( + (string) $pool['network_selector'] + ); + $attachment = $attachmentsByKey[$attachmentKey] ?? null; + if ($attachment === null) { + $attachment = new CloudHostNetworkAttachment(); + $attachment->attachment_key = $attachmentKey; + $attachment->host_id = $hostId; + $attachment->vlan_id = null; + $attachment->routing_table = null; + $attachment->created_at = $now; + $attachmentsByKey[$attachmentKey] = $attachment; + } + $attachment->line_id = $lineId; + $attachment->name = mb_substr( + (string) $pool['network_selector'], 0, 80 + ); + $attachment->network_mode = str_starts_with( + (string) $pool['network_selector'], 'bridge:' + ) ? 'bridge' : 'network'; + $attachment->network_selector = $pool['network_selector']; + $attachment->state = self::attachmentState($pools, + (string) $pool['network_selector']); + $attachment->priority = 0; + $attachment->desired_revision = max( + (int) ($attachment->desired_revision ?? 0), + (int) $pool['revision'] + ); + $attachment->observed_revision = max( + (int) ($attachment->observed_revision ?? 0), + (int) $pool['revision'] + ); + $attachment->sync_state = 'fresh'; + $attachment->last_error_code = ''; + $attachment->updated_at = $now; + $attachment->save(); + $observedAttachmentKeys[$attachmentKey] = true; + + $publicPool = $publicPoolsByKey[$poolKey] ?? null; + if ($publicPool === null) { + $publicPool = new CloudPublicIpPool(); + $publicPool->pool_key = $poolKey; + $publicPool->host_id = $hostId; + $publicPool->name = (string) $pool['cidr']; + $publicPool->created_at = $now; + $publicPoolsByKey[$poolKey] = $publicPool; + } + if ((int) $publicPool->host_id !== $hostId) { + throw new \DomainException('public pool host mismatch'); + } + $publicPool->network_attachment_id = (int) $attachment->id; + $publicPool->cidr = $pool['cidr']; + $publicPool->gateway = $pool['gateway']; + $publicPool->guest_prefix_length = $pool['guest_prefix']; + $publicPool->dns_servers_json = json_encode( + $pool['dns_servers'], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES + ); + if ($pool['reserved_ips_known']) { + $publicPool->reserved_ips_json = json_encode( + $pool['reserved_ips'], + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES + ); + } elseif ($publicPool->reserved_ips_json === null) { + // A pre-catalog engine cannot reveal its explicit + // reserved list. Preserve known metadata and use an + // empty placeholder only for a brand-new legacy row. + $publicPool->reserved_ips_json = '[]'; + } + $publicPool->state = $pool['state']; + $publicPool->priority = 0; + if ($publicPool->exists + && (int) $publicPool->observed_revision + > (int) $pool['revision']) { + throw new \UnexpectedValueException( + 'public pool revision regressed' + ); + } + $publicPool->desired_revision = max( + (int) ($publicPool->desired_revision ?? 0), + (int) $pool['revision'] + ); + $publicPool->observed_revision = (int) $pool['revision']; + $publicPool->sync_state = (int) $publicPool->observed_revision + >= (int) $publicPool->desired_revision + ? 'fresh' : 'pending'; + $publicPool->last_error_code = ''; + $publicPool->updated_at = $now; + $publicPool->save(); + $poolModels[$poolKey] = $publicPool; + $observedPoolKeys[$poolKey] = true; + } + + foreach ($attachmentRows as $row) { + if (!isset($observedAttachmentKeys[(string) $row->attachment_key])) { + $row->sync_state = 'failed'; + $row->last_error_code = 'host_network_attachment_missing'; + $row->updated_at = $now; + $row->save(); + } + } + $missingPoolIds = []; + foreach ($publicPoolRows as $row) { + if (!isset($observedPoolKeys[(string) $row->pool_key])) { + $row->sync_state = 'failed'; + $row->last_error_code = 'host_public_pool_missing'; + $row->updated_at = $now; + $row->save(); + $missingPoolIds[(int) $row->id] = true; + } + } + + // A successful full catalog is authoritative. Free rows + // owned by a pool that disappeared must stop advertising + // capacity immediately; stronger local holds remain intact. + foreach ($ipRows as $row) { + if ($row->public_ip_pool_id !== null + && isset($missingPoolIds[(int) $row->public_ip_pool_id]) + && (string) $row->state === 'available') { + $row->state = 'disabled'; + $row->updated_at = $now; + $row->save(); + } + } + + $observed = []; + foreach ($pools as $poolKey => $pool) { + $publicPool = $poolModels[$poolKey]; + foreach ($pool['available'] as $address => $prefixLength) { + if (isset($observed[$address])) { + throw new \UnexpectedValueException( + 'public IP belongs to multiple observed pools' + ); + } + $observed[$address] = [ + 'prefix_length' => $prefixLength, + 'public_ip_pool_id' => (int) $publicPool->id, + 'allocatable' => (string) $pool['state'] === 'active', + ]; + } + } + $addresses = array_keys($observed); // Never lock another host's inventory after locking this host; // that would allow two concurrent host polls to deadlock. The // global unique key closes an insert race after this check. @@ -289,21 +507,56 @@ final class CloudHostInventorySyncService ->where('host_id', '<>', $hostId)->exists()) { throw new \DomainException('public IP belongs to another host'); } - $rows = CloudPublicIp::where('host_id', $hostId) - ->lockForUpdate()->get(); $byAddress = []; - foreach ($rows as $row) { + foreach ($ipRows as $row) { $byAddress[(string) $row->address] = $row; } + $observedPoolKeyById = []; + foreach ($poolModels as $poolKey => $poolModel) { + $observedPoolKeyById[(int) $poolModel->id] = $poolKey; + } + + // Backfill pool identity for every local row, including held + // and assigned addresses that are intentionally absent from + // the engine's advisory free lists. + foreach ($ipRows as $row) { + $poolKey = self::poolKeyForAddress((string) $row->address, $pools); + $held = in_array((string) $row->state, + self::LOCAL_IP_HOLDS, true); + $currentPoolKey = $row->public_ip_pool_id === null ? null + : ($observedPoolKeyById[(int) $row->public_ip_pool_id] ?? null); + if ($held && $currentPoolKey !== null + && $poolKey !== $currentPoolKey) { + throw new \DomainException( + 'held public IP no longer belongs to its pool' + ); + } + if ($poolKey === null) { + continue; + } + $poolId = (int) $poolModels[$poolKey]->id; + if ($row->public_ip_pool_id !== null + && (int) $row->public_ip_pool_id !== $poolId + && $held) { + throw new \DomainException('held public IP changed pools'); + } + if ((int) ($row->public_ip_pool_id ?? 0) !== $poolId) { + $row->public_ip_pool_id = $poolId; + $row->updated_at = $now; + $row->save(); + } + } - foreach ($available as $address => $prefixLength) { + foreach ($observed as $address => $identity) { $row = $byAddress[$address] ?? null; if ($row === null) { $row = new CloudPublicIp(); $row->host_id = $hostId; + $row->public_ip_pool_id = $identity['public_ip_pool_id']; $row->address = $address; - $row->prefix_length = $prefixLength; - $row->state = 'available'; + $row->prefix_length = $identity['prefix_length']; + $row->state = $identity['allocatable'] + ? 'available' : 'disabled'; $row->reservation_id = null; $row->instance_id = null; $row->operation_id = null; @@ -313,6 +566,14 @@ final class CloudHostInventorySyncService continue; } + if ($row->public_ip_pool_id !== null + && (int) $row->public_ip_pool_id + !== (int) $identity['public_ip_pool_id'] + && in_array((string) $row->state, + self::LOCAL_IP_HOLDS, true)) { + throw new \DomainException('held public IP changed pools'); + } + // An engine free-list is advisory. A PHP hold, assignment, // change or uncertain operation must remain unavailable. if (in_array((string) $row->state, @@ -320,19 +581,32 @@ final class CloudHostInventorySyncService && $row->reservation_id === null && $row->instance_id === null && $row->operation_id === null) { - $row->prefix_length = $prefixLength; - $row->state = 'available'; + $row->public_ip_pool_id = $identity['public_ip_pool_id']; + $row->prefix_length = $identity['prefix_length']; + $row->state = $identity['allocatable'] + ? 'available' : 'disabled'; + $row->updated_at = $now; + $row->save(); + } elseif ($row->public_ip_pool_id === null) { + // A legacy local hold may be associated without + // changing its stronger occupancy state. + $row->public_ip_pool_id = $identity['public_ip_pool_id']; $row->updated_at = $now; $row->save(); } } - $missing = CloudPublicIp::where('host_id', $hostId) - ->where('state', 'available'); - if ($addresses !== []) { - $missing->whereNotIn('address', $addresses); + foreach ($ipRows as $row) { + $poolObserved = $row->public_ip_pool_id !== null + && isset($observedPoolKeyById[(int) $row->public_ip_pool_id]); + if ($poolObserved + && !isset($observed[(string) $row->address]) + && (string) $row->state === 'available') { + $row->state = 'disabled'; + $row->updated_at = $now; + $row->save(); + } } - $missing->update(['state' => 'disabled', 'updated_at' => $now]); foreach ($capacity as $field => $value) { $snapshot->{$field} = $value; @@ -348,6 +622,7 @@ final class CloudHostInventorySyncService $host->last_seen_at = $now; $host->health_checked_at = $now; $host->health_error_code = ''; + CloudHostPoolMembershipService::mirrorLocked($host, $membership); $host->updated_at = $now; $host->save(); return true; @@ -435,36 +710,82 @@ final class CloudHostInventorySyncService if (!is_array($public) || !is_bool($public['configured'] ?? null)) { throw new \UnexpectedValueException('invalid public IP capacity'); } - $publicCounts = []; - foreach (['total', 'reserved', 'allocated', 'available'] as $field) { - $publicCounts[$field] = self::integer( - $public[$field] ?? null, self::UINT32_MAX - ); - } + $publicCounts = self::counts($public, [ + 'total' => self::MAX_PUBLIC_IPS_PER_HOST, + 'reserved' => self::MAX_PUBLIC_IPS_PER_HOST, + 'allocated' => self::MAX_PUBLIC_IPS_PER_HOST, + 'available' => self::MAX_PUBLIC_IPS_PER_HOST, + ]); if ($publicCounts['total'] !== $publicCounts['reserved'] - + $publicCounts['allocated'] + $publicCounts['available'] - || (!$public['configured'] && array_sum($publicCounts) !== 0)) { + + $publicCounts['allocated'] + $publicCounts['available']) { throw new \UnexpectedValueException('inconsistent public IP capacity'); } + $pools = null; + $compatibilityMode = null; + if (array_key_exists('pools', $public)) { + if (!is_array($public['pools']) || !array_is_list($public['pools']) + || count($public['pools']) > self::MAX_PUBLIC_POOLS + || !is_bool($public['compatibility_mode'] ?? null)) { + throw new \UnexpectedValueException('invalid public pool capacity'); + } + $compatibilityMode = $public['compatibility_mode']; + $pools = []; + $aggregate = ['total' => 0, 'reserved' => 0, + 'allocated' => 0, 'available' => 0]; + foreach ($public['pools'] as $rawPool) { + $pool = self::capacityPool($rawPool); + if (isset($pools[$pool['pool_key']])) { + throw new \UnexpectedValueException('duplicate public pool key'); + } + $pools[$pool['pool_key']] = $pool; + foreach ($aggregate as $field => $value) { + $aggregate[$field] += $pool[$field]; + if ($aggregate[$field] > self::MAX_PUBLIC_IPS_PER_HOST) { + throw new \UnexpectedValueException( + 'public pool capacity exceeds host bound' + ); + } + } + } + self::validateCompatibilityMode($pools, $compatibilityMode); + if ($aggregate !== $publicCounts + || (bool) $public['configured'] !== ($pools !== [])) { + throw new \UnexpectedValueException( + 'public capacity does not match pool catalog' + ); + } + } elseif (array_key_exists('compatibility_mode', $public) + || (!(bool) $public['configured'] && array_sum($publicCounts) !== 0)) { + throw new \UnexpectedValueException('invalid legacy public capacity'); + } + return [ - 'cpu_total_cores' => $cpu['total'], - 'cpu_allocated_cores' => $cpu['allocated'], - 'cpu_available_cores' => $cpu['available'], - 'memory_total_mib' => $memory['total'], - 'memory_allocated_mib' => $memory['allocated'], - 'memory_available_mib' => $memory['available'], - 'storage_total_bytes' => $storage['total'], - 'storage_allocated_bytes' => $storage['allocated'], - 'storage_available_bytes' => $storage['available'], - 'instance_total_count' => $instances['total'], - 'instance_running_count' => $instances['running'], - 'public_ip_configured' => $public['configured'], - 'public_ip_total_count' => $publicCounts['total'], - 'public_ip_reserved_count' => $publicCounts['reserved'], - 'public_ip_allocated_count' => $publicCounts['allocated'], - 'public_ip_available_count' => $publicCounts['available'], - 'observed_at' => $observedAt, + 'snapshot' => [ + 'cpu_total_cores' => $cpu['total'], + 'cpu_allocated_cores' => $cpu['allocated'], + 'cpu_available_cores' => $cpu['available'], + 'memory_total_mib' => $memory['total'], + 'memory_allocated_mib' => $memory['allocated'], + 'memory_available_mib' => $memory['available'], + 'storage_total_bytes' => $storage['total'], + 'storage_allocated_bytes' => $storage['allocated'], + 'storage_available_bytes' => $storage['available'], + 'instance_total_count' => $instances['total'], + 'instance_running_count' => $instances['running'], + 'public_ip_configured' => (bool) $public['configured'], + 'public_ip_total_count' => $publicCounts['total'], + 'public_ip_reserved_count' => $publicCounts['reserved'], + 'public_ip_allocated_count' => $publicCounts['allocated'], + 'public_ip_available_count' => $publicCounts['available'], + 'observed_at' => $observedAt, + ], + 'public' => [ + 'configured' => (bool) $public['configured'], + 'counts' => $publicCounts, + 'pools' => $pools, + 'compatibility_mode' => $compatibilityMode, + ], ]; } @@ -482,55 +803,453 @@ final class CloudHostInventorySyncService || !array_is_list($payload['dns_servers'])) { throw new \UnexpectedValueException('invalid network response'); } - $availableCount = self::integer( - $payload['available_count'] ?? null, 256 - ); - if ($state === 'not_configured') { - if (($payload['pool_cidr'] ?? null) !== '' - || ($payload['guest_prefix'] ?? false) !== null - || ($payload['gateway'] ?? null) !== '' - || $payload['dns_servers'] !== [] - || $payload['available_ips'] !== [] - || $availableCount !== 0) { - throw new \UnexpectedValueException('invalid empty network response'); + $top = $state === 'ready' + ? self::networkProjection($payload) + : self::emptyNetworkProjection($payload); + $pools = null; + $compatibilityMode = null; + if (array_key_exists('pools', $payload)) { + if (!is_array($payload['pools']) || !array_is_list($payload['pools']) + || count($payload['pools']) > self::MAX_PUBLIC_POOLS + || !is_bool($payload['compatibility_mode'] ?? null)) { + throw new \UnexpectedValueException('invalid public pool catalog'); + } + $compatibilityMode = $payload['compatibility_mode']; + $pools = []; + $networks = []; + $allAvailable = []; + foreach ($payload['pools'] as $rawPool) { + $pool = self::networkPool($rawPool); + $poolKey = $pool['pool_key']; + if (isset($pools[$poolKey])) { + throw new \UnexpectedValueException('duplicate public pool key'); + } + foreach ($networks as $network) { + if (self::networksOverlap($pool['network'], $network)) { + throw new \UnexpectedValueException( + 'overlapping public pool CIDRs' + ); + } + } + foreach ($pool['available'] as $address => $prefix) { + if (isset($allAvailable[$address])) { + throw new \UnexpectedValueException( + 'public IP appears in multiple pools' + ); + } + $allAvailable[$address] = true; + } + if (count($allAvailable) > self::MAX_PUBLIC_IPS_PER_HOST) { + throw new \UnexpectedValueException( + 'public address catalog exceeds host bound' + ); + } + $pools[$poolKey] = $pool; + $networks[] = $pool['network']; + } + self::validateCompatibilityMode($pools, $compatibilityMode); + + $primary = null; + foreach ($pools as $pool) { + if ($pool['state'] === 'active') { + $primary = $pool; + break; + } + } + if (($state === 'ready') !== ($primary !== null)) { + throw new \UnexpectedValueException( + 'network state does not match active pools' + ); + } + if ($primary !== null + && ($top['cidr'] !== $primary['cidr'] + || $top['guest_prefix'] !== $primary['guest_prefix'] + || $top['gateway'] !== $primary['gateway'] + || $top['dns_servers'] !== $primary['dns_servers'] + || $top['available'] !== $primary['available'])) { + throw new \UnexpectedValueException( + 'network projection does not match primary pool' + ); + } + } elseif (array_key_exists('compatibility_mode', $payload)) { + throw new \UnexpectedValueException('invalid legacy network response'); + } + + return [ + 'configured' => $state === 'ready', + 'top' => $top, + 'pools' => $pools, + 'compatibility_mode' => $compatibilityMode, + ]; + } + + /** Reconcile two independently fetched engine views into one observation. */ + private static function reconcilePublicInventory(array $capacity, + array $network): array + { + $capacityPools = $capacity['public']['pools']; + $networkPools = $network['pools']; + if (($capacityPools === null) !== ($networkPools === null)) { + throw new \UnexpectedValueException('mixed public pool contracts'); + } + + if ($capacityPools === null) { + if ((bool) $capacity['public']['configured'] !== $network['configured'] + || $capacity['public']['counts']['available'] + !== count($network['top']['available'])) { + throw new \UnexpectedValueException( + 'legacy public inventory endpoints disagree' + ); + } + if (!$network['configured']) { + return [$capacity['snapshot'], []]; + } + $counts = $capacity['public']['counts']; + return [$capacity['snapshot'], [self::LEGACY_POOL_KEY => [ + 'pool_key' => self::LEGACY_POOL_KEY, + 'revision' => 1, + 'state' => 'active', + 'network_selector' => '', + 'cidr' => $network['top']['cidr'], + 'network' => $network['top']['network'], + 'gateway' => $network['top']['gateway'], + 'dns_servers' => $network['top']['dns_servers'], + 'reserved_ips' => [], + 'reserved_ips_known' => false, + 'guest_prefix' => $network['top']['guest_prefix'], + 'usable_count' => $counts['total'], + 'reserved_count' => $counts['reserved'], + 'allocated_count' => $counts['allocated'], + 'available_count' => $counts['available'], + 'available' => $network['top']['available'], + ]]]; + } + + if ($capacity['public']['compatibility_mode'] + !== $network['compatibility_mode'] + || count($capacityPools) !== count($networkPools) + || array_diff_key($capacityPools, $networkPools) !== [] + || array_diff_key($networkPools, $capacityPools) !== []) { + throw new \UnexpectedValueException('public pool catalogs disagree'); + } + foreach ($networkPools as $poolKey => $networkPool) { + $capacityPool = $capacityPools[$poolKey]; + if ($capacityPool['revision'] !== $networkPool['revision'] + || $capacityPool['state'] !== $networkPool['state'] + || $capacityPool['network_selector'] + !== $networkPool['network_selector'] + || $capacityPool['total'] !== $networkPool['usable_count'] + || $capacityPool['reserved'] !== $networkPool['reserved_count'] + || $capacityPool['allocated'] !== $networkPool['allocated_count'] + || $capacityPool['available'] !== $networkPool['available_count']) { + throw new \UnexpectedValueException( + 'public pool observations disagree' + ); } - return ['configured' => false, 'available' => []]; } + return [$capacity['snapshot'], $networkPools]; + } + + private static function capacityPool(mixed $value): array + { + if (!is_array($value) || array_is_list($value)) { + throw new \UnexpectedValueException('invalid capacity pool'); + } + $counts = self::counts($value, [ + 'total' => self::MAX_PUBLIC_IPS_PER_POOL, + 'reserved' => self::MAX_PUBLIC_IPS_PER_POOL, + 'allocated' => self::MAX_PUBLIC_IPS_PER_POOL, + 'available' => self::MAX_PUBLIC_IPS_PER_POOL, + ]); + if ($counts['total'] !== $counts['reserved'] + + $counts['allocated'] + $counts['available']) { + throw new \UnexpectedValueException('inconsistent capacity pool'); + } + return [ + 'pool_key' => self::poolKey($value['pool_key'] ?? null), + 'revision' => self::revision($value['revision'] ?? null), + 'state' => self::poolState($value['state'] ?? null), + 'network_selector' => self::networkSelector( + $value['network_selector'] ?? null + ), + ...$counts, + ]; + } + + private static function networkPool(mixed $value): array + { + if (!is_array($value) || array_is_list($value) + || ($value['configuration_state'] ?? null) !== 'ready' + || !is_array($value['reserved_ips'] ?? null) + || !array_is_list($value['reserved_ips'])) { + throw new \UnexpectedValueException('invalid network pool'); + } + $projection = self::networkProjection($value); + $reserved = []; + foreach ($value['reserved_ips'] as $rawAddress) { + $address = self::ipv4($rawAddress); + if (!self::inCidr($address, $projection['network']['address'], + $projection['network']['prefix']) + || self::isBoundary($address, $projection['network']['address'], + $projection['network']['prefix']) + || isset($reserved[$address])) { + throw new \UnexpectedValueException('invalid reserved public IP'); + } + $reserved[$address] = true; + } + if (count($reserved) > self::MAX_PUBLIC_IPS_PER_POOL) { + throw new \UnexpectedValueException('reserved public IP list too large'); + } + foreach ($projection['available'] as $address => $prefix) { + if (isset($reserved[$address])) { + throw new \UnexpectedValueException( + 'reserved public IP appears as available' + ); + } + } + + $counts = self::counts($value, [ + 'usable_count' => self::MAX_PUBLIC_IPS_PER_POOL, + 'reserved_count' => self::MAX_PUBLIC_IPS_PER_POOL, + 'allocated_count' => self::MAX_PUBLIC_IPS_PER_POOL, + 'available_count' => self::MAX_PUBLIC_IPS_PER_POOL, + ]); + $expectedUsable = (1 << (32 - $projection['network']['prefix'])) - 2; + if ($counts['usable_count'] !== $expectedUsable + || $counts['usable_count'] !== $counts['reserved_count'] + + $counts['allocated_count'] + $counts['available_count'] + || $counts['available_count'] !== count($projection['available'])) { + throw new \UnexpectedValueException('inconsistent network pool counts'); + } + + return [ + 'pool_key' => self::poolKey($value['pool_key'] ?? null), + 'revision' => self::revision($value['revision'] ?? null), + 'state' => self::poolState($value['state'] ?? null), + 'network_selector' => self::networkSelector( + $value['network_selector'] ?? null + ), + 'cidr' => $projection['cidr'], + 'network' => $projection['network'], + 'gateway' => $projection['gateway'], + 'dns_servers' => $projection['dns_servers'], + 'reserved_ips' => array_keys($reserved), + 'reserved_ips_known' => true, + 'guest_prefix' => $projection['guest_prefix'], + 'usable_count' => $counts['usable_count'], + 'reserved_count' => $counts['reserved_count'], + 'allocated_count' => $counts['allocated_count'], + 'available_count' => $counts['available_count'], + 'available' => $projection['available'], + ]; + } - $pool = self::cidr($payload['pool_cidr'] ?? null, true); - $guestPrefix = self::integer($payload['guest_prefix'] ?? null, 32); - if ($pool['prefix'] < 24 || $pool['prefix'] > 30 - || $guestPrefix < 24 || $guestPrefix > $pool['prefix']) { + private static function networkProjection(array $value): array + { + if (!is_array($value['available_ips'] ?? null) + || !array_is_list($value['available_ips']) + || count($value['available_ips']) > self::MAX_PUBLIC_IPS_PER_POOL + || !is_array($value['dns_servers'] ?? null) + || !array_is_list($value['dns_servers'])) { + throw new \UnexpectedValueException('invalid network projection'); + } + $network = self::cidr($value['pool_cidr'] ?? null, true); + $guestPrefix = self::integer($value['guest_prefix'] ?? null, 32); + if ($network['prefix'] < 24 || $network['prefix'] > 30 + || $guestPrefix < 24 || $guestPrefix > $network['prefix']) { throw new \UnexpectedValueException('invalid public network prefix'); } - $gateway = self::ipv4($payload['gateway'] ?? null); - if (!self::inCidr($gateway, $pool['address'], $pool['prefix'])) { - throw new \UnexpectedValueException('gateway outside public network'); + $gateway = self::ipv4($value['gateway'] ?? null); + if (!self::inCidr($gateway, $network['address'], $network['prefix']) + || self::isBoundary($gateway, $network['address'], $network['prefix'])) { + throw new \UnexpectedValueException('invalid public gateway'); } - if (count($payload['dns_servers']) < 1 - || count($payload['dns_servers']) > 4) { - throw new \UnexpectedValueException('invalid DNS list'); + + $dnsServers = []; + foreach ($value['dns_servers'] as $rawServer) { + $server = self::ipv4($rawServer); + if (isset($dnsServers[$server])) { + throw new \UnexpectedValueException('duplicate DNS server'); + } + $dnsServers[$server] = true; } - foreach ($payload['dns_servers'] as $server) { - self::ipv4($server); + if (count($dnsServers) < 1 || count($dnsServers) > 4) { + throw new \UnexpectedValueException('invalid DNS list'); } $available = []; - foreach ($payload['available_ips'] as $value) { - $item = self::cidr($value, false); + foreach ($value['available_ips'] as $rawAddress) { + $item = self::cidr($rawAddress, false); if ($item['prefix'] !== $guestPrefix - || !self::inCidr($item['address'], $pool['address'], $pool['prefix']) - || self::isBoundary($item['address'], $pool['address'], $pool['prefix']) + || !self::inCidr($item['address'], $network['address'], + $network['prefix']) + || self::isBoundary($item['address'], $network['address'], + $network['prefix']) + || $item['address'] === $gateway + || isset($dnsServers[$item['address']]) || isset($available[$item['address']])) { throw new \UnexpectedValueException('invalid available public IP'); } $available[$item['address']] = $guestPrefix; } + $availableCount = self::integer( + $value['available_count'] ?? null, self::MAX_PUBLIC_IPS_PER_POOL + ); if ($availableCount !== count($available)) { - throw new \UnexpectedValueException('inconsistent available public IP count'); + throw new \UnexpectedValueException( + 'inconsistent available public IP count' + ); + } + return [ + 'cidr' => $network['address'] . '/' . $network['prefix'], + 'network' => $network, + 'guest_prefix' => $guestPrefix, + 'gateway' => $gateway, + 'dns_servers' => array_keys($dnsServers), + 'available' => $available, + ]; + } + + private static function emptyNetworkProjection(array $payload): array + { + $availableCount = self::integer( + $payload['available_count'] ?? null, self::MAX_PUBLIC_IPS_PER_POOL + ); + if (($payload['pool_cidr'] ?? null) !== '' + || ($payload['guest_prefix'] ?? false) !== null + || ($payload['gateway'] ?? null) !== '' + || $payload['dns_servers'] !== [] + || $payload['available_ips'] !== [] + || $availableCount !== 0) { + throw new \UnexpectedValueException('invalid empty network response'); + } + return [ + 'cidr' => '', + 'network' => null, + 'guest_prefix' => null, + 'gateway' => '', + 'dns_servers' => [], + 'available' => [], + ]; + } + + private static function counts(array $value, array $limits): array + { + $counts = []; + foreach ($limits as $field => $maximum) { + $counts[$field] = self::integer($value[$field] ?? null, $maximum); + } + return $counts; + } + + private static function poolKey(mixed $value): string + { + if (!is_string($value) + || ($value !== self::LEGACY_POOL_KEY + && preg_match('/^[0-9a-f]{32}$/D', $value) !== 1)) { + throw new \UnexpectedValueException('invalid public pool key'); } - return ['configured' => true, 'available' => $available]; + return $value; + } + + private static function revision(mixed $value): int + { + $revision = self::integer($value, PHP_INT_MAX); + if ($revision < 1) { + throw new \UnexpectedValueException('invalid public pool revision'); + } + return $revision; + } + + private static function poolState(mixed $value): string + { + if (!is_string($value) + || !in_array($value, self::PUBLIC_POOL_STATES, true)) { + throw new \UnexpectedValueException('invalid public pool state'); + } + return $value; + } + + private static function networkSelector(mixed $value): string + { + if (!is_string($value) || strlen($value) > 128) { + throw new \UnexpectedValueException('invalid network selector'); + } + if (str_starts_with($value, 'bridge:')) { + $name = substr($value, 7); + if (preg_match('/^[A-Za-z0-9][A-Za-z0-9_.-]{0,14}$/D', $name) !== 1) { + throw new \UnexpectedValueException('invalid bridge selector'); + } + return $value; + } + if (preg_match('/^[A-Za-z0-9_.-]{1,128}$/D', $value) !== 1 + || str_starts_with($value, '-') || $value === '.' || $value === '..') { + throw new \UnexpectedValueException('invalid network selector'); + } + return $value; + } + + private static function validateCompatibilityMode(array $pools, + bool $compatibilityMode): void + { + $legacy = isset($pools[self::LEGACY_POOL_KEY]); + if (($compatibilityMode && (!$legacy || count($pools) !== 1)) + || (!$compatibilityMode && $legacy)) { + throw new \UnexpectedValueException( + 'public pool compatibility mode is inconsistent' + ); + } + } + + private static function networksOverlap(array $left, array $right): bool + { + return self::inCidr($left['address'], $right['address'], $right['prefix']) + || self::inCidr($right['address'], $left['address'], $left['prefix']); + } + + private static function attachmentKey(string $networkSelector): string + { + return substr(hash('sha256', $networkSelector), 0, 32); + } + + private static function attachmentState(array $pools, + string $networkSelector): string + { + $state = 'disabled'; + foreach ($pools as $pool) { + if ((string) $pool['network_selector'] !== $networkSelector) { + continue; + } + if ((string) $pool['state'] === 'active') { + return 'active'; + } + if ((string) $pool['state'] === 'draining') { + $state = 'draining'; + } + } + return $state; + } + + private static function poolKeyForAddress(string $address, array $pools): ?string + { + if (filter_var($address, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) === false) { + return null; + } + $match = null; + foreach ($pools as $poolKey => $pool) { + if (!self::inCidr($address, $pool['network']['address'], + $pool['network']['prefix'])) { + continue; + } + if ($match !== null) { + throw new \UnexpectedValueException( + 'public IP belongs to multiple pool CIDRs' + ); + } + $match = $poolKey; + } + return $match; } private static function group(array $payload, string $name, array $limits): array @@ -652,6 +1371,8 @@ final class CloudHostInventorySyncService try { $schema = (new CloudHost())->getConnection()->getSchemaBuilder(); foreach (['cloud_hosts', 'cloud_host_capacity_snapshots', + 'cloud_host_resource_pool_memberships', 'cloud_resource_pools', + 'cloud_host_network_attachments', 'cloud_public_ip_pools', 'cloud_public_ips'] as $table) { if (!$schema->hasTable($table)) { return false; @@ -663,6 +1384,9 @@ final class CloudHostInventorySyncService return false; } } + if (!$schema->hasColumn('cloud_public_ips', 'public_ip_pool_id')) { + return false; + } return true; } catch (\Throwable $error) { return false; diff --git a/web/app/Services/Cloud/CloudHostPoolMembershipService.php b/web/app/Services/Cloud/CloudHostPoolMembershipService.php new file mode 100644 index 00000000..4fe2c475 --- /dev/null +++ b/web/app/Services/Cloud/CloudHostPoolMembershipService.php @@ -0,0 +1,158 @@ +id) + ->orderBy('resource_pool_id')->orderBy('id')->lockForUpdate()->get(); + if ($rows->isEmpty() && $host->resource_pool_id !== null) { + $row = new CloudHostResourcePoolMembership(); + $row->host_id = (int) $host->id; + $row->resource_pool_id = (int) $host->resource_pool_id; + $row->scheduler_state = self::state((string) $host->scheduler_state); + $row->priority = self::SAFE_PRIORITY; + $row->weight = self::SAFE_WEIGHT; + $row->created_at = $row->updated_at = $now; + $row->save(); + $rows->push($row); + } + return $rows; + } + + /** Select the first-release placement relation deterministically. */ + public static function selected(CloudHost $host, $rows): ?CloudHostResourcePoolMembership + { + $active = null; + $draining = null; + $latest = null; + $activeCount = 0; + foreach ($rows as $row) { + if ((string) $row->scheduler_state === 'active') { + $activeCount++; + $active = $row; + } + if ((string) $row->scheduler_state === 'draining' + && ($draining === null || self::newer($row, $draining))) { + $draining = $row; + } + if ($latest === null || self::newer($row, $latest)) { + $latest = $row; + } + } + if ($activeCount > 1) { + throw new CloudError('宿主机存在多个启用中的资源池成员关系,请先修复调度配置', 409); + } + return $active ?? $draining ?? $latest; + } + + /** + * Replace the first-release selected membership. Until the scheduler can + * express several memberships in its write API, one host has zero or one + * authoritative row; the relation schema itself remains ready for a later + * multi-pool rollout. + */ + public static function replaceLocked(CloudHost $host, ?int $poolId, + string $schedulerState, int $now): ?CloudHostResourcePoolMembership + { + if (!in_array($schedulerState, self::STATES, true)) { + throw new CloudError('调度状态无效'); + } + if ($poolId === null && $schedulerState !== 'disabled') { + throw new CloudError('未选择资源池时只能停用调度', 409); + } + + $rows = self::lockedRows($host, $now); + $target = null; + foreach ($rows as $row) { + if ($poolId !== null && (int) $row->resource_pool_id === $poolId) { + $target = $row; + } + } + if ($poolId !== null && $target === null) { + $target = new CloudHostResourcePoolMembership(); + $target->host_id = (int) $host->id; + $target->resource_pool_id = $poolId; + $target->created_at = $now; + $rows->push($target); + } + + foreach ($rows as $row) { + if ($target !== null + && (int) $row->resource_pool_id === (int) $target->resource_pool_id) { + continue; + } + $row->delete(); + } + if ($target !== null) { + $target->scheduler_state = $schedulerState; + $target->priority = self::SAFE_PRIORITY; + $target->weight = self::SAFE_WEIGHT; + $target->updated_at = $now; + $target->save(); + self::assertSingleActive([$target]); + } + + // Temporary mirror only. Membership rows above are the authority. + $host->resource_pool_id = $target?->resource_pool_id; + $host->scheduler_state = $target === null ? 'disabled' : $schedulerState; + return $target; + } + + /** Heal an old scalar projection after a successful relation backfill. */ + public static function mirrorLocked(CloudHost $host, + ?CloudHostResourcePoolMembership $selected): void + { + $host->resource_pool_id = $selected?->resource_pool_id; + $host->scheduler_state = $selected === null + ? 'disabled' : self::state((string) $selected->scheduler_state); + } + + private static function assertSingleActive($rows): void + { + $active = 0; + foreach ($rows as $row) { + if ((string) $row->scheduler_state === 'active') { + $active++; + } + } + if ($active > 1) { + throw new CloudError('宿主机存在多个启用中的资源池成员关系,请先修复调度配置', 409); + } + } + + private static function state(string $state): string + { + return in_array($state, self::STATES, true) ? $state : 'disabled'; + } + + private static function newer($left, $right): bool + { + return (int) $left->updated_at > (int) $right->updated_at + || ((int) $left->updated_at === (int) $right->updated_at + && (int) $left->id > (int) $right->id); + } +} diff --git a/web/sql/cloud.sql b/web/sql/cloud.sql index 85533cf1..3694c12f 100644 --- a/web/sql/cloud.sql +++ b/web/sql/cloud.sql @@ -129,6 +129,20 @@ CREATE TABLE IF NOT EXISTS cloud_host_resource_pool_memberships ( KEY cloud_host_membership_scheduler (host_id, scheduler_state, id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +-- One-time compatibility import for hosts configured before membership rows +-- existed. Once any relation is present for a host, that relation is the +-- authority and rerunning cloudInit must never recreate a stale scalar link. +INSERT IGNORE INTO cloud_host_resource_pool_memberships + (host_id, resource_pool_id, scheduler_state, priority, weight, created_at, updated_at) +SELECT h.id, h.resource_pool_id, + CASE WHEN h.scheduler_state IN ('active', 'draining', 'disabled') + THEN h.scheduler_state ELSE 'disabled' END, + 0, 100, GREATEST(0, h.created_at), GREATEST(0, h.updated_at) +FROM cloud_hosts AS h +LEFT JOIN cloud_host_resource_pool_memberships AS membership + ON membership.host_id = h.id +WHERE h.resource_pool_id IS NOT NULL AND membership.id IS NULL; + -- One attachment is one real host egress such as bridge:br0. Several CIDR -- blocks may share the same attachment. Binding the attachment to a network -- line keeps physical egress separate from compute capacity and lets a future diff --git a/web/tests/AdminHostSchedulingContractTest.php b/web/tests/AdminHostSchedulingContractTest.php index ca63e018..e0cd7d88 100644 --- a/web/tests/AdminHostSchedulingContractTest.php +++ b/web/tests/AdminHostSchedulingContractTest.php @@ -15,6 +15,9 @@ $assert = static function (bool $condition, string $message): void { }; $service = $read('app/Services/Cloud/CloudAdminHostSchedulingService.php'); +$membershipService = $read( + 'app/Services/Cloud/CloudHostPoolMembershipService.php' +); $controller = $read('app/Controllers/CloudAdminController.php'); $routes = $read('config/routes_cloud.php'); $middleware = $read('app/Middleware/CloudApi.php'); @@ -36,11 +39,30 @@ $assert(str_contains($controller, "'resource_pool_id', 'scheduler_state'") - strpos($controller, 'public function hostScheduling')), 'public_ip'), 'The scheduling allowlist must not accept manual public IP fields.'); -$hostLock = strpos($service, "CloudHost::where('id', \$hostId)->lockForUpdate()"); -$policyLock = strpos($service, "CloudHostCapacityPolicy::where('host_id', \$hostId)"); -$assert($hostLock !== false && $policyLock !== false && $hostLock < $policyLock +$saveStart = strpos($service, 'public static function save'); +$syncStart = strpos($service, 'public static function sync', $saveStart); +$save = $saveStart === false || $syncStart === false ? '' + : substr($service, $saveStart, $syncStart - $saveStart); +$hostLock = strpos($save, "CloudHost::where('id', \$hostId)->lockForUpdate()"); +$membershipLock = strpos($save, 'CloudHostPoolMembershipService::lockedRows'); +$policyLock = strpos($save, "CloudHostCapacityPolicy::where('host_id', \$hostId)"); +$assert($save !== '' && $hostLock !== false && $membershipLock !== false + && $policyLock !== false && $hostLock < $membershipLock + && $membershipLock < $policyLock && str_contains($service, '->lockForUpdate()->first();'), - 'Scheduling writes must lock host before capacity policy.'); + 'Scheduling writes must lock host, membership and capacity policy in order.'); +$assert(str_contains($service, 'CloudHostPoolMembershipService::replaceLocked(') + && str_contains($service, "'memberships' => \$memberships->map") + && str_contains($membershipService, + 'CloudHostResourcePoolMembership::where(\'host_id\'') + && str_contains($membershipService, 'if ($rows->isEmpty()' + . ' && $host->resource_pool_id !== null)') + && str_contains($membershipService, 'public const SAFE_PRIORITY = 0;') + && str_contains($membershipService, 'public const SAFE_WEIGHT = 100;') + && str_contains($membershipService, '$row->delete();') + && str_contains($membershipService, + '宿主机存在多个启用中的资源池成员关系'), + 'Membership authority, legacy backfill or first-release single-row policy is missing.'); $assert(str_contains($service, "\$schedulerState === 'active'") && str_contains($service, 'CloudNetworkLineFeature::enabled()') && str_contains($service, "(string) \$pool->status !== 'active'") diff --git a/web/tests/CloudHostInventorySyncContractTest.php b/web/tests/CloudHostInventorySyncContractTest.php index 12c8f0f6..b31bb502 100644 --- a/web/tests/CloudHostInventorySyncContractTest.php +++ b/web/tests/CloudHostInventorySyncContractTest.php @@ -21,8 +21,10 @@ $assert = static function ($condition, string $message): void { $assert(str_contains($service, "'GET', 'host/capacity'") && str_contains($service, "'GET', 'host/network-options'") && str_contains($service, 'public_ip_available_count') - && str_contains($service, "->where('state', 'available')") && str_contains($service, "['available', 'disabled']") + && str_contains($service, 'self::LOCAL_IP_HOLDS') + && str_contains($service, '$poolObserved') + && str_contains($service, 'public_ip_pool_id') && str_contains($service, "['degraded', self::ERROR_INVALID_NETWORK]") && str_contains($service, "['unavailable', self::ERROR_UNAVAILABLE]") && str_contains($service, '$attemptedAt = $this->now();') @@ -34,7 +36,38 @@ $assert(str_contains($service, "'GET', 'host/capacity'") && substr_count($service, "\$host->last_seen_at =") === 1, 'Capacity/IP observation, local-state preservation or last-seen boundary changed.'); +$assert(str_contains($service, 'private const MAX_PUBLIC_POOLS = 32;') + && str_contains($service, 'private const MAX_PUBLIC_IPS_PER_POOL = 254;') + && str_contains($service, 'private const MAX_PUBLIC_IPS_PER_HOST = 8128;') + && str_contains($service, 'private static function reconcilePublicInventory') + && str_contains($service, 'private static function networksOverlap') + && str_contains($service, "'reserved_ips_known' => true") + && str_contains($service, '$publicPool->reserved_ips_json = json_encode(') + && str_contains($service, "preg_match('/^[0-9a-f]{32}$/D'") + && str_contains($service, 'public capacity does not match pool catalog') + && str_contains($service, 'public pool observations disagree'), + 'Bounded multi-pool parsing, identity or cross-endpoint count checks are missing.'); + +$applyStart = strpos($service, 'private function apply'); $failureStart = strpos($service, 'private function markFailure'); +$apply = $applyStart === false || $failureStart === false ? '' + : substr($service, $applyStart, $failureStart - $applyStart); +$hostLock = strpos($apply, "CloudHost::where('id', \$hostId)->lockForUpdate()"); +$snapshotLock = strpos($apply, + "CloudHostCapacitySnapshot::where('host_id', \$hostId)"); +$membershipLock = strpos($apply, 'CloudHostPoolMembershipService::lockedRows'); +$attachmentLock = strpos($apply, + "CloudHostNetworkAttachment::where('host_id', \$hostId)"); +$poolLock = strpos($apply, "CloudPublicIpPool::where('host_id', \$hostId)"); +$ipLock = strpos($apply, "CloudPublicIp::where('host_id', \$hostId)"); +$assert($apply !== '' && $hostLock !== false && $snapshotLock !== false + && $membershipLock !== false && $attachmentLock !== false + && $poolLock !== false && $ipLock !== false + && $hostLock < $snapshotLock && $snapshotLock < $membershipLock + && $membershipLock < $attachmentLock && $attachmentLock < $poolLock + && $poolLock < $ipLock, + 'Inventory lock order must remain host, snapshot fence, membership, attachment, pool, IP.'); + $validationStart = strpos($service, 'private static function capacity', $failureStart); $failure = $failureStart === false || $validationStart === false ? '' : substr($service, $failureStart, $validationStart - $failureStart); @@ -42,8 +75,10 @@ $assert($failure !== '' && str_contains($failure, "\$snapshot->sync_state = 'failed';") && str_contains($failure, "\$snapshot->last_error_code = \$code;") && !str_contains($failure, 'cpu_total_cores') - && !str_contains($failure, 'CloudPublicIp::'), - 'Failure handling may clear last-known capacity or mutate the IP library.'); + && !str_contains($failure, 'CloudPublicIp::') + && !str_contains($failure, 'CloudPublicIpPool::') + && !str_contains($failure, 'CloudHostNetworkAttachment::'), + 'Failure handling may clear last-known capacity or mutate pool/IP inventory.'); $assert(str_contains($actor, "Config::get('super_admin', '')") && str_contains($actor, "->where('enable', 1)->where('is_admin', 1)") @@ -60,6 +95,11 @@ $assert(str_contains($lifecycle, "busy('host_inventory_sync')") && str_contains($service, 'public function syncOne(int $hostId): array') && str_contains($service, "Config::get('cloud_host_inventory_interval_seconds', 30)") && str_contains($service, 'private const MAX_HOSTS_PER_PASS = 10;') + && str_contains($service, "'cloud_host_resource_pool_memberships'") + && str_contains($service, "'cloud_host_network_attachments'") + && str_contains($service, "'cloud_public_ip_pools'") + && str_contains($service, + "hasColumn('cloud_public_ips', 'public_ip_pool_id')") && str_contains($heartbeat, "'host_inventory_sync' => '宿主容量与公网 IP 同步'") && str_contains($heartbeat, "'host_inventory_sync_failed'") diff --git a/web/tests/CloudHostInventorySyncServiceTest.php b/web/tests/CloudHostInventorySyncServiceTest.php index 4483657c..4a207152 100644 --- a/web/tests/CloudHostInventorySyncServiceTest.php +++ b/web/tests/CloudHostInventorySyncServiceTest.php @@ -30,10 +30,15 @@ namespace { use App\Models\CloudHost; use App\Models\CloudHostCapacitySnapshot; + use App\Models\CloudHostNetworkAttachment; + use App\Models\CloudHostResourcePoolMembership; use App\Models\CloudPublicIp; + use App\Models\CloudPublicIpPool; + use App\Models\CloudResourcePool; use App\Models\User; use App\Services\Cloud\CloudEngineClient; use App\Services\Cloud\CloudHostInventorySyncService; + use App\Services\Cloud\CloudHostPoolMembershipService; use GuzzleHttp\Client; use GuzzleHttp\Handler\MockHandler; use GuzzleHttp\HandlerStack; @@ -50,7 +55,13 @@ namespace { $json = static fn (array $value): string => json_encode( $value, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES ); - $capacity = static function (int $observedAt, int $cpu = 16): array { + $poolA = str_repeat('a', 32); + $poolB = str_repeat('b', 32); + $missingPoolKey = str_repeat('c', 32); + + $capacity = static function (int $observedAt, int $cpu = 16) use ( + $poolA, $poolB + ): array { return [ 'health' => 'healthy', 'observed_at' => $observedAt, @@ -61,23 +72,85 @@ namespace { 'available' => 24576], 'storage_bytes' => ['total' => 1099511627776, 'allocated' => 274877906944, 'available' => 824633720832], - 'public_ipv4' => ['configured' => true, 'total' => 6, - 'reserved' => 1, 'allocated' => 2, 'available' => 3], + 'public_ipv4' => [ + 'configured' => true, + 'total' => 8, + 'reserved' => 3, + 'allocated' => 1, + 'available' => 4, + 'compatibility_mode' => false, + 'pools' => [[ + 'pool_key' => $poolA, + 'revision' => 4, + 'state' => 'active', + 'network_selector' => 'default', + 'total' => 6, + 'reserved' => 2, + 'allocated' => 1, + 'available' => 3, + ], [ + 'pool_key' => $poolB, + 'revision' => 2, + 'state' => 'draining', + 'network_selector' => 'default', + 'total' => 2, + 'reserved' => 1, + 'allocated' => 0, + 'available' => 1, + ]], + ], ]; }; - $network = static function (int $count = 3): array { - return [ + $network = static function () use ($poolA, $poolB): array { + $first = [ 'configuration_state' => 'ready', + 'pool_key' => $poolA, + 'revision' => 4, + 'state' => 'active', + 'network_selector' => 'default', 'pool_cidr' => '8.8.8.0/29', 'guest_prefix' => 29, 'gateway' => '8.8.8.1', 'dns_servers' => ['1.1.1.1'], + 'reserved_ips' => ['8.8.8.2'], + 'usable_count' => 6, + 'reserved_count' => 2, + 'allocated_count' => 1, 'available_ips' => ['8.8.8.3/29', '8.8.8.5/29', '8.8.8.6/29'], - 'available_count' => $count, + 'available_count' => 3, + ]; + $second = [ + 'configuration_state' => 'ready', + 'pool_key' => $poolB, + 'revision' => 2, + 'state' => 'draining', + 'network_selector' => 'default', + 'pool_cidr' => '9.9.9.0/30', + 'guest_prefix' => 30, + 'gateway' => '9.9.9.1', + 'dns_servers' => ['1.0.0.1'], + 'reserved_ips' => [], + 'usable_count' => 2, + 'reserved_count' => 1, + 'allocated_count' => 0, + 'available_ips' => ['9.9.9.2/30'], + 'available_count' => 1, + ]; + return [ + 'configuration_state' => 'ready', + 'pool_cidr' => $first['pool_cidr'], + 'guest_prefix' => $first['guest_prefix'], + 'gateway' => $first['gateway'], + 'dns_servers' => $first['dns_servers'], + 'available_ips' => $first['available_ips'], + 'available_count' => $first['available_count'], + 'pools' => [$first, $second], + 'compatibility_mode' => false, 'message' => 'validated test response', ]; }; - $client = static function (array $responses, ?array &$history = null): CloudEngineClient { + $client = static function (array $responses, + ?array &$history = null): CloudEngineClient { $stack = HandlerStack::create(new MockHandler($responses)); if ($history !== null) { $stack->push(Middleware::history($history)); @@ -95,17 +168,44 @@ namespace { $table->integer('enable')->default(1); $table->integer('is_admin')->default(0); }); + $schema->create('cloud_resource_pools', function (Blueprint $table): void { + $table->increments('id'); + $table->integer('line_id'); + $table->string('code'); + $table->string('name'); + $table->string('status')->default('active'); + $table->string('scheduling_policy')->default('balanced'); + $table->integer('sort')->default(0); + $table->integer('created_at'); + $table->integer('updated_at'); + }); $schema->create('cloud_hosts', function (Blueprint $table): void { $table->increments('id'); $table->string('name'); $table->integer('enabled')->default(0); + $table->integer('resource_pool_id')->nullable(); + $table->string('scheduler_state')->default('disabled'); $table->string('health_state')->default('unknown'); $table->integer('last_seen_at')->nullable(); $table->integer('health_checked_at')->nullable(); $table->string('health_error_code')->default(''); + $table->string('default_network')->default('default'); $table->integer('created_at'); $table->integer('updated_at'); }); + $schema->create('cloud_host_resource_pool_memberships', + function (Blueprint $table): void { + $table->increments('id'); + $table->integer('host_id'); + $table->integer('resource_pool_id'); + $table->string('scheduler_state')->default('disabled'); + $table->integer('priority')->default(0); + $table->integer('weight')->default(100); + $table->integer('created_at'); + $table->integer('updated_at'); + $table->unique(['host_id', 'resource_pool_id']); + } + ); $schema->create('cloud_host_capacity_snapshots', function (Blueprint $table): void { $table->integer('host_id')->primary(); foreach (['cpu_total_cores', 'cpu_allocated_cores', 'cpu_available_cores', @@ -124,9 +224,53 @@ namespace { $table->integer('created_at'); $table->integer('updated_at'); }); + $schema->create('cloud_host_network_attachments', + function (Blueprint $table): void { + $table->increments('id'); + $table->string('attachment_key'); + $table->integer('host_id'); + $table->integer('line_id'); + $table->string('name'); + $table->string('network_mode'); + $table->string('network_selector'); + $table->integer('vlan_id')->nullable(); + $table->integer('routing_table')->nullable(); + $table->string('state'); + $table->integer('priority'); + $table->integer('desired_revision'); + $table->integer('observed_revision'); + $table->string('sync_state'); + $table->string('last_error_code'); + $table->integer('created_at'); + $table->integer('updated_at'); + $table->unique(['host_id', 'attachment_key']); + } + ); + $schema->create('cloud_public_ip_pools', function (Blueprint $table): void { + $table->increments('id'); + $table->string('pool_key'); + $table->integer('host_id'); + $table->integer('network_attachment_id'); + $table->string('name'); + $table->string('cidr'); + $table->string('gateway'); + $table->integer('guest_prefix_length'); + $table->text('dns_servers_json'); + $table->text('reserved_ips_json'); + $table->string('state'); + $table->integer('priority'); + $table->integer('desired_revision'); + $table->integer('observed_revision'); + $table->string('sync_state'); + $table->string('last_error_code'); + $table->integer('created_at'); + $table->integer('updated_at'); + $table->unique(['host_id', 'pool_key']); + }); $schema->create('cloud_public_ips', function (Blueprint $table): void { $table->increments('id'); $table->integer('host_id'); + $table->integer('public_ip_pool_id')->nullable(); $table->string('address')->unique(); $table->integer('prefix_length'); $table->string('state')->default('available'); @@ -142,39 +286,93 @@ namespace { $admin->enable = 1; $admin->is_admin = 1; $admin->save(); + $resourcePool = new CloudResourcePool(); + $resourcePool->id = 1; + $resourcePool->line_id = 77; + $resourcePool->code = 'primary'; + $resourcePool->name = 'Primary'; + $resourcePool->status = 'active'; + $resourcePool->scheduling_policy = 'balanced'; + $resourcePool->sort = 0; + $resourcePool->created_at = $resourcePool->updated_at = 900; + $resourcePool->save(); $host = new CloudHost(); $host->name = 'test-host'; $host->enabled = 1; + $host->resource_pool_id = 1; + $host->scheduler_state = 'active'; $host->health_state = 'unknown'; $host->health_error_code = ''; + $host->default_network = 'default'; $host->created_at = 900; $host->updated_at = 900; $host->save(); + // This pool is deliberately omitted by the engine's successful complete + // catalog. Its free row must stop advertising stale capacity. + $missingAttachment = new CloudHostNetworkAttachment(); + $missingAttachment->attachment_key = str_repeat('d', 32); + $missingAttachment->host_id = 1; + $missingAttachment->line_id = 77; + $missingAttachment->name = 'missing-network'; + $missingAttachment->network_mode = 'network'; + $missingAttachment->network_selector = 'missing-network'; + $missingAttachment->state = 'active'; + $missingAttachment->priority = 0; + $missingAttachment->desired_revision = 1; + $missingAttachment->observed_revision = 1; + $missingAttachment->sync_state = 'fresh'; + $missingAttachment->last_error_code = ''; + $missingAttachment->created_at = $missingAttachment->updated_at = 900; + $missingAttachment->save(); + $missingPool = new CloudPublicIpPool(); + $missingPool->pool_key = $missingPoolKey; + $missingPool->host_id = 1; + $missingPool->network_attachment_id = $missingAttachment->id; + $missingPool->name = 'Missing pool'; + $missingPool->cidr = '11.0.0.0/30'; + $missingPool->gateway = '11.0.0.1'; + $missingPool->guest_prefix_length = 30; + $missingPool->dns_servers_json = '["1.1.1.1"]'; + $missingPool->reserved_ips_json = '[]'; + $missingPool->state = 'active'; + $missingPool->priority = 0; + $missingPool->desired_revision = 1; + $missingPool->observed_revision = 1; + $missingPool->sync_state = 'fresh'; + $missingPool->last_error_code = ''; + $missingPool->created_at = $missingPool->updated_at = 900; + $missingPool->save(); + $insertIp = static function (string $address, string $state, - ?int $reservationId = null, ?int $instanceId = null): void { - $row = new CloudPublicIp(); - $row->host_id = 1; - $row->address = $address; - $row->prefix_length = 29; - $row->state = $state; - $row->reservation_id = $reservationId; - $row->instance_id = $instanceId; - $row->operation_id = null; - $row->created_at = 900; - $row->updated_at = 900; - $row->save(); + ?int $poolId = null, ?int $reservationId = null, + ?int $instanceId = null) use ($capsule): void { + // Raw insert intentionally represents an upgraded legacy row whose + // nullable pool identity has not yet been backfilled. + $capsule->getConnection()->table('cloud_public_ips')->insert([ + 'host_id' => 1, + 'public_ip_pool_id' => $poolId, + 'address' => $address, + 'prefix_length' => str_starts_with($address, '8.') ? 29 : 30, + 'state' => $state, + 'reservation_id' => $reservationId, + 'instance_id' => $instanceId, + 'operation_id' => null, + 'created_at' => 900, + 'updated_at' => 900, + ]); }; $insertIp('8.8.8.2', 'available'); - $insertIp('8.8.8.3', 'reserved', 501); - $insertIp('8.8.8.4', 'assigned', null, 601); + $insertIp('8.8.8.3', 'reserved', null, 501); + $insertIp('8.8.8.4', 'assigned', null, null, 601); $insertIp('8.8.8.5', 'disabled'); + $insertIp('11.0.0.2', 'available', (int) $missingPool->id); $now = 1000; $history = []; $successClient = $client([ // A host clock may lead PHP by the accepted 300-second window. The - // local attempt boundary, not this engine timestamp, must drive later + // local attempt boundary, not this engine timestamp, drives later // reservation accounting. new Response(200, [], $json($capacity($now + 300))), new Response(200, [], $json($network())), @@ -186,26 +384,68 @@ namespace { $host->refresh(); $snapshot = CloudHostCapacitySnapshot::find(1); $states = CloudPublicIp::orderBy('address')->pluck('state', 'address')->all(); + $membership = CloudHostResourcePoolMembership::where('host_id', 1)->first(); + $attachments = CloudHostNetworkAttachment::orderBy('id')->get(); + $publicPools = CloudPublicIpPool::whereIn('pool_key', [$poolA, $poolB]) + ->orderBy('pool_key')->get()->keyBy('pool_key'); $assert($success['checked'] === 1 && $success['succeeded'] === 1 - && $success['errors'] === [], 'A valid host observation did not succeed.'); + && $success['errors'] === [], 'A valid multi-pool observation did not succeed.'); $assert($host->health_state === 'healthy' && $host->last_seen_at === 1000 && $host->health_checked_at === 1000 && $host->health_error_code === '', 'Successful host health markers are incorrect.'); + $assert($membership !== null && (int) $membership->resource_pool_id === 1 + && $membership->scheduler_state === 'active' + && (int) $membership->priority === 0 && (int) $membership->weight === 100, + 'The legacy scalar host membership was not safely backfilled.'); $assert($snapshot !== null && $snapshot->sync_state === 'fresh' && $snapshot->cpu_total_cores === 16 && $snapshot->storage_total_bytes === 1099511627776 - && $snapshot->public_ip_available_count === 3 + && $snapshot->public_ip_total_count === 8 + && $snapshot->public_ip_reserved_count === 3 + && $snapshot->public_ip_allocated_count === 1 + && $snapshot->public_ip_available_count === 4 && $snapshot->observed_at === 1300 && $snapshot->last_attempt_at === 1000 && $snapshot->attempt_generation === 1, - 'Capacity or public-IP counts were not persisted exactly.'); + 'Aggregated capacity or public-IP counts were not persisted exactly.'); + $assert($publicPools->count() === 2 + && (int) $publicPools[$poolA]->network_attachment_id + === (int) $publicPools[$poolB]->network_attachment_id + && $publicPools[$poolA]->reserved_ips_json === '["8.8.8.2"]' + && $publicPools[$poolA]->state === 'active' + && $publicPools[$poolB]->state === 'draining' + && $publicPools[$poolA]->sync_state === 'fresh' + && $publicPools[$poolB]->sync_state === 'fresh', + 'Stable pool identities, reserved metadata or shared attachment were not synced.'); + $defaultAttachment = $attachments->firstWhere( + 'attachment_key', substr(hash('sha256', 'default'), 0, 32) + ); + $missingAttachment->refresh(); + $missingPool->refresh(); + $assert($defaultAttachment !== null + && (int) $defaultAttachment->line_id === 77 + && $defaultAttachment->network_selector === 'default' + && $defaultAttachment->state === 'active' + && $defaultAttachment->sync_state === 'fresh' + && $missingAttachment->sync_state === 'failed' + && $missingPool->sync_state === 'failed', + 'Attachment upsert or missing-catalog diagnostics are incorrect.'); $assert($states === [ + '11.0.0.2' => 'disabled', '8.8.8.2' => 'disabled', '8.8.8.3' => 'reserved', '8.8.8.4' => 'assigned', '8.8.8.5' => 'available', '8.8.8.6' => 'available', - ], 'Free-list sync overwrote local occupancy or retained a missing free address.'); + '9.9.9.2' => 'disabled', + ], 'Pool sync overwrote a local hold or retained stale missing-pool capacity.'); + foreach (CloudPublicIp::where('address', 'like', '8.8.8.%')->get() as $ip) { + $assert((int) $ip->public_ip_pool_id === (int) $publicPools[$poolA]->id, + 'A legacy address was not associated with its unique public pool.'); + } + $assert((int) CloudPublicIp::where('address', '9.9.9.2')->first() + ->public_ip_pool_id === (int) $publicPools[$poolB]->id, + 'A new address was written without its public-pool association.'); $assert(count($history) === 2 && (string) $history[0]['request']->getUri() === 'http://127.0.0.1:8000/internal/v1/hosts/1/host/capacity' @@ -222,15 +462,20 @@ namespace { $beforeCapacity = $snapshot->toArray(); $beforeStates = $states; + $beforePools = CloudPublicIpPool::orderBy('id')->get()->map( + static fn ($row): array => $row->toArray() + )->all(); $now = 1010; $notDue = (new CloudHostInventorySyncService( $client([]), static function () use (&$now): int { return $now; } ))->sync(10); $assert($notDue['checked'] === 0 && $notDue['errors'] === [], 'A host inside the configured due-time window was polled again.'); + $invalidNetwork = $network(); + $invalidNetwork['pools'][1]['available_count'] = 0; $invalidClient = $client([ new Response(200, [], $json($capacity($now, 32))), - new Response(200, [], $json($network(2))), + new Response(200, [], $json($invalidNetwork)), ]); $invalid = (new CloudHostInventorySyncService( $invalidClient, static function () use (&$now): int { return $now; } @@ -238,20 +483,23 @@ namespace { $host->refresh(); $snapshot->refresh(); $states = CloudPublicIp::orderBy('address')->pluck('state', 'address')->all(); + $afterPools = CloudPublicIpPool::orderBy('id')->get()->map( + static fn ($row): array => $row->toArray() + )->all(); $assert($invalid['degraded'] === 1 && $invalid['errors'] === ['host_inventory_sync_failed'] && $host->health_state === 'degraded' && $host->health_error_code === 'host_inventory_network_invalid' && $host->last_seen_at === 1000 && $host->health_checked_at === 1010, - 'Invalid network response did not preserve last success and mark degraded.'); + 'Invalid pool response did not preserve last success and mark degraded.'); foreach (['cpu_total_cores', 'storage_total_bytes', 'observed_at'] as $field) { $assert($snapshot->{$field} === $beforeCapacity[$field], 'Failure replaced last-known capacity field: ' . $field); } $assert($snapshot->sync_state === 'failed' && $snapshot->last_error_code === 'host_inventory_network_invalid' - && $states === $beforeStates, - 'Failure cleared capacity/IP data or exposed an unfixed diagnostic.'); + && $states === $beforeStates && $afterPools === $beforePools, + 'Failure cleared capacity, pool metadata or concrete IP state.'); $now = 1020; $unavailableClient = $client([ @@ -273,33 +521,93 @@ namespace { && !str_contains($persisted, 'private engine path'), 'Engine failure cleared last success or persisted an upstream error.'); - // Same-second polls need a monotonic fence rather than timestamp ordering. - // Prove both dangerous completion orders: stale success after a newer - // failure, then stale failure after a newer success. $reflection = new ReflectionClass(CloudHostInventorySyncService::class); $beginAttempt = $reflection->getMethod('beginAttempt'); $apply = $reflection->getMethod('apply'); $markFailure = $reflection->getMethod('markFailure'); $parseCapacity = $reflection->getMethod('capacity'); $parseNetwork = $reflection->getMethod('network'); + $reconcile = $reflection->getMethod('reconcilePublicInventory'); foreach ([$beginAttempt, $apply, $markFailure, $parseCapacity, - $parseNetwork] as $method) { + $parseNetwork, $reconcile] as $method) { $method->setAccessible(true); } + + // Structural rejection happens before any persistent observation changes. + $expectInvalidNetwork = static function (array $payload, string $message) use ( + $assert, $parseNetwork + ): void { + try { + $parseNetwork->invoke(null, $payload); + } catch (Throwable $error) { + $assert($error instanceof UnexpectedValueException, $message); + return; + } + throw new RuntimeException($message); + }; + $duplicateKey = $network(); + $duplicateKey['pools'][1]['pool_key'] = $poolA; + $expectInvalidNetwork($duplicateKey, 'Duplicate pool keys were accepted.'); + $overlap = $network(); + $overlap['pools'][1]['pool_cidr'] = '8.8.8.0/29'; + $overlap['pools'][1]['guest_prefix'] = 29; + $overlap['pools'][1]['gateway'] = '8.8.8.1'; + $overlap['pools'][1]['reserved_ips'] = ['8.8.8.2']; + $overlap['pools'][1]['usable_count'] = 6; + $overlap['pools'][1]['reserved_count'] = 2; + $overlap['pools'][1]['allocated_count'] = 3; + $overlap['pools'][1]['available_ips'] = ['8.8.8.5/29']; + $overlap['pools'][1]['available_count'] = 1; + $expectInvalidNetwork($overlap, 'Overlapping CIDRs were accepted.'); + $reservedAsFree = $network(); + $reservedAsFree['pools'][0]['available_ips'][0] = '8.8.8.2/29'; + $reservedAsFree['available_ips'][0] = '8.8.8.2/29'; + $expectInvalidNetwork($reservedAsFree, + 'An explicitly reserved address was accepted as free.'); + $badKey = $network(); + $badKey['pools'][0]['pool_key'] = '../../not-stable'; + $expectInvalidNetwork($badKey, 'An unstable pool key was accepted.'); + + // A pre-catalog engine remains readable as one synthetic legacy pool. + $legacyCapacityPayload = $capacity(1020); + $legacyCapacityPayload['public_ipv4'] = [ + 'configured' => true, + 'total' => 6, + 'reserved' => 2, + 'allocated' => 1, + 'available' => 3, + ]; + $legacyNetworkPayload = $network(); + unset($legacyNetworkPayload['pools'], $legacyNetworkPayload['compatibility_mode']); + $legacyCapacity = $parseCapacity->invoke(null, $legacyCapacityPayload, 1020); + $legacyNetwork = $parseNetwork->invoke(null, $legacyNetworkPayload); + [$legacySnapshot, $legacyPools] = $reconcile->invoke( + null, $legacyCapacity, $legacyNetwork + ); + $assert(isset($legacyPools['legacy-default']) + && $legacyPools['legacy-default']['reserved_ips_known'] === false + && $legacySnapshot['public_ip_available_count'] === 3, + 'The bounded pre-catalog compatibility path was lost.'); + + // Same-second polls need a monotonic generation fence rather than + // timestamp ordering. Prove both dangerous completion orders. $now = 1030; $fenced = new CloudHostInventorySyncService( $client([]), static function () use (&$now): int { return $now; } ); $parsedCapacity = $parseCapacity->invoke(null, $capacity($now, 24), $now); $parsedNetwork = $parseNetwork->invoke(null, $network()); + [$snapshotData, $poolData] = $reconcile->invoke( + null, $parsedCapacity, $parsedNetwork + ); $oldSuccess = $beginAttempt->invoke($fenced, 1, $now); $newFailure = $beginAttempt->invoke($fenced, 1, $now); $assert($newFailure === $oldSuccess + 1 && $markFailure->invoke($fenced, 1, 'unavailable', 'host_inventory_unavailable', $now, $newFailure) === true - && $apply->invoke($fenced, 1, $parsedCapacity, - $parsedNetwork['available'], $now, $oldSuccess) === false, + && $apply->invoke($fenced, 1, $snapshotData, + $poolData, $now, $oldSuccess) === false, 'A stale success was not fenced after a newer same-second failure.'); $host->refresh(); $snapshot->refresh(); @@ -310,8 +618,8 @@ namespace { $oldFailure = $beginAttempt->invoke($fenced, 1, $now); $newSuccess = $beginAttempt->invoke($fenced, 1, $now); - $assert($apply->invoke($fenced, 1, $parsedCapacity, - $parsedNetwork['available'], $now, $newSuccess) === true + $assert($apply->invoke($fenced, 1, $snapshotData, + $poolData, $now, $newSuccess) === true && $markFailure->invoke($fenced, 1, 'unavailable', 'host_inventory_unavailable', $now, $oldFailure) === false, 'A stale failure was not fenced after a newer same-second success.'); @@ -323,5 +631,49 @@ namespace { && $snapshot->attempt_generation === $newSuccess, 'The stale failure overwrote a newer successful observation.'); - echo "Cloud host capacity and public-IP lifecycle sync checks passed.\n"; + // The first-release writer retains exactly one canonical relation, fixes + // priority/weight, and can explicitly clear the assignment. + $secondResourcePool = new CloudResourcePool(); + $secondResourcePool->id = 2; + $secondResourcePool->line_id = 77; + $secondResourcePool->code = 'secondary'; + $secondResourcePool->name = 'Secondary'; + $secondResourcePool->status = 'active'; + $secondResourcePool->scheduling_policy = 'balanced'; + $secondResourcePool->sort = 1; + $secondResourcePool->created_at = $secondResourcePool->updated_at = 1030; + $secondResourcePool->save(); + $capsule->getConnection()->transaction(static function (): void { + $lockedHost = CloudHost::where('id', 1)->lockForUpdate()->firstOrFail(); + CloudHostPoolMembershipService::replaceLocked( + $lockedHost, 2, 'disabled', 1040 + ); + $lockedHost->updated_at = 1040; + $lockedHost->save(); + }); + $relations = CloudHostResourcePoolMembership::where('host_id', 1)->get(); + $host->refresh(); + $assert($relations->count() === 1 + && (int) $relations->first()->resource_pool_id === 2 + && $relations->first()->scheduler_state === 'disabled' + && (int) $relations->first()->priority === 0 + && (int) $relations->first()->weight === 100 + && (int) $host->resource_pool_id === 2 + && $host->scheduler_state === 'disabled', + 'Membership replacement retained a stale authority row or unsafe values.'); + $capsule->getConnection()->transaction(static function (): void { + $lockedHost = CloudHost::where('id', 1)->lockForUpdate()->firstOrFail(); + CloudHostPoolMembershipService::replaceLocked( + $lockedHost, null, 'disabled', 1050 + ); + $lockedHost->updated_at = 1050; + $lockedHost->save(); + }); + $host->refresh(); + $assert(CloudHostResourcePoolMembership::where('host_id', 1)->count() === 0 + && $host->resource_pool_id === null + && $host->scheduler_state === 'disabled', + 'Clearing the authoritative membership recreated a stale scalar link.'); + + echo "Cloud host multi-pool inventory and membership checks passed.\n"; }