feat: sync multi-pool host inventory

dev
admin 2 months ago
parent 7f804125b5
commit f3822ac66d

@ -17,4 +17,16 @@ final class CloudPublicIp extends Model
'created_at' => 'int',
'updated_at' => 'int',
];
/** New ledger rows must always belong to one validated public pool. */
protected static function booted(): void
{
static::creating(static function (self $row): void {
if ((int) ($row->public_ip_pool_id ?? 0) < 1) {
throw new \LogicException(
'New public IP rows require a public pool association.'
);
}
});
}
}

@ -17,17 +17,31 @@ final class CloudAdminHostSchedulingService
public static function read(int $hostId): array
{
$host = CloudHost::find($hostId);
if ($host === null) {
throw new CloudError('宿主机不存在', 404);
}
[$host, $membership, $memberships] = (new CloudHost())->getConnection()
->transaction(function () use ($hostId): array {
$host = CloudHost::where('id', $hostId)->lockForUpdate()->first();
if ($host === null) {
throw new CloudError('宿主机不存在', 404);
}
$rows = CloudHostPoolMembershipService::lockedRows($host, time());
$selected = CloudHostPoolMembershipService::selected($host, $rows);
$poolId = $host->resource_pool_id;
$state = (string) $host->scheduler_state;
CloudHostPoolMembershipService::mirrorLocked($host, $selected);
if ($poolId !== $host->resource_pool_id
|| $state !== (string) $host->scheduler_state) {
$host->updated_at = time();
$host->save();
}
return [$host, $selected, $rows];
}, 3);
$policy = CloudHostCapacityPolicy::find($hostId);
$snapshot = CloudHostCapacitySnapshot::find($hostId);
$pools = self::poolOptions();
$selected = null;
foreach ($pools as $pool) {
if ((int) $pool['id'] === (int) ($host->resource_pool_id ?? 0)) {
if ((int) $pool['id'] === (int) ($membership?->resource_pool_id ?? 0)) {
$selected = $pool;
break;
}
@ -39,11 +53,12 @@ final class CloudAdminHostSchedulingService
'id' => (int) $host->id,
'name' => self::safeText((string) $host->name, 80),
'enabled' => (bool) $host->enabled,
'resource_pool_id' => $host->resource_pool_id === null
? null : (int) $host->resource_pool_id,
'scheduler_state' => in_array((string) $host->scheduler_state,
'resource_pool_id' => $membership === null
? null : (int) $membership->resource_pool_id,
'scheduler_state' => in_array((string) ($membership?->scheduler_state
?? 'disabled'),
self::SCHEDULER_STATES, true)
? (string) $host->scheduler_state : 'disabled',
? (string) ($membership?->scheduler_state ?? 'disabled') : 'disabled',
'health_state' => self::safeText((string) ($host->health_state ?? 'unknown'), 32),
'last_seen_at' => self::nullableInt($host->last_seen_at),
'health_checked_at' => self::nullableInt($host->health_checked_at),
@ -52,6 +67,13 @@ final class CloudAdminHostSchedulingService
'policy' => self::policyDto($policy),
'snapshot' => $snapshotDto,
'placement' => $selected,
'memberships' => $memberships->map(static fn ($row): array => [
'id' => (int) $row->id,
'resource_pool_id' => (int) $row->resource_pool_id,
'scheduler_state' => (string) $row->scheduler_state,
'priority' => (int) $row->priority,
'weight' => (int) $row->weight,
])->values()->all(),
'pools' => $pools,
'network_lines_enabled' => CloudNetworkLineFeature::enabled(),
'warnings' => self::warnings($host, $snapshotDto, $selected),
@ -85,6 +107,9 @@ final class CloudAdminHostSchedulingService
if ($host === null) {
throw new CloudError('宿主机不存在', 404);
}
// Membership is the scheduling authority. Lock it directly
// after the host sentinel and before host-global capacity.
CloudHostPoolMembershipService::lockedRows($host, time());
$policy = CloudHostCapacityPolicy::where('host_id', $hostId)
->lockForUpdate()->first();
if ($policy === null) {
@ -104,8 +129,9 @@ final class CloudAdminHostSchedulingService
}
$now = time();
$host->resource_pool_id = $poolId;
$host->scheduler_state = $schedulerState;
CloudHostPoolMembershipService::replaceLocked(
$host, $poolId, $schedulerState, $now
);
$host->updated_at = $now;
$host->save();
foreach ($values as $field => $value) {

@ -5,6 +5,7 @@ namespace App\Services\Cloud;
use App\Models\CloudHost;
use App\Models\CloudHostCapacityPolicy;
use App\Models\CloudHostCapacitySnapshot;
use App\Models\CloudHostResourcePoolMembership;
use App\Models\CloudNetworkLine;
use App\Models\CloudOffer;
use App\Models\CloudPublicIp;
@ -240,7 +241,9 @@ final class CloudAdminResourceService
throw new CloudError('所选线路不存在', 409);
}
if ($pool->exists && (int) $pool->line_id !== $lineId
&& (CloudHost::where('resource_pool_id', (int) $pool->id)->exists()
&& (CloudHostResourcePoolMembership::where(
'resource_pool_id', (int) $pool->id)->exists()
|| CloudHost::where('resource_pool_id', (int) $pool->id)->exists()
|| CloudOffer::where('resource_pool_id', (int) $pool->id)->exists()
|| CloudResourceReservation::where('resource_pool_id', (int) $pool->id)->exists())) {
throw new CloudError('已被节点、销售方案或预留引用的资源池不能更换线路', 409);
@ -367,8 +370,14 @@ final class CloudAdminResourceService
$offerCounts[(int) $row->resource_pool_id] = (int) $row->row_count;
}
$hosts = CloudHost::whereNotNull('resource_pool_id')->orderBy('id')->get();
$hostIds = $hosts->pluck('id')->map(static fn ($id): int => (int) $id)->all();
$memberships = CloudHostResourcePoolMembership::orderBy('host_id')
->orderBy('resource_pool_id')->orderBy('id')->get();
$hostIds = $memberships->pluck('host_id')->map(
static fn ($id): int => (int) $id
)->unique()->values()->all();
$hosts = $hostIds === [] ? collect()
: CloudHost::whereIn('id', $hostIds)->orderBy('id')->get();
$hostById = self::keyById($hosts);
$policies = $hostIds === [] ? collect() : CloudHostCapacityPolicy::whereIn('host_id', $hostIds)->get();
$snapshots = $hostIds === [] ? collect() : CloudHostCapacitySnapshot::whereIn('host_id', $hostIds)->get();
$policyByHost = self::keyBy($policies, 'host_id');
@ -422,8 +431,18 @@ final class CloudAdminResourceService
}
$poolHosts = [];
foreach ($hosts as $host) {
$poolHosts[(int) $host->resource_pool_id][] = $host;
foreach ($memberships->groupBy('host_id') as $hostId => $rows) {
$host = $hostById[(int) $hostId] ?? null;
if ($host === null) {
continue;
}
$membership = CloudHostPoolMembershipService::selected($host, $rows);
if ($membership !== null) {
$poolHosts[(int) $membership->resource_pool_id][] = [
'host' => $host,
'membership' => $membership,
];
}
}
$poolRows = [];
foreach ($pools as $pool) {
@ -434,7 +453,9 @@ final class CloudAdminResourceService
$schedulable = 0;
$syncState = 'never';
$syncRank = 0;
foreach ($poolHosts[$poolId] ?? [] as $host) {
foreach ($poolHosts[$poolId] ?? [] as $placement) {
$host = $placement['host'];
$membership = $placement['membership'];
$hostId = (int) $host->id;
$policy = $policyByHost[$hostId] ?? null;
$snapshot = $snapshotByHost[$hostId] ?? null;
@ -504,6 +525,7 @@ final class CloudAdminResourceService
&& $region !== null && (string) $region->status === 'active'
&& (bool) $host->enabled
&& (string) $host->scheduler_state === 'active'
&& (string) $membership->scheduler_state === 'active'
&& (string) $host->health_state === 'healthy'
&& $state === 'fresh'
&& ($maxInstances <= 0 || $instanceCount < $maxInstances)

File diff suppressed because it is too large Load Diff

@ -0,0 +1,158 @@
<?php
namespace App\Services\Cloud;
use App\Models\CloudHost;
use App\Models\CloudHostResourcePoolMembership;
/**
* Transitional authority for host-to-resource-pool scheduling membership.
*
* The membership table is canonical. cloud_hosts.resource_pool_id and the
* host scheduler state are maintained only as a compatibility projection for
* placement code that has not moved to the relation yet. Every writer first
* locks the host sentinel and then locks membership rows in resource-pool/id
* order.
*/
final class CloudHostPoolMembershipService
{
public const SAFE_PRIORITY = 0;
public const SAFE_WEIGHT = 100;
private const STATES = ['active', 'draining', 'disabled'];
/**
* Lock the authoritative rows and perform the one-time legacy backfill.
* The caller must already hold a FOR UPDATE lock on the host row.
*/
public static function lockedRows(CloudHost $host, int $now)
{
$rows = CloudHostResourcePoolMembership::where('host_id', (int) $host->id)
->orderBy('resource_pool_id')->orderBy('id')->lockForUpdate()->get();
if ($rows->isEmpty() && $host->resource_pool_id !== null) {
$row = new CloudHostResourcePoolMembership();
$row->host_id = (int) $host->id;
$row->resource_pool_id = (int) $host->resource_pool_id;
$row->scheduler_state = self::state((string) $host->scheduler_state);
$row->priority = self::SAFE_PRIORITY;
$row->weight = self::SAFE_WEIGHT;
$row->created_at = $row->updated_at = $now;
$row->save();
$rows->push($row);
}
return $rows;
}
/** Select the first-release placement relation deterministically. */
public static function selected(CloudHost $host, $rows): ?CloudHostResourcePoolMembership
{
$active = null;
$draining = null;
$latest = null;
$activeCount = 0;
foreach ($rows as $row) {
if ((string) $row->scheduler_state === 'active') {
$activeCount++;
$active = $row;
}
if ((string) $row->scheduler_state === 'draining'
&& ($draining === null || self::newer($row, $draining))) {
$draining = $row;
}
if ($latest === null || self::newer($row, $latest)) {
$latest = $row;
}
}
if ($activeCount > 1) {
throw new CloudError('宿主机存在多个启用中的资源池成员关系,请先修复调度配置', 409);
}
return $active ?? $draining ?? $latest;
}
/**
* Replace the first-release selected membership. Until the scheduler can
* express several memberships in its write API, one host has zero or one
* authoritative row; the relation schema itself remains ready for a later
* multi-pool rollout.
*/
public static function replaceLocked(CloudHost $host, ?int $poolId,
string $schedulerState, int $now): ?CloudHostResourcePoolMembership
{
if (!in_array($schedulerState, self::STATES, true)) {
throw new CloudError('调度状态无效');
}
if ($poolId === null && $schedulerState !== 'disabled') {
throw new CloudError('未选择资源池时只能停用调度', 409);
}
$rows = self::lockedRows($host, $now);
$target = null;
foreach ($rows as $row) {
if ($poolId !== null && (int) $row->resource_pool_id === $poolId) {
$target = $row;
}
}
if ($poolId !== null && $target === null) {
$target = new CloudHostResourcePoolMembership();
$target->host_id = (int) $host->id;
$target->resource_pool_id = $poolId;
$target->created_at = $now;
$rows->push($target);
}
foreach ($rows as $row) {
if ($target !== null
&& (int) $row->resource_pool_id === (int) $target->resource_pool_id) {
continue;
}
$row->delete();
}
if ($target !== null) {
$target->scheduler_state = $schedulerState;
$target->priority = self::SAFE_PRIORITY;
$target->weight = self::SAFE_WEIGHT;
$target->updated_at = $now;
$target->save();
self::assertSingleActive([$target]);
}
// Temporary mirror only. Membership rows above are the authority.
$host->resource_pool_id = $target?->resource_pool_id;
$host->scheduler_state = $target === null ? 'disabled' : $schedulerState;
return $target;
}
/** Heal an old scalar projection after a successful relation backfill. */
public static function mirrorLocked(CloudHost $host,
?CloudHostResourcePoolMembership $selected): void
{
$host->resource_pool_id = $selected?->resource_pool_id;
$host->scheduler_state = $selected === null
? 'disabled' : self::state((string) $selected->scheduler_state);
}
private static function assertSingleActive($rows): void
{
$active = 0;
foreach ($rows as $row) {
if ((string) $row->scheduler_state === 'active') {
$active++;
}
}
if ($active > 1) {
throw new CloudError('宿主机存在多个启用中的资源池成员关系,请先修复调度配置', 409);
}
}
private static function state(string $state): string
{
return in_array($state, self::STATES, true) ? $state : 'disabled';
}
private static function newer($left, $right): bool
{
return (int) $left->updated_at > (int) $right->updated_at
|| ((int) $left->updated_at === (int) $right->updated_at
&& (int) $left->id > (int) $right->id);
}
}

@ -129,6 +129,20 @@ CREATE TABLE IF NOT EXISTS cloud_host_resource_pool_memberships (
KEY cloud_host_membership_scheduler (host_id, scheduler_state, id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
-- One-time compatibility import for hosts configured before membership rows
-- existed. Once any relation is present for a host, that relation is the
-- authority and rerunning cloudInit must never recreate a stale scalar link.
INSERT IGNORE INTO cloud_host_resource_pool_memberships
(host_id, resource_pool_id, scheduler_state, priority, weight, created_at, updated_at)
SELECT h.id, h.resource_pool_id,
CASE WHEN h.scheduler_state IN ('active', 'draining', 'disabled')
THEN h.scheduler_state ELSE 'disabled' END,
0, 100, GREATEST(0, h.created_at), GREATEST(0, h.updated_at)
FROM cloud_hosts AS h
LEFT JOIN cloud_host_resource_pool_memberships AS membership
ON membership.host_id = h.id
WHERE h.resource_pool_id IS NOT NULL AND membership.id IS NULL;
-- One attachment is one real host egress such as bridge:br0. Several CIDR
-- blocks may share the same attachment. Binding the attachment to a network
-- line keeps physical egress separate from compute capacity and lets a future

@ -15,6 +15,9 @@ $assert = static function (bool $condition, string $message): void {
};
$service = $read('app/Services/Cloud/CloudAdminHostSchedulingService.php');
$membershipService = $read(
'app/Services/Cloud/CloudHostPoolMembershipService.php'
);
$controller = $read('app/Controllers/CloudAdminController.php');
$routes = $read('config/routes_cloud.php');
$middleware = $read('app/Middleware/CloudApi.php');
@ -36,11 +39,30 @@ $assert(str_contains($controller, "'resource_pool_id', 'scheduler_state'")
- strpos($controller, 'public function hostScheduling')), 'public_ip'),
'The scheduling allowlist must not accept manual public IP fields.');
$hostLock = strpos($service, "CloudHost::where('id', \$hostId)->lockForUpdate()");
$policyLock = strpos($service, "CloudHostCapacityPolicy::where('host_id', \$hostId)");
$assert($hostLock !== false && $policyLock !== false && $hostLock < $policyLock
$saveStart = strpos($service, 'public static function save');
$syncStart = strpos($service, 'public static function sync', $saveStart);
$save = $saveStart === false || $syncStart === false ? ''
: substr($service, $saveStart, $syncStart - $saveStart);
$hostLock = strpos($save, "CloudHost::where('id', \$hostId)->lockForUpdate()");
$membershipLock = strpos($save, 'CloudHostPoolMembershipService::lockedRows');
$policyLock = strpos($save, "CloudHostCapacityPolicy::where('host_id', \$hostId)");
$assert($save !== '' && $hostLock !== false && $membershipLock !== false
&& $policyLock !== false && $hostLock < $membershipLock
&& $membershipLock < $policyLock
&& str_contains($service, '->lockForUpdate()->first();'),
'Scheduling writes must lock host before capacity policy.');
'Scheduling writes must lock host, membership and capacity policy in order.');
$assert(str_contains($service, 'CloudHostPoolMembershipService::replaceLocked(')
&& str_contains($service, "'memberships' => \$memberships->map")
&& str_contains($membershipService,
'CloudHostResourcePoolMembership::where(\'host_id\'')
&& str_contains($membershipService, 'if ($rows->isEmpty()'
. ' && $host->resource_pool_id !== null)')
&& str_contains($membershipService, 'public const SAFE_PRIORITY = 0;')
&& str_contains($membershipService, 'public const SAFE_WEIGHT = 100;')
&& str_contains($membershipService, '$row->delete();')
&& str_contains($membershipService,
'宿主机存在多个启用中的资源池成员关系'),
'Membership authority, legacy backfill or first-release single-row policy is missing.');
$assert(str_contains($service, "\$schedulerState === 'active'")
&& str_contains($service, 'CloudNetworkLineFeature::enabled()')
&& str_contains($service, "(string) \$pool->status !== 'active'")

@ -21,8 +21,10 @@ $assert = static function ($condition, string $message): void {
$assert(str_contains($service, "'GET', 'host/capacity'")
&& str_contains($service, "'GET', 'host/network-options'")
&& str_contains($service, 'public_ip_available_count')
&& str_contains($service, "->where('state', 'available')")
&& str_contains($service, "['available', 'disabled']")
&& str_contains($service, 'self::LOCAL_IP_HOLDS')
&& str_contains($service, '$poolObserved')
&& str_contains($service, 'public_ip_pool_id')
&& str_contains($service, "['degraded', self::ERROR_INVALID_NETWORK]")
&& str_contains($service, "['unavailable', self::ERROR_UNAVAILABLE]")
&& str_contains($service, '$attemptedAt = $this->now();')
@ -34,7 +36,38 @@ $assert(str_contains($service, "'GET', 'host/capacity'")
&& substr_count($service, "\$host->last_seen_at =") === 1,
'Capacity/IP observation, local-state preservation or last-seen boundary changed.');
$assert(str_contains($service, 'private const MAX_PUBLIC_POOLS = 32;')
&& str_contains($service, 'private const MAX_PUBLIC_IPS_PER_POOL = 254;')
&& str_contains($service, 'private const MAX_PUBLIC_IPS_PER_HOST = 8128;')
&& str_contains($service, 'private static function reconcilePublicInventory')
&& str_contains($service, 'private static function networksOverlap')
&& str_contains($service, "'reserved_ips_known' => true")
&& str_contains($service, '$publicPool->reserved_ips_json = json_encode(')
&& str_contains($service, "preg_match('/^[0-9a-f]{32}$/D'")
&& str_contains($service, 'public capacity does not match pool catalog')
&& str_contains($service, 'public pool observations disagree'),
'Bounded multi-pool parsing, identity or cross-endpoint count checks are missing.');
$applyStart = strpos($service, 'private function apply');
$failureStart = strpos($service, 'private function markFailure');
$apply = $applyStart === false || $failureStart === false ? ''
: substr($service, $applyStart, $failureStart - $applyStart);
$hostLock = strpos($apply, "CloudHost::where('id', \$hostId)->lockForUpdate()");
$snapshotLock = strpos($apply,
"CloudHostCapacitySnapshot::where('host_id', \$hostId)");
$membershipLock = strpos($apply, 'CloudHostPoolMembershipService::lockedRows');
$attachmentLock = strpos($apply,
"CloudHostNetworkAttachment::where('host_id', \$hostId)");
$poolLock = strpos($apply, "CloudPublicIpPool::where('host_id', \$hostId)");
$ipLock = strpos($apply, "CloudPublicIp::where('host_id', \$hostId)");
$assert($apply !== '' && $hostLock !== false && $snapshotLock !== false
&& $membershipLock !== false && $attachmentLock !== false
&& $poolLock !== false && $ipLock !== false
&& $hostLock < $snapshotLock && $snapshotLock < $membershipLock
&& $membershipLock < $attachmentLock && $attachmentLock < $poolLock
&& $poolLock < $ipLock,
'Inventory lock order must remain host, snapshot fence, membership, attachment, pool, IP.');
$validationStart = strpos($service, 'private static function capacity', $failureStart);
$failure = $failureStart === false || $validationStart === false ? ''
: substr($service, $failureStart, $validationStart - $failureStart);
@ -42,8 +75,10 @@ $assert($failure !== ''
&& str_contains($failure, "\$snapshot->sync_state = 'failed';")
&& str_contains($failure, "\$snapshot->last_error_code = \$code;")
&& !str_contains($failure, 'cpu_total_cores')
&& !str_contains($failure, 'CloudPublicIp::'),
'Failure handling may clear last-known capacity or mutate the IP library.');
&& !str_contains($failure, 'CloudPublicIp::')
&& !str_contains($failure, 'CloudPublicIpPool::')
&& !str_contains($failure, 'CloudHostNetworkAttachment::'),
'Failure handling may clear last-known capacity or mutate pool/IP inventory.');
$assert(str_contains($actor, "Config::get('super_admin', '')")
&& str_contains($actor, "->where('enable', 1)->where('is_admin', 1)")
@ -60,6 +95,11 @@ $assert(str_contains($lifecycle, "busy('host_inventory_sync')")
&& str_contains($service, 'public function syncOne(int $hostId): array')
&& str_contains($service, "Config::get('cloud_host_inventory_interval_seconds', 30)")
&& str_contains($service, 'private const MAX_HOSTS_PER_PASS = 10;')
&& str_contains($service, "'cloud_host_resource_pool_memberships'")
&& str_contains($service, "'cloud_host_network_attachments'")
&& str_contains($service, "'cloud_public_ip_pools'")
&& str_contains($service,
"hasColumn('cloud_public_ips', 'public_ip_pool_id')")
&& str_contains($heartbeat,
"'host_inventory_sync' => '宿主容量与公网 IP 同步'")
&& str_contains($heartbeat, "'host_inventory_sync_failed'")

@ -30,10 +30,15 @@ namespace {
use App\Models\CloudHost;
use App\Models\CloudHostCapacitySnapshot;
use App\Models\CloudHostNetworkAttachment;
use App\Models\CloudHostResourcePoolMembership;
use App\Models\CloudPublicIp;
use App\Models\CloudPublicIpPool;
use App\Models\CloudResourcePool;
use App\Models\User;
use App\Services\Cloud\CloudEngineClient;
use App\Services\Cloud\CloudHostInventorySyncService;
use App\Services\Cloud\CloudHostPoolMembershipService;
use GuzzleHttp\Client;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
@ -50,7 +55,13 @@ namespace {
$json = static fn (array $value): string => json_encode(
$value, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
);
$capacity = static function (int $observedAt, int $cpu = 16): array {
$poolA = str_repeat('a', 32);
$poolB = str_repeat('b', 32);
$missingPoolKey = str_repeat('c', 32);
$capacity = static function (int $observedAt, int $cpu = 16) use (
$poolA, $poolB
): array {
return [
'health' => 'healthy',
'observed_at' => $observedAt,
@ -61,23 +72,85 @@ namespace {
'available' => 24576],
'storage_bytes' => ['total' => 1099511627776,
'allocated' => 274877906944, 'available' => 824633720832],
'public_ipv4' => ['configured' => true, 'total' => 6,
'reserved' => 1, 'allocated' => 2, 'available' => 3],
'public_ipv4' => [
'configured' => true,
'total' => 8,
'reserved' => 3,
'allocated' => 1,
'available' => 4,
'compatibility_mode' => false,
'pools' => [[
'pool_key' => $poolA,
'revision' => 4,
'state' => 'active',
'network_selector' => 'default',
'total' => 6,
'reserved' => 2,
'allocated' => 1,
'available' => 3,
], [
'pool_key' => $poolB,
'revision' => 2,
'state' => 'draining',
'network_selector' => 'default',
'total' => 2,
'reserved' => 1,
'allocated' => 0,
'available' => 1,
]],
],
];
};
$network = static function (int $count = 3): array {
return [
$network = static function () use ($poolA, $poolB): array {
$first = [
'configuration_state' => 'ready',
'pool_key' => $poolA,
'revision' => 4,
'state' => 'active',
'network_selector' => 'default',
'pool_cidr' => '8.8.8.0/29',
'guest_prefix' => 29,
'gateway' => '8.8.8.1',
'dns_servers' => ['1.1.1.1'],
'reserved_ips' => ['8.8.8.2'],
'usable_count' => 6,
'reserved_count' => 2,
'allocated_count' => 1,
'available_ips' => ['8.8.8.3/29', '8.8.8.5/29', '8.8.8.6/29'],
'available_count' => $count,
'available_count' => 3,
];
$second = [
'configuration_state' => 'ready',
'pool_key' => $poolB,
'revision' => 2,
'state' => 'draining',
'network_selector' => 'default',
'pool_cidr' => '9.9.9.0/30',
'guest_prefix' => 30,
'gateway' => '9.9.9.1',
'dns_servers' => ['1.0.0.1'],
'reserved_ips' => [],
'usable_count' => 2,
'reserved_count' => 1,
'allocated_count' => 0,
'available_ips' => ['9.9.9.2/30'],
'available_count' => 1,
];
return [
'configuration_state' => 'ready',
'pool_cidr' => $first['pool_cidr'],
'guest_prefix' => $first['guest_prefix'],
'gateway' => $first['gateway'],
'dns_servers' => $first['dns_servers'],
'available_ips' => $first['available_ips'],
'available_count' => $first['available_count'],
'pools' => [$first, $second],
'compatibility_mode' => false,
'message' => 'validated test response',
];
};
$client = static function (array $responses, ?array &$history = null): CloudEngineClient {
$client = static function (array $responses,
?array &$history = null): CloudEngineClient {
$stack = HandlerStack::create(new MockHandler($responses));
if ($history !== null) {
$stack->push(Middleware::history($history));
@ -95,17 +168,44 @@ namespace {
$table->integer('enable')->default(1);
$table->integer('is_admin')->default(0);
});
$schema->create('cloud_resource_pools', function (Blueprint $table): void {
$table->increments('id');
$table->integer('line_id');
$table->string('code');
$table->string('name');
$table->string('status')->default('active');
$table->string('scheduling_policy')->default('balanced');
$table->integer('sort')->default(0);
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_hosts', function (Blueprint $table): void {
$table->increments('id');
$table->string('name');
$table->integer('enabled')->default(0);
$table->integer('resource_pool_id')->nullable();
$table->string('scheduler_state')->default('disabled');
$table->string('health_state')->default('unknown');
$table->integer('last_seen_at')->nullable();
$table->integer('health_checked_at')->nullable();
$table->string('health_error_code')->default('');
$table->string('default_network')->default('default');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_host_resource_pool_memberships',
function (Blueprint $table): void {
$table->increments('id');
$table->integer('host_id');
$table->integer('resource_pool_id');
$table->string('scheduler_state')->default('disabled');
$table->integer('priority')->default(0);
$table->integer('weight')->default(100);
$table->integer('created_at');
$table->integer('updated_at');
$table->unique(['host_id', 'resource_pool_id']);
}
);
$schema->create('cloud_host_capacity_snapshots', function (Blueprint $table): void {
$table->integer('host_id')->primary();
foreach (['cpu_total_cores', 'cpu_allocated_cores', 'cpu_available_cores',
@ -124,9 +224,53 @@ namespace {
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_host_network_attachments',
function (Blueprint $table): void {
$table->increments('id');
$table->string('attachment_key');
$table->integer('host_id');
$table->integer('line_id');
$table->string('name');
$table->string('network_mode');
$table->string('network_selector');
$table->integer('vlan_id')->nullable();
$table->integer('routing_table')->nullable();
$table->string('state');
$table->integer('priority');
$table->integer('desired_revision');
$table->integer('observed_revision');
$table->string('sync_state');
$table->string('last_error_code');
$table->integer('created_at');
$table->integer('updated_at');
$table->unique(['host_id', 'attachment_key']);
}
);
$schema->create('cloud_public_ip_pools', function (Blueprint $table): void {
$table->increments('id');
$table->string('pool_key');
$table->integer('host_id');
$table->integer('network_attachment_id');
$table->string('name');
$table->string('cidr');
$table->string('gateway');
$table->integer('guest_prefix_length');
$table->text('dns_servers_json');
$table->text('reserved_ips_json');
$table->string('state');
$table->integer('priority');
$table->integer('desired_revision');
$table->integer('observed_revision');
$table->string('sync_state');
$table->string('last_error_code');
$table->integer('created_at');
$table->integer('updated_at');
$table->unique(['host_id', 'pool_key']);
});
$schema->create('cloud_public_ips', function (Blueprint $table): void {
$table->increments('id');
$table->integer('host_id');
$table->integer('public_ip_pool_id')->nullable();
$table->string('address')->unique();
$table->integer('prefix_length');
$table->string('state')->default('available');
@ -142,39 +286,93 @@ namespace {
$admin->enable = 1;
$admin->is_admin = 1;
$admin->save();
$resourcePool = new CloudResourcePool();
$resourcePool->id = 1;
$resourcePool->line_id = 77;
$resourcePool->code = 'primary';
$resourcePool->name = 'Primary';
$resourcePool->status = 'active';
$resourcePool->scheduling_policy = 'balanced';
$resourcePool->sort = 0;
$resourcePool->created_at = $resourcePool->updated_at = 900;
$resourcePool->save();
$host = new CloudHost();
$host->name = 'test-host';
$host->enabled = 1;
$host->resource_pool_id = 1;
$host->scheduler_state = 'active';
$host->health_state = 'unknown';
$host->health_error_code = '';
$host->default_network = 'default';
$host->created_at = 900;
$host->updated_at = 900;
$host->save();
// This pool is deliberately omitted by the engine's successful complete
// catalog. Its free row must stop advertising stale capacity.
$missingAttachment = new CloudHostNetworkAttachment();
$missingAttachment->attachment_key = str_repeat('d', 32);
$missingAttachment->host_id = 1;
$missingAttachment->line_id = 77;
$missingAttachment->name = 'missing-network';
$missingAttachment->network_mode = 'network';
$missingAttachment->network_selector = 'missing-network';
$missingAttachment->state = 'active';
$missingAttachment->priority = 0;
$missingAttachment->desired_revision = 1;
$missingAttachment->observed_revision = 1;
$missingAttachment->sync_state = 'fresh';
$missingAttachment->last_error_code = '';
$missingAttachment->created_at = $missingAttachment->updated_at = 900;
$missingAttachment->save();
$missingPool = new CloudPublicIpPool();
$missingPool->pool_key = $missingPoolKey;
$missingPool->host_id = 1;
$missingPool->network_attachment_id = $missingAttachment->id;
$missingPool->name = 'Missing pool';
$missingPool->cidr = '11.0.0.0/30';
$missingPool->gateway = '11.0.0.1';
$missingPool->guest_prefix_length = 30;
$missingPool->dns_servers_json = '["1.1.1.1"]';
$missingPool->reserved_ips_json = '[]';
$missingPool->state = 'active';
$missingPool->priority = 0;
$missingPool->desired_revision = 1;
$missingPool->observed_revision = 1;
$missingPool->sync_state = 'fresh';
$missingPool->last_error_code = '';
$missingPool->created_at = $missingPool->updated_at = 900;
$missingPool->save();
$insertIp = static function (string $address, string $state,
?int $reservationId = null, ?int $instanceId = null): void {
$row = new CloudPublicIp();
$row->host_id = 1;
$row->address = $address;
$row->prefix_length = 29;
$row->state = $state;
$row->reservation_id = $reservationId;
$row->instance_id = $instanceId;
$row->operation_id = null;
$row->created_at = 900;
$row->updated_at = 900;
$row->save();
?int $poolId = null, ?int $reservationId = null,
?int $instanceId = null) use ($capsule): void {
// Raw insert intentionally represents an upgraded legacy row whose
// nullable pool identity has not yet been backfilled.
$capsule->getConnection()->table('cloud_public_ips')->insert([
'host_id' => 1,
'public_ip_pool_id' => $poolId,
'address' => $address,
'prefix_length' => str_starts_with($address, '8.') ? 29 : 30,
'state' => $state,
'reservation_id' => $reservationId,
'instance_id' => $instanceId,
'operation_id' => null,
'created_at' => 900,
'updated_at' => 900,
]);
};
$insertIp('8.8.8.2', 'available');
$insertIp('8.8.8.3', 'reserved', 501);
$insertIp('8.8.8.4', 'assigned', null, 601);
$insertIp('8.8.8.3', 'reserved', null, 501);
$insertIp('8.8.8.4', 'assigned', null, null, 601);
$insertIp('8.8.8.5', 'disabled');
$insertIp('11.0.0.2', 'available', (int) $missingPool->id);
$now = 1000;
$history = [];
$successClient = $client([
// A host clock may lead PHP by the accepted 300-second window. The
// local attempt boundary, not this engine timestamp, must drive later
// local attempt boundary, not this engine timestamp, drives later
// reservation accounting.
new Response(200, [], $json($capacity($now + 300))),
new Response(200, [], $json($network())),
@ -186,26 +384,68 @@ namespace {
$host->refresh();
$snapshot = CloudHostCapacitySnapshot::find(1);
$states = CloudPublicIp::orderBy('address')->pluck('state', 'address')->all();
$membership = CloudHostResourcePoolMembership::where('host_id', 1)->first();
$attachments = CloudHostNetworkAttachment::orderBy('id')->get();
$publicPools = CloudPublicIpPool::whereIn('pool_key', [$poolA, $poolB])
->orderBy('pool_key')->get()->keyBy('pool_key');
$assert($success['checked'] === 1 && $success['succeeded'] === 1
&& $success['errors'] === [], 'A valid host observation did not succeed.');
&& $success['errors'] === [], 'A valid multi-pool observation did not succeed.');
$assert($host->health_state === 'healthy' && $host->last_seen_at === 1000
&& $host->health_checked_at === 1000 && $host->health_error_code === '',
'Successful host health markers are incorrect.');
$assert($membership !== null && (int) $membership->resource_pool_id === 1
&& $membership->scheduler_state === 'active'
&& (int) $membership->priority === 0 && (int) $membership->weight === 100,
'The legacy scalar host membership was not safely backfilled.');
$assert($snapshot !== null && $snapshot->sync_state === 'fresh'
&& $snapshot->cpu_total_cores === 16
&& $snapshot->storage_total_bytes === 1099511627776
&& $snapshot->public_ip_available_count === 3
&& $snapshot->public_ip_total_count === 8
&& $snapshot->public_ip_reserved_count === 3
&& $snapshot->public_ip_allocated_count === 1
&& $snapshot->public_ip_available_count === 4
&& $snapshot->observed_at === 1300
&& $snapshot->last_attempt_at === 1000
&& $snapshot->attempt_generation === 1,
'Capacity or public-IP counts were not persisted exactly.');
'Aggregated capacity or public-IP counts were not persisted exactly.');
$assert($publicPools->count() === 2
&& (int) $publicPools[$poolA]->network_attachment_id
=== (int) $publicPools[$poolB]->network_attachment_id
&& $publicPools[$poolA]->reserved_ips_json === '["8.8.8.2"]'
&& $publicPools[$poolA]->state === 'active'
&& $publicPools[$poolB]->state === 'draining'
&& $publicPools[$poolA]->sync_state === 'fresh'
&& $publicPools[$poolB]->sync_state === 'fresh',
'Stable pool identities, reserved metadata or shared attachment were not synced.');
$defaultAttachment = $attachments->firstWhere(
'attachment_key', substr(hash('sha256', 'default'), 0, 32)
);
$missingAttachment->refresh();
$missingPool->refresh();
$assert($defaultAttachment !== null
&& (int) $defaultAttachment->line_id === 77
&& $defaultAttachment->network_selector === 'default'
&& $defaultAttachment->state === 'active'
&& $defaultAttachment->sync_state === 'fresh'
&& $missingAttachment->sync_state === 'failed'
&& $missingPool->sync_state === 'failed',
'Attachment upsert or missing-catalog diagnostics are incorrect.');
$assert($states === [
'11.0.0.2' => 'disabled',
'8.8.8.2' => 'disabled',
'8.8.8.3' => 'reserved',
'8.8.8.4' => 'assigned',
'8.8.8.5' => 'available',
'8.8.8.6' => 'available',
], 'Free-list sync overwrote local occupancy or retained a missing free address.');
'9.9.9.2' => 'disabled',
], 'Pool sync overwrote a local hold or retained stale missing-pool capacity.');
foreach (CloudPublicIp::where('address', 'like', '8.8.8.%')->get() as $ip) {
$assert((int) $ip->public_ip_pool_id === (int) $publicPools[$poolA]->id,
'A legacy address was not associated with its unique public pool.');
}
$assert((int) CloudPublicIp::where('address', '9.9.9.2')->first()
->public_ip_pool_id === (int) $publicPools[$poolB]->id,
'A new address was written without its public-pool association.');
$assert(count($history) === 2
&& (string) $history[0]['request']->getUri()
=== 'http://127.0.0.1:8000/internal/v1/hosts/1/host/capacity'
@ -222,15 +462,20 @@ namespace {
$beforeCapacity = $snapshot->toArray();
$beforeStates = $states;
$beforePools = CloudPublicIpPool::orderBy('id')->get()->map(
static fn ($row): array => $row->toArray()
)->all();
$now = 1010;
$notDue = (new CloudHostInventorySyncService(
$client([]), static function () use (&$now): int { return $now; }
))->sync(10);
$assert($notDue['checked'] === 0 && $notDue['errors'] === [],
'A host inside the configured due-time window was polled again.');
$invalidNetwork = $network();
$invalidNetwork['pools'][1]['available_count'] = 0;
$invalidClient = $client([
new Response(200, [], $json($capacity($now, 32))),
new Response(200, [], $json($network(2))),
new Response(200, [], $json($invalidNetwork)),
]);
$invalid = (new CloudHostInventorySyncService(
$invalidClient, static function () use (&$now): int { return $now; }
@ -238,20 +483,23 @@ namespace {
$host->refresh();
$snapshot->refresh();
$states = CloudPublicIp::orderBy('address')->pluck('state', 'address')->all();
$afterPools = CloudPublicIpPool::orderBy('id')->get()->map(
static fn ($row): array => $row->toArray()
)->all();
$assert($invalid['degraded'] === 1
&& $invalid['errors'] === ['host_inventory_sync_failed']
&& $host->health_state === 'degraded'
&& $host->health_error_code === 'host_inventory_network_invalid'
&& $host->last_seen_at === 1000 && $host->health_checked_at === 1010,
'Invalid network response did not preserve last success and mark degraded.');
'Invalid pool response did not preserve last success and mark degraded.');
foreach (['cpu_total_cores', 'storage_total_bytes', 'observed_at'] as $field) {
$assert($snapshot->{$field} === $beforeCapacity[$field],
'Failure replaced last-known capacity field: ' . $field);
}
$assert($snapshot->sync_state === 'failed'
&& $snapshot->last_error_code === 'host_inventory_network_invalid'
&& $states === $beforeStates,
'Failure cleared capacity/IP data or exposed an unfixed diagnostic.');
&& $states === $beforeStates && $afterPools === $beforePools,
'Failure cleared capacity, pool metadata or concrete IP state.');
$now = 1020;
$unavailableClient = $client([
@ -273,33 +521,93 @@ namespace {
&& !str_contains($persisted, 'private engine path'),
'Engine failure cleared last success or persisted an upstream error.');
// Same-second polls need a monotonic fence rather than timestamp ordering.
// Prove both dangerous completion orders: stale success after a newer
// failure, then stale failure after a newer success.
$reflection = new ReflectionClass(CloudHostInventorySyncService::class);
$beginAttempt = $reflection->getMethod('beginAttempt');
$apply = $reflection->getMethod('apply');
$markFailure = $reflection->getMethod('markFailure');
$parseCapacity = $reflection->getMethod('capacity');
$parseNetwork = $reflection->getMethod('network');
$reconcile = $reflection->getMethod('reconcilePublicInventory');
foreach ([$beginAttempt, $apply, $markFailure, $parseCapacity,
$parseNetwork] as $method) {
$parseNetwork, $reconcile] as $method) {
$method->setAccessible(true);
}
// Structural rejection happens before any persistent observation changes.
$expectInvalidNetwork = static function (array $payload, string $message) use (
$assert, $parseNetwork
): void {
try {
$parseNetwork->invoke(null, $payload);
} catch (Throwable $error) {
$assert($error instanceof UnexpectedValueException, $message);
return;
}
throw new RuntimeException($message);
};
$duplicateKey = $network();
$duplicateKey['pools'][1]['pool_key'] = $poolA;
$expectInvalidNetwork($duplicateKey, 'Duplicate pool keys were accepted.');
$overlap = $network();
$overlap['pools'][1]['pool_cidr'] = '8.8.8.0/29';
$overlap['pools'][1]['guest_prefix'] = 29;
$overlap['pools'][1]['gateway'] = '8.8.8.1';
$overlap['pools'][1]['reserved_ips'] = ['8.8.8.2'];
$overlap['pools'][1]['usable_count'] = 6;
$overlap['pools'][1]['reserved_count'] = 2;
$overlap['pools'][1]['allocated_count'] = 3;
$overlap['pools'][1]['available_ips'] = ['8.8.8.5/29'];
$overlap['pools'][1]['available_count'] = 1;
$expectInvalidNetwork($overlap, 'Overlapping CIDRs were accepted.');
$reservedAsFree = $network();
$reservedAsFree['pools'][0]['available_ips'][0] = '8.8.8.2/29';
$reservedAsFree['available_ips'][0] = '8.8.8.2/29';
$expectInvalidNetwork($reservedAsFree,
'An explicitly reserved address was accepted as free.');
$badKey = $network();
$badKey['pools'][0]['pool_key'] = '../../not-stable';
$expectInvalidNetwork($badKey, 'An unstable pool key was accepted.');
// A pre-catalog engine remains readable as one synthetic legacy pool.
$legacyCapacityPayload = $capacity(1020);
$legacyCapacityPayload['public_ipv4'] = [
'configured' => true,
'total' => 6,
'reserved' => 2,
'allocated' => 1,
'available' => 3,
];
$legacyNetworkPayload = $network();
unset($legacyNetworkPayload['pools'], $legacyNetworkPayload['compatibility_mode']);
$legacyCapacity = $parseCapacity->invoke(null, $legacyCapacityPayload, 1020);
$legacyNetwork = $parseNetwork->invoke(null, $legacyNetworkPayload);
[$legacySnapshot, $legacyPools] = $reconcile->invoke(
null, $legacyCapacity, $legacyNetwork
);
$assert(isset($legacyPools['legacy-default'])
&& $legacyPools['legacy-default']['reserved_ips_known'] === false
&& $legacySnapshot['public_ip_available_count'] === 3,
'The bounded pre-catalog compatibility path was lost.');
// Same-second polls need a monotonic generation fence rather than
// timestamp ordering. Prove both dangerous completion orders.
$now = 1030;
$fenced = new CloudHostInventorySyncService(
$client([]), static function () use (&$now): int { return $now; }
);
$parsedCapacity = $parseCapacity->invoke(null, $capacity($now, 24), $now);
$parsedNetwork = $parseNetwork->invoke(null, $network());
[$snapshotData, $poolData] = $reconcile->invoke(
null, $parsedCapacity, $parsedNetwork
);
$oldSuccess = $beginAttempt->invoke($fenced, 1, $now);
$newFailure = $beginAttempt->invoke($fenced, 1, $now);
$assert($newFailure === $oldSuccess + 1
&& $markFailure->invoke($fenced, 1, 'unavailable',
'host_inventory_unavailable', $now, $newFailure) === true
&& $apply->invoke($fenced, 1, $parsedCapacity,
$parsedNetwork['available'], $now, $oldSuccess) === false,
&& $apply->invoke($fenced, 1, $snapshotData,
$poolData, $now, $oldSuccess) === false,
'A stale success was not fenced after a newer same-second failure.');
$host->refresh();
$snapshot->refresh();
@ -310,8 +618,8 @@ namespace {
$oldFailure = $beginAttempt->invoke($fenced, 1, $now);
$newSuccess = $beginAttempt->invoke($fenced, 1, $now);
$assert($apply->invoke($fenced, 1, $parsedCapacity,
$parsedNetwork['available'], $now, $newSuccess) === true
$assert($apply->invoke($fenced, 1, $snapshotData,
$poolData, $now, $newSuccess) === true
&& $markFailure->invoke($fenced, 1, 'unavailable',
'host_inventory_unavailable', $now, $oldFailure) === false,
'A stale failure was not fenced after a newer same-second success.');
@ -323,5 +631,49 @@ namespace {
&& $snapshot->attempt_generation === $newSuccess,
'The stale failure overwrote a newer successful observation.');
echo "Cloud host capacity and public-IP lifecycle sync checks passed.\n";
// The first-release writer retains exactly one canonical relation, fixes
// priority/weight, and can explicitly clear the assignment.
$secondResourcePool = new CloudResourcePool();
$secondResourcePool->id = 2;
$secondResourcePool->line_id = 77;
$secondResourcePool->code = 'secondary';
$secondResourcePool->name = 'Secondary';
$secondResourcePool->status = 'active';
$secondResourcePool->scheduling_policy = 'balanced';
$secondResourcePool->sort = 1;
$secondResourcePool->created_at = $secondResourcePool->updated_at = 1030;
$secondResourcePool->save();
$capsule->getConnection()->transaction(static function (): void {
$lockedHost = CloudHost::where('id', 1)->lockForUpdate()->firstOrFail();
CloudHostPoolMembershipService::replaceLocked(
$lockedHost, 2, 'disabled', 1040
);
$lockedHost->updated_at = 1040;
$lockedHost->save();
});
$relations = CloudHostResourcePoolMembership::where('host_id', 1)->get();
$host->refresh();
$assert($relations->count() === 1
&& (int) $relations->first()->resource_pool_id === 2
&& $relations->first()->scheduler_state === 'disabled'
&& (int) $relations->first()->priority === 0
&& (int) $relations->first()->weight === 100
&& (int) $host->resource_pool_id === 2
&& $host->scheduler_state === 'disabled',
'Membership replacement retained a stale authority row or unsafe values.');
$capsule->getConnection()->transaction(static function (): void {
$lockedHost = CloudHost::where('id', 1)->lockForUpdate()->firstOrFail();
CloudHostPoolMembershipService::replaceLocked(
$lockedHost, null, 'disabled', 1050
);
$lockedHost->updated_at = 1050;
$lockedHost->save();
});
$host->refresh();
$assert(CloudHostResourcePoolMembership::where('host_id', 1)->count() === 0
&& $host->resource_pool_id === null
&& $host->scheduler_state === 'disabled',
'Clearing the authoritative membership recreated a stale scalar link.');
echo "Cloud host multi-pool inventory and membership checks passed.\n";
}

Loading…
Cancel
Save