You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudHostInventorySyncServi...

799 lines
36 KiB
PHP

<?php
namespace App\Services {
final class Config
{
public static array $values = [
'cloud_engine_url' => 'http://127.0.0.1:8000',
'super_admin' => '99',
];
public static function get($name, $default = null)
{
return self::$values[$name] ?? $default;
}
}
}
namespace App\Services\Cloud {
final class CloudSecrets
{
public static function key(string $purpose): string
{
return str_repeat('k', 32);
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
require __DIR__ . '/CloudHostInventoryDiagnosticsFixture.php';
use App\Models\CloudHost;
use App\Models\CloudHostCapacitySnapshot;
use App\Models\CloudHostInventoryAttempt;
use App\Models\CloudHostNetworkAttachment;
use App\Models\CloudHostResourcePoolMembership;
use App\Models\CloudPublicIp;
use App\Models\CloudPublicIpPool;
use App\Models\CloudResourcePool;
use App\Models\User;
use App\Services\Cloud\CloudEngineClient;
use App\Services\Cloud\CloudHostInventorySyncService;
use App\Services\Cloud\CloudHostPoolMembershipService;
use GuzzleHttp\Client;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Middleware;
use GuzzleHttp\Psr7\Response;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function ($condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$json = static fn (array $value): string => json_encode(
$value, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
);
$poolA = str_repeat('a', 32);
$poolB = str_repeat('b', 32);
$missingPoolKey = str_repeat('c', 32);
$capacity = static function (int $observedAt, int $cpu = 16) use (
$poolA, $poolB
): array {
return [
'health' => 'healthy',
'observed_at' => $observedAt,
'instances' => ['total' => 3, 'running' => 2],
'cpu' => ['total' => $cpu, 'allocated' => 4,
'available' => max(0, $cpu - 4)],
'memory_mib' => ['total' => 32768, 'allocated' => 8192,
'available' => 24576],
'storage_bytes' => ['total' => 1099511627776,
'allocated' => 274877906944, 'available' => 824633720832],
'public_ipv4' => [
'configured' => true,
'total' => 8,
'reserved' => 3,
'allocated' => 1,
'available' => 4,
'compatibility_mode' => false,
'pools' => [[
'pool_key' => $poolA,
'revision' => 4,
'state' => 'active',
'network_selector' => 'default',
'total' => 6,
'reserved' => 2,
'allocated' => 1,
'available' => 3,
], [
'pool_key' => $poolB,
'revision' => 2,
'state' => 'draining',
'network_selector' => 'default',
'total' => 2,
'reserved' => 1,
'allocated' => 0,
'available' => 1,
]],
],
];
};
$network = static function () use ($poolA, $poolB): array {
$first = [
'configuration_state' => 'ready',
'pool_key' => $poolA,
'revision' => 4,
'state' => 'active',
'network_selector' => 'default',
'pool_cidr' => '8.8.8.0/29',
'guest_prefix' => 29,
'gateway' => '8.8.8.1',
'dns_servers' => ['1.1.1.1'],
'reserved_ips' => ['8.8.8.2'],
'usable_count' => 6,
'reserved_count' => 2,
'allocated_count' => 1,
'available_ips' => ['8.8.8.3/29', '8.8.8.5/29', '8.8.8.6/29'],
'available_count' => 3,
];
$second = [
'configuration_state' => 'ready',
'pool_key' => $poolB,
'revision' => 2,
'state' => 'draining',
'network_selector' => 'default',
'pool_cidr' => '9.9.9.0/30',
'guest_prefix' => 30,
'gateway' => '9.9.9.1',
'dns_servers' => ['1.0.0.1'],
'reserved_ips' => [],
'usable_count' => 2,
'reserved_count' => 1,
'allocated_count' => 0,
'available_ips' => ['9.9.9.2/30'],
'available_count' => 1,
];
return [
'configuration_state' => 'ready',
'pool_cidr' => $first['pool_cidr'],
'guest_prefix' => $first['guest_prefix'],
'gateway' => $first['gateway'],
'dns_servers' => $first['dns_servers'],
'available_ips' => $first['available_ips'],
'available_count' => $first['available_count'],
'pools' => [$first, $second],
'compatibility_mode' => false,
'message' => 'validated test response',
];
};
$client = static function (array $responses,
?array &$history = null): CloudEngineClient {
$stack = HandlerStack::create(new MockHandler($responses));
if ($history !== null) {
$stack->push(Middleware::history($history));
}
return new CloudEngineClient(new Client(['handler' => $stack]), 3.0);
};
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
createCloudHostInventoryAttemptTable($schema);
$schema->create('user', function (Blueprint $table): void {
$table->increments('id');
$table->integer('enable')->default(1);
$table->integer('is_admin')->default(0);
});
$schema->create('cloud_resource_pools', function (Blueprint $table): void {
$table->increments('id');
$table->integer('line_id');
$table->string('code');
$table->string('name');
$table->string('status')->default('active');
$table->string('scheduling_policy')->default('balanced');
$table->integer('sort')->default(0);
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_hosts', function (Blueprint $table): void {
$table->increments('id');
$table->string('name');
$table->integer('enabled')->default(0);
$table->integer('resource_pool_id')->nullable();
$table->string('scheduler_state')->default('disabled');
$table->string('health_state')->default('unknown');
$table->integer('last_seen_at')->nullable();
$table->integer('health_checked_at')->nullable();
$table->string('health_error_code')->default('');
$table->string('default_network')->default('default');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_host_resource_pool_memberships',
function (Blueprint $table): void {
$table->increments('id');
$table->integer('host_id');
$table->integer('resource_pool_id');
$table->string('scheduler_state')->default('disabled');
$table->integer('priority')->default(0);
$table->integer('weight')->default(100);
$table->integer('created_at');
$table->integer('updated_at');
$table->unique(['host_id', 'resource_pool_id']);
}
);
$schema->create('cloud_host_capacity_snapshots', function (Blueprint $table): void {
$table->integer('host_id')->primary();
foreach (['cpu_total_cores', 'cpu_allocated_cores', 'cpu_available_cores',
'memory_total_mib', 'memory_allocated_mib', 'memory_available_mib',
'storage_total_bytes', 'storage_allocated_bytes', 'storage_available_bytes',
'instance_total_count', 'instance_running_count',
'public_ip_total_count', 'public_ip_reserved_count',
'public_ip_allocated_count', 'public_ip_available_count',
'observed_at', 'last_attempt_at', 'attempt_generation',
'revision'] as $field) {
$table->integer($field)->default(0);
}
$table->integer('public_ip_configured')->default(0);
$table->string('sync_state')->default('never');
$table->string('last_error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_host_network_attachments',
function (Blueprint $table): void {
$table->increments('id');
$table->string('attachment_key');
$table->integer('host_id');
$table->integer('line_id');
$table->string('name');
$table->string('network_mode');
$table->string('network_selector');
$table->integer('vlan_id')->nullable();
$table->integer('routing_table')->nullable();
$table->string('state');
$table->integer('priority');
$table->integer('desired_revision');
$table->integer('observed_revision');
$table->string('sync_state');
$table->string('last_error_code');
$table->integer('created_at');
$table->integer('updated_at');
$table->unique(['host_id', 'attachment_key']);
}
);
$schema->create('cloud_public_ip_pools', function (Blueprint $table): void {
$table->increments('id');
$table->string('pool_key');
$table->integer('host_id');
$table->integer('network_attachment_id');
$table->string('name');
$table->string('cidr');
$table->string('gateway');
$table->integer('guest_prefix_length');
$table->text('dns_servers_json');
$table->text('reserved_ips_json');
$table->string('state');
$table->integer('priority');
$table->integer('desired_revision');
$table->integer('observed_revision');
$table->string('sync_state');
$table->string('last_error_code');
$table->integer('created_at');
$table->integer('updated_at');
$table->unique(['host_id', 'pool_key']);
});
$schema->create('cloud_public_ips', function (Blueprint $table): void {
$table->increments('id');
$table->integer('host_id');
$table->integer('public_ip_pool_id')->nullable();
$table->string('address')->unique();
$table->integer('prefix_length');
$table->string('state')->default('available');
$table->integer('reservation_id')->nullable();
$table->integer('instance_id')->nullable();
$table->string('operation_id')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$admin = new User();
$admin->id = 99;
$admin->enable = 1;
$admin->is_admin = 1;
$admin->save();
$resourcePool = new CloudResourcePool();
$resourcePool->id = 1;
$resourcePool->line_id = 77;
$resourcePool->code = 'primary';
$resourcePool->name = 'Primary';
$resourcePool->status = 'active';
$resourcePool->scheduling_policy = 'balanced';
$resourcePool->sort = 0;
$resourcePool->created_at = $resourcePool->updated_at = 900;
$resourcePool->save();
$host = new CloudHost();
$host->name = 'test-host';
$host->enabled = 1;
$host->resource_pool_id = 1;
$host->scheduler_state = 'active';
$host->health_state = 'unknown';
$host->health_error_code = '';
$host->default_network = 'default';
$host->created_at = 900;
$host->updated_at = 900;
$host->save();
// This pool is deliberately omitted by the engine's successful complete
// catalog. Its free row must stop advertising stale capacity.
$missingAttachment = new CloudHostNetworkAttachment();
$missingAttachment->attachment_key = str_repeat('d', 32);
$missingAttachment->host_id = 1;
$missingAttachment->line_id = 77;
$missingAttachment->name = 'missing-network';
$missingAttachment->network_mode = 'network';
$missingAttachment->network_selector = 'missing-network';
$missingAttachment->state = 'active';
$missingAttachment->priority = 0;
$missingAttachment->desired_revision = 1;
$missingAttachment->observed_revision = 1;
$missingAttachment->sync_state = 'fresh';
$missingAttachment->last_error_code = '';
$missingAttachment->created_at = $missingAttachment->updated_at = 900;
$missingAttachment->save();
$missingPool = new CloudPublicIpPool();
$missingPool->pool_key = $missingPoolKey;
$missingPool->host_id = 1;
$missingPool->network_attachment_id = $missingAttachment->id;
$missingPool->name = 'Missing pool';
$missingPool->cidr = '11.0.0.0/30';
$missingPool->gateway = '11.0.0.1';
$missingPool->guest_prefix_length = 30;
$missingPool->dns_servers_json = '["1.1.1.1"]';
$missingPool->reserved_ips_json = '[]';
$missingPool->state = 'active';
$missingPool->priority = 0;
$missingPool->desired_revision = 1;
$missingPool->observed_revision = 1;
$missingPool->sync_state = 'fresh';
$missingPool->last_error_code = '';
$missingPool->created_at = $missingPool->updated_at = 900;
$missingPool->save();
$insertIp = static function (string $address, string $state,
?int $poolId = null, ?int $reservationId = null,
?int $instanceId = null) use ($capsule): void {
// Raw insert intentionally represents an upgraded legacy row whose
// nullable pool identity has not yet been backfilled.
$capsule->getConnection()->table('cloud_public_ips')->insert([
'host_id' => 1,
'public_ip_pool_id' => $poolId,
'address' => $address,
'prefix_length' => str_starts_with($address, '8.') ? 29 : 30,
'state' => $state,
'reservation_id' => $reservationId,
'instance_id' => $instanceId,
'operation_id' => null,
'created_at' => 900,
'updated_at' => 900,
]);
};
$insertIp('8.8.8.2', 'available');
$insertIp('8.8.8.3', 'reserved', null, 501);
$insertIp('8.8.8.4', 'assigned', null, null, 601);
$insertIp('8.8.8.5', 'disabled');
$insertIp('11.0.0.2', 'available', (int) $missingPool->id);
$now = 1000;
$history = [];
$successClient = $client([
// A host clock may lead PHP by the accepted 300-second window. The
// local attempt boundary, not this engine timestamp, drives later
// reservation accounting.
new Response(200, [], $json($capacity($now + 300))),
new Response(200, [], $json($network())),
], $history);
$success = (new CloudHostInventorySyncService(
$successClient, static function () use (&$now): int { return $now; }
))->sync(10);
$host->refresh();
$snapshot = CloudHostCapacitySnapshot::find(1);
$states = CloudPublicIp::orderBy('address')->pluck('state', 'address')->all();
$membership = CloudHostResourcePoolMembership::where('host_id', 1)->first();
$attachments = CloudHostNetworkAttachment::orderBy('id')->get();
$publicPools = CloudPublicIpPool::whereIn('pool_key', [$poolA, $poolB])
->orderBy('pool_key')->get()->keyBy('pool_key');
$assert($success['checked'] === 1 && $success['succeeded'] === 1
&& $success['errors'] === [], 'A valid multi-pool observation did not succeed.');
$assert($host->health_state === 'healthy' && $host->last_seen_at === 1000
&& $host->health_checked_at === 1000 && $host->health_error_code === '',
'Successful host health markers are incorrect.');
$assert($membership !== null && (int) $membership->resource_pool_id === 1
&& $membership->scheduler_state === 'active'
&& (int) $membership->priority === 0 && (int) $membership->weight === 100,
'The legacy scalar host membership was not safely backfilled.');
$assert($snapshot !== null && $snapshot->sync_state === 'fresh'
&& $snapshot->cpu_total_cores === 16
&& $snapshot->storage_total_bytes === 1099511627776
&& $snapshot->public_ip_total_count === 8
&& $snapshot->public_ip_reserved_count === 3
&& $snapshot->public_ip_allocated_count === 1
&& $snapshot->public_ip_available_count === 4
&& $snapshot->observed_at === 1300
&& $snapshot->last_attempt_at === 1000
&& $snapshot->attempt_generation === 1,
'Aggregated capacity or public-IP counts were not persisted exactly.');
$assert($publicPools->count() === 2
&& (int) $publicPools[$poolA]->network_attachment_id
=== (int) $publicPools[$poolB]->network_attachment_id
&& $publicPools[$poolA]->reserved_ips_json === '["8.8.8.2"]'
&& $publicPools[$poolA]->state === 'active'
&& $publicPools[$poolB]->state === 'draining'
&& $publicPools[$poolA]->sync_state === 'fresh'
&& $publicPools[$poolB]->sync_state === 'fresh',
'Stable pool identities, reserved metadata or shared attachment were not synced.');
$defaultAttachment = $attachments->firstWhere(
'attachment_key', substr(hash('sha256', 'default'), 0, 32)
);
$missingAttachment->refresh();
$missingPool->refresh();
$assert($defaultAttachment !== null
&& (int) $defaultAttachment->line_id === 77
&& $defaultAttachment->network_selector === 'default'
&& $defaultAttachment->state === 'active'
&& $defaultAttachment->sync_state === 'fresh'
&& $missingAttachment->sync_state === 'failed'
&& $missingPool->sync_state === 'failed',
'Attachment upsert or missing-catalog diagnostics are incorrect.');
$assert($states === [
'11.0.0.2' => 'disabled',
'8.8.8.2' => 'disabled',
'8.8.8.3' => 'reserved',
'8.8.8.4' => 'assigned',
'8.8.8.5' => 'available',
'8.8.8.6' => 'available',
'9.9.9.2' => 'disabled',
], 'Pool sync overwrote a local hold or retained stale missing-pool capacity.');
foreach (CloudPublicIp::where('address', 'like', '8.8.8.%')->get() as $ip) {
$assert((int) $ip->public_ip_pool_id === (int) $publicPools[$poolA]->id,
'A legacy address was not associated with its unique public pool.');
}
$assert((int) CloudPublicIp::where('address', '9.9.9.2')->first()
->public_ip_pool_id === (int) $publicPools[$poolB]->id,
'A new address was written without its public-pool association.');
$assert(count($history) === 2
&& $history[0]['request']->getUri()->getPath()
=== '/internal/v1/hosts/1/host/capacity'
&& $history[1]['request']->getUri()->getPath()
=== '/internal/v1/hosts/1/host/network-options'
&& preg_match('/^inventory_attempt_id=[0-9a-f]{32}$/D',
$history[0]['request']->getUri()->getQuery()) === 1
&& $history[0]['request']->getUri()->getQuery()
=== $history[1]['request']->getUri()->getQuery(),
'Lifecycle inventory did not use both bounded host endpoints.');
$actorHeader = $history[0]['request']->getHeaderLine('X-Cloud-Actor');
$actorHeader .= str_repeat('=', (4 - strlen($actorHeader) % 4) % 4);
$actor = json_decode(base64_decode(strtr($actorHeader, '-_', '+/'), true), true);
$record = CloudHostInventoryAttempt::orderByDesc('sequence_id')->firstOrFail();
$assert($record->outcome === 'succeeded' && $record->applied
&& $record->source === 'auto' && $record->stage === 'database_apply'
&& (int) $record->timeout_ms === 3000
&& (int) $record->started_at === 1000
&& (int) $record->finished_at >= 1000
&& $history[0]['request']->getUri()->getQuery()
=== 'inventory_attempt_id=' . $record->attempt_id,
'A full committed inventory needs its own successful correlated history record.');
$assert(($actor['user_id'] ?? null) === 99
&& ($actor['session_id'] ?? null) === 'cloud-host-inventory-sync'
&& ($actor['is_super_admin'] ?? null) === true,
'Shared daemon system actor was not used for inventory requests.');
$beforeCapacity = $snapshot->toArray();
$beforeStates = $states;
$beforePools = CloudPublicIpPool::orderBy('id')->get()->map(
static fn ($row): array => $row->toArray()
)->all();
$now = 1010;
$notDue = (new CloudHostInventorySyncService(
$client([]), static function () use (&$now): int { return $now; }
))->sync(10);
$assert($notDue['checked'] === 0 && $notDue['errors'] === [],
'A host inside the configured due-time window was polled again.');
$invalidNetwork = $network();
$invalidNetwork['pools'][1]['available_count'] = 0;
$invalidClient = $client([
new Response(200, [], $json($capacity($now, 32))),
new Response(200, [], $json($invalidNetwork)),
]);
$invalid = (new CloudHostInventorySyncService(
$invalidClient, static function () use (&$now): int { return $now; }
))->syncOne(1);
$host->refresh();
$snapshot->refresh();
$states = CloudPublicIp::orderBy('address')->pluck('state', 'address')->all();
$afterPools = CloudPublicIpPool::orderBy('id')->get()->map(
static fn ($row): array => $row->toArray()
)->all();
$assert($invalid['degraded'] === 1
&& $invalid['errors'] === ['host_inventory_sync_failed']
&& $host->health_state === 'degraded'
&& $host->health_error_code === 'host_inventory_network_invalid'
&& $host->last_seen_at === 1000 && $host->health_checked_at === 1010,
'Invalid pool response did not preserve last success and mark degraded.');
foreach (['cpu_total_cores', 'storage_total_bytes', 'observed_at'] as $field) {
$assert($snapshot->{$field} === $beforeCapacity[$field],
'Failure replaced last-known capacity field: ' . $field);
}
$assert($snapshot->sync_state === 'failed'
&& $snapshot->last_error_code === 'host_inventory_network_invalid'
&& $states === $beforeStates && $afterPools === $beforePools,
'Failure cleared capacity, pool metadata or concrete IP state.');
$record = CloudHostInventoryAttempt::orderByDesc('sequence_id')->firstOrFail();
$assert($record->outcome === 'failed' && !$record->applied
&& $record->source === 'manual' && $record->stage === 'network_validate'
&& $record->error_code === 'host_inventory_network_invalid',
'Network validation failure must retain its layer and outcome.');
$now = 1020;
$unavailableClient = $client([
new Response(503, [], '{"error":"private engine path must never persist"}'),
]);
$unavailable = (new CloudHostInventorySyncService(
$unavailableClient, static function () use (&$now): int { return $now; }
))->syncOne(1);
$host->refresh();
$snapshot->refresh();
$persisted = json_encode([
$host->health_error_code,
$snapshot->last_error_code,
], JSON_THROW_ON_ERROR);
$assert($unavailable['unavailable'] === 1
&& $host->health_state === 'unavailable'
&& $host->health_error_code === 'host_inventory_unavailable'
&& $host->last_seen_at === 1000 && $host->health_checked_at === 1020
&& !str_contains($persisted, 'private engine path'),
'Engine failure cleared last success or persisted an upstream error.');
$record = CloudHostInventoryAttempt::orderByDesc('sequence_id')->firstOrFail();
$assert($record->error_code === 'engine_service_error' && (int) $record->http_status === 503
&& $record->stage === 'capacity_request' && !$record->applied
&& !str_contains(json_encode($record->toArray()), 'private engine path'),
'Transport and upstream HTTP evidence must survive safe classification.');
$reflection = new ReflectionClass(CloudHostInventorySyncService::class);
$beginAttempt = $reflection->getMethod('beginAttempt');
$apply = $reflection->getMethod('apply');
$markFailure = $reflection->getMethod('markFailure');
$parseCapacity = $reflection->getMethod('capacity');
$parseNetwork = $reflection->getMethod('network');
$reconcile = $reflection->getMethod('reconcilePublicInventory');
foreach ([$beginAttempt, $apply, $markFailure, $parseCapacity,
$parseNetwork, $reconcile] as $method) {
$method->setAccessible(true);
}
// Structural rejection happens before any persistent observation changes.
$expectInvalidNetwork = static function (array $payload, string $message) use (
$assert, $parseNetwork
): void {
try {
$parseNetwork->invoke(null, $payload);
} catch (Throwable $error) {
$assert($error instanceof UnexpectedValueException, $message);
return;
}
throw new RuntimeException($message);
};
$duplicateKey = $network();
$duplicateKey['pools'][1]['pool_key'] = $poolA;
$expectInvalidNetwork($duplicateKey, 'Duplicate pool keys were accepted.');
$overlap = $network();
$overlap['pools'][1]['pool_cidr'] = '8.8.8.0/29';
$overlap['pools'][1]['guest_prefix'] = 29;
$overlap['pools'][1]['gateway'] = '8.8.8.1';
$overlap['pools'][1]['reserved_ips'] = ['8.8.8.2'];
$overlap['pools'][1]['usable_count'] = 6;
$overlap['pools'][1]['reserved_count'] = 2;
$overlap['pools'][1]['allocated_count'] = 3;
$overlap['pools'][1]['available_ips'] = ['8.8.8.5/29'];
$overlap['pools'][1]['available_count'] = 1;
$expectInvalidNetwork($overlap, 'Overlapping CIDRs were accepted.');
$reservedAsFree = $network();
$reservedAsFree['pools'][0]['available_ips'][0] = '8.8.8.2/29';
$reservedAsFree['available_ips'][0] = '8.8.8.2/29';
$expectInvalidNetwork($reservedAsFree,
'An explicitly reserved address was accepted as free.');
$badKey = $network();
$badKey['pools'][0]['pool_key'] = '../../not-stable';
$expectInvalidNetwork($badKey, 'An unstable pool key was accepted.');
// A pre-catalog engine remains readable as one synthetic legacy pool.
$legacyCapacityPayload = $capacity(1020);
$legacyCapacityPayload['public_ipv4'] = [
'configured' => true,
'total' => 6,
'reserved' => 2,
'allocated' => 1,
'available' => 3,
];
$legacyNetworkPayload = $network();
unset($legacyNetworkPayload['pools'], $legacyNetworkPayload['compatibility_mode']);
$legacyCapacity = $parseCapacity->invoke(null, $legacyCapacityPayload, 1020);
$legacyNetwork = $parseNetwork->invoke(null, $legacyNetworkPayload);
[$legacySnapshot, $legacyPools] = $reconcile->invoke(
null, $legacyCapacity, $legacyNetwork
);
$assert(isset($legacyPools['legacy-default'])
&& $legacyPools['legacy-default']['reserved_ips_known'] === false
&& $legacySnapshot['public_ip_available_count'] === 3,
'The bounded pre-catalog compatibility path was lost.');
// Same-second polls need a monotonic generation fence rather than
// timestamp ordering. Prove both dangerous completion orders.
$now = 1030;
$fenced = new CloudHostInventorySyncService(
$client([]), static function () use (&$now): int { return $now; }
);
$parsedCapacity = $parseCapacity->invoke(null, $capacity($now, 24), $now);
$parsedNetwork = $parseNetwork->invoke(null, $network());
[$snapshotData, $poolData] = $reconcile->invoke(
null, $parsedCapacity, $parsedNetwork
);
$oldSuccess = $beginAttempt->invoke($fenced, 1, $now);
$newFailure = $beginAttempt->invoke($fenced, 1, $now);
$assert($newFailure === $oldSuccess + 1
&& $markFailure->invoke($fenced, 1, 'unavailable',
'host_inventory_unavailable', $now, $newFailure) === true
&& $apply->invoke($fenced, 1, $snapshotData,
$poolData, $now, $oldSuccess) === false,
'A stale success was not fenced after a newer same-second failure.');
$host->refresh();
$snapshot->refresh();
$assert($host->health_state === 'unavailable'
&& $snapshot->sync_state === 'failed'
&& $snapshot->attempt_generation === $newFailure,
'The stale success overwrote a newer failed observation.');
$oldFailure = $beginAttempt->invoke($fenced, 1, $now);
$newSuccess = $beginAttempt->invoke($fenced, 1, $now);
$assert($apply->invoke($fenced, 1, $snapshotData,
$poolData, $now, $newSuccess) === true
&& $markFailure->invoke($fenced, 1, 'unavailable',
'host_inventory_unavailable', $now, $oldFailure) === false,
'A stale failure was not fenced after a newer same-second success.');
$host->refresh();
$snapshot->refresh();
$assert($host->health_state === 'healthy'
&& $snapshot->sync_state === 'fresh'
&& $snapshot->cpu_total_cores === 24
&& $snapshot->attempt_generation === $newSuccess,
'The stale failure overwrote a newer successful observation.');
// An older engine observation may reconcile the concrete IP ledger, but
// it must never erase a newer administrator-authored desired definition.
$pendingPool = CloudPublicIpPool::where('pool_key', $poolB)->firstOrFail();
$pendingPool->cidr = '9.9.10.0/30';
$pendingPool->gateway = '9.9.10.1';
$pendingPool->guest_prefix_length = 30;
$pendingPool->dns_servers_json = '["9.9.9.9"]';
$pendingPool->reserved_ips_json = '[]';
$pendingPool->state = 'draining';
$pendingPool->desired_revision = 3;
$pendingPool->observed_revision = 2;
$pendingPool->sync_state = 'pending';
$pendingPool->last_error_code = 'host_public_pool_outcome_unknown';
$pendingPool->save();
$staleAttempt = $beginAttempt->invoke($fenced, 1, ++$now);
$assert($apply->invoke($fenced, 1, $snapshotData,
$poolData, $now, $staleAttempt) === true,
'The older engine catalog could not be applied safely.');
$pendingPool->refresh();
$assert($pendingPool->cidr === '9.9.10.0/30'
&& $pendingPool->gateway === '9.9.10.1'
&& $pendingPool->dns_servers_json === '["9.9.9.9"]'
&& $pendingPool->reserved_ips_json === '[]'
&& $pendingPool->state === 'draining'
&& (int) $pendingPool->desired_revision === 3
&& (int) $pendingPool->observed_revision === 2
&& $pendingPool->sync_state === 'pending'
&& $pendingPool->last_error_code
=== 'host_public_pool_outcome_unknown',
'An older observation erased desired pool intent or its diagnostic.');
$assert(CloudPublicIp::where('address', '9.9.9.2')->first()->state
=== 'disabled',
'The older observed IP ledger was not retained while intent was pending.');
$confirmedCapacityPayload = $capacity($now, 24);
$confirmedCapacityPayload['public_ipv4']['pools'][1]['revision'] = 3;
$confirmedNetworkPayload = $network();
$confirmedNetworkPayload['pools'][1]['revision'] = 3;
$confirmedNetworkPayload['pools'][1]['pool_cidr'] = '9.9.10.0/30';
$confirmedNetworkPayload['pools'][1]['gateway'] = '9.9.10.1';
$confirmedNetworkPayload['pools'][1]['dns_servers'] = ['9.9.9.9'];
$confirmedNetworkPayload['pools'][1]['available_ips'] = ['9.9.10.2/30'];
$confirmedCapacity = $parseCapacity->invoke(
null, $confirmedCapacityPayload, $now
);
$confirmedNetwork = $parseNetwork->invoke(null, $confirmedNetworkPayload);
[$confirmedSnapshot, $confirmedPools] = $reconcile->invoke(
null, $confirmedCapacity, $confirmedNetwork
);
$confirmedAttempt = $beginAttempt->invoke($fenced, 1, ++$now);
$assert($apply->invoke($fenced, 1, $confirmedSnapshot,
$confirmedPools, $now, $confirmedAttempt) === true,
'The desired engine revision could not be confirmed.');
$pendingPool->refresh();
$assert((int) $pendingPool->desired_revision === 3
&& (int) $pendingPool->observed_revision === 3
&& $pendingPool->sync_state === 'fresh'
&& $pendingPool->last_error_code === '',
'A matching engine revision did not make the desired pool fresh.');
// The first-release writer retains exactly one canonical relation, fixes
// priority/weight, and can explicitly clear the assignment.
$secondResourcePool = new CloudResourcePool();
$secondResourcePool->id = 2;
$secondResourcePool->line_id = 77;
$secondResourcePool->code = 'secondary';
$secondResourcePool->name = 'Secondary';
$secondResourcePool->status = 'active';
$secondResourcePool->scheduling_policy = 'balanced';
$secondResourcePool->sort = 1;
$secondResourcePool->created_at = $secondResourcePool->updated_at = 1030;
$secondResourcePool->save();
$capsule->getConnection()->transaction(static function (): void {
$lockedHost = CloudHost::where('id', 1)->lockForUpdate()->firstOrFail();
CloudHostPoolMembershipService::replaceLocked(
$lockedHost, 2, 'disabled', 1040
);
$lockedHost->updated_at = 1040;
$lockedHost->save();
});
$relations = CloudHostResourcePoolMembership::where('host_id', 1)->get();
$host->refresh();
$assert($relations->count() === 1
&& (int) $relations->first()->resource_pool_id === 2
&& $relations->first()->scheduler_state === 'disabled'
&& (int) $relations->first()->priority === 0
&& (int) $relations->first()->weight === 100
&& (int) $host->resource_pool_id === 2
&& $host->scheduler_state === 'disabled',
'Membership replacement retained a stale authority row or unsafe values.');
$capsule->getConnection()->transaction(static function (): void {
$lockedHost = CloudHost::where('id', 1)->lockForUpdate()->firstOrFail();
CloudHostPoolMembershipService::replaceLocked(
$lockedHost, null, 'disabled', 1050
);
$lockedHost->updated_at = 1050;
$lockedHost->save();
});
$host->refresh();
$assert(CloudHostResourcePoolMembership::where('host_id', 1)->count() === 0
&& $host->resource_pool_id === null
&& $host->scheduler_state === 'disabled',
'Clearing the authoritative membership recreated a stale scalar link.');
$now = 1060;
$timeoutClient = $client([new \GuzzleHttp\Exception\ConnectException('SECRET timeout',
new \GuzzleHttp\Psr7\Request('GET', 'http://127.0.0.1'), null, ['errno' => 28])]);
$timedOut = (new CloudHostInventorySyncService($timeoutClient,
static function () use (&$now): int { return $now; }))->syncOne(1);
$record = CloudHostInventoryAttempt::orderByDesc('sequence_id')->firstOrFail();
$assert($timedOut['unavailable'] === 1 && $record->outcome === 'timeout'
&& $record->error_code === 'engine_http_timeout' && !$record->applied
&& $record->stage === 'capacity_request' && $record->http_status === null,
'A timeout needs durable evidence without inventing an engine failure phase.');
$now = 1070;
$supersededClient = $client([
static function () use ($capsule, $capacity, $json, &$now) {
$capsule->getConnection()->table('cloud_host_capacity_snapshots')
->where('host_id', 1)->increment('attempt_generation');
return new Response(200, [], $json($capacity($now)));
},
new Response(200, [], $json($network())),
]);
$superseded = (new CloudHostInventorySyncService($supersededClient,
static function () use (&$now): int { return $now; }))->syncOne(1);
$record = CloudHostInventoryAttempt::orderByDesc('sequence_id')->firstOrFail();
$assert($superseded['skipped'] === 1 && $record->outcome === 'superseded' && !$record->applied,
'A stale generation must be retained as superseded, never successful.');
$schema->drop('cloud_host_inventory_attempts');
$withoutHistory = (new CloudHostInventorySyncService($client([
new Response(503, [], '{"error":"SECRET"}'),
]), static function () use (&$now): int { return $now; }))->syncOne(1);
$assert($withoutHistory['unavailable'] === 1,
'Unavailable diagnostic storage must not prevent ordinary failure handling.');
echo "Cloud host multi-pool inventory, diagnostic history and membership checks passed.\n";
}