You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudVpcTombstoneConvergenc...

2154 lines
109 KiB
PHP

<?php
use App\Models\CloudVpc;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudVpcService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$root = dirname(__DIR__);
$assert = static function ($condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
if (!is_file($root . '/vendor/autoload.php')) {
echo "VPC tombstone convergence runtime skipped (vendor missing).\n";
return;
}
require $root . '/vendor/autoload.php';
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_regions', static function (Blueprint $table): void {
$table->increments('id');
});
$schema->create('cloud_vpcs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('vpc_key', 32)->unique();
$table->unsignedInteger('user_id');
$table->unsignedInteger('region_id');
$table->unsignedInteger('private_network_id')->default(0);
$table->string('name', 63);
$table->string('cidr', 18);
$table->string('fabric_id', 64);
$table->string('fabric_region', 64);
$table->unsignedBigInteger('fabric_revision');
$table->unsignedInteger('max_members');
$table->string('state', 16);
$table->unsignedBigInteger('desired_revision');
$table->unsignedBigInteger('observed_revision')->default(0);
$table->string('desired_digest', 64);
$table->string('observed_digest', 64)->nullable();
$table->string('sync_state', 16);
$table->string('last_operation_id', 32)->nullable();
$table->string('last_error_code', 64)->nullable();
$table->unsignedBigInteger('last_success_at')->default(0);
$table->unsignedBigInteger('last_failure_at')->default(0);
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$schema->create('cloud_vpc_subnets', static function (Blueprint $table): void {
$table->increments('id');
$table->string('subnet_key', 32);
$table->unsignedInteger('vpc_id');
$table->string('cidr', 18);
$table->unsignedBigInteger('network_address');
$table->unsignedBigInteger('broadcast_address');
$table->unsignedTinyInteger('prefix_length');
$table->string('gateway_ip', 15)->nullable();
$table->string('state', 16);
$table->unsignedBigInteger('desired_revision');
$table->unsignedBigInteger('observed_revision')->default(0);
$table->string('desired_digest', 64)->nullable();
$table->string('observed_digest', 64)->nullable();
$table->string('last_error_code', 64)->nullable();
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
$table->unique(['vpc_id', 'subnet_key']);
$table->unique(['vpc_id', 'cidr']);
});
$schema->create('cloud_instances', static function (Blueprint $table): void {
$table->increments('id');
$table->unsignedInteger('host_id');
$table->string('uuid', 36);
$table->string('status', 24);
$table->string('network_status', 24)->default('active');
});
$schema->create('cloud_instance_runtime', static function (Blueprint $table): void {
$table->unsignedInteger('instance_id')->primary();
$table->unsignedInteger('host_id');
$table->string('uuid', 36);
$table->string('power_state', 16)->default('unknown');
$table->string('sync_state', 16)->default('never');
$table->unsignedBigInteger('last_success_at')->default(0);
$table->unsignedBigInteger('last_attempt_at')->default(0);
$table->unsignedBigInteger('revision')->default(0);
$table->string('last_error_code', 64)->default('');
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
$table->unique(['host_id', 'uuid']);
});
$schema->create('cloud_vpc_members', static function (Blueprint $table): void {
$table->increments('id');
$table->string('member_key', 32);
$table->unsignedInteger('vpc_id');
$table->unsignedInteger('subnet_id');
$table->unsignedInteger('private_ip_id')->nullable();
$table->unsignedInteger('instance_id');
$table->unsignedInteger('host_id');
$table->string('private_ip', 15);
$table->unsignedBigInteger('private_ip_int');
$table->string('state', 16);
$table->unsignedBigInteger('desired_revision');
$table->unsignedBigInteger('observed_revision')->default(0);
$table->string('desired_digest', 64)->nullable();
$table->string('observed_digest', 64)->nullable();
$table->string('last_operation_id', 32)->nullable();
$table->string('last_error_code', 64)->nullable();
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
$table->unique('instance_id');
$table->unique(['vpc_id', 'private_ip']);
$table->unique('private_ip_id');
});
$schema->create('cloud_vpc_private_ips', static function (Blueprint $table): void {
$table->increments('id');
$table->string('ip_key', 32);
$table->unsignedInteger('vpc_id');
$table->unsignedInteger('subnet_id');
$table->unsignedInteger('member_id')->nullable();
$table->unsignedInteger('instance_id')->nullable();
$table->string('private_ip', 15);
$table->unsignedBigInteger('private_ip_int');
$table->string('state', 16);
$table->unsignedBigInteger('desired_revision');
$table->unsignedBigInteger('observed_revision')->default(0);
$table->string('desired_digest', 64)->nullable();
$table->string('observed_digest', 64)->nullable();
$table->string('last_error_code', 64)->nullable();
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
$table->unique(['vpc_id', 'private_ip']);
$table->unique('member_id');
});
foreach (['cloud_vpc_routes' => 'route_key', 'cloud_vpc_acl_rules' => 'rule_key']
as $tableName => $keyColumn) {
$schema->create($tableName, static function (Blueprint $table) use (
$keyColumn, $tableName
): void {
$table->increments('id');
$table->string($keyColumn, 32);
$table->unsignedInteger('vpc_id');
if ($tableName === 'cloud_vpc_routes') {
$table->unsignedInteger('member_id')->nullable();
$table->string('destination_cidr', 18)->nullable();
$table->string('next_hop_ip', 15)->nullable();
$table->unsignedSmallInteger('metric')->nullable();
} else {
$table->unsignedInteger('priority')->nullable();
$table->string('direction', 8)->nullable();
$table->string('action', 8)->nullable();
$table->string('protocol', 8)->nullable();
$table->string('source_cidr', 18)->nullable();
$table->string('destination_cidr', 18)->nullable();
$table->unsignedSmallInteger('port_from')->nullable();
$table->unsignedSmallInteger('port_to')->nullable();
}
$table->string('state', 16);
$table->unsignedBigInteger('desired_revision');
$table->unsignedBigInteger('observed_revision')->default(0);
$table->string('desired_digest', 64)->nullable();
$table->string('observed_digest', 64)->nullable();
$table->string('last_error_code', 64)->nullable();
$table->unsignedBigInteger('updated_at');
});
}
$schema->create('cloud_vpc_host_deployments', static function (Blueprint $table): void {
$table->increments('id');
$table->unsignedInteger('vpc_id');
$table->unsignedInteger('host_id');
$table->unsignedBigInteger('fabric_revision')->default(1);
$table->string('state', 16);
$table->unsignedBigInteger('desired_revision');
$table->unsignedBigInteger('observed_revision')->default(0);
$table->string('desired_digest', 64)->nullable();
$table->string('observed_digest', 64)->nullable();
$table->string('last_operation_id', 32)->nullable();
$table->string('last_error_code', 64)->nullable();
$table->unsignedBigInteger('last_success_at')->default(0);
$table->unsignedBigInteger('last_failure_at')->default(0);
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
$table->unique(['vpc_id', 'host_id']);
});
$schema->create('cloud_vpc_operations', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id', 32)->unique();
$table->unsignedInteger('vpc_id');
$table->unsignedInteger('user_id');
$table->string('kind', 24);
$table->unsignedBigInteger('desired_revision');
$table->string('status', 24);
$table->text('payload_json');
$table->text('observed_json')->nullable();
$table->unsignedSmallInteger('evidence_version')->default(0);
$table->string('error_code', 64)->nullable();
$table->string('lease_token', 32)->nullable();
$table->unsignedBigInteger('lease_until')->default(0);
$table->unsignedBigInteger('attempts')->default(1);
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
$table->unsignedBigInteger('finished_at')->nullable();
});
Capsule::table('cloud_regions')->insert(['id' => 1]);
$now = time();
$privateMacMethod = new ReflectionMethod(CloudVpcService::class, 'privateMacForMember');
$bridgeMethod = new ReflectionMethod(CloudVpcService::class, 'privateBridgeForVpc');
$nftMethod = new ReflectionMethod(CloudVpcService::class, 'privateNftTableForVpc');
$routeTableMethod = new ReflectionMethod(CloudVpcService::class, 'privateRouteTableForVpc');
$publicRootMethod = new ReflectionMethod(CloudVpcService::class, 'publicRoot');
$canonicalJsonMethod = new ReflectionMethod(CloudVpcService::class, 'canonicalJson');
$validateHostMethod = new ReflectionMethod(
CloudVpcService::class, 'validateHostDataplaneEvidence'
);
$assert(CloudVpcService::ENGINE_TIMEOUT_SECONDS > 40.0
&& CloudVpcService::OPERATION_LEASE_SECONDS
>= CloudVpcService::ENGINE_TIMEOUT_SECONDS + 30.0,
'VPC timeout/lease do not contain the engine request-wide probe budget.');
$seedVpc = static function (
int $id, string $vpcKey, int $revision, string $digest,
string $operationId, string $kind
) use ($now): void {
Capsule::table('cloud_vpcs')->insert([
'id' => $id, 'vpc_key' => $vpcKey, 'user_id' => 9, 'region_id' => 1,
'private_network_id' => 0, 'name' => 'tombstone-' . $id,
'cidr' => '10.' . (20 + $id) . '.0.0/24', 'fabric_id' => 'fabric-test',
'fabric_region' => 'test-region', 'fabric_revision' => 7,
'max_members' => 32, 'state' => 'applying',
'desired_revision' => $revision, 'observed_revision' => $revision - 1,
'desired_digest' => $digest, 'observed_digest' => null,
'sync_state' => 'pending', 'last_operation_id' => $operationId,
'last_error_code' => null, 'last_success_at' => 0,
'last_failure_at' => 0, 'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_vpc_operations')->insert([
'operation_id' => $operationId, 'vpc_id' => $id, 'user_id' => 9,
'kind' => $kind, 'desired_revision' => $revision, 'status' => 'running',
'payload_json' => '{}', 'observed_json' => null, 'error_code' => null,
'lease_token' => null, 'lease_until' => $now + 120, 'attempts' => 1,
'created_at' => $now, 'updated_at' => $now, 'finished_at' => null,
]);
};
$seedSubnet = static function (
int $id, int $vpcId, string $key, string $cidr, string $state,
int $revision, string $digest
) use ($now): void {
[$ip, $prefix] = explode('/', $cidr);
$packed = unpack('Nvalue', inet_pton($ip));
$network = (int) sprintf('%u', (int) $packed['value']);
$size = 2 ** (32 - (int) $prefix);
Capsule::table('cloud_vpc_subnets')->insert([
'id' => $id, 'subnet_key' => $key, 'vpc_id' => $vpcId,
'cidr' => $cidr, 'network_address' => $network,
'broadcast_address' => $network + $size - 1,
'prefix_length' => (int) $prefix, 'gateway_ip' => null,
'state' => $state, 'desired_revision' => $revision,
'observed_revision' => max(0, $revision - 1),
'desired_digest' => $digest, 'observed_digest' => null,
'last_error_code' => null, 'created_at' => $now, 'updated_at' => $now,
]);
};
$seedMember = static function (
int $memberId, int $leaseId, int $vpcId, int $subnetId,
string $memberKey, int $instanceId, string $uuid, int $hostId,
string $ip, string $state, int $revision, string $digest
) use ($now): void {
$packed = unpack('Nvalue', inet_pton($ip));
$ipInt = (int) sprintf('%u', (int) $packed['value']);
Capsule::table('cloud_instances')->insert([
'id' => $instanceId, 'host_id' => $hostId,
'uuid' => $uuid, 'status' => 'active',
'network_status' => 'active',
]);
Capsule::table('cloud_instance_runtime')->insert([
'instance_id' => $instanceId, 'host_id' => $hostId, 'uuid' => $uuid,
'power_state' => 'running', 'sync_state' => 'ok',
'last_success_at' => $now, 'last_attempt_at' => $now,
'revision' => 1, 'last_error_code' => '',
'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_vpc_members')->insert([
'id' => $memberId, 'member_key' => $memberKey, 'vpc_id' => $vpcId,
'subnet_id' => $subnetId, 'private_ip_id' => $leaseId,
'instance_id' => $instanceId, 'host_id' => $hostId,
'private_ip' => $ip, 'private_ip_int' => $ipInt, 'state' => $state,
'desired_revision' => $revision,
'observed_revision' => max(0, $revision - 1),
'desired_digest' => $digest, 'observed_digest' => null,
'last_operation_id' => null, 'last_error_code' => null,
'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_vpc_private_ips')->insert([
'id' => $leaseId, 'ip_key' => str_pad(dechex($leaseId), 32, '0', STR_PAD_LEFT),
'vpc_id' => $vpcId, 'subnet_id' => $subnetId,
'member_id' => $memberId, 'instance_id' => $instanceId,
'private_ip' => $ip, 'private_ip_int' => $ipInt, 'state' => $state,
'desired_revision' => $revision,
'observed_revision' => max(0, $revision - 1),
'desired_digest' => $digest, 'observed_digest' => null,
'last_error_code' => null, 'created_at' => $now, 'updated_at' => $now,
]);
};
$seedHost = static function (
int $vpcId, int $hostId, string $state, int $revision,
string $digest, string $operationId
) use ($now): void {
Capsule::table('cloud_vpc_host_deployments')->insert([
'vpc_id' => $vpcId, 'host_id' => $hostId, 'fabric_revision' => 7,
'state' => $state, 'desired_revision' => $revision,
'observed_revision' => max(0, $revision - 1),
'desired_digest' => $digest, 'observed_digest' => null,
'last_operation_id' => $operationId, 'last_error_code' => null,
'last_success_at' => 0, 'last_failure_at' => 0,
'created_at' => $now, 'updated_at' => $now,
]);
};
$activeHost = static function (
string $vpcKey, string $vpcCidr, int $hostId, int $revision, string $digest,
string $memberKey, string $uuid, string $ip
) use ($privateMacMethod, $bridgeMethod, $nftMethod, $routeTableMethod): array {
$bridge = $bridgeMethod->invoke(null, $vpcKey);
return [
'vpc_id' => $vpcKey, 'host_id' => (string) $hostId,
'desired_revision' => $revision, 'desired_digest' => $digest,
'observed_revision' => $revision, 'observed_digest' => $digest,
'state' => 'active', 'transport_verified' => true,
'private_dataplane_verified' => true, 'removed' => false,
'config_verified' => true, 'config_observed_revision' => $revision,
'config_observed_digest' => $digest,
'bridge' => $bridge, 'bridge_owner' => 'virt-cloud-vpc-dataplane-v1',
'nft_table' => $nftMethod->invoke(null, $vpcKey),
'anti_spoof' => true, 'rp_filter' => 2, 'proxy_arp' => 0,
'proxy_arp_pvlan' => 1, 'proxy_ips' => [$ip],
'acl_revision' => $revision, 'rollback' => 'none',
'readback' => 'confirmed',
'member_nics' => [[
'member_id' => $memberKey, 'instance_uuid' => $uuid,
'interface' => 'private0', 'private_ip_cidr' => $ip . '/32',
'private_mac' => $privateMacMethod->invoke(null, $vpcKey, $memberKey),
'xml_verified' => true, 'config_verified' => true,
'seed_verified' => true,
'restart_required' => false, 'domain_running' => true,
'guest_network_verified' => true,
'guest_network_readback' => 'confirmed',
'guest_reason_code' => null,
'guest_private_mac' => $privateMacMethod->invoke(null, $vpcKey, $memberKey),
'guest_private_ip_cidr' => $ip . '/32',
'guest_routes' => [$vpcCidr],
'link_state' => 'up', 'guest_link_state' => 'up',
]],
'routes' => [[
'destination_cidr' => $ip . '/32', 'device' => $bridge,
'table' => $routeTableMethod->invoke(null, $vpcKey),
'owner' => 'virt-cloud-vpc-dataplane-v1', 'via' => null,
'metric' => 100,
]],
'wg_allowed_ips' => [],
];
};
$removedHost = static function (
string $vpcKey, int $hostId, int $revision, string $digest
): array {
return [
'vpc_id' => $vpcKey, 'host_id' => (string) $hostId,
'desired_revision' => $revision, 'desired_digest' => $digest,
'observed_revision' => $revision, 'observed_digest' => $digest,
'state' => 'active', 'transport_verified' => true,
'private_dataplane_verified' => true, 'removed' => true,
'bridge' => null, 'bridge_owner' => null, 'nft_table' => null,
'anti_spoof' => false, 'rp_filter' => null, 'proxy_arp' => null,
'proxy_arp_pvlan' => null, 'proxy_ips' => [], 'rollback' => 'none',
'readback' => 'confirmed', 'member_nics' => [], 'routes' => [],
'wg_allowed_ips' => [],
];
};
$observed = static function (int $revision, string $digest, array $hosts): array {
return [
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
'revision' => $revision, 'digest' => $digest, 'state' => 'active',
'next_stage_allowed' => true, 'private_dataplane_verified' => true,
'fabric_gate' => ['active' => true], 'hosts' => $hosts,
];
};
// Creating an empty VPC completes only its PHP-owned control-plane
// reservation. Both engine terminal-gate spellings are accepted, but neither
// may make the VPC data plane usable without a participating member host.
foreach ([
['id' => 20, 'subnet_id' => 2020, 'gate' => 'vpc_no_participating_hosts'],
['id' => 21, 'subnet_id' => 2021, 'gate' => 'vpc_no_members'],
] as $emptyCreate) {
$id = (int) $emptyCreate['id'];
$revision = $id * 2;
$vpcKey = substr(hash('sha256', 'empty-vpc-' . $id), 0, 32);
$digest = hash('sha256', 'empty-vpc-digest-' . $id);
$operationId = substr(hash('sha256', 'empty-vpc-operation-' . $id), 0, 32);
$subnetKey = substr(hash('sha256', 'empty-vpc-subnet-' . $id), 0, 32);
$seedVpc($id, $vpcKey, $revision, $digest, $operationId, 'vpc_create');
$seedSubnet((int) $emptyCreate['subnet_id'], $id, $subnetKey,
'10.' . (20 + $id) . '.0.0/24', 'applying', $revision, $digest);
$desiredJson = $canonicalJsonMethod->invoke(
null,
CloudVpcService::desiredDocument(CloudVpc::find($id))
);
$digest = hash('sha256', $desiredJson);
Capsule::table('cloud_vpcs')->where('id', $id)->update([
'desired_digest' => $digest,
]);
Capsule::table('cloud_vpc_subnets')->where('vpc_id', $id)->update([
'desired_digest' => $digest,
]);
Capsule::table('cloud_vpc_operations')->where('operation_id', $operationId)->update([
'payload_json' => $desiredJson,
]);
CloudVpcService::recordObserved($vpcKey, $revision, $digest, [
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
'revision' => 0, 'digest' => null, 'state' => 'degraded',
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
'fabric_gate' => [
'active' => false, 'error_code' => (string) $emptyCreate['gate'],
],
'hosts' => [],
], $operationId);
$root = CloudVpc::where('id', $id)->first();
$operation = Capsule::table('cloud_vpc_operations')
->where('operation_id', $operationId)->first();
$subnet = Capsule::table('cloud_vpc_subnets')
->where('id', (int) $emptyCreate['subnet_id'])->first();
$receipt = json_decode((string) ($operation->observed_json ?? ''), true);
$projection = $root === null ? [] : $publicRootMethod->invoke(null, $root);
$assert($root !== null
&& (string) ($operation->status ?? '') === 'succeeded'
&& (int) ($operation->evidence_version ?? 0)
=== CloudVpcService::OBSERVATION_EVIDENCE_VERSION
&& ($operation->error_code ?? null) === null
&& $operation->finished_at !== null
&& (string) $root->state === 'degraded'
&& (string) $root->sync_state === 'applied'
&& (int) $root->observed_revision === $revision
&& (string) $root->observed_digest === $digest
&& $root->last_error_code === null
&& ($projection['usable'] ?? true) === false
&& ($projection['member_count'] ?? null) === 0
&& ($projection['setup_state'] ?? null) === 'awaiting_members'
&& (string) ($subnet->state ?? '') === 'active'
&& (int) ($subnet->observed_revision ?? 0) === $revision
&& (string) ($subnet->observed_digest ?? '') === $digest
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', $id)->count() === 0
&& Capsule::table('cloud_vpc_members')->where('vpc_id', $id)->count() === 0
&& Capsule::table('cloud_vpc_private_ips')->where('vpc_id', $id)->count() === 0
&& ($receipt['no_data_plane_terminal_proof']['gate'] ?? null)
=== (string) $emptyCreate['gate'],
'An exact empty-VPC create receipt did not complete as a non-usable control-plane reservation.');
// Backward compatibility is projection-only: an old failed root with one
// of the reviewed empty gate codes is shown as awaiting members without
// rewriting its persisted operation or sync status.
Capsule::table('cloud_vpcs')->where('id', $id)->update([
'sync_state' => 'failed',
'last_error_code' => (string) $emptyCreate['gate'],
]);
Capsule::table('cloud_vpc_operations')->where('operation_id', $operationId)->update([
'status' => 'failed',
'error_code' => (string) $emptyCreate['gate'],
]);
$legacyProjection = $publicRootMethod->invoke(null, CloudVpc::find($id));
$assert(($legacyProjection['member_count'] ?? null) === 0
&& ($legacyProjection['setup_state'] ?? null) === 'awaiting_members'
&& (string) Capsule::table('cloud_vpcs')->where('id', $id)
->value('sync_state') === 'failed'
&& (string) Capsule::table('cloud_vpc_operations')
->where('operation_id', $operationId)->value('status') === 'failed',
'A legacy empty gate was not projected safely or GET rewrote its ledger.');
$seedHost($id, 900 + $id, 'degraded', $revision, $digest, $operationId);
$hostProjection = $publicRootMethod->invoke(null, CloudVpc::find($id));
$assert(($hostProjection['member_count'] ?? null) === 0
&& ($hostProjection['setup_state'] ?? null) === null,
'A VPC with a non-deleted host deployment was projected as awaiting members.');
}
// Legacy workers persisted a safe empty create as failed before the control-
// plane reservation became a supported terminal outcome. Its GET projection
// is compatibility-only: revision/digest and child state remain untouched.
$vpcKey27 = substr(hash('sha256', 'empty-vpc-27'), 0, 32);
$operation27 = substr(hash('sha256', 'empty-vpc-operation-27'), 0, 32);
$placeholderDigest27 = hash('sha256', 'empty-vpc-digest-27');
$seedVpc(27, $vpcKey27, 54, $placeholderDigest27, $operation27, 'vpc_create');
$seedSubnet(2027, 27, substr(hash('sha256', 'empty-vpc-subnet-27'), 0, 32),
'10.47.0.0/24', 'applying', 54, $placeholderDigest27);
$desiredJson27 = $canonicalJsonMethod->invoke(
null,
CloudVpcService::desiredDocument(CloudVpc::find(27))
);
$digest27 = hash('sha256', $desiredJson27);
Capsule::table('cloud_vpcs')->where('id', 27)->update([
'state' => 'degraded', 'sync_state' => 'failed',
'desired_digest' => $digest27, 'observed_revision' => 53,
'observed_digest' => null,
'last_error_code' => 'vpc_no_participating_hosts',
]);
Capsule::table('cloud_vpc_subnets')->where('id', 2027)->update([
'state' => 'applying', 'desired_digest' => $digest27,
'observed_revision' => 53, 'observed_digest' => null,
]);
Capsule::table('cloud_vpc_operations')->where('operation_id', $operation27)->update([
'status' => 'failed', 'payload_json' => $desiredJson27,
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
'error_code' => 'vpc_no_participating_hosts',
'observed_json' => json_encode([
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
'revision' => 0, 'digest' => null, 'state' => 'degraded',
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
'fabric_gate' => [
'active' => false,
'error_code' => 'vpc_no_participating_hosts',
],
'hosts' => [],
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES),
'finished_at' => $now,
]);
$legacyProjection27 = $publicRootMethod->invoke(null, CloudVpc::find(27));
$assert(($legacyProjection27['setup_state'] ?? null) === 'awaiting_members'
&& Capsule::table('cloud_vpcs')->where('id', 27)
->value('observed_revision') === 53
&& Capsule::table('cloud_vpcs')->where('id', 27)
->value('observed_digest') === null
&& Capsule::table('cloud_vpc_subnets')->where('id', 2027)
->value('state') === 'applying'
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation27)
->value('status') === 'failed',
'A real legacy empty-create failure was not projected read-only as awaiting members.');
// A matching gate code is insufficient when the engine envelope contradicts
// the no-data-plane claim. Never complete even the control-plane reservation.
$vpcKey28 = substr(hash('sha256', 'empty-vpc-28'), 0, 32);
$operation28 = substr(hash('sha256', 'empty-vpc-operation-28'), 0, 32);
$placeholderDigest28 = hash('sha256', 'empty-vpc-digest-28');
$seedVpc(28, $vpcKey28, 56, $placeholderDigest28, $operation28, 'vpc_create');
$seedSubnet(2028, 28, substr(hash('sha256', 'empty-vpc-subnet-28'), 0, 32),
'10.48.0.0/24', 'applying', 56, $placeholderDigest28);
$desiredJson28 = $canonicalJsonMethod->invoke(
null,
CloudVpcService::desiredDocument(CloudVpc::find(28))
);
$digest28 = hash('sha256', $desiredJson28);
Capsule::table('cloud_vpcs')->where('id', 28)->update([
'desired_digest' => $digest28,
]);
Capsule::table('cloud_vpc_subnets')->where('id', 2028)->update([
'desired_digest' => $digest28,
]);
Capsule::table('cloud_vpc_operations')->where('operation_id', $operation28)->update([
'payload_json' => $desiredJson28,
]);
CloudVpcService::recordObserved($vpcKey28, 56, $digest28, [
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
'revision' => 0, 'digest' => null, 'state' => 'degraded',
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
'fabric_gate' => [
'active' => true, 'error_code' => 'vpc_no_participating_hosts',
],
'hosts' => [],
], $operation28);
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation28)
->value('status') !== 'succeeded'
&& Capsule::table('cloud_vpcs')->where('id', 28)
->value('observed_revision') === 55
&& Capsule::table('cloud_vpc_subnets')->where('id', 2028)
->value('state') === 'applying',
'A contradictory no-host envelope completed an empty VPC reservation.');
// The empty-create completion is closed to malformed desired documents.
$vpcKey22 = substr(hash('sha256', 'empty-vpc-22'), 0, 32);
$digest22 = hash('sha256', 'empty-vpc-digest-22');
$operation22 = substr(hash('sha256', 'empty-vpc-operation-22'), 0, 32);
$seedVpc(22, $vpcKey22, 44, $digest22, $operation22, 'vpc_create');
$seedSubnet(2022, 22, substr(hash('sha256', 'empty-vpc-subnet-22'), 0, 32),
'10.42.0.0/24', 'applying', 44, $digest22);
Capsule::table('cloud_vpcs')->where('id', 22)->update(['fabric_region' => '']);
CloudVpcService::recordObserved($vpcKey22, 44, $digest22, [
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
'revision' => 0, 'digest' => null, 'state' => 'degraded',
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_participating_hosts'],
'hosts' => [],
], $operation22);
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation22)
->value('status') === 'failed'
&& Capsule::table('cloud_vpcs')->where('id', 22)->value('observed_revision') === 43
&& Capsule::table('cloud_vpc_subnets')->where('id', 2022)->value('state') === 'applying',
'A malformed empty desired document was accepted as a completed VPC reservation.');
$malformedProjection22 = $publicRootMethod->invoke(null, CloudVpc::find(22));
$assert(($malformedProjection22['setup_state'] ?? null) === null,
'A malformed desired document was hidden behind the neutral empty-VPC projection.');
// An unrelated failure is not the neutral empty-VPC setup state, and even an
// allowed legacy gate must not override an explicit root deletion lifecycle.
Capsule::table('cloud_vpcs')->where('id', 22)->update([
'last_error_code' => 'vpc_observation_incomplete',
]);
$realErrorProjection22 = $publicRootMethod->invoke(null, CloudVpc::find(22));
$assert(($realErrorProjection22['member_count'] ?? null) === 0
&& ($realErrorProjection22['setup_state'] ?? null) === null
&& Capsule::table('cloud_vpcs')->where('id', 22)
->value('last_error_code') === 'vpc_observation_incomplete',
'A real empty-VPC failure was projected as awaiting members or GET mutated it.');
Capsule::table('cloud_vpcs')->where('id', 22)->update([
'state' => 'deleting',
'last_error_code' => 'vpc_no_members',
]);
$deletingProjection22 = $publicRootMethod->invoke(null, CloudVpc::find(22));
$assert(($deletingProjection22['member_count'] ?? null) === 0
&& ($deletingProjection22['setup_state'] ?? null) === null
&& Capsule::table('cloud_vpcs')->where('id', 22)->value('state') === 'deleting',
'A deleting empty VPC was projected as awaiting members or GET mutated it.');
// A contradictory terminal envelope cannot hide a desired member or advance
// its member/private-IP identity without host evidence.
$vpcKey23 = substr(hash('sha256', 'empty-vpc-23'), 0, 32);
$digest23 = hash('sha256', 'empty-vpc-digest-23');
$operation23 = substr(hash('sha256', 'empty-vpc-operation-23'), 0, 32);
$seedVpc(23, $vpcKey23, 46, $digest23, $operation23, 'vpc_create');
$seedSubnet(2023, 23, substr(hash('sha256', 'empty-vpc-subnet-23'), 0, 32),
'10.43.0.0/24', 'applying', 46, $digest23);
$seedMember(128, 228, 23, 2023,
substr(hash('sha256', 'empty-vpc-member-23'), 0, 32), 23001,
'23232323-2323-4323-8323-232323232323', 230, '10.43.0.10',
'applying', 46, $digest23);
CloudVpcService::recordObserved($vpcKey23, 46, $digest23, [
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
'revision' => 0, 'digest' => null, 'state' => 'degraded',
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_participating_hosts'],
'hosts' => [],
], $operation23);
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation23)
->value('status') === 'failed'
&& Capsule::table('cloud_vpc_members')->where('id', 128)->value('state') === 'applying'
&& Capsule::table('cloud_vpc_members')->where('id', 128)
->value('observed_revision') === 45
&& Capsule::table('cloud_vpc_private_ips')->where('id', 228)->value('state') === 'applying'
&& Capsule::table('cloud_vpc_private_ips')->where('id', 228)
->value('observed_revision') === 45
&& ($publicRootMethod->invoke(null, CloudVpc::find(23))['member_count'] ?? null) === 1
&& ($publicRootMethod->invoke(null, CloudVpc::find(23))['setup_state'] ?? null) === null,
'A terminal empty-host envelope advanced a desired member/private-IP without host proof.');
// Capture the pending-child gate before terminal route/ACL cleanup. A corrupt
// create ledger with a tombstone must remain failed even if that generic proof
// closes the individual policy tombstone during the same transaction.
$vpcKey24 = substr(hash('sha256', 'empty-vpc-24'), 0, 32);
$digest24 = hash('sha256', 'empty-vpc-digest-24');
$operation24 = substr(hash('sha256', 'empty-vpc-operation-24'), 0, 32);
$seedVpc(24, $vpcKey24, 48, $digest24, $operation24, 'vpc_create');
$seedSubnet(2024, 24, substr(hash('sha256', 'empty-vpc-subnet-24'), 0, 32),
'10.44.0.0/24', 'applying', 48, $digest24);
Capsule::table('cloud_vpc_acl_rules')->insert([
'id' => 240, 'rule_key' => substr(hash('sha256', 'empty-vpc-acl-24'), 0, 32),
'vpc_id' => 24, 'state' => 'deleting', 'desired_revision' => 48,
'observed_revision' => 47, 'desired_digest' => $digest24,
'observed_digest' => null, 'last_error_code' => null, 'updated_at' => $now,
]);
$pendingChildProjection24 = $publicRootMethod->invoke(null, CloudVpc::find(24));
$assert(($pendingChildProjection24['member_count'] ?? null) === 0
&& ($pendingChildProjection24['setup_state'] ?? null) === null,
'A pending child tombstone was projected as an empty VPC ready for setup.');
CloudVpcService::recordObserved($vpcKey24, 48, $digest24, [
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
'revision' => 0, 'digest' => null, 'state' => 'degraded',
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_participating_hosts'],
'hosts' => [],
], $operation24);
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation24)
->value('status') === 'failed'
&& Capsule::table('cloud_vpcs')->where('id', 24)->value('observed_revision') === 47
&& Capsule::table('cloud_vpc_subnets')->where('id', 2024)->value('state') === 'applying'
&& Capsule::table('cloud_vpc_acl_rules')->where('id', 240)->value('state') === 'deleted',
'A create operation with an initial child tombstone was accepted as an empty reservation.');
// An orphan lease is not represented in desiredDocument(). It must still
// close both empty-create completion and the read-only awaiting projection.
$vpcKey25 = substr(hash('sha256', 'empty-vpc-25'), 0, 32);
$operation25 = substr(hash('sha256', 'empty-vpc-operation-25'), 0, 32);
$placeholderDigest25 = hash('sha256', 'empty-vpc-digest-25');
$seedVpc(25, $vpcKey25, 50, $placeholderDigest25, $operation25, 'vpc_create');
$seedSubnet(2025, 25, substr(hash('sha256', 'empty-vpc-subnet-25'), 0, 32),
'10.45.0.0/24', 'applying', 50, $placeholderDigest25);
$orphanIp = '10.45.0.10';
$orphanPacked = unpack('Nvalue', inet_pton($orphanIp));
Capsule::table('cloud_vpc_private_ips')->insert([
'id' => 250, 'ip_key' => substr(hash('sha256', 'empty-vpc-ip-25'), 0, 32),
'vpc_id' => 25, 'subnet_id' => 2025, 'member_id' => null,
'instance_id' => null, 'private_ip' => $orphanIp,
'private_ip_int' => (int) sprintf('%u', (int) $orphanPacked['value']),
'state' => 'applying', 'desired_revision' => 50, 'observed_revision' => 49,
'desired_digest' => $placeholderDigest25, 'observed_digest' => null,
'last_error_code' => null, 'created_at' => $now, 'updated_at' => $now,
]);
$desiredJson25 = $canonicalJsonMethod->invoke(
null,
CloudVpcService::desiredDocument(CloudVpc::find(25))
);
$digest25 = hash('sha256', $desiredJson25);
Capsule::table('cloud_vpcs')->where('id', 25)->update(['desired_digest' => $digest25]);
Capsule::table('cloud_vpc_subnets')->where('vpc_id', 25)
->update(['desired_digest' => $digest25]);
Capsule::table('cloud_vpc_private_ips')->where('vpc_id', 25)
->update(['desired_digest' => $digest25]);
Capsule::table('cloud_vpc_operations')->where('operation_id', $operation25)
->update(['payload_json' => $desiredJson25]);
CloudVpcService::recordObserved($vpcKey25, 50, $digest25, [
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
'revision' => 0, 'digest' => null, 'state' => 'degraded',
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_participating_hosts'],
'hosts' => [],
], $operation25);
$projection25 = $publicRootMethod->invoke(null, CloudVpc::find(25));
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation25)
->value('status') === 'failed'
&& Capsule::table('cloud_vpc_subnets')->where('id', 2025)->value('state') === 'applying'
&& Capsule::table('cloud_vpc_private_ips')->where('id', 250)->value('state') === 'applying'
&& ($projection25['setup_state'] ?? null) === null,
'An orphan private-IP lease was hidden by empty-VPC completion or projection.');
// An ACL is host configuration. A no-host terminal gate cannot mark it, or
// the enclosing empty create, as applied.
$vpcKey26 = substr(hash('sha256', 'empty-vpc-26'), 0, 32);
$operation26 = substr(hash('sha256', 'empty-vpc-operation-26'), 0, 32);
$placeholderDigest26 = hash('sha256', 'empty-vpc-digest-26');
$seedVpc(26, $vpcKey26, 52, $placeholderDigest26, $operation26, 'vpc_create');
$seedSubnet(2026, 26, substr(hash('sha256', 'empty-vpc-subnet-26'), 0, 32),
'10.46.0.0/24', 'applying', 52, $placeholderDigest26);
Capsule::table('cloud_vpc_acl_rules')->insert([
'id' => 260, 'rule_key' => substr(hash('sha256', 'empty-vpc-acl-26'), 0, 32),
'vpc_id' => 26, 'priority' => 100, 'direction' => 'ingress',
'action' => 'allow', 'protocol' => 'tcp', 'source_cidr' => '0.0.0.0/0',
'destination_cidr' => '10.46.0.0/24', 'port_from' => 22, 'port_to' => 22,
'state' => 'applying', 'desired_revision' => 52, 'observed_revision' => 51,
'desired_digest' => $placeholderDigest26, 'observed_digest' => null,
'last_error_code' => null, 'updated_at' => $now,
]);
$desiredJson26 = $canonicalJsonMethod->invoke(
null,
CloudVpcService::desiredDocument(CloudVpc::find(26))
);
$digest26 = hash('sha256', $desiredJson26);
Capsule::table('cloud_vpcs')->where('id', 26)->update(['desired_digest' => $digest26]);
Capsule::table('cloud_vpc_subnets')->where('vpc_id', 26)
->update(['desired_digest' => $digest26]);
Capsule::table('cloud_vpc_acl_rules')->where('vpc_id', 26)
->update(['desired_digest' => $digest26]);
Capsule::table('cloud_vpc_operations')->where('operation_id', $operation26)
->update(['payload_json' => $desiredJson26]);
CloudVpcService::recordObserved($vpcKey26, 52, $digest26, [
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
'revision' => 0, 'digest' => null, 'state' => 'degraded',
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_members'],
'hosts' => [],
], $operation26);
$projection26 = $publicRootMethod->invoke(null, CloudVpc::find(26));
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation26)
->value('status') === 'failed'
&& Capsule::table('cloud_vpc_subnets')->where('id', 2026)->value('state') === 'applying'
&& Capsule::table('cloud_vpc_acl_rules')->where('id', 260)->value('state') === 'applying'
&& ($projection26['setup_state'] ?? null) === null,
'A no-host receipt falsely applied an ACL-bearing VPC create.');
$validatorVpcKey = str_repeat('a', 32);
$validatorDigest = str_repeat('d', 64);
$validatorMemberKey = str_repeat('b', 32);
$validatorMember = [
'member_id' => $validatorMemberKey,
'instance_uuid' => 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
'host_id' => '901', 'private_ip_cidr' => '10.90.0.10/32',
'private_mac' => $privateMacMethod->invoke(
null, $validatorVpcKey, $validatorMemberKey
),
];
$validatorHost = $activeHost(
$validatorVpcKey, '10.90.0.0/24', 901, 2, $validatorDigest,
$validatorMemberKey,
$validatorMember['instance_uuid'], '10.90.0.10'
);
$validatorArgs = [
'901', [$validatorMemberKey => $validatorMember], [$validatorMember],
['10.90.0.10/32'], '10.90.0.0/24', [],
$validatorVpcKey, 2, $validatorDigest,
];
$assert($validateHostMethod->invoke(null, $validatorHost, ...$validatorArgs) === null,
'An exact guest address/route proof did not open the typed host gate.');
$validatorRouteMismatch = $validatorHost;
$validatorRouteMismatch['member_nics'][0]['guest_routes'] = ['10.91.0.0/24'];
$assert($validateHostMethod->invoke(
null, $validatorRouteMismatch, ...$validatorArgs
) === 'vpc_private_guest_route_mismatch',
'A mismatched guest route opened the typed host gate.');
$validatorMissingGuest = $validatorHost;
unset($validatorMissingGuest['member_nics'][0]['domain_running']);
$assert($validateHostMethod->invoke(
null, $validatorMissingGuest, ...$validatorArgs
) === 'vpc_private_guest_agent_unavailable',
'Missing guest runtime evidence did not fail closed.');
$validatorDeadline = $validatorHost;
$validatorDeadline['member_nics'][0]['guest_network_verified'] = false;
$validatorDeadline['member_nics'][0]['guest_network_readback'] = 'unavailable';
$validatorDeadline['member_nics'][0]['guest_reason_code'] =
'vpc_private_guest_probe_deadline_exceeded';
$assert($validateHostMethod->invoke(
null, $validatorDeadline, ...$validatorArgs
) === 'vpc_private_guest_probe_deadline_exceeded',
'The bounded guest-probe deadline reason was not safely propagated.');
$validatorReap = $validatorHost;
$validatorReap['member_nics'][0]['guest_network_verified'] = false;
$validatorReap['member_nics'][0]['guest_network_readback'] = 'unavailable';
$validatorReap['member_nics'][0]['guest_reason_code'] =
'vpc_private_guest_probe_reap_required';
$assert($validateHostMethod->invoke(
null, $validatorReap, ...$validatorArgs
) === 'vpc_private_guest_probe_reap_required',
'Guest-probe reap-required was not retained as a safe retryable reason.');
$validatorLinkDown = $validatorHost;
$validatorLinkDown['member_nics'][0]['link_state'] = 'down';
$assert($validateHostMethod->invoke(
null, $validatorLinkDown, ...$validatorArgs
) === 'vpc_private_guest_link_mismatch',
'A down guest private link opened the typed host gate.');
$validatorLinkMissing = $validatorHost;
unset($validatorLinkMissing['member_nics'][0]['link_state']);
$assert($validateHostMethod->invoke(
null, $validatorLinkMissing, ...$validatorArgs
) === 'vpc_private_guest_link_mismatch',
'Missing guest private link evidence did not fail closed.');
$validatorMissingXml = $validatorHost;
$validatorMissingXml['member_nics'][0]['xml_verified'] = false;
$configValidatorArgs = [...$validatorArgs, false];
$assert($validateHostMethod->invoke(
null, $validatorMissingXml, ...$configValidatorArgs
) === 'vpc_private_nic_readback_mismatch',
'Configuration-only convergence accepted a private NIC without exact XML.');
// Same-host partial detach: exact current member_nics proves the old private0
// was consumed before the host became active.
$vpcKey1 = str_repeat('1', 32);
$digest1 = str_repeat('a', 64);
$operation1 = str_repeat('a', 32);
$seedVpc(1, $vpcKey1, 2, $digest1, $operation1, 'member_detach');
$seedSubnet(101, 1, str_repeat('2', 32), '10.21.0.0/25', 'active', 2, $digest1);
$seedMember(101, 201, 1, 101, str_repeat('3', 32), 1001,
'11111111-1111-4111-8111-111111111111', 1, '10.21.0.10', 'deleting', 2, $digest1);
$liveKey1 = str_repeat('4', 32);
$liveUuid1 = '11111111-1111-4111-8111-111111111112';
$seedMember(102, 202, 1, 101, $liveKey1, 1002, $liveUuid1,
1, '10.21.0.11', 'active', 2, $digest1);
$seedHost(1, 1, 'applying', 2, $digest1, $operation1);
$host1 = $activeHost($vpcKey1, '10.21.0.0/24', 1, 2, $digest1,
$liveKey1, $liveUuid1, '10.21.0.11');
CloudVpcService::recordObserved($vpcKey1, 2, $digest1,
$observed(2, $digest1, ['1' => $host1]), $operation1);
$assert(Capsule::table('cloud_vpc_members')->where('id', 101)->count() === 0
&& Capsule::table('cloud_vpc_private_ips')->where('id', 201)->count() === 0
&& Capsule::table('cloud_vpc_members')->where('id', 102)->value('state') === 'active'
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation1)
->value('status') === 'succeeded',
'A same-host member/IP tombstone did not converge while its sibling stayed active.');
// The exact proof must release both database conflict keys.
Capsule::table('cloud_vpc_members')->insert([
'member_key' => str_repeat('5', 32), 'vpc_id' => 1, 'subnet_id' => 101,
'private_ip_id' => null, 'instance_id' => 1001, 'host_id' => 1,
'private_ip' => '10.21.0.10', 'private_ip_int' => 169148426,
'state' => 'applying', 'desired_revision' => 3, 'observed_revision' => 0,
'desired_digest' => null, 'observed_digest' => null,
'last_operation_id' => null, 'last_error_code' => null,
'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_vpc_members')->where('member_key', str_repeat('5', 32))->delete();
Capsule::table('cloud_vpc_private_ips')->insert([
'ip_key' => str_repeat('6', 32), 'vpc_id' => 1, 'subnet_id' => 101,
'member_id' => null, 'instance_id' => 1001, 'private_ip' => '10.21.0.10',
'private_ip_int' => 169148426, 'state' => 'applying',
'desired_revision' => 3, 'observed_revision' => 0,
'desired_digest' => null, 'observed_digest' => null,
'last_error_code' => null, 'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_vpc_private_ips')->where('ip_key', str_repeat('6', 32))->delete();
// Host A retires while host B remains live. Both target forms must be exact in
// the same revision before A/member/address are released.
$vpcKey2 = str_repeat('2', 32);
$digest2 = str_repeat('b', 64);
$operation2 = str_repeat('b', 32);
$seedVpc(2, $vpcKey2, 4, $digest2, $operation2, 'member_detach');
$seedSubnet(102, 2, str_repeat('7', 32), '10.22.0.0/25', 'active', 4, $digest2);
$seedMember(103, 203, 2, 102, str_repeat('8', 32), 2001,
'22222222-2222-4222-8222-222222222221', 10, '10.22.0.10', 'deleting', 4, $digest2);
$liveKey2 = str_repeat('9', 32);
$liveUuid2 = '22222222-2222-4222-8222-222222222222';
$seedMember(104, 204, 2, 102, $liveKey2, 2002, $liveUuid2,
20, '10.22.0.11', 'active', 4, $digest2);
$seedHost(2, 10, 'deleting', 4, $digest2, $operation2);
$seedHost(2, 20, 'applying', 4, $digest2, $operation2);
$hosts2 = [
'10' => $removedHost($vpcKey2, 10, 4, $digest2),
'20' => $activeHost($vpcKey2, '10.22.0.0/24', 20, 4, $digest2,
$liveKey2, $liveUuid2, '10.22.0.11'),
];
CloudVpcService::recordObserved($vpcKey2, 4, $digest2,
$observed(4, $digest2, $hosts2), $operation2);
$assert(Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 2)
->where('host_id', 10)->value('state') === 'deleted'
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 2)
->where('host_id', 20)->value('state') === 'active'
&& Capsule::table('cloud_vpc_members')->where('id', 103)->count() === 0
&& Capsule::table('cloud_vpc_private_ips')->where('id', 203)->count() === 0
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation2)
->value('status') === 'succeeded',
'A retiring host plus a live sibling did not converge as one exact host set.');
// An empty subnet is not a libvirt/host object. A live-host exact revision can
// release it, including its unique CIDR key.
$vpcKey3 = str_repeat('3', 32);
$digest3 = str_repeat('c', 64);
$operation3 = str_repeat('c', 32);
$seedVpc(3, $vpcKey3, 6, $digest3, $operation3, 'subnet_remove');
$seedSubnet(103, 3, str_repeat('a', 32), '10.23.0.0/25', 'active', 6, $digest3);
$seedSubnet(104, 3, str_repeat('b', 32), '10.23.0.128/25', 'deleting', 6, $digest3);
$liveKey3 = str_repeat('c', 32);
$liveUuid3 = '33333333-3333-4333-8333-333333333333';
$seedMember(105, 205, 3, 103, $liveKey3, 3001, $liveUuid3,
30, '10.23.0.10', 'active', 6, $digest3);
$seedHost(3, 30, 'applying', 6, $digest3, $operation3);
$host3 = $activeHost($vpcKey3, '10.23.0.0/24', 30, 6, $digest3,
$liveKey3, $liveUuid3, '10.23.0.10');
CloudVpcService::recordObserved($vpcKey3, 6, $digest3,
$observed(6, $digest3, ['30' => $host3]), $operation3);
$assert(Capsule::table('cloud_vpc_subnets')->where('id', 104)->count() === 0
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation3)
->value('status') === 'succeeded',
'An unreferenced subnet did not converge behind an exact live-host gate.');
$seedSubnet(1004, 3, str_repeat('d', 32), '10.23.0.128/25', 'applying', 7, $digest3);
$assert(Capsule::table('cloud_vpc_subnets')->where('id', 1004)->count() === 1,
'A converged subnet still occupied its unique CIDR key.');
// A VPC with no participating host can still complete a subnet-only removal;
// this proof is deliberately not usable for a member/private-IP tombstone.
$vpcKey4 = str_repeat('4', 32);
$digest4 = str_repeat('d', 64);
$operation4 = str_repeat('d', 32);
$seedVpc(4, $vpcKey4, 8, $digest4, $operation4, 'subnet_remove');
$seedSubnet(105, 4, str_repeat('e', 32), '10.24.0.0/25', 'deleting', 8, $digest4);
CloudVpcService::recordObserved($vpcKey4, 8, $digest4, [
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
'revision' => 0, 'digest' => null, 'state' => 'degraded',
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_members'],
'hosts' => [],
], $operation4);
$root4 = CloudVpc::where('id', 4)->first();
$assert($root4 !== null, 'The no-host subnet operation lost its VPC root.');
$rootProjection4 = $publicRootMethod->invoke(null, $root4);
$assert(Capsule::table('cloud_vpc_subnets')->where('id', 105)->count() === 0
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation4)
->value('status') === 'succeeded'
&& Capsule::table('cloud_vpcs')->where('id', 4)->value('state') !== 'active'
&& ($rootProjection4['usable'] ?? true) === false,
'A no-host subnet-only removal either failed or made an empty VPC active/usable.');
// Missing A removal evidence must not partially delete A or its member. This
// also keeps the same operation payload retryable instead of turning A into an
// unknown extra host on the next attempt.
$vpcKey5 = str_repeat('5', 32);
$digest5 = str_repeat('e', 64);
$operation5 = str_repeat('e', 32);
$seedVpc(5, $vpcKey5, 10, $digest5, $operation5, 'member_detach');
$seedSubnet(106, 5, str_repeat('f', 32), '10.25.0.0/25', 'active', 10, $digest5);
$seedMember(106, 206, 5, 106, str_repeat('0', 32), 5001,
'55555555-5555-4555-8555-555555555551', 50, '10.25.0.10', 'deleting', 10, $digest5);
$liveKey5 = str_repeat('1', 31) . '0';
$liveUuid5 = '55555555-5555-4555-8555-555555555552';
$seedMember(107, 207, 5, 106, $liveKey5, 5002, $liveUuid5,
60, '10.25.0.11', 'active', 10, $digest5);
$seedHost(5, 50, 'deleting', 10, $digest5, $operation5);
$seedHost(5, 60, 'applying', 10, $digest5, $operation5);
$host5 = $activeHost($vpcKey5, '10.25.0.0/24', 60, 10, $digest5,
$liveKey5, $liveUuid5, '10.25.0.11');
CloudVpcService::recordObserved($vpcKey5, 10, $digest5,
$observed(10, $digest5, ['60' => $host5]), $operation5);
$assert(Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 5)
->where('host_id', 50)->value('state') === 'deleting'
&& Capsule::table('cloud_vpc_members')->where('id', 106)->value('state') === 'deleting'
&& Capsule::table('cloud_vpc_private_ips')->where('id', 206)->value('state') === 'deleting'
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation5)
->value('status') === 'uncertain',
'A partial multi-host observation released tombstones without A removal proof.');
// Removing the last member has one required deployment: an exact typed host
// removal receipt is sufficient even though no live host remains afterward.
$vpcKey6 = str_repeat('6', 32);
$digest6 = str_repeat('f', 64);
$operation6 = str_repeat('6', 32);
$seedVpc(6, $vpcKey6, 12, $digest6, $operation6, 'member_detach');
$seedSubnet(107, 6, str_repeat('2', 32), '10.26.0.0/25', 'active', 12, $digest6);
$seedMember(108, 208, 6, 107, str_repeat('3', 32), 6001,
'66666666-6666-4666-8666-666666666661', 70, '10.26.0.10',
'deleting', 12, $digest6);
$seedHost(6, 70, 'deleting', 12, $digest6, $operation6);
CloudVpcService::recordObserved($vpcKey6, 12, $digest6,
$observed(12, $digest6, [
'70' => $removedHost($vpcKey6, 70, 12, $digest6),
]), $operation6);
$projection6 = $publicRootMethod->invoke(null, CloudVpc::find(6));
$assert(Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 6)
->where('host_id', 70)->value('state') === 'deleted'
&& Capsule::table('cloud_vpc_members')->where('id', 108)->count() === 0
&& Capsule::table('cloud_vpc_private_ips')->where('id', 208)->count() === 0
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation6)
->value('status') === 'succeeded'
&& ($projection6['member_count'] ?? null) === 0
&& ($projection6['setup_state'] ?? null) === 'awaiting_members',
'The last member did not converge behind its only host removal proof.');
// A terminal Fabric gate is not libvirt removal evidence. Even with no host
// deployment row, it must never consume a member/address tombstone.
$vpcKey7 = str_repeat('7', 32);
$digest7 = str_repeat('0', 64);
$operation7 = str_repeat('7', 32);
$seedVpc(7, $vpcKey7, 14, $digest7, $operation7, 'member_detach');
$seedSubnet(108, 7, str_repeat('4', 32), '10.27.0.0/25', 'active', 14, $digest7);
$seedMember(109, 209, 7, 108, str_repeat('5', 32), 7001,
'77777777-7777-4777-8777-777777777771', 71, '10.27.0.10',
'deleting', 14, $digest7);
try {
CloudVpcService::recordObserved($vpcKey7, 14, $digest7, [
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
'revision' => 0, 'digest' => null, 'state' => 'degraded',
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_members'],
'hosts' => [],
], $operation7);
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
'A terminal member proof failed with an unexpected error.');
}
$assert(Capsule::table('cloud_vpc_members')->where('id', 109)
->value('state') === 'deleting'
&& Capsule::table('cloud_vpc_private_ips')->where('id', 209)
->value('state') === 'deleting'
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation7)
->value('status') !== 'succeeded',
'A terminal no-host proof released a member/private-IP tombstone.');
// An exact live sibling cannot stand in for the missing deployment that owns
// a retiring member. The ledger may reject the observation or stay uncertain,
// but it must retain both sides of the member/address identity.
$vpcKey8 = str_repeat('8', 32);
$digest8 = str_repeat('1', 64);
$operation8 = str_repeat('8', 32);
$seedVpc(8, $vpcKey8, 16, $digest8, $operation8, 'member_detach');
$seedSubnet(109, 8, str_repeat('6', 32), '10.28.0.0/25', 'active', 16, $digest8);
$seedMember(110, 210, 8, 109, str_repeat('7', 32), 8001,
'88888888-8888-4888-8888-888888888881', 80, '10.28.0.10',
'deleting', 16, $digest8);
$liveKey8 = str_repeat('8', 31) . '0';
$liveUuid8 = '88888888-8888-4888-8888-888888888882';
$seedMember(111, 211, 8, 109, $liveKey8, 8002, $liveUuid8,
81, '10.28.0.11', 'active', 16, $digest8);
$seedHost(8, 81, 'applying', 16, $digest8, $operation8);
$host8 = $activeHost($vpcKey8, '10.28.0.0/24', 81, 16, $digest8,
$liveKey8, $liveUuid8, '10.28.0.11');
try {
CloudVpcService::recordObserved($vpcKey8, 16, $digest8,
$observed(16, $digest8, ['81' => $host8]), $operation8);
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
'A missing owner deployment failed with an unexpected error.');
}
$assert(Capsule::table('cloud_vpc_members')->where('id', 110)
->value('state') === 'deleting'
&& Capsule::table('cloud_vpc_private_ips')->where('id', 210)
->value('state') === 'deleting'
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation8)
->value('status') !== 'succeeded',
'A live sibling released member/IP identities whose owner deployment was missing.');
// Exact proofs are monotonic across retries. The first attempt proves A was
// removed but lacks B; the retry supplies only B and must reuse A's persisted
// proof instead of requiring the already-removed host to report again.
$vpcKey9 = str_repeat('9', 32);
$digest9 = str_repeat('2', 64);
$operation9 = str_repeat('9', 32);
$seedVpc(9, $vpcKey9, 18, $digest9, $operation9, 'member_detach');
$seedSubnet(110, 9, str_repeat('9', 31) . '0', '10.29.0.0/25',
'active', 18, $digest9);
$seedMember(112, 212, 9, 110, str_repeat('a', 31) . '0', 9001,
'99999999-9999-4999-8999-999999999991', 90, '10.29.0.10',
'deleting', 18, $digest9);
$liveKey9 = str_repeat('b', 31) . '0';
$liveUuid9 = '99999999-9999-4999-8999-999999999992';
$seedMember(113, 213, 9, 110, $liveKey9, 9002, $liveUuid9,
91, '10.29.0.11', 'active', 18, $digest9);
$seedHost(9, 90, 'deleting', 18, $digest9, $operation9);
$seedHost(9, 91, 'applying', 18, $digest9, $operation9);
CloudVpcService::recordObserved($vpcKey9, 18, $digest9,
$observed(18, $digest9, [
'90' => $removedHost($vpcKey9, 90, 18, $digest9),
]), $operation9);
$assert(Capsule::table('cloud_vpc_members')->where('id', 112)
->value('state') === 'deleting'
&& Capsule::table('cloud_vpc_private_ips')->where('id', 212)
->value('state') === 'deleting'
&& (int) Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 9)
->where('host_id', 90)->value('observed_revision') === 18
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 9)
->where('host_id', 90)->value('observed_digest') === $digest9
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation9)
->value('status') !== 'succeeded',
'The incomplete first retry lost A proof or released its identities early.');
$host9 = $activeHost($vpcKey9, '10.29.0.0/24', 91, 18, $digest9,
$liveKey9, $liveUuid9, '10.29.0.11');
CloudVpcService::recordObserved($vpcKey9, 18, $digest9,
$observed(18, $digest9, ['91' => $host9]), $operation9);
$assert(Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 9)
->where('host_id', 90)->value('state') === 'deleted'
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 9)
->where('host_id', 91)->value('state') === 'active'
&& Capsule::table('cloud_vpc_members')->where('id', 112)->count() === 0
&& Capsule::table('cloud_vpc_private_ips')->where('id', 212)->count() === 0
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation9)
->value('status') === 'succeeded',
'A persisted A-removal proof did not converge after B supplied the retry evidence.');
// If every owner proof was persisted by earlier partial attempts, a later
// no-host response may finish the member operation without pretending the
// terminal Fabric gate itself removed a NIC.
$vpcKey10 = str_repeat('a', 32);
$digest10 = str_repeat('3', 64);
$operation10 = str_repeat('a', 31) . '0';
$seedVpc(10, $vpcKey10, 20, $digest10, $operation10, 'member_detach');
$seedSubnet(111, 10, str_repeat('c', 31) . '0', '10.30.0.0/25',
'active', 20, $digest10);
$seedMember(114, 214, 10, 111, str_repeat('d', 31) . '0', 10001,
'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1', 100, '10.30.0.10',
'deleting', 20, $digest10);
$seedHost(10, 100, 'deleted', 20, $digest10, $operation10);
Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 10)
->where('host_id', 100)->update([
'observed_revision' => 20, 'observed_digest' => $digest10,
'last_error_code' => null, 'last_success_at' => $now,
]);
CloudVpcService::recordObserved($vpcKey10, 20, $digest10, [
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
'revision' => 0, 'digest' => null, 'state' => 'degraded',
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_members'],
'hosts' => [],
], $operation10);
$assert(Capsule::table('cloud_vpc_members')->where('id', 114)->count() === 0
&& Capsule::table('cloud_vpc_private_ips')->where('id', 214)->count() === 0
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation10)
->value('status') === 'succeeded'
&& Capsule::table('cloud_vpcs')->where('id', 10)->value('state') !== 'active',
'Stored owner removal proofs did not finish the no-host member retry safely.');
// Old workers could leave an already-proven member/lease pair in `deleted`.
// A subnet-only terminal operation may garbage-collect that pair only when its
// owner deployment still carries an exact removal proof, then release the CIDR.
$vpcKey11 = str_repeat('b', 32);
$digest11 = str_repeat('4', 64);
$operation11 = str_repeat('b', 31) . '0';
$seedVpc(11, $vpcKey11, 22, $digest11, $operation11, 'subnet_remove');
$seedSubnet(112, 11, str_repeat('e', 31) . '0', '10.31.0.0/25',
'deleting', 22, $digest11);
$seedMember(115, 215, 11, 112, str_repeat('f', 31) . '0', 11001,
'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1', 110, '10.31.0.10',
'deleted', 22, $digest11);
foreach (['cloud_vpc_members' => 115, 'cloud_vpc_private_ips' => 215] as $table => $id) {
Capsule::table($table)->where('id', $id)->update([
'observed_revision' => 22, 'observed_digest' => $digest11,
'last_error_code' => null,
]);
}
$seedHost(11, 110, 'deleted', 22, $digest11, $operation11);
Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 11)
->where('host_id', 110)->update([
'observed_revision' => 22, 'observed_digest' => $digest11,
'last_error_code' => null, 'last_success_at' => $now,
]);
CloudVpcService::recordObserved($vpcKey11, 22, $digest11, [
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
'revision' => 0, 'digest' => null, 'state' => 'degraded',
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_members'],
'hosts' => [],
], $operation11);
$assert(Capsule::table('cloud_vpc_members')->where('id', 115)->count() === 0
&& Capsule::table('cloud_vpc_private_ips')->where('id', 215)->count() === 0
&& Capsule::table('cloud_vpc_subnets')->where('id', 112)->count() === 0
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation11)
->value('status') === 'succeeded',
'A proven historical member/IP pair kept an empty subnet CIDR occupied.');
// A no-members gate must not make a temporarily unreachable retiring host
// terminal. Keep the same operation retryable until its exact libvirt/data-
// plane removal receipt arrives, then release both conflict keys.
$vpcKey12 = str_repeat('c', 32);
$digest12 = str_repeat('5', 64);
$operation12 = str_repeat('c', 31) . '0';
$seedVpc(12, $vpcKey12, 24, $digest12, $operation12, 'member_detach');
$seedSubnet(113, 12, str_repeat('1', 31) . '2', '10.32.0.0/25',
'active', 24, $digest12);
$seedMember(116, 216, 12, 113, str_repeat('2', 31) . '1', 12001,
'cccccccc-cccc-4ccc-8ccc-ccccccccccc1', 120, '10.32.0.10',
'deleting', 24, $digest12);
$seedHost(12, 120, 'deleting', 24, $digest12, $operation12);
CloudVpcService::recordObserved($vpcKey12, 24, $digest12, [
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
'revision' => 0, 'digest' => null, 'state' => 'degraded',
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_members'],
'hosts' => [],
], $operation12);
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation12)
->value('status') === 'uncertain'
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation12)
->value('finished_at') === null
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 12)
->where('host_id', 120)->value('state') === 'deleting'
&& Capsule::table('cloud_vpc_members')->where('id', 116)
->value('state') === 'deleting'
&& Capsule::table('cloud_vpc_private_ips')->where('id', 216)
->value('state') === 'deleting',
'A transient no-members response froze an unresolved host/member tombstone.');
CloudVpcService::recordObserved($vpcKey12, 24, $digest12, [
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
'revision' => 0, 'digest' => null, 'state' => 'degraded',
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_members'],
'hosts' => [
'120' => $removedHost($vpcKey12, 120, 24, $digest12),
],
], $operation12);
$assert(Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 12)
->where('host_id', 120)->value('state') === 'deleted'
&& Capsule::table('cloud_vpc_members')->where('id', 116)->count() === 0
&& Capsule::table('cloud_vpc_private_ips')->where('id', 216)->count() === 0
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation12)
->value('status') === 'succeeded',
'An exact retry did not converge the previously unreachable retiring host.');
// A current owner proof is not enough when another historical deployment has
// lost its exact absence receipt: that host could still retain a route, proxy,
// WireGuard peer or nft rule for the retiring address. Refresh that historical
// proof explicitly before releasing the member/IP identities.
$vpcKey13 = str_repeat('d', 32);
$digest13 = str_repeat('6', 64);
$operation13 = str_repeat('d', 31) . '0';
$seedVpc(13, $vpcKey13, 26, $digest13, $operation13, 'member_detach');
$seedSubnet(114, 13, str_repeat('3', 31) . '1', '10.33.0.0/25',
'active', 26, $digest13);
$seedMember(117, 217, 13, 114, str_repeat('4', 31) . '1', 13001,
'dddddddd-dddd-4ddd-8ddd-ddddddddddd1', 130, '10.33.0.10',
'deleting', 26, $digest13);
$liveKey13 = str_repeat('5', 31) . '1';
$liveUuid13 = 'dddddddd-dddd-4ddd-8ddd-ddddddddddd2';
$seedMember(118, 218, 13, 114, $liveKey13, 13002, $liveUuid13,
130, '10.33.0.11', 'active', 26, $digest13);
$seedHost(13, 130, 'applying', 26, $digest13, $operation13);
$seedHost(13, 131, 'deleted', 26, $digest13, $operation13);
$host13 = $activeHost($vpcKey13, '10.33.0.0/24', 130, 26, $digest13,
$liveKey13, $liveUuid13, '10.33.0.11');
CloudVpcService::recordObserved($vpcKey13, 26, $digest13,
$observed(26, $digest13, ['130' => $host13]), $operation13);
$assert(Capsule::table('cloud_vpc_members')->where('id', 117)
->value('state') === 'deleting'
&& Capsule::table('cloud_vpc_private_ips')->where('id', 217)
->value('state') === 'deleting'
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation13)
->value('status') === 'uncertain',
'A stale unrelated host proof did not block global member/IP release.');
CloudVpcService::recordObserved($vpcKey13, 26, $digest13,
$observed(26, $digest13, [
'130' => $host13,
'131' => $removedHost($vpcKey13, 131, 26, $digest13),
]), $operation13);
$assert(Capsule::table('cloud_vpc_members')->where('id', 117)->count() === 0
&& Capsule::table('cloud_vpc_private_ips')->where('id', 217)->count() === 0
&& Capsule::table('cloud_vpc_members')->where('id', 118)
->value('state') === 'active'
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 13)
->where('host_id', 131)->value('observed_digest') === $digest13
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation13)
->value('status') === 'succeeded',
'Refreshing the stale historical removal proof did not unblock convergence.');
// A running host/libvirt configuration is not usable until the guest itself
// proves the exact address and routes. Preserve one actionable restart reason
// through the host, member and root projections while configuration operation
// completion remains terminal.
$vpcKey14 = str_repeat('e', 32);
$digest14 = str_repeat('7', 64);
$operation14 = str_repeat('e', 31) . '0';
$memberKey14 = str_repeat('6', 31) . '1';
$uuid14 = 'eeeeeeee-eeee-4eee-8eee-eeeeeeeeeee1';
$seedVpc(14, $vpcKey14, 28, $digest14, $operation14, 'member_attach');
$seedSubnet(115, 14, str_repeat('7', 31) . '1', '10.34.0.0/25',
'active', 28, $digest14);
$seedMember(119, 219, 14, 115, $memberKey14, 14001, $uuid14,
140, '10.34.0.10', 'applying', 28, $digest14);
$seedHost(14, 140, 'applying', 28, $digest14, $operation14);
$host14 = $activeHost(
$vpcKey14, '10.34.0.0/24', 140, 28, $digest14,
$memberKey14, $uuid14, '10.34.0.10'
);
$host14['state'] = 'applying';
$host14['private_dataplane_verified'] = false;
$host14['last_error_code'] = 'vpc_private_guest_restart_required';
$host14['member_nics'][0]['restart_required'] = true;
$host14['member_nics'][0]['guest_network_verified'] = false;
$host14['member_nics'][0]['guest_network_readback'] = 'pending_restart';
$host14['member_nics'][0]['guest_reason_code'] =
'vpc_private_guest_restart_required';
$deferred14 = $observed(28, $digest14, ['140' => $host14]);
$deferred14['state'] = 'applying';
$deferred14['next_stage_allowed'] = false;
$deferred14['private_dataplane_verified'] = false;
CloudVpcService::recordObserved(
$vpcKey14, 28, $digest14, $deferred14, $operation14
);
$rootProjection14 = $publicRootMethod->invoke(null, CloudVpc::find(14));
$assert(Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 14)
->where('host_id', 140)->value('last_error_code')
=== 'vpc_private_guest_restart_required'
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation14)
->value('status') === 'succeeded'
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation14)
->value('error_code') === null
&& Capsule::table('cloud_vpc_members')->where('id', 119)
->value('state') === 'degraded'
&& Capsule::table('cloud_vpc_members')->where('id', 119)
->value('last_error_code') === 'vpc_private_guest_restart_required'
&& Capsule::table('cloud_vpcs')->where('id', 14)
->value('last_error_code') === 'vpc_private_guest_restart_required'
&& ($rootProjection14['usable'] ?? true) === false,
'A config-complete guest restart was not retained as non-usable runtime state.');
// Each deployment must retain its own host item. A stale loop variable used
// to copy the final host's error onto every prior host. Mixed typed reasons
// remain per-host and collapse to the generic operation/root code.
$vpcKey15 = str_repeat('f', 32);
$digest15 = str_repeat('8', 64);
$operation15 = str_repeat('f', 31) . '0';
$memberKey15a = str_repeat('8', 31) . '1';
$memberKey15b = str_repeat('9', 31) . '1';
$uuid15a = 'ffffffff-ffff-4fff-8fff-fffffffffff1';
$uuid15b = 'ffffffff-ffff-4fff-8fff-fffffffffff2';
$seedVpc(15, $vpcKey15, 30, $digest15, $operation15, 'member_attach');
$seedSubnet(116, 15, str_repeat('a', 31) . '1', '10.35.0.0/25',
'active', 30, $digest15);
$seedMember(120, 220, 15, 116, $memberKey15a, 15001, $uuid15a,
150, '10.35.0.10', 'applying', 30, $digest15);
$seedMember(121, 221, 15, 116, $memberKey15b, 15002, $uuid15b,
151, '10.35.0.11', 'applying', 30, $digest15);
$seedHost(15, 150, 'applying', 30, $digest15, $operation15);
$seedHost(15, 151, 'applying', 30, $digest15, $operation15);
$host15a = $activeHost(
$vpcKey15, '10.35.0.0/24', 150, 30, $digest15,
$memberKey15a, $uuid15a, '10.35.0.10'
);
$host15b = $activeHost(
$vpcKey15, '10.35.0.0/24', 151, 30, $digest15,
$memberKey15b, $uuid15b, '10.35.0.11'
);
$routeTable15 = $routeTableMethod->invoke(null, $vpcKey15);
$peerKey15 = base64_encode(str_repeat("\x15", 32));
$host15a['proxy_ips'] = ['10.35.0.10', '10.35.0.11'];
$host15b['proxy_ips'] = ['10.35.0.10', '10.35.0.11'];
$host15a['member_nics'][0]['guest_routes'] = ['10.35.0.0/24'];
$host15b['member_nics'][0]['guest_routes'] = ['10.35.0.0/24'];
$host15a['routes'][] = [
'destination_cidr' => '10.35.0.11/32', 'device' => 'wg-fabric0',
'table' => $routeTable15, 'owner' => 'virt-cloud-vpc-dataplane-v1',
'via' => null, 'metric' => 100,
];
$host15b['routes'][] = [
'destination_cidr' => '10.35.0.10/32', 'device' => 'wg-fabric0',
'table' => $routeTable15, 'owner' => 'virt-cloud-vpc-dataplane-v1',
'via' => null, 'metric' => 100,
];
$host15a['wg_allowed_ips'] = [[
'peer_host_id' => '151', 'public_key' => $peerKey15,
'tenant_allowed_ips' => ['10.35.0.11/32'],
]];
$host15b['wg_allowed_ips'] = [[
'peer_host_id' => '150', 'public_key' => $peerKey15,
'tenant_allowed_ips' => ['10.35.0.10/32'],
]];
$host15a['state'] = $host15b['state'] = 'applying';
$host15a['private_dataplane_verified'] = false;
$host15b['private_dataplane_verified'] = false;
$host15a['member_nics'][0]['guest_network_verified'] = false;
$host15b['member_nics'][0]['guest_network_verified'] = false;
$host15a['last_error_code'] = 'vpc_private_guest_restart_required';
$host15b['last_error_code'] = 'vpc_private_guest_agent_unavailable';
$deferred15 = $observed(30, $digest15, [
'150' => $host15a, '151' => $host15b,
]);
$deferred15['state'] = 'applying';
$deferred15['next_stage_allowed'] = false;
$deferred15['private_dataplane_verified'] = false;
CloudVpcService::recordObserved(
$vpcKey15, 30, $digest15, $deferred15, $operation15
);
$assert(Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 15)
->where('host_id', 150)->value('last_error_code')
=== 'vpc_private_guest_restart_required'
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 15)
->where('host_id', 151)->value('last_error_code')
=== 'vpc_private_guest_agent_unavailable'
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation15)
->value('status') === 'succeeded'
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation15)
->value('error_code') === null
&& Capsule::table('cloud_vpcs')->where('id', 15)
->value('last_error_code') === 'vpc_data_plane_deferred',
'Multi-host guest errors were cross-contaminated or unsafely collapsed.');
// A pre-upgrade succeeded receipt has no guest-evidence contract version. It
// must fail closed immediately; the worker contract separately reclaims only
// this current last_operation_id for fresh observation.
$vpcKey16 = str_repeat('0', 32);
$digest16 = str_repeat('9', 64);
$operation16 = str_repeat('0', 31) . '1';
$memberKey16 = str_repeat('b', 31) . '1';
$uuid16 = '00000000-0000-4000-8000-000000000001';
$seedVpc(16, $vpcKey16, 32, $digest16, $operation16, 'member_attach');
$seedSubnet(117, 16, str_repeat('c', 31) . '1', '10.36.0.0/25',
'active', 32, $digest16);
$seedMember(122, 222, 16, 117, $memberKey16, 16001, $uuid16,
160, '10.36.0.10', 'active', 32, $digest16);
$seedHost(16, 160, 'active', 32, $digest16, $operation16);
Capsule::table('cloud_vpc_operations')->where('operation_id', $operation16)->update([
'status' => 'succeeded', 'evidence_version' => 0, 'finished_at' => $now,
]);
Capsule::table('cloud_vpcs')->where('id', 16)->update([
'state' => 'active', 'sync_state' => 'applied', 'observed_revision' => 32,
'observed_digest' => $digest16, 'last_success_at' => $now,
]);
Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 16)
->where('host_id', 160)->update([
'state' => 'active', 'observed_revision' => 32,
'observed_digest' => $digest16, 'last_success_at' => $now,
]);
$rootProjection16 = $publicRootMethod->invoke(null, CloudVpc::find(16));
$assert(($rootProjection16['usable'] ?? true) === false,
'A legacy succeeded observation without guest evidence remained usable.');
// Retiring member A on a host with live sibling B must converge from exact
// current configuration even when B's guest probe is unsupported. The proof
// releases only A/IP; B, the host and the root remain fail-closed for runtime
// usability until B supplies exact guest evidence.
$vpcKey17 = str_repeat('1', 31) . '7';
$digest17 = str_repeat('a', 63) . '7';
$operation17 = str_repeat('2', 31) . '7';
$retiringKey17 = str_repeat('3', 31) . '7';
$liveKey17 = str_repeat('4', 31) . '7';
$liveUuid17 = '17171717-1717-4717-8717-171717171717';
$seedVpc(17, $vpcKey17, 34, $digest17, $operation17, 'member_detach');
$seedSubnet(118, 17, str_repeat('5', 31) . '7', '10.37.0.0/25',
'active', 34, $digest17);
$seedMember(123, 223, 17, 118, $retiringKey17, 17001,
'17171717-1717-4717-8717-171717171701', 170, '10.37.0.10',
'deleting', 34, $digest17);
$seedMember(124, 224, 17, 118, $liveKey17, 17002, $liveUuid17,
170, '10.37.0.11', 'applying', 34, $digest17);
$seedHost(17, 170, 'applying', 34, $digest17, $operation17);
$detachRuntimePayload17 = json_encode([
'schema_version' => 1, 'vpc_id' => $vpcKey17,
'desired_revision' => 34, 'desired_digest' => $digest17,
'operation_id' => $operation17,
'members' => [[
'member_id' => $liveKey17, 'instance_uuid' => $liveUuid17,
'host_id' => '170', 'private_ip_cidr' => '10.37.0.11/32',
]],
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES);
Capsule::table('cloud_vpc_operations')->where('operation_id', $operation17)
->update(['payload_json' => $detachRuntimePayload17]);
$host17 = $activeHost(
$vpcKey17, '10.37.0.0/24', 170, 34, $digest17,
$liveKey17, $liveUuid17, '10.37.0.11'
);
$host17['state'] = 'applying';
$host17['private_dataplane_verified'] = false;
$host17['readback'] = 'pending_guest';
$host17['last_error_code'] = 'vpc_private_guest_probe_unsupported';
$host17['member_nics'][0]['guest_network_verified'] = false;
$host17['member_nics'][0]['guest_network_readback'] = 'unsupported';
$host17['member_nics'][0]['guest_reason_code'] =
'vpc_private_guest_probe_unsupported';
$deferred17 = $observed(34, $digest17, ['170' => $host17]);
$deferred17['state'] = 'applying';
$deferred17['next_stage_allowed'] = false;
$deferred17['private_dataplane_verified'] = false;
CloudVpcService::recordObserved(
$vpcKey17, 34, $digest17, $deferred17, $operation17
);
$rootProjection17 = $publicRootMethod->invoke(null, CloudVpc::find(17));
$assert(Capsule::table('cloud_vpc_members')->where('id', 123)->count() === 0
&& Capsule::table('cloud_vpc_private_ips')->where('id', 223)->count() === 0
&& Capsule::table('cloud_vpc_members')->where('id', 124)->count() === 1
&& Capsule::table('cloud_vpc_private_ips')->where('id', 224)->count() === 1
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 17)
->where('host_id', 170)->value('state') === 'degraded'
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation17)
->value('status') === 'succeeded'
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation17)
->value('error_code') === null
&& Capsule::table('cloud_vpcs')->where('id', 17)
->value('state') === 'degraded'
&& Capsule::table('cloud_vpcs')->where('id', 17)
->value('sync_state') === 'waiting'
&& Capsule::table('cloud_vpcs')->where('id', 17)
->value('last_error_code') === 'vpc_private_guest_probe_unsupported'
&& ($rootProjection17['usable'] ?? true) === false,
'Config-exact detach did not release only A while B guest readiness stayed closed.');
// No new mutation is required once B becomes observable. The worker reuses
// the current, idempotent detach operation and the exact guest receipt opens
// only the runtime gate; the already released A/IP rows stay absent.
$readyHost17 = $activeHost(
$vpcKey17, '10.37.0.0/24', 170, 34, $digest17,
$liveKey17, $liveUuid17, '10.37.0.11'
);
CloudVpcService::recordObserved(
$vpcKey17, 34, $digest17,
$observed(34, $digest17, ['170' => $readyHost17]),
$operation17
);
$readyRootProjection17 = $publicRootMethod->invoke(null, CloudVpc::find(17));
$assert(Capsule::table('cloud_vpc_members')->where('id', 123)->count() === 0
&& Capsule::table('cloud_vpc_private_ips')->where('id', 223)->count() === 0
&& Capsule::table('cloud_vpc_operations')->where('vpc_id', 17)
->where('operation_id', $operation17)->count() === 1
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation17)
->value('payload_json') === $detachRuntimePayload17
&& !str_contains($detachRuntimePayload17, $retiringKey17)
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation17)
->value('status') === 'succeeded'
&& Capsule::table('cloud_vpcs')->where('id', 17)
->value('state') === 'active'
&& Capsule::table('cloud_vpcs')->where('id', 17)
->value('sync_state') === 'applied'
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 17)
->where('host_id', 170)->value('state') === 'active'
&& ($readyRootProjection17['usable'] ?? false) === true,
'A current detach operation could not finish sibling runtime reconciliation.');
// Runtime evidence is member-scoped. A running/exact A remains usable when B
// is stopped after the same host configuration has completed. The operation
// is terminal for configuration, while the same ID remains the worker's
// idempotent runtime-reconcile target for B.
$vpcKey19 = str_repeat('9', 31) . 'a';
$digest19 = str_repeat('c', 63) . '9';
$operation19 = str_repeat('a', 31) . '9';
$memberKey19a = str_repeat('b', 31) . '9';
$memberKey19b = str_repeat('c', 31) . '8';
$uuid19a = '19191919-1919-4919-8919-191919191901';
$uuid19b = '19191919-1919-4919-8919-191919191902';
$seedVpc(19, $vpcKey19, 38, $digest19, $operation19, 'member_attach');
$seedSubnet(120, 19, str_repeat('d', 31) . '9', '10.39.0.0/25',
'applying', 38, $digest19);
$seedMember(126, 226, 19, 120, $memberKey19a, 19001, $uuid19a,
190, '10.39.0.10', 'applying', 38, $digest19);
$seedMember(127, 227, 19, 120, $memberKey19b, 19002, $uuid19b,
190, '10.39.0.11', 'applying', 38, $digest19);
$seedHost(19, 190, 'applying', 38, $digest19, $operation19);
$host19 = $activeHost(
$vpcKey19, '10.39.0.0/24', 190, 38, $digest19,
$memberKey19a, $uuid19a, '10.39.0.10'
);
$bridge19 = $bridgeMethod->invoke(null, $vpcKey19);
$mac19b = $privateMacMethod->invoke(null, $vpcKey19, $memberKey19b);
$host19['state'] = 'applying';
$host19['private_dataplane_verified'] = false;
$host19['readback'] = 'pending_guest';
$host19['last_error_code'] = 'vpc_private_guest_restart_required';
$host19['proxy_ips'][] = '10.39.0.11';
$host19['member_nics'][0]['guest_routes'] = ['10.39.0.0/24'];
$host19['member_nics'][] = [
'member_id' => $memberKey19b, 'instance_uuid' => $uuid19b,
'interface' => 'private0', 'private_ip_cidr' => '10.39.0.11/32',
'private_mac' => $mac19b, 'xml_verified' => true,
'config_verified' => true, 'seed_verified' => true,
'restart_required' => true,
'domain_running' => false, 'guest_network_verified' => false,
'guest_network_readback' => 'stopped',
'guest_reason_code' => 'vpc_private_guest_restart_required',
'guest_private_mac' => null, 'guest_private_ip_cidr' => null,
'guest_routes' => [], 'link_state' => 'up', 'guest_link_state' => null,
];
$host19['routes'][] = [
'destination_cidr' => '10.39.0.11/32', 'device' => $bridge19,
'table' => $routeTableMethod->invoke(null, $vpcKey19),
'owner' => 'virt-cloud-vpc-dataplane-v1', 'via' => null, 'metric' => 100,
];
$partial19 = $observed(38, $digest19, ['190' => $host19]);
$partial19['state'] = 'applying';
$partial19['next_stage_allowed'] = false;
$partial19['private_dataplane_verified'] = false;
CloudVpcService::recordObserved(
$vpcKey19, 38, $digest19, $partial19, $operation19
);
$projection19a = CloudVpcService::instancePrivateProjection(19001);
$projection19b = CloudVpcService::instancePrivateProjection(19002);
$rootProjection19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation19)
->value('status') === 'succeeded'
&& Capsule::table('cloud_vpc_members')->where('id', 126)
->value('state') === 'active'
&& Capsule::table('cloud_vpc_private_ips')->where('id', 226)
->value('state') === 'active'
&& Capsule::table('cloud_vpc_members')->where('id', 127)
->value('state') === 'degraded'
&& Capsule::table('cloud_vpc_members')->where('id', 127)
->value('last_error_code') === 'vpc_private_guest_restart_required'
&& ($projection19a['state'] ?? null) === 'active'
&& ($projection19a['readback'] ?? null) === 'confirmed'
&& ($projection19b['reason_code'] ?? null)
=== 'vpc_private_guest_restart_required'
&& ($rootProjection19['usable'] ?? true) === false,
'A stopped sibling blocked an independently proven running member.');
$ready19 = $host19;
$ready19['state'] = 'active';
$ready19['private_dataplane_verified'] = true;
$ready19['readback'] = 'confirmed';
$ready19['last_error_code'] = null;
$ready19['member_nics'][1] = [
'member_id' => $memberKey19b, 'instance_uuid' => $uuid19b,
'interface' => 'private0', 'private_ip_cidr' => '10.39.0.11/32',
'private_mac' => $mac19b, 'xml_verified' => true,
'config_verified' => true, 'seed_verified' => true,
'restart_required' => false,
'domain_running' => true, 'guest_network_verified' => true,
'guest_network_readback' => 'confirmed', 'guest_reason_code' => null,
'guest_private_mac' => $mac19b,
'guest_private_ip_cidr' => '10.39.0.11/32',
'guest_routes' => ['10.39.0.0/24'],
'link_state' => 'up', 'guest_link_state' => 'up',
];
CloudVpcService::recordObserved(
$vpcKey19, 38, $digest19,
$observed(38, $digest19, ['190' => $ready19]),
$operation19
);
$readyProjection19b = CloudVpcService::instancePrivateProjection(19002);
$readyRootProjection19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation19)
->value('operation_id') === $operation19
&& Capsule::table('cloud_vpc_members')->where('id', 126)
->value('state') === 'active'
&& Capsule::table('cloud_vpc_members')->where('id', 127)
->value('state') === 'active'
&& ($readyProjection19b['state'] ?? null) === 'active'
&& ($readyProjection19b['readback'] ?? null) === 'confirmed'
&& ($readyRootProjection19['usable'] ?? false) === true,
'Starting B did not converge only its runtime gate on the same operation.');
// Runtime usability also depends on the instance network-policy state. The
// VPC/member proof is still exact while quota/expiry enforcement is applying,
// but the public root must fail closed immediately in that window.
Capsule::table('cloud_instances')->where('id', 19001)->update([
'network_status' => 'blocking',
]);
$blockedRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
$assert(($blockedRoot19['usable'] ?? true) === false,
'A blocking instance network policy left an exact VPC publicly usable.');
Capsule::table('cloud_instances')->where('id', 19001)->update([
'network_status' => 'active',
]);
$restoredPolicyRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
$assert(($restoredPolicyRoot19['usable'] ?? false) === true,
'An active instance network policy did not restore the unchanged VPC proof.');
// A graceful shutdown can be accepted while the VM is still running. Once
// the ordinary runtime heartbeat observes the later shutoff, public usability
// must close even though no VPC mutation or new VPC operation occurred.
Capsule::table('cloud_instance_runtime')->where('instance_id', 19002)->update([
'power_state' => 'stopped', 'sync_state' => 'ok',
'last_success_at' => time(), 'updated_at' => time(),
]);
$heartbeatStoppedRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
$assert(($heartbeatStoppedRoot19['usable'] ?? true) === false,
'A fresh stopped runtime heartbeat left an old VPC proof publicly usable.');
Capsule::table('cloud_instance_runtime')->where('instance_id', 19002)->update([
'power_state' => 'running', 'sync_state' => 'ok',
'last_success_at' => time(), 'updated_at' => time(),
]);
$heartbeatRunningRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
$assert(($heartbeatRunningRoot19['usable'] ?? false) === true,
'A fresh running heartbeat did not restore the unchanged exact VPC proof.');
$expectation19b = CloudVpcService::instancePrivateExpectation(19002);
$stoppedObservation19b = [
'version' => 1, 'required' => true, 'vm_uuid' => $uuid19b,
'vpc_id' => $vpcKey19, 'member_id' => $memberKey19b,
'interface' => 'private0', 'state' => 'stopped',
'desired_revision' => 38, 'observed_revision' => 38,
'desired_digest' => $digest19, 'observed_digest' => $digest19,
'readback' => 'confirmed', 'link_state' => 'up',
'private_ip_cidr' => '10.39.0.11/32', 'private_mac' => $mac19b,
'xml_verified' => true, 'config_verified' => true,
'seed_verified' => true,
'domain_running' => false, 'restart_required' => false,
'guest_network_verified' => false,
'guest_network_readback' => 'stopped', 'guest_link_state' => null,
'reason_code' => null, 'observed_at' => time(),
];
$shutdownRunningObservation19b = $stoppedObservation19b;
$shutdownRunningObservation19b['state'] = 'active';
$shutdownRunningObservation19b['domain_running'] = true;
$shutdownRunningObservation19b['guest_network_verified'] = true;
$shutdownRunningObservation19b['guest_network_readback'] = 'confirmed';
$shutdownRunningObservation19b['guest_link_state'] = 'up';
$shutdownAccepted19b = CloudVpcService::validatePrivateObservation(
$expectation19b, $shutdownRunningObservation19b,
'action_shutdown_policy_active', $uuid19b
);
$assert(($shutdownAccepted19b['ok'] ?? false) === true
&& ($shutdownAccepted19b['state']['state'] ?? null) === 'stopping'
&& CloudVpcService::recordPrivateObservation(
19002, $shutdownAccepted19b, str_repeat('0', 31) . '9'
) === true,
'Accepted graceful shutdown did not enter a non-usable transition.');
Capsule::table('cloud_instance_runtime')->where('instance_id', 19002)->update([
'power_state' => 'shutdown', 'sync_state' => 'ok',
'last_success_at' => time(), 'updated_at' => time(),
]);
$shutdownPendingProjection19b = CloudVpcService::instancePrivateProjection(19002);
$shutdownPendingRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
$assert(Capsule::table('cloud_vpc_members')->where('id', 127)
->value('state') === 'stopping'
&& Capsule::table('cloud_vpc_private_ips')->where('id', 227)
->value('state') === 'stopping'
&& ($shutdownPendingProjection19b['state'] ?? null) === 'stopping'
&& ($shutdownPendingRoot19['usable'] ?? true) === false,
'Graceful shutdown acceptance left member, lease or root runtime usable.');
$periodicShutdownPending19 = $ready19;
$periodicShutdownPending19['state'] = 'active';
$periodicShutdownPending19['private_dataplane_verified'] = true;
CloudVpcService::recordObserved(
$vpcKey19, 38, $digest19,
$observed(38, $digest19, ['190' => $periodicShutdownPending19]),
$operation19
);
$assert(Capsule::table('cloud_vpc_members')->where('id', 127)
->value('state') === 'stopping'
&& Capsule::table('cloud_vpcs')->where('id', 19)
->value('sync_state') === 'waiting'
&& Capsule::table('cloud_vpcs')->where('id', 19)
->value('last_error_code') === 'vpc_private_guest_shutdown_pending',
'A periodic running/QGA receipt reopened a graceful-shutdown transition.');
$shutdownValidation19b = CloudVpcService::validatePrivateObservation(
$expectation19b, $stoppedObservation19b,
'action_shutdown_policy_active', $uuid19b
);
$poweroffValidation19b = CloudVpcService::validatePrivateObservation(
$expectation19b, $stoppedObservation19b,
'action_poweroff_policy_active', $uuid19b
);
$assert(($shutdownValidation19b['ok'] ?? false) === true
&& ($poweroffValidation19b['ok'] ?? false) === true
&& CloudVpcService::recordPrivateObservation(
19002, $shutdownValidation19b, str_repeat('1', 31) . '9'
) === true,
'Shutdown with a preserved configured-up link was not stored as stopped.');
Capsule::table('cloud_instance_runtime')->where('instance_id', 19002)->update([
'power_state' => 'stopped', 'sync_state' => 'ok',
'last_success_at' => time(), 'updated_at' => time(),
]);
$stoppedProjection19b = CloudVpcService::instancePrivateProjection(19002);
$stoppedRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
$assert(Capsule::table('cloud_vpc_members')->where('id', 127)
->value('state') === 'stopped'
&& Capsule::table('cloud_vpc_private_ips')->where('id', 227)
->value('state') === 'stopped'
&& ($stoppedProjection19b['state'] ?? null) === 'stopped'
&& ($stoppedProjection19b['link_state'] ?? null) === 'up'
&& ($stoppedRoot19['usable'] ?? true) === false,
'A confirmed stopped guest remained active or publicly usable.');
// Periodic VPC observation must preserve a lifecycle-confirmed stopped state;
// the strict guest probe is intentionally absent for an inactive domain.
$periodicStopped19 = $ready19;
$periodicStopped19['state'] = 'applying';
$periodicStopped19['private_dataplane_verified'] = false;
$periodicStopped19['last_error_code'] = 'vpc_private_guest_stopped';
$periodicStopped19['member_nics'][1]['domain_running'] = false;
$periodicStopped19['member_nics'][1]['restart_required'] = false;
$periodicStopped19['member_nics'][1]['guest_network_verified'] = false;
$periodicStopped19['member_nics'][1]['guest_network_readback'] = 'stopped';
$periodicStopped19['member_nics'][1]['guest_reason_code'] =
'vpc_private_guest_stopped';
$periodicStopped19['member_nics'][1]['guest_private_mac'] = null;
$periodicStopped19['member_nics'][1]['guest_private_ip_cidr'] = null;
$periodicStopped19['member_nics'][1]['guest_routes'] = [];
$periodicStoppedObservation19 = $observed(38, $digest19, [
'190' => $periodicStopped19,
]);
$periodicStoppedObservation19['state'] = 'applying';
$periodicStoppedObservation19['next_stage_allowed'] = false;
$periodicStoppedObservation19['private_dataplane_verified'] = false;
CloudVpcService::recordObserved(
$vpcKey19, 38, $digest19, $periodicStoppedObservation19, $operation19
);
$assert(Capsule::table('cloud_vpc_members')->where('id', 127)
->value('state') === 'stopped'
&& Capsule::table('cloud_vpc_private_ips')->where('id', 227)
->value('state') === 'stopped'
&& Capsule::table('cloud_vpcs')->where('id', 19)
->value('last_error_code') === 'vpc_private_guest_stopped',
'Periodic reconciliation overwrote exact stopped evidence with degraded.');
// Service policy preserves both the stopped power target and the independent
// network policy. A tenant-stopped guest must remain stopped across expiry and
// renewal when start_if_suspended=false.
$serviceSuspendStopped19b = CloudVpcService::validatePrivateObservation(
$expectation19b, $stoppedObservation19b,
'service_suspend_policy_active', $uuid19b
);
$serviceResumeStopped19b = CloudVpcService::validatePrivateObservation(
$expectation19b, $stoppedObservation19b,
'service_resume_stopped_policy_active', $uuid19b
);
$unboundServiceResume19b = CloudVpcService::validatePrivateObservation(
$expectation19b, $stoppedObservation19b,
'service_resume_unbound_policy_active', $uuid19b
);
$assert(($serviceSuspendStopped19b['ok'] ?? false) === true
&& ($serviceResumeStopped19b['ok'] ?? false) === true
&& ($unboundServiceResume19b['ok'] ?? true) === false,
'Stopped expiry/resume policy was not bound to its explicit payload target.');
// Link policy may change while the domain stays stopped. Both operations are
// exact configuration transitions, but neither makes runtime usable.
$stoppedDownObservation19b = $stoppedObservation19b;
$stoppedDownObservation19b['link_state'] = 'down';
$stoppedSuspendValidation19b = CloudVpcService::validatePrivateObservation(
$expectation19b, $stoppedDownObservation19b,
'network_suspend', $uuid19b
);
$blockedPoweroffValidation19b = CloudVpcService::validatePrivateObservation(
$expectation19b, $stoppedDownObservation19b,
'action_poweroff_policy_blocked', $uuid19b
);
$blockedServiceSuspend19b = CloudVpcService::validatePrivateObservation(
$expectation19b, $stoppedDownObservation19b,
'service_suspend_policy_blocked', $uuid19b
);
$blockedServiceResumeStopped19b = CloudVpcService::validatePrivateObservation(
$expectation19b, $stoppedDownObservation19b,
'service_resume_stopped_policy_blocked', $uuid19b
);
Capsule::table('cloud_instances')->where('id', 19002)->update([
'network_status' => 'blocked_traffic',
]);
$assert(($stoppedSuspendValidation19b['ok'] ?? false) === true
&& ($blockedPoweroffValidation19b['ok'] ?? false) === true
&& ($blockedServiceSuspend19b['ok'] ?? false) === true
&& ($blockedServiceResumeStopped19b['ok'] ?? false) === true
&& CloudVpcService::recordPrivateObservation(
19002, $stoppedSuspendValidation19b, str_repeat('2', 31) . '9'
) === true
&& (CloudVpcService::instancePrivateProjection(19002)['state'] ?? null)
=== 'stopped'
&& (CloudVpcService::instancePrivateProjection(19002)['link_state'] ?? null)
=== 'down',
'Stopped network suspend did not persist configured-down without activation.');
$stoppedResumeValidation19b = CloudVpcService::validatePrivateObservation(
$expectation19b, $stoppedObservation19b,
'network_resume', $uuid19b
);
Capsule::table('cloud_instances')->where('id', 19002)->update([
'network_status' => 'active',
]);
$assert(($stoppedResumeValidation19b['ok'] ?? false) === true
&& CloudVpcService::recordPrivateObservation(
19002, $stoppedResumeValidation19b, str_repeat('3', 31) . '9'
) === true
&& (CloudVpcService::instancePrivateProjection(19002)['state'] ?? null)
=== 'stopped'
&& (CloudVpcService::instancePrivateProjection(19002)['link_state'] ?? null)
=== 'up',
'Stopped network resume became usable or lost configured-up evidence.');
// Starting an active-policy VM needs a new, guest-visible QGA proof. The same
// config-only stopped receipt is insufficient.
$activeObservation19b = $stoppedObservation19b;
$activeObservation19b['state'] = 'active';
$activeObservation19b['domain_running'] = true;
$activeObservation19b['guest_network_verified'] = true;
$activeObservation19b['guest_network_readback'] = 'confirmed';
$activeObservation19b['guest_link_state'] = 'up';
$startValidation19b = CloudVpcService::validatePrivateObservation(
$expectation19b, $activeObservation19b,
'action_start_policy_active', $uuid19b
);
$missingStartGuest19b = $activeObservation19b;
$missingStartGuest19b['guest_network_verified'] = false;
$missingStartValidation19b = CloudVpcService::validatePrivateObservation(
$expectation19b, $missingStartGuest19b,
'action_start_policy_active', $uuid19b
);
$genericActiveValidation19b = CloudVpcService::validatePrivateObservation(
$expectation19b, $activeObservation19b,
'snapshot_restore', $uuid19b
);
$legacyActiveObservation19b = $activeObservation19b;
unset($legacyActiveObservation19b['seed_verified']);
unset($legacyActiveObservation19b['restart_required']);
unset($legacyActiveObservation19b['guest_network_verified']);
unset($legacyActiveObservation19b['guest_network_readback']);
unset($legacyActiveObservation19b['guest_link_state']);
$genericMissingGuest19b = CloudVpcService::validatePrivateObservation(
$expectation19b, $legacyActiveObservation19b,
'snapshot_restore', $uuid19b
);
$assert(($startValidation19b['ok'] ?? false) === true
&& ($missingStartValidation19b['ok'] ?? true) === false
&& ($genericActiveValidation19b['ok'] ?? false) === true
&& ($genericMissingGuest19b['ok'] ?? true) === false
&& CloudVpcService::recordPrivateObservation(
19002, $startValidation19b, str_repeat('4', 31) . '9'
) === true,
'An active lifecycle result did not require fresh guest-visible private readiness.');
Capsule::table('cloud_instance_runtime')->where('instance_id', 19002)->update([
'power_state' => 'running', 'sync_state' => 'ok',
'last_success_at' => time(), 'updated_at' => time(),
]);
$startedRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
$assert(($startedRoot19['usable'] ?? false) === true,
'Fresh QGA start did not restore VPC usability after stopped state.');
$suspendedObservation19b = [
'version' => 1, 'required' => true, 'vm_uuid' => $uuid19b,
'vpc_id' => $vpcKey19, 'member_id' => $memberKey19b,
'interface' => 'private0', 'state' => 'suspended',
'desired_revision' => 38, 'observed_revision' => 38,
'desired_digest' => $digest19, 'observed_digest' => $digest19,
'readback' => 'confirmed', 'link_state' => 'down',
'private_ip_cidr' => '10.39.0.11/32', 'private_mac' => $mac19b,
'xml_verified' => true, 'config_verified' => true,
'seed_verified' => true,
'domain_running' => true, 'restart_required' => false,
'guest_network_verified' => false,
'guest_network_readback' => 'suspended', 'guest_link_state' => null,
'reason_code' => null, 'observed_at' => time(),
];
$suspendValidation19b = CloudVpcService::validatePrivateObservation(
$expectation19b, $suspendedObservation19b, 'network_suspend', $uuid19b
);
$assert(($suspendValidation19b['ok'] ?? false) === true
&& CloudVpcService::recordPrivateObservation(
19002, $suspendValidation19b, str_repeat('d', 31) . '9'
) === true,
'An exact lifecycle link-down observation was not persisted.');
Capsule::table('cloud_instances')->where('id', 19002)->update([
'network_status' => 'blocked_traffic',
]);
$suspendedProjection19b = CloudVpcService::instancePrivateProjection(19002);
$suspendedRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
$assert(Capsule::table('cloud_vpc_members')->where('id', 127)
->value('state') === 'suspended'
&& Capsule::table('cloud_vpc_private_ips')->where('id', 227)
->value('state') === 'suspended'
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 19)
->where('host_id', 190)->value('state') === 'degraded'
&& Capsule::table('cloud_vpcs')->where('id', 19)
->value('state') === 'degraded'
&& ($suspendedProjection19b['state'] ?? null) === 'suspended'
&& ($suspendedProjection19b['link_state'] ?? null) === 'down'
&& ($suspendedRoot19['usable'] ?? true) === false,
'Lifecycle suspension left member, host or VPC runtime falsely usable.');
$periodicSuspended19 = $ready19;
$periodicSuspended19['state'] = 'applying';
$periodicSuspended19['private_dataplane_verified'] = false;
$periodicSuspended19['last_error_code'] = 'vpc_private_guest_link_mismatch';
$periodicSuspended19['member_nics'][1]['link_state'] = 'down';
$periodicSuspended19['member_nics'][1]['guest_network_verified'] = false;
$periodicSuspended19['member_nics'][1]['guest_network_readback'] = 'suspended';
$periodicSuspended19['member_nics'][1]['guest_reason_code'] =
'vpc_private_guest_link_mismatch';
$periodicObserved19 = $observed(38, $digest19, [
'190' => $periodicSuspended19,
]);
$periodicObserved19['state'] = 'applying';
$periodicObserved19['next_stage_allowed'] = false;
$periodicObserved19['private_dataplane_verified'] = false;
CloudVpcService::recordObserved(
$vpcKey19, 38, $digest19, $periodicObserved19, $operation19
);
$periodicRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
$assert(Capsule::table('cloud_vpc_members')->where('id', 127)
->value('state') === 'suspended'
&& Capsule::table('cloud_vpc_private_ips')->where('id', 227)
->value('state') === 'suspended'
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation19)
->value('status') === 'succeeded'
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 19)
->where('host_id', 190)->value('last_error_code')
=== 'vpc_private_network_suspended'
&& Capsule::table('cloud_vpcs')->where('id', 19)
->value('sync_state') === 'applied'
&& Capsule::table('cloud_vpcs')->where('id', 19)
->value('last_error_code') === 'vpc_private_network_suspended'
&& ($periodicRoot19['usable'] ?? true) === false,
'Periodic VPC observation overwrote intentional suspended runtime state.');
// User power actions and service resume preserve a blocked network target.
// A running domain is represented as suspended/config-exact, and an active/up
// receipt is rejected so neither path can bypass quota/expiry enforcement.
$blockedReboot19b = CloudVpcService::validatePrivateObservation(
$expectation19b, $suspendedObservation19b,
'action_reboot_policy_blocked', $uuid19b
);
$blockedStart19b = CloudVpcService::validatePrivateObservation(
$expectation19b, $suspendedObservation19b,
'action_start_policy_blocked', $uuid19b
);
$blockedServiceStart19b = CloudVpcService::validatePrivateObservation(
$expectation19b, $suspendedObservation19b,
'service_resume_start_policy_blocked', $uuid19b
);
$blockedButUp19b = $activeObservation19b;
$blockedButUpValidation19b = CloudVpcService::validatePrivateObservation(
$expectation19b, $blockedButUp19b,
'action_reboot_policy_blocked', $uuid19b
);
$transitioningPower19b = CloudVpcService::validatePrivateObservation(
$expectation19b, $suspendedObservation19b,
'action_reboot_policy_unready', $uuid19b
);
$assert(($blockedReboot19b['ok'] ?? false) === true
&& ($blockedStart19b['ok'] ?? false) === true
&& ($blockedServiceStart19b['ok'] ?? false) === true
&& ($blockedButUpValidation19b['ok'] ?? true) === false
&& ($transitioningPower19b['ok'] ?? true) === false
&& CloudVpcService::recordPrivateObservation(
19002, $blockedReboot19b, str_repeat('5', 31) . '9'
) === true
&& Capsule::table('cloud_vpc_members')->where('id', 127)
->value('state') === 'suspended'
&& Capsule::table('cloud_instances')->where('id', 19002)
->value('network_status') === 'blocked_traffic',
'A blocked power/service action raised private0 or failed to converge safely.');
$resumedObservation19b = $suspendedObservation19b;
$resumedObservation19b['state'] = 'active';
$resumedObservation19b['link_state'] = 'up';
$resumedObservation19b['guest_network_verified'] = true;
$resumedObservation19b['guest_network_readback'] = 'confirmed';
$resumedObservation19b['guest_link_state'] = 'up';
$resumeValidation19b = CloudVpcService::validatePrivateObservation(
$expectation19b, $resumedObservation19b, 'network_resume', $uuid19b
);
$assert(($resumeValidation19b['ok'] ?? false) === true
&& CloudVpcService::recordPrivateObservation(
19002, $resumeValidation19b, str_repeat('e', 31) . '9'
) === true,
'An exact lifecycle link-up observation was not persisted.');
Capsule::table('cloud_instances')->where('id', 19002)->update([
'network_status' => 'active',
]);
$resumedProjection19b = CloudVpcService::instancePrivateProjection(19002);
$resumedRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
$assert(Capsule::table('cloud_vpc_members')->where('id', 127)
->value('state') === 'active'
&& Capsule::table('cloud_vpc_private_ips')->where('id', 227)
->value('state') === 'active'
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 19)
->where('host_id', 190)->value('state') === 'active'
&& Capsule::table('cloud_vpcs')->where('id', 19)
->value('state') === 'active'
&& ($resumedProjection19b['state'] ?? null) === 'active'
&& ($resumedProjection19b['link_state'] ?? null) === 'up'
&& ($resumedRoot19['usable'] ?? false) === true,
'Lifecycle resume became usable before exact link-up or failed to recover.');
// A running last member cannot be detached safely. Keep its tombstone and the
// operation retryable; after the VM is stopped, the exact removal receipt must
// converge the same operation ID instead of requiring a second detach request.
$vpcKey18 = str_repeat('6', 31) . '8';
$digest18 = str_repeat('b', 63) . '8';
$operation18 = str_repeat('7', 31) . '8';
$memberKey18 = str_repeat('8', 31) . '8';
$seedVpc(18, $vpcKey18, 36, $digest18, $operation18, 'member_detach');
$seedSubnet(119, 18, str_repeat('9', 31) . '8', '10.38.0.0/25',
'active', 36, $digest18);
$seedMember(125, 225, 18, 119, $memberKey18, 18001,
'18181818-1818-4818-8818-181818181818', 180, '10.38.0.10',
'deleting', 36, $digest18);
$seedHost(18, 180, 'deleting', 36, $digest18, $operation18);
CloudVpcService::recordObserved($vpcKey18, 36, $digest18, [
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
'revision' => 0, 'digest' => null, 'state' => 'degraded',
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
'fabric_gate' => ['active' => true],
'hosts' => ['180' => [
'vpc_id' => $vpcKey18, 'host_id' => '180',
'desired_revision' => 36, 'desired_digest' => $digest18,
'observed_revision' => 0, 'observed_digest' => null,
'state' => 'uncertain', 'transport_verified' => false,
'private_dataplane_verified' => false, 'removed' => false,
'last_error_code' => 'vpc_private_remove_requires_stopped',
]],
], $operation18);
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation18)
->value('status') === 'uncertain'
&& Capsule::table('cloud_vpc_members')->where('id', 125)
->value('state') === 'deleting'
&& Capsule::table('cloud_vpc_private_ips')->where('id', 225)
->value('state') === 'deleting',
'A running-member detach became terminal or released its tombstone early.');
CloudVpcService::recordObserved(
$vpcKey18, 36, $digest18,
$observed(36, $digest18, [
'180' => $removedHost($vpcKey18, 180, 36, $digest18),
]),
$operation18
);
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation18)
->value('status') === 'succeeded'
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation18)
->value('operation_id') === $operation18
&& Capsule::table('cloud_vpc_members')->where('id', 125)->count() === 0
&& Capsule::table('cloud_vpc_private_ips')->where('id', 225)->count() === 0
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 18)
->where('host_id', 180)->value('state') === 'deleted',
'Stopping the VM did not converge the original detach operation.');
echo "VPC member/IP/subnet tombstone convergence contracts passed.\n";