You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2378 lines
121 KiB
PHP
2378 lines
121 KiB
PHP
<?php
|
|
|
|
use App\Models\CloudVpc;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\CloudVpcService;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$root = dirname(__DIR__);
|
|
$assert = static function ($condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
|
|
if (!is_file($root . '/vendor/autoload.php')) {
|
|
echo "VPC tombstone convergence runtime skipped (vendor missing).\n";
|
|
return;
|
|
}
|
|
require $root . '/vendor/autoload.php';
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
|
|
$schema->create('cloud_regions', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
});
|
|
$schema->create('cloud_vpcs', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('vpc_key', 32)->unique();
|
|
$table->unsignedInteger('user_id');
|
|
$table->unsignedInteger('region_id');
|
|
$table->unsignedInteger('private_network_id')->default(0);
|
|
$table->string('name', 63);
|
|
$table->string('cidr', 18);
|
|
$table->string('fabric_id', 64);
|
|
$table->string('fabric_region', 64);
|
|
$table->unsignedBigInteger('fabric_revision');
|
|
$table->unsignedInteger('max_members');
|
|
$table->string('state', 16);
|
|
$table->unsignedBigInteger('desired_revision');
|
|
$table->unsignedBigInteger('observed_revision')->default(0);
|
|
$table->string('desired_digest', 64);
|
|
$table->string('observed_digest', 64)->nullable();
|
|
$table->string('sync_state', 16);
|
|
$table->string('last_operation_id', 32)->nullable();
|
|
$table->string('last_error_code', 64)->nullable();
|
|
$table->unsignedBigInteger('last_success_at')->default(0);
|
|
$table->unsignedBigInteger('last_failure_at')->default(0);
|
|
$table->unsignedBigInteger('created_at');
|
|
$table->unsignedBigInteger('updated_at');
|
|
});
|
|
$schema->create('cloud_vpc_subnets', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('subnet_key', 32);
|
|
$table->unsignedInteger('vpc_id');
|
|
$table->string('cidr', 18);
|
|
$table->unsignedBigInteger('network_address');
|
|
$table->unsignedBigInteger('broadcast_address');
|
|
$table->unsignedTinyInteger('prefix_length');
|
|
$table->string('gateway_ip', 15)->nullable();
|
|
$table->string('state', 16);
|
|
$table->unsignedBigInteger('desired_revision');
|
|
$table->unsignedBigInteger('observed_revision')->default(0);
|
|
$table->string('desired_digest', 64)->nullable();
|
|
$table->string('observed_digest', 64)->nullable();
|
|
$table->string('last_error_code', 64)->nullable();
|
|
$table->unsignedBigInteger('created_at');
|
|
$table->unsignedBigInteger('updated_at');
|
|
$table->unique(['vpc_id', 'subnet_key']);
|
|
$table->unique(['vpc_id', 'cidr']);
|
|
});
|
|
$schema->create('cloud_instances', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->unsignedInteger('host_id');
|
|
$table->string('uuid', 36);
|
|
$table->string('status', 24);
|
|
$table->string('network_status', 24)->default('active');
|
|
});
|
|
$schema->create('cloud_instance_runtime', static function (Blueprint $table): void {
|
|
$table->unsignedInteger('instance_id')->primary();
|
|
$table->unsignedInteger('host_id');
|
|
$table->string('uuid', 36);
|
|
$table->string('power_state', 16)->default('unknown');
|
|
$table->string('sync_state', 16)->default('never');
|
|
$table->unsignedBigInteger('last_success_at')->default(0);
|
|
$table->unsignedBigInteger('last_attempt_at')->default(0);
|
|
$table->unsignedBigInteger('revision')->default(0);
|
|
$table->string('last_error_code', 64)->default('');
|
|
$table->unsignedBigInteger('created_at');
|
|
$table->unsignedBigInteger('updated_at');
|
|
$table->unique(['host_id', 'uuid']);
|
|
});
|
|
$schema->create('cloud_vpc_members', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('member_key', 32);
|
|
$table->unsignedInteger('vpc_id');
|
|
$table->unsignedInteger('subnet_id');
|
|
$table->unsignedInteger('private_ip_id')->nullable();
|
|
$table->unsignedInteger('instance_id');
|
|
$table->unsignedInteger('host_id');
|
|
$table->string('private_ip', 15);
|
|
$table->unsignedBigInteger('private_ip_int');
|
|
$table->string('state', 16);
|
|
$table->unsignedBigInteger('desired_revision');
|
|
$table->unsignedBigInteger('observed_revision')->default(0);
|
|
$table->string('desired_digest', 64)->nullable();
|
|
$table->string('observed_digest', 64)->nullable();
|
|
$table->string('last_operation_id', 32)->nullable();
|
|
$table->string('last_error_code', 64)->nullable();
|
|
$table->unsignedBigInteger('created_at');
|
|
$table->unsignedBigInteger('updated_at');
|
|
$table->unique('instance_id');
|
|
$table->unique(['vpc_id', 'private_ip']);
|
|
$table->unique('private_ip_id');
|
|
});
|
|
$schema->create('cloud_vpc_private_ips', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('ip_key', 32);
|
|
$table->unsignedInteger('vpc_id');
|
|
$table->unsignedInteger('subnet_id');
|
|
$table->unsignedInteger('member_id')->nullable();
|
|
$table->unsignedInteger('instance_id')->nullable();
|
|
$table->string('private_ip', 15);
|
|
$table->unsignedBigInteger('private_ip_int');
|
|
$table->string('state', 16);
|
|
$table->unsignedBigInteger('desired_revision');
|
|
$table->unsignedBigInteger('observed_revision')->default(0);
|
|
$table->string('desired_digest', 64)->nullable();
|
|
$table->string('observed_digest', 64)->nullable();
|
|
$table->string('last_error_code', 64)->nullable();
|
|
$table->unsignedBigInteger('created_at');
|
|
$table->unsignedBigInteger('updated_at');
|
|
$table->unique(['vpc_id', 'private_ip']);
|
|
$table->unique('member_id');
|
|
});
|
|
foreach (['cloud_vpc_routes' => 'route_key', 'cloud_vpc_acl_rules' => 'rule_key']
|
|
as $tableName => $keyColumn) {
|
|
$schema->create($tableName, static function (Blueprint $table) use (
|
|
$keyColumn, $tableName
|
|
): void {
|
|
$table->increments('id');
|
|
$table->string($keyColumn, 32);
|
|
$table->unsignedInteger('vpc_id');
|
|
if ($tableName === 'cloud_vpc_routes') {
|
|
$table->unsignedInteger('member_id')->nullable();
|
|
$table->string('destination_cidr', 18)->nullable();
|
|
$table->string('next_hop_ip', 15)->nullable();
|
|
$table->unsignedSmallInteger('metric')->nullable();
|
|
} else {
|
|
$table->unsignedInteger('priority')->nullable();
|
|
$table->string('direction', 8)->nullable();
|
|
$table->string('action', 8)->nullable();
|
|
$table->string('protocol', 8)->nullable();
|
|
$table->string('source_cidr', 18)->nullable();
|
|
$table->string('destination_cidr', 18)->nullable();
|
|
$table->unsignedSmallInteger('port_from')->nullable();
|
|
$table->unsignedSmallInteger('port_to')->nullable();
|
|
}
|
|
$table->string('state', 16);
|
|
$table->unsignedBigInteger('desired_revision');
|
|
$table->unsignedBigInteger('observed_revision')->default(0);
|
|
$table->string('desired_digest', 64)->nullable();
|
|
$table->string('observed_digest', 64)->nullable();
|
|
$table->string('last_error_code', 64)->nullable();
|
|
$table->unsignedBigInteger('updated_at');
|
|
});
|
|
}
|
|
$schema->create('cloud_vpc_host_deployments', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->unsignedInteger('vpc_id');
|
|
$table->unsignedInteger('host_id');
|
|
$table->unsignedBigInteger('fabric_revision')->default(1);
|
|
$table->string('state', 16);
|
|
$table->unsignedBigInteger('desired_revision');
|
|
$table->unsignedBigInteger('observed_revision')->default(0);
|
|
$table->string('desired_digest', 64)->nullable();
|
|
$table->string('observed_digest', 64)->nullable();
|
|
$table->string('last_operation_id', 32)->nullable();
|
|
$table->string('last_error_code', 64)->nullable();
|
|
$table->unsignedBigInteger('last_success_at')->default(0);
|
|
$table->unsignedBigInteger('last_failure_at')->default(0);
|
|
$table->unsignedBigInteger('created_at');
|
|
$table->unsignedBigInteger('updated_at');
|
|
$table->unique(['vpc_id', 'host_id']);
|
|
});
|
|
$schema->create('cloud_vpc_operations', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('operation_id', 32)->unique();
|
|
$table->unsignedInteger('vpc_id');
|
|
$table->unsignedInteger('user_id');
|
|
$table->string('kind', 24);
|
|
$table->unsignedBigInteger('desired_revision');
|
|
$table->string('status', 24);
|
|
$table->text('payload_json');
|
|
$table->text('observed_json')->nullable();
|
|
$table->unsignedSmallInteger('evidence_version')->default(0);
|
|
$table->string('error_code', 64)->nullable();
|
|
$table->string('lease_token', 32)->nullable();
|
|
$table->unsignedBigInteger('lease_until')->default(0);
|
|
$table->unsignedBigInteger('attempts')->default(1);
|
|
$table->unsignedBigInteger('created_at');
|
|
$table->unsignedBigInteger('updated_at');
|
|
$table->unsignedBigInteger('finished_at')->nullable();
|
|
});
|
|
|
|
Capsule::table('cloud_regions')->insert(['id' => 1]);
|
|
$now = time();
|
|
$privateMacMethod = new ReflectionMethod(CloudVpcService::class, 'privateMacForMember');
|
|
$bridgeMethod = new ReflectionMethod(CloudVpcService::class, 'privateBridgeForVpc');
|
|
$nftMethod = new ReflectionMethod(CloudVpcService::class, 'privateNftTableForVpc');
|
|
$routeTableMethod = new ReflectionMethod(CloudVpcService::class, 'privateRouteTableForVpc');
|
|
$publicRootMethod = new ReflectionMethod(CloudVpcService::class, 'publicRoot');
|
|
$canonicalJsonMethod = new ReflectionMethod(CloudVpcService::class, 'canonicalJson');
|
|
$validateHostMethod = new ReflectionMethod(
|
|
CloudVpcService::class, 'validateHostDataplaneEvidence'
|
|
);
|
|
$assert(CloudVpcService::ENGINE_TIMEOUT_SECONDS > 40.0
|
|
&& CloudVpcService::OPERATION_LEASE_SECONDS
|
|
>= CloudVpcService::ENGINE_TIMEOUT_SECONDS + 30.0,
|
|
'VPC timeout/lease do not contain the engine request-wide probe budget.');
|
|
|
|
$seedVpc = static function (
|
|
int $id, string $vpcKey, int $revision, string $digest,
|
|
string $operationId, string $kind
|
|
) use ($now): void {
|
|
Capsule::table('cloud_vpcs')->insert([
|
|
'id' => $id, 'vpc_key' => $vpcKey, 'user_id' => 9, 'region_id' => 1,
|
|
'private_network_id' => 0, 'name' => 'tombstone-' . $id,
|
|
'cidr' => '10.' . (20 + $id) . '.0.0/24', 'fabric_id' => 'fabric-test',
|
|
'fabric_region' => 'test-region', 'fabric_revision' => 7,
|
|
'max_members' => 32, 'state' => 'applying',
|
|
'desired_revision' => $revision, 'observed_revision' => $revision - 1,
|
|
'desired_digest' => $digest, 'observed_digest' => null,
|
|
'sync_state' => 'pending', 'last_operation_id' => $operationId,
|
|
'last_error_code' => null, 'last_success_at' => 0,
|
|
'last_failure_at' => 0, 'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_vpc_operations')->insert([
|
|
'operation_id' => $operationId, 'vpc_id' => $id, 'user_id' => 9,
|
|
'kind' => $kind, 'desired_revision' => $revision, 'status' => 'running',
|
|
'payload_json' => '{}', 'observed_json' => null, 'error_code' => null,
|
|
'lease_token' => null, 'lease_until' => $now + 120, 'attempts' => 1,
|
|
'created_at' => $now, 'updated_at' => $now, 'finished_at' => null,
|
|
]);
|
|
};
|
|
$seedSubnet = static function (
|
|
int $id, int $vpcId, string $key, string $cidr, string $state,
|
|
int $revision, string $digest
|
|
) use ($now): void {
|
|
[$ip, $prefix] = explode('/', $cidr);
|
|
$packed = unpack('Nvalue', inet_pton($ip));
|
|
$network = (int) sprintf('%u', (int) $packed['value']);
|
|
$size = 2 ** (32 - (int) $prefix);
|
|
Capsule::table('cloud_vpc_subnets')->insert([
|
|
'id' => $id, 'subnet_key' => $key, 'vpc_id' => $vpcId,
|
|
'cidr' => $cidr, 'network_address' => $network,
|
|
'broadcast_address' => $network + $size - 1,
|
|
'prefix_length' => (int) $prefix, 'gateway_ip' => null,
|
|
'state' => $state, 'desired_revision' => $revision,
|
|
'observed_revision' => max(0, $revision - 1),
|
|
'desired_digest' => $digest, 'observed_digest' => null,
|
|
'last_error_code' => null, 'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
};
|
|
$seedMember = static function (
|
|
int $memberId, int $leaseId, int $vpcId, int $subnetId,
|
|
string $memberKey, int $instanceId, string $uuid, int $hostId,
|
|
string $ip, string $state, int $revision, string $digest
|
|
) use ($now): void {
|
|
$packed = unpack('Nvalue', inet_pton($ip));
|
|
$ipInt = (int) sprintf('%u', (int) $packed['value']);
|
|
Capsule::table('cloud_instances')->insert([
|
|
'id' => $instanceId, 'host_id' => $hostId,
|
|
'uuid' => $uuid, 'status' => 'active',
|
|
'network_status' => 'active',
|
|
]);
|
|
Capsule::table('cloud_instance_runtime')->insert([
|
|
'instance_id' => $instanceId, 'host_id' => $hostId, 'uuid' => $uuid,
|
|
'power_state' => 'running', 'sync_state' => 'ok',
|
|
'last_success_at' => $now, 'last_attempt_at' => $now,
|
|
'revision' => 1, 'last_error_code' => '',
|
|
'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_vpc_members')->insert([
|
|
'id' => $memberId, 'member_key' => $memberKey, 'vpc_id' => $vpcId,
|
|
'subnet_id' => $subnetId, 'private_ip_id' => $leaseId,
|
|
'instance_id' => $instanceId, 'host_id' => $hostId,
|
|
'private_ip' => $ip, 'private_ip_int' => $ipInt, 'state' => $state,
|
|
'desired_revision' => $revision,
|
|
'observed_revision' => max(0, $revision - 1),
|
|
'desired_digest' => $digest, 'observed_digest' => null,
|
|
'last_operation_id' => null, 'last_error_code' => null,
|
|
'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_vpc_private_ips')->insert([
|
|
'id' => $leaseId, 'ip_key' => str_pad(dechex($leaseId), 32, '0', STR_PAD_LEFT),
|
|
'vpc_id' => $vpcId, 'subnet_id' => $subnetId,
|
|
'member_id' => $memberId, 'instance_id' => $instanceId,
|
|
'private_ip' => $ip, 'private_ip_int' => $ipInt, 'state' => $state,
|
|
'desired_revision' => $revision,
|
|
'observed_revision' => max(0, $revision - 1),
|
|
'desired_digest' => $digest, 'observed_digest' => null,
|
|
'last_error_code' => null, 'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
};
|
|
$seedHost = static function (
|
|
int $vpcId, int $hostId, string $state, int $revision,
|
|
string $digest, string $operationId
|
|
) use ($now): void {
|
|
Capsule::table('cloud_vpc_host_deployments')->insert([
|
|
'vpc_id' => $vpcId, 'host_id' => $hostId, 'fabric_revision' => 7,
|
|
'state' => $state, 'desired_revision' => $revision,
|
|
'observed_revision' => max(0, $revision - 1),
|
|
'desired_digest' => $digest, 'observed_digest' => null,
|
|
'last_operation_id' => $operationId, 'last_error_code' => null,
|
|
'last_success_at' => 0, 'last_failure_at' => 0,
|
|
'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
};
|
|
$activeHost = static function (
|
|
string $vpcKey, string $vpcCidr, int $hostId, int $revision, string $digest,
|
|
string $memberKey, string $uuid, string $ip
|
|
) use ($privateMacMethod, $bridgeMethod, $nftMethod, $routeTableMethod): array {
|
|
$bridge = $bridgeMethod->invoke(null, $vpcKey);
|
|
return [
|
|
'vpc_id' => $vpcKey, 'host_id' => (string) $hostId,
|
|
'desired_revision' => $revision, 'desired_digest' => $digest,
|
|
'observed_revision' => $revision, 'observed_digest' => $digest,
|
|
'state' => 'active', 'transport_verified' => true,
|
|
'private_dataplane_verified' => true, 'removed' => false,
|
|
'config_verified' => true, 'config_observed_revision' => $revision,
|
|
'config_observed_digest' => $digest,
|
|
'bridge' => $bridge, 'bridge_owner' => 'virt-cloud-vpc-dataplane-v1',
|
|
'nft_table' => $nftMethod->invoke(null, $vpcKey),
|
|
'anti_spoof' => true, 'rp_filter' => 2, 'proxy_arp' => 0,
|
|
'proxy_arp_pvlan' => 1, 'proxy_ips' => [$ip],
|
|
'acl_revision' => $revision, 'rollback' => 'none',
|
|
'readback' => 'confirmed',
|
|
'member_nics' => [[
|
|
'member_id' => $memberKey, 'instance_uuid' => $uuid,
|
|
'interface' => 'private0', 'private_ip_cidr' => $ip . '/32',
|
|
'private_mac' => $privateMacMethod->invoke(null, $vpcKey, $memberKey),
|
|
'xml_verified' => true, 'config_verified' => true,
|
|
'seed_verified' => true,
|
|
'restart_required' => false, 'domain_running' => true,
|
|
'guest_network_verified' => true,
|
|
'guest_network_readback' => 'confirmed',
|
|
'guest_reason_code' => null,
|
|
'guest_private_mac' => $privateMacMethod->invoke(null, $vpcKey, $memberKey),
|
|
'guest_private_ip_cidr' => $ip . '/32',
|
|
'guest_routes' => [$vpcCidr],
|
|
'link_state' => 'up', 'guest_link_state' => 'up',
|
|
]],
|
|
'routes' => [[
|
|
'destination_cidr' => $ip . '/32', 'device' => $bridge,
|
|
'table' => $routeTableMethod->invoke(null, $vpcKey),
|
|
'owner' => 'virt-cloud-vpc-dataplane-v1', 'via' => null,
|
|
'metric' => 100,
|
|
]],
|
|
'wg_allowed_ips' => [],
|
|
];
|
|
};
|
|
$removedHost = static function (
|
|
string $vpcKey, int $hostId, int $revision, string $digest
|
|
): array {
|
|
return [
|
|
'vpc_id' => $vpcKey, 'host_id' => (string) $hostId,
|
|
'desired_revision' => $revision, 'desired_digest' => $digest,
|
|
'observed_revision' => $revision, 'observed_digest' => $digest,
|
|
'state' => 'active', 'transport_verified' => true,
|
|
'private_dataplane_verified' => true, 'removed' => true,
|
|
'bridge' => null, 'bridge_owner' => null, 'nft_table' => null,
|
|
'anti_spoof' => false, 'rp_filter' => null, 'proxy_arp' => null,
|
|
'proxy_arp_pvlan' => null, 'proxy_ips' => [], 'rollback' => 'none',
|
|
'readback' => 'confirmed', 'member_nics' => [], 'routes' => [],
|
|
'wg_allowed_ips' => [],
|
|
];
|
|
};
|
|
$observed = static function (int $revision, string $digest, array $hosts): array {
|
|
return [
|
|
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
|
|
'revision' => $revision, 'digest' => $digest, 'state' => 'active',
|
|
'next_stage_allowed' => true, 'private_dataplane_verified' => true,
|
|
'fabric_gate' => ['active' => true], 'hosts' => $hosts,
|
|
];
|
|
};
|
|
|
|
// Creating an empty VPC completes only its PHP-owned control-plane
|
|
// reservation. Both engine terminal-gate spellings are accepted, but neither
|
|
// may make the VPC data plane usable without a participating member host.
|
|
foreach ([
|
|
['id' => 20, 'subnet_id' => 2020, 'gate' => 'vpc_no_participating_hosts'],
|
|
['id' => 21, 'subnet_id' => 2021, 'gate' => 'vpc_no_members'],
|
|
] as $emptyCreate) {
|
|
$id = (int) $emptyCreate['id'];
|
|
$revision = $id * 2;
|
|
$vpcKey = substr(hash('sha256', 'empty-vpc-' . $id), 0, 32);
|
|
$digest = hash('sha256', 'empty-vpc-digest-' . $id);
|
|
$operationId = substr(hash('sha256', 'empty-vpc-operation-' . $id), 0, 32);
|
|
$subnetKey = substr(hash('sha256', 'empty-vpc-subnet-' . $id), 0, 32);
|
|
$seedVpc($id, $vpcKey, $revision, $digest, $operationId, 'vpc_create');
|
|
$seedSubnet((int) $emptyCreate['subnet_id'], $id, $subnetKey,
|
|
'10.' . (20 + $id) . '.0.0/24', 'applying', $revision, $digest);
|
|
$desiredJson = $canonicalJsonMethod->invoke(
|
|
null,
|
|
CloudVpcService::desiredDocument(CloudVpc::find($id))
|
|
);
|
|
$digest = hash('sha256', $desiredJson);
|
|
Capsule::table('cloud_vpcs')->where('id', $id)->update([
|
|
'desired_digest' => $digest,
|
|
]);
|
|
Capsule::table('cloud_vpc_subnets')->where('vpc_id', $id)->update([
|
|
'desired_digest' => $digest,
|
|
]);
|
|
Capsule::table('cloud_vpc_operations')->where('operation_id', $operationId)->update([
|
|
'payload_json' => $desiredJson,
|
|
]);
|
|
|
|
CloudVpcService::recordObserved($vpcKey, $revision, $digest, [
|
|
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
|
|
'revision' => 0, 'digest' => null, 'state' => 'degraded',
|
|
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
|
|
'fabric_gate' => [
|
|
'active' => false, 'error_code' => (string) $emptyCreate['gate'],
|
|
],
|
|
'hosts' => [],
|
|
], $operationId);
|
|
|
|
$root = CloudVpc::where('id', $id)->first();
|
|
$operation = Capsule::table('cloud_vpc_operations')
|
|
->where('operation_id', $operationId)->first();
|
|
$subnet = Capsule::table('cloud_vpc_subnets')
|
|
->where('id', (int) $emptyCreate['subnet_id'])->first();
|
|
$receipt = json_decode((string) ($operation->observed_json ?? ''), true);
|
|
$projection = $root === null ? [] : $publicRootMethod->invoke(null, $root);
|
|
$assert($root !== null
|
|
&& (string) ($operation->status ?? '') === 'succeeded'
|
|
&& (int) ($operation->evidence_version ?? 0)
|
|
=== CloudVpcService::OBSERVATION_EVIDENCE_VERSION
|
|
&& ($operation->error_code ?? null) === null
|
|
&& $operation->finished_at !== null
|
|
&& (string) $root->state === 'degraded'
|
|
&& (string) $root->sync_state === 'applied'
|
|
&& (int) $root->observed_revision === $revision
|
|
&& (string) $root->observed_digest === $digest
|
|
&& $root->last_error_code === null
|
|
&& ($projection['usable'] ?? true) === false
|
|
&& ($projection['member_count'] ?? null) === 0
|
|
&& ($projection['setup_state'] ?? null) === 'awaiting_members'
|
|
&& (string) ($subnet->state ?? '') === 'active'
|
|
&& (int) ($subnet->observed_revision ?? 0) === $revision
|
|
&& (string) ($subnet->observed_digest ?? '') === $digest
|
|
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', $id)->count() === 0
|
|
&& Capsule::table('cloud_vpc_members')->where('vpc_id', $id)->count() === 0
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('vpc_id', $id)->count() === 0
|
|
&& ($receipt['no_data_plane_terminal_proof']['gate'] ?? null)
|
|
=== (string) $emptyCreate['gate'],
|
|
'An exact empty-VPC create receipt did not complete as a non-usable control-plane reservation.');
|
|
|
|
// Backward compatibility is projection-only: an old failed root with one
|
|
// of the reviewed empty gate codes is shown as awaiting members without
|
|
// rewriting its persisted operation or sync status.
|
|
Capsule::table('cloud_vpcs')->where('id', $id)->update([
|
|
'sync_state' => 'failed',
|
|
'last_error_code' => (string) $emptyCreate['gate'],
|
|
]);
|
|
Capsule::table('cloud_vpc_operations')->where('operation_id', $operationId)->update([
|
|
'status' => 'failed',
|
|
'error_code' => (string) $emptyCreate['gate'],
|
|
]);
|
|
$legacyProjection = $publicRootMethod->invoke(null, CloudVpc::find($id));
|
|
$assert(($legacyProjection['member_count'] ?? null) === 0
|
|
&& ($legacyProjection['setup_state'] ?? null) === 'awaiting_members'
|
|
&& (string) Capsule::table('cloud_vpcs')->where('id', $id)
|
|
->value('sync_state') === 'failed'
|
|
&& (string) Capsule::table('cloud_vpc_operations')
|
|
->where('operation_id', $operationId)->value('status') === 'failed',
|
|
'A legacy empty gate was not projected safely or GET rewrote its ledger.');
|
|
|
|
$seedHost($id, 900 + $id, 'degraded', $revision, $digest, $operationId);
|
|
$hostProjection = $publicRootMethod->invoke(null, CloudVpc::find($id));
|
|
$assert(($hostProjection['member_count'] ?? null) === 0
|
|
&& ($hostProjection['setup_state'] ?? null) === null,
|
|
'A VPC with a non-deleted host deployment was projected as awaiting members.');
|
|
}
|
|
|
|
// Legacy workers persisted a safe empty create as failed before the control-
|
|
// plane reservation became a supported terminal outcome. Its GET projection
|
|
// is compatibility-only: revision/digest and child state remain untouched.
|
|
$vpcKey27 = substr(hash('sha256', 'empty-vpc-27'), 0, 32);
|
|
$operation27 = substr(hash('sha256', 'empty-vpc-operation-27'), 0, 32);
|
|
$placeholderDigest27 = hash('sha256', 'empty-vpc-digest-27');
|
|
$seedVpc(27, $vpcKey27, 54, $placeholderDigest27, $operation27, 'vpc_create');
|
|
$seedSubnet(2027, 27, substr(hash('sha256', 'empty-vpc-subnet-27'), 0, 32),
|
|
'10.47.0.0/24', 'applying', 54, $placeholderDigest27);
|
|
$desiredJson27 = $canonicalJsonMethod->invoke(
|
|
null,
|
|
CloudVpcService::desiredDocument(CloudVpc::find(27))
|
|
);
|
|
$digest27 = hash('sha256', $desiredJson27);
|
|
Capsule::table('cloud_vpcs')->where('id', 27)->update([
|
|
'state' => 'degraded', 'sync_state' => 'failed',
|
|
'desired_digest' => $digest27, 'observed_revision' => 53,
|
|
'observed_digest' => null,
|
|
'last_error_code' => 'vpc_no_participating_hosts',
|
|
]);
|
|
Capsule::table('cloud_vpc_subnets')->where('id', 2027)->update([
|
|
'state' => 'applying', 'desired_digest' => $digest27,
|
|
'observed_revision' => 53, 'observed_digest' => null,
|
|
]);
|
|
Capsule::table('cloud_vpc_operations')->where('operation_id', $operation27)->update([
|
|
'status' => 'failed', 'payload_json' => $desiredJson27,
|
|
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
|
|
'error_code' => 'vpc_no_participating_hosts',
|
|
'observed_json' => json_encode([
|
|
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
|
|
'revision' => 0, 'digest' => null, 'state' => 'degraded',
|
|
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
|
|
'fabric_gate' => [
|
|
'active' => false,
|
|
'error_code' => 'vpc_no_participating_hosts',
|
|
],
|
|
'hosts' => [],
|
|
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES),
|
|
'finished_at' => $now,
|
|
]);
|
|
$legacyProjection27 = $publicRootMethod->invoke(null, CloudVpc::find(27));
|
|
$assert(($legacyProjection27['setup_state'] ?? null) === 'awaiting_members'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 27)
|
|
->value('observed_revision') === 53
|
|
&& Capsule::table('cloud_vpcs')->where('id', 27)
|
|
->value('observed_digest') === null
|
|
&& Capsule::table('cloud_vpc_subnets')->where('id', 2027)
|
|
->value('state') === 'applying'
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation27)
|
|
->value('status') === 'failed',
|
|
'A real legacy empty-create failure was not projected read-only as awaiting members.');
|
|
|
|
// A matching gate code is insufficient when the engine envelope contradicts
|
|
// the no-data-plane claim. Never complete even the control-plane reservation.
|
|
$vpcKey28 = substr(hash('sha256', 'empty-vpc-28'), 0, 32);
|
|
$operation28 = substr(hash('sha256', 'empty-vpc-operation-28'), 0, 32);
|
|
$placeholderDigest28 = hash('sha256', 'empty-vpc-digest-28');
|
|
$seedVpc(28, $vpcKey28, 56, $placeholderDigest28, $operation28, 'vpc_create');
|
|
$seedSubnet(2028, 28, substr(hash('sha256', 'empty-vpc-subnet-28'), 0, 32),
|
|
'10.48.0.0/24', 'applying', 56, $placeholderDigest28);
|
|
$desiredJson28 = $canonicalJsonMethod->invoke(
|
|
null,
|
|
CloudVpcService::desiredDocument(CloudVpc::find(28))
|
|
);
|
|
$digest28 = hash('sha256', $desiredJson28);
|
|
Capsule::table('cloud_vpcs')->where('id', 28)->update([
|
|
'desired_digest' => $digest28,
|
|
]);
|
|
Capsule::table('cloud_vpc_subnets')->where('id', 2028)->update([
|
|
'desired_digest' => $digest28,
|
|
]);
|
|
Capsule::table('cloud_vpc_operations')->where('operation_id', $operation28)->update([
|
|
'payload_json' => $desiredJson28,
|
|
]);
|
|
CloudVpcService::recordObserved($vpcKey28, 56, $digest28, [
|
|
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
|
|
'revision' => 0, 'digest' => null, 'state' => 'degraded',
|
|
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
|
|
'fabric_gate' => [
|
|
'active' => true, 'error_code' => 'vpc_no_participating_hosts',
|
|
],
|
|
'hosts' => [],
|
|
], $operation28);
|
|
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation28)
|
|
->value('status') !== 'succeeded'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 28)
|
|
->value('observed_revision') === 55
|
|
&& Capsule::table('cloud_vpc_subnets')->where('id', 2028)
|
|
->value('state') === 'applying',
|
|
'A contradictory no-host envelope completed an empty VPC reservation.');
|
|
|
|
// The empty-create completion is closed to malformed desired documents.
|
|
$vpcKey22 = substr(hash('sha256', 'empty-vpc-22'), 0, 32);
|
|
$digest22 = hash('sha256', 'empty-vpc-digest-22');
|
|
$operation22 = substr(hash('sha256', 'empty-vpc-operation-22'), 0, 32);
|
|
$seedVpc(22, $vpcKey22, 44, $digest22, $operation22, 'vpc_create');
|
|
$seedSubnet(2022, 22, substr(hash('sha256', 'empty-vpc-subnet-22'), 0, 32),
|
|
'10.42.0.0/24', 'applying', 44, $digest22);
|
|
Capsule::table('cloud_vpcs')->where('id', 22)->update(['fabric_region' => '']);
|
|
CloudVpcService::recordObserved($vpcKey22, 44, $digest22, [
|
|
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
|
|
'revision' => 0, 'digest' => null, 'state' => 'degraded',
|
|
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
|
|
'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_participating_hosts'],
|
|
'hosts' => [],
|
|
], $operation22);
|
|
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation22)
|
|
->value('status') === 'failed'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 22)->value('observed_revision') === 43
|
|
&& Capsule::table('cloud_vpc_subnets')->where('id', 2022)->value('state') === 'applying',
|
|
'A malformed empty desired document was accepted as a completed VPC reservation.');
|
|
$malformedProjection22 = $publicRootMethod->invoke(null, CloudVpc::find(22));
|
|
$assert(($malformedProjection22['setup_state'] ?? null) === null,
|
|
'A malformed desired document was hidden behind the neutral empty-VPC projection.');
|
|
|
|
// An unrelated failure is not the neutral empty-VPC setup state, and even an
|
|
// allowed legacy gate must not override an explicit root deletion lifecycle.
|
|
Capsule::table('cloud_vpcs')->where('id', 22)->update([
|
|
'last_error_code' => 'vpc_observation_incomplete',
|
|
]);
|
|
$realErrorProjection22 = $publicRootMethod->invoke(null, CloudVpc::find(22));
|
|
$assert(($realErrorProjection22['member_count'] ?? null) === 0
|
|
&& ($realErrorProjection22['setup_state'] ?? null) === null
|
|
&& Capsule::table('cloud_vpcs')->where('id', 22)
|
|
->value('last_error_code') === 'vpc_observation_incomplete',
|
|
'A real empty-VPC failure was projected as awaiting members or GET mutated it.');
|
|
Capsule::table('cloud_vpcs')->where('id', 22)->update([
|
|
'state' => 'deleting',
|
|
'last_error_code' => 'vpc_no_members',
|
|
]);
|
|
$deletingProjection22 = $publicRootMethod->invoke(null, CloudVpc::find(22));
|
|
$assert(($deletingProjection22['member_count'] ?? null) === 0
|
|
&& ($deletingProjection22['setup_state'] ?? null) === null
|
|
&& Capsule::table('cloud_vpcs')->where('id', 22)->value('state') === 'deleting',
|
|
'A deleting empty VPC was projected as awaiting members or GET mutated it.');
|
|
|
|
// A contradictory terminal envelope cannot hide a desired member or advance
|
|
// its member/private-IP identity without host evidence.
|
|
$vpcKey23 = substr(hash('sha256', 'empty-vpc-23'), 0, 32);
|
|
$digest23 = hash('sha256', 'empty-vpc-digest-23');
|
|
$operation23 = substr(hash('sha256', 'empty-vpc-operation-23'), 0, 32);
|
|
$seedVpc(23, $vpcKey23, 46, $digest23, $operation23, 'vpc_create');
|
|
$seedSubnet(2023, 23, substr(hash('sha256', 'empty-vpc-subnet-23'), 0, 32),
|
|
'10.43.0.0/24', 'applying', 46, $digest23);
|
|
$seedMember(128, 228, 23, 2023,
|
|
substr(hash('sha256', 'empty-vpc-member-23'), 0, 32), 23001,
|
|
'23232323-2323-4323-8323-232323232323', 230, '10.43.0.10',
|
|
'applying', 46, $digest23);
|
|
CloudVpcService::recordObserved($vpcKey23, 46, $digest23, [
|
|
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
|
|
'revision' => 0, 'digest' => null, 'state' => 'degraded',
|
|
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
|
|
'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_participating_hosts'],
|
|
'hosts' => [],
|
|
], $operation23);
|
|
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation23)
|
|
->value('status') === 'failed'
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 128)->value('state') === 'applying'
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 128)
|
|
->value('observed_revision') === 45
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 228)->value('state') === 'applying'
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 228)
|
|
->value('observed_revision') === 45
|
|
&& ($publicRootMethod->invoke(null, CloudVpc::find(23))['member_count'] ?? null) === 1
|
|
&& ($publicRootMethod->invoke(null, CloudVpc::find(23))['setup_state'] ?? null) === null,
|
|
'A terminal empty-host envelope advanced a desired member/private-IP without host proof.');
|
|
|
|
// Capture the pending-child gate before terminal route/ACL cleanup. A corrupt
|
|
// create ledger with a tombstone must remain failed even if that generic proof
|
|
// closes the individual policy tombstone during the same transaction.
|
|
$vpcKey24 = substr(hash('sha256', 'empty-vpc-24'), 0, 32);
|
|
$digest24 = hash('sha256', 'empty-vpc-digest-24');
|
|
$operation24 = substr(hash('sha256', 'empty-vpc-operation-24'), 0, 32);
|
|
$seedVpc(24, $vpcKey24, 48, $digest24, $operation24, 'vpc_create');
|
|
$seedSubnet(2024, 24, substr(hash('sha256', 'empty-vpc-subnet-24'), 0, 32),
|
|
'10.44.0.0/24', 'applying', 48, $digest24);
|
|
Capsule::table('cloud_vpc_acl_rules')->insert([
|
|
'id' => 240, 'rule_key' => substr(hash('sha256', 'empty-vpc-acl-24'), 0, 32),
|
|
'vpc_id' => 24, 'state' => 'deleting', 'desired_revision' => 48,
|
|
'observed_revision' => 47, 'desired_digest' => $digest24,
|
|
'observed_digest' => null, 'last_error_code' => null, 'updated_at' => $now,
|
|
]);
|
|
$pendingChildProjection24 = $publicRootMethod->invoke(null, CloudVpc::find(24));
|
|
$assert(($pendingChildProjection24['member_count'] ?? null) === 0
|
|
&& ($pendingChildProjection24['setup_state'] ?? null) === null,
|
|
'A pending child tombstone was projected as an empty VPC ready for setup.');
|
|
CloudVpcService::recordObserved($vpcKey24, 48, $digest24, [
|
|
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
|
|
'revision' => 0, 'digest' => null, 'state' => 'degraded',
|
|
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
|
|
'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_participating_hosts'],
|
|
'hosts' => [],
|
|
], $operation24);
|
|
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation24)
|
|
->value('status') === 'failed'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 24)->value('observed_revision') === 47
|
|
&& Capsule::table('cloud_vpc_subnets')->where('id', 2024)->value('state') === 'applying'
|
|
&& Capsule::table('cloud_vpc_acl_rules')->where('id', 240)->value('state') === 'deleted',
|
|
'A create operation with an initial child tombstone was accepted as an empty reservation.');
|
|
|
|
// An orphan lease is not represented in desiredDocument(). It must still
|
|
// close both empty-create completion and the read-only awaiting projection.
|
|
$vpcKey25 = substr(hash('sha256', 'empty-vpc-25'), 0, 32);
|
|
$operation25 = substr(hash('sha256', 'empty-vpc-operation-25'), 0, 32);
|
|
$placeholderDigest25 = hash('sha256', 'empty-vpc-digest-25');
|
|
$seedVpc(25, $vpcKey25, 50, $placeholderDigest25, $operation25, 'vpc_create');
|
|
$seedSubnet(2025, 25, substr(hash('sha256', 'empty-vpc-subnet-25'), 0, 32),
|
|
'10.45.0.0/24', 'applying', 50, $placeholderDigest25);
|
|
$orphanIp = '10.45.0.10';
|
|
$orphanPacked = unpack('Nvalue', inet_pton($orphanIp));
|
|
Capsule::table('cloud_vpc_private_ips')->insert([
|
|
'id' => 250, 'ip_key' => substr(hash('sha256', 'empty-vpc-ip-25'), 0, 32),
|
|
'vpc_id' => 25, 'subnet_id' => 2025, 'member_id' => null,
|
|
'instance_id' => null, 'private_ip' => $orphanIp,
|
|
'private_ip_int' => (int) sprintf('%u', (int) $orphanPacked['value']),
|
|
'state' => 'applying', 'desired_revision' => 50, 'observed_revision' => 49,
|
|
'desired_digest' => $placeholderDigest25, 'observed_digest' => null,
|
|
'last_error_code' => null, 'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
$desiredJson25 = $canonicalJsonMethod->invoke(
|
|
null,
|
|
CloudVpcService::desiredDocument(CloudVpc::find(25))
|
|
);
|
|
$digest25 = hash('sha256', $desiredJson25);
|
|
Capsule::table('cloud_vpcs')->where('id', 25)->update(['desired_digest' => $digest25]);
|
|
Capsule::table('cloud_vpc_subnets')->where('vpc_id', 25)
|
|
->update(['desired_digest' => $digest25]);
|
|
Capsule::table('cloud_vpc_private_ips')->where('vpc_id', 25)
|
|
->update(['desired_digest' => $digest25]);
|
|
Capsule::table('cloud_vpc_operations')->where('operation_id', $operation25)
|
|
->update(['payload_json' => $desiredJson25]);
|
|
CloudVpcService::recordObserved($vpcKey25, 50, $digest25, [
|
|
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
|
|
'revision' => 0, 'digest' => null, 'state' => 'degraded',
|
|
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
|
|
'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_participating_hosts'],
|
|
'hosts' => [],
|
|
], $operation25);
|
|
$projection25 = $publicRootMethod->invoke(null, CloudVpc::find(25));
|
|
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation25)
|
|
->value('status') === 'failed'
|
|
&& Capsule::table('cloud_vpc_subnets')->where('id', 2025)->value('state') === 'applying'
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 250)->value('state') === 'applying'
|
|
&& ($projection25['setup_state'] ?? null) === null,
|
|
'An orphan private-IP lease was hidden by empty-VPC completion or projection.');
|
|
|
|
// An ACL is host configuration. A no-host terminal gate cannot mark it, or
|
|
// the enclosing empty create, as applied.
|
|
$vpcKey26 = substr(hash('sha256', 'empty-vpc-26'), 0, 32);
|
|
$operation26 = substr(hash('sha256', 'empty-vpc-operation-26'), 0, 32);
|
|
$placeholderDigest26 = hash('sha256', 'empty-vpc-digest-26');
|
|
$seedVpc(26, $vpcKey26, 52, $placeholderDigest26, $operation26, 'vpc_create');
|
|
$seedSubnet(2026, 26, substr(hash('sha256', 'empty-vpc-subnet-26'), 0, 32),
|
|
'10.46.0.0/24', 'applying', 52, $placeholderDigest26);
|
|
Capsule::table('cloud_vpc_acl_rules')->insert([
|
|
'id' => 260, 'rule_key' => substr(hash('sha256', 'empty-vpc-acl-26'), 0, 32),
|
|
'vpc_id' => 26, 'priority' => 100, 'direction' => 'ingress',
|
|
'action' => 'allow', 'protocol' => 'tcp', 'source_cidr' => '0.0.0.0/0',
|
|
'destination_cidr' => '10.46.0.0/24', 'port_from' => 22, 'port_to' => 22,
|
|
'state' => 'applying', 'desired_revision' => 52, 'observed_revision' => 51,
|
|
'desired_digest' => $placeholderDigest26, 'observed_digest' => null,
|
|
'last_error_code' => null, 'updated_at' => $now,
|
|
]);
|
|
$desiredJson26 = $canonicalJsonMethod->invoke(
|
|
null,
|
|
CloudVpcService::desiredDocument(CloudVpc::find(26))
|
|
);
|
|
$digest26 = hash('sha256', $desiredJson26);
|
|
Capsule::table('cloud_vpcs')->where('id', 26)->update(['desired_digest' => $digest26]);
|
|
Capsule::table('cloud_vpc_subnets')->where('vpc_id', 26)
|
|
->update(['desired_digest' => $digest26]);
|
|
Capsule::table('cloud_vpc_acl_rules')->where('vpc_id', 26)
|
|
->update(['desired_digest' => $digest26]);
|
|
Capsule::table('cloud_vpc_operations')->where('operation_id', $operation26)
|
|
->update(['payload_json' => $desiredJson26]);
|
|
CloudVpcService::recordObserved($vpcKey26, 52, $digest26, [
|
|
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
|
|
'revision' => 0, 'digest' => null, 'state' => 'degraded',
|
|
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
|
|
'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_members'],
|
|
'hosts' => [],
|
|
], $operation26);
|
|
$projection26 = $publicRootMethod->invoke(null, CloudVpc::find(26));
|
|
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation26)
|
|
->value('status') === 'failed'
|
|
&& Capsule::table('cloud_vpc_subnets')->where('id', 2026)->value('state') === 'applying'
|
|
&& Capsule::table('cloud_vpc_acl_rules')->where('id', 260)->value('state') === 'applying'
|
|
&& ($projection26['setup_state'] ?? null) === null,
|
|
'A no-host receipt falsely applied an ACL-bearing VPC create.');
|
|
|
|
$validatorVpcKey = str_repeat('a', 32);
|
|
$validatorDigest = str_repeat('d', 64);
|
|
$validatorMemberKey = str_repeat('b', 32);
|
|
$validatorMember = [
|
|
'member_id' => $validatorMemberKey,
|
|
'instance_uuid' => 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
|
|
'host_id' => '901', 'private_ip_cidr' => '10.90.0.10/32',
|
|
'private_mac' => $privateMacMethod->invoke(
|
|
null, $validatorVpcKey, $validatorMemberKey
|
|
),
|
|
];
|
|
$validatorHost = $activeHost(
|
|
$validatorVpcKey, '10.90.0.0/24', 901, 2, $validatorDigest,
|
|
$validatorMemberKey,
|
|
$validatorMember['instance_uuid'], '10.90.0.10'
|
|
);
|
|
$validatorArgs = [
|
|
'901', [$validatorMemberKey => $validatorMember], [$validatorMember],
|
|
['10.90.0.10/32'], '10.90.0.0/24', [],
|
|
$validatorVpcKey, 2, $validatorDigest,
|
|
];
|
|
$assert($validateHostMethod->invoke(null, $validatorHost, ...$validatorArgs) === null,
|
|
'An exact guest address/route proof did not open the typed host gate.');
|
|
$validatorRouteMismatch = $validatorHost;
|
|
$validatorRouteMismatch['member_nics'][0]['guest_routes'] = ['10.91.0.0/24'];
|
|
$assert($validateHostMethod->invoke(
|
|
null, $validatorRouteMismatch, ...$validatorArgs
|
|
) === 'vpc_private_guest_route_mismatch',
|
|
'A mismatched guest route opened the typed host gate.');
|
|
$validatorMissingGuest = $validatorHost;
|
|
unset($validatorMissingGuest['member_nics'][0]['domain_running']);
|
|
$assert($validateHostMethod->invoke(
|
|
null, $validatorMissingGuest, ...$validatorArgs
|
|
) === 'vpc_private_guest_agent_unavailable',
|
|
'Missing guest runtime evidence did not fail closed.');
|
|
$validatorDeadline = $validatorHost;
|
|
$validatorDeadline['member_nics'][0]['guest_network_verified'] = false;
|
|
$validatorDeadline['member_nics'][0]['guest_network_readback'] = 'unavailable';
|
|
$validatorDeadline['member_nics'][0]['guest_reason_code'] =
|
|
'vpc_private_guest_probe_deadline_exceeded';
|
|
$assert($validateHostMethod->invoke(
|
|
null, $validatorDeadline, ...$validatorArgs
|
|
) === 'vpc_private_guest_probe_deadline_exceeded',
|
|
'The bounded guest-probe deadline reason was not safely propagated.');
|
|
$validatorReap = $validatorHost;
|
|
$validatorReap['member_nics'][0]['guest_network_verified'] = false;
|
|
$validatorReap['member_nics'][0]['guest_network_readback'] = 'unavailable';
|
|
$validatorReap['member_nics'][0]['guest_reason_code'] =
|
|
'vpc_private_guest_probe_reap_required';
|
|
$assert($validateHostMethod->invoke(
|
|
null, $validatorReap, ...$validatorArgs
|
|
) === 'vpc_private_guest_probe_reap_required',
|
|
'Guest-probe reap-required was not retained as a safe retryable reason.');
|
|
$validatorLinkDown = $validatorHost;
|
|
$validatorLinkDown['member_nics'][0]['link_state'] = 'down';
|
|
$assert($validateHostMethod->invoke(
|
|
null, $validatorLinkDown, ...$validatorArgs
|
|
) === 'vpc_private_guest_link_mismatch',
|
|
'A down guest private link opened the typed host gate.');
|
|
$validatorLinkMissing = $validatorHost;
|
|
unset($validatorLinkMissing['member_nics'][0]['link_state']);
|
|
$assert($validateHostMethod->invoke(
|
|
null, $validatorLinkMissing, ...$validatorArgs
|
|
) === 'vpc_private_guest_link_mismatch',
|
|
'Missing guest private link evidence did not fail closed.');
|
|
$validatorMissingXml = $validatorHost;
|
|
$validatorMissingXml['member_nics'][0]['xml_verified'] = false;
|
|
$configValidatorArgs = [...$validatorArgs, false];
|
|
$assert($validateHostMethod->invoke(
|
|
null, $validatorMissingXml, ...$configValidatorArgs
|
|
) === 'vpc_private_nic_readback_mismatch',
|
|
'Configuration-only convergence accepted a private NIC without exact XML.');
|
|
|
|
// Same-host partial detach: exact current member_nics proves the old private0
|
|
// was consumed before the host became active.
|
|
$vpcKey1 = str_repeat('1', 32);
|
|
$digest1 = str_repeat('a', 64);
|
|
$operation1 = str_repeat('a', 32);
|
|
$seedVpc(1, $vpcKey1, 2, $digest1, $operation1, 'member_detach');
|
|
$seedSubnet(101, 1, str_repeat('2', 32), '10.21.0.0/25', 'active', 2, $digest1);
|
|
$seedMember(101, 201, 1, 101, str_repeat('3', 32), 1001,
|
|
'11111111-1111-4111-8111-111111111111', 1, '10.21.0.10', 'deleting', 2, $digest1);
|
|
$liveKey1 = str_repeat('4', 32);
|
|
$liveUuid1 = '11111111-1111-4111-8111-111111111112';
|
|
$seedMember(102, 202, 1, 101, $liveKey1, 1002, $liveUuid1,
|
|
1, '10.21.0.11', 'active', 2, $digest1);
|
|
$seedHost(1, 1, 'applying', 2, $digest1, $operation1);
|
|
$host1 = $activeHost($vpcKey1, '10.21.0.0/24', 1, 2, $digest1,
|
|
$liveKey1, $liveUuid1, '10.21.0.11');
|
|
CloudVpcService::recordObserved($vpcKey1, 2, $digest1,
|
|
$observed(2, $digest1, ['1' => $host1]), $operation1);
|
|
$assert(Capsule::table('cloud_vpc_members')->where('id', 101)->count() === 0
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 201)->count() === 0
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 102)->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation1)
|
|
->value('status') === 'succeeded',
|
|
'A same-host member/IP tombstone did not converge while its sibling stayed active.');
|
|
// The exact proof must release both database conflict keys.
|
|
Capsule::table('cloud_vpc_members')->insert([
|
|
'member_key' => str_repeat('5', 32), 'vpc_id' => 1, 'subnet_id' => 101,
|
|
'private_ip_id' => null, 'instance_id' => 1001, 'host_id' => 1,
|
|
'private_ip' => '10.21.0.10', 'private_ip_int' => 169148426,
|
|
'state' => 'applying', 'desired_revision' => 3, 'observed_revision' => 0,
|
|
'desired_digest' => null, 'observed_digest' => null,
|
|
'last_operation_id' => null, 'last_error_code' => null,
|
|
'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_vpc_members')->where('member_key', str_repeat('5', 32))->delete();
|
|
Capsule::table('cloud_vpc_private_ips')->insert([
|
|
'ip_key' => str_repeat('6', 32), 'vpc_id' => 1, 'subnet_id' => 101,
|
|
'member_id' => null, 'instance_id' => 1001, 'private_ip' => '10.21.0.10',
|
|
'private_ip_int' => 169148426, 'state' => 'applying',
|
|
'desired_revision' => 3, 'observed_revision' => 0,
|
|
'desired_digest' => null, 'observed_digest' => null,
|
|
'last_error_code' => null, 'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_vpc_private_ips')->where('ip_key', str_repeat('6', 32))->delete();
|
|
|
|
// Host A retires while host B remains live. Both target forms must be exact in
|
|
// the same revision before A/member/address are released.
|
|
$vpcKey2 = str_repeat('2', 32);
|
|
$digest2 = str_repeat('b', 64);
|
|
$operation2 = str_repeat('b', 32);
|
|
$seedVpc(2, $vpcKey2, 4, $digest2, $operation2, 'member_detach');
|
|
$seedSubnet(102, 2, str_repeat('7', 32), '10.22.0.0/25', 'active', 4, $digest2);
|
|
$seedMember(103, 203, 2, 102, str_repeat('8', 32), 2001,
|
|
'22222222-2222-4222-8222-222222222221', 10, '10.22.0.10', 'deleting', 4, $digest2);
|
|
$liveKey2 = str_repeat('9', 32);
|
|
$liveUuid2 = '22222222-2222-4222-8222-222222222222';
|
|
$seedMember(104, 204, 2, 102, $liveKey2, 2002, $liveUuid2,
|
|
20, '10.22.0.11', 'active', 4, $digest2);
|
|
$seedHost(2, 10, 'deleting', 4, $digest2, $operation2);
|
|
$seedHost(2, 20, 'applying', 4, $digest2, $operation2);
|
|
$hosts2 = [
|
|
'10' => $removedHost($vpcKey2, 10, 4, $digest2),
|
|
'20' => $activeHost($vpcKey2, '10.22.0.0/24', 20, 4, $digest2,
|
|
$liveKey2, $liveUuid2, '10.22.0.11'),
|
|
];
|
|
CloudVpcService::recordObserved($vpcKey2, 4, $digest2,
|
|
$observed(4, $digest2, $hosts2), $operation2);
|
|
$assert(Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 2)
|
|
->where('host_id', 10)->value('state') === 'deleted'
|
|
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 2)
|
|
->where('host_id', 20)->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 103)->count() === 0
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 203)->count() === 0
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation2)
|
|
->value('status') === 'succeeded',
|
|
'A retiring host plus a live sibling did not converge as one exact host set.');
|
|
|
|
// An empty subnet is not a libvirt/host object. A live-host exact revision can
|
|
// release it, including its unique CIDR key.
|
|
$vpcKey3 = str_repeat('3', 32);
|
|
$digest3 = str_repeat('c', 64);
|
|
$operation3 = str_repeat('c', 32);
|
|
$seedVpc(3, $vpcKey3, 6, $digest3, $operation3, 'subnet_remove');
|
|
$seedSubnet(103, 3, str_repeat('a', 32), '10.23.0.0/25', 'active', 6, $digest3);
|
|
$seedSubnet(104, 3, str_repeat('b', 32), '10.23.0.128/25', 'deleting', 6, $digest3);
|
|
$liveKey3 = str_repeat('c', 32);
|
|
$liveUuid3 = '33333333-3333-4333-8333-333333333333';
|
|
$seedMember(105, 205, 3, 103, $liveKey3, 3001, $liveUuid3,
|
|
30, '10.23.0.10', 'active', 6, $digest3);
|
|
$seedHost(3, 30, 'applying', 6, $digest3, $operation3);
|
|
$host3 = $activeHost($vpcKey3, '10.23.0.0/24', 30, 6, $digest3,
|
|
$liveKey3, $liveUuid3, '10.23.0.10');
|
|
CloudVpcService::recordObserved($vpcKey3, 6, $digest3,
|
|
$observed(6, $digest3, ['30' => $host3]), $operation3);
|
|
$assert(Capsule::table('cloud_vpc_subnets')->where('id', 104)->count() === 0
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation3)
|
|
->value('status') === 'succeeded',
|
|
'An unreferenced subnet did not converge behind an exact live-host gate.');
|
|
$seedSubnet(1004, 3, str_repeat('d', 32), '10.23.0.128/25', 'applying', 7, $digest3);
|
|
$assert(Capsule::table('cloud_vpc_subnets')->where('id', 1004)->count() === 1,
|
|
'A converged subnet still occupied its unique CIDR key.');
|
|
|
|
// A VPC with no participating host can still complete a subnet-only removal;
|
|
// this proof is deliberately not usable for a member/private-IP tombstone.
|
|
$vpcKey4 = str_repeat('4', 32);
|
|
$digest4 = str_repeat('d', 64);
|
|
$operation4 = str_repeat('d', 32);
|
|
$seedVpc(4, $vpcKey4, 8, $digest4, $operation4, 'subnet_remove');
|
|
$seedSubnet(105, 4, str_repeat('e', 32), '10.24.0.0/25', 'deleting', 8, $digest4);
|
|
CloudVpcService::recordObserved($vpcKey4, 8, $digest4, [
|
|
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
|
|
'revision' => 0, 'digest' => null, 'state' => 'degraded',
|
|
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
|
|
'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_members'],
|
|
'hosts' => [],
|
|
], $operation4);
|
|
$root4 = CloudVpc::where('id', 4)->first();
|
|
$assert($root4 !== null, 'The no-host subnet operation lost its VPC root.');
|
|
$rootProjection4 = $publicRootMethod->invoke(null, $root4);
|
|
$assert(Capsule::table('cloud_vpc_subnets')->where('id', 105)->count() === 0
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation4)
|
|
->value('status') === 'succeeded'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 4)->value('state') !== 'active'
|
|
&& ($rootProjection4['usable'] ?? true) === false,
|
|
'A no-host subnet-only removal either failed or made an empty VPC active/usable.');
|
|
|
|
// Missing A removal evidence must not partially delete A or its member. This
|
|
// also keeps the same operation payload retryable instead of turning A into an
|
|
// unknown extra host on the next attempt.
|
|
$vpcKey5 = str_repeat('5', 32);
|
|
$digest5 = str_repeat('e', 64);
|
|
$operation5 = str_repeat('e', 32);
|
|
$seedVpc(5, $vpcKey5, 10, $digest5, $operation5, 'member_detach');
|
|
$seedSubnet(106, 5, str_repeat('f', 32), '10.25.0.0/25', 'active', 10, $digest5);
|
|
$seedMember(106, 206, 5, 106, str_repeat('0', 32), 5001,
|
|
'55555555-5555-4555-8555-555555555551', 50, '10.25.0.10', 'deleting', 10, $digest5);
|
|
$liveKey5 = str_repeat('1', 31) . '0';
|
|
$liveUuid5 = '55555555-5555-4555-8555-555555555552';
|
|
$seedMember(107, 207, 5, 106, $liveKey5, 5002, $liveUuid5,
|
|
60, '10.25.0.11', 'active', 10, $digest5);
|
|
$seedHost(5, 50, 'deleting', 10, $digest5, $operation5);
|
|
$seedHost(5, 60, 'applying', 10, $digest5, $operation5);
|
|
$host5 = $activeHost($vpcKey5, '10.25.0.0/24', 60, 10, $digest5,
|
|
$liveKey5, $liveUuid5, '10.25.0.11');
|
|
CloudVpcService::recordObserved($vpcKey5, 10, $digest5,
|
|
$observed(10, $digest5, ['60' => $host5]), $operation5);
|
|
$assert(Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 5)
|
|
->where('host_id', 50)->value('state') === 'deleting'
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 106)->value('state') === 'deleting'
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 206)->value('state') === 'deleting'
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation5)
|
|
->value('status') === 'uncertain',
|
|
'A partial multi-host observation released tombstones without A removal proof.');
|
|
|
|
// Removing the last member has one required deployment: an exact typed host
|
|
// removal receipt is sufficient even though no live host remains afterward.
|
|
$vpcKey6 = str_repeat('6', 32);
|
|
$digest6 = str_repeat('f', 64);
|
|
$operation6 = str_repeat('6', 32);
|
|
$seedVpc(6, $vpcKey6, 12, $digest6, $operation6, 'member_detach');
|
|
$seedSubnet(107, 6, str_repeat('2', 32), '10.26.0.0/25', 'active', 12, $digest6);
|
|
$seedMember(108, 208, 6, 107, str_repeat('3', 32), 6001,
|
|
'66666666-6666-4666-8666-666666666661', 70, '10.26.0.10',
|
|
'deleting', 12, $digest6);
|
|
$seedHost(6, 70, 'deleting', 12, $digest6, $operation6);
|
|
CloudVpcService::recordObserved($vpcKey6, 12, $digest6,
|
|
$observed(12, $digest6, [
|
|
'70' => $removedHost($vpcKey6, 70, 12, $digest6),
|
|
]), $operation6);
|
|
$projection6 = $publicRootMethod->invoke(null, CloudVpc::find(6));
|
|
$assert(Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 6)
|
|
->where('host_id', 70)->value('state') === 'deleted'
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 108)->count() === 0
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 208)->count() === 0
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation6)
|
|
->value('status') === 'succeeded'
|
|
&& ($projection6['member_count'] ?? null) === 0
|
|
&& ($projection6['setup_state'] ?? null) === 'awaiting_members',
|
|
'The last member did not converge behind its only host removal proof.');
|
|
|
|
// A terminal Fabric gate is not libvirt removal evidence. Even with no host
|
|
// deployment row, it must never consume a member/address tombstone.
|
|
$vpcKey7 = str_repeat('7', 32);
|
|
$digest7 = str_repeat('0', 64);
|
|
$operation7 = str_repeat('7', 32);
|
|
$seedVpc(7, $vpcKey7, 14, $digest7, $operation7, 'member_detach');
|
|
$seedSubnet(108, 7, str_repeat('4', 32), '10.27.0.0/25', 'active', 14, $digest7);
|
|
$seedMember(109, 209, 7, 108, str_repeat('5', 32), 7001,
|
|
'77777777-7777-4777-8777-777777777771', 71, '10.27.0.10',
|
|
'deleting', 14, $digest7);
|
|
try {
|
|
CloudVpcService::recordObserved($vpcKey7, 14, $digest7, [
|
|
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
|
|
'revision' => 0, 'digest' => null, 'state' => 'degraded',
|
|
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
|
|
'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_members'],
|
|
'hosts' => [],
|
|
], $operation7);
|
|
} catch (CloudError $error) {
|
|
$assert($error->httpStatus === 409,
|
|
'A terminal member proof failed with an unexpected error.');
|
|
}
|
|
$assert(Capsule::table('cloud_vpc_members')->where('id', 109)
|
|
->value('state') === 'deleting'
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 209)
|
|
->value('state') === 'deleting'
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation7)
|
|
->value('status') !== 'succeeded',
|
|
'A terminal no-host proof released a member/private-IP tombstone.');
|
|
|
|
// An exact live sibling cannot stand in for the missing deployment that owns
|
|
// a retiring member. The ledger may reject the observation or stay uncertain,
|
|
// but it must retain both sides of the member/address identity.
|
|
$vpcKey8 = str_repeat('8', 32);
|
|
$digest8 = str_repeat('1', 64);
|
|
$operation8 = str_repeat('8', 32);
|
|
$seedVpc(8, $vpcKey8, 16, $digest8, $operation8, 'member_detach');
|
|
$seedSubnet(109, 8, str_repeat('6', 32), '10.28.0.0/25', 'active', 16, $digest8);
|
|
$seedMember(110, 210, 8, 109, str_repeat('7', 32), 8001,
|
|
'88888888-8888-4888-8888-888888888881', 80, '10.28.0.10',
|
|
'deleting', 16, $digest8);
|
|
$liveKey8 = str_repeat('8', 31) . '0';
|
|
$liveUuid8 = '88888888-8888-4888-8888-888888888882';
|
|
$seedMember(111, 211, 8, 109, $liveKey8, 8002, $liveUuid8,
|
|
81, '10.28.0.11', 'active', 16, $digest8);
|
|
$seedHost(8, 81, 'applying', 16, $digest8, $operation8);
|
|
$host8 = $activeHost($vpcKey8, '10.28.0.0/24', 81, 16, $digest8,
|
|
$liveKey8, $liveUuid8, '10.28.0.11');
|
|
try {
|
|
CloudVpcService::recordObserved($vpcKey8, 16, $digest8,
|
|
$observed(16, $digest8, ['81' => $host8]), $operation8);
|
|
} catch (CloudError $error) {
|
|
$assert($error->httpStatus === 409,
|
|
'A missing owner deployment failed with an unexpected error.');
|
|
}
|
|
$assert(Capsule::table('cloud_vpc_members')->where('id', 110)
|
|
->value('state') === 'deleting'
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 210)
|
|
->value('state') === 'deleting'
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation8)
|
|
->value('status') !== 'succeeded',
|
|
'A live sibling released member/IP identities whose owner deployment was missing.');
|
|
|
|
// Exact proofs are monotonic across retries. The first attempt proves A was
|
|
// removed but lacks B; the retry supplies only B and must reuse A's persisted
|
|
// proof instead of requiring the already-removed host to report again.
|
|
$vpcKey9 = str_repeat('9', 32);
|
|
$digest9 = str_repeat('2', 64);
|
|
$operation9 = str_repeat('9', 32);
|
|
$seedVpc(9, $vpcKey9, 18, $digest9, $operation9, 'member_detach');
|
|
$seedSubnet(110, 9, str_repeat('9', 31) . '0', '10.29.0.0/25',
|
|
'active', 18, $digest9);
|
|
$seedMember(112, 212, 9, 110, str_repeat('a', 31) . '0', 9001,
|
|
'99999999-9999-4999-8999-999999999991', 90, '10.29.0.10',
|
|
'deleting', 18, $digest9);
|
|
$liveKey9 = str_repeat('b', 31) . '0';
|
|
$liveUuid9 = '99999999-9999-4999-8999-999999999992';
|
|
$seedMember(113, 213, 9, 110, $liveKey9, 9002, $liveUuid9,
|
|
91, '10.29.0.11', 'active', 18, $digest9);
|
|
$seedHost(9, 90, 'deleting', 18, $digest9, $operation9);
|
|
$seedHost(9, 91, 'applying', 18, $digest9, $operation9);
|
|
CloudVpcService::recordObserved($vpcKey9, 18, $digest9,
|
|
$observed(18, $digest9, [
|
|
'90' => $removedHost($vpcKey9, 90, 18, $digest9),
|
|
]), $operation9);
|
|
$assert(Capsule::table('cloud_vpc_members')->where('id', 112)
|
|
->value('state') === 'deleting'
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 212)
|
|
->value('state') === 'deleting'
|
|
&& (int) Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 9)
|
|
->where('host_id', 90)->value('observed_revision') === 18
|
|
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 9)
|
|
->where('host_id', 90)->value('observed_digest') === $digest9
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation9)
|
|
->value('status') !== 'succeeded',
|
|
'The incomplete first retry lost A proof or released its identities early.');
|
|
$host9 = $activeHost($vpcKey9, '10.29.0.0/24', 91, 18, $digest9,
|
|
$liveKey9, $liveUuid9, '10.29.0.11');
|
|
CloudVpcService::recordObserved($vpcKey9, 18, $digest9,
|
|
$observed(18, $digest9, ['91' => $host9]), $operation9);
|
|
$assert(Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 9)
|
|
->where('host_id', 90)->value('state') === 'deleted'
|
|
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 9)
|
|
->where('host_id', 91)->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 112)->count() === 0
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 212)->count() === 0
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation9)
|
|
->value('status') === 'succeeded',
|
|
'A persisted A-removal proof did not converge after B supplied the retry evidence.');
|
|
|
|
// If every owner proof was persisted by earlier partial attempts, a later
|
|
// no-host response may finish the member operation without pretending the
|
|
// terminal Fabric gate itself removed a NIC.
|
|
$vpcKey10 = str_repeat('a', 32);
|
|
$digest10 = str_repeat('3', 64);
|
|
$operation10 = str_repeat('a', 31) . '0';
|
|
$seedVpc(10, $vpcKey10, 20, $digest10, $operation10, 'member_detach');
|
|
$seedSubnet(111, 10, str_repeat('c', 31) . '0', '10.30.0.0/25',
|
|
'active', 20, $digest10);
|
|
$seedMember(114, 214, 10, 111, str_repeat('d', 31) . '0', 10001,
|
|
'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1', 100, '10.30.0.10',
|
|
'deleting', 20, $digest10);
|
|
$seedHost(10, 100, 'deleted', 20, $digest10, $operation10);
|
|
Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 10)
|
|
->where('host_id', 100)->update([
|
|
'observed_revision' => 20, 'observed_digest' => $digest10,
|
|
'last_error_code' => null, 'last_success_at' => $now,
|
|
]);
|
|
CloudVpcService::recordObserved($vpcKey10, 20, $digest10, [
|
|
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
|
|
'revision' => 0, 'digest' => null, 'state' => 'degraded',
|
|
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
|
|
'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_members'],
|
|
'hosts' => [],
|
|
], $operation10);
|
|
$assert(Capsule::table('cloud_vpc_members')->where('id', 114)->count() === 0
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 214)->count() === 0
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation10)
|
|
->value('status') === 'succeeded'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 10)->value('state') !== 'active',
|
|
'Stored owner removal proofs did not finish the no-host member retry safely.');
|
|
|
|
// Old workers could leave an already-proven member/lease pair in `deleted`.
|
|
// A subnet-only terminal operation may garbage-collect that pair only when its
|
|
// owner deployment still carries an exact removal proof, then release the CIDR.
|
|
$vpcKey11 = str_repeat('b', 32);
|
|
$digest11 = str_repeat('4', 64);
|
|
$operation11 = str_repeat('b', 31) . '0';
|
|
$seedVpc(11, $vpcKey11, 22, $digest11, $operation11, 'subnet_remove');
|
|
$seedSubnet(112, 11, str_repeat('e', 31) . '0', '10.31.0.0/25',
|
|
'deleting', 22, $digest11);
|
|
$seedMember(115, 215, 11, 112, str_repeat('f', 31) . '0', 11001,
|
|
'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1', 110, '10.31.0.10',
|
|
'deleted', 22, $digest11);
|
|
foreach (['cloud_vpc_members' => 115, 'cloud_vpc_private_ips' => 215] as $table => $id) {
|
|
Capsule::table($table)->where('id', $id)->update([
|
|
'observed_revision' => 22, 'observed_digest' => $digest11,
|
|
'last_error_code' => null,
|
|
]);
|
|
}
|
|
$seedHost(11, 110, 'deleted', 22, $digest11, $operation11);
|
|
Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 11)
|
|
->where('host_id', 110)->update([
|
|
'observed_revision' => 22, 'observed_digest' => $digest11,
|
|
'last_error_code' => null, 'last_success_at' => $now,
|
|
]);
|
|
CloudVpcService::recordObserved($vpcKey11, 22, $digest11, [
|
|
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
|
|
'revision' => 0, 'digest' => null, 'state' => 'degraded',
|
|
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
|
|
'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_members'],
|
|
'hosts' => [],
|
|
], $operation11);
|
|
$assert(Capsule::table('cloud_vpc_members')->where('id', 115)->count() === 0
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 215)->count() === 0
|
|
&& Capsule::table('cloud_vpc_subnets')->where('id', 112)->count() === 0
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation11)
|
|
->value('status') === 'succeeded',
|
|
'A proven historical member/IP pair kept an empty subnet CIDR occupied.');
|
|
|
|
// A no-members gate must not make a temporarily unreachable retiring host
|
|
// terminal. Keep the same operation retryable until its exact libvirt/data-
|
|
// plane removal receipt arrives, then release both conflict keys.
|
|
$vpcKey12 = str_repeat('c', 32);
|
|
$digest12 = str_repeat('5', 64);
|
|
$operation12 = str_repeat('c', 31) . '0';
|
|
$seedVpc(12, $vpcKey12, 24, $digest12, $operation12, 'member_detach');
|
|
$seedSubnet(113, 12, str_repeat('1', 31) . '2', '10.32.0.0/25',
|
|
'active', 24, $digest12);
|
|
$seedMember(116, 216, 12, 113, str_repeat('2', 31) . '1', 12001,
|
|
'cccccccc-cccc-4ccc-8ccc-ccccccccccc1', 120, '10.32.0.10',
|
|
'deleting', 24, $digest12);
|
|
$seedHost(12, 120, 'deleting', 24, $digest12, $operation12);
|
|
CloudVpcService::recordObserved($vpcKey12, 24, $digest12, [
|
|
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
|
|
'revision' => 0, 'digest' => null, 'state' => 'degraded',
|
|
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
|
|
'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_members'],
|
|
'hosts' => [],
|
|
], $operation12);
|
|
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation12)
|
|
->value('status') === 'uncertain'
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation12)
|
|
->value('finished_at') === null
|
|
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 12)
|
|
->where('host_id', 120)->value('state') === 'deleting'
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 116)
|
|
->value('state') === 'deleting'
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 216)
|
|
->value('state') === 'deleting',
|
|
'A transient no-members response froze an unresolved host/member tombstone.');
|
|
CloudVpcService::recordObserved($vpcKey12, 24, $digest12, [
|
|
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
|
|
'revision' => 0, 'digest' => null, 'state' => 'degraded',
|
|
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
|
|
'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_members'],
|
|
'hosts' => [
|
|
'120' => $removedHost($vpcKey12, 120, 24, $digest12),
|
|
],
|
|
], $operation12);
|
|
$assert(Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 12)
|
|
->where('host_id', 120)->value('state') === 'deleted'
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 116)->count() === 0
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 216)->count() === 0
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation12)
|
|
->value('status') === 'succeeded',
|
|
'An exact retry did not converge the previously unreachable retiring host.');
|
|
|
|
// A current owner proof is not enough when another historical deployment has
|
|
// lost its exact absence receipt: that host could still retain a route, proxy,
|
|
// WireGuard peer or nft rule for the retiring address. Refresh that historical
|
|
// proof explicitly before releasing the member/IP identities.
|
|
$vpcKey13 = str_repeat('d', 32);
|
|
$digest13 = str_repeat('6', 64);
|
|
$operation13 = str_repeat('d', 31) . '0';
|
|
$seedVpc(13, $vpcKey13, 26, $digest13, $operation13, 'member_detach');
|
|
$seedSubnet(114, 13, str_repeat('3', 31) . '1', '10.33.0.0/25',
|
|
'active', 26, $digest13);
|
|
$seedMember(117, 217, 13, 114, str_repeat('4', 31) . '1', 13001,
|
|
'dddddddd-dddd-4ddd-8ddd-ddddddddddd1', 130, '10.33.0.10',
|
|
'deleting', 26, $digest13);
|
|
$liveKey13 = str_repeat('5', 31) . '1';
|
|
$liveUuid13 = 'dddddddd-dddd-4ddd-8ddd-ddddddddddd2';
|
|
$seedMember(118, 218, 13, 114, $liveKey13, 13002, $liveUuid13,
|
|
130, '10.33.0.11', 'active', 26, $digest13);
|
|
$seedHost(13, 130, 'applying', 26, $digest13, $operation13);
|
|
$seedHost(13, 131, 'deleted', 26, $digest13, $operation13);
|
|
$host13 = $activeHost($vpcKey13, '10.33.0.0/24', 130, 26, $digest13,
|
|
$liveKey13, $liveUuid13, '10.33.0.11');
|
|
CloudVpcService::recordObserved($vpcKey13, 26, $digest13,
|
|
$observed(26, $digest13, ['130' => $host13]), $operation13);
|
|
$assert(Capsule::table('cloud_vpc_members')->where('id', 117)
|
|
->value('state') === 'deleting'
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 217)
|
|
->value('state') === 'deleting'
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation13)
|
|
->value('status') === 'uncertain',
|
|
'A stale unrelated host proof did not block global member/IP release.');
|
|
CloudVpcService::recordObserved($vpcKey13, 26, $digest13,
|
|
$observed(26, $digest13, [
|
|
'130' => $host13,
|
|
'131' => $removedHost($vpcKey13, 131, 26, $digest13),
|
|
]), $operation13);
|
|
$assert(Capsule::table('cloud_vpc_members')->where('id', 117)->count() === 0
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 217)->count() === 0
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 118)
|
|
->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 13)
|
|
->where('host_id', 131)->value('observed_digest') === $digest13
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation13)
|
|
->value('status') === 'succeeded',
|
|
'Refreshing the stale historical removal proof did not unblock convergence.');
|
|
|
|
// A running host/libvirt configuration is applied independently from guest
|
|
// health. Preserve one actionable restart reason on the member/lease while the
|
|
// host, root and operation remain terminal for the exact configuration.
|
|
$vpcKey14 = str_repeat('e', 32);
|
|
$digest14 = str_repeat('7', 64);
|
|
$operation14 = str_repeat('e', 31) . '0';
|
|
$memberKey14 = str_repeat('6', 31) . '1';
|
|
$uuid14 = 'eeeeeeee-eeee-4eee-8eee-eeeeeeeeeee1';
|
|
$seedVpc(14, $vpcKey14, 28, $digest14, $operation14, 'member_attach');
|
|
$seedSubnet(115, 14, str_repeat('7', 31) . '1', '10.34.0.0/25',
|
|
'active', 28, $digest14);
|
|
$seedMember(119, 219, 14, 115, $memberKey14, 14001, $uuid14,
|
|
140, '10.34.0.10', 'applying', 28, $digest14);
|
|
$seedHost(14, 140, 'applying', 28, $digest14, $operation14);
|
|
$host14 = $activeHost(
|
|
$vpcKey14, '10.34.0.0/24', 140, 28, $digest14,
|
|
$memberKey14, $uuid14, '10.34.0.10'
|
|
);
|
|
$host14['state'] = 'applying';
|
|
$host14['private_dataplane_verified'] = false;
|
|
$host14['last_error_code'] = 'vpc_private_guest_restart_required';
|
|
$host14['member_nics'][0]['restart_required'] = true;
|
|
$host14['member_nics'][0]['guest_network_verified'] = false;
|
|
$host14['member_nics'][0]['guest_network_readback'] = 'pending_restart';
|
|
$host14['member_nics'][0]['guest_reason_code'] =
|
|
'vpc_private_guest_restart_required';
|
|
$deferred14 = $observed(28, $digest14, ['140' => $host14]);
|
|
$deferred14['state'] = 'applying';
|
|
$deferred14['next_stage_allowed'] = false;
|
|
$deferred14['private_dataplane_verified'] = false;
|
|
CloudVpcService::recordObserved(
|
|
$vpcKey14, 28, $digest14, $deferred14, $operation14
|
|
);
|
|
$rootProjection14 = $publicRootMethod->invoke(null, CloudVpc::find(14));
|
|
$assert(Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 14)
|
|
->where('host_id', 140)->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 14)
|
|
->where('host_id', 140)->value('last_error_code') === null
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation14)
|
|
->value('status') === 'succeeded'
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation14)
|
|
->value('error_code') === null
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 119)
|
|
->value('state') === 'degraded'
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 119)
|
|
->value('last_error_code') === 'vpc_private_guest_restart_required'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 14)
|
|
->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 14)
|
|
->value('sync_state') === 'applied'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 14)
|
|
->value('last_error_code') === null
|
|
&& ($rootProjection14['configured'] ?? false) === true
|
|
&& ($rootProjection14['usable'] ?? true) === false,
|
|
'A config-complete guest restart was not retained as non-usable runtime state.');
|
|
|
|
// A guest execution policy denial is actionable deferred evidence, not a
|
|
// usable receipt and not a loss of the already-applied host configuration.
|
|
$host14['last_error_code'] = 'vpc_private_guest_probe_permission_denied';
|
|
$host14['member_nics'][0]['restart_required'] = false;
|
|
$host14['member_nics'][0]['guest_network_readback'] = 'unavailable';
|
|
$host14['member_nics'][0]['guest_reason_code'] = 'vpc_private_guest_probe_permission_denied';
|
|
$deferred14['hosts']['140'] = $host14;
|
|
CloudVpcService::recordObserved($vpcKey14, 28, $digest14, $deferred14, $operation14);
|
|
$deniedProjection14 = $publicRootMethod->invoke(null, CloudVpc::find(14));
|
|
$assert(Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 14)
|
|
->where('host_id', 140)->value('last_error_code') === null
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 119)
|
|
->value('last_error_code') === 'vpc_private_guest_probe_permission_denied'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 14)
|
|
->value('last_error_code') === null
|
|
&& ($deniedProjection14['usable'] ?? true) === false,
|
|
'A guest permission denial was lost or marked usable.');
|
|
|
|
// Mixed guest reasons remain isolated on their member rows. They do not leak
|
|
// into either exact host projection or the applied root configuration.
|
|
$vpcKey15 = str_repeat('f', 32);
|
|
$digest15 = str_repeat('8', 64);
|
|
$operation15 = str_repeat('f', 31) . '0';
|
|
$memberKey15a = str_repeat('8', 31) . '1';
|
|
$memberKey15b = str_repeat('9', 31) . '1';
|
|
$uuid15a = 'ffffffff-ffff-4fff-8fff-fffffffffff1';
|
|
$uuid15b = 'ffffffff-ffff-4fff-8fff-fffffffffff2';
|
|
$seedVpc(15, $vpcKey15, 30, $digest15, $operation15, 'member_attach');
|
|
$seedSubnet(116, 15, str_repeat('a', 31) . '1', '10.35.0.0/25',
|
|
'active', 30, $digest15);
|
|
$seedMember(120, 220, 15, 116, $memberKey15a, 15001, $uuid15a,
|
|
150, '10.35.0.10', 'applying', 30, $digest15);
|
|
$seedMember(121, 221, 15, 116, $memberKey15b, 15002, $uuid15b,
|
|
151, '10.35.0.11', 'applying', 30, $digest15);
|
|
$seedHost(15, 150, 'applying', 30, $digest15, $operation15);
|
|
$seedHost(15, 151, 'applying', 30, $digest15, $operation15);
|
|
$host15a = $activeHost(
|
|
$vpcKey15, '10.35.0.0/24', 150, 30, $digest15,
|
|
$memberKey15a, $uuid15a, '10.35.0.10'
|
|
);
|
|
$host15b = $activeHost(
|
|
$vpcKey15, '10.35.0.0/24', 151, 30, $digest15,
|
|
$memberKey15b, $uuid15b, '10.35.0.11'
|
|
);
|
|
$routeTable15 = $routeTableMethod->invoke(null, $vpcKey15);
|
|
$peerKey15 = base64_encode(str_repeat("\x15", 32));
|
|
$host15a['proxy_ips'] = ['10.35.0.10', '10.35.0.11'];
|
|
$host15b['proxy_ips'] = ['10.35.0.10', '10.35.0.11'];
|
|
$host15a['member_nics'][0]['guest_routes'] = ['10.35.0.0/24'];
|
|
$host15b['member_nics'][0]['guest_routes'] = ['10.35.0.0/24'];
|
|
$host15a['routes'][] = [
|
|
'destination_cidr' => '10.35.0.11/32', 'device' => 'wg-fabric0',
|
|
'table' => $routeTable15, 'owner' => 'virt-cloud-vpc-dataplane-v1',
|
|
'via' => null, 'metric' => 100,
|
|
];
|
|
$host15b['routes'][] = [
|
|
'destination_cidr' => '10.35.0.10/32', 'device' => 'wg-fabric0',
|
|
'table' => $routeTable15, 'owner' => 'virt-cloud-vpc-dataplane-v1',
|
|
'via' => null, 'metric' => 100,
|
|
];
|
|
$host15a['wg_allowed_ips'] = [[
|
|
'peer_host_id' => '151', 'public_key' => $peerKey15,
|
|
'tenant_allowed_ips' => ['10.35.0.11/32'],
|
|
]];
|
|
$host15b['wg_allowed_ips'] = [[
|
|
'peer_host_id' => '150', 'public_key' => $peerKey15,
|
|
'tenant_allowed_ips' => ['10.35.0.10/32'],
|
|
]];
|
|
$host15a['state'] = $host15b['state'] = 'applying';
|
|
$host15a['private_dataplane_verified'] = false;
|
|
$host15b['private_dataplane_verified'] = false;
|
|
$host15a['member_nics'][0]['guest_network_verified'] = false;
|
|
$host15b['member_nics'][0]['guest_network_verified'] = false;
|
|
$host15a['member_nics'][0]['restart_required'] = true;
|
|
$host15a['member_nics'][0]['guest_reason_code'] =
|
|
'vpc_private_guest_restart_required';
|
|
$host15b['member_nics'][0]['guest_reason_code'] =
|
|
'vpc_private_guest_agent_unavailable';
|
|
$host15a['last_error_code'] = 'vpc_private_guest_restart_required';
|
|
$host15b['last_error_code'] = 'vpc_private_guest_agent_unavailable';
|
|
$deferred15 = $observed(30, $digest15, [
|
|
'150' => $host15a, '151' => $host15b,
|
|
]);
|
|
$deferred15['state'] = 'applying';
|
|
$deferred15['next_stage_allowed'] = false;
|
|
$deferred15['private_dataplane_verified'] = false;
|
|
Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 15)
|
|
->where('host_id', 151)->delete();
|
|
$missingDeployment15 = $deferred15;
|
|
$missingDeployment15['hosts'] = ['150' => $host15a];
|
|
CloudVpcService::recordObserved(
|
|
$vpcKey15, 30, $digest15, $missingDeployment15, $operation15
|
|
);
|
|
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation15)
|
|
->value('status') !== 'succeeded'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 15)->value('state') !== 'active',
|
|
'A partial deployment/observed host subset completed the configuration.');
|
|
$seedHost(15, 151, 'applying', 30, $digest15, $operation15);
|
|
CloudVpcService::recordObserved(
|
|
$vpcKey15, 30, $digest15, $deferred15, $operation15
|
|
);
|
|
$assert(Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 15)
|
|
->where('host_id', 150)->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 15)
|
|
->where('host_id', 151)->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 15)
|
|
->whereIn('host_id', [150, 151])->whereNotNull('last_error_code')->count() === 0
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 120)
|
|
->value('last_error_code') === 'vpc_private_guest_restart_required'
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 121)
|
|
->value('last_error_code') === 'vpc_private_guest_agent_unavailable'
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation15)
|
|
->value('status') === 'succeeded'
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation15)
|
|
->value('error_code') === null
|
|
&& Capsule::table('cloud_vpcs')->where('id', 15)
|
|
->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 15)
|
|
->value('sync_state') === 'applied'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 15)
|
|
->value('last_error_code') === null,
|
|
'Multi-host guest errors leaked out of member-scoped health.');
|
|
$configuredProjection15 = $publicRootMethod->invoke(null, CloudVpc::find(15));
|
|
Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 15)
|
|
->where('host_id', 151)->delete();
|
|
$missingProjection15 = $publicRootMethod->invoke(null, CloudVpc::find(15));
|
|
$assert(($configuredProjection15['configured'] ?? false) === true
|
|
&& ($missingProjection15['configured'] ?? true) === false,
|
|
'The public configured projection ignored a live member host missing from deployment rows.');
|
|
|
|
// A pre-upgrade succeeded receipt has no guest-evidence contract version. It
|
|
// must fail closed immediately; the worker contract separately reclaims only
|
|
// this current last_operation_id for fresh observation.
|
|
$vpcKey16 = str_repeat('0', 32);
|
|
$digest16 = str_repeat('9', 64);
|
|
$operation16 = str_repeat('0', 31) . '1';
|
|
$memberKey16 = str_repeat('b', 31) . '1';
|
|
$uuid16 = '00000000-0000-4000-8000-000000000001';
|
|
$seedVpc(16, $vpcKey16, 32, $digest16, $operation16, 'member_attach');
|
|
$seedSubnet(117, 16, str_repeat('c', 31) . '1', '10.36.0.0/25',
|
|
'active', 32, $digest16);
|
|
$seedMember(122, 222, 16, 117, $memberKey16, 16001, $uuid16,
|
|
160, '10.36.0.10', 'active', 32, $digest16);
|
|
$seedHost(16, 160, 'active', 32, $digest16, $operation16);
|
|
Capsule::table('cloud_vpc_operations')->where('operation_id', $operation16)->update([
|
|
'status' => 'succeeded', 'evidence_version' => 0, 'finished_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_vpcs')->where('id', 16)->update([
|
|
'state' => 'active', 'sync_state' => 'applied', 'observed_revision' => 32,
|
|
'observed_digest' => $digest16, 'last_success_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 16)
|
|
->where('host_id', 160)->update([
|
|
'state' => 'active', 'observed_revision' => 32,
|
|
'observed_digest' => $digest16, 'last_success_at' => $now,
|
|
]);
|
|
$rootProjection16 = $publicRootMethod->invoke(null, CloudVpc::find(16));
|
|
$assert(($rootProjection16['usable'] ?? true) === false,
|
|
'A legacy succeeded observation without guest evidence remained usable.');
|
|
|
|
// Retiring member A on a host with live sibling B must converge from exact
|
|
// current configuration even when B's guest probe is unsupported. The proof
|
|
// releases only A/IP; B remains fail-closed for runtime usability while the
|
|
// host and root retain their exact applied configuration projection.
|
|
$vpcKey17 = str_repeat('1', 31) . '7';
|
|
$digest17 = str_repeat('a', 63) . '7';
|
|
$operation17 = str_repeat('2', 31) . '7';
|
|
$retiringKey17 = str_repeat('3', 31) . '7';
|
|
$liveKey17 = str_repeat('4', 31) . '7';
|
|
$liveUuid17 = '17171717-1717-4717-8717-171717171717';
|
|
$seedVpc(17, $vpcKey17, 34, $digest17, $operation17, 'member_detach');
|
|
$seedSubnet(118, 17, str_repeat('5', 31) . '7', '10.37.0.0/25',
|
|
'active', 34, $digest17);
|
|
$seedMember(123, 223, 17, 118, $retiringKey17, 17001,
|
|
'17171717-1717-4717-8717-171717171701', 170, '10.37.0.10',
|
|
'deleting', 34, $digest17);
|
|
$seedMember(124, 224, 17, 118, $liveKey17, 17002, $liveUuid17,
|
|
170, '10.37.0.11', 'applying', 34, $digest17);
|
|
$seedHost(17, 170, 'applying', 34, $digest17, $operation17);
|
|
$detachRuntimePayload17 = json_encode([
|
|
'schema_version' => 1, 'vpc_id' => $vpcKey17,
|
|
'desired_revision' => 34, 'desired_digest' => $digest17,
|
|
'operation_id' => $operation17,
|
|
'members' => [[
|
|
'member_id' => $liveKey17, 'instance_uuid' => $liveUuid17,
|
|
'host_id' => '170', 'private_ip_cidr' => '10.37.0.11/32',
|
|
]],
|
|
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES);
|
|
Capsule::table('cloud_vpc_operations')->where('operation_id', $operation17)
|
|
->update(['payload_json' => $detachRuntimePayload17]);
|
|
$host17 = $activeHost(
|
|
$vpcKey17, '10.37.0.0/24', 170, 34, $digest17,
|
|
$liveKey17, $liveUuid17, '10.37.0.11'
|
|
);
|
|
$host17['state'] = 'applying';
|
|
$host17['private_dataplane_verified'] = false;
|
|
$host17['readback'] = 'pending_guest';
|
|
$host17['last_error_code'] = 'vpc_private_guest_probe_unsupported';
|
|
$host17['member_nics'][0]['guest_network_verified'] = false;
|
|
$host17['member_nics'][0]['guest_network_readback'] = 'unsupported';
|
|
$host17['member_nics'][0]['guest_reason_code'] =
|
|
'vpc_private_guest_probe_unsupported';
|
|
$deferred17 = $observed(34, $digest17, ['170' => $host17]);
|
|
$deferred17['state'] = 'applying';
|
|
$deferred17['next_stage_allowed'] = false;
|
|
$deferred17['private_dataplane_verified'] = false;
|
|
CloudVpcService::recordObserved(
|
|
$vpcKey17, 34, $digest17, $deferred17, $operation17
|
|
);
|
|
$rootProjection17 = $publicRootMethod->invoke(null, CloudVpc::find(17));
|
|
$assert(Capsule::table('cloud_vpc_members')->where('id', 123)->count() === 0
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 223)->count() === 0
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 124)->count() === 1
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 224)->count() === 1
|
|
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 17)
|
|
->where('host_id', 170)->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation17)
|
|
->value('status') === 'succeeded'
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation17)
|
|
->value('error_code') === null
|
|
&& Capsule::table('cloud_vpcs')->where('id', 17)
|
|
->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 17)
|
|
->value('sync_state') === 'applied'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 17)
|
|
->value('last_error_code') === null
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 124)
|
|
->value('last_error_code') === 'vpc_private_guest_probe_unsupported'
|
|
&& ($rootProjection17['configured'] ?? false) === true
|
|
&& ($rootProjection17['usable'] ?? true) === false,
|
|
'Config-exact detach did not release only A while B guest readiness stayed closed.');
|
|
|
|
// No new mutation is required once B becomes observable. A later guest-health
|
|
// receipt may use the same immutable operation identity without reopening the
|
|
// applied configuration; the already released A/IP rows stay absent.
|
|
$readyHost17 = $activeHost(
|
|
$vpcKey17, '10.37.0.0/24', 170, 34, $digest17,
|
|
$liveKey17, $liveUuid17, '10.37.0.11'
|
|
);
|
|
CloudVpcService::recordObserved(
|
|
$vpcKey17, 34, $digest17,
|
|
$observed(34, $digest17, ['170' => $readyHost17]),
|
|
$operation17
|
|
);
|
|
$readyRootProjection17 = $publicRootMethod->invoke(null, CloudVpc::find(17));
|
|
$assert(Capsule::table('cloud_vpc_members')->where('id', 123)->count() === 0
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 223)->count() === 0
|
|
&& Capsule::table('cloud_vpc_operations')->where('vpc_id', 17)
|
|
->where('operation_id', $operation17)->count() === 1
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation17)
|
|
->value('payload_json') === $detachRuntimePayload17
|
|
&& !str_contains($detachRuntimePayload17, $retiringKey17)
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation17)
|
|
->value('status') === 'succeeded'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 17)
|
|
->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 17)
|
|
->value('sync_state') === 'applied'
|
|
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 17)
|
|
->where('host_id', 170)->value('state') === 'active'
|
|
&& ($readyRootProjection17['usable'] ?? false) === true,
|
|
'A current detach operation could not finish sibling runtime reconciliation.');
|
|
|
|
// Runtime evidence is member-scoped. A running/exact A remains usable when B
|
|
// is stopped after the same host configuration has completed. The operation
|
|
// is terminal for configuration and is not left as a worker retry target.
|
|
$vpcKey19 = str_repeat('9', 31) . 'a';
|
|
$digest19 = str_repeat('c', 63) . '9';
|
|
$operation19 = str_repeat('a', 31) . '9';
|
|
$memberKey19a = str_repeat('b', 31) . '9';
|
|
$memberKey19b = str_repeat('c', 31) . '8';
|
|
$uuid19a = '19191919-1919-4919-8919-191919191901';
|
|
$uuid19b = '19191919-1919-4919-8919-191919191902';
|
|
$seedVpc(19, $vpcKey19, 38, $digest19, $operation19, 'member_attach');
|
|
$seedSubnet(120, 19, str_repeat('d', 31) . '9', '10.39.0.0/25',
|
|
'applying', 38, $digest19);
|
|
$seedMember(126, 226, 19, 120, $memberKey19a, 19001, $uuid19a,
|
|
190, '10.39.0.10', 'applying', 38, $digest19);
|
|
$seedMember(127, 227, 19, 120, $memberKey19b, 19002, $uuid19b,
|
|
190, '10.39.0.11', 'applying', 38, $digest19);
|
|
$seedHost(19, 190, 'applying', 38, $digest19, $operation19);
|
|
$host19 = $activeHost(
|
|
$vpcKey19, '10.39.0.0/24', 190, 38, $digest19,
|
|
$memberKey19a, $uuid19a, '10.39.0.10'
|
|
);
|
|
$bridge19 = $bridgeMethod->invoke(null, $vpcKey19);
|
|
$mac19b = $privateMacMethod->invoke(null, $vpcKey19, $memberKey19b);
|
|
$host19['state'] = 'applying';
|
|
$host19['private_dataplane_verified'] = false;
|
|
$host19['readback'] = 'pending_guest';
|
|
$host19['last_error_code'] = 'vpc_private_guest_restart_required';
|
|
$host19['proxy_ips'][] = '10.39.0.11';
|
|
$host19['member_nics'][0]['guest_routes'] = ['10.39.0.0/24'];
|
|
$host19['member_nics'][] = [
|
|
'member_id' => $memberKey19b, 'instance_uuid' => $uuid19b,
|
|
'interface' => 'private0', 'private_ip_cidr' => '10.39.0.11/32',
|
|
'private_mac' => $mac19b, 'xml_verified' => true,
|
|
'config_verified' => true, 'seed_verified' => true,
|
|
'restart_required' => true,
|
|
'domain_running' => false, 'guest_network_verified' => false,
|
|
'guest_network_readback' => 'stopped',
|
|
'guest_reason_code' => 'vpc_private_guest_restart_required',
|
|
'guest_private_mac' => null, 'guest_private_ip_cidr' => null,
|
|
'guest_routes' => [], 'link_state' => 'up', 'guest_link_state' => null,
|
|
];
|
|
$host19['routes'][] = [
|
|
'destination_cidr' => '10.39.0.11/32', 'device' => $bridge19,
|
|
'table' => $routeTableMethod->invoke(null, $vpcKey19),
|
|
'owner' => 'virt-cloud-vpc-dataplane-v1', 'via' => null, 'metric' => 100,
|
|
];
|
|
$partial19 = $observed(38, $digest19, ['190' => $host19]);
|
|
$partial19['state'] = 'applying';
|
|
$partial19['next_stage_allowed'] = false;
|
|
$partial19['private_dataplane_verified'] = false;
|
|
CloudVpcService::recordObserved(
|
|
$vpcKey19, 38, $digest19, $partial19, $operation19
|
|
);
|
|
$projection19a = CloudVpcService::instancePrivateProjection(19001);
|
|
$projection19b = CloudVpcService::instancePrivateProjection(19002);
|
|
$rootProjection19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
|
|
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation19)
|
|
->value('status') === 'succeeded'
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 126)
|
|
->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 226)
|
|
->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 127)
|
|
->value('state') === 'degraded'
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 127)
|
|
->value('last_error_code') === 'vpc_private_guest_restart_required'
|
|
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 19)
|
|
->where('host_id', 190)->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 19)
|
|
->where('host_id', 190)->value('last_error_code') === null
|
|
&& Capsule::table('cloud_vpcs')->where('id', 19)
|
|
->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 19)
|
|
->value('sync_state') === 'applied'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 19)
|
|
->value('last_error_code') === null
|
|
&& ($projection19a['state'] ?? null) === 'active'
|
|
&& ($projection19a['readback'] ?? null) === 'confirmed'
|
|
&& ($projection19b['reason_code'] ?? null)
|
|
=== 'vpc_private_guest_restart_required'
|
|
&& ($rootProjection19['configured'] ?? false) === true
|
|
&& ($rootProjection19['usable'] ?? true) === false,
|
|
'A stopped sibling blocked an independently proven running member.');
|
|
|
|
// Age every operation as if the daemon polled again. Guest-only health must
|
|
// not make a succeeded, config-exact operation eligible for another apply.
|
|
Capsule::table('cloud_vpc_operations')->where('operation_id', '<>', $operation19)
|
|
->update(['status' => 'superseded', 'lease_until' => 0]);
|
|
Capsule::table('cloud_vpc_operations')->where('operation_id', $operation19)
|
|
->update(['updated_at' => time() - 31, 'lease_until' => 0]);
|
|
$assert((new \App\Services\Cloud\CloudVpcWorkerService())->tick() === false,
|
|
'A config-exact root with guest-only health remained an automatic retry target.');
|
|
|
|
$ready19 = $host19;
|
|
$ready19['state'] = 'active';
|
|
$ready19['private_dataplane_verified'] = true;
|
|
$ready19['readback'] = 'confirmed';
|
|
$ready19['last_error_code'] = null;
|
|
$ready19['member_nics'][1] = [
|
|
'member_id' => $memberKey19b, 'instance_uuid' => $uuid19b,
|
|
'interface' => 'private0', 'private_ip_cidr' => '10.39.0.11/32',
|
|
'private_mac' => $mac19b, 'xml_verified' => true,
|
|
'config_verified' => true, 'seed_verified' => true,
|
|
'restart_required' => false,
|
|
'domain_running' => true, 'guest_network_verified' => true,
|
|
'guest_network_readback' => 'confirmed', 'guest_reason_code' => null,
|
|
'guest_private_mac' => $mac19b,
|
|
'guest_private_ip_cidr' => '10.39.0.11/32',
|
|
'guest_routes' => ['10.39.0.0/24'],
|
|
'link_state' => 'up', 'guest_link_state' => 'up',
|
|
];
|
|
CloudVpcService::recordObserved(
|
|
$vpcKey19, 38, $digest19,
|
|
$observed(38, $digest19, ['190' => $ready19]),
|
|
$operation19
|
|
);
|
|
$readyProjection19b = CloudVpcService::instancePrivateProjection(19002);
|
|
$readyRootProjection19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
|
|
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation19)
|
|
->value('operation_id') === $operation19
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 126)
|
|
->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 127)
|
|
->value('state') === 'active'
|
|
&& ($readyProjection19b['state'] ?? null) === 'active'
|
|
&& ($readyProjection19b['readback'] ?? null) === 'confirmed'
|
|
&& ($readyRootProjection19['usable'] ?? false) === true,
|
|
'Starting B did not converge only its runtime gate on the same operation.');
|
|
|
|
// Runtime usability also depends on the instance network-policy state. The
|
|
// VPC/member proof is still exact while quota/expiry enforcement is applying,
|
|
// but the public root must fail closed immediately in that window.
|
|
Capsule::table('cloud_instances')->where('id', 19001)->update([
|
|
'network_status' => 'blocking',
|
|
]);
|
|
$blockedRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
|
|
$assert(($blockedRoot19['usable'] ?? true) === false,
|
|
'A blocking instance network policy left an exact VPC publicly usable.');
|
|
Capsule::table('cloud_instances')->where('id', 19001)->update([
|
|
'network_status' => 'active',
|
|
]);
|
|
$restoredPolicyRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
|
|
$assert(($restoredPolicyRoot19['usable'] ?? false) === true,
|
|
'An active instance network policy did not restore the unchanged VPC proof.');
|
|
|
|
// A graceful shutdown can be accepted while the VM is still running. Once
|
|
// the ordinary runtime heartbeat observes the later shutoff, public usability
|
|
// must close even though no VPC mutation or new VPC operation occurred.
|
|
Capsule::table('cloud_instance_runtime')->where('instance_id', 19002)->update([
|
|
'power_state' => 'stopped', 'sync_state' => 'ok',
|
|
'last_success_at' => time(), 'updated_at' => time(),
|
|
]);
|
|
$heartbeatStoppedRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
|
|
$assert(($heartbeatStoppedRoot19['usable'] ?? true) === false,
|
|
'A fresh stopped runtime heartbeat left an old VPC proof publicly usable.');
|
|
Capsule::table('cloud_instance_runtime')->where('instance_id', 19002)->update([
|
|
'power_state' => 'running', 'sync_state' => 'ok',
|
|
'last_success_at' => time(), 'updated_at' => time(),
|
|
]);
|
|
$heartbeatRunningRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
|
|
$assert(($heartbeatRunningRoot19['usable'] ?? false) === true,
|
|
'A fresh running heartbeat did not restore the unchanged exact VPC proof.');
|
|
|
|
$expectation19b = CloudVpcService::instancePrivateExpectation(19002);
|
|
$stoppedObservation19b = [
|
|
'version' => 1, 'required' => true, 'vm_uuid' => $uuid19b,
|
|
'vpc_id' => $vpcKey19, 'member_id' => $memberKey19b,
|
|
'interface' => 'private0', 'state' => 'stopped',
|
|
'desired_revision' => 38, 'observed_revision' => 38,
|
|
'desired_digest' => $digest19, 'observed_digest' => $digest19,
|
|
'readback' => 'confirmed', 'link_state' => 'up',
|
|
'private_ip_cidr' => '10.39.0.11/32', 'private_mac' => $mac19b,
|
|
'xml_verified' => true, 'config_verified' => true,
|
|
'seed_verified' => true,
|
|
'domain_running' => false, 'restart_required' => false,
|
|
'guest_network_verified' => false,
|
|
'guest_network_readback' => 'stopped', 'guest_link_state' => null,
|
|
'reason_code' => null, 'observed_at' => time(),
|
|
];
|
|
|
|
// A current VM lifecycle receipt separates persistent host configuration from
|
|
// guest/QGA health. Guest-only failure stays on member/IP, while a positive
|
|
// XML/config/seed mismatch invalidates the host/root configuration projection.
|
|
$guestOnlyObservation19b = $stoppedObservation19b;
|
|
$guestOnlyObservation19b['state'] = 'active';
|
|
$guestOnlyObservation19b['domain_running'] = true;
|
|
$guestOnlyObservation19b['guest_network_verified'] = false;
|
|
$guestOnlyObservation19b['guest_network_readback'] = 'unavailable';
|
|
$guestOnlyObservation19b['guest_link_state'] = null;
|
|
$guestOnlyValidation19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $guestOnlyObservation19b,
|
|
'action_start_policy_active', $uuid19b
|
|
);
|
|
$assert(($guestOnlyValidation19b['ok'] ?? true) === false
|
|
&& CloudVpcService::recordPrivateObservation(
|
|
19002, $guestOnlyValidation19b, str_repeat('e', 31) . '9'
|
|
) === true,
|
|
'Guest-only lifecycle failure could not be persisted.');
|
|
$guestOnlyRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
|
|
$assert(Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 19)
|
|
->where('host_id', 190)->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 19)->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 19)->value('sync_state') === 'applied'
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 127)
|
|
->value('state') === 'uncertain'
|
|
&& ($guestOnlyRoot19['configured'] ?? false) === true
|
|
&& ($guestOnlyRoot19['usable'] ?? true) === false,
|
|
'Guest-only lifecycle health revoked the exact configuration receipt.');
|
|
CloudVpcService::recordObserved(
|
|
$vpcKey19, 38, $digest19,
|
|
$observed(38, $digest19, ['190' => $ready19]),
|
|
$operation19
|
|
);
|
|
|
|
$configDriftObservation19b = $guestOnlyObservation19b;
|
|
$configDriftObservation19b['xml_verified'] = false;
|
|
$configDriftValidation19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $configDriftObservation19b,
|
|
'action_start_policy_active', $uuid19b
|
|
);
|
|
$assert(($configDriftValidation19b['ok'] ?? true) === false
|
|
&& CloudVpcService::recordPrivateObservation(
|
|
19002, $configDriftValidation19b, str_repeat('f', 31) . '9'
|
|
) === true,
|
|
'Persistent lifecycle configuration drift could not be persisted.');
|
|
$configDriftRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
|
|
$assert(Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 19)
|
|
->where('host_id', 190)->value('state') === 'degraded'
|
|
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 19)
|
|
->where('host_id', 190)->value('last_error_code')
|
|
=== 'vpc_private_observation_mismatch'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 19)->value('state') === 'degraded'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 19)->value('sync_state') === 'uncertain'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 19)->value('last_error_code')
|
|
=== 'vpc_private_observation_mismatch'
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation19)
|
|
->value('status') === 'succeeded'
|
|
&& ($configDriftRoot19['configured'] ?? true) === false,
|
|
'Persistent lifecycle configuration drift left host/root configured.');
|
|
CloudVpcService::recordObserved(
|
|
$vpcKey19, 38, $digest19,
|
|
$observed(38, $digest19, ['190' => $ready19]),
|
|
$operation19
|
|
);
|
|
|
|
$shutdownRunningObservation19b = $stoppedObservation19b;
|
|
$shutdownRunningObservation19b['state'] = 'active';
|
|
$shutdownRunningObservation19b['domain_running'] = true;
|
|
$shutdownRunningObservation19b['guest_network_verified'] = true;
|
|
$shutdownRunningObservation19b['guest_network_readback'] = 'confirmed';
|
|
$shutdownRunningObservation19b['guest_link_state'] = 'up';
|
|
$shutdownAccepted19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $shutdownRunningObservation19b,
|
|
'action_shutdown_policy_active', $uuid19b
|
|
);
|
|
$assert(($shutdownAccepted19b['ok'] ?? false) === true
|
|
&& ($shutdownAccepted19b['state']['state'] ?? null) === 'stopping'
|
|
&& CloudVpcService::recordPrivateObservation(
|
|
19002, $shutdownAccepted19b, str_repeat('0', 31) . '9'
|
|
) === true,
|
|
'Accepted graceful shutdown did not enter a non-usable transition.');
|
|
Capsule::table('cloud_instance_runtime')->where('instance_id', 19002)->update([
|
|
'power_state' => 'shutdown', 'sync_state' => 'ok',
|
|
'last_success_at' => time(), 'updated_at' => time(),
|
|
]);
|
|
$shutdownPendingProjection19b = CloudVpcService::instancePrivateProjection(19002);
|
|
$shutdownPendingRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
|
|
$assert(Capsule::table('cloud_vpc_members')->where('id', 127)
|
|
->value('state') === 'stopping'
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 227)
|
|
->value('state') === 'stopping'
|
|
&& ($shutdownPendingProjection19b['state'] ?? null) === 'stopping'
|
|
&& ($shutdownPendingRoot19['usable'] ?? true) === false,
|
|
'Graceful shutdown acceptance left member, lease or root runtime usable.');
|
|
$periodicShutdownPending19 = $ready19;
|
|
$periodicShutdownPending19['state'] = 'active';
|
|
$periodicShutdownPending19['private_dataplane_verified'] = true;
|
|
CloudVpcService::recordObserved(
|
|
$vpcKey19, 38, $digest19,
|
|
$observed(38, $digest19, ['190' => $periodicShutdownPending19]),
|
|
$operation19
|
|
);
|
|
$assert(Capsule::table('cloud_vpc_members')->where('id', 127)
|
|
->value('state') === 'stopping'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 19)
|
|
->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 19)
|
|
->value('sync_state') === 'applied'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 19)
|
|
->value('last_error_code') === null
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 227)
|
|
->value('last_error_code') === 'vpc_private_guest_shutdown_pending',
|
|
'A periodic running/QGA receipt reopened a graceful-shutdown transition.');
|
|
$shutdownValidation19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $stoppedObservation19b,
|
|
'action_shutdown_policy_active', $uuid19b
|
|
);
|
|
$poweroffValidation19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $stoppedObservation19b,
|
|
'action_poweroff_policy_active', $uuid19b
|
|
);
|
|
$assert(($shutdownValidation19b['ok'] ?? false) === true
|
|
&& ($poweroffValidation19b['ok'] ?? false) === true
|
|
&& CloudVpcService::recordPrivateObservation(
|
|
19002, $shutdownValidation19b, str_repeat('1', 31) . '9'
|
|
) === true,
|
|
'Shutdown with a preserved configured-up link was not stored as stopped.');
|
|
Capsule::table('cloud_instance_runtime')->where('instance_id', 19002)->update([
|
|
'power_state' => 'stopped', 'sync_state' => 'ok',
|
|
'last_success_at' => time(), 'updated_at' => time(),
|
|
]);
|
|
$stoppedProjection19b = CloudVpcService::instancePrivateProjection(19002);
|
|
$stoppedRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
|
|
$assert(Capsule::table('cloud_vpc_members')->where('id', 127)
|
|
->value('state') === 'stopped'
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 227)
|
|
->value('state') === 'stopped'
|
|
&& ($stoppedProjection19b['state'] ?? null) === 'stopped'
|
|
&& ($stoppedProjection19b['link_state'] ?? null) === 'up'
|
|
&& ($stoppedRoot19['usable'] ?? true) === false,
|
|
'A confirmed stopped guest remained active or publicly usable.');
|
|
|
|
// Periodic VPC observation must preserve a lifecycle-confirmed stopped state;
|
|
// the strict guest probe is intentionally absent for an inactive domain.
|
|
$periodicStopped19 = $ready19;
|
|
$periodicStopped19['state'] = 'applying';
|
|
$periodicStopped19['private_dataplane_verified'] = false;
|
|
$periodicStopped19['last_error_code'] = 'vpc_private_guest_stopped';
|
|
$periodicStopped19['member_nics'][1]['domain_running'] = false;
|
|
$periodicStopped19['member_nics'][1]['restart_required'] = false;
|
|
$periodicStopped19['member_nics'][1]['guest_network_verified'] = false;
|
|
$periodicStopped19['member_nics'][1]['guest_network_readback'] = 'stopped';
|
|
$periodicStopped19['member_nics'][1]['guest_reason_code'] =
|
|
'vpc_private_guest_stopped';
|
|
$periodicStopped19['member_nics'][1]['guest_private_mac'] = null;
|
|
$periodicStopped19['member_nics'][1]['guest_private_ip_cidr'] = null;
|
|
$periodicStopped19['member_nics'][1]['guest_routes'] = [];
|
|
$periodicStoppedObservation19 = $observed(38, $digest19, [
|
|
'190' => $periodicStopped19,
|
|
]);
|
|
$periodicStoppedObservation19['state'] = 'applying';
|
|
$periodicStoppedObservation19['next_stage_allowed'] = false;
|
|
$periodicStoppedObservation19['private_dataplane_verified'] = false;
|
|
CloudVpcService::recordObserved(
|
|
$vpcKey19, 38, $digest19, $periodicStoppedObservation19, $operation19
|
|
);
|
|
$assert(Capsule::table('cloud_vpc_members')->where('id', 127)
|
|
->value('state') === 'stopped'
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 227)
|
|
->value('state') === 'stopped'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 19)
|
|
->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 19)
|
|
->value('sync_state') === 'applied'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 19)
|
|
->value('last_error_code') === null
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 227)
|
|
->value('last_error_code') === 'vpc_private_guest_stopped',
|
|
'Periodic reconciliation overwrote exact stopped evidence with degraded.');
|
|
|
|
// Service policy preserves both the stopped power target and the independent
|
|
// network policy. A tenant-stopped guest must remain stopped across expiry and
|
|
// renewal when start_if_suspended=false.
|
|
$serviceSuspendStopped19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $stoppedObservation19b,
|
|
'service_suspend_policy_active', $uuid19b
|
|
);
|
|
$serviceResumeStopped19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $stoppedObservation19b,
|
|
'service_resume_stopped_policy_active', $uuid19b
|
|
);
|
|
$unboundServiceResume19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $stoppedObservation19b,
|
|
'service_resume_unbound_policy_active', $uuid19b
|
|
);
|
|
$assert(($serviceSuspendStopped19b['ok'] ?? false) === true
|
|
&& ($serviceResumeStopped19b['ok'] ?? false) === true
|
|
&& ($unboundServiceResume19b['ok'] ?? true) === false,
|
|
'Stopped expiry/resume policy was not bound to its explicit payload target.');
|
|
|
|
// Link policy may change while the domain stays stopped. Both operations are
|
|
// exact configuration transitions, but neither makes runtime usable.
|
|
$stoppedDownObservation19b = $stoppedObservation19b;
|
|
$stoppedDownObservation19b['link_state'] = 'down';
|
|
$stoppedSuspendValidation19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $stoppedDownObservation19b,
|
|
'network_suspend', $uuid19b
|
|
);
|
|
$blockedPoweroffValidation19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $stoppedDownObservation19b,
|
|
'action_poweroff_policy_blocked', $uuid19b
|
|
);
|
|
$blockedServiceSuspend19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $stoppedDownObservation19b,
|
|
'service_suspend_policy_blocked', $uuid19b
|
|
);
|
|
$blockedServiceResumeStopped19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $stoppedDownObservation19b,
|
|
'service_resume_stopped_policy_blocked', $uuid19b
|
|
);
|
|
Capsule::table('cloud_instances')->where('id', 19002)->update([
|
|
'network_status' => 'blocked_traffic',
|
|
]);
|
|
$assert(($stoppedSuspendValidation19b['ok'] ?? false) === true
|
|
&& ($blockedPoweroffValidation19b['ok'] ?? false) === true
|
|
&& ($blockedServiceSuspend19b['ok'] ?? false) === true
|
|
&& ($blockedServiceResumeStopped19b['ok'] ?? false) === true
|
|
&& CloudVpcService::recordPrivateObservation(
|
|
19002, $stoppedSuspendValidation19b, str_repeat('2', 31) . '9'
|
|
) === true
|
|
&& (CloudVpcService::instancePrivateProjection(19002)['state'] ?? null)
|
|
=== 'stopped'
|
|
&& (CloudVpcService::instancePrivateProjection(19002)['link_state'] ?? null)
|
|
=== 'down',
|
|
'Stopped network suspend did not persist configured-down without activation.');
|
|
$stoppedResumeValidation19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $stoppedObservation19b,
|
|
'network_resume', $uuid19b
|
|
);
|
|
Capsule::table('cloud_instances')->where('id', 19002)->update([
|
|
'network_status' => 'active',
|
|
]);
|
|
$assert(($stoppedResumeValidation19b['ok'] ?? false) === true
|
|
&& CloudVpcService::recordPrivateObservation(
|
|
19002, $stoppedResumeValidation19b, str_repeat('3', 31) . '9'
|
|
) === true
|
|
&& (CloudVpcService::instancePrivateProjection(19002)['state'] ?? null)
|
|
=== 'stopped'
|
|
&& (CloudVpcService::instancePrivateProjection(19002)['link_state'] ?? null)
|
|
=== 'up',
|
|
'Stopped network resume became usable or lost configured-up evidence.');
|
|
|
|
// Starting an active-policy VM needs a new, guest-visible QGA proof. The same
|
|
// config-only stopped receipt is insufficient.
|
|
$activeObservation19b = $stoppedObservation19b;
|
|
$activeObservation19b['state'] = 'active';
|
|
$activeObservation19b['domain_running'] = true;
|
|
$activeObservation19b['guest_network_verified'] = true;
|
|
$activeObservation19b['guest_network_readback'] = 'confirmed';
|
|
$activeObservation19b['guest_link_state'] = 'up';
|
|
$startValidation19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $activeObservation19b,
|
|
'action_start_policy_active', $uuid19b
|
|
);
|
|
$missingStartGuest19b = $activeObservation19b;
|
|
$missingStartGuest19b['guest_network_verified'] = false;
|
|
$missingStartValidation19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $missingStartGuest19b,
|
|
'action_start_policy_active', $uuid19b
|
|
);
|
|
$genericActiveValidation19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $activeObservation19b,
|
|
'snapshot_restore', $uuid19b
|
|
);
|
|
$legacyActiveObservation19b = $activeObservation19b;
|
|
unset($legacyActiveObservation19b['seed_verified']);
|
|
unset($legacyActiveObservation19b['restart_required']);
|
|
unset($legacyActiveObservation19b['guest_network_verified']);
|
|
unset($legacyActiveObservation19b['guest_network_readback']);
|
|
unset($legacyActiveObservation19b['guest_link_state']);
|
|
$genericMissingGuest19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $legacyActiveObservation19b,
|
|
'snapshot_restore', $uuid19b
|
|
);
|
|
$assert(($startValidation19b['ok'] ?? false) === true
|
|
&& ($missingStartValidation19b['ok'] ?? true) === false
|
|
&& ($genericActiveValidation19b['ok'] ?? false) === true
|
|
&& ($genericMissingGuest19b['ok'] ?? true) === false
|
|
&& CloudVpcService::recordPrivateObservation(
|
|
19002, $startValidation19b, str_repeat('4', 31) . '9'
|
|
) === true,
|
|
'An active lifecycle result did not require fresh guest-visible private readiness.');
|
|
Capsule::table('cloud_instance_runtime')->where('instance_id', 19002)->update([
|
|
'power_state' => 'running', 'sync_state' => 'ok',
|
|
'last_success_at' => time(), 'updated_at' => time(),
|
|
]);
|
|
$startedRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
|
|
$assert(($startedRoot19['usable'] ?? false) === true,
|
|
'Fresh QGA start did not restore VPC usability after stopped state.');
|
|
|
|
$suspendedObservation19b = [
|
|
'version' => 1, 'required' => true, 'vm_uuid' => $uuid19b,
|
|
'vpc_id' => $vpcKey19, 'member_id' => $memberKey19b,
|
|
'interface' => 'private0', 'state' => 'suspended',
|
|
'desired_revision' => 38, 'observed_revision' => 38,
|
|
'desired_digest' => $digest19, 'observed_digest' => $digest19,
|
|
'readback' => 'confirmed', 'link_state' => 'down',
|
|
'private_ip_cidr' => '10.39.0.11/32', 'private_mac' => $mac19b,
|
|
'xml_verified' => true, 'config_verified' => true,
|
|
'seed_verified' => true,
|
|
'domain_running' => true, 'restart_required' => false,
|
|
'guest_network_verified' => false,
|
|
'guest_network_readback' => 'suspended', 'guest_link_state' => null,
|
|
'reason_code' => null, 'observed_at' => time(),
|
|
];
|
|
$suspendValidation19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $suspendedObservation19b, 'network_suspend', $uuid19b
|
|
);
|
|
$assert(($suspendValidation19b['ok'] ?? false) === true
|
|
&& CloudVpcService::recordPrivateObservation(
|
|
19002, $suspendValidation19b, str_repeat('d', 31) . '9'
|
|
) === true,
|
|
'An exact lifecycle link-down observation was not persisted.');
|
|
Capsule::table('cloud_instances')->where('id', 19002)->update([
|
|
'network_status' => 'blocked_traffic',
|
|
]);
|
|
$suspendedProjection19b = CloudVpcService::instancePrivateProjection(19002);
|
|
$suspendedRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
|
|
$assert(Capsule::table('cloud_vpc_members')->where('id', 127)
|
|
->value('state') === 'suspended'
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 227)
|
|
->value('state') === 'suspended'
|
|
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 19)
|
|
->where('host_id', 190)->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 19)
|
|
->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 19)
|
|
->value('sync_state') === 'applied'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 19)
|
|
->value('last_error_code') === null
|
|
&& ($suspendedProjection19b['state'] ?? null) === 'suspended'
|
|
&& ($suspendedProjection19b['link_state'] ?? null) === 'down'
|
|
&& ($suspendedRoot19['usable'] ?? true) === false,
|
|
'Lifecycle suspension left member, host or VPC runtime falsely usable.');
|
|
|
|
$periodicSuspended19 = $ready19;
|
|
$periodicSuspended19['state'] = 'applying';
|
|
$periodicSuspended19['private_dataplane_verified'] = false;
|
|
$periodicSuspended19['last_error_code'] = 'vpc_private_guest_link_mismatch';
|
|
$periodicSuspended19['member_nics'][1]['link_state'] = 'down';
|
|
$periodicSuspended19['member_nics'][1]['guest_network_verified'] = false;
|
|
$periodicSuspended19['member_nics'][1]['guest_network_readback'] = 'suspended';
|
|
$periodicSuspended19['member_nics'][1]['guest_reason_code'] =
|
|
'vpc_private_guest_link_mismatch';
|
|
$periodicObserved19 = $observed(38, $digest19, [
|
|
'190' => $periodicSuspended19,
|
|
]);
|
|
$periodicObserved19['state'] = 'applying';
|
|
$periodicObserved19['next_stage_allowed'] = false;
|
|
$periodicObserved19['private_dataplane_verified'] = false;
|
|
CloudVpcService::recordObserved(
|
|
$vpcKey19, 38, $digest19, $periodicObserved19, $operation19
|
|
);
|
|
$periodicRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
|
|
$assert(Capsule::table('cloud_vpc_members')->where('id', 127)
|
|
->value('state') === 'suspended'
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 227)
|
|
->value('state') === 'suspended'
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation19)
|
|
->value('status') === 'succeeded'
|
|
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 19)
|
|
->where('host_id', 190)->value('last_error_code') === null
|
|
&& Capsule::table('cloud_vpcs')->where('id', 19)
|
|
->value('sync_state') === 'applied'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 19)
|
|
->value('last_error_code') === null
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 227)
|
|
->value('last_error_code') === 'vpc_private_network_suspended'
|
|
&& ($periodicRoot19['usable'] ?? true) === false,
|
|
'Periodic VPC observation overwrote intentional suspended runtime state.');
|
|
|
|
// User power actions and service resume preserve a blocked network target.
|
|
// A running domain is represented as suspended/config-exact, and an active/up
|
|
// receipt is rejected so neither path can bypass quota/expiry enforcement.
|
|
$blockedReboot19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $suspendedObservation19b,
|
|
'action_reboot_policy_blocked', $uuid19b
|
|
);
|
|
$blockedStart19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $suspendedObservation19b,
|
|
'action_start_policy_blocked', $uuid19b
|
|
);
|
|
$blockedServiceStart19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $suspendedObservation19b,
|
|
'service_resume_start_policy_blocked', $uuid19b
|
|
);
|
|
$blockedButUp19b = $activeObservation19b;
|
|
$blockedButUpValidation19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $blockedButUp19b,
|
|
'action_reboot_policy_blocked', $uuid19b
|
|
);
|
|
$transitioningPower19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $suspendedObservation19b,
|
|
'action_reboot_policy_unready', $uuid19b
|
|
);
|
|
$assert(($blockedReboot19b['ok'] ?? false) === true
|
|
&& ($blockedStart19b['ok'] ?? false) === true
|
|
&& ($blockedServiceStart19b['ok'] ?? false) === true
|
|
&& ($blockedButUpValidation19b['ok'] ?? true) === false
|
|
&& ($transitioningPower19b['ok'] ?? true) === false
|
|
&& CloudVpcService::recordPrivateObservation(
|
|
19002, $blockedReboot19b, str_repeat('5', 31) . '9'
|
|
) === true
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 127)
|
|
->value('state') === 'suspended'
|
|
&& Capsule::table('cloud_instances')->where('id', 19002)
|
|
->value('network_status') === 'blocked_traffic',
|
|
'A blocked power/service action raised private0 or failed to converge safely.');
|
|
|
|
$resumedObservation19b = $suspendedObservation19b;
|
|
$resumedObservation19b['state'] = 'active';
|
|
$resumedObservation19b['link_state'] = 'up';
|
|
$resumedObservation19b['guest_network_verified'] = true;
|
|
$resumedObservation19b['guest_network_readback'] = 'confirmed';
|
|
$resumedObservation19b['guest_link_state'] = 'up';
|
|
$resumeValidation19b = CloudVpcService::validatePrivateObservation(
|
|
$expectation19b, $resumedObservation19b, 'network_resume', $uuid19b
|
|
);
|
|
$assert(($resumeValidation19b['ok'] ?? false) === true
|
|
&& CloudVpcService::recordPrivateObservation(
|
|
19002, $resumeValidation19b, str_repeat('e', 31) . '9'
|
|
) === true,
|
|
'An exact lifecycle link-up observation was not persisted.');
|
|
Capsule::table('cloud_instances')->where('id', 19002)->update([
|
|
'network_status' => 'active',
|
|
]);
|
|
$resumedProjection19b = CloudVpcService::instancePrivateProjection(19002);
|
|
$resumedRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19));
|
|
$assert(Capsule::table('cloud_vpc_members')->where('id', 127)
|
|
->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 227)
|
|
->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 19)
|
|
->where('host_id', 190)->value('state') === 'active'
|
|
&& Capsule::table('cloud_vpcs')->where('id', 19)
|
|
->value('state') === 'active'
|
|
&& ($resumedProjection19b['state'] ?? null) === 'active'
|
|
&& ($resumedProjection19b['link_state'] ?? null) === 'up'
|
|
&& ($resumedRoot19['usable'] ?? false) === true,
|
|
'Lifecycle resume became usable before exact link-up or failed to recover.');
|
|
|
|
// A running last member cannot be detached safely. Keep its tombstone and the
|
|
// operation retryable; after the VM is stopped, the exact removal receipt must
|
|
// converge the same operation ID instead of requiring a second detach request.
|
|
$vpcKey18 = str_repeat('6', 31) . '8';
|
|
$digest18 = str_repeat('b', 63) . '8';
|
|
$operation18 = str_repeat('7', 31) . '8';
|
|
$memberKey18 = str_repeat('8', 31) . '8';
|
|
$seedVpc(18, $vpcKey18, 36, $digest18, $operation18, 'member_detach');
|
|
$seedSubnet(119, 18, str_repeat('9', 31) . '8', '10.38.0.0/25',
|
|
'active', 36, $digest18);
|
|
$seedMember(125, 225, 18, 119, $memberKey18, 18001,
|
|
'18181818-1818-4818-8818-181818181818', 180, '10.38.0.10',
|
|
'deleting', 36, $digest18);
|
|
$seedHost(18, 180, 'deleting', 36, $digest18, $operation18);
|
|
CloudVpcService::recordObserved($vpcKey18, 36, $digest18, [
|
|
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
|
|
'revision' => 0, 'digest' => null, 'state' => 'degraded',
|
|
'next_stage_allowed' => false, 'private_dataplane_verified' => false,
|
|
'fabric_gate' => ['active' => true],
|
|
'hosts' => ['180' => [
|
|
'vpc_id' => $vpcKey18, 'host_id' => '180',
|
|
'desired_revision' => 36, 'desired_digest' => $digest18,
|
|
'observed_revision' => 0, 'observed_digest' => null,
|
|
'state' => 'uncertain', 'transport_verified' => false,
|
|
'private_dataplane_verified' => false, 'removed' => false,
|
|
'last_error_code' => 'vpc_private_remove_requires_stopped',
|
|
]],
|
|
], $operation18);
|
|
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation18)
|
|
->value('status') === 'uncertain'
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 125)
|
|
->value('state') === 'deleting'
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 225)
|
|
->value('state') === 'deleting',
|
|
'A running-member detach became terminal or released its tombstone early.');
|
|
CloudVpcService::recordObserved(
|
|
$vpcKey18, 36, $digest18,
|
|
$observed(36, $digest18, [
|
|
'180' => $removedHost($vpcKey18, 180, 36, $digest18),
|
|
]),
|
|
$operation18
|
|
);
|
|
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation18)
|
|
->value('status') === 'succeeded'
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $operation18)
|
|
->value('operation_id') === $operation18
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 125)->count() === 0
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 225)->count() === 0
|
|
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 18)
|
|
->where('host_id', 180)->value('state') === 'deleted',
|
|
'Stopping the VM did not converge the original detach operation.');
|
|
|
|
// A stopped-only admission failure is an exact, pre-write result. Preserve
|
|
// its reviewed reason so the tenant sees the required action instead of a
|
|
// generic readback mismatch; the same immutable operation remains retryable.
|
|
foreach ([
|
|
29 => 'vpc_member_requires_stopped',
|
|
30 => 'vpc_member_power_unavailable',
|
|
] as $preflightVpcId => $preflightCode) {
|
|
$revision = $preflightVpcId * 2;
|
|
$vpcKey = substr(hash('sha256', 'preflight-vpc-' . $preflightVpcId), 0, 32);
|
|
$digest = hash('sha256', 'preflight-digest-' . $preflightVpcId);
|
|
$operationId = substr(hash('sha256', 'preflight-operation-' . $preflightVpcId), 0, 32);
|
|
$memberKey = substr(hash('sha256', 'preflight-member-' . $preflightVpcId), 0, 32);
|
|
$subnetId = 3000 + $preflightVpcId;
|
|
$memberId = 3000 + $preflightVpcId;
|
|
$leaseId = 4000 + $preflightVpcId;
|
|
$instanceId = 28000 + $preflightVpcId;
|
|
$hostId = 280 + $preflightVpcId;
|
|
$uuid = sprintf('00000000-0000-4000-8000-%012d', $preflightVpcId);
|
|
|
|
$seedVpc($preflightVpcId, $vpcKey, $revision, $digest,
|
|
$operationId, 'member_attach');
|
|
$seedSubnet($subnetId, $preflightVpcId,
|
|
substr(hash('sha256', 'preflight-subnet-' . $preflightVpcId), 0, 32),
|
|
'10.' . (20 + $preflightVpcId) . '.0.0/25', 'applying',
|
|
$revision, $digest);
|
|
$seedMember($memberId, $leaseId, $preflightVpcId, $subnetId,
|
|
$memberKey, $instanceId, $uuid, $hostId,
|
|
'10.' . (20 + $preflightVpcId) . '.0.10', 'applying',
|
|
$revision, $digest);
|
|
$seedHost($preflightVpcId, $hostId, 'applying', $revision,
|
|
$digest, $operationId);
|
|
|
|
CloudVpcService::recordObserved($vpcKey, $revision, $digest, [
|
|
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
|
|
'revision' => 0, 'digest' => null, 'state' => 'applying',
|
|
'next_stage_allowed' => false,
|
|
'private_dataplane_verified' => false,
|
|
'fabric_gate' => ['active' => true],
|
|
'hosts' => [(string) $hostId => [
|
|
'vpc_id' => $vpcKey, 'host_id' => (string) $hostId,
|
|
'desired_revision' => $revision, 'desired_digest' => $digest,
|
|
'observed_revision' => 0, 'observed_digest' => null,
|
|
'state' => 'applying', 'transport_verified' => true,
|
|
'private_dataplane_verified' => false,
|
|
'config_verified' => false,
|
|
'config_observed_revision' => 0,
|
|
'config_observed_digest' => null,
|
|
'removed' => false, 'last_error_code' => $preflightCode,
|
|
]],
|
|
], $operationId);
|
|
|
|
$assert(Capsule::table('cloud_vpc_operations')
|
|
->where('operation_id', $operationId)->value('status')
|
|
=== 'waiting_observed'
|
|
&& Capsule::table('cloud_vpc_operations')
|
|
->where('operation_id', $operationId)->value('error_code')
|
|
=== $preflightCode
|
|
&& Capsule::table('cloud_vpcs')->where('id', $preflightVpcId)
|
|
->value('sync_state') === 'waiting'
|
|
&& Capsule::table('cloud_vpcs')->where('id', $preflightVpcId)
|
|
->value('last_error_code') === $preflightCode
|
|
&& Capsule::table('cloud_vpc_host_deployments')
|
|
->where('vpc_id', $preflightVpcId)->where('host_id', $hostId)
|
|
->value('last_error_code') === $preflightCode,
|
|
'A stopped-only preflight reason was replaced by a generic readback error.');
|
|
}
|
|
|
|
echo "VPC member/IP/subnet tombstone convergence contracts passed.\n";
|