You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudVpcDeleteRestoreTest.php

693 lines
31 KiB
PHP

<?php
use App\Models\CloudVpc;
use App\Models\CloudVpcOperation;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudVpcService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$root = dirname(__DIR__);
$source = (string) file_get_contents(
$root . '/app/Services/Cloud/CloudVpcService.php'
);
$assert = static function ($condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$assert(str_contains($source, "'vpc_delete_restore'")
&& str_contains($source, "'fencing'")
&& str_contains($source, 'withoutTerminalEmptyHostTombstones')
&& str_contains($source, 'publishDeleteRestoreLocked'),
'Delete-restore source contract is incomplete.');
if (!is_file($root . '/vendor/autoload.php')) {
echo "VPC delete restore source contracts passed (runtime skipped).\n";
return;
}
require $root . '/vendor/autoload.php';
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_regions', static function (Blueprint $table): void {
$table->increments('id');
});
$schema->create('cloud_vpcs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('vpc_key', 32)->unique();
$table->unsignedInteger('user_id');
$table->unsignedInteger('region_id');
$table->unsignedInteger('private_network_id');
$table->string('name', 63);
$table->string('cidr', 18);
$table->string('fabric_id', 64);
$table->string('fabric_region', 64);
$table->unsignedBigInteger('fabric_revision');
$table->unsignedInteger('max_members');
$table->string('state', 16);
$table->unsignedBigInteger('desired_revision');
$table->unsignedBigInteger('observed_revision');
$table->string('desired_digest', 64);
$table->string('observed_digest', 64)->nullable();
$table->string('sync_state', 16);
$table->string('last_operation_id', 32)->nullable();
$table->string('last_error_code', 64)->nullable();
$table->unsignedBigInteger('last_success_at')->default(0);
$table->unsignedBigInteger('last_failure_at')->default(0);
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$schema->create('cloud_vpc_subnets', static function (Blueprint $table): void {
$table->increments('id');
$table->string('subnet_key', 32);
$table->unsignedInteger('vpc_id');
$table->string('cidr', 18);
$table->unsignedBigInteger('network_address')->default(0);
$table->unsignedBigInteger('broadcast_address')->default(0);
$table->unsignedTinyInteger('prefix_length')->default(24);
$table->string('gateway_ip', 15)->nullable();
$table->string('state', 16);
$table->unsignedBigInteger('desired_revision')->default(0);
$table->unsignedBigInteger('observed_revision')->default(0);
$table->string('desired_digest', 64)->nullable();
$table->string('observed_digest', 64)->nullable();
$table->string('last_error_code', 64)->nullable();
$table->unsignedBigInteger('created_at')->default(0);
$table->unsignedBigInteger('updated_at')->default(0);
});
$schema->create('cloud_instances', static function (Blueprint $table): void {
$table->increments('id');
$table->string('uuid', 36);
$table->string('status', 16)->default('active');
$table->unsignedInteger('host_id')->default(1);
});
$schema->create('cloud_vpc_members', static function (Blueprint $table): void {
$table->increments('id');
$table->string('member_key', 32);
$table->unsignedInteger('vpc_id');
$table->unsignedInteger('subnet_id')->nullable();
$table->unsignedInteger('private_ip_id')->nullable();
$table->unsignedInteger('instance_id')->nullable();
$table->unsignedInteger('host_id')->nullable();
$table->string('private_ip', 15)->nullable();
$table->string('state', 16);
$table->unsignedBigInteger('desired_revision')->default(0);
$table->unsignedBigInteger('observed_revision')->default(0);
$table->string('desired_digest', 64)->nullable();
$table->string('observed_digest', 64)->nullable();
$table->string('last_operation_id', 32)->nullable();
$table->string('last_error_code', 64)->nullable();
$table->unsignedBigInteger('created_at')->default(0);
$table->unsignedBigInteger('updated_at')->default(0);
});
$schema->create('cloud_vpc_private_ips', static function (Blueprint $table): void {
$table->increments('id');
$table->unsignedInteger('vpc_id');
$table->unsignedInteger('subnet_id')->nullable();
$table->unsignedInteger('member_id')->nullable();
$table->unsignedInteger('instance_id')->nullable();
$table->string('private_ip', 15)->nullable();
$table->string('state', 16);
$table->unsignedBigInteger('desired_revision')->default(0);
$table->unsignedBigInteger('observed_revision')->default(0);
$table->string('desired_digest', 64)->nullable();
$table->string('observed_digest', 64)->nullable();
$table->string('last_error_code', 64)->nullable();
$table->unsignedBigInteger('created_at')->default(0);
$table->unsignedBigInteger('updated_at')->default(0);
});
$schema->create('cloud_vpc_routes', static function (Blueprint $table): void {
$table->increments('id');
$table->string('route_key', 32);
$table->unsignedInteger('vpc_id');
$table->string('destination_cidr', 18)->nullable();
$table->unsignedInteger('member_id')->nullable();
$table->unsignedInteger('metric')->default(100);
$table->string('state', 16);
$table->unsignedBigInteger('desired_revision')->default(0);
$table->unsignedBigInteger('observed_revision')->default(0);
$table->string('desired_digest', 64)->nullable();
$table->string('observed_digest', 64)->nullable();
$table->string('last_error_code', 64)->nullable();
$table->unsignedBigInteger('created_at')->default(0);
$table->unsignedBigInteger('updated_at')->default(0);
});
$schema->create('cloud_vpc_acl_rules', static function (Blueprint $table): void {
$table->increments('id');
$table->string('rule_key', 32);
$table->unsignedInteger('vpc_id');
$table->unsignedInteger('priority')->default(1);
$table->string('direction', 8)->default('ingress');
$table->string('action', 8)->default('allow');
$table->string('protocol', 8)->default('all');
$table->string('source_cidr', 18)->default('10.0.0.0/24');
$table->string('destination_cidr', 18)->default('10.0.0.0/24');
$table->unsignedInteger('port_from')->nullable();
$table->unsignedInteger('port_to')->nullable();
$table->string('state', 16);
$table->unsignedBigInteger('desired_revision')->default(0);
$table->unsignedBigInteger('observed_revision')->default(0);
$table->string('desired_digest', 64)->nullable();
$table->string('observed_digest', 64)->nullable();
$table->string('last_error_code', 64)->nullable();
$table->unsignedBigInteger('created_at')->default(0);
$table->unsignedBigInteger('updated_at')->default(0);
});
$schema->create('cloud_vpc_host_deployments', static function (Blueprint $table): void {
$table->increments('id');
$table->unsignedInteger('vpc_id');
$table->unsignedInteger('host_id');
$table->unsignedBigInteger('fabric_revision')->default(0);
$table->string('fabric_digest', 64)->nullable();
$table->unsignedBigInteger('desired_revision')->default(0);
$table->unsignedBigInteger('observed_revision')->default(0);
$table->string('desired_digest', 64)->nullable();
$table->string('observed_digest', 64)->nullable();
$table->string('state', 16);
$table->string('last_operation_id', 32)->nullable();
$table->string('last_error_code', 64)->nullable();
$table->unsignedBigInteger('last_success_at')->default(0);
$table->unsignedBigInteger('last_failure_at')->default(0);
$table->unsignedBigInteger('created_at')->default(0);
$table->unsignedBigInteger('updated_at')->default(0);
});
$schema->create('cloud_vpc_operations', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id', 32)->unique();
$table->unsignedInteger('vpc_id');
$table->unsignedInteger('user_id');
$table->string('kind', 24);
$table->unsignedBigInteger('desired_revision');
$table->string('idempotency_key', 128);
$table->string('request_hash', 64);
$table->string('status', 16);
$table->text('payload_json');
$table->text('observed_json')->nullable();
$table->unsignedSmallInteger('evidence_version')->default(0);
$table->string('error_code', 64)->nullable();
$table->unsignedBigInteger('created_at')->default(0);
$table->unsignedBigInteger('updated_at')->default(0);
$table->unsignedBigInteger('finished_at')->nullable();
$table->string('lease_token', 32)->nullable();
$table->unsignedBigInteger('lease_until')->default(0);
$table->unsignedInteger('attempts')->default(0);
$table->unique(['vpc_id', 'idempotency_key']);
$table->unique(['vpc_id', 'desired_revision']);
});
$schema->create('cloud_private_networks', static function (Blueprint $table): void {
$table->increments('id');
$table->string('vpc_key', 32)->unique();
$table->string('allocation_key', 32)->unique();
$table->unsignedInteger('user_id');
$table->unsignedInteger('region_id');
$table->string('cidr', 18);
$table->string('active_cidr', 18)->nullable();
$table->unsignedBigInteger('network_address');
$table->unsignedBigInteger('broadcast_address');
$table->unsignedTinyInteger('prefix_length');
$table->string('state', 16);
$table->unsignedBigInteger('revision');
$table->unsignedBigInteger('released_at')->nullable();
$table->unsignedBigInteger('updated_at');
});
$schema->create('cloud_private_cidr_claims', static function (Blueprint $table): void {
$table->increments('id');
$table->string('block_cidr', 18)->unique();
$table->string('allocation_key', 32);
});
$canonicalize = new ReflectionMethod(CloudVpcService::class, 'canonicalize');
$canonicalize->setAccessible(true);
$requestHashMethod = new ReflectionMethod(CloudVpcService::class, 'requestHash');
$requestHashMethod->setAccessible(true);
$prepare = new ReflectionMethod(CloudVpcService::class, 'prepareDeleteRestoreLocked');
$prepare->setAccessible(true);
$release = new ReflectionMethod(CloudVpcService::class, 'releaseDeleteRestoreFence');
$release->setAccessible(true);
$publish = new ReflectionMethod(CloudVpcService::class, 'publishDeleteRestoreLocked');
$publish->setAccessible(true);
$resumable = new ReflectionMethod(CloudVpcService::class, 'hasResumableDeleteRestore');
$resumable->setAccessible(true);
$privateMacMethod = new ReflectionMethod(CloudVpcService::class, 'privateMacForMember');
$privateMacMethod->setAccessible(true);
$vpcKey = str_repeat('a', 32);
$sourceOperationId = str_repeat('1', 32);
$deleteOperationId = str_repeat('2', 32);
$subnetKey = str_repeat('3', 32);
$deletedSubnetKey = str_repeat('4', 32);
$allocationKey = str_repeat('5', 32);
$memberKey = str_repeat('8', 32);
$instanceUuid = '11111111-2222-4333-8444-555555555555';
$privateMac = $privateMacMethod->invoke(null, $vpcKey, $memberKey);
$now = time();
Capsule::table('cloud_regions')->insert(['id' => 1]);
$sourceDocument = $canonicalize->invoke(null, [
'schema_version' => 1,
'vpc_id' => $vpcKey,
'region' => 'hk01',
'fabric_id' => 'fabric-hk01',
'fabric_revision' => 1,
'cidr' => '10.0.0.0/24',
'desired_revision' => 1,
// This was a detach removal target. It has since reached terminal proof
// and must validate as history without being revived by restore.
'hosts' => [
['host_id' => '11', 'members' => []],
['host_id' => '2', 'members' => [$memberKey]],
],
'subnets' => [['subnet_id' => $subnetKey, 'cidr' => '10.0.0.0/24']],
'members' => [[
'member_id' => $memberKey,
'instance_uuid' => $instanceUuid,
'host_id' => '2',
'private_ip_cidr' => '10.0.0.10/32',
'subnet_id' => $subnetKey,
'private_mac' => $privateMac,
]],
'routes' => [],
'acls' => [],
]);
$sourceDigest = hash('sha256', json_encode(
$sourceDocument, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
));
$sourcePayload = $sourceDocument;
$sourcePayload['desired_digest'] = $sourceDigest;
$sourcePayload['operation_id'] = $sourceOperationId;
$deleteDocument = $canonicalize->invoke(null, [
'schema_version' => 1,
'vpc_id' => $vpcKey,
'region' => 'hk01',
'fabric_id' => 'fabric-hk01',
'fabric_revision' => 1,
'cidr' => '10.0.0.0/24',
'desired_revision' => 2,
'hosts' => [['host_id' => '2', 'members' => []]],
'subnets' => [],
'members' => [],
'routes' => [],
'acls' => [],
'retiring_members' => [[
'member_id' => $memberKey,
'instance_uuid' => $instanceUuid,
'host_id' => '2',
'private_ip_cidr' => '10.0.0.10/32',
'private_mac' => $privateMac,
]],
]);
$deleteDigest = hash('sha256', json_encode(
$deleteDocument, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
));
$deletePayload = $deleteDocument;
$deletePayload['desired_digest'] = $deleteDigest;
$deletePayload['operation_id'] = $deleteOperationId;
Capsule::table('cloud_vpcs')->insert([
'id' => 1, 'vpc_key' => $vpcKey, 'user_id' => 9, 'region_id' => 1,
'private_network_id' => 1, 'name' => 'restore-me',
'cidr' => '10.0.0.0/24', 'fabric_id' => 'fabric-hk01',
'fabric_region' => 'hk01', 'fabric_revision' => 1, 'max_members' => 200,
'state' => 'deleting', 'desired_revision' => 2, 'observed_revision' => 1,
'desired_digest' => $deleteDigest, 'observed_digest' => $sourceDigest,
'sync_state' => 'uncertain', 'last_operation_id' => $deleteOperationId,
'last_error_code' => 'vpc_result_uncertain', 'last_success_at' => 0,
'last_failure_at' => $now, 'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_vpc_subnets')->insert([
[
'id' => 1, 'subnet_key' => $subnetKey, 'vpc_id' => 1,
'cidr' => '10.0.0.0/24', 'state' => 'deleting',
'desired_revision' => 2, 'desired_digest' => $deleteDigest,
],
[
'id' => 2, 'subnet_key' => $deletedSubnetKey, 'vpc_id' => 1,
'cidr' => '10.0.1.0/24', 'state' => 'deleted',
'desired_revision' => 1, 'desired_digest' => $sourceDigest,
],
]);
Capsule::table('cloud_instances')->insert([
'id' => 20, 'uuid' => $instanceUuid, 'status' => 'active', 'host_id' => 2,
]);
Capsule::table('cloud_vpc_members')->insert([
'id' => 30, 'member_key' => $memberKey, 'vpc_id' => 1,
'subnet_id' => 1, 'private_ip_id' => 40, 'instance_id' => 20,
'host_id' => 2, 'private_ip' => '10.0.0.10', 'state' => 'deleting',
'desired_revision' => 2, 'desired_digest' => $deleteDigest,
]);
Capsule::table('cloud_vpc_private_ips')->insert([
'id' => 40, 'vpc_id' => 1, 'subnet_id' => 1, 'member_id' => 30,
'instance_id' => 20, 'private_ip' => '10.0.0.10',
'state' => 'deleting', 'desired_revision' => 2,
'desired_digest' => $deleteDigest,
]);
Capsule::table('cloud_vpc_host_deployments')->insert([
[
'vpc_id' => 1, 'host_id' => 11, 'fabric_revision' => 1,
'desired_revision' => 1, 'desired_digest' => $sourceDigest,
'state' => 'deleted', 'last_operation_id' => $sourceOperationId,
],
[
'vpc_id' => 1, 'host_id' => 2, 'fabric_revision' => 1,
'desired_revision' => 2, 'desired_digest' => $deleteDigest,
'state' => 'deleting', 'last_operation_id' => $deleteOperationId,
],
]);
Capsule::table('cloud_vpc_operations')->insert([
[
'operation_id' => $sourceOperationId, 'vpc_id' => 1, 'user_id' => 9,
'kind' => 'member_detach', 'desired_revision' => 1,
'idempotency_key' => 'source-operation',
'request_hash' => str_repeat('6', 64), 'status' => 'succeeded',
'payload_json' => json_encode($sourcePayload,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES),
'evidence_version' => 2, 'created_at' => $now, 'updated_at' => $now,
],
[
'operation_id' => $deleteOperationId, 'vpc_id' => 1, 'user_id' => 9,
'kind' => 'vpc_delete', 'desired_revision' => 2,
'idempotency_key' => 'delete-operation',
'request_hash' => str_repeat('7', 64), 'status' => 'uncertain',
'payload_json' => json_encode($deletePayload,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES),
'created_at' => $now, 'updated_at' => $now,
],
]);
Capsule::table('cloud_private_networks')->insert([
'id' => 1, 'vpc_key' => $vpcKey, 'allocation_key' => $allocationKey,
'user_id' => 9, 'region_id' => 1, 'cidr' => '10.0.0.0/24',
'active_cidr' => '10.0.0.0/24', 'network_address' => 167772160,
'broadcast_address' => 167772415, 'prefix_length' => 24,
'state' => 'active', 'revision' => 1, 'released_at' => null,
'updated_at' => $now,
]);
Capsule::table('cloud_private_cidr_claims')->insert([
'block_cidr' => '10.0.0.0/24', 'allocation_key' => $allocationKey,
]);
$actor = ['user_id' => 9, 'is_admin' => false, 'can_manage' => false];
$hash = $requestHashMethod->invoke(null, 'vpc_delete_restore', [
'confirmation_name' => 'restore-me',
'expected_revision' => 2,
'expected_operation_id' => $deleteOperationId,
]);
$operationCount = static fn (): int => (int) Capsule::table(
'cloud_vpc_operations'
)->count();
$expectNoWrite = static function (callable $callback, string $message) use (
$assert, $operationCount
): void {
$before = $operationCount();
try {
$callback();
throw new RuntimeException($message);
} catch (CloudError) {
// Expected fail-closed rejection.
}
$assert($operationCount() === $before
&& Capsule::table('cloud_vpc_operations')
->where('operation_id', str_repeat('2', 32))->value('status')
=== 'uncertain',
$message . ' caused a partial write.');
};
$expectNoWrite(static function () use (
$prepare, $actor, $vpcKey, $hash, $deleteOperationId
): void {
$prepare->invoke(null, $vpcKey, $actor, 'wrong-name',
'restore-wrong-name', $hash, 2, $deleteOperationId);
}, 'Wrong confirmation name');
$expectNoWrite(static function () use (
$prepare, $actor, $vpcKey, $hash, $deleteOperationId
): void {
$prepare->invoke(null, $vpcKey, $actor, 'restore-me',
'restore-wrong-cas', $hash, 3, $deleteOperationId);
}, 'Wrong revision CAS');
$wrongOperationId = str_repeat('9', 32);
$wrongOperationHash = $requestHashMethod->invoke(null, 'vpc_delete_restore', [
'confirmation_name' => 'restore-me',
'expected_revision' => 2,
'expected_operation_id' => $wrongOperationId,
]);
$expectNoWrite(static function () use (
$prepare, $actor, $vpcKey, $wrongOperationHash, $wrongOperationId
): void {
$prepare->invoke(null, $vpcKey, $actor, 'restore-me',
'restore-wrong-operation', $wrongOperationHash, 2, $wrongOperationId);
}, 'Wrong operation CAS');
try {
CloudVpcService::restoreDelete($vpcKey, [
'user_id' => 10, 'is_admin' => false, 'can_manage' => false,
], [
'confirmation_name' => 'restore-me',
'request_key' => 'restore-permission',
'expected_revision' => 2,
'expected_operation_id' => $deleteOperationId,
]);
throw new RuntimeException('A non-owner restored the delete.');
} catch (CloudError $error) {
$assert($error->httpStatus === 404 && $operationCount() === 2,
'Permission rejection wrote state or returned the wrong status.');
}
Capsule::table('cloud_vpc_operations')->where('operation_id', $sourceOperationId)
->update(['desired_revision' => 0]);
$expectNoWrite(static function () use (
$prepare, $actor, $vpcKey, $hash, $deleteOperationId
): void {
$prepare->invoke(null, $vpcKey, $actor, 'restore-me',
'restore-missing-adjacent-source', $hash, 2, $deleteOperationId);
}, 'Missing adjacent source revision');
Capsule::table('cloud_vpc_operations')->where('operation_id', $sourceOperationId)
->update(['desired_revision' => 1]);
Capsule::table('cloud_vpc_operations')->where('operation_id', $sourceOperationId)
->update(['payload_json' => '{}']);
$expectNoWrite(static function () use (
$prepare, $actor, $vpcKey, $hash, $deleteOperationId
): void {
$prepare->invoke(null, $vpcKey, $actor, 'restore-me',
'restore-corrupt', $hash, 2, $deleteOperationId);
}, 'Corrupt source payload');
Capsule::table('cloud_vpc_operations')->where('operation_id', $sourceOperationId)
->update(['payload_json' => json_encode($sourcePayload,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES)]);
Capsule::table('cloud_vpc_subnets')->where('id', 1)
->update(['state' => 'deleted']);
$expectNoWrite(static function () use (
$prepare, $actor, $vpcKey, $hash, $deleteOperationId
): void {
$prepare->invoke(null, $vpcKey, $actor, 'restore-me',
'restore-terminal-source', $hash, 2, $deleteOperationId);
}, 'Terminal selected source identity');
Capsule::table('cloud_vpc_subnets')->where('id', 1)
->update(['state' => 'deleting']);
// A deleted label without the matching observed revision/digest is not a
// terminal absence proof and must not silently shrink the engine host fence.
$expectNoWrite(static function () use (
$prepare, $actor, $vpcKey, $hash, $deleteOperationId
): void {
$prepare->invoke(null, $vpcKey, $actor, 'restore-me',
'restore-incomplete-host-proof', $hash, 2, $deleteOperationId);
}, 'Incomplete terminal empty-host proof');
Capsule::table('cloud_vpc_host_deployments')->where('host_id', 11)->update([
'observed_revision' => 1,
'observed_digest' => $sourceDigest,
'last_error_code' => null,
'last_success_at' => $now,
]);
$prepared = $prepare->invoke(null, $vpcKey, $actor, 'restore-me',
'restore-request-one', $hash, 2, $deleteOperationId);
$restoreOperationId = (string) $prepared['operation']->operation_id;
$restorePayload = $prepared['payload'];
$assert(($restorePayload['hosts'] ?? null) === [[
'host_id' => '2', 'members' => [$memberKey],
]]
&& ($restorePayload['members'][0]['member_id'] ?? null) === $memberKey
&& ($restorePayload['members'][0]['private_ip_cidr'] ?? null)
=== '10.0.0.10/32'
&& ($restorePayload['members'][0]['private_mac'] ?? null) === $privateMac
&& Capsule::table('cloud_vpc_host_deployments')
->where('host_id', 11)->value('state') === 'deleted',
'Restore payload lost the active member or revived a terminal empty-host tombstone.');
$assert(Capsule::table('cloud_vpcs')->where('id', 1)->value('state') === 'deleting'
&& (int) Capsule::table('cloud_vpcs')->where('id', 1)
->value('desired_revision') === 2
&& Capsule::table('cloud_vpc_subnets')->where('id', 1)
->value('state') === 'deleting'
&& Capsule::table('cloud_vpc_members')->where('id', 30)
->value('state') === 'deleting'
&& Capsule::table('cloud_vpc_private_ips')->where('id', 40)
->value('state') === 'deleting'
&& Capsule::table('cloud_vpc_host_deployments')->where('host_id', 2)
->value('state') === 'deleting'
&& Capsule::table('cloud_vpc_operations')
->where('operation_id', $deleteOperationId)->value('status') === 'fenced'
&& Capsule::table('cloud_vpc_operations')
->where('operation_id', $restoreOperationId)->value('status') === 'fencing',
'Phase one changed root/children or failed to fence both operations atomically.');
$assert($resumable->invoke(
null,
CloudVpc::findOrFail(1),
CloudVpcOperation::where('operation_id', $deleteOperationId)->firstOrFail()
) === true,
'A phase-one crash was not exposed as safely resumable.');
// A deterministic engine rejection releases the old delete and leaves the
// restore row terminal/non-claimable.
$release->invoke(null, 1, $restoreOperationId,
new CloudError('conflict', 409, 'vpc_delete_restore_conflict'));
$assert(Capsule::table('cloud_vpc_operations')
->where('operation_id', $deleteOperationId)->value('status') === 'uncertain'
&& Capsule::table('cloud_vpc_operations')
->where('operation_id', $restoreOperationId)->value('status') === 'failed',
'Deterministic engine failure did not reopen the exact old delete lane.');
// A new browser request key must take over the same reserved operation.
$retry = $prepare->invoke(null, $vpcKey, $actor, 'restore-me',
'restore-request-two', $hash, 2, $deleteOperationId);
$assert((string) $retry['operation']->operation_id === $restoreOperationId
&& $operationCount() === 3,
'New request key created a second restore operation instead of takeover.');
$release->invoke(null, 1, $restoreOperationId,
new CloudError('timeout', 502));
$assert(Capsule::table('cloud_vpc_operations')
->where('operation_id', $deleteOperationId)->value('status') === 'uncertain'
&& Capsule::table('cloud_vpc_operations')
->where('operation_id', $restoreOperationId)
->value('status') === 'fence_uncertain',
'Ambiguous transport failure was claimable or left the old delete fenced.');
$acceptedReplay = $prepare->invoke(null, $vpcKey, $actor, 'restore-me',
'restore-request-three', $hash, 2, $deleteOperationId);
$assert((string) $acceptedReplay['operation']->operation_id
=== $restoreOperationId
&& $operationCount() === 3,
'Engine-accepted replay did not retain one operation identity.');
$restorePayload = $acceptedReplay['payload'];
$engineResult = [
'vpc_id' => $vpcKey,
'operation_id' => $restoreOperationId,
'desired_revision' => 3,
'desired_digest' => $restorePayload['desired_digest'],
'state' => 'applying',
'observed' => [],
'hosts' => [],
];
$publish->invoke(null, 1, $restoreOperationId, $restorePayload,
$engineResult, 2, $deleteOperationId);
$storedRestorePayload = json_decode((string) Capsule::table(
'cloud_vpc_operations'
)->where('operation_id', $restoreOperationId)->value('payload_json'), true);
$assert(Capsule::table('cloud_vpcs')->where('id', 1)->value('state') === 'applying'
&& (int) Capsule::table('cloud_vpcs')->where('id', 1)
->value('desired_revision') === 3
&& Capsule::table('cloud_vpcs')->where('id', 1)
->value('last_operation_id') === $restoreOperationId
&& Capsule::table('cloud_vpc_subnets')->where('id', 1)
->value('state') === 'applying'
&& (int) Capsule::table('cloud_vpc_members')->where('id', 30)
->value('desired_revision') === 3
&& Capsule::table('cloud_vpc_members')->where('id', 30)
->value('state') === 'applying'
&& Capsule::table('cloud_vpc_members')->where('id', 30)
->value('desired_digest') === $restorePayload['desired_digest']
&& Capsule::table('cloud_vpc_members')->where('id', 30)
->value('last_operation_id') === $restoreOperationId
&& (int) Capsule::table('cloud_vpc_private_ips')->where('id', 40)
->value('desired_revision') === 3
&& Capsule::table('cloud_vpc_private_ips')->where('id', 40)
->value('state') === 'applying'
&& Capsule::table('cloud_vpc_private_ips')->where('id', 40)
->value('desired_digest') === $restorePayload['desired_digest']
&& Capsule::table('cloud_vpc_subnets')->where('id', 2)
->value('state') === 'deleted'
&& (int) Capsule::table('cloud_vpc_subnets')->where('id', 2)
->value('desired_revision') === 1
&& (int) Capsule::table('cloud_vpc_host_deployments')->where('host_id', 2)
->value('desired_revision') === 3
&& Capsule::table('cloud_vpc_host_deployments')->where('host_id', 2)
->value('state') === 'applying'
&& Capsule::table('cloud_vpc_host_deployments')->where('host_id', 2)
->value('desired_digest') === $restorePayload['desired_digest']
&& Capsule::table('cloud_vpc_host_deployments')->where('host_id', 2)
->value('last_operation_id') === $restoreOperationId
&& Capsule::table('cloud_vpc_host_deployments')->where('host_id', 11)
->value('state') === 'deleted'
&& (int) Capsule::table('cloud_vpc_host_deployments')->where('host_id', 11)
->value('desired_revision') === 1
&& Capsule::table('cloud_vpc_host_deployments')->where('host_id', 11)
->value('last_operation_id') === $sourceOperationId
&& Capsule::table('cloud_vpc_operations')
->where('operation_id', $deleteOperationId)->value('status') === 'superseded'
&& Capsule::table('cloud_vpc_operations')
->where('operation_id', $restoreOperationId)->value('status') === 'queued'
&& !array_key_exists('cancelled_delete', $storedRestorePayload),
'Engine success did not publish exactly the verified normal target.');
$assert(Capsule::table('cloud_private_networks')->where('id', 1)
->value('state') === 'active'
&& Capsule::table('cloud_private_networks')->where('id', 1)
->value('active_cidr') === '10.0.0.0/24'
&& Capsule::table('cloud_private_cidr_claims')->count() === 1,
'Delete restore released the CIDR reservation.');
try {
CloudVpcService::recordDeleted($vpcKey, 2, $deleteDigest, [
'vpc_id' => $vpcKey, 'operation_id' => $deleteOperationId,
'desired_revision' => 2, 'desired_digest' => $deleteDigest,
'deleted' => true, 'state' => 'deleted', 'readback' => 'confirmed',
'removed_host_ids' => [], 'completed_at' => time(),
], $deleteOperationId);
throw new RuntimeException('A late old delete receipt won after restore publish.');
} catch (CloudError $error) {
$assert($error->httpStatus === 409
&& Capsule::table('cloud_vpcs')->where('id', 1)->value('state')
=== 'applying',
'Late delete receipt did not fail the root revision/operation CAS.');
}
// A completed phase two remains idempotent even after the ordinary apply lane
// fails or a later mutation supersedes it. The immutable normal payload plus
// the exact superseded delete row prove publish without trusting root state.
Capsule::table('cloud_vpc_operations')
->where('operation_id', $restoreOperationId)
->update(['status' => 'failed', 'error_code' => 'vpc_observation_incomplete']);
Capsule::table('cloud_vpcs')->where('id', 1)->update([
'state' => 'degraded', 'sync_state' => 'failed',
'last_error_code' => 'vpc_observation_incomplete',
]);
$failedReplay = $prepare->invoke(null, $vpcKey, $actor, 'restore-me',
'restore-request-one', $hash, 2, $deleteOperationId);
$assert(($failedReplay['published'] ?? false) === true
&& (string) $failedReplay['operation']->operation_id === $restoreOperationId
&& $operationCount() === 3,
'Published restore lost idempotence after ordinary apply failed.');
Capsule::table('cloud_vpc_operations')
->where('operation_id', $restoreOperationId)
->update(['status' => 'superseded']);
Capsule::table('cloud_vpcs')->where('id', 1)->update([
'state' => 'applying', 'desired_revision' => 4,
'last_operation_id' => str_repeat('f', 32),
]);
$supersededReplay = $prepare->invoke(null, $vpcKey, $actor, 'restore-me',
'restore-request-after-reload', $hash, 2, $deleteOperationId);
$assert(($supersededReplay['published'] ?? false) === true
&& (string) $supersededReplay['operation']->operation_id
=== $restoreOperationId
&& $operationCount() === 3,
'Reload with a new key did not recognize a previously published restore.');
echo "VPC delete restore workflow contracts passed.\n";