You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudVpcMutationAdmissionTe...

265 lines
14 KiB
PHP

<?php
use App\Services\Cloud\CloudVpcService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\QueryException;
// Reuse the real SQLite/Eloquent fixture, including its lifecycle proofs.
require __DIR__ . '/CloudVpcMemberReuseTest.php';
if (!isset($capsule)) {
return;
}
$admissionVpc = $seedVpc(901, '10.90.1.0/24', '10.90.1.0/24', 10);
$seedInstance(90101, 10);
$input = ['instance_id' => 90101, 'request_key' => 'admission-attach-90101'];
foreach (['queued', 'running', 'uncertain'] as $status) {
Capsule::table('cloud_jobs')->insert([
'instance_id' => 90101, 'kind' => 'power', 'status' => $status,
]);
$error = $expectCloudError(static fn () => CloudVpcService::attachMember(
$admissionVpc['vpc_key'], $actor, $input
), 409);
$assert($error->errorCode === 'vpc_instance_operation_pending',
'The active instance task must be reported as a precise admission conflict.');
$assert((int) Capsule::table('cloud_vpcs')->where('id', 901)
->value('desired_revision') === 1
&& Capsule::table('cloud_vpc_members')->where('vpc_id', 901)->count() === 0
&& Capsule::table('cloud_vpc_private_ips')->where('vpc_id', 901)->count() === 0,
'Rejected attachment changed its revision or reserved a private address.');
Capsule::table('cloud_jobs')->where('instance_id', 90101)->delete();
}
CloudVpcService::attachMember($admissionVpc['vpc_key'], $actor, $input);
$memberKey = Capsule::table('cloud_vpc_members')->where('instance_id', 90101)
->value('member_key');
Capsule::table('cloud_jobs')->insert([
'instance_id' => 90101, 'kind' => 'power', 'status' => 'uncertain',
]);
// A lost HTTP response may be retried without changing the desired target.
CloudVpcService::attachMember($admissionVpc['vpc_key'], $actor, $input);
$seedInstance(90102, 10);
$mutations = [
static fn () => CloudVpcService::attachMember($admissionVpc['vpc_key'], $actor, [
'instance_id' => 90102, 'request_key' => 'admission-attach-90102',
]),
static fn () => CloudVpcService::detachMember($admissionVpc['vpc_key'],
$memberKey, $actor, ['request_key' => 'admission-detach-90101']),
static fn () => CloudVpcService::replaceRoutes($admissionVpc['vpc_key'], $actor,
['routes' => [], 'request_key' => 'admission-routes-90101']),
static fn () => CloudVpcService::replaceAcl($admissionVpc['vpc_key'], $actor,
['rules' => [], 'request_key' => 'admission-acl-90101']),
static fn () => CloudVpcService::delete($admissionVpc['vpc_key'], $actor,
['confirmation_name' => 'reuse-vpc-901', 'request_key' => 'admission-delete-90101']),
];
foreach ($mutations as $mutation) {
$error = $expectCloudError($mutation, 409);
$assert($error->errorCode === 'vpc_instance_operation_pending',
'A VPC document change bypassed the existing member task.');
$assert((int) Capsule::table('cloud_vpcs')->where('id', 901)
->value('desired_revision') === 2
&& Capsule::table('cloud_vpc_members')->where('instance_id', 90101)
->value('state') === 'applying'
&& Capsule::table('cloud_vpc_private_ips')->where('instance_id', 90101)
->value('state') === 'applying'
&& Capsule::table('cloud_vpc_operations')->where('vpc_id', 901)->count() === 2,
'The admission conflict failed to roll back member/IP/root/operation changes.');
}
Capsule::table('cloud_jobs')->where('instance_id', 90101)->update(['status' => 'succeeded']);
// An unrelated task must not serialize independent VPCs or instances.
Capsule::table('cloud_jobs')->insert([
'instance_id' => 99999, 'kind' => 'power', 'status' => 'running',
]);
CloudVpcService::detachMember($admissionVpc['vpc_key'], $memberKey, $actor,
['request_key' => 'admission-detach-90101']);
$assert((int) Capsule::table('cloud_vpcs')->where('id', 901)
->value('desired_revision') === 3
&& Capsule::table('cloud_vpc_members')->where('instance_id', 90101)
->value('state') === 'deleting'
&& Capsule::table('cloud_vpc_private_ips')->where('instance_id', 90101)
->value('state') === 'deleting',
'A completed task did not admit detach or detach released its address prematurely.');
// Execute real mutation transactions while injecting the database's NOWAIT
// refusal at the task read. SQLite cannot reproduce MariaDB row contention;
// retain its transaction engine and compile only this lock clause for capture.
$lockVpc = $seedVpc(902, '10.90.2.0/24', '10.90.2.0/24', 10);
$seedInstance(90201, 10);
$connection = $capsule->getConnection('default');
$originalGrammar = $connection->getQueryGrammar();
$configProperty = new ReflectionProperty($connection, 'config');
$originalConfig = $configProperty->getValue($connection);
$fault = null;
$seenTaskSql = [];
$connection->beforeExecuting(static function ($sql, $bindings) use (
&$fault, &$seenTaskSql, $assert
): void {
if ($fault === null || preg_match('/\bfrom\s+["`]?cloud_jobs["`]?(?:\s|$)/i', $sql) !== 1
|| !str_starts_with(strtolower($sql), 'select')) {
return;
}
$seenTaskSql[] = $sql;
$assert(str_ends_with($sql, 'FOR UPDATE NOWAIT')
&& !str_contains($sql, 'SKIP LOCKED'),
'Task admission waited on the worker lock or skipped a locked task.');
$previous = new PDOException('Synthetic internal database diagnostic');
$previous->errorInfo = [$fault === 1213 ? '40001' : 'HY000', $fault, 'private'];
throw new QueryException('default', $sql, $bindings, $previous);
});
$connection->setQueryGrammar(new class($connection) extends
\Illuminate\Database\Query\Grammars\SQLiteGrammar {
protected function compileLock(\Illuminate\Database\Query\Builder $query, $value)
{
// This fixture injects contention only for cloud_jobs. Other current
// reads (including the new cross-VPC workflow fence) use SQLite locks.
return $query->from === 'cloud_jobs' && is_string($value) ? $value : '';
}
});
$configProperty->setValue($connection, array_replace($originalConfig, ['driver' => 'mysql']));
try {
foreach ([1205, 1213, 3572, 1064, 2006] as $driverCode) {
$fault = $driverCode;
$beforeReads = count($seenTaskSql);
$conflict = in_array($driverCode, [1205, 1213, 3572], true);
$error = $expectCloudError(static fn () => CloudVpcService::attachMember(
$lockVpc['vpc_key'], $actor, [
'instance_id' => 90201, 'request_key' => 'admission-nowait-90201',
]
), $conflict ? 409 : 503);
$assert($error->errorCode === ($conflict ? 'vpc_instance_operation_pending'
: 'vpc_instance_operation_unavailable')
&& !str_contains($error->getMessage(), 'private')
&& count($seenTaskSql) === $beforeReads + 1
&& (int) Capsule::table('cloud_vpcs')->where('id', 902)
->value('desired_revision') === 1
&& Capsule::table('cloud_vpc_members')->where('vpc_id', 902)->count() === 0
&& Capsule::table('cloud_vpc_private_ips')->where('vpc_id', 902)->count() === 0
&& Capsule::table('cloud_vpc_operations')->where('vpc_id', 902)->count() === 1,
'A task-lock refusal retried without protection or leaked partial VPC/IP changes.');
}
} finally {
$fault = null;
$configProperty->setValue($connection, $originalConfig);
$connection->setQueryGrammar($originalGrammar);
}
$mysqlBuilder = new \Illuminate\Database\Query\Builder($connection,
new \Illuminate\Database\Query\Grammars\MySqlGrammar($connection));
$assert(str_ends_with($mysqlBuilder->from('cloud_jobs')->where('instance_id', 90201)
->lock('FOR UPDATE NOWAIT')->toSql(), 'FOR UPDATE NOWAIT'),
'The production MariaDB/MySQL grammar discarded the no-wait current read.');
CloudVpcService::attachMember($lockVpc['vpc_key'], $actor, [
'instance_id' => 90201, 'request_key' => 'admission-nowait-90201',
]);
$assert((int) Capsule::table('cloud_vpcs')->where('id', 902)
->value('desired_revision') === 2,
'A rolled-back lock conflict prevented a later safe retry.');
// Root deletion is also a member mutation. It must capture the complete
// member identity set, prove every runtime freshly stopped in one batch, then
// re-lock and compare the same set before it persists any tombstone.
$deleteVpc = $seedVpc(903, '10.90.3.0/24', '10.90.3.0/24', 10);
$seedInstance(90301, 10);
$seedInstance(90302, 10);
CloudVpcService::attachMember($deleteVpc['vpc_key'], $actor, [
'instance_id' => 90301, 'request_key' => 'delete-attach-90301',
]);
CloudVpcService::attachMember($deleteVpc['vpc_key'], $actor, [
'instance_id' => 90302, 'request_key' => 'delete-attach-90302',
]);
$deleteInput = [
'confirmation_name' => 'reuse-vpc-903',
'request_key' => 'delete-stopped-members-903',
];
$deleteRevision = (int) Capsule::table('cloud_vpcs')->where('id', 903)
->value('desired_revision');
$deleteOperationCount = Capsule::table('cloud_vpc_operations')
->where('vpc_id', 903)->count();
Capsule::table('cloud_instance_runtime')->where('instance_id', 90302)->update([
'power_state' => 'running', 'sync_state' => 'ok',
'last_success_at' => time(), 'updated_at' => time(),
]);
$runningDelete = $expectCloudError(static fn () => CloudVpcService::delete(
$deleteVpc['vpc_key'], $actor, $deleteInput
), 409);
$assert($runningDelete->errorCode === 'vpc_member_requires_stopped'
&& (string) Capsule::table('cloud_vpcs')->where('id', 903)->value('state')
!== 'deleting'
&& (int) Capsule::table('cloud_vpcs')->where('id', 903)
->value('desired_revision') === $deleteRevision
&& Capsule::table('cloud_vpc_operations')->where('vpc_id', 903)->count()
=== $deleteOperationCount
&& Capsule::table('cloud_vpc_members')->where('vpc_id', 903)
->where('state', 'deleting')->count() === 0,
'A running member allowed root-delete intent to change the durable ledger.');
Capsule::table('cloud_instance_runtime')->where('instance_id', 90302)->update([
'power_state' => 'shutoff', 'sync_state' => 'ok',
'last_success_at' => time() - 60, 'updated_at' => time(),
]);
$staleDelete = $expectCloudError(static fn () => CloudVpcService::delete(
$deleteVpc['vpc_key'], $actor, $deleteInput
), 409);
$assert($staleDelete->errorCode === 'vpc_member_power_unavailable'
&& (int) Capsule::table('cloud_vpcs')->where('id', 903)
->value('desired_revision') === $deleteRevision
&& Capsule::table('cloud_vpc_operations')->where('vpc_id', 903)->count()
=== $deleteOperationCount,
'A stale member power observation allowed root deletion to be persisted.');
Capsule::table('cloud_instance_runtime')->whereIn('instance_id', [90301, 90302])
->update([
'power_state' => 'shutoff', 'sync_state' => 'ok',
'last_success_at' => time(), 'updated_at' => time(),
]);
$deleteSnapshotMutationArmed = true;
$deleteSnapshotMutationSeen = false;
$connection->beforeExecuting(static function ($sql) use (
&$deleteSnapshotMutationArmed, &$deleteSnapshotMutationSeen
): void {
if (!$deleteSnapshotMutationArmed
|| !str_starts_with(strtolower(ltrim($sql)), 'select')
|| preg_match('/\bfrom\s+["`]?cloud_instance_runtime["`]?(?:\s|$)/i', $sql) !== 1) {
return;
}
// Simulate a concurrent member transition after the first locked closure
// has committed and while the out-of-lock batch power read is starting.
$deleteSnapshotMutationArmed = false;
Capsule::table('cloud_vpc_members')->where('vpc_id', 903)
->where('instance_id', 90302)->update(['state' => 'active']);
$deleteSnapshotMutationSeen = true;
});
$changedClosureInput = [
'confirmation_name' => 'reuse-vpc-903',
'request_key' => 'delete-changed-closure-903',
];
$changedClosure = $expectCloudError(static fn () => CloudVpcService::delete(
$deleteVpc['vpc_key'], $actor, $changedClosureInput
), 409);
$assert($deleteSnapshotMutationSeen
&& $changedClosure->errorCode === 'vpc_member_power_unavailable'
&& (int) Capsule::table('cloud_vpcs')->where('id', 903)
->value('desired_revision') === $deleteRevision
&& Capsule::table('cloud_vpc_operations')->where('vpc_id', 903)->count()
=== $deleteOperationCount
&& Capsule::table('cloud_vpc_operations')->where('vpc_id', 903)
->where('idempotency_key', 'delete-changed-closure-903')->count() === 0,
'A member identity transition between delete phases created an operation.');
Capsule::table('cloud_vpc_members')->where('vpc_id', 903)
->where('instance_id', 90302)->update(['state' => 'applying']);
CloudVpcService::delete($deleteVpc['vpc_key'], $actor, $deleteInput);
$assert((string) Capsule::table('cloud_vpcs')->where('id', 903)->value('state')
=== 'deleting'
&& (int) Capsule::table('cloud_vpcs')->where('id', 903)
->value('desired_revision') === $deleteRevision + 1
&& Capsule::table('cloud_vpc_members')->where('vpc_id', 903)
->where('state', 'deleting')->count() === 2,
'A fully stopped member closure did not admit one root-delete revision.');
Capsule::table('cloud_instance_runtime')->where('instance_id', 90301)->update([
'power_state' => 'running', 'last_success_at' => time(), 'updated_at' => time(),
]);
CloudVpcService::delete($deleteVpc['vpc_key'], $actor, $deleteInput);
$assert((int) Capsule::table('cloud_vpcs')->where('id', 903)
->value('desired_revision') === $deleteRevision + 1
&& Capsule::table('cloud_vpc_operations')->where('vpc_id', 903)->count()
=== $deleteOperationCount + 1,
'An exact root-delete replay rechecked power or created a second operation.');
echo "VPC instance-task admission database regressions passed.\n";