You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
265 lines
14 KiB
PHP
265 lines
14 KiB
PHP
<?php
|
|
|
|
use App\Services\Cloud\CloudVpcService;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\QueryException;
|
|
|
|
// Reuse the real SQLite/Eloquent fixture, including its lifecycle proofs.
|
|
require __DIR__ . '/CloudVpcMemberReuseTest.php';
|
|
if (!isset($capsule)) {
|
|
return;
|
|
}
|
|
|
|
$admissionVpc = $seedVpc(901, '10.90.1.0/24', '10.90.1.0/24', 10);
|
|
$seedInstance(90101, 10);
|
|
$input = ['instance_id' => 90101, 'request_key' => 'admission-attach-90101'];
|
|
foreach (['queued', 'running', 'uncertain'] as $status) {
|
|
Capsule::table('cloud_jobs')->insert([
|
|
'instance_id' => 90101, 'kind' => 'power', 'status' => $status,
|
|
]);
|
|
$error = $expectCloudError(static fn () => CloudVpcService::attachMember(
|
|
$admissionVpc['vpc_key'], $actor, $input
|
|
), 409);
|
|
$assert($error->errorCode === 'vpc_instance_operation_pending',
|
|
'The active instance task must be reported as a precise admission conflict.');
|
|
$assert((int) Capsule::table('cloud_vpcs')->where('id', 901)
|
|
->value('desired_revision') === 1
|
|
&& Capsule::table('cloud_vpc_members')->where('vpc_id', 901)->count() === 0
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('vpc_id', 901)->count() === 0,
|
|
'Rejected attachment changed its revision or reserved a private address.');
|
|
Capsule::table('cloud_jobs')->where('instance_id', 90101)->delete();
|
|
}
|
|
CloudVpcService::attachMember($admissionVpc['vpc_key'], $actor, $input);
|
|
$memberKey = Capsule::table('cloud_vpc_members')->where('instance_id', 90101)
|
|
->value('member_key');
|
|
Capsule::table('cloud_jobs')->insert([
|
|
'instance_id' => 90101, 'kind' => 'power', 'status' => 'uncertain',
|
|
]);
|
|
// A lost HTTP response may be retried without changing the desired target.
|
|
CloudVpcService::attachMember($admissionVpc['vpc_key'], $actor, $input);
|
|
$seedInstance(90102, 10);
|
|
$mutations = [
|
|
static fn () => CloudVpcService::attachMember($admissionVpc['vpc_key'], $actor, [
|
|
'instance_id' => 90102, 'request_key' => 'admission-attach-90102',
|
|
]),
|
|
static fn () => CloudVpcService::detachMember($admissionVpc['vpc_key'],
|
|
$memberKey, $actor, ['request_key' => 'admission-detach-90101']),
|
|
static fn () => CloudVpcService::replaceRoutes($admissionVpc['vpc_key'], $actor,
|
|
['routes' => [], 'request_key' => 'admission-routes-90101']),
|
|
static fn () => CloudVpcService::replaceAcl($admissionVpc['vpc_key'], $actor,
|
|
['rules' => [], 'request_key' => 'admission-acl-90101']),
|
|
static fn () => CloudVpcService::delete($admissionVpc['vpc_key'], $actor,
|
|
['confirmation_name' => 'reuse-vpc-901', 'request_key' => 'admission-delete-90101']),
|
|
];
|
|
foreach ($mutations as $mutation) {
|
|
$error = $expectCloudError($mutation, 409);
|
|
$assert($error->errorCode === 'vpc_instance_operation_pending',
|
|
'A VPC document change bypassed the existing member task.');
|
|
$assert((int) Capsule::table('cloud_vpcs')->where('id', 901)
|
|
->value('desired_revision') === 2
|
|
&& Capsule::table('cloud_vpc_members')->where('instance_id', 90101)
|
|
->value('state') === 'applying'
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('instance_id', 90101)
|
|
->value('state') === 'applying'
|
|
&& Capsule::table('cloud_vpc_operations')->where('vpc_id', 901)->count() === 2,
|
|
'The admission conflict failed to roll back member/IP/root/operation changes.');
|
|
}
|
|
Capsule::table('cloud_jobs')->where('instance_id', 90101)->update(['status' => 'succeeded']);
|
|
// An unrelated task must not serialize independent VPCs or instances.
|
|
Capsule::table('cloud_jobs')->insert([
|
|
'instance_id' => 99999, 'kind' => 'power', 'status' => 'running',
|
|
]);
|
|
CloudVpcService::detachMember($admissionVpc['vpc_key'], $memberKey, $actor,
|
|
['request_key' => 'admission-detach-90101']);
|
|
$assert((int) Capsule::table('cloud_vpcs')->where('id', 901)
|
|
->value('desired_revision') === 3
|
|
&& Capsule::table('cloud_vpc_members')->where('instance_id', 90101)
|
|
->value('state') === 'deleting'
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('instance_id', 90101)
|
|
->value('state') === 'deleting',
|
|
'A completed task did not admit detach or detach released its address prematurely.');
|
|
|
|
// Execute real mutation transactions while injecting the database's NOWAIT
|
|
// refusal at the task read. SQLite cannot reproduce MariaDB row contention;
|
|
// retain its transaction engine and compile only this lock clause for capture.
|
|
$lockVpc = $seedVpc(902, '10.90.2.0/24', '10.90.2.0/24', 10);
|
|
$seedInstance(90201, 10);
|
|
$connection = $capsule->getConnection('default');
|
|
$originalGrammar = $connection->getQueryGrammar();
|
|
$configProperty = new ReflectionProperty($connection, 'config');
|
|
$originalConfig = $configProperty->getValue($connection);
|
|
$fault = null;
|
|
$seenTaskSql = [];
|
|
$connection->beforeExecuting(static function ($sql, $bindings) use (
|
|
&$fault, &$seenTaskSql, $assert
|
|
): void {
|
|
if ($fault === null || preg_match('/\bfrom\s+["`]?cloud_jobs["`]?(?:\s|$)/i', $sql) !== 1
|
|
|| !str_starts_with(strtolower($sql), 'select')) {
|
|
return;
|
|
}
|
|
$seenTaskSql[] = $sql;
|
|
$assert(str_ends_with($sql, 'FOR UPDATE NOWAIT')
|
|
&& !str_contains($sql, 'SKIP LOCKED'),
|
|
'Task admission waited on the worker lock or skipped a locked task.');
|
|
$previous = new PDOException('Synthetic internal database diagnostic');
|
|
$previous->errorInfo = [$fault === 1213 ? '40001' : 'HY000', $fault, 'private'];
|
|
throw new QueryException('default', $sql, $bindings, $previous);
|
|
});
|
|
$connection->setQueryGrammar(new class($connection) extends
|
|
\Illuminate\Database\Query\Grammars\SQLiteGrammar {
|
|
protected function compileLock(\Illuminate\Database\Query\Builder $query, $value)
|
|
{
|
|
// This fixture injects contention only for cloud_jobs. Other current
|
|
// reads (including the new cross-VPC workflow fence) use SQLite locks.
|
|
return $query->from === 'cloud_jobs' && is_string($value) ? $value : '';
|
|
}
|
|
});
|
|
$configProperty->setValue($connection, array_replace($originalConfig, ['driver' => 'mysql']));
|
|
try {
|
|
foreach ([1205, 1213, 3572, 1064, 2006] as $driverCode) {
|
|
$fault = $driverCode;
|
|
$beforeReads = count($seenTaskSql);
|
|
$conflict = in_array($driverCode, [1205, 1213, 3572], true);
|
|
$error = $expectCloudError(static fn () => CloudVpcService::attachMember(
|
|
$lockVpc['vpc_key'], $actor, [
|
|
'instance_id' => 90201, 'request_key' => 'admission-nowait-90201',
|
|
]
|
|
), $conflict ? 409 : 503);
|
|
$assert($error->errorCode === ($conflict ? 'vpc_instance_operation_pending'
|
|
: 'vpc_instance_operation_unavailable')
|
|
&& !str_contains($error->getMessage(), 'private')
|
|
&& count($seenTaskSql) === $beforeReads + 1
|
|
&& (int) Capsule::table('cloud_vpcs')->where('id', 902)
|
|
->value('desired_revision') === 1
|
|
&& Capsule::table('cloud_vpc_members')->where('vpc_id', 902)->count() === 0
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('vpc_id', 902)->count() === 0
|
|
&& Capsule::table('cloud_vpc_operations')->where('vpc_id', 902)->count() === 1,
|
|
'A task-lock refusal retried without protection or leaked partial VPC/IP changes.');
|
|
}
|
|
} finally {
|
|
$fault = null;
|
|
$configProperty->setValue($connection, $originalConfig);
|
|
$connection->setQueryGrammar($originalGrammar);
|
|
}
|
|
$mysqlBuilder = new \Illuminate\Database\Query\Builder($connection,
|
|
new \Illuminate\Database\Query\Grammars\MySqlGrammar($connection));
|
|
$assert(str_ends_with($mysqlBuilder->from('cloud_jobs')->where('instance_id', 90201)
|
|
->lock('FOR UPDATE NOWAIT')->toSql(), 'FOR UPDATE NOWAIT'),
|
|
'The production MariaDB/MySQL grammar discarded the no-wait current read.');
|
|
CloudVpcService::attachMember($lockVpc['vpc_key'], $actor, [
|
|
'instance_id' => 90201, 'request_key' => 'admission-nowait-90201',
|
|
]);
|
|
$assert((int) Capsule::table('cloud_vpcs')->where('id', 902)
|
|
->value('desired_revision') === 2,
|
|
'A rolled-back lock conflict prevented a later safe retry.');
|
|
|
|
// Root deletion is also a member mutation. It must capture the complete
|
|
// member identity set, prove every runtime freshly stopped in one batch, then
|
|
// re-lock and compare the same set before it persists any tombstone.
|
|
$deleteVpc = $seedVpc(903, '10.90.3.0/24', '10.90.3.0/24', 10);
|
|
$seedInstance(90301, 10);
|
|
$seedInstance(90302, 10);
|
|
CloudVpcService::attachMember($deleteVpc['vpc_key'], $actor, [
|
|
'instance_id' => 90301, 'request_key' => 'delete-attach-90301',
|
|
]);
|
|
CloudVpcService::attachMember($deleteVpc['vpc_key'], $actor, [
|
|
'instance_id' => 90302, 'request_key' => 'delete-attach-90302',
|
|
]);
|
|
$deleteInput = [
|
|
'confirmation_name' => 'reuse-vpc-903',
|
|
'request_key' => 'delete-stopped-members-903',
|
|
];
|
|
$deleteRevision = (int) Capsule::table('cloud_vpcs')->where('id', 903)
|
|
->value('desired_revision');
|
|
$deleteOperationCount = Capsule::table('cloud_vpc_operations')
|
|
->where('vpc_id', 903)->count();
|
|
Capsule::table('cloud_instance_runtime')->where('instance_id', 90302)->update([
|
|
'power_state' => 'running', 'sync_state' => 'ok',
|
|
'last_success_at' => time(), 'updated_at' => time(),
|
|
]);
|
|
$runningDelete = $expectCloudError(static fn () => CloudVpcService::delete(
|
|
$deleteVpc['vpc_key'], $actor, $deleteInput
|
|
), 409);
|
|
$assert($runningDelete->errorCode === 'vpc_member_requires_stopped'
|
|
&& (string) Capsule::table('cloud_vpcs')->where('id', 903)->value('state')
|
|
!== 'deleting'
|
|
&& (int) Capsule::table('cloud_vpcs')->where('id', 903)
|
|
->value('desired_revision') === $deleteRevision
|
|
&& Capsule::table('cloud_vpc_operations')->where('vpc_id', 903)->count()
|
|
=== $deleteOperationCount
|
|
&& Capsule::table('cloud_vpc_members')->where('vpc_id', 903)
|
|
->where('state', 'deleting')->count() === 0,
|
|
'A running member allowed root-delete intent to change the durable ledger.');
|
|
Capsule::table('cloud_instance_runtime')->where('instance_id', 90302)->update([
|
|
'power_state' => 'shutoff', 'sync_state' => 'ok',
|
|
'last_success_at' => time() - 60, 'updated_at' => time(),
|
|
]);
|
|
$staleDelete = $expectCloudError(static fn () => CloudVpcService::delete(
|
|
$deleteVpc['vpc_key'], $actor, $deleteInput
|
|
), 409);
|
|
$assert($staleDelete->errorCode === 'vpc_member_power_unavailable'
|
|
&& (int) Capsule::table('cloud_vpcs')->where('id', 903)
|
|
->value('desired_revision') === $deleteRevision
|
|
&& Capsule::table('cloud_vpc_operations')->where('vpc_id', 903)->count()
|
|
=== $deleteOperationCount,
|
|
'A stale member power observation allowed root deletion to be persisted.');
|
|
Capsule::table('cloud_instance_runtime')->whereIn('instance_id', [90301, 90302])
|
|
->update([
|
|
'power_state' => 'shutoff', 'sync_state' => 'ok',
|
|
'last_success_at' => time(), 'updated_at' => time(),
|
|
]);
|
|
$deleteSnapshotMutationArmed = true;
|
|
$deleteSnapshotMutationSeen = false;
|
|
$connection->beforeExecuting(static function ($sql) use (
|
|
&$deleteSnapshotMutationArmed, &$deleteSnapshotMutationSeen
|
|
): void {
|
|
if (!$deleteSnapshotMutationArmed
|
|
|| !str_starts_with(strtolower(ltrim($sql)), 'select')
|
|
|| preg_match('/\bfrom\s+["`]?cloud_instance_runtime["`]?(?:\s|$)/i', $sql) !== 1) {
|
|
return;
|
|
}
|
|
// Simulate a concurrent member transition after the first locked closure
|
|
// has committed and while the out-of-lock batch power read is starting.
|
|
$deleteSnapshotMutationArmed = false;
|
|
Capsule::table('cloud_vpc_members')->where('vpc_id', 903)
|
|
->where('instance_id', 90302)->update(['state' => 'active']);
|
|
$deleteSnapshotMutationSeen = true;
|
|
});
|
|
$changedClosureInput = [
|
|
'confirmation_name' => 'reuse-vpc-903',
|
|
'request_key' => 'delete-changed-closure-903',
|
|
];
|
|
$changedClosure = $expectCloudError(static fn () => CloudVpcService::delete(
|
|
$deleteVpc['vpc_key'], $actor, $changedClosureInput
|
|
), 409);
|
|
$assert($deleteSnapshotMutationSeen
|
|
&& $changedClosure->errorCode === 'vpc_member_power_unavailable'
|
|
&& (int) Capsule::table('cloud_vpcs')->where('id', 903)
|
|
->value('desired_revision') === $deleteRevision
|
|
&& Capsule::table('cloud_vpc_operations')->where('vpc_id', 903)->count()
|
|
=== $deleteOperationCount
|
|
&& Capsule::table('cloud_vpc_operations')->where('vpc_id', 903)
|
|
->where('idempotency_key', 'delete-changed-closure-903')->count() === 0,
|
|
'A member identity transition between delete phases created an operation.');
|
|
Capsule::table('cloud_vpc_members')->where('vpc_id', 903)
|
|
->where('instance_id', 90302)->update(['state' => 'applying']);
|
|
CloudVpcService::delete($deleteVpc['vpc_key'], $actor, $deleteInput);
|
|
$assert((string) Capsule::table('cloud_vpcs')->where('id', 903)->value('state')
|
|
=== 'deleting'
|
|
&& (int) Capsule::table('cloud_vpcs')->where('id', 903)
|
|
->value('desired_revision') === $deleteRevision + 1
|
|
&& Capsule::table('cloud_vpc_members')->where('vpc_id', 903)
|
|
->where('state', 'deleting')->count() === 2,
|
|
'A fully stopped member closure did not admit one root-delete revision.');
|
|
Capsule::table('cloud_instance_runtime')->where('instance_id', 90301)->update([
|
|
'power_state' => 'running', 'last_success_at' => time(), 'updated_at' => time(),
|
|
]);
|
|
CloudVpcService::delete($deleteVpc['vpc_key'], $actor, $deleteInput);
|
|
$assert((int) Capsule::table('cloud_vpcs')->where('id', 903)
|
|
->value('desired_revision') === $deleteRevision + 1
|
|
&& Capsule::table('cloud_vpc_operations')->where('vpc_id', 903)->count()
|
|
=== $deleteOperationCount + 1,
|
|
'An exact root-delete replay rechecked power or created a second operation.');
|
|
|
|
echo "VPC instance-task admission database regressions passed.\n";
|