You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudPublicNetworkMysqlConc...

579 lines
20 KiB
PHP

<?php
use Illuminate\Database\Capsule\Manager as Capsule;
require __DIR__ . '/Support/CloudPublicNetworkMysqlFixture.php';
$dsn = trim((string) getenv(MYSQL_CONCURRENCY_DSN));
$user = (string) getenv(MYSQL_CONCURRENCY_USER);
$password = getenv(MYSQL_CONCURRENCY_PASSWORD);
$allow = (string) getenv(MYSQL_CONCURRENCY_ALLOW);
$deadlockGate = (string) getenv(
MYSQL_CONCURRENCY_ASSERT_GLOBAL_DEADLOCKS
);
if ($dsn === '' && $user === '' && $password === false && $allow === ''
&& $deadlockGate === '') {
echo "SKIP Cloud public-network MySQL concurrency: opt-in env is absent.\n";
exit(0);
}
mysqlConcurrencyRequireGate();
$sentinel = mysqlConcurrencyPdo();
$assertGlobalDeadlocks = mysqlConcurrencyGlobalDeadlockAssertionEnabled();
echo $assertGlobalDeadlocks
? "INFO MySQL global deadlock counter is enforced; use an exclusive temporary MySQL server.\n"
: "INFO MySQL global deadlock counter is not enforced; this run "
. "validates final serialization only and does not claim deadlock "
. "freedom.\n";
$createdKey = false;
$keyFile = trim((string) getenv(MYSQL_CONCURRENCY_KEY_FILE));
if ($keyFile === '') {
$keyFile = tempnam(sys_get_temp_dir(), 'virt-concurrency-key-');
if (!is_string($keyFile)) {
throw new RuntimeException('Unable to create the test data key.');
}
if (file_put_contents(
$keyFile,
base64_encode(random_bytes(32)) . "\n",
LOCK_EX
) === false
|| !chmod($keyFile, 0600)) {
@unlink($keyFile);
throw new RuntimeException('Unable to create the test data key.');
}
putenv(MYSQL_CONCURRENCY_KEY_FILE . '=' . $keyFile);
$createdKey = true;
} elseif (!is_file($keyFile) || !is_readable($keyFile)) {
throw new RuntimeException('Configured test data key is unreadable.');
}
$barrierDirectory = sys_get_temp_dir() . '/virt-concurrency-'
. bin2hex(random_bytes(8));
if (!mkdir($barrierDirectory, 0700)) {
if ($createdKey && is_file($keyFile)) {
unlink($keyFile);
}
throw new RuntimeException('Unable to create child barrier directory.');
}
$reset = static function () use ($sentinel): void {
mysqlConcurrencyResetSchema($sentinel);
mysqlConcurrencySeed();
};
$race = static function (
string $operationA,
array $payloadA,
string $operationB,
array $payloadB
) use ($sentinel, $barrierDirectory, $assertGlobalDeadlocks): array {
$before = $assertGlobalDeadlocks
? mysqlConcurrencyGlobalDeadlocks($sentinel) : null;
$result = mysqlConcurrencyRace(
$sentinel,
$barrierDirectory,
$operationA,
$payloadA,
$operationB,
$payloadB
);
if ($assertGlobalDeadlocks) {
mysqlConcurrencyAssert(
mysqlConcurrencyGlobalDeadlocks($sentinel) === $before,
'InnoDB recorded a deadlock during the concurrency scenario.'
);
}
return $result;
};
$hostRace = static function (
string $operationA,
array $payloadA,
string $operationB,
array $payloadB
) use ($sentinel, $barrierDirectory, $assertGlobalDeadlocks): array {
$before = $assertGlobalDeadlocks
? mysqlConcurrencyGlobalDeadlocks($sentinel) : null;
$result = mysqlConcurrencyHostBoundaryRace(
$sentinel,
$barrierDirectory,
$operationA,
$payloadA,
$operationB,
$payloadB
);
if ($assertGlobalDeadlocks) {
mysqlConcurrencyAssert(
mysqlConcurrencyGlobalDeadlocks($sentinel) === $before,
'InnoDB recorded a deadlock during the host-boundary scenario.'
);
}
return $result;
};
$oracle = static function (string $operation, array $payload) use (
$reset
): array {
$reset();
mysqlConcurrencyChildOperation($operation, $payload);
mysqlConcurrencyAssertLedger();
return mysqlConcurrencyCanonicalSnapshot();
};
$admin = static function (
int $actorId,
string $requestKey,
string $poolCharacter,
string $address,
int $family = 4,
int $instanceId = 101
): array {
return [
'instance_id' => $instanceId,
'actor_id' => $actorId,
'request_key' => $requestKey,
'family' => $family,
'pool_key' => str_repeat($poolCharacter, 32),
'address_cidr' => $address . ($family === 4 ? '/29' : '/64'),
];
};
$allocator = static function (int $reservationId): array {
return [
'reservation_id' => $reservationId,
'operation_id' => str_pad(
dechex($reservationId),
32,
'0',
STR_PAD_LEFT
),
];
};
$reservationAddress = static function (
int $reservationId,
int $family
): ?string {
$address = Capsule::table('cloud_resource_reservation_ips as claim')
->join(
'cloud_public_ips as ip',
'ip.id',
'=',
'claim.public_ip_id'
)->where('claim.reservation_id', $reservationId)
->where('claim.address_family', $family)
->where('claim.active_claim', 1)->value('ip.address');
return is_string($address) ? $address : null;
};
$tenant = static fn (string $requestKey): array => [
'instance_id' => 101,
'actor_id' => 10,
'request_key' => $requestKey,
'expected_amount' => '5.00',
];
$batch = static fn (int $actorId, string $requestKey): array => [
'actor_id' => $actorId,
'request_key' => $requestKey,
'source_pool_key' => str_repeat('a', 32),
'target_pool_key' => str_repeat('c', 32),
'source_revision' => 11,
'target_revision' => 13,
];
$assertWinnerLoser = static function (
array $winner,
array $loser,
string $context
): void {
mysqlConcurrencyAssert(($winner['ok'] ?? false) === true,
$context . ': expected winner failed.');
mysqlConcurrencyAssert(($loser['ok'] ?? true) === false
&& (int) ($loser['http_status'] ?? 0) === 409,
$context . ': expected loser was not rejected with 409.');
};
try {
// Scenario 1: different actors contend on one instance. Both possible
// winners are exercised; the loser must leave finance and IPAM untouched.
$adminPayload = $admin(
99,
'mysql-admin-wins-0001',
'c',
'8.8.9.2'
);
$expected = $oracle('admin_begin', $adminPayload);
$reset();
[$adminWin, $tenantLose] = $race(
'admin_begin',
$adminPayload,
'user_begin',
$tenant('mysql-tenant-loses-0001')
);
$assertWinnerLoser($adminWin, $tenantLose, 'admin versus tenant');
mysqlConcurrencyAssertTransitionResponseOperation(
$adminWin,
'admin versus tenant winner'
);
mysqlConcurrencyAssert(
Capsule::table('cloud_public_network_transitions')->count() === 1
&& Capsule::table('cloud_jobs')->count() === 1
&& Capsule::table('cloud_invoices')->count() === 0
&& (string) Capsule::table('user')->where('id', 10)
->value('money') === '20.00',
'Rejected tenant mutation changed finance or transition state.'
);
mysqlConcurrencyAssertCanonical($expected, 'admin versus tenant');
mysqlConcurrencyAssertLedger();
$tenantPayload = $tenant('mysql-tenant-wins-0001');
$expected = $oracle('user_begin', $tenantPayload);
$reset();
[$tenantWin, $adminLose] = $race(
'user_begin',
$tenantPayload,
'admin_begin',
$admin(99, 'mysql-admin-loses-0001', 'c', '8.8.9.2')
);
$assertWinnerLoser($tenantWin, $adminLose, 'tenant versus admin');
mysqlConcurrencyAssertTransitionResponseOperation(
$tenantWin,
'tenant versus admin winner'
);
mysqlConcurrencyAssert(
Capsule::table('cloud_public_network_transitions')->count() === 1
&& Capsule::table('cloud_jobs')->count() === 1
&& Capsule::table('cloud_invoices')->count() === 1
&& (string) Capsule::table('user')->where('id', 10)
->value('money') === '15.00',
'Winning tenant mutation was not charged exactly once.'
);
mysqlConcurrencyAssertCanonical($expected, 'tenant versus admin');
mysqlConcurrencyAssertLedger();
// Scenario 2: exact duplicate replays one generation; same idempotency
// key with a different immutable target is rejected with zero extra rows.
$same = $admin(99, 'mysql-exact-replay-0001', 'c', '8.8.9.2');
$expected = $oracle('admin_begin', $same);
$reset();
[$first, $replay] = $race(
'admin_begin', $same, 'admin_begin', $same
);
mysqlConcurrencyAssert(($first['ok'] ?? false) === true
&& ($replay['ok'] ?? false) === true,
'Exact duplicate did not return two successful responses.');
$firstOperationId = mysqlConcurrencyAssertTransitionResponseOperation(
$first,
'exact replay first response'
);
$replayOperationId = mysqlConcurrencyAssertTransitionResponseOperation(
$replay,
'exact replay second response'
);
$replayFlags = [
(bool) ($first['result']['replayed'] ?? null),
(bool) ($replay['result']['replayed'] ?? null),
];
sort($replayFlags);
mysqlConcurrencyAssert(
(int) $first['result']['transition_id']
=== (int) $replay['result']['transition_id']
&& (int) $first['result']['job_id']
=== (int) $replay['result']['job_id']
&& $first['result']['operation_id']
=== $replay['result']['operation_id']
&& hash_equals($firstOperationId, $replayOperationId)
&& $replayFlags === [false, true]
&& Capsule::table('cloud_public_network_transitions')->count() === 1
&& Capsule::table('cloud_jobs')->count() === 1,
'Exact replay duplicated or changed the durable generation.'
);
mysqlConcurrencyAssertCanonical($expected, 'exact idempotent replay');
mysqlConcurrencyAssertLedger();
$conflictWinnerPayload = $admin(
99,
'mysql-key-conflict-0001',
'c',
'8.8.9.2'
);
$conflictOtherPayload = $admin(
99,
'mysql-key-conflict-0001',
'e',
'8.8.10.2'
);
$expectedConflictA = $oracle(
'admin_begin',
$conflictWinnerPayload
);
$expectedConflictB = $oracle(
'admin_begin',
$conflictOtherPayload
);
$reset();
[$conflictA, $conflictB] = $race(
'admin_begin',
$conflictWinnerPayload,
'admin_begin',
$conflictOtherPayload
);
// The actor-row release is unbiased: either conflicting payload may win.
// Its matching sequential oracle must describe the durable result.
if (($conflictA['ok'] ?? false) === true) {
$conflictWinner = $conflictA;
$conflictLoser = $conflictB;
$expected = $expectedConflictA;
} else {
$conflictWinner = $conflictB;
$conflictLoser = $conflictA;
$expected = $expectedConflictB;
}
$assertWinnerLoser($conflictWinner, $conflictLoser,
'idempotency target conflict');
mysqlConcurrencyAssertTransitionResponseOperation(
$conflictWinner,
'idempotency target-conflict winner'
);
mysqlConcurrencyAssert(
Capsule::table('cloud_public_network_transitions')->count() === 1
&& Capsule::table('cloud_jobs')->count() === 1,
'Conflicting replay left more than one durable generation.'
);
mysqlConcurrencyAssertCanonical($expected, 'idempotency target conflict');
mysqlConcurrencyAssertLedger();
// Scenario 3: a batch and a single transition serialize on the same
// instance. Whichever owns the instance first wins without partial batch
// rows or sibling transitions from the loser.
$singlePayload = $admin(
99,
'mysql-single-wins-0001',
'e',
'8.8.10.2'
);
$expected = $oracle('admin_begin', $singlePayload);
$reset();
[$singleWin, $batchLose] = $race(
'admin_begin',
$singlePayload,
'batch_begin',
$batch(98, 'mysql-batch-loses-0001')
);
$assertWinnerLoser($singleWin, $batchLose, 'single versus batch');
mysqlConcurrencyAssertTransitionResponseOperation(
$singleWin,
'single versus batch winner'
);
mysqlConcurrencyAssert(
Capsule::table('cloud_public_ip_pool_replacement_batches')->count()
=== 0
&& Capsule::table('cloud_public_ip_pool_replacement_batch_items')
->count() === 0
&& Capsule::table('cloud_public_network_transitions')->count() === 1,
'Rejected batch left a parent, item, or sibling transition.'
);
mysqlConcurrencyAssertCanonical($expected, 'single versus batch');
mysqlConcurrencyAssertLedger();
$batchPayload = $batch(98, 'mysql-batch-wins-0001');
$expected = $oracle('batch_begin', $batchPayload);
$reset();
[$batchWin, $singleLose] = $race(
'batch_begin',
$batchPayload,
'admin_begin',
$admin(99, 'mysql-single-loses-0001', 'e', '8.8.10.2')
);
$assertWinnerLoser($batchWin, $singleLose, 'batch versus single');
mysqlConcurrencyAssertBatchResponseOperation(
$batchWin,
'batch versus single winner'
);
mysqlConcurrencyAssert(
Capsule::table('cloud_public_ip_pool_replacement_batches')->count()
=== 1
&& Capsule::table('cloud_public_ip_pool_replacement_batch_items')
->count() === 2
&& Capsule::table('cloud_public_network_transitions')->count() === 2
&& Capsule::table('cloud_jobs')->count() === 3,
'Winning batch did not create one atomic parent and two children.'
);
mysqlConcurrencyAssertCanonical($expected, 'batch versus single');
mysqlConcurrencyAssertLedger();
// Scenario 4: two first-time allocators contend on one sparse dual-stack
// catalog. Both reservations must commit a complete and unique pair.
$reset();
[$allocatorA, $allocatorB] = $hostRace(
'reserve_bundle',
$allocator(2001),
'reserve_bundle',
$allocator(2002)
);
mysqlConcurrencyAssert(
($allocatorA['ok'] ?? false) === true
&& ($allocatorB['ok'] ?? false) === true
&& (int) ($allocatorA['result']['reservation_id'] ?? 0) === 2001
&& (int) ($allocatorB['result']['reservation_id'] ?? 0) === 2002,
'allocator versus allocator did not reserve both bundles.'
);
$allocatedV4 = [
$reservationAddress(2001, 4),
$reservationAddress(2002, 4),
];
$allocatedV6 = [
$reservationAddress(2001, 6),
$reservationAddress(2002, 6),
];
sort($allocatedV4);
sort($allocatedV6);
mysqlConcurrencyAssert(
$allocatedV4 === ['8.8.8.4', '8.8.8.5']
&& $allocatedV6 === [
'2404:c140:156:3a::1002',
'2404:c140:156:3a::1003',
]
&& (string) Capsule::table('cloud_public_ip_pools')
->where('id', 12)->value('allocation_cursor')
=== '2404:c140:156:3a::1004'
&& Capsule::table('cloud_public_network_transitions')->count() === 0
&& Capsule::table('cloud_jobs')->count() === 0,
'allocator versus allocator produced duplicate or partial capacity.'
);
mysqlConcurrencyAssertLedger();
// Scenario 5: first-time allocation and an exact sparse IPv6 transition
// contend for ::1002. Allocation always succeeds; the transition either
// owns ::1002 first or is rejected after the allocator owns it.
$reset();
[$allocatorResult, $transitionResult] = $hostRace(
'reserve_bundle',
$allocator(2001),
'admin_begin',
$admin(
99,
'mysql-allocator-v6-race-0001',
'b',
'2404:c140:156:3a::1002',
6
)
);
mysqlConcurrencyAssert(
($allocatorResult['ok'] ?? false) === true
&& (int) ($allocatorResult['result']['reservation_id'] ?? 0) === 2001,
'allocator versus exact IPv6 transition lost the complete bundle.'
);
$allocatorV6 = $reservationAddress(2001, 6);
$exactRows = Capsule::table('cloud_public_ips')
->where('address_family', 6)
->where('address_binary', inet_pton('2404:c140:156:3a::1002'))
->get();
mysqlConcurrencyAssert(
$exactRows->count() === 1,
'allocator versus exact IPv6 transition duplicated ::1002.'
);
if (($transitionResult['ok'] ?? false) === true) {
mysqlConcurrencyAssertTransitionResponseOperation(
$transitionResult,
'allocator versus exact IPv6 transition winner'
);
mysqlConcurrencyAssert(
$allocatorV6 === '2404:c140:156:3a::1003'
&& (string) Capsule::table('cloud_public_ip_pools')
->where('id', 12)->value('allocation_cursor')
=== '2404:c140:156:3a::1004'
&& Capsule::table('cloud_public_network_transitions')->count()
=== 1
&& Capsule::table('cloud_jobs')->count() === 1,
'Exact IPv6 winner did not move the allocator past ::1002.'
);
} else {
mysqlConcurrencyAssert(
(int) ($transitionResult['http_status'] ?? 0) === 409
&& $allocatorV6 === '2404:c140:156:3a::1002'
&& (string) Capsule::table('cloud_public_ip_pools')
->where('id', 12)->value('allocation_cursor')
=== '2404:c140:156:3a::1003'
&& Capsule::table('cloud_public_network_transitions')->count()
=== 0
&& Capsule::table('cloud_jobs')->count() === 0,
'Allocator winner left exact-transition residue or wrong cursor.'
);
}
mysqlConcurrencyAssertLedger();
// Scenario 6: two distinct instances request the same exact sparse IPv6
// address. Exactly one durable transition may own the target generation.
$reset();
[$exactA, $exactB] = $hostRace(
'admin_begin',
$admin(
99,
'mysql-same-v6-instance-a-0001',
'b',
'2404:c140:156:3a::1002',
6,
101
),
'admin_begin',
$admin(
98,
'mysql-same-v6-instance-b-0001',
'b',
'2404:c140:156:3a::1002',
6,
102
)
);
if (($exactA['ok'] ?? false) === true) {
$exactWinner = $exactA;
$exactLoser = $exactB;
$winningInstanceId = 101;
} else {
$exactWinner = $exactB;
$exactLoser = $exactA;
$winningInstanceId = 102;
}
$assertWinnerLoser(
$exactWinner,
$exactLoser,
'two transitions request the same exact IPv6'
);
mysqlConcurrencyAssertTransitionResponseOperation(
$exactWinner,
'same exact IPv6 winner'
);
$exactIp = Capsule::table('cloud_public_ips')
->where('address_family', 6)
->where('address_binary', inet_pton('2404:c140:156:3a::1002'))
->first();
mysqlConcurrencyAssert(
Capsule::table('cloud_public_network_transitions')->count() === 1
&& Capsule::table('cloud_public_network_transitions')
->where('instance_id', $winningInstanceId)->count() === 1
&& Capsule::table('cloud_jobs')->count() === 1
&& Capsule::table('cloud_resource_reservation_ips')
->whereIn('reservation_id', [1001, 1002])
->where('address_family', 6)
->where('state', 'reserved')
->whereNull('active_claim')->count() === 1
&& $exactIp !== null
&& (string) $exactIp->state === 'reserved'
&& $exactIp->instance_id === null
&& (string) Capsule::table('cloud_public_ip_pools')
->where('id', 12)->value('allocation_cursor')
=== '2404:c140:156:3a::1002',
'Same exact IPv6 race left more than one generation or loser residue.'
);
mysqlConcurrencyAssertLedger();
echo "Cloud public-network MySQL concurrency scenarios passed.\n";
echo "C11 will extend this harness with Worker-finalizer and unknown-effect recovery races.\n";
} finally {
foreach (['*.go', '*.entered', '*.tmp-*'] as $pattern) {
foreach (glob($barrierDirectory . '/' . $pattern) ?: [] as $barrier) {
if (is_file($barrier)) {
unlink($barrier);
}
}
}
if (is_dir($barrierDirectory)) {
rmdir($barrierDirectory);
}
if ($createdKey && is_file($keyFile)) {
unlink($keyFile);
}
}