90101, 'request_key' => 'admission-attach-90101']; foreach (['queued', 'running', 'uncertain'] as $status) { Capsule::table('cloud_jobs')->insert([ 'instance_id' => 90101, 'kind' => 'power', 'status' => $status, ]); $error = $expectCloudError(static fn () => CloudVpcService::attachMember( $admissionVpc['vpc_key'], $actor, $input ), 409); $assert($error->errorCode === 'vpc_instance_operation_pending', 'The active instance task must be reported as a precise admission conflict.'); $assert((int) Capsule::table('cloud_vpcs')->where('id', 901) ->value('desired_revision') === 1 && Capsule::table('cloud_vpc_members')->where('vpc_id', 901)->count() === 0 && Capsule::table('cloud_vpc_private_ips')->where('vpc_id', 901)->count() === 0, 'Rejected attachment changed its revision or reserved a private address.'); Capsule::table('cloud_jobs')->where('instance_id', 90101)->delete(); } CloudVpcService::attachMember($admissionVpc['vpc_key'], $actor, $input); $memberKey = Capsule::table('cloud_vpc_members')->where('instance_id', 90101) ->value('member_key'); Capsule::table('cloud_jobs')->insert([ 'instance_id' => 90101, 'kind' => 'power', 'status' => 'uncertain', ]); // A lost HTTP response may be retried without changing the desired target. CloudVpcService::attachMember($admissionVpc['vpc_key'], $actor, $input); $seedInstance(90102, 10); $mutations = [ static fn () => CloudVpcService::attachMember($admissionVpc['vpc_key'], $actor, [ 'instance_id' => 90102, 'request_key' => 'admission-attach-90102', ]), static fn () => CloudVpcService::detachMember($admissionVpc['vpc_key'], $memberKey, $actor, ['request_key' => 'admission-detach-90101']), static fn () => CloudVpcService::replaceRoutes($admissionVpc['vpc_key'], $actor, ['routes' => [], 'request_key' => 'admission-routes-90101']), static fn () => CloudVpcService::replaceAcl($admissionVpc['vpc_key'], $actor, ['rules' => [], 'request_key' => 'admission-acl-90101']), static fn () => CloudVpcService::delete($admissionVpc['vpc_key'], $actor, ['confirmation_name' => 'reuse-vpc-901', 'request_key' => 'admission-delete-90101']), ]; foreach ($mutations as $mutation) { $error = $expectCloudError($mutation, 409); $assert($error->errorCode === 'vpc_instance_operation_pending', 'A VPC document change bypassed the existing member task.'); $assert((int) Capsule::table('cloud_vpcs')->where('id', 901) ->value('desired_revision') === 2 && Capsule::table('cloud_vpc_members')->where('instance_id', 90101) ->value('state') === 'applying' && Capsule::table('cloud_vpc_private_ips')->where('instance_id', 90101) ->value('state') === 'applying' && Capsule::table('cloud_vpc_operations')->where('vpc_id', 901)->count() === 2, 'The admission conflict failed to roll back member/IP/root/operation changes.'); } Capsule::table('cloud_jobs')->where('instance_id', 90101)->update(['status' => 'succeeded']); // An unrelated task must not serialize independent VPCs or instances. Capsule::table('cloud_jobs')->insert([ 'instance_id' => 99999, 'kind' => 'power', 'status' => 'running', ]); CloudVpcService::detachMember($admissionVpc['vpc_key'], $memberKey, $actor, ['request_key' => 'admission-detach-90101']); $assert((int) Capsule::table('cloud_vpcs')->where('id', 901) ->value('desired_revision') === 3 && Capsule::table('cloud_vpc_members')->where('instance_id', 90101) ->value('state') === 'deleting' && Capsule::table('cloud_vpc_private_ips')->where('instance_id', 90101) ->value('state') === 'deleting', 'A completed task did not admit detach or detach released its address prematurely.'); // Execute real mutation transactions while injecting the database's NOWAIT // refusal at the task read. SQLite cannot reproduce MariaDB row contention; // retain its transaction engine and compile only this lock clause for capture. $lockVpc = $seedVpc(902, '10.90.2.0/24', '10.90.2.0/24', 10); $seedInstance(90201, 10); $connection = $capsule->getConnection('default'); $originalGrammar = $connection->getQueryGrammar(); $configProperty = new ReflectionProperty($connection, 'config'); $originalConfig = $configProperty->getValue($connection); $fault = null; $seenTaskSql = []; $connection->beforeExecuting(static function ($sql, $bindings) use ( &$fault, &$seenTaskSql, $assert ): void { if ($fault === null || preg_match('/\bfrom\s+["`]?cloud_jobs["`]?(?:\s|$)/i', $sql) !== 1 || !str_starts_with(strtolower($sql), 'select')) { return; } $seenTaskSql[] = $sql; $assert(str_ends_with($sql, 'FOR UPDATE NOWAIT') && !str_contains($sql, 'SKIP LOCKED'), 'Task admission waited on the worker lock or skipped a locked task.'); $previous = new PDOException('Synthetic internal database diagnostic'); $previous->errorInfo = [$fault === 1213 ? '40001' : 'HY000', $fault, 'private']; throw new QueryException('default', $sql, $bindings, $previous); }); $connection->setQueryGrammar(new class($connection) extends \Illuminate\Database\Query\Grammars\SQLiteGrammar { protected function compileLock(\Illuminate\Database\Query\Builder $query, $value) { // This fixture injects contention only for cloud_jobs. Other current // reads (including the new cross-VPC workflow fence) use SQLite locks. return $query->from === 'cloud_jobs' && is_string($value) ? $value : ''; } }); $configProperty->setValue($connection, array_replace($originalConfig, ['driver' => 'mysql'])); try { foreach ([1205, 1213, 3572, 1064, 2006] as $driverCode) { $fault = $driverCode; $beforeReads = count($seenTaskSql); $conflict = in_array($driverCode, [1205, 1213, 3572], true); $error = $expectCloudError(static fn () => CloudVpcService::attachMember( $lockVpc['vpc_key'], $actor, [ 'instance_id' => 90201, 'request_key' => 'admission-nowait-90201', ] ), $conflict ? 409 : 503); $assert($error->errorCode === ($conflict ? 'vpc_instance_operation_pending' : 'vpc_instance_operation_unavailable') && !str_contains($error->getMessage(), 'private') && count($seenTaskSql) === $beforeReads + 1 && (int) Capsule::table('cloud_vpcs')->where('id', 902) ->value('desired_revision') === 1 && Capsule::table('cloud_vpc_members')->where('vpc_id', 902)->count() === 0 && Capsule::table('cloud_vpc_private_ips')->where('vpc_id', 902)->count() === 0 && Capsule::table('cloud_vpc_operations')->where('vpc_id', 902)->count() === 1, 'A task-lock refusal retried without protection or leaked partial VPC/IP changes.'); } } finally { $fault = null; $configProperty->setValue($connection, $originalConfig); $connection->setQueryGrammar($originalGrammar); } $mysqlBuilder = new \Illuminate\Database\Query\Builder($connection, new \Illuminate\Database\Query\Grammars\MySqlGrammar($connection)); $assert(str_ends_with($mysqlBuilder->from('cloud_jobs')->where('instance_id', 90201) ->lock('FOR UPDATE NOWAIT')->toSql(), 'FOR UPDATE NOWAIT'), 'The production MariaDB/MySQL grammar discarded the no-wait current read.'); CloudVpcService::attachMember($lockVpc['vpc_key'], $actor, [ 'instance_id' => 90201, 'request_key' => 'admission-nowait-90201', ]); $assert((int) Capsule::table('cloud_vpcs')->where('id', 902) ->value('desired_revision') === 2, 'A rolled-back lock conflict prevented a later safe retry.'); // Root deletion is also a member mutation. It must capture the complete // member identity set, prove every runtime freshly stopped in one batch, then // re-lock and compare the same set before it persists any tombstone. $deleteVpc = $seedVpc(903, '10.90.3.0/24', '10.90.3.0/24', 10); $seedInstance(90301, 10); $seedInstance(90302, 10); CloudVpcService::attachMember($deleteVpc['vpc_key'], $actor, [ 'instance_id' => 90301, 'request_key' => 'delete-attach-90301', ]); CloudVpcService::attachMember($deleteVpc['vpc_key'], $actor, [ 'instance_id' => 90302, 'request_key' => 'delete-attach-90302', ]); $deleteInput = [ 'confirmation_name' => 'reuse-vpc-903', 'request_key' => 'delete-stopped-members-903', ]; $deleteRevision = (int) Capsule::table('cloud_vpcs')->where('id', 903) ->value('desired_revision'); $deleteOperationCount = Capsule::table('cloud_vpc_operations') ->where('vpc_id', 903)->count(); Capsule::table('cloud_instance_runtime')->where('instance_id', 90302)->update([ 'power_state' => 'running', 'sync_state' => 'ok', 'last_success_at' => time(), 'updated_at' => time(), ]); $runningDelete = $expectCloudError(static fn () => CloudVpcService::delete( $deleteVpc['vpc_key'], $actor, $deleteInput ), 409); $assert($runningDelete->errorCode === 'vpc_member_requires_stopped' && (string) Capsule::table('cloud_vpcs')->where('id', 903)->value('state') !== 'deleting' && (int) Capsule::table('cloud_vpcs')->where('id', 903) ->value('desired_revision') === $deleteRevision && Capsule::table('cloud_vpc_operations')->where('vpc_id', 903)->count() === $deleteOperationCount && Capsule::table('cloud_vpc_members')->where('vpc_id', 903) ->where('state', 'deleting')->count() === 0, 'A running member allowed root-delete intent to change the durable ledger.'); Capsule::table('cloud_instance_runtime')->where('instance_id', 90302)->update([ 'power_state' => 'shutoff', 'sync_state' => 'ok', 'last_success_at' => time() - 60, 'updated_at' => time(), ]); $staleDelete = $expectCloudError(static fn () => CloudVpcService::delete( $deleteVpc['vpc_key'], $actor, $deleteInput ), 409); $assert($staleDelete->errorCode === 'vpc_member_power_unavailable' && (int) Capsule::table('cloud_vpcs')->where('id', 903) ->value('desired_revision') === $deleteRevision && Capsule::table('cloud_vpc_operations')->where('vpc_id', 903)->count() === $deleteOperationCount, 'A stale member power observation allowed root deletion to be persisted.'); Capsule::table('cloud_instance_runtime')->whereIn('instance_id', [90301, 90302]) ->update([ 'power_state' => 'shutoff', 'sync_state' => 'ok', 'last_success_at' => time(), 'updated_at' => time(), ]); $deleteSnapshotMutationArmed = true; $deleteSnapshotMutationSeen = false; $connection->beforeExecuting(static function ($sql) use ( &$deleteSnapshotMutationArmed, &$deleteSnapshotMutationSeen ): void { if (!$deleteSnapshotMutationArmed || !str_starts_with(strtolower(ltrim($sql)), 'select') || preg_match('/\bfrom\s+["`]?cloud_instance_runtime["`]?(?:\s|$)/i', $sql) !== 1) { return; } // Simulate a concurrent member transition after the first locked closure // has committed and while the out-of-lock batch power read is starting. $deleteSnapshotMutationArmed = false; Capsule::table('cloud_vpc_members')->where('vpc_id', 903) ->where('instance_id', 90302)->update(['state' => 'active']); $deleteSnapshotMutationSeen = true; }); $changedClosureInput = [ 'confirmation_name' => 'reuse-vpc-903', 'request_key' => 'delete-changed-closure-903', ]; $changedClosure = $expectCloudError(static fn () => CloudVpcService::delete( $deleteVpc['vpc_key'], $actor, $changedClosureInput ), 409); $assert($deleteSnapshotMutationSeen && $changedClosure->errorCode === 'vpc_member_power_unavailable' && (int) Capsule::table('cloud_vpcs')->where('id', 903) ->value('desired_revision') === $deleteRevision && Capsule::table('cloud_vpc_operations')->where('vpc_id', 903)->count() === $deleteOperationCount && Capsule::table('cloud_vpc_operations')->where('vpc_id', 903) ->where('idempotency_key', 'delete-changed-closure-903')->count() === 0, 'A member identity transition between delete phases created an operation.'); Capsule::table('cloud_vpc_members')->where('vpc_id', 903) ->where('instance_id', 90302)->update(['state' => 'applying']); CloudVpcService::delete($deleteVpc['vpc_key'], $actor, $deleteInput); $assert((string) Capsule::table('cloud_vpcs')->where('id', 903)->value('state') === 'deleting' && (int) Capsule::table('cloud_vpcs')->where('id', 903) ->value('desired_revision') === $deleteRevision + 1 && Capsule::table('cloud_vpc_members')->where('vpc_id', 903) ->where('state', 'deleting')->count() === 2, 'A fully stopped member closure did not admit one root-delete revision.'); Capsule::table('cloud_instance_runtime')->where('instance_id', 90301)->update([ 'power_state' => 'running', 'last_success_at' => time(), 'updated_at' => time(), ]); CloudVpcService::delete($deleteVpc['vpc_key'], $actor, $deleteInput); $assert((int) Capsule::table('cloud_vpcs')->where('id', 903) ->value('desired_revision') === $deleteRevision + 1 && Capsule::table('cloud_vpc_operations')->where('vpc_id', 903)->count() === $deleteOperationCount + 1, 'An exact root-delete replay rechecked power or created a second operation.'); echo "VPC instance-task admission database regressions passed.\n";