From 13132bc4193dd69845e25b5010677e09b5de7a97 Mon Sep 17 00:00:00 2001 From: codex Date: Tue, 3 Nov 2026 18:55:36 +0800 Subject: [PATCH] refactor: unify public network transition lock order --- ...oudPublicIpPoolReplacementBatchService.php | 158 +-- .../CloudPublicNetworkTransitionWorkflow.php | 934 ++++++++++-------- ...ublicIpPoolReplacementBatchServiceTest.php | 96 ++ ...licIpPoolReplacementSchemaContractTest.php | 75 ++ ...ublicNetworkTransitionFinalizationTest.php | 101 +- ...cNetworkTransitionWorkflowContractTest.php | 116 ++- ...oudPublicNetworkTransitionWorkflowTest.php | 194 ++++ 7 files changed, 1191 insertions(+), 483 deletions(-) diff --git a/web/app/Services/Cloud/CloudPublicIpPoolReplacementBatchService.php b/web/app/Services/Cloud/CloudPublicIpPoolReplacementBatchService.php index fca4d9a7..518c97d8 100644 --- a/web/app/Services/Cloud/CloudPublicIpPoolReplacementBatchService.php +++ b/web/app/Services/Cloud/CloudPublicIpPoolReplacementBatchService.php @@ -14,6 +14,7 @@ use App\Models\CloudPublicIpPoolReplacementBatchItem; use App\Models\CloudResourcePool; use App\Models\CloudResourceReservation; use App\Models\CloudResourceReservationIp; +use App\Models\User; /** * Durable same-host replacement of every assigned address in one public pool. @@ -268,48 +269,59 @@ final class CloudPublicIpPoolReplacementBatchService $batchKey = substr(hash('sha256', "public-pool-replacement\0" . $actorUserId . "\0" . $hostId . "\0" . $sourcePoolKey . "\0" . $requestKey), 0, 32); - $prior = CloudJob::where('user_id', $actorUserId) - ->where('request_idempotency_key', $requestKey)->first(); - if ($prior !== null) { - return self::existingResult($prior, $fingerprint, $batchKey); - } - - $sourceRef = CloudPublicIpPool::where('host_id', $hostId) - ->where('pool_key', $sourcePoolKey)->first(); - $targetRef = CloudPublicIpPool::where('host_id', $hostId) - ->where('pool_key', $targetPoolKey)->first(); - if ($sourceRef === null || $targetRef === null) { - throw new CloudError('源或目标公网地址池不存在', 404); - } - $instanceIds = CloudResourceReservationIp::query() - ->join( - 'cloud_resource_reservations as reservation', - 'reservation.id', - '=', - 'cloud_resource_reservation_ips.reservation_id' - )->where('reservation.host_id', $hostId) - ->where( - 'cloud_resource_reservation_ips.public_ip_pool_id', - (int) $sourceRef->id - )->where('cloud_resource_reservation_ips.active_claim', 1) - ->where('cloud_resource_reservation_ips.state', 'committed') - ->where('reservation.state', 'committed') - ->whereNotNull('reservation.instance_id') - ->orderBy('reservation.instance_id') - ->pluck('reservation.instance_id')->map( - static fn ($id): int => (int) $id - )->unique()->values()->all(); - if ($instanceIds === []) { - throw new CloudError('源公网地址池没有可迁移实例', 409); - } - return (new CloudPublicIpPoolReplacementBatch())->getConnection() ->transaction(static function () use ( $hostId, $sourcePoolKey, $targetPoolKey, $actorUserId, - $requestKey, $fingerprint, $batchKey, $sourceRef, - $targetRef, $instanceIds, $expectedSourceRevision, + $requestKey, $fingerprint, $batchKey, $expectedSourceRevision, $expectedTargetRevision ): array { + $actorUser = User::where('id', $actorUserId) + ->lockForUpdate()->first(); + if (!$actorUser instanceof User) { + throw new CloudError('批量更换公网 IP 的操作账户不存在', 409); + } + $prior = CloudJob::where('user_id', $actorUserId) + ->where('request_idempotency_key', $requestKey) + ->lockForUpdate()->first(); + if ($prior !== null) { + return self::existingResult( + $prior, + $fingerprint, + $batchKey + ); + } + $sourceRef = CloudPublicIpPool::where('host_id', $hostId) + ->where('pool_key', $sourcePoolKey)->first(); + $targetRef = CloudPublicIpPool::where('host_id', $hostId) + ->where('pool_key', $targetPoolKey)->first(); + if ($sourceRef === null || $targetRef === null) { + throw new CloudError('源或目标公网地址池不存在', 404); + } + $instanceIds = CloudResourceReservationIp::query() + ->join( + 'cloud_resource_reservations as reservation', + 'reservation.id', + '=', + 'cloud_resource_reservation_ips.reservation_id' + )->where('reservation.host_id', $hostId) + ->where( + 'cloud_resource_reservation_ips.public_ip_pool_id', + (int) $sourceRef->id + )->where( + 'cloud_resource_reservation_ips.active_claim', + 1 + )->where( + 'cloud_resource_reservation_ips.state', + 'committed' + )->where('reservation.state', 'committed') + ->whereNotNull('reservation.instance_id') + ->orderBy('reservation.instance_id') + ->pluck('reservation.instance_id')->map( + static fn ($id): int => (int) $id + )->unique()->values()->all(); + if ($instanceIds === []) { + throw new CloudError('源公网地址池没有可迁移实例', 409); + } $instances = CloudInstance::whereIn('id', $instanceIds) ->orderBy('id')->lockForUpdate()->get()->keyBy('id'); if ($instances->count() !== count($instanceIds)) { @@ -390,16 +402,6 @@ final class CloudPublicIpPoolReplacementBatchService } self::assertTargetPoolConfiguration($host, $targetPool); - $prior = CloudJob::where('user_id', $actorUserId) - ->where('request_idempotency_key', $requestKey) - ->lockForUpdate()->first(); - if ($prior !== null) { - return self::existingResult( - $prior, - $fingerprint, - $batchKey - ); - } $replacementPoolIds = [ (int) $sourcePool->id, (int) $targetPool->id, @@ -633,18 +635,45 @@ final class CloudPublicIpPoolReplacementBatchService if ($parent === null || (string) $parent->kind !== self::PARENT_KIND) { return; } - $item = CloudPublicIpPoolReplacementBatchItem::where( + // Hints discover immutable IDs only. The mutable catalog is locked + // before batch/item rows so convergence never takes pool after batch. + $itemHint = CloudPublicIpPoolReplacementBatchItem::where( 'child_job_id', (int) $job->id - )->lockForUpdate()->first(); - if ($item === null) { + )->first(['id', 'batch_id']); + if ($itemHint === null) { throw new CloudError('公网地址池迁移子任务缺少批次明细', 409); } + $batchHint = CloudPublicIpPoolReplacementBatch::find( + (int) $itemHint->batch_id + ); + if ($batchHint === null) { + throw new CloudError('公网地址池迁移批次账本不一致', 409); + } + $source = CloudPublicIpPool::where( + 'id', (int) $batchHint->source_public_ip_pool_id + )->lockForUpdate()->first(); + if ($source === null + || (int) $source->host_id !== (int) $batchHint->host_id) { + throw new CloudError('公网地址池迁移源池账本缺失', 409); + } $batch = CloudPublicIpPoolReplacementBatch::where( - 'id', (int) $item->batch_id + 'id', (int) $batchHint->id )->lockForUpdate()->first(); - if ($batch === null + if ($batch === null) { + throw new CloudError('公网地址池迁移批次账本不一致', 409); + } + $items = CloudPublicIpPoolReplacementBatchItem::where( + 'batch_id', (int) $batch->id + )->orderBy('id')->lockForUpdate()->get(); + $item = $items->first(static fn ($candidate): bool => + (int) $candidate->id === (int) $itemHint->id + && (int) ($candidate->child_job_id ?? 0) === (int) $job->id + ); + if ($item === null || (int) $batch->parent_job_id !== (int) $parent->id || (int) $batch->host_id !== (int) $job->host_id + || (int) $batch->source_public_ip_pool_id !== (int) $source->id + || (int) $item->batch_id !== (int) $batch->id || (int) $item->instance_id !== (int) $job->instance_id) { throw new CloudError('公网地址池迁移批次账本不一致', 409); } @@ -678,7 +707,7 @@ final class CloudPublicIpPoolReplacementBatchService : substr((string) ($job->error_code ?: $catalogStatus), 0, 64); $item->updated_at = time(); $item->save(); - self::convergeLocked($batch, $parent); + self::convergeLocked($batch, $parent, $source, $items); } /** @@ -701,14 +730,8 @@ final class CloudPublicIpPoolReplacementBatchService || (string) $parentSnapshot->kind !== self::PARENT_KIND) { return; } - if ($job->host_id === null - || CloudHost::where('id', (int) $job->host_id) - ->lockForUpdate()->first() === null) { - throw new CloudError( - '公网地址池迁移子任务宿主机账本缺失', - 409 - ); - } + // The caller owns actor -> parent -> child -> instance -> host and the + // transition ledger. Do not reacquire host here after parent/job. self::finishChildLocked( $job, (string) $job->effect_resolution === 'applied' @@ -787,11 +810,10 @@ final class CloudPublicIpPoolReplacementBatchService private static function convergeLocked( CloudPublicIpPoolReplacementBatch $batch, - CloudJob $parent + CloudJob $parent, + CloudPublicIpPool $source, + \Illuminate\Support\Collection $items ): void { - $items = CloudPublicIpPoolReplacementBatchItem::where( - 'batch_id', (int) $batch->id - )->orderBy('id')->lockForUpdate()->get(); if ($items->contains(static fn ($item): bool => (string) $item->state === 'unknown')) { $batch->state = 'blocked'; @@ -810,10 +832,8 @@ final class CloudPublicIpPoolReplacementBatchService } $succeeded = $items->where('state', 'succeeded')->count(); $failed = $items->where('state', 'failed')->count(); - $source = CloudPublicIpPool::where( - 'id', (int) $batch->source_public_ip_pool_id - )->lockForUpdate()->first(); - if ($source === null || (int) $source->host_id !== (int) $batch->host_id) { + if ((int) $source->id !== (int) $batch->source_public_ip_pool_id + || (int) $source->host_id !== (int) $batch->host_id) { throw new CloudError('公网地址池迁移源池账本缺失', 409); } $occupied = CloudPublicIp::where( diff --git a/web/app/Services/Cloud/CloudPublicNetworkTransitionWorkflow.php b/web/app/Services/Cloud/CloudPublicNetworkTransitionWorkflow.php index d3ae2ce9..232a6c29 100644 --- a/web/app/Services/Cloud/CloudPublicNetworkTransitionWorkflow.php +++ b/web/app/Services/Cloud/CloudPublicNetworkTransitionWorkflow.php @@ -53,433 +53,490 @@ final class CloudPublicNetworkTransitionWorkflow throw new CloudError('账户不可用', 403); } - $instance = CloudInstance::where('id', $command->instanceId()) - ->lockForUpdate()->first(); - if (!$instance instanceof CloudInstance - || ($command->kind() === 'user_paid' - && (int) $instance->user_id - !== $command->actorUserId())) { - throw new CloudError('虚拟机不满足公网网络变更条件', 409); - } + return self::reserveTransitionLocked($command, $actorUser); + }, 3); + } - // Exact replay is resolved before consulting mutable host, pool or - // address catalogs. A previously accepted command remains - // replayable after its target pool is disabled or removed. - $existing = CloudPublicNetworkTransition::where( - 'transition_key', $command->transitionKey() - )->lockForUpdate()->first(); - if ($existing instanceof CloudPublicNetworkTransition) { - return self::replayLocked($existing, $command); + /** + * Reserve a transition while the caller owns the actor row and the outer + * transaction. This internal seam must never create a nested transaction: + * billing, transition, claim, address and job writes commit or roll back + * together. + */ + private static function reserveTransitionLocked( + CloudPublicNetworkTransitionCommand $command, + User $actorUser + ): CloudPublicNetworkTransitionDispatch { + // Non-locking discovery may only identify the immutable rows. Every + // identity is revalidated after the canonical actor -> parent/job -> + // instance -> host/agent -> transition lock chain is held. + $existingHint = CloudPublicNetworkTransition::where( + 'transition_key', $command->transitionKey() + )->first(['id', 'instance_id', 'host_id']); + $existingJob = null; + $existingParent = null; + if ($existingHint instanceof CloudPublicNetworkTransition) { + $jobHint = CloudJob::where( + 'public_network_transition_id', (int) $existingHint->id + )->orderBy('id')->first(['id', 'parent_job_id']); + if ($jobHint instanceof CloudJob + && (int) ($jobHint->parent_job_id ?? 0) > 0) { + $existingParent = CloudJob::where( + 'id', (int) $jobHint->parent_job_id + )->lockForUpdate()->first(); + } + if ($jobHint instanceof CloudJob) { + $existingJob = CloudJob::where('id', (int) $jobHint->id) + ->lockForUpdate()->first(); } + } else { $priorJob = CloudJob::where( 'user_id', $command->actorUserId() )->where('request_idempotency_key', $command->requestKey()) - ->first(['id']); + ->lockForUpdate()->first(['id']); if ($priorJob instanceof CloudJob) { - throw new CloudError('公网网络变更幂等键已被其他任务占用', 409); - } - $active = CloudPublicNetworkTransition::where( - 'instance_id', (int) $instance->id - )->where('active_transition', 1)->lockForUpdate()->first(); - if ($active instanceof CloudPublicNetworkTransition) { - throw new CloudError('虚拟机已有公网网络变更正在处理', 409); - } - try { - $spec = json_decode( - (string) $instance->spec_json, - true, - 64, - JSON_THROW_ON_ERROR + throw new CloudError( + '公网网络变更幂等键已被其他任务占用', + 409 ); - } catch (\Throwable) { - $spec = null; - } - if ((string) $instance->status !== 'active' - || !$instance->serviceAvailable() - || (int) ($instance->host_id ?? 0) < 1 - || (int) ($instance->reservation_id ?? 0) < 1 - || (int) ($instance->resource_pool_id ?? 0) < 1 - || !is_array($spec) || array_is_list($spec) - || ($spec['managed'] ?? false) !== true - || ($spec['os_source'] ?? null) !== 'cloud_template') { - throw new CloudError('虚拟机不满足公网网络变更条件', 409); - } - CloudInstanceMigrationService::assertInstanceMutationAvailable( - $instance - ); - if (DB::table('cloud_backup_recoveries') - ->where('instance_id', (int) $instance->id) - ->whereIn('state', ['recovering', 'uncertain'])->exists()) { - throw new CloudError('虚拟机正在恢复备份,请等待完成', 409); - } - // The instance row serializes all supported mutation writers. - // This is intentionally an unlocked predicate: Worker finalizers - // lock job before instance, so taking job locks here would reverse - // that global order and create a job/instance deadlock. - if (CloudJobService::blockingQueryForInstance( - (int) $instance->id - )->exists()) { - throw new CloudError('虚拟机已有任务,请等待完成', 409); } + } - $host = CloudHost::where('id', (int) $instance->host_id) - ->lockForUpdate()->first(); - $agent = CloudHostAgent::where('host_id', (int) $instance->host_id) + $lockedInstanceId = $existingHint instanceof CloudPublicNetworkTransition + ? (int) $existingHint->instance_id + : $command->instanceId(); + $instance = CloudInstance::where('id', $lockedInstanceId) + ->lockForUpdate()->first(); + if (!$instance instanceof CloudInstance + || ($command->kind() === 'user_paid' + && (int) $instance->user_id + !== $command->actorUserId())) { + throw new CloudError('虚拟机不满足公网网络变更条件', 409); + } + + if ($existingHint instanceof CloudPublicNetworkTransition) { + [$replayHost] = self::lockHostBoundaryLocked( + (int) $existingHint->host_id + ); + $existing = CloudPublicNetworkTransition::where( + 'id', (int) $existingHint->id + )->where('transition_key', $command->transitionKey()) ->lockForUpdate()->first(); - $resourcePool = CloudResourcePool::where( - 'id', (int) $instance->resource_pool_id - )->lockForUpdate()->first(); - if (!$host instanceof CloudHost - || !(bool) $host->enabled - || !$agent instanceof CloudHostAgent || !$agent->isReady() - || !$resourcePool instanceof CloudResourcePool - || (int) ($resourcePool->line_id ?? 0) < 1) { - throw new CloudError('宿主机公网网络变更能力不可用', 409); - } - - // Hints identify the rows that must be locked in canonical order. - // Every identity is revalidated after all locks are acquired. - $reservationId = (int) $instance->reservation_id; - $claimHints = DB::table('cloud_resource_reservation_ips') - ->where('reservation_id', $reservationId) - ->orderBy('id')->get()->all(); - $sourceAttachmentIds = []; - $sourcePoolIds = []; - foreach ($claimHints as $hint) { - if ((int) ($hint->active_claim ?? 0) !== 1) { - continue; - } - $sourceAttachmentIds[(int) $hint->network_attachment_id] = true; - $sourcePoolIds[(int) $hint->public_ip_pool_id] = true; - } - if (count($sourceAttachmentIds) !== 1 - || count($sourcePoolIds) !== 2) { - throw new CloudError('虚拟机活动双栈公网账本不完整', 409); - } - $sourceAttachmentId = (int) array_key_first($sourceAttachmentIds); + if (!$existing instanceof CloudPublicNetworkTransition) { + throw new CloudError('公网网络变更幂等身份冲突', 409); + } + return self::replayLocked( + $existing, + $command, + $existingParent, + $existingJob, + $instance, + $replayHost + ); + } - $targetPoolHints = []; - foreach ($command->targets() as $target) { - if ($target === null) { - continue; - } - $pool = CloudPublicIpPool::where( - 'host_id', (int) $instance->host_id - )->where('pool_key', $target['pool_key'])->first(); - if (!$pool instanceof CloudPublicIpPool) { - throw new CloudError('目标公网地址池不存在', 409); - } - $targetPoolHints[(int) $pool->id] = $pool; - } - if ($command->kind() === 'user_paid') { - foreach (CloudPublicIpPool::where( - 'host_id', (int) $instance->host_id - )->where('network_attachment_id', $sourceAttachmentId) - ->where('address_family', 4)->orderBy('id')->get() as $pool) { - $targetPoolHints[(int) $pool->id] = $pool; - } - } + try { + $spec = json_decode( + (string) $instance->spec_json, + true, + 64, + JSON_THROW_ON_ERROR + ); + } catch (\Throwable) { + $spec = null; + } + if ((string) $instance->status !== 'active' + || !$instance->serviceAvailable() + || (int) ($instance->host_id ?? 0) < 1 + || (int) ($instance->reservation_id ?? 0) < 1 + || (int) ($instance->resource_pool_id ?? 0) < 1 + || !is_array($spec) || array_is_list($spec) + || ($spec['managed'] ?? false) !== true + || ($spec['os_source'] ?? null) !== 'cloud_template') { + throw new CloudError('虚拟机不满足公网网络变更条件', 409); + } + CloudInstanceMigrationService::assertInstanceMutationAvailable( + $instance + ); + if (DB::table('cloud_backup_recoveries') + ->where('instance_id', (int) $instance->id) + ->whereIn('state', ['recovering', 'uncertain'])->exists()) { + throw new CloudError('虚拟机正在恢复备份,请等待完成', 409); + } + // The instance row serializes all supported mutation writers. + // This is intentionally an unlocked predicate: Worker finalizers + // lock job before instance, so taking job locks here would reverse + // that global order and create a job/instance deadlock. + if (CloudJobService::blockingQueryForInstance( + (int) $instance->id + )->exists()) { + throw new CloudError('虚拟机已有任务,请等待完成', 409); + } - $attachmentIds = array_keys($sourceAttachmentIds); - $poolIds = array_keys($sourcePoolIds); - foreach ($targetPoolHints as $pool) { - $attachmentIds[] = (int) $pool->network_attachment_id; - $poolIds[] = (int) $pool->id; - } - $attachmentIds = array_values(array_unique($attachmentIds)); - $poolIds = array_values(array_unique($poolIds)); - sort($attachmentIds, SORT_NUMERIC); - sort($poolIds, SORT_NUMERIC); - $attachments = CloudHostNetworkAttachment::whereIn( - 'id', $attachmentIds - )->orderBy('id')->lockForUpdate()->get()->keyBy('id'); - $pools = CloudPublicIpPool::whereIn('id', $poolIds) - ->orderBy('id')->lockForUpdate()->get()->keyBy('id'); - if ($attachments->count() !== count($attachmentIds) - || $pools->count() !== count($poolIds)) { - throw new CloudError('公网网络候选目录发生变化', 409); - } + [$host, $agent] = self::lockHostBoundaryLocked( + (int) $instance->host_id + ); - if ($command->kind() === 'user_paid' - && $command->priceCents() !== - CloudIpChangeBillingService::priceCents()) { - throw new CloudError( - '付费更换 IP 的价格已发生变化,请刷新后重试', - 409 - ); + $active = CloudPublicNetworkTransition::where( + 'instance_id', (int) $instance->id + )->where('active_transition', 1)->lockForUpdate()->first(); + if ($active instanceof CloudPublicNetworkTransition) { + throw new CloudError('虚拟机已有公网网络变更正在处理', 409); + } + + $resourcePool = CloudResourcePool::where( + 'id', (int) $instance->resource_pool_id + )->lockForUpdate()->first(); + if (!$host instanceof CloudHost + || !(bool) $host->enabled + || !$agent instanceof CloudHostAgent || !$agent->isReady() + || !$resourcePool instanceof CloudResourcePool + || (int) ($resourcePool->line_id ?? 0) < 1) { + throw new CloudError('宿主机公网网络变更能力不可用', 409); + } + + // Hints identify the rows that must be locked in canonical order. + // Every identity is revalidated after all locks are acquired. + $reservationId = (int) $instance->reservation_id; + $claimHints = DB::table('cloud_resource_reservation_ips') + ->where('reservation_id', $reservationId) + ->orderBy('id')->get()->all(); + $sourceAttachmentIds = []; + $sourcePoolIds = []; + foreach ($claimHints as $hint) { + if ((int) ($hint->active_claim ?? 0) !== 1) { + continue; } + $sourceAttachmentIds[(int) $hint->network_attachment_id] = true; + $sourcePoolIds[(int) $hint->public_ip_pool_id] = true; + } + if (count($sourceAttachmentIds) !== 1 + || count($sourcePoolIds) !== 2) { + throw new CloudError('虚拟机活动双栈公网账本不完整', 409); + } + $sourceAttachmentId = (int) array_key_first($sourceAttachmentIds); - $reservation = CloudResourceReservation::where( - 'id', $reservationId - )->lockForUpdate()->first(); - $allClaims = DB::table('cloud_resource_reservation_ips') - ->where('reservation_id', $reservationId) - ->orderBy('address_family')->orderBy('id') - ->lockForUpdate()->get()->all(); - if (!$reservation instanceof CloudResourceReservation - || (string) $reservation->state !== 'committed' - || (int) $reservation->host_id !== (int) $instance->host_id - || (int) $reservation->instance_id !== (int) $instance->id - || (int) $reservation->resource_pool_id - !== (int) $instance->resource_pool_id) { - throw new CloudError('虚拟机公网资源预留账本不一致', 409); - } - $activeClaims = []; - foreach ($allClaims as $claim) { - if ((int) ($claim->active_claim ?? 0) === 1) { - $activeClaims[(int) $claim->address_family] = $claim; - } + $targetPoolHints = []; + foreach ($command->targets() as $target) { + if ($target === null) { + continue; } - if (count($activeClaims) !== 2 - || !isset($activeClaims[4], $activeClaims[6])) { - throw new CloudError('虚拟机活动双栈公网声明不完整', 409); + $pool = CloudPublicIpPool::where( + 'host_id', (int) $instance->host_id + )->where('pool_key', $target['pool_key'])->first(); + if (!$pool instanceof CloudPublicIpPool) { + throw new CloudError('目标公网地址池不存在', 409); } + $targetPoolHints[(int) $pool->id] = $pool; + } + if ($command->kind() === 'user_paid') { + foreach (CloudPublicIpPool::where( + 'host_id', (int) $instance->host_id + )->where('network_attachment_id', $sourceAttachmentId) + ->where('address_family', 4)->orderBy('id')->get() as $pool) { + $targetPoolHints[(int) $pool->id] = $pool; + } + } - // The host lock serializes every allocator. Lock all concrete rows - // for the exact candidate pools in primary-key order so automatic - // selection and explicit selection share one race-free ledger. - $ips = CloudPublicIp::where('host_id', (int) $instance->host_id) - ->whereIn('public_ip_pool_id', $poolIds) - ->orderBy('id')->lockForUpdate()->get()->keyBy('id'); - $sourceIps = []; - foreach (self::FAMILIES as $family) { - $sourceId = (int) $activeClaims[$family]->public_ip_id; - $sourceIp = $ips->get($sourceId); - if (!$sourceIp instanceof CloudPublicIp) { - // A corrupt cross-catalog source must not be hidden by an - // extra out-of-order lock. - throw new CloudError('活动公网地址不在锁定目录中', 409); - } - self::assertCommittedSourceLeg( - $activeClaims[$family], - $sourceIp, - $reservationId, - (int) $instance->id - ); - $sourceIps[$family] = $sourceIp; + $attachmentIds = array_keys($sourceAttachmentIds); + $poolIds = array_keys($sourcePoolIds); + foreach ($targetPoolHints as $pool) { + $attachmentIds[] = (int) $pool->network_attachment_id; + $poolIds[] = (int) $pool->id; + } + $attachmentIds = array_values(array_unique($attachmentIds)); + $poolIds = array_values(array_unique($poolIds)); + sort($attachmentIds, SORT_NUMERIC); + sort($poolIds, SORT_NUMERIC); + $attachments = CloudHostNetworkAttachment::whereIn( + 'id', $attachmentIds + )->orderBy('id')->lockForUpdate()->get()->keyBy('id'); + $pools = CloudPublicIpPool::whereIn('id', $poolIds) + ->orderBy('id')->lockForUpdate()->get()->keyBy('id'); + if ($attachments->count() !== count($attachmentIds) + || $pools->count() !== count($poolIds)) { + throw new CloudError('公网网络候选目录发生变化', 409); + } + + if ($command->kind() === 'user_paid' + && $command->priceCents() !== + CloudIpChangeBillingService::priceCents()) { + throw new CloudError( + '付费更换 IP 的价格已发生变化,请刷新后重试', + 409 + ); + } + + $reservation = CloudResourceReservation::where( + 'id', $reservationId + )->lockForUpdate()->first(); + $allClaims = DB::table('cloud_resource_reservation_ips') + ->where('reservation_id', $reservationId) + ->orderBy('address_family')->orderBy('id') + ->lockForUpdate()->get()->all(); + if (!$reservation instanceof CloudResourceReservation + || (string) $reservation->state !== 'committed' + || (int) $reservation->host_id !== (int) $instance->host_id + || (int) $reservation->instance_id !== (int) $instance->id + || (int) $reservation->resource_pool_id + !== (int) $instance->resource_pool_id) { + throw new CloudError('虚拟机公网资源预留账本不一致', 409); + } + $activeClaims = []; + foreach ($allClaims as $claim) { + if ((int) ($claim->active_claim ?? 0) === 1) { + $activeClaims[(int) $claim->address_family] = $claim; } + } + if (count($activeClaims) !== 2 + || !isset($activeClaims[4], $activeClaims[6])) { + throw new CloudError('虚拟机活动双栈公网声明不完整', 409); + } + + // The host lock serializes every allocator. Lock all concrete rows + // for the exact candidate pools in primary-key order so automatic + // selection and explicit selection share one race-free ledger. + $ips = CloudPublicIp::where('host_id', (int) $instance->host_id) + ->whereIn('public_ip_pool_id', $poolIds) + ->orderBy('id')->lockForUpdate()->get()->keyBy('id'); + $sourceIps = []; + foreach (self::FAMILIES as $family) { + $sourceId = (int) $activeClaims[$family]->public_ip_id; + $sourceIp = $ips->get($sourceId); + if (!$sourceIp instanceof CloudPublicIp) { + // A corrupt cross-catalog source must not be hidden by an + // extra out-of-order lock. + throw new CloudError('活动公网地址不在锁定目录中', 409); + } + self::assertCommittedSourceLeg( + $activeClaims[$family], + $sourceIp, + $reservationId, + (int) $instance->id + ); + $sourceIps[$family] = $sourceIp; + } + + $sourceAttachment = $attachments->get($sourceAttachmentId); + $sourceSnapshot = self::snapshot( + $instance, + $agent, + $sourceAttachment, + $activeClaims, + $sourceIps, + $pools, + true, + (int) $resourcePool->line_id + ); + $sourceSet = CloudActivePublicNetworkClaimSet::fromLedger( + $reservationId, + array_values($activeClaims), + [ + (int) $sourceIps[4]->id => $sourceIps[4], + (int) $sourceIps[6]->id => $sourceIps[6], + ], + $sourceSnapshot + ); + self::assertInstanceSnapshot($instance, $sourceSnapshot); - $sourceAttachment = $attachments->get($sourceAttachmentId); - $sourceSnapshot = self::snapshot( + [$targetAttachment, $targetClaims, $targetIps, $changed] = + self::selectTargetLocked( + $command, $instance, $agent, $sourceAttachment, $activeClaims, $sourceIps, + $attachments, $pools, - true, + $ips, (int) $resourcePool->line_id ); - $sourceSet = CloudActivePublicNetworkClaimSet::fromLedger( - $reservationId, - array_values($activeClaims), - [ - (int) $sourceIps[4]->id => $sourceIps[4], - (int) $sourceIps[6]->id => $sourceIps[6], - ], - $sourceSnapshot - ); - self::assertInstanceSnapshot($instance, $sourceSnapshot); + $targetSnapshot = self::snapshot( + $instance, + $agent, + $targetAttachment, + $targetClaims, + $targetIps, + $pools, + false, + (int) $resourcePool->line_id + ); + if (hash_equals( + $sourceSnapshot->fingerprint(), + $targetSnapshot->fingerprint() + )) { + throw new CloudError('目标公网网络与当前配置相同', 409); + } - [$targetAttachment, $targetClaims, $targetIps, $changed] = - self::selectTargetLocked( - $command, - $instance, - $agent, - $sourceAttachment, - $activeClaims, - $sourceIps, - $attachments, - $pools, - $ips, - (int) $resourcePool->line_id - ); - $targetSnapshot = self::snapshot( + $operationId = bin2hex(random_bytes(16)); + $now = time(); + $invoice = null; + if ($command->kind() === 'user_paid') { + $invoice = CloudIpChangeBillingService::chargeTransitionLocked( $instance, - $agent, - $targetAttachment, - $targetClaims, - $targetIps, - $pools, - false, - (int) $resourcePool->line_id + $actorUser, + $command ); - if (hash_equals( - $sourceSnapshot->fingerprint(), - $targetSnapshot->fingerprint() - )) { - throw new CloudError('目标公网网络与当前配置相同', 409); - } - - $operationId = bin2hex(random_bytes(16)); - $now = time(); - $invoice = null; - if ($command->kind() === 'user_paid') { - $invoice = CloudIpChangeBillingService::chargeTransitionLocked( - $instance, - $actorUser, - $command - ); - } + } - $transition = new CloudPublicNetworkTransition(); - $transition->transition_key = $command->transitionKey(); - $transition->operation_id = $operationId; - $transition->kind = $command->kind(); - $transition->instance_id = (int) $instance->id; - $transition->host_id = (int) $instance->host_id; - $transition->reservation_id = $reservationId; - $transition->invoice_id = $invoice?->id; - $transition->pool_replacement_batch_item_id = - $command->batchItemId(); - $transition->requested_by_user_id = $command->actorUserId(); - $transition->source_network_attachment_id = - (int) $sourceAttachment->id; - $transition->target_network_attachment_id = - (int) $targetAttachment->id; - $transition->source_claim_set_sha256 = $sourceSet->digest(); - $transition->target_claim_set_sha256 = str_repeat('0', 64); - $transition->source_public_network_json = self::json($sourceSnapshot); - $transition->source_public_network_sha256 = - $sourceSnapshot->fingerprint(); - $transition->target_public_network_json = self::json($targetSnapshot); - $transition->target_public_network_sha256 = - $targetSnapshot->fingerprint(); - $transition->state = 'reserved'; - $transition->effect_resolution = 'not_applicable'; - $transition->active_transition = 1; - $transition->last_error_code = ''; - $transition->effect_resolved_at = null; - $transition->terminal_at = null; - $transition->created_at = $transition->updated_at = $now; - $transition->save(); + $transition = new CloudPublicNetworkTransition(); + $transition->transition_key = $command->transitionKey(); + $transition->operation_id = $operationId; + $transition->kind = $command->kind(); + $transition->instance_id = (int) $instance->id; + $transition->host_id = (int) $instance->host_id; + $transition->reservation_id = $reservationId; + $transition->invoice_id = $invoice?->id; + $transition->pool_replacement_batch_item_id = + $command->batchItemId(); + $transition->requested_by_user_id = $command->actorUserId(); + $transition->source_network_attachment_id = + (int) $sourceAttachment->id; + $transition->target_network_attachment_id = + (int) $targetAttachment->id; + $transition->source_claim_set_sha256 = $sourceSet->digest(); + $transition->target_claim_set_sha256 = str_repeat('0', 64); + $transition->source_public_network_json = self::json($sourceSnapshot); + $transition->source_public_network_sha256 = + $sourceSnapshot->fingerprint(); + $transition->target_public_network_json = self::json($targetSnapshot); + $transition->target_public_network_sha256 = + $targetSnapshot->fingerprint(); + $transition->state = 'reserved'; + $transition->effect_resolution = 'not_applicable'; + $transition->active_transition = 1; + $transition->last_error_code = ''; + $transition->effect_resolved_at = null; + $transition->terminal_at = null; + $transition->created_at = $transition->updated_at = $now; + $transition->save(); - $effectiveClaims = $activeClaims; - $effectiveIps = $sourceIps; - foreach (self::FAMILIES as $family) { - $sourceClaim = $activeClaims[$family]; - if (!isset($changed[$family])) { - self::insertTransitionLeg( - (int) $transition->id, - $family, - (int) $sourceClaim->id, - null, - false, - $now - ); - continue; - } - $targetIp = $targetIps[$family]; - $targetPool = $pools->get( - (int) $targetIp->public_ip_pool_id - ); - $generation = self::nextGeneration($allClaims, $family); - $claimId = DB::table('cloud_resource_reservation_ips') - ->insertGetId([ - 'reservation_id' => $reservationId, - 'address_family' => $family, - 'claim_generation' => $generation, - 'active_claim' => null, - 'network_attachment_id' => - (int) $targetPool->network_attachment_id, - 'public_ip_pool_id' => (int) $targetPool->id, - 'public_ip_id' => (int) $targetIp->id, - 'operation_id' => $operationId, - 'state' => 'reserved', - 'released_at' => null, - 'release_reason' => '', - 'created_at' => $now, - 'updated_at' => $now, - ]); - $updated = CloudPublicIp::where('id', (int) $targetIp->id) - ->where('state', 'available') - ->whereNull('reservation_id') - ->whereNull('reservation_ip_claim_id') - ->whereNull('instance_id') - ->whereNull('operation_id')->update([ - 'state' => 'reserved', - 'reservation_id' => $reservationId, - 'reservation_ip_claim_id' => $claimId, - 'instance_id' => null, - 'operation_id' => $operationId, - 'updated_at' => $now, - ]); - if ($updated !== 1) { - throw new CloudError('目标公网地址声明绑定失败', 409); - } - $targetClaim = DB::table('cloud_resource_reservation_ips') - ->where('id', $claimId)->first(); - $boundIp = CloudPublicIp::where('id', (int) $targetIp->id) - ->first(); - $effectiveClaims[$family] = $targetClaim; - $effectiveIps[$family] = $boundIp; + $effectiveClaims = $activeClaims; + $effectiveIps = $sourceIps; + foreach (self::FAMILIES as $family) { + $sourceClaim = $activeClaims[$family]; + if (!isset($changed[$family])) { self::insertTransitionLeg( (int) $transition->id, $family, (int) $sourceClaim->id, - $claimId, - true, + null, + false, $now ); + continue; } - - $targetIdentity = CloudPublicNetworkClaimSetIdentity::fromLedger( - $reservationId, - $effectiveClaims, - [ - (int) $effectiveIps[4]->id => $effectiveIps[4], - (int) $effectiveIps[6]->id => $effectiveIps[6], - ], - $targetSnapshot + $targetIp = $targetIps[$family]; + $targetPool = $pools->get( + (int) $targetIp->public_ip_pool_id ); - $transition->target_claim_set_sha256 = $targetIdentity->digest(); - - $job = new CloudJob(); - $job->operation_id = $operationId; - $job->request_idempotency_key = $command->requestKey(); - $job->request_fingerprint = $command->fingerprint(); - $job->parent_job_id = null; - $job->invoice_id = $invoice?->id; - $job->instance_id = (int) $instance->id; - $job->public_network_transition_id = (int) $transition->id; - $job->user_id = $command->actorUserId(); - $job->host_id = (int) $instance->host_id; - $job->kind = 'network_change'; - $job->execution_mode = 'agent'; - $job->status = 'queued'; - $job->sealed_payload = CloudSecrets::encrypt([ - 'method' => 'POST', - 'request_target' => '/v1/vms/' - . rawurlencode((string) $instance->uuid) . '/network', - 'body' => [ - 'public_network' => $targetSnapshot->toArray(), - 'confirm_restart' => true, - ], - 'owner_id' => (int) $instance->user_id, - 'can_manage' => true, - 'public_network_transition_id' => (int) $transition->id, - 'source_claim_set_sha256' => $sourceSet->digest(), - 'target_claim_set_sha256' => $targetIdentity->digest(), - 'price_cents' => $command->priceCents(), - 'charged_amount' => $invoice === null - ? null : (string) $invoice->amount, - 'currency' => $invoice === null - ? null : (string) $invoice->currency, - ], 'job-payload'); - $job->available_at = $job->created_at = $job->updated_at = $now; - $job->save(); - - $transition->state = 'queued'; - $transition->updated_at = $now; - $transition->save(); - return new CloudPublicNetworkTransitionDispatch( + $generation = self::nextGeneration($allClaims, $family); + $claimId = DB::table('cloud_resource_reservation_ips') + ->insertGetId([ + 'reservation_id' => $reservationId, + 'address_family' => $family, + 'claim_generation' => $generation, + 'active_claim' => null, + 'network_attachment_id' => + (int) $targetPool->network_attachment_id, + 'public_ip_pool_id' => (int) $targetPool->id, + 'public_ip_id' => (int) $targetIp->id, + 'operation_id' => $operationId, + 'state' => 'reserved', + 'released_at' => null, + 'release_reason' => '', + 'created_at' => $now, + 'updated_at' => $now, + ]); + $updated = CloudPublicIp::where('id', (int) $targetIp->id) + ->where('state', 'available') + ->whereNull('reservation_id') + ->whereNull('reservation_ip_claim_id') + ->whereNull('instance_id') + ->whereNull('operation_id')->update([ + 'state' => 'reserved', + 'reservation_id' => $reservationId, + 'reservation_ip_claim_id' => $claimId, + 'instance_id' => null, + 'operation_id' => $operationId, + 'updated_at' => $now, + ]); + if ($updated !== 1) { + throw new CloudError('目标公网地址声明绑定失败', 409); + } + $targetClaim = DB::table('cloud_resource_reservation_ips') + ->where('id', $claimId)->first(); + $boundIp = CloudPublicIp::where('id', (int) $targetIp->id) + ->first(); + $effectiveClaims[$family] = $targetClaim; + $effectiveIps[$family] = $boundIp; + self::insertTransitionLeg( (int) $transition->id, - (int) $job->id, - $operationId, - $targetSnapshot, - false + $family, + (int) $sourceClaim->id, + $claimId, + true, + $now ); - }, 3); + } + + $targetIdentity = CloudPublicNetworkClaimSetIdentity::fromLedger( + $reservationId, + $effectiveClaims, + [ + (int) $effectiveIps[4]->id => $effectiveIps[4], + (int) $effectiveIps[6]->id => $effectiveIps[6], + ], + $targetSnapshot + ); + $transition->target_claim_set_sha256 = $targetIdentity->digest(); + + $job = new CloudJob(); + $job->operation_id = $operationId; + $job->request_idempotency_key = $command->requestKey(); + $job->request_fingerprint = $command->fingerprint(); + $job->parent_job_id = null; + $job->invoice_id = $invoice?->id; + $job->instance_id = (int) $instance->id; + $job->public_network_transition_id = (int) $transition->id; + $job->user_id = $command->actorUserId(); + $job->host_id = (int) $instance->host_id; + $job->kind = 'network_change'; + $job->execution_mode = 'agent'; + $job->status = 'queued'; + $job->sealed_payload = CloudSecrets::encrypt([ + 'method' => 'POST', + 'request_target' => '/v1/vms/' + . rawurlencode((string) $instance->uuid) . '/network', + 'body' => [ + 'public_network' => $targetSnapshot->toArray(), + 'confirm_restart' => true, + ], + 'owner_id' => (int) $instance->user_id, + 'can_manage' => true, + 'public_network_transition_id' => (int) $transition->id, + 'source_claim_set_sha256' => $sourceSet->digest(), + 'target_claim_set_sha256' => $targetIdentity->digest(), + 'price_cents' => $command->priceCents(), + 'charged_amount' => $invoice === null + ? null : (string) $invoice->amount, + 'currency' => $invoice === null + ? null : (string) $invoice->currency, + ], 'job-payload'); + $job->available_at = $job->created_at = $job->updated_at = $now; + $job->save(); + + $transition->state = 'queued'; + $transition->updated_at = $now; + $transition->save(); + return new CloudPublicNetworkTransitionDispatch( + (int) $transition->id, + (int) $job->id, + $operationId, + $targetSnapshot, + false + ); } /** Publish the dispatch boundary without consulting mutable catalogs. */ @@ -559,6 +616,7 @@ final class CloudPublicNetworkTransitionWorkflow } $instance = CloudInstance::where('id', (int) $job->instance_id) ->lockForUpdate()->first(); + self::lockHostBoundaryLocked((int) ($job->host_id ?? 0)); $transition = CloudPublicNetworkTransition::where( 'id', (int) $job->public_network_transition_id @@ -691,6 +749,7 @@ final class CloudPublicNetworkTransitionWorkflow } $instance = CloudInstance::where('id', (int) $job->instance_id) ->lockForUpdate()->first(); + self::lockHostBoundaryLocked((int) ($job->host_id ?? 0)); $transition = CloudPublicNetworkTransition::where( 'id', (int) $job->public_network_transition_id @@ -910,6 +969,7 @@ final class CloudPublicNetworkTransitionWorkflow }; $instance = CloudInstance::where('id', (int) $job->instance_id) ->lockForUpdate()->first(); + self::lockHostBoundaryLocked((int) ($job->host_id ?? 0)); $transition = CloudPublicNetworkTransition::where( 'id', (int) $job->public_network_transition_id @@ -1055,6 +1115,7 @@ final class CloudPublicNetworkTransitionWorkflow } $instance = CloudInstance::where('id', (int) $job->instance_id) ->lockForUpdate()->first(); + self::lockHostBoundaryLocked((int) ($job->host_id ?? 0)); $transition = CloudPublicNetworkTransition::where( 'id', (int) $job->public_network_transition_id @@ -1262,23 +1323,55 @@ final class CloudPublicNetworkTransitionWorkflow return CloudJob::find($jobId) ?? $snapshot; } + /** @return array{0:CloudHost,1:CloudHostAgent} */ + private static function lockHostBoundaryLocked(int $hostId): array + { + if ($hostId < 1) { + throw new CloudError('公网网络变更宿主机身份缺失', 409); + } + $host = CloudHost::where('id', $hostId) + ->lockForUpdate()->first(); + $agent = CloudHostAgent::where('host_id', $hostId) + ->lockForUpdate()->first(); + if (!$host instanceof CloudHost + || !$agent instanceof CloudHostAgent) { + throw new CloudError('公网网络变更宿主机账本缺失', 409); + } + return [$host, $agent]; + } + private static function replayLocked( CloudPublicNetworkTransition $transition, - CloudPublicNetworkTransitionCommand $command + CloudPublicNetworkTransitionCommand $command, + ?CloudJob $parent, + ?CloudJob $job, + CloudInstance $instance, + CloudHost $host ): CloudPublicNetworkTransitionDispatch { - // Worker finalization locks job before instance/transition. The actor, - // instance and transition already fence this immutable association, so - // replay reads the job without reversing the global lock order. - $job = CloudJob::where( - 'public_network_transition_id', (int) $transition->id - )->where('operation_id', (string) $transition->operation_id) - ->orderBy('id')->first(); $transitionInvoiceId = $transition->invoice_id === null ? null : (int) $transition->invoice_id; $jobInvoiceId = $job?->invoice_id === null ? null : (int) $job->invoice_id; + $terminalResolution = (string) $transition->effect_resolution; + $terminalReplay = $job instanceof CloudJob + && in_array($terminalResolution, + ['applied', 'not_applied'], true) + && self::terminalMatches($transition, $terminalResolution) + && (string) $job->phase === 'terminal' + && in_array((string) $job->outcome, + ['succeeded', 'failed'], true) + && (string) $job->status === (string) $job->outcome + && $job->terminal_at !== null + && hash_equals( + (string) $job->effect_resolution, + $terminalResolution + ); if (!$job instanceof CloudJob || (int) $transition->instance_id !== $command->instanceId() + || (int) $transition->instance_id !== (int) $instance->id + || (int) $transition->host_id !== (int) $host->id + || (!$terminalReplay + && (int) $instance->host_id !== (int) $host->id) || (int) $transition->requested_by_user_id !== $command->actorUserId() || (string) $transition->kind !== $command->kind() @@ -1293,6 +1386,20 @@ final class CloudPublicNetworkTransitionWorkflow $command->fingerprint() ) || (string) $job->kind !== 'network_change' + || ($command->kind() !== 'pool_replacement' + && $job->parent_job_id !== null) + || ($command->kind() === 'pool_replacement' + && (!$parent instanceof CloudJob + || (int) $parent->id !== (int) $command->parentJobId() + || (string) $parent->kind + !== 'public_ip_pool_replacement_batch' + || (int) $parent->user_id !== $command->actorUserId() + || (string) $parent->execution_mode !== 'local' + || $parent->parent_job_id !== null + || $parent->host_id !== null + || $parent->public_network_transition_id !== null + || (int) ($job->parent_job_id ?? 0) + !== (int) $command->parentJobId())) || (int) $job->instance_id !== (int) $transition->instance_id || (int) $job->user_id !== (int) $transition->requested_by_user_id @@ -1356,8 +1463,12 @@ final class CloudPublicNetworkTransitionWorkflow )) { throw new CloudError('公网网络变更目标快照摘要不一致', 409); } - if (in_array((string) $transition->state, - ['succeeded', 'failed'], true)) { + $terminalState = in_array((string) $transition->state, + ['succeeded', 'failed'], true); + if ($terminalState && !$terminalReplay) { + throw new CloudError('公网网络变更终态形状不一致', 409); + } + if ($terminalReplay) { try { $identity = CloudOperationRecoveryIdentity::expected($job); } catch (\Throwable) { @@ -1610,6 +1721,7 @@ final class CloudPublicNetworkTransitionWorkflow CloudPublicNetworkSnapshot $source, CloudPublicNetworkSnapshot $target ): array { + self::lockFrozenTopologyLocked($transition, $source, $target); $reservation = CloudResourceReservation::where( 'id', (int) $transition->reservation_id @@ -1784,6 +1896,40 @@ final class CloudPublicNetworkTransitionWorkflow ]; } + private static function lockFrozenTopologyLocked( + CloudPublicNetworkTransition $transition, + CloudPublicNetworkSnapshot $source, + CloudPublicNetworkSnapshot $target + ): void { + $attachmentIds = array_values(array_unique([ + (int) $transition->source_network_attachment_id, + (int) $transition->target_network_attachment_id, + ])); + sort($attachmentIds, SORT_NUMERIC); + $attachments = CloudHostNetworkAttachment::whereIn( + 'id', $attachmentIds + )->orderBy('id')->lockForUpdate()->get(); + if ($attachments->count() !== count($attachmentIds)) { + throw new CloudError('公网网络变更冻结附件缺失', 409); + } + + $poolKeys = []; + foreach ([$source->toArray(), $target->toArray()] as $snapshot) { + foreach (['ipv4', 'ipv6'] as $leg) { + $poolKeys[(string) $snapshot[$leg]['pool_key']] = true; + } + } + $poolKeys = array_keys($poolKeys); + sort($poolKeys, SORT_STRING); + $pools = CloudPublicIpPool::where( + 'host_id', (int) $transition->host_id + )->whereIn('pool_key', $poolKeys)->orderBy('id') + ->lockForUpdate()->get(); + if ($pools->count() !== count($poolKeys)) { + throw new CloudError('公网网络变更冻结地址池缺失', 409); + } + } + private static function assertFrozenRowIdentity( object $claim, CloudPublicIp $ip, diff --git a/web/tests/CloudPublicIpPoolReplacementBatchServiceTest.php b/web/tests/CloudPublicIpPoolReplacementBatchServiceTest.php index 52fbcac9..558c75b6 100644 --- a/web/tests/CloudPublicIpPoolReplacementBatchServiceTest.php +++ b/web/tests/CloudPublicIpPoolReplacementBatchServiceTest.php @@ -525,6 +525,102 @@ namespace { && Capsule::table('cloud_public_ips')->whereIn('id', [71, 72, 73, 74]) ->where('state', 'available')->count() === 4, 'Gated pool replacement created durable or address-side effects.'); + + // Lock-order convergence must use the same locked item collection it + // updates. One unknown projection followed by one resolved projection is + // sufficient to release the blocked batch; no second hook may be needed. + $job = static function (int $id, string $operationId, string $requestKey, + string $kind, ?int $parentId, ?int $instanceId + ) use ($now): array { + return [ + 'id' => $id, + 'operation_id' => $operationId, + 'request_idempotency_key' => $requestKey, + 'request_fingerprint' => hash('sha256', $requestKey), + 'parent_job_id' => $parentId, + 'instance_id' => $instanceId, + 'user_id' => 99, + 'host_id' => $parentId === null ? null : 1, + 'kind' => $kind, + 'execution_mode' => $parentId === null ? 'local' : 'agent', + 'phase' => 'accepted', + 'outcome' => null, + 'status' => 'queued', + 'sealed_payload' => '{}', + 'attempts' => 0, + 'dispatch_attempts' => 0, + 'dispatch_deadline_at' => $now + 300, + 'effect_resolution' => 'not_applicable', + 'finalization_incident' => 0, + 'lease_until' => 0, + 'available_at' => $now, + 'created_at' => $now, + 'updated_at' => $now, + ]; + }; + Capsule::table('cloud_jobs')->insert([ + $job(900, str_repeat('9', 32), 'batch-lock-parent-0001', + 'public_ip_pool_replacement_batch', null, null), + $job(901, str_repeat('8', 32), 'batch-lock-child-0001', + 'network_change', 900, 51), + ]); + Capsule::table('cloud_public_ip_pool_replacement_batches')->insert([ + 'id' => 900, + 'batch_key' => str_repeat('7', 32), + 'host_id' => 1, + 'source_public_ip_pool_id' => 31, + 'target_public_ip_pool_id' => 32, + 'source_initial_state' => 'active', + 'parent_job_id' => 900, + 'requested_by_user_id' => 99, + 'state' => 'running', + 'active_claim' => 1, + 'last_error_code' => '', + 'terminal_at' => null, + 'created_at' => $now, + 'updated_at' => $now, + ]); + Capsule::table('cloud_public_ip_pool_replacement_batch_items')->insert([ + 'id' => 900, + 'batch_id' => 900, + 'instance_id' => 51, + 'source_reservation_id' => 41, + 'source_public_ip_id' => 61, + 'target_public_ip_id' => 71, + 'source_network_attachment_id' => 11, + 'target_network_attachment_id' => 12, + 'child_job_id' => 901, + 'state' => 'queued', + 'last_error_code' => '', + 'created_at' => $now, + 'updated_at' => $now, + ]); + $lockedChild = CloudJob::find(901); + CloudPublicIpPoolReplacementBatchService::finishChildLocked( + $lockedChild, + 'uncertain' + ); + $assert(CloudPublicIpPoolReplacementBatch::find(900)->state === 'blocked' + && Capsule::table('cloud_public_ip_pool_replacement_batch_items') + ->where('id', 900)->value('state') === 'unknown', + 'One unknown child projection did not block the batch immediately.'); + $lockedChild->phase = 'terminal'; + $lockedChild->outcome = 'failed'; + $lockedChild->status = 'failed'; + $lockedChild->effect_may_have_applied = 1; + $lockedChild->safe_to_resubmit = 0; + $lockedChild->effect_resolution = 'applied'; + $lockedChild->save(); + CloudPublicIpPoolReplacementBatchService::finishResolvedChildLocked( + $lockedChild->fresh() + ); + $resolvedBatch = CloudPublicIpPoolReplacementBatch::find(900); + $assert((string) $resolvedBatch->state === 'failed' + && $resolvedBatch->active_claim === null + && Capsule::table('cloud_public_ip_pool_replacement_batch_items') + ->where('id', 900)->value('state') === 'succeeded' + && (string) CloudJob::find(900)->phase === 'terminal', + 'One resolved child projection left the batch permanently blocked.'); echo "Cloud public IP pool replacement C10 gate tests passed\n"; return; diff --git a/web/tests/CloudPublicIpPoolReplacementSchemaContractTest.php b/web/tests/CloudPublicIpPoolReplacementSchemaContractTest.php index b1aa2f1b..ba74408e 100644 --- a/web/tests/CloudPublicIpPoolReplacementSchemaContractTest.php +++ b/web/tests/CloudPublicIpPoolReplacementSchemaContractTest.php @@ -17,6 +17,15 @@ $assert = static function (bool $condition, string $message): void { throw new RuntimeException($message); } }; +$method = static function (string $contents, string $start, string $end) use ( + $assert +): string { + $offset = strpos($contents, $start); + $limit = $offset === false ? false : strpos($contents, $end, $offset + 1); + $assert($offset !== false && $limit !== false && $limit > $offset, + 'Replacement method source is missing: ' . $start); + return substr($contents, $offset, $limit - $offset); +}; $assert(is_string($schema) && str_contains($schema, @@ -96,4 +105,70 @@ $assert(is_string($worker) && str_contains($worker, "? \$catalogStatus : 'uncertain'"), 'Worker no longer converges replacement children or excludes the local parent from dispatch.'); +$finishChild = $method( + $service, + 'public static function finishChildLocked(', + 'public static function finishResolvedChildLocked(' +); +$start = $method( + $service, + 'public static function start(', + 'public static function finishChildLocked(' +); +$transactionOffset = strpos( + $start, + '->transaction(static function () use (' +); +$assert($transactionOffset !== false, + 'Replacement start transaction source is missing.'); +$lockedStart = substr($start, $transactionOffset); +$startPositions = array_map( + static fn (string $needle): int|false => strpos($lockedStart, $needle), + [ + '$actorUser = User::where(', + '$prior = CloudJob::where(', + '$instances = CloudInstance::whereIn(', + '$host = CloudHost::where(', + '$agent = CloudHostAgent::where(', + '$attachments = CloudHostNetworkAttachment::whereIn(', + '$pools = CloudPublicIpPool::whereIn(', + '$reservations = CloudResourceReservation::whereIn(', + '$ips = CloudPublicIp::whereIn(', + ] +); +$sortedStartPositions = $startPositions; +sort($sortedStartPositions, SORT_NUMERIC); +$assert(!in_array(false, $startPositions, true) + && $startPositions === $sortedStartPositions + && count(array_unique($startPositions)) === count($startPositions), + 'Replacement start reversed actor/job/instance/host/catalog locks.'); +$finishResolved = $method( + $service, + 'public static function finishResolvedChildLocked(', + 'public static function display(' +); +$converge = $method( + $service, + 'private static function convergeLocked(', + 'private static function setPoolStateLocked(' +); +$positions = array_map( + static fn (string $needle): int|false => strpos($finishChild, $needle), + [ + '$parent = CloudJob::where(', + '$source = CloudPublicIpPool::where(', + '$batch = CloudPublicIpPoolReplacementBatch::where(', + '$items = CloudPublicIpPoolReplacementBatchItem::where(', + '$item = $items->first(', + ] +); +$sortedPositions = $positions; +sort($sortedPositions, SORT_NUMERIC); +$assert(!in_array(false, $positions, true) + && $positions === $sortedPositions + && count(array_unique($positions)) === count($positions) + && !str_contains($finishResolved, 'CloudHost::where(') + && !str_contains($converge, 'lockForUpdate()'), + 'Replacement convergence reversed parent/pool/batch locks.'); + echo "Cloud public IP pool replacement schema contract tests passed\n"; diff --git a/web/tests/CloudPublicNetworkTransitionFinalizationTest.php b/web/tests/CloudPublicNetworkTransitionFinalizationTest.php index df09cade..18df56f3 100644 --- a/web/tests/CloudPublicNetworkTransitionFinalizationTest.php +++ b/web/tests/CloudPublicNetworkTransitionFinalizationTest.php @@ -152,6 +152,38 @@ $assert($source4->state === 'released' ->value('state') === 'assigned', 'Applied transition did not atomically switch only the changed leg.'); +// Once the transition is terminal, a later maintenance migration may change +// current placement. Exact replay still returns the frozen historical result +// while an active transition (covered by begin tests) rejects the same drift. +$terminalPayload = $appliedPayload; +$terminalPayload[CloudOperationRecoveryIdentity::payloadKey()] = + CloudOperationRecoveryIdentity::persisted([ + 'action' => $appliedOperation['action'], + 'resource_id' => $appliedOperation['resource_id'], + 'request_fingerprint' => $appliedOperation['request_fingerprint'], + ]); +Capsule::table('cloud_jobs')->where('id', (int) $appliedJob->id)->update([ + 'phase' => 'terminal', + 'outcome' => 'succeeded', + 'status' => 'succeeded', + 'terminal_at' => time(), + 'effect_may_have_applied' => 0, + 'safe_to_resubmit' => 0, + 'effect_resolution' => 'applied', + 'sealed_payload' => CloudSecrets::encrypt( + $terminalPayload, + 'job-payload' + ), +]); +Capsule::table('cloud_instances')->where('id', 1) + ->update(['host_id' => 2]); +$terminalMigratedReplay = CloudPublicNetworkTransitionWorkflow::begin($admin); +$assert($terminalMigratedReplay->replayed() + && $terminalMigratedReplay->transitionId() === $dispatch->transitionId(), + 'Terminal exact replay was coupled to later instance placement.'); +Capsule::table('cloud_instances')->where('id', 1) + ->update(['host_id' => 1]); + // A successful transition must make the active claim pair authoritative even // when intentionally stale singular reservation mirrors remain present. $reservationRow = Capsule::table('cloud_resource_reservations') @@ -366,7 +398,20 @@ Capsule::table('cloud_invoices')->where('id', $normalPaidInvoiceId) // Unknown evidence can also converge to exact not-applied without releasing // the source or leaking the reserved target. -[$lateFailedJob, $lateFailedPayload] = $dispatchJob($batch->jobId()); +$lateFailedDispatch = CloudPublicNetworkTransitionWorkflow::begin( + CloudPublicNetworkTransitionCommand::adminExact( + 10, + 99, + 'admin-network-late-failed-0010', + [4 => [ + 'pool_key' => str_repeat('c', 32), + 'address_cidr' => '8.8.9.22/24', + ]] + ) +); +[$lateFailedJob, $lateFailedPayload] = $dispatchJob( + $lateFailedDispatch->jobId() +); $lateUnknownOperation = $agentOperation( $lateFailedJob, $lateFailedPayload, @@ -396,7 +441,8 @@ CloudPublicNetworkTransitionWorkflow::recordEvidence( ); $finishLocked($lateFailedJob, $lateFailedPayload, 'failed', 99, false); $assert(Capsule::table('cloud_public_network_transitions') - ->where('id', $batch->transitionId())->value('state') === 'failed' + ->where('id', $lateFailedDispatch->transitionId()) + ->value('state') === 'failed' && Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', 110)->where('active_claim', 1) ->count() === 2 @@ -586,4 +632,55 @@ $assert($terminalLedgerRejected, Capsule::table('cloud_public_ips')->where('id', 504) ->update(['host_id' => 1]); +// A finalizer cannot silently skip the global host/agent sentinel. Missing +// topology must fail before transition, claim or address projection changes. +$hostFenceDispatch = CloudPublicNetworkTransitionWorkflow::begin( + CloudPublicNetworkTransitionCommand::adminExact( + 12, + 99, + 'admin-network-host-fence-0012', + [4 => [ + 'pool_key' => str_repeat('c', 32), + 'address_cidr' => '8.8.9.23/24', + ]] + ) +); +[$hostFenceJob, $hostFencePayload] = $dispatchJob( + $hostFenceDispatch->jobId() +); +$hostFenceOperation = $agentOperation( + $hostFenceJob, + $hostFencePayload, + 'failed', + [] +); +$agentRow = Capsule::table('cloud_host_agents')->where('host_id', 1)->first(); +Capsule::table('cloud_host_agents')->where('host_id', 1)->delete(); +$hostFenceRejected = false; +try { + CloudPublicNetworkTransitionWorkflow::recordEvidence( + $hostFenceJob, + 'failed', + [], + $hostFencePayload, + $hostFenceOperation + ); +} catch (Throwable) { + $hostFenceRejected = true; +} +Capsule::table('cloud_host_agents')->insert((array) $agentRow); +$hostFenceTargetClaim = Capsule::table( + 'cloud_public_network_transition_ips' +)->where('transition_id', $hostFenceDispatch->transitionId()) + ->where('address_family', 4)->value('target_claim_id'); +$assert($hostFenceRejected + && Capsule::table('cloud_public_network_transitions') + ->where('id', $hostFenceDispatch->transitionId()) + ->value('state') === 'awaiting_receipt' + && Capsule::table('cloud_resource_reservation_ips') + ->where('id', $hostFenceTargetClaim)->value('state') === 'reserved' + && Capsule::table('cloud_public_ips')->where('id', 204) + ->value('state') === 'reserved', + 'Missing host-agent sentinel allowed partial finalization writes.'); + echo "Public-network transition finalization tests passed.\n"; diff --git a/web/tests/CloudPublicNetworkTransitionWorkflowContractTest.php b/web/tests/CloudPublicNetworkTransitionWorkflowContractTest.php index 49314011..11f5e529 100644 --- a/web/tests/CloudPublicNetworkTransitionWorkflowContractTest.php +++ b/web/tests/CloudPublicNetworkTransitionWorkflowContractTest.php @@ -10,6 +10,15 @@ $assert = static function (bool $condition, string $message): void { throw new RuntimeException($message); } }; +$method = static function (string $contents, string $start, string $end) use ( + $assert +): string { + $offset = strpos($contents, $start); + $limit = $offset === false ? false : strpos($contents, $end, $offset + 1); + $assert($offset !== false && $limit !== false && $limit > $offset, + 'Workflow method source is missing: ' . $start); + return substr($contents, $offset, $limit - $offset); +}; $root = dirname(__DIR__); $source = file_get_contents( $root . '/app/Services/Cloud/CloudPublicNetworkTransitionWorkflow.php' @@ -64,26 +73,97 @@ foreach (['userPaidAutomatic', 'adminExact', 'poolReplacement'] as $factory) { 'Transition command factory is missing: ' . $factory); } -$ordered = [ +$begin = $method( + $source, + 'public static function begin(', + 'private static function reserveTransitionLocked(' +); +$reserve = $method( + $source, + 'private static function reserveTransitionLocked(', + 'public static function markDispatched(' +); +$ordered = static function (string $body, array $needles) use ($assert): void { + $cursor = -1; + foreach ($needles as $needle) { + $position = strpos($body, $needle); + $assert($position !== false && $position > $cursor, + 'Transition lock order drifted at: ' . $needle); + $cursor = $position; + } +}; +$ordered($begin, [ "User::where('id', \$command->actorUserId())", - "CloudInstance::where('id', \$command->instanceId())", - "'transition_key', \$command->transitionKey()", - "CloudHost::where('id', (int) \$instance->host_id)", - "CloudHostAgent::where('host_id', (int) \$instance->host_id)", - "CloudResourcePool::where(\n 'id', (int) \$instance->resource_pool_id", - 'CloudHostNetworkAttachment::whereIn(', - "CloudPublicIpPool::whereIn('id', \$poolIds)", - "CloudResourceReservation::where(\n 'id', \$reservationId", - "->where('reservation_id', \$reservationId)\n ->orderBy('address_family')", - "CloudPublicIp::where('host_id', (int) \$instance->host_id)", -]; -$cursor = -1; -foreach ($ordered as $needle) { - $position = strpos($source, $needle); - $assert($position !== false && $position > $cursor, - 'Transition lock order drifted at: ' . $needle); - $cursor = $position; + 'self::reserveTransitionLocked($command, $actorUser)', +]); +$assert(substr_count($begin, 'transaction(') === 1 + && !str_contains($reserve, 'transaction('), + 'Reservation seam introduced a nested transaction.'); +$ordered($reserve, [ + '$jobHint = CloudJob::where(', + '$existingParent = CloudJob::where(', + '$existingJob = CloudJob::where(', + '$instance = CloudInstance::where(', + '[$replayHost] = self::lockHostBoundaryLocked(', + '$existing = CloudPublicNetworkTransition::where(', +]); +$assert(str_contains($source, + '(int) $parent->id !== (int) $command->parentJobId()') + && str_contains($source, + '(int) ($job->parent_job_id ?? 0)') + && str_contains($source, + '!== (int) $command->parentJobId()'), + 'Batch replay no longer binds the locked parent identity.'); +$ordered($reserve, [ + '$priorJob = CloudJob::where(', + '$instance = CloudInstance::where(', + '[$host, $agent] = self::lockHostBoundaryLocked(', + '$active = CloudPublicNetworkTransition::where(', + '$resourcePool = CloudResourcePool::where(', + '$attachments = CloudHostNetworkAttachment::whereIn(', + '$pools = CloudPublicIpPool::whereIn(\'id\', $poolIds)', + '$reservation = CloudResourceReservation::where(', + '$allClaims = DB::table(\'cloud_resource_reservation_ips\')', + '$ips = CloudPublicIp::where(\'host_id\', (int) $instance->host_id)', +]); +foreach ([ + ['public static function markJobUncertain(', + 'public static function recordEvidence('], + ['public static function recordEvidence(', + 'public static function finishJobLocked('], + ['public static function finishJobLocked(', + 'public static function publishPreparedLocked('], + ['public static function publishPreparedLocked(', + 'public static function retryPendingFinalization('], +] as [$start, $end]) { + $ordered($method($source, $start, $end), [ + '$instance = CloudInstance::where(', + 'self::lockHostBoundaryLocked(', + '$transition = CloudPublicNetworkTransition::where(', + ]); } +$retry = $method( + $source, + 'public static function retryPendingFinalization(', + 'private static function lockHostBoundaryLocked(' +); +$ordered($retry, [ + '$actor = User::where(\'id\', $userId)', + '$parent = $parentId > 0', + '$job = CloudJob::where(\'id\', $jobId)', + 'self::finishJobLocked(', +]); +$topology = $method( + $source, + 'private static function lockFrozenTopologyLocked(', + 'private static function assertFrozenRowIdentity(' +); +$ordered($topology, [ + '$attachments = CloudHostNetworkAttachment::whereIn(', + "->orderBy('id')->lockForUpdate()->get()", + '$pools = CloudPublicIpPool::where(', + "->whereIn('pool_key', \$poolKeys)->orderBy('id')", +]); $assert(str_contains($source, "\$transition->state = 'reserved';") && str_contains($source, "'active_claim' => null") diff --git a/web/tests/CloudPublicNetworkTransitionWorkflowTest.php b/web/tests/CloudPublicNetworkTransitionWorkflowTest.php index 4a9a603b..b2708c2f 100644 --- a/web/tests/CloudPublicNetworkTransitionWorkflowTest.php +++ b/web/tests/CloudPublicNetworkTransitionWorkflowTest.php @@ -492,6 +492,7 @@ namespace { $addIp(201, 13, 4, '8.8.9.20'); $addIp(202, 13, 4, '8.8.9.21'); $addIp(203, 13, 4, '8.8.9.22'); + $addIp(204, 13, 4, '8.8.9.23'); $addIp(301, 21, 4, '9.9.9.30'); $addIp(302, 21, 4, '9.9.9.31'); $addIp(401, 22, 6, '2404:c140:156:3c::1000'); @@ -638,6 +639,8 @@ namespace { '8.8.8.19', '2404:c140:156:3a::1009', 10); $addInstance(11, 111, 1021, 1022, 121, 122, '8.8.8.20', '2404:c140:156:3a::100a', 10); + $addInstance(12, 112, 1023, 1024, 123, 124, + '8.8.8.21', '2404:c140:156:3a::100b', 10); $sourceClaims = Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', 101)->where('active_claim', 1) @@ -731,6 +734,58 @@ namespace { && Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', 101)->count() === 3, 'Exact replay consulted mutable target catalog or duplicated resources.'); + Capsule::table('cloud_hosts')->where('id', 1) + ->update(['enabled' => 0]); + Capsule::table('cloud_instances')->where('id', 1)->update([ + 'status' => 'suspended', + 'service_status' => 'expired_grace', + 'spec_json' => '{}', + ]); + $mutableReplay = CloudPublicNetworkTransitionWorkflow::begin($admin); + $assert($mutableReplay->replayed() + && $mutableReplay->transitionId() === $dispatch->transitionId() + && Capsule::table('cloud_jobs')->count() === 1, + 'Exact replay was coupled to mutable host or instance eligibility.'); + Capsule::table('cloud_hosts')->where('id', 1) + ->update(['enabled' => 1]); + Capsule::table('cloud_instances')->where('id', 1)->update([ + 'status' => 'active', + 'service_status' => 'active', + 'spec_json' => json_encode([ + 'managed' => true, + 'os_source' => 'cloud_template', + 'public_network' => $sourceSnapshot( + '8.8.8.10', + '2404:c140:156:3a::1000' + ), + ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES), + ]); + Capsule::table('cloud_instances')->where('id', 1) + ->update(['host_id' => 2]); + $rejects( + static fn () => CloudPublicNetworkTransitionWorkflow::begin($admin), + 'Exact replay accepted a drifted immutable host association.' + ); + Capsule::table('cloud_instances')->where('id', 1) + ->update(['host_id' => 1]); + Capsule::table('cloud_public_network_transitions') + ->where('id', $dispatch->transitionId())->update([ + 'state' => 'succeeded', + 'effect_resolution' => 'applied', + ]); + Capsule::table('cloud_instances')->where('id', 1) + ->update(['host_id' => 2]); + $rejects( + static fn () => CloudPublicNetworkTransitionWorkflow::begin($admin), + 'Corrupt terminal-shaped replay bypassed the active host fence.' + ); + Capsule::table('cloud_instances')->where('id', 1) + ->update(['host_id' => 1]); + Capsule::table('cloud_public_network_transitions') + ->where('id', $dispatch->transitionId())->update([ + 'state' => 'queued', + 'effect_resolution' => 'not_applicable', + ]); Capsule::table('cloud_public_ip_pools')->where('id', 13) ->update(['state' => 'active']); @@ -1303,6 +1358,64 @@ namespace { Capsule::table('cloud_invoices')->where('id', $invoice->id) ->update(['amount' => '5.00']); + // The private reservation seam must share begin()'s one transaction. A + // failure at the final job insert rolls back wallet, invoice, transition, + // target claim and address fencing as one unit. + $rollbackCommand = CloudPublicNetworkTransitionCommand::userPaidAutomatic( + 12, + 10, + 'tenant-network-rollback-0012', + '5.00' + ); + $rollbackBefore = [ + 'money' => (string) Capsule::table('user')->where('id', 10) + ->value('money'), + 'invoices' => Capsule::table('cloud_invoices')->count(), + 'transitions' => Capsule::table( + 'cloud_public_network_transitions' + )->count(), + 'legs' => Capsule::table( + 'cloud_public_network_transition_ips' + )->count(), + 'jobs' => Capsule::table('cloud_jobs')->count(), + 'claims' => Capsule::table('cloud_resource_reservation_ips') + ->where('reservation_id', 112)->count(), + ]; + Capsule::connection()->statement( + 'CREATE TRIGGER fail_transition_job_insert ' + . 'BEFORE INSERT ON cloud_jobs ' + . "WHEN NEW.request_idempotency_key = " + . "'tenant-network-rollback-0012' " + . "BEGIN SELECT RAISE(ABORT, 'fault injection'); END" + ); + $rollbackRejected = false; + try { + CloudPublicNetworkTransitionWorkflow::begin($rollbackCommand); + } catch (Throwable) { + $rollbackRejected = true; + } + Capsule::connection()->statement('DROP TRIGGER fail_transition_job_insert'); + $assert($rollbackRejected + && (string) Capsule::table('user')->where('id', 10)->value('money') + === $rollbackBefore['money'] + && Capsule::table('cloud_invoices')->count() + === $rollbackBefore['invoices'] + && Capsule::table('cloud_public_network_transitions')->count() + === $rollbackBefore['transitions'] + && Capsule::table('cloud_public_network_transition_ips')->count() + === $rollbackBefore['legs'] + && Capsule::table('cloud_jobs')->count() === $rollbackBefore['jobs'] + && Capsule::table('cloud_resource_reservation_ips') + ->where('reservation_id', 112)->count() + === $rollbackBefore['claims'] + && Capsule::table('cloud_public_ips')->whereIn('id', [203, 204]) + ->where('state', 'available') + ->whereNull('reservation_id') + ->whereNull('reservation_ip_claim_id') + ->whereNull('instance_id') + ->whereNull('operation_id')->count() === 2, + 'Extracted reservation seam committed a partial transition.'); + $batchCommand = CloudPublicNetworkTransitionCommand::poolReplacement( 10, 99, @@ -1346,5 +1459,86 @@ namespace { && Capsule::table('cloud_jobs')->count() === $beforeBatchJobs, 'Pool-replacement parent identity was mutable or left writes.'); + // The internal batch caller may replay only the exact locked parent. A + // drifted parent FK must fail even when the frozen request fingerprint is + // otherwise unchanged. + $seedJob = CloudJob::find($dispatch->jobId()); + $parentA = $seedJob->replicate(); + $parentA->operation_id = str_repeat('d', 32); + $parentA->request_idempotency_key = 'pool-parent-replay-a-0001'; + $parentA->request_fingerprint = hash('sha256', 'parent-a'); + $parentA->parent_job_id = null; + $parentA->instance_id = null; + $parentA->host_id = null; + $parentA->public_network_transition_id = null; + $parentA->kind = 'public_ip_pool_replacement_batch'; + $parentA->execution_mode = 'local'; + $parentA->created_at = $parentA->updated_at = time(); + $parentA->save(); + $parentB = $parentA->replicate(); + $parentB->operation_id = str_repeat('e', 32); + $parentB->request_idempotency_key = 'pool-parent-replay-b-0001'; + $parentB->request_fingerprint = hash('sha256', 'parent-b'); + $parentB->created_at = $parentB->updated_at = time(); + $parentB->save(); + $batchReplayCommand = CloudPublicNetworkTransitionCommand::poolReplacement( + 1, + 99, + 'admin-network-0001', + (int) $parentA->id, + 77, + [4 => [ + 'pool_key' => str_repeat('c', 32), + 'address_cidr' => '8.8.9.20/24', + ]] + ); + Capsule::table('cloud_public_network_transitions') + ->where('id', $dispatch->transitionId())->update([ + 'kind' => 'pool_replacement', + 'pool_replacement_batch_item_id' => 77, + ]); + Capsule::table('cloud_jobs')->where('id', $dispatch->jobId())->update([ + 'parent_job_id' => (int) $parentA->id, + 'request_fingerprint' => $batchReplayCommand->fingerprint(), + ]); + $reserve = new ReflectionMethod( + CloudPublicNetworkTransitionWorkflow::class, + 'reserveTransitionLocked' + ); + $invokeBatchReplay = static function () use ( + $reserve, + $batchReplayCommand + ) { + return Capsule::connection()->transaction(static function () use ( + $reserve, + $batchReplayCommand + ) { + $actor = \App\Models\User::where('id', 99) + ->lockForUpdate()->first(); + return $reserve->invoke(null, $batchReplayCommand, $actor); + }); + }; + $batchReplay = $invokeBatchReplay(); + $assert($batchReplay->replayed() + && $batchReplay->transitionId() === $dispatch->transitionId(), + 'Exact batch parent replay did not return its frozen dispatch.'); + Capsule::table('cloud_jobs')->where('id', $dispatch->jobId()) + ->update(['parent_job_id' => (int) $parentB->id]); + $rejects( + $invokeBatchReplay, + 'Batch replay accepted a drifted parent job association.' + ); + Capsule::table('cloud_jobs')->where('id', $dispatch->jobId())->update([ + 'parent_job_id' => null, + 'request_fingerprint' => $admin->fingerprint(), + ]); + Capsule::table('cloud_public_network_transitions') + ->where('id', $dispatch->transitionId())->update([ + 'kind' => 'admin_manual', + 'pool_replacement_batch_item_id' => null, + ]); + CloudJob::whereIn('id', [(int) $parentA->id, (int) $parentB->id]) + ->delete(); + echo "Public-network transition begin workflow tests passed.\n"; }