diff --git a/web/app/Services/Cloud/CloudJobService.php b/web/app/Services/Cloud/CloudJobService.php index c1c49ef0..ca47466d 100644 --- a/web/app/Services/Cloud/CloudJobService.php +++ b/web/app/Services/Cloud/CloudJobService.php @@ -1043,13 +1043,6 @@ final class CloudJobService CloudJob $child, ?int $resolvedBy ): void { - // A public-pool replacement parent is local rather than aggregate, - // but its ordinary network_change children use the same durable - // post-resolution seam. Recovery prepare may update the PHP resource - // projection while the Agent fence is still unknown; only the later - // applied/not_applied commit is allowed to converge the batch. - CloudPublicIpPoolReplacementBatchService:: - finishResolvedChildLocked($child); if (!in_array( (string) $child->kind, array_values(self::AGGREGATE_CHILD_KIND_MAP), diff --git a/web/app/Services/Cloud/CloudOperationRecoveryService.php b/web/app/Services/Cloud/CloudOperationRecoveryService.php index 332d9eb9..ea696ba1 100644 --- a/web/app/Services/Cloud/CloudOperationRecoveryService.php +++ b/web/app/Services/Cloud/CloudOperationRecoveryService.php @@ -176,6 +176,7 @@ final class CloudOperationRecoveryService 409 ); } + $aggregateParent = null; if ($aggregateParentId > 0) { $aggregateParent = CloudJob::where( 'id', @@ -239,7 +240,8 @@ final class CloudOperationRecoveryService publishPreparedLocked( $job, $resolution, - $transitionActor + $transitionActor, + $aggregateParent ); } $finalReason = $evidenceSource; diff --git a/web/app/Services/Cloud/CloudPublicIpPoolReplacementBatchService.php b/web/app/Services/Cloud/CloudPublicIpPoolReplacementBatchService.php index e0aab599..46d87f91 100644 --- a/web/app/Services/Cloud/CloudPublicIpPoolReplacementBatchService.php +++ b/web/app/Services/Cloud/CloudPublicIpPoolReplacementBatchService.php @@ -764,27 +764,45 @@ final class CloudPublicIpPoolReplacementBatchService }, 3); } - public static function finishChildLocked(CloudJob $job, - string $catalogStatus): void + /** + * Project one pool-replacement child from its durable transition ledger. + * + * The caller already owns actor -> parent -> child -> instance -> host -> + * transition and the transition resource ledger. Mutable job catalog + * fields are deliberately not accepted here: only the frozen transition + * shape may advance an item or its aggregate parent. + */ + public static function projectTransitionEvidenceLocked( + ?CloudJob $parent, + CloudJob $job, + CloudPublicNetworkTransition $transition + ): void { - if ((string) $job->kind !== 'network_change' - || $job->parent_job_id === null - || !in_array($catalogStatus, - ['succeeded', 'failed', 'uncertain'], true)) { - return; + if (DB::connection()->transactionLevel() < 1) { + throw new \LogicException( + 'Pool-replacement projection requires an outer transaction' + ); } - $parent = CloudJob::where('id', (int) $job->parent_job_id) - ->lockForUpdate()->first(); - if ($parent === null || (string) $parent->kind !== self::PARENT_KIND) { + if ((string) $transition->kind !== 'pool_replacement') { + if (($parent instanceof CloudJob + && (string) $parent->kind === self::PARENT_KIND) + || $transition->pool_replacement_batch_item_id !== null) { + throw new CloudError( + '公网地址池迁移网络变更种类不一致', + 409 + ); + } return; } - if ($job->public_network_transition_id === null) { - throw new CloudError('公网地址池迁移子任务缺少网络变更账本', 409); + if (!$parent instanceof CloudJob + || (string) $parent->kind !== self::PARENT_KIND + || (int) ($job->parent_job_id ?? 0) !== (int) $parent->id + || (int) ($job->public_network_transition_id ?? 0) + !== (int) $transition->id) { + throw new CloudError('公网地址池迁移父子任务关联不一致', 409); } - // Hints discover immutable IDs only. The mutable catalog is locked - // before batch/item rows so convergence never takes pool after batch. $itemHint = CloudPublicIpPoolReplacementBatchItem::where( - 'child_job_id', (int) $job->id + 'id', (int) ($transition->pool_replacement_batch_item_id ?? 0) )->first(['id', 'batch_id']); if ($itemHint === null) { throw new CloudError('公网地址池迁移子任务缺少批次明细', 409); @@ -823,67 +841,44 @@ final class CloudPublicIpPoolReplacementBatchService || (int) $item->instance_id !== (int) $job->instance_id) { throw new CloudError('公网地址池迁移批次账本不一致', 409); } - self::assertItemTransitionTopology($item, $batch, $parent, $job); - $nextState = match ($catalogStatus) { - 'succeeded' => 'succeeded', - 'failed' => 'failed', - default => 'unknown', - }; - if (in_array((string) $batch->state, + self::assertItemTransitionTopology( + $item, + $batch, + $parent, + $job, + $transition + ); + [$nextState, $errorCode] = self::transitionProjection($transition); + if (in_array((string) $item->state, ['succeeded', 'failed'], true)) { - // The normal Worker projection runs before the child publishes - // its final public error. Its post-save resolution hook may only - // enrich an already-identical terminal item, never reopen or - // recalculate the batch. - if ((string) $item->state === $nextState - && $catalogStatus !== 'succeeded' - && is_string($job->error_code) - && $job->error_code !== '') { - $item->last_error_code = substr( - (string) $job->error_code, - 0, - 64 + if (!in_array($nextState, ['succeeded', 'failed'], true) + || (string) $item->state !== $nextState) { + throw new CloudError( + '公网地址池迁移明细终态与持久证据冲突', + 409 ); - $item->updated_at = time(); - $item->save(); } return; } - $item->state = $nextState; - $item->last_error_code = $catalogStatus === 'succeeded' ? '' - : substr((string) ($job->error_code ?: $catalogStatus), 0, 64); - $item->updated_at = time(); - $item->save(); - self::convergeLocked($batch, $parent, $source, $items); - } - - /** - * Publish a recovered child only after the generic operation ledger has - * released its unknown-effect fence. Recovery prepare deliberately calls - * finishChildLocked(..., 'uncertain'); this post-resolution hook is the - * sole transition from that durable unknown item to a known conclusion. - */ - public static function finishResolvedChildLocked(CloudJob $job): void - { - if ((string) $job->kind !== 'network_change' - || $job->parent_job_id === null - || (string) $job->phase !== 'terminal' - || !in_array((string) $job->effect_resolution, - ['applied', 'not_applied'], true)) { + if (in_array((string) $batch->state, + ['succeeded', 'failed'], true)) { + if (!in_array($nextState, ['succeeded', 'failed'], true) + || (string) $item->state !== $nextState) { + throw new CloudError( + '公网地址池迁移终态与子任务证据冲突', + 409 + ); + } return; } - $parentSnapshot = CloudJob::find((int) $job->parent_job_id); - if ($parentSnapshot === null - || (string) $parentSnapshot->kind !== self::PARENT_KIND) { - return; + if ((string) $item->state !== $nextState + || !hash_equals((string) $item->last_error_code, $errorCode)) { + $item->state = $nextState; + $item->last_error_code = $errorCode; + $item->updated_at = time(); + $item->save(); } - // The caller owns actor -> parent -> child -> instance -> host and the - // transition ledger. Do not reacquire host here after parent/job. - self::finishChildLocked( - $job, - (string) $job->effect_resolution === 'applied' - ? 'succeeded' : 'failed' - ); + self::convergeLocked($batch, $parent, $source, $items); } /** @return array */ @@ -955,6 +950,64 @@ final class CloudPublicIpPoolReplacementBatchService ]; } + /** @return array{0:string,1:string} */ + private static function transitionProjection( + CloudPublicNetworkTransition $transition + ): array { + $state = (string) $transition->state; + $resolution = (string) $transition->effect_resolution; + $active = $transition->active_transition; + $resolvedAt = $transition->effect_resolved_at; + $terminalAt = $transition->terminal_at; + if (in_array($state, + ['reserved', 'queued', 'awaiting_receipt'], true) + && $resolution === 'not_applicable' + && (int) $active === 1 + && $resolvedAt === null + && $terminalAt === null) { + return ['queued', '']; + } + if ($state === 'effect_unknown' + && $resolution === 'unknown' + && (int) $active === 1 + && $resolvedAt === null + && $terminalAt === null) { + $code = trim((string) $transition->last_error_code); + return [ + 'unknown', + substr($code !== '' ? $code + : 'network_change_effect_unknown', 0, 64), + ]; + } + if ($state === 'finalization_pending' + && in_array($resolution, ['applied', 'not_applied'], true) + && (int) $active === 1 + && $resolvedAt !== null + && $terminalAt === null) { + return ['queued', '']; + } + if ($state === 'succeeded' + && $resolution === 'applied' + && $active === null + && $resolvedAt !== null + && $terminalAt !== null) { + return ['succeeded', '']; + } + if ($state === 'failed' + && $resolution === 'not_applied' + && $active === null + && $resolvedAt !== null + && $terminalAt !== null) { + $code = trim((string) $transition->last_error_code); + return [ + 'failed', + substr($code !== '' ? $code + : 'network_change_not_applied', 0, 64), + ]; + } + throw new CloudError('公网地址池迁移网络变更状态形状损坏', 409); + } + private static function convergeLocked( CloudPublicIpPoolReplacementBatch $batch, CloudJob $parent, @@ -1128,7 +1181,13 @@ final class CloudPublicIpPoolReplacementBatchService || (string) $child->kind !== 'network_change') { throw new CloudError('公网地址池迁移幂等拓扑不一致', 409); } - self::assertItemTransitionTopology($item, $batch, $job, $child); + self::assertItemTransitionTopology( + $item, + $batch, + $job, + $child, + $transition + ); } return [ 'job' => $job, @@ -1141,12 +1200,9 @@ final class CloudPublicIpPoolReplacementBatchService CloudPublicIpPoolReplacementBatchItem $item, CloudPublicIpPoolReplacementBatch $batch, CloudJob $parent, - CloudJob $child + CloudJob $child, + CloudPublicNetworkTransition $transition ): CloudPublicNetworkTransition { - $transition = $child->public_network_transition_id === null ? null - : CloudPublicNetworkTransition::find( - (int) $child->public_network_transition_id - ); if ((string) $parent->kind !== self::PARENT_KIND || (string) $parent->execution_mode !== 'local' || $parent->parent_job_id !== null @@ -1155,7 +1211,6 @@ final class CloudPublicIpPoolReplacementBatchService || (int) $batch->parent_job_id !== (int) $parent->id || (int) $batch->requested_by_user_id !== (int) $parent->user_id || (int) $item->batch_id !== (int) $batch->id - || !$transition instanceof CloudPublicNetworkTransition || (string) $transition->kind !== 'pool_replacement' || (int) $transition->pool_replacement_batch_item_id !== (int) $item->id @@ -1224,16 +1279,14 @@ final class CloudPublicIpPoolReplacementBatchService $source = $claims->get((int) $changed->source_claim_id); $target = $claims->get((int) $changed->target_claim_id); $companion = $claims->get((int) $unchanged->source_claim_id); - // A terminal child no longer owns the instance mutation fence even - // while siblings keep the aggregate batch running. Historical replay - // therefore validates only its frozen companion identity; queued or - // unknown children with an active transition still require the live - // companion claim that will be sent to the Agent. - $requireActiveCompanion = in_array( - (string) $item->state, - ['queued', 'unknown'], - true - ) && (int) ($transition->active_transition ?? 0) === 1; + // A terminal transition no longer owns the instance mutation fence + // even while siblings keep the aggregate batch running. Historical + // replay therefore validates only its frozen companion identity; + // every active transition still requires the live companion claim + // that was frozen for the Agent operation. + $requireActiveCompanion = (int) ( + $transition->active_transition ?? 0 + ) === 1; if ($claims->count() !== 3 || !is_object($source) || !is_object($target) || !is_object($companion) diff --git a/web/app/Services/Cloud/CloudPublicNetworkTransitionWorkflow.php b/web/app/Services/Cloud/CloudPublicNetworkTransitionWorkflow.php index d9b6d37d..65f213b6 100644 --- a/web/app/Services/Cloud/CloudPublicNetworkTransitionWorkflow.php +++ b/web/app/Services/Cloud/CloudPublicNetworkTransitionWorkflow.php @@ -878,6 +878,30 @@ final class CloudPublicNetworkTransitionWorkflow if ((string) $transition->state === 'finalization_pending' || in_array((string) $transition->state, ['succeeded', 'failed'], true)) { + $ledger = self::lockLedgerLocked( + $job, + $instance, + $transition, + $source, + $target + ); + $resolution = (string) $transition->effect_resolution; + $allowed = (string) $transition->state + === 'finalization_pending' + ? ['prepared', $resolution] + : [$resolution]; + if (!in_array($ledger['projection'], $allowed, true)) { + throw new CloudError( + '公网网络变更持久证据与资源投影不一致', + 409 + ); + } + CloudPublicIpPoolReplacementBatchService:: + projectTransitionEvidenceLocked( + $parent, + $job, + $transition + ); return; } $ledger = self::lockLedgerLocked( @@ -919,6 +943,8 @@ final class CloudPublicNetworkTransitionWorkflow $transition->last_error_code = 'local_finalization_failed'; $transition->updated_at = $now; $transition->save(); + CloudPublicIpPoolReplacementBatchService:: + projectTransitionEvidenceLocked($parent, $job, $transition); }, 3); } @@ -1039,6 +1065,12 @@ final class CloudPublicNetworkTransitionWorkflow $payload, $resolution === 'not_applied' ); + CloudPublicIpPoolReplacementBatchService:: + projectTransitionEvidenceLocked( + $parent, + $job, + $transition + ); return; } if ((string) $transition->state === 'finalization_pending') { @@ -1086,6 +1118,12 @@ final class CloudPublicNetworkTransitionWorkflow $transition->updated_at = time(); $transition->save(); } + CloudPublicIpPoolReplacementBatchService:: + projectTransitionEvidenceLocked( + $parent, + $job, + $transition + ); return; } if (!in_array((string) $transition->state, [ @@ -1127,6 +1165,12 @@ final class CloudPublicNetworkTransitionWorkflow $payload, false ); + CloudPublicIpPoolReplacementBatchService:: + projectTransitionEvidenceLocked( + $parent, + $job, + $transition + ); return; } @@ -1183,6 +1227,8 @@ final class CloudPublicNetworkTransitionWorkflow : self::errorCode($operation); $transition->updated_at = $now; $transition->save(); + CloudPublicIpPoolReplacementBatchService:: + projectTransitionEvidenceLocked($parent, $job, $transition); }, 3); } @@ -1196,7 +1242,8 @@ final class CloudPublicNetworkTransitionWorkflow string $catalogStatus, array $payload, ?User $actor, - bool $deferPublish + bool $deferPublish, + ?CloudJob $parent = null ): void { if ($job->public_network_transition_id === null) { return; @@ -1263,6 +1310,8 @@ final class CloudPublicNetworkTransitionWorkflow $payload, false ); + CloudPublicIpPoolReplacementBatchService:: + projectTransitionEvidenceLocked($parent, $job, $transition); return; } if ((string) $transition->state !== 'finalization_pending' @@ -1293,6 +1342,12 @@ final class CloudPublicNetworkTransitionWorkflow $payload, $resolution === 'not_applied' ); + CloudPublicIpPoolReplacementBatchService:: + projectTransitionEvidenceLocked( + $parent, + $job, + $transition + ); return; } throw new CloudError('公网网络变更最终化证据不一致', 409); @@ -1342,13 +1397,16 @@ final class CloudPublicNetworkTransitionWorkflow if (!$deferPublish) { self::publishLocked($transition, $resolution, $now); } + CloudPublicIpPoolReplacementBatchService:: + projectTransitionEvidenceLocked($parent, $job, $transition); } /** Publish a recovery-prepared projection after the Agent fence release. */ public static function publishPreparedLocked( CloudJob $job, string $resolution, - ?User $actor + ?User $actor, + ?CloudJob $parent = null ): void { if ($job->public_network_transition_id === null) { return; @@ -1395,6 +1453,8 @@ final class CloudPublicNetworkTransitionWorkflow null, $resolution === 'not_applied' ); + CloudPublicIpPoolReplacementBatchService:: + projectTransitionEvidenceLocked($parent, $job, $transition); return; } if ((string) $transition->state !== 'finalization_pending' @@ -1433,6 +1493,8 @@ final class CloudPublicNetworkTransitionWorkflow $resolution === 'not_applied' ); self::publishLocked($transition, $resolution, time()); + CloudPublicIpPoolReplacementBatchService:: + projectTransitionEvidenceLocked($parent, $job, $transition); } /** Retry only PHP-owned writes after evidence survived a Worker crash. */ @@ -1530,7 +1592,8 @@ final class CloudPublicNetworkTransitionWorkflow $resolution === 'applied' ? 'succeeded' : 'failed', $payload, $actor, - false + false, + $parent ); $now = time(); $job->effect_resolution = $resolution; @@ -1554,10 +1617,6 @@ final class CloudPublicNetworkTransitionWorkflow $job->available_at = 0; $job->updated_at = $now; $job->save(); - CloudPublicIpPoolReplacementBatchService::finishChildLocked( - $job, - $resolution === 'applied' ? 'succeeded' : 'failed' - ); CloudJobService::reconcileAggregateParentEffectLocked( $job, null diff --git a/web/app/Services/Cloud/CloudWorkerService.php b/web/app/Services/Cloud/CloudWorkerService.php index 7778fa77..631ed0b9 100644 --- a/web/app/Services/Cloud/CloudWorkerService.php +++ b/web/app/Services/Cloud/CloudWorkerService.php @@ -2366,6 +2366,7 @@ final class CloudWorkerService // parent -> children lock order. Never lock a leaf first and then // wait for its parent: concurrent sibling completions otherwise // form a deadlock cycle with the parent-led sweep. + $aggregateParent = null; if ($aggregateParentId > 0) { $aggregateParent = CloudJob::where('id', $aggregateParentId) ->lockForUpdate()->first(); @@ -2618,12 +2619,8 @@ final class CloudWorkerService $catalogStatus, $payload, $user, - $preparedRecoveryMarker !== null - ); - CloudPublicIpPoolReplacementBatchService::finishChildLocked( - $job, - $preparedRecoveryMarker === null - ? $catalogStatus : 'uncertain' + $preparedRecoveryMarker !== null, + $aggregateParent ); } elseif ($job->kind === 'network_change' && $job->public_network_transition_id === null @@ -2712,11 +2709,6 @@ final class CloudWorkerService ); } } - CloudPublicIpPoolReplacementBatchService::finishChildLocked( - $job, - $preparedRecoveryMarker === null - ? $catalogStatus : 'uncertain' - ); } elseif ($status === 'succeeded' && $job->kind === 'create') { $vm = is_array($result) ? ($result['vm'] ?? $result) : []; $uuid = $vm['uuid'] ?? ''; diff --git a/web/tests/CloudPublicIpPoolReplacementBatchServiceTest.php b/web/tests/CloudPublicIpPoolReplacementBatchServiceTest.php index f2081290..20b1177c 100644 --- a/web/tests/CloudPublicIpPoolReplacementBatchServiceTest.php +++ b/web/tests/CloudPublicIpPoolReplacementBatchServiceTest.php @@ -40,6 +40,7 @@ namespace { require dirname(__DIR__) . '/vendor/autoload.php'; use App\Models\CloudJob; + use App\Models\CloudPublicNetworkTransition; use App\Models\CloudPublicIpPoolReplacementBatch; use App\Services\Cloud\CloudError; use App\Services\Cloud\CloudIpAddress; @@ -844,6 +845,33 @@ SQL); )->value('id'); $items = DB::table('cloud_public_ip_pool_replacement_batch_items') ->where('batch_id', $batchId)->orderBy('instance_id')->get(); + $projectChild = static function (int $childId): void { + $hint = CloudJob::find($childId); + if (!$hint instanceof CloudJob) { + throw new RuntimeException('Projection child fixture is missing.'); + } + DB::connection()->transaction(static function () use ( + $hint, + $childId + ): void { + $parent = CloudJob::where( + 'id', + (int) $hint->parent_job_id + )->lockForUpdate()->first(); + $child = CloudJob::where('id', $childId) + ->lockForUpdate()->first(); + $transition = CloudPublicNetworkTransition::where( + 'id', + (int) $child->public_network_transition_id + )->lockForUpdate()->first(); + CloudPublicIpPoolReplacementBatchService:: + projectTransitionEvidenceLocked( + $parent, + $child, + $transition + ); + }); + }; $assert($started['unchanged'] === false && $items->count() === 2 && $items->every(static fn ($item): bool => (int) $item->address_family === 4 @@ -931,28 +959,117 @@ SQL); DB::table('cloud_jobs')->where('id', $firstChildId)->update([ 'host_id' => 2, ]); - $rejects(static fn () => - CloudPublicIpPoolReplacementBatchService::finishChildLocked( - CloudJob::find($firstChildId), - 'succeeded' - ), 'Finish accepted a child with a drifted frozen host.'); + $rejects(static fn () => $projectChild($firstChildId), + 'Projection accepted a child with a drifted frozen host.'); DB::table('cloud_jobs')->where('id', $firstChildId)->update([ 'host_id' => 1, ]); - $firstChild = CloudJob::find($firstChildId); - CloudPublicIpPoolReplacementBatchService::finishChildLocked( - $firstChild, - 'uncertain' - ); + $firstTransitionId = (int) DB::table('cloud_jobs') + ->where('id', $firstChildId) + ->value('public_network_transition_id'); + foreach (['reserved', 'queued', 'awaiting_receipt'] as $activeState) { + DB::table('cloud_public_network_transitions') + ->where('id', $firstTransitionId)->update([ + 'state' => $activeState, + 'effect_resolution' => 'not_applicable', + 'active_transition' => 1, + 'effect_resolved_at' => null, + 'terminal_at' => null, + 'last_error_code' => '', + ]); + $projectChild($firstChildId); + $assert(DB::table('cloud_public_ip_pool_replacement_batch_items') + ->where('child_job_id', $firstChildId) + ->value('state') === 'queued' + && (string) CloudPublicIpPoolReplacementBatch::find($batchId)->state + === 'running', + 'Active transition state ' . $activeState + . ' prematurely terminated its batch item.'); + } + DB::table('cloud_public_network_transitions') + ->where('id', $firstTransitionId)->update([ + 'kind' => 'admin_manual', + ]); + $corruptKindSnapshot = $snapshot(); + $rejects(static fn () => $projectChild($firstChildId), + 'Projection silently ignored a corrupted replacement kind.'); + $assert($snapshot() === $corruptKindSnapshot, + 'Corrupted replacement kind caused partial projection writes.'); + DB::table('cloud_public_network_transitions') + ->where('id', $firstTransitionId)->update([ + 'kind' => 'pool_replacement', + ]); + + DB::table('cloud_public_network_transitions') + ->where('id', $firstTransitionId)->update([ + 'state' => 'succeeded', + 'effect_resolution' => 'applied', + 'active_transition' => 1, + 'effect_resolved_at' => null, + 'terminal_at' => null, + ]); + $invalidShapeSnapshot = $snapshot(); + $rejects(static fn () => $projectChild($firstChildId), + 'Projection accepted an invalid pseudo-terminal transition shape.'); + $assert($snapshot() === $invalidShapeSnapshot, + 'Invalid transition shape caused partial batch writes.'); + + $resolvedAt = time(); + DB::table('cloud_public_network_transitions') + ->where('id', $firstTransitionId)->update([ + 'state' => 'finalization_pending', + 'effect_resolution' => 'applied', + 'active_transition' => 1, + 'effect_resolved_at' => $resolvedAt, + 'terminal_at' => null, + 'last_error_code' => '', + ]); + DB::table('cloud_jobs')->where('id', $firstChildId)->update([ + 'status' => 'failed', + 'error_code' => 'catalog_poison', + ]); + $projectChild($firstChildId); + $assert(DB::table('cloud_public_ip_pool_replacement_batch_items') + ->where('child_job_id', $firstChildId)->value('state') === 'queued' + && (string) CloudPublicIpPoolReplacementBatch::find($batchId)->state + === 'running', + 'Finalization-pending evidence prematurely terminated a batch item.'); + + DB::table('cloud_public_network_transitions') + ->where('id', $firstTransitionId)->update([ + 'state' => 'effect_unknown', + 'effect_resolution' => 'unknown', + 'active_transition' => 1, + 'effect_resolved_at' => null, + 'terminal_at' => null, + 'last_error_code' => 'durable_effect_unknown', + ]); + $unknownIdentity = [ + 'job_count' => DB::table('cloud_jobs')->count(), + 'transition_count' => DB::table( + 'cloud_public_network_transitions' + )->count(), + 'operation_id' => (string) DB::table('cloud_jobs') + ->where('id', $firstChildId)->value('operation_id'), + ]; + $projectChild($firstChildId); $assert((string) CloudPublicIpPoolReplacementBatch::find($batchId)->state === 'blocked' && (int) CloudPublicIpPoolReplacementBatch::find($batchId)->active_claim === 1 && DB::table('cloud_public_ip_pool_replacement_batch_items') ->where('child_job_id', $firstChildId)->value('state') === 'unknown' + && DB::table('cloud_public_ip_pool_replacement_batch_items') + ->where('child_job_id', $firstChildId) + ->value('last_error_code') === 'durable_effect_unknown' + && DB::table('cloud_jobs')->count() === $unknownIdentity['job_count'] + && DB::table('cloud_public_network_transitions')->count() + === $unknownIdentity['transition_count'] + && (string) DB::table('cloud_jobs')->where('id', $firstChildId) + ->value('operation_id') === $unknownIdentity['operation_id'] && (string) CloudJob::find($parentId)->phase !== 'terminal', - 'Unknown child did not retain the batch and parent fences.'); + 'Durable unknown evidence did not retain batch and parent fences.'); $applyChild = static function (int $childId): CloudJob { $child = CloudJob::find($childId); @@ -1060,10 +1177,8 @@ SQL); return CloudJob::find($childId); }; - $resolvedFirst = $applyChild($firstChildId); - CloudPublicIpPoolReplacementBatchService::finishResolvedChildLocked( - $resolvedFirst - ); + $applyChild($firstChildId); + $projectChild($firstChildId); $assert(DB::table('cloud_public_ip_pool_replacement_batch_items') ->where('child_job_id', $firstChildId) ->value('state') === 'succeeded' @@ -1072,12 +1187,25 @@ SQL); && (string) CloudJob::find($parentId)->phase !== 'terminal', 'One resolved child did not release blocked state in one hook.'); + DB::table('cloud_public_network_transitions') + ->where('id', $firstTransitionId)->update([ + 'state' => 'failed', + 'effect_resolution' => 'not_applied', + ]); + $terminalConflictSnapshot = $snapshot(); + $rejects(static fn () => $projectChild($firstChildId), + 'Projection rewrote a terminal item while its sibling was pending.'); + $assert($snapshot() === $terminalConflictSnapshot, + 'Conflicting terminal item evidence caused partial writes.'); + DB::table('cloud_public_network_transitions') + ->where('id', $firstTransitionId)->update([ + 'state' => 'succeeded', + 'effect_resolution' => 'applied', + ]); + $secondChildId = (int) $items->last()->child_job_id; - $resolvedSecond = $applyChild($secondChildId); - CloudPublicIpPoolReplacementBatchService::finishChildLocked( - $resolvedSecond, - 'succeeded' - ); + $applyChild($secondChildId); + $projectChild($secondChildId); $terminalBatch = CloudPublicIpPoolReplacementBatch::find($batchId); $terminalParent = CloudJob::find($parentId); $assert((string) $terminalBatch->state === 'succeeded' @@ -1096,6 +1224,11 @@ SQL); })->count() === 0, 'All-success children did not converge parent/batch/source in one pass.'); + $terminalProjectionSnapshot = $snapshot(); + $projectChild($secondChildId); + $assert($snapshot() === $terminalProjectionSnapshot, + 'Repeated terminal transition projection was not idempotent.'); + // A strict terminal topology remains replayable after the instance has // legitimately acquired a later companion-family generation. $terminalAt = time() + 10; @@ -1271,6 +1404,58 @@ SQL); 'Sparse IPv6 child did not freeze the companion IPv4 leg.'); } + $failedV6ChildId = (int) $v6Items->first()->child_job_id; + $failedV6TransitionId = (int) DB::table('cloud_jobs') + ->where('id', $failedV6ChildId) + ->value('public_network_transition_id'); + $failedV6Leg = DB::table('cloud_public_network_transition_ips') + ->where('transition_id', $failedV6TransitionId) + ->where('changed', 1)->first(); + $failedV6TargetClaim = DB::table('cloud_resource_reservation_ips') + ->where('id', (int) $failedV6Leg->target_claim_id)->first(); + $failedAt = time() + 20; + DB::table('cloud_resource_reservation_ips') + ->where('id', (int) $failedV6TargetClaim->id)->update([ + 'active_claim' => null, + 'state' => 'released', + 'released_at' => $failedAt, + 'release_reason' => 'public_network_transition_not_applied', + 'updated_at' => $failedAt, + ]); + DB::table('cloud_public_ips') + ->where('id', (int) $failedV6TargetClaim->public_ip_id)->update([ + 'state' => 'available', + 'reservation_id' => null, + 'reservation_ip_claim_id' => null, + 'instance_id' => null, + 'operation_id' => null, + 'updated_at' => $failedAt, + ]); + DB::table('cloud_public_network_transitions') + ->where('id', $failedV6TransitionId)->update([ + 'state' => 'failed', + 'effect_resolution' => 'not_applied', + 'active_transition' => null, + 'effect_resolved_at' => $failedAt, + 'terminal_at' => $failedAt, + 'last_error_code' => 'durable_not_applied', + 'updated_at' => $failedAt, + ]); + DB::table('cloud_jobs')->where('id', $failedV6ChildId)->update([ + 'status' => 'succeeded', + 'error_code' => null, + ]); + $projectChild($failedV6ChildId); + $assert(DB::table('cloud_public_ip_pool_replacement_batch_items') + ->where('child_job_id', $failedV6ChildId)->value('state') + === 'failed' + && DB::table('cloud_public_ip_pool_replacement_batch_items') + ->where('child_job_id', $failedV6ChildId) + ->value('last_error_code') === 'durable_not_applied' + && (string) CloudPublicIpPoolReplacementBatch::find($v6BatchId)->state + === 'running', + 'Strict failed/not-applied transition did not override job catalog drift.'); + $jobService = file_get_contents( dirname(__DIR__) . '/app/Services/Cloud/CloudJobService.php' ); diff --git a/web/tests/CloudPublicIpPoolReplacementSchemaContractTest.php b/web/tests/CloudPublicIpPoolReplacementSchemaContractTest.php index 918c77bc..eb2c047d 100644 --- a/web/tests/CloudPublicIpPoolReplacementSchemaContractTest.php +++ b/web/tests/CloudPublicIpPoolReplacementSchemaContractTest.php @@ -17,6 +17,12 @@ $batchPlan = file_get_contents( $root . '/app/Services/Cloud/CloudPublicNetworkTransitionBatchPlan.php' ); $worker = file_get_contents($root . '/app/Services/Cloud/CloudWorkerService.php'); +$recovery = file_get_contents( + $root . '/app/Services/Cloud/CloudOperationRecoveryService.php' +); +$automaticRecovery = file_get_contents( + $root . '/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php' +); $assert = static function (bool $condition, string $message): void { if (!$condition) { @@ -85,7 +91,8 @@ $assert(is_string($service) && str_contains($service, "'target_public_ip_pool_id', \$replacementPoolIds") && str_contains($service, "\$batch->state = 'blocked'") - && str_contains($service, 'finishResolvedChildLocked') + && str_contains($service, 'projectTransitionEvidenceLocked') + && !str_contains($service, 'finishResolvedChildLocked') && str_contains($service, "self::setPoolStateLocked(\$source, 'disabled')") && str_contains($service, 'self::setPoolStateLocked($source, $sourceInitialState)') @@ -106,24 +113,25 @@ $assert(is_string($jobs) && str_contains($batchPlan, 'final class CloudPublicNetworkTransitionBatchPlan'), 'Replacement children do not use the canonical dual-stack transition seam.'); $assert(is_string($worker) - && str_contains($worker, + && !str_contains($worker, 'CloudPublicIpPoolReplacementBatchService::finishChildLocked') && str_contains($worker, - "'public_ip_pool_replacement_batch'") + 'CloudPublicNetworkTransitionWorkflow::finishJobLocked') + && str_contains($worker, '$aggregateParent') && str_contains($worker, - "\$preparedRecoveryMarker === null") - && str_contains($worker, "? \$catalogStatus : 'uncertain'"), - 'Worker no longer converges replacement children or excludes the local parent from dispatch.'); + "'public_ip_pool_replacement_batch'") + && !str_contains($jobs, 'finishResolvedChildLocked'), + 'Worker no longer uses the transition projector or excludes the local parent from dispatch.'); -$finishChild = $method( +$projector = $method( $service, - 'public static function finishChildLocked(', - 'public static function finishResolvedChildLocked(' + 'public static function projectTransitionEvidenceLocked(', + 'public static function display(' ); $start = $method( $service, 'public static function start(', - 'public static function finishChildLocked(' + 'public static function projectTransitionEvidenceLocked(' ); $transactionOffset = strpos( $start, @@ -169,20 +177,39 @@ $assert($ipCatalogLock !== '' && str_contains($ipCatalogLock, "->orderBy('id')->lockForUpdate()->get()->keyBy('id')"), 'Replacement start can discover IP catalog rows outside its current locking read.'); -$finishResolved = $method( - $service, - 'public static function finishResolvedChildLocked(', - 'public static function display(' -); $converge = $method( $service, 'private static function convergeLocked(', 'private static function setPoolStateLocked(' ); +$markUnknown = $method( + $workflow, + 'public static function markJobUncertain(', + 'public static function recordEvidence(' +); +$recordEvidence = $method( + $workflow, + 'public static function recordEvidence(', + 'public static function finishJobLocked(' +); +$finishTransition = $method( + $workflow, + 'public static function finishJobLocked(', + 'public static function publishPreparedLocked(' +); +$publishPrepared = $method( + $workflow, + 'public static function publishPreparedLocked(', + 'public static function retryPendingFinalization(' +); +$retryFinalization = $method( + $workflow, + 'public static function retryPendingFinalization(', + 'private static function lockHostBoundaryLocked(' +); $positions = array_map( - static fn (string $needle): int|false => strpos($finishChild, $needle), + static fn (string $needle): int|false => strpos($projector, $needle), [ - '$parent = CloudJob::where(', '$source = CloudPublicIpPool::where(', '$batch = CloudPublicIpPoolReplacementBatch::where(', '$items = CloudPublicIpPoolReplacementBatchItem::where(', @@ -194,12 +221,28 @@ sort($sortedPositions, SORT_NUMERIC); $assert(!in_array(false, $positions, true) && $positions === $sortedPositions && count(array_unique($positions)) === count($positions) - && str_contains($finishChild, 'assertItemTransitionTopology(') + && str_contains($projector, 'assertItemTransitionTopology(') + && str_contains($projector, 'transitionProjection($transition)') + && str_contains($projector, 'transactionLevel() < 1') + && !str_contains($projector, '$catalogStatus') + && !str_contains($projector, '$job->status') + && !str_contains($projector, '$job->error_code') && str_contains($service, "(string) \$transition->kind !== 'pool_replacement'") && str_contains($service, "(string) \$child->execution_mode !== 'agent'") - && !str_contains($finishResolved, 'CloudHost::where(') && !str_contains($converge, 'lockForUpdate()'), - 'Replacement convergence reversed parent/pool/batch locks.'); + 'Replacement projector trusts mutable job catalogs or reverses pool/batch locks.'); +$assert(str_contains($markUnknown, 'projectTransitionEvidenceLocked(') + && str_contains($recordEvidence, 'projectTransitionEvidenceLocked(') + && str_contains($finishTransition, 'projectTransitionEvidenceLocked(') + && str_contains($publishPrepared, 'projectTransitionEvidenceLocked(') + && str_contains($retryFinalization, 'self::finishJobLocked(') + && str_contains($retryFinalization, '$parent') + && is_string($recovery) + && str_contains($recovery, 'publishPreparedLocked(') + && str_contains($recovery, '$aggregateParent') + && is_string($automaticRecovery) + && str_contains($automaticRecovery, 'retryPendingFinalization('), + 'Worker, late receipt, or recovery paths bypass the transition projector.'); echo "Cloud public IP pool replacement schema contract tests passed\n";