fix: converge pool replacement from durable evidence

go
codex 2 weeks ago
parent 61c99a4680
commit 718bfaef31

@ -1043,13 +1043,6 @@ final class CloudJobService
CloudJob $child,
?int $resolvedBy
): void {
// A public-pool replacement parent is local rather than aggregate,
// but its ordinary network_change children use the same durable
// post-resolution seam. Recovery prepare may update the PHP resource
// projection while the Agent fence is still unknown; only the later
// applied/not_applied commit is allowed to converge the batch.
CloudPublicIpPoolReplacementBatchService::
finishResolvedChildLocked($child);
if (!in_array(
(string) $child->kind,
array_values(self::AGGREGATE_CHILD_KIND_MAP),

@ -176,6 +176,7 @@ final class CloudOperationRecoveryService
409
);
}
$aggregateParent = null;
if ($aggregateParentId > 0) {
$aggregateParent = CloudJob::where(
'id',
@ -239,7 +240,8 @@ final class CloudOperationRecoveryService
publishPreparedLocked(
$job,
$resolution,
$transitionActor
$transitionActor,
$aggregateParent
);
}
$finalReason = $evidenceSource;

@ -764,27 +764,45 @@ final class CloudPublicIpPoolReplacementBatchService
}, 3);
}
public static function finishChildLocked(CloudJob $job,
string $catalogStatus): void
/**
* Project one pool-replacement child from its durable transition ledger.
*
* The caller already owns actor -> parent -> child -> instance -> host ->
* transition and the transition resource ledger. Mutable job catalog
* fields are deliberately not accepted here: only the frozen transition
* shape may advance an item or its aggregate parent.
*/
public static function projectTransitionEvidenceLocked(
?CloudJob $parent,
CloudJob $job,
CloudPublicNetworkTransition $transition
): void
{
if ((string) $job->kind !== 'network_change'
|| $job->parent_job_id === null
|| !in_array($catalogStatus,
['succeeded', 'failed', 'uncertain'], true)) {
return;
if (DB::connection()->transactionLevel() < 1) {
throw new \LogicException(
'Pool-replacement projection requires an outer transaction'
);
}
$parent = CloudJob::where('id', (int) $job->parent_job_id)
->lockForUpdate()->first();
if ($parent === null || (string) $parent->kind !== self::PARENT_KIND) {
if ((string) $transition->kind !== 'pool_replacement') {
if (($parent instanceof CloudJob
&& (string) $parent->kind === self::PARENT_KIND)
|| $transition->pool_replacement_batch_item_id !== null) {
throw new CloudError(
'公网地址池迁移网络变更种类不一致',
409
);
}
return;
}
if ($job->public_network_transition_id === null) {
throw new CloudError('公网地址池迁移子任务缺少网络变更账本', 409);
if (!$parent instanceof CloudJob
|| (string) $parent->kind !== self::PARENT_KIND
|| (int) ($job->parent_job_id ?? 0) !== (int) $parent->id
|| (int) ($job->public_network_transition_id ?? 0)
!== (int) $transition->id) {
throw new CloudError('公网地址池迁移父子任务关联不一致', 409);
}
// Hints discover immutable IDs only. The mutable catalog is locked
// before batch/item rows so convergence never takes pool after batch.
$itemHint = CloudPublicIpPoolReplacementBatchItem::where(
'child_job_id', (int) $job->id
'id', (int) ($transition->pool_replacement_batch_item_id ?? 0)
)->first(['id', 'batch_id']);
if ($itemHint === null) {
throw new CloudError('公网地址池迁移子任务缺少批次明细', 409);
@ -823,67 +841,44 @@ final class CloudPublicIpPoolReplacementBatchService
|| (int) $item->instance_id !== (int) $job->instance_id) {
throw new CloudError('公网地址池迁移批次账本不一致', 409);
}
self::assertItemTransitionTopology($item, $batch, $parent, $job);
$nextState = match ($catalogStatus) {
'succeeded' => 'succeeded',
'failed' => 'failed',
default => 'unknown',
};
if (in_array((string) $batch->state,
self::assertItemTransitionTopology(
$item,
$batch,
$parent,
$job,
$transition
);
[$nextState, $errorCode] = self::transitionProjection($transition);
if (in_array((string) $item->state,
['succeeded', 'failed'], true)) {
// The normal Worker projection runs before the child publishes
// its final public error. Its post-save resolution hook may only
// enrich an already-identical terminal item, never reopen or
// recalculate the batch.
if ((string) $item->state === $nextState
&& $catalogStatus !== 'succeeded'
&& is_string($job->error_code)
&& $job->error_code !== '') {
$item->last_error_code = substr(
(string) $job->error_code,
0,
64
if (!in_array($nextState, ['succeeded', 'failed'], true)
|| (string) $item->state !== $nextState) {
throw new CloudError(
'公网地址池迁移明细终态与持久证据冲突',
409
);
$item->updated_at = time();
$item->save();
}
return;
}
$item->state = $nextState;
$item->last_error_code = $catalogStatus === 'succeeded' ? ''
: substr((string) ($job->error_code ?: $catalogStatus), 0, 64);
$item->updated_at = time();
$item->save();
self::convergeLocked($batch, $parent, $source, $items);
}
/**
* Publish a recovered child only after the generic operation ledger has
* released its unknown-effect fence. Recovery prepare deliberately calls
* finishChildLocked(..., 'uncertain'); this post-resolution hook is the
* sole transition from that durable unknown item to a known conclusion.
*/
public static function finishResolvedChildLocked(CloudJob $job): void
{
if ((string) $job->kind !== 'network_change'
|| $job->parent_job_id === null
|| (string) $job->phase !== 'terminal'
|| !in_array((string) $job->effect_resolution,
['applied', 'not_applied'], true)) {
if (in_array((string) $batch->state,
['succeeded', 'failed'], true)) {
if (!in_array($nextState, ['succeeded', 'failed'], true)
|| (string) $item->state !== $nextState) {
throw new CloudError(
'公网地址池迁移终态与子任务证据冲突',
409
);
}
return;
}
$parentSnapshot = CloudJob::find((int) $job->parent_job_id);
if ($parentSnapshot === null
|| (string) $parentSnapshot->kind !== self::PARENT_KIND) {
return;
if ((string) $item->state !== $nextState
|| !hash_equals((string) $item->last_error_code, $errorCode)) {
$item->state = $nextState;
$item->last_error_code = $errorCode;
$item->updated_at = time();
$item->save();
}
// The caller owns actor -> parent -> child -> instance -> host and the
// transition ledger. Do not reacquire host here after parent/job.
self::finishChildLocked(
$job,
(string) $job->effect_resolution === 'applied'
? 'succeeded' : 'failed'
);
self::convergeLocked($batch, $parent, $source, $items);
}
/** @return array<string,mixed> */
@ -955,6 +950,64 @@ final class CloudPublicIpPoolReplacementBatchService
];
}
/** @return array{0:string,1:string} */
private static function transitionProjection(
CloudPublicNetworkTransition $transition
): array {
$state = (string) $transition->state;
$resolution = (string) $transition->effect_resolution;
$active = $transition->active_transition;
$resolvedAt = $transition->effect_resolved_at;
$terminalAt = $transition->terminal_at;
if (in_array($state,
['reserved', 'queued', 'awaiting_receipt'], true)
&& $resolution === 'not_applicable'
&& (int) $active === 1
&& $resolvedAt === null
&& $terminalAt === null) {
return ['queued', ''];
}
if ($state === 'effect_unknown'
&& $resolution === 'unknown'
&& (int) $active === 1
&& $resolvedAt === null
&& $terminalAt === null) {
$code = trim((string) $transition->last_error_code);
return [
'unknown',
substr($code !== '' ? $code
: 'network_change_effect_unknown', 0, 64),
];
}
if ($state === 'finalization_pending'
&& in_array($resolution, ['applied', 'not_applied'], true)
&& (int) $active === 1
&& $resolvedAt !== null
&& $terminalAt === null) {
return ['queued', ''];
}
if ($state === 'succeeded'
&& $resolution === 'applied'
&& $active === null
&& $resolvedAt !== null
&& $terminalAt !== null) {
return ['succeeded', ''];
}
if ($state === 'failed'
&& $resolution === 'not_applied'
&& $active === null
&& $resolvedAt !== null
&& $terminalAt !== null) {
$code = trim((string) $transition->last_error_code);
return [
'failed',
substr($code !== '' ? $code
: 'network_change_not_applied', 0, 64),
];
}
throw new CloudError('公网地址池迁移网络变更状态形状损坏', 409);
}
private static function convergeLocked(
CloudPublicIpPoolReplacementBatch $batch,
CloudJob $parent,
@ -1128,7 +1181,13 @@ final class CloudPublicIpPoolReplacementBatchService
|| (string) $child->kind !== 'network_change') {
throw new CloudError('公网地址池迁移幂等拓扑不一致', 409);
}
self::assertItemTransitionTopology($item, $batch, $job, $child);
self::assertItemTransitionTopology(
$item,
$batch,
$job,
$child,
$transition
);
}
return [
'job' => $job,
@ -1141,12 +1200,9 @@ final class CloudPublicIpPoolReplacementBatchService
CloudPublicIpPoolReplacementBatchItem $item,
CloudPublicIpPoolReplacementBatch $batch,
CloudJob $parent,
CloudJob $child
CloudJob $child,
CloudPublicNetworkTransition $transition
): CloudPublicNetworkTransition {
$transition = $child->public_network_transition_id === null ? null
: CloudPublicNetworkTransition::find(
(int) $child->public_network_transition_id
);
if ((string) $parent->kind !== self::PARENT_KIND
|| (string) $parent->execution_mode !== 'local'
|| $parent->parent_job_id !== null
@ -1155,7 +1211,6 @@ final class CloudPublicIpPoolReplacementBatchService
|| (int) $batch->parent_job_id !== (int) $parent->id
|| (int) $batch->requested_by_user_id !== (int) $parent->user_id
|| (int) $item->batch_id !== (int) $batch->id
|| !$transition instanceof CloudPublicNetworkTransition
|| (string) $transition->kind !== 'pool_replacement'
|| (int) $transition->pool_replacement_batch_item_id
!== (int) $item->id
@ -1224,16 +1279,14 @@ final class CloudPublicIpPoolReplacementBatchService
$source = $claims->get((int) $changed->source_claim_id);
$target = $claims->get((int) $changed->target_claim_id);
$companion = $claims->get((int) $unchanged->source_claim_id);
// A terminal child no longer owns the instance mutation fence even
// while siblings keep the aggregate batch running. Historical replay
// therefore validates only its frozen companion identity; queued or
// unknown children with an active transition still require the live
// companion claim that will be sent to the Agent.
$requireActiveCompanion = in_array(
(string) $item->state,
['queued', 'unknown'],
true
) && (int) ($transition->active_transition ?? 0) === 1;
// A terminal transition no longer owns the instance mutation fence
// even while siblings keep the aggregate batch running. Historical
// replay therefore validates only its frozen companion identity;
// every active transition still requires the live companion claim
// that was frozen for the Agent operation.
$requireActiveCompanion = (int) (
$transition->active_transition ?? 0
) === 1;
if ($claims->count() !== 3
|| !is_object($source) || !is_object($target)
|| !is_object($companion)

@ -878,6 +878,30 @@ final class CloudPublicNetworkTransitionWorkflow
if ((string) $transition->state === 'finalization_pending'
|| in_array((string) $transition->state,
['succeeded', 'failed'], true)) {
$ledger = self::lockLedgerLocked(
$job,
$instance,
$transition,
$source,
$target
);
$resolution = (string) $transition->effect_resolution;
$allowed = (string) $transition->state
=== 'finalization_pending'
? ['prepared', $resolution]
: [$resolution];
if (!in_array($ledger['projection'], $allowed, true)) {
throw new CloudError(
'公网网络变更持久证据与资源投影不一致',
409
);
}
CloudPublicIpPoolReplacementBatchService::
projectTransitionEvidenceLocked(
$parent,
$job,
$transition
);
return;
}
$ledger = self::lockLedgerLocked(
@ -919,6 +943,8 @@ final class CloudPublicNetworkTransitionWorkflow
$transition->last_error_code = 'local_finalization_failed';
$transition->updated_at = $now;
$transition->save();
CloudPublicIpPoolReplacementBatchService::
projectTransitionEvidenceLocked($parent, $job, $transition);
}, 3);
}
@ -1039,6 +1065,12 @@ final class CloudPublicNetworkTransitionWorkflow
$payload,
$resolution === 'not_applied'
);
CloudPublicIpPoolReplacementBatchService::
projectTransitionEvidenceLocked(
$parent,
$job,
$transition
);
return;
}
if ((string) $transition->state === 'finalization_pending') {
@ -1086,6 +1118,12 @@ final class CloudPublicNetworkTransitionWorkflow
$transition->updated_at = time();
$transition->save();
}
CloudPublicIpPoolReplacementBatchService::
projectTransitionEvidenceLocked(
$parent,
$job,
$transition
);
return;
}
if (!in_array((string) $transition->state, [
@ -1127,6 +1165,12 @@ final class CloudPublicNetworkTransitionWorkflow
$payload,
false
);
CloudPublicIpPoolReplacementBatchService::
projectTransitionEvidenceLocked(
$parent,
$job,
$transition
);
return;
}
@ -1183,6 +1227,8 @@ final class CloudPublicNetworkTransitionWorkflow
: self::errorCode($operation);
$transition->updated_at = $now;
$transition->save();
CloudPublicIpPoolReplacementBatchService::
projectTransitionEvidenceLocked($parent, $job, $transition);
}, 3);
}
@ -1196,7 +1242,8 @@ final class CloudPublicNetworkTransitionWorkflow
string $catalogStatus,
array $payload,
?User $actor,
bool $deferPublish
bool $deferPublish,
?CloudJob $parent = null
): void {
if ($job->public_network_transition_id === null) {
return;
@ -1263,6 +1310,8 @@ final class CloudPublicNetworkTransitionWorkflow
$payload,
false
);
CloudPublicIpPoolReplacementBatchService::
projectTransitionEvidenceLocked($parent, $job, $transition);
return;
}
if ((string) $transition->state !== 'finalization_pending'
@ -1293,6 +1342,12 @@ final class CloudPublicNetworkTransitionWorkflow
$payload,
$resolution === 'not_applied'
);
CloudPublicIpPoolReplacementBatchService::
projectTransitionEvidenceLocked(
$parent,
$job,
$transition
);
return;
}
throw new CloudError('公网网络变更最终化证据不一致', 409);
@ -1342,13 +1397,16 @@ final class CloudPublicNetworkTransitionWorkflow
if (!$deferPublish) {
self::publishLocked($transition, $resolution, $now);
}
CloudPublicIpPoolReplacementBatchService::
projectTransitionEvidenceLocked($parent, $job, $transition);
}
/** Publish a recovery-prepared projection after the Agent fence release. */
public static function publishPreparedLocked(
CloudJob $job,
string $resolution,
?User $actor
?User $actor,
?CloudJob $parent = null
): void {
if ($job->public_network_transition_id === null) {
return;
@ -1395,6 +1453,8 @@ final class CloudPublicNetworkTransitionWorkflow
null,
$resolution === 'not_applied'
);
CloudPublicIpPoolReplacementBatchService::
projectTransitionEvidenceLocked($parent, $job, $transition);
return;
}
if ((string) $transition->state !== 'finalization_pending'
@ -1433,6 +1493,8 @@ final class CloudPublicNetworkTransitionWorkflow
$resolution === 'not_applied'
);
self::publishLocked($transition, $resolution, time());
CloudPublicIpPoolReplacementBatchService::
projectTransitionEvidenceLocked($parent, $job, $transition);
}
/** Retry only PHP-owned writes after evidence survived a Worker crash. */
@ -1530,7 +1592,8 @@ final class CloudPublicNetworkTransitionWorkflow
$resolution === 'applied' ? 'succeeded' : 'failed',
$payload,
$actor,
false
false,
$parent
);
$now = time();
$job->effect_resolution = $resolution;
@ -1554,10 +1617,6 @@ final class CloudPublicNetworkTransitionWorkflow
$job->available_at = 0;
$job->updated_at = $now;
$job->save();
CloudPublicIpPoolReplacementBatchService::finishChildLocked(
$job,
$resolution === 'applied' ? 'succeeded' : 'failed'
);
CloudJobService::reconcileAggregateParentEffectLocked(
$job,
null

@ -2366,6 +2366,7 @@ final class CloudWorkerService
// parent -> children lock order. Never lock a leaf first and then
// wait for its parent: concurrent sibling completions otherwise
// form a deadlock cycle with the parent-led sweep.
$aggregateParent = null;
if ($aggregateParentId > 0) {
$aggregateParent = CloudJob::where('id', $aggregateParentId)
->lockForUpdate()->first();
@ -2618,12 +2619,8 @@ final class CloudWorkerService
$catalogStatus,
$payload,
$user,
$preparedRecoveryMarker !== null
);
CloudPublicIpPoolReplacementBatchService::finishChildLocked(
$job,
$preparedRecoveryMarker === null
? $catalogStatus : 'uncertain'
$preparedRecoveryMarker !== null,
$aggregateParent
);
} elseif ($job->kind === 'network_change'
&& $job->public_network_transition_id === null
@ -2712,11 +2709,6 @@ final class CloudWorkerService
);
}
}
CloudPublicIpPoolReplacementBatchService::finishChildLocked(
$job,
$preparedRecoveryMarker === null
? $catalogStatus : 'uncertain'
);
} elseif ($status === 'succeeded' && $job->kind === 'create') {
$vm = is_array($result) ? ($result['vm'] ?? $result) : [];
$uuid = $vm['uuid'] ?? '';

@ -40,6 +40,7 @@ namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudJob;
use App\Models\CloudPublicNetworkTransition;
use App\Models\CloudPublicIpPoolReplacementBatch;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudIpAddress;
@ -844,6 +845,33 @@ SQL);
)->value('id');
$items = DB::table('cloud_public_ip_pool_replacement_batch_items')
->where('batch_id', $batchId)->orderBy('instance_id')->get();
$projectChild = static function (int $childId): void {
$hint = CloudJob::find($childId);
if (!$hint instanceof CloudJob) {
throw new RuntimeException('Projection child fixture is missing.');
}
DB::connection()->transaction(static function () use (
$hint,
$childId
): void {
$parent = CloudJob::where(
'id',
(int) $hint->parent_job_id
)->lockForUpdate()->first();
$child = CloudJob::where('id', $childId)
->lockForUpdate()->first();
$transition = CloudPublicNetworkTransition::where(
'id',
(int) $child->public_network_transition_id
)->lockForUpdate()->first();
CloudPublicIpPoolReplacementBatchService::
projectTransitionEvidenceLocked(
$parent,
$child,
$transition
);
});
};
$assert($started['unchanged'] === false && $items->count() === 2
&& $items->every(static fn ($item): bool =>
(int) $item->address_family === 4
@ -931,28 +959,117 @@ SQL);
DB::table('cloud_jobs')->where('id', $firstChildId)->update([
'host_id' => 2,
]);
$rejects(static fn () =>
CloudPublicIpPoolReplacementBatchService::finishChildLocked(
CloudJob::find($firstChildId),
'succeeded'
), 'Finish accepted a child with a drifted frozen host.');
$rejects(static fn () => $projectChild($firstChildId),
'Projection accepted a child with a drifted frozen host.');
DB::table('cloud_jobs')->where('id', $firstChildId)->update([
'host_id' => 1,
]);
$firstChild = CloudJob::find($firstChildId);
CloudPublicIpPoolReplacementBatchService::finishChildLocked(
$firstChild,
'uncertain'
);
$firstTransitionId = (int) DB::table('cloud_jobs')
->where('id', $firstChildId)
->value('public_network_transition_id');
foreach (['reserved', 'queued', 'awaiting_receipt'] as $activeState) {
DB::table('cloud_public_network_transitions')
->where('id', $firstTransitionId)->update([
'state' => $activeState,
'effect_resolution' => 'not_applicable',
'active_transition' => 1,
'effect_resolved_at' => null,
'terminal_at' => null,
'last_error_code' => '',
]);
$projectChild($firstChildId);
$assert(DB::table('cloud_public_ip_pool_replacement_batch_items')
->where('child_job_id', $firstChildId)
->value('state') === 'queued'
&& (string) CloudPublicIpPoolReplacementBatch::find($batchId)->state
=== 'running',
'Active transition state ' . $activeState
. ' prematurely terminated its batch item.');
}
DB::table('cloud_public_network_transitions')
->where('id', $firstTransitionId)->update([
'kind' => 'admin_manual',
]);
$corruptKindSnapshot = $snapshot();
$rejects(static fn () => $projectChild($firstChildId),
'Projection silently ignored a corrupted replacement kind.');
$assert($snapshot() === $corruptKindSnapshot,
'Corrupted replacement kind caused partial projection writes.');
DB::table('cloud_public_network_transitions')
->where('id', $firstTransitionId)->update([
'kind' => 'pool_replacement',
]);
DB::table('cloud_public_network_transitions')
->where('id', $firstTransitionId)->update([
'state' => 'succeeded',
'effect_resolution' => 'applied',
'active_transition' => 1,
'effect_resolved_at' => null,
'terminal_at' => null,
]);
$invalidShapeSnapshot = $snapshot();
$rejects(static fn () => $projectChild($firstChildId),
'Projection accepted an invalid pseudo-terminal transition shape.');
$assert($snapshot() === $invalidShapeSnapshot,
'Invalid transition shape caused partial batch writes.');
$resolvedAt = time();
DB::table('cloud_public_network_transitions')
->where('id', $firstTransitionId)->update([
'state' => 'finalization_pending',
'effect_resolution' => 'applied',
'active_transition' => 1,
'effect_resolved_at' => $resolvedAt,
'terminal_at' => null,
'last_error_code' => '',
]);
DB::table('cloud_jobs')->where('id', $firstChildId)->update([
'status' => 'failed',
'error_code' => 'catalog_poison',
]);
$projectChild($firstChildId);
$assert(DB::table('cloud_public_ip_pool_replacement_batch_items')
->where('child_job_id', $firstChildId)->value('state') === 'queued'
&& (string) CloudPublicIpPoolReplacementBatch::find($batchId)->state
=== 'running',
'Finalization-pending evidence prematurely terminated a batch item.');
DB::table('cloud_public_network_transitions')
->where('id', $firstTransitionId)->update([
'state' => 'effect_unknown',
'effect_resolution' => 'unknown',
'active_transition' => 1,
'effect_resolved_at' => null,
'terminal_at' => null,
'last_error_code' => 'durable_effect_unknown',
]);
$unknownIdentity = [
'job_count' => DB::table('cloud_jobs')->count(),
'transition_count' => DB::table(
'cloud_public_network_transitions'
)->count(),
'operation_id' => (string) DB::table('cloud_jobs')
->where('id', $firstChildId)->value('operation_id'),
];
$projectChild($firstChildId);
$assert((string) CloudPublicIpPoolReplacementBatch::find($batchId)->state
=== 'blocked'
&& (int) CloudPublicIpPoolReplacementBatch::find($batchId)->active_claim
=== 1
&& DB::table('cloud_public_ip_pool_replacement_batch_items')
->where('child_job_id', $firstChildId)->value('state') === 'unknown'
&& DB::table('cloud_public_ip_pool_replacement_batch_items')
->where('child_job_id', $firstChildId)
->value('last_error_code') === 'durable_effect_unknown'
&& DB::table('cloud_jobs')->count() === $unknownIdentity['job_count']
&& DB::table('cloud_public_network_transitions')->count()
=== $unknownIdentity['transition_count']
&& (string) DB::table('cloud_jobs')->where('id', $firstChildId)
->value('operation_id') === $unknownIdentity['operation_id']
&& (string) CloudJob::find($parentId)->phase !== 'terminal',
'Unknown child did not retain the batch and parent fences.');
'Durable unknown evidence did not retain batch and parent fences.');
$applyChild = static function (int $childId): CloudJob {
$child = CloudJob::find($childId);
@ -1060,10 +1177,8 @@ SQL);
return CloudJob::find($childId);
};
$resolvedFirst = $applyChild($firstChildId);
CloudPublicIpPoolReplacementBatchService::finishResolvedChildLocked(
$resolvedFirst
);
$applyChild($firstChildId);
$projectChild($firstChildId);
$assert(DB::table('cloud_public_ip_pool_replacement_batch_items')
->where('child_job_id', $firstChildId)
->value('state') === 'succeeded'
@ -1072,12 +1187,25 @@ SQL);
&& (string) CloudJob::find($parentId)->phase !== 'terminal',
'One resolved child did not release blocked state in one hook.');
DB::table('cloud_public_network_transitions')
->where('id', $firstTransitionId)->update([
'state' => 'failed',
'effect_resolution' => 'not_applied',
]);
$terminalConflictSnapshot = $snapshot();
$rejects(static fn () => $projectChild($firstChildId),
'Projection rewrote a terminal item while its sibling was pending.');
$assert($snapshot() === $terminalConflictSnapshot,
'Conflicting terminal item evidence caused partial writes.');
DB::table('cloud_public_network_transitions')
->where('id', $firstTransitionId)->update([
'state' => 'succeeded',
'effect_resolution' => 'applied',
]);
$secondChildId = (int) $items->last()->child_job_id;
$resolvedSecond = $applyChild($secondChildId);
CloudPublicIpPoolReplacementBatchService::finishChildLocked(
$resolvedSecond,
'succeeded'
);
$applyChild($secondChildId);
$projectChild($secondChildId);
$terminalBatch = CloudPublicIpPoolReplacementBatch::find($batchId);
$terminalParent = CloudJob::find($parentId);
$assert((string) $terminalBatch->state === 'succeeded'
@ -1096,6 +1224,11 @@ SQL);
})->count() === 0,
'All-success children did not converge parent/batch/source in one pass.');
$terminalProjectionSnapshot = $snapshot();
$projectChild($secondChildId);
$assert($snapshot() === $terminalProjectionSnapshot,
'Repeated terminal transition projection was not idempotent.');
// A strict terminal topology remains replayable after the instance has
// legitimately acquired a later companion-family generation.
$terminalAt = time() + 10;
@ -1271,6 +1404,58 @@ SQL);
'Sparse IPv6 child did not freeze the companion IPv4 leg.');
}
$failedV6ChildId = (int) $v6Items->first()->child_job_id;
$failedV6TransitionId = (int) DB::table('cloud_jobs')
->where('id', $failedV6ChildId)
->value('public_network_transition_id');
$failedV6Leg = DB::table('cloud_public_network_transition_ips')
->where('transition_id', $failedV6TransitionId)
->where('changed', 1)->first();
$failedV6TargetClaim = DB::table('cloud_resource_reservation_ips')
->where('id', (int) $failedV6Leg->target_claim_id)->first();
$failedAt = time() + 20;
DB::table('cloud_resource_reservation_ips')
->where('id', (int) $failedV6TargetClaim->id)->update([
'active_claim' => null,
'state' => 'released',
'released_at' => $failedAt,
'release_reason' => 'public_network_transition_not_applied',
'updated_at' => $failedAt,
]);
DB::table('cloud_public_ips')
->where('id', (int) $failedV6TargetClaim->public_ip_id)->update([
'state' => 'available',
'reservation_id' => null,
'reservation_ip_claim_id' => null,
'instance_id' => null,
'operation_id' => null,
'updated_at' => $failedAt,
]);
DB::table('cloud_public_network_transitions')
->where('id', $failedV6TransitionId)->update([
'state' => 'failed',
'effect_resolution' => 'not_applied',
'active_transition' => null,
'effect_resolved_at' => $failedAt,
'terminal_at' => $failedAt,
'last_error_code' => 'durable_not_applied',
'updated_at' => $failedAt,
]);
DB::table('cloud_jobs')->where('id', $failedV6ChildId)->update([
'status' => 'succeeded',
'error_code' => null,
]);
$projectChild($failedV6ChildId);
$assert(DB::table('cloud_public_ip_pool_replacement_batch_items')
->where('child_job_id', $failedV6ChildId)->value('state')
=== 'failed'
&& DB::table('cloud_public_ip_pool_replacement_batch_items')
->where('child_job_id', $failedV6ChildId)
->value('last_error_code') === 'durable_not_applied'
&& (string) CloudPublicIpPoolReplacementBatch::find($v6BatchId)->state
=== 'running',
'Strict failed/not-applied transition did not override job catalog drift.');
$jobService = file_get_contents(
dirname(__DIR__) . '/app/Services/Cloud/CloudJobService.php'
);

@ -17,6 +17,12 @@ $batchPlan = file_get_contents(
$root . '/app/Services/Cloud/CloudPublicNetworkTransitionBatchPlan.php'
);
$worker = file_get_contents($root . '/app/Services/Cloud/CloudWorkerService.php');
$recovery = file_get_contents(
$root . '/app/Services/Cloud/CloudOperationRecoveryService.php'
);
$automaticRecovery = file_get_contents(
$root . '/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php'
);
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
@ -85,7 +91,8 @@ $assert(is_string($service)
&& str_contains($service,
"'target_public_ip_pool_id', \$replacementPoolIds")
&& str_contains($service, "\$batch->state = 'blocked'")
&& str_contains($service, 'finishResolvedChildLocked')
&& str_contains($service, 'projectTransitionEvidenceLocked')
&& !str_contains($service, 'finishResolvedChildLocked')
&& str_contains($service, "self::setPoolStateLocked(\$source, 'disabled')")
&& str_contains($service,
'self::setPoolStateLocked($source, $sourceInitialState)')
@ -106,24 +113,25 @@ $assert(is_string($jobs)
&& str_contains($batchPlan, 'final class CloudPublicNetworkTransitionBatchPlan'),
'Replacement children do not use the canonical dual-stack transition seam.');
$assert(is_string($worker)
&& str_contains($worker,
&& !str_contains($worker,
'CloudPublicIpPoolReplacementBatchService::finishChildLocked')
&& str_contains($worker,
"'public_ip_pool_replacement_batch'")
'CloudPublicNetworkTransitionWorkflow::finishJobLocked')
&& str_contains($worker, '$aggregateParent')
&& str_contains($worker,
"\$preparedRecoveryMarker === null")
&& str_contains($worker, "? \$catalogStatus : 'uncertain'"),
'Worker no longer converges replacement children or excludes the local parent from dispatch.');
"'public_ip_pool_replacement_batch'")
&& !str_contains($jobs, 'finishResolvedChildLocked'),
'Worker no longer uses the transition projector or excludes the local parent from dispatch.');
$finishChild = $method(
$projector = $method(
$service,
'public static function finishChildLocked(',
'public static function finishResolvedChildLocked('
'public static function projectTransitionEvidenceLocked(',
'public static function display('
);
$start = $method(
$service,
'public static function start(',
'public static function finishChildLocked('
'public static function projectTransitionEvidenceLocked('
);
$transactionOffset = strpos(
$start,
@ -169,20 +177,39 @@ $assert($ipCatalogLock !== ''
&& str_contains($ipCatalogLock,
"->orderBy('id')->lockForUpdate()->get()->keyBy('id')"),
'Replacement start can discover IP catalog rows outside its current locking read.');
$finishResolved = $method(
$service,
'public static function finishResolvedChildLocked(',
'public static function display('
);
$converge = $method(
$service,
'private static function convergeLocked(',
'private static function setPoolStateLocked('
);
$markUnknown = $method(
$workflow,
'public static function markJobUncertain(',
'public static function recordEvidence('
);
$recordEvidence = $method(
$workflow,
'public static function recordEvidence(',
'public static function finishJobLocked('
);
$finishTransition = $method(
$workflow,
'public static function finishJobLocked(',
'public static function publishPreparedLocked('
);
$publishPrepared = $method(
$workflow,
'public static function publishPreparedLocked(',
'public static function retryPendingFinalization('
);
$retryFinalization = $method(
$workflow,
'public static function retryPendingFinalization(',
'private static function lockHostBoundaryLocked('
);
$positions = array_map(
static fn (string $needle): int|false => strpos($finishChild, $needle),
static fn (string $needle): int|false => strpos($projector, $needle),
[
'$parent = CloudJob::where(',
'$source = CloudPublicIpPool::where(',
'$batch = CloudPublicIpPoolReplacementBatch::where(',
'$items = CloudPublicIpPoolReplacementBatchItem::where(',
@ -194,12 +221,28 @@ sort($sortedPositions, SORT_NUMERIC);
$assert(!in_array(false, $positions, true)
&& $positions === $sortedPositions
&& count(array_unique($positions)) === count($positions)
&& str_contains($finishChild, 'assertItemTransitionTopology(')
&& str_contains($projector, 'assertItemTransitionTopology(')
&& str_contains($projector, 'transitionProjection($transition)')
&& str_contains($projector, 'transactionLevel() < 1')
&& !str_contains($projector, '$catalogStatus')
&& !str_contains($projector, '$job->status')
&& !str_contains($projector, '$job->error_code')
&& str_contains($service, "(string) \$transition->kind !== 'pool_replacement'")
&& str_contains($service,
"(string) \$child->execution_mode !== 'agent'")
&& !str_contains($finishResolved, 'CloudHost::where(')
&& !str_contains($converge, 'lockForUpdate()'),
'Replacement convergence reversed parent/pool/batch locks.');
'Replacement projector trusts mutable job catalogs or reverses pool/batch locks.');
$assert(str_contains($markUnknown, 'projectTransitionEvidenceLocked(')
&& str_contains($recordEvidence, 'projectTransitionEvidenceLocked(')
&& str_contains($finishTransition, 'projectTransitionEvidenceLocked(')
&& str_contains($publishPrepared, 'projectTransitionEvidenceLocked(')
&& str_contains($retryFinalization, 'self::finishJobLocked(')
&& str_contains($retryFinalization, '$parent')
&& is_string($recovery)
&& str_contains($recovery, 'publishPreparedLocked(')
&& str_contains($recovery, '$aggregateParent')
&& is_string($automaticRecovery)
&& str_contains($automaticRecovery, 'retryPendingFinalization('),
'Worker, late receipt, or recovery paths bypass the transition projector.');
echo "Cloud public IP pool replacement schema contract tests passed\n";

Loading…
Cancel
Save