refactor: unify public network transition lock order

go
codex 3 weeks ago
parent fa941beb3c
commit 13132bc419

@ -14,6 +14,7 @@ use App\Models\CloudPublicIpPoolReplacementBatchItem;
use App\Models\CloudResourcePool;
use App\Models\CloudResourceReservation;
use App\Models\CloudResourceReservationIp;
use App\Models\User;
/**
* Durable same-host replacement of every assigned address in one public pool.
@ -268,48 +269,59 @@ final class CloudPublicIpPoolReplacementBatchService
$batchKey = substr(hash('sha256', "public-pool-replacement\0"
. $actorUserId . "\0" . $hostId . "\0" . $sourcePoolKey
. "\0" . $requestKey), 0, 32);
$prior = CloudJob::where('user_id', $actorUserId)
->where('request_idempotency_key', $requestKey)->first();
if ($prior !== null) {
return self::existingResult($prior, $fingerprint, $batchKey);
}
$sourceRef = CloudPublicIpPool::where('host_id', $hostId)
->where('pool_key', $sourcePoolKey)->first();
$targetRef = CloudPublicIpPool::where('host_id', $hostId)
->where('pool_key', $targetPoolKey)->first();
if ($sourceRef === null || $targetRef === null) {
throw new CloudError('源或目标公网地址池不存在', 404);
}
$instanceIds = CloudResourceReservationIp::query()
->join(
'cloud_resource_reservations as reservation',
'reservation.id',
'=',
'cloud_resource_reservation_ips.reservation_id'
)->where('reservation.host_id', $hostId)
->where(
'cloud_resource_reservation_ips.public_ip_pool_id',
(int) $sourceRef->id
)->where('cloud_resource_reservation_ips.active_claim', 1)
->where('cloud_resource_reservation_ips.state', 'committed')
->where('reservation.state', 'committed')
->whereNotNull('reservation.instance_id')
->orderBy('reservation.instance_id')
->pluck('reservation.instance_id')->map(
static fn ($id): int => (int) $id
)->unique()->values()->all();
if ($instanceIds === []) {
throw new CloudError('源公网地址池没有可迁移实例', 409);
}
return (new CloudPublicIpPoolReplacementBatch())->getConnection()
->transaction(static function () use (
$hostId, $sourcePoolKey, $targetPoolKey, $actorUserId,
$requestKey, $fingerprint, $batchKey, $sourceRef,
$targetRef, $instanceIds, $expectedSourceRevision,
$requestKey, $fingerprint, $batchKey, $expectedSourceRevision,
$expectedTargetRevision
): array {
$actorUser = User::where('id', $actorUserId)
->lockForUpdate()->first();
if (!$actorUser instanceof User) {
throw new CloudError('批量更换公网 IP 的操作账户不存在', 409);
}
$prior = CloudJob::where('user_id', $actorUserId)
->where('request_idempotency_key', $requestKey)
->lockForUpdate()->first();
if ($prior !== null) {
return self::existingResult(
$prior,
$fingerprint,
$batchKey
);
}
$sourceRef = CloudPublicIpPool::where('host_id', $hostId)
->where('pool_key', $sourcePoolKey)->first();
$targetRef = CloudPublicIpPool::where('host_id', $hostId)
->where('pool_key', $targetPoolKey)->first();
if ($sourceRef === null || $targetRef === null) {
throw new CloudError('源或目标公网地址池不存在', 404);
}
$instanceIds = CloudResourceReservationIp::query()
->join(
'cloud_resource_reservations as reservation',
'reservation.id',
'=',
'cloud_resource_reservation_ips.reservation_id'
)->where('reservation.host_id', $hostId)
->where(
'cloud_resource_reservation_ips.public_ip_pool_id',
(int) $sourceRef->id
)->where(
'cloud_resource_reservation_ips.active_claim',
1
)->where(
'cloud_resource_reservation_ips.state',
'committed'
)->where('reservation.state', 'committed')
->whereNotNull('reservation.instance_id')
->orderBy('reservation.instance_id')
->pluck('reservation.instance_id')->map(
static fn ($id): int => (int) $id
)->unique()->values()->all();
if ($instanceIds === []) {
throw new CloudError('源公网地址池没有可迁移实例', 409);
}
$instances = CloudInstance::whereIn('id', $instanceIds)
->orderBy('id')->lockForUpdate()->get()->keyBy('id');
if ($instances->count() !== count($instanceIds)) {
@ -390,16 +402,6 @@ final class CloudPublicIpPoolReplacementBatchService
}
self::assertTargetPoolConfiguration($host, $targetPool);
$prior = CloudJob::where('user_id', $actorUserId)
->where('request_idempotency_key', $requestKey)
->lockForUpdate()->first();
if ($prior !== null) {
return self::existingResult(
$prior,
$fingerprint,
$batchKey
);
}
$replacementPoolIds = [
(int) $sourcePool->id,
(int) $targetPool->id,
@ -633,18 +635,45 @@ final class CloudPublicIpPoolReplacementBatchService
if ($parent === null || (string) $parent->kind !== self::PARENT_KIND) {
return;
}
$item = CloudPublicIpPoolReplacementBatchItem::where(
// Hints discover immutable IDs only. The mutable catalog is locked
// before batch/item rows so convergence never takes pool after batch.
$itemHint = CloudPublicIpPoolReplacementBatchItem::where(
'child_job_id', (int) $job->id
)->lockForUpdate()->first();
if ($item === null) {
)->first(['id', 'batch_id']);
if ($itemHint === null) {
throw new CloudError('公网地址池迁移子任务缺少批次明细', 409);
}
$batchHint = CloudPublicIpPoolReplacementBatch::find(
(int) $itemHint->batch_id
);
if ($batchHint === null) {
throw new CloudError('公网地址池迁移批次账本不一致', 409);
}
$source = CloudPublicIpPool::where(
'id', (int) $batchHint->source_public_ip_pool_id
)->lockForUpdate()->first();
if ($source === null
|| (int) $source->host_id !== (int) $batchHint->host_id) {
throw new CloudError('公网地址池迁移源池账本缺失', 409);
}
$batch = CloudPublicIpPoolReplacementBatch::where(
'id', (int) $item->batch_id
'id', (int) $batchHint->id
)->lockForUpdate()->first();
if ($batch === null
if ($batch === null) {
throw new CloudError('公网地址池迁移批次账本不一致', 409);
}
$items = CloudPublicIpPoolReplacementBatchItem::where(
'batch_id', (int) $batch->id
)->orderBy('id')->lockForUpdate()->get();
$item = $items->first(static fn ($candidate): bool =>
(int) $candidate->id === (int) $itemHint->id
&& (int) ($candidate->child_job_id ?? 0) === (int) $job->id
);
if ($item === null
|| (int) $batch->parent_job_id !== (int) $parent->id
|| (int) $batch->host_id !== (int) $job->host_id
|| (int) $batch->source_public_ip_pool_id !== (int) $source->id
|| (int) $item->batch_id !== (int) $batch->id
|| (int) $item->instance_id !== (int) $job->instance_id) {
throw new CloudError('公网地址池迁移批次账本不一致', 409);
}
@ -678,7 +707,7 @@ final class CloudPublicIpPoolReplacementBatchService
: substr((string) ($job->error_code ?: $catalogStatus), 0, 64);
$item->updated_at = time();
$item->save();
self::convergeLocked($batch, $parent);
self::convergeLocked($batch, $parent, $source, $items);
}
/**
@ -701,14 +730,8 @@ final class CloudPublicIpPoolReplacementBatchService
|| (string) $parentSnapshot->kind !== self::PARENT_KIND) {
return;
}
if ($job->host_id === null
|| CloudHost::where('id', (int) $job->host_id)
->lockForUpdate()->first() === null) {
throw new CloudError(
'公网地址池迁移子任务宿主机账本缺失',
409
);
}
// The caller owns actor -> parent -> child -> instance -> host and the
// transition ledger. Do not reacquire host here after parent/job.
self::finishChildLocked(
$job,
(string) $job->effect_resolution === 'applied'
@ -787,11 +810,10 @@ final class CloudPublicIpPoolReplacementBatchService
private static function convergeLocked(
CloudPublicIpPoolReplacementBatch $batch,
CloudJob $parent
CloudJob $parent,
CloudPublicIpPool $source,
\Illuminate\Support\Collection $items
): void {
$items = CloudPublicIpPoolReplacementBatchItem::where(
'batch_id', (int) $batch->id
)->orderBy('id')->lockForUpdate()->get();
if ($items->contains(static fn ($item): bool =>
(string) $item->state === 'unknown')) {
$batch->state = 'blocked';
@ -810,10 +832,8 @@ final class CloudPublicIpPoolReplacementBatchService
}
$succeeded = $items->where('state', 'succeeded')->count();
$failed = $items->where('state', 'failed')->count();
$source = CloudPublicIpPool::where(
'id', (int) $batch->source_public_ip_pool_id
)->lockForUpdate()->first();
if ($source === null || (int) $source->host_id !== (int) $batch->host_id) {
if ((int) $source->id !== (int) $batch->source_public_ip_pool_id
|| (int) $source->host_id !== (int) $batch->host_id) {
throw new CloudError('公网地址池迁移源池账本缺失', 409);
}
$occupied = CloudPublicIp::where(

@ -525,6 +525,102 @@ namespace {
&& Capsule::table('cloud_public_ips')->whereIn('id', [71, 72, 73, 74])
->where('state', 'available')->count() === 4,
'Gated pool replacement created durable or address-side effects.');
// Lock-order convergence must use the same locked item collection it
// updates. One unknown projection followed by one resolved projection is
// sufficient to release the blocked batch; no second hook may be needed.
$job = static function (int $id, string $operationId, string $requestKey,
string $kind, ?int $parentId, ?int $instanceId
) use ($now): array {
return [
'id' => $id,
'operation_id' => $operationId,
'request_idempotency_key' => $requestKey,
'request_fingerprint' => hash('sha256', $requestKey),
'parent_job_id' => $parentId,
'instance_id' => $instanceId,
'user_id' => 99,
'host_id' => $parentId === null ? null : 1,
'kind' => $kind,
'execution_mode' => $parentId === null ? 'local' : 'agent',
'phase' => 'accepted',
'outcome' => null,
'status' => 'queued',
'sealed_payload' => '{}',
'attempts' => 0,
'dispatch_attempts' => 0,
'dispatch_deadline_at' => $now + 300,
'effect_resolution' => 'not_applicable',
'finalization_incident' => 0,
'lease_until' => 0,
'available_at' => $now,
'created_at' => $now,
'updated_at' => $now,
];
};
Capsule::table('cloud_jobs')->insert([
$job(900, str_repeat('9', 32), 'batch-lock-parent-0001',
'public_ip_pool_replacement_batch', null, null),
$job(901, str_repeat('8', 32), 'batch-lock-child-0001',
'network_change', 900, 51),
]);
Capsule::table('cloud_public_ip_pool_replacement_batches')->insert([
'id' => 900,
'batch_key' => str_repeat('7', 32),
'host_id' => 1,
'source_public_ip_pool_id' => 31,
'target_public_ip_pool_id' => 32,
'source_initial_state' => 'active',
'parent_job_id' => 900,
'requested_by_user_id' => 99,
'state' => 'running',
'active_claim' => 1,
'last_error_code' => '',
'terminal_at' => null,
'created_at' => $now,
'updated_at' => $now,
]);
Capsule::table('cloud_public_ip_pool_replacement_batch_items')->insert([
'id' => 900,
'batch_id' => 900,
'instance_id' => 51,
'source_reservation_id' => 41,
'source_public_ip_id' => 61,
'target_public_ip_id' => 71,
'source_network_attachment_id' => 11,
'target_network_attachment_id' => 12,
'child_job_id' => 901,
'state' => 'queued',
'last_error_code' => '',
'created_at' => $now,
'updated_at' => $now,
]);
$lockedChild = CloudJob::find(901);
CloudPublicIpPoolReplacementBatchService::finishChildLocked(
$lockedChild,
'uncertain'
);
$assert(CloudPublicIpPoolReplacementBatch::find(900)->state === 'blocked'
&& Capsule::table('cloud_public_ip_pool_replacement_batch_items')
->where('id', 900)->value('state') === 'unknown',
'One unknown child projection did not block the batch immediately.');
$lockedChild->phase = 'terminal';
$lockedChild->outcome = 'failed';
$lockedChild->status = 'failed';
$lockedChild->effect_may_have_applied = 1;
$lockedChild->safe_to_resubmit = 0;
$lockedChild->effect_resolution = 'applied';
$lockedChild->save();
CloudPublicIpPoolReplacementBatchService::finishResolvedChildLocked(
$lockedChild->fresh()
);
$resolvedBatch = CloudPublicIpPoolReplacementBatch::find(900);
$assert((string) $resolvedBatch->state === 'failed'
&& $resolvedBatch->active_claim === null
&& Capsule::table('cloud_public_ip_pool_replacement_batch_items')
->where('id', 900)->value('state') === 'succeeded'
&& (string) CloudJob::find(900)->phase === 'terminal',
'One resolved child projection left the batch permanently blocked.');
echo "Cloud public IP pool replacement C10 gate tests passed\n";
return;

@ -17,6 +17,15 @@ $assert = static function (bool $condition, string $message): void {
throw new RuntimeException($message);
}
};
$method = static function (string $contents, string $start, string $end) use (
$assert
): string {
$offset = strpos($contents, $start);
$limit = $offset === false ? false : strpos($contents, $end, $offset + 1);
$assert($offset !== false && $limit !== false && $limit > $offset,
'Replacement method source is missing: ' . $start);
return substr($contents, $offset, $limit - $offset);
};
$assert(is_string($schema)
&& str_contains($schema,
@ -96,4 +105,70 @@ $assert(is_string($worker)
&& str_contains($worker, "? \$catalogStatus : 'uncertain'"),
'Worker no longer converges replacement children or excludes the local parent from dispatch.');
$finishChild = $method(
$service,
'public static function finishChildLocked(',
'public static function finishResolvedChildLocked('
);
$start = $method(
$service,
'public static function start(',
'public static function finishChildLocked('
);
$transactionOffset = strpos(
$start,
'->transaction(static function () use ('
);
$assert($transactionOffset !== false,
'Replacement start transaction source is missing.');
$lockedStart = substr($start, $transactionOffset);
$startPositions = array_map(
static fn (string $needle): int|false => strpos($lockedStart, $needle),
[
'$actorUser = User::where(',
'$prior = CloudJob::where(',
'$instances = CloudInstance::whereIn(',
'$host = CloudHost::where(',
'$agent = CloudHostAgent::where(',
'$attachments = CloudHostNetworkAttachment::whereIn(',
'$pools = CloudPublicIpPool::whereIn(',
'$reservations = CloudResourceReservation::whereIn(',
'$ips = CloudPublicIp::whereIn(',
]
);
$sortedStartPositions = $startPositions;
sort($sortedStartPositions, SORT_NUMERIC);
$assert(!in_array(false, $startPositions, true)
&& $startPositions === $sortedStartPositions
&& count(array_unique($startPositions)) === count($startPositions),
'Replacement start reversed actor/job/instance/host/catalog locks.');
$finishResolved = $method(
$service,
'public static function finishResolvedChildLocked(',
'public static function display('
);
$converge = $method(
$service,
'private static function convergeLocked(',
'private static function setPoolStateLocked('
);
$positions = array_map(
static fn (string $needle): int|false => strpos($finishChild, $needle),
[
'$parent = CloudJob::where(',
'$source = CloudPublicIpPool::where(',
'$batch = CloudPublicIpPoolReplacementBatch::where(',
'$items = CloudPublicIpPoolReplacementBatchItem::where(',
'$item = $items->first(',
]
);
$sortedPositions = $positions;
sort($sortedPositions, SORT_NUMERIC);
$assert(!in_array(false, $positions, true)
&& $positions === $sortedPositions
&& count(array_unique($positions)) === count($positions)
&& !str_contains($finishResolved, 'CloudHost::where(')
&& !str_contains($converge, 'lockForUpdate()'),
'Replacement convergence reversed parent/pool/batch locks.');
echo "Cloud public IP pool replacement schema contract tests passed\n";

@ -152,6 +152,38 @@ $assert($source4->state === 'released'
->value('state') === 'assigned',
'Applied transition did not atomically switch only the changed leg.');
// Once the transition is terminal, a later maintenance migration may change
// current placement. Exact replay still returns the frozen historical result
// while an active transition (covered by begin tests) rejects the same drift.
$terminalPayload = $appliedPayload;
$terminalPayload[CloudOperationRecoveryIdentity::payloadKey()] =
CloudOperationRecoveryIdentity::persisted([
'action' => $appliedOperation['action'],
'resource_id' => $appliedOperation['resource_id'],
'request_fingerprint' => $appliedOperation['request_fingerprint'],
]);
Capsule::table('cloud_jobs')->where('id', (int) $appliedJob->id)->update([
'phase' => 'terminal',
'outcome' => 'succeeded',
'status' => 'succeeded',
'terminal_at' => time(),
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'sealed_payload' => CloudSecrets::encrypt(
$terminalPayload,
'job-payload'
),
]);
Capsule::table('cloud_instances')->where('id', 1)
->update(['host_id' => 2]);
$terminalMigratedReplay = CloudPublicNetworkTransitionWorkflow::begin($admin);
$assert($terminalMigratedReplay->replayed()
&& $terminalMigratedReplay->transitionId() === $dispatch->transitionId(),
'Terminal exact replay was coupled to later instance placement.');
Capsule::table('cloud_instances')->where('id', 1)
->update(['host_id' => 1]);
// A successful transition must make the active claim pair authoritative even
// when intentionally stale singular reservation mirrors remain present.
$reservationRow = Capsule::table('cloud_resource_reservations')
@ -366,7 +398,20 @@ Capsule::table('cloud_invoices')->where('id', $normalPaidInvoiceId)
// Unknown evidence can also converge to exact not-applied without releasing
// the source or leaking the reserved target.
[$lateFailedJob, $lateFailedPayload] = $dispatchJob($batch->jobId());
$lateFailedDispatch = CloudPublicNetworkTransitionWorkflow::begin(
CloudPublicNetworkTransitionCommand::adminExact(
10,
99,
'admin-network-late-failed-0010',
[4 => [
'pool_key' => str_repeat('c', 32),
'address_cidr' => '8.8.9.22/24',
]]
)
);
[$lateFailedJob, $lateFailedPayload] = $dispatchJob(
$lateFailedDispatch->jobId()
);
$lateUnknownOperation = $agentOperation(
$lateFailedJob,
$lateFailedPayload,
@ -396,7 +441,8 @@ CloudPublicNetworkTransitionWorkflow::recordEvidence(
);
$finishLocked($lateFailedJob, $lateFailedPayload, 'failed', 99, false);
$assert(Capsule::table('cloud_public_network_transitions')
->where('id', $batch->transitionId())->value('state') === 'failed'
->where('id', $lateFailedDispatch->transitionId())
->value('state') === 'failed'
&& Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', 110)->where('active_claim', 1)
->count() === 2
@ -586,4 +632,55 @@ $assert($terminalLedgerRejected,
Capsule::table('cloud_public_ips')->where('id', 504)
->update(['host_id' => 1]);
// A finalizer cannot silently skip the global host/agent sentinel. Missing
// topology must fail before transition, claim or address projection changes.
$hostFenceDispatch = CloudPublicNetworkTransitionWorkflow::begin(
CloudPublicNetworkTransitionCommand::adminExact(
12,
99,
'admin-network-host-fence-0012',
[4 => [
'pool_key' => str_repeat('c', 32),
'address_cidr' => '8.8.9.23/24',
]]
)
);
[$hostFenceJob, $hostFencePayload] = $dispatchJob(
$hostFenceDispatch->jobId()
);
$hostFenceOperation = $agentOperation(
$hostFenceJob,
$hostFencePayload,
'failed',
[]
);
$agentRow = Capsule::table('cloud_host_agents')->where('host_id', 1)->first();
Capsule::table('cloud_host_agents')->where('host_id', 1)->delete();
$hostFenceRejected = false;
try {
CloudPublicNetworkTransitionWorkflow::recordEvidence(
$hostFenceJob,
'failed',
[],
$hostFencePayload,
$hostFenceOperation
);
} catch (Throwable) {
$hostFenceRejected = true;
}
Capsule::table('cloud_host_agents')->insert((array) $agentRow);
$hostFenceTargetClaim = Capsule::table(
'cloud_public_network_transition_ips'
)->where('transition_id', $hostFenceDispatch->transitionId())
->where('address_family', 4)->value('target_claim_id');
$assert($hostFenceRejected
&& Capsule::table('cloud_public_network_transitions')
->where('id', $hostFenceDispatch->transitionId())
->value('state') === 'awaiting_receipt'
&& Capsule::table('cloud_resource_reservation_ips')
->where('id', $hostFenceTargetClaim)->value('state') === 'reserved'
&& Capsule::table('cloud_public_ips')->where('id', 204)
->value('state') === 'reserved',
'Missing host-agent sentinel allowed partial finalization writes.');
echo "Public-network transition finalization tests passed.\n";

@ -10,6 +10,15 @@ $assert = static function (bool $condition, string $message): void {
throw new RuntimeException($message);
}
};
$method = static function (string $contents, string $start, string $end) use (
$assert
): string {
$offset = strpos($contents, $start);
$limit = $offset === false ? false : strpos($contents, $end, $offset + 1);
$assert($offset !== false && $limit !== false && $limit > $offset,
'Workflow method source is missing: ' . $start);
return substr($contents, $offset, $limit - $offset);
};
$root = dirname(__DIR__);
$source = file_get_contents(
$root . '/app/Services/Cloud/CloudPublicNetworkTransitionWorkflow.php'
@ -64,26 +73,97 @@ foreach (['userPaidAutomatic', 'adminExact', 'poolReplacement'] as $factory) {
'Transition command factory is missing: ' . $factory);
}
$ordered = [
$begin = $method(
$source,
'public static function begin(',
'private static function reserveTransitionLocked('
);
$reserve = $method(
$source,
'private static function reserveTransitionLocked(',
'public static function markDispatched('
);
$ordered = static function (string $body, array $needles) use ($assert): void {
$cursor = -1;
foreach ($needles as $needle) {
$position = strpos($body, $needle);
$assert($position !== false && $position > $cursor,
'Transition lock order drifted at: ' . $needle);
$cursor = $position;
}
};
$ordered($begin, [
"User::where('id', \$command->actorUserId())",
"CloudInstance::where('id', \$command->instanceId())",
"'transition_key', \$command->transitionKey()",
"CloudHost::where('id', (int) \$instance->host_id)",
"CloudHostAgent::where('host_id', (int) \$instance->host_id)",
"CloudResourcePool::where(\n 'id', (int) \$instance->resource_pool_id",
'CloudHostNetworkAttachment::whereIn(',
"CloudPublicIpPool::whereIn('id', \$poolIds)",
"CloudResourceReservation::where(\n 'id', \$reservationId",
"->where('reservation_id', \$reservationId)\n ->orderBy('address_family')",
"CloudPublicIp::where('host_id', (int) \$instance->host_id)",
];
$cursor = -1;
foreach ($ordered as $needle) {
$position = strpos($source, $needle);
$assert($position !== false && $position > $cursor,
'Transition lock order drifted at: ' . $needle);
$cursor = $position;
'self::reserveTransitionLocked($command, $actorUser)',
]);
$assert(substr_count($begin, 'transaction(') === 1
&& !str_contains($reserve, 'transaction('),
'Reservation seam introduced a nested transaction.');
$ordered($reserve, [
'$jobHint = CloudJob::where(',
'$existingParent = CloudJob::where(',
'$existingJob = CloudJob::where(',
'$instance = CloudInstance::where(',
'[$replayHost] = self::lockHostBoundaryLocked(',
'$existing = CloudPublicNetworkTransition::where(',
]);
$assert(str_contains($source,
'(int) $parent->id !== (int) $command->parentJobId()')
&& str_contains($source,
'(int) ($job->parent_job_id ?? 0)')
&& str_contains($source,
'!== (int) $command->parentJobId()'),
'Batch replay no longer binds the locked parent identity.');
$ordered($reserve, [
'$priorJob = CloudJob::where(',
'$instance = CloudInstance::where(',
'[$host, $agent] = self::lockHostBoundaryLocked(',
'$active = CloudPublicNetworkTransition::where(',
'$resourcePool = CloudResourcePool::where(',
'$attachments = CloudHostNetworkAttachment::whereIn(',
'$pools = CloudPublicIpPool::whereIn(\'id\', $poolIds)',
'$reservation = CloudResourceReservation::where(',
'$allClaims = DB::table(\'cloud_resource_reservation_ips\')',
'$ips = CloudPublicIp::where(\'host_id\', (int) $instance->host_id)',
]);
foreach ([
['public static function markJobUncertain(',
'public static function recordEvidence('],
['public static function recordEvidence(',
'public static function finishJobLocked('],
['public static function finishJobLocked(',
'public static function publishPreparedLocked('],
['public static function publishPreparedLocked(',
'public static function retryPendingFinalization('],
] as [$start, $end]) {
$ordered($method($source, $start, $end), [
'$instance = CloudInstance::where(',
'self::lockHostBoundaryLocked(',
'$transition = CloudPublicNetworkTransition::where(',
]);
}
$retry = $method(
$source,
'public static function retryPendingFinalization(',
'private static function lockHostBoundaryLocked('
);
$ordered($retry, [
'$actor = User::where(\'id\', $userId)',
'$parent = $parentId > 0',
'$job = CloudJob::where(\'id\', $jobId)',
'self::finishJobLocked(',
]);
$topology = $method(
$source,
'private static function lockFrozenTopologyLocked(',
'private static function assertFrozenRowIdentity('
);
$ordered($topology, [
'$attachments = CloudHostNetworkAttachment::whereIn(',
"->orderBy('id')->lockForUpdate()->get()",
'$pools = CloudPublicIpPool::where(',
"->whereIn('pool_key', \$poolKeys)->orderBy('id')",
]);
$assert(str_contains($source, "\$transition->state = 'reserved';")
&& str_contains($source, "'active_claim' => null")

@ -492,6 +492,7 @@ namespace {
$addIp(201, 13, 4, '8.8.9.20');
$addIp(202, 13, 4, '8.8.9.21');
$addIp(203, 13, 4, '8.8.9.22');
$addIp(204, 13, 4, '8.8.9.23');
$addIp(301, 21, 4, '9.9.9.30');
$addIp(302, 21, 4, '9.9.9.31');
$addIp(401, 22, 6, '2404:c140:156:3c::1000');
@ -638,6 +639,8 @@ namespace {
'8.8.8.19', '2404:c140:156:3a::1009', 10);
$addInstance(11, 111, 1021, 1022, 121, 122,
'8.8.8.20', '2404:c140:156:3a::100a', 10);
$addInstance(12, 112, 1023, 1024, 123, 124,
'8.8.8.21', '2404:c140:156:3a::100b', 10);
$sourceClaims = Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', 101)->where('active_claim', 1)
@ -731,6 +734,58 @@ namespace {
&& Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', 101)->count() === 3,
'Exact replay consulted mutable target catalog or duplicated resources.');
Capsule::table('cloud_hosts')->where('id', 1)
->update(['enabled' => 0]);
Capsule::table('cloud_instances')->where('id', 1)->update([
'status' => 'suspended',
'service_status' => 'expired_grace',
'spec_json' => '{}',
]);
$mutableReplay = CloudPublicNetworkTransitionWorkflow::begin($admin);
$assert($mutableReplay->replayed()
&& $mutableReplay->transitionId() === $dispatch->transitionId()
&& Capsule::table('cloud_jobs')->count() === 1,
'Exact replay was coupled to mutable host or instance eligibility.');
Capsule::table('cloud_hosts')->where('id', 1)
->update(['enabled' => 1]);
Capsule::table('cloud_instances')->where('id', 1)->update([
'status' => 'active',
'service_status' => 'active',
'spec_json' => json_encode([
'managed' => true,
'os_source' => 'cloud_template',
'public_network' => $sourceSnapshot(
'8.8.8.10',
'2404:c140:156:3a::1000'
),
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES),
]);
Capsule::table('cloud_instances')->where('id', 1)
->update(['host_id' => 2]);
$rejects(
static fn () => CloudPublicNetworkTransitionWorkflow::begin($admin),
'Exact replay accepted a drifted immutable host association.'
);
Capsule::table('cloud_instances')->where('id', 1)
->update(['host_id' => 1]);
Capsule::table('cloud_public_network_transitions')
->where('id', $dispatch->transitionId())->update([
'state' => 'succeeded',
'effect_resolution' => 'applied',
]);
Capsule::table('cloud_instances')->where('id', 1)
->update(['host_id' => 2]);
$rejects(
static fn () => CloudPublicNetworkTransitionWorkflow::begin($admin),
'Corrupt terminal-shaped replay bypassed the active host fence.'
);
Capsule::table('cloud_instances')->where('id', 1)
->update(['host_id' => 1]);
Capsule::table('cloud_public_network_transitions')
->where('id', $dispatch->transitionId())->update([
'state' => 'queued',
'effect_resolution' => 'not_applicable',
]);
Capsule::table('cloud_public_ip_pools')->where('id', 13)
->update(['state' => 'active']);
@ -1303,6 +1358,64 @@ namespace {
Capsule::table('cloud_invoices')->where('id', $invoice->id)
->update(['amount' => '5.00']);
// The private reservation seam must share begin()'s one transaction. A
// failure at the final job insert rolls back wallet, invoice, transition,
// target claim and address fencing as one unit.
$rollbackCommand = CloudPublicNetworkTransitionCommand::userPaidAutomatic(
12,
10,
'tenant-network-rollback-0012',
'5.00'
);
$rollbackBefore = [
'money' => (string) Capsule::table('user')->where('id', 10)
->value('money'),
'invoices' => Capsule::table('cloud_invoices')->count(),
'transitions' => Capsule::table(
'cloud_public_network_transitions'
)->count(),
'legs' => Capsule::table(
'cloud_public_network_transition_ips'
)->count(),
'jobs' => Capsule::table('cloud_jobs')->count(),
'claims' => Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', 112)->count(),
];
Capsule::connection()->statement(
'CREATE TRIGGER fail_transition_job_insert '
. 'BEFORE INSERT ON cloud_jobs '
. "WHEN NEW.request_idempotency_key = "
. "'tenant-network-rollback-0012' "
. "BEGIN SELECT RAISE(ABORT, 'fault injection'); END"
);
$rollbackRejected = false;
try {
CloudPublicNetworkTransitionWorkflow::begin($rollbackCommand);
} catch (Throwable) {
$rollbackRejected = true;
}
Capsule::connection()->statement('DROP TRIGGER fail_transition_job_insert');
$assert($rollbackRejected
&& (string) Capsule::table('user')->where('id', 10)->value('money')
=== $rollbackBefore['money']
&& Capsule::table('cloud_invoices')->count()
=== $rollbackBefore['invoices']
&& Capsule::table('cloud_public_network_transitions')->count()
=== $rollbackBefore['transitions']
&& Capsule::table('cloud_public_network_transition_ips')->count()
=== $rollbackBefore['legs']
&& Capsule::table('cloud_jobs')->count() === $rollbackBefore['jobs']
&& Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', 112)->count()
=== $rollbackBefore['claims']
&& Capsule::table('cloud_public_ips')->whereIn('id', [203, 204])
->where('state', 'available')
->whereNull('reservation_id')
->whereNull('reservation_ip_claim_id')
->whereNull('instance_id')
->whereNull('operation_id')->count() === 2,
'Extracted reservation seam committed a partial transition.');
$batchCommand = CloudPublicNetworkTransitionCommand::poolReplacement(
10,
99,
@ -1346,5 +1459,86 @@ namespace {
&& Capsule::table('cloud_jobs')->count() === $beforeBatchJobs,
'Pool-replacement parent identity was mutable or left writes.');
// The internal batch caller may replay only the exact locked parent. A
// drifted parent FK must fail even when the frozen request fingerprint is
// otherwise unchanged.
$seedJob = CloudJob::find($dispatch->jobId());
$parentA = $seedJob->replicate();
$parentA->operation_id = str_repeat('d', 32);
$parentA->request_idempotency_key = 'pool-parent-replay-a-0001';
$parentA->request_fingerprint = hash('sha256', 'parent-a');
$parentA->parent_job_id = null;
$parentA->instance_id = null;
$parentA->host_id = null;
$parentA->public_network_transition_id = null;
$parentA->kind = 'public_ip_pool_replacement_batch';
$parentA->execution_mode = 'local';
$parentA->created_at = $parentA->updated_at = time();
$parentA->save();
$parentB = $parentA->replicate();
$parentB->operation_id = str_repeat('e', 32);
$parentB->request_idempotency_key = 'pool-parent-replay-b-0001';
$parentB->request_fingerprint = hash('sha256', 'parent-b');
$parentB->created_at = $parentB->updated_at = time();
$parentB->save();
$batchReplayCommand = CloudPublicNetworkTransitionCommand::poolReplacement(
1,
99,
'admin-network-0001',
(int) $parentA->id,
77,
[4 => [
'pool_key' => str_repeat('c', 32),
'address_cidr' => '8.8.9.20/24',
]]
);
Capsule::table('cloud_public_network_transitions')
->where('id', $dispatch->transitionId())->update([
'kind' => 'pool_replacement',
'pool_replacement_batch_item_id' => 77,
]);
Capsule::table('cloud_jobs')->where('id', $dispatch->jobId())->update([
'parent_job_id' => (int) $parentA->id,
'request_fingerprint' => $batchReplayCommand->fingerprint(),
]);
$reserve = new ReflectionMethod(
CloudPublicNetworkTransitionWorkflow::class,
'reserveTransitionLocked'
);
$invokeBatchReplay = static function () use (
$reserve,
$batchReplayCommand
) {
return Capsule::connection()->transaction(static function () use (
$reserve,
$batchReplayCommand
) {
$actor = \App\Models\User::where('id', 99)
->lockForUpdate()->first();
return $reserve->invoke(null, $batchReplayCommand, $actor);
});
};
$batchReplay = $invokeBatchReplay();
$assert($batchReplay->replayed()
&& $batchReplay->transitionId() === $dispatch->transitionId(),
'Exact batch parent replay did not return its frozen dispatch.');
Capsule::table('cloud_jobs')->where('id', $dispatch->jobId())
->update(['parent_job_id' => (int) $parentB->id]);
$rejects(
$invokeBatchReplay,
'Batch replay accepted a drifted parent job association.'
);
Capsule::table('cloud_jobs')->where('id', $dispatch->jobId())->update([
'parent_job_id' => null,
'request_fingerprint' => $admin->fingerprint(),
]);
Capsule::table('cloud_public_network_transitions')
->where('id', $dispatch->transitionId())->update([
'kind' => 'admin_manual',
'pool_replacement_batch_item_id' => null,
]);
CloudJob::whereIn('id', [(int) $parentA->id, (int) $parentB->id])
->delete();
echo "Public-network transition begin workflow tests passed.\n";
}

Loading…
Cancel
Save