|
|
|
|
@ -40,6 +40,7 @@ namespace {
|
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
|
|
|
|
|
|
use App\Models\CloudJob;
|
|
|
|
|
use App\Models\CloudPublicNetworkTransition;
|
|
|
|
|
use App\Models\CloudPublicIpPoolReplacementBatch;
|
|
|
|
|
use App\Services\Cloud\CloudError;
|
|
|
|
|
use App\Services\Cloud\CloudIpAddress;
|
|
|
|
|
@ -844,6 +845,33 @@ SQL);
|
|
|
|
|
)->value('id');
|
|
|
|
|
$items = DB::table('cloud_public_ip_pool_replacement_batch_items')
|
|
|
|
|
->where('batch_id', $batchId)->orderBy('instance_id')->get();
|
|
|
|
|
$projectChild = static function (int $childId): void {
|
|
|
|
|
$hint = CloudJob::find($childId);
|
|
|
|
|
if (!$hint instanceof CloudJob) {
|
|
|
|
|
throw new RuntimeException('Projection child fixture is missing.');
|
|
|
|
|
}
|
|
|
|
|
DB::connection()->transaction(static function () use (
|
|
|
|
|
$hint,
|
|
|
|
|
$childId
|
|
|
|
|
): void {
|
|
|
|
|
$parent = CloudJob::where(
|
|
|
|
|
'id',
|
|
|
|
|
(int) $hint->parent_job_id
|
|
|
|
|
)->lockForUpdate()->first();
|
|
|
|
|
$child = CloudJob::where('id', $childId)
|
|
|
|
|
->lockForUpdate()->first();
|
|
|
|
|
$transition = CloudPublicNetworkTransition::where(
|
|
|
|
|
'id',
|
|
|
|
|
(int) $child->public_network_transition_id
|
|
|
|
|
)->lockForUpdate()->first();
|
|
|
|
|
CloudPublicIpPoolReplacementBatchService::
|
|
|
|
|
projectTransitionEvidenceLocked(
|
|
|
|
|
$parent,
|
|
|
|
|
$child,
|
|
|
|
|
$transition
|
|
|
|
|
);
|
|
|
|
|
});
|
|
|
|
|
};
|
|
|
|
|
$assert($started['unchanged'] === false && $items->count() === 2
|
|
|
|
|
&& $items->every(static fn ($item): bool =>
|
|
|
|
|
(int) $item->address_family === 4
|
|
|
|
|
@ -931,28 +959,117 @@ SQL);
|
|
|
|
|
DB::table('cloud_jobs')->where('id', $firstChildId)->update([
|
|
|
|
|
'host_id' => 2,
|
|
|
|
|
]);
|
|
|
|
|
$rejects(static fn () =>
|
|
|
|
|
CloudPublicIpPoolReplacementBatchService::finishChildLocked(
|
|
|
|
|
CloudJob::find($firstChildId),
|
|
|
|
|
'succeeded'
|
|
|
|
|
), 'Finish accepted a child with a drifted frozen host.');
|
|
|
|
|
$rejects(static fn () => $projectChild($firstChildId),
|
|
|
|
|
'Projection accepted a child with a drifted frozen host.');
|
|
|
|
|
DB::table('cloud_jobs')->where('id', $firstChildId)->update([
|
|
|
|
|
'host_id' => 1,
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
$firstChild = CloudJob::find($firstChildId);
|
|
|
|
|
CloudPublicIpPoolReplacementBatchService::finishChildLocked(
|
|
|
|
|
$firstChild,
|
|
|
|
|
'uncertain'
|
|
|
|
|
);
|
|
|
|
|
$firstTransitionId = (int) DB::table('cloud_jobs')
|
|
|
|
|
->where('id', $firstChildId)
|
|
|
|
|
->value('public_network_transition_id');
|
|
|
|
|
foreach (['reserved', 'queued', 'awaiting_receipt'] as $activeState) {
|
|
|
|
|
DB::table('cloud_public_network_transitions')
|
|
|
|
|
->where('id', $firstTransitionId)->update([
|
|
|
|
|
'state' => $activeState,
|
|
|
|
|
'effect_resolution' => 'not_applicable',
|
|
|
|
|
'active_transition' => 1,
|
|
|
|
|
'effect_resolved_at' => null,
|
|
|
|
|
'terminal_at' => null,
|
|
|
|
|
'last_error_code' => '',
|
|
|
|
|
]);
|
|
|
|
|
$projectChild($firstChildId);
|
|
|
|
|
$assert(DB::table('cloud_public_ip_pool_replacement_batch_items')
|
|
|
|
|
->where('child_job_id', $firstChildId)
|
|
|
|
|
->value('state') === 'queued'
|
|
|
|
|
&& (string) CloudPublicIpPoolReplacementBatch::find($batchId)->state
|
|
|
|
|
=== 'running',
|
|
|
|
|
'Active transition state ' . $activeState
|
|
|
|
|
. ' prematurely terminated its batch item.');
|
|
|
|
|
}
|
|
|
|
|
DB::table('cloud_public_network_transitions')
|
|
|
|
|
->where('id', $firstTransitionId)->update([
|
|
|
|
|
'kind' => 'admin_manual',
|
|
|
|
|
]);
|
|
|
|
|
$corruptKindSnapshot = $snapshot();
|
|
|
|
|
$rejects(static fn () => $projectChild($firstChildId),
|
|
|
|
|
'Projection silently ignored a corrupted replacement kind.');
|
|
|
|
|
$assert($snapshot() === $corruptKindSnapshot,
|
|
|
|
|
'Corrupted replacement kind caused partial projection writes.');
|
|
|
|
|
DB::table('cloud_public_network_transitions')
|
|
|
|
|
->where('id', $firstTransitionId)->update([
|
|
|
|
|
'kind' => 'pool_replacement',
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
DB::table('cloud_public_network_transitions')
|
|
|
|
|
->where('id', $firstTransitionId)->update([
|
|
|
|
|
'state' => 'succeeded',
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'active_transition' => 1,
|
|
|
|
|
'effect_resolved_at' => null,
|
|
|
|
|
'terminal_at' => null,
|
|
|
|
|
]);
|
|
|
|
|
$invalidShapeSnapshot = $snapshot();
|
|
|
|
|
$rejects(static fn () => $projectChild($firstChildId),
|
|
|
|
|
'Projection accepted an invalid pseudo-terminal transition shape.');
|
|
|
|
|
$assert($snapshot() === $invalidShapeSnapshot,
|
|
|
|
|
'Invalid transition shape caused partial batch writes.');
|
|
|
|
|
|
|
|
|
|
$resolvedAt = time();
|
|
|
|
|
DB::table('cloud_public_network_transitions')
|
|
|
|
|
->where('id', $firstTransitionId)->update([
|
|
|
|
|
'state' => 'finalization_pending',
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'active_transition' => 1,
|
|
|
|
|
'effect_resolved_at' => $resolvedAt,
|
|
|
|
|
'terminal_at' => null,
|
|
|
|
|
'last_error_code' => '',
|
|
|
|
|
]);
|
|
|
|
|
DB::table('cloud_jobs')->where('id', $firstChildId)->update([
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
'error_code' => 'catalog_poison',
|
|
|
|
|
]);
|
|
|
|
|
$projectChild($firstChildId);
|
|
|
|
|
$assert(DB::table('cloud_public_ip_pool_replacement_batch_items')
|
|
|
|
|
->where('child_job_id', $firstChildId)->value('state') === 'queued'
|
|
|
|
|
&& (string) CloudPublicIpPoolReplacementBatch::find($batchId)->state
|
|
|
|
|
=== 'running',
|
|
|
|
|
'Finalization-pending evidence prematurely terminated a batch item.');
|
|
|
|
|
|
|
|
|
|
DB::table('cloud_public_network_transitions')
|
|
|
|
|
->where('id', $firstTransitionId)->update([
|
|
|
|
|
'state' => 'effect_unknown',
|
|
|
|
|
'effect_resolution' => 'unknown',
|
|
|
|
|
'active_transition' => 1,
|
|
|
|
|
'effect_resolved_at' => null,
|
|
|
|
|
'terminal_at' => null,
|
|
|
|
|
'last_error_code' => 'durable_effect_unknown',
|
|
|
|
|
]);
|
|
|
|
|
$unknownIdentity = [
|
|
|
|
|
'job_count' => DB::table('cloud_jobs')->count(),
|
|
|
|
|
'transition_count' => DB::table(
|
|
|
|
|
'cloud_public_network_transitions'
|
|
|
|
|
)->count(),
|
|
|
|
|
'operation_id' => (string) DB::table('cloud_jobs')
|
|
|
|
|
->where('id', $firstChildId)->value('operation_id'),
|
|
|
|
|
];
|
|
|
|
|
$projectChild($firstChildId);
|
|
|
|
|
$assert((string) CloudPublicIpPoolReplacementBatch::find($batchId)->state
|
|
|
|
|
=== 'blocked'
|
|
|
|
|
&& (int) CloudPublicIpPoolReplacementBatch::find($batchId)->active_claim
|
|
|
|
|
=== 1
|
|
|
|
|
&& DB::table('cloud_public_ip_pool_replacement_batch_items')
|
|
|
|
|
->where('child_job_id', $firstChildId)->value('state') === 'unknown'
|
|
|
|
|
&& DB::table('cloud_public_ip_pool_replacement_batch_items')
|
|
|
|
|
->where('child_job_id', $firstChildId)
|
|
|
|
|
->value('last_error_code') === 'durable_effect_unknown'
|
|
|
|
|
&& DB::table('cloud_jobs')->count() === $unknownIdentity['job_count']
|
|
|
|
|
&& DB::table('cloud_public_network_transitions')->count()
|
|
|
|
|
=== $unknownIdentity['transition_count']
|
|
|
|
|
&& (string) DB::table('cloud_jobs')->where('id', $firstChildId)
|
|
|
|
|
->value('operation_id') === $unknownIdentity['operation_id']
|
|
|
|
|
&& (string) CloudJob::find($parentId)->phase !== 'terminal',
|
|
|
|
|
'Unknown child did not retain the batch and parent fences.');
|
|
|
|
|
'Durable unknown evidence did not retain batch and parent fences.');
|
|
|
|
|
|
|
|
|
|
$applyChild = static function (int $childId): CloudJob {
|
|
|
|
|
$child = CloudJob::find($childId);
|
|
|
|
|
@ -1060,10 +1177,8 @@ SQL);
|
|
|
|
|
return CloudJob::find($childId);
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$resolvedFirst = $applyChild($firstChildId);
|
|
|
|
|
CloudPublicIpPoolReplacementBatchService::finishResolvedChildLocked(
|
|
|
|
|
$resolvedFirst
|
|
|
|
|
);
|
|
|
|
|
$applyChild($firstChildId);
|
|
|
|
|
$projectChild($firstChildId);
|
|
|
|
|
$assert(DB::table('cloud_public_ip_pool_replacement_batch_items')
|
|
|
|
|
->where('child_job_id', $firstChildId)
|
|
|
|
|
->value('state') === 'succeeded'
|
|
|
|
|
@ -1072,12 +1187,25 @@ SQL);
|
|
|
|
|
&& (string) CloudJob::find($parentId)->phase !== 'terminal',
|
|
|
|
|
'One resolved child did not release blocked state in one hook.');
|
|
|
|
|
|
|
|
|
|
DB::table('cloud_public_network_transitions')
|
|
|
|
|
->where('id', $firstTransitionId)->update([
|
|
|
|
|
'state' => 'failed',
|
|
|
|
|
'effect_resolution' => 'not_applied',
|
|
|
|
|
]);
|
|
|
|
|
$terminalConflictSnapshot = $snapshot();
|
|
|
|
|
$rejects(static fn () => $projectChild($firstChildId),
|
|
|
|
|
'Projection rewrote a terminal item while its sibling was pending.');
|
|
|
|
|
$assert($snapshot() === $terminalConflictSnapshot,
|
|
|
|
|
'Conflicting terminal item evidence caused partial writes.');
|
|
|
|
|
DB::table('cloud_public_network_transitions')
|
|
|
|
|
->where('id', $firstTransitionId)->update([
|
|
|
|
|
'state' => 'succeeded',
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
$secondChildId = (int) $items->last()->child_job_id;
|
|
|
|
|
$resolvedSecond = $applyChild($secondChildId);
|
|
|
|
|
CloudPublicIpPoolReplacementBatchService::finishChildLocked(
|
|
|
|
|
$resolvedSecond,
|
|
|
|
|
'succeeded'
|
|
|
|
|
);
|
|
|
|
|
$applyChild($secondChildId);
|
|
|
|
|
$projectChild($secondChildId);
|
|
|
|
|
$terminalBatch = CloudPublicIpPoolReplacementBatch::find($batchId);
|
|
|
|
|
$terminalParent = CloudJob::find($parentId);
|
|
|
|
|
$assert((string) $terminalBatch->state === 'succeeded'
|
|
|
|
|
@ -1096,6 +1224,11 @@ SQL);
|
|
|
|
|
})->count() === 0,
|
|
|
|
|
'All-success children did not converge parent/batch/source in one pass.');
|
|
|
|
|
|
|
|
|
|
$terminalProjectionSnapshot = $snapshot();
|
|
|
|
|
$projectChild($secondChildId);
|
|
|
|
|
$assert($snapshot() === $terminalProjectionSnapshot,
|
|
|
|
|
'Repeated terminal transition projection was not idempotent.');
|
|
|
|
|
|
|
|
|
|
// A strict terminal topology remains replayable after the instance has
|
|
|
|
|
// legitimately acquired a later companion-family generation.
|
|
|
|
|
$terminalAt = time() + 10;
|
|
|
|
|
@ -1271,6 +1404,58 @@ SQL);
|
|
|
|
|
'Sparse IPv6 child did not freeze the companion IPv4 leg.');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$failedV6ChildId = (int) $v6Items->first()->child_job_id;
|
|
|
|
|
$failedV6TransitionId = (int) DB::table('cloud_jobs')
|
|
|
|
|
->where('id', $failedV6ChildId)
|
|
|
|
|
->value('public_network_transition_id');
|
|
|
|
|
$failedV6Leg = DB::table('cloud_public_network_transition_ips')
|
|
|
|
|
->where('transition_id', $failedV6TransitionId)
|
|
|
|
|
->where('changed', 1)->first();
|
|
|
|
|
$failedV6TargetClaim = DB::table('cloud_resource_reservation_ips')
|
|
|
|
|
->where('id', (int) $failedV6Leg->target_claim_id)->first();
|
|
|
|
|
$failedAt = time() + 20;
|
|
|
|
|
DB::table('cloud_resource_reservation_ips')
|
|
|
|
|
->where('id', (int) $failedV6TargetClaim->id)->update([
|
|
|
|
|
'active_claim' => null,
|
|
|
|
|
'state' => 'released',
|
|
|
|
|
'released_at' => $failedAt,
|
|
|
|
|
'release_reason' => 'public_network_transition_not_applied',
|
|
|
|
|
'updated_at' => $failedAt,
|
|
|
|
|
]);
|
|
|
|
|
DB::table('cloud_public_ips')
|
|
|
|
|
->where('id', (int) $failedV6TargetClaim->public_ip_id)->update([
|
|
|
|
|
'state' => 'available',
|
|
|
|
|
'reservation_id' => null,
|
|
|
|
|
'reservation_ip_claim_id' => null,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'operation_id' => null,
|
|
|
|
|
'updated_at' => $failedAt,
|
|
|
|
|
]);
|
|
|
|
|
DB::table('cloud_public_network_transitions')
|
|
|
|
|
->where('id', $failedV6TransitionId)->update([
|
|
|
|
|
'state' => 'failed',
|
|
|
|
|
'effect_resolution' => 'not_applied',
|
|
|
|
|
'active_transition' => null,
|
|
|
|
|
'effect_resolved_at' => $failedAt,
|
|
|
|
|
'terminal_at' => $failedAt,
|
|
|
|
|
'last_error_code' => 'durable_not_applied',
|
|
|
|
|
'updated_at' => $failedAt,
|
|
|
|
|
]);
|
|
|
|
|
DB::table('cloud_jobs')->where('id', $failedV6ChildId)->update([
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'error_code' => null,
|
|
|
|
|
]);
|
|
|
|
|
$projectChild($failedV6ChildId);
|
|
|
|
|
$assert(DB::table('cloud_public_ip_pool_replacement_batch_items')
|
|
|
|
|
->where('child_job_id', $failedV6ChildId)->value('state')
|
|
|
|
|
=== 'failed'
|
|
|
|
|
&& DB::table('cloud_public_ip_pool_replacement_batch_items')
|
|
|
|
|
->where('child_job_id', $failedV6ChildId)
|
|
|
|
|
->value('last_error_code') === 'durable_not_applied'
|
|
|
|
|
&& (string) CloudPublicIpPoolReplacementBatch::find($v6BatchId)->state
|
|
|
|
|
=== 'running',
|
|
|
|
|
'Strict failed/not-applied transition did not override job catalog drift.');
|
|
|
|
|
|
|
|
|
|
$jobService = file_get_contents(
|
|
|
|
|
dirname(__DIR__) . '/app/Services/Cloud/CloudJobService.php'
|
|
|
|
|
);
|
|
|
|
|
|