You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
751 lines
28 KiB
PHP
751 lines
28 KiB
PHP
<?php
|
|
|
|
require __DIR__ . '/CloudPublicNetworkTransitionWorkflowTest.php';
|
|
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudJob;
|
|
use App\Models\User;
|
|
use App\Services\Cloud\CloudAdminOperationRecoveryService;
|
|
use App\Services\Cloud\CloudOperationRecoveryIdentity;
|
|
use App\Services\Cloud\CloudNetworkChangeOptionsService;
|
|
use App\Services\Cloud\CloudPublicNetworkAllocationService;
|
|
use App\Services\Cloud\CloudPublicNetworkTransitionCommand;
|
|
use App\Services\Cloud\CloudPublicNetworkTransitionWorkflow;
|
|
use App\Services\Cloud\CloudSecrets;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
|
|
$dispatchJob = static function (int $jobId): array {
|
|
$now = time();
|
|
$token = bin2hex(random_bytes(16));
|
|
Capsule::table('cloud_jobs')->where('id', $jobId)->update([
|
|
'phase' => 'dispatching', 'status' => 'running',
|
|
'attempts' => 1, 'dispatch_attempts' => 1,
|
|
'dispatch_started_at' => $now, 'dispatched_at' => $now,
|
|
'receipt_deadline_at' => $now + 300,
|
|
'lease_token' => $token, 'lease_until' => $now + 300,
|
|
'updated_at' => $now,
|
|
]);
|
|
$job = CloudJob::find($jobId);
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $job->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
CloudPublicNetworkTransitionWorkflow::markDispatched($job);
|
|
return [$job, $payload];
|
|
};
|
|
|
|
$agentOperation = static function (CloudJob $job, array $payload,
|
|
string $status, array $result): array {
|
|
$identity = CloudOperationRecoveryIdentity::expectedFromPayload(
|
|
$job,
|
|
$payload,
|
|
$status === 'succeeded'
|
|
);
|
|
return [
|
|
'operation_id' => (string) $job->operation_id,
|
|
'action' => $identity['action'],
|
|
'resource_id' => $identity['resource_id'],
|
|
'request_fingerprint' => $identity['request_fingerprint'],
|
|
'status' => $status,
|
|
'effect_may_have_applied' => false,
|
|
'safe_to_resubmit' => false,
|
|
'result' => $result,
|
|
];
|
|
};
|
|
|
|
$finishLocked = static function (CloudJob $source, array $payload,
|
|
string $status, int $userId, bool $defer): void {
|
|
Capsule::connection()->transaction(static function () use (
|
|
$source, $payload, $status, $userId, $defer
|
|
): void {
|
|
$user = User::where('id', $userId)->lockForUpdate()->first();
|
|
$job = CloudJob::where('id', (int) $source->id)
|
|
->lockForUpdate()->first();
|
|
CloudPublicNetworkTransitionWorkflow::finishJobLocked(
|
|
$job,
|
|
$status,
|
|
$payload,
|
|
$user,
|
|
$defer
|
|
);
|
|
});
|
|
};
|
|
|
|
// Recovery accepts only the transition-owned typed public-network projection.
|
|
// Retired scalar tasks have no recovery authority after the clean break.
|
|
$projectionMethod = new ReflectionMethod(
|
|
CloudAdminOperationRecoveryService::class,
|
|
'effectProjection'
|
|
);
|
|
$assert($projectionMethod->getNumberOfParameters() === 3,
|
|
'Recovery projection retained the retired transition compatibility flag.');
|
|
$legacyProjection = [
|
|
'uuid' => (string) CloudInstance::find(1)->uuid,
|
|
'network_changed' => true,
|
|
];
|
|
$legacyProjectionRejected = false;
|
|
try {
|
|
$projectionMethod->invoke(
|
|
null,
|
|
$legacyProjection,
|
|
'network_change',
|
|
'applied'
|
|
);
|
|
} catch (Throwable) {
|
|
$legacyProjectionRejected = true;
|
|
}
|
|
$assert($legacyProjectionRejected,
|
|
'Recovery accepted a retired scalar network-change projection.');
|
|
$typedProjection = $legacyProjection + [
|
|
'public_network' => $dispatch->target()->toArray(),
|
|
];
|
|
$assert($projectionMethod->invoke(
|
|
null,
|
|
$typedProjection,
|
|
'network_change',
|
|
'applied'
|
|
) === $typedProjection,
|
|
'Transition recovery rejected its canonical public_network projection.');
|
|
|
|
// Applied single-family transition preserves the unchanged IPv6 leg.
|
|
[$appliedJob, $appliedPayload] = $dispatchJob($dispatch->jobId());
|
|
$appliedResult = [
|
|
'uuid' => (string) CloudInstance::find(1)->uuid,
|
|
'network_changed' => true,
|
|
'public_network' => $dispatch->target()->toArray(),
|
|
];
|
|
$appliedOperation = $agentOperation(
|
|
$appliedJob, $appliedPayload, 'succeeded', $appliedResult
|
|
);
|
|
CloudPublicNetworkTransitionWorkflow::recordEvidence(
|
|
$appliedJob, 'succeeded', $appliedResult, $appliedPayload,
|
|
$appliedOperation
|
|
);
|
|
$assert(Capsule::table('cloud_public_network_transitions')
|
|
->where('id', $dispatch->transitionId())
|
|
->value('state') === 'finalization_pending',
|
|
'Applied evidence did not survive independently of finalization.');
|
|
$finishLocked($appliedJob, $appliedPayload, 'succeeded', 99, false);
|
|
$source4 = Capsule::table('cloud_resource_reservation_ips')
|
|
->where('id', 1001)->first();
|
|
$unchanged6 = Capsule::table('cloud_resource_reservation_ips')
|
|
->where('id', 1002)->first();
|
|
$target4 = Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', 101)->where('address_family', 4)
|
|
->where('active_claim', 1)->first();
|
|
$assert($source4->state === 'released'
|
|
&& $source4->release_reason === 'public_network_transition_succeeded'
|
|
&& (int) $unchanged6->active_claim === 1
|
|
&& $unchanged6->state === 'committed'
|
|
&& (int) $target4->public_ip_id === 201
|
|
&& Capsule::table('cloud_public_ips')->where('id', 101)
|
|
->value('state') === 'available'
|
|
&& Capsule::table('cloud_public_ips')->where('id', 201)
|
|
->value('state') === 'assigned',
|
|
'Applied transition did not atomically switch only the changed leg.');
|
|
|
|
// Once the transition is terminal, a later maintenance migration may change
|
|
// current placement. Exact replay still returns the frozen historical result
|
|
// while an active transition (covered by begin tests) rejects the same drift.
|
|
$terminalPayload = $appliedPayload;
|
|
$terminalPayload[CloudOperationRecoveryIdentity::payloadKey()] =
|
|
CloudOperationRecoveryIdentity::persisted([
|
|
'action' => $appliedOperation['action'],
|
|
'resource_id' => $appliedOperation['resource_id'],
|
|
'request_fingerprint' => $appliedOperation['request_fingerprint'],
|
|
]);
|
|
Capsule::table('cloud_jobs')->where('id', (int) $appliedJob->id)->update([
|
|
'phase' => 'terminal',
|
|
'outcome' => 'succeeded',
|
|
'status' => 'succeeded',
|
|
'terminal_at' => time(),
|
|
'effect_may_have_applied' => 0,
|
|
'safe_to_resubmit' => 0,
|
|
'effect_resolution' => 'applied',
|
|
'sealed_payload' => CloudSecrets::encrypt(
|
|
$terminalPayload,
|
|
'job-payload'
|
|
),
|
|
]);
|
|
Capsule::table('cloud_instances')->where('id', 1)
|
|
->update(['host_id' => 2]);
|
|
$terminalMigratedReplay = CloudPublicNetworkTransitionWorkflow::begin($admin);
|
|
$assert($terminalMigratedReplay->replayed()
|
|
&& $terminalMigratedReplay->transitionId() === $dispatch->transitionId(),
|
|
'Terminal exact replay was coupled to later instance placement.');
|
|
Capsule::table('cloud_instances')->where('id', 1)
|
|
->update(['host_id' => 1]);
|
|
|
|
// A successful transition must make the active claim pair authoritative even
|
|
// when intentionally stale singular reservation mirrors remain present.
|
|
$reservationRow = Capsule::table('cloud_resource_reservations')
|
|
->where('id', 101)->first();
|
|
$postTransitionOptions = CloudNetworkChangeOptionsService::adminOptions(
|
|
CloudInstance::find(1)
|
|
);
|
|
$assert((int) $reservationRow->network_attachment_id === 999
|
|
&& (int) $reservationRow->public_ip_pool_id === 999
|
|
&& (int) $reservationRow->public_ip_id === 999
|
|
&& ($postTransitionOptions['current_attachment_id'] ?? null) === '1'
|
|
&& ($postTransitionOptions['current_public_network']['ipv4']
|
|
['address_cidr'] ?? null) === '8.8.9.20/24'
|
|
&& ($postTransitionOptions['current_public_network']['ipv6']
|
|
['address_cidr'] ?? null) === '2404:c140:156:3a::1000/64',
|
|
'Post-transition options fell back to stale singular reservation data.');
|
|
|
|
CloudPublicNetworkAllocationService::releaseCommittedBundle(
|
|
101,
|
|
1,
|
|
(string) $reservationOperation,
|
|
'instance_deleted',
|
|
true
|
|
);
|
|
$assert(Capsule::table('cloud_resource_reservation_ips')
|
|
->whereIn('id', [(int) $target4->id, 1002])
|
|
->where('state', 'released')->count() === 2
|
|
&& Capsule::table('cloud_public_ips')->whereIn('id', [201, 102])
|
|
->where('state', 'available')->count() === 2
|
|
&& Capsule::table('cloud_public_ips')->where('instance_id', 1)
|
|
->where('state', 'assigned')->count() === 0,
|
|
'Committed release used stale scalar mirrors instead of active claims.');
|
|
|
|
// Recovery prepare may release its target projection, but wallet settlement
|
|
// and transition publication wait for the later recovery commit.
|
|
[$failedJob, $failedPayload] = $dispatchJob($paidDispatch->jobId());
|
|
$failedOperation = $agentOperation(
|
|
$failedJob, $failedPayload, 'failed', []
|
|
);
|
|
CloudPublicNetworkTransitionWorkflow::recordEvidence(
|
|
$failedJob, 'failed', [], $failedPayload, $failedOperation, true
|
|
);
|
|
$finishLocked($failedJob, $failedPayload, 'failed', 10, true);
|
|
$paidTransitionId = $paidDispatch->transitionId();
|
|
$paidInvoiceId = Capsule::table('cloud_public_network_transitions')
|
|
->where('id', $paidTransitionId)->value('invoice_id');
|
|
$assert(Capsule::table('cloud_public_network_transitions')
|
|
->where('id', $paidTransitionId)->value('state')
|
|
=== 'finalization_pending'
|
|
&& (int) Capsule::table('cloud_public_network_transitions')
|
|
->where('id', $paidTransitionId)->value('active_transition') === 1
|
|
&& Capsule::table('cloud_invoices')->where('id', $paidInvoiceId)
|
|
->value('status') === 'paid'
|
|
&& (string) Capsule::table('user')->where('id', 10)
|
|
->value('money') === '15.00',
|
|
'Recovery prepare refunded or published before fence commit.');
|
|
Capsule::connection()->transaction(static function () use ($failedJob): void {
|
|
$user = User::where('id', 10)->lockForUpdate()->first();
|
|
$job = CloudJob::where('id', (int) $failedJob->id)
|
|
->lockForUpdate()->first();
|
|
CloudPublicNetworkTransitionWorkflow::publishPreparedLocked(
|
|
$job,
|
|
'not_applied',
|
|
$user
|
|
);
|
|
});
|
|
$assert(Capsule::table('cloud_invoices')->where('id', $paidInvoiceId)
|
|
->value('status') === 'refunded'
|
|
&& (string) Capsule::table('user')->where('id', 10)
|
|
->value('money') === '20.00'
|
|
&& Capsule::table('cloud_public_network_transitions')
|
|
->where('id', $paidTransitionId)->value('state') === 'failed',
|
|
'Recovery commit did not refund exactly once and publish failure.');
|
|
|
|
// Unknown evidence retains both targets and may later converge the same
|
|
// operation from an exact applied result without creating another claim.
|
|
[$unknownJob, $unknownPayload] = $dispatchJob($cross->jobId());
|
|
$unknownOperation = $agentOperation(
|
|
$unknownJob, $unknownPayload, 'failed', []
|
|
);
|
|
$unknownOperation['effect_may_have_applied'] = true;
|
|
CloudPublicNetworkTransitionWorkflow::recordEvidence(
|
|
$unknownJob, 'uncertain', [], $unknownPayload, $unknownOperation
|
|
);
|
|
$assert(Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', 102)->whereNull('active_claim')
|
|
->where('state', 'uncertain')->count() === 2,
|
|
'Unknown effect did not retain both target claims as uncertain.');
|
|
$lateResult = [
|
|
'uuid' => (string) CloudInstance::find(2)->uuid,
|
|
'network_changed' => true,
|
|
'public_network' => $cross->target()->toArray(),
|
|
];
|
|
$lateOperation = $agentOperation(
|
|
$unknownJob, $unknownPayload, 'succeeded', $lateResult
|
|
);
|
|
CloudPublicNetworkTransitionWorkflow::recordEvidence(
|
|
$unknownJob, 'succeeded', $lateResult, $unknownPayload, $lateOperation
|
|
);
|
|
$finishLocked($unknownJob, $unknownPayload, 'succeeded', 99, false);
|
|
$assert(Capsule::table('cloud_public_network_transitions')
|
|
->where('id', $cross->transitionId())->value('state') === 'succeeded',
|
|
'Late exact applied evidence did not converge the same transition.');
|
|
|
|
// A normal receipt can stage evidence, crash, and later enter recovery
|
|
// prepare. The pending marker must be promoted so commit can publish.
|
|
[$crashJob, $crashPayload] = $dispatchJob($sparse->jobId());
|
|
$crashResult = [
|
|
'uuid' => (string) CloudInstance::find(8)->uuid,
|
|
'network_changed' => true,
|
|
'public_network' => $sparse->target()->toArray(),
|
|
];
|
|
$crashOperation = $agentOperation(
|
|
$crashJob, $crashPayload, 'succeeded', $crashResult
|
|
);
|
|
CloudPublicNetworkTransitionWorkflow::recordEvidence(
|
|
$crashJob, 'succeeded', $crashResult, $crashPayload, $crashOperation
|
|
);
|
|
$now = time();
|
|
Capsule::table('cloud_jobs')->where('id', (int) $crashJob->id)->update([
|
|
'phase' => 'terminal', 'outcome' => 'failed', 'status' => 'failed',
|
|
'terminal_at' => $now, 'effect_may_have_applied' => 1,
|
|
'safe_to_resubmit' => 0, 'effect_resolution' => 'unknown',
|
|
'error_code' => 'local_finalization_failed',
|
|
'finalization_incident' => 1,
|
|
'finalization_error_code' => 'recovery_internal_error',
|
|
'lease_token' => null, 'lease_until' => 0,
|
|
'available_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
$crashJob = CloudJob::find((int) $crashJob->id);
|
|
CloudPublicNetworkTransitionWorkflow::recordEvidence(
|
|
$crashJob, 'succeeded', $crashResult, $crashPayload,
|
|
$crashOperation, true
|
|
);
|
|
$assert(Capsule::table('cloud_public_network_transitions')
|
|
->where('id', $sparse->transitionId())->value('last_error_code')
|
|
=== 'recovery_prepare_pending',
|
|
'Late recovery did not promote an already-pending evidence marker.');
|
|
$finishLocked($crashJob, $crashPayload, 'succeeded', 99, true);
|
|
Capsule::connection()->transaction(static function () use ($crashJob): void {
|
|
$user = User::where('id', 99)->lockForUpdate()->first();
|
|
$job = CloudJob::where('id', (int) $crashJob->id)
|
|
->lockForUpdate()->first();
|
|
CloudPublicNetworkTransitionWorkflow::publishPreparedLocked(
|
|
$job,
|
|
'applied',
|
|
$user
|
|
);
|
|
});
|
|
$assert(Capsule::table('cloud_public_network_transitions')
|
|
->where('id', $sparse->transitionId())->value('state') === 'succeeded',
|
|
'Recovery commit left transition permanently pending.');
|
|
|
|
// Ordinary not-applied finalization refunds immediately and exact terminal
|
|
// replay cannot credit the wallet a second time.
|
|
$normalPaid = CloudPublicNetworkTransitionCommand::userPaidAutomatic(
|
|
4,
|
|
10,
|
|
'tenant-network-finalize-0002',
|
|
'5.00'
|
|
);
|
|
$normalPaidDispatch = CloudPublicNetworkTransitionWorkflow::begin($normalPaid);
|
|
[$normalPaidJob, $normalPaidPayload] = $dispatchJob(
|
|
$normalPaidDispatch->jobId()
|
|
);
|
|
$normalFailedOperation = $agentOperation(
|
|
$normalPaidJob,
|
|
$normalPaidPayload,
|
|
'failed',
|
|
[]
|
|
);
|
|
CloudPublicNetworkTransitionWorkflow::recordEvidence(
|
|
$normalPaidJob,
|
|
'failed',
|
|
[],
|
|
$normalPaidPayload,
|
|
$normalFailedOperation
|
|
);
|
|
$finishLocked($normalPaidJob, $normalPaidPayload, 'failed', 10, false);
|
|
$normalPaidTransition = Capsule::table('cloud_public_network_transitions')
|
|
->where('id', $normalPaidDispatch->transitionId())->first();
|
|
$normalPaidInvoiceId = (int) $normalPaidTransition->invoice_id;
|
|
$assert((string) Capsule::table('user')->where('id', 10)->value('money')
|
|
=== '20.00'
|
|
&& Capsule::table('cloud_invoices')->where('id', $normalPaidInvoiceId)
|
|
->value('status') === 'refunded'
|
|
&& $normalPaidTransition->state === 'failed',
|
|
'Ordinary not-applied finalization did not refund atomically.');
|
|
CloudPublicNetworkTransitionWorkflow::recordEvidence(
|
|
$normalPaidJob,
|
|
'failed',
|
|
[],
|
|
$normalPaidPayload,
|
|
$normalFailedOperation
|
|
);
|
|
$finishLocked($normalPaidJob, $normalPaidPayload, 'failed', 10, false);
|
|
$assert((string) Capsule::table('user')->where('id', 10)->value('money')
|
|
=== '20.00',
|
|
'Terminal not-applied replay credited the wallet twice.');
|
|
Capsule::table('cloud_invoices')->where('id', $normalPaidInvoiceId)
|
|
->update(['amount' => '6.00']);
|
|
$terminalInvoiceRejected = false;
|
|
try {
|
|
$finishLocked($normalPaidJob, $normalPaidPayload, 'failed', 10, false);
|
|
} catch (Throwable) {
|
|
$terminalInvoiceRejected = true;
|
|
}
|
|
$assert($terminalInvoiceRejected,
|
|
'Terminal replay accepted a drifted refunded invoice.');
|
|
Capsule::table('cloud_invoices')->where('id', $normalPaidInvoiceId)
|
|
->update(['amount' => '5.00']);
|
|
|
|
// Unknown evidence can also converge to exact not-applied without releasing
|
|
// the source or leaking the reserved target.
|
|
$lateFailedDispatch = CloudPublicNetworkTransitionWorkflow::begin(
|
|
CloudPublicNetworkTransitionCommand::adminExact(
|
|
10,
|
|
99,
|
|
'admin-network-late-failed-0010',
|
|
[4 => [
|
|
'pool_key' => str_repeat('c', 32),
|
|
'address_cidr' => '8.8.9.22/24',
|
|
]]
|
|
)
|
|
);
|
|
[$lateFailedJob, $lateFailedPayload] = $dispatchJob(
|
|
$lateFailedDispatch->jobId()
|
|
);
|
|
$lateUnknownOperation = $agentOperation(
|
|
$lateFailedJob,
|
|
$lateFailedPayload,
|
|
'failed',
|
|
[]
|
|
);
|
|
$lateUnknownOperation['effect_may_have_applied'] = true;
|
|
CloudPublicNetworkTransitionWorkflow::recordEvidence(
|
|
$lateFailedJob,
|
|
'uncertain',
|
|
[],
|
|
$lateFailedPayload,
|
|
$lateUnknownOperation
|
|
);
|
|
$lateFailedOperation = $agentOperation(
|
|
$lateFailedJob,
|
|
$lateFailedPayload,
|
|
'failed',
|
|
[]
|
|
);
|
|
CloudPublicNetworkTransitionWorkflow::recordEvidence(
|
|
$lateFailedJob,
|
|
'failed',
|
|
[],
|
|
$lateFailedPayload,
|
|
$lateFailedOperation
|
|
);
|
|
$finishLocked($lateFailedJob, $lateFailedPayload, 'failed', 99, false);
|
|
$assert(Capsule::table('cloud_public_network_transitions')
|
|
->where('id', $lateFailedDispatch->transitionId())
|
|
->value('state') === 'failed'
|
|
&& Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', 110)->where('active_claim', 1)
|
|
->count() === 2
|
|
&& Capsule::table('cloud_public_ips')->where('id', 203)
|
|
->value('state') === 'available',
|
|
'Late not-applied evidence changed the source or leaked its target.');
|
|
|
|
// Evidence can survive an ordinary Worker finalizer crash and converge via
|
|
// the PHP-only automatic finalizer. The original terminal audit remains a
|
|
// failure while only the effect resolution and local ledger are repaired.
|
|
$retryDispatch = CloudPublicNetworkTransitionWorkflow::begin(
|
|
CloudPublicNetworkTransitionCommand::adminExact(
|
|
11,
|
|
99,
|
|
'admin-network-retry-0004',
|
|
[4 => [
|
|
'pool_key' => str_repeat('c', 32),
|
|
'address_cidr' => '8.8.9.22/24',
|
|
]]
|
|
)
|
|
);
|
|
[$retryJob, $retryPayload] = $dispatchJob($retryDispatch->jobId());
|
|
$retryResult = [
|
|
'uuid' => (string) CloudInstance::find(11)->uuid,
|
|
'network_changed' => true,
|
|
'public_network' => $retryDispatch->target()->toArray(),
|
|
];
|
|
$retryOperation = $agentOperation(
|
|
$retryJob,
|
|
$retryPayload,
|
|
'succeeded',
|
|
$retryResult
|
|
);
|
|
CloudPublicNetworkTransitionWorkflow::recordEvidence(
|
|
$retryJob,
|
|
'succeeded',
|
|
$retryResult,
|
|
$retryPayload,
|
|
$retryOperation
|
|
);
|
|
$retryLease = bin2hex(random_bytes(16));
|
|
$retryTerminalAt = time();
|
|
Capsule::table('cloud_jobs')->where('id', (int) $retryJob->id)->update([
|
|
'phase' => 'terminal',
|
|
'outcome' => 'failed',
|
|
'status' => 'failed',
|
|
'terminal_at' => $retryTerminalAt,
|
|
'effect_may_have_applied' => 1,
|
|
'safe_to_resubmit' => 0,
|
|
'effect_resolution' => 'unknown',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolved_by' => null,
|
|
'effect_resolution_reason' => '',
|
|
'error_code' => 'local_finalization_failed',
|
|
'finalization_incident' => 1,
|
|
'finalization_error_code' => 'recovery_internal_error',
|
|
'lease_token' => $retryLease,
|
|
'lease_until' => time() + 300,
|
|
'available_at' => time(),
|
|
'updated_at' => time(),
|
|
]);
|
|
$retried = CloudPublicNetworkTransitionWorkflow::retryPendingFinalization(
|
|
(int) $retryJob->id,
|
|
$retryLease
|
|
);
|
|
$retriedPayload = CloudSecrets::decrypt(
|
|
(string) $retried->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
$retriedIdentity = CloudOperationRecoveryIdentity::expected($retried, true);
|
|
$assert((string) $retried->phase === 'terminal'
|
|
&& (string) $retried->outcome === 'failed'
|
|
&& (string) $retried->status === 'failed'
|
|
&& (int) $retried->terminal_at === $retryTerminalAt
|
|
&& (string) $retried->error_code === 'local_finalization_failed'
|
|
&& (string) $retried->effect_resolution === 'applied'
|
|
&& !(bool) $retried->finalization_incident
|
|
&& $retried->lease_token === null
|
|
&& !array_key_exists('_operation_recovery', $retriedPayload)
|
|
&& isset($retriedPayload[
|
|
CloudOperationRecoveryIdentity::payloadKey()
|
|
])
|
|
&& $retriedIdentity === CloudOperationRecoveryIdentity::expectedFromPayload(
|
|
$retried,
|
|
$retriedPayload,
|
|
true
|
|
)
|
|
&& Capsule::table('cloud_public_network_transitions')
|
|
->where('id', $retryDispatch->transitionId())->value('state')
|
|
=== 'succeeded',
|
|
'PHP-only finalizer changed terminal audit or failed to converge.');
|
|
|
|
// A database failure after the first dual-stack leg starts switching must
|
|
// roll the whole local projection back. Retrying the same persisted evidence
|
|
// then succeeds without another Agent dispatch.
|
|
$addIp(503, 21, 4, '9.9.9.32');
|
|
$addIp(504, 22, 6, '2404:c140:156:3c::1004');
|
|
Capsule::table('cloud_host_network_attachments')->where('id', 1)
|
|
->update(['state' => 'draining']);
|
|
$atomic = CloudPublicNetworkTransitionWorkflow::begin(
|
|
CloudPublicNetworkTransitionCommand::adminExact(
|
|
5,
|
|
99,
|
|
'admin-network-atomic-0005',
|
|
[
|
|
4 => [
|
|
'pool_key' => str_repeat('e', 32),
|
|
'address_cidr' => '9.9.9.32/24',
|
|
],
|
|
6 => [
|
|
'pool_key' => str_repeat('f', 32),
|
|
'address_cidr' => '2404:c140:156:3c::1004/64',
|
|
],
|
|
]
|
|
)
|
|
);
|
|
Capsule::table('cloud_host_network_attachments')->where('id', 1)
|
|
->update(['state' => 'active']);
|
|
[$atomicJob, $atomicPayload] = $dispatchJob($atomic->jobId());
|
|
$atomicResult = [
|
|
'uuid' => (string) CloudInstance::find(5)->uuid,
|
|
'network_changed' => true,
|
|
'public_network' => $atomic->target()->toArray(),
|
|
];
|
|
$atomicOperation = $agentOperation(
|
|
$atomicJob,
|
|
$atomicPayload,
|
|
'succeeded',
|
|
$atomicResult
|
|
);
|
|
CloudPublicNetworkTransitionWorkflow::recordEvidence(
|
|
$atomicJob,
|
|
'succeeded',
|
|
$atomicResult,
|
|
$atomicPayload,
|
|
$atomicOperation
|
|
);
|
|
$atomicTarget6 = (int) Capsule::table(
|
|
'cloud_public_network_transition_ips'
|
|
)->where('transition_id', $atomic->transitionId())
|
|
->where('address_family', 6)->value('target_claim_id');
|
|
Capsule::connection()->statement(
|
|
'CREATE TRIGGER fail_atomic_target_commit '
|
|
. 'BEFORE UPDATE OF state ON cloud_resource_reservation_ips '
|
|
. 'WHEN NEW.id = ' . $atomicTarget6
|
|
. " AND NEW.state = 'committed' "
|
|
. "BEGIN SELECT RAISE(ABORT, 'fault injection'); END"
|
|
);
|
|
$atomicRejected = false;
|
|
try {
|
|
$finishLocked($atomicJob, $atomicPayload, 'succeeded', 99, false);
|
|
} catch (Throwable) {
|
|
$atomicRejected = true;
|
|
}
|
|
$assert($atomicRejected
|
|
&& Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', 105)->where('active_claim', 1)
|
|
->where('state', 'committed')->count() === 2
|
|
&& Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', 105)->whereNull('active_claim')
|
|
->where('state', 'reserved')->count() === 2
|
|
&& Capsule::table('cloud_public_network_transitions')
|
|
->where('id', $atomic->transitionId())->value('state')
|
|
=== 'finalization_pending',
|
|
'A mid-finalizer database failure partially switched the dual-stack ledger.');
|
|
Capsule::connection()->statement('DROP TRIGGER fail_atomic_target_commit');
|
|
$finishLocked($atomicJob, $atomicPayload, 'succeeded', 99, false);
|
|
$assert(Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', 105)->where('active_claim', 1)
|
|
->where('state', 'committed')->count() === 2
|
|
&& Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', 105)->whereNull('active_claim')
|
|
->where('state', 'released')->count() === 2
|
|
&& Capsule::table('cloud_public_network_transitions')
|
|
->where('id', $atomic->transitionId())->value('state') === 'succeeded',
|
|
'Dual-stack retry did not atomically publish the persisted Agent effect.');
|
|
Capsule::table('cloud_public_ips')->where('id', 504)
|
|
->update(['host_id' => 2]);
|
|
$terminalLedgerRejected = false;
|
|
try {
|
|
$finishLocked($atomicJob, $atomicPayload, 'succeeded', 99, false);
|
|
} catch (Throwable) {
|
|
$terminalLedgerRejected = true;
|
|
}
|
|
$assert($terminalLedgerRejected,
|
|
'Terminal replay accepted a target IP moved to another host.');
|
|
Capsule::table('cloud_public_ips')->where('id', 504)
|
|
->update(['host_id' => 1]);
|
|
|
|
// A finalizer cannot silently skip the global host/agent sentinel. Missing
|
|
// topology must fail before transition, claim or address projection changes.
|
|
$hostFenceDispatch = CloudPublicNetworkTransitionWorkflow::begin(
|
|
CloudPublicNetworkTransitionCommand::adminExact(
|
|
12,
|
|
99,
|
|
'admin-network-host-fence-0012',
|
|
[4 => [
|
|
'pool_key' => str_repeat('c', 32),
|
|
'address_cidr' => '8.8.9.23/24',
|
|
]]
|
|
)
|
|
);
|
|
[$hostFenceJob, $hostFencePayload] = $dispatchJob(
|
|
$hostFenceDispatch->jobId()
|
|
);
|
|
$hostFenceOperation = $agentOperation(
|
|
$hostFenceJob,
|
|
$hostFencePayload,
|
|
'failed',
|
|
[]
|
|
);
|
|
$agentRow = Capsule::table('cloud_host_agents')->where('host_id', 1)->first();
|
|
Capsule::table('cloud_host_agents')->where('host_id', 1)->delete();
|
|
$hostFenceRejected = false;
|
|
try {
|
|
CloudPublicNetworkTransitionWorkflow::recordEvidence(
|
|
$hostFenceJob,
|
|
'failed',
|
|
[],
|
|
$hostFencePayload,
|
|
$hostFenceOperation
|
|
);
|
|
} catch (Throwable) {
|
|
$hostFenceRejected = true;
|
|
}
|
|
Capsule::table('cloud_host_agents')->insert((array) $agentRow);
|
|
$hostFenceTargetClaim = Capsule::table(
|
|
'cloud_public_network_transition_ips'
|
|
)->where('transition_id', $hostFenceDispatch->transitionId())
|
|
->where('address_family', 4)->value('target_claim_id');
|
|
$assert($hostFenceRejected
|
|
&& Capsule::table('cloud_public_network_transitions')
|
|
->where('id', $hostFenceDispatch->transitionId())
|
|
->value('state') === 'awaiting_receipt'
|
|
&& Capsule::table('cloud_resource_reservation_ips')
|
|
->where('id', $hostFenceTargetClaim)->value('state') === 'reserved'
|
|
&& Capsule::table('cloud_public_ips')->where('id', 204)
|
|
->value('state') === 'reserved',
|
|
'Missing host-agent sentinel allowed partial finalization writes.');
|
|
|
|
// Frozen operation and claim-set identities are fail-closed contracts. A
|
|
// tampered durable transition must be rejected before any resource projection
|
|
// changes, regardless of which identity field drifted.
|
|
$frozenTransition = Capsule::table('cloud_public_network_transitions')
|
|
->where('id', $hostFenceDispatch->transitionId())->first();
|
|
$hostFenceTargetIpId = Capsule::table('cloud_resource_reservation_ips')
|
|
->where('id', $hostFenceTargetClaim)->value('public_ip_id');
|
|
$tamperProjection = static function () use (
|
|
$hostFenceDispatch,
|
|
$hostFenceTargetClaim,
|
|
$hostFenceTargetIpId
|
|
): array {
|
|
$transition = Capsule::table('cloud_public_network_transitions')
|
|
->where('id', $hostFenceDispatch->transitionId())->first();
|
|
$claim = Capsule::table('cloud_resource_reservation_ips')
|
|
->where('id', $hostFenceTargetClaim)->first();
|
|
$ip = Capsule::table('cloud_public_ips')
|
|
->where('id', $hostFenceTargetIpId)->first();
|
|
return [
|
|
'transition' => [
|
|
$transition->state,
|
|
$transition->effect_resolution,
|
|
$transition->effect_resolved_at,
|
|
$transition->active_transition,
|
|
$transition->terminal_at,
|
|
$transition->last_error_code,
|
|
],
|
|
'claim' => [
|
|
$claim->state,
|
|
$claim->active_claim,
|
|
$claim->released_at,
|
|
$claim->release_reason,
|
|
],
|
|
'ip' => [
|
|
$ip->state,
|
|
$ip->reservation_id,
|
|
$ip->reservation_ip_claim_id,
|
|
$ip->instance_id,
|
|
$ip->operation_id,
|
|
],
|
|
];
|
|
};
|
|
foreach ([
|
|
'operation_id' => str_repeat('e', 32),
|
|
'target_claim_set_sha256' => str_repeat('0', 64),
|
|
] as $field => $tamperedValue) {
|
|
Capsule::table('cloud_public_network_transitions')
|
|
->where('id', $hostFenceDispatch->transitionId())
|
|
->update([$field => $tamperedValue]);
|
|
$beforeTamperAttempt = $tamperProjection();
|
|
$tamperRejected = false;
|
|
try {
|
|
CloudPublicNetworkTransitionWorkflow::recordEvidence(
|
|
$hostFenceJob,
|
|
'failed',
|
|
[],
|
|
$hostFencePayload,
|
|
$hostFenceOperation
|
|
);
|
|
} catch (Throwable) {
|
|
$tamperRejected = true;
|
|
}
|
|
$assert(
|
|
$tamperRejected && $tamperProjection() === $beforeTamperAttempt,
|
|
'Tampered transition identity changed the durable resource projection: '
|
|
. $field
|
|
);
|
|
Capsule::table('cloud_public_network_transitions')
|
|
->where('id', $hostFenceDispatch->transitionId())
|
|
->update([$field => $frozenTransition->{$field}]);
|
|
}
|
|
|
|
echo "Public-network transition finalization tests passed.\n";
|