You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudPublicNetworkTransitio...

751 lines
28 KiB
PHP

<?php
require __DIR__ . '/CloudPublicNetworkTransitionWorkflowTest.php';
use App\Models\CloudInstance;
use App\Models\CloudJob;
use App\Models\User;
use App\Services\Cloud\CloudAdminOperationRecoveryService;
use App\Services\Cloud\CloudOperationRecoveryIdentity;
use App\Services\Cloud\CloudNetworkChangeOptionsService;
use App\Services\Cloud\CloudPublicNetworkAllocationService;
use App\Services\Cloud\CloudPublicNetworkTransitionCommand;
use App\Services\Cloud\CloudPublicNetworkTransitionWorkflow;
use App\Services\Cloud\CloudSecrets;
use Illuminate\Database\Capsule\Manager as Capsule;
$dispatchJob = static function (int $jobId): array {
$now = time();
$token = bin2hex(random_bytes(16));
Capsule::table('cloud_jobs')->where('id', $jobId)->update([
'phase' => 'dispatching', 'status' => 'running',
'attempts' => 1, 'dispatch_attempts' => 1,
'dispatch_started_at' => $now, 'dispatched_at' => $now,
'receipt_deadline_at' => $now + 300,
'lease_token' => $token, 'lease_until' => $now + 300,
'updated_at' => $now,
]);
$job = CloudJob::find($jobId);
$payload = CloudSecrets::decrypt(
(string) $job->sealed_payload,
'job-payload'
);
CloudPublicNetworkTransitionWorkflow::markDispatched($job);
return [$job, $payload];
};
$agentOperation = static function (CloudJob $job, array $payload,
string $status, array $result): array {
$identity = CloudOperationRecoveryIdentity::expectedFromPayload(
$job,
$payload,
$status === 'succeeded'
);
return [
'operation_id' => (string) $job->operation_id,
'action' => $identity['action'],
'resource_id' => $identity['resource_id'],
'request_fingerprint' => $identity['request_fingerprint'],
'status' => $status,
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'result' => $result,
];
};
$finishLocked = static function (CloudJob $source, array $payload,
string $status, int $userId, bool $defer): void {
Capsule::connection()->transaction(static function () use (
$source, $payload, $status, $userId, $defer
): void {
$user = User::where('id', $userId)->lockForUpdate()->first();
$job = CloudJob::where('id', (int) $source->id)
->lockForUpdate()->first();
CloudPublicNetworkTransitionWorkflow::finishJobLocked(
$job,
$status,
$payload,
$user,
$defer
);
});
};
// Recovery accepts only the transition-owned typed public-network projection.
// Retired scalar tasks have no recovery authority after the clean break.
$projectionMethod = new ReflectionMethod(
CloudAdminOperationRecoveryService::class,
'effectProjection'
);
$assert($projectionMethod->getNumberOfParameters() === 3,
'Recovery projection retained the retired transition compatibility flag.');
$legacyProjection = [
'uuid' => (string) CloudInstance::find(1)->uuid,
'network_changed' => true,
];
$legacyProjectionRejected = false;
try {
$projectionMethod->invoke(
null,
$legacyProjection,
'network_change',
'applied'
);
} catch (Throwable) {
$legacyProjectionRejected = true;
}
$assert($legacyProjectionRejected,
'Recovery accepted a retired scalar network-change projection.');
$typedProjection = $legacyProjection + [
'public_network' => $dispatch->target()->toArray(),
];
$assert($projectionMethod->invoke(
null,
$typedProjection,
'network_change',
'applied'
) === $typedProjection,
'Transition recovery rejected its canonical public_network projection.');
// Applied single-family transition preserves the unchanged IPv6 leg.
[$appliedJob, $appliedPayload] = $dispatchJob($dispatch->jobId());
$appliedResult = [
'uuid' => (string) CloudInstance::find(1)->uuid,
'network_changed' => true,
'public_network' => $dispatch->target()->toArray(),
];
$appliedOperation = $agentOperation(
$appliedJob, $appliedPayload, 'succeeded', $appliedResult
);
CloudPublicNetworkTransitionWorkflow::recordEvidence(
$appliedJob, 'succeeded', $appliedResult, $appliedPayload,
$appliedOperation
);
$assert(Capsule::table('cloud_public_network_transitions')
->where('id', $dispatch->transitionId())
->value('state') === 'finalization_pending',
'Applied evidence did not survive independently of finalization.');
$finishLocked($appliedJob, $appliedPayload, 'succeeded', 99, false);
$source4 = Capsule::table('cloud_resource_reservation_ips')
->where('id', 1001)->first();
$unchanged6 = Capsule::table('cloud_resource_reservation_ips')
->where('id', 1002)->first();
$target4 = Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', 101)->where('address_family', 4)
->where('active_claim', 1)->first();
$assert($source4->state === 'released'
&& $source4->release_reason === 'public_network_transition_succeeded'
&& (int) $unchanged6->active_claim === 1
&& $unchanged6->state === 'committed'
&& (int) $target4->public_ip_id === 201
&& Capsule::table('cloud_public_ips')->where('id', 101)
->value('state') === 'available'
&& Capsule::table('cloud_public_ips')->where('id', 201)
->value('state') === 'assigned',
'Applied transition did not atomically switch only the changed leg.');
// Once the transition is terminal, a later maintenance migration may change
// current placement. Exact replay still returns the frozen historical result
// while an active transition (covered by begin tests) rejects the same drift.
$terminalPayload = $appliedPayload;
$terminalPayload[CloudOperationRecoveryIdentity::payloadKey()] =
CloudOperationRecoveryIdentity::persisted([
'action' => $appliedOperation['action'],
'resource_id' => $appliedOperation['resource_id'],
'request_fingerprint' => $appliedOperation['request_fingerprint'],
]);
Capsule::table('cloud_jobs')->where('id', (int) $appliedJob->id)->update([
'phase' => 'terminal',
'outcome' => 'succeeded',
'status' => 'succeeded',
'terminal_at' => time(),
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'sealed_payload' => CloudSecrets::encrypt(
$terminalPayload,
'job-payload'
),
]);
Capsule::table('cloud_instances')->where('id', 1)
->update(['host_id' => 2]);
$terminalMigratedReplay = CloudPublicNetworkTransitionWorkflow::begin($admin);
$assert($terminalMigratedReplay->replayed()
&& $terminalMigratedReplay->transitionId() === $dispatch->transitionId(),
'Terminal exact replay was coupled to later instance placement.');
Capsule::table('cloud_instances')->where('id', 1)
->update(['host_id' => 1]);
// A successful transition must make the active claim pair authoritative even
// when intentionally stale singular reservation mirrors remain present.
$reservationRow = Capsule::table('cloud_resource_reservations')
->where('id', 101)->first();
$postTransitionOptions = CloudNetworkChangeOptionsService::adminOptions(
CloudInstance::find(1)
);
$assert((int) $reservationRow->network_attachment_id === 999
&& (int) $reservationRow->public_ip_pool_id === 999
&& (int) $reservationRow->public_ip_id === 999
&& ($postTransitionOptions['current_attachment_id'] ?? null) === '1'
&& ($postTransitionOptions['current_public_network']['ipv4']
['address_cidr'] ?? null) === '8.8.9.20/24'
&& ($postTransitionOptions['current_public_network']['ipv6']
['address_cidr'] ?? null) === '2404:c140:156:3a::1000/64',
'Post-transition options fell back to stale singular reservation data.');
CloudPublicNetworkAllocationService::releaseCommittedBundle(
101,
1,
(string) $reservationOperation,
'instance_deleted',
true
);
$assert(Capsule::table('cloud_resource_reservation_ips')
->whereIn('id', [(int) $target4->id, 1002])
->where('state', 'released')->count() === 2
&& Capsule::table('cloud_public_ips')->whereIn('id', [201, 102])
->where('state', 'available')->count() === 2
&& Capsule::table('cloud_public_ips')->where('instance_id', 1)
->where('state', 'assigned')->count() === 0,
'Committed release used stale scalar mirrors instead of active claims.');
// Recovery prepare may release its target projection, but wallet settlement
// and transition publication wait for the later recovery commit.
[$failedJob, $failedPayload] = $dispatchJob($paidDispatch->jobId());
$failedOperation = $agentOperation(
$failedJob, $failedPayload, 'failed', []
);
CloudPublicNetworkTransitionWorkflow::recordEvidence(
$failedJob, 'failed', [], $failedPayload, $failedOperation, true
);
$finishLocked($failedJob, $failedPayload, 'failed', 10, true);
$paidTransitionId = $paidDispatch->transitionId();
$paidInvoiceId = Capsule::table('cloud_public_network_transitions')
->where('id', $paidTransitionId)->value('invoice_id');
$assert(Capsule::table('cloud_public_network_transitions')
->where('id', $paidTransitionId)->value('state')
=== 'finalization_pending'
&& (int) Capsule::table('cloud_public_network_transitions')
->where('id', $paidTransitionId)->value('active_transition') === 1
&& Capsule::table('cloud_invoices')->where('id', $paidInvoiceId)
->value('status') === 'paid'
&& (string) Capsule::table('user')->where('id', 10)
->value('money') === '15.00',
'Recovery prepare refunded or published before fence commit.');
Capsule::connection()->transaction(static function () use ($failedJob): void {
$user = User::where('id', 10)->lockForUpdate()->first();
$job = CloudJob::where('id', (int) $failedJob->id)
->lockForUpdate()->first();
CloudPublicNetworkTransitionWorkflow::publishPreparedLocked(
$job,
'not_applied',
$user
);
});
$assert(Capsule::table('cloud_invoices')->where('id', $paidInvoiceId)
->value('status') === 'refunded'
&& (string) Capsule::table('user')->where('id', 10)
->value('money') === '20.00'
&& Capsule::table('cloud_public_network_transitions')
->where('id', $paidTransitionId)->value('state') === 'failed',
'Recovery commit did not refund exactly once and publish failure.');
// Unknown evidence retains both targets and may later converge the same
// operation from an exact applied result without creating another claim.
[$unknownJob, $unknownPayload] = $dispatchJob($cross->jobId());
$unknownOperation = $agentOperation(
$unknownJob, $unknownPayload, 'failed', []
);
$unknownOperation['effect_may_have_applied'] = true;
CloudPublicNetworkTransitionWorkflow::recordEvidence(
$unknownJob, 'uncertain', [], $unknownPayload, $unknownOperation
);
$assert(Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', 102)->whereNull('active_claim')
->where('state', 'uncertain')->count() === 2,
'Unknown effect did not retain both target claims as uncertain.');
$lateResult = [
'uuid' => (string) CloudInstance::find(2)->uuid,
'network_changed' => true,
'public_network' => $cross->target()->toArray(),
];
$lateOperation = $agentOperation(
$unknownJob, $unknownPayload, 'succeeded', $lateResult
);
CloudPublicNetworkTransitionWorkflow::recordEvidence(
$unknownJob, 'succeeded', $lateResult, $unknownPayload, $lateOperation
);
$finishLocked($unknownJob, $unknownPayload, 'succeeded', 99, false);
$assert(Capsule::table('cloud_public_network_transitions')
->where('id', $cross->transitionId())->value('state') === 'succeeded',
'Late exact applied evidence did not converge the same transition.');
// A normal receipt can stage evidence, crash, and later enter recovery
// prepare. The pending marker must be promoted so commit can publish.
[$crashJob, $crashPayload] = $dispatchJob($sparse->jobId());
$crashResult = [
'uuid' => (string) CloudInstance::find(8)->uuid,
'network_changed' => true,
'public_network' => $sparse->target()->toArray(),
];
$crashOperation = $agentOperation(
$crashJob, $crashPayload, 'succeeded', $crashResult
);
CloudPublicNetworkTransitionWorkflow::recordEvidence(
$crashJob, 'succeeded', $crashResult, $crashPayload, $crashOperation
);
$now = time();
Capsule::table('cloud_jobs')->where('id', (int) $crashJob->id)->update([
'phase' => 'terminal', 'outcome' => 'failed', 'status' => 'failed',
'terminal_at' => $now, 'effect_may_have_applied' => 1,
'safe_to_resubmit' => 0, 'effect_resolution' => 'unknown',
'error_code' => 'local_finalization_failed',
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_internal_error',
'lease_token' => null, 'lease_until' => 0,
'available_at' => $now, 'updated_at' => $now,
]);
$crashJob = CloudJob::find((int) $crashJob->id);
CloudPublicNetworkTransitionWorkflow::recordEvidence(
$crashJob, 'succeeded', $crashResult, $crashPayload,
$crashOperation, true
);
$assert(Capsule::table('cloud_public_network_transitions')
->where('id', $sparse->transitionId())->value('last_error_code')
=== 'recovery_prepare_pending',
'Late recovery did not promote an already-pending evidence marker.');
$finishLocked($crashJob, $crashPayload, 'succeeded', 99, true);
Capsule::connection()->transaction(static function () use ($crashJob): void {
$user = User::where('id', 99)->lockForUpdate()->first();
$job = CloudJob::where('id', (int) $crashJob->id)
->lockForUpdate()->first();
CloudPublicNetworkTransitionWorkflow::publishPreparedLocked(
$job,
'applied',
$user
);
});
$assert(Capsule::table('cloud_public_network_transitions')
->where('id', $sparse->transitionId())->value('state') === 'succeeded',
'Recovery commit left transition permanently pending.');
// Ordinary not-applied finalization refunds immediately and exact terminal
// replay cannot credit the wallet a second time.
$normalPaid = CloudPublicNetworkTransitionCommand::userPaidAutomatic(
4,
10,
'tenant-network-finalize-0002',
'5.00'
);
$normalPaidDispatch = CloudPublicNetworkTransitionWorkflow::begin($normalPaid);
[$normalPaidJob, $normalPaidPayload] = $dispatchJob(
$normalPaidDispatch->jobId()
);
$normalFailedOperation = $agentOperation(
$normalPaidJob,
$normalPaidPayload,
'failed',
[]
);
CloudPublicNetworkTransitionWorkflow::recordEvidence(
$normalPaidJob,
'failed',
[],
$normalPaidPayload,
$normalFailedOperation
);
$finishLocked($normalPaidJob, $normalPaidPayload, 'failed', 10, false);
$normalPaidTransition = Capsule::table('cloud_public_network_transitions')
->where('id', $normalPaidDispatch->transitionId())->first();
$normalPaidInvoiceId = (int) $normalPaidTransition->invoice_id;
$assert((string) Capsule::table('user')->where('id', 10)->value('money')
=== '20.00'
&& Capsule::table('cloud_invoices')->where('id', $normalPaidInvoiceId)
->value('status') === 'refunded'
&& $normalPaidTransition->state === 'failed',
'Ordinary not-applied finalization did not refund atomically.');
CloudPublicNetworkTransitionWorkflow::recordEvidence(
$normalPaidJob,
'failed',
[],
$normalPaidPayload,
$normalFailedOperation
);
$finishLocked($normalPaidJob, $normalPaidPayload, 'failed', 10, false);
$assert((string) Capsule::table('user')->where('id', 10)->value('money')
=== '20.00',
'Terminal not-applied replay credited the wallet twice.');
Capsule::table('cloud_invoices')->where('id', $normalPaidInvoiceId)
->update(['amount' => '6.00']);
$terminalInvoiceRejected = false;
try {
$finishLocked($normalPaidJob, $normalPaidPayload, 'failed', 10, false);
} catch (Throwable) {
$terminalInvoiceRejected = true;
}
$assert($terminalInvoiceRejected,
'Terminal replay accepted a drifted refunded invoice.');
Capsule::table('cloud_invoices')->where('id', $normalPaidInvoiceId)
->update(['amount' => '5.00']);
// Unknown evidence can also converge to exact not-applied without releasing
// the source or leaking the reserved target.
$lateFailedDispatch = CloudPublicNetworkTransitionWorkflow::begin(
CloudPublicNetworkTransitionCommand::adminExact(
10,
99,
'admin-network-late-failed-0010',
[4 => [
'pool_key' => str_repeat('c', 32),
'address_cidr' => '8.8.9.22/24',
]]
)
);
[$lateFailedJob, $lateFailedPayload] = $dispatchJob(
$lateFailedDispatch->jobId()
);
$lateUnknownOperation = $agentOperation(
$lateFailedJob,
$lateFailedPayload,
'failed',
[]
);
$lateUnknownOperation['effect_may_have_applied'] = true;
CloudPublicNetworkTransitionWorkflow::recordEvidence(
$lateFailedJob,
'uncertain',
[],
$lateFailedPayload,
$lateUnknownOperation
);
$lateFailedOperation = $agentOperation(
$lateFailedJob,
$lateFailedPayload,
'failed',
[]
);
CloudPublicNetworkTransitionWorkflow::recordEvidence(
$lateFailedJob,
'failed',
[],
$lateFailedPayload,
$lateFailedOperation
);
$finishLocked($lateFailedJob, $lateFailedPayload, 'failed', 99, false);
$assert(Capsule::table('cloud_public_network_transitions')
->where('id', $lateFailedDispatch->transitionId())
->value('state') === 'failed'
&& Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', 110)->where('active_claim', 1)
->count() === 2
&& Capsule::table('cloud_public_ips')->where('id', 203)
->value('state') === 'available',
'Late not-applied evidence changed the source or leaked its target.');
// Evidence can survive an ordinary Worker finalizer crash and converge via
// the PHP-only automatic finalizer. The original terminal audit remains a
// failure while only the effect resolution and local ledger are repaired.
$retryDispatch = CloudPublicNetworkTransitionWorkflow::begin(
CloudPublicNetworkTransitionCommand::adminExact(
11,
99,
'admin-network-retry-0004',
[4 => [
'pool_key' => str_repeat('c', 32),
'address_cidr' => '8.8.9.22/24',
]]
)
);
[$retryJob, $retryPayload] = $dispatchJob($retryDispatch->jobId());
$retryResult = [
'uuid' => (string) CloudInstance::find(11)->uuid,
'network_changed' => true,
'public_network' => $retryDispatch->target()->toArray(),
];
$retryOperation = $agentOperation(
$retryJob,
$retryPayload,
'succeeded',
$retryResult
);
CloudPublicNetworkTransitionWorkflow::recordEvidence(
$retryJob,
'succeeded',
$retryResult,
$retryPayload,
$retryOperation
);
$retryLease = bin2hex(random_bytes(16));
$retryTerminalAt = time();
Capsule::table('cloud_jobs')->where('id', (int) $retryJob->id)->update([
'phase' => 'terminal',
'outcome' => 'failed',
'status' => 'failed',
'terminal_at' => $retryTerminalAt,
'effect_may_have_applied' => 1,
'safe_to_resubmit' => 0,
'effect_resolution' => 'unknown',
'effect_resolved_at' => null,
'effect_resolved_by' => null,
'effect_resolution_reason' => '',
'error_code' => 'local_finalization_failed',
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_internal_error',
'lease_token' => $retryLease,
'lease_until' => time() + 300,
'available_at' => time(),
'updated_at' => time(),
]);
$retried = CloudPublicNetworkTransitionWorkflow::retryPendingFinalization(
(int) $retryJob->id,
$retryLease
);
$retriedPayload = CloudSecrets::decrypt(
(string) $retried->sealed_payload,
'job-payload'
);
$retriedIdentity = CloudOperationRecoveryIdentity::expected($retried, true);
$assert((string) $retried->phase === 'terminal'
&& (string) $retried->outcome === 'failed'
&& (string) $retried->status === 'failed'
&& (int) $retried->terminal_at === $retryTerminalAt
&& (string) $retried->error_code === 'local_finalization_failed'
&& (string) $retried->effect_resolution === 'applied'
&& !(bool) $retried->finalization_incident
&& $retried->lease_token === null
&& !array_key_exists('_operation_recovery', $retriedPayload)
&& isset($retriedPayload[
CloudOperationRecoveryIdentity::payloadKey()
])
&& $retriedIdentity === CloudOperationRecoveryIdentity::expectedFromPayload(
$retried,
$retriedPayload,
true
)
&& Capsule::table('cloud_public_network_transitions')
->where('id', $retryDispatch->transitionId())->value('state')
=== 'succeeded',
'PHP-only finalizer changed terminal audit or failed to converge.');
// A database failure after the first dual-stack leg starts switching must
// roll the whole local projection back. Retrying the same persisted evidence
// then succeeds without another Agent dispatch.
$addIp(503, 21, 4, '9.9.9.32');
$addIp(504, 22, 6, '2404:c140:156:3c::1004');
Capsule::table('cloud_host_network_attachments')->where('id', 1)
->update(['state' => 'draining']);
$atomic = CloudPublicNetworkTransitionWorkflow::begin(
CloudPublicNetworkTransitionCommand::adminExact(
5,
99,
'admin-network-atomic-0005',
[
4 => [
'pool_key' => str_repeat('e', 32),
'address_cidr' => '9.9.9.32/24',
],
6 => [
'pool_key' => str_repeat('f', 32),
'address_cidr' => '2404:c140:156:3c::1004/64',
],
]
)
);
Capsule::table('cloud_host_network_attachments')->where('id', 1)
->update(['state' => 'active']);
[$atomicJob, $atomicPayload] = $dispatchJob($atomic->jobId());
$atomicResult = [
'uuid' => (string) CloudInstance::find(5)->uuid,
'network_changed' => true,
'public_network' => $atomic->target()->toArray(),
];
$atomicOperation = $agentOperation(
$atomicJob,
$atomicPayload,
'succeeded',
$atomicResult
);
CloudPublicNetworkTransitionWorkflow::recordEvidence(
$atomicJob,
'succeeded',
$atomicResult,
$atomicPayload,
$atomicOperation
);
$atomicTarget6 = (int) Capsule::table(
'cloud_public_network_transition_ips'
)->where('transition_id', $atomic->transitionId())
->where('address_family', 6)->value('target_claim_id');
Capsule::connection()->statement(
'CREATE TRIGGER fail_atomic_target_commit '
. 'BEFORE UPDATE OF state ON cloud_resource_reservation_ips '
. 'WHEN NEW.id = ' . $atomicTarget6
. " AND NEW.state = 'committed' "
. "BEGIN SELECT RAISE(ABORT, 'fault injection'); END"
);
$atomicRejected = false;
try {
$finishLocked($atomicJob, $atomicPayload, 'succeeded', 99, false);
} catch (Throwable) {
$atomicRejected = true;
}
$assert($atomicRejected
&& Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', 105)->where('active_claim', 1)
->where('state', 'committed')->count() === 2
&& Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', 105)->whereNull('active_claim')
->where('state', 'reserved')->count() === 2
&& Capsule::table('cloud_public_network_transitions')
->where('id', $atomic->transitionId())->value('state')
=== 'finalization_pending',
'A mid-finalizer database failure partially switched the dual-stack ledger.');
Capsule::connection()->statement('DROP TRIGGER fail_atomic_target_commit');
$finishLocked($atomicJob, $atomicPayload, 'succeeded', 99, false);
$assert(Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', 105)->where('active_claim', 1)
->where('state', 'committed')->count() === 2
&& Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', 105)->whereNull('active_claim')
->where('state', 'released')->count() === 2
&& Capsule::table('cloud_public_network_transitions')
->where('id', $atomic->transitionId())->value('state') === 'succeeded',
'Dual-stack retry did not atomically publish the persisted Agent effect.');
Capsule::table('cloud_public_ips')->where('id', 504)
->update(['host_id' => 2]);
$terminalLedgerRejected = false;
try {
$finishLocked($atomicJob, $atomicPayload, 'succeeded', 99, false);
} catch (Throwable) {
$terminalLedgerRejected = true;
}
$assert($terminalLedgerRejected,
'Terminal replay accepted a target IP moved to another host.');
Capsule::table('cloud_public_ips')->where('id', 504)
->update(['host_id' => 1]);
// A finalizer cannot silently skip the global host/agent sentinel. Missing
// topology must fail before transition, claim or address projection changes.
$hostFenceDispatch = CloudPublicNetworkTransitionWorkflow::begin(
CloudPublicNetworkTransitionCommand::adminExact(
12,
99,
'admin-network-host-fence-0012',
[4 => [
'pool_key' => str_repeat('c', 32),
'address_cidr' => '8.8.9.23/24',
]]
)
);
[$hostFenceJob, $hostFencePayload] = $dispatchJob(
$hostFenceDispatch->jobId()
);
$hostFenceOperation = $agentOperation(
$hostFenceJob,
$hostFencePayload,
'failed',
[]
);
$agentRow = Capsule::table('cloud_host_agents')->where('host_id', 1)->first();
Capsule::table('cloud_host_agents')->where('host_id', 1)->delete();
$hostFenceRejected = false;
try {
CloudPublicNetworkTransitionWorkflow::recordEvidence(
$hostFenceJob,
'failed',
[],
$hostFencePayload,
$hostFenceOperation
);
} catch (Throwable) {
$hostFenceRejected = true;
}
Capsule::table('cloud_host_agents')->insert((array) $agentRow);
$hostFenceTargetClaim = Capsule::table(
'cloud_public_network_transition_ips'
)->where('transition_id', $hostFenceDispatch->transitionId())
->where('address_family', 4)->value('target_claim_id');
$assert($hostFenceRejected
&& Capsule::table('cloud_public_network_transitions')
->where('id', $hostFenceDispatch->transitionId())
->value('state') === 'awaiting_receipt'
&& Capsule::table('cloud_resource_reservation_ips')
->where('id', $hostFenceTargetClaim)->value('state') === 'reserved'
&& Capsule::table('cloud_public_ips')->where('id', 204)
->value('state') === 'reserved',
'Missing host-agent sentinel allowed partial finalization writes.');
// Frozen operation and claim-set identities are fail-closed contracts. A
// tampered durable transition must be rejected before any resource projection
// changes, regardless of which identity field drifted.
$frozenTransition = Capsule::table('cloud_public_network_transitions')
->where('id', $hostFenceDispatch->transitionId())->first();
$hostFenceTargetIpId = Capsule::table('cloud_resource_reservation_ips')
->where('id', $hostFenceTargetClaim)->value('public_ip_id');
$tamperProjection = static function () use (
$hostFenceDispatch,
$hostFenceTargetClaim,
$hostFenceTargetIpId
): array {
$transition = Capsule::table('cloud_public_network_transitions')
->where('id', $hostFenceDispatch->transitionId())->first();
$claim = Capsule::table('cloud_resource_reservation_ips')
->where('id', $hostFenceTargetClaim)->first();
$ip = Capsule::table('cloud_public_ips')
->where('id', $hostFenceTargetIpId)->first();
return [
'transition' => [
$transition->state,
$transition->effect_resolution,
$transition->effect_resolved_at,
$transition->active_transition,
$transition->terminal_at,
$transition->last_error_code,
],
'claim' => [
$claim->state,
$claim->active_claim,
$claim->released_at,
$claim->release_reason,
],
'ip' => [
$ip->state,
$ip->reservation_id,
$ip->reservation_ip_claim_id,
$ip->instance_id,
$ip->operation_id,
],
];
};
foreach ([
'operation_id' => str_repeat('e', 32),
'target_claim_set_sha256' => str_repeat('0', 64),
] as $field => $tamperedValue) {
Capsule::table('cloud_public_network_transitions')
->where('id', $hostFenceDispatch->transitionId())
->update([$field => $tamperedValue]);
$beforeTamperAttempt = $tamperProjection();
$tamperRejected = false;
try {
CloudPublicNetworkTransitionWorkflow::recordEvidence(
$hostFenceJob,
'failed',
[],
$hostFencePayload,
$hostFenceOperation
);
} catch (Throwable) {
$tamperRejected = true;
}
$assert(
$tamperRejected && $tamperProjection() === $beforeTamperAttempt,
'Tampered transition identity changed the durable resource projection: '
. $field
);
Capsule::table('cloud_public_network_transitions')
->where('id', $hostFenceDispatch->transitionId())
->update([$field => $frozenTransition->{$field}]);
}
echo "Public-network transition finalization tests passed.\n";