You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudAutomaticOperationReco...

1929 lines
83 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudAgentCompletionEvent;
use App\Models\CloudJob;
use App\Models\CloudTemplate;
use App\Models\CloudUserIso;
/**
* Safely converges one timed-out Agent operation from read-only evidence.
*
* This coordinator never calls the original VM mutation endpoint. Its only
* Agent write is the operation-bound journal fence resolution, after PHP has
* durably prepared the corresponding ledger conclusion.
*/
final class CloudAutomaticOperationRecoveryService
{
// One recovery may spend up to four 30-second Agent calls (inspection,
// fence read, one write, and lost-response verification).
private const CLAIM_SECONDS = 180;
private const RETRY_SECONDS = 300;
private const INSPECTION_MAX_AGE_SECONDS = 120;
private const LANE_RECEIPT = 0;
private const LANE_LEAF = 1;
private const LANE_AGGREGATE = 2;
private const LANE_ADMISSION = 3;
private const LANE_COUNT = 4;
/** Next recovery lane for the long-lived recovery-only worker. */
private int $nextLane = self::LANE_RECEIPT;
/** Claim and inspect at most one unknown-effect or corrupt-ledger task. */
public function tick(): bool
{
// Rotate the three independent recovery lanes. A lane with no ready
// work is skipped in this same tick, so sparse queues retain full
// throughput. Under sustained mixed load every lane receives one
// bounded opportunity per three completed ticks; immutable receipts,
// Agent leaves and aggregate projection can no longer starve one
// another.
$claimed = null;
for ($offset = 0; $offset < self::LANE_COUNT; $offset++) {
$lane = ($this->nextLane + $offset) % self::LANE_COUNT;
if ($lane === self::LANE_RECEIPT) {
$claimed = $this->claim(true);
} elseif ($lane === self::LANE_LEAF) {
// Do not let a due job with a received event consume both the
// receipt lane and the ordinary leaf lane.
$claimed = $this->claim(false, true);
} elseif ($lane === self::LANE_AGGREGATE) {
if (CloudJobService::reconcileOneLegacyAggregateEffect()) {
$this->nextLane = self::LANE_ADMISSION;
return true;
}
} elseif (CloudPortableRestoreAdmissionRecoveryService::
recoverOne()) {
$this->nextLane = self::LANE_RECEIPT;
return true;
}
if ($claimed !== null) {
$this->nextLane = ($lane + 1) % self::LANE_COUNT;
break;
}
}
if ($claimed === null) {
return false;
}
$job = $claimed['job'];
if (CloudDispatchLedger::isInvalid($job)) {
// A corrupt generation counter is neither proof that the request
// stayed local nor a valid dispatched generation. Keep the job in
// the recovery queue, retain any immutable receipt, and wait for
// an operator to repair the counter to exactly one.
CloudOperationCompletionService::deferDispatchLedgerConflict(
$job
);
return true;
}
if ($claimed['mode'] === 'dispatch_repair') {
$this->resumeDispatchLedgerRepair($job);
return true;
}
if ($claimed['mode'] === 'public_network_finalizer') {
try {
CloudPublicNetworkTransitionWorkflow::
retryPendingFinalization(
(int) $job->id,
(string) $job->lease_token
);
} catch (\Throwable $error) {
$this->releaseLocalClaim(
$job,
self::recoveryErrorCode($error)
);
}
return true;
}
if ($claimed['mode'] === 'portable_restore_finalizer') {
try {
CloudPortableRestoreFinalizer::retryPendingFinalization(
(int) $job->id,
(string) $job->lease_token
);
} catch (\Throwable $error) {
$this->releaseLocalClaim(
$job,
self::recoveryErrorCode($error)
);
}
return true;
}
if (in_array($claimed['mode'], [
'invalid_envelope', 'invalid_state', 'resolved_incident',
'aggregate_state', 'invalid_identity',
'dispatch_repair_invalid_envelope',
'dispatch_repair_invalid_identity',
], true)) {
$incidentCode = strtolower(trim(
(string) ($job->finalization_error_code ?? '')
));
$releaseCode = 'recovery_instance_state_conflict';
if ($claimed['mode'] === 'invalid_identity') {
$releaseCode = 'recovery_operation_identity_conflict';
} elseif ($claimed['mode']
=== 'dispatch_repair_invalid_identity') {
$releaseCode =
'recovery_dispatch_repair_operation_identity_conflict';
} elseif ($claimed['mode']
=== 'dispatch_repair_invalid_envelope') {
$releaseCode =
'recovery_dispatch_repair_instance_state_conflict';
} elseif ($claimed['mode'] === 'resolved_incident'
&& preg_match(
'/^recovery_[a-z0-9_]{1,55}$/D',
$incidentCode
) === 1) {
$releaseCode = $incidentCode;
} elseif (in_array($claimed['mode'], [
'aggregate_state', 'invalid_state',
], true) && ((string) $job->execution_mode === 'aggregate'
|| CloudJobService::isAggregateKind((string) $job->kind))) {
$releaseCode = 'recovery_aggregate_state_conflict';
}
$this->releaseLocalClaim($job, $releaseCode);
return true;
}
if ($claimed['mode'] === 'local_finalizer') {
try {
(new CloudWorkerService())->retryUndispatchedFinalization(
(int) $job->id,
(string) $job->lease_token
);
} catch (\Throwable $error) {
$this->releaseLocalClaim(
$job,
self::recoveryErrorCode($error)
);
}
return true;
}
if ($claimed['event'] !== null) {
try {
$this->settleReceivedEvent($job, $claimed['event']);
} catch (\Throwable $error) {
$this->releaseReceivedEventClaim(
$job,
self::receiptRetryDelay($claimed['event']),
self::recoveryErrorCode($error)
);
error_log(sprintf(
'cloud received completion recovery pending job_id=%d kind=%s error=%s',
(int) $job->id,
(string) $job->kind,
get_class($error)
));
}
return true;
}
try {
$pending = $this->recover($job);
if ($pending !== null) {
$this->releaseForRetry(
$job,
$pending['delay'],
$pending['error_code']
);
}
} catch (\Throwable $error) {
// Agent downtime, stale evidence, malformed host state and a PHP
// crash-window retry all remain fenced. A later worker performs a
// new read-only inspection; the original mutation is never sent.
$this->releaseForRetry(
$job,
self::RETRY_SECONDS,
self::recoveryErrorCode($error)
);
error_log(sprintf(
'cloud automatic operation recovery pending job_id=%d kind=%s error=%s',
(int) $job->id,
(string) $job->kind,
get_class($error)
));
}
return true;
}
private function settleReceivedEvent(
CloudJob $job,
CloudAgentCompletionEvent $event
): void {
if (!CloudOperationCompletionService::isExactStoredEvent($event)) {
CloudOperationCompletionService::mark($event, 'rejected');
$this->releaseReceivedEventClaim(
$job,
self::RETRY_SECONDS,
'recovery_receipt_invalid'
);
return;
}
$expectedIdentity = CloudOperationRecoveryIdentity::expected(
$job,
(string) $event->outcome === 'succeeded'
);
if (!CloudOperationRecoveryIdentity::same(
$expectedIdentity,
CloudOperationCompletionService::identity($event)
)) {
CloudOperationCompletionService::isolateIdentityMismatch(
$job,
$event
);
return;
}
$disposition = CloudOperationRecoveryService::settleLateReceipt(
$job,
$event
);
if ($disposition === 'applied') {
return;
}
$this->releaseReceivedEventClaim(
$job,
$disposition === 'received'
? self::receiptRetryDelay($event) : 0
);
}
private static function receiptRetryDelay(
CloudAgentCompletionEvent $event
): int {
return self::RETRY_SECONDS;
}
private function releaseReceivedEventClaim(
CloudJob $claimed,
int $delay,
?string $errorCode = null
): void {
$now = time();
CloudJob::where('id', (int) $claimed->id)
->where('effect_resolution', 'unknown')
->where('lease_token', (string) $claimed->lease_token)
->update(array_filter([
'lease_token' => null,
// Receipt evidence bypasses an older inspection backoff, so
// use the lease clock as the retry gate if local settlement
// itself fails. This prevents a received event hot loop.
'lease_until' => $delay > 0 ? $now + $delay : 0,
'available_at' => $delay > 0 ? $now + $delay : 0,
'finalization_incident' => $errorCode === null ? null : 1,
'finalization_error_code' => $errorCode,
'read_at' => $errorCode === null ? null : null,
'updated_at' => $now,
], static fn ($value, $key): bool => $errorCode !== null
|| !in_array($key, [
'finalization_incident',
'finalization_error_code',
'read_at',
], true), ARRAY_FILTER_USE_BOTH));
}
/**
* Select one fair recovery candidate. A received completion event makes
* an otherwise exhausted job eligible, but is loaded only by its indexed
* host+operation key; the event table is never scanned by disposition.
*
* @return array{job:CloudJob,event:?CloudAgentCompletionEvent,mode:string}|null
*/
private function claim(
bool $receivedOnly,
bool $excludeReceived = false
): ?array
{
return (new CloudJob())->getConnection()->transaction(
static function () use (
$receivedOnly,
$excludeReceived
): ?array {
$now = time();
$job = null;
$mode = 'agent';
if (!$receivedOnly) {
// Recover historical fail-open rows regardless of their
// apparent terminal outcome. A corrupt non-zero dispatch
// counter invalidates both "succeeded" and "not applied"
// projections until an operator repairs it to exactly 1.
// They are claimed only on the ordinary pass so a damaged
// row cannot starve a newly received immutable receipt.
$job = CloudJob::query()
->where('execution_mode', 'agent')
->whereIn('phase', [
'accepted',
'dispatching',
'awaiting_receipt',
'terminal',
])
->where(static function ($invalid): void {
$invalid->whereNull('dispatch_attempts')
->orWhereNotIn(
'dispatch_attempts',
[0, 1]
);
})
->where('available_at', '<=', $now)
->where('lease_until', '<=', $now)
->orderBy('available_at')
->orderBy('id')
->lockForUpdate()
->first();
}
if ($job === null && !$receivedOnly) {
// Agent evidence was committed independently before the
// PHP finalizer crashed. Retry only the transition-owned
// local ledger; never inspect or re-dispatch the mutation.
$job = CloudJob::query()
->where('execution_mode', 'agent')
->where('kind', 'network_change')
->whereNotNull('public_network_transition_id')
->where('phase', 'terminal')
->where('outcome', 'failed')
->where('status', 'failed')
->where('effect_may_have_applied', 1)
->where('safe_to_resubmit', 0)
->where('effect_resolution', 'unknown')
->where('finalization_incident', 1)
->whereExists(static function ($transition): void {
$transition->selectRaw('1')
->from(
'cloud_public_network_transitions as pnt'
)->whereColumn(
'pnt.id',
'cloud_jobs.public_network_transition_id'
)->where('pnt.state', 'finalization_pending')
->whereIn('pnt.effect_resolution', [
'applied', 'not_applied',
])->where(
'pnt.last_error_code',
'<>',
'recovery_prepare_pending'
);
})->where('available_at', '<=', $now)
->where('lease_until', '<=', $now)
->orderBy('available_at')->orderBy('id')
->lockForUpdate()->first();
if ($job !== null) {
$mode = 'public_network_finalizer';
}
}
if ($job === null && !$receivedOnly) {
// Portable restore evidence is committed independently
// before its larger PHP ledger finalizer. Resume only
// that local finalizer; a received immutable completion
// event always remains in the higher-priority receipt
// lane and the restore mutation is never replayed.
$job = CloudJob::query()
->where('execution_mode', 'agent')
->where('kind', 'backup_restore')
->where('phase', 'terminal')
->where('outcome', 'failed')
->where('status', 'failed')
->where('effect_may_have_applied', 1)
->where('safe_to_resubmit', 0)
->where('effect_resolution', 'unknown')
->where('finalization_incident', 1)
->whereExists(static function ($attempt): void {
$attempt->selectRaw('1')
->from(
'cloud_backup_restore_attempts as bra'
)->whereColumn(
'bra.job_id',
'cloud_jobs.id'
)->where(
'bra.state',
'finalization_pending'
)->whereIn(
'bra.effect_resolution',
['applied', 'not_applied']
)->where('bra.active_attempt', 1)
->where(
'bra.last_error_code',
'<>',
'recovery_prepare_pending'
);
})->when(
$excludeReceived,
static function ($candidate): void {
$candidate->whereNotExists(
static function ($events): void {
self::receivedEventExists($events);
}
);
}
)->where('available_at', '<=', $now)
->where('lease_until', '<=', $now)
->orderBy('available_at')->orderBy('id')
->lockForUpdate()->first();
if ($job !== null) {
$mode = 'portable_restore_finalizer';
}
}
if ($job === null && !$receivedOnly) {
// A zero counter or the worker's exact persisted
// dispatch_state=not_dispatched proof establishes that no
// Agent mutation was sent. Retry only PHP's kind-specific
// not-applied ledger finalizer; never inspect or contact
// the host.
$job = CloudJob::query()
->where('execution_mode', 'agent')
->where('phase', 'terminal')
->where('outcome', 'failed')
->where('status', 'failed')
->where(static function ($proof): void {
$proof->where('dispatch_attempts', 0)
->orWhere(static function ($transport): void {
$transport->where('dispatch_attempts', 1)
->where('attempts', 1)
->where('dispatch_started_at', '>', 0)
->where('dispatched_at', '>', 0)
->where('receipt_deadline_at', '>', 0)
->whereNull('agent_accepted_at')
->whereNull('completion_event_id')
->where('effect_may_have_applied', 0)
->where('safe_to_resubmit', 0)
->where(
'effect_resolution',
'not_applied'
)->where(
'effect_resolution_reason',
CloudDispatchLedger::
TRANSPORT_NOT_DISPATCHED_REASON
)->where(
'error_code',
'local_finalization_failed'
);
});
})
->where('finalization_incident', 1)
->where(static function ($incident): void {
$incident->whereNull('finalization_error_code')
->orWhereNotIn(
'finalization_error_code',
CloudDispatchLedger::repairIncidentCodes()
);
})
->where(static function ($finalizer): void {
$finalizer->whereIn(
'kind',
CloudOperationRecoveryPolicy::ledgerFinalizerKinds()
)->orWhere(static function ($create): void {
// Proven local no-effect is authoritative for
// create too. Its financial and placement
// ledgers use the specialised atomic local
// finalizer, never the Agent mutation.
$create->where('kind', 'create')
->whereNotNull('order_id');
});
})->where('available_at', '<=', $now)
->where('lease_until', '<=', $now)
->orderBy('available_at')
->orderBy('id')
->lockForUpdate()
->first();
if ($job !== null) {
$mode = 'local_finalizer';
}
}
if ($job === null) {
// A terminal row whose invalid counter was repaired keeps
// a sticky blocker until its immutable state is checked.
$repair = CloudJob::query()
->where('execution_mode', 'agent')
->where('phase', 'terminal')
->where('finalization_incident', 1)
->whereIn(
'finalization_error_code',
CloudDispatchLedger::repairIncidentCodes()
)->whereIn('dispatch_attempts', [0, 1])
->where('lease_until', '<=', $now);
if ($receivedOnly) {
$repair->whereExists(
static function ($events): void {
self::receivedEventExists($events);
}
);
} else {
$repair->where('available_at', '<=', $now);
}
$job = $repair->orderBy('available_at')
->orderBy('id')->lockForUpdate()->first();
if ($job !== null) {
$mode = 'dispatch_repair';
}
}
if ($job === null && !$receivedOnly) {
// A terminal row may have a final effect and still carry a
// failed PHP projection/finalizer incident. It is already
// resolved on the host, so only retain the local incident
// with backoff; never inspect or mutate Agent state.
$job = CloudJob::query()
->where('execution_mode', 'agent')
->where('phase', 'terminal')
->whereIn('dispatch_attempts', [0, 1])
->whereIn(
'effect_resolution',
['applied', 'not_applied']
)->where('finalization_incident', 1)
->where(static function ($incident): void {
$incident->whereNull('finalization_error_code')
->orWhereNotIn(
'finalization_error_code',
CloudDispatchLedger::repairIncidentCodes()
);
})->whereIn(
'kind',
CloudOperationRecoveryPolicy::ledgerFinalizerKinds()
)->where('available_at', '<=', $now)
->where('lease_until', '<=', $now)
->orderBy('available_at')
->orderBy('id')
->lockForUpdate()
->first();
if ($job !== null) {
$mode = 'resolved_incident';
}
}
if ($job === null && !$receivedOnly) {
// A published effect with a contradictory immutable
// terminal tuple is a PHP-ledger conflict. Keep it in a
// local-only retry loop; never ask the Agent to reinterpret
// or repeat the already concluded mutation.
$job = CloudJob::query()
->where('phase', 'terminal')
->whereIn(
'kind',
CloudOperationRecoveryPolicy::ledgerFinalizerKinds()
)->where(static function ($invalid): void {
CloudJobService::applyInvalidResolvedTerminalTuple(
$invalid
);
})->where('available_at', '<=', $now)
->where('lease_until', '<=', $now)
->orderBy('available_at')
->orderBy('id')
->lockForUpdate()
->first();
if ($job !== null) {
$mode = 'invalid_state';
}
}
if ($job === null && !$receivedOnly) {
// A valid dispatched generation with a damaged immutable
// envelope must not disappear from the recovery queue.
// This mode records the concrete conflict without making
// any Agent request.
$job = CloudJob::query()
->where('execution_mode', 'agent')
->where('phase', 'terminal')
->where('outcome', 'failed')
->where('status', 'failed')
->where('dispatch_attempts', 1)
->where('effect_may_have_applied', 1)
->where('safe_to_resubmit', 0)
->where('effect_resolution', 'unknown')
->whereIn(
'kind',
CloudOperationRecoveryPolicy::ledgerFinalizerKinds()
)->where(static function ($repair): void {
$repair->where('finalization_incident', 0)
->orWhereNull('finalization_error_code')
->orWhereNotIn(
'finalization_error_code',
CloudDispatchLedger::repairIncidentCodes()
);
})->where(static function ($broken): void {
$broken->whereNull('host_id')
->orWhere('host_id', '<=', 0)
->orWhereNull('operation_id')
->orWhereRaw('LENGTH(operation_id) <> 32')
->orWhereNull('dispatched_at')
->orWhere('dispatched_at', '<=', 0);
})->where('available_at', '<=', $now)
->where('lease_until', '<=', $now)
->orderBy('available_at')
->orderBy('id')
->lockForUpdate()
->first();
if ($job !== null) {
$mode = 'invalid_envelope';
}
}
if ($job === null && !$receivedOnly) {
// A terminal unknown-effect row has exactly one trustworthy
// safety tuple. Contradictory historical/data-damaged rows
// stay in a PHP-only incident loop and never contact Agent.
$job = CloudJob::query()
->where('phase', 'terminal')
->where('effect_resolution', 'unknown')
->where(static function ($scope): void {
$scope->where(static function ($leaf): void {
$leaf->whereIn('dispatch_attempts', [0, 1])
->whereIn(
'kind',
CloudOperationRecoveryPolicy::ledgerFinalizerKinds()
);
})->orWhere(
static function ($aggregate): void {
$aggregate->where(
'execution_mode',
'aggregate'
)->whereNull('parent_job_id')
->whereIn(
'kind',
CloudJobService::aggregateKinds()
);
}
);
})->where(static function ($invalid): void {
$invalid->where(static function ($mode): void {
$mode->whereIn(
'kind',
CloudOperationRecoveryPolicy::ledgerFinalizerKinds()
)->where(static function ($broken): void {
$broken->whereNull('execution_mode')
->orWhere(
'execution_mode',
'<>',
'agent'
);
});
})->orWhere(static function ($tuple): void {
self::applyInvalidTerminalTuple($tuple);
});
})->where('available_at', '<=', $now)
->where('lease_until', '<=', $now)
->orderBy('available_at')
->orderBy('id')
->lockForUpdate()
->first();
if ($job !== null) {
$mode = 'invalid_state';
}
}
if ($job === null) {
$job = CloudJob::query()
->where('execution_mode', 'agent')
->where('phase', 'terminal')
->where('outcome', 'failed')
->where('status', 'failed')
->where('dispatch_attempts', 1)
->where('effect_may_have_applied', 1)
->where('safe_to_resubmit', 0)
->where('effect_resolution', 'unknown')
->where(static function ($repair): void {
$repair->where('finalization_incident', 0)
->orWhereNull('finalization_error_code')
->orWhereNotIn(
'finalization_error_code',
CloudDispatchLedger::repairIncidentCodes()
);
})
->whereNotNull('operation_id')
->whereNotNull('host_id')
->whereNotNull('dispatched_at')
->where(static function ($eligible) use (
$now,
$receivedOnly,
$excludeReceived
): void {
$eligible->where(
static function ($policy) use (
$now,
$receivedOnly,
$excludeReceived
): void {
$policy->whereIn(
'kind',
// Create uses a separate financial
// finalizer, never the generic replay or
// effect-projection path below.
CloudOperationRecoveryPolicy::automaticKinds()
)->where(
static function ($ready) use (
$now,
$receivedOnly,
$excludeReceived
): void {
if ($receivedOnly) {
$ready->whereExists(
static function (
$events
): void {
self::receivedEventExists(
$events
);
}
);
return;
}
$ready->where(
'available_at',
'<=',
$now
);
if (!$excludeReceived) {
$ready->orWhereExists(
static function ($events): void {
self::receivedEventExists(
$events
);
}
);
}
}
);
}
);
})
->when(
$excludeReceived,
static function ($candidate): void {
$candidate->whereNotExists(
static function ($events): void {
self::receivedEventExists($events);
}
);
}
)
->where('lease_until', '<=', $now)
->orderByRaw(
'CASE WHEN EXISTS (SELECT 1 FROM cloud_agent_completion_events AS recovery_priority_events WHERE recovery_priority_events.host_id = cloud_jobs.host_id AND recovery_priority_events.operation_id = cloud_jobs.operation_id AND recovery_priority_events.disposition = ?) THEN 0 ELSE 1 END',
['received']
)
->orderBy('available_at')
->orderBy('id')
->lockForUpdate()
->first();
}
if ($job === null) {
return null;
}
// SQL only finds a bounded candidate. Revalidate the complete
// immutable Agent envelope on the hydrated row before loading
// a receipt or allowing any Agent/fence path. A damaged field
// is a local ledger incident; immutable evidence stays queued.
$envelopeConflict =
CloudJobService::hasAgentRecoveryEnvelopeConflict($job);
if ($envelopeConflict) {
$identityConflict = CloudJobService::
hasAgentRecoveryIdentityConflict($job);
if ($mode === 'dispatch_repair') {
$mode = $identityConflict
? 'dispatch_repair_invalid_identity'
: 'dispatch_repair_invalid_envelope';
} else {
$mode = $identityConflict
? 'invalid_identity' : 'invalid_envelope';
}
}
if (!$envelopeConflict
&& (string) $job->phase === 'terminal'
&& (string) $job->execution_mode === 'aggregate'
&& (CloudTemplate::where(
'active_job_id',
(int) $job->id
)->exists() || CloudUserIso::where(
'active_job_id',
(int) $job->id
)->exists())) {
$job->lease_token = bin2hex(random_bytes(16));
$job->lease_until = $now + self::CLAIM_SECONDS;
$job->save();
return [
'job' => $job,
'event' => null,
'mode' => 'aggregate_state',
];
}
if (CloudDispatchLedger::isInvalid($job)) {
// Do not inspect or consume a receipt while the dispatch
// generation is corrupt. Lease only long enough to hand
// the row to the fail-closed normalizer after commit.
$job->lease_token = bin2hex(random_bytes(16));
$job->lease_until = $now + self::CLAIM_SECONDS;
$job->save();
return [
'job' => $job,
'event' => null,
'mode' => 'invalid',
];
}
if ($mode === 'local_finalizer'
|| $mode === 'public_network_finalizer'
|| $mode === 'portable_restore_finalizer'
|| $mode === 'invalid_envelope'
|| $mode === 'invalid_state'
|| $mode === 'resolved_incident'
|| $mode === 'aggregate_state'
|| $mode === 'invalid_identity'
|| $mode === 'dispatch_repair_invalid_envelope'
|| $mode === 'dispatch_repair_invalid_identity'
|| $mode === 'dispatch_repair') {
$event = null;
if ($mode === 'dispatch_repair'
&& is_string($job->completion_event_id)
&& $job->completion_event_id !== '') {
$event = CloudAgentCompletionEvent::where(
'event_id',
(string) $job->completion_event_id
)->lockForUpdate()->first();
}
$job->lease_token = bin2hex(random_bytes(16));
$job->lease_until = $now + self::CLAIM_SECONDS;
$job->save();
return [
'job' => $job,
'event' => $event,
'mode' => $mode,
];
}
$event = CloudAgentCompletionEvent::where(
'host_id',
(int) $job->host_id
)->where(
'operation_id',
(string) $job->operation_id
)->where('disposition', 'received')
->lockForUpdate()
->first();
if ($event !== null) {
$job->lease_token = bin2hex(random_bytes(16));
$job->lease_until = $now + self::CLAIM_SECONDS;
$job->save();
return [
'job' => $job,
'event' => $event,
'mode' => 'agent',
];
}
$attempt = self::nextAttemptMarker(
(string) $job->effect_resolution_reason,
$now
);
// The dedicated create coordinator owns its encrypted
// prepare/commit binding. Never wrap or replace that marker
// when resuming a fence-write/financial-commit crash window.
if ((string) $job->kind === 'create'
&& preg_match(
'/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D',
(string) $job->effect_resolution_reason
) === 1) {
$attempt = (string) $job->effect_resolution_reason;
}
if ($attempt === false) {
$job->finalization_incident = 1;
$job->finalization_error_code =
'recovery_state_invalid';
$job->read_at = null;
} else {
$job->effect_resolution_reason = $attempt;
if (in_array(
(string) ($job->finalization_error_code ?? ''),
[
'recovery_limit_reached',
'recovery_receipt_limit_reached',
],
true
)) {
$job->finalization_incident = 0;
$job->finalization_error_code = null;
$job->read_at = null;
}
}
$job->lease_token = bin2hex(random_bytes(16));
$job->lease_until = $now + self::CLAIM_SECONDS;
$job->save();
return [
'job' => $job,
'event' => null,
'mode' => 'agent',
];
},
3
);
}
/** Add the negation of the canonical failed/effectful/non-retryable tuple. */
private static function applyInvalidTerminalTuple($invalid): void
{
$invalid->where(static function ($tuple): void {
$tuple->whereNull('outcome')
->orWhere('outcome', '<>', 'failed')
->orWhereNull('status')
->orWhere('status', '<>', 'failed')
->orWhereNull('effect_may_have_applied')
->orWhere('effect_may_have_applied', '<>', 1)
->orWhereNull('safe_to_resubmit')
->orWhere('safe_to_resubmit', '<>', 0);
})->orWhere(static function ($neverDispatched): void {
$neverDispatched->where('dispatch_attempts', 0)
->where('finalization_incident', 0);
});
}
private static function receivedEventExists($events): void
{
$events->selectRaw('1')
->from('cloud_agent_completion_events as recovery_events')
->whereColumn(
'recovery_events.host_id',
'cloud_jobs.host_id'
)
->whereColumn(
'recovery_events.operation_id',
'cloud_jobs.operation_id'
)
->where('recovery_events.disposition', 'received');
}
/**
* @return array{delay:int,error_code:string}|null Retry state, or null
* after a committed resolution.
*/
private function recover(CloudJob $claimed): ?array
{
$job = CloudJob::find((int) $claimed->id);
if ($job === null
|| !is_string($job->lease_token)
|| !hash_equals(
(string) $claimed->lease_token,
(string) $job->lease_token
)) {
throw new CloudError('自动核对租约已失效', 409);
}
if ((string) $job->kind === 'create') {
CloudCreateFailureRecoveryService::recover($job);
return null;
}
$context = self::recoveryContext($job);
// A late immutable completion receipt is a PHP-local two-phase
// recovery. If the accepting request crashed after prepare, rebuild
// the durable event and finish without contacting the Agent.
$lateEventId = self::preparedLateReceiptEventId($job);
if ($lateEventId !== null) {
$event = CloudAgentCompletionEvent::where(
'event_id',
$lateEventId
)->first();
if ($event === null
|| (string) $event->disposition !== 'received') {
throw new CloudError('晚到回执恢复记录不完整', 409);
}
$disposition = CloudOperationRecoveryService::settleLateReceipt(
$job,
$event
);
$resolved = CloudJob::find((int) $job->id);
$appliedEvent = CloudAgentCompletionEvent::where(
'event_id',
$lateEventId
)->first();
if ($disposition !== 'applied'
|| $resolved === null
|| !in_array(
(string) $resolved->effect_resolution,
['applied', 'not_applied'],
true
)
|| $appliedEvent === null
|| (string) $appliedEvent->disposition !== 'applied') {
throw new CloudError('晚到回执尚未完成安全收敛', 409);
}
return null;
}
$prepared = self::preparedAutomaticIntent($job, $context);
if ($prepared !== null) {
$reason = self::recoveryReason(
$job,
$prepared['source'],
$prepared['resolution']
);
if ($prepared['fence_resolution_required']) {
try {
CloudAdminOperationRecoveryService::
preparedFenceResolutionForRecovery(
(string) $job->id,
$prepared['resolution'],
$reason
);
self::commitPrepared($job, $prepared);
return null;
} catch (CloudAgentError $error) {
if ($error->agentHttpStatus !== 404) {
throw $error;
}
// The prior process did not commit a fence resolution.
// Continue with a fresh inspection before any new write.
}
} else {
// Exact Journal success and exact no-effect failure never
// retained an Agent fence. PHP may finish an already-prepared
// intent without inventing a fence read or write.
self::commitPrepared($job, $prepared);
return null;
}
}
$inspection = CloudAdminOperationRecoveryService::inspectForRecovery(
(string) $claimed->id
);
self::assertFreshInspection($job, $inspection);
// A poisoned completion outbox means the Agent cannot prove that its
// immutable result was delivered intact. Even if the journal's local
// projection says applied/not-applied, that projection must remain
// inconclusive until the outbox/receipt record is repaired.
$deliveryIncident = self::completionDeliveryIncident($inspection);
if ($deliveryIncident !== null) {
return self::pending($deliveryIncident);
}
// A resolved Agent fence proves that some coordinator already wrote
// an immutable verdict; it does not prove that PHP prepared the same
// verdict. Only the prepared-intent resume path above may consume it.
if (($inspection['fence_state'] ?? null) === 'resolved') {
return self::pending('recovery_evidence_conflict');
}
$resolution = (string) ($inspection['effect_resolution'] ?? '');
if ($resolution === 'partial'
&& (string) $job->kind === 'instance_migration_transfer'
&& CloudInstanceMigrationService::ensureTargetResidueCleanup(
$job,
$inspection
)) {
return self::pending('recovery_target_cleanup_pending');
}
if (in_array($resolution, ['partial', 'inconclusive'], true)) {
return self::pending(
$resolution === 'partial'
? 'recovery_evidence_partial'
: 'recovery_evidence_inconclusive'
);
}
if (!in_array($resolution, ['applied', 'not_applied'], true)) {
throw new CloudError('宿主机效果核对结论无效', 409);
}
$operation = self::operationFromInspection(
$job,
$context,
$inspection,
$resolution
);
$evidenceSource = self::evidenceSource(
$job,
$context,
$inspection,
$resolution
);
$reason = self::recoveryReason($job, $evidenceSource, $resolution);
$journalStatus = (string) $inspection['journal_status'];
$fenceResolutionRequired =
(bool) $inspection['fence_resolution_required'];
// A system fence verdict is an authorization boundary, not merely a
// validation performed after PHP has changed its ledgers. In
// particular, backup restore may be resolved automatically only as
// not-applied while a fence is active. Reject a host-state inference
// before prepareApplied()/prepareNotApplied() can materialise it.
if ($fenceResolutionRequired
&& !CloudOperationRecoveryPolicy::allowsSystemResolution(
(string) $job->kind,
$resolution
)) {
return self::pending('recovery_system_resolution_not_allowed');
}
if ($resolution === 'applied') {
CloudOperationRecoveryService::prepareApplied(
(int) $job->id,
$operation,
$evidenceSource
);
} else {
CloudOperationRecoveryService::prepareNotApplied(
(int) $job->id,
$operation,
$evidenceSource
);
}
$preparedIntent = [
'resolution' => $resolution,
'journal_status' => $journalStatus,
'fence_resolution_required' => $fenceResolutionRequired,
'source' => $evidenceSource,
'operation' => $operation,
];
if ($fenceResolutionRequired) {
self::resolveFenceAndCommit(
$job,
$preparedIntent,
$reason,
$inspection
);
return null;
} elseif (!(($journalStatus === 'succeeded'
&& $resolution === 'applied')
|| ($journalStatus === 'failed'
&& $resolution === 'not_applied'))) {
throw new CloudError('宿主机操作终态不允许自动收敛', 409);
}
self::commitPrepared($job, $preparedIntent);
return null;
}
/**
* Serialize the only Agent fence write with PHP's effect commit.
*
* The job row remains locked across the bounded Agent metadata request.
* Receipt ingestion locks the same row first, so either the immutable
* receipt wins before this transaction (and no fence write occurs), or
* this prepared recovery commits before the receipt is classified late.
* Unrelated jobs and ordinary workers are not locked.
*
* @param array{resolution:string,journal_status:string,fence_resolution_required:bool,source:string,operation:array} $prepared
*/
private static function resolveFenceAndCommit(
CloudJob $claimed,
array $prepared,
string $reason,
array $inspection
): void {
// Snapshot the immutable parent link before opening the locking
// transaction. Aggregate paths take parent -> catalog -> leaf so the
// nested commitPrepared() transaction can safely reacquire that same
// canonical lock sequence.
$aggregateParentId = (int) (CloudJob::where(
'id',
(int) $claimed->id
)->value('parent_job_id') ?? 0);
(new CloudJob())->getConnection()->transaction(
static function () use (
$claimed,
$prepared,
$reason,
$inspection,
$aggregateParentId
): void {
$aggregateChildren = null;
if ($aggregateParentId > 0) {
$aggregateParent = CloudJob::where(
'id',
$aggregateParentId
)
->lockForUpdate()->first();
if ($aggregateParent === null) {
throw new CloudRecoveryAggregateStateConflict(
'任务的汇总父任务不存在',
409
);
}
if ((string) $aggregateParent->execution_mode
!== 'aggregate'
|| CloudJobService::hasAggregateLocalStateConflict(
$aggregateParent
)) {
throw new CloudRecoveryAggregateStateConflict(
'任务的汇总父任务账本状态异常',
409
);
}
CloudTemplateCatalogService::lockAggregateCatalogLocked(
$aggregateParent
);
CloudUserIsoService::lockAggregateCatalogLocked(
$aggregateParent
);
$aggregateChildren = CloudJob::where(
'parent_job_id',
$aggregateParentId
)->orderBy('id')->lockForUpdate()->get();
$job = $aggregateChildren->first(
static fn (CloudJob $child): bool =>
(int) $child->id === (int) $claimed->id
);
} else {
$job = CloudJob::where('id', (int) $claimed->id)
->lockForUpdate()->first();
}
if ($job === null
|| !hash_equals(
(string) ($job->lease_token ?? ''),
(string) $claimed->lease_token
)
|| (int) $job->lease_until <= time()
|| (string) $job->effect_resolution !== 'unknown'
|| !$prepared['fence_resolution_required']) {
throw new CloudError('自动核对租约已失效', 409);
}
if ((int) ($job->parent_job_id ?? 0)
!== $aggregateParentId) {
throw new CloudRecoveryAggregateStateConflict(
'任务的汇总父级关系在栅栏提交期间发生变化',
409
);
}
if ($aggregateParentId > 0) {
CloudJobService::assertAggregateChildRelationshipLocked(
$aggregateParent,
$job,
$aggregateChildren
);
}
$context = self::recoveryContext($job);
$current = self::preparedAutomaticIntent($job, $context);
if ($current === null
|| $current['resolution'] !== $prepared['resolution']
|| $current['journal_status']
!== $prepared['journal_status']
|| !$current['fence_resolution_required']
|| !hash_equals(
$current['source'],
$prepared['source']
)
|| self::canonicalize($current['operation'])
!== self::canonicalize($prepared['operation'])) {
throw new CloudError('自动核对预提交证据已改变', 409);
}
$pending = CloudAgentCompletionEvent::where(
'host_id',
(int) $job->host_id
)->where(
'operation_id',
(string) $job->operation_id
)->where('disposition', 'received')
->lockForUpdate()->first();
if ($pending !== null) {
throw new CloudError('晚到完成回执正在等待优先收敛', 409);
}
CloudAdminOperationRecoveryService::resolveFenceForRecovery(
(string) $job->id,
$prepared['resolution'],
$reason,
$inspection
);
self::commitPrepared($job, $prepared);
},
3
);
}
private static function assertFreshInspection(
CloudJob $job,
array $inspection
): void {
$checkedAt = $inspection['checked_at'] ?? null;
$now = time();
if (($inspection['operation_id'] ?? null)
!== (string) $job->operation_id
|| !is_int($checkedAt)
|| $checkedAt < $now - self::INSPECTION_MAX_AGE_SECONDS
|| $checkedAt > $now + 5) {
throw new CloudError('宿主机只读核对证据已失效', 409);
}
$expected = CloudOperationRecoveryIdentity::expected(
$job,
($inspection['effect_resolution'] ?? null) === 'applied'
);
CloudOperationRecoveryIdentity::assertMatches($expected, [
'action' => $inspection['action'] ?? null,
'resource_id' => $inspection['resource_id'] ?? null,
'request_fingerprint' =>
$inspection['request_fingerprint'] ?? null,
]);
}
private static function completionDeliveryIncident(
array $inspection
): ?string {
if (($inspection['completion_delivery_state'] ?? null)
!== 'quarantined') {
return null;
}
return match ($inspection['completion_delivery_error_code'] ?? null) {
'completion_outbox_invalid' =>
'recovery_completion_outbox_invalid',
'completion_receipt_missing' =>
'recovery_completion_receipt_missing',
'completion_receipt_invalid' =>
'recovery_completion_receipt_invalid',
'completion_operation_missing' =>
'recovery_completion_operation_missing',
'completion_operation_invalid' =>
'recovery_completion_operation_invalid',
'completion_identity_mismatch' =>
'recovery_completion_identity_mismatch',
default => 'recovery_evidence_invalid',
};
}
private static function operationFromInspection(
CloudJob $job,
array $context,
array $inspection,
string $resolution
): array {
$journalStatus = $inspection['journal_status'] ?? null;
$fenceState = $inspection['fence_state'] ?? null;
$fenceResolutionRequired =
$inspection['fence_resolution_required'] ?? null;
if (!in_array($journalStatus, ['succeeded', 'failed'], true)) {
throw new CloudError('宿主机操作日志终态无效', 409);
}
if (!is_bool($fenceResolutionRequired)
|| !in_array($fenceState,
['not_required', 'active'], true)
|| $fenceResolutionRequired !== ($fenceState === 'active')
|| ($journalStatus === 'succeeded'
&& $fenceState !== 'not_required')
|| ($journalStatus === 'failed' && $resolution === 'applied'
&& $fenceState !== 'active')) {
throw new CloudError('宿主机栅栏证据与操作日志不匹配', 409);
}
if ($resolution === 'not_applied') {
$errorCode = $inspection['error_code'] ?? null;
$createLike = CloudOperationRecoveryPolicy::isCreateLike(
(string) $job->kind
);
$projection = $inspection['effect_projection'] ?? null;
if ($journalStatus !== 'failed'
|| ($inspection['outcome'] ?? null) !== 'failed'
|| ($inspection['summary_code'] ?? null)
!== 'operation_failed'
|| !is_string($errorCode)
|| preg_match('/^[a-z][a-z0-9_]{0,63}$/D',
$errorCode) !== 1
|| ($createLike
? (!is_array($projection)
|| array_is_list($projection)
|| ($projection['creation_absent'] ?? null) !== true
|| ($projection['managed_artifacts_absent'] ?? null)
!== true)
: array_key_exists('effect_projection', $inspection))) {
throw new CloudError('宿主机未生效证据不完整', 409);
}
return [
'operation_id' => (string) $job->operation_id,
'action' => (string) $inspection['action'],
'resource_id' => (string) $inspection['resource_id'],
'request_fingerprint' =>
(string) $inspection['request_fingerprint'],
'status' => 'failed',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'retryable' => false,
'error_code' => $errorCode,
'result' => $createLike ? $projection : [],
];
}
$projection = $inspection['effect_projection'] ?? null;
if (($inspection['outcome'] ?? null) !== 'succeeded'
|| ($inspection['summary_code'] ?? null)
!== 'operation_succeeded'
|| array_key_exists('error_code', $inspection)
|| !is_array($projection) || array_is_list($projection)) {
throw new CloudError('宿主机已生效证据不完整', 409);
}
if ((string) $job->kind === 'instance_settings') {
if (($projection['uuid'] ?? null)
!== (string) $context['instance']->uuid
|| ($projection['settings_changed'] ?? null) !== true) {
throw new CloudError('宿主机实例设置证据不完整', 409);
}
if (array_key_exists('hostname', $context['body'])
&& ($projection['hostname'] ?? null)
!== $context['body']['hostname']) {
throw new CloudError('宿主机主机名证据与请求不匹配', 409);
}
} elseif ((string) $job->kind === 'network_change'
&& $job->public_network_transition_id !== null) {
try {
$requested = CloudPublicNetworkSnapshot::fromArray(
$context['payload']['body']['public_network'] ?? null
);
$confirmed = CloudPublicNetworkSnapshot::fromArray(
$projection['public_network'] ?? null
);
} catch (\Throwable) {
throw new CloudError('宿主机公网网络证据无效', 409);
}
if (!$context['instance'] instanceof \App\Models\CloudInstance
|| ($projection['uuid'] ?? null)
!== (string) $context['instance']->uuid
|| ($projection['network_changed'] ?? null) !== true
|| !hash_equals(
$requested->fingerprint(),
$confirmed->fingerprint()
)) {
throw new CloudError(
'宿主机公网网络证据与冻结目标不匹配',
409
);
}
}
return [
'operation_id' => (string) $job->operation_id,
'action' => (string) $inspection['action'],
'resource_id' => (string) $inspection['resource_id'],
'request_fingerprint' =>
(string) $inspection['request_fingerprint'],
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'retryable' => false,
'error_code' => '',
// CloudAdminOperationRecoveryService already applied the strict
// per-kind projection schema. The existing worker finalizer now
// validates it against PHP's encrypted request and ledgers.
'result' => $projection,
];
}
private static function evidenceSource(
CloudJob $job,
array $context,
array $inspection,
string $resolution
): string {
$projection = $inspection['effect_projection'] ?? [];
$criticalProjection = is_array($projection) ? $projection : [];
$instance = $context['instance'] ?? null;
$evidence = self::canonicalize([
'operation_id' => (string) $job->operation_id,
'kind' => (string) $job->kind,
'instance_uuid' => $instance === null
? null : (string) $instance->uuid,
'journal_status' => $inspection['journal_status'] ?? null,
'fence_resolution_required' =>
$inspection['fence_resolution_required'] ?? null,
'resolution' => $resolution,
'error_code' => $inspection['error_code'] ?? null,
'projection' => $criticalProjection,
]);
$journalStatus = (string) ($inspection['journal_status'] ?? '');
$fence = ($inspection['fence_resolution_required'] ?? false) === true
? 'fence' : 'nofence';
$prefix = $resolution === 'applied'
? 'automatic_applied_' . $journalStatus . '_' . $fence
: 'automatic_not_applied_' . $journalStatus . '_' . $fence;
return $prefix . ':' . hash(
'sha256',
json_encode(
$evidence,
JSON_THROW_ON_ERROR
| JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
)
);
}
/**
* Rebuild the exact stable operation which was durably prepared earlier.
* The prepared source records both resolution and Journal terminal state;
* commitApplied()/commitNotApplied() re-check its operation hash.
*
* @return array{resolution:string,journal_status:string,fence_resolution_required:bool,source:string,operation:array}|null
*/
private static function preparedAutomaticIntent(
CloudJob $job,
array $context
): ?array {
$marker = self::preparedPayload(
(string) $job->effect_resolution_reason
);
if ($marker === null) {
return null;
}
if (preg_match(
'/^prepared:(automatic_(applied|not_applied)_(succeeded|failed)_(fence|nofence):[a-f0-9]{64}):([a-f0-9]{64})$/D',
$marker,
$matches
) !== 1) {
return null;
}
$resolution = $matches[2];
$journalStatus = $matches[3];
$fenceResolutionRequired = $matches[4] === 'fence';
$validShape = ($resolution === 'applied'
&& $journalStatus === 'succeeded'
&& !$fenceResolutionRequired)
|| ($resolution === 'applied'
&& $journalStatus === 'failed'
&& $fenceResolutionRequired)
|| ($resolution === 'not_applied'
&& $journalStatus === 'failed');
if (!$validShape) {
return null;
}
$operation = CloudWorkerService::preparedRecoveryOperation($job);
if ($operation === null && (string) $job->kind !== 'instance_settings') {
throw new CloudRecoveryPayloadUnavailable(
'任务缺少可恢复的预提交结果',
409
);
}
// Compatibility for settings intents prepared before the generic
// encrypted operation envelope was introduced.
if ($operation === null && $resolution === 'not_applied') {
$operation = [
'operation_id' => (string) $job->operation_id,
'status' => 'failed',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => 'settings_not_committed',
'result' => [],
];
} elseif ($operation === null) {
$result = [
'uuid' => (string) $context['instance']->uuid,
'settings_changed' => true,
];
if (array_key_exists('hostname', $context['body'])) {
$spec = json_decode(
(string) $context['instance']->spec_json,
true
);
if (!is_array($spec) || array_is_list($spec)
|| ($spec['guest_hostname'] ?? null)
!== $context['body']['hostname']) {
throw new CloudError(
'已预提交的主机名账本与请求不匹配',
409
);
}
$result['hostname'] = $context['body']['hostname'];
}
$operation = [
'operation_id' => (string) $job->operation_id,
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => $result,
];
}
$operationHash = CloudOperationRecoveryService::
preparedOperationHash(
$operation
);
$validOperation = $resolution === 'applied'
? (($operation['status'] ?? null) === 'succeeded'
&& ($operation['effect_may_have_applied'] ?? null) === false
&& ($operation['safe_to_resubmit'] ?? null) === false)
: (($operation['status'] ?? null) === 'failed'
&& ($operation['effect_may_have_applied'] ?? null) === false
&& is_bool($operation['safe_to_resubmit'] ?? null)
&& is_string($operation['error_code'] ?? null)
&& preg_match('/^[a-z][a-z0-9_]{0,63}$/D',
$operation['error_code']) === 1);
if (!$validOperation || !hash_equals($matches[5], $operationHash)) {
throw new CloudRecoveryPayloadUnavailable(
'任务预提交结果与恢复标记不匹配',
409
);
}
return [
'resolution' => $resolution,
'journal_status' => $journalStatus,
'fence_resolution_required' => $fenceResolutionRequired,
'source' => $matches[1],
'operation' => $operation,
];
}
private static function recoveryReason(
CloudJob $job,
string $source,
string $resolution
): string {
$separator = strrpos($source, ':');
$evidence = $separator === false
? $source : substr($source, $separator + 1);
return sprintf(
'automatic %s recovery job=%d operation=%s evidence=%s resolution=%s',
(string) $job->kind,
(int) $job->id,
(string) $job->operation_id,
$evidence,
$resolution
);
}
/**
* @param array{resolution:string,journal_status:string,fence_resolution_required:bool,source:string,operation:array} $prepared
*/
private static function commitPrepared(
CloudJob $job,
array $prepared
): void {
if ($prepared['resolution'] === 'applied') {
CloudOperationRecoveryService::commitApplied(
(int) $job->id,
$prepared['operation'],
$prepared['source'],
0
);
return;
}
CloudOperationRecoveryService::commitNotApplied(
(int) $job->id,
$prepared['operation'],
$prepared['source'],
0
);
}
private static function canonicalize(mixed $value): mixed
{
if (!is_array($value)) {
return $value;
}
if (!array_is_list($value)) {
ksort($value, SORT_STRING);
}
foreach ($value as $key => $item) {
$value[$key] = self::canonicalize($item);
}
return $value;
}
private static function recoveryContext(CloudJob $job): array
{
return (string) $job->kind === 'instance_settings'
? CloudWorkerService::settingsRecoveryContext($job)
: CloudWorkerService::recoveryContext($job);
}
/** @return array{delay:int,error_code:string} */
private static function pending(string $errorCode): array
{
return [
'delay' => self::RETRY_SECONDS,
'error_code' => $errorCode,
];
}
private static function recoveryErrorCode(\Throwable $error): string
{
if ($error instanceof CloudRecoveryDispatchLedgerConflict) {
return CloudDispatchLedger::CONFLICT_CODE;
}
if ($error instanceof CloudRecoveryAggregateStateConflict) {
return 'recovery_aggregate_state_conflict';
}
if ($error instanceof CloudRecoveryPayloadUnavailable) {
return 'recovery_payload_unavailable';
}
if ($error instanceof CloudRecoveryOperationIdentityConflict) {
return 'recovery_operation_identity_conflict';
}
if ($error instanceof CloudRecoveryEvidenceConflict) {
return 'recovery_evidence_conflict';
}
if ($error instanceof CloudRecoveryReceiptInvalid) {
return 'recovery_receipt_invalid';
}
if ($error instanceof CloudRecoveryInstanceStateConflict) {
return 'recovery_instance_state_conflict';
}
if ($error instanceof CloudAgentError) {
if (in_array($error->agentCode, [
'agent_invalid_inspection',
'agent_invalid_fence_resolution',
'fence_resolution_read_failed',
], true) && $error->agentHttpStatus === 200) {
return 'recovery_evidence_invalid';
}
if ($error->agentHttpStatus === 404) {
return 'recovery_operation_missing';
}
if (in_array($error->agentHttpStatus, [401, 403], true)) {
return 'recovery_authentication_failed';
}
if ($error->agentHttpStatus === 409) {
return 'recovery_operation_conflict';
}
return 'recovery_waiting_connection';
}
if ($error instanceof CloudError) {
return 'recovery_ledger_conflict';
}
return 'recovery_internal_error';
}
private function releaseForRetry(
CloudJob $claimed,
int $delay,
string $errorCode
): void
{
(new CloudJob())->getConnection()->transaction(
static function () use ($claimed, $delay, $errorCode): void {
$job = CloudJob::where('id', (int) $claimed->id)
->where('effect_resolution', 'unknown')
->where(
'lease_token',
(string) $claimed->lease_token
)
->lockForUpdate()
->first();
if ($job === null) {
return;
}
$now = time();
$job->lease_token = null;
$job->lease_until = 0;
$job->available_at = $now + max(
self::RETRY_SECONDS,
$delay
);
if (!hash_equals(
(string) ($job->finalization_error_code ?? ''),
$errorCode
)) {
$job->read_at = null;
}
$job->finalization_incident = 1;
$job->finalization_error_code = $errorCode;
$job->updated_at = $now;
$job->save();
},
3
);
}
private function resumeDispatchLedgerRepair(CloudJob $claimed): void
{
try {
$result = CloudOperationCompletionService::
resumeRepairedDispatchLedger($claimed);
if ($result === 'missing') {
$this->releaseLocalClaim(
$claimed,
'recovery_dispatch_repair_completion_receipt_missing'
);
} elseif ($result === 'invalid') {
$this->releaseLocalClaim(
$claimed,
'recovery_dispatch_repair_receipt_invalid'
);
}
} catch (\Throwable $error) {
$this->releaseLocalClaim(
$claimed,
self::dispatchRepairErrorCode($error)
);
}
}
private static function dispatchRepairErrorCode(
\Throwable $error
): string {
return match (self::recoveryErrorCode($error)) {
CloudDispatchLedger::CONFLICT_CODE =>
CloudDispatchLedger::CONFLICT_CODE,
'recovery_aggregate_state_conflict' =>
'recovery_dispatch_repair_aggregate_state_conflict',
'recovery_payload_unavailable' =>
'recovery_dispatch_repair_payload_unavailable',
'recovery_operation_identity_conflict' =>
'recovery_dispatch_repair_operation_identity_conflict',
'recovery_evidence_conflict' =>
'recovery_dispatch_repair_evidence_conflict',
'recovery_receipt_invalid' =>
'recovery_dispatch_repair_receipt_invalid',
'recovery_instance_state_conflict' =>
'recovery_dispatch_repair_instance_state_conflict',
'recovery_ledger_conflict' =>
'recovery_dispatch_repair_ledger_conflict',
default => 'recovery_dispatch_repair_internal_error',
};
}
/** Release a PHP-only recovery claim without requiring unknown effect. */
private function releaseLocalClaim(
CloudJob $claimed,
string $errorCode
): void {
(new CloudJob())->getConnection()->transaction(
static function () use ($claimed, $errorCode): void {
$job = CloudJob::where('id', (int) $claimed->id)
->where(
'lease_token',
(string) $claimed->lease_token
)->lockForUpdate()->first();
if ($job === null) {
return;
}
$now = time();
$job->lease_token = null;
// claim(true) intentionally prioritizes a matching received
// event over ordinary available_at backoff. Keep the lease
// clock as the retry gate as well, otherwise an invalid bound
// receipt would be reclaimed on every worker tick.
$job->lease_until = $now + self::RETRY_SECONDS;
$job->available_at = $now + self::RETRY_SECONDS;
if (!hash_equals(
(string) ($job->finalization_error_code ?? ''),
$errorCode
)) {
$job->read_at = null;
}
$job->finalization_incident = 1;
$job->finalization_error_code = $errorCode;
$job->updated_at = $now;
$job->save();
},
3
);
}
/** @return array{first_at:int,attempts:int,payload:string}|null */
private static function attemptState(string $marker): ?array
{
if (preg_match(
'/^automatic_recovery:v1:([1-9][0-9]{0,10}):([1-9][0-9]{0,9}):(pending|prepared:(?:automatic_(?:applied|not_applied)_(?:succeeded|failed)_(?:fence|nofence):[a-f0-9]{64}|late_completion_receipt:[A-Za-z0-9_.-]{1,160}):[a-f0-9]{64})$/D',
$marker,
$matches
) !== 1) {
return null;
}
return [
'first_at' => (int) $matches[1],
'attempts' => (int) $matches[2],
'payload' => $matches[3],
];
}
private static function nextAttemptMarker(
string $marker,
int $now
): string|false {
if ($marker === '') {
return sprintf(
'automatic_recovery:v1:%d:1:pending',
$now
);
}
$state = self::attemptState($marker);
if ($state === null) {
if (preg_match(
'/^prepared:(?:automatic_(?:applied|not_applied)_(?:succeeded|failed)_(?:fence|nofence):[a-f0-9]{64}|late_completion_receipt:[A-Za-z0-9_.-]{1,160}):[a-f0-9]{64}$/D',
$marker
) === 1) {
return sprintf(
'automatic_recovery:v1:%d:1:%s',
$now,
$marker
);
}
return false;
}
return sprintf(
'automatic_recovery:v1:%d:%d:%s',
$state['first_at'],
min(9999999999, $state['attempts'] + 1),
$state['payload']
);
}
private static function preparedPayload(string $marker): ?string
{
$state = self::attemptState($marker);
if ($state !== null && str_starts_with(
$state['payload'],
'prepared:'
)) {
return $state['payload'];
}
return preg_match(
'/^prepared:(?:automatic_(?:applied|not_applied)_(?:succeeded|failed)_(?:fence|nofence):[a-f0-9]{64}|late_completion_receipt:[A-Za-z0-9_.-]{1,160}):[a-f0-9]{64}$/D',
$marker
) === 1 ? $marker : null;
}
private static function preparedLateReceiptEventId(
CloudJob $job
): ?string {
$payload = self::preparedPayload(
(string) $job->effect_resolution_reason
);
if (!is_string($payload)
|| preg_match(
'/^prepared:late_completion_receipt:([A-Za-z0-9_.-]{1,160}):[a-f0-9]{64}$/D',
$payload,
$matches
) !== 1) {
return null;
}
return $matches[1];
}
}