nextLane + $offset) % self::LANE_COUNT; if ($lane === self::LANE_RECEIPT) { $claimed = $this->claim(true); } elseif ($lane === self::LANE_LEAF) { // Do not let a due job with a received event consume both the // receipt lane and the ordinary leaf lane. $claimed = $this->claim(false, true); } elseif ($lane === self::LANE_AGGREGATE) { if (CloudJobService::reconcileOneLegacyAggregateEffect()) { $this->nextLane = self::LANE_ADMISSION; return true; } } elseif (CloudPortableRestoreAdmissionRecoveryService:: recoverOne()) { $this->nextLane = self::LANE_RECEIPT; return true; } if ($claimed !== null) { $this->nextLane = ($lane + 1) % self::LANE_COUNT; break; } } if ($claimed === null) { return false; } $job = $claimed['job']; if (CloudDispatchLedger::isInvalid($job)) { // A corrupt generation counter is neither proof that the request // stayed local nor a valid dispatched generation. Keep the job in // the recovery queue, retain any immutable receipt, and wait for // an operator to repair the counter to exactly one. CloudOperationCompletionService::deferDispatchLedgerConflict( $job ); return true; } if ($claimed['mode'] === 'dispatch_repair') { $this->resumeDispatchLedgerRepair($job); return true; } if ($claimed['mode'] === 'public_network_finalizer') { try { CloudPublicNetworkTransitionWorkflow:: retryPendingFinalization( (int) $job->id, (string) $job->lease_token ); } catch (\Throwable $error) { $this->releaseLocalClaim( $job, self::recoveryErrorCode($error) ); } return true; } if ($claimed['mode'] === 'portable_restore_finalizer') { try { CloudPortableRestoreFinalizer::retryPendingFinalization( (int) $job->id, (string) $job->lease_token ); } catch (\Throwable $error) { $this->releaseLocalClaim( $job, self::recoveryErrorCode($error) ); } return true; } if (in_array($claimed['mode'], [ 'invalid_envelope', 'invalid_state', 'resolved_incident', 'aggregate_state', 'invalid_identity', 'dispatch_repair_invalid_envelope', 'dispatch_repair_invalid_identity', ], true)) { $incidentCode = strtolower(trim( (string) ($job->finalization_error_code ?? '') )); $releaseCode = 'recovery_instance_state_conflict'; if ($claimed['mode'] === 'invalid_identity') { $releaseCode = 'recovery_operation_identity_conflict'; } elseif ($claimed['mode'] === 'dispatch_repair_invalid_identity') { $releaseCode = 'recovery_dispatch_repair_operation_identity_conflict'; } elseif ($claimed['mode'] === 'dispatch_repair_invalid_envelope') { $releaseCode = 'recovery_dispatch_repair_instance_state_conflict'; } elseif ($claimed['mode'] === 'resolved_incident' && preg_match( '/^recovery_[a-z0-9_]{1,55}$/D', $incidentCode ) === 1) { $releaseCode = $incidentCode; } elseif (in_array($claimed['mode'], [ 'aggregate_state', 'invalid_state', ], true) && ((string) $job->execution_mode === 'aggregate' || CloudJobService::isAggregateKind((string) $job->kind))) { $releaseCode = 'recovery_aggregate_state_conflict'; } $this->releaseLocalClaim($job, $releaseCode); return true; } if ($claimed['mode'] === 'local_finalizer') { try { (new CloudWorkerService())->retryUndispatchedFinalization( (int) $job->id, (string) $job->lease_token ); } catch (\Throwable $error) { $this->releaseLocalClaim( $job, self::recoveryErrorCode($error) ); } return true; } if ($claimed['event'] !== null) { try { $this->settleReceivedEvent($job, $claimed['event']); } catch (\Throwable $error) { $this->releaseReceivedEventClaim( $job, self::receiptRetryDelay($claimed['event']), self::recoveryErrorCode($error) ); error_log(sprintf( 'cloud received completion recovery pending job_id=%d kind=%s error=%s', (int) $job->id, (string) $job->kind, get_class($error) )); } return true; } try { $pending = $this->recover($job); if ($pending !== null) { $this->releaseForRetry( $job, $pending['delay'], $pending['error_code'] ); } } catch (\Throwable $error) { // Agent downtime, stale evidence, malformed host state and a PHP // crash-window retry all remain fenced. A later worker performs a // new read-only inspection; the original mutation is never sent. $this->releaseForRetry( $job, self::RETRY_SECONDS, self::recoveryErrorCode($error) ); error_log(sprintf( 'cloud automatic operation recovery pending job_id=%d kind=%s error=%s', (int) $job->id, (string) $job->kind, get_class($error) )); } return true; } private function settleReceivedEvent( CloudJob $job, CloudAgentCompletionEvent $event ): void { if (!CloudOperationCompletionService::isExactStoredEvent($event)) { CloudOperationCompletionService::mark($event, 'rejected'); $this->releaseReceivedEventClaim( $job, self::RETRY_SECONDS, 'recovery_receipt_invalid' ); return; } $expectedIdentity = CloudOperationRecoveryIdentity::expected( $job, (string) $event->outcome === 'succeeded' ); if (!CloudOperationRecoveryIdentity::same( $expectedIdentity, CloudOperationCompletionService::identity($event) )) { CloudOperationCompletionService::isolateIdentityMismatch( $job, $event ); return; } $disposition = CloudOperationRecoveryService::settleLateReceipt( $job, $event ); if ($disposition === 'applied') { return; } $this->releaseReceivedEventClaim( $job, $disposition === 'received' ? self::receiptRetryDelay($event) : 0 ); } private static function receiptRetryDelay( CloudAgentCompletionEvent $event ): int { return self::RETRY_SECONDS; } private function releaseReceivedEventClaim( CloudJob $claimed, int $delay, ?string $errorCode = null ): void { $now = time(); CloudJob::where('id', (int) $claimed->id) ->where('effect_resolution', 'unknown') ->where('lease_token', (string) $claimed->lease_token) ->update(array_filter([ 'lease_token' => null, // Receipt evidence bypasses an older inspection backoff, so // use the lease clock as the retry gate if local settlement // itself fails. This prevents a received event hot loop. 'lease_until' => $delay > 0 ? $now + $delay : 0, 'available_at' => $delay > 0 ? $now + $delay : 0, 'finalization_incident' => $errorCode === null ? null : 1, 'finalization_error_code' => $errorCode, 'read_at' => $errorCode === null ? null : null, 'updated_at' => $now, ], static fn ($value, $key): bool => $errorCode !== null || !in_array($key, [ 'finalization_incident', 'finalization_error_code', 'read_at', ], true), ARRAY_FILTER_USE_BOTH)); } /** * Select one fair recovery candidate. A received completion event makes * an otherwise exhausted job eligible, but is loaded only by its indexed * host+operation key; the event table is never scanned by disposition. * * @return array{job:CloudJob,event:?CloudAgentCompletionEvent,mode:string}|null */ private function claim( bool $receivedOnly, bool $excludeReceived = false ): ?array { return (new CloudJob())->getConnection()->transaction( static function () use ( $receivedOnly, $excludeReceived ): ?array { $now = time(); $job = null; $mode = 'agent'; if (!$receivedOnly) { // Recover historical fail-open rows regardless of their // apparent terminal outcome. A corrupt non-zero dispatch // counter invalidates both "succeeded" and "not applied" // projections until an operator repairs it to exactly 1. // They are claimed only on the ordinary pass so a damaged // row cannot starve a newly received immutable receipt. $job = CloudJob::query() ->where('execution_mode', 'agent') ->whereIn('phase', [ 'accepted', 'dispatching', 'awaiting_receipt', 'terminal', ]) ->where(static function ($invalid): void { $invalid->whereNull('dispatch_attempts') ->orWhereNotIn( 'dispatch_attempts', [0, 1] ); }) ->where('available_at', '<=', $now) ->where('lease_until', '<=', $now) ->orderBy('available_at') ->orderBy('id') ->lockForUpdate() ->first(); } if ($job === null && !$receivedOnly) { // Agent evidence was committed independently before the // PHP finalizer crashed. Retry only the transition-owned // local ledger; never inspect or re-dispatch the mutation. $job = CloudJob::query() ->where('execution_mode', 'agent') ->where('kind', 'network_change') ->whereNotNull('public_network_transition_id') ->where('phase', 'terminal') ->where('outcome', 'failed') ->where('status', 'failed') ->where('effect_may_have_applied', 1) ->where('safe_to_resubmit', 0) ->where('effect_resolution', 'unknown') ->where('finalization_incident', 1) ->whereExists(static function ($transition): void { $transition->selectRaw('1') ->from( 'cloud_public_network_transitions as pnt' )->whereColumn( 'pnt.id', 'cloud_jobs.public_network_transition_id' )->where('pnt.state', 'finalization_pending') ->whereIn('pnt.effect_resolution', [ 'applied', 'not_applied', ])->where( 'pnt.last_error_code', '<>', 'recovery_prepare_pending' ); })->where('available_at', '<=', $now) ->where('lease_until', '<=', $now) ->orderBy('available_at')->orderBy('id') ->lockForUpdate()->first(); if ($job !== null) { $mode = 'public_network_finalizer'; } } if ($job === null && !$receivedOnly) { // Portable restore evidence is committed independently // before its larger PHP ledger finalizer. Resume only // that local finalizer; a received immutable completion // event always remains in the higher-priority receipt // lane and the restore mutation is never replayed. $job = CloudJob::query() ->where('execution_mode', 'agent') ->where('kind', 'backup_restore') ->where('phase', 'terminal') ->where('outcome', 'failed') ->where('status', 'failed') ->where('effect_may_have_applied', 1) ->where('safe_to_resubmit', 0) ->where('effect_resolution', 'unknown') ->where('finalization_incident', 1) ->whereExists(static function ($attempt): void { $attempt->selectRaw('1') ->from( 'cloud_backup_restore_attempts as bra' )->whereColumn( 'bra.job_id', 'cloud_jobs.id' )->where( 'bra.state', 'finalization_pending' )->whereIn( 'bra.effect_resolution', ['applied', 'not_applied'] )->where('bra.active_attempt', 1) ->where( 'bra.last_error_code', '<>', 'recovery_prepare_pending' ); })->when( $excludeReceived, static function ($candidate): void { $candidate->whereNotExists( static function ($events): void { self::receivedEventExists($events); } ); } )->where('available_at', '<=', $now) ->where('lease_until', '<=', $now) ->orderBy('available_at')->orderBy('id') ->lockForUpdate()->first(); if ($job !== null) { $mode = 'portable_restore_finalizer'; } } if ($job === null && !$receivedOnly) { // A zero counter or the worker's exact persisted // dispatch_state=not_dispatched proof establishes that no // Agent mutation was sent. Retry only PHP's kind-specific // not-applied ledger finalizer; never inspect or contact // the host. $job = CloudJob::query() ->where('execution_mode', 'agent') ->where('phase', 'terminal') ->where('outcome', 'failed') ->where('status', 'failed') ->where(static function ($proof): void { $proof->where('dispatch_attempts', 0) ->orWhere(static function ($transport): void { $transport->where('dispatch_attempts', 1) ->where('attempts', 1) ->where('dispatch_started_at', '>', 0) ->where('dispatched_at', '>', 0) ->where('receipt_deadline_at', '>', 0) ->whereNull('agent_accepted_at') ->whereNull('completion_event_id') ->where('effect_may_have_applied', 0) ->where('safe_to_resubmit', 0) ->where( 'effect_resolution', 'not_applied' )->where( 'effect_resolution_reason', CloudDispatchLedger:: TRANSPORT_NOT_DISPATCHED_REASON )->where( 'error_code', 'local_finalization_failed' ); }); }) ->where('finalization_incident', 1) ->where(static function ($incident): void { $incident->whereNull('finalization_error_code') ->orWhereNotIn( 'finalization_error_code', CloudDispatchLedger::repairIncidentCodes() ); }) ->where(static function ($finalizer): void { $finalizer->whereIn( 'kind', CloudOperationRecoveryPolicy::ledgerFinalizerKinds() )->orWhere(static function ($create): void { // Proven local no-effect is authoritative for // create too. Its financial and placement // ledgers use the specialised atomic local // finalizer, never the Agent mutation. $create->where('kind', 'create') ->whereNotNull('order_id'); }); })->where('available_at', '<=', $now) ->where('lease_until', '<=', $now) ->orderBy('available_at') ->orderBy('id') ->lockForUpdate() ->first(); if ($job !== null) { $mode = 'local_finalizer'; } } if ($job === null) { // A terminal row whose invalid counter was repaired keeps // a sticky blocker until its immutable state is checked. $repair = CloudJob::query() ->where('execution_mode', 'agent') ->where('phase', 'terminal') ->where('finalization_incident', 1) ->whereIn( 'finalization_error_code', CloudDispatchLedger::repairIncidentCodes() )->whereIn('dispatch_attempts', [0, 1]) ->where('lease_until', '<=', $now); if ($receivedOnly) { $repair->whereExists( static function ($events): void { self::receivedEventExists($events); } ); } else { $repair->where('available_at', '<=', $now); } $job = $repair->orderBy('available_at') ->orderBy('id')->lockForUpdate()->first(); if ($job !== null) { $mode = 'dispatch_repair'; } } if ($job === null && !$receivedOnly) { // A terminal row may have a final effect and still carry a // failed PHP projection/finalizer incident. It is already // resolved on the host, so only retain the local incident // with backoff; never inspect or mutate Agent state. $job = CloudJob::query() ->where('execution_mode', 'agent') ->where('phase', 'terminal') ->whereIn('dispatch_attempts', [0, 1]) ->whereIn( 'effect_resolution', ['applied', 'not_applied'] )->where('finalization_incident', 1) ->where(static function ($incident): void { $incident->whereNull('finalization_error_code') ->orWhereNotIn( 'finalization_error_code', CloudDispatchLedger::repairIncidentCodes() ); })->whereIn( 'kind', CloudOperationRecoveryPolicy::ledgerFinalizerKinds() )->where('available_at', '<=', $now) ->where('lease_until', '<=', $now) ->orderBy('available_at') ->orderBy('id') ->lockForUpdate() ->first(); if ($job !== null) { $mode = 'resolved_incident'; } } if ($job === null && !$receivedOnly) { // A published effect with a contradictory immutable // terminal tuple is a PHP-ledger conflict. Keep it in a // local-only retry loop; never ask the Agent to reinterpret // or repeat the already concluded mutation. $job = CloudJob::query() ->where('phase', 'terminal') ->whereIn( 'kind', CloudOperationRecoveryPolicy::ledgerFinalizerKinds() )->where(static function ($invalid): void { CloudJobService::applyInvalidResolvedTerminalTuple( $invalid ); })->where('available_at', '<=', $now) ->where('lease_until', '<=', $now) ->orderBy('available_at') ->orderBy('id') ->lockForUpdate() ->first(); if ($job !== null) { $mode = 'invalid_state'; } } if ($job === null && !$receivedOnly) { // A valid dispatched generation with a damaged immutable // envelope must not disappear from the recovery queue. // This mode records the concrete conflict without making // any Agent request. $job = CloudJob::query() ->where('execution_mode', 'agent') ->where('phase', 'terminal') ->where('outcome', 'failed') ->where('status', 'failed') ->where('dispatch_attempts', 1) ->where('effect_may_have_applied', 1) ->where('safe_to_resubmit', 0) ->where('effect_resolution', 'unknown') ->whereIn( 'kind', CloudOperationRecoveryPolicy::ledgerFinalizerKinds() )->where(static function ($repair): void { $repair->where('finalization_incident', 0) ->orWhereNull('finalization_error_code') ->orWhereNotIn( 'finalization_error_code', CloudDispatchLedger::repairIncidentCodes() ); })->where(static function ($broken): void { $broken->whereNull('host_id') ->orWhere('host_id', '<=', 0) ->orWhereNull('operation_id') ->orWhereRaw('LENGTH(operation_id) <> 32') ->orWhereNull('dispatched_at') ->orWhere('dispatched_at', '<=', 0); })->where('available_at', '<=', $now) ->where('lease_until', '<=', $now) ->orderBy('available_at') ->orderBy('id') ->lockForUpdate() ->first(); if ($job !== null) { $mode = 'invalid_envelope'; } } if ($job === null && !$receivedOnly) { // A terminal unknown-effect row has exactly one trustworthy // safety tuple. Contradictory historical/data-damaged rows // stay in a PHP-only incident loop and never contact Agent. $job = CloudJob::query() ->where('phase', 'terminal') ->where('effect_resolution', 'unknown') ->where(static function ($scope): void { $scope->where(static function ($leaf): void { $leaf->whereIn('dispatch_attempts', [0, 1]) ->whereIn( 'kind', CloudOperationRecoveryPolicy::ledgerFinalizerKinds() ); })->orWhere( static function ($aggregate): void { $aggregate->where( 'execution_mode', 'aggregate' )->whereNull('parent_job_id') ->whereIn( 'kind', CloudJobService::aggregateKinds() ); } ); })->where(static function ($invalid): void { $invalid->where(static function ($mode): void { $mode->whereIn( 'kind', CloudOperationRecoveryPolicy::ledgerFinalizerKinds() )->where(static function ($broken): void { $broken->whereNull('execution_mode') ->orWhere( 'execution_mode', '<>', 'agent' ); }); })->orWhere(static function ($tuple): void { self::applyInvalidTerminalTuple($tuple); }); })->where('available_at', '<=', $now) ->where('lease_until', '<=', $now) ->orderBy('available_at') ->orderBy('id') ->lockForUpdate() ->first(); if ($job !== null) { $mode = 'invalid_state'; } } if ($job === null) { $job = CloudJob::query() ->where('execution_mode', 'agent') ->where('phase', 'terminal') ->where('outcome', 'failed') ->where('status', 'failed') ->where('dispatch_attempts', 1) ->where('effect_may_have_applied', 1) ->where('safe_to_resubmit', 0) ->where('effect_resolution', 'unknown') ->where(static function ($repair): void { $repair->where('finalization_incident', 0) ->orWhereNull('finalization_error_code') ->orWhereNotIn( 'finalization_error_code', CloudDispatchLedger::repairIncidentCodes() ); }) ->whereNotNull('operation_id') ->whereNotNull('host_id') ->whereNotNull('dispatched_at') ->where(static function ($eligible) use ( $now, $receivedOnly, $excludeReceived ): void { $eligible->where( static function ($policy) use ( $now, $receivedOnly, $excludeReceived ): void { $policy->whereIn( 'kind', // Create uses a separate financial // finalizer, never the generic replay or // effect-projection path below. CloudOperationRecoveryPolicy::automaticKinds() )->where( static function ($ready) use ( $now, $receivedOnly, $excludeReceived ): void { if ($receivedOnly) { $ready->whereExists( static function ( $events ): void { self::receivedEventExists( $events ); } ); return; } $ready->where( 'available_at', '<=', $now ); if (!$excludeReceived) { $ready->orWhereExists( static function ($events): void { self::receivedEventExists( $events ); } ); } } ); } ); }) ->when( $excludeReceived, static function ($candidate): void { $candidate->whereNotExists( static function ($events): void { self::receivedEventExists($events); } ); } ) ->where('lease_until', '<=', $now) ->orderByRaw( 'CASE WHEN EXISTS (SELECT 1 FROM cloud_agent_completion_events AS recovery_priority_events WHERE recovery_priority_events.host_id = cloud_jobs.host_id AND recovery_priority_events.operation_id = cloud_jobs.operation_id AND recovery_priority_events.disposition = ?) THEN 0 ELSE 1 END', ['received'] ) ->orderBy('available_at') ->orderBy('id') ->lockForUpdate() ->first(); } if ($job === null) { return null; } // SQL only finds a bounded candidate. Revalidate the complete // immutable Agent envelope on the hydrated row before loading // a receipt or allowing any Agent/fence path. A damaged field // is a local ledger incident; immutable evidence stays queued. $envelopeConflict = CloudJobService::hasAgentRecoveryEnvelopeConflict($job); if ($envelopeConflict) { $identityConflict = CloudJobService:: hasAgentRecoveryIdentityConflict($job); if ($mode === 'dispatch_repair') { $mode = $identityConflict ? 'dispatch_repair_invalid_identity' : 'dispatch_repair_invalid_envelope'; } else { $mode = $identityConflict ? 'invalid_identity' : 'invalid_envelope'; } } if (!$envelopeConflict && (string) $job->phase === 'terminal' && (string) $job->execution_mode === 'aggregate' && (CloudTemplate::where( 'active_job_id', (int) $job->id )->exists() || CloudUserIso::where( 'active_job_id', (int) $job->id )->exists())) { $job->lease_token = bin2hex(random_bytes(16)); $job->lease_until = $now + self::CLAIM_SECONDS; $job->save(); return [ 'job' => $job, 'event' => null, 'mode' => 'aggregate_state', ]; } if (CloudDispatchLedger::isInvalid($job)) { // Do not inspect or consume a receipt while the dispatch // generation is corrupt. Lease only long enough to hand // the row to the fail-closed normalizer after commit. $job->lease_token = bin2hex(random_bytes(16)); $job->lease_until = $now + self::CLAIM_SECONDS; $job->save(); return [ 'job' => $job, 'event' => null, 'mode' => 'invalid', ]; } if ($mode === 'local_finalizer' || $mode === 'public_network_finalizer' || $mode === 'portable_restore_finalizer' || $mode === 'invalid_envelope' || $mode === 'invalid_state' || $mode === 'resolved_incident' || $mode === 'aggregate_state' || $mode === 'invalid_identity' || $mode === 'dispatch_repair_invalid_envelope' || $mode === 'dispatch_repair_invalid_identity' || $mode === 'dispatch_repair') { $event = null; if ($mode === 'dispatch_repair' && is_string($job->completion_event_id) && $job->completion_event_id !== '') { $event = CloudAgentCompletionEvent::where( 'event_id', (string) $job->completion_event_id )->lockForUpdate()->first(); } $job->lease_token = bin2hex(random_bytes(16)); $job->lease_until = $now + self::CLAIM_SECONDS; $job->save(); return [ 'job' => $job, 'event' => $event, 'mode' => $mode, ]; } $event = CloudAgentCompletionEvent::where( 'host_id', (int) $job->host_id )->where( 'operation_id', (string) $job->operation_id )->where('disposition', 'received') ->lockForUpdate() ->first(); if ($event !== null) { $job->lease_token = bin2hex(random_bytes(16)); $job->lease_until = $now + self::CLAIM_SECONDS; $job->save(); return [ 'job' => $job, 'event' => $event, 'mode' => 'agent', ]; } $attempt = self::nextAttemptMarker( (string) $job->effect_resolution_reason, $now ); // The dedicated create coordinator owns its encrypted // prepare/commit binding. Never wrap or replace that marker // when resuming a fence-write/financial-commit crash window. if ((string) $job->kind === 'create' && preg_match( '/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D', (string) $job->effect_resolution_reason ) === 1) { $attempt = (string) $job->effect_resolution_reason; } if ($attempt === false) { $job->finalization_incident = 1; $job->finalization_error_code = 'recovery_state_invalid'; $job->read_at = null; } else { $job->effect_resolution_reason = $attempt; if (in_array( (string) ($job->finalization_error_code ?? ''), [ 'recovery_limit_reached', 'recovery_receipt_limit_reached', ], true )) { $job->finalization_incident = 0; $job->finalization_error_code = null; $job->read_at = null; } } $job->lease_token = bin2hex(random_bytes(16)); $job->lease_until = $now + self::CLAIM_SECONDS; $job->save(); return [ 'job' => $job, 'event' => null, 'mode' => 'agent', ]; }, 3 ); } /** Add the negation of the canonical failed/effectful/non-retryable tuple. */ private static function applyInvalidTerminalTuple($invalid): void { $invalid->where(static function ($tuple): void { $tuple->whereNull('outcome') ->orWhere('outcome', '<>', 'failed') ->orWhereNull('status') ->orWhere('status', '<>', 'failed') ->orWhereNull('effect_may_have_applied') ->orWhere('effect_may_have_applied', '<>', 1) ->orWhereNull('safe_to_resubmit') ->orWhere('safe_to_resubmit', '<>', 0); })->orWhere(static function ($neverDispatched): void { $neverDispatched->where('dispatch_attempts', 0) ->where('finalization_incident', 0); }); } private static function receivedEventExists($events): void { $events->selectRaw('1') ->from('cloud_agent_completion_events as recovery_events') ->whereColumn( 'recovery_events.host_id', 'cloud_jobs.host_id' ) ->whereColumn( 'recovery_events.operation_id', 'cloud_jobs.operation_id' ) ->where('recovery_events.disposition', 'received'); } /** * @return array{delay:int,error_code:string}|null Retry state, or null * after a committed resolution. */ private function recover(CloudJob $claimed): ?array { $job = CloudJob::find((int) $claimed->id); if ($job === null || !is_string($job->lease_token) || !hash_equals( (string) $claimed->lease_token, (string) $job->lease_token )) { throw new CloudError('自动核对租约已失效', 409); } if ((string) $job->kind === 'create') { CloudCreateFailureRecoveryService::recover($job); return null; } $context = self::recoveryContext($job); // A late immutable completion receipt is a PHP-local two-phase // recovery. If the accepting request crashed after prepare, rebuild // the durable event and finish without contacting the Agent. $lateEventId = self::preparedLateReceiptEventId($job); if ($lateEventId !== null) { $event = CloudAgentCompletionEvent::where( 'event_id', $lateEventId )->first(); if ($event === null || (string) $event->disposition !== 'received') { throw new CloudError('晚到回执恢复记录不完整', 409); } $disposition = CloudOperationRecoveryService::settleLateReceipt( $job, $event ); $resolved = CloudJob::find((int) $job->id); $appliedEvent = CloudAgentCompletionEvent::where( 'event_id', $lateEventId )->first(); if ($disposition !== 'applied' || $resolved === null || !in_array( (string) $resolved->effect_resolution, ['applied', 'not_applied'], true ) || $appliedEvent === null || (string) $appliedEvent->disposition !== 'applied') { throw new CloudError('晚到回执尚未完成安全收敛', 409); } return null; } $prepared = self::preparedAutomaticIntent($job, $context); if ($prepared !== null) { $reason = self::recoveryReason( $job, $prepared['source'], $prepared['resolution'] ); if ($prepared['fence_resolution_required']) { try { CloudAdminOperationRecoveryService:: preparedFenceResolutionForRecovery( (string) $job->id, $prepared['resolution'], $reason ); self::commitPrepared($job, $prepared); return null; } catch (CloudAgentError $error) { if ($error->agentHttpStatus !== 404) { throw $error; } // The prior process did not commit a fence resolution. // Continue with a fresh inspection before any new write. } } else { // Exact Journal success and exact no-effect failure never // retained an Agent fence. PHP may finish an already-prepared // intent without inventing a fence read or write. self::commitPrepared($job, $prepared); return null; } } $inspection = CloudAdminOperationRecoveryService::inspectForRecovery( (string) $claimed->id ); self::assertFreshInspection($job, $inspection); // A poisoned completion outbox means the Agent cannot prove that its // immutable result was delivered intact. Even if the journal's local // projection says applied/not-applied, that projection must remain // inconclusive until the outbox/receipt record is repaired. $deliveryIncident = self::completionDeliveryIncident($inspection); if ($deliveryIncident !== null) { return self::pending($deliveryIncident); } // A resolved Agent fence proves that some coordinator already wrote // an immutable verdict; it does not prove that PHP prepared the same // verdict. Only the prepared-intent resume path above may consume it. if (($inspection['fence_state'] ?? null) === 'resolved') { return self::pending('recovery_evidence_conflict'); } $resolution = (string) ($inspection['effect_resolution'] ?? ''); if ($resolution === 'partial' && (string) $job->kind === 'instance_migration_transfer' && CloudInstanceMigrationService::ensureTargetResidueCleanup( $job, $inspection )) { return self::pending('recovery_target_cleanup_pending'); } if (in_array($resolution, ['partial', 'inconclusive'], true)) { return self::pending( $resolution === 'partial' ? 'recovery_evidence_partial' : 'recovery_evidence_inconclusive' ); } if (!in_array($resolution, ['applied', 'not_applied'], true)) { throw new CloudError('宿主机效果核对结论无效', 409); } $operation = self::operationFromInspection( $job, $context, $inspection, $resolution ); $evidenceSource = self::evidenceSource( $job, $context, $inspection, $resolution ); $reason = self::recoveryReason($job, $evidenceSource, $resolution); $journalStatus = (string) $inspection['journal_status']; $fenceResolutionRequired = (bool) $inspection['fence_resolution_required']; // A system fence verdict is an authorization boundary, not merely a // validation performed after PHP has changed its ledgers. In // particular, backup restore may be resolved automatically only as // not-applied while a fence is active. Reject a host-state inference // before prepareApplied()/prepareNotApplied() can materialise it. if ($fenceResolutionRequired && !CloudOperationRecoveryPolicy::allowsSystemResolution( (string) $job->kind, $resolution )) { return self::pending('recovery_system_resolution_not_allowed'); } if ($resolution === 'applied') { CloudOperationRecoveryService::prepareApplied( (int) $job->id, $operation, $evidenceSource ); } else { CloudOperationRecoveryService::prepareNotApplied( (int) $job->id, $operation, $evidenceSource ); } $preparedIntent = [ 'resolution' => $resolution, 'journal_status' => $journalStatus, 'fence_resolution_required' => $fenceResolutionRequired, 'source' => $evidenceSource, 'operation' => $operation, ]; if ($fenceResolutionRequired) { self::resolveFenceAndCommit( $job, $preparedIntent, $reason, $inspection ); return null; } elseif (!(($journalStatus === 'succeeded' && $resolution === 'applied') || ($journalStatus === 'failed' && $resolution === 'not_applied'))) { throw new CloudError('宿主机操作终态不允许自动收敛', 409); } self::commitPrepared($job, $preparedIntent); return null; } /** * Serialize the only Agent fence write with PHP's effect commit. * * The job row remains locked across the bounded Agent metadata request. * Receipt ingestion locks the same row first, so either the immutable * receipt wins before this transaction (and no fence write occurs), or * this prepared recovery commits before the receipt is classified late. * Unrelated jobs and ordinary workers are not locked. * * @param array{resolution:string,journal_status:string,fence_resolution_required:bool,source:string,operation:array} $prepared */ private static function resolveFenceAndCommit( CloudJob $claimed, array $prepared, string $reason, array $inspection ): void { // Snapshot the immutable parent link before opening the locking // transaction. Aggregate paths take parent -> catalog -> leaf so the // nested commitPrepared() transaction can safely reacquire that same // canonical lock sequence. $aggregateParentId = (int) (CloudJob::where( 'id', (int) $claimed->id )->value('parent_job_id') ?? 0); (new CloudJob())->getConnection()->transaction( static function () use ( $claimed, $prepared, $reason, $inspection, $aggregateParentId ): void { $aggregateChildren = null; if ($aggregateParentId > 0) { $aggregateParent = CloudJob::where( 'id', $aggregateParentId ) ->lockForUpdate()->first(); if ($aggregateParent === null) { throw new CloudRecoveryAggregateStateConflict( '任务的汇总父任务不存在', 409 ); } if ((string) $aggregateParent->execution_mode !== 'aggregate' || CloudJobService::hasAggregateLocalStateConflict( $aggregateParent )) { throw new CloudRecoveryAggregateStateConflict( '任务的汇总父任务账本状态异常', 409 ); } CloudTemplateCatalogService::lockAggregateCatalogLocked( $aggregateParent ); CloudUserIsoService::lockAggregateCatalogLocked( $aggregateParent ); $aggregateChildren = CloudJob::where( 'parent_job_id', $aggregateParentId )->orderBy('id')->lockForUpdate()->get(); $job = $aggregateChildren->first( static fn (CloudJob $child): bool => (int) $child->id === (int) $claimed->id ); } else { $job = CloudJob::where('id', (int) $claimed->id) ->lockForUpdate()->first(); } if ($job === null || !hash_equals( (string) ($job->lease_token ?? ''), (string) $claimed->lease_token ) || (int) $job->lease_until <= time() || (string) $job->effect_resolution !== 'unknown' || !$prepared['fence_resolution_required']) { throw new CloudError('自动核对租约已失效', 409); } if ((int) ($job->parent_job_id ?? 0) !== $aggregateParentId) { throw new CloudRecoveryAggregateStateConflict( '任务的汇总父级关系在栅栏提交期间发生变化', 409 ); } if ($aggregateParentId > 0) { CloudJobService::assertAggregateChildRelationshipLocked( $aggregateParent, $job, $aggregateChildren ); } $context = self::recoveryContext($job); $current = self::preparedAutomaticIntent($job, $context); if ($current === null || $current['resolution'] !== $prepared['resolution'] || $current['journal_status'] !== $prepared['journal_status'] || !$current['fence_resolution_required'] || !hash_equals( $current['source'], $prepared['source'] ) || self::canonicalize($current['operation']) !== self::canonicalize($prepared['operation'])) { throw new CloudError('自动核对预提交证据已改变', 409); } $pending = CloudAgentCompletionEvent::where( 'host_id', (int) $job->host_id )->where( 'operation_id', (string) $job->operation_id )->where('disposition', 'received') ->lockForUpdate()->first(); if ($pending !== null) { throw new CloudError('晚到完成回执正在等待优先收敛', 409); } CloudAdminOperationRecoveryService::resolveFenceForRecovery( (string) $job->id, $prepared['resolution'], $reason, $inspection ); self::commitPrepared($job, $prepared); }, 3 ); } private static function assertFreshInspection( CloudJob $job, array $inspection ): void { $checkedAt = $inspection['checked_at'] ?? null; $now = time(); if (($inspection['operation_id'] ?? null) !== (string) $job->operation_id || !is_int($checkedAt) || $checkedAt < $now - self::INSPECTION_MAX_AGE_SECONDS || $checkedAt > $now + 5) { throw new CloudError('宿主机只读核对证据已失效', 409); } $expected = CloudOperationRecoveryIdentity::expected( $job, ($inspection['effect_resolution'] ?? null) === 'applied' ); CloudOperationRecoveryIdentity::assertMatches($expected, [ 'action' => $inspection['action'] ?? null, 'resource_id' => $inspection['resource_id'] ?? null, 'request_fingerprint' => $inspection['request_fingerprint'] ?? null, ]); } private static function completionDeliveryIncident( array $inspection ): ?string { if (($inspection['completion_delivery_state'] ?? null) !== 'quarantined') { return null; } return match ($inspection['completion_delivery_error_code'] ?? null) { 'completion_outbox_invalid' => 'recovery_completion_outbox_invalid', 'completion_receipt_missing' => 'recovery_completion_receipt_missing', 'completion_receipt_invalid' => 'recovery_completion_receipt_invalid', 'completion_operation_missing' => 'recovery_completion_operation_missing', 'completion_operation_invalid' => 'recovery_completion_operation_invalid', 'completion_identity_mismatch' => 'recovery_completion_identity_mismatch', default => 'recovery_evidence_invalid', }; } private static function operationFromInspection( CloudJob $job, array $context, array $inspection, string $resolution ): array { $journalStatus = $inspection['journal_status'] ?? null; $fenceState = $inspection['fence_state'] ?? null; $fenceResolutionRequired = $inspection['fence_resolution_required'] ?? null; if (!in_array($journalStatus, ['succeeded', 'failed'], true)) { throw new CloudError('宿主机操作日志终态无效', 409); } if (!is_bool($fenceResolutionRequired) || !in_array($fenceState, ['not_required', 'active'], true) || $fenceResolutionRequired !== ($fenceState === 'active') || ($journalStatus === 'succeeded' && $fenceState !== 'not_required') || ($journalStatus === 'failed' && $resolution === 'applied' && $fenceState !== 'active')) { throw new CloudError('宿主机栅栏证据与操作日志不匹配', 409); } if ($resolution === 'not_applied') { $errorCode = $inspection['error_code'] ?? null; $createLike = CloudOperationRecoveryPolicy::isCreateLike( (string) $job->kind ); $projection = $inspection['effect_projection'] ?? null; if ($journalStatus !== 'failed' || ($inspection['outcome'] ?? null) !== 'failed' || ($inspection['summary_code'] ?? null) !== 'operation_failed' || !is_string($errorCode) || preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $errorCode) !== 1 || ($createLike ? (!is_array($projection) || array_is_list($projection) || ($projection['creation_absent'] ?? null) !== true || ($projection['managed_artifacts_absent'] ?? null) !== true) : array_key_exists('effect_projection', $inspection))) { throw new CloudError('宿主机未生效证据不完整', 409); } return [ 'operation_id' => (string) $job->operation_id, 'action' => (string) $inspection['action'], 'resource_id' => (string) $inspection['resource_id'], 'request_fingerprint' => (string) $inspection['request_fingerprint'], 'status' => 'failed', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'retryable' => false, 'error_code' => $errorCode, 'result' => $createLike ? $projection : [], ]; } $projection = $inspection['effect_projection'] ?? null; if (($inspection['outcome'] ?? null) !== 'succeeded' || ($inspection['summary_code'] ?? null) !== 'operation_succeeded' || array_key_exists('error_code', $inspection) || !is_array($projection) || array_is_list($projection)) { throw new CloudError('宿主机已生效证据不完整', 409); } if ((string) $job->kind === 'instance_settings') { if (($projection['uuid'] ?? null) !== (string) $context['instance']->uuid || ($projection['settings_changed'] ?? null) !== true) { throw new CloudError('宿主机实例设置证据不完整', 409); } if (array_key_exists('hostname', $context['body']) && ($projection['hostname'] ?? null) !== $context['body']['hostname']) { throw new CloudError('宿主机主机名证据与请求不匹配', 409); } } elseif ((string) $job->kind === 'network_change' && $job->public_network_transition_id !== null) { try { $requested = CloudPublicNetworkSnapshot::fromArray( $context['payload']['body']['public_network'] ?? null ); $confirmed = CloudPublicNetworkSnapshot::fromArray( $projection['public_network'] ?? null ); } catch (\Throwable) { throw new CloudError('宿主机公网网络证据无效', 409); } if (!$context['instance'] instanceof \App\Models\CloudInstance || ($projection['uuid'] ?? null) !== (string) $context['instance']->uuid || ($projection['network_changed'] ?? null) !== true || !hash_equals( $requested->fingerprint(), $confirmed->fingerprint() )) { throw new CloudError( '宿主机公网网络证据与冻结目标不匹配', 409 ); } } return [ 'operation_id' => (string) $job->operation_id, 'action' => (string) $inspection['action'], 'resource_id' => (string) $inspection['resource_id'], 'request_fingerprint' => (string) $inspection['request_fingerprint'], 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'retryable' => false, 'error_code' => '', // CloudAdminOperationRecoveryService already applied the strict // per-kind projection schema. The existing worker finalizer now // validates it against PHP's encrypted request and ledgers. 'result' => $projection, ]; } private static function evidenceSource( CloudJob $job, array $context, array $inspection, string $resolution ): string { $projection = $inspection['effect_projection'] ?? []; $criticalProjection = is_array($projection) ? $projection : []; $instance = $context['instance'] ?? null; $evidence = self::canonicalize([ 'operation_id' => (string) $job->operation_id, 'kind' => (string) $job->kind, 'instance_uuid' => $instance === null ? null : (string) $instance->uuid, 'journal_status' => $inspection['journal_status'] ?? null, 'fence_resolution_required' => $inspection['fence_resolution_required'] ?? null, 'resolution' => $resolution, 'error_code' => $inspection['error_code'] ?? null, 'projection' => $criticalProjection, ]); $journalStatus = (string) ($inspection['journal_status'] ?? ''); $fence = ($inspection['fence_resolution_required'] ?? false) === true ? 'fence' : 'nofence'; $prefix = $resolution === 'applied' ? 'automatic_applied_' . $journalStatus . '_' . $fence : 'automatic_not_applied_' . $journalStatus . '_' . $fence; return $prefix . ':' . hash( 'sha256', json_encode( $evidence, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE ) ); } /** * Rebuild the exact stable operation which was durably prepared earlier. * The prepared source records both resolution and Journal terminal state; * commitApplied()/commitNotApplied() re-check its operation hash. * * @return array{resolution:string,journal_status:string,fence_resolution_required:bool,source:string,operation:array}|null */ private static function preparedAutomaticIntent( CloudJob $job, array $context ): ?array { $marker = self::preparedPayload( (string) $job->effect_resolution_reason ); if ($marker === null) { return null; } if (preg_match( '/^prepared:(automatic_(applied|not_applied)_(succeeded|failed)_(fence|nofence):[a-f0-9]{64}):([a-f0-9]{64})$/D', $marker, $matches ) !== 1) { return null; } $resolution = $matches[2]; $journalStatus = $matches[3]; $fenceResolutionRequired = $matches[4] === 'fence'; $validShape = ($resolution === 'applied' && $journalStatus === 'succeeded' && !$fenceResolutionRequired) || ($resolution === 'applied' && $journalStatus === 'failed' && $fenceResolutionRequired) || ($resolution === 'not_applied' && $journalStatus === 'failed'); if (!$validShape) { return null; } $operation = CloudWorkerService::preparedRecoveryOperation($job); if ($operation === null && (string) $job->kind !== 'instance_settings') { throw new CloudRecoveryPayloadUnavailable( '任务缺少可恢复的预提交结果', 409 ); } // Compatibility for settings intents prepared before the generic // encrypted operation envelope was introduced. if ($operation === null && $resolution === 'not_applied') { $operation = [ 'operation_id' => (string) $job->operation_id, 'status' => 'failed', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => 'settings_not_committed', 'result' => [], ]; } elseif ($operation === null) { $result = [ 'uuid' => (string) $context['instance']->uuid, 'settings_changed' => true, ]; if (array_key_exists('hostname', $context['body'])) { $spec = json_decode( (string) $context['instance']->spec_json, true ); if (!is_array($spec) || array_is_list($spec) || ($spec['guest_hostname'] ?? null) !== $context['body']['hostname']) { throw new CloudError( '已预提交的主机名账本与请求不匹配', 409 ); } $result['hostname'] = $context['body']['hostname']; } $operation = [ 'operation_id' => (string) $job->operation_id, 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => $result, ]; } $operationHash = CloudOperationRecoveryService:: preparedOperationHash( $operation ); $validOperation = $resolution === 'applied' ? (($operation['status'] ?? null) === 'succeeded' && ($operation['effect_may_have_applied'] ?? null) === false && ($operation['safe_to_resubmit'] ?? null) === false) : (($operation['status'] ?? null) === 'failed' && ($operation['effect_may_have_applied'] ?? null) === false && is_bool($operation['safe_to_resubmit'] ?? null) && is_string($operation['error_code'] ?? null) && preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $operation['error_code']) === 1); if (!$validOperation || !hash_equals($matches[5], $operationHash)) { throw new CloudRecoveryPayloadUnavailable( '任务预提交结果与恢复标记不匹配', 409 ); } return [ 'resolution' => $resolution, 'journal_status' => $journalStatus, 'fence_resolution_required' => $fenceResolutionRequired, 'source' => $matches[1], 'operation' => $operation, ]; } private static function recoveryReason( CloudJob $job, string $source, string $resolution ): string { $separator = strrpos($source, ':'); $evidence = $separator === false ? $source : substr($source, $separator + 1); return sprintf( 'automatic %s recovery job=%d operation=%s evidence=%s resolution=%s', (string) $job->kind, (int) $job->id, (string) $job->operation_id, $evidence, $resolution ); } /** * @param array{resolution:string,journal_status:string,fence_resolution_required:bool,source:string,operation:array} $prepared */ private static function commitPrepared( CloudJob $job, array $prepared ): void { if ($prepared['resolution'] === 'applied') { CloudOperationRecoveryService::commitApplied( (int) $job->id, $prepared['operation'], $prepared['source'], 0 ); return; } CloudOperationRecoveryService::commitNotApplied( (int) $job->id, $prepared['operation'], $prepared['source'], 0 ); } private static function canonicalize(mixed $value): mixed { if (!is_array($value)) { return $value; } if (!array_is_list($value)) { ksort($value, SORT_STRING); } foreach ($value as $key => $item) { $value[$key] = self::canonicalize($item); } return $value; } private static function recoveryContext(CloudJob $job): array { return (string) $job->kind === 'instance_settings' ? CloudWorkerService::settingsRecoveryContext($job) : CloudWorkerService::recoveryContext($job); } /** @return array{delay:int,error_code:string} */ private static function pending(string $errorCode): array { return [ 'delay' => self::RETRY_SECONDS, 'error_code' => $errorCode, ]; } private static function recoveryErrorCode(\Throwable $error): string { if ($error instanceof CloudRecoveryDispatchLedgerConflict) { return CloudDispatchLedger::CONFLICT_CODE; } if ($error instanceof CloudRecoveryAggregateStateConflict) { return 'recovery_aggregate_state_conflict'; } if ($error instanceof CloudRecoveryPayloadUnavailable) { return 'recovery_payload_unavailable'; } if ($error instanceof CloudRecoveryOperationIdentityConflict) { return 'recovery_operation_identity_conflict'; } if ($error instanceof CloudRecoveryEvidenceConflict) { return 'recovery_evidence_conflict'; } if ($error instanceof CloudRecoveryReceiptInvalid) { return 'recovery_receipt_invalid'; } if ($error instanceof CloudRecoveryInstanceStateConflict) { return 'recovery_instance_state_conflict'; } if ($error instanceof CloudAgentError) { if (in_array($error->agentCode, [ 'agent_invalid_inspection', 'agent_invalid_fence_resolution', 'fence_resolution_read_failed', ], true) && $error->agentHttpStatus === 200) { return 'recovery_evidence_invalid'; } if ($error->agentHttpStatus === 404) { return 'recovery_operation_missing'; } if (in_array($error->agentHttpStatus, [401, 403], true)) { return 'recovery_authentication_failed'; } if ($error->agentHttpStatus === 409) { return 'recovery_operation_conflict'; } return 'recovery_waiting_connection'; } if ($error instanceof CloudError) { return 'recovery_ledger_conflict'; } return 'recovery_internal_error'; } private function releaseForRetry( CloudJob $claimed, int $delay, string $errorCode ): void { (new CloudJob())->getConnection()->transaction( static function () use ($claimed, $delay, $errorCode): void { $job = CloudJob::where('id', (int) $claimed->id) ->where('effect_resolution', 'unknown') ->where( 'lease_token', (string) $claimed->lease_token ) ->lockForUpdate() ->first(); if ($job === null) { return; } $now = time(); $job->lease_token = null; $job->lease_until = 0; $job->available_at = $now + max( self::RETRY_SECONDS, $delay ); if (!hash_equals( (string) ($job->finalization_error_code ?? ''), $errorCode )) { $job->read_at = null; } $job->finalization_incident = 1; $job->finalization_error_code = $errorCode; $job->updated_at = $now; $job->save(); }, 3 ); } private function resumeDispatchLedgerRepair(CloudJob $claimed): void { try { $result = CloudOperationCompletionService:: resumeRepairedDispatchLedger($claimed); if ($result === 'missing') { $this->releaseLocalClaim( $claimed, 'recovery_dispatch_repair_completion_receipt_missing' ); } elseif ($result === 'invalid') { $this->releaseLocalClaim( $claimed, 'recovery_dispatch_repair_receipt_invalid' ); } } catch (\Throwable $error) { $this->releaseLocalClaim( $claimed, self::dispatchRepairErrorCode($error) ); } } private static function dispatchRepairErrorCode( \Throwable $error ): string { return match (self::recoveryErrorCode($error)) { CloudDispatchLedger::CONFLICT_CODE => CloudDispatchLedger::CONFLICT_CODE, 'recovery_aggregate_state_conflict' => 'recovery_dispatch_repair_aggregate_state_conflict', 'recovery_payload_unavailable' => 'recovery_dispatch_repair_payload_unavailable', 'recovery_operation_identity_conflict' => 'recovery_dispatch_repair_operation_identity_conflict', 'recovery_evidence_conflict' => 'recovery_dispatch_repair_evidence_conflict', 'recovery_receipt_invalid' => 'recovery_dispatch_repair_receipt_invalid', 'recovery_instance_state_conflict' => 'recovery_dispatch_repair_instance_state_conflict', 'recovery_ledger_conflict' => 'recovery_dispatch_repair_ledger_conflict', default => 'recovery_dispatch_repair_internal_error', }; } /** Release a PHP-only recovery claim without requiring unknown effect. */ private function releaseLocalClaim( CloudJob $claimed, string $errorCode ): void { (new CloudJob())->getConnection()->transaction( static function () use ($claimed, $errorCode): void { $job = CloudJob::where('id', (int) $claimed->id) ->where( 'lease_token', (string) $claimed->lease_token )->lockForUpdate()->first(); if ($job === null) { return; } $now = time(); $job->lease_token = null; // claim(true) intentionally prioritizes a matching received // event over ordinary available_at backoff. Keep the lease // clock as the retry gate as well, otherwise an invalid bound // receipt would be reclaimed on every worker tick. $job->lease_until = $now + self::RETRY_SECONDS; $job->available_at = $now + self::RETRY_SECONDS; if (!hash_equals( (string) ($job->finalization_error_code ?? ''), $errorCode )) { $job->read_at = null; } $job->finalization_incident = 1; $job->finalization_error_code = $errorCode; $job->updated_at = $now; $job->save(); }, 3 ); } /** @return array{first_at:int,attempts:int,payload:string}|null */ private static function attemptState(string $marker): ?array { if (preg_match( '/^automatic_recovery:v1:([1-9][0-9]{0,10}):([1-9][0-9]{0,9}):(pending|prepared:(?:automatic_(?:applied|not_applied)_(?:succeeded|failed)_(?:fence|nofence):[a-f0-9]{64}|late_completion_receipt:[A-Za-z0-9_.-]{1,160}):[a-f0-9]{64})$/D', $marker, $matches ) !== 1) { return null; } return [ 'first_at' => (int) $matches[1], 'attempts' => (int) $matches[2], 'payload' => $matches[3], ]; } private static function nextAttemptMarker( string $marker, int $now ): string|false { if ($marker === '') { return sprintf( 'automatic_recovery:v1:%d:1:pending', $now ); } $state = self::attemptState($marker); if ($state === null) { if (preg_match( '/^prepared:(?:automatic_(?:applied|not_applied)_(?:succeeded|failed)_(?:fence|nofence):[a-f0-9]{64}|late_completion_receipt:[A-Za-z0-9_.-]{1,160}):[a-f0-9]{64}$/D', $marker ) === 1) { return sprintf( 'automatic_recovery:v1:%d:1:%s', $now, $marker ); } return false; } return sprintf( 'automatic_recovery:v1:%d:%d:%s', $state['first_at'], min(9999999999, $state['attempts'] + 1), $state['payload'] ); } private static function preparedPayload(string $marker): ?string { $state = self::attemptState($marker); if ($state !== null && str_starts_with( $state['payload'], 'prepared:' )) { return $state['payload']; } return preg_match( '/^prepared:(?:automatic_(?:applied|not_applied)_(?:succeeded|failed)_(?:fence|nofence):[a-f0-9]{64}|late_completion_receipt:[A-Za-z0-9_.-]{1,160}):[a-f0-9]{64}$/D', $marker ) === 1 ? $marker : null; } private static function preparedLateReceiptEventId( CloudJob $job ): ?string { $payload = self::preparedPayload( (string) $job->effect_resolution_reason ); if (!is_string($payload) || preg_match( '/^prepared:late_completion_receipt:([A-Za-z0-9_.-]{1,160}):[a-f0-9]{64}$/D', $payload, $matches ) !== 1) { return null; } return $matches[1]; } }