effect_resolution_reason, $prepared, $evidenceSource, $resolution, self::legacyCompletionPreparedMarker( $operation, $evidenceSource ) ); return (new CloudWorkerService())->prepareRecoveredEffect( $jobId, $operation, $marker ); } /** Publish one prepared resolution through the shared locked commit seam. */ private static function commitResolution( int $jobId, array $operation, string $evidenceSource, string $resolution, ?int $resolvedBy, ?CloudAgentCompletionEvent $completionEvent ): CloudJob { self::assertResolution($resolution); self::assertResolvedBy($resolvedBy); $jobSnapshot = CloudJob::find($jobId); $aggregateParentId = (int) ($jobSnapshot?->parent_job_id ?? 0); $transitionUserId = $jobSnapshot instanceof CloudJob && (string) $jobSnapshot->kind === 'network_change' && $jobSnapshot->public_network_transition_id !== null ? (int) $jobSnapshot->user_id : 0; $portableRestoreUserId = $jobSnapshot instanceof CloudJob && CloudPortableRestoreFinalizer::owns($jobSnapshot) ? (int) $jobSnapshot->user_id : 0; $recoveryUserId = $transitionUserId > 0 ? $transitionUserId : $portableRestoreUserId; return (new CloudJob())->getConnection()->transaction( function () use ( $jobId, $operation, $evidenceSource, $resolution, $resolvedBy, $completionEvent, $aggregateParentId, $transitionUserId, $portableRestoreUserId, $recoveryUserId ): CloudJob { $recoveryActor = $recoveryUserId > 0 ? User::where('id', $recoveryUserId) ->lockForUpdate()->first() : null; if ($recoveryUserId > 0 && !$recoveryActor instanceof User) { throw new CloudError( '任务效果恢复账户不存在', 409 ); } $transitionActor = $transitionUserId > 0 ? $recoveryActor : null; $aggregateParent = null; if ($aggregateParentId > 0) { $aggregateParent = CloudJob::where( 'id', $aggregateParentId ) ->lockForUpdate()->first(); if ($aggregateParent === null) { throw new CloudRecoveryAggregateStateConflict( '任务的汇总父任务不存在', 409 ); } CloudTemplateCatalogService::lockAggregateCatalogLocked( $aggregateParent ); CloudUserIsoService::lockAggregateCatalogLocked( $aggregateParent ); } $job = CloudJob::where('id', $jobId) ->lockForUpdate()->first(); if ($job === null) { throw new CloudError('任务不存在', 404); } if ((int) ($job->parent_job_id ?? 0) !== $aggregateParentId) { throw new CloudRecoveryAggregateStateConflict( '任务的汇总父级关系在提交期间发生变化', 409 ); } if ($transitionUserId > 0 && ((int) $job->user_id !== $transitionUserId || (string) $job->kind !== 'network_change' || $job->public_network_transition_id === null)) { throw new CloudRecoveryInstanceStateConflict( '公网网络变更恢复账户关联发生变化', 409 ); } if ($portableRestoreUserId > 0 && ((int) $job->user_id !== $portableRestoreUserId || !CloudPortableRestoreFinalizer::owns($job))) { throw new CloudRecoveryInstanceStateConflict( '备份恢复账户或尝试关联发生变化', 409 ); } self::assertLockedAgentRecoveryEnvelope($job); self::assertEvidenceSource($evidenceSource); self::assertOperationIdentity( $job, $operation, $resolution ); $lockedEvent = self::lockLateReceiptEvent( $job, $completionEvent, $evidenceSource ); self::assertRecoveryCommitMayPublish( $job, $evidenceSource, $resolvedBy, $completionEvent ); $finalReason = $evidenceSource; if ((string) $job->effect_resolution === $resolution && hash_equals( $finalReason, (string) $job->effect_resolution_reason )) { self::markLateReceiptApplied($job, $lockedEvent); return $job; } if ($job->public_network_transition_id !== null) { CloudPublicNetworkTransitionWorkflow:: publishPreparedLocked( $job, $resolution, $transitionActor, $aggregateParent ); } if ($portableRestoreUserId > 0) { CloudPortableRestoreFinalizer::publishPreparedLocked( $job, $resolution ); } self::assertRecoverableEvidence( $job, $operation, $evidenceSource, $resolution ); $marker = self::preparedMarker( $operation, $evidenceSource ); if (!self::preparedMarkerMatches( (string) $job->effect_resolution_reason, $marker )) { throw new CloudError('任务效果尚未完成账本预提交', 409); } $job->effect_resolution = $resolution; $job->effect_resolved_at = time(); $job->effect_resolved_by = $resolvedBy; $job->effect_resolution_reason = $finalReason; $job->sealed_payload = self::finalizedRecoveryPayload($job); $job->lease_token = null; $job->lease_until = 0; $job->available_at = 0; $job->finalization_incident = 0; $job->finalization_error_code = null; $job->read_at = null; $job->updated_at = time(); $job->save(); CloudJobService::reconcileAggregateParentEffectLocked( $job, $resolvedBy ); self::markLateReceiptApplied($job, $lockedEvent); return $job; }, 3 ); } /** * @return string One of the existing completion-event dispositions. */ public static function settleLateReceipt( CloudJob $source, CloudAgentCompletionEvent $event ): string { $disposition = (string) $event->disposition; if ($disposition !== 'received') { return $disposition; } if (!self::sameOperation($source, $event)) { return self::rejectInvalidLateReceipt( (int) $source->id, (string) $event->event_id ); } if (CloudDispatchLedger::isInvalid($source)) { CloudOperationCompletionService::deferDispatchLedgerConflict( $source ); return 'received'; } if (CloudDispatchLedger::isNeverDispatched($source)) { CloudOperationCompletionService:: deferNeverDispatchedEvidenceConflict($source); return 'received'; } if (CloudDispatchLedger::hasRepairIncident($source)) { return 'received'; } $current = $source; try { $current = CloudJob::find((int) $source->id) ?? $source; CloudOperationCompletionService::assertEventIdentityForJob( $current, $event ); if (self::wasReconciledBy($current, $event)) { return self::acknowledgeReconciledLateReceipt( (int) $current->id, (string) $event->event_id ); } if ((string) $current->kind === 'create') { if ((string) $event->outcome === 'failed') { return CloudCreateFailureRecoveryService::settleReceipt( $current, $event ); } if ((string) $event->outcome !== 'succeeded' || (bool) $event->effect_may_have_applied || (bool) $event->safe_to_resubmit || !CloudOperationCompletionService::isExactStoredEvent( $event )) { CloudOperationCompletionService::mark($event, 'late'); return 'late'; } (new CloudWorkerService())->settleRecoveredCreateCompletion( (int) $current->id, $event ); $reloadedCreateEvent = CloudAgentCompletionEvent::where( 'event_id', (string) $event->event_id )->first(); return $reloadedCreateEvent === null ? 'received' : (string) $reloadedCreateEvent->disposition; } $lateResolution = self::lateReceiptResolution($current, $event); if ($lateResolution === null) { CloudOperationCompletionService::mark($event, 'late'); return 'late'; } if (!CloudOperationCompletionService::isExactStoredEvent( $event )) { return self::rejectInvalidLateReceipt( (int) $current->id, (string) $event->event_id ); } $evidenceSource = 'late_completion_receipt:' . (string) $event->event_id; $operation = self::lateReceiptOperation( $current, $event, $lateResolution ); if ($lateResolution === 'applied') { self::prepareApplied( (int) $current->id, $operation, $evidenceSource ); } else { self::prepareNotApplied( (int) $current->id, $operation, $evidenceSource ); } // A terminal success receipt means the Agent journal has already // committed its terminal record and no longer retains its // resource fence. The same is true for an exact no-effect failed // receipt. No Agent request is needed for this receipt-only path. if ($lateResolution === 'applied') { self::commitApplied( (int) $current->id, $operation, $evidenceSource, null, $event ); } else { self::commitNotApplied( (int) $current->id, $operation, $evidenceSource, null, $event ); } } catch (CloudRecoveryEvidenceConflict $error) { try { return self::parkLateReceiptEvidenceConflict( (int) $current->id, (string) $event->event_id ); } catch (\Throwable $parkingError) { error_log(sprintf( 'cloud late completion conflict parking pending job_id=%d kind=%s error=%s', (int) $current->id, (string) $current->kind, get_class($parkingError) )); return 'received'; } } catch (CloudRecoveryReceiptInvalid $error) { return self::isolateLateReceiptValidationFailure( $current, $event ); } catch (CloudError $error) { if ($error->httpStatus === 502) { return self::isolateLateReceiptValidationFailure( $current, $event ); } return self::retryOrParkLateReceiptFailure( $current, $event, $error ); } catch (\Throwable $error) { return self::retryOrParkLateReceiptFailure( $current, $event, $error ); } $reloaded = CloudAgentCompletionEvent::where( 'event_id', (string) $event->event_id )->first(); return $reloaded === null ? 'received' : (string) $reloaded->disposition; } private static function lateReceiptOperation( CloudJob $job, CloudAgentCompletionEvent $event, string $resolution ): array { if (!in_array($resolution, ['applied', 'not_applied'], true) || !CloudOperationRecoveryPolicy::supportsLedgerFinalizer( (string) $job->kind )) { throw new CloudRecoveryReceiptInvalid( 'Agent 完成回执不支持该任务的效果收敛', 409 ); } // Keep the exact authenticated Agent operation. Every supported kind's // existing worker finalizer validates the result fields it consumes; // replacing those fields with a settings-shaped synthetic result would // discard snapshot, network, firewall and catalog ledger evidence. return CloudOperationCompletionService::operation($event); } private static function acknowledgeReconciledLateReceipt( int $jobId, string $eventId ): string { return (new CloudJob())->getConnection()->transaction( static function () use ($jobId, $eventId): string { $job = CloudJob::where('id', $jobId) ->lockForUpdate()->first(); $event = CloudAgentCompletionEvent::where( 'event_id', $eventId )->lockForUpdate()->first(); if ($event === null) { return 'received'; } $disposition = (string) $event->disposition; if ($disposition !== 'received') { return $disposition; } if ($job !== null && self::sameOperation($job, $event) && CloudDispatchLedger::isInvalid($job)) { self::holdDispatchLedgerConflictLocked($job); return 'received'; } if ($job === null || !self::sameOperation($job, $event) || !CloudOperationCompletionService::isExactStoredEvent( $event )) { $event->disposition = 'rejected'; $event->applied_at = null; $event->save(); return 'rejected'; } if (!self::wasReconciledBy($job, $event)) { // Another recovery source may have committed after the // caller loaded its snapshot. Once this job is no longer // eligible for the immutable receipt, terminate the event // instead of leaving an unclaimable "received" row. if (self::retryableCreateCompletion($job, $event)) { // Create uses dedicated financial finalizers. Keep // either exact outcome retryable when PHP could not // finish inspecting or settling its ledger. An // incomplete failure is ACKed late by the dedicated // finalizer only after those local checks succeed. return 'received'; } if (self::lateReceiptResolution($job, $event) === null) { $event->disposition = 'late'; $event->applied_at = null; $event->save(); return 'late'; } return 'received'; } $event->disposition = 'applied'; $event->applied_at = time(); $event->save(); return 'applied'; }, 3 ); } private static function rejectInvalidLateReceipt( int $jobId, string $eventId ): string { return (new CloudJob())->getConnection()->transaction( static function () use ($jobId, $eventId): string { CloudJob::where('id', $jobId)->lockForUpdate()->first(); $event = CloudAgentCompletionEvent::where( 'event_id', $eventId )->lockForUpdate()->first(); if ($event === null) { return 'rejected'; } if ((string) $event->disposition === 'received') { $event->disposition = 'rejected'; $event->applied_at = null; $event->save(); } return (string) $event->disposition; }, 3 ); } /** * Isolate a receipt which contradicts already prepared durable evidence. * The job itself remains scheduled for fresh inspection; only this exact * conflicting event leaves the recovery queue. */ private static function parkLateReceiptEvidenceConflict( int $jobId, string $eventId ): string { return self::parkLateReceiptIncident( $jobId, $eventId, 'recovery_evidence_conflict', true ); } private static function parkLateReceiptValidationFailure( int $jobId, string $eventId ): string { return self::parkLateReceiptIncident( $jobId, $eventId, 'recovery_receipt_invalid', false ); } private static function isolateLateReceiptValidationFailure( CloudJob $job, CloudAgentCompletionEvent $event ): string { try { $repaired = self::acknowledgeReconciledLateReceipt( (int) $job->id, (string) $event->event_id ); if ($repaired !== 'received') { return $repaired; } return self::parkLateReceiptValidationFailure( (int) $job->id, (string) $event->event_id ); } catch (\Throwable $parkingError) { error_log(sprintf( 'cloud late completion invalid receipt parking pending job_id=%d kind=%s error=%s', (int) $job->id, (string) $job->kind, get_class($parkingError) )); return 'received'; } } private static function retryOrParkLateReceiptFailure( CloudJob $job, CloudAgentCompletionEvent $event, \Throwable $error ): string { try { $repaired = self::acknowledgeReconciledLateReceipt( (int) $job->id, (string) $event->event_id ); if ($repaired !== 'received') { return $repaired; } } catch (\Throwable $parkingError) { error_log(sprintf( 'cloud late completion retry parking pending job_id=%d kind=%s error=%s', (int) $job->id, (string) $job->kind, get_class($parkingError) )); return 'received'; } // Keep the immutable receipt in "received" without an age limit so a // retransmission or later worker can retry PHP-only ledger convergence. // The original Agent mutation is never dispatched again. error_log(sprintf( 'cloud late completion recovery pending job_id=%d kind=%s error=%s', (int) $job->id, (string) $job->kind, get_class($error) )); return 'received'; } private static function parkLateReceiptIncident( int $jobId, string $eventId, string $errorCode, bool $requireEvidenceConflict ): string { return (new CloudJob())->getConnection()->transaction( static function () use ( $jobId, $eventId, $errorCode, $requireEvidenceConflict ): string { $job = CloudJob::where('id', $jobId) ->lockForUpdate()->first(); $event = CloudAgentCompletionEvent::where( 'event_id', $eventId )->lockForUpdate()->first(); if ($event === null) { return 'received'; } $disposition = (string) $event->disposition; if ($disposition !== 'received') { return $disposition; } if ($job !== null && self::sameOperation($job, $event) && CloudDispatchLedger::isInvalid($job)) { self::holdDispatchLedgerConflictLocked($job); return 'received'; } if ($job === null || !self::sameOperation($job, $event) || !CloudOperationCompletionService::isExactStoredEvent( $event )) { $event->disposition = 'rejected'; $event->applied_at = null; $event->save(); return 'rejected'; } if (self::wasReconciledBy($job, $event)) { $event->disposition = 'applied'; $event->applied_at = time(); $event->save(); return 'applied'; } $resolution = self::lateReceiptResolution($job, $event); if ($resolution === null) { $event->disposition = 'late'; $event->applied_at = null; $event->save(); return 'late'; } if ($requireEvidenceConflict) { $source = 'late_completion_receipt:' . (string) $event->event_id; $operation = self::lateReceiptOperation( $job, $event, $resolution ); $prepared = self::preparedMarker( $operation, $source ); try { self::transitionPreparedMarker( (string) $job->effect_resolution_reason, $prepared, $source, $resolution, self::legacyCompletionPreparedMarker( $operation, $source ) ); // The conflict disappeared before both rows were // locked. Leave the receipt retryable rather than // inventing an incident from a stale snapshot. return 'received'; } catch (CloudRecoveryEvidenceConflict) { // The typed conflict was reproduced under both locks. } } if (!hash_equals( (string) ($job->finalization_error_code ?? ''), $errorCode )) { $job->read_at = null; } $job->finalization_incident = 1; $job->finalization_error_code = $errorCode; $job->lease_token = null; $job->lease_until = 0; $now = time(); $job->available_at = $now + self::INCIDENT_RETRY_SECONDS; $job->updated_at = $now; $job->save(); $event->disposition = 'late'; $event->applied_at = null; $event->save(); return 'late'; }, 3 ); } private static function lockLateReceiptEvent( CloudJob $job, ?CloudAgentCompletionEvent $event, string $source ): ?CloudAgentCompletionEvent { $latePrefix = 'late_completion_receipt:'; if ($event === null) { if (str_starts_with($source, $latePrefix)) { throw new CloudError('晚到回执提交缺少事件记录', 409); } return null; } if (!str_starts_with($source, $latePrefix) || !hash_equals( substr($source, strlen($latePrefix)), (string) $event->event_id )) { throw new CloudError('晚到回执与恢复证据来源不匹配', 409); } $locked = CloudAgentCompletionEvent::where( 'event_id', (string) $event->event_id )->lockForUpdate()->first(); if ($locked === null || !self::sameOperation($job, $locked) || !in_array( (string) $locked->disposition, ['received', 'applied'], true ) || ((string) $locked->disposition === 'received' && !CloudOperationCompletionService::isExactStoredEvent( $locked ))) { throw new CloudError('晚到回执提交记录无效', 409); } return $locked; } private static function markLateReceiptApplied( CloudJob $job, ?CloudAgentCompletionEvent $event ): void { if ($event === null) { return; } if (!self::wasReconciledBy($job, $event)) { throw new CloudError('晚到回执尚未完成效果收敛', 409); } if ((string) $event->disposition === 'received') { $event->disposition = 'applied'; $event->applied_at = time(); $event->save(); return; } if ((string) $event->disposition !== 'applied') { throw new CloudError('晚到回执已被其他流程终结', 409); } } private static function assertRecoveryCommitMayPublish( CloudJob $job, string $source, ?int $resolvedBy, ?CloudAgentCompletionEvent $completionEvent ): void { self::assertRecoveryIncidentAllowsPrepare($job); if ($completionEvent !== null || $resolvedBy !== 0 || !str_starts_with($source, 'automatic_')) { return; } // A completion receipt can arrive while this worker is outside the // database performing its bounded Agent fence call. Serialize again // on job -> event before publishing: the immutable receipt must win // the next recovery turn, where compatibility is checked explicitly. $pending = CloudAgentCompletionEvent::where( 'host_id', (int) $job->host_id )->where( 'operation_id', (string) $job->operation_id )->where('disposition', 'received') ->lockForUpdate()->first(); if ($pending !== null) { throw new CloudError('晚到完成回执正在等待优先收敛', 409); } } /** Fail closed on immutable generation damage at the locked publish seam. */ private static function assertLockedAgentRecoveryEnvelope( CloudJob $job ): void { CloudDispatchLedger::assertValid($job); if (CloudJobService::hasAgentRecoveryIdentityConflict($job)) { throw new CloudRecoveryOperationIdentityConflict( '任务不可变 Agent 请求身份异常', 409 ); } if (CloudJobService::hasAgentRecoveryEnvelopeConflict($job)) { throw new CloudRecoveryInstanceStateConflict( '任务不可变 Agent 派发信封状态异常', 409 ); } } public static function assertRecoveryIncidentAllowsPrepare( CloudJob $job ): void { // Recovery incidents are operator-visible diagnostics, not terminal // queue states. Exact marker/evidence comparisons below still prevent // conflicting conclusions from publishing, while a later fresh Agent // inspection remains free to converge repaired host or panel state. } private static function sameOperation( CloudJob $job, CloudAgentCompletionEvent $event ): bool { return (string) $job->execution_mode === 'agent' && $job->host_id !== null && (int) $job->host_id === (int) $event->host_id && is_string($job->operation_id) && is_string($event->operation_id) && hash_equals( (string) $job->operation_id, (string) $event->operation_id ); } private static function lateReceiptResolution( CloudJob $job, CloudAgentCompletionEvent $event ): ?string { if (!self::recoverableUnknownEffect($job) || !CloudOperationRecoveryPolicy::supportsLedgerFinalizer( (string) $job->kind )) { return null; } if ((string) $event->outcome === 'succeeded' && !(bool) $event->effect_may_have_applied && !(bool) $event->safe_to_resubmit) { return 'applied'; } if ((string) $event->outcome === 'failed' && !(bool) $event->effect_may_have_applied) { // safe_to_resubmit belongs to the completed Agent generation. It // never authorizes replay here; both true and false settle only // this original immutable job as not-applied. return 'not_applied'; } return null; } private static function retryableCreateCompletion( CloudJob $job, CloudAgentCompletionEvent $event ): bool { return (string) $job->kind === 'create' && (string) $job->execution_mode === 'agent' && (string) $job->phase === 'terminal' && (string) $job->outcome === 'failed' && (string) $job->status === 'failed' && (int) $job->dispatch_attempts === 1 && (bool) $job->effect_may_have_applied && !(bool) $job->safe_to_resubmit && (string) $job->effect_resolution === 'unknown' && ((string) $event->outcome === 'failed' || ((string) $event->outcome === 'succeeded' && !(bool) $event->effect_may_have_applied && !(bool) $event->safe_to_resubmit)); } private static function assertResolution(string $resolution): void { if (!in_array($resolution, ['applied', 'not_applied'], true)) { throw new \LogicException('Unsupported recovery resolution.'); } } private static function assertRecoverableEvidence( CloudJob $job, array $operation, string $evidenceSource, string $resolution ): void { self::assertResolution($resolution); if ($resolution === 'applied') { self::assertRecoverableAppliedEvidence( $job, $operation, $evidenceSource ); return; } self::assertRecoverableNotAppliedEvidence( $job, $operation, $evidenceSource ); } private static function assertOperationIdentity( CloudJob $job, array $operation, string $resolution ): void { self::assertResolution($resolution); if ($resolution === 'applied') { self::assertAppliedOperationIdentity($job, $operation); return; } self::assertNotAppliedOperationIdentity($job, $operation); } private static function assertRecoverableAppliedEvidence( CloudJob $job, array $operation, string $evidenceSource ): void { self::assertEvidenceSource($evidenceSource); self::assertAppliedOperationIdentity($job, $operation); if (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer( (string) $job->kind ) || !self::recoverableUnknownEffect($job)) { throw new CloudError('任务效果恢复证据不满足安全策略', 409); } } private static function assertRecoverableNotAppliedEvidence( CloudJob $job, array $operation, string $evidenceSource ): void { self::assertEvidenceSource($evidenceSource); self::assertNotAppliedOperationIdentity($job, $operation); if (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer( (string) $job->kind ) || !self::recoverableUnknownEffect($job)) { throw new CloudError('任务效果恢复证据不满足安全策略', 409); } } private static function recoverableUnknownEffect(CloudJob $job): bool { return (string) $job->execution_mode === 'agent' && (string) $job->phase === 'terminal' && (string) $job->outcome === 'failed' && (string) $job->status === 'failed' && (int) $job->dispatch_attempts === 1 && (bool) $job->effect_may_have_applied && !(bool) $job->safe_to_resubmit && (string) $job->effect_resolution === 'unknown'; } private static function assertAppliedOperationIdentity( CloudJob $job, array $operation ): void { $operationId = $operation['operation_id'] ?? null; if (!is_string($operationId) || !is_string($job->operation_id) || !hash_equals((string) $job->operation_id, $operationId) || ($operation['status'] ?? null) !== 'succeeded' || ($operation['effect_may_have_applied'] ?? null) !== false || ($operation['safe_to_resubmit'] ?? null) !== false) { throw new CloudError('任务效果恢复证据与原操作不匹配', 409); } self::assertOperationBinding($job, $operation, true); } private static function assertNotAppliedOperationIdentity( CloudJob $job, array $operation ): void { $operationId = $operation['operation_id'] ?? null; $errorCode = $operation['error_code'] ?? null; $result = $operation['result'] ?? null; if (!is_string($operationId) || !is_string($job->operation_id) || !hash_equals((string) $job->operation_id, $operationId) || ($operation['status'] ?? null) !== 'failed' || ($operation['effect_may_have_applied'] ?? null) !== false || !is_bool($operation['safe_to_resubmit'] ?? null) || !is_string($errorCode) || preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $errorCode) !== 1 || ($result !== null && !is_array($result))) { throw new CloudError('任务效果恢复证据与原操作不匹配', 409); } self::assertOperationBinding($job, $operation, false); } private static function assertOperationBinding( CloudJob $job, array $operation, bool $forAppliedEffect ): void { $expected = CloudOperationRecoveryIdentity::expected( $job, $forAppliedEffect ); CloudOperationRecoveryIdentity::assertMatches($expected, [ 'action' => $operation['action'] ?? null, 'resource_id' => $operation['resource_id'] ?? null, 'request_fingerprint' => $operation['request_fingerprint'] ?? $operation['fingerprint'] ?? null, ]); } private static function assertEvidenceSource(string $source): void { if (preg_match( '/^[a-z][a-z0-9_]{2,63}:[A-Za-z0-9_.-]{1,160}$/D', $source ) !== 1) { throw new CloudError('任务效果恢复证据来源无效', 409); } } private static function assertResolvedBy(?int $resolvedBy): void { if ($resolvedBy !== null && $resolvedBy < 0) { throw new CloudError('任务效果恢复审计主体无效', 409); } } private static function preparedMarker( array $operation, string $source ): string { self::assertEvidenceSource($source); return 'prepared:' . $source . ':' . self::preparedOperationHash($operation); } /** Hash the exact normalized operation bound into a prepared marker. */ public static function preparedOperationHash(array $operation): string { $canonical = self::canonicalize( self::recoveryCriticalOperation($operation) ); return hash( 'sha256', json_encode( $canonical, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE ) ); } /** Hash the sole legacy omission accepted from terminal completion wire. */ public static function legacyCompletionOperationHash( array $operation ): ?string { if (($operation['retryable'] ?? null) !== false || !in_array( $operation['status'] ?? null, ['succeeded', 'failed'], true )) { return null; } unset($operation['retryable']); return self::preparedOperationHash($operation); } /** * Preserve the durable automatic-attempt envelope while replacing its * pending payload with the exact prepared evidence. An immutable late * completion may replace only a still-pending legacy envelope; it may * never overwrite a different prepared conclusion. */ private static function transitionPreparedMarker( string $current, string $prepared, string $source, string $resolution, ?string $legacyCompatiblePrepared = null ): string { if ($current === '' || hash_equals($prepared, $current)) { return $prepared; } if ($legacyCompatiblePrepared !== null && hash_equals($legacyCompatiblePrepared, $current)) { return $prepared; } $state = self::automaticRecoveryState($current); if ($state === null) { self::throwPreparedEvidenceConflict($source); } if (hash_equals($prepared, $state['payload'])) { return $current; } if ($legacyCompatiblePrepared !== null && hash_equals( $legacyCompatiblePrepared, $state['payload'] )) { return $prepared; } if ($state['payload'] !== 'pending') { $existing = []; $late = []; $legacy = []; $journalStatus = $resolution === 'applied' ? 'succeeded' : 'failed'; $compatibleOperationHash = false; if ($legacyCompatiblePrepared !== null && preg_match( '/^prepared:late_completion_receipt:' . '[A-Za-z0-9_.-]{1,160}:([a-f0-9]{64})$/D', $legacyCompatiblePrepared, $legacy ) === 1) { $compatibleOperationHash = $legacy[1]; } if (str_starts_with($source, 'late_completion_receipt:') && preg_match( '/^prepared:automatic_' . preg_quote($resolution, '/') . '_' . $journalStatus . '_nofence:[a-f0-9]{64}:([a-f0-9]{64})$/D', $state['payload'], $existing ) === 1 && preg_match( '/^prepared:late_completion_receipt:[A-Za-z0-9_.-]{1,160}:([a-f0-9]{64})$/D', $prepared, $late ) === 1 && (hash_equals($existing[1], $late[1]) || (is_string($compatibleOperationHash) && hash_equals( $existing[1], $compatibleOperationHash )))) { return $prepared; } self::throwPreparedEvidenceConflict($source); } if (str_starts_with($source, 'automatic_')) { return sprintf( 'automatic_recovery:v1:%d:%d:%s', time(), 1, $prepared ); } if (str_starts_with($source, 'late_completion_receipt:')) { return $prepared; } throw new CloudError('任务已有不同的效果恢复证据', 409); } private static function throwPreparedEvidenceConflict( string $source ): never { if (str_starts_with($source, 'late_completion_receipt:')) { throw new CloudRecoveryEvidenceConflict( '任务已有冲突的效果恢复证据', 409 ); } throw new CloudError('任务已有不同的效果恢复证据', 409); } private static function preparedMarkerMatches( string $current, string $prepared ): bool { if (hash_equals($prepared, $current)) { return true; } $state = self::automaticRecoveryState($current); return $state !== null && hash_equals($prepared, $state['payload']); } /** @return array{first_at:int,attempts:int,payload:string}|null */ private static function automaticRecoveryState(string $marker): ?array { if (preg_match( '/^automatic_recovery:v1:([1-9][0-9]*):([1-9][0-9]*):(pending|prepared:(?:automatic_(?:applied|not_applied)_(?:succeeded|failed)_(?:fence|nofence):[a-f0-9]{64}|late_completion_receipt:[A-Za-z0-9_.-]{1,160}):[a-f0-9]{64})$/D', $marker, $matches ) !== 1) { return null; } return [ 'first_at' => (int) $matches[1], 'attempts' => (int) $matches[2], 'payload' => $matches[3], ]; } private static function finalizedRecoveryPayload(CloudJob $job): string { $payload = CloudSecrets::decrypt( (string) $job->sealed_payload, 'job-payload' ); $identity = CloudOperationRecoveryIdentity::expected( $job, (string) $job->effect_resolution === 'applied' ); // The prepared operation is retained only across the PHP prepare -> // Agent fence -> PHP commit crash window. It must never survive in the // ordinary terminal audit payload. unset($payload['_operation_recovery']); $payload[CloudOperationRecoveryIdentity::payloadKey()] = CloudOperationRecoveryIdentity::persisted($identity); return CloudSecrets::encrypt( CloudInput::safeResult($payload), 'job-payload' ); } private static function recoveryCriticalOperation(array $operation): array { // The normalized operation is bounded by the completion/inspection // schemas. Hash every allow-listed result field because different kinds // consume different immutable evidence (snapshot keys, policy digests, // catalog generation identifiers, deletion proofs, and so on). return $operation; } /** * Completion receipts before retryable was restored at the PHP adapter * produced prepared hashes without that canonical terminal field. Accept * only that one historical omission while upgrading the durable marker to * the fully bound operation; every other field remains hash-identical. */ private static function legacyCompletionPreparedMarker( array $operation, string $source ): ?string { if (!str_starts_with($source, 'late_completion_receipt:')) { return null; } $legacyHash = self::legacyCompletionOperationHash($operation); return $legacyHash === null ? null : 'prepared:' . $source . ':' . $legacyHash; } private static function canonicalize($value) { if (!is_array($value)) { return $value; } if (!array_is_list($value)) { ksort($value, SORT_STRING); } foreach ($value as $key => $item) { $value[$key] = self::canonicalize($item); } return $value; } private static function wasReconciledBy( CloudJob $job, CloudAgentCompletionEvent $event ): bool { return CloudDispatchLedger::isDispatched($job) && (string) $job->phase === 'terminal' && (string) $job->outcome === 'failed' && in_array((string) $job->effect_resolution, ['applied', 'not_applied'], true) && hash_equals( 'late_completion_receipt:' . (string) $event->event_id, (string) $job->effect_resolution_reason ); } private static function holdDispatchLedgerConflictLocked( CloudJob $job ): void { $now = time(); if ((string) $job->phase === 'terminal' && (string) $job->outcome === 'failed') { $job->effect_may_have_applied = 1; $job->safe_to_resubmit = 0; $job->effect_resolution = 'unknown'; } if (!hash_equals( (string) ($job->finalization_error_code ?? ''), CloudDispatchLedger::CONFLICT_CODE )) { $job->read_at = null; } $job->finalization_incident = 1; $job->finalization_error_code = CloudDispatchLedger::CONFLICT_CODE; $job->lease_token = null; $job->lease_until = 0; $job->available_at = $now + self::INCIDENT_RETRY_SECONDS; $job->updated_at = $now; $job->save(); } }