getConnection()->transaction( static function () use ($source, $sourceEvent): string { [$user, $order, $job] = self::lockLedger($source); $event = CloudAgentCompletionEvent::where( 'event_id', (string) $sourceEvent->event_id )->lockForUpdate()->first(); if ($event === null) { return 'received'; } if ((string) $event->disposition !== 'received') { return (string) $event->disposition; } if ((int) $event->host_id !== (int) $job->host_id || !hash_equals( (string) $job->operation_id, (string) $event->operation_id )) { throw new CloudRecoveryOperationIdentityConflict( '开通失败回执与原任务不匹配', 409 ); } CloudOperationCompletionService::assertEventIdentityForJob( $job, $event ); $operation = CloudOperationCompletionService::operation($event); if (self::settledNotApplied($job, $order, $event)) { // A fresh inspection may have completed the atomic PHP // finalizer before the Agent could deliver its immutable // failure. The receipt no longer decides the ledger, but // the exact identity and committed audit allow it to be // ACKed without reopening or rewriting that resolution. $event->disposition = 'applied'; $event->applied_at = time(); $event->save(); return 'applied'; } self::assertRecoverableJob($job); if (!self::receiptProvesAbsence($event, $operation)) { $event->disposition = 'late'; $event->applied_at = null; $event->save(); return 'late'; } self::assertNoSuccessfulReceiptLocked($job); self::finalizeLocked( $job, $user, $order, 'late_completion_receipt:' . (string) $event->event_id, $event ); return 'applied'; }, 3 ); } /** Recover one lease-owned create without replaying its Agent mutation. */ public static function recover(CloudJob $claimed): void { $job = self::claimedJob($claimed); $prepared = self::preparedEvidence($job); if ($prepared !== null) { if (!$prepared['fence_resolution_required']) { self::finishPrepared($claimed, $prepared, false); return; } try { self::finishPrepared($claimed, $prepared, null); return; } catch (CloudAgentError $error) { if ($error->agentHttpStatus !== 404) { throw $error; } // No fence verdict was committed before the prior PHP crash. // A new write requires new, bounded read-only evidence. } } $inspection = CloudAdminOperationRecoveryService::inspectForRecovery( (string) $job->id ); $prepared = self::prepareInspection($claimed, $inspection); self::finishPrepared( $claimed, $prepared, $prepared['fence_resolution_required'] ? $inspection : false ); } /** * Authenticate the durable PHP intent before an automatic Agent fence read * or write. This is deliberately create-specific and does not broaden the * generic recovery allow-list. */ public static function assertPreparedForFence( CloudJob $job, string $resolution, string $reason ): void { if ($resolution !== 'not_applied') { throw new CloudError('开通任务只支持确认未应用', 409); } self::assertRecoverableJob($job); $prepared = self::preparedEvidence($job); if ($prepared === null || !hash_equals($prepared['reason'], $reason)) { throw new CloudRecoveryEvidenceConflict( '开通任务的恢复预提交证据不匹配', 409 ); } } /** * @return array{marker:string,reason:string,identity:array,inspection:array,fence_resolution_required:bool} */ private static function prepareInspection( CloudJob $claimed, array $inspection ): array { $job = self::claimedJob($claimed); $identity = CloudOperationRecoveryIdentity::expected($job, false); CloudOperationRecoveryIdentity::assertMatches($identity, [ 'action' => $inspection['action'] ?? null, 'resource_id' => $inspection['resource_id'] ?? null, 'request_fingerprint' => $inspection['request_fingerprint'] ?? null, ]); self::assertInspectionProvesAbsence($job, $inspection); $evidence = self::inspectionEvidence($inspection); $marker = self::inspectionMarker($evidence); $reason = self::fenceReason($job, $marker); return (new CloudJob())->getConnection()->transaction( static function () use ( $claimed, $identity, $evidence, $marker, $reason ): array { [$user, $order, $locked] = self::lockLedger($claimed); self::assertClaimedJob($locked, $claimed); self::assertNoSuccessfulReceiptLocked($locked); $payload = self::payload($locked); $lockedIdentity = CloudOperationRecoveryIdentity:: expectedFromPayload($locked, $payload, false); CloudOperationRecoveryIdentity::assertMatches( $lockedIdentity, $identity ); $payload[self::ESCROW_KEY] = [ 'version' => self::ESCROW_VERSION, 'marker' => $marker, 'reason' => $reason, 'identity' => CloudOperationRecoveryIdentity::persisted( $lockedIdentity ), 'inspection' => $evidence, ]; $encoded = json_encode( $payload, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE ); if (strlen($encoded) > 196608) { throw new CloudRecoveryPayloadUnavailable( '开通任务恢复预提交数据过大', 409 ); } $locked->effect_resolution_reason = $marker; $locked->sealed_payload = CloudSecrets::encrypt( $payload, 'job-payload' ); $locked->updated_at = time(); $locked->save(); return [ 'marker' => $marker, 'reason' => $reason, 'identity' => $lockedIdentity, 'inspection' => $evidence, 'fence_resolution_required' => (bool) $evidence['fence_resolution_required'], ]; }, 3 ); } /** * Resolve or verify the Agent fence while holding the canonical financial * locks, then commit every PHP ledger transition in that same transaction. * * @param array{marker:string,reason:string,identity:array,inspection:array,fence_resolution_required:bool} $prepared * @param array|false|null $inspection Fresh evidence, false when no fence * is required, null when resuming an already committed fence resolution. */ private static function finishPrepared( CloudJob $claimed, array $prepared, array|false|null $inspection ): void { (new CloudJob())->getConnection()->transaction( static function () use ($claimed, $prepared, $inspection): void { [$user, $order, $job] = self::lockLedger($claimed); self::assertClaimedJob($job, $claimed); $lockedPrepared = self::preparedEvidence($job); if ($lockedPrepared === null || !hash_equals( $prepared['marker'], $lockedPrepared['marker'] ) || !hash_equals( $prepared['reason'], $lockedPrepared['reason'] )) { throw new CloudRecoveryEvidenceConflict( '开通任务的恢复预提交已变化', 409 ); } self::assertNoSuccessfulReceiptLocked($job); if ($inspection === null) { CloudAdminOperationRecoveryService:: preparedFenceResolutionForRecovery( (string) $job->id, 'not_applied', $lockedPrepared['reason'] ); } elseif ($inspection !== false) { CloudAdminOperationRecoveryService::resolveFenceForRecovery( (string) $job->id, 'not_applied', $lockedPrepared['reason'], $inspection ); } elseif ($lockedPrepared['fence_resolution_required']) { throw new CloudRecoveryEvidenceConflict( '开通任务仍需解除宿主机资源栅栏', 409 ); } self::assertNoSuccessfulReceiptLocked($job); self::finalizeLocked( $job, $user, $order, 'automatic_create_not_applied:' . substr($lockedPrepared['marker'], -64) ); }, 3 ); } /** * @return array{marker:string,reason:string,identity:array,inspection:array,fence_resolution_required:bool}|null */ private static function preparedEvidence(CloudJob $job): ?array { $marker = (string) ($job->effect_resolution_reason ?? ''); if (preg_match( '/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D', $marker ) !== 1) { return null; } $payload = self::payload($job); $value = $payload[self::ESCROW_KEY] ?? null; $identity = is_array($value) ? ($value['identity'] ?? null) : null; $inspection = is_array($value) ? ($value['inspection'] ?? null) : null; if (!is_array($value) || array_is_list($value) || array_diff(array_keys($value), [ 'version', 'marker', 'reason', 'identity', 'inspection', ]) !== [] || ($value['version'] ?? null) !== self::ESCROW_VERSION || !is_string($value['marker'] ?? null) || !hash_equals($marker, $value['marker']) || !is_string($value['reason'] ?? null) || mb_strlen($value['reason']) < 20 || mb_strlen($value['reason']) > 500 || !is_array($identity) || array_is_list($identity) || !is_array($inspection) || array_is_list($inspection)) { throw new CloudRecoveryPayloadUnavailable( '开通任务恢复预提交数据无效', 409 ); } $expected = CloudOperationRecoveryIdentity::expectedFromPayload( $job, $payload, false ); CloudOperationRecoveryIdentity::assertMatches($expected, $identity); self::assertInspectionProvesAbsence($job, $inspection, false); CloudOperationRecoveryIdentity::assertMatches($expected, [ 'action' => $inspection['action'] ?? null, 'resource_id' => $inspection['resource_id'] ?? null, 'request_fingerprint' => $inspection['request_fingerprint'] ?? null, ]); $evidence = self::inspectionEvidence($inspection); $expectedMarker = self::inspectionMarker($evidence); if (!hash_equals($expectedMarker, $marker) || !hash_equals(self::fenceReason($job, $marker), $value['reason'])) { throw new CloudRecoveryEvidenceConflict( '开通任务恢复预提交摘要不匹配', 409 ); } return [ 'marker' => $marker, 'reason' => $value['reason'], 'identity' => $expected, 'inspection' => $evidence, 'fence_resolution_required' => (bool) $evidence['fence_resolution_required'], ]; } private static function assertInspectionProvesAbsence( CloudJob $job, array $inspection, bool $requireFresh = true ): void { $checkedAt = $inspection['checked_at'] ?? null; $projection = $inspection['effect_projection'] ?? null; if (($inspection['operation_id'] ?? null) !== (string) $job->operation_id || ($inspection['journal_status'] ?? null) !== 'failed' || ($inspection['outcome'] ?? null) !== 'failed' || ($inspection['summary_code'] ?? null) !== 'operation_failed' || ($inspection['effect_resolution'] ?? null) !== 'not_applied' || !is_int($checkedAt) || ($requireFresh && ($checkedAt < time() - self::INSPECTION_MAX_AGE_SECONDS || $checkedAt > time() + 5)) || !is_array($projection) || array_is_list($projection) || ($projection['creation_absent'] ?? null) !== true || ($projection['managed_artifacts_absent'] ?? null) !== true || !in_array( $inspection['fence_state'] ?? null, ['not_required', 'active'], true ) || !is_bool($inspection['fence_resolution_required'] ?? null) || (bool) $inspection['fence_resolution_required'] !== (($inspection['fence_state'] ?? null) === 'active') || array_key_exists('completion_delivery_state', $inspection)) { throw new CloudRecoveryEvidenceConflict( '宿主机检查没有同时确认虚拟机及面板管理资源均不存在', 409 ); } CloudCreateFailureFinalizer::assertResolutionSupported( $job, 'not_applied', $inspection ); } private static function receiptProvesAbsence( CloudAgentCompletionEvent $event, array $operation ): bool { $result = $operation['result'] ?? null; return (string) $event->outcome === 'failed' && !(bool) $event->effect_may_have_applied && is_array($result) && !array_is_list($result) && ($result['creation_absent'] ?? null) === true && ($result['managed_artifacts_absent'] ?? null) === true; } /** @return array{0:User,1:CloudOrder,2:CloudJob} */ private static function lockLedger(CloudJob $source): array { $user = User::where('id', (int) $source->user_id) ->lockForUpdate()->first(); $order = CloudOrder::where('id', (int) $source->order_id) ->lockForUpdate()->first(); $job = CloudJob::where('id', (int) $source->id) ->lockForUpdate()->first(); if ($user === null || $order === null || $job === null || (int) $job->user_id !== (int) $user->id || (int) $job->order_id !== (int) $order->id || (int) $order->user_id !== (int) $user->id) { throw new CloudRecoveryInstanceStateConflict( '开通任务与资金账本关联不一致', 409 ); } return [$user, $order, $job]; } private static function assertRecoverableJob(CloudJob $job): void { self::assertDispatchedEnvelope($job); if ((string) $job->kind !== 'create' || $job->order_id === null || (string) $job->execution_mode !== 'agent' || (string) $job->phase !== 'terminal' || (string) $job->outcome !== 'failed' || (string) $job->status !== 'failed' || !(bool) $job->effect_may_have_applied || (bool) $job->safe_to_resubmit || (string) $job->effect_resolution !== 'unknown') { throw new CloudRecoveryInstanceStateConflict( '开通任务不处于可自动核对状态', 409 ); } } private static function settledNotApplied( CloudJob $job, CloudOrder $order, CloudAgentCompletionEvent $event ): bool { self::assertDispatchedEnvelope($job); $reason = $job->effect_resolution_reason; $completionEventId = $job->completion_event_id; return (string) $event->outcome === 'failed' && (string) $job->kind === 'create' && $job->order_id !== null && (string) $job->execution_mode === 'agent' && (string) $job->phase === 'terminal' && (string) $job->outcome === 'failed' && (string) $job->status === 'failed' && (int) $job->dispatch_attempts === 1 && !(bool) $job->safe_to_resubmit && (string) $job->effect_resolution === 'not_applied' && (int) ($job->effect_resolved_at ?? 0) > 0 && is_string($reason) && mb_strlen($reason) >= 20 && mb_strlen($reason) <= 500 && !(bool) $job->finalization_incident && $job->finalization_error_code === null && $job->instance_id === null && $order->instance_id === null && (string) $order->status === 'failed' && $order->refunded_at !== null && ($completionEventId === null || (is_string($completionEventId) && hash_equals( $completionEventId, (string) $event->event_id ))); } /** Validate create's immutable generation without broadening policy. */ private static function assertDispatchedEnvelope(CloudJob $job): void { CloudDispatchLedger::assertDispatched($job); if ((string) $job->attempts !== '1' || (int) ($job->host_id ?? 0) < 1 || (int) ($job->dispatch_deadline_at ?? 0) < 1 || (int) ($job->dispatch_started_at ?? 0) < 1 || (($job->agent_accepted_at ?? null) !== null && (int) $job->agent_accepted_at < 1) || (int) ($job->dispatched_at ?? 0) < 1 || (int) ($job->receipt_deadline_at ?? 0) < 1 || (int) ($job->terminal_at ?? 0) < 1 || preg_match( '/^[a-f0-9]{32}$/D', (string) ($job->operation_id ?? '') ) !== 1 || preg_match( '/^[a-f0-9]{64}$/D', (string) ($job->request_fingerprint ?? '') ) !== 1 || preg_match( '/^[A-Za-z0-9_.-]{16,128}$/D', (string) ($job->request_idempotency_key ?? '') ) !== 1) { throw new CloudRecoveryInstanceStateConflict( '开通任务单次派发信封不完整', 409 ); } } private static function assertClaimedJob( CloudJob $job, CloudJob $claimed ): void { self::assertRecoverableJob($job); if (!is_string($claimed->lease_token) || !is_string($job->lease_token) || !hash_equals( (string) $claimed->lease_token, (string) $job->lease_token ) || (int) $job->lease_until <= time()) { throw new CloudRecoveryInstanceStateConflict( '开通任务自动核对租约已失效', 409 ); } } private static function claimedJob(CloudJob $claimed): CloudJob { $job = CloudJob::find((int) $claimed->id); if ($job === null) { throw new CloudError('任务不存在', 404); } self::assertClaimedJob($job, $claimed); return $job; } private static function assertNoSuccessfulReceiptLocked(CloudJob $job): void { $event = CloudAgentCompletionEvent::where( 'host_id', (int) $job->host_id )->where( 'operation_id', (string) $job->operation_id )->where('outcome', 'succeeded')->lockForUpdate()->first(); if ($event === null) { return; } $exact = CloudOperationCompletionService::isExactStoredEvent($event) || CloudOperationCompletionService::isExactAppliedEvent($event); if (!$exact) { throw new CloudRecoveryOperationIdentityConflict( '开通成功回执状态异常,不能执行退款', 409 ); } $payload = self::payload($job); $identity = CloudOperationRecoveryIdentity::expectedFromPayload( $job, $payload, true ); if (CloudOperationRecoveryIdentity::same( $identity, CloudOperationCompletionService::identity($event) )) { throw new CloudRecoveryEvidenceConflict( '已收到匹配的实例创建成功回执,不能按未应用退款', 409 ); } throw new CloudRecoveryOperationIdentityConflict( '开通成功回执身份冲突,不能执行退款', 409 ); } private static function finalizeLocked( CloudJob $job, User $user, CloudOrder $order, string $resolutionReason, ?CloudAgentCompletionEvent $event = null ): void { $payload = self::payload($job); $identity = CloudOperationRecoveryIdentity::expectedFromPayload( $job, $payload, false ); CloudCreateFailureFinalizer::refundAndReleaseLocked( $job, $user, $order, 'create_effect_not_applied' ); $safePayload = CloudInput::safeResult($payload); unset($safePayload[self::ESCROW_KEY]); $safePayload[CloudOperationRecoveryIdentity::payloadKey()] = CloudOperationRecoveryIdentity::persisted($identity); $now = time(); $job->effect_resolution = 'not_applied'; $job->effect_resolved_at = $now; $job->effect_resolved_by = null; $job->effect_resolution_reason = $resolutionReason; if ($event !== null) { $job->completion_event_id = (string) $event->event_id; } $job->sealed_payload = CloudSecrets::encrypt( $safePayload, 'job-payload' ); $job->finalization_incident = 0; $job->finalization_error_code = null; $job->read_at = null; $job->lease_token = null; $job->lease_until = 0; $job->available_at = 0; $job->updated_at = $now; $job->save(); if ($event !== null) { $event->disposition = 'applied'; $event->applied_at = $now; $event->save(); } } private static function payload(CloudJob $job): array { try { $payload = CloudSecrets::decrypt( (string) $job->sealed_payload, 'job-payload' ); } catch (\Throwable $error) { throw new CloudRecoveryPayloadUnavailable( '开通任务恢复数据无法解密', 409 ); } if (!is_array($payload) || array_is_list($payload)) { throw new CloudRecoveryPayloadUnavailable( '开通任务恢复数据结构不完整', 409 ); } return $payload; } private static function fenceReason(CloudJob $job, string $marker): string { return '自动恢复已确认原开通操作未创建虚拟机或面板管理资源;operation_id=' . (string) $job->operation_id . ';evidence=' . substr($marker, -64); } /** Hash only stable evidence so a lost fence response can be resumed. */ private static function inspectionMarker(array $evidence): string { unset($evidence['checked_at']); return self::MARKER_PREFIX . hash( 'sha256', json_encode( $evidence, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE ) ); } /** Return the bounded, canonical evidence persisted in encrypted escrow. */ private static function inspectionEvidence(array $inspection): array { return [ 'operation_id' => $inspection['operation_id'], 'action' => $inspection['action'], 'resource_id' => $inspection['resource_id'], 'request_fingerprint' => $inspection['request_fingerprint'], 'journal_status' => $inspection['journal_status'], 'checked_at' => $inspection['checked_at'], 'outcome' => $inspection['outcome'], 'summary_code' => $inspection['summary_code'], 'error_code' => $inspection['error_code'] ?? '', 'effect_resolution' => $inspection['effect_resolution'], 'fence_state' => $inspection['fence_state'], 'fence_resolution_required' => $inspection['fence_resolution_required'], 'effect_projection' => [ 'creation_absent' => true, 'managed_artifacts_absent' => true, ], ]; } }