You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudOperationRecoveryServi...

1410 lines
51 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudAgentCompletionEvent;
use App\Models\CloudJob;
use App\Models\User;
/**
* Converges a terminal task's unresolved host effect without replaying it.
*
* The original job remains an immutable failed audit record. Only an exact
* immutable Agent completion receipt or a freshly validated, operation-bound
* read-only inspection for an explicitly registered kind may update PHP's
* resource ledger and release the effect fence.
*/
final class CloudOperationRecoveryService
{
private const INCIDENT_RETRY_SECONDS = 300;
/**
* Apply the kind-specific PHP ledger finalizer while the job remains
* effect_resolution=unknown. The returned job therefore still blocks
* conflicting work until the caller has released the matching Agent
* resource fence and invokes commitApplied().
*
* This method performs PHP-only convergence and never contacts an Agent.
*/
public static function prepareApplied(
int $jobId,
array $operation,
string $evidenceSource
): CloudJob {
return self::prepareResolution(
$jobId,
$operation,
$evidenceSource,
'applied'
);
}
/**
* Publish the prepared resolution and erase the encrypted recovery escrow.
* The caller must release the matching Agent fence before this commit.
*/
public static function commitApplied(
int $jobId,
array $operation,
string $evidenceSource,
?int $resolvedBy = null,
?CloudAgentCompletionEvent $completionEvent = null
): CloudJob {
return self::commitResolution(
$jobId,
$operation,
$evidenceSource,
'applied',
$resolvedBy,
$completionEvent
);
}
/**
* Roll back the kind-specific PHP pending ledger and persist a no-effect
* intent while the original unknown-effect blocker remains active.
*/
public static function prepareNotApplied(
int $jobId,
array $operation,
string $evidenceSource
): CloudJob {
// A proved no-effect operation must still drive the original failed
// finalizer. Network reservations, snapshots, firewall revisions and
// catalog replicas all have PHP-owned pending/uncertain ledger state
// which only that finalizer can roll back safely.
return self::prepareResolution(
$jobId,
$operation,
$evidenceSource,
'not_applied'
);
}
/**
* Publish a prepared no-effect conclusion after the matching Agent fence
* has been durably released, then erase the encrypted recovery escrow.
*/
public static function commitNotApplied(
int $jobId,
array $operation,
string $evidenceSource,
?int $resolvedBy = null,
?CloudAgentCompletionEvent $completionEvent = null
): CloudJob {
return self::commitResolution(
$jobId,
$operation,
$evidenceSource,
'not_applied',
$resolvedBy,
$completionEvent
);
}
/** Prepare one validated resolution without publishing its effect. */
private static function prepareResolution(
int $jobId,
array $operation,
string $evidenceSource,
string $resolution
): CloudJob {
$job = CloudJob::find($jobId);
if ($job === null) {
throw new CloudError('任务不存在', 404);
}
self::assertRecoverableEvidence(
$job,
$operation,
$evidenceSource,
$resolution
);
$prepared = self::preparedMarker($operation, $evidenceSource);
$marker = self::transitionPreparedMarker(
(string) $job->effect_resolution_reason,
$prepared,
$evidenceSource,
$resolution,
self::legacyCompletionPreparedMarker(
$operation,
$evidenceSource
)
);
return (new CloudWorkerService())->prepareRecoveredEffect(
$jobId,
$operation,
$marker
);
}
/** Publish one prepared resolution through the shared locked commit seam. */
private static function commitResolution(
int $jobId,
array $operation,
string $evidenceSource,
string $resolution,
?int $resolvedBy,
?CloudAgentCompletionEvent $completionEvent
): CloudJob {
self::assertResolution($resolution);
self::assertResolvedBy($resolvedBy);
$jobSnapshot = CloudJob::find($jobId);
$aggregateParentId = (int) ($jobSnapshot?->parent_job_id ?? 0);
$transitionUserId = $jobSnapshot instanceof CloudJob
&& (string) $jobSnapshot->kind === 'network_change'
&& $jobSnapshot->public_network_transition_id !== null
? (int) $jobSnapshot->user_id : 0;
$portableRestoreUserId = $jobSnapshot instanceof CloudJob
&& CloudPortableRestoreFinalizer::owns($jobSnapshot)
? (int) $jobSnapshot->user_id : 0;
$recoveryUserId = $transitionUserId > 0
? $transitionUserId : $portableRestoreUserId;
return (new CloudJob())->getConnection()->transaction(
function () use (
$jobId,
$operation,
$evidenceSource,
$resolution,
$resolvedBy,
$completionEvent,
$aggregateParentId,
$transitionUserId,
$portableRestoreUserId,
$recoveryUserId
): CloudJob {
$recoveryActor = $recoveryUserId > 0
? User::where('id', $recoveryUserId)
->lockForUpdate()->first()
: null;
if ($recoveryUserId > 0
&& !$recoveryActor instanceof User) {
throw new CloudError(
'任务效果恢复账户不存在',
409
);
}
$transitionActor = $transitionUserId > 0
? $recoveryActor : null;
$aggregateParent = null;
if ($aggregateParentId > 0) {
$aggregateParent = CloudJob::where(
'id',
$aggregateParentId
)
->lockForUpdate()->first();
if ($aggregateParent === null) {
throw new CloudRecoveryAggregateStateConflict(
'任务的汇总父任务不存在',
409
);
}
CloudTemplateCatalogService::lockAggregateCatalogLocked(
$aggregateParent
);
CloudUserIsoService::lockAggregateCatalogLocked(
$aggregateParent
);
}
$job = CloudJob::where('id', $jobId)
->lockForUpdate()->first();
if ($job === null) {
throw new CloudError('任务不存在', 404);
}
if ((int) ($job->parent_job_id ?? 0)
!== $aggregateParentId) {
throw new CloudRecoveryAggregateStateConflict(
'任务的汇总父级关系在提交期间发生变化',
409
);
}
if ($transitionUserId > 0
&& ((int) $job->user_id !== $transitionUserId
|| (string) $job->kind !== 'network_change'
|| $job->public_network_transition_id === null)) {
throw new CloudRecoveryInstanceStateConflict(
'公网网络变更恢复账户关联发生变化',
409
);
}
if ($portableRestoreUserId > 0
&& ((int) $job->user_id !== $portableRestoreUserId
|| !CloudPortableRestoreFinalizer::owns($job))) {
throw new CloudRecoveryInstanceStateConflict(
'备份恢复账户或尝试关联发生变化',
409
);
}
self::assertLockedAgentRecoveryEnvelope($job);
self::assertEvidenceSource($evidenceSource);
self::assertOperationIdentity(
$job,
$operation,
$resolution
);
$lockedEvent = self::lockLateReceiptEvent(
$job,
$completionEvent,
$evidenceSource
);
self::assertRecoveryCommitMayPublish(
$job,
$evidenceSource,
$resolvedBy,
$completionEvent
);
$finalReason = $evidenceSource;
if ((string) $job->effect_resolution === $resolution
&& hash_equals(
$finalReason,
(string) $job->effect_resolution_reason
)) {
self::markLateReceiptApplied($job, $lockedEvent);
return $job;
}
if ($job->public_network_transition_id !== null) {
CloudPublicNetworkTransitionWorkflow::
publishPreparedLocked(
$job,
$resolution,
$transitionActor,
$aggregateParent
);
}
if ($portableRestoreUserId > 0) {
CloudPortableRestoreFinalizer::publishPreparedLocked(
$job,
$resolution
);
}
self::assertRecoverableEvidence(
$job,
$operation,
$evidenceSource,
$resolution
);
$marker = self::preparedMarker(
$operation,
$evidenceSource
);
if (!self::preparedMarkerMatches(
(string) $job->effect_resolution_reason,
$marker
)) {
throw new CloudError('任务效果尚未完成账本预提交', 409);
}
$job->effect_resolution = $resolution;
$job->effect_resolved_at = time();
$job->effect_resolved_by = $resolvedBy;
$job->effect_resolution_reason = $finalReason;
$job->sealed_payload = self::finalizedRecoveryPayload($job);
$job->lease_token = null;
$job->lease_until = 0;
$job->available_at = 0;
$job->finalization_incident = 0;
$job->finalization_error_code = null;
$job->read_at = null;
$job->updated_at = time();
$job->save();
CloudJobService::reconcileAggregateParentEffectLocked(
$job,
$resolvedBy
);
self::markLateReceiptApplied($job, $lockedEvent);
return $job;
},
3
);
}
/**
* @return string One of the existing completion-event dispositions.
*/
public static function settleLateReceipt(
CloudJob $source,
CloudAgentCompletionEvent $event
): string {
$disposition = (string) $event->disposition;
if ($disposition !== 'received') {
return $disposition;
}
if (!self::sameOperation($source, $event)) {
return self::rejectInvalidLateReceipt(
(int) $source->id,
(string) $event->event_id
);
}
if (CloudDispatchLedger::isInvalid($source)) {
CloudOperationCompletionService::deferDispatchLedgerConflict(
$source
);
return 'received';
}
if (CloudDispatchLedger::isNeverDispatched($source)) {
CloudOperationCompletionService::
deferNeverDispatchedEvidenceConflict($source);
return 'received';
}
if (CloudDispatchLedger::hasRepairIncident($source)) {
return 'received';
}
$current = $source;
try {
$current = CloudJob::find((int) $source->id) ?? $source;
CloudOperationCompletionService::assertEventIdentityForJob(
$current,
$event
);
if (self::wasReconciledBy($current, $event)) {
return self::acknowledgeReconciledLateReceipt(
(int) $current->id,
(string) $event->event_id
);
}
if ((string) $current->kind === 'create') {
if ((string) $event->outcome === 'failed') {
return CloudCreateFailureRecoveryService::settleReceipt(
$current,
$event
);
}
if ((string) $event->outcome !== 'succeeded'
|| (bool) $event->effect_may_have_applied
|| (bool) $event->safe_to_resubmit
|| !CloudOperationCompletionService::isExactStoredEvent(
$event
)) {
CloudOperationCompletionService::mark($event, 'late');
return 'late';
}
(new CloudWorkerService())->settleRecoveredCreateCompletion(
(int) $current->id,
$event
);
$reloadedCreateEvent = CloudAgentCompletionEvent::where(
'event_id',
(string) $event->event_id
)->first();
return $reloadedCreateEvent === null
? 'received'
: (string) $reloadedCreateEvent->disposition;
}
$lateResolution = self::lateReceiptResolution($current, $event);
if ($lateResolution === null) {
CloudOperationCompletionService::mark($event, 'late');
return 'late';
}
if (!CloudOperationCompletionService::isExactStoredEvent(
$event
)) {
return self::rejectInvalidLateReceipt(
(int) $current->id,
(string) $event->event_id
);
}
$evidenceSource = 'late_completion_receipt:'
. (string) $event->event_id;
$operation = self::lateReceiptOperation(
$current,
$event,
$lateResolution
);
if ($lateResolution === 'applied') {
self::prepareApplied(
(int) $current->id,
$operation,
$evidenceSource
);
} else {
self::prepareNotApplied(
(int) $current->id,
$operation,
$evidenceSource
);
}
// A terminal success receipt means the Agent journal has already
// committed its terminal record and no longer retains its
// resource fence. The same is true for an exact no-effect failed
// receipt. No Agent request is needed for this receipt-only path.
if ($lateResolution === 'applied') {
self::commitApplied(
(int) $current->id,
$operation,
$evidenceSource,
null,
$event
);
} else {
self::commitNotApplied(
(int) $current->id,
$operation,
$evidenceSource,
null,
$event
);
}
} catch (CloudRecoveryEvidenceConflict $error) {
try {
return self::parkLateReceiptEvidenceConflict(
(int) $current->id,
(string) $event->event_id
);
} catch (\Throwable $parkingError) {
error_log(sprintf(
'cloud late completion conflict parking pending job_id=%d kind=%s error=%s',
(int) $current->id,
(string) $current->kind,
get_class($parkingError)
));
return 'received';
}
} catch (CloudRecoveryReceiptInvalid $error) {
return self::isolateLateReceiptValidationFailure(
$current,
$event
);
} catch (CloudError $error) {
if ($error->httpStatus === 502) {
return self::isolateLateReceiptValidationFailure(
$current,
$event
);
}
return self::retryOrParkLateReceiptFailure(
$current,
$event,
$error
);
} catch (\Throwable $error) {
return self::retryOrParkLateReceiptFailure(
$current,
$event,
$error
);
}
$reloaded = CloudAgentCompletionEvent::where(
'event_id',
(string) $event->event_id
)->first();
return $reloaded === null
? 'received' : (string) $reloaded->disposition;
}
private static function lateReceiptOperation(
CloudJob $job,
CloudAgentCompletionEvent $event,
string $resolution
): array {
if (!in_array($resolution, ['applied', 'not_applied'], true)
|| !CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
(string) $job->kind
)) {
throw new CloudRecoveryReceiptInvalid(
'Agent 完成回执不支持该任务的效果收敛',
409
);
}
// Keep the exact authenticated Agent operation. Every supported kind's
// existing worker finalizer validates the result fields it consumes;
// replacing those fields with a settings-shaped synthetic result would
// discard snapshot, network, firewall and catalog ledger evidence.
return CloudOperationCompletionService::operation($event);
}
private static function acknowledgeReconciledLateReceipt(
int $jobId,
string $eventId
): string {
return (new CloudJob())->getConnection()->transaction(
static function () use ($jobId, $eventId): string {
$job = CloudJob::where('id', $jobId)
->lockForUpdate()->first();
$event = CloudAgentCompletionEvent::where(
'event_id',
$eventId
)->lockForUpdate()->first();
if ($event === null) {
return 'received';
}
$disposition = (string) $event->disposition;
if ($disposition !== 'received') {
return $disposition;
}
if ($job !== null
&& self::sameOperation($job, $event)
&& CloudDispatchLedger::isInvalid($job)) {
self::holdDispatchLedgerConflictLocked($job);
return 'received';
}
if ($job === null
|| !self::sameOperation($job, $event)
|| !CloudOperationCompletionService::isExactStoredEvent(
$event
)) {
$event->disposition = 'rejected';
$event->applied_at = null;
$event->save();
return 'rejected';
}
if (!self::wasReconciledBy($job, $event)) {
// Another recovery source may have committed after the
// caller loaded its snapshot. Once this job is no longer
// eligible for the immutable receipt, terminate the event
// instead of leaving an unclaimable "received" row.
if (self::retryableCreateCompletion($job, $event)) {
// Create uses dedicated financial finalizers. Keep
// either exact outcome retryable when PHP could not
// finish inspecting or settling its ledger. An
// incomplete failure is ACKed late by the dedicated
// finalizer only after those local checks succeed.
return 'received';
}
if (self::lateReceiptResolution($job, $event) === null) {
$event->disposition = 'late';
$event->applied_at = null;
$event->save();
return 'late';
}
return 'received';
}
$event->disposition = 'applied';
$event->applied_at = time();
$event->save();
return 'applied';
},
3
);
}
private static function rejectInvalidLateReceipt(
int $jobId,
string $eventId
): string {
return (new CloudJob())->getConnection()->transaction(
static function () use ($jobId, $eventId): string {
CloudJob::where('id', $jobId)->lockForUpdate()->first();
$event = CloudAgentCompletionEvent::where(
'event_id',
$eventId
)->lockForUpdate()->first();
if ($event === null) {
return 'rejected';
}
if ((string) $event->disposition === 'received') {
$event->disposition = 'rejected';
$event->applied_at = null;
$event->save();
}
return (string) $event->disposition;
},
3
);
}
/**
* Isolate a receipt which contradicts already prepared durable evidence.
* The job itself remains scheduled for fresh inspection; only this exact
* conflicting event leaves the recovery queue.
*/
private static function parkLateReceiptEvidenceConflict(
int $jobId,
string $eventId
): string {
return self::parkLateReceiptIncident(
$jobId,
$eventId,
'recovery_evidence_conflict',
true
);
}
private static function parkLateReceiptValidationFailure(
int $jobId,
string $eventId
): string {
return self::parkLateReceiptIncident(
$jobId,
$eventId,
'recovery_receipt_invalid',
false
);
}
private static function isolateLateReceiptValidationFailure(
CloudJob $job,
CloudAgentCompletionEvent $event
): string {
try {
$repaired = self::acknowledgeReconciledLateReceipt(
(int) $job->id,
(string) $event->event_id
);
if ($repaired !== 'received') {
return $repaired;
}
return self::parkLateReceiptValidationFailure(
(int) $job->id,
(string) $event->event_id
);
} catch (\Throwable $parkingError) {
error_log(sprintf(
'cloud late completion invalid receipt parking pending job_id=%d kind=%s error=%s',
(int) $job->id,
(string) $job->kind,
get_class($parkingError)
));
return 'received';
}
}
private static function retryOrParkLateReceiptFailure(
CloudJob $job,
CloudAgentCompletionEvent $event,
\Throwable $error
): string {
try {
$repaired = self::acknowledgeReconciledLateReceipt(
(int) $job->id,
(string) $event->event_id
);
if ($repaired !== 'received') {
return $repaired;
}
} catch (\Throwable $parkingError) {
error_log(sprintf(
'cloud late completion retry parking pending job_id=%d kind=%s error=%s',
(int) $job->id,
(string) $job->kind,
get_class($parkingError)
));
return 'received';
}
// Keep the immutable receipt in "received" without an age limit so a
// retransmission or later worker can retry PHP-only ledger convergence.
// The original Agent mutation is never dispatched again.
error_log(sprintf(
'cloud late completion recovery pending job_id=%d kind=%s error=%s',
(int) $job->id,
(string) $job->kind,
get_class($error)
));
return 'received';
}
private static function parkLateReceiptIncident(
int $jobId,
string $eventId,
string $errorCode,
bool $requireEvidenceConflict
): string {
return (new CloudJob())->getConnection()->transaction(
static function () use (
$jobId,
$eventId,
$errorCode,
$requireEvidenceConflict
): string {
$job = CloudJob::where('id', $jobId)
->lockForUpdate()->first();
$event = CloudAgentCompletionEvent::where(
'event_id',
$eventId
)->lockForUpdate()->first();
if ($event === null) {
return 'received';
}
$disposition = (string) $event->disposition;
if ($disposition !== 'received') {
return $disposition;
}
if ($job !== null
&& self::sameOperation($job, $event)
&& CloudDispatchLedger::isInvalid($job)) {
self::holdDispatchLedgerConflictLocked($job);
return 'received';
}
if ($job === null
|| !self::sameOperation($job, $event)
|| !CloudOperationCompletionService::isExactStoredEvent(
$event
)) {
$event->disposition = 'rejected';
$event->applied_at = null;
$event->save();
return 'rejected';
}
if (self::wasReconciledBy($job, $event)) {
$event->disposition = 'applied';
$event->applied_at = time();
$event->save();
return 'applied';
}
$resolution = self::lateReceiptResolution($job, $event);
if ($resolution === null) {
$event->disposition = 'late';
$event->applied_at = null;
$event->save();
return 'late';
}
if ($requireEvidenceConflict) {
$source = 'late_completion_receipt:'
. (string) $event->event_id;
$operation = self::lateReceiptOperation(
$job,
$event,
$resolution
);
$prepared = self::preparedMarker(
$operation,
$source
);
try {
self::transitionPreparedMarker(
(string) $job->effect_resolution_reason,
$prepared,
$source,
$resolution,
self::legacyCompletionPreparedMarker(
$operation,
$source
)
);
// The conflict disappeared before both rows were
// locked. Leave the receipt retryable rather than
// inventing an incident from a stale snapshot.
return 'received';
} catch (CloudRecoveryEvidenceConflict) {
// The typed conflict was reproduced under both locks.
}
}
if (!hash_equals(
(string) ($job->finalization_error_code ?? ''),
$errorCode
)) {
$job->read_at = null;
}
$job->finalization_incident = 1;
$job->finalization_error_code = $errorCode;
$job->lease_token = null;
$job->lease_until = 0;
$now = time();
$job->available_at = $now + self::INCIDENT_RETRY_SECONDS;
$job->updated_at = $now;
$job->save();
$event->disposition = 'late';
$event->applied_at = null;
$event->save();
return 'late';
},
3
);
}
private static function lockLateReceiptEvent(
CloudJob $job,
?CloudAgentCompletionEvent $event,
string $source
): ?CloudAgentCompletionEvent {
$latePrefix = 'late_completion_receipt:';
if ($event === null) {
if (str_starts_with($source, $latePrefix)) {
throw new CloudError('晚到回执提交缺少事件记录', 409);
}
return null;
}
if (!str_starts_with($source, $latePrefix)
|| !hash_equals(
substr($source, strlen($latePrefix)),
(string) $event->event_id
)) {
throw new CloudError('晚到回执与恢复证据来源不匹配', 409);
}
$locked = CloudAgentCompletionEvent::where(
'event_id',
(string) $event->event_id
)->lockForUpdate()->first();
if ($locked === null
|| !self::sameOperation($job, $locked)
|| !in_array(
(string) $locked->disposition,
['received', 'applied'],
true
)
|| ((string) $locked->disposition === 'received'
&& !CloudOperationCompletionService::isExactStoredEvent(
$locked
))) {
throw new CloudError('晚到回执提交记录无效', 409);
}
return $locked;
}
private static function markLateReceiptApplied(
CloudJob $job,
?CloudAgentCompletionEvent $event
): void {
if ($event === null) {
return;
}
if (!self::wasReconciledBy($job, $event)) {
throw new CloudError('晚到回执尚未完成效果收敛', 409);
}
if ((string) $event->disposition === 'received') {
$event->disposition = 'applied';
$event->applied_at = time();
$event->save();
return;
}
if ((string) $event->disposition !== 'applied') {
throw new CloudError('晚到回执已被其他流程终结', 409);
}
}
private static function assertRecoveryCommitMayPublish(
CloudJob $job,
string $source,
?int $resolvedBy,
?CloudAgentCompletionEvent $completionEvent
): void {
self::assertRecoveryIncidentAllowsPrepare($job);
if ($completionEvent !== null
|| $resolvedBy !== 0
|| !str_starts_with($source, 'automatic_')) {
return;
}
// A completion receipt can arrive while this worker is outside the
// database performing its bounded Agent fence call. Serialize again
// on job -> event before publishing: the immutable receipt must win
// the next recovery turn, where compatibility is checked explicitly.
$pending = CloudAgentCompletionEvent::where(
'host_id',
(int) $job->host_id
)->where(
'operation_id',
(string) $job->operation_id
)->where('disposition', 'received')
->lockForUpdate()->first();
if ($pending !== null) {
throw new CloudError('晚到完成回执正在等待优先收敛', 409);
}
}
/** Fail closed on immutable generation damage at the locked publish seam. */
private static function assertLockedAgentRecoveryEnvelope(
CloudJob $job
): void {
CloudDispatchLedger::assertValid($job);
if (CloudJobService::hasAgentRecoveryIdentityConflict($job)) {
throw new CloudRecoveryOperationIdentityConflict(
'任务不可变 Agent 请求身份异常',
409
);
}
if (CloudJobService::hasAgentRecoveryEnvelopeConflict($job)) {
throw new CloudRecoveryInstanceStateConflict(
'任务不可变 Agent 派发信封状态异常',
409
);
}
}
public static function assertRecoveryIncidentAllowsPrepare(
CloudJob $job
): void {
// Recovery incidents are operator-visible diagnostics, not terminal
// queue states. Exact marker/evidence comparisons below still prevent
// conflicting conclusions from publishing, while a later fresh Agent
// inspection remains free to converge repaired host or panel state.
}
private static function sameOperation(
CloudJob $job,
CloudAgentCompletionEvent $event
): bool {
return (string) $job->execution_mode === 'agent'
&& $job->host_id !== null
&& (int) $job->host_id === (int) $event->host_id
&& is_string($job->operation_id)
&& is_string($event->operation_id)
&& hash_equals(
(string) $job->operation_id,
(string) $event->operation_id
);
}
private static function lateReceiptResolution(
CloudJob $job,
CloudAgentCompletionEvent $event
): ?string {
if (!self::recoverableUnknownEffect($job)
|| !CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
(string) $job->kind
)) {
return null;
}
if ((string) $event->outcome === 'succeeded'
&& !(bool) $event->effect_may_have_applied
&& !(bool) $event->safe_to_resubmit) {
return 'applied';
}
if ((string) $event->outcome === 'failed'
&& !(bool) $event->effect_may_have_applied) {
// safe_to_resubmit belongs to the completed Agent generation. It
// never authorizes replay here; both true and false settle only
// this original immutable job as not-applied.
return 'not_applied';
}
return null;
}
private static function retryableCreateCompletion(
CloudJob $job,
CloudAgentCompletionEvent $event
): bool {
return (string) $job->kind === 'create'
&& (string) $job->execution_mode === 'agent'
&& (string) $job->phase === 'terminal'
&& (string) $job->outcome === 'failed'
&& (string) $job->status === 'failed'
&& (int) $job->dispatch_attempts === 1
&& (bool) $job->effect_may_have_applied
&& !(bool) $job->safe_to_resubmit
&& (string) $job->effect_resolution === 'unknown'
&& ((string) $event->outcome === 'failed'
|| ((string) $event->outcome === 'succeeded'
&& !(bool) $event->effect_may_have_applied
&& !(bool) $event->safe_to_resubmit));
}
private static function assertResolution(string $resolution): void
{
if (!in_array($resolution, ['applied', 'not_applied'], true)) {
throw new \LogicException('Unsupported recovery resolution.');
}
}
private static function assertRecoverableEvidence(
CloudJob $job,
array $operation,
string $evidenceSource,
string $resolution
): void {
self::assertResolution($resolution);
if ($resolution === 'applied') {
self::assertRecoverableAppliedEvidence(
$job,
$operation,
$evidenceSource
);
return;
}
self::assertRecoverableNotAppliedEvidence(
$job,
$operation,
$evidenceSource
);
}
private static function assertOperationIdentity(
CloudJob $job,
array $operation,
string $resolution
): void {
self::assertResolution($resolution);
if ($resolution === 'applied') {
self::assertAppliedOperationIdentity($job, $operation);
return;
}
self::assertNotAppliedOperationIdentity($job, $operation);
}
private static function assertRecoverableAppliedEvidence(
CloudJob $job,
array $operation,
string $evidenceSource
): void {
self::assertEvidenceSource($evidenceSource);
self::assertAppliedOperationIdentity($job, $operation);
if (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
(string) $job->kind
)
|| !self::recoverableUnknownEffect($job)) {
throw new CloudError('任务效果恢复证据不满足安全策略', 409);
}
}
private static function assertRecoverableNotAppliedEvidence(
CloudJob $job,
array $operation,
string $evidenceSource
): void {
self::assertEvidenceSource($evidenceSource);
self::assertNotAppliedOperationIdentity($job, $operation);
if (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
(string) $job->kind
)
|| !self::recoverableUnknownEffect($job)) {
throw new CloudError('任务效果恢复证据不满足安全策略', 409);
}
}
private static function recoverableUnknownEffect(CloudJob $job): bool
{
return (string) $job->execution_mode === 'agent'
&& (string) $job->phase === 'terminal'
&& (string) $job->outcome === 'failed'
&& (string) $job->status === 'failed'
&& (int) $job->dispatch_attempts === 1
&& (bool) $job->effect_may_have_applied
&& !(bool) $job->safe_to_resubmit
&& (string) $job->effect_resolution === 'unknown';
}
private static function assertAppliedOperationIdentity(
CloudJob $job,
array $operation
): void {
$operationId = $operation['operation_id'] ?? null;
if (!is_string($operationId)
|| !is_string($job->operation_id)
|| !hash_equals((string) $job->operation_id, $operationId)
|| ($operation['status'] ?? null) !== 'succeeded'
|| ($operation['effect_may_have_applied'] ?? null) !== false
|| ($operation['safe_to_resubmit'] ?? null) !== false) {
throw new CloudError('任务效果恢复证据与原操作不匹配', 409);
}
self::assertOperationBinding($job, $operation, true);
}
private static function assertNotAppliedOperationIdentity(
CloudJob $job,
array $operation
): void {
$operationId = $operation['operation_id'] ?? null;
$errorCode = $operation['error_code'] ?? null;
$result = $operation['result'] ?? null;
if (!is_string($operationId)
|| !is_string($job->operation_id)
|| !hash_equals((string) $job->operation_id, $operationId)
|| ($operation['status'] ?? null) !== 'failed'
|| ($operation['effect_may_have_applied'] ?? null) !== false
|| !is_bool($operation['safe_to_resubmit'] ?? null)
|| !is_string($errorCode)
|| preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $errorCode) !== 1
|| ($result !== null && !is_array($result))) {
throw new CloudError('任务效果恢复证据与原操作不匹配', 409);
}
self::assertOperationBinding($job, $operation, false);
}
private static function assertOperationBinding(
CloudJob $job,
array $operation,
bool $forAppliedEffect
): void {
$expected = CloudOperationRecoveryIdentity::expected(
$job,
$forAppliedEffect
);
CloudOperationRecoveryIdentity::assertMatches($expected, [
'action' => $operation['action'] ?? null,
'resource_id' => $operation['resource_id'] ?? null,
'request_fingerprint' => $operation['request_fingerprint']
?? $operation['fingerprint'] ?? null,
]);
}
private static function assertEvidenceSource(string $source): void
{
if (preg_match(
'/^[a-z][a-z0-9_]{2,63}:[A-Za-z0-9_.-]{1,160}$/D',
$source
) !== 1) {
throw new CloudError('任务效果恢复证据来源无效', 409);
}
}
private static function assertResolvedBy(?int $resolvedBy): void
{
if ($resolvedBy !== null && $resolvedBy < 0) {
throw new CloudError('任务效果恢复审计主体无效', 409);
}
}
private static function preparedMarker(
array $operation,
string $source
): string {
self::assertEvidenceSource($source);
return 'prepared:' . $source . ':'
. self::preparedOperationHash($operation);
}
/** Hash the exact normalized operation bound into a prepared marker. */
public static function preparedOperationHash(array $operation): string
{
$canonical = self::canonicalize(
self::recoveryCriticalOperation($operation)
);
return hash(
'sha256',
json_encode(
$canonical,
JSON_THROW_ON_ERROR
| JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
)
);
}
/** Hash the sole legacy omission accepted from terminal completion wire. */
public static function legacyCompletionOperationHash(
array $operation
): ?string {
if (($operation['retryable'] ?? null) !== false
|| !in_array(
$operation['status'] ?? null,
['succeeded', 'failed'],
true
)) {
return null;
}
unset($operation['retryable']);
return self::preparedOperationHash($operation);
}
/**
* Preserve the durable automatic-attempt envelope while replacing its
* pending payload with the exact prepared evidence. An immutable late
* completion may replace only a still-pending legacy envelope; it may
* never overwrite a different prepared conclusion.
*/
private static function transitionPreparedMarker(
string $current,
string $prepared,
string $source,
string $resolution,
?string $legacyCompatiblePrepared = null
): string {
if ($current === '' || hash_equals($prepared, $current)) {
return $prepared;
}
if ($legacyCompatiblePrepared !== null
&& hash_equals($legacyCompatiblePrepared, $current)) {
return $prepared;
}
$state = self::automaticRecoveryState($current);
if ($state === null) {
self::throwPreparedEvidenceConflict($source);
}
if (hash_equals($prepared, $state['payload'])) {
return $current;
}
if ($legacyCompatiblePrepared !== null
&& hash_equals(
$legacyCompatiblePrepared,
$state['payload']
)) {
return $prepared;
}
if ($state['payload'] !== 'pending') {
$existing = [];
$late = [];
$legacy = [];
$journalStatus = $resolution === 'applied'
? 'succeeded' : 'failed';
$compatibleOperationHash = false;
if ($legacyCompatiblePrepared !== null
&& preg_match(
'/^prepared:late_completion_receipt:'
. '[A-Za-z0-9_.-]{1,160}:([a-f0-9]{64})$/D',
$legacyCompatiblePrepared,
$legacy
) === 1) {
$compatibleOperationHash = $legacy[1];
}
if (str_starts_with($source, 'late_completion_receipt:')
&& preg_match(
'/^prepared:automatic_'
. preg_quote($resolution, '/') . '_'
. $journalStatus
. '_nofence:[a-f0-9]{64}:([a-f0-9]{64})$/D',
$state['payload'],
$existing
) === 1
&& preg_match(
'/^prepared:late_completion_receipt:[A-Za-z0-9_.-]{1,160}:([a-f0-9]{64})$/D',
$prepared,
$late
) === 1
&& (hash_equals($existing[1], $late[1])
|| (is_string($compatibleOperationHash)
&& hash_equals(
$existing[1],
$compatibleOperationHash
)))) {
return $prepared;
}
self::throwPreparedEvidenceConflict($source);
}
if (str_starts_with($source, 'automatic_')) {
return sprintf(
'automatic_recovery:v1:%d:%d:%s',
time(),
1,
$prepared
);
}
if (str_starts_with($source, 'late_completion_receipt:')) {
return $prepared;
}
throw new CloudError('任务已有不同的效果恢复证据', 409);
}
private static function throwPreparedEvidenceConflict(
string $source
): never {
if (str_starts_with($source, 'late_completion_receipt:')) {
throw new CloudRecoveryEvidenceConflict(
'任务已有冲突的效果恢复证据',
409
);
}
throw new CloudError('任务已有不同的效果恢复证据', 409);
}
private static function preparedMarkerMatches(
string $current,
string $prepared
): bool {
if (hash_equals($prepared, $current)) {
return true;
}
$state = self::automaticRecoveryState($current);
return $state !== null
&& hash_equals($prepared, $state['payload']);
}
/** @return array{first_at:int,attempts:int,payload:string}|null */
private static function automaticRecoveryState(string $marker): ?array
{
if (preg_match(
'/^automatic_recovery:v1:([1-9][0-9]*):([1-9][0-9]*):(pending|prepared:(?:automatic_(?:applied|not_applied)_(?:succeeded|failed)_(?:fence|nofence):[a-f0-9]{64}|late_completion_receipt:[A-Za-z0-9_.-]{1,160}):[a-f0-9]{64})$/D',
$marker,
$matches
) !== 1) {
return null;
}
return [
'first_at' => (int) $matches[1],
'attempts' => (int) $matches[2],
'payload' => $matches[3],
];
}
private static function finalizedRecoveryPayload(CloudJob $job): string
{
$payload = CloudSecrets::decrypt(
(string) $job->sealed_payload,
'job-payload'
);
$identity = CloudOperationRecoveryIdentity::expected(
$job,
(string) $job->effect_resolution === 'applied'
);
// The prepared operation is retained only across the PHP prepare ->
// Agent fence -> PHP commit crash window. It must never survive in the
// ordinary terminal audit payload.
unset($payload['_operation_recovery']);
$payload[CloudOperationRecoveryIdentity::payloadKey()] =
CloudOperationRecoveryIdentity::persisted($identity);
return CloudSecrets::encrypt(
CloudInput::safeResult($payload),
'job-payload'
);
}
private static function recoveryCriticalOperation(array $operation): array
{
// The normalized operation is bounded by the completion/inspection
// schemas. Hash every allow-listed result field because different kinds
// consume different immutable evidence (snapshot keys, policy digests,
// catalog generation identifiers, deletion proofs, and so on).
return $operation;
}
/**
* Completion receipts before retryable was restored at the PHP adapter
* produced prepared hashes without that canonical terminal field. Accept
* only that one historical omission while upgrading the durable marker to
* the fully bound operation; every other field remains hash-identical.
*/
private static function legacyCompletionPreparedMarker(
array $operation,
string $source
): ?string {
if (!str_starts_with($source, 'late_completion_receipt:')) {
return null;
}
$legacyHash = self::legacyCompletionOperationHash($operation);
return $legacyHash === null
? null : 'prepared:' . $source . ':' . $legacyHash;
}
private static function canonicalize($value)
{
if (!is_array($value)) {
return $value;
}
if (!array_is_list($value)) {
ksort($value, SORT_STRING);
}
foreach ($value as $key => $item) {
$value[$key] = self::canonicalize($item);
}
return $value;
}
private static function wasReconciledBy(
CloudJob $job,
CloudAgentCompletionEvent $event
): bool {
return CloudDispatchLedger::isDispatched($job)
&& (string) $job->phase === 'terminal'
&& (string) $job->outcome === 'failed'
&& in_array((string) $job->effect_resolution,
['applied', 'not_applied'], true)
&& hash_equals(
'late_completion_receipt:' . (string) $event->event_id,
(string) $job->effect_resolution_reason
);
}
private static function holdDispatchLedgerConflictLocked(
CloudJob $job
): void {
$now = time();
if ((string) $job->phase === 'terminal'
&& (string) $job->outcome === 'failed') {
$job->effect_may_have_applied = 1;
$job->safe_to_resubmit = 0;
$job->effect_resolution = 'unknown';
}
if (!hash_equals(
(string) ($job->finalization_error_code ?? ''),
CloudDispatchLedger::CONFLICT_CODE
)) {
$job->read_at = null;
}
$job->finalization_incident = 1;
$job->finalization_error_code =
CloudDispatchLedger::CONFLICT_CODE;
$job->lease_token = null;
$job->lease_until = 0;
$job->available_at = $now + self::INCIDENT_RETRY_SECONDS;
$job->updated_at = $now;
$job->save();
}
}