You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1410 lines
51 KiB
PHP
1410 lines
51 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\CloudAgentCompletionEvent;
|
|
use App\Models\CloudJob;
|
|
use App\Models\User;
|
|
|
|
/**
|
|
* Converges a terminal task's unresolved host effect without replaying it.
|
|
*
|
|
* The original job remains an immutable failed audit record. Only an exact
|
|
* immutable Agent completion receipt or a freshly validated, operation-bound
|
|
* read-only inspection for an explicitly registered kind may update PHP's
|
|
* resource ledger and release the effect fence.
|
|
*/
|
|
final class CloudOperationRecoveryService
|
|
{
|
|
private const INCIDENT_RETRY_SECONDS = 300;
|
|
|
|
/**
|
|
* Apply the kind-specific PHP ledger finalizer while the job remains
|
|
* effect_resolution=unknown. The returned job therefore still blocks
|
|
* conflicting work until the caller has released the matching Agent
|
|
* resource fence and invokes commitApplied().
|
|
*
|
|
* This method performs PHP-only convergence and never contacts an Agent.
|
|
*/
|
|
public static function prepareApplied(
|
|
int $jobId,
|
|
array $operation,
|
|
string $evidenceSource
|
|
): CloudJob {
|
|
return self::prepareResolution(
|
|
$jobId,
|
|
$operation,
|
|
$evidenceSource,
|
|
'applied'
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Publish the prepared resolution and erase the encrypted recovery escrow.
|
|
* The caller must release the matching Agent fence before this commit.
|
|
*/
|
|
public static function commitApplied(
|
|
int $jobId,
|
|
array $operation,
|
|
string $evidenceSource,
|
|
?int $resolvedBy = null,
|
|
?CloudAgentCompletionEvent $completionEvent = null
|
|
): CloudJob {
|
|
return self::commitResolution(
|
|
$jobId,
|
|
$operation,
|
|
$evidenceSource,
|
|
'applied',
|
|
$resolvedBy,
|
|
$completionEvent
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Roll back the kind-specific PHP pending ledger and persist a no-effect
|
|
* intent while the original unknown-effect blocker remains active.
|
|
*/
|
|
public static function prepareNotApplied(
|
|
int $jobId,
|
|
array $operation,
|
|
string $evidenceSource
|
|
): CloudJob {
|
|
// A proved no-effect operation must still drive the original failed
|
|
// finalizer. Network reservations, snapshots, firewall revisions and
|
|
// catalog replicas all have PHP-owned pending/uncertain ledger state
|
|
// which only that finalizer can roll back safely.
|
|
return self::prepareResolution(
|
|
$jobId,
|
|
$operation,
|
|
$evidenceSource,
|
|
'not_applied'
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Publish a prepared no-effect conclusion after the matching Agent fence
|
|
* has been durably released, then erase the encrypted recovery escrow.
|
|
*/
|
|
public static function commitNotApplied(
|
|
int $jobId,
|
|
array $operation,
|
|
string $evidenceSource,
|
|
?int $resolvedBy = null,
|
|
?CloudAgentCompletionEvent $completionEvent = null
|
|
): CloudJob {
|
|
return self::commitResolution(
|
|
$jobId,
|
|
$operation,
|
|
$evidenceSource,
|
|
'not_applied',
|
|
$resolvedBy,
|
|
$completionEvent
|
|
);
|
|
}
|
|
|
|
/** Prepare one validated resolution without publishing its effect. */
|
|
private static function prepareResolution(
|
|
int $jobId,
|
|
array $operation,
|
|
string $evidenceSource,
|
|
string $resolution
|
|
): CloudJob {
|
|
$job = CloudJob::find($jobId);
|
|
if ($job === null) {
|
|
throw new CloudError('任务不存在', 404);
|
|
}
|
|
self::assertRecoverableEvidence(
|
|
$job,
|
|
$operation,
|
|
$evidenceSource,
|
|
$resolution
|
|
);
|
|
$prepared = self::preparedMarker($operation, $evidenceSource);
|
|
$marker = self::transitionPreparedMarker(
|
|
(string) $job->effect_resolution_reason,
|
|
$prepared,
|
|
$evidenceSource,
|
|
$resolution,
|
|
self::legacyCompletionPreparedMarker(
|
|
$operation,
|
|
$evidenceSource
|
|
)
|
|
);
|
|
return (new CloudWorkerService())->prepareRecoveredEffect(
|
|
$jobId,
|
|
$operation,
|
|
$marker
|
|
);
|
|
}
|
|
|
|
/** Publish one prepared resolution through the shared locked commit seam. */
|
|
private static function commitResolution(
|
|
int $jobId,
|
|
array $operation,
|
|
string $evidenceSource,
|
|
string $resolution,
|
|
?int $resolvedBy,
|
|
?CloudAgentCompletionEvent $completionEvent
|
|
): CloudJob {
|
|
self::assertResolution($resolution);
|
|
self::assertResolvedBy($resolvedBy);
|
|
$jobSnapshot = CloudJob::find($jobId);
|
|
$aggregateParentId = (int) ($jobSnapshot?->parent_job_id ?? 0);
|
|
$transitionUserId = $jobSnapshot instanceof CloudJob
|
|
&& (string) $jobSnapshot->kind === 'network_change'
|
|
&& $jobSnapshot->public_network_transition_id !== null
|
|
? (int) $jobSnapshot->user_id : 0;
|
|
$portableRestoreUserId = $jobSnapshot instanceof CloudJob
|
|
&& CloudPortableRestoreFinalizer::owns($jobSnapshot)
|
|
? (int) $jobSnapshot->user_id : 0;
|
|
$recoveryUserId = $transitionUserId > 0
|
|
? $transitionUserId : $portableRestoreUserId;
|
|
return (new CloudJob())->getConnection()->transaction(
|
|
function () use (
|
|
$jobId,
|
|
$operation,
|
|
$evidenceSource,
|
|
$resolution,
|
|
$resolvedBy,
|
|
$completionEvent,
|
|
$aggregateParentId,
|
|
$transitionUserId,
|
|
$portableRestoreUserId,
|
|
$recoveryUserId
|
|
): CloudJob {
|
|
$recoveryActor = $recoveryUserId > 0
|
|
? User::where('id', $recoveryUserId)
|
|
->lockForUpdate()->first()
|
|
: null;
|
|
if ($recoveryUserId > 0
|
|
&& !$recoveryActor instanceof User) {
|
|
throw new CloudError(
|
|
'任务效果恢复账户不存在',
|
|
409
|
|
);
|
|
}
|
|
$transitionActor = $transitionUserId > 0
|
|
? $recoveryActor : null;
|
|
$aggregateParent = null;
|
|
if ($aggregateParentId > 0) {
|
|
$aggregateParent = CloudJob::where(
|
|
'id',
|
|
$aggregateParentId
|
|
)
|
|
->lockForUpdate()->first();
|
|
if ($aggregateParent === null) {
|
|
throw new CloudRecoveryAggregateStateConflict(
|
|
'任务的汇总父任务不存在',
|
|
409
|
|
);
|
|
}
|
|
CloudTemplateCatalogService::lockAggregateCatalogLocked(
|
|
$aggregateParent
|
|
);
|
|
CloudUserIsoService::lockAggregateCatalogLocked(
|
|
$aggregateParent
|
|
);
|
|
}
|
|
$job = CloudJob::where('id', $jobId)
|
|
->lockForUpdate()->first();
|
|
if ($job === null) {
|
|
throw new CloudError('任务不存在', 404);
|
|
}
|
|
if ((int) ($job->parent_job_id ?? 0)
|
|
!== $aggregateParentId) {
|
|
throw new CloudRecoveryAggregateStateConflict(
|
|
'任务的汇总父级关系在提交期间发生变化',
|
|
409
|
|
);
|
|
}
|
|
if ($transitionUserId > 0
|
|
&& ((int) $job->user_id !== $transitionUserId
|
|
|| (string) $job->kind !== 'network_change'
|
|
|| $job->public_network_transition_id === null)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'公网网络变更恢复账户关联发生变化',
|
|
409
|
|
);
|
|
}
|
|
if ($portableRestoreUserId > 0
|
|
&& ((int) $job->user_id !== $portableRestoreUserId
|
|
|| !CloudPortableRestoreFinalizer::owns($job))) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'备份恢复账户或尝试关联发生变化',
|
|
409
|
|
);
|
|
}
|
|
self::assertLockedAgentRecoveryEnvelope($job);
|
|
self::assertEvidenceSource($evidenceSource);
|
|
self::assertOperationIdentity(
|
|
$job,
|
|
$operation,
|
|
$resolution
|
|
);
|
|
$lockedEvent = self::lockLateReceiptEvent(
|
|
$job,
|
|
$completionEvent,
|
|
$evidenceSource
|
|
);
|
|
self::assertRecoveryCommitMayPublish(
|
|
$job,
|
|
$evidenceSource,
|
|
$resolvedBy,
|
|
$completionEvent
|
|
);
|
|
$finalReason = $evidenceSource;
|
|
if ((string) $job->effect_resolution === $resolution
|
|
&& hash_equals(
|
|
$finalReason,
|
|
(string) $job->effect_resolution_reason
|
|
)) {
|
|
self::markLateReceiptApplied($job, $lockedEvent);
|
|
return $job;
|
|
}
|
|
if ($job->public_network_transition_id !== null) {
|
|
CloudPublicNetworkTransitionWorkflow::
|
|
publishPreparedLocked(
|
|
$job,
|
|
$resolution,
|
|
$transitionActor,
|
|
$aggregateParent
|
|
);
|
|
}
|
|
if ($portableRestoreUserId > 0) {
|
|
CloudPortableRestoreFinalizer::publishPreparedLocked(
|
|
$job,
|
|
$resolution
|
|
);
|
|
}
|
|
self::assertRecoverableEvidence(
|
|
$job,
|
|
$operation,
|
|
$evidenceSource,
|
|
$resolution
|
|
);
|
|
$marker = self::preparedMarker(
|
|
$operation,
|
|
$evidenceSource
|
|
);
|
|
if (!self::preparedMarkerMatches(
|
|
(string) $job->effect_resolution_reason,
|
|
$marker
|
|
)) {
|
|
throw new CloudError('任务效果尚未完成账本预提交', 409);
|
|
}
|
|
$job->effect_resolution = $resolution;
|
|
$job->effect_resolved_at = time();
|
|
$job->effect_resolved_by = $resolvedBy;
|
|
$job->effect_resolution_reason = $finalReason;
|
|
$job->sealed_payload = self::finalizedRecoveryPayload($job);
|
|
$job->lease_token = null;
|
|
$job->lease_until = 0;
|
|
$job->available_at = 0;
|
|
$job->finalization_incident = 0;
|
|
$job->finalization_error_code = null;
|
|
$job->read_at = null;
|
|
$job->updated_at = time();
|
|
$job->save();
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
$job,
|
|
$resolvedBy
|
|
);
|
|
self::markLateReceiptApplied($job, $lockedEvent);
|
|
return $job;
|
|
},
|
|
3
|
|
);
|
|
}
|
|
|
|
/**
|
|
* @return string One of the existing completion-event dispositions.
|
|
*/
|
|
public static function settleLateReceipt(
|
|
CloudJob $source,
|
|
CloudAgentCompletionEvent $event
|
|
): string {
|
|
$disposition = (string) $event->disposition;
|
|
if ($disposition !== 'received') {
|
|
return $disposition;
|
|
}
|
|
if (!self::sameOperation($source, $event)) {
|
|
return self::rejectInvalidLateReceipt(
|
|
(int) $source->id,
|
|
(string) $event->event_id
|
|
);
|
|
}
|
|
if (CloudDispatchLedger::isInvalid($source)) {
|
|
CloudOperationCompletionService::deferDispatchLedgerConflict(
|
|
$source
|
|
);
|
|
return 'received';
|
|
}
|
|
if (CloudDispatchLedger::isNeverDispatched($source)) {
|
|
CloudOperationCompletionService::
|
|
deferNeverDispatchedEvidenceConflict($source);
|
|
return 'received';
|
|
}
|
|
if (CloudDispatchLedger::hasRepairIncident($source)) {
|
|
return 'received';
|
|
}
|
|
|
|
$current = $source;
|
|
try {
|
|
$current = CloudJob::find((int) $source->id) ?? $source;
|
|
CloudOperationCompletionService::assertEventIdentityForJob(
|
|
$current,
|
|
$event
|
|
);
|
|
if (self::wasReconciledBy($current, $event)) {
|
|
return self::acknowledgeReconciledLateReceipt(
|
|
(int) $current->id,
|
|
(string) $event->event_id
|
|
);
|
|
}
|
|
if ((string) $current->kind === 'create') {
|
|
if ((string) $event->outcome === 'failed') {
|
|
return CloudCreateFailureRecoveryService::settleReceipt(
|
|
$current,
|
|
$event
|
|
);
|
|
}
|
|
if ((string) $event->outcome !== 'succeeded'
|
|
|| (bool) $event->effect_may_have_applied
|
|
|| (bool) $event->safe_to_resubmit
|
|
|| !CloudOperationCompletionService::isExactStoredEvent(
|
|
$event
|
|
)) {
|
|
CloudOperationCompletionService::mark($event, 'late');
|
|
return 'late';
|
|
}
|
|
(new CloudWorkerService())->settleRecoveredCreateCompletion(
|
|
(int) $current->id,
|
|
$event
|
|
);
|
|
$reloadedCreateEvent = CloudAgentCompletionEvent::where(
|
|
'event_id',
|
|
(string) $event->event_id
|
|
)->first();
|
|
return $reloadedCreateEvent === null
|
|
? 'received'
|
|
: (string) $reloadedCreateEvent->disposition;
|
|
}
|
|
$lateResolution = self::lateReceiptResolution($current, $event);
|
|
if ($lateResolution === null) {
|
|
CloudOperationCompletionService::mark($event, 'late');
|
|
return 'late';
|
|
}
|
|
if (!CloudOperationCompletionService::isExactStoredEvent(
|
|
$event
|
|
)) {
|
|
return self::rejectInvalidLateReceipt(
|
|
(int) $current->id,
|
|
(string) $event->event_id
|
|
);
|
|
}
|
|
|
|
$evidenceSource = 'late_completion_receipt:'
|
|
. (string) $event->event_id;
|
|
$operation = self::lateReceiptOperation(
|
|
$current,
|
|
$event,
|
|
$lateResolution
|
|
);
|
|
if ($lateResolution === 'applied') {
|
|
self::prepareApplied(
|
|
(int) $current->id,
|
|
$operation,
|
|
$evidenceSource
|
|
);
|
|
} else {
|
|
self::prepareNotApplied(
|
|
(int) $current->id,
|
|
$operation,
|
|
$evidenceSource
|
|
);
|
|
}
|
|
// A terminal success receipt means the Agent journal has already
|
|
// committed its terminal record and no longer retains its
|
|
// resource fence. The same is true for an exact no-effect failed
|
|
// receipt. No Agent request is needed for this receipt-only path.
|
|
if ($lateResolution === 'applied') {
|
|
self::commitApplied(
|
|
(int) $current->id,
|
|
$operation,
|
|
$evidenceSource,
|
|
null,
|
|
$event
|
|
);
|
|
} else {
|
|
self::commitNotApplied(
|
|
(int) $current->id,
|
|
$operation,
|
|
$evidenceSource,
|
|
null,
|
|
$event
|
|
);
|
|
}
|
|
} catch (CloudRecoveryEvidenceConflict $error) {
|
|
try {
|
|
return self::parkLateReceiptEvidenceConflict(
|
|
(int) $current->id,
|
|
(string) $event->event_id
|
|
);
|
|
} catch (\Throwable $parkingError) {
|
|
error_log(sprintf(
|
|
'cloud late completion conflict parking pending job_id=%d kind=%s error=%s',
|
|
(int) $current->id,
|
|
(string) $current->kind,
|
|
get_class($parkingError)
|
|
));
|
|
return 'received';
|
|
}
|
|
} catch (CloudRecoveryReceiptInvalid $error) {
|
|
return self::isolateLateReceiptValidationFailure(
|
|
$current,
|
|
$event
|
|
);
|
|
} catch (CloudError $error) {
|
|
if ($error->httpStatus === 502) {
|
|
return self::isolateLateReceiptValidationFailure(
|
|
$current,
|
|
$event
|
|
);
|
|
}
|
|
return self::retryOrParkLateReceiptFailure(
|
|
$current,
|
|
$event,
|
|
$error
|
|
);
|
|
} catch (\Throwable $error) {
|
|
return self::retryOrParkLateReceiptFailure(
|
|
$current,
|
|
$event,
|
|
$error
|
|
);
|
|
}
|
|
$reloaded = CloudAgentCompletionEvent::where(
|
|
'event_id',
|
|
(string) $event->event_id
|
|
)->first();
|
|
return $reloaded === null
|
|
? 'received' : (string) $reloaded->disposition;
|
|
}
|
|
|
|
private static function lateReceiptOperation(
|
|
CloudJob $job,
|
|
CloudAgentCompletionEvent $event,
|
|
string $resolution
|
|
): array {
|
|
if (!in_array($resolution, ['applied', 'not_applied'], true)
|
|
|| !CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
|
|
(string) $job->kind
|
|
)) {
|
|
throw new CloudRecoveryReceiptInvalid(
|
|
'Agent 完成回执不支持该任务的效果收敛',
|
|
409
|
|
);
|
|
}
|
|
// Keep the exact authenticated Agent operation. Every supported kind's
|
|
// existing worker finalizer validates the result fields it consumes;
|
|
// replacing those fields with a settings-shaped synthetic result would
|
|
// discard snapshot, network, firewall and catalog ledger evidence.
|
|
return CloudOperationCompletionService::operation($event);
|
|
}
|
|
|
|
private static function acknowledgeReconciledLateReceipt(
|
|
int $jobId,
|
|
string $eventId
|
|
): string {
|
|
return (new CloudJob())->getConnection()->transaction(
|
|
static function () use ($jobId, $eventId): string {
|
|
$job = CloudJob::where('id', $jobId)
|
|
->lockForUpdate()->first();
|
|
$event = CloudAgentCompletionEvent::where(
|
|
'event_id',
|
|
$eventId
|
|
)->lockForUpdate()->first();
|
|
if ($event === null) {
|
|
return 'received';
|
|
}
|
|
$disposition = (string) $event->disposition;
|
|
if ($disposition !== 'received') {
|
|
return $disposition;
|
|
}
|
|
if ($job !== null
|
|
&& self::sameOperation($job, $event)
|
|
&& CloudDispatchLedger::isInvalid($job)) {
|
|
self::holdDispatchLedgerConflictLocked($job);
|
|
return 'received';
|
|
}
|
|
if ($job === null
|
|
|| !self::sameOperation($job, $event)
|
|
|| !CloudOperationCompletionService::isExactStoredEvent(
|
|
$event
|
|
)) {
|
|
$event->disposition = 'rejected';
|
|
$event->applied_at = null;
|
|
$event->save();
|
|
return 'rejected';
|
|
}
|
|
if (!self::wasReconciledBy($job, $event)) {
|
|
// Another recovery source may have committed after the
|
|
// caller loaded its snapshot. Once this job is no longer
|
|
// eligible for the immutable receipt, terminate the event
|
|
// instead of leaving an unclaimable "received" row.
|
|
if (self::retryableCreateCompletion($job, $event)) {
|
|
// Create uses dedicated financial finalizers. Keep
|
|
// either exact outcome retryable when PHP could not
|
|
// finish inspecting or settling its ledger. An
|
|
// incomplete failure is ACKed late by the dedicated
|
|
// finalizer only after those local checks succeed.
|
|
return 'received';
|
|
}
|
|
if (self::lateReceiptResolution($job, $event) === null) {
|
|
$event->disposition = 'late';
|
|
$event->applied_at = null;
|
|
$event->save();
|
|
return 'late';
|
|
}
|
|
return 'received';
|
|
}
|
|
$event->disposition = 'applied';
|
|
$event->applied_at = time();
|
|
$event->save();
|
|
return 'applied';
|
|
},
|
|
3
|
|
);
|
|
}
|
|
|
|
private static function rejectInvalidLateReceipt(
|
|
int $jobId,
|
|
string $eventId
|
|
): string {
|
|
return (new CloudJob())->getConnection()->transaction(
|
|
static function () use ($jobId, $eventId): string {
|
|
CloudJob::where('id', $jobId)->lockForUpdate()->first();
|
|
$event = CloudAgentCompletionEvent::where(
|
|
'event_id',
|
|
$eventId
|
|
)->lockForUpdate()->first();
|
|
if ($event === null) {
|
|
return 'rejected';
|
|
}
|
|
if ((string) $event->disposition === 'received') {
|
|
$event->disposition = 'rejected';
|
|
$event->applied_at = null;
|
|
$event->save();
|
|
}
|
|
return (string) $event->disposition;
|
|
},
|
|
3
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Isolate a receipt which contradicts already prepared durable evidence.
|
|
* The job itself remains scheduled for fresh inspection; only this exact
|
|
* conflicting event leaves the recovery queue.
|
|
*/
|
|
private static function parkLateReceiptEvidenceConflict(
|
|
int $jobId,
|
|
string $eventId
|
|
): string {
|
|
return self::parkLateReceiptIncident(
|
|
$jobId,
|
|
$eventId,
|
|
'recovery_evidence_conflict',
|
|
true
|
|
);
|
|
}
|
|
|
|
private static function parkLateReceiptValidationFailure(
|
|
int $jobId,
|
|
string $eventId
|
|
): string {
|
|
return self::parkLateReceiptIncident(
|
|
$jobId,
|
|
$eventId,
|
|
'recovery_receipt_invalid',
|
|
false
|
|
);
|
|
}
|
|
|
|
private static function isolateLateReceiptValidationFailure(
|
|
CloudJob $job,
|
|
CloudAgentCompletionEvent $event
|
|
): string {
|
|
try {
|
|
$repaired = self::acknowledgeReconciledLateReceipt(
|
|
(int) $job->id,
|
|
(string) $event->event_id
|
|
);
|
|
if ($repaired !== 'received') {
|
|
return $repaired;
|
|
}
|
|
return self::parkLateReceiptValidationFailure(
|
|
(int) $job->id,
|
|
(string) $event->event_id
|
|
);
|
|
} catch (\Throwable $parkingError) {
|
|
error_log(sprintf(
|
|
'cloud late completion invalid receipt parking pending job_id=%d kind=%s error=%s',
|
|
(int) $job->id,
|
|
(string) $job->kind,
|
|
get_class($parkingError)
|
|
));
|
|
return 'received';
|
|
}
|
|
}
|
|
|
|
private static function retryOrParkLateReceiptFailure(
|
|
CloudJob $job,
|
|
CloudAgentCompletionEvent $event,
|
|
\Throwable $error
|
|
): string {
|
|
try {
|
|
$repaired = self::acknowledgeReconciledLateReceipt(
|
|
(int) $job->id,
|
|
(string) $event->event_id
|
|
);
|
|
if ($repaired !== 'received') {
|
|
return $repaired;
|
|
}
|
|
} catch (\Throwable $parkingError) {
|
|
error_log(sprintf(
|
|
'cloud late completion retry parking pending job_id=%d kind=%s error=%s',
|
|
(int) $job->id,
|
|
(string) $job->kind,
|
|
get_class($parkingError)
|
|
));
|
|
return 'received';
|
|
}
|
|
// Keep the immutable receipt in "received" without an age limit so a
|
|
// retransmission or later worker can retry PHP-only ledger convergence.
|
|
// The original Agent mutation is never dispatched again.
|
|
error_log(sprintf(
|
|
'cloud late completion recovery pending job_id=%d kind=%s error=%s',
|
|
(int) $job->id,
|
|
(string) $job->kind,
|
|
get_class($error)
|
|
));
|
|
return 'received';
|
|
}
|
|
|
|
private static function parkLateReceiptIncident(
|
|
int $jobId,
|
|
string $eventId,
|
|
string $errorCode,
|
|
bool $requireEvidenceConflict
|
|
): string {
|
|
return (new CloudJob())->getConnection()->transaction(
|
|
static function () use (
|
|
$jobId,
|
|
$eventId,
|
|
$errorCode,
|
|
$requireEvidenceConflict
|
|
): string {
|
|
$job = CloudJob::where('id', $jobId)
|
|
->lockForUpdate()->first();
|
|
$event = CloudAgentCompletionEvent::where(
|
|
'event_id',
|
|
$eventId
|
|
)->lockForUpdate()->first();
|
|
if ($event === null) {
|
|
return 'received';
|
|
}
|
|
$disposition = (string) $event->disposition;
|
|
if ($disposition !== 'received') {
|
|
return $disposition;
|
|
}
|
|
if ($job !== null
|
|
&& self::sameOperation($job, $event)
|
|
&& CloudDispatchLedger::isInvalid($job)) {
|
|
self::holdDispatchLedgerConflictLocked($job);
|
|
return 'received';
|
|
}
|
|
if ($job === null
|
|
|| !self::sameOperation($job, $event)
|
|
|| !CloudOperationCompletionService::isExactStoredEvent(
|
|
$event
|
|
)) {
|
|
$event->disposition = 'rejected';
|
|
$event->applied_at = null;
|
|
$event->save();
|
|
return 'rejected';
|
|
}
|
|
if (self::wasReconciledBy($job, $event)) {
|
|
$event->disposition = 'applied';
|
|
$event->applied_at = time();
|
|
$event->save();
|
|
return 'applied';
|
|
}
|
|
$resolution = self::lateReceiptResolution($job, $event);
|
|
if ($resolution === null) {
|
|
$event->disposition = 'late';
|
|
$event->applied_at = null;
|
|
$event->save();
|
|
return 'late';
|
|
}
|
|
if ($requireEvidenceConflict) {
|
|
$source = 'late_completion_receipt:'
|
|
. (string) $event->event_id;
|
|
$operation = self::lateReceiptOperation(
|
|
$job,
|
|
$event,
|
|
$resolution
|
|
);
|
|
$prepared = self::preparedMarker(
|
|
$operation,
|
|
$source
|
|
);
|
|
try {
|
|
self::transitionPreparedMarker(
|
|
(string) $job->effect_resolution_reason,
|
|
$prepared,
|
|
$source,
|
|
$resolution,
|
|
self::legacyCompletionPreparedMarker(
|
|
$operation,
|
|
$source
|
|
)
|
|
);
|
|
// The conflict disappeared before both rows were
|
|
// locked. Leave the receipt retryable rather than
|
|
// inventing an incident from a stale snapshot.
|
|
return 'received';
|
|
} catch (CloudRecoveryEvidenceConflict) {
|
|
// The typed conflict was reproduced under both locks.
|
|
}
|
|
}
|
|
if (!hash_equals(
|
|
(string) ($job->finalization_error_code ?? ''),
|
|
$errorCode
|
|
)) {
|
|
$job->read_at = null;
|
|
}
|
|
$job->finalization_incident = 1;
|
|
$job->finalization_error_code = $errorCode;
|
|
$job->lease_token = null;
|
|
$job->lease_until = 0;
|
|
$now = time();
|
|
$job->available_at = $now + self::INCIDENT_RETRY_SECONDS;
|
|
$job->updated_at = $now;
|
|
$job->save();
|
|
$event->disposition = 'late';
|
|
$event->applied_at = null;
|
|
$event->save();
|
|
return 'late';
|
|
},
|
|
3
|
|
);
|
|
}
|
|
|
|
private static function lockLateReceiptEvent(
|
|
CloudJob $job,
|
|
?CloudAgentCompletionEvent $event,
|
|
string $source
|
|
): ?CloudAgentCompletionEvent {
|
|
$latePrefix = 'late_completion_receipt:';
|
|
if ($event === null) {
|
|
if (str_starts_with($source, $latePrefix)) {
|
|
throw new CloudError('晚到回执提交缺少事件记录', 409);
|
|
}
|
|
return null;
|
|
}
|
|
if (!str_starts_with($source, $latePrefix)
|
|
|| !hash_equals(
|
|
substr($source, strlen($latePrefix)),
|
|
(string) $event->event_id
|
|
)) {
|
|
throw new CloudError('晚到回执与恢复证据来源不匹配', 409);
|
|
}
|
|
$locked = CloudAgentCompletionEvent::where(
|
|
'event_id',
|
|
(string) $event->event_id
|
|
)->lockForUpdate()->first();
|
|
if ($locked === null
|
|
|| !self::sameOperation($job, $locked)
|
|
|| !in_array(
|
|
(string) $locked->disposition,
|
|
['received', 'applied'],
|
|
true
|
|
)
|
|
|| ((string) $locked->disposition === 'received'
|
|
&& !CloudOperationCompletionService::isExactStoredEvent(
|
|
$locked
|
|
))) {
|
|
throw new CloudError('晚到回执提交记录无效', 409);
|
|
}
|
|
return $locked;
|
|
}
|
|
|
|
private static function markLateReceiptApplied(
|
|
CloudJob $job,
|
|
?CloudAgentCompletionEvent $event
|
|
): void {
|
|
if ($event === null) {
|
|
return;
|
|
}
|
|
if (!self::wasReconciledBy($job, $event)) {
|
|
throw new CloudError('晚到回执尚未完成效果收敛', 409);
|
|
}
|
|
if ((string) $event->disposition === 'received') {
|
|
$event->disposition = 'applied';
|
|
$event->applied_at = time();
|
|
$event->save();
|
|
return;
|
|
}
|
|
if ((string) $event->disposition !== 'applied') {
|
|
throw new CloudError('晚到回执已被其他流程终结', 409);
|
|
}
|
|
}
|
|
|
|
private static function assertRecoveryCommitMayPublish(
|
|
CloudJob $job,
|
|
string $source,
|
|
?int $resolvedBy,
|
|
?CloudAgentCompletionEvent $completionEvent
|
|
): void {
|
|
self::assertRecoveryIncidentAllowsPrepare($job);
|
|
if ($completionEvent !== null
|
|
|| $resolvedBy !== 0
|
|
|| !str_starts_with($source, 'automatic_')) {
|
|
return;
|
|
}
|
|
// A completion receipt can arrive while this worker is outside the
|
|
// database performing its bounded Agent fence call. Serialize again
|
|
// on job -> event before publishing: the immutable receipt must win
|
|
// the next recovery turn, where compatibility is checked explicitly.
|
|
$pending = CloudAgentCompletionEvent::where(
|
|
'host_id',
|
|
(int) $job->host_id
|
|
)->where(
|
|
'operation_id',
|
|
(string) $job->operation_id
|
|
)->where('disposition', 'received')
|
|
->lockForUpdate()->first();
|
|
if ($pending !== null) {
|
|
throw new CloudError('晚到完成回执正在等待优先收敛', 409);
|
|
}
|
|
}
|
|
|
|
/** Fail closed on immutable generation damage at the locked publish seam. */
|
|
private static function assertLockedAgentRecoveryEnvelope(
|
|
CloudJob $job
|
|
): void {
|
|
CloudDispatchLedger::assertValid($job);
|
|
if (CloudJobService::hasAgentRecoveryIdentityConflict($job)) {
|
|
throw new CloudRecoveryOperationIdentityConflict(
|
|
'任务不可变 Agent 请求身份异常',
|
|
409
|
|
);
|
|
}
|
|
if (CloudJobService::hasAgentRecoveryEnvelopeConflict($job)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'任务不可变 Agent 派发信封状态异常',
|
|
409
|
|
);
|
|
}
|
|
}
|
|
|
|
public static function assertRecoveryIncidentAllowsPrepare(
|
|
CloudJob $job
|
|
): void {
|
|
// Recovery incidents are operator-visible diagnostics, not terminal
|
|
// queue states. Exact marker/evidence comparisons below still prevent
|
|
// conflicting conclusions from publishing, while a later fresh Agent
|
|
// inspection remains free to converge repaired host or panel state.
|
|
}
|
|
|
|
private static function sameOperation(
|
|
CloudJob $job,
|
|
CloudAgentCompletionEvent $event
|
|
): bool {
|
|
return (string) $job->execution_mode === 'agent'
|
|
&& $job->host_id !== null
|
|
&& (int) $job->host_id === (int) $event->host_id
|
|
&& is_string($job->operation_id)
|
|
&& is_string($event->operation_id)
|
|
&& hash_equals(
|
|
(string) $job->operation_id,
|
|
(string) $event->operation_id
|
|
);
|
|
}
|
|
|
|
private static function lateReceiptResolution(
|
|
CloudJob $job,
|
|
CloudAgentCompletionEvent $event
|
|
): ?string {
|
|
if (!self::recoverableUnknownEffect($job)
|
|
|| !CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
|
|
(string) $job->kind
|
|
)) {
|
|
return null;
|
|
}
|
|
if ((string) $event->outcome === 'succeeded'
|
|
&& !(bool) $event->effect_may_have_applied
|
|
&& !(bool) $event->safe_to_resubmit) {
|
|
return 'applied';
|
|
}
|
|
if ((string) $event->outcome === 'failed'
|
|
&& !(bool) $event->effect_may_have_applied) {
|
|
// safe_to_resubmit belongs to the completed Agent generation. It
|
|
// never authorizes replay here; both true and false settle only
|
|
// this original immutable job as not-applied.
|
|
return 'not_applied';
|
|
}
|
|
return null;
|
|
}
|
|
|
|
private static function retryableCreateCompletion(
|
|
CloudJob $job,
|
|
CloudAgentCompletionEvent $event
|
|
): bool {
|
|
return (string) $job->kind === 'create'
|
|
&& (string) $job->execution_mode === 'agent'
|
|
&& (string) $job->phase === 'terminal'
|
|
&& (string) $job->outcome === 'failed'
|
|
&& (string) $job->status === 'failed'
|
|
&& (int) $job->dispatch_attempts === 1
|
|
&& (bool) $job->effect_may_have_applied
|
|
&& !(bool) $job->safe_to_resubmit
|
|
&& (string) $job->effect_resolution === 'unknown'
|
|
&& ((string) $event->outcome === 'failed'
|
|
|| ((string) $event->outcome === 'succeeded'
|
|
&& !(bool) $event->effect_may_have_applied
|
|
&& !(bool) $event->safe_to_resubmit));
|
|
}
|
|
|
|
private static function assertResolution(string $resolution): void
|
|
{
|
|
if (!in_array($resolution, ['applied', 'not_applied'], true)) {
|
|
throw new \LogicException('Unsupported recovery resolution.');
|
|
}
|
|
}
|
|
|
|
private static function assertRecoverableEvidence(
|
|
CloudJob $job,
|
|
array $operation,
|
|
string $evidenceSource,
|
|
string $resolution
|
|
): void {
|
|
self::assertResolution($resolution);
|
|
if ($resolution === 'applied') {
|
|
self::assertRecoverableAppliedEvidence(
|
|
$job,
|
|
$operation,
|
|
$evidenceSource
|
|
);
|
|
return;
|
|
}
|
|
self::assertRecoverableNotAppliedEvidence(
|
|
$job,
|
|
$operation,
|
|
$evidenceSource
|
|
);
|
|
}
|
|
|
|
private static function assertOperationIdentity(
|
|
CloudJob $job,
|
|
array $operation,
|
|
string $resolution
|
|
): void {
|
|
self::assertResolution($resolution);
|
|
if ($resolution === 'applied') {
|
|
self::assertAppliedOperationIdentity($job, $operation);
|
|
return;
|
|
}
|
|
self::assertNotAppliedOperationIdentity($job, $operation);
|
|
}
|
|
|
|
private static function assertRecoverableAppliedEvidence(
|
|
CloudJob $job,
|
|
array $operation,
|
|
string $evidenceSource
|
|
): void {
|
|
self::assertEvidenceSource($evidenceSource);
|
|
self::assertAppliedOperationIdentity($job, $operation);
|
|
if (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
|
|
(string) $job->kind
|
|
)
|
|
|| !self::recoverableUnknownEffect($job)) {
|
|
throw new CloudError('任务效果恢复证据不满足安全策略', 409);
|
|
}
|
|
}
|
|
|
|
private static function assertRecoverableNotAppliedEvidence(
|
|
CloudJob $job,
|
|
array $operation,
|
|
string $evidenceSource
|
|
): void {
|
|
self::assertEvidenceSource($evidenceSource);
|
|
self::assertNotAppliedOperationIdentity($job, $operation);
|
|
if (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
|
|
(string) $job->kind
|
|
)
|
|
|| !self::recoverableUnknownEffect($job)) {
|
|
throw new CloudError('任务效果恢复证据不满足安全策略', 409);
|
|
}
|
|
}
|
|
|
|
private static function recoverableUnknownEffect(CloudJob $job): bool
|
|
{
|
|
return (string) $job->execution_mode === 'agent'
|
|
&& (string) $job->phase === 'terminal'
|
|
&& (string) $job->outcome === 'failed'
|
|
&& (string) $job->status === 'failed'
|
|
&& (int) $job->dispatch_attempts === 1
|
|
&& (bool) $job->effect_may_have_applied
|
|
&& !(bool) $job->safe_to_resubmit
|
|
&& (string) $job->effect_resolution === 'unknown';
|
|
}
|
|
|
|
private static function assertAppliedOperationIdentity(
|
|
CloudJob $job,
|
|
array $operation
|
|
): void {
|
|
$operationId = $operation['operation_id'] ?? null;
|
|
if (!is_string($operationId)
|
|
|| !is_string($job->operation_id)
|
|
|| !hash_equals((string) $job->operation_id, $operationId)
|
|
|| ($operation['status'] ?? null) !== 'succeeded'
|
|
|| ($operation['effect_may_have_applied'] ?? null) !== false
|
|
|| ($operation['safe_to_resubmit'] ?? null) !== false) {
|
|
throw new CloudError('任务效果恢复证据与原操作不匹配', 409);
|
|
}
|
|
self::assertOperationBinding($job, $operation, true);
|
|
}
|
|
|
|
private static function assertNotAppliedOperationIdentity(
|
|
CloudJob $job,
|
|
array $operation
|
|
): void {
|
|
$operationId = $operation['operation_id'] ?? null;
|
|
$errorCode = $operation['error_code'] ?? null;
|
|
$result = $operation['result'] ?? null;
|
|
if (!is_string($operationId)
|
|
|| !is_string($job->operation_id)
|
|
|| !hash_equals((string) $job->operation_id, $operationId)
|
|
|| ($operation['status'] ?? null) !== 'failed'
|
|
|| ($operation['effect_may_have_applied'] ?? null) !== false
|
|
|| !is_bool($operation['safe_to_resubmit'] ?? null)
|
|
|| !is_string($errorCode)
|
|
|| preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $errorCode) !== 1
|
|
|| ($result !== null && !is_array($result))) {
|
|
throw new CloudError('任务效果恢复证据与原操作不匹配', 409);
|
|
}
|
|
self::assertOperationBinding($job, $operation, false);
|
|
}
|
|
|
|
private static function assertOperationBinding(
|
|
CloudJob $job,
|
|
array $operation,
|
|
bool $forAppliedEffect
|
|
): void {
|
|
$expected = CloudOperationRecoveryIdentity::expected(
|
|
$job,
|
|
$forAppliedEffect
|
|
);
|
|
CloudOperationRecoveryIdentity::assertMatches($expected, [
|
|
'action' => $operation['action'] ?? null,
|
|
'resource_id' => $operation['resource_id'] ?? null,
|
|
'request_fingerprint' => $operation['request_fingerprint']
|
|
?? $operation['fingerprint'] ?? null,
|
|
]);
|
|
}
|
|
|
|
private static function assertEvidenceSource(string $source): void
|
|
{
|
|
if (preg_match(
|
|
'/^[a-z][a-z0-9_]{2,63}:[A-Za-z0-9_.-]{1,160}$/D',
|
|
$source
|
|
) !== 1) {
|
|
throw new CloudError('任务效果恢复证据来源无效', 409);
|
|
}
|
|
}
|
|
|
|
private static function assertResolvedBy(?int $resolvedBy): void
|
|
{
|
|
if ($resolvedBy !== null && $resolvedBy < 0) {
|
|
throw new CloudError('任务效果恢复审计主体无效', 409);
|
|
}
|
|
}
|
|
|
|
private static function preparedMarker(
|
|
array $operation,
|
|
string $source
|
|
): string {
|
|
self::assertEvidenceSource($source);
|
|
return 'prepared:' . $source . ':'
|
|
. self::preparedOperationHash($operation);
|
|
}
|
|
|
|
/** Hash the exact normalized operation bound into a prepared marker. */
|
|
public static function preparedOperationHash(array $operation): string
|
|
{
|
|
$canonical = self::canonicalize(
|
|
self::recoveryCriticalOperation($operation)
|
|
);
|
|
return hash(
|
|
'sha256',
|
|
json_encode(
|
|
$canonical,
|
|
JSON_THROW_ON_ERROR
|
|
| JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE
|
|
)
|
|
);
|
|
}
|
|
|
|
/** Hash the sole legacy omission accepted from terminal completion wire. */
|
|
public static function legacyCompletionOperationHash(
|
|
array $operation
|
|
): ?string {
|
|
if (($operation['retryable'] ?? null) !== false
|
|
|| !in_array(
|
|
$operation['status'] ?? null,
|
|
['succeeded', 'failed'],
|
|
true
|
|
)) {
|
|
return null;
|
|
}
|
|
unset($operation['retryable']);
|
|
return self::preparedOperationHash($operation);
|
|
}
|
|
|
|
/**
|
|
* Preserve the durable automatic-attempt envelope while replacing its
|
|
* pending payload with the exact prepared evidence. An immutable late
|
|
* completion may replace only a still-pending legacy envelope; it may
|
|
* never overwrite a different prepared conclusion.
|
|
*/
|
|
private static function transitionPreparedMarker(
|
|
string $current,
|
|
string $prepared,
|
|
string $source,
|
|
string $resolution,
|
|
?string $legacyCompatiblePrepared = null
|
|
): string {
|
|
if ($current === '' || hash_equals($prepared, $current)) {
|
|
return $prepared;
|
|
}
|
|
if ($legacyCompatiblePrepared !== null
|
|
&& hash_equals($legacyCompatiblePrepared, $current)) {
|
|
return $prepared;
|
|
}
|
|
$state = self::automaticRecoveryState($current);
|
|
if ($state === null) {
|
|
self::throwPreparedEvidenceConflict($source);
|
|
}
|
|
if (hash_equals($prepared, $state['payload'])) {
|
|
return $current;
|
|
}
|
|
if ($legacyCompatiblePrepared !== null
|
|
&& hash_equals(
|
|
$legacyCompatiblePrepared,
|
|
$state['payload']
|
|
)) {
|
|
return $prepared;
|
|
}
|
|
if ($state['payload'] !== 'pending') {
|
|
$existing = [];
|
|
$late = [];
|
|
$legacy = [];
|
|
$journalStatus = $resolution === 'applied'
|
|
? 'succeeded' : 'failed';
|
|
$compatibleOperationHash = false;
|
|
if ($legacyCompatiblePrepared !== null
|
|
&& preg_match(
|
|
'/^prepared:late_completion_receipt:'
|
|
. '[A-Za-z0-9_.-]{1,160}:([a-f0-9]{64})$/D',
|
|
$legacyCompatiblePrepared,
|
|
$legacy
|
|
) === 1) {
|
|
$compatibleOperationHash = $legacy[1];
|
|
}
|
|
if (str_starts_with($source, 'late_completion_receipt:')
|
|
&& preg_match(
|
|
'/^prepared:automatic_'
|
|
. preg_quote($resolution, '/') . '_'
|
|
. $journalStatus
|
|
. '_nofence:[a-f0-9]{64}:([a-f0-9]{64})$/D',
|
|
$state['payload'],
|
|
$existing
|
|
) === 1
|
|
&& preg_match(
|
|
'/^prepared:late_completion_receipt:[A-Za-z0-9_.-]{1,160}:([a-f0-9]{64})$/D',
|
|
$prepared,
|
|
$late
|
|
) === 1
|
|
&& (hash_equals($existing[1], $late[1])
|
|
|| (is_string($compatibleOperationHash)
|
|
&& hash_equals(
|
|
$existing[1],
|
|
$compatibleOperationHash
|
|
)))) {
|
|
return $prepared;
|
|
}
|
|
self::throwPreparedEvidenceConflict($source);
|
|
}
|
|
if (str_starts_with($source, 'automatic_')) {
|
|
return sprintf(
|
|
'automatic_recovery:v1:%d:%d:%s',
|
|
time(),
|
|
1,
|
|
$prepared
|
|
);
|
|
}
|
|
if (str_starts_with($source, 'late_completion_receipt:')) {
|
|
return $prepared;
|
|
}
|
|
throw new CloudError('任务已有不同的效果恢复证据', 409);
|
|
}
|
|
|
|
private static function throwPreparedEvidenceConflict(
|
|
string $source
|
|
): never {
|
|
if (str_starts_with($source, 'late_completion_receipt:')) {
|
|
throw new CloudRecoveryEvidenceConflict(
|
|
'任务已有冲突的效果恢复证据',
|
|
409
|
|
);
|
|
}
|
|
throw new CloudError('任务已有不同的效果恢复证据', 409);
|
|
}
|
|
|
|
private static function preparedMarkerMatches(
|
|
string $current,
|
|
string $prepared
|
|
): bool {
|
|
if (hash_equals($prepared, $current)) {
|
|
return true;
|
|
}
|
|
$state = self::automaticRecoveryState($current);
|
|
return $state !== null
|
|
&& hash_equals($prepared, $state['payload']);
|
|
}
|
|
|
|
/** @return array{first_at:int,attempts:int,payload:string}|null */
|
|
private static function automaticRecoveryState(string $marker): ?array
|
|
{
|
|
if (preg_match(
|
|
'/^automatic_recovery:v1:([1-9][0-9]*):([1-9][0-9]*):(pending|prepared:(?:automatic_(?:applied|not_applied)_(?:succeeded|failed)_(?:fence|nofence):[a-f0-9]{64}|late_completion_receipt:[A-Za-z0-9_.-]{1,160}):[a-f0-9]{64})$/D',
|
|
$marker,
|
|
$matches
|
|
) !== 1) {
|
|
return null;
|
|
}
|
|
return [
|
|
'first_at' => (int) $matches[1],
|
|
'attempts' => (int) $matches[2],
|
|
'payload' => $matches[3],
|
|
];
|
|
}
|
|
|
|
private static function finalizedRecoveryPayload(CloudJob $job): string
|
|
{
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $job->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
$identity = CloudOperationRecoveryIdentity::expected(
|
|
$job,
|
|
(string) $job->effect_resolution === 'applied'
|
|
);
|
|
// The prepared operation is retained only across the PHP prepare ->
|
|
// Agent fence -> PHP commit crash window. It must never survive in the
|
|
// ordinary terminal audit payload.
|
|
unset($payload['_operation_recovery']);
|
|
$payload[CloudOperationRecoveryIdentity::payloadKey()] =
|
|
CloudOperationRecoveryIdentity::persisted($identity);
|
|
return CloudSecrets::encrypt(
|
|
CloudInput::safeResult($payload),
|
|
'job-payload'
|
|
);
|
|
}
|
|
|
|
private static function recoveryCriticalOperation(array $operation): array
|
|
{
|
|
// The normalized operation is bounded by the completion/inspection
|
|
// schemas. Hash every allow-listed result field because different kinds
|
|
// consume different immutable evidence (snapshot keys, policy digests,
|
|
// catalog generation identifiers, deletion proofs, and so on).
|
|
return $operation;
|
|
}
|
|
|
|
/**
|
|
* Completion receipts before retryable was restored at the PHP adapter
|
|
* produced prepared hashes without that canonical terminal field. Accept
|
|
* only that one historical omission while upgrading the durable marker to
|
|
* the fully bound operation; every other field remains hash-identical.
|
|
*/
|
|
private static function legacyCompletionPreparedMarker(
|
|
array $operation,
|
|
string $source
|
|
): ?string {
|
|
if (!str_starts_with($source, 'late_completion_receipt:')) {
|
|
return null;
|
|
}
|
|
$legacyHash = self::legacyCompletionOperationHash($operation);
|
|
return $legacyHash === null
|
|
? null : 'prepared:' . $source . ':' . $legacyHash;
|
|
}
|
|
|
|
private static function canonicalize($value)
|
|
{
|
|
if (!is_array($value)) {
|
|
return $value;
|
|
}
|
|
if (!array_is_list($value)) {
|
|
ksort($value, SORT_STRING);
|
|
}
|
|
foreach ($value as $key => $item) {
|
|
$value[$key] = self::canonicalize($item);
|
|
}
|
|
return $value;
|
|
}
|
|
|
|
private static function wasReconciledBy(
|
|
CloudJob $job,
|
|
CloudAgentCompletionEvent $event
|
|
): bool {
|
|
return CloudDispatchLedger::isDispatched($job)
|
|
&& (string) $job->phase === 'terminal'
|
|
&& (string) $job->outcome === 'failed'
|
|
&& in_array((string) $job->effect_resolution,
|
|
['applied', 'not_applied'], true)
|
|
&& hash_equals(
|
|
'late_completion_receipt:' . (string) $event->event_id,
|
|
(string) $job->effect_resolution_reason
|
|
);
|
|
}
|
|
|
|
private static function holdDispatchLedgerConflictLocked(
|
|
CloudJob $job
|
|
): void {
|
|
$now = time();
|
|
if ((string) $job->phase === 'terminal'
|
|
&& (string) $job->outcome === 'failed') {
|
|
$job->effect_may_have_applied = 1;
|
|
$job->safe_to_resubmit = 0;
|
|
$job->effect_resolution = 'unknown';
|
|
}
|
|
if (!hash_equals(
|
|
(string) ($job->finalization_error_code ?? ''),
|
|
CloudDispatchLedger::CONFLICT_CODE
|
|
)) {
|
|
$job->read_at = null;
|
|
}
|
|
$job->finalization_incident = 1;
|
|
$job->finalization_error_code =
|
|
CloudDispatchLedger::CONFLICT_CODE;
|
|
$job->lease_token = null;
|
|
$job->lease_until = 0;
|
|
$job->available_at = $now + self::INCIDENT_RETRY_SECONDS;
|
|
$job->updated_at = $now;
|
|
$job->save();
|
|
}
|
|
}
|