You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1718 lines
72 KiB
PHP
1718 lines
72 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\CloudAgentCompletionEvent;
|
|
use App\Models\CloudJob;
|
|
use App\Models\CloudTemplate;
|
|
use App\Models\CloudUserIso;
|
|
|
|
/**
|
|
* Safely converges one timed-out Agent operation from read-only evidence.
|
|
*
|
|
* This coordinator never calls the original VM mutation endpoint. Its only
|
|
* Agent write is the operation-bound journal fence resolution, after PHP has
|
|
* durably prepared the corresponding ledger conclusion.
|
|
*/
|
|
final class CloudAutomaticOperationRecoveryService
|
|
{
|
|
// One recovery may spend up to four 30-second Agent calls (inspection,
|
|
// fence read, one write, and lost-response verification).
|
|
private const CLAIM_SECONDS = 180;
|
|
private const RETRY_SECONDS = 300;
|
|
private const INSPECTION_MAX_AGE_SECONDS = 120;
|
|
private const LANE_RECEIPT = 0;
|
|
private const LANE_LEAF = 1;
|
|
private const LANE_AGGREGATE = 2;
|
|
private const LANE_COUNT = 3;
|
|
|
|
/** Next recovery lane for the long-lived recovery-only worker. */
|
|
private int $nextLane = self::LANE_RECEIPT;
|
|
|
|
/** Claim and inspect at most one unknown-effect or corrupt-ledger task. */
|
|
public function tick(): bool
|
|
{
|
|
// Rotate the three independent recovery lanes. A lane with no ready
|
|
// work is skipped in this same tick, so sparse queues retain full
|
|
// throughput. Under sustained mixed load every lane receives one
|
|
// bounded opportunity per three completed ticks; immutable receipts,
|
|
// Agent leaves and aggregate projection can no longer starve one
|
|
// another.
|
|
$claimed = null;
|
|
for ($offset = 0; $offset < self::LANE_COUNT; $offset++) {
|
|
$lane = ($this->nextLane + $offset) % self::LANE_COUNT;
|
|
if ($lane === self::LANE_RECEIPT) {
|
|
$claimed = $this->claim(true);
|
|
} elseif ($lane === self::LANE_LEAF) {
|
|
// Do not let a due job with a received event consume both the
|
|
// receipt lane and the ordinary leaf lane.
|
|
$claimed = $this->claim(false, true);
|
|
} elseif (CloudJobService::reconcileOneLegacyAggregateEffect()) {
|
|
$this->nextLane = self::LANE_RECEIPT;
|
|
return true;
|
|
}
|
|
if ($claimed !== null) {
|
|
$this->nextLane = ($lane + 1) % self::LANE_COUNT;
|
|
break;
|
|
}
|
|
}
|
|
if ($claimed === null) {
|
|
return false;
|
|
}
|
|
$job = $claimed['job'];
|
|
if (CloudDispatchLedger::isInvalid($job)) {
|
|
// A corrupt generation counter is neither proof that the request
|
|
// stayed local nor a valid dispatched generation. Keep the job in
|
|
// the recovery queue, retain any immutable receipt, and wait for
|
|
// an operator to repair the counter to exactly one.
|
|
CloudOperationCompletionService::deferDispatchLedgerConflict(
|
|
$job
|
|
);
|
|
return true;
|
|
}
|
|
if ($claimed['mode'] === 'dispatch_repair') {
|
|
$this->resumeDispatchLedgerRepair($job);
|
|
return true;
|
|
}
|
|
if (in_array($claimed['mode'], [
|
|
'invalid_envelope', 'invalid_state', 'resolved_incident',
|
|
'aggregate_state', 'invalid_identity',
|
|
'dispatch_repair_invalid_envelope',
|
|
'dispatch_repair_invalid_identity',
|
|
], true)) {
|
|
$incidentCode = strtolower(trim(
|
|
(string) ($job->finalization_error_code ?? '')
|
|
));
|
|
$releaseCode = 'recovery_instance_state_conflict';
|
|
if ($claimed['mode'] === 'invalid_identity') {
|
|
$releaseCode = 'recovery_operation_identity_conflict';
|
|
} elseif ($claimed['mode']
|
|
=== 'dispatch_repair_invalid_identity') {
|
|
$releaseCode =
|
|
'recovery_dispatch_repair_operation_identity_conflict';
|
|
} elseif ($claimed['mode']
|
|
=== 'dispatch_repair_invalid_envelope') {
|
|
$releaseCode =
|
|
'recovery_dispatch_repair_instance_state_conflict';
|
|
} elseif ($claimed['mode'] === 'resolved_incident'
|
|
&& preg_match(
|
|
'/^recovery_[a-z0-9_]{1,55}$/D',
|
|
$incidentCode
|
|
) === 1) {
|
|
$releaseCode = $incidentCode;
|
|
} elseif (in_array($claimed['mode'], [
|
|
'aggregate_state', 'invalid_state',
|
|
], true) && ((string) $job->execution_mode === 'aggregate'
|
|
|| CloudJobService::isAggregateKind((string) $job->kind))) {
|
|
$releaseCode = 'recovery_aggregate_state_conflict';
|
|
}
|
|
$this->releaseLocalClaim($job, $releaseCode);
|
|
return true;
|
|
}
|
|
if ($claimed['mode'] === 'local_finalizer') {
|
|
try {
|
|
(new CloudWorkerService())->retryUndispatchedFinalization(
|
|
(int) $job->id,
|
|
(string) $job->lease_token
|
|
);
|
|
} catch (\Throwable $error) {
|
|
$this->releaseLocalClaim(
|
|
$job,
|
|
self::recoveryErrorCode($error)
|
|
);
|
|
}
|
|
return true;
|
|
}
|
|
if ($claimed['event'] !== null) {
|
|
try {
|
|
$this->settleReceivedEvent($job, $claimed['event']);
|
|
} catch (\Throwable $error) {
|
|
$this->releaseReceivedEventClaim(
|
|
$job,
|
|
self::receiptRetryDelay($claimed['event']),
|
|
self::recoveryErrorCode($error)
|
|
);
|
|
error_log(sprintf(
|
|
'cloud received completion recovery pending job_id=%d kind=%s error=%s',
|
|
(int) $job->id,
|
|
(string) $job->kind,
|
|
get_class($error)
|
|
));
|
|
}
|
|
return true;
|
|
}
|
|
|
|
try {
|
|
$pending = $this->recover($job);
|
|
if ($pending !== null) {
|
|
$this->releaseForRetry(
|
|
$job,
|
|
$pending['delay'],
|
|
$pending['error_code']
|
|
);
|
|
}
|
|
} catch (\Throwable $error) {
|
|
// Agent downtime, stale evidence, malformed host state and a PHP
|
|
// crash-window retry all remain fenced. A later worker performs a
|
|
// new read-only inspection; the original mutation is never sent.
|
|
$this->releaseForRetry(
|
|
$job,
|
|
self::RETRY_SECONDS,
|
|
self::recoveryErrorCode($error)
|
|
);
|
|
error_log(sprintf(
|
|
'cloud automatic operation recovery pending job_id=%d kind=%s error=%s',
|
|
(int) $job->id,
|
|
(string) $job->kind,
|
|
get_class($error)
|
|
));
|
|
}
|
|
return true;
|
|
}
|
|
|
|
private function settleReceivedEvent(
|
|
CloudJob $job,
|
|
CloudAgentCompletionEvent $event
|
|
): void {
|
|
if (!CloudOperationCompletionService::isExactStoredEvent($event)) {
|
|
CloudOperationCompletionService::mark($event, 'rejected');
|
|
$this->releaseReceivedEventClaim(
|
|
$job,
|
|
self::RETRY_SECONDS,
|
|
'recovery_receipt_invalid'
|
|
);
|
|
return;
|
|
}
|
|
$expectedIdentity = CloudOperationRecoveryIdentity::expected(
|
|
$job,
|
|
(string) $event->outcome === 'succeeded'
|
|
);
|
|
if (!CloudOperationRecoveryIdentity::same(
|
|
$expectedIdentity,
|
|
CloudOperationCompletionService::identity($event)
|
|
)) {
|
|
CloudOperationCompletionService::isolateIdentityMismatch(
|
|
$job,
|
|
$event
|
|
);
|
|
return;
|
|
}
|
|
$disposition = CloudOperationRecoveryService::settleLateReceipt(
|
|
$job,
|
|
$event
|
|
);
|
|
if ($disposition === 'applied') {
|
|
return;
|
|
}
|
|
$this->releaseReceivedEventClaim(
|
|
$job,
|
|
$disposition === 'received'
|
|
? self::receiptRetryDelay($event) : 0
|
|
);
|
|
}
|
|
|
|
private static function receiptRetryDelay(
|
|
CloudAgentCompletionEvent $event
|
|
): int {
|
|
return self::RETRY_SECONDS;
|
|
}
|
|
|
|
private function releaseReceivedEventClaim(
|
|
CloudJob $claimed,
|
|
int $delay,
|
|
?string $errorCode = null
|
|
): void {
|
|
$now = time();
|
|
CloudJob::where('id', (int) $claimed->id)
|
|
->where('effect_resolution', 'unknown')
|
|
->where('lease_token', (string) $claimed->lease_token)
|
|
->update(array_filter([
|
|
'lease_token' => null,
|
|
// Receipt evidence bypasses an older inspection backoff, so
|
|
// use the lease clock as the retry gate if local settlement
|
|
// itself fails. This prevents a received event hot loop.
|
|
'lease_until' => $delay > 0 ? $now + $delay : 0,
|
|
'available_at' => $delay > 0 ? $now + $delay : 0,
|
|
'finalization_incident' => $errorCode === null ? null : 1,
|
|
'finalization_error_code' => $errorCode,
|
|
'read_at' => $errorCode === null ? null : null,
|
|
'updated_at' => $now,
|
|
], static fn ($value, $key): bool => $errorCode !== null
|
|
|| !in_array($key, [
|
|
'finalization_incident',
|
|
'finalization_error_code',
|
|
'read_at',
|
|
], true), ARRAY_FILTER_USE_BOTH));
|
|
}
|
|
|
|
/**
|
|
* Select one fair recovery candidate. A received completion event makes
|
|
* an otherwise exhausted job eligible, but is loaded only by its indexed
|
|
* host+operation key; the event table is never scanned by disposition.
|
|
*
|
|
* @return array{job:CloudJob,event:?CloudAgentCompletionEvent,mode:string}|null
|
|
*/
|
|
private function claim(
|
|
bool $receivedOnly,
|
|
bool $excludeReceived = false
|
|
): ?array
|
|
{
|
|
return (new CloudJob())->getConnection()->transaction(
|
|
static function () use (
|
|
$receivedOnly,
|
|
$excludeReceived
|
|
): ?array {
|
|
$now = time();
|
|
$job = null;
|
|
$mode = 'agent';
|
|
if (!$receivedOnly) {
|
|
// Recover historical fail-open rows regardless of their
|
|
// apparent terminal outcome. A corrupt non-zero dispatch
|
|
// counter invalidates both "succeeded" and "not applied"
|
|
// projections until an operator repairs it to exactly 1.
|
|
// They are claimed only on the ordinary pass so a damaged
|
|
// row cannot starve a newly received immutable receipt.
|
|
$job = CloudJob::query()
|
|
->where('execution_mode', 'agent')
|
|
->whereIn('phase', [
|
|
'accepted',
|
|
'dispatching',
|
|
'awaiting_receipt',
|
|
'terminal',
|
|
])
|
|
->where(static function ($invalid): void {
|
|
$invalid->whereNull('dispatch_attempts')
|
|
->orWhereNotIn(
|
|
'dispatch_attempts',
|
|
[0, 1]
|
|
);
|
|
})
|
|
->where('available_at', '<=', $now)
|
|
->where('lease_until', '<=', $now)
|
|
->orderBy('available_at')
|
|
->orderBy('id')
|
|
->lockForUpdate()
|
|
->first();
|
|
}
|
|
if ($job === null && !$receivedOnly) {
|
|
// A valid zero counter proves that no Agent mutation was
|
|
// sent. Retry only PHP's kind-specific not-applied ledger
|
|
// finalizer; never inspect or contact the host.
|
|
$job = CloudJob::query()
|
|
->where('execution_mode', 'agent')
|
|
->where('phase', 'terminal')
|
|
->where('outcome', 'failed')
|
|
->where('status', 'failed')
|
|
->where('dispatch_attempts', 0)
|
|
->where('finalization_incident', 1)
|
|
->where(static function ($incident): void {
|
|
$incident->whereNull('finalization_error_code')
|
|
->orWhereNotIn(
|
|
'finalization_error_code',
|
|
CloudDispatchLedger::repairIncidentCodes()
|
|
);
|
|
})
|
|
->where(static function ($finalizer): void {
|
|
$finalizer->whereIn(
|
|
'kind',
|
|
CloudOperationRecoveryPolicy::kinds()
|
|
)->orWhere(static function ($create): void {
|
|
// Generation zero is definitive no-effect
|
|
// evidence for create too. Its financial and
|
|
// placement ledgers use the specialised atomic
|
|
// local finalizer, never the Agent mutation.
|
|
$create->where('kind', 'create')
|
|
->whereNotNull('order_id');
|
|
});
|
|
})->where('available_at', '<=', $now)
|
|
->where('lease_until', '<=', $now)
|
|
->orderBy('available_at')
|
|
->orderBy('id')
|
|
->lockForUpdate()
|
|
->first();
|
|
if ($job !== null) {
|
|
$mode = 'local_finalizer';
|
|
}
|
|
}
|
|
if ($job === null) {
|
|
// A terminal row whose invalid counter was repaired keeps
|
|
// a sticky blocker until its immutable state is checked.
|
|
$repair = CloudJob::query()
|
|
->where('execution_mode', 'agent')
|
|
->where('phase', 'terminal')
|
|
->where('finalization_incident', 1)
|
|
->whereIn(
|
|
'finalization_error_code',
|
|
CloudDispatchLedger::repairIncidentCodes()
|
|
)->whereIn('dispatch_attempts', [0, 1])
|
|
->where('lease_until', '<=', $now);
|
|
if ($receivedOnly) {
|
|
$repair->whereExists(
|
|
static function ($events): void {
|
|
self::receivedEventExists($events);
|
|
}
|
|
);
|
|
} else {
|
|
$repair->where('available_at', '<=', $now);
|
|
}
|
|
$job = $repair->orderBy('available_at')
|
|
->orderBy('id')->lockForUpdate()->first();
|
|
if ($job !== null) {
|
|
$mode = 'dispatch_repair';
|
|
}
|
|
}
|
|
if ($job === null && !$receivedOnly) {
|
|
// A terminal row may have a final effect and still carry a
|
|
// failed PHP projection/finalizer incident. It is already
|
|
// resolved on the host, so only retain the local incident
|
|
// with backoff; never inspect or mutate Agent state.
|
|
$job = CloudJob::query()
|
|
->where('execution_mode', 'agent')
|
|
->where('phase', 'terminal')
|
|
->whereIn('dispatch_attempts', [0, 1])
|
|
->whereIn(
|
|
'effect_resolution',
|
|
['applied', 'not_applied']
|
|
)->where('finalization_incident', 1)
|
|
->where(static function ($incident): void {
|
|
$incident->whereNull('finalization_error_code')
|
|
->orWhereNotIn(
|
|
'finalization_error_code',
|
|
CloudDispatchLedger::repairIncidentCodes()
|
|
);
|
|
})->whereIn(
|
|
'kind',
|
|
CloudOperationRecoveryPolicy::kinds()
|
|
)->where('available_at', '<=', $now)
|
|
->where('lease_until', '<=', $now)
|
|
->orderBy('available_at')
|
|
->orderBy('id')
|
|
->lockForUpdate()
|
|
->first();
|
|
if ($job !== null) {
|
|
$mode = 'resolved_incident';
|
|
}
|
|
}
|
|
if ($job === null && !$receivedOnly) {
|
|
// A published effect with a contradictory immutable
|
|
// terminal tuple is a PHP-ledger conflict. Keep it in a
|
|
// local-only retry loop; never ask the Agent to reinterpret
|
|
// or repeat the already concluded mutation.
|
|
$job = CloudJob::query()
|
|
->where('phase', 'terminal')
|
|
->whereIn(
|
|
'kind',
|
|
CloudOperationRecoveryPolicy::kinds()
|
|
)->where(static function ($invalid): void {
|
|
CloudJobService::applyInvalidResolvedTerminalTuple(
|
|
$invalid
|
|
);
|
|
})->where('available_at', '<=', $now)
|
|
->where('lease_until', '<=', $now)
|
|
->orderBy('available_at')
|
|
->orderBy('id')
|
|
->lockForUpdate()
|
|
->first();
|
|
if ($job !== null) {
|
|
$mode = 'invalid_state';
|
|
}
|
|
}
|
|
if ($job === null && !$receivedOnly) {
|
|
// A valid dispatched generation with a damaged immutable
|
|
// envelope must not disappear from the recovery queue.
|
|
// This mode records the concrete conflict without making
|
|
// any Agent request.
|
|
$job = CloudJob::query()
|
|
->where('execution_mode', 'agent')
|
|
->where('phase', 'terminal')
|
|
->where('outcome', 'failed')
|
|
->where('status', 'failed')
|
|
->where('dispatch_attempts', 1)
|
|
->where('effect_may_have_applied', 1)
|
|
->where('safe_to_resubmit', 0)
|
|
->where('effect_resolution', 'unknown')
|
|
->whereIn(
|
|
'kind',
|
|
CloudOperationRecoveryPolicy::kinds()
|
|
)->where(static function ($repair): void {
|
|
$repair->where('finalization_incident', 0)
|
|
->orWhereNull('finalization_error_code')
|
|
->orWhereNotIn(
|
|
'finalization_error_code',
|
|
CloudDispatchLedger::repairIncidentCodes()
|
|
);
|
|
})->where(static function ($broken): void {
|
|
$broken->whereNull('host_id')
|
|
->orWhere('host_id', '<=', 0)
|
|
->orWhereNull('operation_id')
|
|
->orWhereRaw('LENGTH(operation_id) <> 32')
|
|
->orWhereNull('dispatched_at')
|
|
->orWhere('dispatched_at', '<=', 0);
|
|
})->where('available_at', '<=', $now)
|
|
->where('lease_until', '<=', $now)
|
|
->orderBy('available_at')
|
|
->orderBy('id')
|
|
->lockForUpdate()
|
|
->first();
|
|
if ($job !== null) {
|
|
$mode = 'invalid_envelope';
|
|
}
|
|
}
|
|
if ($job === null && !$receivedOnly) {
|
|
// A terminal unknown-effect row has exactly one trustworthy
|
|
// safety tuple. Contradictory historical/data-damaged rows
|
|
// stay in a PHP-only incident loop and never contact Agent.
|
|
$job = CloudJob::query()
|
|
->where('phase', 'terminal')
|
|
->where('effect_resolution', 'unknown')
|
|
->where(static function ($scope): void {
|
|
$scope->where(static function ($leaf): void {
|
|
$leaf->whereIn('dispatch_attempts', [0, 1])
|
|
->whereIn(
|
|
'kind',
|
|
CloudOperationRecoveryPolicy::kinds()
|
|
);
|
|
})->orWhere(
|
|
static function ($aggregate): void {
|
|
$aggregate->where(
|
|
'execution_mode',
|
|
'aggregate'
|
|
)->whereNull('parent_job_id')
|
|
->whereIn(
|
|
'kind',
|
|
CloudJobService::aggregateKinds()
|
|
);
|
|
}
|
|
);
|
|
})->where(static function ($invalid): void {
|
|
$invalid->where(static function ($mode): void {
|
|
$mode->whereIn(
|
|
'kind',
|
|
CloudOperationRecoveryPolicy::kinds()
|
|
)->where(static function ($broken): void {
|
|
$broken->whereNull('execution_mode')
|
|
->orWhere(
|
|
'execution_mode',
|
|
'<>',
|
|
'agent'
|
|
);
|
|
});
|
|
})->orWhere(static function ($tuple): void {
|
|
self::applyInvalidTerminalTuple($tuple);
|
|
});
|
|
})->where('available_at', '<=', $now)
|
|
->where('lease_until', '<=', $now)
|
|
->orderBy('available_at')
|
|
->orderBy('id')
|
|
->lockForUpdate()
|
|
->first();
|
|
if ($job !== null) {
|
|
$mode = 'invalid_state';
|
|
}
|
|
}
|
|
if ($job === null) {
|
|
$job = CloudJob::query()
|
|
->where('execution_mode', 'agent')
|
|
->where('phase', 'terminal')
|
|
->where('outcome', 'failed')
|
|
->where('status', 'failed')
|
|
->where('dispatch_attempts', 1)
|
|
->where('effect_may_have_applied', 1)
|
|
->where('safe_to_resubmit', 0)
|
|
->where('effect_resolution', 'unknown')
|
|
->where(static function ($repair): void {
|
|
$repair->where('finalization_incident', 0)
|
|
->orWhereNull('finalization_error_code')
|
|
->orWhereNotIn(
|
|
'finalization_error_code',
|
|
CloudDispatchLedger::repairIncidentCodes()
|
|
);
|
|
})
|
|
->whereNotNull('operation_id')
|
|
->whereNotNull('host_id')
|
|
->whereNotNull('dispatched_at')
|
|
->where(static function ($eligible) use (
|
|
$now,
|
|
$receivedOnly,
|
|
$excludeReceived
|
|
): void {
|
|
$eligible->where(
|
|
static function ($policy) use (
|
|
$now,
|
|
$receivedOnly,
|
|
$excludeReceived
|
|
): void {
|
|
$policy->whereIn(
|
|
'kind',
|
|
// Create uses a separate financial
|
|
// finalizer, never the generic replay or
|
|
// effect-projection path below.
|
|
CloudOperationRecoveryPolicy::automaticKinds()
|
|
)->where(
|
|
static function ($ready) use (
|
|
$now,
|
|
$receivedOnly,
|
|
$excludeReceived
|
|
): void {
|
|
if ($receivedOnly) {
|
|
$ready->whereExists(
|
|
static function (
|
|
$events
|
|
): void {
|
|
self::receivedEventExists(
|
|
$events
|
|
);
|
|
}
|
|
);
|
|
return;
|
|
}
|
|
$ready->where(
|
|
'available_at',
|
|
'<=',
|
|
$now
|
|
);
|
|
if (!$excludeReceived) {
|
|
$ready->orWhereExists(
|
|
static function ($events): void {
|
|
self::receivedEventExists(
|
|
$events
|
|
);
|
|
}
|
|
);
|
|
}
|
|
}
|
|
);
|
|
}
|
|
);
|
|
})
|
|
->when(
|
|
$excludeReceived,
|
|
static function ($candidate): void {
|
|
$candidate->whereNotExists(
|
|
static function ($events): void {
|
|
self::receivedEventExists($events);
|
|
}
|
|
);
|
|
}
|
|
)
|
|
->where('lease_until', '<=', $now)
|
|
->orderByRaw(
|
|
'CASE WHEN EXISTS (SELECT 1 FROM cloud_agent_completion_events AS recovery_priority_events WHERE recovery_priority_events.host_id = cloud_jobs.host_id AND recovery_priority_events.operation_id = cloud_jobs.operation_id AND recovery_priority_events.disposition = ?) THEN 0 ELSE 1 END',
|
|
['received']
|
|
)
|
|
->orderBy('available_at')
|
|
->orderBy('id')
|
|
->lockForUpdate()
|
|
->first();
|
|
}
|
|
if ($job === null) {
|
|
return null;
|
|
}
|
|
// SQL only finds a bounded candidate. Revalidate the complete
|
|
// immutable Agent envelope on the hydrated row before loading
|
|
// a receipt or allowing any Agent/fence path. A damaged field
|
|
// is a local ledger incident; immutable evidence stays queued.
|
|
$envelopeConflict =
|
|
CloudJobService::hasAgentRecoveryEnvelopeConflict($job);
|
|
if ($envelopeConflict) {
|
|
$identityConflict = CloudJobService::
|
|
hasAgentRecoveryIdentityConflict($job);
|
|
if ($mode === 'dispatch_repair') {
|
|
$mode = $identityConflict
|
|
? 'dispatch_repair_invalid_identity'
|
|
: 'dispatch_repair_invalid_envelope';
|
|
} else {
|
|
$mode = $identityConflict
|
|
? 'invalid_identity' : 'invalid_envelope';
|
|
}
|
|
}
|
|
if (!$envelopeConflict
|
|
&& (string) $job->phase === 'terminal'
|
|
&& (CloudTemplate::where(
|
|
'active_job_id',
|
|
(int) $job->id
|
|
)->exists() || CloudUserIso::where(
|
|
'active_job_id',
|
|
(int) $job->id
|
|
)->exists())) {
|
|
$job->lease_token = bin2hex(random_bytes(16));
|
|
$job->lease_until = $now + self::CLAIM_SECONDS;
|
|
$job->save();
|
|
return [
|
|
'job' => $job,
|
|
'event' => null,
|
|
'mode' => 'aggregate_state',
|
|
];
|
|
}
|
|
if (CloudDispatchLedger::isInvalid($job)) {
|
|
// Do not inspect or consume a receipt while the dispatch
|
|
// generation is corrupt. Lease only long enough to hand
|
|
// the row to the fail-closed normalizer after commit.
|
|
$job->lease_token = bin2hex(random_bytes(16));
|
|
$job->lease_until = $now + self::CLAIM_SECONDS;
|
|
$job->save();
|
|
return [
|
|
'job' => $job,
|
|
'event' => null,
|
|
'mode' => 'invalid',
|
|
];
|
|
}
|
|
if ($mode === 'local_finalizer'
|
|
|| $mode === 'invalid_envelope'
|
|
|| $mode === 'invalid_state'
|
|
|| $mode === 'resolved_incident'
|
|
|| $mode === 'aggregate_state'
|
|
|| $mode === 'invalid_identity'
|
|
|| $mode === 'dispatch_repair_invalid_envelope'
|
|
|| $mode === 'dispatch_repair_invalid_identity'
|
|
|| $mode === 'dispatch_repair') {
|
|
$event = null;
|
|
if ($mode === 'dispatch_repair'
|
|
&& is_string($job->completion_event_id)
|
|
&& $job->completion_event_id !== '') {
|
|
$event = CloudAgentCompletionEvent::where(
|
|
'event_id',
|
|
(string) $job->completion_event_id
|
|
)->lockForUpdate()->first();
|
|
}
|
|
$job->lease_token = bin2hex(random_bytes(16));
|
|
$job->lease_until = $now + self::CLAIM_SECONDS;
|
|
$job->save();
|
|
return [
|
|
'job' => $job,
|
|
'event' => $event,
|
|
'mode' => $mode,
|
|
];
|
|
}
|
|
$event = CloudAgentCompletionEvent::where(
|
|
'host_id',
|
|
(int) $job->host_id
|
|
)->where(
|
|
'operation_id',
|
|
(string) $job->operation_id
|
|
)->where('disposition', 'received')
|
|
->lockForUpdate()
|
|
->first();
|
|
if ($event !== null) {
|
|
$job->lease_token = bin2hex(random_bytes(16));
|
|
$job->lease_until = $now + self::CLAIM_SECONDS;
|
|
$job->save();
|
|
return [
|
|
'job' => $job,
|
|
'event' => $event,
|
|
'mode' => 'agent',
|
|
];
|
|
}
|
|
$attempt = self::nextAttemptMarker(
|
|
(string) $job->effect_resolution_reason,
|
|
$now
|
|
);
|
|
// The dedicated create coordinator owns its encrypted
|
|
// prepare/commit binding. Never wrap or replace that marker
|
|
// when resuming a fence-write/financial-commit crash window.
|
|
if ((string) $job->kind === 'create'
|
|
&& preg_match(
|
|
'/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D',
|
|
(string) $job->effect_resolution_reason
|
|
) === 1) {
|
|
$attempt = (string) $job->effect_resolution_reason;
|
|
}
|
|
if ($attempt === false) {
|
|
$job->finalization_incident = 1;
|
|
$job->finalization_error_code =
|
|
'recovery_state_invalid';
|
|
$job->read_at = null;
|
|
} else {
|
|
$job->effect_resolution_reason = $attempt;
|
|
if (in_array(
|
|
(string) ($job->finalization_error_code ?? ''),
|
|
[
|
|
'recovery_limit_reached',
|
|
'recovery_receipt_limit_reached',
|
|
],
|
|
true
|
|
)) {
|
|
$job->finalization_incident = 0;
|
|
$job->finalization_error_code = null;
|
|
$job->read_at = null;
|
|
}
|
|
}
|
|
$job->lease_token = bin2hex(random_bytes(16));
|
|
$job->lease_until = $now + self::CLAIM_SECONDS;
|
|
$job->save();
|
|
return [
|
|
'job' => $job,
|
|
'event' => null,
|
|
'mode' => 'agent',
|
|
];
|
|
},
|
|
3
|
|
);
|
|
}
|
|
|
|
/** Add the negation of the canonical failed/effectful/non-retryable tuple. */
|
|
private static function applyInvalidTerminalTuple($invalid): void
|
|
{
|
|
$invalid->where(static function ($tuple): void {
|
|
$tuple->whereNull('outcome')
|
|
->orWhere('outcome', '<>', 'failed')
|
|
->orWhereNull('status')
|
|
->orWhere('status', '<>', 'failed')
|
|
->orWhereNull('effect_may_have_applied')
|
|
->orWhere('effect_may_have_applied', '<>', 1)
|
|
->orWhereNull('safe_to_resubmit')
|
|
->orWhere('safe_to_resubmit', '<>', 0);
|
|
})->orWhere(static function ($neverDispatched): void {
|
|
$neverDispatched->where('dispatch_attempts', 0)
|
|
->where('finalization_incident', 0);
|
|
});
|
|
}
|
|
|
|
private static function receivedEventExists($events): void
|
|
{
|
|
$events->selectRaw('1')
|
|
->from('cloud_agent_completion_events as recovery_events')
|
|
->whereColumn(
|
|
'recovery_events.host_id',
|
|
'cloud_jobs.host_id'
|
|
)
|
|
->whereColumn(
|
|
'recovery_events.operation_id',
|
|
'cloud_jobs.operation_id'
|
|
)
|
|
->where('recovery_events.disposition', 'received');
|
|
}
|
|
|
|
/**
|
|
* @return array{delay:int,error_code:string}|null Retry state, or null
|
|
* after a committed resolution.
|
|
*/
|
|
private function recover(CloudJob $claimed): ?array
|
|
{
|
|
$job = CloudJob::find((int) $claimed->id);
|
|
if ($job === null
|
|
|| !is_string($job->lease_token)
|
|
|| !hash_equals(
|
|
(string) $claimed->lease_token,
|
|
(string) $job->lease_token
|
|
)) {
|
|
throw new CloudError('自动核对租约已失效', 409);
|
|
}
|
|
if ((string) $job->kind === 'create') {
|
|
CloudCreateFailureRecoveryService::recover($job);
|
|
return null;
|
|
}
|
|
$context = self::recoveryContext($job);
|
|
|
|
// A late immutable completion receipt is a PHP-local two-phase
|
|
// recovery. If the accepting request crashed after prepare, rebuild
|
|
// the durable event and finish without contacting the Agent.
|
|
$lateEventId = self::preparedLateReceiptEventId($job);
|
|
if ($lateEventId !== null) {
|
|
$event = CloudAgentCompletionEvent::where(
|
|
'event_id',
|
|
$lateEventId
|
|
)->first();
|
|
if ($event === null
|
|
|| (string) $event->disposition !== 'received') {
|
|
throw new CloudError('晚到回执恢复记录不完整', 409);
|
|
}
|
|
$disposition = CloudOperationRecoveryService::settleLateReceipt(
|
|
$job,
|
|
$event
|
|
);
|
|
$resolved = CloudJob::find((int) $job->id);
|
|
$appliedEvent = CloudAgentCompletionEvent::where(
|
|
'event_id',
|
|
$lateEventId
|
|
)->first();
|
|
if ($disposition !== 'applied'
|
|
|| $resolved === null
|
|
|| !in_array(
|
|
(string) $resolved->effect_resolution,
|
|
['applied', 'not_applied'],
|
|
true
|
|
)
|
|
|| $appliedEvent === null
|
|
|| (string) $appliedEvent->disposition !== 'applied') {
|
|
throw new CloudError('晚到回执尚未完成安全收敛', 409);
|
|
}
|
|
return null;
|
|
}
|
|
$prepared = self::preparedAutomaticIntent($job, $context);
|
|
if ($prepared !== null) {
|
|
$reason = self::recoveryReason(
|
|
$job,
|
|
$prepared['source'],
|
|
$prepared['resolution']
|
|
);
|
|
if ($prepared['fence_resolution_required']) {
|
|
try {
|
|
CloudAdminOperationRecoveryService::
|
|
preparedFenceResolutionForRecovery(
|
|
(string) $job->id,
|
|
$prepared['resolution'],
|
|
$reason
|
|
);
|
|
self::commitPrepared($job, $prepared);
|
|
return null;
|
|
} catch (CloudAgentError $error) {
|
|
if ($error->agentHttpStatus !== 404) {
|
|
throw $error;
|
|
}
|
|
// The prior process did not commit a fence resolution.
|
|
// Continue with a fresh inspection before any new write.
|
|
}
|
|
} else {
|
|
// Exact Journal success and exact no-effect failure never
|
|
// retained an Agent fence. PHP may finish an already-prepared
|
|
// intent without inventing a fence read or write.
|
|
self::commitPrepared($job, $prepared);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
$inspection = CloudAdminOperationRecoveryService::inspectForRecovery(
|
|
(string) $claimed->id
|
|
);
|
|
self::assertFreshInspection($job, $inspection);
|
|
|
|
// A poisoned completion outbox means the Agent cannot prove that its
|
|
// immutable result was delivered intact. Even if the journal's local
|
|
// projection says applied/not-applied, that projection must remain
|
|
// inconclusive until the outbox/receipt record is repaired.
|
|
$deliveryIncident = self::completionDeliveryIncident($inspection);
|
|
if ($deliveryIncident !== null) {
|
|
return self::pending($deliveryIncident);
|
|
}
|
|
|
|
// A resolved Agent fence proves that some coordinator already wrote
|
|
// an immutable verdict; it does not prove that PHP prepared the same
|
|
// verdict. Only the prepared-intent resume path above may consume it.
|
|
if (($inspection['fence_state'] ?? null) === 'resolved') {
|
|
return self::pending('recovery_evidence_conflict');
|
|
}
|
|
|
|
$resolution = (string) ($inspection['effect_resolution'] ?? '');
|
|
if (in_array($resolution, ['partial', 'inconclusive'], true)) {
|
|
return self::pending(
|
|
$resolution === 'partial'
|
|
? 'recovery_evidence_partial'
|
|
: 'recovery_evidence_inconclusive'
|
|
);
|
|
}
|
|
if (!in_array($resolution, ['applied', 'not_applied'], true)) {
|
|
throw new CloudError('宿主机效果核对结论无效', 409);
|
|
}
|
|
|
|
$operation = self::operationFromInspection(
|
|
$job,
|
|
$context,
|
|
$inspection,
|
|
$resolution
|
|
);
|
|
$evidenceSource = self::evidenceSource(
|
|
$job,
|
|
$context,
|
|
$inspection,
|
|
$resolution
|
|
);
|
|
$reason = self::recoveryReason($job, $evidenceSource, $resolution);
|
|
|
|
if ($resolution === 'applied') {
|
|
CloudOperationRecoveryService::prepareApplied(
|
|
(int) $job->id,
|
|
$operation,
|
|
$evidenceSource
|
|
);
|
|
} else {
|
|
CloudOperationRecoveryService::prepareNotApplied(
|
|
(int) $job->id,
|
|
$operation,
|
|
$evidenceSource
|
|
);
|
|
}
|
|
|
|
$journalStatus = (string) $inspection['journal_status'];
|
|
$fenceResolutionRequired =
|
|
(bool) $inspection['fence_resolution_required'];
|
|
$preparedIntent = [
|
|
'resolution' => $resolution,
|
|
'journal_status' => $journalStatus,
|
|
'fence_resolution_required' => $fenceResolutionRequired,
|
|
'source' => $evidenceSource,
|
|
'operation' => $operation,
|
|
];
|
|
if ($fenceResolutionRequired) {
|
|
self::resolveFenceAndCommit(
|
|
$job,
|
|
$preparedIntent,
|
|
$reason,
|
|
$inspection
|
|
);
|
|
return null;
|
|
} elseif (!(($journalStatus === 'succeeded'
|
|
&& $resolution === 'applied')
|
|
|| ($journalStatus === 'failed'
|
|
&& $resolution === 'not_applied'))) {
|
|
throw new CloudError('宿主机操作终态不允许自动收敛', 409);
|
|
}
|
|
|
|
self::commitPrepared($job, $preparedIntent);
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Serialize the only Agent fence write with PHP's effect commit.
|
|
*
|
|
* The job row remains locked across the bounded Agent metadata request.
|
|
* Receipt ingestion locks the same row first, so either the immutable
|
|
* receipt wins before this transaction (and no fence write occurs), or
|
|
* this prepared recovery commits before the receipt is classified late.
|
|
* Unrelated jobs and ordinary workers are not locked.
|
|
*
|
|
* @param array{resolution:string,journal_status:string,fence_resolution_required:bool,source:string,operation:array} $prepared
|
|
*/
|
|
private static function resolveFenceAndCommit(
|
|
CloudJob $claimed,
|
|
array $prepared,
|
|
string $reason,
|
|
array $inspection
|
|
): void {
|
|
// Snapshot the immutable parent link before opening the locking
|
|
// transaction. Aggregate paths take parent -> catalog -> leaf so the
|
|
// nested commitPrepared() transaction can safely reacquire that same
|
|
// canonical lock sequence.
|
|
$aggregateParentId = (int) (CloudJob::where(
|
|
'id',
|
|
(int) $claimed->id
|
|
)->value('parent_job_id') ?? 0);
|
|
(new CloudJob())->getConnection()->transaction(
|
|
static function () use (
|
|
$claimed,
|
|
$prepared,
|
|
$reason,
|
|
$inspection,
|
|
$aggregateParentId
|
|
): void {
|
|
$aggregateChildren = null;
|
|
if ($aggregateParentId > 0) {
|
|
$aggregateParent = CloudJob::where(
|
|
'id',
|
|
$aggregateParentId
|
|
)
|
|
->lockForUpdate()->first();
|
|
if ($aggregateParent === null) {
|
|
throw new CloudRecoveryAggregateStateConflict(
|
|
'任务的汇总父任务不存在',
|
|
409
|
|
);
|
|
}
|
|
if ((string) $aggregateParent->execution_mode
|
|
!== 'aggregate'
|
|
|| CloudJobService::hasAggregateLocalStateConflict(
|
|
$aggregateParent
|
|
)) {
|
|
throw new CloudRecoveryAggregateStateConflict(
|
|
'任务的汇总父任务账本状态异常',
|
|
409
|
|
);
|
|
}
|
|
CloudTemplateCatalogService::lockAggregateCatalogLocked(
|
|
$aggregateParent
|
|
);
|
|
CloudUserIsoService::lockAggregateCatalogLocked(
|
|
$aggregateParent
|
|
);
|
|
$aggregateChildren = CloudJob::where(
|
|
'parent_job_id',
|
|
$aggregateParentId
|
|
)->orderBy('id')->lockForUpdate()->get();
|
|
$job = $aggregateChildren->first(
|
|
static fn (CloudJob $child): bool =>
|
|
(int) $child->id === (int) $claimed->id
|
|
);
|
|
} else {
|
|
$job = CloudJob::where('id', (int) $claimed->id)
|
|
->lockForUpdate()->first();
|
|
}
|
|
if ($job === null
|
|
|| !hash_equals(
|
|
(string) ($job->lease_token ?? ''),
|
|
(string) $claimed->lease_token
|
|
)
|
|
|| (int) $job->lease_until <= time()
|
|
|| (string) $job->effect_resolution !== 'unknown'
|
|
|| !$prepared['fence_resolution_required']) {
|
|
throw new CloudError('自动核对租约已失效', 409);
|
|
}
|
|
if ((int) ($job->parent_job_id ?? 0)
|
|
!== $aggregateParentId) {
|
|
throw new CloudRecoveryAggregateStateConflict(
|
|
'任务的汇总父级关系在栅栏提交期间发生变化',
|
|
409
|
|
);
|
|
}
|
|
if ($aggregateParentId > 0) {
|
|
CloudJobService::assertAggregateChildRelationshipLocked(
|
|
$aggregateParent,
|
|
$job,
|
|
$aggregateChildren
|
|
);
|
|
}
|
|
$context = self::recoveryContext($job);
|
|
$current = self::preparedAutomaticIntent($job, $context);
|
|
if ($current === null
|
|
|| $current['resolution'] !== $prepared['resolution']
|
|
|| $current['journal_status']
|
|
!== $prepared['journal_status']
|
|
|| !$current['fence_resolution_required']
|
|
|| !hash_equals(
|
|
$current['source'],
|
|
$prepared['source']
|
|
)
|
|
|| self::canonicalize($current['operation'])
|
|
!== self::canonicalize($prepared['operation'])) {
|
|
throw new CloudError('自动核对预提交证据已改变', 409);
|
|
}
|
|
$pending = CloudAgentCompletionEvent::where(
|
|
'host_id',
|
|
(int) $job->host_id
|
|
)->where(
|
|
'operation_id',
|
|
(string) $job->operation_id
|
|
)->where('disposition', 'received')
|
|
->lockForUpdate()->first();
|
|
if ($pending !== null) {
|
|
throw new CloudError('晚到完成回执正在等待优先收敛', 409);
|
|
}
|
|
CloudAdminOperationRecoveryService::resolveFenceForRecovery(
|
|
(string) $job->id,
|
|
$prepared['resolution'],
|
|
$reason,
|
|
$inspection
|
|
);
|
|
self::commitPrepared($job, $prepared);
|
|
},
|
|
3
|
|
);
|
|
}
|
|
|
|
private static function assertFreshInspection(
|
|
CloudJob $job,
|
|
array $inspection
|
|
): void {
|
|
$checkedAt = $inspection['checked_at'] ?? null;
|
|
$now = time();
|
|
if (($inspection['operation_id'] ?? null)
|
|
!== (string) $job->operation_id
|
|
|| !is_int($checkedAt)
|
|
|| $checkedAt < $now - self::INSPECTION_MAX_AGE_SECONDS
|
|
|| $checkedAt > $now + 5) {
|
|
throw new CloudError('宿主机只读核对证据已失效', 409);
|
|
}
|
|
$expected = CloudOperationRecoveryIdentity::expected(
|
|
$job,
|
|
($inspection['effect_resolution'] ?? null) === 'applied'
|
|
);
|
|
CloudOperationRecoveryIdentity::assertMatches($expected, [
|
|
'action' => $inspection['action'] ?? null,
|
|
'resource_id' => $inspection['resource_id'] ?? null,
|
|
'request_fingerprint' =>
|
|
$inspection['request_fingerprint'] ?? null,
|
|
]);
|
|
}
|
|
|
|
private static function completionDeliveryIncident(
|
|
array $inspection
|
|
): ?string {
|
|
if (($inspection['completion_delivery_state'] ?? null)
|
|
!== 'quarantined') {
|
|
return null;
|
|
}
|
|
return match ($inspection['completion_delivery_error_code'] ?? null) {
|
|
'completion_outbox_invalid' =>
|
|
'recovery_completion_outbox_invalid',
|
|
'completion_receipt_missing' =>
|
|
'recovery_completion_receipt_missing',
|
|
'completion_receipt_invalid' =>
|
|
'recovery_completion_receipt_invalid',
|
|
'completion_operation_missing' =>
|
|
'recovery_completion_operation_missing',
|
|
'completion_operation_invalid' =>
|
|
'recovery_completion_operation_invalid',
|
|
'completion_identity_mismatch' =>
|
|
'recovery_completion_identity_mismatch',
|
|
default => 'recovery_evidence_invalid',
|
|
};
|
|
}
|
|
|
|
private static function operationFromInspection(
|
|
CloudJob $job,
|
|
array $context,
|
|
array $inspection,
|
|
string $resolution
|
|
): array {
|
|
$journalStatus = $inspection['journal_status'] ?? null;
|
|
$fenceState = $inspection['fence_state'] ?? null;
|
|
$fenceResolutionRequired =
|
|
$inspection['fence_resolution_required'] ?? null;
|
|
if (!in_array($journalStatus, ['succeeded', 'failed'], true)) {
|
|
throw new CloudError('宿主机操作日志终态无效', 409);
|
|
}
|
|
if (!is_bool($fenceResolutionRequired)
|
|
|| !in_array($fenceState,
|
|
['not_required', 'active'], true)
|
|
|| $fenceResolutionRequired !== ($fenceState === 'active')
|
|
|| ($journalStatus === 'succeeded'
|
|
&& $fenceState !== 'not_required')
|
|
|| ($journalStatus === 'failed' && $resolution === 'applied'
|
|
&& $fenceState !== 'active')) {
|
|
throw new CloudError('宿主机栅栏证据与操作日志不匹配', 409);
|
|
}
|
|
if ($resolution === 'not_applied') {
|
|
$errorCode = $inspection['error_code'] ?? null;
|
|
if ($journalStatus !== 'failed'
|
|
|| ($inspection['outcome'] ?? null) !== 'failed'
|
|
|| ($inspection['summary_code'] ?? null)
|
|
!== 'operation_failed'
|
|
|| !is_string($errorCode)
|
|
|| preg_match('/^[a-z][a-z0-9_]{0,63}$/D',
|
|
$errorCode) !== 1
|
|
|| array_key_exists('effect_projection', $inspection)) {
|
|
throw new CloudError('宿主机未生效证据不完整', 409);
|
|
}
|
|
return [
|
|
'operation_id' => (string) $job->operation_id,
|
|
'action' => (string) $inspection['action'],
|
|
'resource_id' => (string) $inspection['resource_id'],
|
|
'request_fingerprint' =>
|
|
(string) $inspection['request_fingerprint'],
|
|
'status' => 'failed',
|
|
'effect_may_have_applied' => false,
|
|
'safe_to_resubmit' => false,
|
|
'retryable' => false,
|
|
'error_code' => $errorCode,
|
|
'result' => [],
|
|
];
|
|
}
|
|
|
|
$projection = $inspection['effect_projection'] ?? null;
|
|
if (($inspection['outcome'] ?? null) !== 'succeeded'
|
|
|| ($inspection['summary_code'] ?? null)
|
|
!== 'operation_succeeded'
|
|
|| array_key_exists('error_code', $inspection)
|
|
|| !is_array($projection) || array_is_list($projection)) {
|
|
throw new CloudError('宿主机已生效证据不完整', 409);
|
|
}
|
|
if ((string) $job->kind === 'instance_settings') {
|
|
if (($projection['uuid'] ?? null)
|
|
!== (string) $context['instance']->uuid
|
|
|| ($projection['settings_changed'] ?? null) !== true) {
|
|
throw new CloudError('宿主机实例设置证据不完整', 409);
|
|
}
|
|
if (array_key_exists('hostname', $context['body'])
|
|
&& ($projection['hostname'] ?? null)
|
|
!== $context['body']['hostname']) {
|
|
throw new CloudError('宿主机主机名证据与请求不匹配', 409);
|
|
}
|
|
}
|
|
return [
|
|
'operation_id' => (string) $job->operation_id,
|
|
'action' => (string) $inspection['action'],
|
|
'resource_id' => (string) $inspection['resource_id'],
|
|
'request_fingerprint' =>
|
|
(string) $inspection['request_fingerprint'],
|
|
'status' => 'succeeded',
|
|
'effect_may_have_applied' => false,
|
|
'safe_to_resubmit' => false,
|
|
'retryable' => false,
|
|
'error_code' => '',
|
|
// CloudAdminOperationRecoveryService already applied the strict
|
|
// per-kind projection schema. The existing worker finalizer now
|
|
// validates it against PHP's encrypted request and ledgers.
|
|
'result' => $projection,
|
|
];
|
|
}
|
|
|
|
private static function evidenceSource(
|
|
CloudJob $job,
|
|
array $context,
|
|
array $inspection,
|
|
string $resolution
|
|
): string {
|
|
$projection = $inspection['effect_projection'] ?? [];
|
|
$criticalProjection = is_array($projection) ? $projection : [];
|
|
$instance = $context['instance'] ?? null;
|
|
$evidence = self::canonicalize([
|
|
'operation_id' => (string) $job->operation_id,
|
|
'kind' => (string) $job->kind,
|
|
'instance_uuid' => $instance === null
|
|
? null : (string) $instance->uuid,
|
|
'journal_status' => $inspection['journal_status'] ?? null,
|
|
'fence_resolution_required' =>
|
|
$inspection['fence_resolution_required'] ?? null,
|
|
'resolution' => $resolution,
|
|
'error_code' => $inspection['error_code'] ?? null,
|
|
'projection' => $criticalProjection,
|
|
]);
|
|
$journalStatus = (string) ($inspection['journal_status'] ?? '');
|
|
$fence = ($inspection['fence_resolution_required'] ?? false) === true
|
|
? 'fence' : 'nofence';
|
|
$prefix = $resolution === 'applied'
|
|
? 'automatic_applied_' . $journalStatus . '_' . $fence
|
|
: 'automatic_not_applied_' . $journalStatus . '_' . $fence;
|
|
return $prefix . ':' . hash(
|
|
'sha256',
|
|
json_encode(
|
|
$evidence,
|
|
JSON_THROW_ON_ERROR
|
|
| JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE
|
|
)
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Rebuild the exact stable operation which was durably prepared earlier.
|
|
* The prepared source records both resolution and Journal terminal state;
|
|
* commitApplied()/commitNotApplied() re-check its operation hash.
|
|
*
|
|
* @return array{resolution:string,journal_status:string,fence_resolution_required:bool,source:string,operation:array}|null
|
|
*/
|
|
private static function preparedAutomaticIntent(
|
|
CloudJob $job,
|
|
array $context
|
|
): ?array {
|
|
$marker = self::preparedPayload(
|
|
(string) $job->effect_resolution_reason
|
|
);
|
|
if ($marker === null) {
|
|
return null;
|
|
}
|
|
if (preg_match(
|
|
'/^prepared:(automatic_(applied|not_applied)_(succeeded|failed)_(fence|nofence):[a-f0-9]{64}):([a-f0-9]{64})$/D',
|
|
$marker,
|
|
$matches
|
|
) !== 1) {
|
|
return null;
|
|
}
|
|
$resolution = $matches[2];
|
|
$journalStatus = $matches[3];
|
|
$fenceResolutionRequired = $matches[4] === 'fence';
|
|
$validShape = ($resolution === 'applied'
|
|
&& $journalStatus === 'succeeded'
|
|
&& !$fenceResolutionRequired)
|
|
|| ($resolution === 'applied'
|
|
&& $journalStatus === 'failed'
|
|
&& $fenceResolutionRequired)
|
|
|| ($resolution === 'not_applied'
|
|
&& $journalStatus === 'failed');
|
|
if (!$validShape) {
|
|
return null;
|
|
}
|
|
$operation = CloudWorkerService::preparedRecoveryOperation($job);
|
|
if ($operation === null && (string) $job->kind !== 'instance_settings') {
|
|
throw new CloudRecoveryPayloadUnavailable(
|
|
'任务缺少可恢复的预提交结果',
|
|
409
|
|
);
|
|
}
|
|
// Compatibility for settings intents prepared before the generic
|
|
// encrypted operation envelope was introduced.
|
|
if ($operation === null && $resolution === 'not_applied') {
|
|
$operation = [
|
|
'operation_id' => (string) $job->operation_id,
|
|
'status' => 'failed',
|
|
'effect_may_have_applied' => false,
|
|
'safe_to_resubmit' => false,
|
|
'error_code' => 'settings_not_committed',
|
|
'result' => [],
|
|
];
|
|
} elseif ($operation === null) {
|
|
$result = [
|
|
'uuid' => (string) $context['instance']->uuid,
|
|
'settings_changed' => true,
|
|
];
|
|
if (array_key_exists('hostname', $context['body'])) {
|
|
$spec = json_decode(
|
|
(string) $context['instance']->spec_json,
|
|
true
|
|
);
|
|
if (!is_array($spec) || array_is_list($spec)
|
|
|| ($spec['guest_hostname'] ?? null)
|
|
!== $context['body']['hostname']) {
|
|
throw new CloudError(
|
|
'已预提交的主机名账本与请求不匹配',
|
|
409
|
|
);
|
|
}
|
|
$result['hostname'] = $context['body']['hostname'];
|
|
}
|
|
$operation = [
|
|
'operation_id' => (string) $job->operation_id,
|
|
'status' => 'succeeded',
|
|
'effect_may_have_applied' => false,
|
|
'safe_to_resubmit' => false,
|
|
'error_code' => '',
|
|
'result' => $result,
|
|
];
|
|
}
|
|
$operationHash = CloudOperationRecoveryService::
|
|
preparedOperationHash(
|
|
$operation
|
|
);
|
|
$validOperation = $resolution === 'applied'
|
|
? (($operation['status'] ?? null) === 'succeeded'
|
|
&& ($operation['effect_may_have_applied'] ?? null) === false
|
|
&& ($operation['safe_to_resubmit'] ?? null) === false)
|
|
: (($operation['status'] ?? null) === 'failed'
|
|
&& ($operation['effect_may_have_applied'] ?? null) === false
|
|
&& is_bool($operation['safe_to_resubmit'] ?? null)
|
|
&& is_string($operation['error_code'] ?? null)
|
|
&& preg_match('/^[a-z][a-z0-9_]{0,63}$/D',
|
|
$operation['error_code']) === 1);
|
|
if (!$validOperation || !hash_equals($matches[5], $operationHash)) {
|
|
throw new CloudRecoveryPayloadUnavailable(
|
|
'任务预提交结果与恢复标记不匹配',
|
|
409
|
|
);
|
|
}
|
|
return [
|
|
'resolution' => $resolution,
|
|
'journal_status' => $journalStatus,
|
|
'fence_resolution_required' => $fenceResolutionRequired,
|
|
'source' => $matches[1],
|
|
'operation' => $operation,
|
|
];
|
|
}
|
|
|
|
private static function recoveryReason(
|
|
CloudJob $job,
|
|
string $source,
|
|
string $resolution
|
|
): string {
|
|
$separator = strrpos($source, ':');
|
|
$evidence = $separator === false
|
|
? $source : substr($source, $separator + 1);
|
|
return sprintf(
|
|
'automatic %s recovery job=%d operation=%s evidence=%s resolution=%s',
|
|
(string) $job->kind,
|
|
(int) $job->id,
|
|
(string) $job->operation_id,
|
|
$evidence,
|
|
$resolution
|
|
);
|
|
}
|
|
|
|
/**
|
|
* @param array{resolution:string,journal_status:string,fence_resolution_required:bool,source:string,operation:array} $prepared
|
|
*/
|
|
private static function commitPrepared(
|
|
CloudJob $job,
|
|
array $prepared
|
|
): void {
|
|
if ($prepared['resolution'] === 'applied') {
|
|
CloudOperationRecoveryService::commitApplied(
|
|
(int) $job->id,
|
|
$prepared['operation'],
|
|
$prepared['source'],
|
|
0
|
|
);
|
|
return;
|
|
}
|
|
CloudOperationRecoveryService::commitNotApplied(
|
|
(int) $job->id,
|
|
$prepared['operation'],
|
|
$prepared['source'],
|
|
0
|
|
);
|
|
}
|
|
|
|
private static function canonicalize(mixed $value): mixed
|
|
{
|
|
if (!is_array($value)) {
|
|
return $value;
|
|
}
|
|
if (!array_is_list($value)) {
|
|
ksort($value, SORT_STRING);
|
|
}
|
|
foreach ($value as $key => $item) {
|
|
$value[$key] = self::canonicalize($item);
|
|
}
|
|
return $value;
|
|
}
|
|
|
|
private static function recoveryContext(CloudJob $job): array
|
|
{
|
|
return (string) $job->kind === 'instance_settings'
|
|
? CloudWorkerService::settingsRecoveryContext($job)
|
|
: CloudWorkerService::recoveryContext($job);
|
|
}
|
|
|
|
/** @return array{delay:int,error_code:string} */
|
|
private static function pending(string $errorCode): array
|
|
{
|
|
return [
|
|
'delay' => self::RETRY_SECONDS,
|
|
'error_code' => $errorCode,
|
|
];
|
|
}
|
|
|
|
private static function recoveryErrorCode(\Throwable $error): string
|
|
{
|
|
if ($error instanceof CloudRecoveryDispatchLedgerConflict) {
|
|
return CloudDispatchLedger::CONFLICT_CODE;
|
|
}
|
|
if ($error instanceof CloudRecoveryAggregateStateConflict) {
|
|
return 'recovery_aggregate_state_conflict';
|
|
}
|
|
if ($error instanceof CloudRecoveryPayloadUnavailable) {
|
|
return 'recovery_payload_unavailable';
|
|
}
|
|
if ($error instanceof CloudRecoveryOperationIdentityConflict) {
|
|
return 'recovery_operation_identity_conflict';
|
|
}
|
|
if ($error instanceof CloudRecoveryEvidenceConflict) {
|
|
return 'recovery_evidence_conflict';
|
|
}
|
|
if ($error instanceof CloudRecoveryReceiptInvalid) {
|
|
return 'recovery_receipt_invalid';
|
|
}
|
|
if ($error instanceof CloudRecoveryInstanceStateConflict) {
|
|
return 'recovery_instance_state_conflict';
|
|
}
|
|
if ($error instanceof CloudAgentError) {
|
|
if (in_array($error->agentCode, [
|
|
'agent_invalid_inspection',
|
|
'agent_invalid_fence_resolution',
|
|
'fence_resolution_read_failed',
|
|
], true) && $error->agentHttpStatus === 200) {
|
|
return 'recovery_evidence_invalid';
|
|
}
|
|
if ($error->agentHttpStatus === 404) {
|
|
return 'recovery_operation_missing';
|
|
}
|
|
if (in_array($error->agentHttpStatus, [401, 403], true)) {
|
|
return 'recovery_authentication_failed';
|
|
}
|
|
if ($error->agentHttpStatus === 409) {
|
|
return 'recovery_operation_conflict';
|
|
}
|
|
return 'recovery_waiting_connection';
|
|
}
|
|
if ($error instanceof CloudError) {
|
|
return 'recovery_ledger_conflict';
|
|
}
|
|
return 'recovery_internal_error';
|
|
}
|
|
|
|
private function releaseForRetry(
|
|
CloudJob $claimed,
|
|
int $delay,
|
|
string $errorCode
|
|
): void
|
|
{
|
|
(new CloudJob())->getConnection()->transaction(
|
|
static function () use ($claimed, $delay, $errorCode): void {
|
|
$job = CloudJob::where('id', (int) $claimed->id)
|
|
->where('effect_resolution', 'unknown')
|
|
->where(
|
|
'lease_token',
|
|
(string) $claimed->lease_token
|
|
)
|
|
->lockForUpdate()
|
|
->first();
|
|
if ($job === null) {
|
|
return;
|
|
}
|
|
$now = time();
|
|
$job->lease_token = null;
|
|
$job->lease_until = 0;
|
|
$job->available_at = $now + max(
|
|
self::RETRY_SECONDS,
|
|
$delay
|
|
);
|
|
if (!hash_equals(
|
|
(string) ($job->finalization_error_code ?? ''),
|
|
$errorCode
|
|
)) {
|
|
$job->read_at = null;
|
|
}
|
|
$job->finalization_incident = 1;
|
|
$job->finalization_error_code = $errorCode;
|
|
$job->updated_at = $now;
|
|
$job->save();
|
|
},
|
|
3
|
|
);
|
|
}
|
|
|
|
private function resumeDispatchLedgerRepair(CloudJob $claimed): void
|
|
{
|
|
try {
|
|
$result = CloudOperationCompletionService::
|
|
resumeRepairedDispatchLedger($claimed);
|
|
if ($result === 'missing') {
|
|
$this->releaseLocalClaim(
|
|
$claimed,
|
|
'recovery_dispatch_repair_completion_receipt_missing'
|
|
);
|
|
} elseif ($result === 'invalid') {
|
|
$this->releaseLocalClaim(
|
|
$claimed,
|
|
'recovery_dispatch_repair_receipt_invalid'
|
|
);
|
|
}
|
|
} catch (\Throwable $error) {
|
|
$this->releaseLocalClaim(
|
|
$claimed,
|
|
self::dispatchRepairErrorCode($error)
|
|
);
|
|
}
|
|
}
|
|
|
|
private static function dispatchRepairErrorCode(
|
|
\Throwable $error
|
|
): string {
|
|
return match (self::recoveryErrorCode($error)) {
|
|
CloudDispatchLedger::CONFLICT_CODE =>
|
|
CloudDispatchLedger::CONFLICT_CODE,
|
|
'recovery_aggregate_state_conflict' =>
|
|
'recovery_dispatch_repair_aggregate_state_conflict',
|
|
'recovery_payload_unavailable' =>
|
|
'recovery_dispatch_repair_payload_unavailable',
|
|
'recovery_operation_identity_conflict' =>
|
|
'recovery_dispatch_repair_operation_identity_conflict',
|
|
'recovery_evidence_conflict' =>
|
|
'recovery_dispatch_repair_evidence_conflict',
|
|
'recovery_receipt_invalid' =>
|
|
'recovery_dispatch_repair_receipt_invalid',
|
|
'recovery_instance_state_conflict' =>
|
|
'recovery_dispatch_repair_instance_state_conflict',
|
|
'recovery_ledger_conflict' =>
|
|
'recovery_dispatch_repair_ledger_conflict',
|
|
default => 'recovery_dispatch_repair_internal_error',
|
|
};
|
|
}
|
|
|
|
/** Release a PHP-only recovery claim without requiring unknown effect. */
|
|
private function releaseLocalClaim(
|
|
CloudJob $claimed,
|
|
string $errorCode
|
|
): void {
|
|
(new CloudJob())->getConnection()->transaction(
|
|
static function () use ($claimed, $errorCode): void {
|
|
$job = CloudJob::where('id', (int) $claimed->id)
|
|
->where(
|
|
'lease_token',
|
|
(string) $claimed->lease_token
|
|
)->lockForUpdate()->first();
|
|
if ($job === null) {
|
|
return;
|
|
}
|
|
$now = time();
|
|
$job->lease_token = null;
|
|
// claim(true) intentionally prioritizes a matching received
|
|
// event over ordinary available_at backoff. Keep the lease
|
|
// clock as the retry gate as well, otherwise an invalid bound
|
|
// receipt would be reclaimed on every worker tick.
|
|
$job->lease_until = $now + self::RETRY_SECONDS;
|
|
$job->available_at = $now + self::RETRY_SECONDS;
|
|
if (!hash_equals(
|
|
(string) ($job->finalization_error_code ?? ''),
|
|
$errorCode
|
|
)) {
|
|
$job->read_at = null;
|
|
}
|
|
$job->finalization_incident = 1;
|
|
$job->finalization_error_code = $errorCode;
|
|
$job->updated_at = $now;
|
|
$job->save();
|
|
},
|
|
3
|
|
);
|
|
}
|
|
|
|
/** @return array{first_at:int,attempts:int,payload:string}|null */
|
|
private static function attemptState(string $marker): ?array
|
|
{
|
|
if (preg_match(
|
|
'/^automatic_recovery:v1:([1-9][0-9]{0,10}):([1-9][0-9]{0,9}):(pending|prepared:(?:automatic_(?:applied|not_applied)_(?:succeeded|failed)_(?:fence|nofence):[a-f0-9]{64}|late_completion_receipt:[A-Za-z0-9_.-]{1,160}):[a-f0-9]{64})$/D',
|
|
$marker,
|
|
$matches
|
|
) !== 1) {
|
|
return null;
|
|
}
|
|
return [
|
|
'first_at' => (int) $matches[1],
|
|
'attempts' => (int) $matches[2],
|
|
'payload' => $matches[3],
|
|
];
|
|
}
|
|
|
|
private static function nextAttemptMarker(
|
|
string $marker,
|
|
int $now
|
|
): string|false {
|
|
if ($marker === '') {
|
|
return sprintf(
|
|
'automatic_recovery:v1:%d:1:pending',
|
|
$now
|
|
);
|
|
}
|
|
$state = self::attemptState($marker);
|
|
if ($state === null) {
|
|
if (preg_match(
|
|
'/^prepared:(?:automatic_(?:applied|not_applied)_(?:succeeded|failed)_(?:fence|nofence):[a-f0-9]{64}|late_completion_receipt:[A-Za-z0-9_.-]{1,160}):[a-f0-9]{64}$/D',
|
|
$marker
|
|
) === 1) {
|
|
return sprintf(
|
|
'automatic_recovery:v1:%d:1:%s',
|
|
$now,
|
|
$marker
|
|
);
|
|
}
|
|
return false;
|
|
}
|
|
return sprintf(
|
|
'automatic_recovery:v1:%d:%d:%s',
|
|
$state['first_at'],
|
|
min(9999999999, $state['attempts'] + 1),
|
|
$state['payload']
|
|
);
|
|
}
|
|
|
|
private static function preparedPayload(string $marker): ?string
|
|
{
|
|
$state = self::attemptState($marker);
|
|
if ($state !== null && str_starts_with(
|
|
$state['payload'],
|
|
'prepared:'
|
|
)) {
|
|
return $state['payload'];
|
|
}
|
|
return preg_match(
|
|
'/^prepared:(?:automatic_(?:applied|not_applied)_(?:succeeded|failed)_(?:fence|nofence):[a-f0-9]{64}|late_completion_receipt:[A-Za-z0-9_.-]{1,160}):[a-f0-9]{64}$/D',
|
|
$marker
|
|
) === 1 ? $marker : null;
|
|
}
|
|
|
|
private static function preparedLateReceiptEventId(
|
|
CloudJob $job
|
|
): ?string {
|
|
$payload = self::preparedPayload(
|
|
(string) $job->effect_resolution_reason
|
|
);
|
|
if (!is_string($payload)
|
|
|| preg_match(
|
|
'/^prepared:late_completion_receipt:([A-Za-z0-9_.-]{1,160}):[a-f0-9]{64}$/D',
|
|
$payload,
|
|
$matches
|
|
) !== 1) {
|
|
return null;
|
|
}
|
|
return $matches[1];
|
|
}
|
|
}
|