You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudOperationCompletionSer...

1274 lines
53 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudAgentCompletionEvent;
use App\Models\CloudJob;
/** Applies immutable Agent receipts without ever re-dispatching a mutation. */
final class CloudOperationCompletionService
{
private const RESULT_ENVELOPE = 'virt_cloud_completion_event';
private const RESULT_ENVELOPE_VERSION = 2;
private const LEGACY_RESULT_ENVELOPE_VERSION = 1;
private const IDENTITY_RETRY_SECONDS = 300;
public static function accept(array $input): array
{
try {
$receipt = self::validate($input);
} catch (CloudError $error) {
self::recordInputIncident($input, 'recovery_receipt_invalid');
throw $error;
}
$hash = hash('sha256', json_encode($receipt,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE));
try {
$event = (new CloudAgentCompletionEvent())->getConnection()
->transaction(function () use ($receipt, $hash) {
// Serialize durable receipt publication with the deadline
// finalizer. If expiry owns the job row first, this receipt is
// necessarily late; if receipt ingestion owns it first, expiry
// cannot pass the event before the event commit becomes
// visible. This makes received_at a real durable boundary.
$receiptJob = CloudJob::where('host_id', $receipt['host_id'])
->where('execution_mode', 'agent')
->where('operation_id', $receipt['operation_id'])
->lockForUpdate()->first();
$existing = CloudAgentCompletionEvent::where(
'event_id', $receipt['event_id']
)->lockForUpdate()->first();
if ($existing === null) {
$existing = CloudAgentCompletionEvent::where(
'host_id', $receipt['host_id']
)->where('operation_id', $receipt['operation_id'])
->lockForUpdate()->first();
}
if ($existing !== null) {
if (!hash_equals((string) $existing->payload_sha256,
$hash)) {
throw new CloudError(
'同一完成回执标识不能承载不同结果', 409
);
}
if ($receiptJob !== null
&& CloudDispatchLedger::isNeverDispatched(
$receiptJob
)) {
self::applyDispatchLedgerConflictLocked(
$receiptJob,
time(),
'recovery_dispatch_repair_instance_state_conflict',
true
);
}
return $existing;
}
$event = new CloudAgentCompletionEvent();
$event->event_id = $receipt['event_id'];
$event->host_id = $receipt['host_id'];
$event->operation_id = $receipt['operation_id'];
$event->payload_sha256 = $hash;
$event->outcome = $receipt['status'];
$event->effect_may_have_applied =
$receipt['effect_may_have_applied'] ? 1 : 0;
$event->safe_to_resubmit = $receipt['safe_to_resubmit'] ? 1 : 0;
$event->error_code = $receipt['error_code'];
// No schema change is needed: result_json stores an internal,
// versioned envelope while operation() keeps exposing only the
// original business result to existing finalizers.
$event->result_json = json_encode([
'schema' => self::RESULT_ENVELOPE,
'version' => self::RESULT_ENVELOPE_VERSION,
'identity' => [
'action' => $receipt['action'],
'resource_id' => $receipt['resource_id'],
'request_fingerprint' =>
$receipt['request_fingerprint'],
],
'cause_code' => $receipt['cause_code'] ?? null,
'result' => $receipt['result'],
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE);
$event->disposition = 'received';
$event->received_at = time();
$event->applied_at = null;
$event->save();
// Receipt evidence and a contradictory generation-zero fence
// commit atomically. The ordinary worker must never observe a
// durable completion while the mutation remains dispatchable.
if ($receiptJob !== null
&& CloudDispatchLedger::isNeverDispatched($receiptJob)) {
self::applyDispatchLedgerConflictLocked(
$receiptJob,
time(),
'recovery_dispatch_repair_instance_state_conflict',
true
);
}
return $event;
}, 3);
} catch (CloudError $error) {
if (in_array($error->httpStatus, [400, 409], true)) {
self::recordReceiptIncident(
(int) $receipt['host_id'],
(string) $receipt['operation_id'],
$error->httpStatus === 400
? 'recovery_receipt_invalid'
: 'recovery_operation_identity_conflict'
);
}
throw $error;
}
$job = CloudJob::where('host_id', $receipt['host_id'])
->where('execution_mode', 'agent')
->where('operation_id', $receipt['operation_id'])->first();
if ($job === null) {
self::mark($event, 'rejected');
$event = self::reload($event);
return ['accepted' => true,
'disposition' => (string) $event->disposition];
}
if (CloudDispatchLedger::isInvalid($job)) {
// The receipt may be genuine, but a corrupt PHP dispatch ledger
// cannot prove which generation it belongs to. Keep it received
// so the Agent outbox is not ACKed; repairing the counter to one
// lets the same immutable receipt resume normal convergence.
self::deferDispatchLedgerConflict($job);
$event = self::reload($event);
return ['accepted' => true,
'disposition' => (string) $event->disposition];
}
if (CloudDispatchLedger::isNeverDispatched($job)) {
// An authenticated completion for generation zero is positive
// evidence that the PHP ledger is wrong. Never reject/ACK it and
// never let the ordinary worker dispatch the mutation again.
self::deferNeverDispatchedEvidenceConflict($job);
$event = self::reload($event);
return ['accepted' => true,
'disposition' => (string) $event->disposition];
}
if (CloudDispatchLedger::hasRepairIncident($job)) {
// The repair coordinator owns both receipt validation and ACK. In
// particular, a retransmission between operator repair and the
// next recovery tick must not turn the only evidence into late or
// rejected data.
return ['accepted' => true,
'disposition' => (string) $event->disposition];
}
if (CloudOperationRecoveryIdentity::supports((string) $job->kind)) {
if (!self::isExactStoredEvent($event)) {
self::mark($event, 'rejected');
$event = self::reload($event);
return ['accepted' => true,
'disposition' => (string) $event->disposition];
}
try {
$expectedIdentity = CloudOperationRecoveryIdentity::expected(
$job,
(string) $event->outcome === 'succeeded'
);
} catch (CloudRecoveryPayloadUnavailable $error) {
if ((string) $job->phase === 'terminal'
&& in_array((string) $job->effect_resolution,
['applied', 'not_applied'], true)) {
// An already-resolved pre-upgrade task cannot be changed by
// this late receipt. Rejecting it is a terminal ACK only;
// no result or ledger field is consumed.
self::mark($event, 'rejected');
} else {
self::deferIdentityConflict(
$job,
'recovery_payload_unavailable'
);
}
$event = self::reload($event);
return ['accepted' => true,
'disposition' => (string) $event->disposition];
} catch (CloudRecoveryOperationIdentityConflict $error) {
if ((string) $job->phase === 'terminal'
&& in_array((string) $job->effect_resolution,
['applied', 'not_applied'], true)) {
self::mark($event, 'rejected');
} else {
// This exception arose while rebuilding PHP's expected
// identity, before comparing the Agent triple. Keep the
// event retryable because repairing the durable request or
// stored binding may make this exact receipt usable.
self::deferIdentityConflict(
$job,
'recovery_operation_identity_conflict'
);
}
$event = self::reload($event);
return ['accepted' => true,
'disposition' => (string) $event->disposition];
} catch (CloudRecoveryInstanceStateConflict $error) {
if ((string) $job->phase === 'terminal'
&& in_array((string) $job->effect_resolution,
['applied', 'not_applied'], true)) {
self::mark($event, 'rejected');
} else {
self::deferIdentityConflict(
$job,
'recovery_instance_state_conflict'
);
}
$event = self::reload($event);
return ['accepted' => true,
'disposition' => (string) $event->disposition];
}
if (!CloudOperationRecoveryIdentity::same(
$expectedIdentity,
self::identity($event)
)) {
// PHP successfully rebuilt the immutable request, so this is a
// definitive wrong-operation receipt rather than temporarily
// unavailable metadata. Isolate it so fresh inspection (or a
// prepared crash resume) is not starved by receipt priority.
self::isolateIdentityMismatch($job, $event);
$event = self::reload($event);
return ['accepted' => true,
'disposition' => (string) $event->disposition];
}
}
if ((string) $job->phase === 'terminal') {
// finish() stores completion_event_id before mark(applied). Repair
// that ACK only when the terminal effect fields prove that this
// exact receipt actually settled the task. A failed business
// finalizer retains the event for the recovery-only worker.
if (self::settledTaskWithEvent($job, $event)) {
self::mark($event, 'applied');
} else {
CloudOperationRecoveryService::settleLateReceipt($job, $event);
}
$event = self::reload($event);
return ['accepted' => true,
'disposition' => (string) $event->disposition];
}
if ((int) $job->dispatch_attempts !== 1
|| !in_array((string) $job->phase,
['dispatching', 'awaiting_receipt'], true)) {
self::mark($event, 'rejected');
return ['accepted' => true, 'disposition' => 'rejected'];
}
if (in_array((string) $job->phase,
['dispatching', 'awaiting_receipt'], true)) {
(new CloudWorkerService())->settleCompletionReceipt(
(int) $job->id,
(string) $event->event_id
);
$event = self::reload($event);
}
return ['accepted' => true,
'disposition' => (string) $event->disposition];
}
public static function pendingForJob(CloudJob $job): ?CloudAgentCompletionEvent
{
return CloudAgentCompletionEvent::where('host_id', (int) $job->host_id)
->where('operation_id', (string) $job->operation_id)
->where('disposition', 'received')->first();
}
public static function operation(CloudAgentCompletionEvent $event): array
{
$envelope = self::resultEnvelope($event);
$operation = [
'operation_id' => (string) $event->operation_id,
'status' => (string) $event->outcome,
'effect_may_have_applied' => (bool) $event->effect_may_have_applied,
'safe_to_resubmit' => (bool) $event->safe_to_resubmit,
// Completion delivery contains only terminal Journal records.
// Agent operation responses define retryable as false, but that
// display-only field is intentionally absent from the durable
// completion wire format, so restore its canonical value here.
'retryable' => false,
'error_code' => (string) $event->error_code,
'action' => $envelope['identity']['action'],
'resource_id' => $envelope['identity']['resource_id'],
'request_fingerprint' =>
$envelope['identity']['request_fingerprint'],
'result' => $envelope['result'],
];
if ($envelope['cause_code'] !== null) {
$operation['cause_code'] = $envelope['cause_code'];
}
return $operation;
}
/**
* @return array{action:string,resource_id:string,request_fingerprint:string}
*/
public static function identity(CloudAgentCompletionEvent $event): array
{
return self::resultEnvelope($event)['identity'];
}
public static function assertEventIdentityForJob(
CloudJob $job,
CloudAgentCompletionEvent $event
): void {
if (!self::isExactStoredEvent($event)) {
throw new CloudRecoveryOperationIdentityConflict(
'Agent 完成回执缺少可信操作身份',
409
);
}
$expected = CloudOperationRecoveryIdentity::expected(
$job,
(string) $event->outcome === 'succeeded'
);
CloudOperationRecoveryIdentity::assertMatches(
$expected,
self::identity($event)
);
}
public static function settledTaskWithEvent(
CloudJob $job,
CloudAgentCompletionEvent $event
): bool {
$exactEvent = (string) $job->execution_mode === 'agent'
&& CloudDispatchLedger::isDispatched($job)
&& (string) $job->phase === 'terminal'
&& !(bool) $job->finalization_incident
&& $job->host_id !== null
&& (int) $job->host_id === (int) $event->host_id
&& is_string($job->operation_id)
&& hash_equals(
(string) $job->operation_id,
(string) $event->operation_id
)
&& is_string($job->completion_event_id)
&& hash_equals(
(string) $job->completion_event_id,
(string) $event->event_id
);
if (!$exactEvent) {
return false;
}
if (CloudOperationRecoveryIdentity::supports((string) $job->kind)) {
try {
self::assertEventIdentityForJob($job, $event);
} catch (\Throwable $error) {
return false;
}
}
$outcome = (string) $event->outcome;
$resolution = $outcome === 'succeeded'
? 'applied'
: ((bool) $event->effect_may_have_applied
? 'unknown' : 'not_applied');
// A stored event ID proves identity, not PHP ledger convergence. This
// applies to create/storage as well as the automatic recovery policy:
// their dedicated financial/configuration finalizer must commit before
// the Agent outbox can receive an ACK.
if ((string) $job->effect_resolution === 'unknown') {
return false;
}
if (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
(string) $job->kind
)) {
if ((string) $job->kind === 'create'
&& $outcome === 'succeeded'
&& (string) $job->outcome === 'failed'
&& (string) $job->status === 'failed') {
// A late exact create result materializes the paid order while
// preserving the immutable original timeout/failure receipt.
return $job->instance_id !== null
&& (string) $job->effect_resolution === 'applied';
}
return (string) $job->outcome === $outcome
&& (string) $job->status === $outcome
&& (string) $job->effect_resolution === $resolution;
}
return (string) $job->outcome === $outcome
&& (string) $job->status === $outcome
&& (string) $job->effect_resolution === $resolution;
}
/**
* Rebuild and authenticate a persisted completion event without trusting
* mutable database columns independently. This is used only to close the
* crash window after accept() committed the event but before settlement.
*/
public static function isExactStoredEvent(
CloudAgentCompletionEvent $event
): bool {
return (string) $event->disposition === 'received'
&& $event->applied_at === null
&& self::hasExactStoredEventPayload($event);
}
public static function isExactAppliedEvent(
CloudAgentCompletionEvent $event
): bool {
$appliedAt = filter_var(
$event->applied_at,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
);
return (string) $event->disposition === 'applied'
&& $appliedAt !== false
&& self::hasExactStoredEventPayload($event);
}
private static function hasExactStoredEventPayload(
CloudAgentCompletionEvent $event
): bool {
$eventId = $event->event_id;
$hostId = filter_var(
$event->host_id,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
);
$operationId = $event->operation_id;
$status = $event->outcome;
$effect = self::storedBoolean($event->effect_may_have_applied);
$safe = self::storedBoolean($event->safe_to_resubmit);
$errorCode = $event->error_code;
$hash = $event->payload_sha256;
$receivedAt = filter_var(
$event->received_at,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
);
if (!is_string($eventId)
|| preg_match('/^[A-Za-z0-9_-]{16,80}$/D', $eventId) !== 1
|| $hostId === false
|| !is_string($operationId)
|| preg_match('/^[a-f0-9]{32}$/D', $operationId) !== 1
|| !in_array($status, ['succeeded', 'failed'], true)
|| $effect === null
|| $safe === null
|| !is_string($errorCode)
|| ($errorCode !== ''
&& preg_match('/^[a-z][a-z0-9_]{0,63}$/D',
$errorCode) !== 1)
|| !is_string($hash)
|| preg_match('/^[a-f0-9]{64}$/D', $hash) !== 1
|| $receivedAt === false
|| ($status === 'succeeded'
&& ($effect || $safe || $errorCode !== ''))
|| ($status === 'failed'
&& ($errorCode === '' || ($effect && $safe)))) {
return false;
}
try {
$envelope = self::resultEnvelope($event);
} catch (\Throwable $error) {
return false;
}
$identity = $envelope['identity'];
$result = $envelope['result'];
if ($result !== null && !is_array($result)) {
return false;
}
$receipt = [
'event_id' => $eventId,
'host_id' => $hostId,
'operation_id' => $operationId,
'action' => $identity['action'],
'resource_id' => $identity['resource_id'],
'request_fingerprint' => $identity['request_fingerprint'],
'status' => $status,
'effect_may_have_applied' => $effect,
'safe_to_resubmit' => $safe,
'error_code' => $errorCode,
'result' => $result,
];
if ($envelope['cause_code'] !== null) {
$receipt['cause_code'] = $envelope['cause_code'];
}
$calculated = hash('sha256', json_encode(
$receipt,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
));
return hash_equals($hash, $calculated);
}
public static function mark(CloudAgentCompletionEvent $event,
string $disposition): void
{
CloudAgentCompletionEvent::where('event_id', (string) $event->event_id)
->where('disposition', 'received')->update([
'disposition' => $disposition,
'applied_at' => $disposition === 'applied' ? time() : null,
]);
}
/**
* Atomically quarantine a receipt whose well-formed identity definitively
* differs from PHP's reconstructed request, while keeping the job eligible
* for its normal read-only recovery path.
*/
public static function isolateIdentityMismatch(
CloudJob $source,
CloudAgentCompletionEvent $sourceEvent
): void {
(new CloudJob())->getConnection()->transaction(
static function () use ($source, $sourceEvent): void {
$job = CloudJob::where('id', (int) $source->id)
->lockForUpdate()->first();
$event = CloudAgentCompletionEvent::where(
'event_id',
(string) $sourceEvent->event_id
)->lockForUpdate()->first();
if ($event === null
|| (string) $event->disposition !== 'received') {
return;
}
if ($job !== null && CloudDispatchLedger::isInvalid($job)) {
self::applyDispatchLedgerConflictLocked($job, time());
return;
}
$sameOperation = $job !== null
&& (int) $job->host_id === (int) $event->host_id
&& is_string($job->operation_id)
&& hash_equals(
(string) $job->operation_id,
(string) $event->operation_id
);
$event->disposition = 'rejected';
$event->applied_at = null;
$event->save();
if (!$sameOperation) {
return;
}
$now = time();
$active = in_array((string) $job->phase,
['dispatching', 'awaiting_receipt'], true)
&& (int) $job->dispatch_attempts === 1;
if ($active && !CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
(string) $job->kind
)) {
// create and host-storage have dedicated financial/config
// finalizers. A malformed receipt may be surfaced, but it
// must not bypass those finalizers by terminalizing the job.
$job->finalization_incident = 1;
$job->finalization_error_code =
'recovery_operation_identity_conflict';
$job->read_at = null;
$job->updated_at = $now;
$job->save();
return;
}
if (in_array((string) $job->phase,
['dispatching', 'awaiting_receipt'], true)
&& (int) $job->dispatch_attempts === 1) {
$job->phase = 'terminal';
$job->outcome = 'failed';
$job->status = 'failed';
$job->terminal_at = $now;
$job->effect_may_have_applied = 1;
$job->safe_to_resubmit = 0;
$job->effect_resolution = 'unknown';
$job->error_code =
'recovery_operation_identity_conflict';
}
if ((string) $job->phase !== 'terminal'
|| (string) $job->effect_resolution !== 'unknown') {
return;
}
if (!hash_equals(
(string) ($job->finalization_error_code ?? ''),
'recovery_operation_identity_conflict'
)) {
$job->read_at = null;
}
$job->finalization_incident = 1;
$job->finalization_error_code =
'recovery_operation_identity_conflict';
$job->lease_token = null;
$job->lease_until = 0;
$job->available_at = $now + self::IDENTITY_RETRY_SECONDS;
$job->updated_at = $now;
$job->save();
},
3
);
}
private static function reload(
CloudAgentCompletionEvent $event
): CloudAgentCompletionEvent {
return CloudAgentCompletionEvent::where(
'event_id', (string) $event->event_id
)->first() ?? $event;
}
/**
* @return array{schema:string,version:int,identity:array{action:string,resource_id:string,request_fingerprint:string},cause_code:?string,result:?array}
*/
private static function resultEnvelope(
CloudAgentCompletionEvent $event
): array {
if (!is_string($event->result_json)
|| $event->result_json === '') {
throw new CloudRecoveryOperationIdentityConflict(
'Agent 完成回执没有持久化身份', 409
);
}
try {
$value = json_decode(
(string) $event->result_json,
true,
64,
JSON_THROW_ON_ERROR
);
} catch (\JsonException $error) {
throw new CloudRecoveryOperationIdentityConflict(
'Agent 完成回执身份封装无效', 409
);
}
$identity = is_array($value) ? ($value['identity'] ?? null) : null;
$version = is_array($value) ? ($value['version'] ?? null) : null;
$allowedKeys = $version === self::LEGACY_RESULT_ENVELOPE_VERSION
? ['schema', 'version', 'identity', 'result']
: ['schema', 'version', 'identity', 'cause_code', 'result'];
$causeCode = $version === self::LEGACY_RESULT_ENVELOPE_VERSION
? null : (is_array($value) ? ($value['cause_code'] ?? null) : null);
if (!is_array($value) || array_is_list($value)
|| array_diff(array_keys($value), $allowedKeys) !== []
|| ($value['schema'] ?? null) !== self::RESULT_ENVELOPE
|| !in_array($version, [
self::LEGACY_RESULT_ENVELOPE_VERSION,
self::RESULT_ENVELOPE_VERSION,
], true)
|| !is_array($identity) || array_is_list($identity)
|| array_diff(array_keys($identity), [
'action', 'resource_id', 'request_fingerprint',
]) !== []
|| !array_key_exists('result', $value)
|| ($version === self::RESULT_ENVELOPE_VERSION
&& !array_key_exists('cause_code', $value))
|| ($causeCode !== null
&& (!is_string($causeCode)
|| preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $causeCode)
!== 1))
|| ($value['result'] !== null && !is_array($value['result']))) {
throw new CloudRecoveryOperationIdentityConflict(
'Agent 完成回执身份封装无效', 409
);
}
self::assertIdentityShape($identity);
$value['cause_code'] = $causeCode;
return $value;
}
private static function assertIdentityShape(array $identity): void
{
if (!is_string($identity['action'] ?? null)
|| strlen($identity['action']) > 128
|| preg_match(
'/^[a-z][a-z0-9]*(?:[-:][a-z0-9]+)*$/D',
$identity['action']
) !== 1
|| !is_string($identity['resource_id'] ?? null)
|| strlen($identity['resource_id']) > 255
|| preg_match('/^[A-Za-z0-9][A-Za-z0-9_.:-]*$/D',
$identity['resource_id']) !== 1
|| !is_string($identity['request_fingerprint'] ?? null)
|| preg_match('/^[a-f0-9]{64}$/D',
$identity['request_fingerprint']) !== 1) {
throw new CloudRecoveryOperationIdentityConflict(
'Agent 完成回执操作身份无效', 409
);
}
}
private static function assertCompletionResultSafe(
?array $result,
int $depth = 0
): void {
if ($result === null) {
return;
}
if ($depth > 16) {
throw new CloudError('Agent 完成回执结构过深', 400);
}
foreach ($result as $key => $value) {
if (is_string($key) && preg_match(
'/password|secret|(?:public|private).?key|ssh.?key|passphrase|credential|(?:(?:access|refresh|auth|api|bearer|session|source).?token)|^token$/i',
$key
) === 1) {
throw new CloudError(
'Agent 完成回执包含禁止持久化的敏感字段',
400
);
}
if (is_array($value)) {
self::assertCompletionResultSafe(
$value,
$depth + 1
);
}
}
}
private static function deferIdentityConflict(
CloudJob $source,
string $errorCode
): void {
(new CloudJob())->getConnection()->transaction(
static function () use ($source, $errorCode): void {
$job = CloudJob::where('id', (int) $source->id)
->lockForUpdate()->first();
if ($job !== null && CloudDispatchLedger::isInvalid($job)) {
self::applyDispatchLedgerConflictLocked($job, time());
return;
}
if ($job === null
|| ((string) $job->effect_resolution !== 'unknown'
&& (string) $job->phase === 'terminal')) {
return;
}
$now = time();
$active = in_array((string) $job->phase,
['dispatching', 'awaiting_receipt'], true)
&& (int) $job->dispatch_attempts === 1;
if ($active && !CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
(string) $job->kind
)) {
// Do not replace create's atomic refund/placement finalizer
// or host storage's projection finalizer with a receipt-only
// job update. Their existing deadline remains authoritative.
if (!hash_equals(
(string) ($job->finalization_error_code ?? ''),
$errorCode
)) {
$job->read_at = null;
}
$job->finalization_incident = 1;
$job->finalization_error_code = $errorCode;
$job->updated_at = $now;
$job->save();
return;
}
if ($active) {
$job->phase = 'terminal';
$job->outcome = 'failed';
$job->status = 'failed';
$job->terminal_at = $now;
$job->effect_may_have_applied = 1;
$job->safe_to_resubmit = 0;
$job->effect_resolution = 'unknown';
$job->error_code = $errorCode;
}
if ((string) $job->phase !== 'terminal'
|| (string) $job->effect_resolution !== 'unknown') {
return;
}
if (!hash_equals(
(string) ($job->finalization_error_code ?? ''),
$errorCode
)) {
$job->read_at = null;
}
$job->finalization_incident = 1;
$job->finalization_error_code = $errorCode;
$job->lease_token = null;
$job->lease_until = 0;
$job->available_at = $now + self::IDENTITY_RETRY_SECONDS;
$job->updated_at = $now;
$job->save();
},
3
);
}
/**
* Surface a corrupt single-dispatch counter without consuming Agent proof.
*
* This method deliberately does not touch completion-event disposition.
*/
public static function deferDispatchLedgerConflict(
CloudJob $source
): void {
(new CloudJob())->getConnection()->transaction(
static function () use ($source): void {
$job = CloudJob::where('id', (int) $source->id)
->lockForUpdate()->first();
if ($job === null
|| (!CloudDispatchLedger::isInvalid($source)
&& !CloudDispatchLedger::isInvalid($job))) {
return;
}
// The caller already observed a corrupt generation. If an
// operator repairs the counter before this row lock is taken,
// retain a sticky incident and let the repair coordinator
// validate the immutable terminal state before unlocking.
self::applyDispatchLedgerConflictLocked($job, time());
},
3
);
}
/**
* Preserve an Agent receipt which contradicts a generation-zero ledger.
* The sticky incident also terminalizes an active row so the normal worker
* cannot replay the mutation while an operator repairs the counter.
*/
public static function deferNeverDispatchedEvidenceConflict(
CloudJob $source
): void {
(new CloudJob())->getConnection()->transaction(
static function () use ($source): void {
$job = CloudJob::where('id', (int) $source->id)
->lockForUpdate()->first();
if ($job === null
|| (string) $job->execution_mode !== 'agent'
|| (!CloudDispatchLedger::isNeverDispatched($source)
&& !CloudDispatchLedger::isNeverDispatched($job)
&& !CloudDispatchLedger::hasRepairIncident($job))) {
return;
}
self::applyDispatchLedgerConflictLocked(
$job,
time(),
'recovery_dispatch_repair_instance_state_conflict',
true
);
},
3
);
}
private static function applyDispatchLedgerConflictLocked(
CloudJob $job,
int $now,
string $incidentCode = CloudDispatchLedger::CONFLICT_CODE,
bool $gateReceivedEvidence = false
): void {
if (in_array((string) $job->phase,
['accepted', 'dispatching', 'awaiting_receipt'], true)) {
$job->phase = 'terminal';
$job->outcome = 'failed';
$job->status = 'failed';
$job->terminal_at = $now;
$job->effect_may_have_applied = 1;
$job->safe_to_resubmit = 0;
$job->effect_resolution = 'unknown';
$job->error_code = CloudDispatchLedger::CONFLICT_CODE;
}
if (!hash_equals(
(string) ($job->finalization_error_code ?? ''),
$incidentCode
)) {
$job->read_at = null;
}
$job->finalization_incident = 1;
$job->finalization_error_code = $incidentCode;
$job->lease_token = null;
$job->lease_until = $gateReceivedEvidence
? $now + self::IDENTITY_RETRY_SECONDS : 0;
$job->available_at = $now + self::IDENTITY_RETRY_SECONDS;
$job->updated_at = $now;
$job->save();
}
/**
* Resume only the local ledger consequences of an operator-repaired
* dispatch counter. Terminal task fields and Agent mutations are immutable.
*
* @return string resolved, pending, missing, or invalid
*/
public static function resumeRepairedDispatchLedger(
CloudJob $source
): string {
return (new CloudJob())->getConnection()->transaction(
static function () use ($source): string {
$job = CloudJob::where('id', (int) $source->id)
->lockForUpdate()->first();
if ($job === null) {
throw new CloudError('任务不存在', 404);
}
if (CloudDispatchLedger::isInvalid($job)) {
self::applyDispatchLedgerConflictLocked($job, time());
return 'pending';
}
if (!CloudDispatchLedger::hasRepairIncident($job)) {
return 'resolved';
}
if ((string) $job->phase !== 'terminal'
|| !in_array((string) $job->outcome,
['failed', 'succeeded'], true)
|| (string) $job->status !== (string) $job->outcome) {
throw new CloudRecoveryInstanceStateConflict(
'派发账本修复后的任务终态不一致',
409
);
}
$now = time();
if ((string) $job->outcome === 'failed') {
if (CloudDispatchLedger::isDispatched($job)) {
if (!self::canEnterAutomaticAgentRecovery($job)) {
throw new CloudRecoveryInstanceStateConflict(
'派发账本修复后仍缺少宿主机恢复身份或派发时间',
409
);
}
$job->effect_may_have_applied = 1;
$job->safe_to_resubmit = 0;
$job->effect_resolution = 'unknown';
} else {
self::assertNeverDispatchedRecoveryState($job);
if (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
(string) $job->kind
)) {
throw new CloudRecoveryInstanceStateConflict(
'未派发任务缺少已注册的 PHP 业务终结器',
409
);
}
$job->effect_may_have_applied = 0;
$job->safe_to_resubmit = 0;
$job->effect_resolution = 'not_applied';
}
$job->finalization_incident = 1;
$job->finalization_error_code = 'recovery_pending';
$job->lease_token = null;
$job->lease_until = 0;
$job->available_at = 0;
$job->updated_at = $now;
$job->save();
return 'pending';
}
if (!CloudDispatchLedger::isDispatched($job)
|| (string) $job->effect_resolution !== 'applied') {
throw new CloudRecoveryInstanceStateConflict(
'成功任务与修复后的派发账本状态矛盾',
409
);
}
if (!self::canEnterAutomaticAgentRecovery($job)) {
throw new CloudRecoveryInstanceStateConflict(
'成功任务的派发账本仍缺少宿主机恢复身份、派发时间或受支持操作类型',
409
);
}
$eventId = $job->completion_event_id;
$event = $eventId === null || (string) $eventId === ''
? CloudAgentCompletionEvent::where(
'host_id',
(int) $job->host_id
)->where(
'operation_id',
(string) $job->operation_id
)->lockForUpdate()->first()
: CloudAgentCompletionEvent::where(
'event_id',
(string) $eventId
)->lockForUpdate()->first();
if ($event === null
&& ($eventId === null || (string) $eventId === '')) {
CloudOperationRecoveryIdentity::expected($job, true);
self::clearDispatchLedgerIncidentLocked($job, $now);
return 'resolved';
}
if ($event === null) {
return 'missing';
}
$exact = self::isExactStoredEvent($event)
|| self::isExactAppliedEvent($event);
if (!$exact
|| $job->host_id === null
|| (int) $job->host_id !== (int) $event->host_id
|| !is_string($job->operation_id)
|| !hash_equals(
(string) $job->operation_id,
(string) $event->operation_id
)
|| (string) $event->outcome !== 'succeeded'
|| (bool) $event->effect_may_have_applied
|| (bool) $event->safe_to_resubmit) {
return 'invalid';
}
$expected = CloudOperationRecoveryIdentity::expected(
$job,
true
);
CloudOperationRecoveryIdentity::assertMatches(
$expected,
self::identity($event)
);
self::clearDispatchLedgerIncidentLocked($job, $now);
if ((string) $event->disposition === 'received') {
$event->disposition = 'applied';
$event->applied_at = $now;
$event->save();
}
return 'resolved';
},
3
);
}
private static function canEnterAutomaticAgentRecovery(
CloudJob $job
): bool {
$hostId = filter_var(
$job->host_id,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
);
$dispatchedAt = filter_var(
$job->dispatched_at,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
);
if ($hostId === false || $dispatchedAt === false
|| !is_string($job->operation_id)
|| preg_match(
'/^[a-f0-9]{32}$/D',
(string) $job->operation_id
) !== 1) {
return false;
}
if (CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
(string) $job->kind
)) {
return true;
}
if ((string) $job->kind !== 'create') {
return false;
}
return CloudAgentCompletionEvent::where(
'host_id',
(int) $hostId
)->where(
'operation_id',
(string) $job->operation_id
)->where('outcome', 'succeeded')
->where('disposition', 'received')
->exists();
}
/**
* Accept either generation zero or the worker's exact persisted transport
* proof that a generation-one request never entered the Agent journal.
* This authorizes only a PHP finalizer; any completion evidence keeps the
* resource fenced and requires operator review.
*/
public static function assertLocalNotDispatchedRecoveryState(
CloudJob $job
): void {
if (CloudDispatchLedger::isNeverDispatched($job)) {
self::assertNeverDispatchedRecoveryState($job);
return;
}
if (!CloudDispatchLedger::hasTransportNotDispatchedProof($job)) {
throw new CloudRecoveryDispatchLedgerConflict(
'任务缺少可信的未派发证明',
409
);
}
if (CloudJobService::hasAgentRecoveryEnvelopeConflict($job)) {
throw new CloudRecoveryInstanceStateConflict(
'未派发传输证明与 Agent 请求账本不一致',
409
);
}
$hostId = filter_var(
$job->host_id,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
);
$operationId = $job->operation_id;
if ($hostId === false || !is_string($operationId)
|| preg_match('/^[a-f0-9]{32}$/D', $operationId) !== 1
|| CloudAgentCompletionEvent::where('host_id', (int) $hostId)
->where('operation_id', $operationId)->exists()) {
throw new CloudRecoveryInstanceStateConflict(
'未派发传输证明存在冲突的 Agent 完成证据',
409
);
}
}
/**
* Generation zero may run only a PHP finalizer. Any durable Agent boundary
* or completion evidence means that zero is not trustworthy and must stay
* in the sticky dispatch-repair queue.
*/
public static function assertNeverDispatchedRecoveryState(
CloudJob $job
): void {
if (!CloudDispatchLedger::isNeverDispatched($job)) {
throw new CloudRecoveryDispatchLedgerConflict(
'任务不再是未派发代次',
409
);
}
if ($job->dispatched_at !== null
|| $job->agent_accepted_at !== null
|| $job->receipt_deadline_at !== null
|| (is_string($job->completion_event_id)
&& $job->completion_event_id !== '')) {
throw new CloudRecoveryInstanceStateConflict(
'未派发任务存在 Agent 派发或完成回执标记',
409
);
}
$hostId = filter_var(
$job->host_id,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
);
$operationId = $job->operation_id;
if ($hostId !== false && is_string($operationId)
&& preg_match('/^[a-f0-9]{32}$/D', $operationId) === 1
&& CloudAgentCompletionEvent::where('host_id', (int) $hostId)
->where('operation_id', $operationId)->exists()) {
throw new CloudRecoveryInstanceStateConflict(
'未派发任务存在同一操作的 Agent 完成证据',
409
);
}
}
private static function clearDispatchLedgerIncidentLocked(
CloudJob $job,
int $now
): void {
$job->finalization_incident = 0;
$job->finalization_error_code = null;
$job->read_at = null;
$job->lease_token = null;
$job->lease_until = 0;
$job->available_at = 0;
$job->updated_at = $now;
$job->save();
}
private static function recordInputIncident(
array $input,
string $errorCode
): void {
$hostId = filter_var(
$input['agent_id'] ?? null,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
);
$operationId = $input['operation_id'] ?? null;
if ($hostId === false || !is_string($operationId)
|| preg_match('/^[a-f0-9]{32}$/D', $operationId) !== 1) {
return;
}
self::recordReceiptIncident(
(int) $hostId,
$operationId,
$errorCode
);
}
private static function recordReceiptIncident(
int $hostId,
string $operationId,
string $errorCode
): void {
try {
$job = CloudJob::where('host_id', $hostId)
->where('execution_mode', 'agent')
->where('operation_id', $operationId)
->first();
if ($job !== null) {
self::deferIdentityConflict($job, $errorCode);
}
} catch (\Throwable $ignored) {
// Receipt validation must retain its original bounded 400/409
// contract even if the best-effort operator incident write fails.
}
}
private static function storedBoolean(mixed $value): ?bool
{
if ($value === true || $value === 1 || $value === '1') {
return true;
}
if ($value === false || $value === 0 || $value === '0') {
return false;
}
return null;
}
private static function validate(array $input): array
{
$allowed = ['type', 'id', 'agent_id', 'operation_id', 'action',
'resource_id', 'request_fingerprint', 'status',
'effect_may_have_applied', 'safe_to_resubmit', 'error_code',
'cause_code', 'result'];
$causeCode = $input['cause_code'] ?? '';
if (array_is_list($input)
|| array_diff(array_keys($input), $allowed) !== []
|| ($input['type'] ?? null) !== 'completion'
|| !is_string($input['id'] ?? null)
|| preg_match('/^[A-Za-z0-9_-]{16,80}$/D', $input['id']) !== 1
|| (!is_int($input['agent_id'] ?? null)
&& (!is_string($input['agent_id'] ?? null)
|| preg_match('/^[1-9][0-9]{0,18}$/D',
$input['agent_id']) !== 1))
|| (int) $input['agent_id'] < 1
|| !is_string($input['operation_id'] ?? null)
|| preg_match('/^[a-f0-9]{32}$/D', $input['operation_id']) !== 1
|| !is_string($input['action'] ?? null)
|| strlen($input['action']) > 128
|| preg_match('/^[a-z][a-z0-9]*(?:[-:][a-z0-9]+)*$/D',
$input['action']) !== 1
|| !is_string($input['resource_id'] ?? null)
|| strlen($input['resource_id']) > 255
|| preg_match('/^[A-Za-z0-9][A-Za-z0-9_.:-]*$/D',
$input['resource_id']) !== 1
|| !is_string($input['request_fingerprint'] ?? null)
|| preg_match('/^[a-f0-9]{64}$/D',
$input['request_fingerprint']) !== 1
|| !in_array($input['status'] ?? null,
['succeeded', 'failed'], true)
|| !is_bool($input['effect_may_have_applied'] ?? null)
|| !is_bool($input['safe_to_resubmit'] ?? null)
|| !is_string($input['error_code'] ?? null)
|| (($input['error_code'] ?? '') !== ''
&& preg_match('/^[a-z][a-z0-9_]{0,63}$/D',
$input['error_code']) !== 1)
|| !is_string($causeCode)
|| ($causeCode !== ''
&& preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $causeCode) !== 1)
|| !array_key_exists('result', $input)
|| ($input['result'] !== null && !is_array($input['result']))
|| (($input['status'] ?? null) === 'succeeded'
&& (($input['effect_may_have_applied'] ?? null) !== false
|| ($input['safe_to_resubmit'] ?? null) !== false
|| ($input['error_code'] ?? null) !== ''
|| $causeCode !== ''))
|| (($input['status'] ?? null) === 'failed'
&& (($input['error_code'] ?? '') === ''
|| (($input['effect_may_have_applied'] ?? false) === true
&& ($input['safe_to_resubmit'] ?? false) === true)))) {
throw new CloudError('Agent 完成回执无效', 400);
}
self::assertCompletionResultSafe($input['result']);
$receipt = [
'event_id' => $input['id'],
'host_id' => (int) $input['agent_id'],
'operation_id' => $input['operation_id'],
'action' => $input['action'],
'resource_id' => $input['resource_id'],
'request_fingerprint' => $input['request_fingerprint'],
'status' => $input['status'],
'effect_may_have_applied' => $input['effect_may_have_applied'],
'safe_to_resubmit' => $input['safe_to_resubmit'],
'error_code' => $input['error_code'],
'result' => $input['result'],
];
if ($causeCode !== '') {
$receipt['cause_code'] = $causeCode;
}
return $receipt;
}
}