You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudCreateFailureRecoveryS...

713 lines
27 KiB
PHTML

<?php
namespace App\Services\Cloud;
use App\Models\CloudAgentCompletionEvent;
use App\Models\CloudJob;
use App\Models\CloudOrder;
use App\Models\User;
/**
* Converges one dispatched create only from operation-bound absence evidence.
*
* A completion receipt proves its own immutable Agent generation. Read-only
* inspection instead uses a durable PHP prepare marker before the Agent fence
* is released, so a database failure after that release can safely resume.
* Neither path ever sends the original create request again.
*/
final class CloudCreateFailureRecoveryService
{
private const ESCROW_KEY = '_create_failure_recovery';
private const ESCROW_VERSION = 1;
private const MARKER_PREFIX = 'prepared:create_not_applied:v1:';
private const INSPECTION_MAX_AGE_SECONDS = 120;
/**
* Settle an exact failed completion, or ACK it as non-authoritative audit
* data so a later fresh host inspection may continue recovery.
*/
public static function settleReceipt(
CloudJob $source,
CloudAgentCompletionEvent $sourceEvent
): string {
return (new CloudJob())->getConnection()->transaction(
static function () use ($source, $sourceEvent): string {
[$user, $order, $job] = self::lockLedger($source);
$event = CloudAgentCompletionEvent::where(
'event_id',
(string) $sourceEvent->event_id
)->lockForUpdate()->first();
if ($event === null) {
return 'received';
}
if ((string) $event->disposition !== 'received') {
return (string) $event->disposition;
}
if ((int) $event->host_id !== (int) $job->host_id
|| !hash_equals(
(string) $job->operation_id,
(string) $event->operation_id
)) {
throw new CloudRecoveryOperationIdentityConflict(
'开通失败回执与原任务不匹配',
409
);
}
CloudOperationCompletionService::assertEventIdentityForJob(
$job,
$event
);
$operation = CloudOperationCompletionService::operation($event);
if (self::settledNotApplied($job, $order, $event)) {
// A fresh inspection may have completed the atomic PHP
// finalizer before the Agent could deliver its immutable
// failure. The receipt no longer decides the ledger, but
// the exact identity and committed audit allow it to be
// ACKed without reopening or rewriting that resolution.
$event->disposition = 'applied';
$event->applied_at = time();
$event->save();
return 'applied';
}
self::assertRecoverableJob($job);
if (!self::receiptProvesAbsence($event, $operation)) {
$event->disposition = 'late';
$event->applied_at = null;
$event->save();
return 'late';
}
self::assertNoSuccessfulReceiptLocked($job);
self::finalizeLocked(
$job,
$user,
$order,
'late_completion_receipt:' . (string) $event->event_id,
$event
);
return 'applied';
},
3
);
}
/** Recover one lease-owned create without replaying its Agent mutation. */
public static function recover(CloudJob $claimed): void
{
$job = self::claimedJob($claimed);
$prepared = self::preparedEvidence($job);
if ($prepared !== null) {
if (!$prepared['fence_resolution_required']) {
self::finishPrepared($claimed, $prepared, false);
return;
}
try {
self::finishPrepared($claimed, $prepared, null);
return;
} catch (CloudAgentError $error) {
if ($error->agentHttpStatus !== 404) {
throw $error;
}
// No fence verdict was committed before the prior PHP crash.
// A new write requires new, bounded read-only evidence.
}
}
$inspection = CloudAdminOperationRecoveryService::inspectForRecovery(
(string) $job->id
);
$prepared = self::prepareInspection($claimed, $inspection);
self::finishPrepared(
$claimed,
$prepared,
$prepared['fence_resolution_required'] ? $inspection : false
);
}
/**
* Authenticate the durable PHP intent before an automatic Agent fence read
* or write. This is deliberately create-specific and does not broaden the
* generic recovery allow-list.
*/
public static function assertPreparedForFence(
CloudJob $job,
string $resolution,
string $reason
): void {
if ($resolution !== 'not_applied') {
throw new CloudError('开通任务只支持确认未应用', 409);
}
self::assertRecoverableJob($job);
$prepared = self::preparedEvidence($job);
if ($prepared === null
|| !hash_equals($prepared['reason'], $reason)) {
throw new CloudRecoveryEvidenceConflict(
'开通任务的恢复预提交证据不匹配',
409
);
}
}
/**
* @return array{marker:string,reason:string,identity:array,inspection:array,fence_resolution_required:bool}
*/
private static function prepareInspection(
CloudJob $claimed,
array $inspection
): array {
$job = self::claimedJob($claimed);
$identity = CloudOperationRecoveryIdentity::expected($job, false);
CloudOperationRecoveryIdentity::assertMatches($identity, [
'action' => $inspection['action'] ?? null,
'resource_id' => $inspection['resource_id'] ?? null,
'request_fingerprint' =>
$inspection['request_fingerprint'] ?? null,
]);
self::assertInspectionProvesAbsence($job, $inspection);
$evidence = self::inspectionEvidence($inspection);
$marker = self::inspectionMarker($evidence);
$reason = self::fenceReason($job, $marker);
return (new CloudJob())->getConnection()->transaction(
static function () use (
$claimed,
$identity,
$evidence,
$marker,
$reason
): array {
[$user, $order, $locked] = self::lockLedger($claimed);
self::assertClaimedJob($locked, $claimed);
self::assertNoSuccessfulReceiptLocked($locked);
$payload = self::payload($locked);
$lockedIdentity = CloudOperationRecoveryIdentity::
expectedFromPayload($locked, $payload, false);
CloudOperationRecoveryIdentity::assertMatches(
$lockedIdentity,
$identity
);
$payload[self::ESCROW_KEY] = [
'version' => self::ESCROW_VERSION,
'marker' => $marker,
'reason' => $reason,
'identity' => CloudOperationRecoveryIdentity::persisted(
$lockedIdentity
),
'inspection' => $evidence,
];
$encoded = json_encode(
$payload,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
);
if (strlen($encoded) > 196608) {
throw new CloudRecoveryPayloadUnavailable(
'开通任务恢复预提交数据过大',
409
);
}
$locked->effect_resolution_reason = $marker;
$locked->sealed_payload = CloudSecrets::encrypt(
$payload,
'job-payload'
);
$locked->updated_at = time();
$locked->save();
return [
'marker' => $marker,
'reason' => $reason,
'identity' => $lockedIdentity,
'inspection' => $evidence,
'fence_resolution_required' => (bool)
$evidence['fence_resolution_required'],
];
},
3
);
}
/**
* Resolve or verify the Agent fence while holding the canonical financial
* locks, then commit every PHP ledger transition in that same transaction.
*
* @param array{marker:string,reason:string,identity:array,inspection:array,fence_resolution_required:bool} $prepared
* @param array|false|null $inspection Fresh evidence, false when no fence
* is required, null when resuming an already committed fence resolution.
*/
private static function finishPrepared(
CloudJob $claimed,
array $prepared,
array|false|null $inspection
): void {
(new CloudJob())->getConnection()->transaction(
static function () use ($claimed, $prepared, $inspection): void {
[$user, $order, $job] = self::lockLedger($claimed);
self::assertClaimedJob($job, $claimed);
$lockedPrepared = self::preparedEvidence($job);
if ($lockedPrepared === null
|| !hash_equals(
$prepared['marker'],
$lockedPrepared['marker']
)
|| !hash_equals(
$prepared['reason'],
$lockedPrepared['reason']
)) {
throw new CloudRecoveryEvidenceConflict(
'开通任务的恢复预提交已变化',
409
);
}
self::assertNoSuccessfulReceiptLocked($job);
if ($inspection === null) {
CloudAdminOperationRecoveryService::
preparedFenceResolutionForRecovery(
(string) $job->id,
'not_applied',
$lockedPrepared['reason']
);
} elseif ($inspection !== false) {
CloudAdminOperationRecoveryService::resolveFenceForRecovery(
(string) $job->id,
'not_applied',
$lockedPrepared['reason'],
$inspection
);
} elseif ($lockedPrepared['fence_resolution_required']) {
throw new CloudRecoveryEvidenceConflict(
'开通任务仍需解除宿主机资源栅栏',
409
);
}
self::assertNoSuccessfulReceiptLocked($job);
self::finalizeLocked(
$job,
$user,
$order,
'automatic_create_not_applied:'
. substr($lockedPrepared['marker'], -64)
);
},
3
);
}
/**
* @return array{marker:string,reason:string,identity:array,inspection:array,fence_resolution_required:bool}|null
*/
private static function preparedEvidence(CloudJob $job): ?array
{
$marker = (string) ($job->effect_resolution_reason ?? '');
if (preg_match(
'/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D',
$marker
) !== 1) {
return null;
}
$payload = self::payload($job);
$value = $payload[self::ESCROW_KEY] ?? null;
$identity = is_array($value) ? ($value['identity'] ?? null) : null;
$inspection = is_array($value) ? ($value['inspection'] ?? null) : null;
if (!is_array($value) || array_is_list($value)
|| array_diff(array_keys($value), [
'version', 'marker', 'reason', 'identity', 'inspection',
]) !== []
|| ($value['version'] ?? null) !== self::ESCROW_VERSION
|| !is_string($value['marker'] ?? null)
|| !hash_equals($marker, $value['marker'])
|| !is_string($value['reason'] ?? null)
|| mb_strlen($value['reason']) < 20
|| mb_strlen($value['reason']) > 500
|| !is_array($identity) || array_is_list($identity)
|| !is_array($inspection) || array_is_list($inspection)) {
throw new CloudRecoveryPayloadUnavailable(
'开通任务恢复预提交数据无效',
409
);
}
$expected = CloudOperationRecoveryIdentity::expectedFromPayload(
$job,
$payload,
false
);
CloudOperationRecoveryIdentity::assertMatches($expected, $identity);
self::assertInspectionProvesAbsence($job, $inspection, false);
CloudOperationRecoveryIdentity::assertMatches($expected, [
'action' => $inspection['action'] ?? null,
'resource_id' => $inspection['resource_id'] ?? null,
'request_fingerprint' =>
$inspection['request_fingerprint'] ?? null,
]);
$evidence = self::inspectionEvidence($inspection);
$expectedMarker = self::inspectionMarker($evidence);
if (!hash_equals($expectedMarker, $marker)
|| !hash_equals(self::fenceReason($job, $marker), $value['reason'])) {
throw new CloudRecoveryEvidenceConflict(
'开通任务恢复预提交摘要不匹配',
409
);
}
return [
'marker' => $marker,
'reason' => $value['reason'],
'identity' => $expected,
'inspection' => $evidence,
'fence_resolution_required' =>
(bool) $evidence['fence_resolution_required'],
];
}
private static function assertInspectionProvesAbsence(
CloudJob $job,
array $inspection,
bool $requireFresh = true
): void {
$checkedAt = $inspection['checked_at'] ?? null;
$projection = $inspection['effect_projection'] ?? null;
if (($inspection['operation_id'] ?? null)
!== (string) $job->operation_id
|| ($inspection['journal_status'] ?? null) !== 'failed'
|| ($inspection['outcome'] ?? null) !== 'failed'
|| ($inspection['summary_code'] ?? null) !== 'operation_failed'
|| ($inspection['effect_resolution'] ?? null) !== 'not_applied'
|| !is_int($checkedAt)
|| ($requireFresh && ($checkedAt < time()
- self::INSPECTION_MAX_AGE_SECONDS
|| $checkedAt > time() + 5))
|| !is_array($projection) || array_is_list($projection)
|| ($projection['creation_absent'] ?? null) !== true
|| ($projection['managed_artifacts_absent'] ?? null) !== true
|| !in_array(
$inspection['fence_state'] ?? null,
['not_required', 'active'],
true
)
|| !is_bool($inspection['fence_resolution_required'] ?? null)
|| (bool) $inspection['fence_resolution_required']
!== (($inspection['fence_state'] ?? null) === 'active')
|| array_key_exists('completion_delivery_state', $inspection)) {
throw new CloudRecoveryEvidenceConflict(
'宿主机检查没有同时确认虚拟机及面板管理资源均不存在',
409
);
}
CloudCreateFailureFinalizer::assertResolutionSupported(
$job,
'not_applied',
$inspection
);
}
private static function receiptProvesAbsence(
CloudAgentCompletionEvent $event,
array $operation
): bool {
$result = $operation['result'] ?? null;
return (string) $event->outcome === 'failed'
&& !(bool) $event->effect_may_have_applied
&& is_array($result) && !array_is_list($result)
&& ($result['creation_absent'] ?? null) === true
&& ($result['managed_artifacts_absent'] ?? null) === true;
}
/** @return array{0:User,1:CloudOrder,2:CloudJob} */
private static function lockLedger(CloudJob $source): array
{
$user = User::where('id', (int) $source->user_id)
->lockForUpdate()->first();
$order = CloudOrder::where('id', (int) $source->order_id)
->lockForUpdate()->first();
$job = CloudJob::where('id', (int) $source->id)
->lockForUpdate()->first();
if ($user === null || $order === null || $job === null
|| (int) $job->user_id !== (int) $user->id
|| (int) $job->order_id !== (int) $order->id
|| (int) $order->user_id !== (int) $user->id) {
throw new CloudRecoveryInstanceStateConflict(
'开通任务与资金账本关联不一致',
409
);
}
return [$user, $order, $job];
}
private static function assertRecoverableJob(CloudJob $job): void
{
self::assertDispatchedEnvelope($job);
if ((string) $job->kind !== 'create'
|| $job->order_id === null
|| (string) $job->execution_mode !== 'agent'
|| (string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
|| (string) $job->status !== 'failed'
|| !(bool) $job->effect_may_have_applied
|| (bool) $job->safe_to_resubmit
|| (string) $job->effect_resolution !== 'unknown') {
throw new CloudRecoveryInstanceStateConflict(
'开通任务不处于可自动核对状态',
409
);
}
}
private static function settledNotApplied(
CloudJob $job,
CloudOrder $order,
CloudAgentCompletionEvent $event
): bool {
self::assertDispatchedEnvelope($job);
$reason = $job->effect_resolution_reason;
$completionEventId = $job->completion_event_id;
return (string) $event->outcome === 'failed'
&& (string) $job->kind === 'create'
&& $job->order_id !== null
&& (string) $job->execution_mode === 'agent'
&& (string) $job->phase === 'terminal'
&& (string) $job->outcome === 'failed'
&& (string) $job->status === 'failed'
&& (int) $job->dispatch_attempts === 1
&& !(bool) $job->safe_to_resubmit
&& (string) $job->effect_resolution === 'not_applied'
&& (int) ($job->effect_resolved_at ?? 0) > 0
&& is_string($reason)
&& mb_strlen($reason) >= 20
&& mb_strlen($reason) <= 500
&& !(bool) $job->finalization_incident
&& $job->finalization_error_code === null
&& $job->instance_id === null
&& $order->instance_id === null
&& (string) $order->status === 'failed'
&& $order->refunded_at !== null
&& ($completionEventId === null
|| (is_string($completionEventId)
&& hash_equals(
$completionEventId,
(string) $event->event_id
)));
}
/** Validate create's immutable generation without broadening policy. */
private static function assertDispatchedEnvelope(CloudJob $job): void
{
CloudDispatchLedger::assertDispatched($job);
if ((string) $job->attempts !== '1'
|| (int) ($job->host_id ?? 0) < 1
|| (int) ($job->dispatch_deadline_at ?? 0) < 1
|| (int) ($job->dispatch_started_at ?? 0) < 1
|| (($job->agent_accepted_at ?? null) !== null
&& (int) $job->agent_accepted_at < 1)
|| (int) ($job->dispatched_at ?? 0) < 1
|| (int) ($job->receipt_deadline_at ?? 0) < 1
|| (int) ($job->terminal_at ?? 0) < 1
|| preg_match(
'/^[a-f0-9]{32}$/D',
(string) ($job->operation_id ?? '')
) !== 1
|| preg_match(
'/^[a-f0-9]{64}$/D',
(string) ($job->request_fingerprint ?? '')
) !== 1
|| preg_match(
'/^[A-Za-z0-9_.-]{16,128}$/D',
(string) ($job->request_idempotency_key ?? '')
) !== 1) {
throw new CloudRecoveryInstanceStateConflict(
'开通任务单次派发信封不完整',
409
);
}
}
private static function assertClaimedJob(
CloudJob $job,
CloudJob $claimed
): void {
self::assertRecoverableJob($job);
if (!is_string($claimed->lease_token)
|| !is_string($job->lease_token)
|| !hash_equals(
(string) $claimed->lease_token,
(string) $job->lease_token
)
|| (int) $job->lease_until <= time()) {
throw new CloudRecoveryInstanceStateConflict(
'开通任务自动核对租约已失效',
409
);
}
}
private static function claimedJob(CloudJob $claimed): CloudJob
{
$job = CloudJob::find((int) $claimed->id);
if ($job === null) {
throw new CloudError('任务不存在', 404);
}
self::assertClaimedJob($job, $claimed);
return $job;
}
private static function assertNoSuccessfulReceiptLocked(CloudJob $job): void
{
$event = CloudAgentCompletionEvent::where(
'host_id',
(int) $job->host_id
)->where(
'operation_id',
(string) $job->operation_id
)->where('outcome', 'succeeded')->lockForUpdate()->first();
if ($event === null) {
return;
}
$exact = CloudOperationCompletionService::isExactStoredEvent($event)
|| CloudOperationCompletionService::isExactAppliedEvent($event);
if (!$exact) {
throw new CloudRecoveryOperationIdentityConflict(
'开通成功回执状态异常,不能执行退款',
409
);
}
$payload = self::payload($job);
$identity = CloudOperationRecoveryIdentity::expectedFromPayload(
$job,
$payload,
true
);
if (CloudOperationRecoveryIdentity::same(
$identity,
CloudOperationCompletionService::identity($event)
)) {
throw new CloudRecoveryEvidenceConflict(
'已收到匹配的实例创建成功回执,不能按未应用退款',
409
);
}
throw new CloudRecoveryOperationIdentityConflict(
'开通成功回执身份冲突,不能执行退款',
409
);
}
private static function finalizeLocked(
CloudJob $job,
User $user,
CloudOrder $order,
string $resolutionReason,
?CloudAgentCompletionEvent $event = null
): void {
$payload = self::payload($job);
$identity = CloudOperationRecoveryIdentity::expectedFromPayload(
$job,
$payload,
false
);
CloudCreateFailureFinalizer::refundAndReleaseLocked(
$job,
$user,
$order,
'create_effect_not_applied'
);
$safePayload = CloudInput::safeResult($payload);
unset($safePayload[self::ESCROW_KEY]);
$safePayload[CloudOperationRecoveryIdentity::payloadKey()] =
CloudOperationRecoveryIdentity::persisted($identity);
$now = time();
$job->effect_resolution = 'not_applied';
$job->effect_resolved_at = $now;
$job->effect_resolved_by = null;
$job->effect_resolution_reason = $resolutionReason;
if ($event !== null) {
$job->completion_event_id = (string) $event->event_id;
}
$job->sealed_payload = CloudSecrets::encrypt(
$safePayload,
'job-payload'
);
$job->finalization_incident = 0;
$job->finalization_error_code = null;
$job->read_at = null;
$job->lease_token = null;
$job->lease_until = 0;
$job->available_at = 0;
$job->updated_at = $now;
$job->save();
if ($event !== null) {
$event->disposition = 'applied';
$event->applied_at = $now;
$event->save();
}
}
private static function payload(CloudJob $job): array
{
try {
$payload = CloudSecrets::decrypt(
(string) $job->sealed_payload,
'job-payload'
);
} catch (\Throwable $error) {
throw new CloudRecoveryPayloadUnavailable(
'开通任务恢复数据无法解密',
409
);
}
if (!is_array($payload) || array_is_list($payload)) {
throw new CloudRecoveryPayloadUnavailable(
'开通任务恢复数据结构不完整',
409
);
}
return $payload;
}
private static function fenceReason(CloudJob $job, string $marker): string
{
return '自动恢复已确认原开通操作未创建虚拟机或面板管理资源;operation_id='
. (string) $job->operation_id . ';evidence='
. substr($marker, -64);
}
/** Hash only stable evidence so a lost fence response can be resumed. */
private static function inspectionMarker(array $evidence): string
{
unset($evidence['checked_at']);
return self::MARKER_PREFIX . hash(
'sha256',
json_encode(
$evidence,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
)
);
}
/** Return the bounded, canonical evidence persisted in encrypted escrow. */
private static function inspectionEvidence(array $inspection): array
{
return [
'operation_id' => $inspection['operation_id'],
'action' => $inspection['action'],
'resource_id' => $inspection['resource_id'],
'request_fingerprint' => $inspection['request_fingerprint'],
'journal_status' => $inspection['journal_status'],
'checked_at' => $inspection['checked_at'],
'outcome' => $inspection['outcome'],
'summary_code' => $inspection['summary_code'],
'error_code' => $inspection['error_code'] ?? '',
'effect_resolution' => $inspection['effect_resolution'],
'fence_state' => $inspection['fence_state'],
'fence_resolution_required' =>
$inspection['fence_resolution_required'],
'effect_projection' => [
'creation_absent' => true,
'managed_artifacts_absent' => true,
],
];
}
}