You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudCreateFailureRecoveryS...

713 lines
27 KiB
PHP

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

<?php
namespace App\Services\Cloud;
use App\Models\CloudAgentCompletionEvent;
use App\Models\CloudJob;
use App\Models\CloudOrder;
use App\Models\User;
/**
* Converges one dispatched create only from operation-bound absence evidence.
*
* A completion receipt proves its own immutable Agent generation. Read-only
* inspection instead uses a durable PHP prepare marker before the Agent fence
* is released, so a database failure after that release can safely resume.
* Neither path ever sends the original create request again.
*/
final class CloudCreateFailureRecoveryService
{
private const ESCROW_KEY = '_create_failure_recovery';
private const ESCROW_VERSION = 1;
private const MARKER_PREFIX = 'prepared:create_not_applied:v1:';
private const INSPECTION_MAX_AGE_SECONDS = 120;
/**
* Settle an exact failed completion, or ACK it as non-authoritative audit
* data so a later fresh host inspection may continue recovery.
*/
public static function settleReceipt(
CloudJob $source,
CloudAgentCompletionEvent $sourceEvent
): string {
return (new CloudJob())->getConnection()->transaction(
static function () use ($source, $sourceEvent): string {
[$user, $order, $job] = self::lockLedger($source);
$event = CloudAgentCompletionEvent::where(
'event_id',
(string) $sourceEvent->event_id
)->lockForUpdate()->first();
if ($event === null) {
return 'received';
}
if ((string) $event->disposition !== 'received') {
return (string) $event->disposition;
}
if ((int) $event->host_id !== (int) $job->host_id
|| !hash_equals(
(string) $job->operation_id,
(string) $event->operation_id
)) {
throw new CloudRecoveryOperationIdentityConflict(
'开通失败回执与原任务不匹配',
409
);
}
CloudOperationCompletionService::assertEventIdentityForJob(
$job,
$event
);
$operation = CloudOperationCompletionService::operation($event);
if (self::settledNotApplied($job, $order, $event)) {
// A fresh inspection may have completed the atomic PHP
// finalizer before the Agent could deliver its immutable
// failure. The receipt no longer decides the ledger, but
// the exact identity and committed audit allow it to be
// ACKed without reopening or rewriting that resolution.
$event->disposition = 'applied';
$event->applied_at = time();
$event->save();
return 'applied';
}
self::assertRecoverableJob($job);
if (!self::receiptProvesAbsence($event, $operation)) {
$event->disposition = 'late';
$event->applied_at = null;
$event->save();
return 'late';
}
self::assertNoSuccessfulReceiptLocked($job);
self::finalizeLocked(
$job,
$user,
$order,
'late_completion_receipt:' . (string) $event->event_id,
$event
);
return 'applied';
},
3
);
}
/** Recover one lease-owned create without replaying its Agent mutation. */
public static function recover(CloudJob $claimed): void
{
$job = self::claimedJob($claimed);
$prepared = self::preparedEvidence($job);
if ($prepared !== null) {
if (!$prepared['fence_resolution_required']) {
self::finishPrepared($claimed, $prepared, false);
return;
}
try {
self::finishPrepared($claimed, $prepared, null);
return;
} catch (CloudAgentError $error) {
if ($error->agentHttpStatus !== 404) {
throw $error;
}
// No fence verdict was committed before the prior PHP crash.
// A new write requires new, bounded read-only evidence.
}
}
$inspection = CloudAdminOperationRecoveryService::inspectForRecovery(
(string) $job->id
);
$prepared = self::prepareInspection($claimed, $inspection);
self::finishPrepared(
$claimed,
$prepared,
$prepared['fence_resolution_required'] ? $inspection : false
);
}
/**
* Authenticate the durable PHP intent before an automatic Agent fence read
* or write. This is deliberately create-specific and does not broaden the
* generic recovery allow-list.
*/
public static function assertPreparedForFence(
CloudJob $job,
string $resolution,
string $reason
): void {
if ($resolution !== 'not_applied') {
throw new CloudError('开通任务只支持确认未应用', 409);
}
self::assertRecoverableJob($job);
$prepared = self::preparedEvidence($job);
if ($prepared === null
|| !hash_equals($prepared['reason'], $reason)) {
throw new CloudRecoveryEvidenceConflict(
'开通任务的恢复预提交证据不匹配',
409
);
}
}
/**
* @return array{marker:string,reason:string,identity:array,inspection:array,fence_resolution_required:bool}
*/
private static function prepareInspection(
CloudJob $claimed,
array $inspection
): array {
$job = self::claimedJob($claimed);
$identity = CloudOperationRecoveryIdentity::expected($job, false);
CloudOperationRecoveryIdentity::assertMatches($identity, [
'action' => $inspection['action'] ?? null,
'resource_id' => $inspection['resource_id'] ?? null,
'request_fingerprint' =>
$inspection['request_fingerprint'] ?? null,
]);
self::assertInspectionProvesAbsence($job, $inspection);
$evidence = self::inspectionEvidence($inspection);
$marker = self::inspectionMarker($evidence);
$reason = self::fenceReason($job, $marker);
return (new CloudJob())->getConnection()->transaction(
static function () use (
$claimed,
$identity,
$evidence,
$marker,
$reason
): array {
[$user, $order, $locked] = self::lockLedger($claimed);
self::assertClaimedJob($locked, $claimed);
self::assertNoSuccessfulReceiptLocked($locked);
$payload = self::payload($locked);
$lockedIdentity = CloudOperationRecoveryIdentity::
expectedFromPayload($locked, $payload, false);
CloudOperationRecoveryIdentity::assertMatches(
$lockedIdentity,
$identity
);
$payload[self::ESCROW_KEY] = [
'version' => self::ESCROW_VERSION,
'marker' => $marker,
'reason' => $reason,
'identity' => CloudOperationRecoveryIdentity::persisted(
$lockedIdentity
),
'inspection' => $evidence,
];
$encoded = json_encode(
$payload,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
);
if (strlen($encoded) > 196608) {
throw new CloudRecoveryPayloadUnavailable(
'开通任务恢复预提交数据过大',
409
);
}
$locked->effect_resolution_reason = $marker;
$locked->sealed_payload = CloudSecrets::encrypt(
$payload,
'job-payload'
);
$locked->updated_at = time();
$locked->save();
return [
'marker' => $marker,
'reason' => $reason,
'identity' => $lockedIdentity,
'inspection' => $evidence,
'fence_resolution_required' => (bool)
$evidence['fence_resolution_required'],
];
},
3
);
}
/**
* Resolve or verify the Agent fence while holding the canonical financial
* locks, then commit every PHP ledger transition in that same transaction.
*
* @param array{marker:string,reason:string,identity:array,inspection:array,fence_resolution_required:bool} $prepared
* @param array|false|null $inspection Fresh evidence, false when no fence
* is required, null when resuming an already committed fence resolution.
*/
private static function finishPrepared(
CloudJob $claimed,
array $prepared,
array|false|null $inspection
): void {
(new CloudJob())->getConnection()->transaction(
static function () use ($claimed, $prepared, $inspection): void {
[$user, $order, $job] = self::lockLedger($claimed);
self::assertClaimedJob($job, $claimed);
$lockedPrepared = self::preparedEvidence($job);
if ($lockedPrepared === null
|| !hash_equals(
$prepared['marker'],
$lockedPrepared['marker']
)
|| !hash_equals(
$prepared['reason'],
$lockedPrepared['reason']
)) {
throw new CloudRecoveryEvidenceConflict(
'开通任务的恢复预提交已变化',
409
);
}
self::assertNoSuccessfulReceiptLocked($job);
if ($inspection === null) {
CloudAdminOperationRecoveryService::
preparedFenceResolutionForRecovery(
(string) $job->id,
'not_applied',
$lockedPrepared['reason']
);
} elseif ($inspection !== false) {
CloudAdminOperationRecoveryService::resolveFenceForRecovery(
(string) $job->id,
'not_applied',
$lockedPrepared['reason'],
$inspection
);
} elseif ($lockedPrepared['fence_resolution_required']) {
throw new CloudRecoveryEvidenceConflict(
'开通任务仍需解除宿主机资源栅栏',
409
);
}
self::assertNoSuccessfulReceiptLocked($job);
self::finalizeLocked(
$job,
$user,
$order,
'automatic_create_not_applied:'
. substr($lockedPrepared['marker'], -64)
);
},
3
);
}
/**
* @return array{marker:string,reason:string,identity:array,inspection:array,fence_resolution_required:bool}|null
*/
private static function preparedEvidence(CloudJob $job): ?array
{
$marker = (string) ($job->effect_resolution_reason ?? '');
if (preg_match(
'/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D',
$marker
) !== 1) {
return null;
}
$payload = self::payload($job);
$value = $payload[self::ESCROW_KEY] ?? null;
$identity = is_array($value) ? ($value['identity'] ?? null) : null;
$inspection = is_array($value) ? ($value['inspection'] ?? null) : null;
if (!is_array($value) || array_is_list($value)
|| array_diff(array_keys($value), [
'version', 'marker', 'reason', 'identity', 'inspection',
]) !== []
|| ($value['version'] ?? null) !== self::ESCROW_VERSION
|| !is_string($value['marker'] ?? null)
|| !hash_equals($marker, $value['marker'])
|| !is_string($value['reason'] ?? null)
|| mb_strlen($value['reason']) < 20
|| mb_strlen($value['reason']) > 500
|| !is_array($identity) || array_is_list($identity)
|| !is_array($inspection) || array_is_list($inspection)) {
throw new CloudRecoveryPayloadUnavailable(
'开通任务恢复预提交数据无效',
409
);
}
$expected = CloudOperationRecoveryIdentity::expectedFromPayload(
$job,
$payload,
false
);
CloudOperationRecoveryIdentity::assertMatches($expected, $identity);
self::assertInspectionProvesAbsence($job, $inspection, false);
CloudOperationRecoveryIdentity::assertMatches($expected, [
'action' => $inspection['action'] ?? null,
'resource_id' => $inspection['resource_id'] ?? null,
'request_fingerprint' =>
$inspection['request_fingerprint'] ?? null,
]);
$evidence = self::inspectionEvidence($inspection);
$expectedMarker = self::inspectionMarker($evidence);
if (!hash_equals($expectedMarker, $marker)
|| !hash_equals(self::fenceReason($job, $marker), $value['reason'])) {
throw new CloudRecoveryEvidenceConflict(
'开通任务恢复预提交摘要不匹配',
409
);
}
return [
'marker' => $marker,
'reason' => $value['reason'],
'identity' => $expected,
'inspection' => $evidence,
'fence_resolution_required' =>
(bool) $evidence['fence_resolution_required'],
];
}
private static function assertInspectionProvesAbsence(
CloudJob $job,
array $inspection,
bool $requireFresh = true
): void {
$checkedAt = $inspection['checked_at'] ?? null;
$projection = $inspection['effect_projection'] ?? null;
if (($inspection['operation_id'] ?? null)
!== (string) $job->operation_id
|| ($inspection['journal_status'] ?? null) !== 'failed'
|| ($inspection['outcome'] ?? null) !== 'failed'
|| ($inspection['summary_code'] ?? null) !== 'operation_failed'
|| ($inspection['effect_resolution'] ?? null) !== 'not_applied'
|| !is_int($checkedAt)
|| ($requireFresh && ($checkedAt < time()
- self::INSPECTION_MAX_AGE_SECONDS
|| $checkedAt > time() + 5))
|| !is_array($projection) || array_is_list($projection)
|| ($projection['creation_absent'] ?? null) !== true
|| ($projection['managed_artifacts_absent'] ?? null) !== true
|| !in_array(
$inspection['fence_state'] ?? null,
['not_required', 'active'],
true
)
|| !is_bool($inspection['fence_resolution_required'] ?? null)
|| (bool) $inspection['fence_resolution_required']
!== (($inspection['fence_state'] ?? null) === 'active')
|| array_key_exists('completion_delivery_state', $inspection)) {
throw new CloudRecoveryEvidenceConflict(
'宿主机检查没有同时确认虚拟机及面板管理资源均不存在',
409
);
}
CloudCreateFailureFinalizer::assertResolutionSupported(
$job,
'not_applied',
$inspection
);
}
private static function receiptProvesAbsence(
CloudAgentCompletionEvent $event,
array $operation
): bool {
$result = $operation['result'] ?? null;
return (string) $event->outcome === 'failed'
&& !(bool) $event->effect_may_have_applied
&& is_array($result) && !array_is_list($result)
&& ($result['creation_absent'] ?? null) === true
&& ($result['managed_artifacts_absent'] ?? null) === true;
}
/** @return array{0:User,1:CloudOrder,2:CloudJob} */
private static function lockLedger(CloudJob $source): array
{
$user = User::where('id', (int) $source->user_id)
->lockForUpdate()->first();
$order = CloudOrder::where('id', (int) $source->order_id)
->lockForUpdate()->first();
$job = CloudJob::where('id', (int) $source->id)
->lockForUpdate()->first();
if ($user === null || $order === null || $job === null
|| (int) $job->user_id !== (int) $user->id
|| (int) $job->order_id !== (int) $order->id
|| (int) $order->user_id !== (int) $user->id) {
throw new CloudRecoveryInstanceStateConflict(
'开通任务与资金账本关联不一致',
409
);
}
return [$user, $order, $job];
}
private static function assertRecoverableJob(CloudJob $job): void
{
self::assertDispatchedEnvelope($job);
if ((string) $job->kind !== 'create'
|| $job->order_id === null
|| (string) $job->execution_mode !== 'agent'
|| (string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
|| (string) $job->status !== 'failed'
|| !(bool) $job->effect_may_have_applied
|| (bool) $job->safe_to_resubmit
|| (string) $job->effect_resolution !== 'unknown') {
throw new CloudRecoveryInstanceStateConflict(
'开通任务不处于可自动核对状态',
409
);
}
}
private static function settledNotApplied(
CloudJob $job,
CloudOrder $order,
CloudAgentCompletionEvent $event
): bool {
self::assertDispatchedEnvelope($job);
$reason = $job->effect_resolution_reason;
$completionEventId = $job->completion_event_id;
return (string) $event->outcome === 'failed'
&& (string) $job->kind === 'create'
&& $job->order_id !== null
&& (string) $job->execution_mode === 'agent'
&& (string) $job->phase === 'terminal'
&& (string) $job->outcome === 'failed'
&& (string) $job->status === 'failed'
&& (int) $job->dispatch_attempts === 1
&& !(bool) $job->safe_to_resubmit
&& (string) $job->effect_resolution === 'not_applied'
&& (int) ($job->effect_resolved_at ?? 0) > 0
&& is_string($reason)
&& mb_strlen($reason) >= 20
&& mb_strlen($reason) <= 500
&& !(bool) $job->finalization_incident
&& $job->finalization_error_code === null
&& $job->instance_id === null
&& $order->instance_id === null
&& (string) $order->status === 'failed'
&& $order->refunded_at !== null
&& ($completionEventId === null
|| (is_string($completionEventId)
&& hash_equals(
$completionEventId,
(string) $event->event_id
)));
}
/** Validate create's immutable generation without broadening policy. */
private static function assertDispatchedEnvelope(CloudJob $job): void
{
CloudDispatchLedger::assertDispatched($job);
if ((string) $job->attempts !== '1'
|| (int) ($job->host_id ?? 0) < 1
|| (int) ($job->dispatch_deadline_at ?? 0) < 1
|| (int) ($job->dispatch_started_at ?? 0) < 1
|| (($job->agent_accepted_at ?? null) !== null
&& (int) $job->agent_accepted_at < 1)
|| (int) ($job->dispatched_at ?? 0) < 1
|| (int) ($job->receipt_deadline_at ?? 0) < 1
|| (int) ($job->terminal_at ?? 0) < 1
|| preg_match(
'/^[a-f0-9]{32}$/D',
(string) ($job->operation_id ?? '')
) !== 1
|| preg_match(
'/^[a-f0-9]{64}$/D',
(string) ($job->request_fingerprint ?? '')
) !== 1
|| preg_match(
'/^[A-Za-z0-9_.-]{16,128}$/D',
(string) ($job->request_idempotency_key ?? '')
) !== 1) {
throw new CloudRecoveryInstanceStateConflict(
'开通任务单次派发信封不完整',
409
);
}
}
private static function assertClaimedJob(
CloudJob $job,
CloudJob $claimed
): void {
self::assertRecoverableJob($job);
if (!is_string($claimed->lease_token)
|| !is_string($job->lease_token)
|| !hash_equals(
(string) $claimed->lease_token,
(string) $job->lease_token
)
|| (int) $job->lease_until <= time()) {
throw new CloudRecoveryInstanceStateConflict(
'开通任务自动核对租约已失效',
409
);
}
}
private static function claimedJob(CloudJob $claimed): CloudJob
{
$job = CloudJob::find((int) $claimed->id);
if ($job === null) {
throw new CloudError('任务不存在', 404);
}
self::assertClaimedJob($job, $claimed);
return $job;
}
private static function assertNoSuccessfulReceiptLocked(CloudJob $job): void
{
$event = CloudAgentCompletionEvent::where(
'host_id',
(int) $job->host_id
)->where(
'operation_id',
(string) $job->operation_id
)->where('outcome', 'succeeded')->lockForUpdate()->first();
if ($event === null) {
return;
}
$exact = CloudOperationCompletionService::isExactStoredEvent($event)
|| CloudOperationCompletionService::isExactAppliedEvent($event);
if (!$exact) {
throw new CloudRecoveryOperationIdentityConflict(
'开通成功回执状态异常,不能执行退款',
409
);
}
$payload = self::payload($job);
$identity = CloudOperationRecoveryIdentity::expectedFromPayload(
$job,
$payload,
true
);
if (CloudOperationRecoveryIdentity::same(
$identity,
CloudOperationCompletionService::identity($event)
)) {
throw new CloudRecoveryEvidenceConflict(
'已收到匹配的实例创建成功回执,不能按未应用退款',
409
);
}
throw new CloudRecoveryOperationIdentityConflict(
'开通成功回执身份冲突,不能执行退款',
409
);
}
private static function finalizeLocked(
CloudJob $job,
User $user,
CloudOrder $order,
string $resolutionReason,
?CloudAgentCompletionEvent $event = null
): void {
$payload = self::payload($job);
$identity = CloudOperationRecoveryIdentity::expectedFromPayload(
$job,
$payload,
false
);
CloudCreateFailureFinalizer::refundAndReleaseLocked(
$job,
$user,
$order,
'create_effect_not_applied'
);
$safePayload = CloudInput::safeResult($payload);
unset($safePayload[self::ESCROW_KEY]);
$safePayload[CloudOperationRecoveryIdentity::payloadKey()] =
CloudOperationRecoveryIdentity::persisted($identity);
$now = time();
$job->effect_resolution = 'not_applied';
$job->effect_resolved_at = $now;
$job->effect_resolved_by = null;
$job->effect_resolution_reason = $resolutionReason;
if ($event !== null) {
$job->completion_event_id = (string) $event->event_id;
}
$job->sealed_payload = CloudSecrets::encrypt(
$safePayload,
'job-payload'
);
$job->finalization_incident = 0;
$job->finalization_error_code = null;
$job->read_at = null;
$job->lease_token = null;
$job->lease_until = 0;
$job->available_at = 0;
$job->updated_at = $now;
$job->save();
if ($event !== null) {
$event->disposition = 'applied';
$event->applied_at = $now;
$event->save();
}
}
private static function payload(CloudJob $job): array
{
try {
$payload = CloudSecrets::decrypt(
(string) $job->sealed_payload,
'job-payload'
);
} catch (\Throwable $error) {
throw new CloudRecoveryPayloadUnavailable(
'开通任务恢复数据无法解密',
409
);
}
if (!is_array($payload) || array_is_list($payload)) {
throw new CloudRecoveryPayloadUnavailable(
'开通任务恢复数据结构不完整',
409
);
}
return $payload;
}
private static function fenceReason(CloudJob $job, string $marker): string
{
return '自动恢复已确认原开通操作未创建虚拟机或面板管理资源;operation_id='
. (string) $job->operation_id . ';evidence='
. substr($marker, -64);
}
/** Hash only stable evidence so a lost fence response can be resumed. */
private static function inspectionMarker(array $evidence): string
{
unset($evidence['checked_at']);
return self::MARKER_PREFIX . hash(
'sha256',
json_encode(
$evidence,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
)
);
}
/** Return the bounded, canonical evidence persisted in encrypted escrow. */
private static function inspectionEvidence(array $inspection): array
{
return [
'operation_id' => $inspection['operation_id'],
'action' => $inspection['action'],
'resource_id' => $inspection['resource_id'],
'request_fingerprint' => $inspection['request_fingerprint'],
'journal_status' => $inspection['journal_status'],
'checked_at' => $inspection['checked_at'],
'outcome' => $inspection['outcome'],
'summary_code' => $inspection['summary_code'],
'error_code' => $inspection['error_code'] ?? '',
'effect_resolution' => $inspection['effect_resolution'],
'fence_state' => $inspection['fence_state'],
'fence_resolution_required' =>
$inspection['fence_resolution_required'],
'effect_projection' => [
'creation_absent' => true,
'managed_artifacts_absent' => true,
],
];
}
}